From c08c7c34f670f2cb2f3cc3532647fe6efc71a4fd Mon Sep 17 00:00:00 2001 From: Reinier Maas Date: Fri, 17 Jul 2026 14:46:06 +0200 Subject: [PATCH 1/7] CI: Double compute performance, halve build jobs More effective caching between the nix jobs. --- .semaphore/semaphore.yml | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/.semaphore/semaphore.yml b/.semaphore/semaphore.yml index 250aa835..435cd524 100644 --- a/.semaphore/semaphore.yml +++ b/.semaphore/semaphore.yml @@ -3,8 +3,8 @@ name: "Opsqueue CI" agent: machine: - type: "f1-standard-2" - os_image: "ubuntu2204" + type: "f1-standard-4" + os_image: "ubuntu2404" # If we push a new build to some branch that isn't master, and another build is # already running, we cancel it. @@ -60,17 +60,14 @@ blocks: when: "branch = 'master'" task: jobs: - - name: "Build Opsqueue binary (in release mode) & run unit tests" + - name: "Build Opsqueue binary & Python library (in release mode) & run unit tests" commands: - - "nix-store -qR --include-outputs $(nix-store -qd $(just nix-build-bin)) | grep -v '\\.drv$' | cachix push channable" + - "nix-store -qR --include-outputs $(nix-store -qd $(just nix-build)) | grep -v '\\.drv$' | cachix push channable" # Once a day, we update the devenv cache. # Except when fresh, this takes < 1 second # but it cannot live in the prologue # as that might corrupt it - cache store nix-store-$(date -u -Idate) /nix - - name: "Build Python library (in release mode)" - commands: - - "nix-store -qR --include-outputs $(nix-store -qd $(just nix-build-python)) | grep -v '\\.drv$' | cachix push channable" - name: "Integration" dependencies: - "Build" From 1095092d4ea711026ed9f8ec773a13d5dd7eee61 Mon Sep 17 00:00:00 2001 From: Reinier Maas Date: Fri, 17 Jul 2026 14:57:55 +0200 Subject: [PATCH 2/7] Add: `cargo hakari` prevents rebuilding libraries using different flags --- .config/hakari.toml | 27 +++++++++ Cargo.lock | 56 +++++++++++++++++ Cargo.toml | 4 ++ default.nix | 1 + justfile | 7 ++- libs/opsqueue_python/Cargo.toml | 2 + opsqueue/Cargo.toml | 1 + workspace-hack/.gitattributes | 4 ++ workspace-hack/Cargo.toml | 103 ++++++++++++++++++++++++++++++++ workspace-hack/build.rs | 2 + workspace-hack/src/lib.rs | 1 + 11 files changed, 207 insertions(+), 1 deletion(-) create mode 100644 .config/hakari.toml create mode 100644 workspace-hack/.gitattributes create mode 100644 workspace-hack/Cargo.toml create mode 100644 workspace-hack/build.rs create mode 100644 workspace-hack/src/lib.rs diff --git a/.config/hakari.toml b/.config/hakari.toml new file mode 100644 index 00000000..bb5d464e --- /dev/null +++ b/.config/hakari.toml @@ -0,0 +1,27 @@ +# This file contains settings for `cargo hakari`. +# See https://docs.rs/cargo-hakari/latest/cargo_hakari/config for a full list of options. + +hakari-package = "workspace-hack" + +# Format version for hakari's output. Version 4 requires cargo-hakari 0.9.22 or above. +dep-format-version = "4" + +# Setting workspace.resolver = "2" or higher in the root Cargo.toml is HIGHLY recommended. +# Hakari works much better with the v2 resolver. (The v2 and v3 resolvers are identical from +# hakari's perspective, so you're welcome to set either.) +# +# For more about the new feature resolver, see: +# https://blog.rust-lang.org/2021/03/25/Rust-1.51.0.html#cargos-new-feature-resolver +resolver = "3" + +# Add triples corresponding to platforms commonly used by developers here. +# https://doc.rust-lang.org/rustc/platform-support.html +platforms = [ + "x86_64-unknown-linux-gnu", + # "x86_64-apple-darwin", + # "aarch64-apple-darwin", + # "x86_64-pc-windows-msvc", +] + +# Write out exact versions rather than a semver range. (Defaults to false.) +# exact-versions = true diff --git a/Cargo.lock b/Cargo.lock index 4ea740a9..f56ae2b7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2241,6 +2241,7 @@ dependencies = [ "url", "uuid", "ux", + "workspace-hack", ] [[package]] @@ -2260,6 +2261,7 @@ dependencies = [ "tracing", "uuid", "ux", + "workspace-hack", ] [[package]] @@ -4521,6 +4523,60 @@ version = "0.57.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1ebf944e87a7c253233ad6766e082e3cd714b5d03812acc24c318f549614536e" +[[package]] +name = "workspace-hack" +version = "0.1.0" +dependencies = [ + "base64", + "bitflags", + "cc", + "chrono", + "crossbeam-epoch", + "crossbeam-utils", + "either", + "event-listener", + "fastrand", + "futures-channel", + "futures-io", + "futures-sink", + "futures-util", + "hyper", + "hyper-util", + "libc", + "libsqlite3-sys", + "log", + "num-traits", + "opentelemetry", + "opentelemetry-http", + "opentelemetry_sdk", + "rand 0.9.5", + "regex", + "regex-automata", + "regex-syntax", + "reqwest", + "rustls-pki-types", + "serde", + "serde_core", + "serde_json", + "sha2 0.10.9", + "slab", + "smallvec", + "sqlx-core", + "sqlx-sqlite", + "syn 3.0.2", + "thiserror", + "tokio", + "tokio-stream", + "tokio-util", + "tower", + "tower-http 0.6.11", + "tracing-core", + "typenum", + "url", + "uuid", + "zerocopy", +] + [[package]] name = "writeable" version = "0.6.3" diff --git a/Cargo.toml b/Cargo.toml index a16b5baa..4d9dda73 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -6,8 +6,12 @@ resolver = "3" members = [ "opsqueue/", "libs/opsqueue_python", + "workspace-hack", ] +[workspace.metadata.hakari] +hakari-package = "workspace-hack" + [workspace.package] version = "0.36.0" diff --git a/default.nix b/default.nix index 2da1a739..f33a05cc 100644 --- a/default.nix +++ b/default.nix @@ -48,6 +48,7 @@ let pkgs.cargo-audit pkgs.cargo-edit pkgs.cargo-insta + pkgs.cargo-hakari pkgs.cargo-nextest pkgs.maturin diff --git a/justfile b/justfile index 0cba2f34..7c8ad88e 100644 --- a/justfile +++ b/justfile @@ -81,7 +81,12 @@ lint-light *ARGS: # Run the slow linters/static analysers that need to look at everything [group('lint')] -lint-heavy: clippy mypy +lint-heavy: clippy hakari mypy + +# Verify the workspace-hack crate is up to date +[group('lint')] +hakari: + cargo hakari verify # Rust static analysis [group('lint')] diff --git a/libs/opsqueue_python/Cargo.toml b/libs/opsqueue_python/Cargo.toml index 67d10548..28335314 100644 --- a/libs/opsqueue_python/Cargo.toml +++ b/libs/opsqueue_python/Cargo.toml @@ -38,5 +38,7 @@ once_cell = "1.21.4" # Only currently used for `unsync::OnceCell` as part of PyO pyo3-log = "0.13.3" tracing = { version = "0.1.41", features = ["log"] } +workspace-hack = { version = "0.1", path = "../../workspace-hack" } + [lints] workspace = true diff --git a/opsqueue/Cargo.toml b/opsqueue/Cargo.toml index 59e3fbae..7cc6c0af 100644 --- a/opsqueue/Cargo.toml +++ b/opsqueue/Cargo.toml @@ -76,6 +76,7 @@ humantime = "2.1.0" dashmap = "6.1.0" crossbeam-skiplist = "0.1.3" sqlformat = "0.5.0" +workspace-hack = { version = "0.1", path = "../workspace-hack" } # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html diff --git a/workspace-hack/.gitattributes b/workspace-hack/.gitattributes new file mode 100644 index 00000000..3e9dba4b --- /dev/null +++ b/workspace-hack/.gitattributes @@ -0,0 +1,4 @@ +# Avoid putting conflict markers in the generated Cargo.toml file, since their presence breaks +# Cargo. +# Also do not check out the file as CRLF on Windows, as that's what hakari needs. +Cargo.toml merge=binary -crlf diff --git a/workspace-hack/Cargo.toml b/workspace-hack/Cargo.toml new file mode 100644 index 00000000..543f78d4 --- /dev/null +++ b/workspace-hack/Cargo.toml @@ -0,0 +1,103 @@ +# This file is generated by `cargo hakari`. +# To regenerate, run: +# cargo hakari generate + +[package] +name = "workspace-hack" +version = "0.1.0" +edition = "2024" +description = "workspace-hack package, managed by hakari" +# You can choose to publish this crate: see https://docs.rs/cargo-hakari/latest/cargo_hakari/publishing. +publish = false + +# The parts of the file between the BEGIN HAKARI SECTION and END HAKARI SECTION comments +# are managed by hakari. + +### BEGIN HAKARI SECTION +[dependencies] +base64 = { version = "0.22" } +chrono = { version = "0.4", features = ["serde"] } +crossbeam-epoch = { version = "0.9" } +crossbeam-utils = { version = "0.8" } +either = { version = "1", features = ["serde", "use_std"] } +event-listener = { version = "5" } +fastrand = { version = "2" } +futures-channel = { version = "0.3", features = ["sink"] } +futures-io = { version = "0.3" } +futures-sink = { version = "0.3" } +futures-util = { version = "0.3", features = ["channel", "io", "sink"] } +hyper = { version = "1", features = ["client", "http1", "http2", "server"] } +libsqlite3-sys = { version = "0.30", default-features = false, features = ["bundled", "pkg-config", "unlock_notify", "vcpkg"] } +log = { version = "0.4", default-features = false, features = ["std"] } +num-traits = { version = "0.2" } +opentelemetry = { version = "0.32" } +opentelemetry-http = { version = "0.32", features = ["reqwest-blocking"] } +opentelemetry_sdk = { version = "0.32", default-features = false, features = ["internal-logs", "logs", "metrics", "rt-tokio", "trace"] } +rand = { version = "0.9" } +regex = { version = "1", default-features = false, features = ["std", "unicode-perl"] } +regex-automata = { version = "0.4", default-features = false, features = ["dfa-build", "meta", "std", "unicode-perl", "unicode-word-boundary"] } +regex-syntax = { version = "0.8", default-features = false, features = ["std", "unicode-perl"] } +reqwest = { version = "0.13", default-features = false, features = ["blocking", "http2", "json", "rustls", "stream"] } +rustls-pki-types = { version = "1", features = ["std"] } +serde = { version = "1", features = ["alloc", "derive", "rc"] } +serde_core = { version = "1", features = ["alloc", "rc"] } +serde_json = { version = "1", features = ["raw_value"] } +sha2 = { version = "0.10" } +slab = { version = "0.4" } +smallvec = { version = "1", default-features = false, features = ["const_new"] } +sqlx-core = { version = "0.9", features = ["_rt-tokio", "any", "chrono", "json", "migrate", "offline"] } +sqlx-sqlite = { version = "0.9", default-features = false, features = ["any", "bundled", "chrono", "deserialize", "json", "load-extension", "migrate", "offline", "unlock-notify"] } +thiserror = { version = "2" } +tokio = { version = "1", features = ["fs", "io-util", "macros", "net", "rt-multi-thread", "signal", "sync", "time"] } +tokio-stream = { version = "0.1", features = ["fs"] } +tokio-util = { version = "0.7", features = ["codec", "io", "rt", "time"] } +tracing-core = { version = "0.1" } +typenum = { version = "1", default-features = false, features = ["const-generics"] } +url = { version = "2", features = ["serde"] } +uuid = { version = "1", features = ["fast-rng", "serde", "v4", "v7"] } +zerocopy = { version = "0.8", default-features = false, features = ["derive", "simd"] } + +[build-dependencies] +base64 = { version = "0.22" } +chrono = { version = "0.4", features = ["serde"] } +crossbeam-utils = { version = "0.8" } +either = { version = "1", features = ["serde", "use_std"] } +event-listener = { version = "5" } +futures-channel = { version = "0.3", features = ["sink"] } +futures-io = { version = "0.3" } +futures-sink = { version = "0.3" } +futures-util = { version = "0.3", features = ["channel", "io", "sink"] } +libsqlite3-sys = { version = "0.30", default-features = false, features = ["bundled", "pkg-config", "unlock_notify", "vcpkg"] } +log = { version = "0.4", default-features = false, features = ["std"] } +num-traits = { version = "0.2" } +serde = { version = "1", features = ["alloc", "derive", "rc"] } +serde_core = { version = "1", features = ["alloc", "rc"] } +serde_json = { version = "1", features = ["raw_value"] } +sha2 = { version = "0.10" } +slab = { version = "0.4" } +smallvec = { version = "1", default-features = false, features = ["const_new"] } +sqlx-core = { version = "0.9", features = ["_rt-tokio", "any", "chrono", "json", "migrate", "offline"] } +sqlx-sqlite = { version = "0.9", default-features = false, features = ["any", "bundled", "chrono", "deserialize", "json", "load-extension", "migrate", "offline", "unlock-notify"] } +syn = { version = "3", features = ["full", "visit-mut"] } +thiserror = { version = "2" } +tokio = { version = "1", features = ["fs", "io-util", "macros", "net", "rt-multi-thread", "signal", "sync", "time"] } +tokio-stream = { version = "0.1", features = ["fs"] } +tracing-core = { version = "0.1" } +typenum = { version = "1", default-features = false, features = ["const-generics"] } +url = { version = "2", features = ["serde"] } + +[target.x86_64-unknown-linux-gnu.dependencies] +bitflags = { version = "2", default-features = false, features = ["std"] } +hyper-util = { version = "0.1", features = ["client-legacy", "client-proxy", "http1", "http2", "server", "service"] } +libc = { version = "0.2", features = ["extra_traits"] } +tower = { version = "0.5", default-features = false, features = ["balance", "buffer", "limit", "load-shed", "log", "retry", "timeout"] } +tower-http = { version = "0.6", features = ["follow-redirect"] } + +[target.x86_64-unknown-linux-gnu.build-dependencies] +cc = { version = "1", default-features = false, features = ["parallel"] } +libc = { version = "0.2", features = ["extra_traits"] } + +### END HAKARI SECTION + +[lints] +workspace = true diff --git a/workspace-hack/build.rs b/workspace-hack/build.rs new file mode 100644 index 00000000..92518ef0 --- /dev/null +++ b/workspace-hack/build.rs @@ -0,0 +1,2 @@ +// A build script is required for cargo to consider build dependencies. +fn main() {} diff --git a/workspace-hack/src/lib.rs b/workspace-hack/src/lib.rs new file mode 100644 index 00000000..22489f63 --- /dev/null +++ b/workspace-hack/src/lib.rs @@ -0,0 +1 @@ +// This is a stub lib.rs. From b49fa148be24239c077dedee18f3086bb6a14ee1 Mon Sep 17 00:00:00 2001 From: Reinier Maas Date: Thu, 23 Jul 2026 17:07:28 +0200 Subject: [PATCH 3/7] Cargo: Move dependencies definitions to workspace --- Cargo.toml | 57 +++++++++++++++++ libs/opsqueue_python/Cargo.toml | 31 +++++----- opsqueue/Cargo.toml | 104 +++++++++++++++----------------- 3 files changed, 121 insertions(+), 71 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 4d9dda73..7aa5d60c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -15,6 +15,63 @@ hakari-package = "workspace-hack" [workspace.package] version = "0.36.0" +[workspace.dependencies] +anyhow = "1.0.102" +arc-swap = "1.9.1" +assert_matches = "1.5.0" +axum = { version = "0.8.9", features = ["ws", "macros"] } +axum-prometheus = "0.10.0" +backon = { version = "1.3.0", features = ["tokio-sleep"] } +ciborium = "0.2.2" +chrono = { version = "0.4.44", features = ["serde"] } +clap = { version = "4.5.60", features = ["derive"] } +criterion = { version = "0.8", features = ["async_tokio"] } +crossbeam-skiplist = "0.1.3" +dashmap = "6.1.0" +either = "1.13.0" +futures = "0.3.32" +http = "1.4.0" +humantime = "2.1.0" +insta = "1.47.2" +itertools = "0.15.0" +moka = { version = "0.12.15", features = ["sync"] } +moro-local = "0.4.0" +object_store = { version = "0.14.0", features = ["gcp", "http"] } +once_cell = "1.21.4" +opentelemetry = { version = "0.32", default-features = false, features = ["trace"] } +opentelemetry-http = "0.32" +opentelemetry-otlp = { version = "0.32", features = ["grpc-tonic"] } +opentelemetry-resource-detectors = "0.11.0" +opentelemetry-semantic-conventions = { version = "0.32.0", features = ["semconv_experimental"] } +opentelemetry_sdk = { version = "0.32", default-features = false, features = ["trace", "rt-tokio"] } +opsqueue = { path = "opsqueue", default-features = false } +pyo3 = { version = "0.28.3", features = ["chrono"] } +pyo3-async-runtimes = { version = "0.28.0", features = ["tokio-runtime", "unstable-streams"] } +pyo3-log = "0.13.3" +rand = "0.10.1" +reqwest = { version = "0.13.2", default-features = false, features = ["json"] } +rustc-hash = "2.1.2" +sentry = { version = "0.48", default-features = false, features = ["rustls", "reqwest", "anyhow", "backtrace", "contexts", "tracing", "panic"] } +sentry-tracing = "0.48" +serde = { version = "1.0.203", features = ["derive"] } +serde-error = "0.1.3" +serde_json = "1.0.149" +snowflaked = { version = "1.0.3", features = ["sync"] } +sqlformat = "0.5.0" +sqlx = { version = "0.9.0", features = ["sqlite", "runtime-tokio", "chrono"] } +thiserror = "2.0.18" +tokio = { version = "1.52.1", features = ["macros", "signal", "rt-multi-thread"] } +tokio-tungstenite = "0.30.0" +tokio-util = { version = "0.7.18", features = ["io", "rt", "time"] } +tower-http = { version = "0.7.0", features = ["trace", "catch-panic"] } +tracing = { version = "0.1.41", features = ["log"] } +tracing-opentelemetry = "0.33.0" +tracing-subscriber = { version = "0.3", features = ["std", "env-filter"] } +url = "2.5.2" +uuid = { version = "1.23.1", features = ["v7", "fast-rng", "serde"] } +ux = "0.1.6" +workspace-hack = { version = "0.1", path = "workspace-hack" } + [workspace.lints.clippy] cargo = { level = "deny", priority = -1 } cargo_common_metadata = "allow" diff --git a/libs/opsqueue_python/Cargo.toml b/libs/opsqueue_python/Cargo.toml index 28335314..b080b1e6 100644 --- a/libs/opsqueue_python/Cargo.toml +++ b/libs/opsqueue_python/Cargo.toml @@ -11,34 +11,31 @@ crate-type = ["cdylib"] [dependencies] # Opsqueue itself: # NOTE: We compile without the default `sqlx` feature, reducing the amount of dependencies -opsqueue = {path = "../../opsqueue/", default-features = false, features = ["client-logic"] } +opsqueue = { workspace = true, features = ["client-logic"] } # General-purpose datatypes: -uuid = {version = "1.23.1", features = [ - "v7", # Temporally orderable UUIDs with random component - "fast-rng", # Use a faster (but still sufficiently random) RNG - ]} -chrono = { version = "0.4.44"} -ux = "0.1.6" +uuid.workspace = true +chrono.workspace = true +ux.workspace = true # Concurrency: -tokio = {version = "1.52", features = ["macros", "rt-multi-thread"]} -futures = "0.3.32" +tokio.workspace = true +futures.workspace = true # Error handling: -anyhow = "1.0.102" -thiserror = "2.0.18" +anyhow.workspace = true +thiserror.workspace = true # Python FFI: -pyo3 = { version = "0.28.3", features = ["chrono"] } -pyo3-async-runtimes = { version = "0.28.0", features = ["tokio-runtime", "unstable-streams"] } -once_cell = "1.21.4" # Only currently used for `unsync::OnceCell` as part of PyO3 async helpers. +pyo3.workspace = true +pyo3-async-runtimes.workspace = true +once_cell.workspace = true # Only currently used for `unsync::OnceCell` as part of PyO3 async helpers. # Logging/tracing: -pyo3-log = "0.13.3" -tracing = { version = "0.1.41", features = ["log"] } +pyo3-log.workspace = true +tracing.workspace = true -workspace-hack = { version = "0.1", path = "../../workspace-hack" } +workspace-hack.workspace = true [lints] workspace = true diff --git a/opsqueue/Cargo.toml b/opsqueue/Cargo.toml index 7cc6c0af..e825613e 100644 --- a/opsqueue/Cargo.toml +++ b/opsqueue/Cargo.toml @@ -17,66 +17,62 @@ required-features = ["server-logic"] [dependencies] # Datatypes and concurrency: -itertools = "0.15.0" -arc-swap = {version = "1.9.1", optional = true} -moka = { version = "0.12.15", features = ["sync"], optional = true } -chrono = { version = "0.4.44", features = ["serde"]} -futures = "0.3.32" -tokio = { version = "1.52.1", features = ["macros", "signal", "rt-multi-thread"] } -uuid = {version = "1.23.1", features = [ - "v7", # Timestamp-sortable UUIDs with random component - "fast-rng", # Use a faster (but still sufficiently random) RNG - "serde", - ]} -ux = "0.1.6" +itertools.workspace = true +arc-swap = { workspace = true, optional = true } +moka = { workspace = true, optional = true } +chrono.workspace = true +futures.workspace = true +tokio.workspace = true +uuid.workspace = true +ux.workspace = true # Error handling: -anyhow = "1.0.102" +anyhow.workspace = true # Database: -sqlx = { version = "0.9.0", features = ["sqlite", "runtime-tokio", "chrono"], optional = true } +sqlx = { workspace = true, optional = true } # Serialization: -serde = { version = "1.0.203", features = ["derive"] } -serde_json = "1.0.149" -ciborium = "0.2.2" +serde.workspace = true +serde_json.workspace = true +ciborium.workspace = true # Webservers/clients -http = "1.4.0" -object_store = {version = "0.14.0", features = ["gcp", "http"]} -snowflaked = {version = "1.0.3", features = ["sync"] } -tokio-tungstenite = {version = "0.30.0", optional = true} -axum = { version = "0.8.9", features = ["ws", "macros"], optional = true } -reqwest = { version = "0.13.2", default-features = false, features = ["json"], optional = true } -url = {version = "2.5.2"} -tokio-util = { version = "0.7.18", features = ["io", "rt", "time"] } -tower-http = { version = "0.7.0", features = ["trace", "catch-panic"], optional = true } +http.workspace = true +object_store.workspace = true +snowflaked.workspace = true +tokio-tungstenite = { workspace = true, optional = true } +axum = { workspace = true, optional = true } +reqwest = { workspace = true, optional = true } +url.workspace = true +tokio-util.workspace = true +tower-http = { workspace = true, optional = true } # Logging and tracing: -tracing = {version = "0.1", features = ["log"] } -tracing-subscriber = {version = "0.3", features = ["std", "env-filter"] } -sentry = {version = "0.48", optional = true, default-features=false, features=["rustls", "reqwest", "anyhow", "backtrace", "contexts", "tracing", "panic"]} -sentry-tracing = {version = "0.48", optional = true} +tracing.workspace = true +tracing-subscriber.workspace = true +sentry = { workspace = true, optional = true } +sentry-tracing = { workspace = true, optional = true } # Exporting traces to Opentelemetry: -opentelemetry = { version = "0.32", default-features = false, features = ["trace"] } -opentelemetry_sdk = { version = "0.32", default-features = false, features = ["trace", "rt-tokio"] } -opentelemetry-resource-detectors = "0.11.0" -opentelemetry-http = { version = "0.32" } -opentelemetry-otlp = { version = "0.32", optional = true, features = ["grpc-tonic"] } -tracing-opentelemetry = "0.33.0" -opentelemetry-semantic-conventions = {version = "0.32.0", features = ["semconv_experimental"], optional = true} -moro-local = "0.4.0" -thiserror = "2.0.18" -either = "1.13.0" -serde-error = "0.1.3" -backon = { version = "1.3.0", features = ["tokio-sleep"] } -rand = "0.10.1" -rustc-hash = "2.1.2" -axum-prometheus = {version = "0.10.0", optional = true} +opentelemetry.workspace = true +opentelemetry_sdk.workspace = true +opentelemetry-resource-detectors.workspace = true +opentelemetry-http.workspace = true +opentelemetry-otlp = { workspace = true, optional = true } +tracing-opentelemetry.workspace = true +opentelemetry-semantic-conventions = { workspace = true, optional = true } +moro-local.workspace = true +thiserror.workspace = true +either.workspace = true +serde-error.workspace = true +backon.workspace = true +rand.workspace = true +rustc-hash.workspace = true +axum-prometheus = { workspace = true, optional = true } # Configuration: -clap = { version = "4.5.60", features = ["derive"] } -humantime = "2.1.0" +clap.workspace = true +humantime.workspace = true -dashmap = "6.1.0" -crossbeam-skiplist = "0.1.3" -sqlformat = "0.5.0" -workspace-hack = { version = "0.1", path = "../workspace-hack" } +dashmap.workspace = true +crossbeam-skiplist.workspace = true +sqlformat.workspace = true +workspace-hack.workspace = true # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html @@ -84,9 +80,9 @@ workspace-hack = { version = "0.1", path = "../workspace-hack" } workspace = true [dev-dependencies] -criterion = {version = "0.8", features = ["async_tokio"]} -insta = { version = "1.47.2" } -assert_matches = { version = "1.5.0" } +criterion.workspace = true +insta.workspace = true +assert_matches.workspace = true # [[bench]] # name = "chunks_select" From 038ecd2223016ecb888e7dac04e8f7b5fd0fd736 Mon Sep 17 00:00:00 2001 From: Reinier Maas Date: Thu, 23 Jul 2026 17:25:35 +0200 Subject: [PATCH 4/7] Cargo: Share edition from workspace --- Cargo.toml | 1 + libs/opsqueue_python/Cargo.toml | 2 +- opsqueue/Cargo.toml | 2 +- 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 7aa5d60c..a56d7f4d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -13,6 +13,7 @@ members = [ hakari-package = "workspace-hack" [workspace.package] +edition = "2024" version = "0.36.0" [workspace.dependencies] diff --git a/libs/opsqueue_python/Cargo.toml b/libs/opsqueue_python/Cargo.toml index b080b1e6..4cd01f26 100644 --- a/libs/opsqueue_python/Cargo.toml +++ b/libs/opsqueue_python/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "opsqueue_python" version.workspace = true -edition = "2024" +edition.workspace = true # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [lib] diff --git a/opsqueue/Cargo.toml b/opsqueue/Cargo.toml index e825613e..0039f700 100644 --- a/opsqueue/Cargo.toml +++ b/opsqueue/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "opsqueue" version.workspace = true -edition = "2024" +edition.workspace = true description = "lightweight batch processing queue for heavy loads" repository = "https://github.com/channable/opsqueue" license = "MIT" From 8416c6df9e49cc2ecd0558ae4312e6cf10ecfcf7 Mon Sep 17 00:00:00 2001 From: Reinier Maas Date: Thu, 23 Jul 2026 17:47:21 +0200 Subject: [PATCH 5/7] Cargo: Remove unused dependencies --- .gitattributes | 3 + Cargo.lock | 169 -------------------------------------- Cargo.toml | 64 +++++++-------- opsqueue/Cargo.toml | 11 --- workspace-hack/Cargo.toml | 5 +- 5 files changed, 37 insertions(+), 215 deletions(-) create mode 100644 .gitattributes diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 00000000..07be7bcf --- /dev/null +++ b/.gitattributes @@ -0,0 +1,3 @@ +# Avoid putting conflict markers in the generated Cargo.toml file, since their presence breaks +# Cargo. +Cargo.lock merge=binary diff --git a/Cargo.lock b/Cargo.lock index f56ae2b7..ef9675ae 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -26,15 +26,6 @@ dependencies = [ "memchr", ] -[[package]] -name = "alloca" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e5a7d05ea6aea7e9e64d25b9156ba2fee3fdd659e34e41063cd2fc7cd020d7f4" -dependencies = [ - "cc", -] - [[package]] name = "allocator-api2" version = "0.2.21" @@ -50,12 +41,6 @@ dependencies = [ "libc", ] -[[package]] -name = "anes" -version = "0.1.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4b46cbb362ab8752921c97e041f5e366ee6297bd428a31275b9fcf1e380f7299" - [[package]] name = "anstream" version = "1.0.0" @@ -121,12 +106,6 @@ dependencies = [ "rustversion", ] -[[package]] -name = "assert_matches" -version = "1.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9b34d609dfbaf33d6889b2b7106d3ca345eacad44200913df5ba02bfd31d2ba9" - [[package]] name = "async-channel" version = "2.5.0" @@ -367,12 +346,6 @@ version = "1.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fc652a48c352aef3ea3aed32080501cf3ef6ed5da78602a020c991775b0aff04" -[[package]] -name = "cast" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5" - [[package]] name = "cc" version = "1.3.0" @@ -595,42 +568,6 @@ version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "217698eaf96b4a3f0bc4f3662aaa55bdf913cd54d7204591faa790070c6d0853" -[[package]] -name = "criterion" -version = "0.8.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "950046b2aa2492f9a536f5f4f9a3de7b9e2476e575e05bd6c333371add4d98f3" -dependencies = [ - "alloca", - "anes", - "cast", - "ciborium", - "clap", - "criterion-plot", - "itertools 0.13.0", - "num-traits", - "oorandom", - "page_size", - "plotters", - "rayon", - "regex", - "serde", - "serde_json", - "tinytemplate", - "tokio", - "walkdir", -] - -[[package]] -name = "criterion-plot" -version = "0.8.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d8d80a2f4f5b554395e47b5d8305bc3d27813bacb73493eb1001e8f76dae29ea" -dependencies = [ - "cast", - "itertools 0.13.0", -] - [[package]] name = "crossbeam-channel" version = "0.5.16" @@ -640,16 +577,6 @@ dependencies = [ "crossbeam-utils", ] -[[package]] -name = "crossbeam-deque" -version = "0.8.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5181e0de7b61eb03a81e347d6dd8797bae9da5146707b51077e2d71a54ec0ceb" -dependencies = [ - "crossbeam-epoch", - "crossbeam-utils", -] - [[package]] name = "crossbeam-epoch" version = "0.9.20" @@ -1516,15 +1443,6 @@ version = "1.70.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a6cb138bb79a146c1bd460005623e142ef0181e3d0219cb493e02f7d08a35695" -[[package]] -name = "itertools" -version = "0.13.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "413ee7dfc52ee1a4949ceeb7dbc8a33f2d6c088194d9f922fb8318faf1f01186" -dependencies = [ - "either", -] - [[package]] name = "itertools" version = "0.14.0" @@ -2082,12 +2000,6 @@ version = "1.70.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe" -[[package]] -name = "oorandom" -version = "11.1.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d6790f58c7ff633d8771f42965289203411a5e5c68388703c06e14f24770b41e" - [[package]] name = "openssl-probe" version = "0.2.1" @@ -2194,14 +2106,12 @@ version = "0.36.0" dependencies = [ "anyhow", "arc-swap", - "assert_matches", "axum", "axum-prometheus", "backon", "chrono", "ciborium", "clap", - "criterion", "crossbeam-skiplist", "dashmap", "either", @@ -2225,7 +2135,6 @@ dependencies = [ "sentry", "sentry-tracing", "serde", - "serde-error", "serde_json", "snowflaked", "sqlformat", @@ -2280,16 +2189,6 @@ dependencies = [ "windows-sys 0.61.2", ] -[[package]] -name = "page_size" -version = "0.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "30d5b2194ed13191c1999ae0704b7839fb18384fa22e49b57eeaa97d79ce40da" -dependencies = [ - "libc", - "winapi", -] - [[package]] name = "parking" version = "2.2.1" @@ -2357,34 +2256,6 @@ version = "0.3.33" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "19f132c84eca552bf34cab8ec81f1c1dcc229b811638f9d283dceabe58c5569e" -[[package]] -name = "plotters" -version = "0.3.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5aeb6f403d7a4911efb1e33402027fc44f29b5bf6def3effcc22d7bb75f2b747" -dependencies = [ - "num-traits", - "plotters-backend", - "plotters-svg", - "wasm-bindgen", - "web-sys", -] - -[[package]] -name = "plotters-backend" -version = "0.3.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "df42e13c12958a16b3f7f4386b9ab1f3e7933914ecea48da7139435263a4172a" - -[[package]] -name = "plotters-svg" -version = "0.3.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "51bae2ac328883f7acdfea3d66a7c35751187f870bc81f94563733a154d7a670" -dependencies = [ - "plotters-backend", -] - [[package]] name = "portable-atomic" version = "1.14.0" @@ -2726,26 +2597,6 @@ dependencies = [ "bitflags", ] -[[package]] -name = "rayon" -version = "1.12.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fb39b166781f92d482534ef4b4b1b2568f42613b53e5b6c160e24cfbfa30926d" -dependencies = [ - "either", - "rayon-core", -] - -[[package]] -name = "rayon-core" -version = "1.13.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "22e18b0f0062d30d4230b2e85ff77fdfe4326feb054b9783a3460d8435c8ab91" -dependencies = [ - "crossbeam-deque", - "crossbeam-utils", -] - [[package]] name = "redox_syscall" version = "0.5.18" @@ -3141,15 +2992,6 @@ dependencies = [ "serde_derive", ] -[[package]] -name = "serde-error" -version = "0.1.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "342110fb7a5d801060c885da03bf91bfa7c7ca936deafcc64bb6706375605d47" -dependencies = [ - "serde", -] - [[package]] name = "serde_core" version = "1.0.229" @@ -3695,16 +3537,6 @@ dependencies = [ "zerovec", ] -[[package]] -name = "tinytemplate" -version = "1.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "be4d6b5f19ff7664e8c98d03e2139cb510db9b0a60b55f8e8709b689d939b6bc" -dependencies = [ - "serde", - "serde_json", -] - [[package]] name = "tinyvec" version = "1.12.0" @@ -4550,7 +4382,6 @@ dependencies = [ "opentelemetry-http", "opentelemetry_sdk", "rand 0.9.5", - "regex", "regex-automata", "regex-syntax", "reqwest", diff --git a/Cargo.toml b/Cargo.toml index a56d7f4d..b45f0577 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -17,60 +17,60 @@ edition = "2024" version = "0.36.0" [workspace.dependencies] -anyhow = "1.0.102" -arc-swap = "1.9.1" -assert_matches = "1.5.0" +anyhow = { version = "1.0.102" } +arc-swap = { version = "1.9.1" } +assert_matches = { version = "1.5.0" } axum = { version = "0.8.9", features = ["ws", "macros"] } -axum-prometheus = "0.10.0" +axum-prometheus = { version = "0.10.0", features = ["prometheus"] } backon = { version = "1.3.0", features = ["tokio-sleep"] } -ciborium = "0.2.2" chrono = { version = "0.4.44", features = ["serde"] } -clap = { version = "4.5.60", features = ["derive"] } +ciborium = { version = "0.2.2", features = ["std"] } +clap = { version = "4.5.60", features = ["derive", "std"] } criterion = { version = "0.8", features = ["async_tokio"] } -crossbeam-skiplist = "0.1.3" -dashmap = "6.1.0" -either = "1.13.0" -futures = "0.3.32" -http = "1.4.0" -humantime = "2.1.0" -insta = "1.47.2" -itertools = "0.15.0" +crossbeam-skiplist = { version = "0.1.3", features = ["std"] } +dashmap = { version = "6.1.0" } +either = { version = "1.13.0" } +futures = { version = "0.3.32" } +http = { version = "1.4.0" } +humantime = { version = "2.1.0" } +insta = { version = "1.47.2" } +itertools = { version = "0.15.0" } moka = { version = "0.12.15", features = ["sync"] } -moro-local = "0.4.0" +moro-local = { version = "0.4.0" } object_store = { version = "0.14.0", features = ["gcp", "http"] } -once_cell = "1.21.4" +once_cell = { version = "1.21.4" } opentelemetry = { version = "0.32", default-features = false, features = ["trace"] } -opentelemetry-http = "0.32" +opentelemetry-http = { version = "0.32" } opentelemetry-otlp = { version = "0.32", features = ["grpc-tonic"] } -opentelemetry-resource-detectors = "0.11.0" +opentelemetry-resource-detectors = { version = "0.11.0" } opentelemetry-semantic-conventions = { version = "0.32.0", features = ["semconv_experimental"] } opentelemetry_sdk = { version = "0.32", default-features = false, features = ["trace", "rt-tokio"] } opsqueue = { path = "opsqueue", default-features = false } pyo3 = { version = "0.28.3", features = ["chrono"] } pyo3-async-runtimes = { version = "0.28.0", features = ["tokio-runtime", "unstable-streams"] } -pyo3-log = "0.13.3" -rand = "0.10.1" +pyo3-log = { version = "0.13.3" } +rand = { version = "0.10.1" } reqwest = { version = "0.13.2", default-features = false, features = ["json"] } -rustc-hash = "2.1.2" +rustc-hash = { version = "2.1.2" } sentry = { version = "0.48", default-features = false, features = ["rustls", "reqwest", "anyhow", "backtrace", "contexts", "tracing", "panic"] } -sentry-tracing = "0.48" +sentry-tracing = { version = "0.48" } serde = { version = "1.0.203", features = ["derive"] } -serde-error = "0.1.3" -serde_json = "1.0.149" +serde-error = { version = "0.1.3" } +serde_json = { version = "1.0.149" } snowflaked = { version = "1.0.3", features = ["sync"] } -sqlformat = "0.5.0" -sqlx = { version = "0.9.0", features = ["sqlite", "runtime-tokio", "chrono"] } -thiserror = "2.0.18" +sqlformat = { version = "0.5.0" } +sqlx = { version = "0.9.0", features = ["sqlite", "runtime-tokio", "chrono", "migrate", "macros", "derive", "json"] } +thiserror = { version = "2.0.18" } tokio = { version = "1.52.1", features = ["macros", "signal", "rt-multi-thread"] } -tokio-tungstenite = "0.30.0" +tokio-tungstenite = { version = "0.30.0", features = ["connect", "stream"] } tokio-util = { version = "0.7.18", features = ["io", "rt", "time"] } tower-http = { version = "0.7.0", features = ["trace", "catch-panic"] } tracing = { version = "0.1.41", features = ["log"] } -tracing-opentelemetry = "0.33.0" -tracing-subscriber = { version = "0.3", features = ["std", "env-filter"] } -url = "2.5.2" +tracing-opentelemetry = { version = "0.33.0" } +tracing-subscriber = { version = "0.3", features = ["std", "env-filter", "fmt"] } +url = { version = "2.5.2" } uuid = { version = "1.23.1", features = ["v7", "fast-rng", "serde"] } -ux = "0.1.6" +ux = { version = "0.1.6" } workspace-hack = { version = "0.1", path = "workspace-hack" } [workspace.lints.clippy] diff --git a/opsqueue/Cargo.toml b/opsqueue/Cargo.toml index 0039f700..310eb645 100644 --- a/opsqueue/Cargo.toml +++ b/opsqueue/Cargo.toml @@ -59,7 +59,6 @@ opentelemetry-semantic-conventions = { workspace = true, optional = true } moro-local.workspace = true thiserror.workspace = true either.workspace = true -serde-error.workspace = true backon.workspace = true rand.workspace = true rustc-hash.workspace = true @@ -80,17 +79,7 @@ workspace-hack.workspace = true workspace = true [dev-dependencies] -criterion.workspace = true insta.workspace = true -assert_matches.workspace = true - -# [[bench]] -# name = "chunks_select" -# harness = false - -# [[bench]] -# name = "submissions_insert" -# harness = false [features] # Dependencies only in use by the server-logic: diff --git a/workspace-hack/Cargo.toml b/workspace-hack/Cargo.toml index 543f78d4..a930c5f6 100644 --- a/workspace-hack/Cargo.toml +++ b/workspace-hack/Cargo.toml @@ -29,12 +29,11 @@ futures-util = { version = "0.3", features = ["channel", "io", "sink"] } hyper = { version = "1", features = ["client", "http1", "http2", "server"] } libsqlite3-sys = { version = "0.30", default-features = false, features = ["bundled", "pkg-config", "unlock_notify", "vcpkg"] } log = { version = "0.4", default-features = false, features = ["std"] } -num-traits = { version = "0.2" } +num-traits = { version = "0.2", default-features = false, features = ["std"] } opentelemetry = { version = "0.32" } opentelemetry-http = { version = "0.32", features = ["reqwest-blocking"] } opentelemetry_sdk = { version = "0.32", default-features = false, features = ["internal-logs", "logs", "metrics", "rt-tokio", "trace"] } rand = { version = "0.9" } -regex = { version = "1", default-features = false, features = ["std", "unicode-perl"] } regex-automata = { version = "0.4", default-features = false, features = ["dfa-build", "meta", "std", "unicode-perl", "unicode-word-boundary"] } regex-syntax = { version = "0.8", default-features = false, features = ["std", "unicode-perl"] } reqwest = { version = "0.13", default-features = false, features = ["blocking", "http2", "json", "rustls", "stream"] } @@ -69,7 +68,7 @@ futures-sink = { version = "0.3" } futures-util = { version = "0.3", features = ["channel", "io", "sink"] } libsqlite3-sys = { version = "0.30", default-features = false, features = ["bundled", "pkg-config", "unlock_notify", "vcpkg"] } log = { version = "0.4", default-features = false, features = ["std"] } -num-traits = { version = "0.2" } +num-traits = { version = "0.2", default-features = false, features = ["std"] } serde = { version = "1", features = ["alloc", "derive", "rc"] } serde_core = { version = "1", features = ["alloc", "rc"] } serde_json = { version = "1", features = ["raw_value"] } From fbb75f3c11dde786418ef362fe9d0dd5c4d5c308 Mon Sep 17 00:00:00 2001 From: Reinier Maas Date: Thu, 23 Jul 2026 18:45:12 +0200 Subject: [PATCH 6/7] Nix: Prebuilt `sqlite` --- .github/workflows/ci.yml | 10 +- Cargo.lock | 3 + Cargo.toml | 2 +- default.nix | 8 + libs/opsqueue_python/opsqueue_python.nix | 10 +- opsqueue/opsqueue.nix | 8 +- opsqueue/src/consumer/strategy.rs | 250 ++++++++++++----------- workspace-hack/Cargo.toml | 7 +- 8 files changed, 165 insertions(+), 133 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1073282e..ef0db1a1 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -6,15 +6,7 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@0c366fd6a839edf440554fa01a7085ccba70ac98 - - # c.f. https://github.com/extractions/setup-just/issues/23 - - uses: extractions/setup-crate@4993624604c307fbca528d28a3c8b60fa5ecc859 # v1.4.0 - with: - repo: casey/just - version: 1.42.4 - - uses: moonrepo/setup-rust@ede6de059f8046a5e236c94046823e2af11ca670 # v1.2.2 - with: - bins: cargo-nextest, cargo-insta@1.28.0 + - uses: ./.github/actions/setup-nix-direnv-rust - run: just test-unit test-integration: name: Integration test (& build all in dev-mode) diff --git a/Cargo.lock b/Cargo.lock index ef9675ae..668797cd 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3234,6 +3234,7 @@ dependencies = [ "log", "memchr", "percent-encoding", + "rustls", "serde", "serde_json", "sha2 0.10.9", @@ -3243,6 +3244,7 @@ dependencies = [ "tokio-stream", "tracing", "url", + "webpki-roots", ] [[package]] @@ -4359,6 +4361,7 @@ checksum = "1ebf944e87a7c253233ad6766e082e3cd714b5d03812acc24c318f549614536e" name = "workspace-hack" version = "0.1.0" dependencies = [ + "aws-lc-rs", "base64", "bitflags", "cc", diff --git a/Cargo.toml b/Cargo.toml index b45f0577..4baa1496 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -59,7 +59,7 @@ serde-error = { version = "0.1.3" } serde_json = { version = "1.0.149" } snowflaked = { version = "1.0.3", features = ["sync"] } sqlformat = { version = "0.5.0" } -sqlx = { version = "0.9.0", features = ["sqlite", "runtime-tokio", "chrono", "migrate", "macros", "derive", "json"] } +sqlx = { version = "0.9.0", features = ["chrono", "derive", "json", "macros", "migrate", "runtime-tokio", "sqlite", "tls-rustls-aws-lc-rs"] } thiserror = { version = "2.0.18" } tokio = { version = "1.52.1", features = ["macros", "signal", "rt-multi-thread"] } tokio-tungstenite = { version = "0.30.0", features = ["connect", "stream"] } diff --git a/default.nix b/default.nix index f33a05cc..ef1e7aed 100644 --- a/default.nix +++ b/default.nix @@ -52,6 +52,8 @@ let pkgs.cargo-nextest pkgs.maturin + # Resolve native sqlite from Nix for libsqlite3-sys + pkgs.pkg-config # sqlite3 binary, for easy debugging/introspection pkgs.sqlite ]; @@ -64,6 +66,12 @@ let # For the shell, libpython needs to be in the search path. pythonEnv ]; + shellHook = '' + # We need Cargo to re-run if the sqlite3 dev package changes, so we set this env var to point to it. + export LIBSQLITE3_SYS_USE_PKG_CONFIG="${pkgs.sqlite.dev}" + # This is needed for the libsqlite3-sys crate to find the correct sqlite3. + export PKG_CONFIG_PATH="${pkgs.sqlite.dev}/lib/pkgconfig''${PKG_CONFIG_PATH:+:''${PKG_CONFIG_PATH}}" + ''; }; }; in diff --git a/libs/opsqueue_python/opsqueue_python.nix b/libs/opsqueue_python/opsqueue_python.nix index 57a3e89f..3645f38e 100644 --- a/libs/opsqueue_python/opsqueue_python.nix +++ b/libs/opsqueue_python/opsqueue_python.nix @@ -32,6 +32,7 @@ let sources = import ../../nix/sources.nix; crane = import sources.crane { pkgs = pkgs; }; craneLib = crane.overrideToolchain (pkgs: rustToolchain); + sqlitePkgConfigPath = lib.makeSearchPathOutput "dev" "lib/pkgconfig" [ pkgs.sqlite ]; # Only the files necessary to build the Rust-side and cache dependencies sqlFileFilter = path: _type: builtins.match "^.*\.(db|sql)$" path != null; @@ -59,7 +60,11 @@ let inherit version pname; src = depsSrc; strictDeps = true; - nativeBuildInputs = [ python ]; + nativeBuildInputs = [ + python + pkgs.pkg-config + ]; + buildInputs = [ pkgs.sqlite ]; cargoExtraArgs = "--package opsqueue_python"; doCheck = false; }; @@ -77,6 +82,9 @@ let // { inherit cargoArtifacts; src = wheelSrc; + env.DATABASE_URL = "sqlite://${depsSrc}/opsqueue/opsqueue_example_database_schema.db"; + env.LIBSQLITE3_SYS_USE_PKG_CONFIG = "${pkgs.sqlite.dev}"; + env.PKG_CONFIG_PATH = sqlitePkgConfigPath; } )).overrideAttrs (old: { diff --git a/opsqueue/opsqueue.nix b/opsqueue/opsqueue.nix index 95e8ac77..c63c5610 100644 --- a/opsqueue/opsqueue.nix +++ b/opsqueue/opsqueue.nix @@ -16,6 +16,7 @@ let sources = import ../nix/sources.nix; crane = import sources.crane { pkgs = pkgs; }; craneLib = crane.overrideToolchain (pkgs: rustToolchain); + sqlitePkgConfigPath = lib.makeSearchPathOutput "dev" "lib/pkgconfig" [ pkgs.sqlite ]; # Only the files necessary to build the Rust-side and cache dependencies sqlFileFilter = path: _type: builtins.match "^.*\.(db|sql)$" path != null; @@ -36,7 +37,8 @@ let commonArgs = { inherit src version pname; strictDeps = true; - nativeBuildInputs = [ ]; + nativeBuildInputs = [ pkgs.pkg-config ]; + buildInputs = [ pkgs.sqlite ]; cargoExtraArgs = "--package opsqueue"; doCheck = true; }; @@ -52,7 +54,9 @@ craneLib.buildPackage ( # Needed for the SQLx macros: env = { - DATABASE_URL = "sqlite:///build/opsqueue/opsqueue/opsqueue_example_database_schema.db"; + DATABASE_URL = "sqlite://${src}/opsqueue/opsqueue_example_database_schema.db"; + LIBSQLITE3_SYS_USE_PKG_CONFIG = "${pkgs.sqlite.dev}"; + PKG_CONFIG_PATH = sqlitePkgConfigPath; }; } diff --git a/opsqueue/src/consumer/strategy.rs b/opsqueue/src/consumer/strategy.rs index 17906e89..a725d138 100644 --- a/opsqueue/src/consumer/strategy.rs +++ b/opsqueue/src/consumer/strategy.rs @@ -309,13 +309,13 @@ pub mod test { 8, 2, CORRELATED SCALAR SUBQUERY 4 12, 8, SEARCH submissions_metadata USING PRIMARY KEY (submission_id=? AND metadata_key=?) 22, 8, LIST SUBQUERY 2 - 24, 22, SCAN json_each VIRTUAL TABLE INDEX 0: - 58, 1, UNION ALL - 61, 58, SCAN chunks - 64, 58, CORRELATED SCALAR SUBQUERY 6 - 68, 64, SEARCH submissions_metadata USING PRIMARY KEY (submission_id=? AND metadata_key=?) - 78, 64, LIST SUBQUERY 2 - 80, 78, SCAN json_each VIRTUAL TABLE INDEX 0: + 25, 22, SCAN json_each VIRTUAL TABLE INDEX 0: + 33, 22, CREATE BLOOM FILTER + 62, 1, UNION ALL + 65, 62, SCAN chunks + 68, 62, CORRELATED SCALAR SUBQUERY 6 + 72, 68, SEARCH submissions_metadata USING PRIMARY KEY (submission_id=? AND metadata_key=?) + 80, 68, REUSE LIST SUBQUERY 2 "); } @@ -399,13 +399,13 @@ pub mod test { 8, 2, CORRELATED SCALAR SUBQUERY 4 12, 8, SEARCH submissions_metadata USING PRIMARY KEY (submission_id=? AND metadata_key=?) 22, 8, LIST SUBQUERY 2 - 24, 22, SCAN json_each VIRTUAL TABLE INDEX 0: - 58, 1, UNION ALL - 61, 58, SCAN chunks - 64, 58, CORRELATED SCALAR SUBQUERY 6 - 68, 64, SEARCH submissions_metadata USING PRIMARY KEY (submission_id=? AND metadata_key=?) - 78, 64, LIST SUBQUERY 2 - 80, 78, SCAN json_each VIRTUAL TABLE INDEX 0: + 25, 22, SCAN json_each VIRTUAL TABLE INDEX 0: + 33, 22, CREATE BLOOM FILTER + 62, 1, UNION ALL + 65, 62, SCAN chunks + 68, 62, CORRELATED SCALAR SUBQUERY 6 + 72, 68, SEARCH submissions_metadata USING PRIMARY KEY (submission_id=? AND metadata_key=?) + 80, 68, REUSE LIST SUBQUERY 2 "); } @@ -498,27 +498,25 @@ pub mod test { 16, 4, CORRELATED SCALAR SUBQUERY 5 20, 16, SEARCH submissions_metadata USING PRIMARY KEY (submission_id=? AND metadata_key=?) 30, 16, LIST SUBQUERY 3 - 32, 30, SCAN json_each VIRTUAL TABLE INDEX 0: - 58, 3, UNION ALL - 61, 58, SEARCH chunks USING INDEX random_chunks_order (random_order?) - 127, 115, CORRELATED SCALAR SUBQUERY 7 - 131, 127, SEARCH submissions_metadata USING PRIMARY KEY (submission_id=? AND metadata_key=?) - 141, 127, LIST SUBQUERY 3 - 143, 141, SCAN json_each VIRTUAL TABLE INDEX 0: - 169, 114, UNION ALL - 172, 169, SEARCH chunks USING INDEX random_chunks_order (random_order?) + 121, 109, CORRELATED SCALAR SUBQUERY 7 + 125, 121, SEARCH submissions_metadata USING PRIMARY KEY (submission_id=? AND metadata_key=?) + 133, 121, REUSE LIST SUBQUERY 3 + 153, 108, UNION ALL + 156, 153, SEARCH chunks USING INDEX random_chunks_order (random_order?) - 197, 185, CORRELATED SCALAR SUBQUERY 7 - 201, 197, SEARCH submissions_metadata USING PRIMARY KEY (submission_id=? AND metadata_key=?) - 211, 197, LIST SUBQUERY 3 - 213, 211, SCAN json_each VIRTUAL TABLE INDEX 0: - 231, 185, CORRELATED SCALAR SUBQUERY 11 - 235, 231, SEARCH submissions_metadata USING PRIMARY KEY (submission_id=? AND metadata_key=?) - 245, 231, LIST SUBQUERY 9 - 247, 245, SCAN json_each VIRTUAL TABLE INDEX 0: - 273, 184, UNION ALL - 276, 273, SEARCH chunks USING INDEX random_chunks_order (random_order?) - 378, 366, CORRELATED SCALAR SUBQUERY 5 - 382, 378, SEARCH submissions_metadata USING PRIMARY KEY (submission_id=? AND metadata_key=?) - 392, 378, LIST SUBQUERY 3 - 394, 392, SCAN json_each VIRTUAL TABLE INDEX 0: - 412, 366, CORRELATED SCALAR SUBQUERY 13 - 416, 412, SEARCH submissions_metadata USING PRIMARY KEY (submission_id=? AND metadata_key=?) - 426, 412, LIST SUBQUERY 9 - 428, 426, SCAN json_each VIRTUAL TABLE INDEX 0: - 454, 365, UNION ALL - 457, 454, SEARCH chunks USING INDEX random_chunks_order (random_order?) - 557, 545, CORRELATED SCALAR SUBQUERY 7 - 561, 557, SEARCH submissions_metadata USING PRIMARY KEY (submission_id=? AND metadata_key=?) - 571, 557, LIST SUBQUERY 3 - 573, 571, SCAN json_each VIRTUAL TABLE INDEX 0: - 591, 545, CORRELATED SCALAR SUBQUERY 13 - 595, 591, SEARCH submissions_metadata USING PRIMARY KEY (submission_id=? AND metadata_key=?) - 605, 591, LIST SUBQUERY 9 - 607, 605, SCAN json_each VIRTUAL TABLE INDEX 0: - 633, 544, UNION ALL - 636, 633, SEARCH chunks USING INDEX random_chunks_order (random_order?) + 213, 201, CORRELATED SCALAR SUBQUERY 7 + 217, 213, SEARCH submissions_metadata USING PRIMARY KEY (submission_id=? AND metadata_key=?) + 227, 213, LIST SUBQUERY 3 + 230, 227, SCAN json_each VIRTUAL TABLE INDEX 0: + 238, 227, CREATE BLOOM FILTER + 251, 201, CORRELATED SCALAR SUBQUERY 11 + 255, 251, SEARCH submissions_metadata USING PRIMARY KEY (submission_id=? AND metadata_key=?) + 265, 251, LIST SUBQUERY 9 + 268, 265, SCAN json_each VIRTUAL TABLE INDEX 0: + 276, 265, CREATE BLOOM FILTER + 297, 200, UNION ALL + 300, 297, SEARCH chunks USING INDEX random_chunks_order (random_order?) + 410, 398, CORRELATED SCALAR SUBQUERY 5 + 414, 410, SEARCH submissions_metadata USING PRIMARY KEY (submission_id=? AND metadata_key=?) + 424, 410, LIST SUBQUERY 3 + 427, 424, SCAN json_each VIRTUAL TABLE INDEX 0: + 435, 424, CREATE BLOOM FILTER + 448, 398, CORRELATED SCALAR SUBQUERY 13 + 452, 448, SEARCH submissions_metadata USING PRIMARY KEY (submission_id=? AND metadata_key=?) + 462, 448, LIST SUBQUERY 9 + 465, 462, SCAN json_each VIRTUAL TABLE INDEX 0: + 473, 462, CREATE BLOOM FILTER + 494, 397, UNION ALL + 497, 494, SEARCH chunks USING INDEX random_chunks_order (random_order?) + 605, 593, CORRELATED SCALAR SUBQUERY 7 + 609, 605, SEARCH submissions_metadata USING PRIMARY KEY (submission_id=? AND metadata_key=?) + 619, 605, LIST SUBQUERY 3 + 622, 619, SCAN json_each VIRTUAL TABLE INDEX 0: + 630, 619, CREATE BLOOM FILTER + 643, 593, CORRELATED SCALAR SUBQUERY 13 + 647, 643, SEARCH submissions_metadata USING PRIMARY KEY (submission_id=? AND metadata_key=?) + 657, 643, LIST SUBQUERY 9 + 660, 657, SCAN json_each VIRTUAL TABLE INDEX 0: + 668, 657, CREATE BLOOM FILTER + 689, 592, UNION ALL + 692, 689, SEARCH chunks USING INDEX random_chunks_order (random_order Date: Fri, 24 Jul 2026 13:04:01 +0200 Subject: [PATCH 7/7] Clippy: Add `itertools` --- clippy.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/clippy.toml b/clippy.toml index 274bb7be..cb9f6293 100644 --- a/clippy.toml +++ b/clippy.toml @@ -5,6 +5,7 @@ allowed-duplicate-crates = [ "digest", "getrandom", "hashbrown", + "itertools", "r-efi", "rand", "rand_core",