Macro is a unified workspace for teams: email, chat, docs, tasks, agents, calls, and CRM — @-linked together with shared AI memory.
TypeScript agentaiai-agentsall-in-onecrmcrm-systememaillinearmcpmessagingnotesnotionofficerustslackslack-alternativestartupstartup-toolstypescriptworkspace
fix(kafka): use statically linked kafka (#4560)
Will Hutchinson0d170c7e8154
diff --git a/nix/cloud-storage.nix b/nix/cloud-storage.nixindex 6ae2400541..937ec00d95 100644--- a/nix/cloud-storage.nix+++ b/nix/cloud-storage.nix@@ -28,21 +28,21 @@[opensslopenssl.dev- rdkafka- rdkafka.devglibglib.devlibclangxz.out]- ++ pkgs.lib.optionals isLinux [- glibc.dev- gcc- ]++ pkgs.lib.optionals isDarwin [libiconv];+ # Do not add glibc.dev directly to buildInputs/libraries on Linux. The+ # GCC wrapper already adds glibc headers as an -idirafter path; adding+ # glibc.dev through the setup hook turns it into an early -isystem path,+ # which breaks libstdc++'s #include_next <stdlib.h> while building C+++ # sources such as bundled librdkafka.+# Include Cargo sources plus the .sqlx offline query cache.sqlxFilter = path: _type: builtins.match ".*\\.sqlx/.*\\.json$" path != null;pdfiumFilter = path: _type: builtins.match ".*pdfium-lib/.*\\.(so|dylib)$" path != null;@@ -154,6 +154,7 @@nativeBuildInputs =with pkgs;[+ cmakegitpkg-config]@@ -769,7 +770,6 @@letpkgConfigPath = pkgs.lib.makeSearchPath "lib/pkgconfig" [pkgs.openssl.dev- pkgs.rdkafka.dev];inpkgs.mkShell (@@ -778,7 +778,8 @@PKG_CONFIG_PATH = pkgConfigPath;# Cargo build scripts use the Nix pkg-config wrapper, which prefers# the target-specific search path when PKG_CONFIG_PATH_FOR_TARGET is set.- # Keep librdkafka visible there so rdkafka-sys can find rdkafka.pc.+ # librdkafka is built from the bundled sources via rdkafka's+ # cmake-build feature, so no system librdkafka.pc is needed here.PKG_CONFIG_PATH_FOR_TARGET = pkgConfigPath;LIBCLANG_PATH = "${pkgs.libclang.lib}/lib";RUSTC_WRAPPER = "${pkgs.sccache}/bin/sccache";diff --git a/rust/cloud-storage/Cargo.lock b/rust/cloud-storage/Cargo.lockindex b3d7bd7eb1..b1c4f588f8 100644--- a/rust/cloud-storage/Cargo.lock+++ b/rust/cloud-storage/Cargo.lock@@ -7446,6 +7446,18 @@ dependencies = ["version_check",]+[[package]]+name = "libz-sys"+version = "1.1.29"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "85bc9657773828b90eeb625adff10eeac83cc21bbfd8e23a03eaa8a33c9e28d9"+dependencies = [+ "cc",+ "libc",+ "pkg-config",+ "vcpkg",+]+[[package]]name = "linux-raw-sys"version = "0.4.15"@@ -9544,6 +9556,18 @@ version = "0.2.1"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "7c87def4c32ab89d880effc9e097653c8da5d6ef28e6b539d313baaacfbafcbe"+[[package]]+name = "openssl-sys"+version = "0.9.112"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "57d55af3b3e226502be1526dfdba67ab0e9c96fc293004e79576b2b9edb0dbdb"+dependencies = [+ "cc",+ "libc",+ "pkg-config",+ "vcpkg",+]+[[package]]name = "opentelemetry"version = "0.31.0"@@ -10947,8 +10971,11 @@ version = "4.10.0+2.12.1"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "e234cf318915c1059d4921ef7f75616b5219b10b46e9f3a511a15eb4b56a3f77"dependencies = [+ "cmake","libc",+ "libz-sys","num_enum",+ "openssl-sys","pkg-config",]diff --git a/rust/cloud-storage/Cargo.toml b/rust/cloud-storage/Cargo.tomlindex ae843d964b..e5052da00b 100644--- a/rust/cloud-storage/Cargo.toml+++ b/rust/cloud-storage/Cargo.toml@@ -166,7 +166,9 @@ pgvector = { version = "0.4", features = ["postgres", "sqlx"] }quote = "1"rand = "0.9.0"rdkafka = { version = "0.37", default-features = false, features = [- "dynamic-linking",+ "cmake-build",+ "libz-static",+ "ssl","tokio",] }recursion = "0.5.4"diff --git a/rust/cloud-storage/tools/xtask/crates/xtask_local/Cargo.toml b/rust/cloud-storage/tools/xtask/crates/xtask_local/Cargo.tomlindex afbc0b9337..d57830f923 100644--- a/rust/cloud-storage/tools/xtask/crates/xtask_local/Cargo.toml+++ b/rust/cloud-storage/tools/xtask/crates/xtask_local/Cargo.toml@@ -26,9 +26,8 @@ tokio = { workspace = true }# Source of truth for the declared Kafka topics (local-stack provisioning).macro_event_topics = { path = "../../../../macro_event_topics" }# AdminClient for creating the declared topics on the local broker. Optional-# (behind `local-stack`): the workspace links librdkafka dynamically, and the-# CI jobs that build this on bare runners have no system librdkafka to link-# against.+# (behind `local-stack`) so plain xtask builds don't spend time compiling the+# bundled static librdkafka dependency.rdkafka = { workspace = true, optional = true }# Typed docker-compose model (build the override as structs, not strings) +# Docker Engine API client for the discrete container/image ops.@@ -48,6 +47,6 @@ xtask_paths = { path = "../xtask_paths" }[features]# Local Kafka topic provisioning (`run-local` / `kafka-provision`). Off by-# default so a plain build works without a system librdkafka; the just recipes+# default so a plain build avoids compiling bundled librdkafka; the just recipes# enable it.local-stack = ["dep:rdkafka"]diff --git a/rust/cloud-storage/tools/xtask/crates/xtask_local/src/local/kafka.rs b/rust/cloud-storage/tools/xtask/crates/xtask_local/src/local/kafka.rsindex c1ca75b12e..2dd5f766f9 100644--- a/rust/cloud-storage/tools/xtask/crates/xtask_local/src/local/kafka.rs+++ b/rust/cloud-storage/tools/xtask/crates/xtask_local/src/local/kafka.rs@@ -5,10 +5,11 @@//! `.github/kafka-cluster-topics.json`) — so a topic declared in the crate//! exists both locally and in the deployed cluster without a second list.//!-//! Gated behind the `local-stack` feature: rdkafka links a system librdkafka,-//! which the bare CI runners that build xtask don't have. The just recipes-//! enable the feature; without it the provisioning entry points fail with-//! instructions instead of the whole binary failing to build.+//! Gated behind the `local-stack` feature: rdkafka builds and statically links+//! bundled librdkafka, which is unnecessary for plain xtask builds. The just+//! recipes enable the feature; without it the provisioning entry points fail+//! with instructions instead of the whole binary spending time on native Kafka+//! dependencies.#[cfg(feature = "local-stack")]use std::time::Duration;