Skip to content

armgit / macro

Public Imported from GitHub
0

Macro is a unified workspace for teams: email, chat, docs, tasks, agents, calls, and CRM — @-linked together with shared AI memory.

fix(kafka): use statically linked kafka (#4560)

Will Hutchinson0d170c7e8154
diff --git a/nix/cloud-storage.nix b/nix/cloud-storage.nix
index 6ae2400541..937ec00d95 100644
--- a/nix/cloud-storage.nix
+++ b/nix/cloud-storage.nix
@@ -28,21 +28,21 @@
[
openssl
openssl.dev
- rdkafka
- rdkafka.dev
glib
glib.dev
libclang
xz.out
]
- ++ pkgs.lib.optionals isLinux [
- glibc.dev
- gcc
- ]
++ pkgs.lib.optionals isDarwin [
libiconv
];
+ # Do not add glibc.dev directly to buildInputs/libraries on Linux. The
+ # GCC wrapper already adds glibc headers as an -idirafter path; adding
+ # glibc.dev through the setup hook turns it into an early -isystem path,
+ # which breaks libstdc++'s #include_next <stdlib.h> while building C++
+ # sources such as bundled librdkafka.
+
# Include Cargo sources plus the .sqlx offline query cache.
sqlxFilter = path: _type: builtins.match ".*\\.sqlx/.*\\.json$" path != null;
pdfiumFilter = path: _type: builtins.match ".*pdfium-lib/.*\\.(so|dylib)$" path != null;
@@ -154,6 +154,7 @@
nativeBuildInputs =
with pkgs;
[
+ cmake
git
pkg-config
]
@@ -769,7 +770,6 @@
let
pkgConfigPath = pkgs.lib.makeSearchPath "lib/pkgconfig" [
pkgs.openssl.dev
- pkgs.rdkafka.dev
];
in
pkgs.mkShell (
@@ -778,7 +778,8 @@
PKG_CONFIG_PATH = pkgConfigPath;
# Cargo build scripts use the Nix pkg-config wrapper, which prefers
# the target-specific search path when PKG_CONFIG_PATH_FOR_TARGET is set.
- # Keep librdkafka visible there so rdkafka-sys can find rdkafka.pc.
+ # librdkafka is built from the bundled sources via rdkafka's
+ # cmake-build feature, so no system librdkafka.pc is needed here.
PKG_CONFIG_PATH_FOR_TARGET = pkgConfigPath;
LIBCLANG_PATH = "${pkgs.libclang.lib}/lib";
RUSTC_WRAPPER = "${pkgs.sccache}/bin/sccache";
diff --git a/rust/cloud-storage/Cargo.lock b/rust/cloud-storage/Cargo.lock
index b3d7bd7eb1..b1c4f588f8 100644
--- a/rust/cloud-storage/Cargo.lock
+++ b/rust/cloud-storage/Cargo.lock
@@ -7446,6 +7446,18 @@ dependencies = [
"version_check",
]
+[[package]]
+name = "libz-sys"
+version = "1.1.29"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "85bc9657773828b90eeb625adff10eeac83cc21bbfd8e23a03eaa8a33c9e28d9"
+dependencies = [
+ "cc",
+ "libc",
+ "pkg-config",
+ "vcpkg",
+]
+
[[package]]
name = "linux-raw-sys"
version = "0.4.15"
@@ -9544,6 +9556,18 @@ version = "0.2.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7c87def4c32ab89d880effc9e097653c8da5d6ef28e6b539d313baaacfbafcbe"
+[[package]]
+name = "openssl-sys"
+version = "0.9.112"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "57d55af3b3e226502be1526dfdba67ab0e9c96fc293004e79576b2b9edb0dbdb"
+dependencies = [
+ "cc",
+ "libc",
+ "pkg-config",
+ "vcpkg",
+]
+
[[package]]
name = "opentelemetry"
version = "0.31.0"
@@ -10947,8 +10971,11 @@ version = "4.10.0+2.12.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e234cf318915c1059d4921ef7f75616b5219b10b46e9f3a511a15eb4b56a3f77"
dependencies = [
+ "cmake",
"libc",
+ "libz-sys",
"num_enum",
+ "openssl-sys",
"pkg-config",
]
diff --git a/rust/cloud-storage/Cargo.toml b/rust/cloud-storage/Cargo.toml
index ae843d964b..e5052da00b 100644
--- a/rust/cloud-storage/Cargo.toml
+++ b/rust/cloud-storage/Cargo.toml
@@ -166,7 +166,9 @@ pgvector = { version = "0.4", features = ["postgres", "sqlx"] }
quote = "1"
rand = "0.9.0"
rdkafka = { version = "0.37", default-features = false, features = [
- "dynamic-linking",
+ "cmake-build",
+ "libz-static",
+ "ssl",
"tokio",
] }
recursion = "0.5.4"
diff --git a/rust/cloud-storage/tools/xtask/crates/xtask_local/Cargo.toml b/rust/cloud-storage/tools/xtask/crates/xtask_local/Cargo.toml
index afbc0b9337..d57830f923 100644
--- a/rust/cloud-storage/tools/xtask/crates/xtask_local/Cargo.toml
+++ b/rust/cloud-storage/tools/xtask/crates/xtask_local/Cargo.toml
@@ -26,9 +26,8 @@ tokio = { workspace = true }
# Source of truth for the declared Kafka topics (local-stack provisioning).
macro_event_topics = { path = "../../../../macro_event_topics" }
# AdminClient for creating the declared topics on the local broker. Optional
-# (behind `local-stack`): the workspace links librdkafka dynamically, and the
-# CI jobs that build this on bare runners have no system librdkafka to link
-# against.
+# (behind `local-stack`) so plain xtask builds don't spend time compiling the
+# bundled static librdkafka dependency.
rdkafka = { workspace = true, optional = true }
# Typed docker-compose model (build the override as structs, not strings) +
# Docker Engine API client for the discrete container/image ops.
@@ -48,6 +47,6 @@ xtask_paths = { path = "../xtask_paths" }
[features]
# Local Kafka topic provisioning (`run-local` / `kafka-provision`). Off by
-# default so a plain build works without a system librdkafka; the just recipes
+# default so a plain build avoids compiling bundled librdkafka; the just recipes
# enable it.
local-stack = ["dep:rdkafka"]
diff --git a/rust/cloud-storage/tools/xtask/crates/xtask_local/src/local/kafka.rs b/rust/cloud-storage/tools/xtask/crates/xtask_local/src/local/kafka.rs
index c1ca75b12e..2dd5f766f9 100644
--- a/rust/cloud-storage/tools/xtask/crates/xtask_local/src/local/kafka.rs
+++ b/rust/cloud-storage/tools/xtask/crates/xtask_local/src/local/kafka.rs
@@ -5,10 +5,11 @@
//! `.github/kafka-cluster-topics.json`) — so a topic declared in the crate
//! exists both locally and in the deployed cluster without a second list.
//!
-//! Gated behind the `local-stack` feature: rdkafka links a system librdkafka,
-//! which the bare CI runners that build xtask don't have. The just recipes
-//! enable the feature; without it the provisioning entry points fail with
-//! instructions instead of the whole binary failing to build.
+//! Gated behind the `local-stack` feature: rdkafka builds and statically links
+//! bundled librdkafka, which is unnecessary for plain xtask builds. The just
+//! recipes enable the feature; without it the provisioning entry points fail
+//! with instructions instead of the whole binary spending time on native Kafka
+//! dependencies.
#[cfg(feature = "local-stack")]
use std::time::Duration;