Skip to content

armgit / macro

Public Imported from GitHub
0

Macro is a unified workspace for teams: email, chat, docs, tasks, agents, calls, and CRM — @-linked together with shared AI memory.

fix(kafka): use vendored ssl to fix lambda build failures (#4574)

Will Hutchinsonf6ade2c26ad1
diff --git a/nix/cloud-storage.nix b/nix/cloud-storage.nix
index 8cec1a0ed1..eb9aeaf1e1 100644
--- a/nix/cloud-storage.nix
+++ b/nix/cloud-storage.nix
@@ -524,6 +524,24 @@
# zig links the final binary (cargo-zigbuild); drop the host-only mold arg.
RUSTFLAGS = "";
CARGO_PROFILE = "release";
+ # cargo-zigbuild compiles C with zig cc, which ignores the Nix
+ # cc-wrapper's NIX_CFLAGS_COMPILE — so the curl.dev headers that
+ # buildInputs provides to host builds (see the note on `libraries`;
+ # librdkafka 2.12 needs curl headers even with WITH_CURL=0) are
+ # invisible here. Pass the include path explicitly; cc-rs and
+ # cmake-rs both honor CFLAGS/CXXFLAGS.
+ CFLAGS = "-I${pkgs.curl.dev}/include";
+ CXXFLAGS = "-I${pkgs.curl.dev}/include";
+ # commonArgs pins OPENSSL_NO_VENDOR=1 so host builds use the Nix
+ # openssl, and openssl-sys honors that env var even when the crate's
+ # `vendored` feature (rdkafka `ssl-vendored`) is enabled. Lambdas
+ # can't use the Nix openssl: its shared libs need glibc >= 2.38
+ # symbols while zig links against the pinned Lambda glibc, and the
+ # /nix/store rpath doesn't exist in the Lambda runtime anyway. Flip
+ # the override off so openssl-src compiles OpenSSL with zig cc
+ # against the pinned glibc and links it statically — same idea as
+ # AWS_LC_SYS_CMAKE_BUILDER above.
+ OPENSSL_NO_VENDOR = "0";
# Lambdas don't need max opt; matches the cargo-lambda CI setting.
CARGO_PROFILE_RELEASE_OPT_LEVEL = "2";
# aws-lc-sys (aws-lc-rs / rustls default crypto, pulled via aws-sdk &
@@ -536,6 +554,9 @@
pkgs.zig
pkgs.cmake
pkgs.nasm
+ # openssl-src (vendored OpenSSL, see OPENSSL_NO_VENDOR below) runs
+ # OpenSSL's perl-based Configure.
+ pkgs.perl
];
# zig needs a writable cache inside the sandbox ($HOME is read-only).
preBuild = ''
@@ -789,6 +810,11 @@
PKG_CONFIG_PATH_FOR_TARGET = pkgConfigPath;
LIBCLANG_PATH = "${pkgs.libclang.lib}/lib";
RUSTC_WRAPPER = "${pkgs.sccache}/bin/sccache";
+ # rdkafka's ssl-vendored feature exists for the Lambda
+ # derivations (see lambdaCommonArgs); local builds keep using
+ # the Nix openssl instead of vendoring, which would need perl
+ # on PATH and a from-source OpenSSL compile.
+ OPENSSL_NO_VENDOR = "1";
# Keep local cargo-lambda builds on the same aws-lc-sys path as
# the Nix lambda derivations. The default cc builder rejects
# cargo-lambda/cargo-zigbuild's Zig cc wrapper.
diff --git a/rust/cloud-storage/Cargo.lock b/rust/cloud-storage/Cargo.lock
index 2e685f510e..64f163193a 100644
--- a/rust/cloud-storage/Cargo.lock
+++ b/rust/cloud-storage/Cargo.lock
@@ -9558,6 +9558,15 @@ version = "0.2.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7c87def4c32ab89d880effc9e097653c8da5d6ef28e6b539d313baaacfbafcbe"
+[[package]]
+name = "openssl-src"
+version = "300.6.1+3.6.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "46eb8fb9fb3b61ce1c0f8a026c4c1a0714d3a9e138e7fbde78753ce2babc3846"
+dependencies = [
+ "cc",
+]
+
[[package]]
name = "openssl-sys"
version = "0.9.112"
@@ -9566,6 +9575,7 @@ checksum = "57d55af3b3e226502be1526dfdba67ab0e9c96fc293004e79576b2b9edb0dbdb"
dependencies = [
"cc",
"libc",
+ "openssl-src",
"pkg-config",
"vcpkg",
]
diff --git a/rust/cloud-storage/Cargo.toml b/rust/cloud-storage/Cargo.toml
index e5052da00b..9e794207ea 100644
--- a/rust/cloud-storage/Cargo.toml
+++ b/rust/cloud-storage/Cargo.toml
@@ -80,13 +80,13 @@ apollo-compiler = "1"
apollo-parser = "0.8"
askama = "0.14"
async-graphql = "7.2.1"
+async-lock = "3"
async-openai = { version = "0.36.1", features = [
"byot",
"chat-completion",
"completions",
"embedding",
] }
-async-lock = "3"
async-stream = "0.3.6"
async-trait = "0.1.86"
automod = "1"
@@ -134,7 +134,6 @@ jsonwebtoken = { git = "https://github.com/whutchinson98/jsonwebtoken", features
"rust_crypto",
] }
lambda_runtime = "0.13.0"
-lru = "0.16"
lazy_static = "1.5.0"
livekit-api = { version = "0.4.16", default-features = false, features = [
"access-token",
@@ -145,6 +144,7 @@ livekit-api = { version = "0.4.16", default-features = false, features = [
livekit-protocol = "0.7.2"
log = "0.4.26"
lol_html = "2.4.0"
+lru = "0.16"
mockall = "0.13"
nom = "8"
opentelemetry = "0.31.0"
@@ -163,12 +163,14 @@ pdfium-render = { version = "0.8.37", default-features = false, features = [
"thread_safe",
] }
pgvector = { version = "0.4", features = ["postgres", "sqlx"] }
+postcard = { version = "1", features = ["use-std"] }
quote = "1"
rand = "0.9.0"
+# uses vendored to fix lambda builds failing.
rdkafka = { version = "0.37", default-features = false, features = [
"cmake-build",
"libz-static",
- "ssl",
+ "ssl-vendored",
"tokio",
] }
recursion = "0.5.4"
@@ -184,11 +186,10 @@ rmcp = "1.6.0"
rootcause = { version = "0.12" }
schemars = { version = "1.0.4", features = ["chrono04", "uuid1"] }
semver = { version = "1", features = ["serde"] }
-postcard = { version = "1", features = ["use-std"] }
serde = { version = "1.0.214", features = ["derive"] }
+serde-wasm-bindgen = "0.6"
serde_dynamo = { version = "4", features = ["aws-sdk-dynamodb+1"] }
serde_json = { version = "1.0.132" }
-serde-wasm-bindgen = "0.6"
serde_repr = "0.1"
serde_with = { version = "3.12.0", features = ["base64"] }
serde_yaml = "0.9"
@@ -202,8 +203,6 @@ sqlx = { version = "0.8.6", features = [
sqlx-core = { version = "0.8.6" }
strum = { version = "0.27.1", features = ["derive"] }
thiserror = "2.0.0"
-wasm-bindgen = "0.2"
-wasm-bindgen-futures = "0.4"
time = "0.3.37"
tokio = { version = "1.43.0", features = ["macros", "rt-multi-thread"] }
tokio-stream = { version = "0.1.17" }
@@ -233,6 +232,8 @@ utoipa = { version = "5.4.0", features = [
] }
utoipa-swagger-ui = { version = "9", features = ["axum", "vendored"] }
uuid = { version = "1.15.1", features = ["fast-rng", "serde", "v4", "v7"] }
+wasm-bindgen = "0.2"
+wasm-bindgen-futures = "0.4"
# this is pinned to match axum 0.7.9 dependency requirements
async-stripe = { version = "0.41", default-features = false, features = [
"billing",