Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

feat: encode v4 native blocks and event bytes in Rust (#17563)

Valery Meleshkinb0608e3a14ed
diff --git a/.github/workflows/pipeline.yml b/.github/workflows/pipeline.yml
index 1f376b4e41..a74c8d83ef 100644
--- a/.github/workflows/pipeline.yml
+++ b/.github/workflows/pipeline.yml
@@ -1002,6 +1002,13 @@ jobs:
pnpm --filter=shared ch:dev-tables
- name: Build
run: pnpm --filter=worker... run build
+ - name: run native codec tests against ClickHouse
+ if: matrix.deploy-mode == ''
+ env:
+ LANGFUSE_NATIVE_FAIL_IF_CLICKHOUSE_UNAVAILABLE: "1"
+ run: |
+ pnpm --filter @langfuse/native run test
+ pnpm --filter worker run test:native-codec
- name: run tests
run: pnpm --filter=worker run test:exclude-llm-connections
env:
diff --git a/packages/native/.gitignore b/packages/native/.gitignore
index 3304cd5642..0202152254 100644
--- a/packages/native/.gitignore
+++ b/packages/native/.gitignore
@@ -1,5 +1,8 @@
# Cargo build output
target/
+# Generated Cargo overlay for the pinned clickhouse source patch
+.cargo/
+
# Platform-specific binaries produced by `napi build`
*.node
diff --git a/packages/native/Cargo.lock b/packages/native/Cargo.lock
index 02b45de9bf..fe9600dfac 100644
--- a/packages/native/Cargo.lock
+++ b/packages/native/Cargo.lock
@@ -11,6 +11,24 @@ dependencies = [
"memchr",
]
+[[package]]
+name = "allocator-api2"
+version = "0.2.21"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "683d7910e743518b0e34f1186f92494becacb047c7b6bf616c96772180fef923"
+
+[[package]]
+name = "arrayvec"
+version = "0.7.8"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d3fb67a6e08acf24fdeccbac2cb6ac4305825bd1f117462e0e6f2f193345ad56"
+
+[[package]]
+name = "atomic-waker"
+version = "1.1.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0"
+
[[package]]
name = "autocfg"
version = "1.5.1"
@@ -23,6 +41,45 @@ version = "2.13.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b588b76d00fde79687d7646a9b5bdf3cc0f655e0bbd080335a95d7e96f3587da"
+[[package]]
+name = "bnum"
+version = "0.13.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "119771309b95163ec7aaf79810da82f7cd0599c19722d48b9c03894dca833966"
+
+[[package]]
+name = "borsh"
+version = "1.8.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "553c5d846a6ba5150c65e3b1b8ec073bcf1abc20f9b7220de384a4443ea4e20a"
+dependencies = [
+ "borsh-derive",
+ "bytes",
+ "cfg_aliases",
+]
+
+[[package]]
+name = "borsh-derive"
+version = "1.8.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "12cdfe656708a01f89b451a7d36466e6fe6c414de0aa18fc54f864f6f9ca9f56"
+dependencies = [
+ "once_cell",
+ "proc-macro-crate",
+ "proc-macro2",
+ "quote",
+ "syn 3.0.4",
+]
+
+[[package]]
+name = "bstr"
+version = "1.13.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6bb31b46c14244e20ee9984b11bf5c992b91fb6939fea616e3512c8baecdbe5f"
+dependencies = [
+ "memchr",
+]
+
[[package]]
name = "bumpalo"
version = "3.20.3"
@@ -34,6 +91,9 @@ name = "bytes"
version = "1.12.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fc652a48c352aef3ea3aed32080501cf3ef6ed5da78602a020c991775b0aff04"
+dependencies = [
+ "serde",
+]
[[package]]
name = "cfg-if"
@@ -41,6 +101,67 @@ version = "1.0.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801"
+[[package]]
+name = "cfg_aliases"
+version = "0.2.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f079e83a288787bcd14a6aea84cee5c87a67c5a3e660c30f557a3d24761b3527"
+
+[[package]]
+name = "chrono"
+version = "0.4.45"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1aa79e62e7697b8e29b513a68abacf485adcd1fe8284a4316c5ae868e6633327"
+dependencies = [
+ "num-traits",
+]
+
+[[package]]
+name = "clickhouse"
+version = "0.15.2"
+dependencies = [
+ "bnum",
+ "bstr",
+ "bytes",
+ "clickhouse-macros",
+ "clickhouse-types",
+ "futures-channel",
+ "futures-util",
+ "half",
+ "hashbrown 0.17.1",
+ "http-body-util",
+ "hyper",
+ "hyper-util",
+ "serde",
+ "serde_json",
+ "thiserror",
+ "tokio",
+ "tracing",
+ "url",
+]
+
+[[package]]
+name = "clickhouse-macros"
+version = "0.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ff6669899e23cb87b43daf7996f0ea3b9c07d0fb933d745bb7b815b052515ae3"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "serde_derive_internals",
+ "syn 2.0.119",
+]
+
+[[package]]
+name = "clickhouse-types"
+version = "0.1.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3966eea13bfc826e24d471e49cb2095727261009be59d0ac5a6c0be0286974d1"
+dependencies = [
+ "bytes",
+ "thiserror",
+]
+
[[package]]
name = "convert_case"
version = "0.11.0"
@@ -65,12 +186,29 @@ version = "0.8.22"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "61803da095bee82a81bb1a452ecc25d3b2f1416d1897eb86430c6159ef717c17"
+[[package]]
+name = "crunchy"
+version = "0.2.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "460fbee9c2c2f33933d720630a6a0bac33ba7053db5344fac858d4b8952d77d5"
+
[[package]]
name = "ctor"
version = "1.0.13"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "914a755b7c2d4af2bdcff7ce1739e2db9a1b81a9b07123d8015786ae03c0980d"
+[[package]]
+name = "displaydoc"
+version = "0.2.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c6232dd377dcc64799954cbd3a9bb882e9cdc1308ccd87b1c098f1fb2eaf82a8"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn 3.0.4",
+]
+
[[package]]
name = "endian-type"
version = "0.1.2"
@@ -89,6 +227,15 @@ version = "0.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "77ce24cb58228fbb8aa041425bb1050850ac19177686ea6e0f41a70416f56fdb"
+[[package]]
+name = "form_urlencoded"
+version = "1.2.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "cb4cb245038516f5f85277875cdaa4f7d2c9a0fa0468de06ed190163b1581fcf"
+dependencies = [
+ "percent-encoding",
+]
+
[[package]]
name = "futures"
version = "0.3.34"
@@ -177,6 +324,17 @@ dependencies = [
"slab",
]
+[[package]]
+name = "getrandom"
+version = "0.2.17"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ff2abc00be7fca6ebc474524697ae276ad847ad0a6b3faa4bcb027e9a4614ad0"
+dependencies = [
+ "cfg-if",
+ "libc",
+ "wasi",
+]
+
[[package]]
name = "getrandom"
version = "0.3.4"
@@ -189,6 +347,17 @@ dependencies = [
"wasip2",
]
+[[package]]
+name = "half"
+version = "2.7.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6ea2d84b969582b4b1864a92dc5d27cd2b77b622a8d79306834f1be5ba20d84b"
+dependencies = [
+ "cfg-if",
+ "crunchy",
+ "zerocopy",
+]
+
[[package]]
name = "hashbrown"
version = "0.16.1"
@@ -203,6 +372,194 @@ name = "hashbrown"
version = "0.17.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a"
+dependencies = [
+ "allocator-api2",
+ "equivalent",
+ "foldhash",
+]
+
+[[package]]
+name = "http"
+version = "1.5.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "918d3568bebf352712bc2ef3d46a8bcf1a75b373be6539de198e9105cbbf9ce0"
+dependencies = [
+ "bytes",
+ "itoa",
+]
+
+[[package]]
+name = "http-body"
+version = "1.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ca2a8f2913ee65f60facd6a5905613afaa448497a0230cc41ce022d93290bc2c"
+dependencies = [
+ "bytes",
+ "http",
+]
+
+[[package]]
+name = "http-body-util"
+version = "0.1.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "23169fe34a5fbcdd3f3862e78fb9b6fccd5f02a6dc6f732547005d45631ce71c"
+dependencies = [
+ "bytes",
+ "futures-core",
+ "http",
+ "http-body",
+ "pin-project-lite",
+]
+
+[[package]]
+name = "httparse"
+version = "1.10.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6dbf3de79e51f3d586ab4cb9d5c3e2c14aa28ed23d180cf89b4df0454a69cc87"
+
+[[package]]
+name = "hyper"
+version = "1.11.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "27b501faa50e7a26c3d3560ca625132f4078a17771f4810baf70475ae48cbe43"
+dependencies = [
+ "atomic-waker",
+ "bytes",
+ "futures-channel",
+ "futures-core",
+ "http",
+ "http-body",
+ "httparse",
+ "itoa",
+ "pin-project-lite",
+ "smallvec",
+ "tokio",
+ "want",
+]
+
+[[package]]
+name = "hyper-util"
+version = "0.1.20"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "96547c2556ec9d12fb1578c4eaf448b04993e7fb79cbaad930a656880a6bdfa0"
+dependencies = [
+ "bytes",
+ "futures-channel",
+ "futures-util",
+ "http",
+ "http-body",
+ "hyper",
+ "libc",
+ "pin-project-lite",
+ "socket2",
+ "tokio",
+ "tower-service",
+ "tracing",
+]
+
+[[package]]
+name = "icu_collections"
+version = "2.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "fa68d21081c4a05d5a901a1c62add574c77048b6a1c67be3b50ce0b60d4ca513"
+dependencies = [
+ "displaydoc",
+ "potential_utf",
+ "utf8_iter",
+ "yoke",
+ "zerofrom",
+ "zerovec",
+]
+
+[[package]]
+name = "icu_locale_core"
+version = "2.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d56e28588da92eee5c3201a6eff33fabdd49b62269c8938d4ff050ce4d900deb"
+dependencies = [
+ "displaydoc",
+ "litemap",
+ "tinystr",
+ "writeable",
+ "zerovec",
+]
+
+[[package]]
+name = "icu_normalizer"
+version = "2.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "12f9cf5f235641ed274641dd81c3f28d870e276763d0797aeeab72317b1c646f"
+dependencies = [
+ "icu_collections",
+ "icu_normalizer_data",
+ "icu_properties",
+ "icu_provider",
+ "smallvec",
+ "zerovec",
+]
+
+[[package]]
+name = "icu_normalizer_data"
+version = "2.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1563da1ed3e0b3bf3d74c9b85917ac9c56464d2f57242270c09c9e752f8021a0"
+
+[[package]]
+name = "icu_properties"
+version = "2.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7e7ca276ad3145661a65914e6daf131ca5120cd3dcee8f8f3214b8875184a148"
+dependencies = [
+ "displaydoc",
+ "icu_collections",
+ "icu_locale_core",
+ "icu_properties_data",
+ "icu_provider",
+ "zerotrie",
+ "zerovec",
+]
+
+[[package]]
+name = "icu_properties_data"
+version = "2.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e590f038c1464a96894fd6d10127e90a8be4509f56ff7ecef851b15cee0b7caa"
+
+[[package]]
+name = "icu_provider"
+version = "2.3.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d27bbb9d3abbefac45d55f647c9de1d44aafcd1186eb91879afef17c396c3e73"
+dependencies = [
+ "displaydoc",
+ "icu_locale_core",
+ "writeable",
+ "yoke",
+ "zerofrom",
+ "zerotrie",
+ "zerovec",
+]
+
+[[package]]
+name = "idna"
+version = "1.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3b0875f23caa03898994f6ddc501886a45c7d3d62d04d2d90788d47be1b1e4de"
+dependencies = [
+ "idna_adapter",
+ "smallvec",
+ "utf8_iter",
+]
+
+[[package]]
+name = "idna_adapter"
+version = "1.2.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "cb68373c0d6620ef8105e855e7745e18b0d00d3bdb07fb532e434244cdb9a714"
+dependencies = [
+ "icu_normalizer",
+ "icu_properties",
+]
[[package]]
name = "indexmap"
@@ -235,11 +592,19 @@ dependencies = [
name = "langfuse-native"
version = "0.0.0"
dependencies = [
+ "bytes",
+ "chrono",
+ "clickhouse",
+ "clickhouse-types",
"metrics",
"metrics-exporter-dogstatsd",
"napi",
"napi-build",
"napi-derive",
+ "proptest",
+ "rust_decimal",
+ "serde",
+ "serde_json",
"tracing",
"tracing-subscriber",
]
@@ -266,6 +631,12 @@ dependencies = [
"windows-link",
]
+[[package]]
+name = "litemap"
+version = "0.8.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "47d9d19d1d6efa0109d2f65ff4c85cddd50bd572e5a00127ab10987290bcefae"
+
[[package]]
name = "log"
version = "0.4.34"
@@ -327,12 +698,23 @@ dependencies = [
"ordered-float",
"quanta",
"radix_trie",
- "rand",
+ "rand 0.9.5",
"rand_xoshiro",
"rapidhash",
"sketches-ddsketch",
]
+[[package]]
+name = "mio"
+version = "1.2.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4b18443e9c262bfe8fa82f51666e2642c53393f7e5c27b3e1aeab922cff5b9d8"
+dependencies = [
+ "libc",
+ "wasi",
+ "windows-sys",
+]
+
[[package]]
name = "napi"
version = "3.12.2"
@@ -439,6 +821,12 @@ dependencies = [
"num-traits",
]
+[[package]]
+name = "percent-encoding"
+version = "2.3.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220"
+
[[package]]
name = "pin-project-lite"
version = "0.2.17"
@@ -451,6 +839,15 @@ version = "1.15.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "05c8b63e8d9609db387f0324918f81d68fe27748f084ef092fb35954d0539a85"
+[[package]]
+name = "potential_utf"
+version = "0.1.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d83eb9bc6d8e5cf568e7a1101d60ee05e81ed50ea106026f3d18deeb046d7661"
+dependencies = [
+ "zerovec",
+]
+
[[package]]
name = "ppv-lite86"
version = "0.2.21"
@@ -460,6 +857,15 @@ dependencies = [
"zerocopy",
]
+[[package]]
+name = "proc-macro-crate"
+version = "3.5.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e67ba7e9b2b56446f1d419b1d807906278ffa1a658a8a5d8a39dcb1f5a78614f"
+dependencies = [
+ "toml_edit",
+]
+
[[package]]
name = "proc-macro2"
version = "1.0.107"
@@ -469,6 +875,21 @@ dependencies = [
"unicode-ident",
]
+[[package]]
+name = "proptest"
+version = "1.11.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4b45fcc2344c680f5025fe57779faef368840d0bd1f42f216291f0dc4ace4744"
+dependencies = [
+ "bitflags",
+ "num-traits",
+ "rand 0.9.5",
+ "rand_chacha 0.9.0",
+ "rand_xorshift",
+ "regex-syntax",
+ "unarray",
+]
+
[[package]]
name = "quanta"
version = "0.12.6"
@@ -509,14 +930,35 @@ dependencies = [
"nibble_vec",
]
+[[package]]
+name = "rand"
+version = "0.8.8"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e058c7de0b26af77780c769414d6257830bb240f3c38477dbc2c16e5f54d6d4c"
+dependencies = [
+ "libc",
+ "rand_chacha 0.3.1",
+ "rand_core 0.6.4",
+]
+
[[package]]
name = "rand"
version = "0.9.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b9ef1d0d795eb7d84685bca4f72f3649f064e6641543d3a8c415898726a57b41"
dependencies = [
- "rand_chacha",
- "rand_core",
+ "rand_chacha 0.9.0",
+ "rand_core 0.9.5",
+]
+
+[[package]]
+name = "rand_chacha"
+version = "0.3.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88"
+dependencies = [
+ "ppv-lite86",
+ "rand_core 0.6.4",
]
[[package]]
@@ -526,7 +968,16 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb"
dependencies = [
"ppv-lite86",
- "rand_core",
+ "rand_core 0.9.5",
+]
+
+[[package]]
+name = "rand_core"
+version = "0.6.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c"
+dependencies = [
+ "getrandom 0.2.17",
]
[[package]]
@@ -535,7 +986,16 @@ version = "0.9.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "76afc826de14238e6e8c374ddcc1fa19e374fd8dd986b0d2af0d02377261d83c"
dependencies = [
- "getrandom",
+ "getrandom 0.3.4",
+]
+
+[[package]]
+name = "rand_xorshift"
+version = "0.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "513962919efc330f829edb2535844d1b912b0fbe2ca165d613e4e8788bb05a5a"
+dependencies = [
+ "rand_core 0.9.5",
]
[[package]]
@@ -544,7 +1004,7 @@ version = "0.7.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f703f4665700daf5512dcca5f43afa6af89f09db47fb56be587f80636bda2d41"
dependencies = [
- "rand_core",
+ "rand_core 0.9.5",
]
[[package]]
@@ -582,6 +1042,23 @@ version = "0.8.11"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4"
+[[package]]
+name = "rust_decimal"
+version = "1.43.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7653272e75dcac41dc199fbea6f5797633994fafd339943c06c9af16bf29cd3a"
+dependencies = [
+ "arrayvec",
+ "borsh",
+ "bytes",
+ "num-traits",
+ "rand 0.8.8",
+ "rand 0.9.5",
+ "serde",
+ "serde_json",
+ "wasm-bindgen",
+]
+
[[package]]
name = "rustc-hash"
version = "2.1.3"
@@ -613,6 +1090,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4148590afebada386688f18773da617792bf2ef03ffc1e4cbd2b1d45b023e0ba"
dependencies = [
"serde_core",
+ "serde_derive",
]
[[package]]
@@ -635,12 +1113,24 @@ dependencies = [
"syn 3.0.4",
]
+[[package]]
+name = "serde_derive_internals"
+version = "0.29.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "18d26a20a969b9e3fdf2fc2d9f21eda6c40e2de84c9408bb5d3b05d499aae711"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn 2.0.119",
+]
+
[[package]]
name = "serde_json"
version = "1.0.151"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c841b55ecdae098c80dcae9cf767f6f8a0c2cdb3416bbef72181df4d0fe73f14"
dependencies = [
+ "indexmap",
"itoa",
"memchr",
"serde",
@@ -675,6 +1165,22 @@ version = "1.16.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b9be42f50aa861c555654aa3a37f52f4b1074bacf4e48fe0ef7fa584e80f1f0f"
+[[package]]
+name = "socket2"
+version = "0.6.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c3d1e2c7f27f8d4cb10542a02c49005dbd6e93095799d6f3be745fae9f8fedd4"
+dependencies = [
+ "libc",
+ "windows-sys",
+]
+
+[[package]]
+name = "stable_deref_trait"
+version = "1.2.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596"
+
[[package]]
name = "syn"
version = "2.0.119"
@@ -697,6 +1203,17 @@ dependencies = [
"unicode-ident",
]
+[[package]]
+name = "synstructure"
+version = "0.14.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "901704edd0dfe137f1987838ee4f259e4e063c31371bdb423f7ae38ec6f77f02"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn 3.0.4",
+]
+
[[package]]
name = "thiserror"
version = "2.0.20"
@@ -726,6 +1243,77 @@ dependencies = [
"cfg-if",
]
+[[package]]
+name = "tinystr"
+version = "0.8.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b1e27c91459209c2986af3dcf603a5a74a4368754ce37414f59acc971167f643"
+dependencies = [
+ "displaydoc",
+ "zerovec",
+]
+
+[[package]]
+name = "tokio"
+version = "1.53.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "202caea871b69668250d242070849eb495be178ed697a3e98aebce5bc81a0bed"
+dependencies = [
+ "libc",
+ "mio",
+ "pin-project-lite",
+ "socket2",
+ "tokio-macros",
+ "windows-sys",
+]
+
+[[package]]
+name = "tokio-macros"
+version = "2.7.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "78773a2a397f451582ce068015985c33193cf6dea8b74d2a639fe457b2f07b0e"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn 3.0.4",
+]
+
+[[package]]
+name = "toml_datetime"
+version = "1.1.1+spec-1.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3165f65f62e28e0115a00b2ebdd37eb6f3b641855f9d636d3cd4103767159ad7"
+dependencies = [
+ "serde_core",
+]
+
+[[package]]
+name = "toml_edit"
+version = "0.25.15+spec-1.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1340ea94a5856333492c9064b02c778b191dd2c853778d9609debdcdfea3a614"
+dependencies = [
+ "indexmap",
+ "toml_datetime",
+ "toml_parser",
+ "winnow",
+]
+
+[[package]]
+name = "toml_parser"
+version = "1.1.3+spec-1.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1d38ac1cf9b95face32296c0a3ede1fdc270627c9d9c02a7274dd6d960dc4d56"
+dependencies = [
+ "winnow",
+]
+
+[[package]]
+name = "tower-service"
+version = "0.3.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8df9b6e13f2d32c91b9bd719c00d1958837bc7dec474d94952798cc8e69eeec3"
+
[[package]]
name = "tracing"
version = "0.1.44"
@@ -800,6 +1388,18 @@ dependencies = [
"tracing-serde",
]
+[[package]]
+name = "try-lock"
+version = "0.2.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b"
+
+[[package]]
+name = "unarray"
+version = "0.1.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "eaea85b334db583fe3274d12b4cd1880032beab409c0d774be044d4480ab9a94"
+
[[package]]
name = "unicode-ident"
version = "1.0.24"
@@ -812,12 +1412,39 @@ version = "1.13.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c6f5d3c3b1bf09027a88a6bc961fc00497d651009560b5463668dc81b0fa87a8"
+[[package]]
+name = "url"
+version = "2.5.8"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ff67a8a4397373c3ef660812acab3268222035010ab8680ec4215f38ba3d0eed"
+dependencies = [
+ "form_urlencoded",
+ "idna",
+ "percent-encoding",
+ "serde",
+]
+
+[[package]]
+name = "utf8_iter"
+version = "1.0.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be"
+
[[package]]
name = "valuable"
version = "0.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ba73ea9cf16a25df0c8caa16c51acb937d5712a8429db78a3ee29d5dcacd3a65"
+[[package]]
+name = "want"
+version = "0.3.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "bfa7760aed19e106de2c7c0b581b509f2f25d3dacaf737cb82ac61bc6d760b0e"
+dependencies = [
+ "try-lock",
+]
+
[[package]]
name = "wasi"
version = "0.11.1+wasi-snapshot-preview1"
@@ -925,12 +1552,50 @@ dependencies = [
"windows-link",
]
+[[package]]
+name = "winnow"
+version = "1.0.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "23b97319f7b8343df12cc98938e5c3eb436064524c8d2b4e30a1d3a36eecdf81"
+dependencies = [
+ "memchr",
+]
+
[[package]]
name = "wit-bindgen"
version = "0.57.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1ebf944e87a7c253233ad6766e082e3cd714b5d03812acc24c318f549614536e"
+[[package]]
+name = "writeable"
+version = "0.6.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3ad82d2a33cdc9674dc7465672f271e096168fcdbe0f799d9e6db8c5892679dc"
+
+[[package]]
+name = "yoke"
+version = "0.8.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "709fe23a0424b6a435d82152b1bd3fdfb0833487d5fa90d05d42762a9891fef5"
+dependencies = [
+ "stable_deref_trait",
+ "yoke-derive",
+ "zerofrom",
+]
+
+[[package]]
+name = "yoke-derive"
+version = "0.8.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "33811428bee40dbceb6d545e95754741d17a6aef9a4849f0fd62e2ba4f412a78"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn 3.0.4",
+ "synstructure",
+]
+
[[package]]
name = "zerocopy"
version = "0.8.56"
@@ -951,6 +1616,60 @@ dependencies = [
"syn 2.0.119",
]
+[[package]]
+name = "zerofrom"
+version = "0.1.8"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0ec05a11813ea801ff6d75110ad09cd0824ddba17dfe17128ea0d5f68e6c5272"
+dependencies = [
+ "zerofrom-derive",
+]
+
+[[package]]
+name = "zerofrom-derive"
+version = "0.1.8"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f75b4683f6c7f45248d4d64056a24298c6281e0993356d7d1b4a1a962ef10d4a"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn 3.0.4",
+ "synstructure",
+]
+
+[[package]]
+name = "zerotrie"
+version = "0.2.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4ea269c3bd32f0a32c321907a2ae912ba6f4649bb0fc764a15627e99a7095a3f"
+dependencies = [
+ "displaydoc",
+ "yoke",
+ "zerofrom",
+]
+
+[[package]]
+name = "zerovec"
+version = "0.11.8"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "bb0464e17806c1d976d5cba29399c7f08e516e279e2ba493f63123b5fca67dd8"
+dependencies = [
+ "yoke",
+ "zerofrom",
+ "zerovec-derive",
+]
+
+[[package]]
+name = "zerovec-derive"
+version = "0.11.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "34df6fc39dbd26ddc9c10e6a2984476e13acce22e64e4487636ef494369225da"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn 3.0.4",
+]
+
[[package]]
name = "zmij"
version = "1.0.23"
diff --git a/packages/native/Cargo.toml b/packages/native/Cargo.toml
index 7e0dedf621..5ca234f133 100644
--- a/packages/native/Cargo.toml
+++ b/packages/native/Cargo.toml
@@ -10,14 +10,28 @@ publish = false
[lib]
crate-type = ["cdylib"]
+[[example]]
+name = "native_codec_test_driver"
+test = false
+
[dependencies]
+bytes = "1"
+chrono = { version = "0.4", default-features = false, features = ["alloc", "std"] }
+clickhouse = { version = "=0.15.2", default-features = false }
+clickhouse-types = "=0.1.3"
metrics = "0.24"
metrics-exporter-dogstatsd = "0.9"
napi = { version = "3", default-features = false, features = ["napi9"] }
napi-derive = "3"
+rust_decimal = { version = "1.37.2", default-features = false, features = ["std"] }
+serde = "1"
+serde_json = { version = "1", features = ["preserve_order", "float_roundtrip"] }
tracing = "0.1"
tracing-subscriber = { version = "0.3", features = ["env-filter", "json"] }
+[dev-dependencies]
+proptest = { version = "1", default-features = false, features = ["std"] }
+
[build-dependencies]
napi-build = "2"
diff --git a/packages/native/README.md b/packages/native/README.md
index 80b09f0f8d..fa3d092575 100644
--- a/packages/native/README.md
+++ b/packages/native/README.md
@@ -8,10 +8,6 @@ other workspace package:
import { hello } from "@langfuse/native";
```
-Today the addon exports a single hello-world function. It exists to put the
-build, packaging, and deployment pipeline for native code in place; real
-functionality lands on top of it.
-
## Layout
| Path | Purpose |
@@ -29,6 +25,12 @@ functionality lands on top of it.
type checks and tooling work without a Rust toolchain. `*.node` binaries and
`target/` are ignored.
+The encoder uses a small local source overlay for the pinned `clickhouse`
+0.15.2 crate. `scripts/prepare-clickhouse-rs.sh` downloads the exact crate
+archive, verifies its checksum, applies the delta in `patches/`, and writes an
+ignored Cargo patch configuration. The repository keeps the delta and setup
+script, not a vendored copy of the crate.
+
## Building
Install Rust through [rustup](https://rustup.rs), the same prerequisite the AI
@@ -44,6 +46,7 @@ Run from the repo root:
pnpm --filter @langfuse/native run build # release build for the current platform
pnpm --filter @langfuse/native run build:debug # unoptimised build
pnpm --filter @langfuse/native run lint # cargo fmt --check && cargo clippy -D warnings
+pnpm --filter @langfuse/native run test # Rust tests; uses ClickHouse when available
```
`pnpm run dev`, `pnpm run build`, `pnpm run test`, and the worker's
diff --git a/packages/native/examples/native_codec_test_driver.rs b/packages/native/examples/native_codec_test_driver.rs
new file mode 100644
index 0000000000..6606257043
--- /dev/null
+++ b/packages/native/examples/native_codec_test_driver.rs
@@ -0,0 +1,58 @@
+//! Test driver used by the worker's TypeScript parity suite; not an addon export.
+
+#[allow(dead_code)]
+#[path = "../src/native_codec.rs"]
+mod native_codec;
+#[allow(dead_code)]
+#[path = "../src/native_schema.rs"]
+mod native_schema;
+
+use native_codec::encode_v4_native_blocks;
+use native_schema::PreparedEvent;
+use serde::Deserialize;
+use serde_json::{json, Value};
+use std::{env, fs, path::Path};
+
+#[derive(Debug, Deserialize)]
+#[serde(rename_all = "camelCase")]
+struct NativeCodecTestInput {
+ rows: Vec<Value>,
+ max_rows_per_block: usize,
+}
+
+fn main() {
+ let input_path = env::var("LANGFUSE_NATIVE_TEST_INPUT")
+ .expect("LANGFUSE_NATIVE_TEST_INPUT must point to the JSON input");
+ let output_dir = env::var("LANGFUSE_NATIVE_TEST_OUTPUT")
+ .expect("LANGFUSE_NATIVE_TEST_OUTPUT must point to an existing directory");
+ let input: NativeCodecTestInput =
+ serde_json::from_slice(&fs::read(&input_path).expect("read LANGFUSE_NATIVE_TEST_INPUT"))
+ .expect("parse native codec test input");
+ let prepared = input
+ .rows
+ .iter()
+ .map(PreparedEvent::from_json)
+ .collect::<Result<Vec<_>, _>>()
+ .expect("prepare native codec test rows");
+ let event_bytes = prepared
+ .iter()
+ .map(|row| row.event_bytes)
+ .collect::<Vec<_>>();
+ let blocks = encode_v4_native_blocks(&prepared, input.max_rows_per_block)
+ .expect("encode native codec test rows");
+ let mut native_payload = Vec::new();
+ for block in blocks {
+ native_payload.extend_from_slice(&block.bytes);
+ }
+
+ let output_dir = Path::new(&output_dir);
+ fs::write(output_dir.join("native.bin"), native_payload).expect("write native.bin");
+ let manifest = json!({
+ "eventBytes": event_bytes,
+ });
+ fs::write(
+ output_dir.join("manifest.json"),
+ serde_json::to_vec(&manifest).expect("serialize manifest"),
+ )
+ .expect("write manifest.json");
+}
diff --git a/packages/native/package.json b/packages/native/package.json
index 1c56e9977b..7a18e6a8c4 100644
--- a/packages/native/package.json
+++ b/packages/native/package.json
@@ -15,10 +15,12 @@
"node": "24"
},
"scripts": {
- "build": "napi build --platform --release",
- "build:debug": "napi build --platform",
- "lint": "cargo fmt --check && cargo clippy --all-targets -- -D warnings",
- "lint:fix": "cargo fmt && cargo clippy --all-targets --fix --allow-dirty -- -D warnings"
+ "prepare-clickhouse-rs": "sh scripts/prepare-clickhouse-rs.sh",
+ "build": "pnpm run prepare-clickhouse-rs && napi build --platform --release",
+ "build:debug": "pnpm run prepare-clickhouse-rs && napi build --platform",
+ "lint": "pnpm run prepare-clickhouse-rs && cargo fmt --check && cargo clippy --all-targets -- -D warnings",
+ "lint:fix": "pnpm run prepare-clickhouse-rs && cargo fmt && cargo clippy --all-targets --fix --allow-dirty -- -D warnings",
+ "test": "pnpm run prepare-clickhouse-rs && cargo test --locked"
},
"napi": {
"binaryName": "langfuse-native",
diff --git a/packages/native/patches/clickhouse-0.15.2-native-encoder.patch b/packages/native/patches/clickhouse-0.15.2-native-encoder.patch
new file mode 100644
index 0000000000..071c653350
--- /dev/null
+++ b/packages/native/patches/clickhouse-0.15.2-native-encoder.patch
@@ -0,0 +1,128 @@
+diff --git a/src/native/mod.rs b/src/native/mod.rs
+index 1ef0339..68181bc 100644
+--- a/src/native/mod.rs
++++ b/src/native/mod.rs
+@@ -47,6 +47,15 @@ impl Block {
+ }
+ }
+
++ /// Encode this block as a self-contained ClickHouse Native-format payload.
++ ///
++ /// The returned bytes contain the block header, column metadata, and all column data. They
++ /// can be sent as one Native block without retaining any references to the builder or its
++ /// source values.
++ pub fn encode(&self) -> Result<Bytes, Error> {
++ writer::encode_block(self)
++ }
++
+ /// The number of rows in this block.
+ ///
+ /// Note that the size of a single block in a query resultset can be influenced by many things
+diff --git a/src/native/writer.rs b/src/native/writer.rs
+index b9553c8..35327f8 100644
+--- a/src/native/writer.rs
++++ b/src/native/writer.rs
+@@ -5,6 +5,103 @@ use crate::native::{Block, Column, Layout, LayoutKind, varuint};
+ use bytes::{BufMut, Bytes, BytesMut};
+ use std::num::Saturating;
+
++/// Encode a complete block synchronously in ClickHouse Native format.
++///
++/// This is kept separate from [`BlockWriter`], whose purpose is streaming an insert request. It
++/// lets callers that already own a block obtain an independent byte buffer with no async sink or
++/// request lifetime involved.
++pub(crate) fn encode_block(block: &Block) -> Result<Bytes, Error> {
++ if block.num_rows == 0 {
++ return Err(Error::Other("attempting to encode an empty block".into()));
++ }
++
++ let mut buf = BytesMut::with_capacity(8192);
++ varuint::write(&mut buf, block.columns.len());
++ varuint::write(&mut buf, block.num_rows);
++
++ for column in &block.columns {
++ encode_column(&mut buf, column)?;
++ }
++
++ Ok(buf.freeze())
++}
++
++fn encode_column(buf: &mut BytesMut, column: &Column) -> Result<(), Error> {
++ varuint::write(&mut *buf, column.name.len());
++ buf.extend_from_slice(column.name.as_bytes());
++
++ let data_type_str = column.data_type.to_str();
++ varuint::write(&mut *buf, data_type_str.len());
++ buf.extend_from_slice(data_type_str.as_bytes());
++
++ encode_layout(buf, &column.layout)
++}
++
++fn encode_layout(buf: &mut BytesMut, layout: &Layout) -> Result<(), Error> {
++ if let Some(nulls) = &layout.nulls {
++ buf.extend_from_slice(nulls);
++ }
++
++ match &layout.kind {
++ LayoutKind::Fixed { data, .. } => buf.extend_from_slice(data),
++ LayoutKind::Variable { end_offsets, data } => {
++ let mut start_offset = 0;
++
++ for &end_offset in end_offsets {
++ let len = end_offset.checked_sub(start_offset).ok_or_else(|| {
++ Error::Other(
++ format!(
++ "BUG: string length underflow in encoding block: {end_offset} - {start_offset}"
++ )
++ .into(),
++ )
++ })?;
++
++ varuint::write(&mut *buf, len);
++ buf.extend_from_slice(&data[start_offset..end_offset]);
++ start_offset = end_offset;
++ }
++ }
++ LayoutKind::LowCardinality(_) => {
++ return Err(Error::Other(
++ "inserting LowCardinality data not yet implemented".into(),
++ ));
++ }
++ LayoutKind::Array {
++ end_indices,
++ elem_layout,
++ } => {
++ for &index in end_indices {
++ buf.put_u64_le(u64::try_from(index).map_err(|_| {
++ Error::Other(format!("array end index out of range: {index}").into())
++ })?);
++ }
++
++ encode_layout(buf, elem_layout)?;
++ }
++ LayoutKind::Tuple { layouts } => {
++ for layout in layouts {
++ encode_layout(buf, layout)?;
++ }
++ }
++ LayoutKind::Map {
++ key_val_layouts,
++ end_indices,
++ } => {
++ for &index in end_indices {
++ buf.put_u64_le(u64::try_from(index).map_err(|_| {
++ Error::Other(format!("map end index out of range: {index}").into())
++ })?);
++ }
++
++ encode_layout(buf, &key_val_layouts[0])?;
++ encode_layout(buf, &key_val_layouts[1])?;
++ }
++ }
++
++ Ok(())
++}
++
+ pub(crate) struct BlockWriter {
+ insert: InsertFormatted,
+ buf: BytesMut,
diff --git a/packages/native/scripts/prepare-clickhouse-rs.sh b/packages/native/scripts/prepare-clickhouse-rs.sh
new file mode 100644
index 0000000000..099ea246c6
--- /dev/null
+++ b/packages/native/scripts/prepare-clickhouse-rs.sh
@@ -0,0 +1,48 @@
+#!/bin/sh
+set -eu
+
+# Apply the local Native encoder patch to the clickhouse-rs client crate.
+
+crate_version=0.15.2
+crate_sha256=7654154fde4d97ec1321bfe4c3965570be6fcd09e091da8afc3166ab5ee27be2
+package_dir=$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd)
+source_root="${package_dir}/target/.clickhouse-source"
+source_dir="${source_root}/clickhouse-${crate_version}"
+archive="${source_root}/clickhouse-${crate_version}.crate"
+patch_file="${package_dir}/patches/clickhouse-${crate_version}-native-encoder.patch"
+marker="${source_dir}/.langfuse-native-encoder-patched"
+config_dir="${package_dir}/.cargo"
+
+checksum() {
+ if command -v sha256sum >/dev/null 2>&1; then
+ sha256sum "$1" | cut -d ' ' -f 1
+ else
+ shasum -a 256 "$1" | cut -d ' ' -f 1
+ fi
+}
+
+patch_sha256="$(checksum "$patch_file")"
+
+if [ ! -f "$marker" ] || [ "$(cat "$marker")" != "$patch_sha256" ]; then
+ mkdir -p "$source_root"
+ curl --fail --location --retry 3 --retry-delay 2 --retry-all-errors --silent --show-error \
+ --user-agent 'langfuse-native-build/0.1 (https://github.com/langfuse/langfuse)' \
+ "https://crates.io/api/v1/crates/clickhouse/${crate_version}/download" \
+ --output "$archive"
+
+ if [ "$(checksum "$archive")" != "$crate_sha256" ]; then
+ echo "clickhouse ${crate_version} archive checksum mismatch" >&2
+ exit 1
+ fi
+
+ rm -rf "$source_dir"
+ tar -xzf "$archive" -C "$source_root"
+ patch --directory="$source_dir" --strip=1 < "$patch_file"
+ printf '%s' "$patch_sha256" > "$marker"
+fi
+
+mkdir -p "$config_dir"
+cat > "$config_dir/config.toml" <<EOF
+[patch.crates-io]
+clickhouse = { path = "${source_dir}" }
+EOF
diff --git a/packages/native/src/lib.rs b/packages/native/src/lib.rs
index cd5701de5a..5231287d2d 100644
--- a/packages/native/src/lib.rs
+++ b/packages/native/src/lib.rs
@@ -5,6 +5,12 @@
//! `metrics` and `tracing` facades set up in [`telemetry`]; it never hands
//! values back for Node to record.
+/// Rust-only encoder for prepared v4 `events_full` rows.
+// These modules are exercised by Rust tests until the ingestion transport calls them.
+#[allow(dead_code)]
+pub mod native_codec;
+#[allow(dead_code)]
+pub(crate) mod native_schema;
mod telemetry;
use std::time::Duration;
diff --git a/packages/native/src/native_codec.rs b/packages/native/src/native_codec.rs
new file mode 100644
index 0000000000..ce5908a943
--- /dev/null
+++ b/packages/native/src/native_codec.rs
@@ -0,0 +1,434 @@
+//! Rust-only encoding for prepared v4 `events_full` rows.
+//!
+//! `PreparedEvent::from_json` accepts the prepared `EventRecordInsertType` shape after
+//! observation-field overflow handling and before the writer's numeric Decimal clamp. Nested
+//! input/output payloads have already been serialized as strings. Native encoding then receives
+//! only typed, owned fields; the adapters below define the JSON conversion policy.
+
+use std::borrow::Cow;
+use std::collections::BTreeMap;
+use std::error::Error as StdError;
+
+use bytes::Bytes;
+use chrono::{DateTime, NaiveDateTime, Utc};
+use clickhouse::native::builder::BlockBuilder;
+use clickhouse::native::encode::{Encode, ValueWriter};
+use clickhouse::native::DataTypeNode;
+use clickhouse_types::data_types::{DateTimePrecision, DecimalType};
+use rust_decimal::{Decimal, RoundingStrategy};
+use serde::ser::{SerializeMap, Serializer}; // codespell:ignore ser
+use serde_json::Value;
+
+use crate::native_schema::{ColumnKind, DefaultPolicy, PreparedEvent};
+
+const DECIMAL_SCALE: u32 = 12;
+const DECIMAL_LIMIT: i128 = 1_000_000_000_000_000_000;
+const DECIMAL_CLAMPED_MAX: i64 = 999_999_999_999_000_000;
+const DECIMAL_CLAMPED_MIN: i64 = -DECIMAL_CLAMPED_MAX;
+const DECIMAL_OVERFLOW_LIMIT: f64 = 1_000_000.0;
+type EncodeError = Box<dyn StdError + Send + Sync>;
+
+#[derive(Clone, Copy, Debug)]
+pub(crate) struct DateTime64Micros(pub(crate) i64);
+
+impl Encode for DateTime64Micros {
+ fn produces() -> DataTypeNode {
+ DataTypeNode::DateTime64(DateTimePrecision::Precision6, None)
+ }
+
+ fn encode(&self, writer: &mut ValueWriter<'_>) -> Result<(), EncodeError> {
+ // clickhouse-rs has no DateTime64 Encode implementation. Keep the DateTime64(6) type tag
+ // while writing ClickHouse's signed microsecond representation as a fixed-width value.
+ writer.write_fixed(&self.0.to_le_bytes())?;
+ Ok(())
+ }
+}
+
+#[derive(Clone, Copy, Debug)]
+pub(crate) struct Decimal64(pub(crate) i64);
+
+impl Encode for Decimal64 {
+ fn produces() -> DataTypeNode {
+ DataTypeNode::Decimal(18, 12, DecimalType::Decimal64)
+ }
+
+ fn encode(&self, writer: &mut ValueWriter<'_>) -> Result<(), EncodeError> {
+ // rust_decimal supplies the policy and coefficient; clickhouse-rs has no Decimal Encode
+ // implementation, so retain the Decimal(18,12) type tag and write its Int64 coefficient.
+ writer.write_fixed(&self.0.to_le_bytes())?;
+ Ok(())
+ }
+}
+
+/// An owned ClickHouse Native-format block and its row count.
+///
+/// Ownership is part of the transport contract: the later JS transport can retry this complete
+/// block after the encoder and source rows have gone out of scope.
+#[derive(Debug)]
+pub struct EncodedBlock {
+ pub bytes: Bytes,
+ pub row_count: usize,
+}
+
+/// Encode prepared v4 event rows into Native blocks containing at most `max_rows_per_block` rows.
+/// The boundary is a row count, not a byte-size or memory limit; the transport can use the returned
+/// owned buffers and counts when it chooses its request boundaries.
+///
+/// Prepared rows already contain the defaults that production-prepared rows rely on when every
+/// insertable column is listed in a Native block. Decimal values use the existing
+/// Decimal(18,12) overflow policy and truncate toward zero at twelve fractional digits.
+/// `event_bytes` is the UTF-8 size of the compact Rust JSON serialization of the prepared row
+/// without the accounting field; it is intentionally a logical size metric and need not be
+/// byte-identical to JavaScript's JSON.stringify output. Preparation stores this value before
+/// applying typed decimal conversion or DEFAULT-backed field filling.
+pub(crate) fn encode_v4_native_blocks(
+ rows: &[PreparedEvent],
+ max_rows_per_block: usize,
+) -> Result<Vec<EncodedBlock>, String> {
+ if max_rows_per_block == 0 {
+ return Err("max_rows_per_block must be greater than zero".to_owned());
+ }
+
+ rows.chunks(max_rows_per_block)
+ .map(|chunk| {
+ let bytes = encode_block(chunk)?;
+ Ok(EncodedBlock {
+ bytes,
+ row_count: chunk.len(),
+ })
+ })
+ .collect()
+}
+
+/// Compute the UTF-8 size of a prepared event's JSONEachRow representation without `event_bytes`.
+pub fn event_bytes(value: &Value) -> Result<u64, String> {
+ let object = value
+ .as_object()
+ .ok_or_else(|| "event row must be a JSON object".to_owned())?;
+ // Borrow the fields and discard serialized bytes immediately: accounting needs the length,
+ // not a cloned row or a second payload buffer.
+ let mut counter = ByteCounter(0);
+ let mut serializer = serde_json::Serializer::new(&mut counter);
+ let result = (|| -> Result<(), serde_json::Error> {
+ let mut map = serializer.serialize_map(None)?;
+ for (key, value) in object {
+ if key != "event_bytes" {
+ map.serialize_entry(key, value)?;
+ }
+ }
+ map.end()
+ })();
+ result.map_err(|error| format!("failed to serialize event row: {error}"))?;
+ Ok(counter.0)
+}
+
+struct ByteCounter(u64);
+
+impl std::io::Write for ByteCounter {
+ fn write(&mut self, bytes: &[u8]) -> std::io::Result<usize> {
+ self.0 = self
+ .0
+ .checked_add(bytes.len() as u64)
+ .ok_or_else(|| std::io::Error::other("serialized event is too large"))?;
+ Ok(bytes.len())
+ }
+
+ fn flush(&mut self) -> std::io::Result<()> {
+ Ok(())
+ }
+}
+
+fn encode_block(rows: &[PreparedEvent]) -> Result<Bytes, String> {
+ let mut builder = BlockBuilder::new();
+ PreparedEvent::encode_into(&mut builder, rows)?;
+
+ builder
+ .build()
+ .map_err(|error| error.to_string())?
+ .encode()
+ .map_err(|error| error.to_string())
+}
+
+/// Resolve values whose ClickHouse defaults would otherwise be bypassed by listing the column in
+/// the Native block. Ordinary columns stay borrowed from the input row; only synthesized defaults
+/// allocate a JSON value. This runs once at the JSON-to-typed preparation boundary.
+fn value_for<'a>(row: &'a Value, key: &str, default_policy: DefaultPolicy) -> Cow<'a, Value> {
+ let explicit = row.get(key);
+ if default_policy == DefaultPolicy::None {
+ return explicit
+ .map(Cow::Borrowed)
+ .unwrap_or(Cow::Owned(Value::Null));
+ }
+ if let Some(value) = explicit.filter(|value| !value.is_null()) {
+ return Cow::Borrowed(value);
+ }
+ // For example, omitted environment gets DEFAULT 'default' in JSONEachRow. Sending an empty
+ // string in a Native column would instead persist that empty string and bypass the default.
+ Cow::Owned(match default_policy {
+ DefaultPolicy::Literal(value) => Value::String(value.to_owned()),
+ DefaultPolicy::Metadata(key) => Value::String(metadata_value(row, key).unwrap_or_default()),
+ DefaultPolicy::MetadataFallback(key, fallback) => Value::String(
+ metadata_value(row, key)
+ .filter(|value| !value.is_empty())
+ .or_else(|| metadata_value(row, fallback))
+ .unwrap_or_default(),
+ ),
+ DefaultPolicy::MetadataEquals(key, expected) => {
+ Value::Bool(metadata_value(row, key).as_deref() == Some(expected))
+ }
+ DefaultPolicy::None => Value::Null,
+ })
+}
+
+pub(crate) fn from_json_field<T: FromJson>(
+ row: &Value,
+ key: &str,
+ default_policy: DefaultPolicy,
+) -> Result<T, String> {
+ T::from_json(value_for(row, key, default_policy).as_ref())
+ .map_err(|error| format!("{key}: {error}"))
+}
+
+fn metadata_value(row: &Value, name: &str) -> Option<String> {
+ // Ingestion stores metadata as parallel arrays; default expressions look up the same pair.
+ let names = row.get("metadata_names")?.as_array()?;
+ let values = row.get("metadata_values")?.as_array()?;
+ names.iter().zip(values).find_map(|(metadata_name, value)| {
+ (metadata_name.as_str() == Some(name)).then(|| string_value(value))
+ })
+}
+
+// Conversion is separate from wire encoding: upstream Encode implementations own nullable, array,
+// map, and primitive layouts, while these implementations define the prepared-row input policy.
+pub(crate) trait FromJson: Encode + Sized {
+ const COLUMN_KIND: ColumnKind;
+
+ fn from_json(value: &Value) -> Result<Self, String>;
+}
+
+impl FromJson for String {
+ const COLUMN_KIND: ColumnKind = ColumnKind::String;
+
+ fn from_json(value: &Value) -> Result<Self, String> {
+ Ok(string_value(value))
+ }
+}
+
+impl FromJson for bool {
+ const COLUMN_KIND: ColumnKind = ColumnKind::Bool;
+
+ fn from_json(value: &Value) -> Result<Self, String> {
+ match value {
+ Value::Null => Ok(false),
+ Value::Bool(value) => Ok(*value),
+ Value::Number(value) => value
+ .as_f64()
+ .filter(|value| value.is_finite())
+ .map(|value| value != 0.0)
+ .ok_or_else(|| "expected a finite boolean number".to_owned()),
+ _ => Err(format!("expected boolean, got {value}")),
+ }
+ }
+}
+
+macro_rules! unsigned {
+ ($(($type:ty, $kind:expr)),+ $(,)?) => {$(
+ impl FromJson for $type {
+ const COLUMN_KIND: ColumnKind = $kind;
+
+ fn from_json(value: &Value) -> Result<Self, String> {
+ <$type>::try_from(json_u64(Some(value), stringify!($type))?)
+ .map_err(|_| format!("expected a {}", stringify!($type)))
+ }
+ }
+ )+};
+}
+unsigned!(
+ (u8, ColumnKind::UInt8),
+ (u16, ColumnKind::UInt16),
+ (u64, ColumnKind::UInt64),
+);
+
+impl FromJson for DateTime64Micros {
+ const COLUMN_KIND: ColumnKind = ColumnKind::DateTime64;
+
+ fn from_json(value: &Value) -> Result<Self, String> {
+ parse_datetime_value(value, "datetime").map(Self)
+ }
+}
+
+impl FromJson for Decimal64 {
+ const COLUMN_KIND: ColumnKind = ColumnKind::Decimal;
+
+ fn from_json(value: &Value) -> Result<Self, String> {
+ parse_decimal_value(value, "decimal").map(Self)
+ }
+}
+
+impl<T: FromJson> FromJson for Option<T> {
+ const COLUMN_KIND: ColumnKind = match T::COLUMN_KIND {
+ ColumnKind::String => ColumnKind::NullableString,
+ ColumnKind::DateTime64 => ColumnKind::NullableDateTime64,
+ ColumnKind::UInt16 => ColumnKind::NullableUInt16,
+ _ => panic!("unsupported nullable events_full column type"),
+ };
+
+ fn from_json(value: &Value) -> Result<Self, String> {
+ if value.is_null() {
+ Ok(None)
+ } else {
+ T::from_json(value).map(Some)
+ }
+ }
+}
+
+impl<T: FromJson> FromJson for Vec<T> {
+ const COLUMN_KIND: ColumnKind = match T::COLUMN_KIND {
+ ColumnKind::String => ColumnKind::ArrayString,
+ _ => panic!("unsupported array events_full column type"),
+ };
+
+ fn from_json(value: &Value) -> Result<Self, String> {
+ match value {
+ Value::Null => Ok(Vec::new()),
+ Value::Array(values) => values.iter().map(T::from_json).collect(),
+ _ => Err(format!("expected an array, got {value}")),
+ }
+ }
+}
+
+impl<T: FromJson> FromJson for BTreeMap<String, T> {
+ const COLUMN_KIND: ColumnKind = match T::COLUMN_KIND {
+ ColumnKind::String => ColumnKind::MapStringString,
+ ColumnKind::UInt64 => ColumnKind::MapStringUInt64,
+ ColumnKind::Decimal => ColumnKind::MapStringDecimal,
+ _ => panic!("unsupported map events_full column type"),
+ };
+
+ fn from_json(value: &Value) -> Result<Self, String> {
+ match value {
+ Value::Null => Ok(BTreeMap::new()),
+ Value::Object(values) => values
+ .iter()
+ .map(|(key, value)| Ok((key.clone(), T::from_json(value)?)))
+ .collect(),
+ _ => Err(format!("expected a map, got {value}")),
+ }
+ }
+}
+
+fn json_u64(value: Option<&Value>, type_name: &str) -> Result<u64, String> {
+ // JSONEachRow inputs can represent whole numbers as either JSON integers or integral floats;
+ // accept both forms while rejecting signs, fractions, and non-finite values for UInt columns.
+ match value {
+ None | Some(Value::Null) => Ok(0),
+ Some(Value::Number(value)) => value
+ .as_u64()
+ .or_else(|| {
+ value
+ .as_f64()
+ .filter(|number| number.is_finite() && *number >= 0.0 && number.fract() == 0.0)
+ .and_then(|number| u64::try_from(number as u128).ok())
+ })
+ .ok_or_else(|| format!("expected a {type_name}")),
+ Some(value) => Err(format!("expected a {type_name}, got {value}")),
+ }
+}
+
+/// Parse the UTC-only datetime values produced by the ingestion pipeline. A ClickHouse datetime
+/// string without an offset is a UTC calendar value; an RFC3339 offset is normalized to that same
+/// UTC instant. The project never interprets these values in the host's local timezone.
+fn parse_datetime_value(value: &Value, key: &str) -> Result<i64, String> {
+ match value {
+ Value::String(value) => {
+ let normalized = if value.contains(' ') && !value.contains('T') {
+ value.replacen(' ', "T", 1)
+ } else {
+ value.clone()
+ };
+ if let Ok(parsed) = DateTime::parse_from_rfc3339(&normalized) {
+ return Ok(parsed.timestamp_micros());
+ }
+ let naive = NaiveDateTime::parse_from_str(&normalized, "%Y-%m-%dT%H:%M:%S%.f")
+ .map_err(|error| {
+ format!("{key} has invalid DateTime64 value {value:?}: {error}")
+ })?;
+ Ok(DateTime::<Utc>::from_naive_utc_and_offset(naive, Utc).timestamp_micros())
+ }
+ _ => Err(format!("{key} must be a UTC timestamp string")),
+ }
+}
+
+fn parse_decimal_value(value: &Value, key: &str) -> Result<i64, String> {
+ let text = match value {
+ Value::String(value) => value.as_str(),
+ Value::Number(value) => return parse_decimal(&value.to_string(), key),
+ _ => return Err(format!("{key} must be a decimal number or string")),
+ };
+ parse_decimal(text, key)
+}
+
+fn parse_decimal(value: &str, key: &str) -> Result<i64, String> {
+ let value = value.trim();
+ if value.is_empty() {
+ return Ok(0);
+ }
+ let number = value
+ .parse::<f64>()
+ .map_err(|_| format!("{key} has an invalid decimal"))?;
+
+ // `UsageCostSchema` converts costs to JavaScript numbers and the writer's existing policy
+ // turns non-finite values into zero and clamps values at the exclusive 1e6 boundary.
+ if !number.is_finite() {
+ return Ok(0);
+ }
+ if number.abs() >= DECIMAL_OVERFLOW_LIMIT {
+ return Ok(if number.is_sign_negative() {
+ DECIMAL_CLAMPED_MIN
+ } else {
+ DECIMAL_CLAMPED_MAX
+ });
+ }
+
+ // Values below the target quantum are zero after ClickHouse's scale conversion. Returning
+ // early also covers f64 underflow values whose scientific exponent exceeds Decimal's scale
+ // range without turning a valid, zero-result input into a parse error.
+ if number == 0.0 || number.abs() < 10_f64.powi(-(DECIMAL_SCALE as i32)) {
+ return Ok(0);
+ }
+
+ // The ingestion schema has already reduced costs to JavaScript numbers. Parse that shortest
+ // round-tripping representation as a fixed-precision decimal so scaling never multiplies a
+ // binary float. Decimal(18,12) fits comfortably within rust_decimal's coefficient range.
+ let normalized = number.to_string();
+ let mut decimal = if normalized.contains(['e', 'E']) {
+ Decimal::from_scientific(&normalized)
+ } else {
+ normalized.parse::<Decimal>()
+ }
+ .map_err(|_| format!("{key} has an invalid decimal"))?;
+
+ decimal = decimal.round_dp_with_strategy(DECIMAL_SCALE, RoundingStrategy::ToZero);
+ decimal.rescale(DECIMAL_SCALE);
+ let coefficient = decimal.mantissa();
+ if coefficient >= DECIMAL_LIMIT || coefficient <= -DECIMAL_LIMIT {
+ return Ok(if coefficient.is_negative() {
+ DECIMAL_CLAMPED_MIN
+ } else {
+ DECIMAL_CLAMPED_MAX
+ });
+ }
+ i64::try_from(coefficient).map_err(|_| format!("{key} decimal is out of range"))
+}
+
+fn string_value(value: &Value) -> String {
+ match value {
+ Value::String(value) => value.clone(),
+ Value::Null => String::new(),
+ value => serde_json::to_string(value).unwrap_or_default(),
+ }
+}
+
+#[cfg(test)]
+#[path = "native_codec_tests.rs"]
+mod tests;
diff --git a/packages/native/src/native_codec_tests.rs b/packages/native/src/native_codec_tests.rs
new file mode 100644
index 0000000000..49bc32ad17
--- /dev/null
+++ b/packages/native/src/native_codec_tests.rs
@@ -0,0 +1,539 @@
+use super::*;
+use crate::native_schema::{
+ find_column, ColumnKind, DefaultPolicy, PreparedEvent, EVENTS_FULL_INSERT_COLUMNS,
+};
+use proptest::prelude::*;
+use proptest::test_runner::{Config as ProptestConfig, TestRunner};
+use serde_json::json;
+use std::collections::BTreeSet;
+use std::io::Write;
+use std::process::{Command, Stdio};
+use std::time::{SystemTime, UNIX_EPOCH};
+
+fn encode_json_rows(rows: &[Value], max_rows: usize) -> Result<Vec<EncodedBlock>, String> {
+ let prepared = rows
+ .iter()
+ .map(PreparedEvent::from_json)
+ .collect::<Result<Vec<_>, _>>()?;
+ encode_v4_native_blocks(&prepared, max_rows)
+}
+
+fn row(input: &str, output: &str, event_bytes: Value) -> Value {
+ json!({
+ "project_id": "project",
+ "trace_id": "trace",
+ "span_id": "span",
+ "start_time": "2026-07-22 00:00:00.000",
+ "created_at": "2026-07-22 00:00:00.000",
+ "updated_at": "2026-07-22 00:00:00.000",
+ "event_ts": "2026-07-22 00:00:00.000",
+ "input": input,
+ "output": output,
+ "metadata_names": ["source"],
+ "metadata_values": ["API"],
+ "provided_usage_details": {"input": 1},
+ "usage_details": {"output": 2},
+ "provided_cost_details": {"input": "0.000000000001"},
+ "cost_details": {"output": 1.25},
+ "event_bytes": event_bytes,
+ })
+}
+
+#[test]
+fn event_bytes_excludes_the_accounting_field_and_counts_utf8() {
+ let value = json!({"input": "🔥", "event_bytes": 123});
+ assert_eq!(
+ event_bytes(&value).expect("event bytes"),
+ "{\"input\":\"🔥\"}".len() as u64
+ );
+ for payload in [
+ json!({}),
+ json!({"event_bytes": 99}),
+ json!({"nested": [null, true, {"event_bytes": 7, "text": "\"\n\\🔥"}], "cost": 1e-13}),
+ ] {
+ let mut expected = payload.clone();
+ expected.as_object_mut().unwrap().remove("event_bytes");
+ assert_eq!(
+ event_bytes(&payload).unwrap(),
+ serde_json::to_vec(&expected).unwrap().len() as u64
+ );
+ }
+}
+
+#[test]
+fn zero_max_rows_per_block_is_rejected() {
+ let error = match encode_json_rows(&[], 0) {
+ Ok(_) => panic!("zero block size must fail"),
+ Err(error) => error,
+ };
+ assert_eq!(error, "max_rows_per_block must be greater than zero");
+}
+
+#[test]
+fn decimal_parser_matches_clickhouse_scale_and_overflow_policy() {
+ assert_eq!(parse_decimal("1.25", "cost").unwrap(), 1_250_000_000_000);
+ let precise_json_number: Value = serde_json::from_str("510407.65505697404").unwrap();
+ assert_eq!(
+ parse_decimal_value(&precise_json_number, "cost").unwrap(),
+ 510_407_655_056_974_040
+ );
+ assert_eq!(parse_decimal("0.0000000000001", "cost").unwrap(), 0);
+ assert_eq!(
+ parse_decimal("0.00004381800000000001", "cost").unwrap(),
+ 43_818_000
+ );
+ assert_eq!(
+ parse_decimal("-1.9999999999999", "cost").unwrap(),
+ -1_999_999_999_999
+ );
+ assert_eq!(parse_decimal("1e6", "cost").unwrap(), DECIMAL_CLAMPED_MAX);
+ assert_eq!(parse_decimal("-1e6", "cost").unwrap(), DECIMAL_CLAMPED_MIN);
+ assert_eq!(parse_decimal("Infinity", "cost").unwrap(), 0);
+ assert_eq!(parse_decimal("-Infinity", "cost").unwrap(), 0);
+ assert_eq!(parse_decimal("1e2147483647", "cost").unwrap(), 0);
+ assert_eq!(parse_decimal("1e-2147483648", "cost").unwrap(), 0);
+ assert_eq!(parse_decimal("", "cost").unwrap(), 0);
+ assert_eq!(u16::from_json(&json!(42.0)), Ok(42));
+ assert!(u16::from_json(&json!(-1.0)).is_err());
+}
+
+#[test]
+fn datetime_strings_are_explicitly_utc() {
+ let naive = parse_datetime_value(&json!("2026-07-22 00:00:00.000"), "timestamp").unwrap();
+ let utc = parse_datetime_value(&json!("2026-07-22T00:00:00.000Z"), "timestamp").unwrap();
+ let offset =
+ parse_datetime_value(&json!("2026-07-22T02:00:00.000+02:00"), "timestamp").unwrap();
+
+ assert_eq!(naive, utc);
+ assert_eq!(offset, utc);
+}
+
+#[test]
+fn invalid_decimal_prevents_a_prepared_event_from_being_returned() {
+ let mut value = row("input", "output", Value::Null);
+ value["cost_details"] = json!({"output": "definitely-not-a-number"});
+
+ let error = PreparedEvent::from_json(&value).expect_err("invalid decimal");
+ assert!(error.contains("cost_details"));
+ assert!(error.contains("invalid decimal"));
+}
+
+#[derive(Debug)]
+struct LiveColumn {
+ name: String,
+ column_type: String,
+ default_kind: String,
+}
+
+fn live_events_full_schema() -> Option<Vec<LiveColumn>> {
+ let output = match run_clickhouse_client(
+ "SELECT name, type, default_kind FROM system.columns WHERE database = currentDatabase() AND table = 'events_full' ORDER BY position FORMAT JSONEachRow",
+ &[],
+ ) {
+ Ok(output) => output,
+ Err(error) => {
+ if clickhouse_is_required() {
+ panic!("ClickHouse schema introspection failed: {error}");
+ }
+ eprintln!("ClickHouse schema unavailable; skipping generated-row test: {error}");
+ return None;
+ }
+ };
+
+ output
+ .split(|byte| *byte == b'\n')
+ .filter(|line| !line.is_empty())
+ .map(|line| {
+ serde_json::from_slice::<Value>(line)
+ .map_err(|error| format!("parse ClickHouse schema row: {error}"))
+ .and_then(|value| {
+ Ok(LiveColumn {
+ name: value["name"]
+ .as_str()
+ .ok_or_else(|| "schema row has no column name".to_owned())?
+ .to_owned(),
+ column_type: value["type"]
+ .as_str()
+ .ok_or_else(|| "schema row has no column type".to_owned())?
+ .to_owned(),
+ default_kind: value["default_kind"]
+ .as_str()
+ .unwrap_or_default()
+ .to_owned(),
+ })
+ })
+ })
+ .collect::<Result<Vec<_>, _>>()
+ .map(Some)
+ .unwrap_or_else(|error| panic!("invalid ClickHouse schema response: {error}"))
+}
+
+fn assert_live_schema_matches_codec(schema: &[LiveColumn]) {
+ let actual_names = schema
+ .iter()
+ .filter(|column| !is_server_owned(column))
+ .map(|column| column.name.clone())
+ .collect::<BTreeSet<_>>();
+ let expected_names = EVENTS_FULL_INSERT_COLUMNS
+ .iter()
+ .map(|column| column.name.to_owned())
+ .collect::<BTreeSet<_>>();
+ assert_eq!(
+ actual_names, expected_names,
+ "events_full insertable columns differ from the Native codec contract"
+ );
+
+ for column in schema.iter().filter(|column| !is_server_owned(column)) {
+ let kind = ColumnKind::from_clickhouse_type(&column.column_type)
+ .unwrap_or_else(|error| panic!("{}: {error}", column.name));
+ let spec = find_column(&column.name).expect("column set checked above");
+ assert_eq!(
+ spec.kind, kind,
+ "{} changed from {:?} to {}",
+ column.name, spec.kind, column.column_type
+ );
+ if spec.default_policy != DefaultPolicy::None {
+ assert_eq!(
+ column.default_kind, "DEFAULT",
+ "{} no longer has the DEFAULT expression required by the Native adapter",
+ column.name
+ );
+ }
+ }
+}
+
+fn is_server_owned(column: &LiveColumn) -> bool {
+ matches!(column.default_kind.as_str(), "MATERIALIZED" | "ALIAS")
+}
+
+fn clickhouse_is_required() -> bool {
+ std::env::var("LANGFUSE_NATIVE_FAIL_IF_CLICKHOUSE_UNAVAILABLE")
+ .ok()
+ .as_deref()
+ == Some("1")
+}
+
+#[derive(Debug)]
+struct ClickhouseComparison {
+ json_count: u64,
+ native_count: u64,
+ json_only: u64,
+ native_only: u64,
+ json_event_bytes: u64,
+ native_event_bytes: u64,
+}
+
+fn clickhouse_server_round_trip(
+ json_payload: &[u8],
+ native_payload: &[u8],
+) -> Option<ClickhouseComparison> {
+ let required = clickhouse_is_required();
+ let suffix = SystemTime::now()
+ .duration_since(UNIX_EPOCH)
+ .expect("system clock")
+ .as_nanos();
+ let json_table = format!("native_codec_json_{}_{}", std::process::id(), suffix);
+ let native_table = format!("native_codec_native_{}_{}", std::process::id(), suffix);
+
+ let create_result = run_clickhouse_client(
+ &format!(
+ "CREATE TABLE {json_table} AS events_full ENGINE = Memory; CREATE TABLE {native_table} AS events_full ENGINE = Memory;"
+ ),
+ &[],
+ );
+ if let Err(error) = create_result {
+ let _ = run_clickhouse_client(
+ &format!("DROP TABLE IF EXISTS {json_table}; DROP TABLE IF EXISTS {native_table};"),
+ &[],
+ );
+ if required {
+ panic!("ClickHouse server parity test failed to create tables: {error}");
+ }
+ eprintln!("ClickHouse server unavailable; skipping parity test: {error}");
+ return None;
+ }
+
+ let result: Result<ClickhouseComparison, String> = (|| {
+ run_clickhouse_client(
+ &format!("INSERT INTO {json_table} FORMAT JSONEachRow"),
+ json_payload,
+ )?;
+ run_clickhouse_client(
+ &format!("INSERT INTO {native_table} FORMAT Native"),
+ native_payload,
+ )?;
+
+ let output = run_clickhouse_client(
+ &format!(
+ "SELECT (SELECT count() FROM {json_table}) AS json_count, (SELECT count() FROM {native_table}) AS native_count, (SELECT count() FROM (SELECT * EXCEPT (event_bytes, provided_usage_details, usage_details, provided_cost_details, cost_details), mapSort((k, v) -> k, provided_usage_details) AS provided_usage_details, mapSort((k, v) -> k, usage_details) AS usage_details, mapSort((k, v) -> k, provided_cost_details) AS provided_cost_details, mapSort((k, v) -> k, cost_details) AS cost_details FROM {json_table} EXCEPT ALL SELECT * EXCEPT (event_bytes, provided_usage_details, usage_details, provided_cost_details, cost_details), mapSort((k, v) -> k, provided_usage_details) AS provided_usage_details, mapSort((k, v) -> k, usage_details) AS usage_details, mapSort((k, v) -> k, provided_cost_details) AS provided_cost_details, mapSort((k, v) -> k, cost_details) AS cost_details FROM {native_table})) AS json_only, (SELECT count() FROM (SELECT * EXCEPT (event_bytes, provided_usage_details, usage_details, provided_cost_details, cost_details), mapSort((k, v) -> k, provided_usage_details) AS provided_usage_details, mapSort((k, v) -> k, usage_details) AS usage_details, mapSort((k, v) -> k, provided_cost_details) AS provided_cost_details, mapSort((k, v) -> k, cost_details) AS cost_details FROM {native_table} EXCEPT ALL SELECT * EXCEPT (event_bytes, provided_usage_details, usage_details, provided_cost_details, cost_details), mapSort((k, v) -> k, provided_usage_details) AS provided_usage_details, mapSort((k, v) -> k, usage_details) AS usage_details, mapSort((k, v) -> k, provided_cost_details) AS provided_cost_details, mapSort((k, v) -> k, cost_details) AS cost_details FROM {json_table})) AS native_only, (SELECT sum(event_bytes) FROM {json_table}) AS json_event_bytes, (SELECT sum(event_bytes) FROM {native_table}) AS native_event_bytes SETTINGS asterisk_include_materialized_columns=1, asterisk_include_alias_columns=1 FORMAT JSONEachRow"
+ ),
+ &[],
+ )?;
+ let line = output
+ .split(|byte| *byte == b'\n')
+ .find(|line| !line.is_empty())
+ .ok_or_else(|| "ClickHouse comparison returned no rows".to_owned())?;
+ let value: Value = serde_json::from_slice(line)
+ .map_err(|error| format!("parse ClickHouse comparison: {error}"))?;
+ let comparison = ClickhouseComparison {
+ json_count: output_u64(&value["json_count"]),
+ native_count: output_u64(&value["native_count"]),
+ json_only: output_u64(&value["json_only"]),
+ native_only: output_u64(&value["native_only"]),
+ json_event_bytes: output_u64(&value["json_event_bytes"]),
+ native_event_bytes: output_u64(&value["native_event_bytes"]),
+ };
+ Ok(comparison)
+ })();
+
+ let _ = run_clickhouse_client(
+ &format!("DROP TABLE IF EXISTS {json_table}; DROP TABLE IF EXISTS {native_table};"),
+ &[],
+ );
+
+ match result {
+ Ok(comparison) => Some(comparison),
+ Err(error) => panic!("ClickHouse server parity test failed: {error}"),
+ }
+}
+
+fn run_clickhouse_client(query: &str, input: &[u8]) -> Result<Vec<u8>, String> {
+ let binary =
+ std::env::var("LANGFUSE_NATIVE_CLICKHOUSE_BIN").unwrap_or_else(|_| "clickhouse".to_owned());
+ let host = std::env::var("LANGFUSE_NATIVE_CLICKHOUSE_HOST")
+ .or_else(|_| std::env::var("CLICKHOUSE_HOST"))
+ .unwrap_or_else(|_| "127.0.0.1".to_owned());
+ let port = std::env::var("LANGFUSE_NATIVE_CLICKHOUSE_PORT")
+ .or_else(|_| std::env::var("CLICKHOUSE_PORT"))
+ .unwrap_or_else(|_| "9000".to_owned());
+ let user = std::env::var("CLICKHOUSE_USER").unwrap_or_else(|_| "clickhouse".to_owned());
+ let password = std::env::var("CLICKHOUSE_PASSWORD").unwrap_or_else(|_| "clickhouse".to_owned());
+ let database = std::env::var("CLICKHOUSE_DB").unwrap_or_else(|_| "default".to_owned());
+
+ let mut child = Command::new(&binary)
+ .args([
+ "client",
+ "--host",
+ &host,
+ "--port",
+ &port,
+ "--user",
+ &user,
+ "--password",
+ &password,
+ "--database",
+ &database,
+ "--multiquery",
+ "--query",
+ query,
+ ])
+ .stdin(Stdio::piped())
+ .stdout(Stdio::piped())
+ .stderr(Stdio::piped())
+ .spawn()
+ .map_err(|error| format!("start {binary} client: {error}"))?;
+ child
+ .stdin
+ .take()
+ .expect("ClickHouse client stdin")
+ .write_all(input)
+ .map_err(|error| format!("write ClickHouse client input: {error}"))?;
+ let output = child
+ .wait_with_output()
+ .map_err(|error| format!("wait for ClickHouse client: {error}"))?;
+ if !output.status.success() {
+ return Err(String::from_utf8_lossy(&output.stderr).trim().to_owned());
+ }
+ Ok(output.stdout)
+}
+
+fn output_u64(value: &Value) -> u64 {
+ value
+ .as_u64()
+ .or_else(|| value.as_str().and_then(|value| value.parse().ok()))
+ .expect("ClickHouse UInt64 output")
+}
+
+proptest! {
+ #[test]
+ fn decimal_inputs_with_extreme_exponents_are_handled(
+ integer in -999_999i64..=999_999,
+ fraction in "[0-9]{1,24}",
+ exponent in -1_000i32..=1_000,
+ ) {
+ let value = format!("{integer}.{fraction}e{exponent}");
+ prop_assert!(parse_decimal(&value, "cost").is_ok());
+ }
+
+ #[test]
+ fn decimal_scale_truncates_toward_zero_for_valid_prepared_costs(
+ whole in -9i64..=9,
+ fraction in 0u64..=9_999_999_999_999,
+ ) {
+ let value = format!("{whole}.{fraction:013}");
+ let sign = if whole.is_negative() { -1 } else { 1 };
+ let expected = whole * 1_000_000_000_000 + sign * (fraction / 10) as i64;
+ prop_assert_eq!(parse_decimal(&value, "cost").unwrap(), expected);
+ }
+}
+
+// Derive wire-format properties from the live table, independently of the native field list.
+// The captured-traffic test in TypeScript separately exercises the production preparation path.
+fn generated_row_strategy(schema: &[LiveColumn]) -> BoxedStrategy<Value> {
+ let fields = schema
+ .iter()
+ .filter(|column| !is_server_owned(column))
+ .map(|column| {
+ let name = column.name.clone();
+ let kind = ColumnKind::from_clickhouse_type(&column.column_type).unwrap();
+ let value = match name.as_str() {
+ "type" => {
+ prop::sample::select(vec![json!("SPAN"), json!("GENERATION"), json!("EVENT")])
+ .boxed()
+ }
+ "level" => {
+ prop::sample::select(vec![json!("DEFAULT"), json!("ERROR"), json!("WARNING")])
+ .boxed()
+ }
+ "source" => Just(json!("API")).boxed(),
+ _ => generated_value_strategy(kind),
+ };
+ (value, any::<bool>()).prop_map(move |(value, omit)| {
+ let defaulted = find_column(&name).unwrap().default_policy != DefaultPolicy::None;
+ (
+ name.clone(),
+ if omit && defaulted { None } else { Some(value) },
+ )
+ })
+ })
+ .collect::<Vec<_>>();
+ fields
+ .prop_map(|fields| {
+ let mut object = fields
+ .into_iter()
+ .filter_map(|(name, value)| value.map(|v| (name, v)))
+ .collect::<serde_json::Map<_, _>>();
+ for prefix in [
+ "metadata",
+ "experiment_metadata",
+ "experiment_item_metadata",
+ ] {
+ let values = object[&format!("{prefix}_values")].as_array().unwrap();
+ let names = [
+ "evaluator_id",
+ "evaluation_rule_id",
+ "job_configuration_id",
+ "evaluator_test",
+ ];
+ object.insert(format!("{prefix}_names"), json!(&names[..values.len()]));
+ }
+ Value::Object(object)
+ })
+ .boxed()
+}
+
+fn generated_value_strategy(kind: ColumnKind) -> BoxedStrategy<Value> {
+ let text = prop_oneof![
+ prop::collection::vec(any::<char>(), 0..24)
+ .prop_map(|chars| chars.into_iter().collect::<String>()),
+ Just("true".to_owned()),
+ Just(String::new()),
+ ]
+ .boxed();
+ let datetime = (0i64..2_000_000_000, 0u32..1_000_000)
+ .prop_map(|(seconds, micros)| {
+ json!(DateTime::<Utc>::from_timestamp(seconds, micros * 1000)
+ .unwrap()
+ .format("%Y-%m-%d %H:%M:%S%.6f")
+ .to_string())
+ })
+ .boxed();
+ match kind {
+ ColumnKind::String => text.prop_map(Value::String).boxed(),
+ ColumnKind::NullableString => prop::option::of(text).prop_map(|v| json!(v)).boxed(),
+ ColumnKind::DateTime64 => datetime,
+ ColumnKind::NullableDateTime64 => prop_oneof![Just(Value::Null), datetime].boxed(),
+ ColumnKind::NullableUInt16 => prop::option::of(any::<u16>())
+ .prop_map(|v| json!(v))
+ .boxed(),
+ ColumnKind::Bool => any::<bool>().prop_map(|v| json!(v)).boxed(),
+ ColumnKind::UInt8 => (0u8..=1).prop_map(|v| json!(v)).boxed(),
+ ColumnKind::UInt16 => any::<u16>().prop_map(|v| json!(v)).boxed(),
+ ColumnKind::UInt64 => (0u64..=9_007_199_254_740_991)
+ .prop_map(|v| json!(v))
+ .boxed(),
+ ColumnKind::Decimal => (-999_999.0f64..999_999.0).prop_map(|v| json!(v)).boxed(),
+ ColumnKind::ArrayString => prop::collection::vec(text, 0..=4)
+ .prop_map(|v| json!(v))
+ .boxed(),
+ ColumnKind::MapStringString => prop::collection::btree_map(text.clone(), text, 0..=4)
+ .prop_map(|v| json!(v))
+ .boxed(),
+ ColumnKind::MapStringUInt64 => {
+ prop::collection::btree_map(text, 0u64..=9_007_199_254_740_991, 0..=4)
+ .prop_map(|v| json!(v))
+ .boxed()
+ }
+ ColumnKind::MapStringDecimal => {
+ prop::collection::btree_map(text, -999_999.0f64..999_999.0, 0..=4)
+ .prop_map(|v| json!(v))
+ .boxed()
+ }
+ }
+}
+
+#[test]
+fn generated_prepared_rows_match_json_each_row_and_native() {
+ let Some(schema) = live_events_full_schema() else {
+ return;
+ };
+ assert_live_schema_matches_codec(&schema);
+ let mut runner = TestRunner::new(ProptestConfig {
+ cases: 32,
+ ..ProptestConfig::default()
+ });
+ runner
+ .run(
+ &(
+ prop::collection::vec(generated_row_strategy(&schema), 1..=8),
+ 1usize..=8,
+ ),
+ |(rows, max_rows)| {
+ let blocks = encode_json_rows(&rows, max_rows).map_err(TestCaseError::fail)?;
+ prop_assert_eq!(blocks.len(), rows.len().div_ceil(max_rows));
+ for (block, expected_rows) in blocks.iter().zip(rows.chunks(max_rows)) {
+ prop_assert_eq!(block.row_count, expected_rows.len());
+ }
+ let native_payload = blocks
+ .into_iter()
+ .flat_map(|block| block.bytes)
+ .collect::<Vec<_>>();
+ let json_payload = rows
+ .iter()
+ .flat_map(|row| {
+ let mut bytes = serde_json::to_vec(row).unwrap();
+ bytes.push(b'\n');
+ bytes
+ })
+ .collect::<Vec<_>>();
+ let comparison = clickhouse_server_round_trip(&json_payload, &native_payload)
+ .expect("schema introspection succeeded, so parity server is available");
+ prop_assert_eq!(comparison.json_count, rows.len() as u64);
+ prop_assert_eq!(comparison.native_count, rows.len() as u64);
+ prop_assert_eq!(comparison.json_only, 0);
+ prop_assert_eq!(comparison.native_only, 0);
+ prop_assert_eq!(
+ comparison.json_event_bytes,
+ rows.iter()
+ .map(|r| output_u64(&r["event_bytes"]))
+ .sum::<u64>()
+ );
+ prop_assert_eq!(
+ comparison.native_event_bytes,
+ rows.iter().map(|r| event_bytes(r).unwrap()).sum::<u64>()
+ );
+ Ok(())
+ },
+ )
+ .expect("generated prepared rows match JSONEachRow and Native");
+}
diff --git a/packages/native/src/native_schema.rs b/packages/native/src/native_schema.rs
new file mode 100644
index 0000000000..918be8a7a1
--- /dev/null
+++ b/packages/native/src/native_schema.rs
@@ -0,0 +1,262 @@
+//! The insertable part of `events_full` and its prepared Rust row.
+//!
+//! The macro invocation below is the only per-column declaration. It drives the prepared row,
+//! the live-schema metadata, JSON conversion, and the typed Native column writes together.
+
+use clickhouse::native::builder::BlockBuilder;
+use serde_json::Value;
+
+use crate::native_codec::{DateTime64Micros, Decimal64, FromJson};
+
+#[derive(Clone, Copy, Debug, Eq, PartialEq)]
+pub(crate) enum ColumnKind {
+ String,
+ DateTime64,
+ NullableDateTime64,
+ NullableString,
+ NullableUInt16,
+ Bool,
+ UInt8,
+ UInt16,
+ UInt64,
+ Decimal,
+ ArrayString,
+ MapStringString,
+ MapStringUInt64,
+ MapStringDecimal,
+}
+
+impl ColumnKind {
+ /// Convert the type spelling returned by `system.columns` to the codec's logical type.
+ #[cfg(test)]
+ pub(crate) fn from_clickhouse_type(value: &str) -> Result<Self, String> {
+ let normalized: String = value
+ .chars()
+ .filter(|character| !character.is_whitespace())
+ .collect();
+ match normalized.as_str() {
+ "String" | "LowCardinality(String)" => Ok(Self::String),
+ "DateTime64(6)" => Ok(Self::DateTime64),
+ "Nullable(DateTime64(6))" => Ok(Self::NullableDateTime64),
+ "Nullable(String)" | "Nullable(LowCardinality(String))" => Ok(Self::NullableString),
+ "Nullable(UInt16)" => Ok(Self::NullableUInt16),
+ "Bool" => Ok(Self::Bool),
+ "UInt8" => Ok(Self::UInt8),
+ "UInt16" => Ok(Self::UInt16),
+ "UInt64" => Ok(Self::UInt64),
+ "Decimal(18,12)" => Ok(Self::Decimal),
+ "Array(String)" | "Array(LowCardinality(String))" => Ok(Self::ArrayString),
+ "Map(String,String)" | "Map(LowCardinality(String),String)" => {
+ Ok(Self::MapStringString)
+ }
+ "Map(String,UInt64)" | "Map(LowCardinality(String),UInt64)" => {
+ Ok(Self::MapStringUInt64)
+ }
+ "Map(String,Decimal(18,12))" | "Map(LowCardinality(String),Decimal(18,12))" => {
+ Ok(Self::MapStringDecimal)
+ }
+ _ => Err(format!("unsupported events_full column type {value:?}")),
+ }
+ }
+}
+
+#[derive(Clone, Copy, Debug, Eq, PartialEq)]
+pub(crate) enum DefaultPolicy {
+ None,
+ Literal(&'static str),
+ Metadata(&'static str),
+ MetadataFallback(&'static str, &'static str),
+ MetadataEquals(&'static str, &'static str),
+}
+
+#[derive(Clone, Copy, Debug, Eq, PartialEq)]
+pub(crate) struct ColumnSpec {
+ pub(crate) name: &'static str,
+ pub(crate) kind: ColumnKind,
+ pub(crate) default_policy: DefaultPolicy,
+}
+
+macro_rules! schema_column_name {
+ ($field:ident) => {
+ stringify!($field)
+ };
+ ($field:ident as $name:literal) => {
+ $name
+ };
+}
+
+macro_rules! prepared_json_value {
+ (
+ $value:ident,
+ $event_bytes:ident,
+ event_bytes $(as $name:literal)?,
+ $ty:ty,
+ $policy:expr,
+ computed
+ ) => {
+ $event_bytes
+ };
+ (
+ $value:ident,
+ $event_bytes:ident,
+ $field:ident $(as $name:literal)?,
+ $ty:ty,
+ $policy:expr
+ ) => {
+ crate::native_codec::from_json_field::<$ty>(
+ $value,
+ schema_column_name!($field $(as $name)?),
+ $policy,
+ )?
+ };
+}
+
+/// Declare the prepared row and every operation that must agree with its shape.
+macro_rules! clickhouse_table_schema {
+ (
+ $(
+ $field:ident $(as $name:literal)? : $ty:ty => $( @ $source:ident )? $policy:expr
+ ),* $(,)?
+ ) => {
+ #[derive(Clone, Debug)]
+ pub(crate) struct PreparedEvent {
+ $(pub(crate) $field: $ty,)*
+ }
+
+pub(crate) const EVENTS_FULL_INSERT_COLUMNS: &[ColumnSpec] = &[
+ $(ColumnSpec {
+ name: schema_column_name!($field $(as $name)?),
+ kind: <$ty as FromJson>::COLUMN_KIND,
+ default_policy: $policy,
+ },)*
+ ];
+
+ impl PreparedEvent {
+ /// Convert one prepared ingestion object and compute accounting before typed defaults.
+ pub(crate) fn from_json(value: &Value) -> Result<Self, String> {
+ // `event_bytes` describes the immutable prepared JSON row. In particular, it
+ // intentionally sees raw decimal values and omitted DEFAULT-backed fields.
+ let event_bytes = crate::native_codec::event_bytes(value)?;
+
+ Ok(Self {
+ $(
+ $field: prepared_json_value!(
+ value,
+ event_bytes,
+ $field $(as $name)?,
+ $ty,
+ $policy $(, $source)?
+ ),
+ )*
+ })
+ }
+
+ /// Write this row type through clickhouse-rs's typed `Encode` implementations.
+ pub(crate) fn encode_into(
+ builder: &mut BlockBuilder,
+ rows: &[Self],
+ ) -> Result<(), String> {
+ $(
+ let name = schema_column_name!($field $(as $name)?);
+ let mut column = builder
+ .upsert_column::<&$ty>(name)
+ .map_err(|error| format!("cannot add column {name}: {error}"))?;
+ for row in rows {
+ column
+ .add(&row.$field)
+ .map_err(|error| format!("cannot encode column {name}: {error}"))?;
+ }
+ )*
+ Ok(())
+ }
+ }
+ };
+}
+
+// Keep this list in the same order as the insertable events_full contract. Materialized and alias
+// columns remain server-owned and therefore are intentionally absent.
+clickhouse_table_schema! {
+ project_id: String => DefaultPolicy::None,
+ trace_id: String => DefaultPolicy::None,
+ span_id: String => DefaultPolicy::None,
+ parent_span_id: String => DefaultPolicy::None,
+ name: String => DefaultPolicy::None,
+ environment: String => DefaultPolicy::Literal("default"),
+ version: String => DefaultPolicy::None,
+ release: String => DefaultPolicy::None,
+ trace_name: String => DefaultPolicy::None,
+ user_id: String => DefaultPolicy::None,
+ session_id: String => DefaultPolicy::None,
+ level: String => DefaultPolicy::None,
+ status_message: String => DefaultPolicy::None,
+ prompt_id: String => DefaultPolicy::None,
+ prompt_name: String => DefaultPolicy::None,
+ model_id: String => DefaultPolicy::None,
+ provided_model_name: String => DefaultPolicy::None,
+ model_parameters: String => DefaultPolicy::None,
+ input: String => DefaultPolicy::None,
+ output: String => DefaultPolicy::None,
+ evaluator_id: String => DefaultPolicy::Metadata("evaluator_id"),
+ evaluation_rule_id: String => {
+ DefaultPolicy::MetadataFallback("evaluation_rule_id", "job_configuration_id")
+ },
+ experiment_id: String => DefaultPolicy::None,
+ experiment_name: String => DefaultPolicy::None,
+ experiment_description: String => DefaultPolicy::None,
+ experiment_dataset_id: String => DefaultPolicy::None,
+ experiment_item_id: String => DefaultPolicy::None,
+ experiment_item_expected_output: String => DefaultPolicy::None,
+ experiment_item_root_span_id: String => DefaultPolicy::None,
+ source: String => DefaultPolicy::None,
+ service_name: String => DefaultPolicy::None,
+ service_version: String => DefaultPolicy::None,
+ scope_name: String => DefaultPolicy::None,
+ scope_version: String => DefaultPolicy::None,
+ telemetry_sdk_language: String => DefaultPolicy::None,
+ telemetry_sdk_name: String => DefaultPolicy::None,
+ telemetry_sdk_version: String => DefaultPolicy::None,
+ blob_storage_file_path: String => DefaultPolicy::None,
+ ingestion_api_key: String => DefaultPolicy::None,
+ ingestion_sdk_name: String => DefaultPolicy::None,
+ ingestion_sdk_version: String => DefaultPolicy::None,
+ event_type as "type": String => DefaultPolicy::None,
+ start_time: DateTime64Micros => DefaultPolicy::None,
+ created_at: DateTime64Micros => DefaultPolicy::None,
+ updated_at: DateTime64Micros => DefaultPolicy::None,
+ event_ts: DateTime64Micros => DefaultPolicy::None,
+ end_time: Option<DateTime64Micros> => DefaultPolicy::None,
+ completion_start_time: Option<DateTime64Micros> => DefaultPolicy::None,
+ experiment_item_version: Option<DateTime64Micros> => DefaultPolicy::None,
+ prompt_version: Option<u16> => DefaultPolicy::None,
+ is_app_root: bool => DefaultPolicy::None,
+ bookmarked: bool => DefaultPolicy::None,
+ public: bool => DefaultPolicy::None,
+ evaluator_execution_is_test: bool => {
+ DefaultPolicy::MetadataEquals("evaluator_test", "true")
+ },
+ is_deleted: u8 => DefaultPolicy::None,
+ event_bytes: u64 => @computed DefaultPolicy::None,
+ tags: Vec<String> => DefaultPolicy::None,
+ tool_calls: Vec<String> => DefaultPolicy::None,
+ tool_call_names: Vec<String> => DefaultPolicy::None,
+ metadata_names: Vec<String> => DefaultPolicy::None,
+ metadata_values: Vec<String> => DefaultPolicy::None,
+ experiment_metadata_names: Vec<String> => DefaultPolicy::None,
+ experiment_metadata_values: Vec<String> => DefaultPolicy::None,
+ experiment_item_metadata_names: Vec<String> => DefaultPolicy::None,
+ experiment_item_metadata_values: Vec<String> => DefaultPolicy::None,
+ provided_usage_details: std::collections::BTreeMap<String, u64> => DefaultPolicy::None,
+ usage_details: std::collections::BTreeMap<String, u64> => DefaultPolicy::None,
+ provided_cost_details: std::collections::BTreeMap<String, Decimal64> => DefaultPolicy::None,
+ cost_details: std::collections::BTreeMap<String, Decimal64> => DefaultPolicy::None,
+ usage_pricing_tier_id: Option<String> => DefaultPolicy::None,
+ usage_pricing_tier_name: Option<String> => DefaultPolicy::None,
+ tool_definitions: std::collections::BTreeMap<String, String> => DefaultPolicy::None,
+}
+
+#[cfg(test)]
+pub(crate) fn find_column(name: &str) -> Option<&'static ColumnSpec> {
+ EVENTS_FULL_INSERT_COLUMNS
+ .iter()
+ .find(|column| column.name == name)
+}
diff --git a/worker/Dockerfile b/worker/Dockerfile
index f83123fbc2..e00649a738 100644
--- a/worker/Dockerfile
+++ b/worker/Dockerfile
@@ -37,14 +37,15 @@ WORKDIR /app
# The toolchain comes from the official Rust image for the same Alpine release,
# so the addon compiles with the exact version pinned in
# packages/native/rust-toolchain.toml; bump the two together. gcc and musl-dev
-# provide the linker and C runtime objects the build needs.
+# provide the linker and C runtime objects the build needs; curl and patch
+# prepare the small clickhouse source overlay used by the native encoder.
COPY --from=rust:1.98.0-alpine3.24 /usr/local/rustup /usr/local/rustup
COPY --from=rust:1.98.0-alpine3.24 /usr/local/cargo /usr/local/cargo
ENV RUSTUP_HOME=/usr/local/rustup CARGO_HOME=/usr/local/cargo PATH=/usr/local/cargo/bin:$PATH
# rustc's musl targets default to a `<arch>-linux-musl-gcc` linker name that
# Alpine does not ship; its gcc is reachable as `cc`.
ENV CARGO_TARGET_AARCH64_UNKNOWN_LINUX_MUSL_LINKER=cc CARGO_TARGET_X86_64_UNKNOWN_LINUX_MUSL_LINKER=cc
-RUN apk add --no-cache gcc musl-dev
+RUN apk add --no-cache gcc musl-dev curl patch
# First install the dependencies (as they change less often)
COPY --from=pruner /app/out/pnpm-lock.yaml ./pnpm-lock.yaml
diff --git a/worker/package.json b/worker/package.json
index 14ce6e48db..38d4e3848c 100644
--- a/worker/package.json
+++ b/worker/package.json
@@ -27,7 +27,8 @@
"refill-ingestion-events": "dotenv -e ../.env -- tsx src/scripts/replayIngestionEvents/s3-ingestion-event-replay.ts",
"refill-billing-event": "dotenv -e ../.env -- tsx src/scripts/refill-billing-event.ts",
"refill-queue-event": "dotenv -e ../.env -- tsx src/scripts/refillQueueEvent/index.ts",
- "retry-monitor-bad-query-errors": "dotenv -e ../.env -- tsx src/scripts/retryMonitorBadQueryErrors.ts"
+ "retry-monitor-bad-query-errors": "dotenv -e ../.env -- tsx src/scripts/retryMonitorBadQueryErrors.ts",
+ "test:native-codec": "LANGFUSE_NATIVE_FAIL_IF_CLICKHOUSE_UNAVAILABLE=1 vitest run nativeCodec.integration"
},
"author": "[email protected]",
"dependencies": {
diff --git a/worker/src/__tests__/helpers/nativeCodecFixtures.ts b/worker/src/__tests__/helpers/nativeCodecFixtures.ts
new file mode 100644
index 0000000000..64286fbd8a
--- /dev/null
+++ b/worker/src/__tests__/helpers/nativeCodecFixtures.ts
@@ -0,0 +1,223 @@
+import { readFileSync, readdirSync } from "node:fs";
+import { resolve } from "node:path";
+import type { EventRecordInsertType } from "@langfuse/shared/src/server";
+import type { EventInput } from "../../services/IngestionService/index.js";
+
+// Read the existing trace captures and exercise production createEventRecord/writeEventRecord.
+// Model lookup responses are replayed from captures; no fixture JSON is copied into Rust.
+type JsonObject = Record<string, unknown>;
+const repoRoot = resolve(__dirname, "../../../..");
+
+const parseObject = (value: unknown): JsonObject => {
+ if (value && typeof value === "object" && !Array.isArray(value)) {
+ return value as JsonObject;
+ }
+ if (typeof value === "string") {
+ try {
+ const parsed: unknown = JSON.parse(value);
+ return parseObject(parsed);
+ } catch {
+ return {};
+ }
+ }
+ return {};
+};
+
+const numberMap = (value: unknown): Record<string, number> =>
+ Object.fromEntries(
+ Object.entries(parseObject(value)).flatMap(([key, item]) => {
+ const number = typeof item === "number" ? item : Number(item);
+ return Number.isFinite(number) ? [[key, number]] : [];
+ }),
+ );
+
+const stringArray = (value: unknown): string[] =>
+ Array.isArray(value)
+ ? value.filter((item): item is string => typeof item === "string")
+ : [];
+
+const stringMap = (value: unknown): Record<string, string> =>
+ Object.fromEntries(
+ Object.entries(parseObject(value)).map(([key, item]) => [
+ key,
+ typeof item === "string" ? item : JSON.stringify(item),
+ ]),
+ );
+
+const optionalString = (value: unknown): string | undefined =>
+ typeof value === "string" ? value : undefined;
+
+const mergeObjects = (...values: unknown[]): JsonObject =>
+ Object.assign({}, ...values.map(parseObject));
+
+const fixtureDirectories = [
+ resolve(repoRoot, "packages/shared/scripts/seeder/utils/framework-traces"),
+ resolve(repoRoot, "worker/src/__tests__/chatml/framework-traces"),
+];
+
+const fixturePaths = fixtureDirectories
+ .flatMap((directory) =>
+ readdirSync(directory)
+ .filter((name) =>
+ directory.endsWith("chatml/framework-traces")
+ ? name.endsWith(".trace.json")
+ : name.endsWith(".json"),
+ )
+ .map((name) => resolve(directory, name)),
+ )
+ .sort();
+
+const eventInputFromCapture = (
+ observation: JsonObject,
+ trace: JsonObject,
+): EventInput => {
+ const isRoot = observation.parentObservationId == null;
+ const usageDetails = numberMap(observation.usageDetails);
+ const costDetails = numberMap(observation.costDetails);
+ const providedUsageDetails = numberMap(observation.providedUsageDetails);
+ const providedCostDetails = numberMap(observation.providedCostDetails);
+
+ return {
+ projectId: String(observation.projectId ?? trace.projectId ?? "fixture"),
+ traceId: String(observation.traceId ?? trace.id),
+ spanId: String(observation.id),
+ parentSpanId:
+ typeof observation.parentObservationId === "string"
+ ? observation.parentObservationId
+ : undefined,
+ startTimeISO: String(observation.startTime),
+ endTimeISO: String(observation.endTime ?? observation.startTime),
+ completionStartTime: optionalString(observation.completionStartTime),
+ name: typeof observation.name === "string" ? observation.name : "",
+ type: typeof observation.type === "string" ? observation.type : "SPAN",
+ environment: String(
+ observation.environment ?? trace.environment ?? "default",
+ ),
+ version:
+ typeof observation.version === "string" ? observation.version : undefined,
+ release: typeof trace.release === "string" ? trace.release : undefined,
+ traceName: typeof trace.name === "string" ? trace.name : undefined,
+ tags: Array.isArray(trace.tags)
+ ? trace.tags.filter((tag): tag is string => typeof tag === "string")
+ : [],
+ public: trace.public === true,
+ bookmarked: isRoot && trace.bookmarked === true,
+ userId: typeof trace.userId === "string" ? trace.userId : undefined,
+ sessionId:
+ typeof trace.sessionId === "string" ? trace.sessionId : undefined,
+ level:
+ typeof observation.level === "string" ? observation.level : undefined,
+ statusMessage:
+ typeof observation.statusMessage === "string"
+ ? observation.statusMessage
+ : undefined,
+ modelName:
+ typeof observation.model === "string" ? observation.model : undefined,
+ modelId: optionalString(observation.internalModelId),
+ modelParameters:
+ observation.modelParameters as EventInput["modelParameters"],
+ providedUsageDetails:
+ Object.keys(providedUsageDetails).length > 0
+ ? providedUsageDetails
+ : usageDetails,
+ usageDetails,
+ providedCostDetails:
+ Object.keys(providedCostDetails).length > 0
+ ? providedCostDetails
+ : costDetails,
+ costDetails,
+ usagePricingTierId: optionalString(observation.usagePricingTierId),
+ usagePricingTierName: optionalString(observation.usagePricingTierName),
+ promptId: optionalString(observation.promptId),
+ promptName: optionalString(observation.promptName),
+ promptVersion:
+ observation.promptVersion == null
+ ? undefined
+ : String(observation.promptVersion),
+ toolDefinitions: stringMap(observation.toolDefinitions),
+ toolCalls: stringArray(observation.toolCalls),
+ toolCallNames: stringArray(observation.toolCallNames),
+ input: observation.input as string | undefined,
+ output: observation.output as string | undefined,
+ metadata: mergeObjects(trace.metadata, observation.metadata),
+ source: "API",
+ ingestionApiKey: "",
+ ingestionSdkName: "",
+ ingestionSdkVersion: "",
+ blobStorageFilePath: "fixtures",
+ eventBytes: 0,
+ };
+};
+
+export const capturedPreparedRows = async () => {
+ const { IngestionService } =
+ await import("../../services/IngestionService/index.js");
+
+ const normalizedRows: EventRecordInsertType[] = [];
+ const capturedGenerationUsage = new Map<
+ string,
+ Pick<
+ EventRecordInsertType,
+ | "model_id"
+ | "usage_pricing_tier_id"
+ | "usage_pricing_tier_name"
+ | "usage_details"
+ | "cost_details"
+ >
+ >();
+ const writer = {
+ addToQueue: (_table: unknown, record: EventRecordInsertType) => {
+ normalizedRows.push(record);
+ },
+ };
+ const service = new IngestionService(
+ {} as never,
+ {} as never,
+ writer as never,
+ {} as never,
+ );
+
+ // Avoid external model and prompt lookups while retaining the production method's field mapping.
+ // Captured generation enrichment is replayed where the source trace already contains it.
+ (service as any).getGenerationUsage = async ({
+ observationRecord,
+ }: {
+ observationRecord: EventRecordInsertType;
+ }) => ({
+ usage_details: capturedGenerationUsage.get(observationRecord.id)
+ ?.usage_details,
+ cost_details: capturedGenerationUsage.get(observationRecord.id)
+ ?.cost_details,
+ internal_model_id: capturedGenerationUsage.get(observationRecord.id)
+ ?.model_id,
+ usage_pricing_tier_id: capturedGenerationUsage.get(observationRecord.id)
+ ?.usage_pricing_tier_id,
+ usage_pricing_tier_name: capturedGenerationUsage.get(observationRecord.id)
+ ?.usage_pricing_tier_name,
+ });
+
+ for (const fixturePath of fixturePaths) {
+ const fixture = JSON.parse(readFileSync(fixturePath, "utf8")) as {
+ trace?: JsonObject;
+ observations?: JsonObject[];
+ };
+ const trace = fixture.trace ?? {};
+ for (const observation of fixture.observations ?? []) {
+ const eventInput = eventInputFromCapture(observation, trace);
+ capturedGenerationUsage.set(eventInput.spanId, {
+ usage_details: eventInput.usageDetails ?? {},
+ cost_details: eventInput.costDetails ?? {},
+ model_id: eventInput.modelId,
+ usage_pricing_tier_id: eventInput.usagePricingTierId,
+ usage_pricing_tier_name: eventInput.usagePricingTierName,
+ });
+ const eventRecord = await service.createEventRecord(
+ eventInput,
+ `fixtures/${fixturePath.split("/").pop()}`,
+ );
+ await service.writeEventRecord(eventRecord);
+ }
+ }
+
+ return normalizedRows;
+};
diff --git a/worker/src/__tests__/nativeCodec.integration.test.ts b/worker/src/__tests__/nativeCodec.integration.test.ts
new file mode 100644
index 0000000000..38f2f6bd95
--- /dev/null
+++ b/worker/src/__tests__/nativeCodec.integration.test.ts
@@ -0,0 +1,184 @@
+import { execFileSync } from "node:child_process";
+import { mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { resolve } from "node:path";
+import { randomUUID } from "node:crypto";
+import { afterAll, beforeAll, describe, expect, it } from "vitest";
+import {
+ clickhouseClient,
+ type ClickhouseClientType,
+ type EventRecordInsertType,
+} from "@langfuse/shared/src/server";
+import { ClickhouseWriter, TableName } from "../services/ClickhouseWriter";
+import { capturedPreparedRows } from "./helpers/nativeCodecFixtures";
+
+type Row = Record<string, unknown>;
+type Column = { name: string; type: string; default_kind: string };
+type Manifest = {
+ eventBytes: number[];
+};
+const nativeDir = resolve(__dirname, "../../../packages/native");
+
+// This suite is run explicitly by test:native-codec and CI. Ordinary worker tests need neither
+// cargo nor a Native-protocol client. The explicit run always fails if ClickHouse is unavailable.
+describe.runIf(
+ process.env.LANGFUSE_NATIVE_FAIL_IF_CLICKHOUSE_UNAVAILABLE === "1",
+)("TypeScript and Rust Native ingestion parity", () => {
+ let directory: string;
+ let columns: Column[];
+ let writer: ClickhouseWriter;
+ let jsonTable: string;
+ let nativeTable: string;
+ const client = clickhouseClient();
+
+ const query = (sql: string, input: Buffer = Buffer.alloc(0)) =>
+ execFileSync(
+ process.env.LANGFUSE_NATIVE_CLICKHOUSE_BIN ?? "clickhouse",
+ [
+ "client",
+ "--host",
+ process.env.LANGFUSE_NATIVE_CLICKHOUSE_HOST ?? "127.0.0.1",
+ "--port",
+ process.env.LANGFUSE_NATIVE_CLICKHOUSE_PORT ?? "9000",
+ "--user",
+ process.env.CLICKHOUSE_USER ?? "clickhouse",
+ "--password",
+ process.env.CLICKHOUSE_PASSWORD ?? "clickhouse",
+ "--database",
+ process.env.CLICKHOUSE_DB ?? "default",
+ "--multiquery",
+ "--query",
+ sql,
+ ],
+ { input, maxBuffer: 16 * 1024 * 1024, timeout: 30_000 },
+ );
+
+ const encode = (rows: Row[], maxRowsPerBlock: number): Manifest => {
+ const input = resolve(directory, "input.json");
+ writeFileSync(input, JSON.stringify({ rows, maxRowsPerBlock }));
+ execFileSync(
+ "cargo",
+ ["run", "--locked", "--example", "native_codec_test_driver"],
+ {
+ cwd: nativeDir,
+ env: {
+ ...process.env,
+ LANGFUSE_NATIVE_TEST_INPUT: input,
+ LANGFUSE_NATIVE_TEST_OUTPUT: directory,
+ },
+ timeout: 120_000,
+ stdio: "pipe",
+ },
+ );
+ return JSON.parse(
+ readFileSync(resolve(directory, "manifest.json"), "utf8"),
+ ) as Manifest;
+ };
+
+ beforeAll(async () => {
+ query("SELECT 1");
+ directory = mkdtempSync(resolve(tmpdir(), "native-codec-"));
+ execFileSync("sh", ["scripts/prepare-clickhouse-rs.sh"], {
+ cwd: nativeDir,
+ timeout: 120_000,
+ });
+ columns = query(
+ "SELECT name, type, default_kind FROM system.columns WHERE database=currentDatabase() AND table='events_full' ORDER BY position FORMAT JSONEachRow",
+ )
+ .toString()
+ .trim()
+ .split("\n")
+ .map((line) => JSON.parse(line) as Column);
+ // Exercise production writer clamping and the real JS client's JSONEachRow serializer.
+ // Only redirect the destination table; leave the records and insert settings untouched.
+ writer = ClickhouseWriter.getInstance({
+ insert: async (params: Parameters<ClickhouseClientType["insert"]>[0]) => {
+ return client.insert({ ...params, table: jsonTable });
+ },
+ } as ClickhouseClientType);
+ if (writer.intervalId) clearInterval(writer.intervalId);
+ writer.intervalId = null;
+ writer.batchSize = Number.MAX_SAFE_INTEGER;
+ }, 120_000);
+
+ afterAll(async () => {
+ if (writer) await writer.shutdown();
+ await client.close();
+ if (directory) rmSync(directory, { recursive: true, force: true });
+ });
+
+ const compare = async (rows: Row[], maxRows: number) => {
+ const suffix = randomUUID().replaceAll("-", "");
+ jsonTable = `native_codec_json_${suffix}`;
+ nativeTable = `native_codec_native_${suffix}`;
+ const manifest = encode(rows, maxRows);
+ try {
+ query(
+ `CREATE TABLE ${jsonTable} AS events_full ENGINE=Memory; CREATE TABLE ${nativeTable} AS events_full ENGINE=Memory`,
+ );
+ for (const row of rows)
+ writer.addToQueue(
+ TableName.EventsFull,
+ structuredClone(row) as EventRecordInsertType,
+ );
+ await writer.flushAll(true);
+ query(
+ `INSERT INTO ${nativeTable} FORMAT Native`,
+ readFileSync(resolve(directory, "native.bin")),
+ );
+ // Compare typed values inside ClickHouse: preserve Decimal/UInt64 precision, duplicates,
+ // defaults, materialized and alias columns. Map key order is not semantically meaningful.
+ const maps = columns
+ .filter((c) => c.type.startsWith("Map("))
+ .map((c) => c.name);
+ const projection = `* EXCEPT (event_bytes, ${maps.join(", ")}), ${maps.map((name) => `mapSort((k, v) -> k, ${name}) AS ${name}`).join(", ")}`;
+ const select = (table: string) => `SELECT ${projection} FROM ${table}`;
+ const result = JSON.parse(
+ query(`SELECT
+ (SELECT count() FROM ${jsonTable}) AS json_count,
+ (SELECT count() FROM ${nativeTable}) AS native_count,
+ (SELECT count() FROM (${select(jsonTable)} EXCEPT ALL ${select(nativeTable)})) AS json_only,
+ (SELECT count() FROM (${select(nativeTable)} EXCEPT ALL ${select(jsonTable)})) AS native_only,
+ (SELECT sum(event_bytes) FROM ${jsonTable}) AS json_bytes,
+ (SELECT sum(event_bytes) FROM ${nativeTable}) AS native_bytes
+ SETTINGS asterisk_include_materialized_columns=1, asterisk_include_alias_columns=1 FORMAT JSONEachRow`).toString(),
+ ) as Record<string, string | number>;
+ expect(Number(result.json_count)).toBe(rows.length);
+ expect(Number(result.native_count)).toBe(rows.length);
+ expect(Number(result.json_only)).toBe(0);
+ expect(Number(result.native_only)).toBe(0);
+ expect(BigInt(result.json_bytes)).toBe(
+ rows.reduce((sum, row) => sum + BigInt(row.event_bytes as number), 0n),
+ );
+ expect(BigInt(result.native_bytes)).toBe(
+ manifest.eventBytes.reduce((sum, bytes) => sum + BigInt(bytes), 0n),
+ );
+ } finally {
+ // Discard failed writes before shutdown can retry against the temporary table.
+ writer.queue[TableName.EventsFull] = [];
+ query(
+ `DROP TABLE IF EXISTS ${jsonTable}; DROP TABLE IF EXISTS ${nativeTable}`,
+ );
+ }
+ };
+
+ it("compares captured traffic through production TS preparation and both writers", async () => {
+ const rows = await capturedPreparedRows();
+ expect(rows.length).toBeGreaterThan(0);
+ // This capture has inferred costs despite explicitly provided zero costs.
+ const enriched = rows.find(
+ (row) => row.span_id === "0d8f5427-c91a-44c8-86c5-e12d806edd57",
+ );
+ expect(enriched?.provided_cost_details).toEqual({
+ input: 0,
+ output: 0,
+ total: 0,
+ });
+ expect(enriched?.cost_details).toEqual({
+ input: 0.00147,
+ output: 0.0024,
+ total: 0.00387,
+ });
+ await compare(rows, 16);
+ }, 120_000);
+});