🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
feat: encode v4 native blocks and event bytes in Rust (#17563)
Valery Meleshkinb0608e3a14ed
diff --git a/.github/workflows/pipeline.yml b/.github/workflows/pipeline.ymlindex 1f376b4e41..a74c8d83ef 100644--- a/.github/workflows/pipeline.yml+++ b/.github/workflows/pipeline.yml@@ -1002,6 +1002,13 @@ jobs:pnpm --filter=shared ch:dev-tables- name: Buildrun: pnpm --filter=worker... run build+ - name: run native codec tests against ClickHouse+ if: matrix.deploy-mode == ''+ env:+ LANGFUSE_NATIVE_FAIL_IF_CLICKHOUSE_UNAVAILABLE: "1"+ run: |+ pnpm --filter @langfuse/native run test+ pnpm --filter worker run test:native-codec- name: run testsrun: pnpm --filter=worker run test:exclude-llm-connectionsenv:diff --git a/packages/native/.gitignore b/packages/native/.gitignoreindex 3304cd5642..0202152254 100644--- a/packages/native/.gitignore+++ b/packages/native/.gitignore@@ -1,5 +1,8 @@# Cargo build outputtarget/+# Generated Cargo overlay for the pinned clickhouse source patch+.cargo/+# Platform-specific binaries produced by `napi build`*.nodediff --git a/packages/native/Cargo.lock b/packages/native/Cargo.lockindex 02b45de9bf..fe9600dfac 100644--- a/packages/native/Cargo.lock+++ b/packages/native/Cargo.lock@@ -11,6 +11,24 @@ dependencies = ["memchr",]+[[package]]+name = "allocator-api2"+version = "0.2.21"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "683d7910e743518b0e34f1186f92494becacb047c7b6bf616c96772180fef923"++[[package]]+name = "arrayvec"+version = "0.7.8"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "d3fb67a6e08acf24fdeccbac2cb6ac4305825bd1f117462e0e6f2f193345ad56"++[[package]]+name = "atomic-waker"+version = "1.1.2"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0"+[[package]]name = "autocfg"version = "1.5.1"@@ -23,6 +41,45 @@ version = "2.13.1"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "b588b76d00fde79687d7646a9b5bdf3cc0f655e0bbd080335a95d7e96f3587da"+[[package]]+name = "bnum"+version = "0.13.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "119771309b95163ec7aaf79810da82f7cd0599c19722d48b9c03894dca833966"++[[package]]+name = "borsh"+version = "1.8.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "553c5d846a6ba5150c65e3b1b8ec073bcf1abc20f9b7220de384a4443ea4e20a"+dependencies = [+ "borsh-derive",+ "bytes",+ "cfg_aliases",+]++[[package]]+name = "borsh-derive"+version = "1.8.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "12cdfe656708a01f89b451a7d36466e6fe6c414de0aa18fc54f864f6f9ca9f56"+dependencies = [+ "once_cell",+ "proc-macro-crate",+ "proc-macro2",+ "quote",+ "syn 3.0.4",+]++[[package]]+name = "bstr"+version = "1.13.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "6bb31b46c14244e20ee9984b11bf5c992b91fb6939fea616e3512c8baecdbe5f"+dependencies = [+ "memchr",+]+[[package]]name = "bumpalo"version = "3.20.3"@@ -34,6 +91,9 @@ name = "bytes"version = "1.12.1"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "fc652a48c352aef3ea3aed32080501cf3ef6ed5da78602a020c991775b0aff04"+dependencies = [+ "serde",+][[package]]name = "cfg-if"@@ -41,6 +101,67 @@ version = "1.0.4"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801"+[[package]]+name = "cfg_aliases"+version = "0.2.2"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "f079e83a288787bcd14a6aea84cee5c87a67c5a3e660c30f557a3d24761b3527"++[[package]]+name = "chrono"+version = "0.4.45"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "1aa79e62e7697b8e29b513a68abacf485adcd1fe8284a4316c5ae868e6633327"+dependencies = [+ "num-traits",+]++[[package]]+name = "clickhouse"+version = "0.15.2"+dependencies = [+ "bnum",+ "bstr",+ "bytes",+ "clickhouse-macros",+ "clickhouse-types",+ "futures-channel",+ "futures-util",+ "half",+ "hashbrown 0.17.1",+ "http-body-util",+ "hyper",+ "hyper-util",+ "serde",+ "serde_json",+ "thiserror",+ "tokio",+ "tracing",+ "url",+]++[[package]]+name = "clickhouse-macros"+version = "0.3.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "ff6669899e23cb87b43daf7996f0ea3b9c07d0fb933d745bb7b815b052515ae3"+dependencies = [+ "proc-macro2",+ "quote",+ "serde_derive_internals",+ "syn 2.0.119",+]++[[package]]+name = "clickhouse-types"+version = "0.1.3"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "3966eea13bfc826e24d471e49cb2095727261009be59d0ac5a6c0be0286974d1"+dependencies = [+ "bytes",+ "thiserror",+]+[[package]]name = "convert_case"version = "0.11.0"@@ -65,12 +186,29 @@ version = "0.8.22"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "61803da095bee82a81bb1a452ecc25d3b2f1416d1897eb86430c6159ef717c17"+[[package]]+name = "crunchy"+version = "0.2.4"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "460fbee9c2c2f33933d720630a6a0bac33ba7053db5344fac858d4b8952d77d5"+[[package]]name = "ctor"version = "1.0.13"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "914a755b7c2d4af2bdcff7ce1739e2db9a1b81a9b07123d8015786ae03c0980d"+[[package]]+name = "displaydoc"+version = "0.2.7"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "c6232dd377dcc64799954cbd3a9bb882e9cdc1308ccd87b1c098f1fb2eaf82a8"+dependencies = [+ "proc-macro2",+ "quote",+ "syn 3.0.4",+]+[[package]]name = "endian-type"version = "0.1.2"@@ -89,6 +227,15 @@ version = "0.2.0"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "77ce24cb58228fbb8aa041425bb1050850ac19177686ea6e0f41a70416f56fdb"+[[package]]+name = "form_urlencoded"+version = "1.2.2"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "cb4cb245038516f5f85277875cdaa4f7d2c9a0fa0468de06ed190163b1581fcf"+dependencies = [+ "percent-encoding",+]+[[package]]name = "futures"version = "0.3.34"@@ -177,6 +324,17 @@ dependencies = ["slab",]+[[package]]+name = "getrandom"+version = "0.2.17"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "ff2abc00be7fca6ebc474524697ae276ad847ad0a6b3faa4bcb027e9a4614ad0"+dependencies = [+ "cfg-if",+ "libc",+ "wasi",+]+[[package]]name = "getrandom"version = "0.3.4"@@ -189,6 +347,17 @@ dependencies = ["wasip2",]+[[package]]+name = "half"+version = "2.7.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "6ea2d84b969582b4b1864a92dc5d27cd2b77b622a8d79306834f1be5ba20d84b"+dependencies = [+ "cfg-if",+ "crunchy",+ "zerocopy",+]+[[package]]name = "hashbrown"version = "0.16.1"@@ -203,6 +372,194 @@ name = "hashbrown"version = "0.17.1"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a"+dependencies = [+ "allocator-api2",+ "equivalent",+ "foldhash",+]++[[package]]+name = "http"+version = "1.5.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "918d3568bebf352712bc2ef3d46a8bcf1a75b373be6539de198e9105cbbf9ce0"+dependencies = [+ "bytes",+ "itoa",+]++[[package]]+name = "http-body"+version = "1.1.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "ca2a8f2913ee65f60facd6a5905613afaa448497a0230cc41ce022d93290bc2c"+dependencies = [+ "bytes",+ "http",+]++[[package]]+name = "http-body-util"+version = "0.1.5"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "23169fe34a5fbcdd3f3862e78fb9b6fccd5f02a6dc6f732547005d45631ce71c"+dependencies = [+ "bytes",+ "futures-core",+ "http",+ "http-body",+ "pin-project-lite",+]++[[package]]+name = "httparse"+version = "1.10.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "6dbf3de79e51f3d586ab4cb9d5c3e2c14aa28ed23d180cf89b4df0454a69cc87"++[[package]]+name = "hyper"+version = "1.11.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "27b501faa50e7a26c3d3560ca625132f4078a17771f4810baf70475ae48cbe43"+dependencies = [+ "atomic-waker",+ "bytes",+ "futures-channel",+ "futures-core",+ "http",+ "http-body",+ "httparse",+ "itoa",+ "pin-project-lite",+ "smallvec",+ "tokio",+ "want",+]++[[package]]+name = "hyper-util"+version = "0.1.20"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "96547c2556ec9d12fb1578c4eaf448b04993e7fb79cbaad930a656880a6bdfa0"+dependencies = [+ "bytes",+ "futures-channel",+ "futures-util",+ "http",+ "http-body",+ "hyper",+ "libc",+ "pin-project-lite",+ "socket2",+ "tokio",+ "tower-service",+ "tracing",+]++[[package]]+name = "icu_collections"+version = "2.3.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "fa68d21081c4a05d5a901a1c62add574c77048b6a1c67be3b50ce0b60d4ca513"+dependencies = [+ "displaydoc",+ "potential_utf",+ "utf8_iter",+ "yoke",+ "zerofrom",+ "zerovec",+]++[[package]]+name = "icu_locale_core"+version = "2.3.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "d56e28588da92eee5c3201a6eff33fabdd49b62269c8938d4ff050ce4d900deb"+dependencies = [+ "displaydoc",+ "litemap",+ "tinystr",+ "writeable",+ "zerovec",+]++[[package]]+name = "icu_normalizer"+version = "2.3.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "12f9cf5f235641ed274641dd81c3f28d870e276763d0797aeeab72317b1c646f"+dependencies = [+ "icu_collections",+ "icu_normalizer_data",+ "icu_properties",+ "icu_provider",+ "smallvec",+ "zerovec",+]++[[package]]+name = "icu_normalizer_data"+version = "2.3.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "1563da1ed3e0b3bf3d74c9b85917ac9c56464d2f57242270c09c9e752f8021a0"++[[package]]+name = "icu_properties"+version = "2.3.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "7e7ca276ad3145661a65914e6daf131ca5120cd3dcee8f8f3214b8875184a148"+dependencies = [+ "displaydoc",+ "icu_collections",+ "icu_locale_core",+ "icu_properties_data",+ "icu_provider",+ "zerotrie",+ "zerovec",+]++[[package]]+name = "icu_properties_data"+version = "2.3.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "e590f038c1464a96894fd6d10127e90a8be4509f56ff7ecef851b15cee0b7caa"++[[package]]+name = "icu_provider"+version = "2.3.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "d27bbb9d3abbefac45d55f647c9de1d44aafcd1186eb91879afef17c396c3e73"+dependencies = [+ "displaydoc",+ "icu_locale_core",+ "writeable",+ "yoke",+ "zerofrom",+ "zerotrie",+ "zerovec",+]++[[package]]+name = "idna"+version = "1.1.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "3b0875f23caa03898994f6ddc501886a45c7d3d62d04d2d90788d47be1b1e4de"+dependencies = [+ "idna_adapter",+ "smallvec",+ "utf8_iter",+]++[[package]]+name = "idna_adapter"+version = "1.2.2"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "cb68373c0d6620ef8105e855e7745e18b0d00d3bdb07fb532e434244cdb9a714"+dependencies = [+ "icu_normalizer",+ "icu_properties",+][[package]]name = "indexmap"@@ -235,11 +592,19 @@ dependencies = [name = "langfuse-native"version = "0.0.0"dependencies = [+ "bytes",+ "chrono",+ "clickhouse",+ "clickhouse-types","metrics","metrics-exporter-dogstatsd","napi","napi-build","napi-derive",+ "proptest",+ "rust_decimal",+ "serde",+ "serde_json","tracing","tracing-subscriber",]@@ -266,6 +631,12 @@ dependencies = ["windows-link",]+[[package]]+name = "litemap"+version = "0.8.3"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "47d9d19d1d6efa0109d2f65ff4c85cddd50bd572e5a00127ab10987290bcefae"+[[package]]name = "log"version = "0.4.34"@@ -327,12 +698,23 @@ dependencies = ["ordered-float","quanta","radix_trie",- "rand",+ "rand 0.9.5","rand_xoshiro","rapidhash","sketches-ddsketch",]+[[package]]+name = "mio"+version = "1.2.3"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "4b18443e9c262bfe8fa82f51666e2642c53393f7e5c27b3e1aeab922cff5b9d8"+dependencies = [+ "libc",+ "wasi",+ "windows-sys",+]+[[package]]name = "napi"version = "3.12.2"@@ -439,6 +821,12 @@ dependencies = ["num-traits",]+[[package]]+name = "percent-encoding"+version = "2.3.2"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220"+[[package]]name = "pin-project-lite"version = "0.2.17"@@ -451,6 +839,15 @@ version = "1.15.0"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "05c8b63e8d9609db387f0324918f81d68fe27748f084ef092fb35954d0539a85"+[[package]]+name = "potential_utf"+version = "0.1.6"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "d83eb9bc6d8e5cf568e7a1101d60ee05e81ed50ea106026f3d18deeb046d7661"+dependencies = [+ "zerovec",+]+[[package]]name = "ppv-lite86"version = "0.2.21"@@ -460,6 +857,15 @@ dependencies = ["zerocopy",]+[[package]]+name = "proc-macro-crate"+version = "3.5.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "e67ba7e9b2b56446f1d419b1d807906278ffa1a658a8a5d8a39dcb1f5a78614f"+dependencies = [+ "toml_edit",+]+[[package]]name = "proc-macro2"version = "1.0.107"@@ -469,6 +875,21 @@ dependencies = ["unicode-ident",]+[[package]]+name = "proptest"+version = "1.11.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "4b45fcc2344c680f5025fe57779faef368840d0bd1f42f216291f0dc4ace4744"+dependencies = [+ "bitflags",+ "num-traits",+ "rand 0.9.5",+ "rand_chacha 0.9.0",+ "rand_xorshift",+ "regex-syntax",+ "unarray",+]+[[package]]name = "quanta"version = "0.12.6"@@ -509,14 +930,35 @@ dependencies = ["nibble_vec",]+[[package]]+name = "rand"+version = "0.8.8"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "e058c7de0b26af77780c769414d6257830bb240f3c38477dbc2c16e5f54d6d4c"+dependencies = [+ "libc",+ "rand_chacha 0.3.1",+ "rand_core 0.6.4",+]+[[package]]name = "rand"version = "0.9.5"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "b9ef1d0d795eb7d84685bca4f72f3649f064e6641543d3a8c415898726a57b41"dependencies = [- "rand_chacha",- "rand_core",+ "rand_chacha 0.9.0",+ "rand_core 0.9.5",+]++[[package]]+name = "rand_chacha"+version = "0.3.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88"+dependencies = [+ "ppv-lite86",+ "rand_core 0.6.4",][[package]]@@ -526,7 +968,16 @@ source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb"dependencies = ["ppv-lite86",- "rand_core",+ "rand_core 0.9.5",+]++[[package]]+name = "rand_core"+version = "0.6.4"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c"+dependencies = [+ "getrandom 0.2.17",][[package]]@@ -535,7 +986,16 @@ version = "0.9.5"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "76afc826de14238e6e8c374ddcc1fa19e374fd8dd986b0d2af0d02377261d83c"dependencies = [- "getrandom",+ "getrandom 0.3.4",+]++[[package]]+name = "rand_xorshift"+version = "0.4.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "513962919efc330f829edb2535844d1b912b0fbe2ca165d613e4e8788bb05a5a"+dependencies = [+ "rand_core 0.9.5",][[package]]@@ -544,7 +1004,7 @@ version = "0.7.0"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "f703f4665700daf5512dcca5f43afa6af89f09db47fb56be587f80636bda2d41"dependencies = [- "rand_core",+ "rand_core 0.9.5",][[package]]@@ -582,6 +1042,23 @@ version = "0.8.11"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4"+[[package]]+name = "rust_decimal"+version = "1.43.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "7653272e75dcac41dc199fbea6f5797633994fafd339943c06c9af16bf29cd3a"+dependencies = [+ "arrayvec",+ "borsh",+ "bytes",+ "num-traits",+ "rand 0.8.8",+ "rand 0.9.5",+ "serde",+ "serde_json",+ "wasm-bindgen",+]+[[package]]name = "rustc-hash"version = "2.1.3"@@ -613,6 +1090,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "4148590afebada386688f18773da617792bf2ef03ffc1e4cbd2b1d45b023e0ba"dependencies = ["serde_core",+ "serde_derive",][[package]]@@ -635,12 +1113,24 @@ dependencies = ["syn 3.0.4",]+[[package]]+name = "serde_derive_internals"+version = "0.29.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "18d26a20a969b9e3fdf2fc2d9f21eda6c40e2de84c9408bb5d3b05d499aae711"+dependencies = [+ "proc-macro2",+ "quote",+ "syn 2.0.119",+]+[[package]]name = "serde_json"version = "1.0.151"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "c841b55ecdae098c80dcae9cf767f6f8a0c2cdb3416bbef72181df4d0fe73f14"dependencies = [+ "indexmap","itoa","memchr","serde",@@ -675,6 +1165,22 @@ version = "1.16.0"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "b9be42f50aa861c555654aa3a37f52f4b1074bacf4e48fe0ef7fa584e80f1f0f"+[[package]]+name = "socket2"+version = "0.6.5"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "c3d1e2c7f27f8d4cb10542a02c49005dbd6e93095799d6f3be745fae9f8fedd4"+dependencies = [+ "libc",+ "windows-sys",+]++[[package]]+name = "stable_deref_trait"+version = "1.2.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596"+[[package]]name = "syn"version = "2.0.119"@@ -697,6 +1203,17 @@ dependencies = ["unicode-ident",]+[[package]]+name = "synstructure"+version = "0.14.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "901704edd0dfe137f1987838ee4f259e4e063c31371bdb423f7ae38ec6f77f02"+dependencies = [+ "proc-macro2",+ "quote",+ "syn 3.0.4",+]+[[package]]name = "thiserror"version = "2.0.20"@@ -726,6 +1243,77 @@ dependencies = ["cfg-if",]+[[package]]+name = "tinystr"+version = "0.8.4"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "b1e27c91459209c2986af3dcf603a5a74a4368754ce37414f59acc971167f643"+dependencies = [+ "displaydoc",+ "zerovec",+]++[[package]]+name = "tokio"+version = "1.53.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "202caea871b69668250d242070849eb495be178ed697a3e98aebce5bc81a0bed"+dependencies = [+ "libc",+ "mio",+ "pin-project-lite",+ "socket2",+ "tokio-macros",+ "windows-sys",+]++[[package]]+name = "tokio-macros"+version = "2.7.2"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "78773a2a397f451582ce068015985c33193cf6dea8b74d2a639fe457b2f07b0e"+dependencies = [+ "proc-macro2",+ "quote",+ "syn 3.0.4",+]++[[package]]+name = "toml_datetime"+version = "1.1.1+spec-1.1.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "3165f65f62e28e0115a00b2ebdd37eb6f3b641855f9d636d3cd4103767159ad7"+dependencies = [+ "serde_core",+]++[[package]]+name = "toml_edit"+version = "0.25.15+spec-1.1.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "1340ea94a5856333492c9064b02c778b191dd2c853778d9609debdcdfea3a614"+dependencies = [+ "indexmap",+ "toml_datetime",+ "toml_parser",+ "winnow",+]++[[package]]+name = "toml_parser"+version = "1.1.3+spec-1.1.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "1d38ac1cf9b95face32296c0a3ede1fdc270627c9d9c02a7274dd6d960dc4d56"+dependencies = [+ "winnow",+]++[[package]]+name = "tower-service"+version = "0.3.3"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "8df9b6e13f2d32c91b9bd719c00d1958837bc7dec474d94952798cc8e69eeec3"+[[package]]name = "tracing"version = "0.1.44"@@ -800,6 +1388,18 @@ dependencies = ["tracing-serde",]+[[package]]+name = "try-lock"+version = "0.2.5"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b"++[[package]]+name = "unarray"+version = "0.1.4"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "eaea85b334db583fe3274d12b4cd1880032beab409c0d774be044d4480ab9a94"+[[package]]name = "unicode-ident"version = "1.0.24"@@ -812,12 +1412,39 @@ version = "1.13.3"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "c6f5d3c3b1bf09027a88a6bc961fc00497d651009560b5463668dc81b0fa87a8"+[[package]]+name = "url"+version = "2.5.8"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "ff67a8a4397373c3ef660812acab3268222035010ab8680ec4215f38ba3d0eed"+dependencies = [+ "form_urlencoded",+ "idna",+ "percent-encoding",+ "serde",+]++[[package]]+name = "utf8_iter"+version = "1.0.4"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be"+[[package]]name = "valuable"version = "0.1.1"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "ba73ea9cf16a25df0c8caa16c51acb937d5712a8429db78a3ee29d5dcacd3a65"+[[package]]+name = "want"+version = "0.3.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "bfa7760aed19e106de2c7c0b581b509f2f25d3dacaf737cb82ac61bc6d760b0e"+dependencies = [+ "try-lock",+]+[[package]]name = "wasi"version = "0.11.1+wasi-snapshot-preview1"@@ -925,12 +1552,50 @@ dependencies = ["windows-link",]+[[package]]+name = "winnow"+version = "1.0.4"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "23b97319f7b8343df12cc98938e5c3eb436064524c8d2b4e30a1d3a36eecdf81"+dependencies = [+ "memchr",+]+[[package]]name = "wit-bindgen"version = "0.57.1"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "1ebf944e87a7c253233ad6766e082e3cd714b5d03812acc24c318f549614536e"+[[package]]+name = "writeable"+version = "0.6.4"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "3ad82d2a33cdc9674dc7465672f271e096168fcdbe0f799d9e6db8c5892679dc"++[[package]]+name = "yoke"+version = "0.8.3"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "709fe23a0424b6a435d82152b1bd3fdfb0833487d5fa90d05d42762a9891fef5"+dependencies = [+ "stable_deref_trait",+ "yoke-derive",+ "zerofrom",+]++[[package]]+name = "yoke-derive"+version = "0.8.3"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "33811428bee40dbceb6d545e95754741d17a6aef9a4849f0fd62e2ba4f412a78"+dependencies = [+ "proc-macro2",+ "quote",+ "syn 3.0.4",+ "synstructure",+]+[[package]]name = "zerocopy"version = "0.8.56"@@ -951,6 +1616,60 @@ dependencies = ["syn 2.0.119",]+[[package]]+name = "zerofrom"+version = "0.1.8"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "0ec05a11813ea801ff6d75110ad09cd0824ddba17dfe17128ea0d5f68e6c5272"+dependencies = [+ "zerofrom-derive",+]++[[package]]+name = "zerofrom-derive"+version = "0.1.8"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "f75b4683f6c7f45248d4d64056a24298c6281e0993356d7d1b4a1a962ef10d4a"+dependencies = [+ "proc-macro2",+ "quote",+ "syn 3.0.4",+ "synstructure",+]++[[package]]+name = "zerotrie"+version = "0.2.5"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "4ea269c3bd32f0a32c321907a2ae912ba6f4649bb0fc764a15627e99a7095a3f"+dependencies = [+ "displaydoc",+ "yoke",+ "zerofrom",+]++[[package]]+name = "zerovec"+version = "0.11.8"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "bb0464e17806c1d976d5cba29399c7f08e516e279e2ba493f63123b5fca67dd8"+dependencies = [+ "yoke",+ "zerofrom",+ "zerovec-derive",+]++[[package]]+name = "zerovec-derive"+version = "0.11.6"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "34df6fc39dbd26ddc9c10e6a2984476e13acce22e64e4487636ef494369225da"+dependencies = [+ "proc-macro2",+ "quote",+ "syn 3.0.4",+]+[[package]]name = "zmij"version = "1.0.23"diff --git a/packages/native/Cargo.toml b/packages/native/Cargo.tomlindex 7e0dedf621..5ca234f133 100644--- a/packages/native/Cargo.toml+++ b/packages/native/Cargo.toml@@ -10,14 +10,28 @@ publish = false[lib]crate-type = ["cdylib"]+[[example]]+name = "native_codec_test_driver"+test = false+[dependencies]+bytes = "1"+chrono = { version = "0.4", default-features = false, features = ["alloc", "std"] }+clickhouse = { version = "=0.15.2", default-features = false }+clickhouse-types = "=0.1.3"metrics = "0.24"metrics-exporter-dogstatsd = "0.9"napi = { version = "3", default-features = false, features = ["napi9"] }napi-derive = "3"+rust_decimal = { version = "1.37.2", default-features = false, features = ["std"] }+serde = "1"+serde_json = { version = "1", features = ["preserve_order", "float_roundtrip"] }tracing = "0.1"tracing-subscriber = { version = "0.3", features = ["env-filter", "json"] }+[dev-dependencies]+proptest = { version = "1", default-features = false, features = ["std"] }+[build-dependencies]napi-build = "2"diff --git a/packages/native/README.md b/packages/native/README.mdindex 80b09f0f8d..fa3d092575 100644--- a/packages/native/README.md+++ b/packages/native/README.md@@ -8,10 +8,6 @@ other workspace package:import { hello } from "@langfuse/native";```-Today the addon exports a single hello-world function. It exists to put the-build, packaging, and deployment pipeline for native code in place; real-functionality lands on top of it.-## Layout| Path | Purpose |@@ -29,6 +25,12 @@ functionality lands on top of it.type checks and tooling work without a Rust toolchain. `*.node` binaries and`target/` are ignored.+The encoder uses a small local source overlay for the pinned `clickhouse`+0.15.2 crate. `scripts/prepare-clickhouse-rs.sh` downloads the exact crate+archive, verifies its checksum, applies the delta in `patches/`, and writes an+ignored Cargo patch configuration. The repository keeps the delta and setup+script, not a vendored copy of the crate.+## BuildingInstall Rust through [rustup](https://rustup.rs), the same prerequisite the AI@@ -44,6 +46,7 @@ Run from the repo root:pnpm --filter @langfuse/native run build # release build for the current platformpnpm --filter @langfuse/native run build:debug # unoptimised buildpnpm --filter @langfuse/native run lint # cargo fmt --check && cargo clippy -D warnings+pnpm --filter @langfuse/native run test # Rust tests; uses ClickHouse when available````pnpm run dev`, `pnpm run build`, `pnpm run test`, and the worker'sdiff --git a/packages/native/examples/native_codec_test_driver.rs b/packages/native/examples/native_codec_test_driver.rsnew file mode 100644index 0000000000..6606257043--- /dev/null+++ b/packages/native/examples/native_codec_test_driver.rs@@ -0,0 +1,58 @@+//! Test driver used by the worker's TypeScript parity suite; not an addon export.++#[allow(dead_code)]+#[path = "../src/native_codec.rs"]+mod native_codec;+#[allow(dead_code)]+#[path = "../src/native_schema.rs"]+mod native_schema;++use native_codec::encode_v4_native_blocks;+use native_schema::PreparedEvent;+use serde::Deserialize;+use serde_json::{json, Value};+use std::{env, fs, path::Path};++#[derive(Debug, Deserialize)]+#[serde(rename_all = "camelCase")]+struct NativeCodecTestInput {+ rows: Vec<Value>,+ max_rows_per_block: usize,+}++fn main() {+ let input_path = env::var("LANGFUSE_NATIVE_TEST_INPUT")+ .expect("LANGFUSE_NATIVE_TEST_INPUT must point to the JSON input");+ let output_dir = env::var("LANGFUSE_NATIVE_TEST_OUTPUT")+ .expect("LANGFUSE_NATIVE_TEST_OUTPUT must point to an existing directory");+ let input: NativeCodecTestInput =+ serde_json::from_slice(&fs::read(&input_path).expect("read LANGFUSE_NATIVE_TEST_INPUT"))+ .expect("parse native codec test input");+ let prepared = input+ .rows+ .iter()+ .map(PreparedEvent::from_json)+ .collect::<Result<Vec<_>, _>>()+ .expect("prepare native codec test rows");+ let event_bytes = prepared+ .iter()+ .map(|row| row.event_bytes)+ .collect::<Vec<_>>();+ let blocks = encode_v4_native_blocks(&prepared, input.max_rows_per_block)+ .expect("encode native codec test rows");+ let mut native_payload = Vec::new();+ for block in blocks {+ native_payload.extend_from_slice(&block.bytes);+ }++ let output_dir = Path::new(&output_dir);+ fs::write(output_dir.join("native.bin"), native_payload).expect("write native.bin");+ let manifest = json!({+ "eventBytes": event_bytes,+ });+ fs::write(+ output_dir.join("manifest.json"),+ serde_json::to_vec(&manifest).expect("serialize manifest"),+ )+ .expect("write manifest.json");+}diff --git a/packages/native/package.json b/packages/native/package.jsonindex 1c56e9977b..7a18e6a8c4 100644--- a/packages/native/package.json+++ b/packages/native/package.json@@ -15,10 +15,12 @@"node": "24"},"scripts": {- "build": "napi build --platform --release",- "build:debug": "napi build --platform",- "lint": "cargo fmt --check && cargo clippy --all-targets -- -D warnings",- "lint:fix": "cargo fmt && cargo clippy --all-targets --fix --allow-dirty -- -D warnings"+ "prepare-clickhouse-rs": "sh scripts/prepare-clickhouse-rs.sh",+ "build": "pnpm run prepare-clickhouse-rs && napi build --platform --release",+ "build:debug": "pnpm run prepare-clickhouse-rs && napi build --platform",+ "lint": "pnpm run prepare-clickhouse-rs && cargo fmt --check && cargo clippy --all-targets -- -D warnings",+ "lint:fix": "pnpm run prepare-clickhouse-rs && cargo fmt && cargo clippy --all-targets --fix --allow-dirty -- -D warnings",+ "test": "pnpm run prepare-clickhouse-rs && cargo test --locked"},"napi": {"binaryName": "langfuse-native",diff --git a/packages/native/patches/clickhouse-0.15.2-native-encoder.patch b/packages/native/patches/clickhouse-0.15.2-native-encoder.patchnew file mode 100644index 0000000000..071c653350--- /dev/null+++ b/packages/native/patches/clickhouse-0.15.2-native-encoder.patch@@ -0,0 +1,128 @@+diff --git a/src/native/mod.rs b/src/native/mod.rs+index 1ef0339..68181bc 100644+--- a/src/native/mod.rs++++ b/src/native/mod.rs+@@ -47,6 +47,15 @@ impl Block {+ }+ }+++ /// Encode this block as a self-contained ClickHouse Native-format payload.++ ///++ /// The returned bytes contain the block header, column metadata, and all column data. They++ /// can be sent as one Native block without retaining any references to the builder or its++ /// source values.++ pub fn encode(&self) -> Result<Bytes, Error> {++ writer::encode_block(self)++ }+++ /// The number of rows in this block.+ ///+ /// Note that the size of a single block in a query resultset can be influenced by many things+diff --git a/src/native/writer.rs b/src/native/writer.rs+index b9553c8..35327f8 100644+--- a/src/native/writer.rs++++ b/src/native/writer.rs+@@ -5,6 +5,103 @@ use crate::native::{Block, Column, Layout, LayoutKind, varuint};+ use bytes::{BufMut, Bytes, BytesMut};+ use std::num::Saturating;+++/// Encode a complete block synchronously in ClickHouse Native format.++///++/// This is kept separate from [`BlockWriter`], whose purpose is streaming an insert request. It++/// lets callers that already own a block obtain an independent byte buffer with no async sink or++/// request lifetime involved.++pub(crate) fn encode_block(block: &Block) -> Result<Bytes, Error> {++ if block.num_rows == 0 {++ return Err(Error::Other("attempting to encode an empty block".into()));++ }++++ let mut buf = BytesMut::with_capacity(8192);++ varuint::write(&mut buf, block.columns.len());++ varuint::write(&mut buf, block.num_rows);++++ for column in &block.columns {++ encode_column(&mut buf, column)?;++ }++++ Ok(buf.freeze())++}++++fn encode_column(buf: &mut BytesMut, column: &Column) -> Result<(), Error> {++ varuint::write(&mut *buf, column.name.len());++ buf.extend_from_slice(column.name.as_bytes());++++ let data_type_str = column.data_type.to_str();++ varuint::write(&mut *buf, data_type_str.len());++ buf.extend_from_slice(data_type_str.as_bytes());++++ encode_layout(buf, &column.layout)++}++++fn encode_layout(buf: &mut BytesMut, layout: &Layout) -> Result<(), Error> {++ if let Some(nulls) = &layout.nulls {++ buf.extend_from_slice(nulls);++ }++++ match &layout.kind {++ LayoutKind::Fixed { data, .. } => buf.extend_from_slice(data),++ LayoutKind::Variable { end_offsets, data } => {++ let mut start_offset = 0;++++ for &end_offset in end_offsets {++ let len = end_offset.checked_sub(start_offset).ok_or_else(|| {++ Error::Other(++ format!(++ "BUG: string length underflow in encoding block: {end_offset} - {start_offset}"++ )++ .into(),++ )++ })?;++++ varuint::write(&mut *buf, len);++ buf.extend_from_slice(&data[start_offset..end_offset]);++ start_offset = end_offset;++ }++ }++ LayoutKind::LowCardinality(_) => {++ return Err(Error::Other(++ "inserting LowCardinality data not yet implemented".into(),++ ));++ }++ LayoutKind::Array {++ end_indices,++ elem_layout,++ } => {++ for &index in end_indices {++ buf.put_u64_le(u64::try_from(index).map_err(|_| {++ Error::Other(format!("array end index out of range: {index}").into())++ })?);++ }++++ encode_layout(buf, elem_layout)?;++ }++ LayoutKind::Tuple { layouts } => {++ for layout in layouts {++ encode_layout(buf, layout)?;++ }++ }++ LayoutKind::Map {++ key_val_layouts,++ end_indices,++ } => {++ for &index in end_indices {++ buf.put_u64_le(u64::try_from(index).map_err(|_| {++ Error::Other(format!("map end index out of range: {index}").into())++ })?);++ }++++ encode_layout(buf, &key_val_layouts[0])?;++ encode_layout(buf, &key_val_layouts[1])?;++ }++ }++++ Ok(())++}+++ pub(crate) struct BlockWriter {+ insert: InsertFormatted,+ buf: BytesMut,diff --git a/packages/native/scripts/prepare-clickhouse-rs.sh b/packages/native/scripts/prepare-clickhouse-rs.shnew file mode 100644index 0000000000..099ea246c6--- /dev/null+++ b/packages/native/scripts/prepare-clickhouse-rs.sh@@ -0,0 +1,48 @@+#!/bin/sh+set -eu++# Apply the local Native encoder patch to the clickhouse-rs client crate.++crate_version=0.15.2+crate_sha256=7654154fde4d97ec1321bfe4c3965570be6fcd09e091da8afc3166ab5ee27be2+package_dir=$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd)+source_root="${package_dir}/target/.clickhouse-source"+source_dir="${source_root}/clickhouse-${crate_version}"+archive="${source_root}/clickhouse-${crate_version}.crate"+patch_file="${package_dir}/patches/clickhouse-${crate_version}-native-encoder.patch"+marker="${source_dir}/.langfuse-native-encoder-patched"+config_dir="${package_dir}/.cargo"++checksum() {+ if command -v sha256sum >/dev/null 2>&1; then+ sha256sum "$1" | cut -d ' ' -f 1+ else+ shasum -a 256 "$1" | cut -d ' ' -f 1+ fi+}++patch_sha256="$(checksum "$patch_file")"++if [ ! -f "$marker" ] || [ "$(cat "$marker")" != "$patch_sha256" ]; then+ mkdir -p "$source_root"+ curl --fail --location --retry 3 --retry-delay 2 --retry-all-errors --silent --show-error \+ --user-agent 'langfuse-native-build/0.1 (https://github.com/langfuse/langfuse)' \+ "https://crates.io/api/v1/crates/clickhouse/${crate_version}/download" \+ --output "$archive"++ if [ "$(checksum "$archive")" != "$crate_sha256" ]; then+ echo "clickhouse ${crate_version} archive checksum mismatch" >&2+ exit 1+ fi++ rm -rf "$source_dir"+ tar -xzf "$archive" -C "$source_root"+ patch --directory="$source_dir" --strip=1 < "$patch_file"+ printf '%s' "$patch_sha256" > "$marker"+fi++mkdir -p "$config_dir"+cat > "$config_dir/config.toml" <<EOF+[patch.crates-io]+clickhouse = { path = "${source_dir}" }+EOFdiff --git a/packages/native/src/lib.rs b/packages/native/src/lib.rsindex cd5701de5a..5231287d2d 100644--- a/packages/native/src/lib.rs+++ b/packages/native/src/lib.rs@@ -5,6 +5,12 @@//! `metrics` and `tracing` facades set up in [`telemetry`]; it never hands//! values back for Node to record.+/// Rust-only encoder for prepared v4 `events_full` rows.+// These modules are exercised by Rust tests until the ingestion transport calls them.+#[allow(dead_code)]+pub mod native_codec;+#[allow(dead_code)]+pub(crate) mod native_schema;mod telemetry;use std::time::Duration;diff --git a/packages/native/src/native_codec.rs b/packages/native/src/native_codec.rsnew file mode 100644index 0000000000..ce5908a943--- /dev/null+++ b/packages/native/src/native_codec.rs@@ -0,0 +1,434 @@+//! Rust-only encoding for prepared v4 `events_full` rows.+//!+//! `PreparedEvent::from_json` accepts the prepared `EventRecordInsertType` shape after+//! observation-field overflow handling and before the writer's numeric Decimal clamp. Nested+//! input/output payloads have already been serialized as strings. Native encoding then receives+//! only typed, owned fields; the adapters below define the JSON conversion policy.++use std::borrow::Cow;+use std::collections::BTreeMap;+use std::error::Error as StdError;++use bytes::Bytes;+use chrono::{DateTime, NaiveDateTime, Utc};+use clickhouse::native::builder::BlockBuilder;+use clickhouse::native::encode::{Encode, ValueWriter};+use clickhouse::native::DataTypeNode;+use clickhouse_types::data_types::{DateTimePrecision, DecimalType};+use rust_decimal::{Decimal, RoundingStrategy};+use serde::ser::{SerializeMap, Serializer}; // codespell:ignore ser+use serde_json::Value;++use crate::native_schema::{ColumnKind, DefaultPolicy, PreparedEvent};++const DECIMAL_SCALE: u32 = 12;+const DECIMAL_LIMIT: i128 = 1_000_000_000_000_000_000;+const DECIMAL_CLAMPED_MAX: i64 = 999_999_999_999_000_000;+const DECIMAL_CLAMPED_MIN: i64 = -DECIMAL_CLAMPED_MAX;+const DECIMAL_OVERFLOW_LIMIT: f64 = 1_000_000.0;+type EncodeError = Box<dyn StdError + Send + Sync>;++#[derive(Clone, Copy, Debug)]+pub(crate) struct DateTime64Micros(pub(crate) i64);++impl Encode for DateTime64Micros {+ fn produces() -> DataTypeNode {+ DataTypeNode::DateTime64(DateTimePrecision::Precision6, None)+ }++ fn encode(&self, writer: &mut ValueWriter<'_>) -> Result<(), EncodeError> {+ // clickhouse-rs has no DateTime64 Encode implementation. Keep the DateTime64(6) type tag+ // while writing ClickHouse's signed microsecond representation as a fixed-width value.+ writer.write_fixed(&self.0.to_le_bytes())?;+ Ok(())+ }+}++#[derive(Clone, Copy, Debug)]+pub(crate) struct Decimal64(pub(crate) i64);++impl Encode for Decimal64 {+ fn produces() -> DataTypeNode {+ DataTypeNode::Decimal(18, 12, DecimalType::Decimal64)+ }++ fn encode(&self, writer: &mut ValueWriter<'_>) -> Result<(), EncodeError> {+ // rust_decimal supplies the policy and coefficient; clickhouse-rs has no Decimal Encode+ // implementation, so retain the Decimal(18,12) type tag and write its Int64 coefficient.+ writer.write_fixed(&self.0.to_le_bytes())?;+ Ok(())+ }+}++/// An owned ClickHouse Native-format block and its row count.+///+/// Ownership is part of the transport contract: the later JS transport can retry this complete+/// block after the encoder and source rows have gone out of scope.+#[derive(Debug)]+pub struct EncodedBlock {+ pub bytes: Bytes,+ pub row_count: usize,+}++/// Encode prepared v4 event rows into Native blocks containing at most `max_rows_per_block` rows.+/// The boundary is a row count, not a byte-size or memory limit; the transport can use the returned+/// owned buffers and counts when it chooses its request boundaries.+///+/// Prepared rows already contain the defaults that production-prepared rows rely on when every+/// insertable column is listed in a Native block. Decimal values use the existing+/// Decimal(18,12) overflow policy and truncate toward zero at twelve fractional digits.+/// `event_bytes` is the UTF-8 size of the compact Rust JSON serialization of the prepared row+/// without the accounting field; it is intentionally a logical size metric and need not be+/// byte-identical to JavaScript's JSON.stringify output. Preparation stores this value before+/// applying typed decimal conversion or DEFAULT-backed field filling.+pub(crate) fn encode_v4_native_blocks(+ rows: &[PreparedEvent],+ max_rows_per_block: usize,+) -> Result<Vec<EncodedBlock>, String> {+ if max_rows_per_block == 0 {+ return Err("max_rows_per_block must be greater than zero".to_owned());+ }++ rows.chunks(max_rows_per_block)+ .map(|chunk| {+ let bytes = encode_block(chunk)?;+ Ok(EncodedBlock {+ bytes,+ row_count: chunk.len(),+ })+ })+ .collect()+}++/// Compute the UTF-8 size of a prepared event's JSONEachRow representation without `event_bytes`.+pub fn event_bytes(value: &Value) -> Result<u64, String> {+ let object = value+ .as_object()+ .ok_or_else(|| "event row must be a JSON object".to_owned())?;+ // Borrow the fields and discard serialized bytes immediately: accounting needs the length,+ // not a cloned row or a second payload buffer.+ let mut counter = ByteCounter(0);+ let mut serializer = serde_json::Serializer::new(&mut counter);+ let result = (|| -> Result<(), serde_json::Error> {+ let mut map = serializer.serialize_map(None)?;+ for (key, value) in object {+ if key != "event_bytes" {+ map.serialize_entry(key, value)?;+ }+ }+ map.end()+ })();+ result.map_err(|error| format!("failed to serialize event row: {error}"))?;+ Ok(counter.0)+}++struct ByteCounter(u64);++impl std::io::Write for ByteCounter {+ fn write(&mut self, bytes: &[u8]) -> std::io::Result<usize> {+ self.0 = self+ .0+ .checked_add(bytes.len() as u64)+ .ok_or_else(|| std::io::Error::other("serialized event is too large"))?;+ Ok(bytes.len())+ }++ fn flush(&mut self) -> std::io::Result<()> {+ Ok(())+ }+}++fn encode_block(rows: &[PreparedEvent]) -> Result<Bytes, String> {+ let mut builder = BlockBuilder::new();+ PreparedEvent::encode_into(&mut builder, rows)?;++ builder+ .build()+ .map_err(|error| error.to_string())?+ .encode()+ .map_err(|error| error.to_string())+}++/// Resolve values whose ClickHouse defaults would otherwise be bypassed by listing the column in+/// the Native block. Ordinary columns stay borrowed from the input row; only synthesized defaults+/// allocate a JSON value. This runs once at the JSON-to-typed preparation boundary.+fn value_for<'a>(row: &'a Value, key: &str, default_policy: DefaultPolicy) -> Cow<'a, Value> {+ let explicit = row.get(key);+ if default_policy == DefaultPolicy::None {+ return explicit+ .map(Cow::Borrowed)+ .unwrap_or(Cow::Owned(Value::Null));+ }+ if let Some(value) = explicit.filter(|value| !value.is_null()) {+ return Cow::Borrowed(value);+ }+ // For example, omitted environment gets DEFAULT 'default' in JSONEachRow. Sending an empty+ // string in a Native column would instead persist that empty string and bypass the default.+ Cow::Owned(match default_policy {+ DefaultPolicy::Literal(value) => Value::String(value.to_owned()),+ DefaultPolicy::Metadata(key) => Value::String(metadata_value(row, key).unwrap_or_default()),+ DefaultPolicy::MetadataFallback(key, fallback) => Value::String(+ metadata_value(row, key)+ .filter(|value| !value.is_empty())+ .or_else(|| metadata_value(row, fallback))+ .unwrap_or_default(),+ ),+ DefaultPolicy::MetadataEquals(key, expected) => {+ Value::Bool(metadata_value(row, key).as_deref() == Some(expected))+ }+ DefaultPolicy::None => Value::Null,+ })+}++pub(crate) fn from_json_field<T: FromJson>(+ row: &Value,+ key: &str,+ default_policy: DefaultPolicy,+) -> Result<T, String> {+ T::from_json(value_for(row, key, default_policy).as_ref())+ .map_err(|error| format!("{key}: {error}"))+}++fn metadata_value(row: &Value, name: &str) -> Option<String> {+ // Ingestion stores metadata as parallel arrays; default expressions look up the same pair.+ let names = row.get("metadata_names")?.as_array()?;+ let values = row.get("metadata_values")?.as_array()?;+ names.iter().zip(values).find_map(|(metadata_name, value)| {+ (metadata_name.as_str() == Some(name)).then(|| string_value(value))+ })+}++// Conversion is separate from wire encoding: upstream Encode implementations own nullable, array,+// map, and primitive layouts, while these implementations define the prepared-row input policy.+pub(crate) trait FromJson: Encode + Sized {+ const COLUMN_KIND: ColumnKind;++ fn from_json(value: &Value) -> Result<Self, String>;+}++impl FromJson for String {+ const COLUMN_KIND: ColumnKind = ColumnKind::String;++ fn from_json(value: &Value) -> Result<Self, String> {+ Ok(string_value(value))+ }+}++impl FromJson for bool {+ const COLUMN_KIND: ColumnKind = ColumnKind::Bool;++ fn from_json(value: &Value) -> Result<Self, String> {+ match value {+ Value::Null => Ok(false),+ Value::Bool(value) => Ok(*value),+ Value::Number(value) => value+ .as_f64()+ .filter(|value| value.is_finite())+ .map(|value| value != 0.0)+ .ok_or_else(|| "expected a finite boolean number".to_owned()),+ _ => Err(format!("expected boolean, got {value}")),+ }+ }+}++macro_rules! unsigned {+ ($(($type:ty, $kind:expr)),+ $(,)?) => {$(+ impl FromJson for $type {+ const COLUMN_KIND: ColumnKind = $kind;++ fn from_json(value: &Value) -> Result<Self, String> {+ <$type>::try_from(json_u64(Some(value), stringify!($type))?)+ .map_err(|_| format!("expected a {}", stringify!($type)))+ }+ }+ )+};+}+unsigned!(+ (u8, ColumnKind::UInt8),+ (u16, ColumnKind::UInt16),+ (u64, ColumnKind::UInt64),+);++impl FromJson for DateTime64Micros {+ const COLUMN_KIND: ColumnKind = ColumnKind::DateTime64;++ fn from_json(value: &Value) -> Result<Self, String> {+ parse_datetime_value(value, "datetime").map(Self)+ }+}++impl FromJson for Decimal64 {+ const COLUMN_KIND: ColumnKind = ColumnKind::Decimal;++ fn from_json(value: &Value) -> Result<Self, String> {+ parse_decimal_value(value, "decimal").map(Self)+ }+}++impl<T: FromJson> FromJson for Option<T> {+ const COLUMN_KIND: ColumnKind = match T::COLUMN_KIND {+ ColumnKind::String => ColumnKind::NullableString,+ ColumnKind::DateTime64 => ColumnKind::NullableDateTime64,+ ColumnKind::UInt16 => ColumnKind::NullableUInt16,+ _ => panic!("unsupported nullable events_full column type"),+ };++ fn from_json(value: &Value) -> Result<Self, String> {+ if value.is_null() {+ Ok(None)+ } else {+ T::from_json(value).map(Some)+ }+ }+}++impl<T: FromJson> FromJson for Vec<T> {+ const COLUMN_KIND: ColumnKind = match T::COLUMN_KIND {+ ColumnKind::String => ColumnKind::ArrayString,+ _ => panic!("unsupported array events_full column type"),+ };++ fn from_json(value: &Value) -> Result<Self, String> {+ match value {+ Value::Null => Ok(Vec::new()),+ Value::Array(values) => values.iter().map(T::from_json).collect(),+ _ => Err(format!("expected an array, got {value}")),+ }+ }+}++impl<T: FromJson> FromJson for BTreeMap<String, T> {+ const COLUMN_KIND: ColumnKind = match T::COLUMN_KIND {+ ColumnKind::String => ColumnKind::MapStringString,+ ColumnKind::UInt64 => ColumnKind::MapStringUInt64,+ ColumnKind::Decimal => ColumnKind::MapStringDecimal,+ _ => panic!("unsupported map events_full column type"),+ };++ fn from_json(value: &Value) -> Result<Self, String> {+ match value {+ Value::Null => Ok(BTreeMap::new()),+ Value::Object(values) => values+ .iter()+ .map(|(key, value)| Ok((key.clone(), T::from_json(value)?)))+ .collect(),+ _ => Err(format!("expected a map, got {value}")),+ }+ }+}++fn json_u64(value: Option<&Value>, type_name: &str) -> Result<u64, String> {+ // JSONEachRow inputs can represent whole numbers as either JSON integers or integral floats;+ // accept both forms while rejecting signs, fractions, and non-finite values for UInt columns.+ match value {+ None | Some(Value::Null) => Ok(0),+ Some(Value::Number(value)) => value+ .as_u64()+ .or_else(|| {+ value+ .as_f64()+ .filter(|number| number.is_finite() && *number >= 0.0 && number.fract() == 0.0)+ .and_then(|number| u64::try_from(number as u128).ok())+ })+ .ok_or_else(|| format!("expected a {type_name}")),+ Some(value) => Err(format!("expected a {type_name}, got {value}")),+ }+}++/// Parse the UTC-only datetime values produced by the ingestion pipeline. A ClickHouse datetime+/// string without an offset is a UTC calendar value; an RFC3339 offset is normalized to that same+/// UTC instant. The project never interprets these values in the host's local timezone.+fn parse_datetime_value(value: &Value, key: &str) -> Result<i64, String> {+ match value {+ Value::String(value) => {+ let normalized = if value.contains(' ') && !value.contains('T') {+ value.replacen(' ', "T", 1)+ } else {+ value.clone()+ };+ if let Ok(parsed) = DateTime::parse_from_rfc3339(&normalized) {+ return Ok(parsed.timestamp_micros());+ }+ let naive = NaiveDateTime::parse_from_str(&normalized, "%Y-%m-%dT%H:%M:%S%.f")+ .map_err(|error| {+ format!("{key} has invalid DateTime64 value {value:?}: {error}")+ })?;+ Ok(DateTime::<Utc>::from_naive_utc_and_offset(naive, Utc).timestamp_micros())+ }+ _ => Err(format!("{key} must be a UTC timestamp string")),+ }+}++fn parse_decimal_value(value: &Value, key: &str) -> Result<i64, String> {+ let text = match value {+ Value::String(value) => value.as_str(),+ Value::Number(value) => return parse_decimal(&value.to_string(), key),+ _ => return Err(format!("{key} must be a decimal number or string")),+ };+ parse_decimal(text, key)+}++fn parse_decimal(value: &str, key: &str) -> Result<i64, String> {+ let value = value.trim();+ if value.is_empty() {+ return Ok(0);+ }+ let number = value+ .parse::<f64>()+ .map_err(|_| format!("{key} has an invalid decimal"))?;++ // `UsageCostSchema` converts costs to JavaScript numbers and the writer's existing policy+ // turns non-finite values into zero and clamps values at the exclusive 1e6 boundary.+ if !number.is_finite() {+ return Ok(0);+ }+ if number.abs() >= DECIMAL_OVERFLOW_LIMIT {+ return Ok(if number.is_sign_negative() {+ DECIMAL_CLAMPED_MIN+ } else {+ DECIMAL_CLAMPED_MAX+ });+ }++ // Values below the target quantum are zero after ClickHouse's scale conversion. Returning+ // early also covers f64 underflow values whose scientific exponent exceeds Decimal's scale+ // range without turning a valid, zero-result input into a parse error.+ if number == 0.0 || number.abs() < 10_f64.powi(-(DECIMAL_SCALE as i32)) {+ return Ok(0);+ }++ // The ingestion schema has already reduced costs to JavaScript numbers. Parse that shortest+ // round-tripping representation as a fixed-precision decimal so scaling never multiplies a+ // binary float. Decimal(18,12) fits comfortably within rust_decimal's coefficient range.+ let normalized = number.to_string();+ let mut decimal = if normalized.contains(['e', 'E']) {+ Decimal::from_scientific(&normalized)+ } else {+ normalized.parse::<Decimal>()+ }+ .map_err(|_| format!("{key} has an invalid decimal"))?;++ decimal = decimal.round_dp_with_strategy(DECIMAL_SCALE, RoundingStrategy::ToZero);+ decimal.rescale(DECIMAL_SCALE);+ let coefficient = decimal.mantissa();+ if coefficient >= DECIMAL_LIMIT || coefficient <= -DECIMAL_LIMIT {+ return Ok(if coefficient.is_negative() {+ DECIMAL_CLAMPED_MIN+ } else {+ DECIMAL_CLAMPED_MAX+ });+ }+ i64::try_from(coefficient).map_err(|_| format!("{key} decimal is out of range"))+}++fn string_value(value: &Value) -> String {+ match value {+ Value::String(value) => value.clone(),+ Value::Null => String::new(),+ value => serde_json::to_string(value).unwrap_or_default(),+ }+}++#[cfg(test)]+#[path = "native_codec_tests.rs"]+mod tests;diff --git a/packages/native/src/native_codec_tests.rs b/packages/native/src/native_codec_tests.rsnew file mode 100644index 0000000000..49bc32ad17--- /dev/null+++ b/packages/native/src/native_codec_tests.rs@@ -0,0 +1,539 @@+use super::*;+use crate::native_schema::{+ find_column, ColumnKind, DefaultPolicy, PreparedEvent, EVENTS_FULL_INSERT_COLUMNS,+};+use proptest::prelude::*;+use proptest::test_runner::{Config as ProptestConfig, TestRunner};+use serde_json::json;+use std::collections::BTreeSet;+use std::io::Write;+use std::process::{Command, Stdio};+use std::time::{SystemTime, UNIX_EPOCH};++fn encode_json_rows(rows: &[Value], max_rows: usize) -> Result<Vec<EncodedBlock>, String> {+ let prepared = rows+ .iter()+ .map(PreparedEvent::from_json)+ .collect::<Result<Vec<_>, _>>()?;+ encode_v4_native_blocks(&prepared, max_rows)+}++fn row(input: &str, output: &str, event_bytes: Value) -> Value {+ json!({+ "project_id": "project",+ "trace_id": "trace",+ "span_id": "span",+ "start_time": "2026-07-22 00:00:00.000",+ "created_at": "2026-07-22 00:00:00.000",+ "updated_at": "2026-07-22 00:00:00.000",+ "event_ts": "2026-07-22 00:00:00.000",+ "input": input,+ "output": output,+ "metadata_names": ["source"],+ "metadata_values": ["API"],+ "provided_usage_details": {"input": 1},+ "usage_details": {"output": 2},+ "provided_cost_details": {"input": "0.000000000001"},+ "cost_details": {"output": 1.25},+ "event_bytes": event_bytes,+ })+}++#[test]+fn event_bytes_excludes_the_accounting_field_and_counts_utf8() {+ let value = json!({"input": "🔥", "event_bytes": 123});+ assert_eq!(+ event_bytes(&value).expect("event bytes"),+ "{\"input\":\"🔥\"}".len() as u64+ );+ for payload in [+ json!({}),+ json!({"event_bytes": 99}),+ json!({"nested": [null, true, {"event_bytes": 7, "text": "\"\n\\🔥"}], "cost": 1e-13}),+ ] {+ let mut expected = payload.clone();+ expected.as_object_mut().unwrap().remove("event_bytes");+ assert_eq!(+ event_bytes(&payload).unwrap(),+ serde_json::to_vec(&expected).unwrap().len() as u64+ );+ }+}++#[test]+fn zero_max_rows_per_block_is_rejected() {+ let error = match encode_json_rows(&[], 0) {+ Ok(_) => panic!("zero block size must fail"),+ Err(error) => error,+ };+ assert_eq!(error, "max_rows_per_block must be greater than zero");+}++#[test]+fn decimal_parser_matches_clickhouse_scale_and_overflow_policy() {+ assert_eq!(parse_decimal("1.25", "cost").unwrap(), 1_250_000_000_000);+ let precise_json_number: Value = serde_json::from_str("510407.65505697404").unwrap();+ assert_eq!(+ parse_decimal_value(&precise_json_number, "cost").unwrap(),+ 510_407_655_056_974_040+ );+ assert_eq!(parse_decimal("0.0000000000001", "cost").unwrap(), 0);+ assert_eq!(+ parse_decimal("0.00004381800000000001", "cost").unwrap(),+ 43_818_000+ );+ assert_eq!(+ parse_decimal("-1.9999999999999", "cost").unwrap(),+ -1_999_999_999_999+ );+ assert_eq!(parse_decimal("1e6", "cost").unwrap(), DECIMAL_CLAMPED_MAX);+ assert_eq!(parse_decimal("-1e6", "cost").unwrap(), DECIMAL_CLAMPED_MIN);+ assert_eq!(parse_decimal("Infinity", "cost").unwrap(), 0);+ assert_eq!(parse_decimal("-Infinity", "cost").unwrap(), 0);+ assert_eq!(parse_decimal("1e2147483647", "cost").unwrap(), 0);+ assert_eq!(parse_decimal("1e-2147483648", "cost").unwrap(), 0);+ assert_eq!(parse_decimal("", "cost").unwrap(), 0);+ assert_eq!(u16::from_json(&json!(42.0)), Ok(42));+ assert!(u16::from_json(&json!(-1.0)).is_err());+}++#[test]+fn datetime_strings_are_explicitly_utc() {+ let naive = parse_datetime_value(&json!("2026-07-22 00:00:00.000"), "timestamp").unwrap();+ let utc = parse_datetime_value(&json!("2026-07-22T00:00:00.000Z"), "timestamp").unwrap();+ let offset =+ parse_datetime_value(&json!("2026-07-22T02:00:00.000+02:00"), "timestamp").unwrap();++ assert_eq!(naive, utc);+ assert_eq!(offset, utc);+}++#[test]+fn invalid_decimal_prevents_a_prepared_event_from_being_returned() {+ let mut value = row("input", "output", Value::Null);+ value["cost_details"] = json!({"output": "definitely-not-a-number"});++ let error = PreparedEvent::from_json(&value).expect_err("invalid decimal");+ assert!(error.contains("cost_details"));+ assert!(error.contains("invalid decimal"));+}++#[derive(Debug)]+struct LiveColumn {+ name: String,+ column_type: String,+ default_kind: String,+}++fn live_events_full_schema() -> Option<Vec<LiveColumn>> {+ let output = match run_clickhouse_client(+ "SELECT name, type, default_kind FROM system.columns WHERE database = currentDatabase() AND table = 'events_full' ORDER BY position FORMAT JSONEachRow",+ &[],+ ) {+ Ok(output) => output,+ Err(error) => {+ if clickhouse_is_required() {+ panic!("ClickHouse schema introspection failed: {error}");+ }+ eprintln!("ClickHouse schema unavailable; skipping generated-row test: {error}");+ return None;+ }+ };++ output+ .split(|byte| *byte == b'\n')+ .filter(|line| !line.is_empty())+ .map(|line| {+ serde_json::from_slice::<Value>(line)+ .map_err(|error| format!("parse ClickHouse schema row: {error}"))+ .and_then(|value| {+ Ok(LiveColumn {+ name: value["name"]+ .as_str()+ .ok_or_else(|| "schema row has no column name".to_owned())?+ .to_owned(),+ column_type: value["type"]+ .as_str()+ .ok_or_else(|| "schema row has no column type".to_owned())?+ .to_owned(),+ default_kind: value["default_kind"]+ .as_str()+ .unwrap_or_default()+ .to_owned(),+ })+ })+ })+ .collect::<Result<Vec<_>, _>>()+ .map(Some)+ .unwrap_or_else(|error| panic!("invalid ClickHouse schema response: {error}"))+}++fn assert_live_schema_matches_codec(schema: &[LiveColumn]) {+ let actual_names = schema+ .iter()+ .filter(|column| !is_server_owned(column))+ .map(|column| column.name.clone())+ .collect::<BTreeSet<_>>();+ let expected_names = EVENTS_FULL_INSERT_COLUMNS+ .iter()+ .map(|column| column.name.to_owned())+ .collect::<BTreeSet<_>>();+ assert_eq!(+ actual_names, expected_names,+ "events_full insertable columns differ from the Native codec contract"+ );++ for column in schema.iter().filter(|column| !is_server_owned(column)) {+ let kind = ColumnKind::from_clickhouse_type(&column.column_type)+ .unwrap_or_else(|error| panic!("{}: {error}", column.name));+ let spec = find_column(&column.name).expect("column set checked above");+ assert_eq!(+ spec.kind, kind,+ "{} changed from {:?} to {}",+ column.name, spec.kind, column.column_type+ );+ if spec.default_policy != DefaultPolicy::None {+ assert_eq!(+ column.default_kind, "DEFAULT",+ "{} no longer has the DEFAULT expression required by the Native adapter",+ column.name+ );+ }+ }+}++fn is_server_owned(column: &LiveColumn) -> bool {+ matches!(column.default_kind.as_str(), "MATERIALIZED" | "ALIAS")+}++fn clickhouse_is_required() -> bool {+ std::env::var("LANGFUSE_NATIVE_FAIL_IF_CLICKHOUSE_UNAVAILABLE")+ .ok()+ .as_deref()+ == Some("1")+}++#[derive(Debug)]+struct ClickhouseComparison {+ json_count: u64,+ native_count: u64,+ json_only: u64,+ native_only: u64,+ json_event_bytes: u64,+ native_event_bytes: u64,+}++fn clickhouse_server_round_trip(+ json_payload: &[u8],+ native_payload: &[u8],+) -> Option<ClickhouseComparison> {+ let required = clickhouse_is_required();+ let suffix = SystemTime::now()+ .duration_since(UNIX_EPOCH)+ .expect("system clock")+ .as_nanos();+ let json_table = format!("native_codec_json_{}_{}", std::process::id(), suffix);+ let native_table = format!("native_codec_native_{}_{}", std::process::id(), suffix);++ let create_result = run_clickhouse_client(+ &format!(+ "CREATE TABLE {json_table} AS events_full ENGINE = Memory; CREATE TABLE {native_table} AS events_full ENGINE = Memory;"+ ),+ &[],+ );+ if let Err(error) = create_result {+ let _ = run_clickhouse_client(+ &format!("DROP TABLE IF EXISTS {json_table}; DROP TABLE IF EXISTS {native_table};"),+ &[],+ );+ if required {+ panic!("ClickHouse server parity test failed to create tables: {error}");+ }+ eprintln!("ClickHouse server unavailable; skipping parity test: {error}");+ return None;+ }++ let result: Result<ClickhouseComparison, String> = (|| {+ run_clickhouse_client(+ &format!("INSERT INTO {json_table} FORMAT JSONEachRow"),+ json_payload,+ )?;+ run_clickhouse_client(+ &format!("INSERT INTO {native_table} FORMAT Native"),+ native_payload,+ )?;++ let output = run_clickhouse_client(+ &format!(+ "SELECT (SELECT count() FROM {json_table}) AS json_count, (SELECT count() FROM {native_table}) AS native_count, (SELECT count() FROM (SELECT * EXCEPT (event_bytes, provided_usage_details, usage_details, provided_cost_details, cost_details), mapSort((k, v) -> k, provided_usage_details) AS provided_usage_details, mapSort((k, v) -> k, usage_details) AS usage_details, mapSort((k, v) -> k, provided_cost_details) AS provided_cost_details, mapSort((k, v) -> k, cost_details) AS cost_details FROM {json_table} EXCEPT ALL SELECT * EXCEPT (event_bytes, provided_usage_details, usage_details, provided_cost_details, cost_details), mapSort((k, v) -> k, provided_usage_details) AS provided_usage_details, mapSort((k, v) -> k, usage_details) AS usage_details, mapSort((k, v) -> k, provided_cost_details) AS provided_cost_details, mapSort((k, v) -> k, cost_details) AS cost_details FROM {native_table})) AS json_only, (SELECT count() FROM (SELECT * EXCEPT (event_bytes, provided_usage_details, usage_details, provided_cost_details, cost_details), mapSort((k, v) -> k, provided_usage_details) AS provided_usage_details, mapSort((k, v) -> k, usage_details) AS usage_details, mapSort((k, v) -> k, provided_cost_details) AS provided_cost_details, mapSort((k, v) -> k, cost_details) AS cost_details FROM {native_table} EXCEPT ALL SELECT * EXCEPT (event_bytes, provided_usage_details, usage_details, provided_cost_details, cost_details), mapSort((k, v) -> k, provided_usage_details) AS provided_usage_details, mapSort((k, v) -> k, usage_details) AS usage_details, mapSort((k, v) -> k, provided_cost_details) AS provided_cost_details, mapSort((k, v) -> k, cost_details) AS cost_details FROM {json_table})) AS native_only, (SELECT sum(event_bytes) FROM {json_table}) AS json_event_bytes, (SELECT sum(event_bytes) FROM {native_table}) AS native_event_bytes SETTINGS asterisk_include_materialized_columns=1, asterisk_include_alias_columns=1 FORMAT JSONEachRow"+ ),+ &[],+ )?;+ let line = output+ .split(|byte| *byte == b'\n')+ .find(|line| !line.is_empty())+ .ok_or_else(|| "ClickHouse comparison returned no rows".to_owned())?;+ let value: Value = serde_json::from_slice(line)+ .map_err(|error| format!("parse ClickHouse comparison: {error}"))?;+ let comparison = ClickhouseComparison {+ json_count: output_u64(&value["json_count"]),+ native_count: output_u64(&value["native_count"]),+ json_only: output_u64(&value["json_only"]),+ native_only: output_u64(&value["native_only"]),+ json_event_bytes: output_u64(&value["json_event_bytes"]),+ native_event_bytes: output_u64(&value["native_event_bytes"]),+ };+ Ok(comparison)+ })();++ let _ = run_clickhouse_client(+ &format!("DROP TABLE IF EXISTS {json_table}; DROP TABLE IF EXISTS {native_table};"),+ &[],+ );++ match result {+ Ok(comparison) => Some(comparison),+ Err(error) => panic!("ClickHouse server parity test failed: {error}"),+ }+}++fn run_clickhouse_client(query: &str, input: &[u8]) -> Result<Vec<u8>, String> {+ let binary =+ std::env::var("LANGFUSE_NATIVE_CLICKHOUSE_BIN").unwrap_or_else(|_| "clickhouse".to_owned());+ let host = std::env::var("LANGFUSE_NATIVE_CLICKHOUSE_HOST")+ .or_else(|_| std::env::var("CLICKHOUSE_HOST"))+ .unwrap_or_else(|_| "127.0.0.1".to_owned());+ let port = std::env::var("LANGFUSE_NATIVE_CLICKHOUSE_PORT")+ .or_else(|_| std::env::var("CLICKHOUSE_PORT"))+ .unwrap_or_else(|_| "9000".to_owned());+ let user = std::env::var("CLICKHOUSE_USER").unwrap_or_else(|_| "clickhouse".to_owned());+ let password = std::env::var("CLICKHOUSE_PASSWORD").unwrap_or_else(|_| "clickhouse".to_owned());+ let database = std::env::var("CLICKHOUSE_DB").unwrap_or_else(|_| "default".to_owned());++ let mut child = Command::new(&binary)+ .args([+ "client",+ "--host",+ &host,+ "--port",+ &port,+ "--user",+ &user,+ "--password",+ &password,+ "--database",+ &database,+ "--multiquery",+ "--query",+ query,+ ])+ .stdin(Stdio::piped())+ .stdout(Stdio::piped())+ .stderr(Stdio::piped())+ .spawn()+ .map_err(|error| format!("start {binary} client: {error}"))?;+ child+ .stdin+ .take()+ .expect("ClickHouse client stdin")+ .write_all(input)+ .map_err(|error| format!("write ClickHouse client input: {error}"))?;+ let output = child+ .wait_with_output()+ .map_err(|error| format!("wait for ClickHouse client: {error}"))?;+ if !output.status.success() {+ return Err(String::from_utf8_lossy(&output.stderr).trim().to_owned());+ }+ Ok(output.stdout)+}++fn output_u64(value: &Value) -> u64 {+ value+ .as_u64()+ .or_else(|| value.as_str().and_then(|value| value.parse().ok()))+ .expect("ClickHouse UInt64 output")+}++proptest! {+ #[test]+ fn decimal_inputs_with_extreme_exponents_are_handled(+ integer in -999_999i64..=999_999,+ fraction in "[0-9]{1,24}",+ exponent in -1_000i32..=1_000,+ ) {+ let value = format!("{integer}.{fraction}e{exponent}");+ prop_assert!(parse_decimal(&value, "cost").is_ok());+ }++ #[test]+ fn decimal_scale_truncates_toward_zero_for_valid_prepared_costs(+ whole in -9i64..=9,+ fraction in 0u64..=9_999_999_999_999,+ ) {+ let value = format!("{whole}.{fraction:013}");+ let sign = if whole.is_negative() { -1 } else { 1 };+ let expected = whole * 1_000_000_000_000 + sign * (fraction / 10) as i64;+ prop_assert_eq!(parse_decimal(&value, "cost").unwrap(), expected);+ }+}++// Derive wire-format properties from the live table, independently of the native field list.+// The captured-traffic test in TypeScript separately exercises the production preparation path.+fn generated_row_strategy(schema: &[LiveColumn]) -> BoxedStrategy<Value> {+ let fields = schema+ .iter()+ .filter(|column| !is_server_owned(column))+ .map(|column| {+ let name = column.name.clone();+ let kind = ColumnKind::from_clickhouse_type(&column.column_type).unwrap();+ let value = match name.as_str() {+ "type" => {+ prop::sample::select(vec![json!("SPAN"), json!("GENERATION"), json!("EVENT")])+ .boxed()+ }+ "level" => {+ prop::sample::select(vec![json!("DEFAULT"), json!("ERROR"), json!("WARNING")])+ .boxed()+ }+ "source" => Just(json!("API")).boxed(),+ _ => generated_value_strategy(kind),+ };+ (value, any::<bool>()).prop_map(move |(value, omit)| {+ let defaulted = find_column(&name).unwrap().default_policy != DefaultPolicy::None;+ (+ name.clone(),+ if omit && defaulted { None } else { Some(value) },+ )+ })+ })+ .collect::<Vec<_>>();+ fields+ .prop_map(|fields| {+ let mut object = fields+ .into_iter()+ .filter_map(|(name, value)| value.map(|v| (name, v)))+ .collect::<serde_json::Map<_, _>>();+ for prefix in [+ "metadata",+ "experiment_metadata",+ "experiment_item_metadata",+ ] {+ let values = object[&format!("{prefix}_values")].as_array().unwrap();+ let names = [+ "evaluator_id",+ "evaluation_rule_id",+ "job_configuration_id",+ "evaluator_test",+ ];+ object.insert(format!("{prefix}_names"), json!(&names[..values.len()]));+ }+ Value::Object(object)+ })+ .boxed()+}++fn generated_value_strategy(kind: ColumnKind) -> BoxedStrategy<Value> {+ let text = prop_oneof![+ prop::collection::vec(any::<char>(), 0..24)+ .prop_map(|chars| chars.into_iter().collect::<String>()),+ Just("true".to_owned()),+ Just(String::new()),+ ]+ .boxed();+ let datetime = (0i64..2_000_000_000, 0u32..1_000_000)+ .prop_map(|(seconds, micros)| {+ json!(DateTime::<Utc>::from_timestamp(seconds, micros * 1000)+ .unwrap()+ .format("%Y-%m-%d %H:%M:%S%.6f")+ .to_string())+ })+ .boxed();+ match kind {+ ColumnKind::String => text.prop_map(Value::String).boxed(),+ ColumnKind::NullableString => prop::option::of(text).prop_map(|v| json!(v)).boxed(),+ ColumnKind::DateTime64 => datetime,+ ColumnKind::NullableDateTime64 => prop_oneof![Just(Value::Null), datetime].boxed(),+ ColumnKind::NullableUInt16 => prop::option::of(any::<u16>())+ .prop_map(|v| json!(v))+ .boxed(),+ ColumnKind::Bool => any::<bool>().prop_map(|v| json!(v)).boxed(),+ ColumnKind::UInt8 => (0u8..=1).prop_map(|v| json!(v)).boxed(),+ ColumnKind::UInt16 => any::<u16>().prop_map(|v| json!(v)).boxed(),+ ColumnKind::UInt64 => (0u64..=9_007_199_254_740_991)+ .prop_map(|v| json!(v))+ .boxed(),+ ColumnKind::Decimal => (-999_999.0f64..999_999.0).prop_map(|v| json!(v)).boxed(),+ ColumnKind::ArrayString => prop::collection::vec(text, 0..=4)+ .prop_map(|v| json!(v))+ .boxed(),+ ColumnKind::MapStringString => prop::collection::btree_map(text.clone(), text, 0..=4)+ .prop_map(|v| json!(v))+ .boxed(),+ ColumnKind::MapStringUInt64 => {+ prop::collection::btree_map(text, 0u64..=9_007_199_254_740_991, 0..=4)+ .prop_map(|v| json!(v))+ .boxed()+ }+ ColumnKind::MapStringDecimal => {+ prop::collection::btree_map(text, -999_999.0f64..999_999.0, 0..=4)+ .prop_map(|v| json!(v))+ .boxed()+ }+ }+}++#[test]+fn generated_prepared_rows_match_json_each_row_and_native() {+ let Some(schema) = live_events_full_schema() else {+ return;+ };+ assert_live_schema_matches_codec(&schema);+ let mut runner = TestRunner::new(ProptestConfig {+ cases: 32,+ ..ProptestConfig::default()+ });+ runner+ .run(+ &(+ prop::collection::vec(generated_row_strategy(&schema), 1..=8),+ 1usize..=8,+ ),+ |(rows, max_rows)| {+ let blocks = encode_json_rows(&rows, max_rows).map_err(TestCaseError::fail)?;+ prop_assert_eq!(blocks.len(), rows.len().div_ceil(max_rows));+ for (block, expected_rows) in blocks.iter().zip(rows.chunks(max_rows)) {+ prop_assert_eq!(block.row_count, expected_rows.len());+ }+ let native_payload = blocks+ .into_iter()+ .flat_map(|block| block.bytes)+ .collect::<Vec<_>>();+ let json_payload = rows+ .iter()+ .flat_map(|row| {+ let mut bytes = serde_json::to_vec(row).unwrap();+ bytes.push(b'\n');+ bytes+ })+ .collect::<Vec<_>>();+ let comparison = clickhouse_server_round_trip(&json_payload, &native_payload)+ .expect("schema introspection succeeded, so parity server is available");+ prop_assert_eq!(comparison.json_count, rows.len() as u64);+ prop_assert_eq!(comparison.native_count, rows.len() as u64);+ prop_assert_eq!(comparison.json_only, 0);+ prop_assert_eq!(comparison.native_only, 0);+ prop_assert_eq!(+ comparison.json_event_bytes,+ rows.iter()+ .map(|r| output_u64(&r["event_bytes"]))+ .sum::<u64>()+ );+ prop_assert_eq!(+ comparison.native_event_bytes,+ rows.iter().map(|r| event_bytes(r).unwrap()).sum::<u64>()+ );+ Ok(())+ },+ )+ .expect("generated prepared rows match JSONEachRow and Native");+}diff --git a/packages/native/src/native_schema.rs b/packages/native/src/native_schema.rsnew file mode 100644index 0000000000..918be8a7a1--- /dev/null+++ b/packages/native/src/native_schema.rs@@ -0,0 +1,262 @@+//! The insertable part of `events_full` and its prepared Rust row.+//!+//! The macro invocation below is the only per-column declaration. It drives the prepared row,+//! the live-schema metadata, JSON conversion, and the typed Native column writes together.++use clickhouse::native::builder::BlockBuilder;+use serde_json::Value;++use crate::native_codec::{DateTime64Micros, Decimal64, FromJson};++#[derive(Clone, Copy, Debug, Eq, PartialEq)]+pub(crate) enum ColumnKind {+ String,+ DateTime64,+ NullableDateTime64,+ NullableString,+ NullableUInt16,+ Bool,+ UInt8,+ UInt16,+ UInt64,+ Decimal,+ ArrayString,+ MapStringString,+ MapStringUInt64,+ MapStringDecimal,+}++impl ColumnKind {+ /// Convert the type spelling returned by `system.columns` to the codec's logical type.+ #[cfg(test)]+ pub(crate) fn from_clickhouse_type(value: &str) -> Result<Self, String> {+ let normalized: String = value+ .chars()+ .filter(|character| !character.is_whitespace())+ .collect();+ match normalized.as_str() {+ "String" | "LowCardinality(String)" => Ok(Self::String),+ "DateTime64(6)" => Ok(Self::DateTime64),+ "Nullable(DateTime64(6))" => Ok(Self::NullableDateTime64),+ "Nullable(String)" | "Nullable(LowCardinality(String))" => Ok(Self::NullableString),+ "Nullable(UInt16)" => Ok(Self::NullableUInt16),+ "Bool" => Ok(Self::Bool),+ "UInt8" => Ok(Self::UInt8),+ "UInt16" => Ok(Self::UInt16),+ "UInt64" => Ok(Self::UInt64),+ "Decimal(18,12)" => Ok(Self::Decimal),+ "Array(String)" | "Array(LowCardinality(String))" => Ok(Self::ArrayString),+ "Map(String,String)" | "Map(LowCardinality(String),String)" => {+ Ok(Self::MapStringString)+ }+ "Map(String,UInt64)" | "Map(LowCardinality(String),UInt64)" => {+ Ok(Self::MapStringUInt64)+ }+ "Map(String,Decimal(18,12))" | "Map(LowCardinality(String),Decimal(18,12))" => {+ Ok(Self::MapStringDecimal)+ }+ _ => Err(format!("unsupported events_full column type {value:?}")),+ }+ }+}++#[derive(Clone, Copy, Debug, Eq, PartialEq)]+pub(crate) enum DefaultPolicy {+ None,+ Literal(&'static str),+ Metadata(&'static str),+ MetadataFallback(&'static str, &'static str),+ MetadataEquals(&'static str, &'static str),+}++#[derive(Clone, Copy, Debug, Eq, PartialEq)]+pub(crate) struct ColumnSpec {+ pub(crate) name: &'static str,+ pub(crate) kind: ColumnKind,+ pub(crate) default_policy: DefaultPolicy,+}++macro_rules! schema_column_name {+ ($field:ident) => {+ stringify!($field)+ };+ ($field:ident as $name:literal) => {+ $name+ };+}++macro_rules! prepared_json_value {+ (+ $value:ident,+ $event_bytes:ident,+ event_bytes $(as $name:literal)?,+ $ty:ty,+ $policy:expr,+ computed+ ) => {+ $event_bytes+ };+ (+ $value:ident,+ $event_bytes:ident,+ $field:ident $(as $name:literal)?,+ $ty:ty,+ $policy:expr+ ) => {+ crate::native_codec::from_json_field::<$ty>(+ $value,+ schema_column_name!($field $(as $name)?),+ $policy,+ )?+ };+}++/// Declare the prepared row and every operation that must agree with its shape.+macro_rules! clickhouse_table_schema {+ (+ $(+ $field:ident $(as $name:literal)? : $ty:ty => $( @ $source:ident )? $policy:expr+ ),* $(,)?+ ) => {+ #[derive(Clone, Debug)]+ pub(crate) struct PreparedEvent {+ $(pub(crate) $field: $ty,)*+ }++pub(crate) const EVENTS_FULL_INSERT_COLUMNS: &[ColumnSpec] = &[+ $(ColumnSpec {+ name: schema_column_name!($field $(as $name)?),+ kind: <$ty as FromJson>::COLUMN_KIND,+ default_policy: $policy,+ },)*+ ];++ impl PreparedEvent {+ /// Convert one prepared ingestion object and compute accounting before typed defaults.+ pub(crate) fn from_json(value: &Value) -> Result<Self, String> {+ // `event_bytes` describes the immutable prepared JSON row. In particular, it+ // intentionally sees raw decimal values and omitted DEFAULT-backed fields.+ let event_bytes = crate::native_codec::event_bytes(value)?;++ Ok(Self {+ $(+ $field: prepared_json_value!(+ value,+ event_bytes,+ $field $(as $name)?,+ $ty,+ $policy $(, $source)?+ ),+ )*+ })+ }++ /// Write this row type through clickhouse-rs's typed `Encode` implementations.+ pub(crate) fn encode_into(+ builder: &mut BlockBuilder,+ rows: &[Self],+ ) -> Result<(), String> {+ $(+ let name = schema_column_name!($field $(as $name)?);+ let mut column = builder+ .upsert_column::<&$ty>(name)+ .map_err(|error| format!("cannot add column {name}: {error}"))?;+ for row in rows {+ column+ .add(&row.$field)+ .map_err(|error| format!("cannot encode column {name}: {error}"))?;+ }+ )*+ Ok(())+ }+ }+ };+}++// Keep this list in the same order as the insertable events_full contract. Materialized and alias+// columns remain server-owned and therefore are intentionally absent.+clickhouse_table_schema! {+ project_id: String => DefaultPolicy::None,+ trace_id: String => DefaultPolicy::None,+ span_id: String => DefaultPolicy::None,+ parent_span_id: String => DefaultPolicy::None,+ name: String => DefaultPolicy::None,+ environment: String => DefaultPolicy::Literal("default"),+ version: String => DefaultPolicy::None,+ release: String => DefaultPolicy::None,+ trace_name: String => DefaultPolicy::None,+ user_id: String => DefaultPolicy::None,+ session_id: String => DefaultPolicy::None,+ level: String => DefaultPolicy::None,+ status_message: String => DefaultPolicy::None,+ prompt_id: String => DefaultPolicy::None,+ prompt_name: String => DefaultPolicy::None,+ model_id: String => DefaultPolicy::None,+ provided_model_name: String => DefaultPolicy::None,+ model_parameters: String => DefaultPolicy::None,+ input: String => DefaultPolicy::None,+ output: String => DefaultPolicy::None,+ evaluator_id: String => DefaultPolicy::Metadata("evaluator_id"),+ evaluation_rule_id: String => {+ DefaultPolicy::MetadataFallback("evaluation_rule_id", "job_configuration_id")+ },+ experiment_id: String => DefaultPolicy::None,+ experiment_name: String => DefaultPolicy::None,+ experiment_description: String => DefaultPolicy::None,+ experiment_dataset_id: String => DefaultPolicy::None,+ experiment_item_id: String => DefaultPolicy::None,+ experiment_item_expected_output: String => DefaultPolicy::None,+ experiment_item_root_span_id: String => DefaultPolicy::None,+ source: String => DefaultPolicy::None,+ service_name: String => DefaultPolicy::None,+ service_version: String => DefaultPolicy::None,+ scope_name: String => DefaultPolicy::None,+ scope_version: String => DefaultPolicy::None,+ telemetry_sdk_language: String => DefaultPolicy::None,+ telemetry_sdk_name: String => DefaultPolicy::None,+ telemetry_sdk_version: String => DefaultPolicy::None,+ blob_storage_file_path: String => DefaultPolicy::None,+ ingestion_api_key: String => DefaultPolicy::None,+ ingestion_sdk_name: String => DefaultPolicy::None,+ ingestion_sdk_version: String => DefaultPolicy::None,+ event_type as "type": String => DefaultPolicy::None,+ start_time: DateTime64Micros => DefaultPolicy::None,+ created_at: DateTime64Micros => DefaultPolicy::None,+ updated_at: DateTime64Micros => DefaultPolicy::None,+ event_ts: DateTime64Micros => DefaultPolicy::None,+ end_time: Option<DateTime64Micros> => DefaultPolicy::None,+ completion_start_time: Option<DateTime64Micros> => DefaultPolicy::None,+ experiment_item_version: Option<DateTime64Micros> => DefaultPolicy::None,+ prompt_version: Option<u16> => DefaultPolicy::None,+ is_app_root: bool => DefaultPolicy::None,+ bookmarked: bool => DefaultPolicy::None,+ public: bool => DefaultPolicy::None,+ evaluator_execution_is_test: bool => {+ DefaultPolicy::MetadataEquals("evaluator_test", "true")+ },+ is_deleted: u8 => DefaultPolicy::None,+ event_bytes: u64 => @computed DefaultPolicy::None,+ tags: Vec<String> => DefaultPolicy::None,+ tool_calls: Vec<String> => DefaultPolicy::None,+ tool_call_names: Vec<String> => DefaultPolicy::None,+ metadata_names: Vec<String> => DefaultPolicy::None,+ metadata_values: Vec<String> => DefaultPolicy::None,+ experiment_metadata_names: Vec<String> => DefaultPolicy::None,+ experiment_metadata_values: Vec<String> => DefaultPolicy::None,+ experiment_item_metadata_names: Vec<String> => DefaultPolicy::None,+ experiment_item_metadata_values: Vec<String> => DefaultPolicy::None,+ provided_usage_details: std::collections::BTreeMap<String, u64> => DefaultPolicy::None,+ usage_details: std::collections::BTreeMap<String, u64> => DefaultPolicy::None,+ provided_cost_details: std::collections::BTreeMap<String, Decimal64> => DefaultPolicy::None,+ cost_details: std::collections::BTreeMap<String, Decimal64> => DefaultPolicy::None,+ usage_pricing_tier_id: Option<String> => DefaultPolicy::None,+ usage_pricing_tier_name: Option<String> => DefaultPolicy::None,+ tool_definitions: std::collections::BTreeMap<String, String> => DefaultPolicy::None,+}++#[cfg(test)]+pub(crate) fn find_column(name: &str) -> Option<&'static ColumnSpec> {+ EVENTS_FULL_INSERT_COLUMNS+ .iter()+ .find(|column| column.name == name)+}diff --git a/worker/Dockerfile b/worker/Dockerfileindex f83123fbc2..e00649a738 100644--- a/worker/Dockerfile+++ b/worker/Dockerfile@@ -37,14 +37,15 @@ WORKDIR /app# The toolchain comes from the official Rust image for the same Alpine release,# so the addon compiles with the exact version pinned in# packages/native/rust-toolchain.toml; bump the two together. gcc and musl-dev-# provide the linker and C runtime objects the build needs.+# provide the linker and C runtime objects the build needs; curl and patch+# prepare the small clickhouse source overlay used by the native encoder.COPY --from=rust:1.98.0-alpine3.24 /usr/local/rustup /usr/local/rustupCOPY --from=rust:1.98.0-alpine3.24 /usr/local/cargo /usr/local/cargoENV RUSTUP_HOME=/usr/local/rustup CARGO_HOME=/usr/local/cargo PATH=/usr/local/cargo/bin:$PATH# rustc's musl targets default to a `<arch>-linux-musl-gcc` linker name that# Alpine does not ship; its gcc is reachable as `cc`.ENV CARGO_TARGET_AARCH64_UNKNOWN_LINUX_MUSL_LINKER=cc CARGO_TARGET_X86_64_UNKNOWN_LINUX_MUSL_LINKER=cc-RUN apk add --no-cache gcc musl-dev+RUN apk add --no-cache gcc musl-dev curl patch# First install the dependencies (as they change less often)COPY --from=pruner /app/out/pnpm-lock.yaml ./pnpm-lock.yamldiff --git a/worker/package.json b/worker/package.jsonindex 14ce6e48db..38d4e3848c 100644--- a/worker/package.json+++ b/worker/package.json@@ -27,7 +27,8 @@"refill-ingestion-events": "dotenv -e ../.env -- tsx src/scripts/replayIngestionEvents/s3-ingestion-event-replay.ts","refill-billing-event": "dotenv -e ../.env -- tsx src/scripts/refill-billing-event.ts","refill-queue-event": "dotenv -e ../.env -- tsx src/scripts/refillQueueEvent/index.ts",- "retry-monitor-bad-query-errors": "dotenv -e ../.env -- tsx src/scripts/retryMonitorBadQueryErrors.ts"+ "retry-monitor-bad-query-errors": "dotenv -e ../.env -- tsx src/scripts/retryMonitorBadQueryErrors.ts",+ "test:native-codec": "LANGFUSE_NATIVE_FAIL_IF_CLICKHOUSE_UNAVAILABLE=1 vitest run nativeCodec.integration"},"author": "[email protected]","dependencies": {diff --git a/worker/src/__tests__/helpers/nativeCodecFixtures.ts b/worker/src/__tests__/helpers/nativeCodecFixtures.tsnew file mode 100644index 0000000000..64286fbd8a--- /dev/null+++ b/worker/src/__tests__/helpers/nativeCodecFixtures.ts@@ -0,0 +1,223 @@+import { readFileSync, readdirSync } from "node:fs";+import { resolve } from "node:path";+import type { EventRecordInsertType } from "@langfuse/shared/src/server";+import type { EventInput } from "../../services/IngestionService/index.js";++// Read the existing trace captures and exercise production createEventRecord/writeEventRecord.+// Model lookup responses are replayed from captures; no fixture JSON is copied into Rust.+type JsonObject = Record<string, unknown>;+const repoRoot = resolve(__dirname, "../../../..");++const parseObject = (value: unknown): JsonObject => {+ if (value && typeof value === "object" && !Array.isArray(value)) {+ return value as JsonObject;+ }+ if (typeof value === "string") {+ try {+ const parsed: unknown = JSON.parse(value);+ return parseObject(parsed);+ } catch {+ return {};+ }+ }+ return {};+};++const numberMap = (value: unknown): Record<string, number> =>+ Object.fromEntries(+ Object.entries(parseObject(value)).flatMap(([key, item]) => {+ const number = typeof item === "number" ? item : Number(item);+ return Number.isFinite(number) ? [[key, number]] : [];+ }),+ );++const stringArray = (value: unknown): string[] =>+ Array.isArray(value)+ ? value.filter((item): item is string => typeof item === "string")+ : [];++const stringMap = (value: unknown): Record<string, string> =>+ Object.fromEntries(+ Object.entries(parseObject(value)).map(([key, item]) => [+ key,+ typeof item === "string" ? item : JSON.stringify(item),+ ]),+ );++const optionalString = (value: unknown): string | undefined =>+ typeof value === "string" ? value : undefined;++const mergeObjects = (...values: unknown[]): JsonObject =>+ Object.assign({}, ...values.map(parseObject));++const fixtureDirectories = [+ resolve(repoRoot, "packages/shared/scripts/seeder/utils/framework-traces"),+ resolve(repoRoot, "worker/src/__tests__/chatml/framework-traces"),+];++const fixturePaths = fixtureDirectories+ .flatMap((directory) =>+ readdirSync(directory)+ .filter((name) =>+ directory.endsWith("chatml/framework-traces")+ ? name.endsWith(".trace.json")+ : name.endsWith(".json"),+ )+ .map((name) => resolve(directory, name)),+ )+ .sort();++const eventInputFromCapture = (+ observation: JsonObject,+ trace: JsonObject,+): EventInput => {+ const isRoot = observation.parentObservationId == null;+ const usageDetails = numberMap(observation.usageDetails);+ const costDetails = numberMap(observation.costDetails);+ const providedUsageDetails = numberMap(observation.providedUsageDetails);+ const providedCostDetails = numberMap(observation.providedCostDetails);++ return {+ projectId: String(observation.projectId ?? trace.projectId ?? "fixture"),+ traceId: String(observation.traceId ?? trace.id),+ spanId: String(observation.id),+ parentSpanId:+ typeof observation.parentObservationId === "string"+ ? observation.parentObservationId+ : undefined,+ startTimeISO: String(observation.startTime),+ endTimeISO: String(observation.endTime ?? observation.startTime),+ completionStartTime: optionalString(observation.completionStartTime),+ name: typeof observation.name === "string" ? observation.name : "",+ type: typeof observation.type === "string" ? observation.type : "SPAN",+ environment: String(+ observation.environment ?? trace.environment ?? "default",+ ),+ version:+ typeof observation.version === "string" ? observation.version : undefined,+ release: typeof trace.release === "string" ? trace.release : undefined,+ traceName: typeof trace.name === "string" ? trace.name : undefined,+ tags: Array.isArray(trace.tags)+ ? trace.tags.filter((tag): tag is string => typeof tag === "string")+ : [],+ public: trace.public === true,+ bookmarked: isRoot && trace.bookmarked === true,+ userId: typeof trace.userId === "string" ? trace.userId : undefined,+ sessionId:+ typeof trace.sessionId === "string" ? trace.sessionId : undefined,+ level:+ typeof observation.level === "string" ? observation.level : undefined,+ statusMessage:+ typeof observation.statusMessage === "string"+ ? observation.statusMessage+ : undefined,+ modelName:+ typeof observation.model === "string" ? observation.model : undefined,+ modelId: optionalString(observation.internalModelId),+ modelParameters:+ observation.modelParameters as EventInput["modelParameters"],+ providedUsageDetails:+ Object.keys(providedUsageDetails).length > 0+ ? providedUsageDetails+ : usageDetails,+ usageDetails,+ providedCostDetails:+ Object.keys(providedCostDetails).length > 0+ ? providedCostDetails+ : costDetails,+ costDetails,+ usagePricingTierId: optionalString(observation.usagePricingTierId),+ usagePricingTierName: optionalString(observation.usagePricingTierName),+ promptId: optionalString(observation.promptId),+ promptName: optionalString(observation.promptName),+ promptVersion:+ observation.promptVersion == null+ ? undefined+ : String(observation.promptVersion),+ toolDefinitions: stringMap(observation.toolDefinitions),+ toolCalls: stringArray(observation.toolCalls),+ toolCallNames: stringArray(observation.toolCallNames),+ input: observation.input as string | undefined,+ output: observation.output as string | undefined,+ metadata: mergeObjects(trace.metadata, observation.metadata),+ source: "API",+ ingestionApiKey: "",+ ingestionSdkName: "",+ ingestionSdkVersion: "",+ blobStorageFilePath: "fixtures",+ eventBytes: 0,+ };+};++export const capturedPreparedRows = async () => {+ const { IngestionService } =+ await import("../../services/IngestionService/index.js");++ const normalizedRows: EventRecordInsertType[] = [];+ const capturedGenerationUsage = new Map<+ string,+ Pick<+ EventRecordInsertType,+ | "model_id"+ | "usage_pricing_tier_id"+ | "usage_pricing_tier_name"+ | "usage_details"+ | "cost_details"+ >+ >();+ const writer = {+ addToQueue: (_table: unknown, record: EventRecordInsertType) => {+ normalizedRows.push(record);+ },+ };+ const service = new IngestionService(+ {} as never,+ {} as never,+ writer as never,+ {} as never,+ );++ // Avoid external model and prompt lookups while retaining the production method's field mapping.+ // Captured generation enrichment is replayed where the source trace already contains it.+ (service as any).getGenerationUsage = async ({+ observationRecord,+ }: {+ observationRecord: EventRecordInsertType;+ }) => ({+ usage_details: capturedGenerationUsage.get(observationRecord.id)+ ?.usage_details,+ cost_details: capturedGenerationUsage.get(observationRecord.id)+ ?.cost_details,+ internal_model_id: capturedGenerationUsage.get(observationRecord.id)+ ?.model_id,+ usage_pricing_tier_id: capturedGenerationUsage.get(observationRecord.id)+ ?.usage_pricing_tier_id,+ usage_pricing_tier_name: capturedGenerationUsage.get(observationRecord.id)+ ?.usage_pricing_tier_name,+ });++ for (const fixturePath of fixturePaths) {+ const fixture = JSON.parse(readFileSync(fixturePath, "utf8")) as {+ trace?: JsonObject;+ observations?: JsonObject[];+ };+ const trace = fixture.trace ?? {};+ for (const observation of fixture.observations ?? []) {+ const eventInput = eventInputFromCapture(observation, trace);+ capturedGenerationUsage.set(eventInput.spanId, {+ usage_details: eventInput.usageDetails ?? {},+ cost_details: eventInput.costDetails ?? {},+ model_id: eventInput.modelId,+ usage_pricing_tier_id: eventInput.usagePricingTierId,+ usage_pricing_tier_name: eventInput.usagePricingTierName,+ });+ const eventRecord = await service.createEventRecord(+ eventInput,+ `fixtures/${fixturePath.split("/").pop()}`,+ );+ await service.writeEventRecord(eventRecord);+ }+ }++ return normalizedRows;+};diff --git a/worker/src/__tests__/nativeCodec.integration.test.ts b/worker/src/__tests__/nativeCodec.integration.test.tsnew file mode 100644index 0000000000..38f2f6bd95--- /dev/null+++ b/worker/src/__tests__/nativeCodec.integration.test.ts@@ -0,0 +1,184 @@+import { execFileSync } from "node:child_process";+import { mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs";+import { tmpdir } from "node:os";+import { resolve } from "node:path";+import { randomUUID } from "node:crypto";+import { afterAll, beforeAll, describe, expect, it } from "vitest";+import {+ clickhouseClient,+ type ClickhouseClientType,+ type EventRecordInsertType,+} from "@langfuse/shared/src/server";+import { ClickhouseWriter, TableName } from "../services/ClickhouseWriter";+import { capturedPreparedRows } from "./helpers/nativeCodecFixtures";++type Row = Record<string, unknown>;+type Column = { name: string; type: string; default_kind: string };+type Manifest = {+ eventBytes: number[];+};+const nativeDir = resolve(__dirname, "../../../packages/native");++// This suite is run explicitly by test:native-codec and CI. Ordinary worker tests need neither+// cargo nor a Native-protocol client. The explicit run always fails if ClickHouse is unavailable.+describe.runIf(+ process.env.LANGFUSE_NATIVE_FAIL_IF_CLICKHOUSE_UNAVAILABLE === "1",+)("TypeScript and Rust Native ingestion parity", () => {+ let directory: string;+ let columns: Column[];+ let writer: ClickhouseWriter;+ let jsonTable: string;+ let nativeTable: string;+ const client = clickhouseClient();++ const query = (sql: string, input: Buffer = Buffer.alloc(0)) =>+ execFileSync(+ process.env.LANGFUSE_NATIVE_CLICKHOUSE_BIN ?? "clickhouse",+ [+ "client",+ "--host",+ process.env.LANGFUSE_NATIVE_CLICKHOUSE_HOST ?? "127.0.0.1",+ "--port",+ process.env.LANGFUSE_NATIVE_CLICKHOUSE_PORT ?? "9000",+ "--user",+ process.env.CLICKHOUSE_USER ?? "clickhouse",+ "--password",+ process.env.CLICKHOUSE_PASSWORD ?? "clickhouse",+ "--database",+ process.env.CLICKHOUSE_DB ?? "default",+ "--multiquery",+ "--query",+ sql,+ ],+ { input, maxBuffer: 16 * 1024 * 1024, timeout: 30_000 },+ );++ const encode = (rows: Row[], maxRowsPerBlock: number): Manifest => {+ const input = resolve(directory, "input.json");+ writeFileSync(input, JSON.stringify({ rows, maxRowsPerBlock }));+ execFileSync(+ "cargo",+ ["run", "--locked", "--example", "native_codec_test_driver"],+ {+ cwd: nativeDir,+ env: {+ ...process.env,+ LANGFUSE_NATIVE_TEST_INPUT: input,+ LANGFUSE_NATIVE_TEST_OUTPUT: directory,+ },+ timeout: 120_000,+ stdio: "pipe",+ },+ );+ return JSON.parse(+ readFileSync(resolve(directory, "manifest.json"), "utf8"),+ ) as Manifest;+ };++ beforeAll(async () => {+ query("SELECT 1");+ directory = mkdtempSync(resolve(tmpdir(), "native-codec-"));+ execFileSync("sh", ["scripts/prepare-clickhouse-rs.sh"], {+ cwd: nativeDir,+ timeout: 120_000,+ });+ columns = query(+ "SELECT name, type, default_kind FROM system.columns WHERE database=currentDatabase() AND table='events_full' ORDER BY position FORMAT JSONEachRow",+ )+ .toString()+ .trim()+ .split("\n")+ .map((line) => JSON.parse(line) as Column);+ // Exercise production writer clamping and the real JS client's JSONEachRow serializer.+ // Only redirect the destination table; leave the records and insert settings untouched.+ writer = ClickhouseWriter.getInstance({+ insert: async (params: Parameters<ClickhouseClientType["insert"]>[0]) => {+ return client.insert({ ...params, table: jsonTable });+ },+ } as ClickhouseClientType);+ if (writer.intervalId) clearInterval(writer.intervalId);+ writer.intervalId = null;+ writer.batchSize = Number.MAX_SAFE_INTEGER;+ }, 120_000);++ afterAll(async () => {+ if (writer) await writer.shutdown();+ await client.close();+ if (directory) rmSync(directory, { recursive: true, force: true });+ });++ const compare = async (rows: Row[], maxRows: number) => {+ const suffix = randomUUID().replaceAll("-", "");+ jsonTable = `native_codec_json_${suffix}`;+ nativeTable = `native_codec_native_${suffix}`;+ const manifest = encode(rows, maxRows);+ try {+ query(+ `CREATE TABLE ${jsonTable} AS events_full ENGINE=Memory; CREATE TABLE ${nativeTable} AS events_full ENGINE=Memory`,+ );+ for (const row of rows)+ writer.addToQueue(+ TableName.EventsFull,+ structuredClone(row) as EventRecordInsertType,+ );+ await writer.flushAll(true);+ query(+ `INSERT INTO ${nativeTable} FORMAT Native`,+ readFileSync(resolve(directory, "native.bin")),+ );+ // Compare typed values inside ClickHouse: preserve Decimal/UInt64 precision, duplicates,+ // defaults, materialized and alias columns. Map key order is not semantically meaningful.+ const maps = columns+ .filter((c) => c.type.startsWith("Map("))+ .map((c) => c.name);+ const projection = `* EXCEPT (event_bytes, ${maps.join(", ")}), ${maps.map((name) => `mapSort((k, v) -> k, ${name}) AS ${name}`).join(", ")}`;+ const select = (table: string) => `SELECT ${projection} FROM ${table}`;+ const result = JSON.parse(+ query(`SELECT+ (SELECT count() FROM ${jsonTable}) AS json_count,+ (SELECT count() FROM ${nativeTable}) AS native_count,+ (SELECT count() FROM (${select(jsonTable)} EXCEPT ALL ${select(nativeTable)})) AS json_only,+ (SELECT count() FROM (${select(nativeTable)} EXCEPT ALL ${select(jsonTable)})) AS native_only,+ (SELECT sum(event_bytes) FROM ${jsonTable}) AS json_bytes,+ (SELECT sum(event_bytes) FROM ${nativeTable}) AS native_bytes+ SETTINGS asterisk_include_materialized_columns=1, asterisk_include_alias_columns=1 FORMAT JSONEachRow`).toString(),+ ) as Record<string, string | number>;+ expect(Number(result.json_count)).toBe(rows.length);+ expect(Number(result.native_count)).toBe(rows.length);+ expect(Number(result.json_only)).toBe(0);+ expect(Number(result.native_only)).toBe(0);+ expect(BigInt(result.json_bytes)).toBe(+ rows.reduce((sum, row) => sum + BigInt(row.event_bytes as number), 0n),+ );+ expect(BigInt(result.native_bytes)).toBe(+ manifest.eventBytes.reduce((sum, bytes) => sum + BigInt(bytes), 0n),+ );+ } finally {+ // Discard failed writes before shutdown can retry against the temporary table.+ writer.queue[TableName.EventsFull] = [];+ query(+ `DROP TABLE IF EXISTS ${jsonTable}; DROP TABLE IF EXISTS ${nativeTable}`,+ );+ }+ };++ it("compares captured traffic through production TS preparation and both writers", async () => {+ const rows = await capturedPreparedRows();+ expect(rows.length).toBeGreaterThan(0);+ // This capture has inferred costs despite explicitly provided zero costs.+ const enriched = rows.find(+ (row) => row.span_id === "0d8f5427-c91a-44c8-86c5-e12d806edd57",+ );+ expect(enriched?.provided_cost_details).toEqual({+ input: 0,+ output: 0,+ total: 0,+ });+ expect(enriched?.cost_details).toEqual({+ input: 0.00147,+ output: 0.0024,+ total: 0.00387,+ });+ await compare(rows, 16);+ }, 120_000);+});