Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

chore(native): replace patched ClickHouse encoder with ch-core (#18968)

Valery Meleshkin358f6237dded
diff --git a/.github/dependabot.yml b/.github/dependabot.yml
index c06c79d703..1363ab4f54 100644
--- a/.github/dependabot.yml
+++ b/.github/dependabot.yml
@@ -34,10 +34,6 @@ updates:
patterns:
- "*"
- # Dependabot resolves without the clickhouse overlay from
- # packages/native/scripts/prepare-clickhouse-rs.sh, so its Cargo.lock adds
- # `source` and `checksum` lines to the `clickhouse` entry. Drop those two
- # lines before merging, or CI's `--locked` builds reject the lockfile.
- package-ecosystem: cargo
directory: "/packages/native"
assignees:
diff --git a/.github/workflows/dependabot-security-maintainer.lock.yml b/.github/workflows/dependabot-security-maintainer.lock.yml
index 6c90039013..3f05cb091f 100644
--- a/.github/workflows/dependabot-security-maintainer.lock.yml
+++ b/.github/workflows/dependabot-security-maintainer.lock.yml
@@ -1,4 +1,4 @@
-# gh-aw-metadata: {"schema_version":"v4","frontmatter_hash":"b2335945e7d0a86f3eb65428aa9a9da8e1ade083dc753acd7a5e19f0549f46ec","body_hash":"cab1aea2a1986d2a160b862f84b3ede2dfbc72b36e735b034c3bdd40cd309725","compiler_version":"v0.86.2","strict":true,"agent_id":"claude","agent_model":"claude-opus-5-5","engine_versions":{"claude":"2.1.277"}}
+# gh-aw-metadata: {"schema_version":"v4","frontmatter_hash":"a1f41154bb81ec8aaf2cee2efebda57a5c33a5999e1be2ea9ff8b46a3fdb5d3f","body_hash":"cab1aea2a1986d2a160b862f84b3ede2dfbc72b36e735b034c3bdd40cd309725","compiler_version":"v0.86.2","strict":true,"agent_id":"claude","agent_model":"claude-opus-5-5","engine_versions":{"claude":"2.1.277"}}
# gh-aw-manifest: {"version":1,"secrets":["CLAUDE_API_KEY","COPILOT_GITHUB_TOKEN","GH_ACCESS_TOKEN","GH_AW_GITHUB_MCP_SERVER_TOKEN","GH_AW_GITHUB_TOKEN","GH_AW_LANGFUSE_OTLP_BASIC_AUTH","GITHUB_TOKEN"],"actions":[{"repo":"actions/cache/restore","sha":"55cc8345863c7cc4c66a329aec7e433d2d1c52a9","version":"v6.1.0"},{"repo":"actions/cache/save","sha":"55cc8345863c7cc4c66a329aec7e433d2d1c52a9","version":"v6.1.0"},{"repo":"actions/checkout","sha":"3d3c42e5aac5ba805825da76410c181273ba90b1","version":"v7.0.1"},{"repo":"actions/download-artifact","sha":"3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c","version":"v8.0.1"},{"repo":"actions/github-script","sha":"3a2844b7e9c422d3c10d287c895573f7108da1b3","version":"v9.0.0"},{"repo":"actions/setup-node","sha":"820762786026740c76f36085b0efc47a31fe5020","version":"v7.0.0"},{"repo":"actions/upload-artifact","sha":"043fb46d1a93c77aae656e7c1c64a875d1fc6a0a","version":"v7.0.1"},{"repo":"github/gh-aw-actions/setup","sha":"6aab9e5b5c91c615506061f09bedd81a23babe3c","version":"v0.86.2"},{"repo":"pnpm/setup","sha":"703c52620218391530e48b9e8870d5c0082e1b9b","version":"v2.1.0"}],"containers":[{"image":"ghcr.io/github/gh-aw-firewall/agent:0.27.44","digest":"sha256:0d727725c737b58c7bdf51f640cffb928385ec46517e0917c7f1a02f1bada8b4","pinned_image":"ghcr.io/github/gh-aw-firewall/agent:0.27.44@sha256:0d727725c737b58c7bdf51f640cffb928385ec46517e0917c7f1a02f1bada8b4"},{"image":"ghcr.io/github/gh-aw-firewall/api-proxy:0.27.44","digest":"sha256:b50fbadba138f6e9aba94aca09711335c489bb3b15861220cb66f6092e042dc7","pinned_image":"ghcr.io/github/gh-aw-firewall/api-proxy:0.27.44@sha256:b50fbadba138f6e9aba94aca09711335c489bb3b15861220cb66f6092e042dc7"},{"image":"ghcr.io/github/gh-aw-firewall/squid:0.27.44","digest":"sha256:83e48bbe12c634be8c228a576832fe45f66c529ac3659db92bddbcf2eeb6d627","pinned_image":"ghcr.io/github/gh-aw-firewall/squid:0.27.44@sha256:83e48bbe12c634be8c228a576832fe45f66c529ac3659db92bddbcf2eeb6d627"},{"image":"ghcr.io/github/gh-aw-mcpg:v0.4.9","digest":"sha256:e5a1569aeaf41820fa7bdee3e94468cae448133cdbf00119ad24f5b74db1ab9f","pinned_image":"ghcr.io/github/gh-aw-mcpg:v0.4.9@sha256:e5a1569aeaf41820fa7bdee3e94468cae448133cdbf00119ad24f5b74db1ab9f"},{"image":"ghcr.io/github/gh-aw-node","digest":"sha256:0d9f1fb5fd6610c0ac1f5194a38e45a8a1e81f8a390d5142d8e4e6f26a4b3196","pinned_image":"ghcr.io/github/gh-aw-node@sha256:0d9f1fb5fd6610c0ac1f5194a38e45a8a1e81f8a390d5142d8e4e6f26a4b3196"},{"image":"ghcr.io/github/github-mcp-server:v1.9.0","digest":"sha256:881b53d6f75f69bdbc1b5b10fc2f1361717c19054143b3a8529fb5c32061a50e","pinned_image":"ghcr.io/github/github-mcp-server:v1.9.0@sha256:881b53d6f75f69bdbc1b5b10fc2f1361717c19054143b3a8529fb5c32061a50e"}]}
# This file was automatically generated by gh-aw (v0.86.2). DO NOT EDIT. To debug this workflow, load the skill at https://github.com/github/gh-aw/blob/main/debug.md
#
@@ -463,7 +463,7 @@ jobs:
INSTALL_DIR: ${{ runner.temp }}/gh-aw/cargo-security
RUSTUP_HOME: ${{ runner.temp }}/gh-aw/cargo-security/rustup
name: Install the cargo-security wrapper and pinned Rust toolchains
- run: "set -euo pipefail\nmkdir -p \"$INSTALL_DIR/toolchains\" \"$INSTALL_DIR/cwd\"\nfor crate in ai-gateway:ai-gateway packages/native:native; do\n dir=\"${crate%%:*}\"\n slug=\"${crate##*:}\"\n channel=\"$(sed -n 's/^channel = \"\\(.*\\)\"$/\\1/p' \"$dir/rust-toolchain.toml\")\"\n rustup toolchain install \"$channel\" --profile minimal --no-self-update\n ln -s \"$(dirname \"$(rustup which --toolchain \"$channel\" cargo)\")\" \"$INSTALL_DIR/toolchains/$slug\"\ndone\nsh packages/native/scripts/prepare-clickhouse-rs.sh\ncp packages/native/.cargo/config.toml \"$INSTALL_DIR/native-cargo-config.toml\"\ninstall -m 0755 scripts/ci/cargo-security.sh \"$INSTALL_DIR/cargo-security\"\necho \"$INSTALL_DIR\" >> \"$GITHUB_PATH\"\n"
+ run: "set -euo pipefail\nmkdir -p \"$INSTALL_DIR/toolchains\" \"$INSTALL_DIR/cwd\"\nfor crate in ai-gateway:ai-gateway packages/native:native; do\n dir=\"${crate%%:*}\"\n slug=\"${crate##*:}\"\n channel=\"$(sed -n 's/^channel = \"\\(.*\\)\"$/\\1/p' \"$dir/rust-toolchain.toml\")\"\n rustup toolchain install \"$channel\" --profile minimal --no-self-update\n ln -s \"$(dirname \"$(rustup which --toolchain \"$channel\" cargo)\")\" \"$INSTALL_DIR/toolchains/$slug\"\ndone\ninstall -m 0755 scripts/ci/cargo-security.sh \"$INSTALL_DIR/cargo-security\"\necho \"$INSTALL_DIR\" >> \"$GITHUB_PATH\"\n"
- env:
ALERTS_PATH: /tmp/gh-aw/agent/dependabot-alerts.json
GH_TOKEN: ${{ github.token }}
diff --git a/.github/workflows/dependabot-security-maintainer.md b/.github/workflows/dependabot-security-maintainer.md
index 6eafcd029a..745c40df0a 100644
--- a/.github/workflows/dependabot-security-maintainer.md
+++ b/.github/workflows/dependabot-security-maintainer.md
@@ -118,9 +118,8 @@ steps:
# The agent runs Cargo only through the cargo-security wrapper. The sandbox
# mounts runner.temp/gh-aw read-only, so the agent can change neither the
- # wrapper, the per-crate pinned toolchains, nor the clickhouse overlay that
- # the packages/native lockfile was resolved against. Only the wrapper's
- # CARGO_HOME, /tmp/gh-aw/rust/cargo, is writable.
+ # wrapper nor the per-crate pinned toolchains. Only the wrapper's CARGO_HOME,
+ # /tmp/gh-aw/rust/cargo, is writable.
- name: Install the cargo-security wrapper and pinned Rust toolchains
env:
INSTALL_DIR: ${{ runner.temp }}/gh-aw/cargo-security
@@ -135,8 +134,6 @@ steps:
rustup toolchain install "$channel" --profile minimal --no-self-update
ln -s "$(dirname "$(rustup which --toolchain "$channel" cargo)")" "$INSTALL_DIR/toolchains/$slug"
done
- sh packages/native/scripts/prepare-clickhouse-rs.sh
- cp packages/native/.cargo/config.toml "$INSTALL_DIR/native-cargo-config.toml"
install -m 0755 scripts/ci/cargo-security.sh "$INSTALL_DIR/cargo-security"
echo "$INSTALL_DIR" >> "$GITHUB_PATH"
diff --git a/.github/workflows/licencecheck.yml b/.github/workflows/licencecheck.yml
index 99ec29e1ad..93c474f4f7 100644
--- a/.github/workflows/licencecheck.yml
+++ b/.github/workflows/licencecheck.yml
@@ -96,11 +96,6 @@ jobs:
working-directory: ai-gateway
run: cargo deny --locked check licenses
- # Cargo.lock is resolved against the patched clickhouse overlay.
- - name: Prepare patched clickhouse crate for the native addon
- working-directory: packages/native
- run: sh scripts/prepare-clickhouse-rs.sh
-
- name: Check native addon dependency licenses
working-directory: packages/native
run: cargo deny --locked check licenses
diff --git a/deny.toml b/deny.toml
index 2166b6a584..cbe24a90f8 100644
--- a/deny.toml
+++ b/deny.toml
@@ -50,4 +50,4 @@ allow-wildcard-paths = true
unknown-registry = "deny"
unknown-git = "deny"
allow-registry = ["https://github.com/rust-lang/crates.io-index"]
-allow-git = []
+allow-git = ["https://github.com/ClickHouse/ch-core-rs.git"]
diff --git a/packages/native/.gitignore b/packages/native/.gitignore
index 0202152254..a9457aed5b 100644
--- a/packages/native/.gitignore
+++ b/packages/native/.gitignore
@@ -1,7 +1,7 @@
# Cargo build output
target/
-# Generated Cargo overlay for the pinned clickhouse source patch
+# Local Cargo configuration
.cargo/
# Platform-specific binaries produced by `napi build`
diff --git a/packages/native/Cargo.lock b/packages/native/Cargo.lock
index 984b7bee5e..3bf99f0f8d 100644
--- a/packages/native/Cargo.lock
+++ b/packages/native/Cargo.lock
@@ -24,12 +24,6 @@ dependencies = [
"memchr",
]
-[[package]]
-name = "allocator-api2"
-version = "0.2.21"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "683d7910e743518b0e34f1186f92494becacb047c7b6bf616c96772180fef923"
-
[[package]]
name = "approx"
version = "0.5.1"
@@ -45,12 +39,6 @@ version = "0.7.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d3fb67a6e08acf24fdeccbac2cb6ac4305825bd1f117462e0e6f2f193345ad56"
-[[package]]
-name = "atomic-waker"
-version = "1.1.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0"
-
[[package]]
name = "autocfg"
version = "1.5.1"
@@ -90,12 +78,6 @@ dependencies = [
"generic-array",
]
-[[package]]
-name = "bnum"
-version = "0.13.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "119771309b95163ec7aaf79810da82f7cd0599c19722d48b9c03894dca833966"
-
[[package]]
name = "borsh"
version = "1.8.1"
@@ -120,15 +102,6 @@ dependencies = [
"syn 3.0.4",
]
-[[package]]
-name = "bstr"
-version = "1.13.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6bb31b46c14244e20ee9984b11bf5c992b91fb6939fea616e3512c8baecdbe5f"
-dependencies = [
- "memchr",
-]
-
[[package]]
name = "bumpalo"
version = "3.20.3"
@@ -146,9 +119,6 @@ name = "bytes"
version = "1.12.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fc652a48c352aef3ea3aed32080501cf3ef6ed5da78602a020c991775b0aff04"
-dependencies = [
- "serde",
-]
[[package]]
name = "cc"
@@ -172,6 +142,11 @@ version = "0.2.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f079e83a288787bcd14a6aea84cee5c87a67c5a3e660c30f557a3d24761b3527"
+[[package]]
+name = "ch-core-rs"
+version = "0.2.0"
+source = "git+https://github.com/ClickHouse/ch-core-rs.git?rev=afb8f67cb71a810b83fac9c7b5a43173589e56c1#afb8f67cb71a810b83fac9c7b5a43173589e56c1"
+
[[package]]
name = "chrono"
version = "0.4.45"
@@ -181,52 +156,6 @@ dependencies = [
"num-traits",
]
-[[package]]
-name = "clickhouse"
-version = "0.15.2"
-dependencies = [
- "bnum",
- "bstr",
- "bytes",
- "clickhouse-macros",
- "clickhouse-types",
- "futures-channel",
- "futures-util",
- "half",
- "hashbrown 0.17.1",
- "http-body-util",
- "hyper",
- "hyper-util",
- "serde",
- "serde_json",
- "thiserror",
- "tokio",
- "tracing",
- "url",
-]
-
-[[package]]
-name = "clickhouse-macros"
-version = "0.3.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ff6669899e23cb87b43daf7996f0ea3b9c07d0fb933d745bb7b815b052515ae3"
-dependencies = [
- "proc-macro2",
- "quote",
- "serde_derive_internals",
- "syn 2.0.119",
-]
-
-[[package]]
-name = "clickhouse-types"
-version = "0.1.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3966eea13bfc826e24d471e49cb2095727261009be59d0ac5a6c0be0286974d1"
-dependencies = [
- "bytes",
- "thiserror",
-]
-
[[package]]
name = "convert_case"
version = "0.11.0"
@@ -260,12 +189,6 @@ version = "0.8.22"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "61803da095bee82a81bb1a452ecc25d3b2f1416d1897eb86430c6159ef717c17"
-[[package]]
-name = "crunchy"
-version = "0.2.4"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "460fbee9c2c2f33933d720630a6a0bac33ba7053db5344fac858d4b8952d77d5"
-
[[package]]
name = "crypto-common"
version = "0.1.7"
@@ -292,17 +215,6 @@ dependencies = [
"crypto-common",
]
-[[package]]
-name = "displaydoc"
-version = "0.2.7"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c6232dd377dcc64799954cbd3a9bb882e9cdc1308ccd87b1c098f1fb2eaf82a8"
-dependencies = [
- "proc-macro2",
- "quote",
- "syn 3.0.4",
-]
-
[[package]]
name = "endian-type"
version = "0.1.2"
@@ -327,15 +239,6 @@ version = "0.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "77ce24cb58228fbb8aa041425bb1050850ac19177686ea6e0f41a70416f56fdb"
-[[package]]
-name = "form_urlencoded"
-version = "1.2.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "cb4cb245038516f5f85277875cdaa4f7d2c9a0fa0468de06ed190163b1581fcf"
-dependencies = [
- "percent-encoding",
-]
-
[[package]]
name = "funty"
version = "2.0.0"
@@ -487,17 +390,6 @@ version = "0.33.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e762b9b188634fc508d4ec54b62ea3d352c43fd431d18d05d46f559f217f4725"
-[[package]]
-name = "half"
-version = "2.7.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6ea2d84b969582b4b1864a92dc5d27cd2b77b622a8d79306834f1be5ba20d84b"
-dependencies = [
- "cfg-if",
- "crunchy",
- "zerocopy",
-]
-
[[package]]
name = "hashbrown"
version = "0.16.1"
@@ -512,11 +404,6 @@ name = "hashbrown"
version = "0.17.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a"
-dependencies = [
- "allocator-api2",
- "equivalent",
- "foldhash",
-]
[[package]]
name = "hdbscan-rs"
@@ -543,189 +430,6 @@ dependencies = [
"rand_pcg",
]
-[[package]]
-name = "http"
-version = "1.5.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "918d3568bebf352712bc2ef3d46a8bcf1a75b373be6539de198e9105cbbf9ce0"
-dependencies = [
- "bytes",
- "itoa",
-]
-
-[[package]]
-name = "http-body"
-version = "1.1.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ca2a8f2913ee65f60facd6a5905613afaa448497a0230cc41ce022d93290bc2c"
-dependencies = [
- "bytes",
- "http",
-]
-
-[[package]]
-name = "http-body-util"
-version = "0.1.5"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "23169fe34a5fbcdd3f3862e78fb9b6fccd5f02a6dc6f732547005d45631ce71c"
-dependencies = [
- "bytes",
- "futures-core",
- "http",
- "http-body",
- "pin-project-lite",
-]
-
-[[package]]
-name = "httparse"
-version = "1.10.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6dbf3de79e51f3d586ab4cb9d5c3e2c14aa28ed23d180cf89b4df0454a69cc87"
-
-[[package]]
-name = "hyper"
-version = "1.11.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "27b501faa50e7a26c3d3560ca625132f4078a17771f4810baf70475ae48cbe43"
-dependencies = [
- "atomic-waker",
- "bytes",
- "futures-channel",
- "futures-core",
- "http",
- "http-body",
- "httparse",
- "itoa",
- "pin-project-lite",
- "smallvec",
- "tokio",
- "want",
-]
-
-[[package]]
-name = "hyper-util"
-version = "0.1.20"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "96547c2556ec9d12fb1578c4eaf448b04993e7fb79cbaad930a656880a6bdfa0"
-dependencies = [
- "bytes",
- "futures-channel",
- "futures-util",
- "http",
- "http-body",
- "hyper",
- "libc",
- "pin-project-lite",
- "socket2",
- "tokio",
- "tower-service",
- "tracing",
-]
-
-[[package]]
-name = "icu_collections"
-version = "2.3.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "fa68d21081c4a05d5a901a1c62add574c77048b6a1c67be3b50ce0b60d4ca513"
-dependencies = [
- "displaydoc",
- "potential_utf",
- "utf8_iter",
- "yoke",
- "zerofrom",
- "zerovec",
-]
-
-[[package]]
-name = "icu_locale_core"
-version = "2.3.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d56e28588da92eee5c3201a6eff33fabdd49b62269c8938d4ff050ce4d900deb"
-dependencies = [
- "displaydoc",
- "litemap",
- "tinystr",
- "writeable",
- "zerovec",
-]
-
-[[package]]
-name = "icu_normalizer"
-version = "2.3.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "12f9cf5f235641ed274641dd81c3f28d870e276763d0797aeeab72317b1c646f"
-dependencies = [
- "icu_collections",
- "icu_normalizer_data",
- "icu_properties",
- "icu_provider",
- "smallvec",
- "zerovec",
-]
-
-[[package]]
-name = "icu_normalizer_data"
-version = "2.3.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1563da1ed3e0b3bf3d74c9b85917ac9c56464d2f57242270c09c9e752f8021a0"
-
-[[package]]
-name = "icu_properties"
-version = "2.3.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7e7ca276ad3145661a65914e6daf131ca5120cd3dcee8f8f3214b8875184a148"
-dependencies = [
- "displaydoc",
- "icu_collections",
- "icu_locale_core",
- "icu_properties_data",
- "icu_provider",
- "zerotrie",
- "zerovec",
-]
-
-[[package]]
-name = "icu_properties_data"
-version = "2.3.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e590f038c1464a96894fd6d10127e90a8be4509f56ff7ecef851b15cee0b7caa"
-
-[[package]]
-name = "icu_provider"
-version = "2.3.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d27bbb9d3abbefac45d55f647c9de1d44aafcd1186eb91879afef17c396c3e73"
-dependencies = [
- "displaydoc",
- "icu_locale_core",
- "writeable",
- "yoke",
- "zerofrom",
- "zerotrie",
- "zerovec",
-]
-
-[[package]]
-name = "idna"
-version = "1.1.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3b0875f23caa03898994f6ddc501886a45c7d3d62d04d2d90788d47be1b1e4de"
-dependencies = [
- "idna_adapter",
- "smallvec",
- "utf8_iter",
-]
-
-[[package]]
-name = "idna_adapter"
-version = "1.2.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "cb68373c0d6620ef8105e855e7745e18b0d00d3bdb07fb532e434244cdb9a714"
-dependencies = [
- "icu_normalizer",
- "icu_properties",
-]
-
[[package]]
name = "indexmap"
version = "2.14.1"
@@ -772,10 +476,8 @@ version = "0.0.0"
dependencies = [
"aho-corasick",
"base64",
- "bytes",
+ "ch-core-rs",
"chrono",
- "clickhouse",
- "clickhouse-types",
"getrandom 0.3.4",
"hdbscan-rs",
"holomap",
@@ -849,12 +551,6 @@ version = "0.2.16"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b6d2cec3eae94f9f509c767b45932f1ada8350c4bdb85af2fcab4a3c14807981"
-[[package]]
-name = "litemap"
-version = "0.8.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "47d9d19d1d6efa0109d2f65ff4c85cddd50bd572e5a00127ab10987290bcefae"
-
[[package]]
name = "log"
version = "0.4.34"
@@ -932,17 +628,6 @@ dependencies = [
"sketches-ddsketch",
]
-[[package]]
-name = "mio"
-version = "1.2.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4b18443e9c262bfe8fa82f51666e2642c53393f7e5c27b3e1aeab922cff5b9d8"
-dependencies = [
- "libc",
- "wasi",
- "windows-sys",
-]
-
[[package]]
name = "nalgebra"
version = "0.35.0"
@@ -1131,12 +816,6 @@ dependencies = [
"num-traits",
]
-[[package]]
-name = "percent-encoding"
-version = "2.3.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220"
-
[[package]]
name = "pin-project-lite"
version = "0.2.17"
@@ -1158,15 +837,6 @@ dependencies = [
"portable-atomic",
]
-[[package]]
-name = "potential_utf"
-version = "0.1.6"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d83eb9bc6d8e5cf568e7a1101d60ee05e81ed50ea106026f3d18deeb046d7661"
-dependencies = [
- "zerovec",
-]
-
[[package]]
name = "ppv-lite86"
version = "0.2.21"
@@ -1439,7 +1109,6 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4148590afebada386688f18773da617792bf2ef03ffc1e4cbd2b1d45b023e0ba"
dependencies = [
"serde_core",
- "serde_derive",
]
[[package]]
@@ -1462,17 +1131,6 @@ dependencies = [
"syn 3.0.4",
]
-[[package]]
-name = "serde_derive_internals"
-version = "0.29.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "18d26a20a969b9e3fdf2fc2d9f21eda6c40e2de84c9408bb5d3b05d499aae711"
-dependencies = [
- "proc-macro2",
- "quote",
- "syn 2.0.119",
-]
-
[[package]]
name = "serde_json"
version = "1.0.151"
@@ -1552,22 +1210,6 @@ version = "1.16.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b9be42f50aa861c555654aa3a37f52f4b1074bacf4e48fe0ef7fa584e80f1f0f"
-[[package]]
-name = "socket2"
-version = "0.6.5"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c3d1e2c7f27f8d4cb10542a02c49005dbd6e93095799d6f3be745fae9f8fedd4"
-dependencies = [
- "libc",
- "windows-sys",
-]
-
-[[package]]
-name = "stable_deref_trait"
-version = "1.2.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596"
-
[[package]]
name = "syn"
version = "2.0.119"
@@ -1590,17 +1232,6 @@ dependencies = [
"unicode-ident",
]
-[[package]]
-name = "synstructure"
-version = "0.14.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "901704edd0dfe137f1987838ee4f259e4e063c31371bdb423f7ae38ec6f77f02"
-dependencies = [
- "proc-macro2",
- "quote",
- "syn 3.0.4",
-]
-
[[package]]
name = "tap"
version = "1.0.1"
@@ -1636,41 +1267,6 @@ dependencies = [
"cfg-if",
]
-[[package]]
-name = "tinystr"
-version = "0.8.4"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b1e27c91459209c2986af3dcf603a5a74a4368754ce37414f59acc971167f643"
-dependencies = [
- "displaydoc",
- "zerovec",
-]
-
-[[package]]
-name = "tokio"
-version = "1.53.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "202caea871b69668250d242070849eb495be178ed697a3e98aebce5bc81a0bed"
-dependencies = [
- "libc",
- "mio",
- "pin-project-lite",
- "socket2",
- "tokio-macros",
- "windows-sys",
-]
-
-[[package]]
-name = "tokio-macros"
-version = "2.7.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "78773a2a397f451582ce068015985c33193cf6dea8b74d2a639fe457b2f07b0e"
-dependencies = [
- "proc-macro2",
- "quote",
- "syn 3.0.4",
-]
-
[[package]]
name = "toml_datetime"
version = "1.1.1+spec-1.1.0"
@@ -1701,12 +1297,6 @@ dependencies = [
"winnow",
]
-[[package]]
-name = "tower-service"
-version = "0.3.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8df9b6e13f2d32c91b9bd719c00d1958837bc7dec474d94952798cc8e69eeec3"
-
[[package]]
name = "tracing"
version = "0.1.44"
@@ -1781,12 +1371,6 @@ dependencies = [
"tracing-serde",
]
-[[package]]
-name = "try-lock"
-version = "0.2.5"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b"
-
[[package]]
name = "typenum"
version = "1.20.1"
@@ -1811,24 +1395,6 @@ version = "1.13.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c6f5d3c3b1bf09027a88a6bc961fc00497d651009560b5463668dc81b0fa87a8"
-[[package]]
-name = "url"
-version = "2.5.8"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ff67a8a4397373c3ef660812acab3268222035010ab8680ec4215f38ba3d0eed"
-dependencies = [
- "form_urlencoded",
- "idna",
- "percent-encoding",
- "serde",
-]
-
-[[package]]
-name = "utf8_iter"
-version = "1.0.4"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be"
-
[[package]]
name = "valuable"
version = "0.1.1"
@@ -1841,15 +1407,6 @@ version = "0.9.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a"
-[[package]]
-name = "want"
-version = "0.3.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "bfa7760aed19e106de2c7c0b581b509f2f25d3dacaf737cb82ac61bc6d760b0e"
-dependencies = [
- "try-lock",
-]
-
[[package]]
name = "wasi"
version = "0.11.1+wasi-snapshot-preview1"
@@ -1982,12 +1539,6 @@ version = "0.57.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1ebf944e87a7c253233ad6766e082e3cd714b5d03812acc24c318f549614536e"
-[[package]]
-name = "writeable"
-version = "0.6.4"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3ad82d2a33cdc9674dc7465672f271e096168fcdbe0f799d9e6db8c5892679dc"
-
[[package]]
name = "wyz"
version = "0.5.1"
@@ -1997,29 +1548,6 @@ dependencies = [
"tap",
]
-[[package]]
-name = "yoke"
-version = "0.8.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "709fe23a0424b6a435d82152b1bd3fdfb0833487d5fa90d05d42762a9891fef5"
-dependencies = [
- "stable_deref_trait",
- "yoke-derive",
- "zerofrom",
-]
-
-[[package]]
-name = "yoke-derive"
-version = "0.8.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "33811428bee40dbceb6d545e95754741d17a6aef9a4849f0fd62e2ba4f412a78"
-dependencies = [
- "proc-macro2",
- "quote",
- "syn 3.0.4",
- "synstructure",
-]
-
[[package]]
name = "zerocopy"
version = "0.8.56"
@@ -2040,60 +1568,6 @@ dependencies = [
"syn 2.0.119",
]
-[[package]]
-name = "zerofrom"
-version = "0.1.8"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0ec05a11813ea801ff6d75110ad09cd0824ddba17dfe17128ea0d5f68e6c5272"
-dependencies = [
- "zerofrom-derive",
-]
-
-[[package]]
-name = "zerofrom-derive"
-version = "0.1.8"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f75b4683f6c7f45248d4d64056a24298c6281e0993356d7d1b4a1a962ef10d4a"
-dependencies = [
- "proc-macro2",
- "quote",
- "syn 3.0.4",
- "synstructure",
-]
-
-[[package]]
-name = "zerotrie"
-version = "0.2.5"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4ea269c3bd32f0a32c321907a2ae912ba6f4649bb0fc764a15627e99a7095a3f"
-dependencies = [
- "displaydoc",
- "yoke",
- "zerofrom",
-]
-
-[[package]]
-name = "zerovec"
-version = "0.11.8"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "bb0464e17806c1d976d5cba29399c7f08e516e279e2ba493f63123b5fca67dd8"
-dependencies = [
- "yoke",
- "zerofrom",
- "zerovec-derive",
-]
-
-[[package]]
-name = "zerovec-derive"
-version = "0.11.6"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "34df6fc39dbd26ddc9c10e6a2984476e13acce22e64e4487636ef494369225da"
-dependencies = [
- "proc-macro2",
- "quote",
- "syn 3.0.4",
-]
-
[[package]]
name = "zmij"
version = "1.0.23"
diff --git a/packages/native/Cargo.toml b/packages/native/Cargo.toml
index e7c261932d..17b9771e6b 100644
--- a/packages/native/Cargo.toml
+++ b/packages/native/Cargo.toml
@@ -13,10 +13,8 @@ crate-type = ["cdylib"]
[dependencies]
aho-corasick = "1.1"
base64 = { version = "0.23.1", features = ["simd-unsafe"] }
-bytes = "1"
chrono = { version = "0.4", default-features = false, features = ["alloc", "std"] }
-clickhouse = { version = "=0.15.2", default-features = false }
-clickhouse-types = "=0.1.3"
+ch-core-rs = { git = "https://github.com/ClickHouse/ch-core-rs.git", rev = "afb8f67cb71a810b83fac9c7b5a43173589e56c1" }
hdbscan-rs = { version = "=0.6.1", default-features = false }
getrandom = "0.3"
memchr = "2"
diff --git a/packages/native/README.md b/packages/native/README.md
index c7080980e7..26ecbb32d5 100644
--- a/packages/native/README.md
+++ b/packages/native/README.md
@@ -31,11 +31,13 @@ CPU-bound fits can be killed without blocking the worker.
type checks and tooling work without a Rust toolchain. `*.node` binaries and
`target/` are ignored.
-The encoder uses a small local source overlay for the pinned `clickhouse`
-0.15.2 crate. `scripts/prepare-clickhouse-rs.sh` downloads the exact crate
-archive, verifies its checksum, applies the delta in `patches/`, and writes an
-ignored Cargo patch configuration. The repository keeps the delta and setup
-script, not a vendored copy of the crate.
+The encoder uses the pinned `ch-core-rs` Git dependency for ClickHouse Native
+format encoding. The HTTP transport remains in the worker.
+
+The `rust:*` scripts remove the obsolete generated `.cargo/config.toml` from
+checkouts that used the patched encoder. Modified configs are preserved; remove
+the old `clickhouse` patch entry manually if warned. Direct Cargo commands bypass
+this cleanup.
## Building
diff --git a/packages/native/package.json b/packages/native/package.json
index 654baed0de..e7bb91ceb6 100644
--- a/packages/native/package.json
+++ b/packages/native/package.json
@@ -15,19 +15,18 @@
"node": "24"
},
"scripts": {
- "prepare-clickhouse-rs": "sh scripts/prepare-clickhouse-rs.sh",
"build": "pnpm run rust:build",
- "rust:build": "pnpm run prepare-clickhouse-rs && napi build --platform --release",
+ "rust:build": "node scripts/remove-generated-cargo-patch.mjs && napi build --platform --release",
"build:debug": "pnpm run rust:build:debug",
- "rust:build:debug": "pnpm run prepare-clickhouse-rs && napi build --platform",
+ "rust:build:debug": "node scripts/remove-generated-cargo-patch.mjs && napi build --platform",
"lint": "pnpm run rust:lint",
- "rust:lint": "pnpm run prepare-clickhouse-rs && cargo fmt --check && cargo clippy --all-targets -- -D warnings",
+ "rust:lint": "node scripts/remove-generated-cargo-patch.mjs && cargo fmt --check && cargo clippy --all-targets -- -D warnings",
"lint:fix": "pnpm run rust:lint:fix",
- "rust:lint:fix": "pnpm run prepare-clickhouse-rs && cargo fmt && cargo clippy --all-targets --fix --allow-dirty -- -D warnings",
+ "rust:lint:fix": "node scripts/remove-generated-cargo-patch.mjs && cargo fmt && cargo clippy --all-targets --fix --allow-dirty -- -D warnings",
"test": "pnpm run rust:test",
- "rust:test": "pnpm run prepare-clickhouse-rs && cargo test --locked --features napi/dyn-symbols",
- "rust:deny": "pnpm run prepare-clickhouse-rs && cargo deny --locked check",
- "rust:unused-deps": "cargo machete"
+ "rust:test": "node scripts/remove-generated-cargo-patch.mjs && cargo test --locked --features napi/dyn-symbols",
+ "rust:deny": "node scripts/remove-generated-cargo-patch.mjs && cargo deny --locked check",
+ "rust:unused-deps": "node scripts/remove-generated-cargo-patch.mjs && cargo machete"
},
"napi": {
"binaryName": "langfuse-native",
diff --git a/packages/native/patches/clickhouse-0.15.2-native-encoder.patch b/packages/native/patches/clickhouse-0.15.2-native-encoder.patch
deleted file mode 100644
index 071c653350..0000000000
--- a/packages/native/patches/clickhouse-0.15.2-native-encoder.patch
+++ /dev/null
@@ -1,128 +0,0 @@
-diff --git a/src/native/mod.rs b/src/native/mod.rs
-index 1ef0339..68181bc 100644
---- a/src/native/mod.rs
-+++ b/src/native/mod.rs
-@@ -47,6 +47,15 @@ impl Block {
- }
- }
-
-+ /// Encode this block as a self-contained ClickHouse Native-format payload.
-+ ///
-+ /// The returned bytes contain the block header, column metadata, and all column data. They
-+ /// can be sent as one Native block without retaining any references to the builder or its
-+ /// source values.
-+ pub fn encode(&self) -> Result<Bytes, Error> {
-+ writer::encode_block(self)
-+ }
-+
- /// The number of rows in this block.
- ///
- /// Note that the size of a single block in a query resultset can be influenced by many things
-diff --git a/src/native/writer.rs b/src/native/writer.rs
-index b9553c8..35327f8 100644
---- a/src/native/writer.rs
-+++ b/src/native/writer.rs
-@@ -5,6 +5,103 @@ use crate::native::{Block, Column, Layout, LayoutKind, varuint};
- use bytes::{BufMut, Bytes, BytesMut};
- use std::num::Saturating;
-
-+/// Encode a complete block synchronously in ClickHouse Native format.
-+///
-+/// This is kept separate from [`BlockWriter`], whose purpose is streaming an insert request. It
-+/// lets callers that already own a block obtain an independent byte buffer with no async sink or
-+/// request lifetime involved.
-+pub(crate) fn encode_block(block: &Block) -> Result<Bytes, Error> {
-+ if block.num_rows == 0 {
-+ return Err(Error::Other("attempting to encode an empty block".into()));
-+ }
-+
-+ let mut buf = BytesMut::with_capacity(8192);
-+ varuint::write(&mut buf, block.columns.len());
-+ varuint::write(&mut buf, block.num_rows);
-+
-+ for column in &block.columns {
-+ encode_column(&mut buf, column)?;
-+ }
-+
-+ Ok(buf.freeze())
-+}
-+
-+fn encode_column(buf: &mut BytesMut, column: &Column) -> Result<(), Error> {
-+ varuint::write(&mut *buf, column.name.len());
-+ buf.extend_from_slice(column.name.as_bytes());
-+
-+ let data_type_str = column.data_type.to_str();
-+ varuint::write(&mut *buf, data_type_str.len());
-+ buf.extend_from_slice(data_type_str.as_bytes());
-+
-+ encode_layout(buf, &column.layout)
-+}
-+
-+fn encode_layout(buf: &mut BytesMut, layout: &Layout) -> Result<(), Error> {
-+ if let Some(nulls) = &layout.nulls {
-+ buf.extend_from_slice(nulls);
-+ }
-+
-+ match &layout.kind {
-+ LayoutKind::Fixed { data, .. } => buf.extend_from_slice(data),
-+ LayoutKind::Variable { end_offsets, data } => {
-+ let mut start_offset = 0;
-+
-+ for &end_offset in end_offsets {
-+ let len = end_offset.checked_sub(start_offset).ok_or_else(|| {
-+ Error::Other(
-+ format!(
-+ "BUG: string length underflow in encoding block: {end_offset} - {start_offset}"
-+ )
-+ .into(),
-+ )
-+ })?;
-+
-+ varuint::write(&mut *buf, len);
-+ buf.extend_from_slice(&data[start_offset..end_offset]);
-+ start_offset = end_offset;
-+ }
-+ }
-+ LayoutKind::LowCardinality(_) => {
-+ return Err(Error::Other(
-+ "inserting LowCardinality data not yet implemented".into(),
-+ ));
-+ }
-+ LayoutKind::Array {
-+ end_indices,
-+ elem_layout,
-+ } => {
-+ for &index in end_indices {
-+ buf.put_u64_le(u64::try_from(index).map_err(|_| {
-+ Error::Other(format!("array end index out of range: {index}").into())
-+ })?);
-+ }
-+
-+ encode_layout(buf, elem_layout)?;
-+ }
-+ LayoutKind::Tuple { layouts } => {
-+ for layout in layouts {
-+ encode_layout(buf, layout)?;
-+ }
-+ }
-+ LayoutKind::Map {
-+ key_val_layouts,
-+ end_indices,
-+ } => {
-+ for &index in end_indices {
-+ buf.put_u64_le(u64::try_from(index).map_err(|_| {
-+ Error::Other(format!("map end index out of range: {index}").into())
-+ })?);
-+ }
-+
-+ encode_layout(buf, &key_val_layouts[0])?;
-+ encode_layout(buf, &key_val_layouts[1])?;
-+ }
-+ }
-+
-+ Ok(())
-+}
-+
- pub(crate) struct BlockWriter {
- insert: InsertFormatted,
- buf: BytesMut,
diff --git a/packages/native/scripts/prepare-clickhouse-rs.sh b/packages/native/scripts/prepare-clickhouse-rs.sh
deleted file mode 100644
index 099ea246c6..0000000000
--- a/packages/native/scripts/prepare-clickhouse-rs.sh
+++ /dev/null
@@ -1,48 +0,0 @@
-#!/bin/sh
-set -eu
-
-# Apply the local Native encoder patch to the clickhouse-rs client crate.
-
-crate_version=0.15.2
-crate_sha256=7654154fde4d97ec1321bfe4c3965570be6fcd09e091da8afc3166ab5ee27be2
-package_dir=$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd)
-source_root="${package_dir}/target/.clickhouse-source"
-source_dir="${source_root}/clickhouse-${crate_version}"
-archive="${source_root}/clickhouse-${crate_version}.crate"
-patch_file="${package_dir}/patches/clickhouse-${crate_version}-native-encoder.patch"
-marker="${source_dir}/.langfuse-native-encoder-patched"
-config_dir="${package_dir}/.cargo"
-
-checksum() {
- if command -v sha256sum >/dev/null 2>&1; then
- sha256sum "$1" | cut -d ' ' -f 1
- else
- shasum -a 256 "$1" | cut -d ' ' -f 1
- fi
-}
-
-patch_sha256="$(checksum "$patch_file")"
-
-if [ ! -f "$marker" ] || [ "$(cat "$marker")" != "$patch_sha256" ]; then
- mkdir -p "$source_root"
- curl --fail --location --retry 3 --retry-delay 2 --retry-all-errors --silent --show-error \
- --user-agent 'langfuse-native-build/0.1 (https://github.com/langfuse/langfuse)' \
- "https://crates.io/api/v1/crates/clickhouse/${crate_version}/download" \
- --output "$archive"
-
- if [ "$(checksum "$archive")" != "$crate_sha256" ]; then
- echo "clickhouse ${crate_version} archive checksum mismatch" >&2
- exit 1
- fi
-
- rm -rf "$source_dir"
- tar -xzf "$archive" -C "$source_root"
- patch --directory="$source_dir" --strip=1 < "$patch_file"
- printf '%s' "$patch_sha256" > "$marker"
-fi
-
-mkdir -p "$config_dir"
-cat > "$config_dir/config.toml" <<EOF
-[patch.crates-io]
-clickhouse = { path = "${source_dir}" }
-EOF
diff --git a/packages/native/scripts/remove-generated-cargo-patch.mjs b/packages/native/scripts/remove-generated-cargo-patch.mjs
new file mode 100644
index 0000000000..429dce5b32
--- /dev/null
+++ b/packages/native/scripts/remove-generated-cargo-patch.mjs
@@ -0,0 +1,21 @@
+import { readFileSync, rmSync } from "node:fs";
+
+const config = new URL("../.cargo/config.toml", import.meta.url);
+try {
+ const contents = readFileSync(config, "utf8");
+ // Only remove the complete two-line file generated for the patched encoder.
+ const generated =
+ /^\[patch\.crates-io\]\r?\nclickhouse = \{ path = "[^"\r\n]+\/target\/\.clickhouse-source\/clickhouse-0\.15\.2" \}\r?\n?$/;
+ if (generated.test(contents)) {
+ rmSync(config, { force: true });
+ console.log("Removed obsolete generated ClickHouse Cargo patch.");
+ } else if (
+ contents.includes("/target/.clickhouse-source/clickhouse-0.15.2")
+ ) {
+ console.warn(
+ "Local .cargo/config.toml still references the obsolete ClickHouse patch. Remove that entry manually; custom configuration was left unchanged.",
+ );
+ }
+} catch (error) {
+ if (error.code !== "ENOENT") throw error;
+}
diff --git a/packages/native/src/lib.rs b/packages/native/src/lib.rs
index 38f125d99d..c3d9c4e093 100644
--- a/packages/native/src/lib.rs
+++ b/packages/native/src/lib.rs
@@ -7,6 +7,7 @@
/// Rust-only encoder for prepared v4 `events_full` rows.
pub mod native_codec;
+mod native_columns;
mod native_js;
mod native_memory;
pub(crate) mod native_schema;
@@ -121,13 +122,10 @@ impl Task for EncodeClickhouseEventsTask {
.map(|block| {
let row_count = u32::try_from(block.row_count)
.map_err(|_| Error::from_reason("Native block row count exceeds UInt32"))?;
- // The encoder returns a fresh, uniquely owned allocation. Vec takes it over,
- // then napi wraps the same memory in a JS Buffer whose GC finalizer frees it.
- // This handoff does not copy the payload. If the runtime forbids external
- // buffers, napi-rs copies it into Node-owned memory instead. Shared or sliced
- // Bytes would not have the same no-copy guarantee when converted to Vec.
+ // napi wraps the owned Vec in a JS Buffer whose GC finalizer frees it.
+ // If the runtime forbids external buffers, napi-rs copies it instead.
Ok(NativeEventBlock {
- bytes: Buffer::from(Vec::from(block.bytes)),
+ bytes: Buffer::from(block.bytes),
row_count,
})
})
diff --git a/packages/native/src/native_codec.rs b/packages/native/src/native_codec.rs
index d65e560383..3a581a7b8c 100644
--- a/packages/native/src/native_codec.rs
+++ b/packages/native/src/native_codec.rs
@@ -2,17 +2,12 @@
//!
//! Encoding receives typed, owned fields from the direct JS adapter in `native_js`.
-use std::error::Error as StdError;
use std::sync::Arc;
-use bytes::Bytes;
use chrono::{DateTime, NaiveDateTime, Utc};
-use clickhouse::native::builder::BlockBuilder;
-use clickhouse::native::encode::{Encode, ValueWriter};
-use clickhouse::native::DataTypeNode;
-use clickhouse_types::data_types::{DateTimePrecision, DecimalType};
use rust_decimal::{Decimal, RoundingStrategy};
+use crate::native_columns::NativeColumnError;
use crate::native_schema::PreparedEventRow;
const DECIMAL_SCALE: u32 = 12;
@@ -20,40 +15,13 @@ const DECIMAL_LIMIT: i128 = 1_000_000_000_000_000_000;
const DECIMAL_CLAMPED_MAX: i64 = 999_999_999_999_000_000;
const DECIMAL_CLAMPED_MIN: i64 = -DECIMAL_CLAMPED_MAX;
const DECIMAL_OVERFLOW_LIMIT: f64 = 1_000_000.0;
-type EncodeError = Box<dyn StdError + Send + Sync>;
-#[derive(Clone, Copy, Debug)]
+#[derive(Clone, Copy, Debug, Default)]
pub(crate) struct DateTime64Micros(pub(crate) i64);
-impl Encode for DateTime64Micros {
- fn produces() -> DataTypeNode {
- DataTypeNode::DateTime64(DateTimePrecision::Precision6, None)
- }
-
- fn encode(&self, writer: &mut ValueWriter<'_>) -> Result<(), EncodeError> {
- // clickhouse-rs has no DateTime64 Encode implementation. Keep the DateTime64(6) type tag
- // while writing ClickHouse's signed microsecond representation as a fixed-width value.
- writer.write_fixed(&self.0.to_le_bytes())?;
- Ok(())
- }
-}
-
#[derive(Clone, Copy, Debug)]
pub(crate) struct Decimal64(pub(crate) i64);
-impl Encode for Decimal64 {
- fn produces() -> DataTypeNode {
- DataTypeNode::Decimal(18, 12, DecimalType::Decimal64)
- }
-
- fn encode(&self, writer: &mut ValueWriter<'_>) -> Result<(), EncodeError> {
- // rust_decimal supplies the policy and coefficient; clickhouse-rs has no Decimal Encode
- // implementation, so retain the Decimal(18,12) type tag and write its Int64 coefficient.
- writer.write_fixed(&self.0.to_le_bytes())?;
- Ok(())
- }
-}
-
#[derive(Debug)]
pub(crate) struct SkillReference {
pub(crate) langfuse_skill_id: Option<String>,
@@ -61,21 +29,6 @@ pub(crate) struct SkillReference {
pub(crate) skill_name: String,
}
-impl Encode for SkillReference {
- fn produces() -> DataTypeNode {
- <(Option<String>, Option<u32>, String)>::produces()
- }
-
- fn encode(&self, writer: &mut ValueWriter<'_>) -> Result<(), EncodeError> {
- (
- &self.langfuse_skill_id,
- &self.skill_version,
- &self.skill_name,
- )
- .encode(writer)
- }
-}
-
#[derive(Debug)]
pub(crate) struct SkillResourceLoaded {
pub(crate) langfuse_skill_id: Option<String>,
@@ -84,29 +37,13 @@ pub(crate) struct SkillResourceLoaded {
pub(crate) file_path: String,
}
-impl Encode for SkillResourceLoaded {
- fn produces() -> DataTypeNode {
- <(Option<String>, Option<u32>, String, String)>::produces()
- }
-
- fn encode(&self, writer: &mut ValueWriter<'_>) -> Result<(), EncodeError> {
- (
- &self.langfuse_skill_id,
- &self.skill_version,
- &self.skill_name,
- &self.file_path,
- )
- .encode(writer)
- }
-}
-
/// An owned ClickHouse Native-format block and its row count.
///
/// Ownership is part of the transport contract: the later JS transport can retry this complete
/// block after the encoder and source rows have gone out of scope.
#[derive(Debug)]
pub struct EncodedBlock {
- pub bytes: Bytes,
+ pub bytes: Vec<u8>,
pub row_count: usize,
}
@@ -119,6 +56,8 @@ pub struct EncodedBlock {
/// Decimal(18,12) overflow policy and truncate toward zero at twelve fractional digits.
/// Encoding preserves the prepared row's `event_bytes`. The NAPI adapter receives that value
/// from TypeScript after overflow handling; it does not recompute accounting in Rust.
+/// A block can still split when ch-core's signed String offsets would overflow; a single row that
+/// exceeds that limit remains an error.
pub(crate) fn encode_v4_native_blocks(
rows: &[Arc<PreparedEventRow>],
max_rows_per_block: usize,
@@ -127,26 +66,45 @@ pub(crate) fn encode_v4_native_blocks(
return Err("max_rows_per_block must be greater than zero".to_owned());
}
- rows.chunks(max_rows_per_block)
- .map(|chunk| {
- let bytes = encode_block(chunk)?;
- Ok(EncodedBlock {
- bytes,
- row_count: chunk.len(),
- })
- })
- .collect()
+ let mut blocks = Vec::new();
+ let mut encode = encode_block;
+ for chunk in rows.chunks(max_rows_per_block) {
+ encode_with_bisection(chunk, &mut encode, &mut blocks)
+ .map_err(|error| error.to_string())?;
+ }
+ Ok(blocks)
}
-fn encode_block(rows: &[Arc<PreparedEventRow>]) -> Result<Bytes, String> {
- let mut builder = BlockBuilder::new();
- PreparedEventRow::encode_arcs(&mut builder, rows)?;
+fn encode_block(rows: &[Arc<PreparedEventRow>]) -> Result<Vec<u8>, NativeColumnError> {
+ let schema = PreparedEventRow::schema();
+ let columns = PreparedEventRow::build_columns(rows)?;
+ let batch = ch_core_rs::batch::ColBatch::new(schema, columns, rows.len());
+ ch_core_rs::native::encode::encode_block(
+ &batch,
+ &ch_core_rs::native::encode::EncodeOptions::default(),
+ )
+ .map_err(|error| NativeColumnError::Message(error.to_string()))
+}
- builder
- .build()
- .map_err(|error| error.to_string())?
- .encode()
- .map_err(|error| error.to_string())
+fn encode_with_bisection<T>(
+ rows: &[T],
+ encode: &mut impl FnMut(&[T]) -> Result<Vec<u8>, NativeColumnError>,
+ blocks: &mut Vec<EncodedBlock>,
+) -> Result<(), NativeColumnError> {
+ match encode(rows) {
+ Ok(bytes) => blocks.push(EncodedBlock {
+ bytes,
+ row_count: rows.len(),
+ }),
+ Err(NativeColumnError::StringColumnTooLarge) if rows.len() > 1 => {
+ let midpoint = rows.len() / 2;
+ let (left, right) = rows.split_at(midpoint);
+ encode_with_bisection(left, encode, blocks)?;
+ encode_with_bisection(right, encode, blocks)?;
+ }
+ Err(error) => return Err(error),
+ }
+ Ok(())
}
/// Parse the UTC-only datetime values produced by the ingestion pipeline. A ClickHouse datetime
diff --git a/packages/native/src/native_codec_tests.rs b/packages/native/src/native_codec_tests.rs
index d680d87eca..61a419762f 100644
--- a/packages/native/src/native_codec_tests.rs
+++ b/packages/native/src/native_codec_tests.rs
@@ -37,6 +37,38 @@ fn datetime_strings_are_explicitly_utc() {
assert_eq!(offset, utc);
}
+#[test]
+fn string_column_overflow_bisects_rows_in_order() {
+ let rows = [0u8, 1, 2, 3, 4];
+ let mut attempts = 0;
+ let mut encode = |chunk: &[u8]| {
+ attempts += 1;
+ if chunk.len() > 1 {
+ Err(NativeColumnError::StringColumnTooLarge)
+ } else {
+ Ok(vec![chunk[0]])
+ }
+ };
+ let mut blocks = Vec::new();
+ encode_with_bisection(&rows, &mut encode, &mut blocks).unwrap();
+
+ assert_eq!(
+ blocks
+ .iter()
+ .map(|block| (block.row_count, block.bytes[0]))
+ .collect::<Vec<_>>(),
+ vec![(1, 0), (1, 1), (1, 2), (1, 3), (1, 4)]
+ );
+ assert_eq!(attempts, 9);
+
+ let mut encode = |_: &[u8]| -> Result<Vec<u8>, NativeColumnError> {
+ Err(NativeColumnError::StringColumnTooLarge)
+ };
+ let mut blocks = Vec::new();
+ let error = encode_with_bisection(&[0u8], &mut encode, &mut blocks).unwrap_err();
+ assert_eq!(error, NativeColumnError::StringColumnTooLarge);
+}
+
proptest! {
#[test]
fn decimal_inputs_with_extreme_exponents_are_handled(
diff --git a/packages/native/src/native_columns.rs b/packages/native/src/native_columns.rs
new file mode 100644
index 0000000000..be1d323ad6
--- /dev/null
+++ b/packages/native/src/native_columns.rs
@@ -0,0 +1,358 @@
+//! Build ch-core-rs columns from prepared event rows.
+
+use std::collections::BTreeMap;
+use std::fmt;
+
+use ch_core_rs::bitmap::Bitmap;
+use ch_core_rs::column::{
+ ArrayColumn, BoolColumn, Column, DecimalColumn, MapColumn, PrimitiveColumn, TupleColumn,
+ Utf8Column,
+};
+use ch_core_rs::schema::ChType;
+
+use crate::native_codec::{DateTime64Micros, Decimal64, SkillReference, SkillResourceLoaded};
+
+#[derive(Debug, PartialEq, Eq)]
+pub(crate) enum NativeColumnError {
+ StringColumnTooLarge,
+ Message(String),
+}
+
+impl fmt::Display for NativeColumnError {
+ fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
+ match self {
+ Self::StringColumnTooLarge => {
+ formatter.write_str("String column exceeds i32::MAX bytes")
+ }
+ Self::Message(message) => formatter.write_str(message),
+ }
+ }
+}
+
+impl From<String> for NativeColumnError {
+ fn from(message: String) -> Self {
+ Self::Message(message)
+ }
+}
+
+/// Maps one prepared field type to its owned ch-core column.
+///
+/// The input is a slice of row references. Nested implementations flatten references and let the
+/// leaf implementation allocate each output buffer once.
+pub(crate) trait NativeColumn: Sized {
+ fn ch_type() -> ChType;
+
+ fn build(values: &[&Self]) -> Result<Column, NativeColumnError>;
+}
+
+trait NullableColumn: NativeColumn + Default {}
+
+pub(crate) fn column_type<T: NativeColumn>() -> ChType {
+ T::ch_type()
+}
+
+pub(crate) fn build_column<'a, T, I>(values: I) -> Result<Column, NativeColumnError>
+where
+ T: NativeColumn + 'a,
+ I: IntoIterator<Item = &'a T>,
+{
+ let values: Vec<&'a T> = values.into_iter().collect();
+ T::build(&values)
+}
+
+fn checked_string_offset(value: usize) -> Result<i32, NativeColumnError> {
+ i32::try_from(value).map_err(|_| NativeColumnError::StringColumnTooLarge)
+}
+
+fn checked_i64_offset(value: usize, kind: &str) -> Result<i64, NativeColumnError> {
+ i64::try_from(value).map_err(|_| format!("{kind} offset exceeds i64::MAX").into())
+}
+
+impl NativeColumn for String {
+ fn ch_type() -> ChType {
+ ChType::String
+ }
+
+ fn build(values: &[&Self]) -> Result<Column, NativeColumnError> {
+ let total_bytes = values.iter().try_fold(0usize, |total, value| {
+ total
+ .checked_add(value.len())
+ .ok_or_else(|| "String data length overflow".to_owned())
+ })?;
+ let _ = checked_string_offset(total_bytes)?;
+
+ let mut offsets = Vec::with_capacity(values.len() + 1);
+ offsets.push(0);
+ let mut data = Vec::with_capacity(total_bytes);
+ for value in values {
+ data.extend_from_slice(value.as_bytes());
+ offsets.push(checked_string_offset(data.len())?);
+ }
+ Ok(Column::Utf8(Utf8Column::new(offsets, data)))
+ }
+}
+
+impl NativeColumn for bool {
+ fn ch_type() -> ChType {
+ ChType::Bool
+ }
+
+ fn build(values: &[&Self]) -> Result<Column, NativeColumnError> {
+ let len = values.len();
+ let mut bitmap = vec![0; len.div_ceil(8)];
+ for (index, value) in values.iter().enumerate() {
+ if **value {
+ bitmap[index / 8] |= 1 << (index % 8);
+ }
+ }
+ Ok(Column::Bool(BoolColumn {
+ bitmap,
+ len,
+ validity: None,
+ }))
+ }
+}
+
+macro_rules! impl_primitive_column {
+ ($ty:ty, $ch_type:ident, $variant:ident) => {
+ impl NativeColumn for $ty {
+ fn ch_type() -> ChType {
+ ChType::$ch_type
+ }
+
+ fn build(values: &[&Self]) -> Result<Column, NativeColumnError> {
+ Ok(Column::$variant(PrimitiveColumn::new(
+ values.iter().map(|value| **value).collect(),
+ )))
+ }
+ }
+ };
+}
+
+impl_primitive_column!(u8, UInt8, UInt8);
+impl_primitive_column!(u16, UInt16, UInt16);
+impl_primitive_column!(u32, UInt32, UInt32);
+impl_primitive_column!(u64, UInt64, UInt64);
+
+impl NativeColumn for DateTime64Micros {
+ fn ch_type() -> ChType {
+ ChType::DateTime64 {
+ precision: 6,
+ timezone: None,
+ }
+ }
+
+ fn build(values: &[&Self]) -> Result<Column, NativeColumnError> {
+ Ok(Column::DateTime64(PrimitiveColumn::new(
+ values.iter().map(|value| value.0).collect(),
+ )))
+ }
+}
+
+impl NullableColumn for String {}
+impl NullableColumn for u16 {}
+impl NullableColumn for u32 {}
+impl NullableColumn for DateTime64Micros {}
+
+impl NativeColumn for Decimal64 {
+ fn ch_type() -> ChType {
+ ChType::Decimal {
+ precision: 18,
+ scale: 12,
+ bits: 64,
+ }
+ }
+
+ fn build(values: &[&Self]) -> Result<Column, NativeColumnError> {
+ let capacity = values
+ .len()
+ .checked_mul(std::mem::size_of::<i64>())
+ .ok_or_else(|| "Decimal data length overflow".to_owned())?;
+ let mut data = Vec::with_capacity(capacity);
+ for value in values {
+ data.extend_from_slice(&value.0.to_le_bytes());
+ }
+ Ok(Column::Decimal(DecimalColumn::new(data, 8, 18, 12)))
+ }
+}
+
+impl<T> NativeColumn for Option<T>
+where
+ T: NullableColumn,
+{
+ fn ch_type() -> ChType {
+ ChType::Nullable(Box::new(T::ch_type()))
+ }
+
+ fn build(values: &[&Self]) -> Result<Column, NativeColumnError> {
+ let default = T::default();
+ let mut null_map = Vec::with_capacity(values.len());
+ let inner: Vec<&T> = values
+ .iter()
+ .map(|value| match value {
+ Some(value) => {
+ null_map.push(0);
+ value
+ }
+ None => {
+ null_map.push(1);
+ &default
+ }
+ })
+ .collect();
+ with_validity(T::build(&inner)?, Bitmap::from_ch_null_map(&null_map))
+ }
+}
+
+fn with_validity(column: Column, validity: Bitmap) -> Result<Column, NativeColumnError> {
+ match column {
+ Column::Utf8(mut column) => {
+ column.validity = Some(validity);
+ Ok(Column::Utf8(column))
+ }
+ Column::UInt16(mut column) => {
+ column.validity = Some(validity);
+ Ok(Column::UInt16(column))
+ }
+ Column::UInt32(mut column) => {
+ column.validity = Some(validity);
+ Ok(Column::UInt32(column))
+ }
+ Column::DateTime64(mut column) => {
+ column.validity = Some(validity);
+ Ok(Column::DateTime64(column))
+ }
+ _ => Err("Nullable wrapper is unsupported for this column"
+ .to_owned()
+ .into()),
+ }
+}
+
+impl<T> NativeColumn for Vec<T>
+where
+ T: NativeColumn,
+{
+ fn ch_type() -> ChType {
+ ChType::Array(Box::new(T::ch_type()))
+ }
+
+ fn build(values: &[&Self]) -> Result<Column, NativeColumnError> {
+ let total_values = values.iter().try_fold(0usize, |total, row| {
+ total
+ .checked_add(row.len())
+ .ok_or_else(|| "Array element count overflow".to_owned())
+ })?;
+ let mut offsets = Vec::with_capacity(values.len() + 1);
+ offsets.push(0);
+ let mut flattened = Vec::with_capacity(total_values);
+ for row in values {
+ flattened.extend(row.iter());
+ offsets.push(checked_i64_offset(flattened.len(), "Array")?);
+ }
+ let child = T::build(&flattened)?;
+ Ok(Column::Array(ArrayColumn::new(offsets, child)))
+ }
+}
+
+impl<V> NativeColumn for BTreeMap<String, V>
+where
+ V: NativeColumn,
+{
+ fn ch_type() -> ChType {
+ ChType::Map(Box::new(ChType::String), Box::new(V::ch_type()))
+ }
+
+ fn build(values: &[&Self]) -> Result<Column, NativeColumnError> {
+ let total_entries = values.iter().try_fold(0usize, |total, map| {
+ total
+ .checked_add(map.len())
+ .ok_or_else(|| "Map entry count overflow".to_owned())
+ })?;
+ let mut offsets = Vec::with_capacity(values.len() + 1);
+ offsets.push(0);
+ let mut keys = Vec::with_capacity(total_entries);
+ let mut map_values = Vec::with_capacity(total_entries);
+ for map in values {
+ keys.extend(map.keys());
+ map_values.extend(map.values());
+ offsets.push(checked_i64_offset(keys.len(), "Map")?);
+ }
+
+ let entries = Column::Tuple(TupleColumn::new(
+ vec![String::build(&keys)?, V::build(&map_values)?],
+ total_entries,
+ ));
+ Ok(Column::Map(MapColumn::new(offsets, entries)))
+ }
+}
+
+impl NativeColumn for SkillReference {
+ fn ch_type() -> ChType {
+ ChType::Tuple(vec![
+ (None, <Option<String>>::ch_type()),
+ (None, <Option<u32>>::ch_type()),
+ (None, <String>::ch_type()),
+ ])
+ }
+
+ fn build(values: &[&Self]) -> Result<Column, NativeColumnError> {
+ let ids: Vec<&Option<String>> = values
+ .iter()
+ .map(|value| &value.langfuse_skill_id)
+ .collect();
+ let versions: Vec<&Option<u32>> = values.iter().map(|value| &value.skill_version).collect();
+ let names: Vec<&String> = values.iter().map(|value| &value.skill_name).collect();
+ Ok(Column::Tuple(TupleColumn::new(
+ vec![
+ Option::<String>::build(&ids)?,
+ Option::<u32>::build(&versions)?,
+ String::build(&names)?,
+ ],
+ values.len(),
+ )))
+ }
+}
+
+impl NativeColumn for SkillResourceLoaded {
+ fn ch_type() -> ChType {
+ ChType::Tuple(vec![
+ (None, <Option<String>>::ch_type()),
+ (None, <Option<u32>>::ch_type()),
+ (None, <String>::ch_type()),
+ (None, <String>::ch_type()),
+ ])
+ }
+
+ fn build(values: &[&Self]) -> Result<Column, NativeColumnError> {
+ let ids: Vec<&Option<String>> = values
+ .iter()
+ .map(|value| &value.langfuse_skill_id)
+ .collect();
+ let versions: Vec<&Option<u32>> = values.iter().map(|value| &value.skill_version).collect();
+ let names: Vec<&String> = values.iter().map(|value| &value.skill_name).collect();
+ let paths: Vec<&String> = values.iter().map(|value| &value.file_path).collect();
+ Ok(Column::Tuple(TupleColumn::new(
+ vec![
+ Option::<String>::build(&ids)?,
+ Option::<u32>::build(&versions)?,
+ String::build(&names)?,
+ String::build(&paths)?,
+ ],
+ values.len(),
+ )))
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ #[test]
+ fn string_offset_limit_is_checked_without_allocating_the_column() {
+ assert_eq!(checked_string_offset(i32::MAX as usize), Ok(i32::MAX));
+ assert_eq!(
+ checked_string_offset(i32::MAX as usize + 1),
+ Err(NativeColumnError::StringColumnTooLarge)
+ );
+ }
+}
diff --git a/packages/native/src/native_js.rs b/packages/native/src/native_js.rs
index 68cc495e7d..7d1c968d8a 100644
--- a/packages/native/src/native_js.rs
+++ b/packages/native/src/native_js.rs
@@ -5,11 +5,30 @@ use crate::native_codec::{
parse_datetime_text, parse_decimal, DateTime64Micros, Decimal64, SkillReference,
SkillResourceLoaded,
};
-use crate::native_schema::DefaultPolicy;
+use crate::native_schema::{DefaultPolicy, PreparedEventRow};
use napi::bindgen_prelude::{Array, JsObjectValue, Object, Unknown};
use napi::{JsNumber, JsValue};
use std::collections::BTreeMap;
+impl PreparedEventRow {
+ /// Convert one finalized JavaScript row directly into owned Rust fields.
+ /// Only declared columns are read; JS handles stay on the calling thread.
+ pub(crate) fn from_js(value: Unknown<'_>) -> Result<Self, String> {
+ let object: Object<'_> = match value.get_type().map_err(|error| error.to_string())? {
+ napi::ValueType::Object => {
+ unsafe { value.cast() }.map_err(|error| error.to_string())?
+ }
+ value_type => {
+ return Err(format!(
+ "prepared event must be an object, got {value_type:?}"
+ ));
+ }
+ };
+ let metadata = MetadataValues::from_js(&object)?;
+ Self::from_js_fields(object, metadata)
+ }
+}
+
/// Convert one schema field by reading its property directly from the JS object.
///
/// This is intentionally a small typed adapter rather than a generic serde bridge. It never
diff --git a/packages/native/src/native_schema.rs b/packages/native/src/native_schema.rs
index 39a8aa3f07..66f8760fac 100644
--- a/packages/native/src/native_schema.rs
+++ b/packages/native/src/native_schema.rs
@@ -1,11 +1,9 @@
//! The insertable part of `events_full` and its prepared Rust row.
//!
//! The macro invocation below is the only per-column declaration. It drives the prepared row,
-//! column type metadata, JS reads, and typed Native column writes.
+//! column type metadata, JS reads, and typed Native column construction.
-use clickhouse::native::builder::BlockBuilder;
-use clickhouse::native::encode::Encode;
-use napi::bindgen_prelude::{Object, Unknown};
+use napi::bindgen_prelude::Object;
use crate::native_codec::{DateTime64Micros, Decimal64, SkillReference, SkillResourceLoaded};
@@ -100,34 +98,26 @@ macro_rules! prepare_js_field {
};
}
-/// Declare the prepared row and every operation that must agree with its shape.
-macro_rules! clickhouse_table_schema {
- (
- $(
- $field:ident $(as $name:literal)? : $ty:ty => $( @ $source:ident )? $policy:expr
- ),* $(,)?
- ) => {
+macro_rules! define_event_row {
+ ($($field:ident : $ty:ty),* $(,)?) => {
#[derive(Debug)]
pub(crate) struct PreparedEventRow {
$(pub(crate) $field: $ty,)*
}
+ };
+}
+macro_rules! impl_js_field_reads {
+ (
+ $(
+ $field:ident $(as $name:literal)? : $ty:ty => $( @ $source:ident )? $policy:expr
+ ),* $(,)?
+ ) => {
impl PreparedEventRow {
- /// Convert one finalized JavaScript row directly into owned Rust fields.
- ///
- /// The schema macro generates typed reads for the declared columns and ignores
- /// unrelated properties. Only owned typed fields leave the NAPI call.
- pub(crate) fn from_js(value: Unknown<'_>) -> Result<Self, String> {
- let object: Object<'_> = match value.get_type().map_err(|error| error.to_string())? {
- napi::ValueType::Object => unsafe { value.cast() }
- .map_err(|error| error.to_string())?,
- value_type => {
- return Err(format!(
- "prepared event must be an object, got {value_type:?}"
- ));
- }
- };
- let mut metadata = crate::native_js::MetadataValues::from_js(&object)?;
+ pub(crate) fn from_js_fields(
+ object: Object<'_>,
+ mut metadata: crate::native_js::MetadataValues,
+ ) -> Result<Self, String> {
$(
prepare_js_field!(
object,
@@ -151,40 +141,59 @@ macro_rules! clickhouse_table_schema {
)*
})
}
+ }
+ };
+}
+macro_rules! impl_native_columns {
+ ($($field:ident $(as $name:literal)? : $ty:ty => $policy:expr),* $(,)?) => {
+ impl PreparedEventRow {
/// Return the insertable columns and their Native type names from this declaration.
pub(crate) fn columns() -> Vec<(String, String, bool)> {
vec![$(
(
schema_column_name!($field $(as $name)?).to_owned(),
- <$ty as Encode>::produces().to_string(),
+ crate::native_columns::column_type::<$ty>().to_string(),
$policy != DefaultPolicy::None,
),
)*]
}
- /// Write shared prepared rows through clickhouse-rs's typed `Encode` implementations.
- pub(crate) fn encode_arcs(
- builder: &mut BlockBuilder,
+ pub(crate) fn schema() -> ch_core_rs::schema::Schema {
+ ch_core_rs::schema::Schema::new(vec![
+ $(
+ ch_core_rs::schema::Field {
+ name: schema_column_name!($field $(as $name)?).to_owned(),
+ ch_type: crate::native_columns::column_type::<$ty>(),
+ },
+ )*
+ ])
+ }
+
+ pub(crate) fn build_columns(
rows: &[std::sync::Arc<Self>],
- ) -> Result<(), String> {
- $(
- let name = schema_column_name!($field $(as $name)?);
- let mut column = builder
- .upsert_column::<&$ty>(name)
- .map_err(|error| format!("cannot add column {name}: {error}"))?;
- for row in rows {
- column
- .add(&row.$field)
- .map_err(|error| format!("cannot encode column {name}: {error}"))?;
- }
- )*
- Ok(())
+ ) -> Result<Vec<ch_core_rs::column::Column>, crate::native_columns::NativeColumnError> {
+ Ok(vec![
+ $(
+ crate::native_columns::build_column::<$ty, _>(
+ rows.iter().map(|row| &row.$field),
+ )?,
+ )*
+ ])
}
}
};
}
+/// Keep the row, JS reads, and Native columns in the same declared order.
+macro_rules! clickhouse_table_schema {
+ ($($field:ident $(as $name:literal)? : $ty:ty => $(@ $source:ident)? $policy:expr),* $(,)?) => {
+ define_event_row! { $($field: $ty,)* }
+ impl_js_field_reads! { $($field $(as $name)?: $ty => $(@ $source)? $policy,)* }
+ impl_native_columns! { $($field $(as $name)?: $ty => $policy,)* }
+ };
+}
+
// Keep this list in the same order as the insertable events_full contract. Materialized and alias
// columns remain server-owned and therefore are intentionally absent.
clickhouse_table_schema! {
diff --git a/scripts/ci/cargo-security.sh b/scripts/ci/cargo-security.sh
index 39bee231e9..2507b4eca3 100755
--- a/scripts/ci/cargo-security.sh
+++ b/scripts/ci/cargo-security.sh
@@ -14,7 +14,6 @@
# The workflow installs this script before the agent starts, in a directory the
# sandbox mounts read-only. That directory contains:
# toolchains/<crate-slug> bin directory of the crate's pinned toolchain
-# native-cargo-config.toml clickhouse patch overlay for packages/native
# cwd/ empty working directory for cargo
#
# Usage:
@@ -64,9 +63,6 @@ if [[ ! "$spec" =~ $spec_re ]]; then
fi
cargo_args=()
-if [[ "$slug" == native ]]; then
- cargo_args+=(--config "$install_dir/native-cargo-config.toml")
-fi
manifest="$workspace/$crate_dir/Cargo.toml"
if [[ "$subcommand" == update ]]; then
diff --git a/worker/Dockerfile b/worker/Dockerfile
index 35974d3a47..e2db9f0019 100644
--- a/worker/Dockerfile
+++ b/worker/Dockerfile
@@ -114,9 +114,8 @@ ARG TARGETARCH
# @langfuse/native is a Rust addon (napi-rs) compiled by `turbo run build` for
# the glibc (`*-unknown-linux-gnu`) target, linked against UBI9's glibc so it
# loads in the runtime image. gcc provides the linker; tar and gzip unpack the
-# pinned toolchain; patch (with the base image's curl-minimal) prepares the
-# small clickhouse source overlay used by the native encoder.
-RUN dnf -y install --setopt=install_weak_deps=0 gcc tar gzip patch && \
+# pinned toolchain.
+RUN dnf -y install --setopt=install_weak_deps=0 gcc tar gzip && \
dnf clean all
# Pinned standalone Rust installer, the exact channel from
@@ -183,11 +182,9 @@ COPY --from=pruner /app/out/full/ .
RUN turbo run build --filter=worker...
-# Runs after the build so the generated clickhouse patch overlay
-# (packages/native/.cargo) resolves the same dependency graph that was
-# compiled. `cargo fetch` adds the sources the build skipped (dev and
-# other-target dependencies) that cargo metadata still reads; `generate` then
-# reads licenses only from local crate sources and fails on any dependency
+# Runs after the build so `cargo fetch` adds the sources the build skipped (dev
+# and other-target dependencies) that cargo metadata still reads; `generate`
+# then reads licenses only from local crate sources and fails on any dependency
# whose license packages/native/about.toml does not accept.
RUN cd packages/native && \
mkdir -p /usr/share/licenses/langfuse-native && \