Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(ai-research): add the v4 native addon for the url policy (#117979)

Nicholas Waltz82722687d06a
diff --git a/.github/scripts/trunk-impacted-targets.js b/.github/scripts/trunk-impacted-targets.js
index be7435045b51..64338571ef49 100644
--- a/.github/scripts/trunk-impacted-targets.js
+++ b/.github/scripts/trunk-impacted-targets.js
@@ -1501,8 +1501,8 @@ const rustCrate = (crate) => `rust:crate:${crate}`
const RUST_DETERMINATOR = 'rust:determinator'
// The lanes that import a native module built from the cargo workspace.
-// nodejs/package.json is the only dependent of the two binding packages
-// (@posthog/hogvm-node, @posthog/replay-anonymizer) today,
+// nodejs/package.json is the only dependent of the three binding packages
+// (@posthog/hogvm-node, @posthog/replay-anonymizer, @posthog/ai-research-v4-native) today,
// and the test suite re-derives that from pnpm-workspace.yaml so a second
// dependent fails there rather than silently going unclaimed here. The
// image-scrub sidecar under nodejs/src/ingestion loads the replay-anonymizer
diff --git a/.github/workflows/ci-nodejs-container.yml b/.github/workflows/ci-nodejs-container.yml
index 217953cb4ccb..b1f6b6f81c83 100644
--- a/.github/workflows/ci-nodejs-container.yml
+++ b/.github/workflows/ci-nodejs-container.yml
@@ -14,6 +14,8 @@ on:
- 'rust/common/hogvm/**'
- 'rust/replay-anonymizer/**'
- 'rust/replay-anonymizer-node/**'
+ - 'rust/ai-research/replay-core-v4/**'
+ - 'rust/ai-research/replay-core-v4-node/**'
- 'Dockerfile.node'
- '.github/workflows/ci-nodejs-container.yml'
- 'turbo.json'
@@ -72,6 +74,8 @@ jobs:
- 'rust/common/hogvm/**'
- 'rust/replay-anonymizer/**'
- 'rust/replay-anonymizer-node/**'
+ - 'rust/ai-research/replay-core-v4/**'
+ - 'rust/ai-research/replay-core-v4-node/**'
- 'Dockerfile.node'
- '.github/workflows/ci-nodejs-container.yml'
- 'turbo.json'
@@ -94,6 +98,8 @@ jobs:
- 'nodejs/package.json'
- 'rust/replay-anonymizer/**'
- 'rust/replay-anonymizer-node/**'
+ - 'rust/ai-research/replay-core-v4/**'
+ - 'rust/ai-research/replay-core-v4-node/**'
- 'Dockerfile.node'
- '.github/workflows/ci-nodejs-container.yml'
- 'turbo.json'
diff --git a/.github/workflows/ci-nodejs.yml b/.github/workflows/ci-nodejs.yml
index a946a3409009..006ed2159f2a 100644
--- a/.github/workflows/ci-nodejs.yml
+++ b/.github/workflows/ci-nodejs.yml
@@ -117,6 +117,9 @@ jobs:
# its Rust tests; a rust-side change (incl. the shared fixtures) must run it.
- 'rust/replay-anonymizer/**'
- 'rust/replay-anonymizer-node/**'
+ # The v4 image fetcher's Jest tests load the URL policy through this addon.
+ - 'rust/ai-research/replay-core-v4/**'
+ - 'rust/ai-research/replay-core-v4-node/**'
# These native addons are built by Turbo before Node.js tests. Their Rust
# dependencies are not part of the ingestion consumer's targeted e2e set.
- 'rust/common/hogvm/**'
diff --git a/.github/workflows/ci-rust.yml b/.github/workflows/ci-rust.yml
index bff89b5f3da7..de4578854e38 100644
--- a/.github/workflows/ci-rust.yml
+++ b/.github/workflows/ci-rust.yml
@@ -220,6 +220,7 @@ jobs:
"property-vals-rs": 70,
"replay-anonymizer-node": 5,
"replay-core-v4": 10,
+ "replay-core-v4-node": 5,
"replay-recorder-v4": 15,
"serve-metrics": 5,
"usage-ingestion": 70,
diff --git a/Dockerfile.node b/Dockerfile.node
index b26f1183c839..5d5fe54065dc 100644
--- a/Dockerfile.node
+++ b/Dockerfile.node
@@ -77,6 +77,9 @@ RUN NODE_OPTIONS="--max-old-space-size=16384" bin/turbo --filter=@posthog/hogvm-
# Build the session-replay anonymizer native addon
RUN NODE_OPTIONS="--max-old-space-size=16384" bin/turbo --filter=@posthog/replay-anonymizer build
+# Build the v4 AI research native addon (the URL policy of the v4 image fetcher)
+RUN NODE_OPTIONS="--max-old-space-size=16384" bin/turbo --filter=@posthog/ai-research-v4-native build
+
# Build replay-headless (dependency of nodejs rasterizer code)
RUN NODE_OPTIONS="--max-old-space-size=16384" bin/turbo --filter=@posthog/replay-headless build
# Then build the nodejs services
@@ -134,6 +137,9 @@ COPY --from=nodejs-build --chown=posthog:posthog /code/rust/common/hogvm/node/in
COPY --from=nodejs-build --chown=posthog:posthog /code/rust/replay-anonymizer-node/dist /code/rust/replay-anonymizer-node/dist
COPY --from=nodejs-build --chown=posthog:posthog /code/rust/replay-anonymizer-node/package.json /code/rust/replay-anonymizer-node/package.json
COPY --from=nodejs-build --chown=posthog:posthog /code/rust/replay-anonymizer-node/index.node /code/rust/replay-anonymizer-node/index.node
+COPY --from=nodejs-build --chown=posthog:posthog /code/rust/ai-research/replay-core-v4-node/dist /code/rust/ai-research/replay-core-v4-node/dist
+COPY --from=nodejs-build --chown=posthog:posthog /code/rust/ai-research/replay-core-v4-node/package.json /code/rust/ai-research/replay-core-v4-node/package.json
+COPY --from=nodejs-build --chown=posthog:posthog /code/rust/ai-research/replay-core-v4-node/index.node /code/rust/ai-research/replay-core-v4-node/index.node
COPY --from=nodejs-build --chown=posthog:posthog /code/common/plugin_transpiler/dist /code/common/plugin_transpiler/dist
COPY --from=nodejs-build --chown=posthog:posthog /code/common/plugin_transpiler/node_modules /code/common/plugin_transpiler/node_modules
COPY --from=nodejs-build --chown=posthog:posthog /code/common/plugin_transpiler/package.json /code/common/plugin_transpiler/package.json
diff --git a/nodejs/package.json b/nodejs/package.json
index d390250255f1..c53880215676 100644
--- a/nodejs/package.json
+++ b/nodejs/package.json
@@ -49,6 +49,8 @@
"build:hogvm-node:dev": "pnpm --filter=@posthog/hogvm-node run build:dev",
"build:replay-anonymizer": "pnpm --filter=@posthog/replay-anonymizer package",
"build:replay-anonymizer:dev": "pnpm --filter=@posthog/replay-anonymizer run build:dev",
+ "build:ai-research-v4-native": "pnpm --filter=@posthog/ai-research-v4-native package",
+ "build:ai-research-v4-native:dev": "pnpm --filter=@posthog/ai-research-v4-native run build:dev",
"generate:personhog-proto": "cd ../proto && buf generate --template ../nodejs/buf.gen.yaml --path personhog/service --path personhog/identity --path personhog/lifecycle --path personhog/types . && cd ../nodejs && eslint --fix src/common/generated/personhog && oxlint -c .oxlintrc.nodejs.json --fix src/common/generated/personhog && prettier --write src/common/generated/personhog",
"generate:ingestion-worker-proto": "cd ../proto && buf generate --template ../nodejs/buf.gen.ingestion-worker.yaml --path ingestion/worker . && cd ../nodejs && eslint --fix src/common/generated/ingestion-worker && oxlint -c .oxlintrc.nodejs.json --fix src/common/generated/ingestion-worker && prettier --write src/common/generated/ingestion-worker",
"generate:usage-ingestion-proto": "cd ../proto && buf generate --template ../nodejs/buf.gen.usage-ingestion.yaml --path usage_ingestion . && cd ../nodejs && eslint --fix src/common/generated/usage-ingestion && oxlint -c .oxlintrc.nodejs.json --fix src/common/generated/usage-ingestion && prettier --write src/common/generated/usage-ingestion",
@@ -91,6 +93,7 @@
"@opentelemetry/sdk-node": "^0.218.0",
"@opentelemetry/sdk-trace-node": "^2.0.1",
"@opentelemetry/semantic-conventions": "^1.36.0",
+ "@posthog/ai-research-v4-native": "workspace:*",
"@posthog/hogvm": "workspace:*",
"@posthog/hogvm-node": "workspace:*",
"@posthog/replay-anonymizer": "workspace:*",
diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml
index 8ff82bfb122d..7cd47f4ff8f4 100644
--- a/pnpm-lock.yaml
+++ b/pnpm-lock.yaml
@@ -1622,6 +1622,9 @@ importers:
'@opentelemetry/semantic-conventions':
specifier: ^1.36.0
version: 1.36.0
+ '@posthog/ai-research-v4-native':
+ specifier: workspace:*
+ version: link:../rust/ai-research/replay-core-v4-node
'@posthog/hogvm':
specifier: workspace:*
version: link:../common/hogvm/typescript
@@ -5720,6 +5723,18 @@ importers:
specifier: 'catalog:'
version: 0.2.4([email protected](patch_hash=139b8d1f1304f9d9da452a9a1244c94ea679dbcb85687d8999563146879fb6f5)([email protected]))
+ rust/ai-research/replay-core-v4-node:
+ devDependencies:
+ '@types/node':
+ specifier: 22.15.17
+ version: 22.15.17
+ typescript:
+ specifier: 6.0.3
+ version: 6.0.3
+ undici-types:
+ specifier: ^7.3.0
+ version: 7.3.0
+
rust/common/hogvm/node: {}
rust/replay-anonymizer-node:
diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml
index 74540d5c9d23..0db74b2cd4d2 100644
--- a/pnpm-workspace.yaml
+++ b/pnpm-workspace.yaml
@@ -30,6 +30,7 @@ packages:
- '!tools/hedgebox-dummy'
- rust/common/hogvm/node
- rust/replay-anonymizer-node
+ - rust/ai-research/replay-core-v4-node
blockExoticSubdeps: true
trustPolicy: no-downgrade
diff --git a/rust/Cargo.lock b/rust/Cargo.lock
index f0aef9f5fc85..22b923751bc7 100644
--- a/rust/Cargo.lock
+++ b/rust/Cargo.lock
@@ -11047,6 +11047,14 @@ dependencies = [
"zstd 0.13.3",
]
+[[package]]
+name = "replay-core-v4-node"
+version = "0.0.1"
+dependencies = [
+ "neon",
+ "replay-core-v4",
+]
+
[[package]]
name = "replay-recorder-v4"
version = "0.1.0"
diff --git a/rust/Cargo.toml b/rust/Cargo.toml
index 7aa27ed9ec09..9f8b436c55ae 100644
--- a/rust/Cargo.toml
+++ b/rust/Cargo.toml
@@ -80,6 +80,7 @@ members = [
"pgcollector",
"pgapi",
"ai-research/replay-core-v4",
+ "ai-research/replay-core-v4-node",
"ai-research/replay-recorder-v4",
]
diff --git a/rust/ai-research/replay-core-v4-node/.gitignore b/rust/ai-research/replay-core-v4-node/.gitignore
new file mode 100644
index 000000000000..d15049d90e52
--- /dev/null
+++ b/rust/ai-research/replay-core-v4-node/.gitignore
@@ -0,0 +1,3 @@
+index.node
+cargo.log
+dist/
diff --git a/rust/ai-research/replay-core-v4-node/Cargo.toml b/rust/ai-research/replay-core-v4-node/Cargo.toml
new file mode 100644
index 000000000000..69fe000d480f
--- /dev/null
+++ b/rust/ai-research/replay-core-v4-node/Cargo.toml
@@ -0,0 +1,17 @@
+[package]
+name = "replay-core-v4-node"
+version = "0.0.1"
+edition = "2021"
+exclude = ["index.node"]
+publish = false
+
+[lints]
+workspace = true
+
+[lib]
+crate-type = ["cdylib"]
+
+[dependencies]
+neon = { workspace = true }
+# default-features = false: the key table and KMS clients have no place in an addon that only checks URLs.
+replay-core-v4 = { path = "../replay-core-v4", default-features = false }
diff --git a/rust/ai-research/replay-core-v4-node/package.json b/rust/ai-research/replay-core-v4-node/package.json
new file mode 100644
index 000000000000..b11dd60d0be8
--- /dev/null
+++ b/rust/ai-research/replay-core-v4-node/package.json
@@ -0,0 +1,31 @@
+{
+ "name": "@posthog/ai-research-v4-native",
+ "version": "0.0.1",
+ "description": "Node bindings for the v4 AI research replay URL policy",
+ "main": "dist/index.js",
+ "types": "dist/index.d.ts",
+ "scripts": {
+ "test": "cargo test -p replay-core-v4 -p replay-core-v4-node",
+ "build": "pnpm run build:cargo --release && pnpm run build:move-lib && pnpm run build:typescript",
+ "build:dev": "pnpm run build:cargo && pnpm run build:move-lib:dev && pnpm run build:typescript",
+ "build:move-lib": "TARGET_DIR=${CARGO_TARGET_DIR:-../../target} && cp \"$TARGET_DIR/release/libreplay_core_v4_node.dylib\" index.node 2>/dev/null || cp \"$TARGET_DIR/release/libreplay_core_v4_node.so\" index.node || (echo \"No valid library file found in $TARGET_DIR/release/\" && exit 1)",
+ "build:move-lib:dev": "TARGET_DIR=${CARGO_TARGET_DIR:-../../target} && cp \"$TARGET_DIR/debug/libreplay_core_v4_node.dylib\" index.node 2>/dev/null || cp \"$TARGET_DIR/debug/libreplay_core_v4_node.so\" index.node || (echo \"No valid library file found in $TARGET_DIR/debug/\" && exit 1)",
+ "build:cargo": "cargo build -p replay-core-v4-node --message-format=json > cargo.log",
+ "build:cargo:debug": "pnpm run build:cargo",
+ "build:typescript": "tsc",
+ "typecheck": "tsc --noEmit",
+ "clean": "rm -rf index.node dist",
+ "package": "NODE_ENV=development pnpm --filter=@posthog/ai-research-v4-native run build"
+ },
+ "author": "",
+ "license": "MIT",
+ "devDependencies": {
+ "@types/node": "22.15.17",
+ "typescript": "6.0.3",
+ "undici-types": "^7.3.0"
+ },
+ "files": [
+ "dist",
+ "index.node"
+ ]
+}
diff --git a/rust/ai-research/replay-core-v4-node/src/index.ts b/rust/ai-research/replay-core-v4-node/src/index.ts
new file mode 100644
index 000000000000..411264409ac5
--- /dev/null
+++ b/rust/ai-research/replay-core-v4-node/src/index.ts
@@ -0,0 +1,76 @@
+/* eslint-disable @typescript-eslint/no-var-requires */
+// The native addon is built from `src/lib.rs` and copied to `index.node` at the package root.
+const native = require('../index.node')
+
+/**
+ * The politeness unit for a host: the registrable domain, or the host itself when it has none.
+ *
+ * The fetch lane rate limits by this value and the fetch topic keys on it, so both must get the
+ * same answer from one public suffix list, which is why the value comes from the Rust crate. The
+ * private section of that list keeps `user.github.io` and `d111.cloudfront.net` out of a shared
+ * budget with the other tenants of the same provider.
+ *
+ * An IP literal has no registrable domain and comes back unchanged, because the address is the
+ * operator.
+ */
+export function politenessKey(host: string): string {
+ return native.politenessKey(host)
+}
+
+/**
+ * Whether the fetch lane may send a request to a host.
+ *
+ * The recorder applies this rule before a URL reaches the topic. A redirect target has not been
+ * through it, so the fetcher calls the same function rather than deriving a second answer.
+ *
+ * It refuses a private or reserved address, a single-label name, and a name under a suffix that
+ * resolves only inside one network, which is the split-horizon DNS case.
+ */
+export function isPublicHost(host: string): boolean {
+ return native.isPublicHost(host)
+}
+
+export interface CanonicalUrl {
+ fetch: string
+ dedup: string
+ host: string
+ domain: string
+}
+
+/** The labels of the Rust `Decline` enum. The fetch lane reports them as metric reasons. */
+export type UrlPolicyDecline =
+ | 'too_long'
+ | 'not_absolute'
+ | 'bad_scheme'
+ | 'bad_port'
+ | 'no_host'
+ | 'non_public_host'
+ | 'credential'
+ | 'invalid_query'
+ | 'tracking_beacon'
+
+/**
+ * The canonical forms of a URL the policy accepts, or the rule that refused it. `unwanted` is true
+ * when the URL is well formed and safe but nobody wants it fetched, so a queue consumer drops only
+ * that job instead of rejecting the record that carries it.
+ */
+export type UrlPolicyVerdict =
+ | { ok: true; url: CanonicalUrl }
+ | { ok: false; decline: UrlPolicyDecline; unwanted: boolean }
+
+export function tryCanonicalizeUrl(url: string): UrlPolicyVerdict {
+ const result = native.tryCanonicalizeUrl(url)
+ if (typeof result.decline === 'string') {
+ return {
+ ok: false,
+ decline: result.decline,
+ unwanted: result.unwanted === true,
+ }
+ }
+ return { ok: true, url: result }
+}
+
+export function canonicalizeUrl(url: string): CanonicalUrl | null {
+ const verdict = tryCanonicalizeUrl(url)
+ return verdict.ok ? verdict.url : null
+}
diff --git a/rust/ai-research/replay-core-v4-node/src/lib.rs b/rust/ai-research/replay-core-v4-node/src/lib.rs
new file mode 100644
index 000000000000..e82ba426e18a
--- /dev/null
+++ b/rust/ai-research/replay-core-v4-node/src/lib.rs
@@ -0,0 +1,57 @@
+//! Neon native addon exposing the v4 URL policy, `replay_core_v4::url_policy`, to the v4 image
+//! fetcher.
+//!
+//! The recorder collects URLs with that module, and the fetcher must refuse and canonicalize them
+//! the same way, so the fetcher calls the module rather than a second implementation. This crate is
+//! glue only: each function converts its arguments and its result, and the rules stay in the core
+//! crate.
+
+use neon::prelude::*;
+use replay_core_v4::url_policy::{is_public_host, politeness_key, try_canonicalize};
+
+/// The registrable domain of a host.
+fn politeness_key_ffi(mut cx: FunctionContext) -> JsResult<JsString> {
+ let host = cx.argument::<JsString>(0)?.value(&mut cx);
+ Ok(cx.string(politeness_key(&host)))
+}
+
+/// Whether the fetch lane may send a request to a host.
+fn is_public_host_ffi(mut cx: FunctionContext) -> JsResult<JsBoolean> {
+ let host = cx.argument::<JsString>(0)?.value(&mut cx);
+ Ok(cx.boolean(is_public_host(&host)))
+}
+
+/// The URL policy's verdict for one URL: its canonical forms, or under `decline` the label of the
+/// rule that refused it. The fetch lane reads the label so that it can drop a beacon job without
+/// rejecting the record that carries it.
+fn try_canonicalize_url_ffi(mut cx: FunctionContext) -> JsResult<JsObject> {
+ let raw = cx.argument::<JsString>(0)?.value(&mut cx);
+ let result = cx.empty_object();
+ match try_canonicalize(&raw) {
+ Ok(canonical) => {
+ let fetch = cx.string(canonical.fetch);
+ result.set(&mut cx, "fetch", fetch)?;
+ let dedup = cx.string(canonical.dedup);
+ result.set(&mut cx, "dedup", dedup)?;
+ let host = cx.string(canonical.host);
+ result.set(&mut cx, "host", host)?;
+ let domain = cx.string(canonical.domain);
+ result.set(&mut cx, "domain", domain)?;
+ }
+ Err(decline) => {
+ let label = cx.string(decline.label());
+ result.set(&mut cx, "decline", label)?;
+ let unwanted = cx.boolean(decline.is_unwanted());
+ result.set(&mut cx, "unwanted", unwanted)?;
+ }
+ }
+ Ok(result)
+}
+
+#[neon::main]
+fn main(mut cx: ModuleContext) -> NeonResult<()> {
+ cx.export_function("politenessKey", politeness_key_ffi)?;
+ cx.export_function("isPublicHost", is_public_host_ffi)?;
+ cx.export_function("tryCanonicalizeUrl", try_canonicalize_url_ffi)?;
+ Ok(())
+}
diff --git a/rust/ai-research/replay-core-v4-node/tsconfig.json b/rust/ai-research/replay-core-v4-node/tsconfig.json
new file mode 100644
index 000000000000..d6d504ce92ae
--- /dev/null
+++ b/rust/ai-research/replay-core-v4-node/tsconfig.json
@@ -0,0 +1,22 @@
+{
+ "compilerOptions": {
+ "module": "nodenext",
+ "target": "esnext",
+ "declaration": true,
+ "removeComments": true,
+ "moduleResolution": "nodenext",
+ "esModuleInterop": true,
+ "allowJs": true,
+ "sourceMap": true,
+ "rootDir": "src/",
+ "outDir": "dist/",
+ "types": ["node"],
+ "resolveJsonModule": true,
+ "strict": true,
+ "noImplicitAny": true,
+ "useUnknownInCatchVariables": false,
+ "skipLibCheck": true
+ },
+ "include": ["src"],
+ "exclude": ["node_modules", "dist", "bin"]
+}
diff --git a/rust/ai-research/replay-core-v4-node/turbo.json b/rust/ai-research/replay-core-v4-node/turbo.json
new file mode 100644
index 000000000000..23f42c5fe207
--- /dev/null
+++ b/rust/ai-research/replay-core-v4-node/turbo.json
@@ -0,0 +1,27 @@
+{
+ "$schema": "https://turbo.build/schema.json",
+ "extends": ["//"],
+ "tasks": {
+ "build": {
+ "inputs": [
+ "src",
+ "Cargo.toml",
+ "$TURBO_ROOT$/rust/ai-research/replay-core-v4/src/**",
+ "$TURBO_ROOT$/rust/ai-research/replay-core-v4/Cargo.toml",
+ "$TURBO_ROOT$/rust/Cargo.toml",
+ "$TURBO_ROOT$/rust/Cargo.lock"
+ ]
+ },
+ "test": {
+ "inputs": [
+ "src",
+ "Cargo.toml",
+ "$TURBO_ROOT$/rust/ai-research/replay-core-v4/src/**",
+ "$TURBO_ROOT$/rust/ai-research/replay-core-v4/tests/**",
+ "$TURBO_ROOT$/rust/ai-research/replay-core-v4/Cargo.toml",
+ "$TURBO_ROOT$/rust/Cargo.toml",
+ "$TURBO_ROOT$/rust/Cargo.lock"
+ ]
+ }
+ }
+}