:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(ai-research): add the v4 native addon for the url policy (#117979)
Nicholas Waltz82722687d06a
diff --git a/.github/scripts/trunk-impacted-targets.js b/.github/scripts/trunk-impacted-targets.jsindex be7435045b51..64338571ef49 100644--- a/.github/scripts/trunk-impacted-targets.js+++ b/.github/scripts/trunk-impacted-targets.js@@ -1501,8 +1501,8 @@ const rustCrate = (crate) => `rust:crate:${crate}`const RUST_DETERMINATOR = 'rust:determinator'// The lanes that import a native module built from the cargo workspace.-// nodejs/package.json is the only dependent of the two binding packages-// (@posthog/hogvm-node, @posthog/replay-anonymizer) today,+// nodejs/package.json is the only dependent of the three binding packages+// (@posthog/hogvm-node, @posthog/replay-anonymizer, @posthog/ai-research-v4-native) today,// and the test suite re-derives that from pnpm-workspace.yaml so a second// dependent fails there rather than silently going unclaimed here. The// image-scrub sidecar under nodejs/src/ingestion loads the replay-anonymizerdiff --git a/.github/workflows/ci-nodejs-container.yml b/.github/workflows/ci-nodejs-container.ymlindex 217953cb4ccb..b1f6b6f81c83 100644--- a/.github/workflows/ci-nodejs-container.yml+++ b/.github/workflows/ci-nodejs-container.yml@@ -14,6 +14,8 @@ on:- 'rust/common/hogvm/**'- 'rust/replay-anonymizer/**'- 'rust/replay-anonymizer-node/**'+ - 'rust/ai-research/replay-core-v4/**'+ - 'rust/ai-research/replay-core-v4-node/**'- 'Dockerfile.node'- '.github/workflows/ci-nodejs-container.yml'- 'turbo.json'@@ -72,6 +74,8 @@ jobs:- 'rust/common/hogvm/**'- 'rust/replay-anonymizer/**'- 'rust/replay-anonymizer-node/**'+ - 'rust/ai-research/replay-core-v4/**'+ - 'rust/ai-research/replay-core-v4-node/**'- 'Dockerfile.node'- '.github/workflows/ci-nodejs-container.yml'- 'turbo.json'@@ -94,6 +98,8 @@ jobs:- 'nodejs/package.json'- 'rust/replay-anonymizer/**'- 'rust/replay-anonymizer-node/**'+ - 'rust/ai-research/replay-core-v4/**'+ - 'rust/ai-research/replay-core-v4-node/**'- 'Dockerfile.node'- '.github/workflows/ci-nodejs-container.yml'- 'turbo.json'diff --git a/.github/workflows/ci-nodejs.yml b/.github/workflows/ci-nodejs.ymlindex a946a3409009..006ed2159f2a 100644--- a/.github/workflows/ci-nodejs.yml+++ b/.github/workflows/ci-nodejs.yml@@ -117,6 +117,9 @@ jobs:# its Rust tests; a rust-side change (incl. the shared fixtures) must run it.- 'rust/replay-anonymizer/**'- 'rust/replay-anonymizer-node/**'+ # The v4 image fetcher's Jest tests load the URL policy through this addon.+ - 'rust/ai-research/replay-core-v4/**'+ - 'rust/ai-research/replay-core-v4-node/**'# These native addons are built by Turbo before Node.js tests. Their Rust# dependencies are not part of the ingestion consumer's targeted e2e set.- 'rust/common/hogvm/**'diff --git a/.github/workflows/ci-rust.yml b/.github/workflows/ci-rust.ymlindex bff89b5f3da7..de4578854e38 100644--- a/.github/workflows/ci-rust.yml+++ b/.github/workflows/ci-rust.yml@@ -220,6 +220,7 @@ jobs:"property-vals-rs": 70,"replay-anonymizer-node": 5,"replay-core-v4": 10,+ "replay-core-v4-node": 5,"replay-recorder-v4": 15,"serve-metrics": 5,"usage-ingestion": 70,diff --git a/Dockerfile.node b/Dockerfile.nodeindex b26f1183c839..5d5fe54065dc 100644--- a/Dockerfile.node+++ b/Dockerfile.node@@ -77,6 +77,9 @@ RUN NODE_OPTIONS="--max-old-space-size=16384" bin/turbo --filter=@posthog/hogvm-# Build the session-replay anonymizer native addonRUN NODE_OPTIONS="--max-old-space-size=16384" bin/turbo --filter=@posthog/replay-anonymizer build+# Build the v4 AI research native addon (the URL policy of the v4 image fetcher)+RUN NODE_OPTIONS="--max-old-space-size=16384" bin/turbo --filter=@posthog/ai-research-v4-native build+# Build replay-headless (dependency of nodejs rasterizer code)RUN NODE_OPTIONS="--max-old-space-size=16384" bin/turbo --filter=@posthog/replay-headless build# Then build the nodejs services@@ -134,6 +137,9 @@ COPY --from=nodejs-build --chown=posthog:posthog /code/rust/common/hogvm/node/inCOPY --from=nodejs-build --chown=posthog:posthog /code/rust/replay-anonymizer-node/dist /code/rust/replay-anonymizer-node/distCOPY --from=nodejs-build --chown=posthog:posthog /code/rust/replay-anonymizer-node/package.json /code/rust/replay-anonymizer-node/package.jsonCOPY --from=nodejs-build --chown=posthog:posthog /code/rust/replay-anonymizer-node/index.node /code/rust/replay-anonymizer-node/index.node+COPY --from=nodejs-build --chown=posthog:posthog /code/rust/ai-research/replay-core-v4-node/dist /code/rust/ai-research/replay-core-v4-node/dist+COPY --from=nodejs-build --chown=posthog:posthog /code/rust/ai-research/replay-core-v4-node/package.json /code/rust/ai-research/replay-core-v4-node/package.json+COPY --from=nodejs-build --chown=posthog:posthog /code/rust/ai-research/replay-core-v4-node/index.node /code/rust/ai-research/replay-core-v4-node/index.nodeCOPY --from=nodejs-build --chown=posthog:posthog /code/common/plugin_transpiler/dist /code/common/plugin_transpiler/distCOPY --from=nodejs-build --chown=posthog:posthog /code/common/plugin_transpiler/node_modules /code/common/plugin_transpiler/node_modulesCOPY --from=nodejs-build --chown=posthog:posthog /code/common/plugin_transpiler/package.json /code/common/plugin_transpiler/package.jsondiff --git a/nodejs/package.json b/nodejs/package.jsonindex d390250255f1..c53880215676 100644--- a/nodejs/package.json+++ b/nodejs/package.json@@ -49,6 +49,8 @@"build:hogvm-node:dev": "pnpm --filter=@posthog/hogvm-node run build:dev","build:replay-anonymizer": "pnpm --filter=@posthog/replay-anonymizer package","build:replay-anonymizer:dev": "pnpm --filter=@posthog/replay-anonymizer run build:dev",+ "build:ai-research-v4-native": "pnpm --filter=@posthog/ai-research-v4-native package",+ "build:ai-research-v4-native:dev": "pnpm --filter=@posthog/ai-research-v4-native run build:dev","generate:personhog-proto": "cd ../proto && buf generate --template ../nodejs/buf.gen.yaml --path personhog/service --path personhog/identity --path personhog/lifecycle --path personhog/types . && cd ../nodejs && eslint --fix src/common/generated/personhog && oxlint -c .oxlintrc.nodejs.json --fix src/common/generated/personhog && prettier --write src/common/generated/personhog","generate:ingestion-worker-proto": "cd ../proto && buf generate --template ../nodejs/buf.gen.ingestion-worker.yaml --path ingestion/worker . && cd ../nodejs && eslint --fix src/common/generated/ingestion-worker && oxlint -c .oxlintrc.nodejs.json --fix src/common/generated/ingestion-worker && prettier --write src/common/generated/ingestion-worker","generate:usage-ingestion-proto": "cd ../proto && buf generate --template ../nodejs/buf.gen.usage-ingestion.yaml --path usage_ingestion . && cd ../nodejs && eslint --fix src/common/generated/usage-ingestion && oxlint -c .oxlintrc.nodejs.json --fix src/common/generated/usage-ingestion && prettier --write src/common/generated/usage-ingestion",@@ -91,6 +93,7 @@"@opentelemetry/sdk-node": "^0.218.0","@opentelemetry/sdk-trace-node": "^2.0.1","@opentelemetry/semantic-conventions": "^1.36.0",+ "@posthog/ai-research-v4-native": "workspace:*","@posthog/hogvm": "workspace:*","@posthog/hogvm-node": "workspace:*","@posthog/replay-anonymizer": "workspace:*",diff --git a/pnpm-lock.yaml b/pnpm-lock.yamlindex 8ff82bfb122d..7cd47f4ff8f4 100644--- a/pnpm-lock.yaml+++ b/pnpm-lock.yaml@@ -1622,6 +1622,9 @@ importers:'@opentelemetry/semantic-conventions':specifier: ^1.36.0version: 1.36.0+ '@posthog/ai-research-v4-native':+ specifier: workspace:*+ version: link:../rust/ai-research/replay-core-v4-node'@posthog/hogvm':specifier: workspace:*version: link:../common/hogvm/typescript@@ -5720,6 +5723,18 @@ importers:specifier: 'catalog:'version: 0.2.4([email protected](patch_hash=139b8d1f1304f9d9da452a9a1244c94ea679dbcb85687d8999563146879fb6f5)([email protected]))+ rust/ai-research/replay-core-v4-node:+ devDependencies:+ '@types/node':+ specifier: 22.15.17+ version: 22.15.17+ typescript:+ specifier: 6.0.3+ version: 6.0.3+ undici-types:+ specifier: ^7.3.0+ version: 7.3.0+rust/common/hogvm/node: {}rust/replay-anonymizer-node:diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yamlindex 74540d5c9d23..0db74b2cd4d2 100644--- a/pnpm-workspace.yaml+++ b/pnpm-workspace.yaml@@ -30,6 +30,7 @@ packages:- '!tools/hedgebox-dummy'- rust/common/hogvm/node- rust/replay-anonymizer-node+ - rust/ai-research/replay-core-v4-nodeblockExoticSubdeps: truetrustPolicy: no-downgradediff --git a/rust/Cargo.lock b/rust/Cargo.lockindex f0aef9f5fc85..22b923751bc7 100644--- a/rust/Cargo.lock+++ b/rust/Cargo.lock@@ -11047,6 +11047,14 @@ dependencies = ["zstd 0.13.3",]+[[package]]+name = "replay-core-v4-node"+version = "0.0.1"+dependencies = [+ "neon",+ "replay-core-v4",+]+[[package]]name = "replay-recorder-v4"version = "0.1.0"diff --git a/rust/Cargo.toml b/rust/Cargo.tomlindex 7aa27ed9ec09..9f8b436c55ae 100644--- a/rust/Cargo.toml+++ b/rust/Cargo.toml@@ -80,6 +80,7 @@ members = ["pgcollector","pgapi","ai-research/replay-core-v4",+ "ai-research/replay-core-v4-node","ai-research/replay-recorder-v4",]diff --git a/rust/ai-research/replay-core-v4-node/.gitignore b/rust/ai-research/replay-core-v4-node/.gitignorenew file mode 100644index 000000000000..d15049d90e52--- /dev/null+++ b/rust/ai-research/replay-core-v4-node/.gitignore@@ -0,0 +1,3 @@+index.node+cargo.log+dist/diff --git a/rust/ai-research/replay-core-v4-node/Cargo.toml b/rust/ai-research/replay-core-v4-node/Cargo.tomlnew file mode 100644index 000000000000..69fe000d480f--- /dev/null+++ b/rust/ai-research/replay-core-v4-node/Cargo.toml@@ -0,0 +1,17 @@+[package]+name = "replay-core-v4-node"+version = "0.0.1"+edition = "2021"+exclude = ["index.node"]+publish = false++[lints]+workspace = true++[lib]+crate-type = ["cdylib"]++[dependencies]+neon = { workspace = true }+# default-features = false: the key table and KMS clients have no place in an addon that only checks URLs.+replay-core-v4 = { path = "../replay-core-v4", default-features = false }diff --git a/rust/ai-research/replay-core-v4-node/package.json b/rust/ai-research/replay-core-v4-node/package.jsonnew file mode 100644index 000000000000..b11dd60d0be8--- /dev/null+++ b/rust/ai-research/replay-core-v4-node/package.json@@ -0,0 +1,31 @@+{+ "name": "@posthog/ai-research-v4-native",+ "version": "0.0.1",+ "description": "Node bindings for the v4 AI research replay URL policy",+ "main": "dist/index.js",+ "types": "dist/index.d.ts",+ "scripts": {+ "test": "cargo test -p replay-core-v4 -p replay-core-v4-node",+ "build": "pnpm run build:cargo --release && pnpm run build:move-lib && pnpm run build:typescript",+ "build:dev": "pnpm run build:cargo && pnpm run build:move-lib:dev && pnpm run build:typescript",+ "build:move-lib": "TARGET_DIR=${CARGO_TARGET_DIR:-../../target} && cp \"$TARGET_DIR/release/libreplay_core_v4_node.dylib\" index.node 2>/dev/null || cp \"$TARGET_DIR/release/libreplay_core_v4_node.so\" index.node || (echo \"No valid library file found in $TARGET_DIR/release/\" && exit 1)",+ "build:move-lib:dev": "TARGET_DIR=${CARGO_TARGET_DIR:-../../target} && cp \"$TARGET_DIR/debug/libreplay_core_v4_node.dylib\" index.node 2>/dev/null || cp \"$TARGET_DIR/debug/libreplay_core_v4_node.so\" index.node || (echo \"No valid library file found in $TARGET_DIR/debug/\" && exit 1)",+ "build:cargo": "cargo build -p replay-core-v4-node --message-format=json > cargo.log",+ "build:cargo:debug": "pnpm run build:cargo",+ "build:typescript": "tsc",+ "typecheck": "tsc --noEmit",+ "clean": "rm -rf index.node dist",+ "package": "NODE_ENV=development pnpm --filter=@posthog/ai-research-v4-native run build"+ },+ "author": "",+ "license": "MIT",+ "devDependencies": {+ "@types/node": "22.15.17",+ "typescript": "6.0.3",+ "undici-types": "^7.3.0"+ },+ "files": [+ "dist",+ "index.node"+ ]+}diff --git a/rust/ai-research/replay-core-v4-node/src/index.ts b/rust/ai-research/replay-core-v4-node/src/index.tsnew file mode 100644index 000000000000..411264409ac5--- /dev/null+++ b/rust/ai-research/replay-core-v4-node/src/index.ts@@ -0,0 +1,76 @@+/* eslint-disable @typescript-eslint/no-var-requires */+// The native addon is built from `src/lib.rs` and copied to `index.node` at the package root.+const native = require('../index.node')++/**+ * The politeness unit for a host: the registrable domain, or the host itself when it has none.+ *+ * The fetch lane rate limits by this value and the fetch topic keys on it, so both must get the+ * same answer from one public suffix list, which is why the value comes from the Rust crate. The+ * private section of that list keeps `user.github.io` and `d111.cloudfront.net` out of a shared+ * budget with the other tenants of the same provider.+ *+ * An IP literal has no registrable domain and comes back unchanged, because the address is the+ * operator.+ */+export function politenessKey(host: string): string {+ return native.politenessKey(host)+}++/**+ * Whether the fetch lane may send a request to a host.+ *+ * The recorder applies this rule before a URL reaches the topic. A redirect target has not been+ * through it, so the fetcher calls the same function rather than deriving a second answer.+ *+ * It refuses a private or reserved address, a single-label name, and a name under a suffix that+ * resolves only inside one network, which is the split-horizon DNS case.+ */+export function isPublicHost(host: string): boolean {+ return native.isPublicHost(host)+}++export interface CanonicalUrl {+ fetch: string+ dedup: string+ host: string+ domain: string+}++/** The labels of the Rust `Decline` enum. The fetch lane reports them as metric reasons. */+export type UrlPolicyDecline =+ | 'too_long'+ | 'not_absolute'+ | 'bad_scheme'+ | 'bad_port'+ | 'no_host'+ | 'non_public_host'+ | 'credential'+ | 'invalid_query'+ | 'tracking_beacon'++/**+ * The canonical forms of a URL the policy accepts, or the rule that refused it. `unwanted` is true+ * when the URL is well formed and safe but nobody wants it fetched, so a queue consumer drops only+ * that job instead of rejecting the record that carries it.+ */+export type UrlPolicyVerdict =+ | { ok: true; url: CanonicalUrl }+ | { ok: false; decline: UrlPolicyDecline; unwanted: boolean }++export function tryCanonicalizeUrl(url: string): UrlPolicyVerdict {+ const result = native.tryCanonicalizeUrl(url)+ if (typeof result.decline === 'string') {+ return {+ ok: false,+ decline: result.decline,+ unwanted: result.unwanted === true,+ }+ }+ return { ok: true, url: result }+}++export function canonicalizeUrl(url: string): CanonicalUrl | null {+ const verdict = tryCanonicalizeUrl(url)+ return verdict.ok ? verdict.url : null+}diff --git a/rust/ai-research/replay-core-v4-node/src/lib.rs b/rust/ai-research/replay-core-v4-node/src/lib.rsnew file mode 100644index 000000000000..e82ba426e18a--- /dev/null+++ b/rust/ai-research/replay-core-v4-node/src/lib.rs@@ -0,0 +1,57 @@+//! Neon native addon exposing the v4 URL policy, `replay_core_v4::url_policy`, to the v4 image+//! fetcher.+//!+//! The recorder collects URLs with that module, and the fetcher must refuse and canonicalize them+//! the same way, so the fetcher calls the module rather than a second implementation. This crate is+//! glue only: each function converts its arguments and its result, and the rules stay in the core+//! crate.++use neon::prelude::*;+use replay_core_v4::url_policy::{is_public_host, politeness_key, try_canonicalize};++/// The registrable domain of a host.+fn politeness_key_ffi(mut cx: FunctionContext) -> JsResult<JsString> {+ let host = cx.argument::<JsString>(0)?.value(&mut cx);+ Ok(cx.string(politeness_key(&host)))+}++/// Whether the fetch lane may send a request to a host.+fn is_public_host_ffi(mut cx: FunctionContext) -> JsResult<JsBoolean> {+ let host = cx.argument::<JsString>(0)?.value(&mut cx);+ Ok(cx.boolean(is_public_host(&host)))+}++/// The URL policy's verdict for one URL: its canonical forms, or under `decline` the label of the+/// rule that refused it. The fetch lane reads the label so that it can drop a beacon job without+/// rejecting the record that carries it.+fn try_canonicalize_url_ffi(mut cx: FunctionContext) -> JsResult<JsObject> {+ let raw = cx.argument::<JsString>(0)?.value(&mut cx);+ let result = cx.empty_object();+ match try_canonicalize(&raw) {+ Ok(canonical) => {+ let fetch = cx.string(canonical.fetch);+ result.set(&mut cx, "fetch", fetch)?;+ let dedup = cx.string(canonical.dedup);+ result.set(&mut cx, "dedup", dedup)?;+ let host = cx.string(canonical.host);+ result.set(&mut cx, "host", host)?;+ let domain = cx.string(canonical.domain);+ result.set(&mut cx, "domain", domain)?;+ }+ Err(decline) => {+ let label = cx.string(decline.label());+ result.set(&mut cx, "decline", label)?;+ let unwanted = cx.boolean(decline.is_unwanted());+ result.set(&mut cx, "unwanted", unwanted)?;+ }+ }+ Ok(result)+}++#[neon::main]+fn main(mut cx: ModuleContext) -> NeonResult<()> {+ cx.export_function("politenessKey", politeness_key_ffi)?;+ cx.export_function("isPublicHost", is_public_host_ffi)?;+ cx.export_function("tryCanonicalizeUrl", try_canonicalize_url_ffi)?;+ Ok(())+}diff --git a/rust/ai-research/replay-core-v4-node/tsconfig.json b/rust/ai-research/replay-core-v4-node/tsconfig.jsonnew file mode 100644index 000000000000..d6d504ce92ae--- /dev/null+++ b/rust/ai-research/replay-core-v4-node/tsconfig.json@@ -0,0 +1,22 @@+{+ "compilerOptions": {+ "module": "nodenext",+ "target": "esnext",+ "declaration": true,+ "removeComments": true,+ "moduleResolution": "nodenext",+ "esModuleInterop": true,+ "allowJs": true,+ "sourceMap": true,+ "rootDir": "src/",+ "outDir": "dist/",+ "types": ["node"],+ "resolveJsonModule": true,+ "strict": true,+ "noImplicitAny": true,+ "useUnknownInCatchVariables": false,+ "skipLibCheck": true+ },+ "include": ["src"],+ "exclude": ["node_modules", "dist", "bin"]+}diff --git a/rust/ai-research/replay-core-v4-node/turbo.json b/rust/ai-research/replay-core-v4-node/turbo.jsonnew file mode 100644index 000000000000..23f42c5fe207--- /dev/null+++ b/rust/ai-research/replay-core-v4-node/turbo.json@@ -0,0 +1,27 @@+{+ "$schema": "https://turbo.build/schema.json",+ "extends": ["//"],+ "tasks": {+ "build": {+ "inputs": [+ "src",+ "Cargo.toml",+ "$TURBO_ROOT$/rust/ai-research/replay-core-v4/src/**",+ "$TURBO_ROOT$/rust/ai-research/replay-core-v4/Cargo.toml",+ "$TURBO_ROOT$/rust/Cargo.toml",+ "$TURBO_ROOT$/rust/Cargo.lock"+ ]+ },+ "test": {+ "inputs": [+ "src",+ "Cargo.toml",+ "$TURBO_ROOT$/rust/ai-research/replay-core-v4/src/**",+ "$TURBO_ROOT$/rust/ai-research/replay-core-v4/tests/**",+ "$TURBO_ROOT$/rust/ai-research/replay-core-v4/Cargo.toml",+ "$TURBO_ROOT$/rust/Cargo.toml",+ "$TURBO_ROOT$/rust/Cargo.lock"+ ]+ }+ }+}