:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
refactor(ai-research-v4): seal fetched images and parse their refs in Rust (#128002)
Robbiefcb21c895f5c
diff --git a/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/collected-urls-record.ts b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/collected-urls-record.tsindex 18a8eb77ed63..c18e6a513567 100644--- a/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/collected-urls-record.ts+++ b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/collected-urls-record.ts@@ -3,8 +3,7 @@ import type { UrlPolicyDecline } from '@posthog/ai-research-v4-native'import { parseJSON } from '~/common/utils/json-parse'import { ImageFetchBlockReason, isImageFetchBlockReason } from './block-reason'-import { tryCanonicalizeUrl } from './politeness-key'-import { parseV4UrlRef } from './v4-url-ref'+import { parseV4UrlRef, tryCanonicalizeUrl } from './v4-native'export const MAX_HOPS = 10export const MAX_JOBS_PER_RECORD = 1_000diff --git a/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/configuration-policy.ts b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/configuration-policy.tsindex 2cf203e1c1ae..8cb7334511f4 100644--- a/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/configuration-policy.ts+++ b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/configuration-policy.ts@@ -6,9 +6,9 @@ import { fetchStreamed } from '~/common/utils/request'import { ConfigurationCacheItem, ConfigurationFile, HttpCacheMetadata, configurationCacheKey } from './crawl-history'import { ImageFetchRequestMetrics } from './metrics'-import { canonicalizeUrl, politenessKey } from './politeness-key'import { ConfigurationFetchReason, ImageFetchProcessingMetrics } from './processing-metrics'import { BOT_NAME, REQUEST_IDENTITY_HEADERS } from './request-identity'+import { canonicalizeUrl, politenessKey } from './v4-native'import { WebBotAuthRequestSigner } from './web-bot-auth'import { wildcardPatternMatchesPathname } from './wildcard-pattern'diff --git a/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/dynamodb-crawl-history.ts b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/dynamodb-crawl-history.tsindex e65ade75df58..9577879d8e49 100644--- a/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/dynamodb-crawl-history.ts+++ b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/dynamodb-crawl-history.ts@@ -20,7 +20,7 @@ import {HttpCacheMetadata,UrlCrawlHistoryItem,} from './crawl-history'-import { isUrlHistoryValue } from './politeness-key'+import { isUrlHistoryValue } from './v4-native'const KEY_ATTRIBUTE = 'key'const EXPIRES_AT_ATTRIBUTE = 'expires_at'diff --git a/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/fetch-runner.ts b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/fetch-runner.tsindex 8b5a750a912c..166725945ca6 100644--- a/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/fetch-runner.ts+++ b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/fetch-runner.ts@@ -25,11 +25,11 @@ import {import type { RefreshingImageSink } from './image-sink'import { ImageFetchPoolMetrics, ImageFetchRequestMetrics } from './metrics'import { OriginRequestScheduler } from './origin-request-scheduler'-import { canonicalizeUrl } from './politeness-key'import { ImageFetchProcessingMetrics } from './processing-metrics'import type { ResponseUrlBinder } from './response-url-binding'import { ImageFetchTopHogMetrics } from './tophog-metrics'import { urlHistoryExpiresAtMs, urlHistoryStorageExpiresAtMs } from './url-history-expiry'+import { canonicalizeUrl } from './v4-native'export type ShedReason =| 'breaker_open'diff --git a/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/image-fetcher.ts b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/image-fetcher.tsindex aa42c215544b..8995c977ffbc 100644--- a/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/image-fetcher.ts+++ b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/image-fetcher.ts@@ -4,8 +4,8 @@ import type { ImageFetchBlockReason } from './block-reason'import { OriginPolicyReason, ResponseOptOutReason, responseOptOutReason } from './configuration-policy'import { HttpCacheMetadata } from './crawl-history'import { ImageFetchRequestMetrics } from './metrics'-import { canonicalizeUrl } from './politeness-key'import { REQUEST_IDENTITY_HEADERS } from './request-identity'+import { canonicalizeUrl } from './v4-native'import { WebBotAuthRequestSigner } from './web-bot-auth'/**diff --git a/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/job-admission.ts b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/job-admission.tsindex aac5f6d56750..d373e12b816e 100644--- a/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/job-admission.ts+++ b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/job-admission.ts@@ -9,7 +9,7 @@ import {} from '~/ingestion/pipelines/sessionreplay/ai-research/v4-keys/team-month-keys'import type { FetchCandidate } from './collected-urls-record'-import { parseV4UrlRef } from './v4-url-ref'+import { parseV4UrlRef } from './v4-native'/** Each lookup's read budget starts when it starts, so a batch of many new team months must not start them all at once. */const KEY_LOOKUP_CONCURRENCY = 16diff --git a/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/origin-request-scheduler.ts b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/origin-request-scheduler.tsindex d7be78697dee..8f13ce92691c 100644--- a/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/origin-request-scheduler.ts+++ b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/origin-request-scheduler.ts@@ -5,9 +5,9 @@ import type { ImageFetchBlockReason } from './block-reason'import { ConfigurationRequestScheduler } from './configuration-policy'import { BudgetBlockReason, BudgetGrant, HostBudget } from './host-budget'import { ImageFetchRequestMetrics, SchedulerWaitScope } from './metrics'-import { politenessKey } from './politeness-key'import { ImageFetchProcessingMetrics } from './processing-metrics'import { ImageFetchTopHogMetrics } from './tophog-metrics'+import { politenessKey } from './v4-native'export type ScheduledRequest<T> =| { ran: true; value: T }diff --git a/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/raw-image-store.test.ts b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/raw-image-store.test.tsindex 5a576ce88eac..9c0f5e5b94af 100644--- a/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/raw-image-store.test.ts+++ b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/raw-image-store.test.ts@@ -117,22 +117,32 @@ describe('RawImageStore', () => {})it('holds a further image back while the write limit is full, so a slow S3 does not pile up images in memory', async () => {- let finishWrite!: () => void+ const finishWrites: (() => void)[] = []+ let writeStarted!: () => void+ let started = new Promise<void>((resolve) => (writeStarted = resolve))const s3 = {- send: jest.fn(() => new Promise((resolve) => (finishWrite = () => resolve({})))),+ send: jest.fn(+ () =>+ new Promise((resolve) => {+ finishWrites.push(() => resolve({}))+ writeStarted()+ })+ ),}const keyReads = jest.fn(() => Promise.resolve(STORED))const sink = new RawImageStore({ get: keyReads }, s3 as any, 'images-v4', 1)const first = sink.publishImage(candidate(), fetched())const second = sink.publishImage(candidate(), fetched())+ await startedawait new Promise((resolve) => setImmediate(resolve))expect([s3.send.mock.calls.length, keyReads.mock.calls.length]).toEqual([1, 1])- finishWrite()+ started = new Promise<void>((resolve) => (writeStarted = resolve))+ finishWrites[0]()await first- await new Promise((resolve) => setImmediate(resolve))- finishWrite()+ await started+ finishWrites[1]()await secondexpect(s3.send).toHaveBeenCalledTimes(2)})diff --git a/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/raw-image-store.ts b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/raw-image-store.tsindex 666e8dfabd44..3d8fcd5e48dd 100644--- a/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/raw-image-store.ts+++ b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/raw-image-store.ts@@ -2,7 +2,6 @@ import { CopyObjectCommand, PutObjectCommand, S3Client, S3ServiceException } froimport { Counter } from 'prom-client'import { ConcurrencyController } from '~/common/utils/concurrencyController'-import { sealV4Frame } from '~/ingestion/pipelines/sessionreplay/ai-research/v4-keys/frame'import type { TeamMonthKeys } from '~/ingestion/pipelines/sessionreplay/ai-research/v4-keys/team-month-keys'import type { FetchCandidate } from './collected-urls-record'@@ -10,7 +9,7 @@ import type { ImageFetchResult } from './image-fetcher'import type { RefreshingImageSink } from './image-sink'import { InvalidImageTransportError, prepareFetchedImage } from './image-transport'import { ImageFetchProcessingMetrics } from './processing-metrics'-import { parseV4UrlRef } from './v4-url-ref'+import { parseV4UrlRef, sealFetchedImage } from './v4-native'const S3_WRITE_TIMEOUT_MS = 30_000const CONTENT_TYPE = 'application/octet-stream'@@ -121,11 +120,7 @@ export class RawImageStore implements RefreshingImageSink {storedImages.labels(key.status === 'deleted' ? 'deleted' : 'missing_key').inc()return}- const frame = sealV4Frame(- key.key,- { teamId, keyMonth, kind: 'image-url', codec: 'none', ref: candidate.originalRef },- image- )+ const frame = await sealFetchedImage(key.key, candidate.originalRef, image)await this.s3.send(new PutObjectCommand({Bucket: this.bucket,diff --git a/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/response-url-binding.ts b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/response-url-binding.tsindex 684805f851f6..0cf0076f838c 100644--- a/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/response-url-binding.ts+++ b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/response-url-binding.ts@@ -1,16 +1,9 @@-import { createHmac, hkdfSync } from 'node:crypto'-import type {TeamMonthKey,TeamMonthKeys,} from '~/ingestion/pipelines/sessionreplay/ai-research/v4-keys/team-month-keys'-import { parseV4UrlRef } from './v4-url-ref'--/** The HKDF info of `replay_core_v4::seal::image_url_key`, the key behind the hash of an `imageurl:v4` reference. */-const IMAGE_URL_KEY_INFO = 'ml-image-url-key'-/** The length `replay_core_v4::refs::reference_hash` truncates a hash to. */-const HASH_CHARS = 22+import { imageUrlHash, parseV4UrlRef } from './v4-native'/** The keyed hash that the crawl history stores in place of a response URL. */export type ResponseUrlBinding = (url: string) => string@@ -42,7 +35,7 @@ export class TeamMonthResponseUrlBinder implements ResponseUrlBinder {if (key.status !== 'ok') {return undefined}- const urlKey = Buffer.from(hkdfSync('sha256', key.key, Buffer.alloc(0), IMAGE_URL_KEY_INFO, 32))- return (url) => createHmac('sha256', urlKey).update(url).digest('base64url').slice(0, HASH_CHARS)+ const teamMonthKey = key.key+ return (url) => imageUrlHash(teamMonthKey, url)}}diff --git a/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/url-fetch-consumer.ts b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/url-fetch-consumer.tsindex 18ad7bd24c1d..910572be3114 100644--- a/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/url-fetch-consumer.ts+++ b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/url-fetch-consumer.ts@@ -28,10 +28,10 @@ import { FrontierDeadLetterReason, FrontierDeadLetterSink } from './frontier-deaimport { FrontierPublisher, RepublishBatch } from './frontier-publisher'import type { RefreshingImageSink } from './image-sink'import { ImageFetchConsumerMetrics, ImageFetchRequestMetrics } from './metrics'-import { urlJob } from './politeness-key'import { ImageFetchProcessingMetrics } from './processing-metrics'import { ImageFetchTopHogMetrics } from './tophog-metrics'import { urlHistoryExpiresAtMs, urlHistoryStorageExpiresAtMs } from './url-history-expiry'+import { urlJob } from './v4-native'const ONE_HOUR_MS = 60 * 60 * 1000const REPUBLISH_DEADLINE_FROM_BATCH_START_MS = 200_000diff --git a/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/url-history-expiry.ts b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/url-history-expiry.tsindex ab3a6b3d186c..f2e01a3ae899 100644--- a/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/url-history-expiry.ts+++ b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/url-history-expiry.ts@@ -1,6 +1,6 @@import { explicitFreshnessLifetimeMs } from './configuration-policy'import { HttpCacheMetadata } from './crawl-history'-import { parseV4UrlRef } from './v4-url-ref'+import { parseV4UrlRef } from './v4-native'/** A reference is fetched again after the crawl history TTL, or later when the response stays fresh for longer. */export function urlHistoryExpiresAtMs(nowMs: number, seenTtlSeconds: number, cache?: HttpCacheMetadata): number {diff --git a/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/politeness-key.test.ts b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/v4-native.test.tssimilarity index 62%rename from nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/politeness-key.test.tsrename to nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/v4-native.test.tsindex 3377d4ea9607..93d5e24e3855 100644--- a/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/politeness-key.test.ts+++ b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/v4-native.test.ts@@ -1,4 +1,4 @@-import { isPublicHost, politenessKey, tryCanonicalizeUrl } from '@posthog/ai-research-v4-native'+import { isPublicHost, parseV4UrlRef, politenessKey, tryCanonicalizeUrl } from '@posthog/ai-research-v4-native'/*** The crate owns the rule and tests it. This file tests only that the fetch lane reaches that@@ -48,20 +48,26 @@ describe('the v4 url policy addon', () => {})})- it('stops the fetch pod at startup when index.node predates a shared rule', () => {- const nativePath = require.resolve('@posthog/ai-research-v4-native/index.node')- jest.isolateModules(() => {- jest.doMock(nativePath, () => {- const {- urlJob: _urlJob,- isUrlHistoryValue: _isUrlHistoryValue,- ...stale- } = jest.requireActual(nativePath)- return stale- })- const { assertUrlPolicyLoaded } = require('./politeness-key')- expect(() => assertUrlPolicyLoaded()).toThrow('rebuild index.node')- })- jest.dontMock(nativePath)+ it('parses only an image URL reference whose team id a number holds exactly', () => {+ const hash = 'a'.repeat(22)+ expect(parseV4UrlRef(`imageurl:v4:7:2026-10:${hash}`)).toEqual({ teamId: 7, keyMonth: '2026-10', hash })+ expect(parseV4UrlRef(`image:v4:7:2026-10:${hash}`)).toBeNull()+ expect(parseV4UrlRef(`imageurl:v4:9007199254740993:2026-10:${hash}`)).toBeNull()})++ it.each(['urlJob', 'isUrlHistoryValue', 'parseV4UrlRef', 'sealFetchedImage', 'imageUrlHash'])(+ 'stops the fetch pod at startup when index.node has no %s',+ (missing) => {+ const nativePath = require.resolve('@posthog/ai-research-v4-native/index.node')+ jest.isolateModules(() => {+ jest.doMock(nativePath, () => {+ const { [missing]: _missing, ...stale } = jest.requireActual(nativePath)+ return stale+ })+ const { assertSharedRulesLoaded } = require('./v4-native')+ expect(() => assertSharedRulesLoaded()).toThrow('rebuild index.node')+ })+ jest.dontMock(nativePath)+ }+ )})diff --git a/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/politeness-key.ts b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/v4-native.tssimilarity index 59%rename from nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/politeness-key.tsrename to nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/v4-native.tsindex 91abeae32ec6..5efa35be2e42 100644--- a/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/politeness-key.ts+++ b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/v4-native.ts@@ -1,7 +1,5 @@/**- * The URL policy and the crawl history rule of the recorder, reached through the v4 native addon- * rather than reimplemented here. One public suffix list keeps producer and consumer- * canonicalization aligned.+ * The rules this lane shares with the recorder, reached through the v4 native addon rather than reimplemented here.** The addon is a native library, and `index.ts` imports every server whatever mode the pod runs.* This loads the addon on first use rather than at import, so only a pod in this lane pays for it.@@ -17,7 +15,10 @@ function addon(): typeof import('@posthog/ai-research-v4-native') {typeof loaded.isPublicHost !== 'function' ||typeof loaded.tryCanonicalizeUrl !== 'function' ||typeof loaded.urlJob !== 'function' ||- typeof loaded.isUrlHistoryValue !== 'function'+ typeof loaded.isUrlHistoryValue !== 'function' ||+ typeof loaded.parseV4UrlRef !== 'function' ||+ typeof loaded.sealFetchedImage !== 'function' ||+ typeof loaded.imageUrlHash !== 'function') {throw new Error('the ai-research-v4-native addon is missing a shared rule: rebuild index.node')}@@ -26,27 +27,33 @@ function addon(): typeof import('@posthog/ai-research-v4-native') {return cached}-/**- * Load the addon now and run one URL and one crawl history entry through it, so a stale `index.node` stops the pod at- * startup.- *- * The parser calls into it for every URL, and the crawl history reader for every URL row. A stale addon must stop the- * pod before it reads a batch.- */-export function assertUrlPolicyLoaded(): void {+/** Runs a probe through each shared rule, so a stale `index.node` stops the pod at startup rather than in its first batch. */+export function assertSharedRulesLoaded(): void {const verdict = addon().tryCanonicalizeUrl('https://example.com/probe.png')if (!verdict.ok) {throw new Error(`the ai-research-v4-native addon refused a plain URL (${verdict.decline}): rebuild index.node`)}- if (!readsCrawlHistory()) {- throw new Error('the ai-research-v4-native addon misread a crawl history entry: rebuild index.node')+ if (!runsSharedRules()) {+ throw new Error('the ai-research-v4-native addon failed a probe: rebuild index.node')}}-function readsCrawlHistory(): boolean {+const PROBE_REF = 'imageurl:v4:1:2026-01:AAAAAAAAAAAAAAAAAAAAAA'++function runsSharedRules(): boolean {const entry = JSON.stringify({ kind: 'url', key: 'probe', nextFetchAtMs: 1, storageExpiresAtMs: 1, outcome: 'ok' })+ const key = Buffer.alloc(32)try {- return addon().urlJob(1, undefined, 0) === 'drop' && addon().isUrlHistoryValue('probe', entry)+ // A stale index.node has no export, so the wrapper throws before it returns the promise.+ void addon()+ .sealFetchedImage(key, PROBE_REF, Buffer.alloc(0))+ .catch(() => undefined)+ return (+ addon().urlJob(1, undefined, 0) === 'drop' &&+ addon().isUrlHistoryValue('probe', entry) &&+ addon().parseV4UrlRef(PROBE_REF)?.teamId === 1 &&+ addon().imageUrlHash(key, 'https://example.com/probe.png').length === 22+ )} catch {return false}@@ -86,3 +93,15 @@ export function urlJob(export function isUrlHistoryValue(key: string, value: string): boolean {return addon().isUrlHistoryValue(key, value)}++export function parseV4UrlRef(reference: string): { teamId: number; keyMonth: string; hash: string } | null {+ return addon().parseV4UrlRef(reference)+}++export function sealFetchedImage(teamMonthKey: Buffer, reference: string, image: Buffer): Promise<Buffer> {+ return addon().sealFetchedImage(teamMonthKey, reference, image)+}++export function imageUrlHash(teamMonthKey: Buffer, url: string): string {+ return addon().imageUrlHash(teamMonthKey, url)+}diff --git a/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/v4-url-ref.test.ts b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/v4-url-ref.test.tsdeleted file mode 100644index f51c2c2bb368..000000000000--- a/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/v4-url-ref.test.ts+++ /dev/null@@ -1,23 +0,0 @@-import { parseImageRef } from '~/ingestion/pipelines/sessionreplay/ml-mirror-image-scrub/content-ref'--import { parseV4UrlRef } from './v4-url-ref'--describe('parseV4UrlRef', () => {- const hash = 'a'.repeat(22)-- it('parses a month-scoped v4 URL ref that no v2 or v3 reader takes', () => {- const ref = `imageurl:v4:7:2026-10:${hash}`- expect(parseV4UrlRef(ref)).toEqual({ teamId: 7, keyMonth: '2026-10', hash })- expect(parseImageRef(ref)).toBeNull()- })-- it.each([- ['a v3 ref', `imageurl:v3:7:2026-10:${hash}`],- ['a content ref', `image:v4:7:2026-10:${hash}`],- ['a month that does not exist', `imageurl:v4:7:2026-13:${hash}`],- ['a day-scoped ref', `imageurl:v4:7:2026-10-15:${hash}`],- ['a team id of zero', `imageurl:v4:0:2026-10:${hash}`],- ])('refuses %s', (_name, ref) => {- expect(parseV4UrlRef(ref)).toBeNull()- })-})diff --git a/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/v4-url-ref.ts b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/v4-url-ref.tsdeleted file mode 100644index 059f79f02b38..000000000000--- a/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/v4-url-ref.ts+++ /dev/null@@ -1,14 +0,0 @@-/** Image URL references of the v4 dataset. */--function isRawTeamId(value: unknown): boolean {- return typeof value === 'string' && /^[1-9][0-9]{0,15}$/.test(value) && Number.isSafeInteger(Number(value))-}--/** Parses `imageurl:v4:{team}:{YYYY-MM}:{hash}`. `parseImageRef` refuses it, so no v2 or v3 reader takes one. */-export function parseV4UrlRef(s: string): { teamId: number; keyMonth: string; hash: string } | null {- const match = /^imageurl:v4:([1-9][0-9]{0,15}):([0-9]{4}-(?:0[1-9]|1[0-2])):([A-Za-z0-9_-]{22})$/.exec(s)- if (!match || !isRawTeamId(match[1])) {- return null- }- return { teamId: Number(match[1]), keyMonth: match[2], hash: match[3] }-}diff --git a/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/v4-keys/canonical-json.ts b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/v4-keys/canonical-json.tsdeleted file mode 100644index 5379d208ba6f..000000000000--- a/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/v4-keys/canonical-json.ts+++ /dev/null@@ -1,15 +0,0 @@-/** Both readers rebuild this byte for byte, so key order is sorted and there is no whitespace. */-export function canonicalJson(value: unknown): string {- if (Array.isArray(value)) {- return `[${value.map(canonicalJson).join(',')}]`- }- if (value && typeof value === 'object') {- // An absent field is left out, as in the Rust context, so `{ ref: undefined }` matches `{}`.- return `{${Object.keys(value)- .filter((k) => (value as Record<string, unknown>)[k] !== undefined)- .sort()- .map((k) => `${JSON.stringify(k)}:${canonicalJson((value as Record<string, unknown>)[k])}`)- .join(',')}}`- }- return JSON.stringify(value)-}diff --git a/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/v4-keys/frame.ts b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/v4-keys/frame.tsdeleted file mode 100644index 64b89031407c..000000000000--- a/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/v4-keys/frame.ts+++ /dev/null@@ -1,38 +0,0 @@-import { createCipheriv, randomBytes } from 'node:crypto'--import { canonicalJson } from './canonical-json'--/** The owner and kind of a v4 object. Field names are the ones the v3 frame uses, plus `keyMonth`. */-export interface V4FrameContext {- teamId: number- keyMonth: string- sessionId?: string- kind: string- codec: 'none' | 'brotli' | 'zstd'- ref?: string-}--const MAGIC = Buffer.from('AISR03')-const NONCE_BYTES = 12--/**- * The v3 AISR03 frame with `"v": 4` in the authenticated data, so a v3 reader never opens a v4 object. The format is- * in `products/ai_training/docs/replay-data.md`, section "v4 dataset".- */-export function sealV4Frame(- key: Buffer,- context: V4FrameContext,- body: Buffer,- nonce = randomBytes(NONCE_BYTES)-): Buffer {- const aad = Buffer.from(canonicalJson({ v: 4, context }))- if (aad.length > 0xffff) {- throw new Error('v4 frame context is too long to frame')- }- const cipher = createCipheriv('aes-256-gcm', key, nonce, { authTagLength: 16 })- cipher.setAAD(aad)- const sealed = Buffer.concat([cipher.update(body), cipher.final(), cipher.getAuthTag()])- const length = Buffer.alloc(2)- length.writeUInt16BE(aad.length)- return Buffer.concat([MAGIC, length, aad, nonce, sealed])-}diff --git a/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/v4-keys/v4-keys.test.ts b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/v4-keys/v4-keys.test.tsindex 02929c0cb3f5..4f9352ed77c9 100644--- a/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/v4-keys/v4-keys.test.ts+++ b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/v4-keys/v4-keys.test.ts@@ -1,52 +1,9 @@import { GetItemCommand } from '@aws-sdk/client-dynamodb'import { DecryptCommand } from '@aws-sdk/client-kms'-import vector from '~/ingestion/pipelines/sessionreplay/ai-research/v4-keys/encryption-vector-v4.json'--import { V4FrameContext, sealV4Frame } from './frame'import { TeamMonthKeys, teamMonthKeyExpiresAtMs, teamMonthWrappingContext } from './team-month-keys'-const base64 = (value: string): Buffer => Buffer.from(value, 'base64')-describe('v4 keys', () => {- it.each([- [- 'an inline image under the team month key',- vector.teamMonthKey,- vector.image.context,- vector.image.bytes,- vector.image.nonce,- vector.image.frame,- ],- [- 'a fetched image under the team month key',- vector.teamMonthKey,- vector.imageUrl.context,- vector.imageUrl.bytes,- vector.imageUrl.nonce,- vector.imageUrl.frame,- ],- [- 'a fetched image whose context sets its absent fields to undefined',- vector.teamMonthKey,- { ...vector.imageUrl.context, sessionId: undefined },- vector.imageUrl.bytes,- vector.imageUrl.nonce,- vector.imageUrl.frame,- ],- [- 'a block under the session key',- vector.sessionKey,- vector.block.context,- vector.block.body,- vector.block.nonce,- vector.block.frame,- ],- ])('seals %s exactly as the Rust recorder does', (_name, key, context, body, nonce, frame) => {- // The Rust crate pins the same file, so the two writers cannot drift apart.- expect(sealV4Frame(base64(key), context as V4FrameContext, base64(body), base64(nonce))).toEqual(base64(frame))- })-const TEAM = 7const MONTH = '2026-10'const wrapped = Buffer.from('wrapped')diff --git a/nodejs/src/servers/ai-research-v4-image-fetch-server.ts b/nodejs/src/servers/ai-research-v4-image-fetch-server.tsindex c141317059c7..c4157e51579e 100644--- a/nodejs/src/servers/ai-research-v4-image-fetch-server.ts+++ b/nodejs/src/servers/ai-research-v4-image-fetch-server.ts@@ -41,7 +41,6 @@ import {v4HostLimits,} from '~/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/lane'import { OriginRequestScheduler } from '~/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/origin-request-scheduler'-import { assertUrlPolicyLoaded } from '~/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/politeness-key'import { createPoolWindowBatchHandler } from '~/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/pool-window-batch-handler'import {AI_RESEARCH_V4_IMAGE_FETCH_PRODUCER,@@ -55,6 +54,7 @@ import { RetryDelayConsumer } from '~/ingestion/pipelines/sessionreplay/ai-reseaimport { SharedImageFetcher } from '~/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/shared-image-fetcher'import { ImageFetchTopHogMetrics } from '~/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/tophog-metrics'import { UrlFetchConsumer } from '~/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/url-fetch-consumer'+import { assertSharedRulesLoaded } from '~/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/v4-native'import { createWebBotAuthRequestSigner } from '~/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/web-bot-auth'import { AiResearchV4ConsumerServer } from '~/ingestion/pipelines/sessionreplay/ai-research/server/v4-consumer-server'import { TeamMonthKeys } from '~/ingestion/pipelines/sessionreplay/ai-research/v4-keys/team-month-keys'@@ -301,7 +301,7 @@ export class AiResearchV4ImageFetchServer extends AiResearchV4ConsumerServer {// Here rather than on the first record. The parser calls into this addon for every URL, and// it must answer with a reason rather than raise, so a build that shipped without the addon// has to stop the pod at startup instead.- assertUrlPolicyLoaded()+ assertSharedRulesLoaded()const consumerConfigs = buildImageFetchConsumerConfigs(this.config)const dryRun = this.config.AI_RESEARCH_V4_IMAGE_FETCH_DRY_RUNdiff --git a/products/ai_training/docs/replay-data.md b/products/ai_training/docs/replay-data.mdindex 71a1b54746d7..2bf7a3ccf3e2 100644--- a/products/ai_training/docs/replay-data.md+++ b/products/ai_training/docs/replay-data.md@@ -669,7 +669,7 @@ It collects URLs from image sources, the kept candidate of each `srcset`, SVG imThe topics hold jobs for 24 hours, because jobs carry raw URLs.The v4 image fetcher is its own copy of the v3 fetcher under `ai-research/image-fetch-v4/`, with the v3 fetch policy, its own consumer groups, its own `AI_RESEARCH_V4_IMAGE_FETCH_*` settings and a share of the v3 per-host limits. It shares no code or table with v3.It applies the recorder's URL policy through `@posthog/ai-research-v4-native`, an addon over `replay_core_v4::url_policy`, so it canonicalizes a URL, groups it by registrable domain and refuses private hosts exactly as the recorder does.-It stores each image, sealed and unscrubbed, under `images/url/`, and it does not send images to the scrub lane.+It stores each image, sealed and unscrubbed, under `images/url/`, and it does not send images to the scrub lane. It parses the image URL reference, seals the frame and computes the response URL HMAC with `replay_core_v4::fetched_image` through the same addon.It stores PNG, JPEG, GIF, WebP and SVG as fetched. It does not rasterize an SVG; redaction downstream of the dataset handles SVG text.The response must have one of these media types and bytes of that format: a signature for a raster image, or an SVG root element for SVG. An SVG may carry a byte order mark, comments, processing instructions and a doctype before its root. The doctype may name no internal subset and no external DTD other than a W3C SVG DTD, so that a parser downstream cannot resolve entities from the document. It refuses AVIF and every other type.diff --git a/rust/ai-research/replay-core-v4-node/Cargo.toml b/rust/ai-research/replay-core-v4-node/Cargo.tomlindex 69fe000d480f..39c8be455854 100644--- a/rust/ai-research/replay-core-v4-node/Cargo.toml+++ b/rust/ai-research/replay-core-v4-node/Cargo.toml@@ -12,6 +12,8 @@ workspace = truecrate-type = ["cdylib"][dependencies]-neon = { workspace = true }+# `external-buffers`: a sealed image crosses the FFI without a copy. The addon runs only in Node+# ingestion workers, never under Electron, where external buffers abort the VM.+neon = { workspace = true, features = ["external-buffers"] }# default-features = false: the key table and KMS clients have no place in an addon that only checks URLs.replay-core-v4 = { path = "../replay-core-v4", default-features = false }diff --git a/rust/ai-research/replay-core-v4-node/src/index.ts b/rust/ai-research/replay-core-v4-node/src/index.tsindex b73411aebc7a..63640badd1f4 100644--- a/rust/ai-research/replay-core-v4-node/src/index.ts+++ b/rust/ai-research/replay-core-v4-node/src/index.ts@@ -84,3 +84,16 @@ export function urlJob(nextFetchAtMs: number, storedAtMs: number | undefined, noexport function isUrlHistoryValue(key: string, value: string): boolean {return native.isUrlHistoryValue(key, value)}++/** Parses `imageurl:v4:{team}:{YYYY-MM}:{hash}`. `parseImageRef` refuses it, so no v2 or v3 reader takes one. */+export function parseV4UrlRef(reference: string): { teamId: number; keyMonth: string; hash: string } | null {+ return native.parseV4UrlRef(reference)+}++export function sealFetchedImage(teamMonthKey: Buffer, reference: string, image: Buffer): Promise<Buffer> {+ return native.sealFetchedImage(teamMonthKey, reference, image)+}++export function imageUrlHash(teamMonthKey: Buffer, url: string): string {+ return native.imageUrlHash(teamMonthKey, url)+}diff --git a/rust/ai-research/replay-core-v4-node/src/lib.rs b/rust/ai-research/replay-core-v4-node/src/lib.rsindex 99d335de9d73..991fd24b67e5 100644--- a/rust/ai-research/replay-core-v4-node/src/lib.rs+++ b/rust/ai-research/replay-core-v4-node/src/lib.rs@@ -1,13 +1,12 @@-//! Neon native addon exposing the v4 URL policy, `replay_core_v4::url_policy`, and the crawl-//! history rule, `replay_core_v4::fetch_history`, to the v4 image fetcher.-//!-//! The recorder collects URLs and drops jobs with those modules, and the fetcher must decide the-//! same way, so the fetcher calls the modules rather than a second implementation. This crate is+//! Neon native addon exposing the `replay_core_v4` modules the recorder writes and reads with, so the+//! v4 image fetcher decides and seals the same way without a second implementation. This crate is//! glue only: each function converts its arguments and its result, and the rules stay in the core//! crate.use neon::prelude::*;+use neon::types::buffer::TypedArray;use replay_core_v4::fetch_history::UrlHistory;+use replay_core_v4::fetched_image::{image_url_hash, image_url_ref, seal_fetched_image};use replay_core_v4::url_policy::{is_public_host, politeness_key, try_canonicalize};/// The registrable domain of a host.@@ -70,6 +69,53 @@ fn is_url_history_value_ffi(mut cx: FunctionContext) -> JsResult<JsBoolean> {Ok(cx.boolean(UrlHistory::parse(&key, &value).is_some()))}+/// `null` also for a team id that a JavaScript number cannot hold exactly.+fn parse_v4_url_ref_ffi(mut cx: FunctionContext) -> JsResult<JsValue> {+ const MAX_SAFE_INTEGER: i64 = (1 << 53) - 1;+ let reference = cx.argument::<JsString>(0)?.value(&mut cx);+ let Some(parsed) =+ image_url_ref(&reference).filter(|parsed| parsed.team_id <= MAX_SAFE_INTEGER)+ else {+ return Ok(cx.null().upcast());+ };+ let result = cx.empty_object();+ let team_id = cx.number(parsed.team_id as f64);+ result.set(&mut cx, "teamId", team_id)?;+ let key_month = cx.string(parsed.key_month);+ result.set(&mut cx, "keyMonth", key_month)?;+ let hash = cx.string(parsed.hash);+ result.set(&mut cx, "hash", hash)?;+ Ok(result.upcast())+}++fn team_month_key(cx: &mut FunctionContext, index: usize) -> NeonResult<[u8; 32]> {+ let key = cx.argument::<JsBuffer>(index)?;+ match <[u8; 32]>::try_from(key.as_slice(cx)) {+ Ok(key) => Ok(key),+ Err(_) => cx.throw_range_error("a team month key is 32 bytes"),+ }+}++/// Seals off the event loop, because a fetched image can be megabytes.+fn seal_fetched_image_ffi(mut cx: FunctionContext) -> JsResult<JsPromise> {+ let key = team_month_key(&mut cx, 0)?;+ let reference = cx.argument::<JsString>(1)?.value(&mut cx);+ let image = cx.argument::<JsBuffer>(2)?.as_slice(&cx).to_vec();+ let promise = cx+ .task(move || seal_fetched_image(&key, &reference, &image))+ .promise(|mut cx, frame| match frame {+ Ok(frame) => Ok(JsBuffer::external(&mut cx, frame)),+ Err(error) => cx.throw_error(error.to_string()),+ });+ Ok(promise)+}++fn image_url_hash_ffi(mut cx: FunctionContext) -> JsResult<JsString> {+ let key = team_month_key(&mut cx, 0)?;+ let url = cx.argument::<JsString>(1)?.value(&mut cx);+ Ok(cx.string(image_url_hash(&key, &url)))+}+#[neon::main]fn main(mut cx: ModuleContext) -> NeonResult<()> {cx.export_function("politenessKey", politeness_key_ffi)?;@@ -77,5 +123,8 @@ fn main(mut cx: ModuleContext) -> NeonResult<()> {cx.export_function("tryCanonicalizeUrl", try_canonicalize_url_ffi)?;cx.export_function("urlJob", url_job_ffi)?;cx.export_function("isUrlHistoryValue", is_url_history_value_ffi)?;+ cx.export_function("parseV4UrlRef", parse_v4_url_ref_ffi)?;+ cx.export_function("sealFetchedImage", seal_fetched_image_ffi)?;+ cx.export_function("imageUrlHash", image_url_hash_ffi)?;Ok(())}diff --git a/rust/ai-research/replay-core-v4/src/fetched_image.rs b/rust/ai-research/replay-core-v4/src/fetched_image.rsnew file mode 100644index 000000000000..da337266b070--- /dev/null+++ b/rust/ai-research/replay-core-v4/src/fetched_image.rs@@ -0,0 +1,79 @@+//! The frame the image fetcher stores for each fetched image. The fetcher seals it through `replay-core-v4-node`, and a+//! reader rebuilds its context from the image's URL reference.++use posthog_replay_anonymizer::v4_image_refs::{parse, Parsed, Ref, RefKind};++use crate::envelope::{seal_frame, Codec, Context, EnvelopeError, Version};+use crate::refs::reference_hash;+use crate::seal::image_url_key;++#[derive(Debug, thiserror::Error, PartialEq, Eq)]+pub enum SealError {+ #[error("not an image URL reference")]+ NotAnImageUrlReference,+ #[error(transparent)]+ Envelope(#[from] EnvelopeError),+}++pub fn image_url_ref(reference: &str) -> Option<Ref<'_>> {+ match parse(reference)? {+ Parsed::Ref(parsed) if parsed.kind == RefKind::ImageUrl => Some(parsed),+ _ => None,+ }+}++pub fn fetched_image_context(reference: &str) -> Option<Context> {+ let parsed = image_url_ref(reference)?;+ Some(Context {+ team_id: parsed.team_id,+ key_month: Some(parsed.key_month.to_owned()),+ kind: "image-url".to_owned(),+ codec: Some(Codec::None),+ reference: Some(reference.to_owned()),+ ..Context::default()+ })+}++pub fn seal_fetched_image(+ team_month_key: &[u8; 32],+ reference: &str,+ image: &[u8],+) -> Result<Vec<u8>, SealError> {+ let context = fetched_image_context(reference).ok_or(SealError::NotAnImageUrlReference)?;+ Ok(seal_frame(team_month_key, Version::V4, &context, image)?)+}++pub fn image_url_hash(team_month_key: &[u8; 32], url: &str) -> String {+ reference_hash(&image_url_key(team_month_key), url.as_bytes())+}++#[cfg(test)]+mod tests {+ use super::*;+ use crate::envelope::open_frame;++ const REFERENCE: &str = "imageurl:v4:7:2026-10:TEX0ajC0UkaOC9js5c7oqr";++ #[test]+ fn seals_an_image_only_under_an_image_url_reference() {+ let key = [3; 32];+ let frame = seal_fetched_image(&key, REFERENCE, b"png").unwrap();+ let context = fetched_image_context(REFERENCE).unwrap();+ assert_eq!(+ open_frame(&key, Version::V4, &context, &frame).unwrap(),+ b"png"+ );+ for reference in [+ "image:v4:7:2026-10:TEX0ajC0UkaOC9js5c7oqr",+ "imageurl:v4:07:2026-10:TEX0ajC0UkaOC9js5c7oqr",+ "imageurl:v4:7:2026-13:TEX0ajC0UkaOC9js5c7oqr",+ "imageurl:v4:declined:credential",+ ] {+ assert_eq!(+ seal_fetched_image(&key, reference, b"png"),+ Err(SealError::NotAnImageUrlReference),+ "{reference}"+ );+ }+ }+}diff --git a/rust/ai-research/replay-core-v4/src/lib.rs b/rust/ai-research/replay-core-v4/src/lib.rsindex 9e7f9d6982b3..07ba84681498 100644--- a/rust/ai-research/replay-core-v4/src/lib.rs+++ b/rust/ai-research/replay-core-v4/src/lib.rs@@ -4,6 +4,7 @@ pub mod bytescan;pub mod compression;pub mod envelope;pub mod fetch_history;+pub mod fetched_image;pub mod gate;pub mod inspect;#[cfg(feature = "store")]diff --git a/rust/ai-research/replay-core-v4/tests/v4_vector.rs b/rust/ai-research/replay-core-v4/tests/v4_vector.rsindex 3fb0b197d50a..704569479a7a 100644--- a/rust/ai-research/replay-core-v4/tests/v4_vector.rs+++ b/rust/ai-research/replay-core-v4/tests/v4_vector.rs@@ -4,6 +4,7 @@ use posthog_replay_anonymizer::v4_image_refs::{image_ref, image_url_ref};use replay_core_v4::envelope::{open_frame, seal_frame_with_nonce, seal_json_envelope_with_nonce, Codec, Context, Version,};+use replay_core_v4::fetched_image::{fetched_image_context, image_url_hash};use replay_core_v4::refs::{reference_hash, url_fingerprint};use replay_core_v4::seal::{image_content_key, image_url_key, open_session_key, seal_session_key_with_nonce,@@ -12,7 +13,7 @@ use replay_core_v4::seal::{use serde_json::Value;// Written with Node's crypto from the spec in replay-data.md, so this checks the Rust code against an independent-// implementation. The v4 Node writers read the same file.+// implementation. The v4 Node tests read the same file.const VECTOR: &str =include_str!("../../../../nodejs/src/ingestion/pipelines/sessionreplay/ai-research/v4-keys/encryption-vector-v4.json");@@ -139,9 +140,13 @@ fn reproduces_every_pinned_v4_value() {assert!(open_frame(&session_key, Version::V4, &image_context, &image_frame).is_err());let url = &vector["url"];- let url_hash = reference_hash(- &image_url_key(&team_month_key),- url["url"].as_str().unwrap().as_bytes(),+ let url_hash = image_url_hash(&team_month_key, url["url"].as_str().unwrap());+ assert_eq!(+ url_hash,+ reference_hash(+ &image_url_key(&team_month_key),+ url["url"].as_str().unwrap().as_bytes()+ ));assert_eq!(image_url_ref(team_id, key_month, &url_hash),@@ -151,7 +156,11 @@ fn reproduces_every_pinned_v4_value() {// The image fetcher seals a fetched image under its URL reference, so a reader rebuilds the context from it.let fetched = &vector["imageUrl"];assert_eq!(fetched["ref"], url["ref"]);- let fetched_context = context(&fetched["context"], Some(Codec::None));+ let fetched_context = fetched_image_context(fetched["ref"].as_str().unwrap()).unwrap();+ assert_eq!(+ fetched_context,+ context(&fetched["context"], Some(Codec::None))+ );let fetched_frame = seal_frame_with_nonce(&team_month_key,Version::V4,