:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(ai-research): keep v4 url history per team month (#117983)
Nicholas Waltz15748768e806
diff --git a/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/crawl-history.ts b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/crawl-history.tsnew file mode 100644index 000000000000..0bfb1e5a1588--- /dev/null+++ b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/crawl-history.ts@@ -0,0 +1,52 @@+export interface HttpCacheMetadata {+ requestTimeMs: number+ responseTimeMs: number+ /**+ * A keyed hash of the URL that sent these headers, never the URL itself. Its validators are sent to that URL only.+ */+ responseUrlHmac?: string+ etag?: string+ lastModified?: string+ date?: string+ age?: string+ cacheControl?: string+ expires?: string+}++export interface UrlCrawlHistoryItem {+ kind: 'url'+ key: string+ nextFetchAtMs: number+ storageExpiresAtMs: number+ outcome: string+ cache?: HttpCacheMetadata+ /** When the lane last stored or refreshed the image of this reference. */+ storedAtMs?: number+}++export type ConfigurationFile = 'robots' | 'tdmrep'+export type ConfigurationStatus = 'available' | 'absent' | 'refused' | 'unreachable'++export interface ConfigurationCacheItem {+ kind: ConfigurationFile+ key: string+ origin: string+ status: ConfigurationStatus+ body?: string+ fetchedAtMs: number+ refreshAtMs: number+ freshUntilMs: number+ retryAtMs: number+ storageExpiresAtMs: number+}++export type CrawlHistoryItem = UrlCrawlHistoryItem | ConfigurationCacheItem++export function configurationCacheKey(origin: string, file: ConfigurationFile): string {+ return `imgfetch:config:${file}:${origin}`+}++export interface CrawlHistoryStore {+ read(keys: string[]): Promise<Map<string, CrawlHistoryItem>>+ write(items: CrawlHistoryItem[]): Promise<void>+}diff --git a/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/dynamodb-crawl-history.integration.test.ts b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/dynamodb-crawl-history.integration.test.tsnew file mode 100644index 000000000000..38b19b5a0a17--- /dev/null+++ b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/dynamodb-crawl-history.integration.test.ts@@ -0,0 +1,66 @@+import {+ CreateTableCommand,+ DeleteTableCommand,+ DynamoDBClient,+ GetItemCommand,+ waitUntilTableExists,+} from '@aws-sdk/client-dynamodb'++import { UrlCrawlHistoryItem } from './crawl-history'+import { DynamoDBCrawlHistory } from './dynamodb-crawl-history'++const DYNAMODB_ENDPOINT = 'http://127.0.0.1:18010'+const TABLE_NAME = `ai-research-image-fetch-crawl-history-test-${process.pid}-${Date.now()}`+const NOW_MS = 1_700_000_000_000+const NOW_SECONDS = Math.floor(NOW_MS / 1000)+const TTL_SECONDS = 30 * 24 * 60 * 60+const KEYS = Array.from({ length: 205 }, (_value, index) => `key-${index}`)++describe('DynamoDBCrawlHistory integration', () => {+ const client = new DynamoDBClient({+ endpoint: DYNAMODB_ENDPOINT,+ region: 'us-east-1',+ credentials: { accessKeyId: 'test', secretAccessKey: 'test' },+ maxAttempts: 1,+ })++ beforeAll(async () => {+ await client.send(+ new CreateTableCommand({+ TableName: TABLE_NAME,+ AttributeDefinitions: [{ AttributeName: 'key', AttributeType: 'S' }],+ KeySchema: [{ AttributeName: 'key', KeyType: 'HASH' }],+ BillingMode: 'PAY_PER_REQUEST',+ })+ )+ await waitUntilTableExists({ client, maxWaitTime: 10, minDelay: 1 }, { TableName: TABLE_NAME })+ })++ afterAll(async () => {+ await client.send(new DeleteTableCommand({ TableName: TABLE_NAME }))+ client.destroy()+ })++ it('records and reads a full ingestion batch through DynamoDB', async () => {+ const crawlHistory = new DynamoDBCrawlHistory(client, TABLE_NAME, 5_000, 30_000)++ await crawlHistory.validateAccess(NOW_MS)+ const storageExpiresAtMs = NOW_MS + TTL_SECONDS * 1000+ const items: UrlCrawlHistoryItem[] = KEYS.map((key) => ({+ kind: 'url',+ key,+ nextFetchAtMs: storageExpiresAtMs,+ storageExpiresAtMs,+ outcome: 'ok',+ }))+ await crawlHistory.write(items)+ const readResult = await crawlHistory.read([...KEYS, 'missing'])+ const storedItem = await client.send(+ new GetItemCommand({ TableName: TABLE_NAME, Key: { key: { S: KEYS[0] } }, ConsistentRead: true })+ )++ expect(readResult.size).toBe(KEYS.length)+ expect([...readResult.keys()].sort()).toEqual([...KEYS].sort())+ expect(storedItem.Item?.expires_at?.N).toBe(String(NOW_SECONDS + TTL_SECONDS))+ })+})diff --git a/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/dynamodb-crawl-history.test.ts b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/dynamodb-crawl-history.test.tsnew file mode 100644index 000000000000..3af76f475b7d--- /dev/null+++ b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/dynamodb-crawl-history.test.ts@@ -0,0 +1,415 @@+import { BatchGetItemCommand, BatchWriteItemCommand } from '@aws-sdk/client-dynamodb'++import { parseJSON } from '~/common/utils/json-parse'+import { logger } from '~/common/utils/logger'++import { ConfigurationCacheItem, CrawlHistoryItem, UrlCrawlHistoryItem } from './crawl-history'+import { DynamoDBCrawlHistory, DynamoDBCrawlHistoryClient } from './dynamodb-crawl-history'++const TABLE = 'crawl-history'+const NOW_MS = 1_700_000_000_123++function urlItem(key: string, overrides: Partial<UrlCrawlHistoryItem> = {}): UrlCrawlHistoryItem {+ return {+ kind: 'url',+ key,+ nextFetchAtMs: NOW_MS + 30 * 24 * 60 * 60 * 1000,+ storageExpiresAtMs: NOW_MS + 30 * 24 * 60 * 60 * 1000,+ outcome: 'ok',+ ...overrides,+ }+}++function stored(item: CrawlHistoryItem): Record<string, { S?: string; N?: string }> {+ return {+ key: { S: item.key },+ expires_at: { N: String(Math.ceil(item.storageExpiresAtMs / 1000)) },+ value: { S: JSON.stringify(item) },+ }+}++function build(send: jest.Mock, batchBudgetMs = 10_000): DynamoDBCrawlHistory {+ return new DynamoDBCrawlHistory({ send } as unknown as DynamoDBCrawlHistoryClient, TABLE, 1_000, batchBudgetMs)+}++function requestedReadKeys(command: BatchGetItemCommand): string[] {+ return command.input.RequestItems?.[TABLE]?.Keys?.flatMap((key) => (key.key?.S ? [key.key.S] : [])) ?? []+}++function requestedWrites(+ command: BatchWriteItemCommand+): NonNullable<BatchWriteItemCommand['input']['RequestItems']>[string] {+ return command.input.RequestItems?.[TABLE] ?? []+}++function hangUntilAborted(_command: unknown, { abortSignal }: { abortSignal: AbortSignal }): Promise<never> {+ return new Promise((_resolve, reject) =>+ abortSignal.addEventListener('abort', () =>+ reject(Object.assign(new Error('Request aborted'), { name: 'AbortError' }))+ )+ )+}++describe('DynamoDBCrawlHistory', () => {+ it('reads each distinct key in batches of at most 100', async () => {+ const send = jest.fn((command: BatchGetItemCommand) => {+ const items = requestedReadKeys(command).map((key) => stored(urlItem(key)))+ return Promise.resolve({ Responses: { [TABLE]: items } })+ })+ const keys = [...Array.from({ length: 205 }, (_, index) => `k${index}`), 'k0']++ const result = await build(send).read(keys)++ expect(result).toHaveProperty('size', 205)+ expect(send).toHaveBeenCalledTimes(3)+ expect(send.mock.calls.map(([command]) => requestedReadKeys(command))).toEqual(+ expect.arrayContaining([expect.any(Array), expect.any(Array), expect.any(Array)])+ )+ expect(send.mock.calls.every(([command]) => requestedReadKeys(command).length <= 100)).toBe(true)+ })++ it('reads a legacy expiry-only URL item', async () => {+ const send = jest.fn(() =>+ Promise.resolve({+ Responses: {+ [TABLE]: [{ key: { S: 'legacy' }, expires_at: { N: String(Math.ceil(NOW_MS / 1000)) } }],+ },+ })+ )++ await expect(build(send).read(['legacy'])).resolves.toEqual(+ new Map([+ [+ 'legacy',+ {+ kind: 'url',+ key: 'legacy',+ nextFetchAtMs: Math.ceil(NOW_MS / 1000) * 1000,+ storageExpiresAtMs: Math.ceil(NOW_MS / 1000) * 1000,+ outcome: 'legacy',+ },+ ],+ ])+ )+ })++ it('reads a URL item whose cache holds a plaintext response URL as unbound', async () => {+ const cache = { requestTimeMs: NOW_MS - 10, responseTimeMs: NOW_MS, etag: '"v1"' }+ const item = stored(urlItem('a'))+ item.value = {+ S: JSON.stringify({ ...urlItem('a'), cache: { ...cache, responseUrl: 'https://cdn.example.com/a.png' } }),+ }+ const send = jest.fn(() => Promise.resolve({ Responses: { [TABLE]: [item] } }))++ const result = await build(send).read(['a'])++ expect((result.get('a') as UrlCrawlHistoryItem).cache).toEqual(cache)+ })++ it('retries only unprocessed read keys', async () => {+ const send = jest+ .fn()+ .mockResolvedValueOnce({+ Responses: { [TABLE]: [stored(urlItem('done'))] },+ UnprocessedKeys: { [TABLE]: { Keys: [{ key: { S: 'retry' } }] } },+ })+ .mockResolvedValueOnce({ Responses: { [TABLE]: [stored(urlItem('retry'))] } })++ const result = await build(send).read(['done', 'retry'])++ expect([...result.keys()].sort()).toEqual(['done', 'retry'])+ expect(requestedReadKeys(send.mock.calls[1][0])).toEqual(['retry'])+ })++ it('throws when a read request fails', async () => {+ const send = jest.fn(() => Promise.reject(new Error('DynamoDB unavailable')))++ await expect(build(send).read(['a'])).rejects.toThrow('DynamoDB unavailable')+ })++ it.each([+ ['an item with no valid expiry', { key: { S: 'a' }, expires_at: { N: 'invalid' } }],+ ['a value that is not JSON', { ...stored(urlItem('a')), value: { S: '{"kind":"url",' } }],+ [+ 'a value with an invalid nested shape',+ {+ ...stored(urlItem('a')),+ value: { S: JSON.stringify({ ...urlItem('a'), cache: { requestTimeMs: 'x' } }) },+ },+ ],+ [+ 'a kind a newer build wrote',+ { ...stored(urlItem('a')), value: { S: JSON.stringify({ ...urlItem('a'), kind: 'sitemap' }) } },+ ],+ ])('reads %s as a cache miss and still returns the other items', async (_name, item) => {+ const warn = jest.spyOn(logger, 'warn').mockImplementation()+ const send = jest.fn(() => Promise.resolve({ Responses: { [TABLE]: [item, stored(urlItem('b'))] } }))++ const result = await build(send).read(['a', 'b'])++ expect([...result.keys()]).toEqual(['b'])+ expect(warn).toHaveBeenCalledWith('🌐', 'ai_research_v4_image_fetch_crawl_history_item_invalid', {+ key: 'a',+ error: expect.any(String),+ })+ expect(JSON.stringify(warn.mock.calls)).not.toContain('"kind')+ warn.mockRestore()+ })++ it('folds repeated updates and writes batches of at most 25', async () => {+ const send: jest.Mock<Promise<unknown>, [BatchWriteItemCommand]> = jest.fn((_command) => Promise.resolve({}))+ const first = urlItem('same', { outcome: 'first' })+ const last = urlItem('same', { outcome: 'last', storageExpiresAtMs: NOW_MS + 1 })+ const items = [first, ...Array.from({ length: 55 }, (_, index) => urlItem(`k${index}`)), last]++ await build(send).write(items)++ expect(send).toHaveBeenCalledTimes(3)+ const requests = send.mock.calls.flatMap(([command]) => requestedWrites(command))+ expect(send.mock.calls.every(([command]) => requestedWrites(command).length <= 25)).toBe(true)+ expect(requests).toHaveLength(56)+ const same = requests.find((request) => request.PutRequest?.Item?.key?.S === 'same')+ expect(parseJSON(same?.PutRequest?.Item?.value?.S ?? '{}')).toMatchObject({ outcome: 'last' })+ expect(same?.PutRequest?.Item?.expires_at?.N).toBe(String(Math.ceil((NOW_MS + 1) / 1000)))+ })++ it('retries only unprocessed writes', async () => {+ const send = jest+ .fn()+ .mockImplementationOnce((command: BatchWriteItemCommand) =>+ Promise.resolve({ UnprocessedItems: { [TABLE]: [requestedWrites(command)[1]] } })+ )+ .mockResolvedValueOnce({})++ await build(send).write([urlItem('done'), urlItem('retry')])++ expect(requestedWrites(send.mock.calls[1][0])).toHaveLength(1)+ expect(requestedWrites(send.mock.calls[1][0])[0].PutRequest?.Item?.key?.S).toBe('retry')+ })++ it('splits and reassembles a 500 KiB configuration body', async () => {+ const dynamoItems = new Map<string, Record<string, { S?: string; N?: string }>>()+ const send = jest.fn((command: BatchWriteItemCommand | BatchGetItemCommand) => {+ if (command instanceof BatchWriteItemCommand) {+ for (const request of requestedWrites(command)) {+ const item = request.PutRequest?.Item as Record<string, { S?: string; N?: string }>+ dynamoItems.set(item.key.S ?? '', item)+ }+ return Promise.resolve({})+ }+ return Promise.resolve({+ Responses: {+ [TABLE]: requestedReadKeys(command).flatMap((key) => {+ const item = dynamoItems.get(key)+ return item ? [item] : []+ }),+ },+ })+ })+ const body = 'abc😀'.repeat(73_000)+ const item: ConfigurationCacheItem = {+ kind: 'robots',+ key: 'imgfetch:config:robots:https://example.com',+ origin: 'https://example.com',+ status: 'available',+ body,+ fetchedAtMs: NOW_MS,+ refreshAtMs: NOW_MS + 1,+ freshUntilMs: NOW_MS + 2,+ retryAtMs: NOW_MS,+ storageExpiresAtMs: NOW_MS + 3,+ }+ const store = build(send)++ await store.write([item])+ const result = await store.read([item.key])++ expect(result.get(item.key)).toEqual(item)+ expect(dynamoItems.size).toBe(4)+ const writeCalls = send.mock.calls.filter(([command]) => command instanceof BatchWriteItemCommand)+ expect(requestedWrites(writeCalls[0][0] as BatchWriteItemCommand)).toHaveLength(3)+ expect(+ requestedWrites(writeCalls[0][0] as BatchWriteItemCommand).every((request) =>+ request.PutRequest?.Item?.key?.S?.startsWith(`${item.key}:body:`)+ )+ ).toBe(true)+ expect(requestedWrites(writeCalls[1][0] as BatchWriteItemCommand)[0].PutRequest?.Item?.key?.S).toBe(item.key)+ expect(+ [...dynamoItems.values()].every((storedItem) => Buffer.byteLength(JSON.stringify(storedItem)) < 400 * 1024)+ ).toBe(true)+ })++ it('treats an incomplete configuration body generation as a cache miss', async () => {+ const sha256 = 'a'.repeat(64)+ const item: ConfigurationCacheItem = {+ kind: 'robots',+ key: 'imgfetch:config:robots:https://example.com',+ origin: 'https://example.com',+ status: 'available',+ fetchedAtMs: NOW_MS,+ refreshAtMs: NOW_MS + 1,+ freshUntilMs: NOW_MS + 2,+ retryAtMs: NOW_MS,+ storageExpiresAtMs: NOW_MS + 3,+ }+ const manifest = stored(item)+ manifest.value = { S: JSON.stringify({ ...item, _bodyChunks: { count: 2, sha256 } }) }+ const send = jest+ .fn()+ .mockResolvedValueOnce({ Responses: { [TABLE]: [manifest] } })+ .mockResolvedValueOnce({ Responses: { [TABLE]: [] } })++ await expect(build(send).read([item.key])).resolves.toEqual(new Map())+ })++ it('does not publish a manifest when its body chunks cannot be written', async () => {+ jest.useFakeTimers()+ jest.spyOn(Math, 'random').mockReturnValue(0)+ const sentKeys: string[] = []+ const send = jest.fn((command: BatchWriteItemCommand) => {+ const writes = requestedWrites(command)+ sentKeys.push(...writes.flatMap((request) => request.PutRequest?.Item?.key?.S ?? []))+ return Promise.resolve({ UnprocessedItems: { [TABLE]: writes } })+ })+ const key = 'imgfetch:config:robots:https://example.com'+ const item: ConfigurationCacheItem = {+ kind: 'robots',+ key,+ origin: 'https://example.com',+ status: 'available',+ body: 'abc😀'.repeat(73_000),+ fetchedAtMs: NOW_MS,+ refreshAtMs: NOW_MS + 1,+ freshUntilMs: NOW_MS + 2,+ retryAtMs: NOW_MS,+ storageExpiresAtMs: NOW_MS + 3,+ }++ const write = build(send).write([item])+ const refused = expect(write).rejects.toThrow('crawl-history writes unprocessed')+ await jest.runAllTimersAsync()++ await refused+ expect(sentKeys.length).toBeGreaterThan(0)+ expect(sentKeys.every((sentKey) => sentKey.startsWith(`${key}:body:`))).toBe(true)+ jest.useRealTimers()+ })++ it.each([+ {+ operation: 'read',+ run: (history: DynamoDBCrawlHistory) => history.read(['slow']),+ retried: { Responses: { [TABLE]: [stored(urlItem('slow'))] } },+ requestedKeys: (command: BatchGetItemCommand | BatchWriteItemCommand) =>+ requestedReadKeys(command as BatchGetItemCommand),+ },+ {+ operation: 'write',+ run: (history: DynamoDBCrawlHistory) => history.write([urlItem('slow')]),+ retried: {},+ requestedKeys: (command: BatchGetItemCommand | BatchWriteItemCommand) =>+ requestedWrites(command as BatchWriteItemCommand).map((request) => request.PutRequest?.Item?.key?.S),+ },+ ])('retries a $operation command that times out', async ({ run, retried, requestedKeys }) => {+ jest.useFakeTimers()+ jest.spyOn(Math, 'random').mockReturnValue(0)+ const send = jest.fn().mockImplementationOnce(hangUntilAborted).mockResolvedValueOnce(retried)++ const outcome = run(build(send)).then(+ () => 'completed',+ (error: unknown) => String(error)+ )+ await jest.runAllTimersAsync()++ expect(await outcome).toBe('completed')+ expect(send).toHaveBeenCalledTimes(2)+ expect(requestedKeys(send.mock.calls[1][0])).toEqual(['slow'])+ jest.useRealTimers()+ })++ it.each([+ { operation: 'read', run: (history: DynamoDBCrawlHistory) => history.read(['slow']) },+ { operation: 'write', run: (history: DynamoDBCrawlHistory) => history.write([urlItem('slow')]) },+ ])('fails a $operation after every attempt times out', async ({ run }) => {+ jest.useFakeTimers()+ jest.spyOn(Math, 'random').mockReturnValue(0)+ const send = jest.fn(hangUntilAborted)++ const outcome = run(build(send)).then(+ () => 'completed',+ (error: unknown) => String(error)+ )+ await jest.runAllTimersAsync()++ expect(await outcome).toBe('Error: DynamoDB crawl-history command timed out after 1000ms')+ expect(send).toHaveBeenCalledTimes(5)+ jest.useRealTimers()+ })++ it('throws after unprocessed write retries are exhausted', async () => {+ jest.useFakeTimers()+ jest.spyOn(Math, 'random').mockReturnValue(0)+ const send = jest.fn((command: BatchWriteItemCommand) =>+ Promise.resolve({ UnprocessedItems: { [TABLE]: requestedWrites(command) } })+ )++ const write = build(send).write([urlItem('retry')])+ const refused = expect(write).rejects.toThrow('crawl-history writes unprocessed: retry')+ await jest.runAllTimersAsync()++ await refused+ expect(send).toHaveBeenCalledTimes(5)+ jest.useRealTimers()+ })++ it.each([+ {+ operation: 'read',+ run: (history: DynamoDBCrawlHistory, keys: string[]) => history.read(keys),+ answerOne: (command: BatchGetItemCommand | BatchWriteItemCommand) => {+ const [first, ...rest] = requestedReadKeys(command as BatchGetItemCommand)+ return {+ Responses: { [TABLE]: [stored(urlItem(first))] },+ UnprocessedKeys: { [TABLE]: { Keys: rest.map((key) => ({ key: { S: key } })) } },+ }+ },+ },+ {+ operation: 'write',+ run: (history: DynamoDBCrawlHistory, keys: string[]) => history.write(keys.map((key) => urlItem(key))),+ answerOne: (command: BatchGetItemCommand | BatchWriteItemCommand) => ({+ UnprocessedItems: { [TABLE]: requestedWrites(command as BatchWriteItemCommand).slice(1) },+ }),+ },+ ])('drains a $operation that makes progress on every call', async ({ run, answerOne }) => {+ jest.useFakeTimers()+ jest.spyOn(Math, 'random').mockReturnValue(0)+ const send = jest.fn((command: BatchGetItemCommand | BatchWriteItemCommand) =>+ Promise.resolve(answerOne(command))+ )+ const keys = Array.from({ length: 8 }, (_, index) => `k${index}`)++ const outcome = run(build(send), keys).then(+ () => 'completed',+ (error: unknown) => String(error)+ )+ await jest.runAllTimersAsync()++ expect(await outcome).toBe('completed')+ expect(send).toHaveBeenCalledTimes(8)+ jest.useRealTimers()+ })++ it('validates write and consistent read access', async () => {+ const send = jest+ .fn()+ .mockResolvedValueOnce({})+ .mockResolvedValueOnce({ Responses: { [TABLE]: [{ key: { S: 'imgfetch:access-probe' } }] } })++ await build(send).validateAccess(NOW_MS)++ expect(send.mock.calls[0][0]).toBeInstanceOf(BatchWriteItemCommand)+ expect(send.mock.calls[1][0]).toBeInstanceOf(BatchGetItemCommand)+ expect(send.mock.calls[1][0].input.RequestItems?.[TABLE]?.ConsistentRead).toBe(true)+ })+})diff --git a/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/dynamodb-crawl-history.ts b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/dynamodb-crawl-history.tsnew file mode 100644index 000000000000..40a44a1fbf0f--- /dev/null+++ b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/dynamodb-crawl-history.ts@@ -0,0 +1,506 @@+import {+ AttributeValue,+ BatchGetItemCommand,+ BatchGetItemCommandOutput,+ BatchWriteItemCommand,+ BatchWriteItemCommandOutput,+ DynamoDBClient,+ WriteRequest,+} from '@aws-sdk/client-dynamodb'+import { createHash } from 'node:crypto'+import { Counter } from 'prom-client'++import { parseJSON } from '~/common/utils/json-parse'+import { logger } from '~/common/utils/logger'++import { ConfigurationCacheItem, CrawlHistoryItem, CrawlHistoryStore, HttpCacheMetadata } from './crawl-history'++const KEY_ATTRIBUTE = 'key'+const EXPIRES_AT_ATTRIBUTE = 'expires_at'+const VALUE_ATTRIBUTE = 'value'+const BATCH_GET_SIZE = 100+const BATCH_WRITE_SIZE = 25+const MAX_CONCURRENT_BATCH_REQUESTS = 8+const UNPROCESSED_MAX_ATTEMPTS = 5+const UNPROCESSED_INITIAL_BACKOFF_MS = 50+const ACCESS_PROBE_KEY = 'imgfetch:access-probe'+const ACCESS_PROBE_TTL_SECONDS = 5 * 60+const SAFE_ITEM_SIZE_BYTES = 350 * 1024+const BODY_CHUNK_SIZE_BYTES = 240 * 1024+const MAX_BODY_CHUNKS = 3++const invalidItems = new Counter({+ name: 'ai_research_v4_image_fetch_crawl_history_invalid_items_total',+ help: 'v4 crawl history items that did not parse and were read as cache misses.',+})++interface StoredBodyChunks {+ count: number+ sha256: string+}++type ParsedStoredItem =+ | { item: CrawlHistoryItem; bodyChunks?: undefined }+ | { item: ConfigurationCacheItem; bodyChunks: StoredBodyChunks }++interface ItemWritePlan {+ bodyChunkRequests: WriteRequest[]+ manifestRequest: WriteRequest+}++export type DynamoDBCrawlHistoryClient = Pick<DynamoDBClient, 'send'>++class CommandTimeoutError extends Error {}++function chunk<T>(items: T[], size: number): T[][] {+ const chunks: T[][] = []+ for (let index = 0; index < items.length; index += size) {+ chunks.push(items.slice(index, index + size))+ }+ return chunks+}++function keyFromWriteRequest(request: WriteRequest): string | undefined {+ return request.PutRequest?.Item?.[KEY_ATTRIBUTE]?.S+}++function isRecord(value: unknown): value is Record<string, unknown> {+ return typeof value === 'object' && value !== null && !Array.isArray(value)+}++function isFiniteNumber(value: unknown): value is number {+ return typeof value === 'number' && Number.isFinite(value)+}++function isOptionalString(value: unknown): value is string | undefined {+ return value === undefined || typeof value === 'string'+}++function isHttpCacheMetadata(value: unknown): boolean {+ if (!isRecord(value) || !isFiniteNumber(value.requestTimeMs) || !isFiniteNumber(value.responseTimeMs)) {+ return false+ }+ return ['responseUrlHmac', 'responseUrl', 'etag', 'lastModified', 'date', 'age', 'cacheControl', 'expires'].every(+ (field) => isOptionalString(value[field])+ )+}++function isCrawlHistoryItem(value: unknown, expectedKey: string): value is CrawlHistoryItem {+ if (!isRecord(value) || value.key !== expectedKey || !isFiniteNumber(value.storageExpiresAtMs)) {+ return false+ }+ if (value.kind === 'url') {+ return (+ isFiniteNumber(value.nextFetchAtMs) &&+ typeof value.outcome === 'string' &&+ (value.cache === undefined || isHttpCacheMetadata(value.cache)) &&+ (value.storedAtMs === undefined || isFiniteNumber(value.storedAtMs))+ )+ }+ return (+ (value.kind === 'robots' || value.kind === 'tdmrep') &&+ typeof value.origin === 'string' &&+ ['available', 'absent', 'refused', 'unreachable'].includes(String(value.status)) &&+ isOptionalString(value.body) &&+ isFiniteNumber(value.fetchedAtMs) &&+ isFiniteNumber(value.refreshAtMs) &&+ isFiniteNumber(value.freshUntilMs) &&+ isFiniteNumber(value.retryAtMs)+ )+}++/**+ * An older row can hold the response URL in plaintext. It reads as unbound, so its validators are not sent, and the+ * next write of the row does not carry the URL.+ */+function withoutPlaintextResponseUrl(item: CrawlHistoryItem): CrawlHistoryItem {+ if (item.kind !== 'url' || !item.cache || !('responseUrl' in item.cache)) {+ return item+ }+ const { responseUrl: _plaintext, ...cache } = item.cache as HttpCacheMetadata & { responseUrl?: string }+ return { ...item, cache }+}++function parseStoredItem(item: Record<string, AttributeValue>): ParsedStoredItem {+ const key = item[KEY_ATTRIBUTE]?.S+ const expiresAtSeconds = Number(item[EXPIRES_AT_ATTRIBUTE]?.N)+ if (!key || !Number.isFinite(expiresAtSeconds)) {+ throw new Error('DynamoDB returned a malformed crawl-history item')+ }+ const serialized = item[VALUE_ATTRIBUTE]?.S+ if (!serialized) {+ return {+ item: {+ kind: 'url',+ key,+ nextFetchAtMs: expiresAtSeconds * 1000,+ storageExpiresAtMs: expiresAtSeconds * 1000,+ outcome: 'legacy',+ },+ }+ }+ const parsed = parseJSON(serialized)+ if (!isRecord(parsed)) {+ throw new Error('DynamoDB returned an invalid crawl-history value')+ }+ const storedBodyChunks = parsed._bodyChunks+ if (storedBodyChunks === undefined) {+ if (!isCrawlHistoryItem(parsed, key)) {+ throw new Error('DynamoDB returned an invalid crawl-history value')+ }+ return { item: withoutPlaintextResponseUrl(parsed) }+ }+ const { _bodyChunks: _storedBodyChunks, ...applicationItem } = parsed+ if (+ !isCrawlHistoryItem(applicationItem, key) ||+ applicationItem.kind === 'url' ||+ applicationItem.body !== undefined ||+ !isRecord(storedBodyChunks) ||+ !Number.isInteger(storedBodyChunks.count) ||+ (storedBodyChunks.count as number) < 1 ||+ (storedBodyChunks.count as number) > MAX_BODY_CHUNKS ||+ typeof storedBodyChunks.sha256 !== 'string' ||+ !/^[0-9a-f]{64}$/.test(storedBodyChunks.sha256)+ ) {+ throw new Error('DynamoDB returned invalid crawl-history body chunks')+ }+ return {+ item: applicationItem,+ bodyChunks: { count: storedBodyChunks.count as number, sha256: storedBodyChunks.sha256 },+ }+}++function bodyChunkKey(key: string, sha256: string, index: number): string {+ return `${key}:body:${sha256}:${index}`+}++function putRequest(key: string, storageExpiresAtMs: number, serialized: string): WriteRequest {+ const expiresAt = String(Math.ceil(storageExpiresAtMs / 1000))+ const sizeBytes =+ Buffer.byteLength(KEY_ATTRIBUTE) ++ Buffer.byteLength(key) ++ Buffer.byteLength(EXPIRES_AT_ATTRIBUTE) ++ Buffer.byteLength(expiresAt) ++ Buffer.byteLength(VALUE_ATTRIBUTE) ++ Buffer.byteLength(serialized)+ if (sizeBytes > SAFE_ITEM_SIZE_BYTES) {+ throw new Error(`crawl-history item ${key} exceeds the safe DynamoDB item size`)+ }+ return {+ PutRequest: {+ Item: {+ [KEY_ATTRIBUTE]: { S: key },+ [EXPIRES_AT_ATTRIBUTE]: { N: expiresAt },+ [VALUE_ATTRIBUTE]: { S: serialized },+ },+ },+ }+}++function writePlanForItem(item: CrawlHistoryItem): ItemWritePlan {+ const serialized = JSON.stringify(item)+ if (Buffer.byteLength(serialized) <= SAFE_ITEM_SIZE_BYTES / 2 || item.kind === 'url' || !item.body) {+ return {+ bodyChunkRequests: [],+ manifestRequest: putRequest(item.key, item.storageExpiresAtMs, serialized),+ }+ }+ const body = Buffer.from(item.body, 'utf8')+ const bodyChunks: Buffer[] = []+ for (let offset = 0; offset < body.length; offset += BODY_CHUNK_SIZE_BYTES) {+ bodyChunks.push(body.subarray(offset, offset + BODY_CHUNK_SIZE_BYTES))+ }+ if (bodyChunks.length > MAX_BODY_CHUNKS) {+ throw new Error(`crawl-history body ${item.key} needs too many DynamoDB items`)+ }+ const sha256 = createHash('sha256').update(body).digest('hex')+ const storedItem = {+ ...item,+ body: undefined,+ _bodyChunks: {+ count: bodyChunks.length,+ sha256,+ },+ }+ return {+ bodyChunkRequests: bodyChunks.map((bodyChunk, index) =>+ putRequest(bodyChunkKey(item.key, sha256, index), item.storageExpiresAtMs, bodyChunk.toString('base64'))+ ),+ manifestRequest: putRequest(item.key, item.storageExpiresAtMs, JSON.stringify(storedItem)),+ }+}++export class DynamoDBCrawlHistory implements CrawlHistoryStore {+ constructor(+ private readonly client: DynamoDBCrawlHistoryClient,+ private readonly tableName: string,+ private readonly commandTimeoutMs: number,+ private readonly batchBudgetMs: number+ ) {}++ public async validateAccess(nowMs: number): Promise<void> {+ const expiresAt = Math.floor(nowMs / 1000) + ACCESS_PROBE_TTL_SECONDS+ const writeResponse = await this.sendBatchWrite(+ new BatchWriteItemCommand({+ RequestItems: {+ [this.tableName]: [+ {+ PutRequest: {+ Item: {+ [KEY_ATTRIBUTE]: { S: ACCESS_PROBE_KEY },+ [EXPIRES_AT_ATTRIBUTE]: { N: String(expiresAt) },+ },+ },+ },+ ],+ },+ })+ )+ if ((writeResponse.UnprocessedItems?.[this.tableName] ?? []).length > 0) {+ throw new Error('DynamoDB crawl-history access probe write was not processed')+ }+ const readResponse = await this.sendBatchGet(+ new BatchGetItemCommand({+ RequestItems: {+ [this.tableName]: {+ Keys: [{ [KEY_ATTRIBUTE]: { S: ACCESS_PROBE_KEY } }],+ ProjectionExpression: '#key',+ ExpressionAttributeNames: { '#key': KEY_ATTRIBUTE },+ ConsistentRead: true,+ },+ },+ })+ )+ if (+ (readResponse.UnprocessedKeys?.[this.tableName]?.Keys ?? []).length > 0 ||+ !(readResponse.Responses?.[this.tableName] ?? []).some(+ (item) => item[KEY_ATTRIBUTE]?.S === ACCESS_PROBE_KEY+ )+ ) {+ throw new Error('DynamoDB crawl-history access probe read was not processed')+ }+ }++ public async read(keys: string[]): Promise<Map<string, CrawlHistoryItem>> {+ const result = new Map<string, CrawlHistoryItem>()+ const startedAt = process.hrtime.bigint()+ const bodyChunks = new Map<string, { item: ConfigurationCacheItem; metadata: StoredBodyChunks }>()+ for (const item of await this.readRawItems([...new Set(keys)], startedAt)) {+ let parsed: ParsedStoredItem+ try {+ parsed = parseStoredItem(item)+ } catch (error) {+ // A throw would fail the batch on every redelivery, as would a row that a newer build wrote before a+ // rollback. A miss lets the lane fetch again and overwrite the row.+ invalidItems.inc()+ logger.warn('🌐', 'ai_research_v4_image_fetch_crawl_history_item_invalid', {+ key: item[KEY_ATTRIBUTE]?.S ?? 'missing',+ // A JSON syntax error quotes the stored value, so only its name is logged.+ error:+ error instanceof SyntaxError ? error.name : error instanceof Error ? error.message : 'unknown',+ })+ continue+ }+ if (parsed.bodyChunks) {+ bodyChunks.set(parsed.item.key, { item: parsed.item, metadata: parsed.bodyChunks })+ } else {+ result.set(parsed.item.key, parsed.item)+ }+ }+ if (bodyChunks.size === 0) {+ return result+ }++ const chunkKeys = [...bodyChunks].flatMap(([key, value]) =>+ Array.from({ length: value.metadata.count }, (_unused, index) =>+ bodyChunkKey(key, value.metadata.sha256, index)+ )+ )+ const encodedChunks = new Map<string, string>()+ for (const item of await this.readRawItems(chunkKeys, startedAt)) {+ const key = item[KEY_ATTRIBUTE]?.S+ const encoded = item[VALUE_ATTRIBUTE]?.S+ if (!key || !encoded) {+ logger.warn('🌐', 'ai_research_v4_image_fetch_crawl_history_chunk_invalid', { key: key ?? 'missing' })+ continue+ }+ encodedChunks.set(key, encoded)+ }+ for (const [key, value] of bodyChunks) {+ try {+ const chunks = Array.from({ length: value.metadata.count }, (_unused, index) => {+ const encoded = encodedChunks.get(bodyChunkKey(key, value.metadata.sha256, index))+ if (!encoded) {+ throw new Error('missing')+ }+ const decoded = Buffer.from(encoded, 'base64')+ if (decoded.toString('base64') !== encoded) {+ throw new Error('invalid base64')+ }+ return decoded+ })+ const body = Buffer.concat(chunks)+ if (createHash('sha256').update(body).digest('hex') !== value.metadata.sha256) {+ throw new Error('checksum mismatch')+ }+ result.set(key, { ...value.item, body: new TextDecoder('utf-8', { fatal: true }).decode(body) })+ } catch (error) {+ logger.warn('🌐', 'ai_research_v4_image_fetch_crawl_history_chunk_incomplete', {+ key,+ error: String(error),+ })+ }+ }+ return result+ }++ private async readRawItems(keys: string[], startedAt: bigint): Promise<Record<string, AttributeValue>[]> {+ const result: Record<string, AttributeValue>[] = []+ await this.runChunks(chunk(keys, BATCH_GET_SIZE), async (batch) => {+ let pending = batch+ let attempt = 1+ while (pending.length > 0) {+ this.requireBudget(startedAt)+ let response: BatchGetItemCommandOutput+ try {+ response = await this.sendBatchGet(+ new BatchGetItemCommand({+ RequestItems: {+ [this.tableName]: {+ Keys: pending.map((key) => ({ [KEY_ATTRIBUTE]: { S: key } })),+ ProjectionExpression: '#key, #expiresAt, #value',+ ExpressionAttributeNames: {+ '#key': KEY_ATTRIBUTE,+ '#expiresAt': EXPIRES_AT_ATTRIBUTE,+ '#value': VALUE_ATTRIBUTE,+ },+ },+ },+ })+ )+ } catch (error) {+ await this.backoffAfterCommandTimeout(error, attempt++)+ continue+ }+ for (const item of response.Responses?.[this.tableName] ?? []) {+ result.push(item)+ }+ const unprocessed = (response.UnprocessedKeys?.[this.tableName]?.Keys ?? []).flatMap((key) =>+ key[KEY_ATTRIBUTE]?.S ? [key[KEY_ATTRIBUTE].S] : []+ )+ // DynamoDB answers at most 1 MiB per partition per call, so only calls without progress count.+ attempt = unprocessed.length < pending.length ? 1 : attempt+ pending = unprocessed+ if (pending.length > 0) {+ if (attempt >= UNPROCESSED_MAX_ATTEMPTS) {+ throw new Error(`DynamoDB left ${pending.length} crawl-history reads unprocessed`)+ }+ await this.backoffBeforeRetry(attempt++)+ }+ }+ })+ return result+ }++ public async write(items: CrawlHistoryItem[]): Promise<void> {+ const byKey = new Map(items.map((item) => [item.key, item]))+ const plans = [...byKey.values()].map(writePlanForItem)+ const startedAt = process.hrtime.bigint()+ await this.writeRequests(+ plans.flatMap((plan) => plan.bodyChunkRequests),+ startedAt+ )+ await this.writeRequests(+ plans.map((plan) => plan.manifestRequest),+ startedAt+ )+ }++ private async writeRequests(requests: WriteRequest[], startedAt: bigint): Promise<void> {+ await this.runChunks(chunk(requests, BATCH_WRITE_SIZE), async (batch) => {+ let pending = batch+ let attempt = 1+ while (pending.length > 0) {+ this.requireBudget(startedAt)+ let response: BatchWriteItemCommandOutput+ try {+ response = await this.sendBatchWrite(+ new BatchWriteItemCommand({ RequestItems: { [this.tableName]: pending } })+ )+ } catch (error) {+ await this.backoffAfterCommandTimeout(error, attempt++)+ continue+ }+ const unprocessed = response.UnprocessedItems?.[this.tableName] ?? []+ attempt = unprocessed.length < pending.length ? 1 : attempt+ pending = unprocessed+ if (pending.length > 0) {+ if (attempt >= UNPROCESSED_MAX_ATTEMPTS) {+ const keys = pending.flatMap((request) => keyFromWriteRequest(request) ?? [])+ throw new Error(`DynamoDB left crawl-history writes unprocessed: ${keys.join(',')}`)+ }+ await this.backoffBeforeRetry(attempt++)+ }+ }+ })+ }++ private async runChunks<T>(chunks: T[][], run: (batch: T[]) => Promise<void>): Promise<void> {+ let nextChunk = 0+ const worker = async (): Promise<void> => {+ while (nextChunk < chunks.length) {+ await run(chunks[nextChunk++])+ }+ }+ await Promise.all(Array.from({ length: Math.min(chunks.length, MAX_CONCURRENT_BATCH_REQUESTS) }, worker))+ }++ private requireBudget(startedAt: bigint): void {+ if (Number(process.hrtime.bigint() - startedAt) / 1e6 > this.batchBudgetMs) {+ throw new Error('DynamoDB crawl-history batch budget was exhausted')+ }+ }++ /** A batch get or batch write of the same items is idempotent, so a timed-out command is retried like unprocessed items; any other failure, or a timeout on the last attempt, is rethrown. */+ private async backoffAfterCommandTimeout(error: unknown, attempt: number): Promise<void> {+ if (!(error instanceof CommandTimeoutError) || attempt >= UNPROCESSED_MAX_ATTEMPTS) {+ throw error+ }+ await this.backoffBeforeRetry(attempt)+ }++ private async backoffBeforeRetry(attempt: number): Promise<void> {+ const maximumDelayMs = UNPROCESSED_INITIAL_BACKOFF_MS * 2 ** (attempt - 1)+ const delayMs = Math.floor(maximumDelayMs / 2 + Math.random() * (maximumDelayMs / 2))+ await new Promise<void>((resolve) => setTimeout(resolve, delayMs))+ }++ private sendBatchGet(command: BatchGetItemCommand): Promise<BatchGetItemCommandOutput> {+ return this.withTimeout((abortSignal) => this.client.send(command, { abortSignal }))+ }++ private sendBatchWrite(command: BatchWriteItemCommand): Promise<BatchWriteItemCommandOutput> {+ return this.withTimeout((abortSignal) => this.client.send(command, { abortSignal }))+ }++ private async withTimeout<T>(send: (abortSignal: AbortSignal) => Promise<T>): Promise<T> {+ const abortController = new AbortController()+ const timer = setTimeout(() => abortController.abort(), this.commandTimeoutMs)+ timer.unref()+ try {+ return await send(abortController.signal)+ } catch (error) {+ if (abortController.signal.aborted) {+ throw new CommandTimeoutError(+ `DynamoDB crawl-history command timed out after ${this.commandTimeoutMs}ms`,+ {+ cause: error,+ }+ )+ }+ throw error+ } finally {+ clearTimeout(timer)+ }+ }+}diff --git a/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/url-history-expiry.test.ts b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/url-history-expiry.test.tsnew file mode 100644index 000000000000..5dab25d4cc28--- /dev/null+++ b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/url-history-expiry.test.ts@@ -0,0 +1,18 @@+import { urlHistoryExpiresAtMs, urlHistoryStorageExpiresAtMs } from './url-history-expiry'++describe('url history expiry', () => {+ const NOW_MS = Date.parse('2026-10-15T12:00:00Z')+ const TTL_SECONDS = 30 * 24 * 60 * 60++ it('fetches a v4 reference again after the TTL, and keeps its entry until 8 days after the end of its month', () => {+ const nextFetchAtMs = urlHistoryExpiresAtMs(NOW_MS, TTL_SECONDS)++ expect(nextFetchAtMs).toBe(NOW_MS + TTL_SECONDS * 1000)+ expect(urlHistoryStorageExpiresAtMs('imageurl:v4:7:2026-10:aaaaaaaaaaaaaaaaaaaaaa', nextFetchAtMs)).toBe(+ Date.parse('2026-11-09T00:00:00Z')+ )+ expect(urlHistoryStorageExpiresAtMs('imageurl:v4:7:2026-12:aaaaaaaaaaaaaaaaaaaaaa', nextFetchAtMs)).toBe(+ Date.parse('2027-01-09T00:00:00Z')+ )+ })+})diff --git a/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/url-history-expiry.ts b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/url-history-expiry.tsnew file mode 100644index 000000000000..a882606482f9--- /dev/null+++ b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/url-history-expiry.ts@@ -0,0 +1,28 @@+import { explicitFreshnessLifetimeMs } from '~/ingestion/pipelines/sessionreplay/ml-mirror-image-fetch/configuration-policy'++import { HttpCacheMetadata } from './crawl-history'+import { parseV4UrlRef } from './v4-url-ref'++/** A reference is fetched again after the crawl history TTL, or later when the response stays fresh for longer. */+export function urlHistoryExpiresAtMs(nowMs: number, seenTtlSeconds: number, cache?: HttpCacheMetadata): number {+ const minimumNextFetchAtMs = nowMs + seenTtlSeconds * 1000+ const explicitNextFetchAtMs = cache ? nowMs + explicitFreshnessLifetimeMs(cache, nowMs) : 0+ return Math.max(minimumNextFetchAtMs, explicitNextFetchAtMs)+}++/** How long the entry of a reference stays after the end of its month, as in the v3 lane. */+const MONTH_GRACE_DAYS = 8++/**+ * When the history entry of a reference may go. A v4 reference carries the month it was written in, so the recorder+ * sends no job for it after that month. Its entry stays 8 more days for jobs that are still on their way, and a later+ * job finds no entry and fetches the image again.+ */+export function urlHistoryStorageExpiresAtMs(ref: string, nextFetchAtMs: number): number {+ const v4 = parseV4UrlRef(ref)+ if (!v4) {+ return nextFetchAtMs+ }+ const [year, month] = v4.keyMonth.split('-').map(Number)+ return Date.UTC(year, month, 1 + MONTH_GRACE_DAYS)+}