Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(ai-research): keep v4 url history per team month (#117983)

Nicholas Waltz15748768e806
diff --git a/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/crawl-history.ts b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/crawl-history.ts
new file mode 100644
index 000000000000..0bfb1e5a1588
--- /dev/null
+++ b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/crawl-history.ts
@@ -0,0 +1,52 @@
+export interface HttpCacheMetadata {
+ requestTimeMs: number
+ responseTimeMs: number
+ /**
+ * A keyed hash of the URL that sent these headers, never the URL itself. Its validators are sent to that URL only.
+ */
+ responseUrlHmac?: string
+ etag?: string
+ lastModified?: string
+ date?: string
+ age?: string
+ cacheControl?: string
+ expires?: string
+}
+
+export interface UrlCrawlHistoryItem {
+ kind: 'url'
+ key: string
+ nextFetchAtMs: number
+ storageExpiresAtMs: number
+ outcome: string
+ cache?: HttpCacheMetadata
+ /** When the lane last stored or refreshed the image of this reference. */
+ storedAtMs?: number
+}
+
+export type ConfigurationFile = 'robots' | 'tdmrep'
+export type ConfigurationStatus = 'available' | 'absent' | 'refused' | 'unreachable'
+
+export interface ConfigurationCacheItem {
+ kind: ConfigurationFile
+ key: string
+ origin: string
+ status: ConfigurationStatus
+ body?: string
+ fetchedAtMs: number
+ refreshAtMs: number
+ freshUntilMs: number
+ retryAtMs: number
+ storageExpiresAtMs: number
+}
+
+export type CrawlHistoryItem = UrlCrawlHistoryItem | ConfigurationCacheItem
+
+export function configurationCacheKey(origin: string, file: ConfigurationFile): string {
+ return `imgfetch:config:${file}:${origin}`
+}
+
+export interface CrawlHistoryStore {
+ read(keys: string[]): Promise<Map<string, CrawlHistoryItem>>
+ write(items: CrawlHistoryItem[]): Promise<void>
+}
diff --git a/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/dynamodb-crawl-history.integration.test.ts b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/dynamodb-crawl-history.integration.test.ts
new file mode 100644
index 000000000000..38b19b5a0a17
--- /dev/null
+++ b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/dynamodb-crawl-history.integration.test.ts
@@ -0,0 +1,66 @@
+import {
+ CreateTableCommand,
+ DeleteTableCommand,
+ DynamoDBClient,
+ GetItemCommand,
+ waitUntilTableExists,
+} from '@aws-sdk/client-dynamodb'
+
+import { UrlCrawlHistoryItem } from './crawl-history'
+import { DynamoDBCrawlHistory } from './dynamodb-crawl-history'
+
+const DYNAMODB_ENDPOINT = 'http://127.0.0.1:18010'
+const TABLE_NAME = `ai-research-image-fetch-crawl-history-test-${process.pid}-${Date.now()}`
+const NOW_MS = 1_700_000_000_000
+const NOW_SECONDS = Math.floor(NOW_MS / 1000)
+const TTL_SECONDS = 30 * 24 * 60 * 60
+const KEYS = Array.from({ length: 205 }, (_value, index) => `key-${index}`)
+
+describe('DynamoDBCrawlHistory integration', () => {
+ const client = new DynamoDBClient({
+ endpoint: DYNAMODB_ENDPOINT,
+ region: 'us-east-1',
+ credentials: { accessKeyId: 'test', secretAccessKey: 'test' },
+ maxAttempts: 1,
+ })
+
+ beforeAll(async () => {
+ await client.send(
+ new CreateTableCommand({
+ TableName: TABLE_NAME,
+ AttributeDefinitions: [{ AttributeName: 'key', AttributeType: 'S' }],
+ KeySchema: [{ AttributeName: 'key', KeyType: 'HASH' }],
+ BillingMode: 'PAY_PER_REQUEST',
+ })
+ )
+ await waitUntilTableExists({ client, maxWaitTime: 10, minDelay: 1 }, { TableName: TABLE_NAME })
+ })
+
+ afterAll(async () => {
+ await client.send(new DeleteTableCommand({ TableName: TABLE_NAME }))
+ client.destroy()
+ })
+
+ it('records and reads a full ingestion batch through DynamoDB', async () => {
+ const crawlHistory = new DynamoDBCrawlHistory(client, TABLE_NAME, 5_000, 30_000)
+
+ await crawlHistory.validateAccess(NOW_MS)
+ const storageExpiresAtMs = NOW_MS + TTL_SECONDS * 1000
+ const items: UrlCrawlHistoryItem[] = KEYS.map((key) => ({
+ kind: 'url',
+ key,
+ nextFetchAtMs: storageExpiresAtMs,
+ storageExpiresAtMs,
+ outcome: 'ok',
+ }))
+ await crawlHistory.write(items)
+ const readResult = await crawlHistory.read([...KEYS, 'missing'])
+ const storedItem = await client.send(
+ new GetItemCommand({ TableName: TABLE_NAME, Key: { key: { S: KEYS[0] } }, ConsistentRead: true })
+ )
+
+ expect(readResult.size).toBe(KEYS.length)
+ expect([...readResult.keys()].sort()).toEqual([...KEYS].sort())
+ expect(storedItem.Item?.expires_at?.N).toBe(String(NOW_SECONDS + TTL_SECONDS))
+ })
+})
diff --git a/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/dynamodb-crawl-history.test.ts b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/dynamodb-crawl-history.test.ts
new file mode 100644
index 000000000000..3af76f475b7d
--- /dev/null
+++ b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/dynamodb-crawl-history.test.ts
@@ -0,0 +1,415 @@
+import { BatchGetItemCommand, BatchWriteItemCommand } from '@aws-sdk/client-dynamodb'
+
+import { parseJSON } from '~/common/utils/json-parse'
+import { logger } from '~/common/utils/logger'
+
+import { ConfigurationCacheItem, CrawlHistoryItem, UrlCrawlHistoryItem } from './crawl-history'
+import { DynamoDBCrawlHistory, DynamoDBCrawlHistoryClient } from './dynamodb-crawl-history'
+
+const TABLE = 'crawl-history'
+const NOW_MS = 1_700_000_000_123
+
+function urlItem(key: string, overrides: Partial<UrlCrawlHistoryItem> = {}): UrlCrawlHistoryItem {
+ return {
+ kind: 'url',
+ key,
+ nextFetchAtMs: NOW_MS + 30 * 24 * 60 * 60 * 1000,
+ storageExpiresAtMs: NOW_MS + 30 * 24 * 60 * 60 * 1000,
+ outcome: 'ok',
+ ...overrides,
+ }
+}
+
+function stored(item: CrawlHistoryItem): Record<string, { S?: string; N?: string }> {
+ return {
+ key: { S: item.key },
+ expires_at: { N: String(Math.ceil(item.storageExpiresAtMs / 1000)) },
+ value: { S: JSON.stringify(item) },
+ }
+}
+
+function build(send: jest.Mock, batchBudgetMs = 10_000): DynamoDBCrawlHistory {
+ return new DynamoDBCrawlHistory({ send } as unknown as DynamoDBCrawlHistoryClient, TABLE, 1_000, batchBudgetMs)
+}
+
+function requestedReadKeys(command: BatchGetItemCommand): string[] {
+ return command.input.RequestItems?.[TABLE]?.Keys?.flatMap((key) => (key.key?.S ? [key.key.S] : [])) ?? []
+}
+
+function requestedWrites(
+ command: BatchWriteItemCommand
+): NonNullable<BatchWriteItemCommand['input']['RequestItems']>[string] {
+ return command.input.RequestItems?.[TABLE] ?? []
+}
+
+function hangUntilAborted(_command: unknown, { abortSignal }: { abortSignal: AbortSignal }): Promise<never> {
+ return new Promise((_resolve, reject) =>
+ abortSignal.addEventListener('abort', () =>
+ reject(Object.assign(new Error('Request aborted'), { name: 'AbortError' }))
+ )
+ )
+}
+
+describe('DynamoDBCrawlHistory', () => {
+ it('reads each distinct key in batches of at most 100', async () => {
+ const send = jest.fn((command: BatchGetItemCommand) => {
+ const items = requestedReadKeys(command).map((key) => stored(urlItem(key)))
+ return Promise.resolve({ Responses: { [TABLE]: items } })
+ })
+ const keys = [...Array.from({ length: 205 }, (_, index) => `k${index}`), 'k0']
+
+ const result = await build(send).read(keys)
+
+ expect(result).toHaveProperty('size', 205)
+ expect(send).toHaveBeenCalledTimes(3)
+ expect(send.mock.calls.map(([command]) => requestedReadKeys(command))).toEqual(
+ expect.arrayContaining([expect.any(Array), expect.any(Array), expect.any(Array)])
+ )
+ expect(send.mock.calls.every(([command]) => requestedReadKeys(command).length <= 100)).toBe(true)
+ })
+
+ it('reads a legacy expiry-only URL item', async () => {
+ const send = jest.fn(() =>
+ Promise.resolve({
+ Responses: {
+ [TABLE]: [{ key: { S: 'legacy' }, expires_at: { N: String(Math.ceil(NOW_MS / 1000)) } }],
+ },
+ })
+ )
+
+ await expect(build(send).read(['legacy'])).resolves.toEqual(
+ new Map([
+ [
+ 'legacy',
+ {
+ kind: 'url',
+ key: 'legacy',
+ nextFetchAtMs: Math.ceil(NOW_MS / 1000) * 1000,
+ storageExpiresAtMs: Math.ceil(NOW_MS / 1000) * 1000,
+ outcome: 'legacy',
+ },
+ ],
+ ])
+ )
+ })
+
+ it('reads a URL item whose cache holds a plaintext response URL as unbound', async () => {
+ const cache = { requestTimeMs: NOW_MS - 10, responseTimeMs: NOW_MS, etag: '"v1"' }
+ const item = stored(urlItem('a'))
+ item.value = {
+ S: JSON.stringify({ ...urlItem('a'), cache: { ...cache, responseUrl: 'https://cdn.example.com/a.png' } }),
+ }
+ const send = jest.fn(() => Promise.resolve({ Responses: { [TABLE]: [item] } }))
+
+ const result = await build(send).read(['a'])
+
+ expect((result.get('a') as UrlCrawlHistoryItem).cache).toEqual(cache)
+ })
+
+ it('retries only unprocessed read keys', async () => {
+ const send = jest
+ .fn()
+ .mockResolvedValueOnce({
+ Responses: { [TABLE]: [stored(urlItem('done'))] },
+ UnprocessedKeys: { [TABLE]: { Keys: [{ key: { S: 'retry' } }] } },
+ })
+ .mockResolvedValueOnce({ Responses: { [TABLE]: [stored(urlItem('retry'))] } })
+
+ const result = await build(send).read(['done', 'retry'])
+
+ expect([...result.keys()].sort()).toEqual(['done', 'retry'])
+ expect(requestedReadKeys(send.mock.calls[1][0])).toEqual(['retry'])
+ })
+
+ it('throws when a read request fails', async () => {
+ const send = jest.fn(() => Promise.reject(new Error('DynamoDB unavailable')))
+
+ await expect(build(send).read(['a'])).rejects.toThrow('DynamoDB unavailable')
+ })
+
+ it.each([
+ ['an item with no valid expiry', { key: { S: 'a' }, expires_at: { N: 'invalid' } }],
+ ['a value that is not JSON', { ...stored(urlItem('a')), value: { S: '{"kind":"url",' } }],
+ [
+ 'a value with an invalid nested shape',
+ {
+ ...stored(urlItem('a')),
+ value: { S: JSON.stringify({ ...urlItem('a'), cache: { requestTimeMs: 'x' } }) },
+ },
+ ],
+ [
+ 'a kind a newer build wrote',
+ { ...stored(urlItem('a')), value: { S: JSON.stringify({ ...urlItem('a'), kind: 'sitemap' }) } },
+ ],
+ ])('reads %s as a cache miss and still returns the other items', async (_name, item) => {
+ const warn = jest.spyOn(logger, 'warn').mockImplementation()
+ const send = jest.fn(() => Promise.resolve({ Responses: { [TABLE]: [item, stored(urlItem('b'))] } }))
+
+ const result = await build(send).read(['a', 'b'])
+
+ expect([...result.keys()]).toEqual(['b'])
+ expect(warn).toHaveBeenCalledWith('🌐', 'ai_research_v4_image_fetch_crawl_history_item_invalid', {
+ key: 'a',
+ error: expect.any(String),
+ })
+ expect(JSON.stringify(warn.mock.calls)).not.toContain('"kind')
+ warn.mockRestore()
+ })
+
+ it('folds repeated updates and writes batches of at most 25', async () => {
+ const send: jest.Mock<Promise<unknown>, [BatchWriteItemCommand]> = jest.fn((_command) => Promise.resolve({}))
+ const first = urlItem('same', { outcome: 'first' })
+ const last = urlItem('same', { outcome: 'last', storageExpiresAtMs: NOW_MS + 1 })
+ const items = [first, ...Array.from({ length: 55 }, (_, index) => urlItem(`k${index}`)), last]
+
+ await build(send).write(items)
+
+ expect(send).toHaveBeenCalledTimes(3)
+ const requests = send.mock.calls.flatMap(([command]) => requestedWrites(command))
+ expect(send.mock.calls.every(([command]) => requestedWrites(command).length <= 25)).toBe(true)
+ expect(requests).toHaveLength(56)
+ const same = requests.find((request) => request.PutRequest?.Item?.key?.S === 'same')
+ expect(parseJSON(same?.PutRequest?.Item?.value?.S ?? '{}')).toMatchObject({ outcome: 'last' })
+ expect(same?.PutRequest?.Item?.expires_at?.N).toBe(String(Math.ceil((NOW_MS + 1) / 1000)))
+ })
+
+ it('retries only unprocessed writes', async () => {
+ const send = jest
+ .fn()
+ .mockImplementationOnce((command: BatchWriteItemCommand) =>
+ Promise.resolve({ UnprocessedItems: { [TABLE]: [requestedWrites(command)[1]] } })
+ )
+ .mockResolvedValueOnce({})
+
+ await build(send).write([urlItem('done'), urlItem('retry')])
+
+ expect(requestedWrites(send.mock.calls[1][0])).toHaveLength(1)
+ expect(requestedWrites(send.mock.calls[1][0])[0].PutRequest?.Item?.key?.S).toBe('retry')
+ })
+
+ it('splits and reassembles a 500 KiB configuration body', async () => {
+ const dynamoItems = new Map<string, Record<string, { S?: string; N?: string }>>()
+ const send = jest.fn((command: BatchWriteItemCommand | BatchGetItemCommand) => {
+ if (command instanceof BatchWriteItemCommand) {
+ for (const request of requestedWrites(command)) {
+ const item = request.PutRequest?.Item as Record<string, { S?: string; N?: string }>
+ dynamoItems.set(item.key.S ?? '', item)
+ }
+ return Promise.resolve({})
+ }
+ return Promise.resolve({
+ Responses: {
+ [TABLE]: requestedReadKeys(command).flatMap((key) => {
+ const item = dynamoItems.get(key)
+ return item ? [item] : []
+ }),
+ },
+ })
+ })
+ const body = 'abc😀'.repeat(73_000)
+ const item: ConfigurationCacheItem = {
+ kind: 'robots',
+ key: 'imgfetch:config:robots:https://example.com',
+ origin: 'https://example.com',
+ status: 'available',
+ body,
+ fetchedAtMs: NOW_MS,
+ refreshAtMs: NOW_MS + 1,
+ freshUntilMs: NOW_MS + 2,
+ retryAtMs: NOW_MS,
+ storageExpiresAtMs: NOW_MS + 3,
+ }
+ const store = build(send)
+
+ await store.write([item])
+ const result = await store.read([item.key])
+
+ expect(result.get(item.key)).toEqual(item)
+ expect(dynamoItems.size).toBe(4)
+ const writeCalls = send.mock.calls.filter(([command]) => command instanceof BatchWriteItemCommand)
+ expect(requestedWrites(writeCalls[0][0] as BatchWriteItemCommand)).toHaveLength(3)
+ expect(
+ requestedWrites(writeCalls[0][0] as BatchWriteItemCommand).every((request) =>
+ request.PutRequest?.Item?.key?.S?.startsWith(`${item.key}:body:`)
+ )
+ ).toBe(true)
+ expect(requestedWrites(writeCalls[1][0] as BatchWriteItemCommand)[0].PutRequest?.Item?.key?.S).toBe(item.key)
+ expect(
+ [...dynamoItems.values()].every((storedItem) => Buffer.byteLength(JSON.stringify(storedItem)) < 400 * 1024)
+ ).toBe(true)
+ })
+
+ it('treats an incomplete configuration body generation as a cache miss', async () => {
+ const sha256 = 'a'.repeat(64)
+ const item: ConfigurationCacheItem = {
+ kind: 'robots',
+ key: 'imgfetch:config:robots:https://example.com',
+ origin: 'https://example.com',
+ status: 'available',
+ fetchedAtMs: NOW_MS,
+ refreshAtMs: NOW_MS + 1,
+ freshUntilMs: NOW_MS + 2,
+ retryAtMs: NOW_MS,
+ storageExpiresAtMs: NOW_MS + 3,
+ }
+ const manifest = stored(item)
+ manifest.value = { S: JSON.stringify({ ...item, _bodyChunks: { count: 2, sha256 } }) }
+ const send = jest
+ .fn()
+ .mockResolvedValueOnce({ Responses: { [TABLE]: [manifest] } })
+ .mockResolvedValueOnce({ Responses: { [TABLE]: [] } })
+
+ await expect(build(send).read([item.key])).resolves.toEqual(new Map())
+ })
+
+ it('does not publish a manifest when its body chunks cannot be written', async () => {
+ jest.useFakeTimers()
+ jest.spyOn(Math, 'random').mockReturnValue(0)
+ const sentKeys: string[] = []
+ const send = jest.fn((command: BatchWriteItemCommand) => {
+ const writes = requestedWrites(command)
+ sentKeys.push(...writes.flatMap((request) => request.PutRequest?.Item?.key?.S ?? []))
+ return Promise.resolve({ UnprocessedItems: { [TABLE]: writes } })
+ })
+ const key = 'imgfetch:config:robots:https://example.com'
+ const item: ConfigurationCacheItem = {
+ kind: 'robots',
+ key,
+ origin: 'https://example.com',
+ status: 'available',
+ body: 'abc😀'.repeat(73_000),
+ fetchedAtMs: NOW_MS,
+ refreshAtMs: NOW_MS + 1,
+ freshUntilMs: NOW_MS + 2,
+ retryAtMs: NOW_MS,
+ storageExpiresAtMs: NOW_MS + 3,
+ }
+
+ const write = build(send).write([item])
+ const refused = expect(write).rejects.toThrow('crawl-history writes unprocessed')
+ await jest.runAllTimersAsync()
+
+ await refused
+ expect(sentKeys.length).toBeGreaterThan(0)
+ expect(sentKeys.every((sentKey) => sentKey.startsWith(`${key}:body:`))).toBe(true)
+ jest.useRealTimers()
+ })
+
+ it.each([
+ {
+ operation: 'read',
+ run: (history: DynamoDBCrawlHistory) => history.read(['slow']),
+ retried: { Responses: { [TABLE]: [stored(urlItem('slow'))] } },
+ requestedKeys: (command: BatchGetItemCommand | BatchWriteItemCommand) =>
+ requestedReadKeys(command as BatchGetItemCommand),
+ },
+ {
+ operation: 'write',
+ run: (history: DynamoDBCrawlHistory) => history.write([urlItem('slow')]),
+ retried: {},
+ requestedKeys: (command: BatchGetItemCommand | BatchWriteItemCommand) =>
+ requestedWrites(command as BatchWriteItemCommand).map((request) => request.PutRequest?.Item?.key?.S),
+ },
+ ])('retries a $operation command that times out', async ({ run, retried, requestedKeys }) => {
+ jest.useFakeTimers()
+ jest.spyOn(Math, 'random').mockReturnValue(0)
+ const send = jest.fn().mockImplementationOnce(hangUntilAborted).mockResolvedValueOnce(retried)
+
+ const outcome = run(build(send)).then(
+ () => 'completed',
+ (error: unknown) => String(error)
+ )
+ await jest.runAllTimersAsync()
+
+ expect(await outcome).toBe('completed')
+ expect(send).toHaveBeenCalledTimes(2)
+ expect(requestedKeys(send.mock.calls[1][0])).toEqual(['slow'])
+ jest.useRealTimers()
+ })
+
+ it.each([
+ { operation: 'read', run: (history: DynamoDBCrawlHistory) => history.read(['slow']) },
+ { operation: 'write', run: (history: DynamoDBCrawlHistory) => history.write([urlItem('slow')]) },
+ ])('fails a $operation after every attempt times out', async ({ run }) => {
+ jest.useFakeTimers()
+ jest.spyOn(Math, 'random').mockReturnValue(0)
+ const send = jest.fn(hangUntilAborted)
+
+ const outcome = run(build(send)).then(
+ () => 'completed',
+ (error: unknown) => String(error)
+ )
+ await jest.runAllTimersAsync()
+
+ expect(await outcome).toBe('Error: DynamoDB crawl-history command timed out after 1000ms')
+ expect(send).toHaveBeenCalledTimes(5)
+ jest.useRealTimers()
+ })
+
+ it('throws after unprocessed write retries are exhausted', async () => {
+ jest.useFakeTimers()
+ jest.spyOn(Math, 'random').mockReturnValue(0)
+ const send = jest.fn((command: BatchWriteItemCommand) =>
+ Promise.resolve({ UnprocessedItems: { [TABLE]: requestedWrites(command) } })
+ )
+
+ const write = build(send).write([urlItem('retry')])
+ const refused = expect(write).rejects.toThrow('crawl-history writes unprocessed: retry')
+ await jest.runAllTimersAsync()
+
+ await refused
+ expect(send).toHaveBeenCalledTimes(5)
+ jest.useRealTimers()
+ })
+
+ it.each([
+ {
+ operation: 'read',
+ run: (history: DynamoDBCrawlHistory, keys: string[]) => history.read(keys),
+ answerOne: (command: BatchGetItemCommand | BatchWriteItemCommand) => {
+ const [first, ...rest] = requestedReadKeys(command as BatchGetItemCommand)
+ return {
+ Responses: { [TABLE]: [stored(urlItem(first))] },
+ UnprocessedKeys: { [TABLE]: { Keys: rest.map((key) => ({ key: { S: key } })) } },
+ }
+ },
+ },
+ {
+ operation: 'write',
+ run: (history: DynamoDBCrawlHistory, keys: string[]) => history.write(keys.map((key) => urlItem(key))),
+ answerOne: (command: BatchGetItemCommand | BatchWriteItemCommand) => ({
+ UnprocessedItems: { [TABLE]: requestedWrites(command as BatchWriteItemCommand).slice(1) },
+ }),
+ },
+ ])('drains a $operation that makes progress on every call', async ({ run, answerOne }) => {
+ jest.useFakeTimers()
+ jest.spyOn(Math, 'random').mockReturnValue(0)
+ const send = jest.fn((command: BatchGetItemCommand | BatchWriteItemCommand) =>
+ Promise.resolve(answerOne(command))
+ )
+ const keys = Array.from({ length: 8 }, (_, index) => `k${index}`)
+
+ const outcome = run(build(send), keys).then(
+ () => 'completed',
+ (error: unknown) => String(error)
+ )
+ await jest.runAllTimersAsync()
+
+ expect(await outcome).toBe('completed')
+ expect(send).toHaveBeenCalledTimes(8)
+ jest.useRealTimers()
+ })
+
+ it('validates write and consistent read access', async () => {
+ const send = jest
+ .fn()
+ .mockResolvedValueOnce({})
+ .mockResolvedValueOnce({ Responses: { [TABLE]: [{ key: { S: 'imgfetch:access-probe' } }] } })
+
+ await build(send).validateAccess(NOW_MS)
+
+ expect(send.mock.calls[0][0]).toBeInstanceOf(BatchWriteItemCommand)
+ expect(send.mock.calls[1][0]).toBeInstanceOf(BatchGetItemCommand)
+ expect(send.mock.calls[1][0].input.RequestItems?.[TABLE]?.ConsistentRead).toBe(true)
+ })
+})
diff --git a/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/dynamodb-crawl-history.ts b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/dynamodb-crawl-history.ts
new file mode 100644
index 000000000000..40a44a1fbf0f
--- /dev/null
+++ b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/dynamodb-crawl-history.ts
@@ -0,0 +1,506 @@
+import {
+ AttributeValue,
+ BatchGetItemCommand,
+ BatchGetItemCommandOutput,
+ BatchWriteItemCommand,
+ BatchWriteItemCommandOutput,
+ DynamoDBClient,
+ WriteRequest,
+} from '@aws-sdk/client-dynamodb'
+import { createHash } from 'node:crypto'
+import { Counter } from 'prom-client'
+
+import { parseJSON } from '~/common/utils/json-parse'
+import { logger } from '~/common/utils/logger'
+
+import { ConfigurationCacheItem, CrawlHistoryItem, CrawlHistoryStore, HttpCacheMetadata } from './crawl-history'
+
+const KEY_ATTRIBUTE = 'key'
+const EXPIRES_AT_ATTRIBUTE = 'expires_at'
+const VALUE_ATTRIBUTE = 'value'
+const BATCH_GET_SIZE = 100
+const BATCH_WRITE_SIZE = 25
+const MAX_CONCURRENT_BATCH_REQUESTS = 8
+const UNPROCESSED_MAX_ATTEMPTS = 5
+const UNPROCESSED_INITIAL_BACKOFF_MS = 50
+const ACCESS_PROBE_KEY = 'imgfetch:access-probe'
+const ACCESS_PROBE_TTL_SECONDS = 5 * 60
+const SAFE_ITEM_SIZE_BYTES = 350 * 1024
+const BODY_CHUNK_SIZE_BYTES = 240 * 1024
+const MAX_BODY_CHUNKS = 3
+
+const invalidItems = new Counter({
+ name: 'ai_research_v4_image_fetch_crawl_history_invalid_items_total',
+ help: 'v4 crawl history items that did not parse and were read as cache misses.',
+})
+
+interface StoredBodyChunks {
+ count: number
+ sha256: string
+}
+
+type ParsedStoredItem =
+ | { item: CrawlHistoryItem; bodyChunks?: undefined }
+ | { item: ConfigurationCacheItem; bodyChunks: StoredBodyChunks }
+
+interface ItemWritePlan {
+ bodyChunkRequests: WriteRequest[]
+ manifestRequest: WriteRequest
+}
+
+export type DynamoDBCrawlHistoryClient = Pick<DynamoDBClient, 'send'>
+
+class CommandTimeoutError extends Error {}
+
+function chunk<T>(items: T[], size: number): T[][] {
+ const chunks: T[][] = []
+ for (let index = 0; index < items.length; index += size) {
+ chunks.push(items.slice(index, index + size))
+ }
+ return chunks
+}
+
+function keyFromWriteRequest(request: WriteRequest): string | undefined {
+ return request.PutRequest?.Item?.[KEY_ATTRIBUTE]?.S
+}
+
+function isRecord(value: unknown): value is Record<string, unknown> {
+ return typeof value === 'object' && value !== null && !Array.isArray(value)
+}
+
+function isFiniteNumber(value: unknown): value is number {
+ return typeof value === 'number' && Number.isFinite(value)
+}
+
+function isOptionalString(value: unknown): value is string | undefined {
+ return value === undefined || typeof value === 'string'
+}
+
+function isHttpCacheMetadata(value: unknown): boolean {
+ if (!isRecord(value) || !isFiniteNumber(value.requestTimeMs) || !isFiniteNumber(value.responseTimeMs)) {
+ return false
+ }
+ return ['responseUrlHmac', 'responseUrl', 'etag', 'lastModified', 'date', 'age', 'cacheControl', 'expires'].every(
+ (field) => isOptionalString(value[field])
+ )
+}
+
+function isCrawlHistoryItem(value: unknown, expectedKey: string): value is CrawlHistoryItem {
+ if (!isRecord(value) || value.key !== expectedKey || !isFiniteNumber(value.storageExpiresAtMs)) {
+ return false
+ }
+ if (value.kind === 'url') {
+ return (
+ isFiniteNumber(value.nextFetchAtMs) &&
+ typeof value.outcome === 'string' &&
+ (value.cache === undefined || isHttpCacheMetadata(value.cache)) &&
+ (value.storedAtMs === undefined || isFiniteNumber(value.storedAtMs))
+ )
+ }
+ return (
+ (value.kind === 'robots' || value.kind === 'tdmrep') &&
+ typeof value.origin === 'string' &&
+ ['available', 'absent', 'refused', 'unreachable'].includes(String(value.status)) &&
+ isOptionalString(value.body) &&
+ isFiniteNumber(value.fetchedAtMs) &&
+ isFiniteNumber(value.refreshAtMs) &&
+ isFiniteNumber(value.freshUntilMs) &&
+ isFiniteNumber(value.retryAtMs)
+ )
+}
+
+/**
+ * An older row can hold the response URL in plaintext. It reads as unbound, so its validators are not sent, and the
+ * next write of the row does not carry the URL.
+ */
+function withoutPlaintextResponseUrl(item: CrawlHistoryItem): CrawlHistoryItem {
+ if (item.kind !== 'url' || !item.cache || !('responseUrl' in item.cache)) {
+ return item
+ }
+ const { responseUrl: _plaintext, ...cache } = item.cache as HttpCacheMetadata & { responseUrl?: string }
+ return { ...item, cache }
+}
+
+function parseStoredItem(item: Record<string, AttributeValue>): ParsedStoredItem {
+ const key = item[KEY_ATTRIBUTE]?.S
+ const expiresAtSeconds = Number(item[EXPIRES_AT_ATTRIBUTE]?.N)
+ if (!key || !Number.isFinite(expiresAtSeconds)) {
+ throw new Error('DynamoDB returned a malformed crawl-history item')
+ }
+ const serialized = item[VALUE_ATTRIBUTE]?.S
+ if (!serialized) {
+ return {
+ item: {
+ kind: 'url',
+ key,
+ nextFetchAtMs: expiresAtSeconds * 1000,
+ storageExpiresAtMs: expiresAtSeconds * 1000,
+ outcome: 'legacy',
+ },
+ }
+ }
+ const parsed = parseJSON(serialized)
+ if (!isRecord(parsed)) {
+ throw new Error('DynamoDB returned an invalid crawl-history value')
+ }
+ const storedBodyChunks = parsed._bodyChunks
+ if (storedBodyChunks === undefined) {
+ if (!isCrawlHistoryItem(parsed, key)) {
+ throw new Error('DynamoDB returned an invalid crawl-history value')
+ }
+ return { item: withoutPlaintextResponseUrl(parsed) }
+ }
+ const { _bodyChunks: _storedBodyChunks, ...applicationItem } = parsed
+ if (
+ !isCrawlHistoryItem(applicationItem, key) ||
+ applicationItem.kind === 'url' ||
+ applicationItem.body !== undefined ||
+ !isRecord(storedBodyChunks) ||
+ !Number.isInteger(storedBodyChunks.count) ||
+ (storedBodyChunks.count as number) < 1 ||
+ (storedBodyChunks.count as number) > MAX_BODY_CHUNKS ||
+ typeof storedBodyChunks.sha256 !== 'string' ||
+ !/^[0-9a-f]{64}$/.test(storedBodyChunks.sha256)
+ ) {
+ throw new Error('DynamoDB returned invalid crawl-history body chunks')
+ }
+ return {
+ item: applicationItem,
+ bodyChunks: { count: storedBodyChunks.count as number, sha256: storedBodyChunks.sha256 },
+ }
+}
+
+function bodyChunkKey(key: string, sha256: string, index: number): string {
+ return `${key}:body:${sha256}:${index}`
+}
+
+function putRequest(key: string, storageExpiresAtMs: number, serialized: string): WriteRequest {
+ const expiresAt = String(Math.ceil(storageExpiresAtMs / 1000))
+ const sizeBytes =
+ Buffer.byteLength(KEY_ATTRIBUTE) +
+ Buffer.byteLength(key) +
+ Buffer.byteLength(EXPIRES_AT_ATTRIBUTE) +
+ Buffer.byteLength(expiresAt) +
+ Buffer.byteLength(VALUE_ATTRIBUTE) +
+ Buffer.byteLength(serialized)
+ if (sizeBytes > SAFE_ITEM_SIZE_BYTES) {
+ throw new Error(`crawl-history item ${key} exceeds the safe DynamoDB item size`)
+ }
+ return {
+ PutRequest: {
+ Item: {
+ [KEY_ATTRIBUTE]: { S: key },
+ [EXPIRES_AT_ATTRIBUTE]: { N: expiresAt },
+ [VALUE_ATTRIBUTE]: { S: serialized },
+ },
+ },
+ }
+}
+
+function writePlanForItem(item: CrawlHistoryItem): ItemWritePlan {
+ const serialized = JSON.stringify(item)
+ if (Buffer.byteLength(serialized) <= SAFE_ITEM_SIZE_BYTES / 2 || item.kind === 'url' || !item.body) {
+ return {
+ bodyChunkRequests: [],
+ manifestRequest: putRequest(item.key, item.storageExpiresAtMs, serialized),
+ }
+ }
+ const body = Buffer.from(item.body, 'utf8')
+ const bodyChunks: Buffer[] = []
+ for (let offset = 0; offset < body.length; offset += BODY_CHUNK_SIZE_BYTES) {
+ bodyChunks.push(body.subarray(offset, offset + BODY_CHUNK_SIZE_BYTES))
+ }
+ if (bodyChunks.length > MAX_BODY_CHUNKS) {
+ throw new Error(`crawl-history body ${item.key} needs too many DynamoDB items`)
+ }
+ const sha256 = createHash('sha256').update(body).digest('hex')
+ const storedItem = {
+ ...item,
+ body: undefined,
+ _bodyChunks: {
+ count: bodyChunks.length,
+ sha256,
+ },
+ }
+ return {
+ bodyChunkRequests: bodyChunks.map((bodyChunk, index) =>
+ putRequest(bodyChunkKey(item.key, sha256, index), item.storageExpiresAtMs, bodyChunk.toString('base64'))
+ ),
+ manifestRequest: putRequest(item.key, item.storageExpiresAtMs, JSON.stringify(storedItem)),
+ }
+}
+
+export class DynamoDBCrawlHistory implements CrawlHistoryStore {
+ constructor(
+ private readonly client: DynamoDBCrawlHistoryClient,
+ private readonly tableName: string,
+ private readonly commandTimeoutMs: number,
+ private readonly batchBudgetMs: number
+ ) {}
+
+ public async validateAccess(nowMs: number): Promise<void> {
+ const expiresAt = Math.floor(nowMs / 1000) + ACCESS_PROBE_TTL_SECONDS
+ const writeResponse = await this.sendBatchWrite(
+ new BatchWriteItemCommand({
+ RequestItems: {
+ [this.tableName]: [
+ {
+ PutRequest: {
+ Item: {
+ [KEY_ATTRIBUTE]: { S: ACCESS_PROBE_KEY },
+ [EXPIRES_AT_ATTRIBUTE]: { N: String(expiresAt) },
+ },
+ },
+ },
+ ],
+ },
+ })
+ )
+ if ((writeResponse.UnprocessedItems?.[this.tableName] ?? []).length > 0) {
+ throw new Error('DynamoDB crawl-history access probe write was not processed')
+ }
+ const readResponse = await this.sendBatchGet(
+ new BatchGetItemCommand({
+ RequestItems: {
+ [this.tableName]: {
+ Keys: [{ [KEY_ATTRIBUTE]: { S: ACCESS_PROBE_KEY } }],
+ ProjectionExpression: '#key',
+ ExpressionAttributeNames: { '#key': KEY_ATTRIBUTE },
+ ConsistentRead: true,
+ },
+ },
+ })
+ )
+ if (
+ (readResponse.UnprocessedKeys?.[this.tableName]?.Keys ?? []).length > 0 ||
+ !(readResponse.Responses?.[this.tableName] ?? []).some(
+ (item) => item[KEY_ATTRIBUTE]?.S === ACCESS_PROBE_KEY
+ )
+ ) {
+ throw new Error('DynamoDB crawl-history access probe read was not processed')
+ }
+ }
+
+ public async read(keys: string[]): Promise<Map<string, CrawlHistoryItem>> {
+ const result = new Map<string, CrawlHistoryItem>()
+ const startedAt = process.hrtime.bigint()
+ const bodyChunks = new Map<string, { item: ConfigurationCacheItem; metadata: StoredBodyChunks }>()
+ for (const item of await this.readRawItems([...new Set(keys)], startedAt)) {
+ let parsed: ParsedStoredItem
+ try {
+ parsed = parseStoredItem(item)
+ } catch (error) {
+ // A throw would fail the batch on every redelivery, as would a row that a newer build wrote before a
+ // rollback. A miss lets the lane fetch again and overwrite the row.
+ invalidItems.inc()
+ logger.warn('🌐', 'ai_research_v4_image_fetch_crawl_history_item_invalid', {
+ key: item[KEY_ATTRIBUTE]?.S ?? 'missing',
+ // A JSON syntax error quotes the stored value, so only its name is logged.
+ error:
+ error instanceof SyntaxError ? error.name : error instanceof Error ? error.message : 'unknown',
+ })
+ continue
+ }
+ if (parsed.bodyChunks) {
+ bodyChunks.set(parsed.item.key, { item: parsed.item, metadata: parsed.bodyChunks })
+ } else {
+ result.set(parsed.item.key, parsed.item)
+ }
+ }
+ if (bodyChunks.size === 0) {
+ return result
+ }
+
+ const chunkKeys = [...bodyChunks].flatMap(([key, value]) =>
+ Array.from({ length: value.metadata.count }, (_unused, index) =>
+ bodyChunkKey(key, value.metadata.sha256, index)
+ )
+ )
+ const encodedChunks = new Map<string, string>()
+ for (const item of await this.readRawItems(chunkKeys, startedAt)) {
+ const key = item[KEY_ATTRIBUTE]?.S
+ const encoded = item[VALUE_ATTRIBUTE]?.S
+ if (!key || !encoded) {
+ logger.warn('🌐', 'ai_research_v4_image_fetch_crawl_history_chunk_invalid', { key: key ?? 'missing' })
+ continue
+ }
+ encodedChunks.set(key, encoded)
+ }
+ for (const [key, value] of bodyChunks) {
+ try {
+ const chunks = Array.from({ length: value.metadata.count }, (_unused, index) => {
+ const encoded = encodedChunks.get(bodyChunkKey(key, value.metadata.sha256, index))
+ if (!encoded) {
+ throw new Error('missing')
+ }
+ const decoded = Buffer.from(encoded, 'base64')
+ if (decoded.toString('base64') !== encoded) {
+ throw new Error('invalid base64')
+ }
+ return decoded
+ })
+ const body = Buffer.concat(chunks)
+ if (createHash('sha256').update(body).digest('hex') !== value.metadata.sha256) {
+ throw new Error('checksum mismatch')
+ }
+ result.set(key, { ...value.item, body: new TextDecoder('utf-8', { fatal: true }).decode(body) })
+ } catch (error) {
+ logger.warn('🌐', 'ai_research_v4_image_fetch_crawl_history_chunk_incomplete', {
+ key,
+ error: String(error),
+ })
+ }
+ }
+ return result
+ }
+
+ private async readRawItems(keys: string[], startedAt: bigint): Promise<Record<string, AttributeValue>[]> {
+ const result: Record<string, AttributeValue>[] = []
+ await this.runChunks(chunk(keys, BATCH_GET_SIZE), async (batch) => {
+ let pending = batch
+ let attempt = 1
+ while (pending.length > 0) {
+ this.requireBudget(startedAt)
+ let response: BatchGetItemCommandOutput
+ try {
+ response = await this.sendBatchGet(
+ new BatchGetItemCommand({
+ RequestItems: {
+ [this.tableName]: {
+ Keys: pending.map((key) => ({ [KEY_ATTRIBUTE]: { S: key } })),
+ ProjectionExpression: '#key, #expiresAt, #value',
+ ExpressionAttributeNames: {
+ '#key': KEY_ATTRIBUTE,
+ '#expiresAt': EXPIRES_AT_ATTRIBUTE,
+ '#value': VALUE_ATTRIBUTE,
+ },
+ },
+ },
+ })
+ )
+ } catch (error) {
+ await this.backoffAfterCommandTimeout(error, attempt++)
+ continue
+ }
+ for (const item of response.Responses?.[this.tableName] ?? []) {
+ result.push(item)
+ }
+ const unprocessed = (response.UnprocessedKeys?.[this.tableName]?.Keys ?? []).flatMap((key) =>
+ key[KEY_ATTRIBUTE]?.S ? [key[KEY_ATTRIBUTE].S] : []
+ )
+ // DynamoDB answers at most 1 MiB per partition per call, so only calls without progress count.
+ attempt = unprocessed.length < pending.length ? 1 : attempt
+ pending = unprocessed
+ if (pending.length > 0) {
+ if (attempt >= UNPROCESSED_MAX_ATTEMPTS) {
+ throw new Error(`DynamoDB left ${pending.length} crawl-history reads unprocessed`)
+ }
+ await this.backoffBeforeRetry(attempt++)
+ }
+ }
+ })
+ return result
+ }
+
+ public async write(items: CrawlHistoryItem[]): Promise<void> {
+ const byKey = new Map(items.map((item) => [item.key, item]))
+ const plans = [...byKey.values()].map(writePlanForItem)
+ const startedAt = process.hrtime.bigint()
+ await this.writeRequests(
+ plans.flatMap((plan) => plan.bodyChunkRequests),
+ startedAt
+ )
+ await this.writeRequests(
+ plans.map((plan) => plan.manifestRequest),
+ startedAt
+ )
+ }
+
+ private async writeRequests(requests: WriteRequest[], startedAt: bigint): Promise<void> {
+ await this.runChunks(chunk(requests, BATCH_WRITE_SIZE), async (batch) => {
+ let pending = batch
+ let attempt = 1
+ while (pending.length > 0) {
+ this.requireBudget(startedAt)
+ let response: BatchWriteItemCommandOutput
+ try {
+ response = await this.sendBatchWrite(
+ new BatchWriteItemCommand({ RequestItems: { [this.tableName]: pending } })
+ )
+ } catch (error) {
+ await this.backoffAfterCommandTimeout(error, attempt++)
+ continue
+ }
+ const unprocessed = response.UnprocessedItems?.[this.tableName] ?? []
+ attempt = unprocessed.length < pending.length ? 1 : attempt
+ pending = unprocessed
+ if (pending.length > 0) {
+ if (attempt >= UNPROCESSED_MAX_ATTEMPTS) {
+ const keys = pending.flatMap((request) => keyFromWriteRequest(request) ?? [])
+ throw new Error(`DynamoDB left crawl-history writes unprocessed: ${keys.join(',')}`)
+ }
+ await this.backoffBeforeRetry(attempt++)
+ }
+ }
+ })
+ }
+
+ private async runChunks<T>(chunks: T[][], run: (batch: T[]) => Promise<void>): Promise<void> {
+ let nextChunk = 0
+ const worker = async (): Promise<void> => {
+ while (nextChunk < chunks.length) {
+ await run(chunks[nextChunk++])
+ }
+ }
+ await Promise.all(Array.from({ length: Math.min(chunks.length, MAX_CONCURRENT_BATCH_REQUESTS) }, worker))
+ }
+
+ private requireBudget(startedAt: bigint): void {
+ if (Number(process.hrtime.bigint() - startedAt) / 1e6 > this.batchBudgetMs) {
+ throw new Error('DynamoDB crawl-history batch budget was exhausted')
+ }
+ }
+
+ /** A batch get or batch write of the same items is idempotent, so a timed-out command is retried like unprocessed items; any other failure, or a timeout on the last attempt, is rethrown. */
+ private async backoffAfterCommandTimeout(error: unknown, attempt: number): Promise<void> {
+ if (!(error instanceof CommandTimeoutError) || attempt >= UNPROCESSED_MAX_ATTEMPTS) {
+ throw error
+ }
+ await this.backoffBeforeRetry(attempt)
+ }
+
+ private async backoffBeforeRetry(attempt: number): Promise<void> {
+ const maximumDelayMs = UNPROCESSED_INITIAL_BACKOFF_MS * 2 ** (attempt - 1)
+ const delayMs = Math.floor(maximumDelayMs / 2 + Math.random() * (maximumDelayMs / 2))
+ await new Promise<void>((resolve) => setTimeout(resolve, delayMs))
+ }
+
+ private sendBatchGet(command: BatchGetItemCommand): Promise<BatchGetItemCommandOutput> {
+ return this.withTimeout((abortSignal) => this.client.send(command, { abortSignal }))
+ }
+
+ private sendBatchWrite(command: BatchWriteItemCommand): Promise<BatchWriteItemCommandOutput> {
+ return this.withTimeout((abortSignal) => this.client.send(command, { abortSignal }))
+ }
+
+ private async withTimeout<T>(send: (abortSignal: AbortSignal) => Promise<T>): Promise<T> {
+ const abortController = new AbortController()
+ const timer = setTimeout(() => abortController.abort(), this.commandTimeoutMs)
+ timer.unref()
+ try {
+ return await send(abortController.signal)
+ } catch (error) {
+ if (abortController.signal.aborted) {
+ throw new CommandTimeoutError(
+ `DynamoDB crawl-history command timed out after ${this.commandTimeoutMs}ms`,
+ {
+ cause: error,
+ }
+ )
+ }
+ throw error
+ } finally {
+ clearTimeout(timer)
+ }
+ }
+}
diff --git a/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/url-history-expiry.test.ts b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/url-history-expiry.test.ts
new file mode 100644
index 000000000000..5dab25d4cc28
--- /dev/null
+++ b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/url-history-expiry.test.ts
@@ -0,0 +1,18 @@
+import { urlHistoryExpiresAtMs, urlHistoryStorageExpiresAtMs } from './url-history-expiry'
+
+describe('url history expiry', () => {
+ const NOW_MS = Date.parse('2026-10-15T12:00:00Z')
+ const TTL_SECONDS = 30 * 24 * 60 * 60
+
+ it('fetches a v4 reference again after the TTL, and keeps its entry until 8 days after the end of its month', () => {
+ const nextFetchAtMs = urlHistoryExpiresAtMs(NOW_MS, TTL_SECONDS)
+
+ expect(nextFetchAtMs).toBe(NOW_MS + TTL_SECONDS * 1000)
+ expect(urlHistoryStorageExpiresAtMs('imageurl:v4:7:2026-10:aaaaaaaaaaaaaaaaaaaaaa', nextFetchAtMs)).toBe(
+ Date.parse('2026-11-09T00:00:00Z')
+ )
+ expect(urlHistoryStorageExpiresAtMs('imageurl:v4:7:2026-12:aaaaaaaaaaaaaaaaaaaaaa', nextFetchAtMs)).toBe(
+ Date.parse('2027-01-09T00:00:00Z')
+ )
+ })
+})
diff --git a/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/url-history-expiry.ts b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/url-history-expiry.ts
new file mode 100644
index 000000000000..a882606482f9
--- /dev/null
+++ b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/url-history-expiry.ts
@@ -0,0 +1,28 @@
+import { explicitFreshnessLifetimeMs } from '~/ingestion/pipelines/sessionreplay/ml-mirror-image-fetch/configuration-policy'
+
+import { HttpCacheMetadata } from './crawl-history'
+import { parseV4UrlRef } from './v4-url-ref'
+
+/** A reference is fetched again after the crawl history TTL, or later when the response stays fresh for longer. */
+export function urlHistoryExpiresAtMs(nowMs: number, seenTtlSeconds: number, cache?: HttpCacheMetadata): number {
+ const minimumNextFetchAtMs = nowMs + seenTtlSeconds * 1000
+ const explicitNextFetchAtMs = cache ? nowMs + explicitFreshnessLifetimeMs(cache, nowMs) : 0
+ return Math.max(minimumNextFetchAtMs, explicitNextFetchAtMs)
+}
+
+/** How long the entry of a reference stays after the end of its month, as in the v3 lane. */
+const MONTH_GRACE_DAYS = 8
+
+/**
+ * When the history entry of a reference may go. A v4 reference carries the month it was written in, so the recorder
+ * sends no job for it after that month. Its entry stays 8 more days for jobs that are still on their way, and a later
+ * job finds no entry and fetches the image again.
+ */
+export function urlHistoryStorageExpiresAtMs(ref: string, nextFetchAtMs: number): number {
+ const v4 = parseV4UrlRef(ref)
+ if (!v4) {
+ return nextFetchAtMs
+ }
+ const [year, month] = v4.keyMonth.split('-').map(Number)
+ return Date.UTC(year, month, 1 + MONTH_GRACE_DAYS)
+}