Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(ai-research-v4): check fetch history before sending image jobs (#127591)

Robbie4ebce8a94cc1
diff --git a/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/dynamodb-crawl-history.ts b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/dynamodb-crawl-history.ts
index 40a44a1fbf0f..e65ade75df58 100644
--- a/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/dynamodb-crawl-history.ts
+++ b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/dynamodb-crawl-history.ts
@@ -13,7 +13,14 @@ import { Counter } from 'prom-client'
import { parseJSON } from '~/common/utils/json-parse'
import { logger } from '~/common/utils/logger'
-import { ConfigurationCacheItem, CrawlHistoryItem, CrawlHistoryStore, HttpCacheMetadata } from './crawl-history'
+import {
+ ConfigurationCacheItem,
+ CrawlHistoryItem,
+ CrawlHistoryStore,
+ HttpCacheMetadata,
+ UrlCrawlHistoryItem,
+} from './crawl-history'
+import { isUrlHistoryValue } from './politeness-key'
const KEY_ATTRIBUTE = 'key'
const EXPIRES_AT_ATTRIBUTE = 'expires_at'
@@ -76,27 +83,10 @@ function isOptionalString(value: unknown): value is string | undefined {
return value === undefined || typeof value === 'string'
}
-function isHttpCacheMetadata(value: unknown): boolean {
- if (!isRecord(value) || !isFiniteNumber(value.requestTimeMs) || !isFiniteNumber(value.responseTimeMs)) {
- return false
- }
- return ['responseUrlHmac', 'responseUrl', 'etag', 'lastModified', 'date', 'age', 'cacheControl', 'expires'].every(
- (field) => isOptionalString(value[field])
- )
-}
-
-function isCrawlHistoryItem(value: unknown, expectedKey: string): value is CrawlHistoryItem {
+function isConfigurationCacheItem(value: unknown, expectedKey: string): value is ConfigurationCacheItem {
if (!isRecord(value) || value.key !== expectedKey || !isFiniteNumber(value.storageExpiresAtMs)) {
return false
}
- if (value.kind === 'url') {
- return (
- isFiniteNumber(value.nextFetchAtMs) &&
- typeof value.outcome === 'string' &&
- (value.cache === undefined || isHttpCacheMetadata(value.cache)) &&
- (value.storedAtMs === undefined || isFiniteNumber(value.storedAtMs))
- )
- }
return (
(value.kind === 'robots' || value.kind === 'tdmrep') &&
typeof value.origin === 'string' &&
@@ -113,8 +103,8 @@ function isCrawlHistoryItem(value: unknown, expectedKey: string): value is Crawl
* An older row can hold the response URL in plaintext. It reads as unbound, so its validators are not sent, and the
* next write of the row does not carry the URL.
*/
-function withoutPlaintextResponseUrl(item: CrawlHistoryItem): CrawlHistoryItem {
- if (item.kind !== 'url' || !item.cache || !('responseUrl' in item.cache)) {
+function withoutPlaintextResponseUrl(item: UrlCrawlHistoryItem): UrlCrawlHistoryItem {
+ if (!item.cache || !('responseUrl' in item.cache)) {
return item
}
const { responseUrl: _plaintext, ...cache } = item.cache as HttpCacheMetadata & { responseUrl?: string }
@@ -143,17 +133,22 @@ function parseStoredItem(item: Record<string, AttributeValue>): ParsedStoredItem
if (!isRecord(parsed)) {
throw new Error('DynamoDB returned an invalid crawl-history value')
}
+ if (parsed.kind === 'url') {
+ if (!isUrlHistoryValue(key, serialized)) {
+ throw new Error('DynamoDB returned an invalid crawl-history value')
+ }
+ return { item: withoutPlaintextResponseUrl(parsed as unknown as UrlCrawlHistoryItem) }
+ }
const storedBodyChunks = parsed._bodyChunks
if (storedBodyChunks === undefined) {
- if (!isCrawlHistoryItem(parsed, key)) {
+ if (!isConfigurationCacheItem(parsed, key)) {
throw new Error('DynamoDB returned an invalid crawl-history value')
}
- return { item: withoutPlaintextResponseUrl(parsed) }
+ return { item: parsed }
}
const { _bodyChunks: _storedBodyChunks, ...applicationItem } = parsed
if (
- !isCrawlHistoryItem(applicationItem, key) ||
- applicationItem.kind === 'url' ||
+ !isConfigurationCacheItem(applicationItem, key) ||
applicationItem.body !== undefined ||
!isRecord(storedBodyChunks) ||
!Number.isInteger(storedBodyChunks.count) ||
diff --git a/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/fetch-runner.test.ts b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/fetch-runner.test.ts
index 324fa4e4cbca..888bc39a969a 100644
--- a/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/fetch-runner.test.ts
+++ b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/fetch-runner.test.ts
@@ -110,7 +110,7 @@ function build(
{ createPass } as unknown as ConfigurationPolicyService,
options,
{ createRepublishBatch } as unknown as FrontierPublisher,
- { refreshIntervalMs: 7 * 24 * 60 * 60 * 1000, publishImage, refreshImage: () => Promise.resolve(true) },
+ { publishImage, refreshImage: () => Promise.resolve(true) },
{ bindingFor: () => Promise.resolve(bind) },
new ImageFetchTopHogMetrics(recordingTopHog.registry)
)
diff --git a/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/image-sink.ts b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/image-sink.ts
index d1692c79214e..81bcfcb65ab4 100644
--- a/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/image-sink.ts
+++ b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/image-sink.ts
@@ -5,11 +5,10 @@ import type { ImageFetchResult } from './image-fetcher'
* Where the lane stores each fetched image. The image counts as handled only once the sink resolves: a rejection keeps
* the URL unfinished, so it is fetched again.
*
- * S3 expires an object by its creation time, so the lane refreshes an image that went `refreshIntervalMs` without one,
- * and the image lives as long as the data that references it.
+ * S3 expires an object by its creation time, so the lane refreshes an image that went the refresh interval of
+ * `replay_core_v4::fetch_history` without one, and the image lives as long as the data that references it.
*/
export interface RefreshingImageSink {
- readonly refreshIntervalMs: number
publishImage(candidate: FetchCandidate, result: ImageFetchResult): Promise<void>
/** Resets the age of the stored image of a reference, with no origin request. Resolves to false if none is stored. */
refreshImage(candidate: FetchCandidate): Promise<boolean>
diff --git a/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/politeness-key.test.ts b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/politeness-key.test.ts
index 3e6348fce64b..3377d4ea9607 100644
--- a/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/politeness-key.test.ts
+++ b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/politeness-key.test.ts
@@ -47,4 +47,21 @@ describe('the v4 url policy addon', () => {
expect(politenessKey(host)).toBe(host)
})
})
+
+ it('stops the fetch pod at startup when index.node predates a shared rule', () => {
+ const nativePath = require.resolve('@posthog/ai-research-v4-native/index.node')
+ jest.isolateModules(() => {
+ jest.doMock(nativePath, () => {
+ const {
+ urlJob: _urlJob,
+ isUrlHistoryValue: _isUrlHistoryValue,
+ ...stale
+ } = jest.requireActual(nativePath)
+ return stale
+ })
+ const { assertUrlPolicyLoaded } = require('./politeness-key')
+ expect(() => assertUrlPolicyLoaded()).toThrow('rebuild index.node')
+ })
+ jest.dontMock(nativePath)
+ })
})
diff --git a/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/politeness-key.ts b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/politeness-key.ts
index c3b30169c8dc..91abeae32ec6 100644
--- a/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/politeness-key.ts
+++ b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/politeness-key.ts
@@ -1,6 +1,7 @@
/**
- * The URL policy of the collector, reached through the v4 native addon rather than reimplemented
- * here. One public suffix list keeps producer and consumer canonicalization aligned.
+ * The URL policy and the crawl history rule of the recorder, reached through the v4 native addon
+ * rather than reimplemented here. One public suffix list keeps producer and consumer
+ * canonicalization aligned.
*
* The addon is a native library, and `index.ts` imports every server whatever mode the pod runs.
* This loads the addon on first use rather than at import, so only a pod in this lane pays for it.
@@ -14,9 +15,11 @@ function addon(): typeof import('@posthog/ai-research-v4-native') {
if (
typeof loaded.politenessKey !== 'function' ||
typeof loaded.isPublicHost !== 'function' ||
- typeof loaded.tryCanonicalizeUrl !== 'function'
+ typeof loaded.tryCanonicalizeUrl !== 'function' ||
+ typeof loaded.urlJob !== 'function' ||
+ typeof loaded.isUrlHistoryValue !== 'function'
) {
- throw new Error('the ai-research-v4-native addon is missing the url policy: rebuild index.node')
+ throw new Error('the ai-research-v4-native addon is missing a shared rule: rebuild index.node')
}
cached = loaded
}
@@ -24,15 +27,29 @@ function addon(): typeof import('@posthog/ai-research-v4-native') {
}
/**
- * Load the addon now and run one URL through it, so a stale `index.node` stops the pod at startup.
+ * Load the addon now and run one URL and one crawl history entry through it, so a stale `index.node` stops the pod at
+ * startup.
*
- * The parser calls into it for every URL. A stale addon must stop the pod before it reads a batch.
+ * The parser calls into it for every URL, and the crawl history reader for every URL row. A stale addon must stop the
+ * pod before it reads a batch.
*/
export function assertUrlPolicyLoaded(): void {
const verdict = addon().tryCanonicalizeUrl('https://example.com/probe.png')
if (!verdict.ok) {
throw new Error(`the ai-research-v4-native addon refused a plain URL (${verdict.decline}): rebuild index.node`)
}
+ if (!readsCrawlHistory()) {
+ throw new Error('the ai-research-v4-native addon misread a crawl history entry: rebuild index.node')
+ }
+}
+
+function readsCrawlHistory(): boolean {
+ const entry = JSON.stringify({ kind: 'url', key: 'probe', nextFetchAtMs: 1, storageExpiresAtMs: 1, outcome: 'ok' })
+ try {
+ return addon().urlJob(1, undefined, 0) === 'drop' && addon().isUrlHistoryValue('probe', entry)
+ } catch {
+ return false
+ }
}
export function politenessKey(host: string): string {
@@ -57,3 +74,15 @@ export function canonicalizeUrl(url: string): import('@posthog/ai-research-v4-na
export function tryCanonicalizeUrl(url: string): import('@posthog/ai-research-v4-native').UrlPolicyVerdict {
return addon().tryCanonicalizeUrl(url)
}
+
+export function urlJob(
+ nextFetchAtMs: number,
+ storedAtMs: number | undefined,
+ nowMs: number
+): import('@posthog/ai-research-v4-native').UrlJob {
+ return addon().urlJob(nextFetchAtMs, storedAtMs, nowMs)
+}
+
+export function isUrlHistoryValue(key: string, value: string): boolean {
+ return addon().isUrlHistoryValue(key, value)
+}
diff --git a/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/raw-image-store.ts b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/raw-image-store.ts
index 2b54b528cdf5..666e8dfabd44 100644
--- a/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/raw-image-store.ts
+++ b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/raw-image-store.ts
@@ -13,8 +13,6 @@ import { ImageFetchProcessingMetrics } from './processing-metrics'
import { parseV4UrlRef } from './v4-url-ref'
const S3_WRITE_TIMEOUT_MS = 30_000
-/** The `images-v4` lifecycle is 98 days: the 90-day raw window plus this interval and one day. */
-const REFRESH_INTERVAL_MS = 7 * 24 * 60 * 60 * 1000
const CONTENT_TYPE = 'application/octet-stream'
const storedImages = new Counter({
@@ -31,7 +29,6 @@ const storedImages = new Counter({
* overwrites the object of its reference, and `refreshImage` copies it onto itself to reset its age.
*/
export class RawImageStore implements RefreshingImageSink {
- public readonly refreshIntervalMs = REFRESH_INTERVAL_MS
private readonly writes: ConcurrencyController
constructor(
diff --git a/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/url-fetch-consumer.ts b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/url-fetch-consumer.ts
index 76f0df1cd571..18ad7bd24c1d 100644
--- a/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/url-fetch-consumer.ts
+++ b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/url-fetch-consumer.ts
@@ -1,6 +1,8 @@
import { Message } from 'node-rdkafka'
import pLimit from 'p-limit'
+import type { UrlJob } from '@posthog/ai-research-v4-native'
+
import { logger } from '~/common/utils/logger'
import {
@@ -26,6 +28,7 @@ import { FrontierDeadLetterReason, FrontierDeadLetterSink } from './frontier-dea
import { FrontierPublisher, RepublishBatch } from './frontier-publisher'
import type { RefreshingImageSink } from './image-sink'
import { ImageFetchConsumerMetrics, ImageFetchRequestMetrics } from './metrics'
+import { urlJob } from './politeness-key'
import { ImageFetchProcessingMetrics } from './processing-metrics'
import { ImageFetchTopHogMetrics } from './tophog-metrics'
import { urlHistoryExpiresAtMs, urlHistoryStorageExpiresAtMs } from './url-history-expiry'
@@ -180,16 +183,19 @@ export class UrlFetchConsumer {
const refreshDue: { candidate: FetchCandidate; history: UrlCrawlHistoryItem }[] = []
for (const candidate of candidates) {
const history = stored.get(candidate.originalRef)
- if (history?.kind === 'url' && history.nextFetchAtMs > nowMs) {
- if (this.isRefreshDue(history, nowMs)) {
+ if (history?.kind === 'url') {
+ const job = this.storedUrlJob(history, nowMs)
+ if (job === 'refresh') {
refreshDue.push({ candidate, history })
continue
}
- ImageFetchConsumerMetrics.incDeduped('store', 1)
- for (const sourcePartition of candidate.sourcePartitions ?? []) {
- ImageFetchConsumerMetrics.incPartitionUrls(sourcePartition, 'store_deduped', 1)
+ if (job === 'drop') {
+ ImageFetchConsumerMetrics.incDeduped('store', 1)
+ for (const sourcePartition of candidate.sourcePartitions ?? []) {
+ ImageFetchConsumerMetrics.incPartitionUrls(sourcePartition, 'store_deduped', 1)
+ }
+ continue
}
- continue
}
if (candidate.notBeforeMs > nowMs) {
notReady.push(candidate)
@@ -411,9 +417,9 @@ export class UrlFetchConsumer {
}
}
- private isRefreshDue(history: UrlCrawlHistoryItem, nowMs: number): boolean {
- const sink = this.options.refreshingSink
- return !!sink && history.storedAtMs !== undefined && nowMs - history.storedAtMs >= sink.refreshIntervalMs
+ private storedUrlJob(history: UrlCrawlHistoryItem, nowMs: number): UrlJob {
+ const job = urlJob(history.nextFetchAtMs, history.storedAtMs, nowMs)
+ return job === 'refresh' && !this.options.refreshingSink ? 'drop' : job
}
private async republishNotReady(
diff --git a/products/ai_training/docs/replay-data.md b/products/ai_training/docs/replay-data.md
index f3b35a9611bc..5fe09ecb6cb5 100644
--- a/products/ai_training/docs/replay-data.md
+++ b/products/ai_training/docs/replay-data.md
@@ -434,7 +434,7 @@ Nothing in v4 is kept for good. Each kind of data has a fixed lifetime, and a de
- Blocks expire by write time, not by session start, so a session whose blocks span several write days loses its first blocks first. Nothing marks the rest as partial, and `cold_start` does not cover it. Readers that need whole sessions follow the rule in [Block index](#block-index).
- The buckets have no versioning, so an expired object leaves no older version behind. S3 removes the parts of an unfinished upload after one day.
- `blocks/` is stored in Glacier Instant Retrieval from the start. The index prefixes and both image prefixes stay in Standard, because readers read them often.
-- A block can use an image only while the image outlives the block. A pack is written in the same flush as its blocks and expires 8 days later. A fetched image lives at least 30 days after the last block that refers to it, because the recorder sends a job for each referenced URL at least once per day.
+- A block can use an image only while the image outlives the block. A pack is written in the same flush as its blocks and expires 8 days later. A fetched image lives at least 30 days after the last block that refers to it. Each day that a block refers to an image, the recorder sends a job for it, or the fetcher's history shows a stored copy that is younger than 7 days at the end of that day.
- A key outlives every object it sealed: a session writes for at most about 15 days after its key month ends, and an object lives at most 38 days plus the S3 delay. The margin lets the object windows go up to 90 days without a change to the key expiry. Raise the object windows in `v4-staging.hcl` in posthog-cloud-infra only together with the key expiry.
- DynamoDB TTL can remove a row days after `expires_at`. Writers and readers treat a row past `expires_at` as unusable, so the delay only costs storage.
- The key table has no point-in-time recovery, backups, exports, streams or replicas, so no copy brings a shredded key back.
@@ -683,5 +683,16 @@ The fetcher keeps a crawl history per URL reference, so per team and month, in i
A URL is therefore fetched from its origin about once per team and month, as v3 fetches it about once in 30 days.
Unlike v3, the history is per team: a sealed image is readable only with its team's key, so two teams that refer to one URL each fetch it.
-The recorder sends a job for each referenced URL at least once per day, so the fetcher sees a reference in every 7-day refresh interval.
+The recorder sends a job for each referenced URL at least once per day, unless the fetcher's history shows that the fetcher would drop it (see below). So the fetcher sees a reference in every 7-day refresh interval.
The fetcher drops a job whose team month row is missing, expired or deleted.
+
+Each recorder pod sends a reference at most once per UTC day. When `AI_RESEARCH_V4_IMAGE_FETCH_HISTORY_TABLE` is set, the recorder also reads the fetcher's history, as the v3 mirror did:
+
+- It reads only the references of the flush that it did not send or mark that day, each one once. The read starts with the flush and runs while the flush seals its blocks.
+- It does not send a job that the fetcher would still drop at the end of the day. That is a job before the refetch time of its reference, when the reference has no stored copy or a stored copy less than 7 days old. It marks that reference as sent for the day.
+- It sends every other job, so the fetcher still refreshes each stored copy within its interval.
+- It parses a history row with the same code as the fetcher. A row that the fetcher reads as a miss is a miss for the recorder too.
+- A read that fails or takes more than 5 seconds checks nothing, and its jobs go out. The read does not retry a failed request.
+- `ai_research_v4_recorder_image_fetch_history_total` counts the references it read by `outcome`: `fresh` (marked), `due` (a refresh or refetch is due today), `absent`, `invalid` and `error`.
+
+`replay_core_v4::fetch_history` holds the drop rule, the 7-day refresh interval and the row parser. The recorder calls it directly, and the fetcher calls it through `@posthog/ai-research-v4-native`.
diff --git a/rust/ai-research/replay-core-v4-node/package.json b/rust/ai-research/replay-core-v4-node/package.json
index b11dd60d0be8..2e16013899f1 100644
--- a/rust/ai-research/replay-core-v4-node/package.json
+++ b/rust/ai-research/replay-core-v4-node/package.json
@@ -1,7 +1,7 @@
{
"name": "@posthog/ai-research-v4-native",
"version": "0.0.1",
- "description": "Node bindings for the v4 AI research replay URL policy",
+ "description": "Node bindings for the v4 AI research replay URL policy and crawl history rule",
"main": "dist/index.js",
"types": "dist/index.d.ts",
"scripts": {
diff --git a/rust/ai-research/replay-core-v4-node/src/index.ts b/rust/ai-research/replay-core-v4-node/src/index.ts
index 411264409ac5..b73411aebc7a 100644
--- a/rust/ai-research/replay-core-v4-node/src/index.ts
+++ b/rust/ai-research/replay-core-v4-node/src/index.ts
@@ -74,3 +74,13 @@ export function canonicalizeUrl(url: string): CanonicalUrl | null {
const verdict = tryCanonicalizeUrl(url)
return verdict.ok ? verdict.url : null
}
+
+export type UrlJob = 'drop' | 'refresh' | 'fetch'
+
+export function urlJob(nextFetchAtMs: number, storedAtMs: number | undefined, nowMs: number): UrlJob {
+ return native.urlJob(nextFetchAtMs, storedAtMs, nowMs)
+}
+
+export function isUrlHistoryValue(key: string, value: string): boolean {
+ return native.isUrlHistoryValue(key, value)
+}
diff --git a/rust/ai-research/replay-core-v4-node/src/lib.rs b/rust/ai-research/replay-core-v4-node/src/lib.rs
index e82ba426e18a..99d335de9d73 100644
--- a/rust/ai-research/replay-core-v4-node/src/lib.rs
+++ b/rust/ai-research/replay-core-v4-node/src/lib.rs
@@ -1,12 +1,13 @@
-//! Neon native addon exposing the v4 URL policy, `replay_core_v4::url_policy`, to the v4 image
-//! fetcher.
+//! Neon native addon exposing the v4 URL policy, `replay_core_v4::url_policy`, and the crawl
+//! history rule, `replay_core_v4::fetch_history`, to the v4 image fetcher.
//!
-//! The recorder collects URLs with that module, and the fetcher must refuse and canonicalize them
-//! the same way, so the fetcher calls the module rather than a second implementation. This crate is
+//! The recorder collects URLs and drops jobs with those modules, and the fetcher must decide the
+//! same way, so the fetcher calls the modules rather than a second implementation. This crate is
//! glue only: each function converts its arguments and its result, and the rules stay in the core
//! crate.
use neon::prelude::*;
+use replay_core_v4::fetch_history::UrlHistory;
use replay_core_v4::url_policy::{is_public_host, politeness_key, try_canonicalize};
/// The registrable domain of a host.
@@ -48,10 +49,33 @@ fn try_canonicalize_url_ffi(mut cx: FunctionContext) -> JsResult<JsObject> {
Ok(result)
}
+fn url_job_ffi(mut cx: FunctionContext) -> JsResult<JsString> {
+ let next_fetch_at_ms = cx.argument::<JsNumber>(0)?.value(&mut cx);
+ let stored_at_ms = cx
+ .argument::<JsValue>(1)?
+ .downcast::<JsNumber, _>(&mut cx)
+ .ok()
+ .map(|stored_at_ms| stored_at_ms.value(&mut cx));
+ let now_ms = cx.argument::<JsNumber>(2)?.value(&mut cx);
+ let history = UrlHistory {
+ next_fetch_at_ms,
+ stored_at_ms,
+ };
+ Ok(cx.string(history.job_at(now_ms).label()))
+}
+
+fn is_url_history_value_ffi(mut cx: FunctionContext) -> JsResult<JsBoolean> {
+ let key = cx.argument::<JsString>(0)?.value(&mut cx);
+ let value = cx.argument::<JsString>(1)?.value(&mut cx);
+ Ok(cx.boolean(UrlHistory::parse(&key, &value).is_some()))
+}
+
#[neon::main]
fn main(mut cx: ModuleContext) -> NeonResult<()> {
cx.export_function("politenessKey", politeness_key_ffi)?;
cx.export_function("isPublicHost", is_public_host_ffi)?;
cx.export_function("tryCanonicalizeUrl", try_canonicalize_url_ffi)?;
+ cx.export_function("urlJob", url_job_ffi)?;
+ cx.export_function("isUrlHistoryValue", is_url_history_value_ffi)?;
Ok(())
}
diff --git a/rust/ai-research/replay-core-v4/src/fetch_history.rs b/rust/ai-research/replay-core-v4/src/fetch_history.rs
new file mode 100644
index 000000000000..2b8c8bd94caf
--- /dev/null
+++ b/rust/ai-research/replay-core-v4/src/fetch_history.rs
@@ -0,0 +1,186 @@
+//! The image fetcher's crawl history entry for a reference, and what the fetcher does with a job for it. The fetcher
+//! calls this through `replay-core-v4-node` and the recorder calls it directly, so both drop the same jobs.
+
+use serde_json::Value;
+
+/// The `images-v4` lifecycle is 98 days: the 90-day raw window plus this interval and one day.
+pub const REFRESH_INTERVAL_MS: f64 = 7.0 * 24.0 * 3600.0 * 1000.0;
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub enum UrlJob {
+ Drop,
+ Refresh,
+ Fetch,
+}
+
+impl UrlJob {
+ pub fn label(self) -> &'static str {
+ match self {
+ UrlJob::Drop => "drop",
+ UrlJob::Refresh => "refresh",
+ UrlJob::Fetch => "fetch",
+ }
+ }
+}
+
+#[derive(Debug, Clone, Copy, PartialEq)]
+pub struct UrlHistory {
+ pub next_fetch_at_ms: f64,
+ pub stored_at_ms: Option<f64>,
+}
+
+impl UrlHistory {
+ /// Parses the `value` attribute of the crawl history row `key`. `None` is a row that the fetcher reads as a miss.
+ pub fn parse(key: &str, value: &str) -> Option<Self> {
+ let value: Value = serde_json::from_str(value).ok()?;
+ let number = |field: &str| value.get(field).and_then(Value::as_f64);
+ let valid = value.get("kind").and_then(Value::as_str) == Some("url")
+ && value.get("key").and_then(Value::as_str) == Some(key)
+ && value.get("outcome").is_some_and(Value::is_string)
+ && number("storageExpiresAtMs").is_some()
+ && value.get("cache").is_none_or(is_http_cache)
+ && value.get("storedAtMs").is_none_or(Value::is_number)
+ && value.get("_bodyChunks").is_none();
+ if !valid {
+ return None;
+ }
+ Some(Self {
+ next_fetch_at_ms: number("nextFetchAtMs")?,
+ stored_at_ms: number("storedAtMs"),
+ })
+ }
+
+ pub fn job_at(&self, at_ms: f64) -> UrlJob {
+ let refresh_due = self
+ .stored_at_ms
+ .is_some_and(|stored_at_ms| at_ms - stored_at_ms >= REFRESH_INTERVAL_MS);
+ if self.next_fetch_at_ms > at_ms {
+ if refresh_due {
+ UrlJob::Refresh
+ } else {
+ UrlJob::Drop
+ }
+ } else {
+ UrlJob::Fetch
+ }
+ }
+}
+
+fn is_http_cache(cache: &Value) -> bool {
+ const TEXT_FIELDS: [&str; 8] = [
+ "responseUrlHmac",
+ "responseUrl",
+ "etag",
+ "lastModified",
+ "date",
+ "age",
+ "cacheControl",
+ "expires",
+ ];
+ cache.get("requestTimeMs").is_some_and(Value::is_number)
+ && cache.get("responseTimeMs").is_some_and(Value::is_number)
+ && TEXT_FIELDS
+ .iter()
+ .all(|field| cache.get(field).is_none_or(Value::is_string))
+}
+
+#[cfg(test)]
+mod tests {
+ use serde_json::json;
+
+ use super::*;
+
+ const NOW_MS: f64 = 1_794_000_000_000.0;
+ const DAY_MS: f64 = 24.0 * 3600.0 * 1000.0;
+
+ #[test]
+ fn decides_each_job_as_the_fetcher_does() {
+ for (name, next_fetch_in_ms, stored_ago_ms, job) in [
+ (
+ "a copy one day old",
+ 20.0 * DAY_MS,
+ Some(DAY_MS),
+ UrlJob::Drop,
+ ),
+ ("no stored copy", 20.0 * DAY_MS, None, UrlJob::Drop),
+ (
+ "a copy 1 ms younger than 7 days",
+ 20.0 * DAY_MS,
+ Some(7.0 * DAY_MS - 1.0),
+ UrlJob::Drop,
+ ),
+ (
+ "a copy 7 days old",
+ 20.0 * DAY_MS,
+ Some(7.0 * DAY_MS),
+ UrlJob::Refresh,
+ ),
+ ("its refetch time", 0.0, Some(DAY_MS), UrlJob::Fetch),
+ (
+ "an old copy past its refetch time",
+ -1.0,
+ Some(14.0 * DAY_MS),
+ UrlJob::Fetch,
+ ),
+ ] {
+ let history = UrlHistory {
+ next_fetch_at_ms: NOW_MS + next_fetch_in_ms,
+ stored_at_ms: stored_ago_ms.map(|ago_ms| NOW_MS - ago_ms),
+ };
+ assert_eq!(history.job_at(NOW_MS), job, "{name}");
+ }
+ }
+
+ #[test]
+ fn parses_only_url_entries_the_fetcher_accepts_for_their_key() {
+ let key = "imageurl:v4:7:2026-10:AAAAAAAAAAAAAAAAAAAAAA";
+ let entry = json!({
+ "kind": "url",
+ "key": key,
+ "nextFetchAtMs": 1_795_000_000_000i64,
+ "storageExpiresAtMs": 1_794_000_000_000i64,
+ "outcome": "ok",
+ "cache": {"requestTimeMs": 1, "responseTimeMs": 2, "etag": "\"x\""},
+ "storedAtMs": 1_792_000_000_000i64,
+ });
+ let with = |field: &str, value: Value| {
+ let mut changed = entry.clone();
+ changed[field] = value;
+ changed.to_string()
+ };
+ let without = |field: &str| {
+ let mut changed = entry.clone();
+ changed.as_object_mut().unwrap().remove(field);
+ changed.to_string()
+ };
+
+ assert_eq!(
+ UrlHistory::parse(key, &entry.to_string()),
+ Some(UrlHistory {
+ next_fetch_at_ms: 1_795_000_000_000.0,
+ stored_at_ms: Some(1_792_000_000_000.0),
+ })
+ );
+ assert_eq!(
+ UrlHistory::parse(key, &without("cache")).map(|history| history.next_fetch_at_ms),
+ Some(1_795_000_000_000.0)
+ );
+ for value in [
+ "{\"kind\":\"url\",".to_owned(),
+ with("kind", json!("robots")),
+ with("key", json!("imageurl:v4:7:2026-10:other")),
+ without("nextFetchAtMs"),
+ without("outcome"),
+ without("storageExpiresAtMs"),
+ with("storedAtMs", Value::Null),
+ with("cache", json!({"requestTimeMs": 1})),
+ with(
+ "cache",
+ json!({"requestTimeMs": 1, "responseTimeMs": 2, "etag": null}),
+ ),
+ with("_bodyChunks", json!({"count": 1, "sha256": "0".repeat(64)})),
+ ] {
+ assert_eq!(UrlHistory::parse(key, &value), None, "{value}");
+ }
+ }
+}
diff --git a/rust/ai-research/replay-core-v4/src/lib.rs b/rust/ai-research/replay-core-v4/src/lib.rs
index b4b0afc7472a..9e7f9d6982b3 100644
--- a/rust/ai-research/replay-core-v4/src/lib.rs
+++ b/rust/ai-research/replay-core-v4/src/lib.rs
@@ -3,6 +3,7 @@
pub mod bytescan;
pub mod compression;
pub mod envelope;
+pub mod fetch_history;
pub mod gate;
pub mod inspect;
#[cfg(feature = "store")]
diff --git a/rust/ai-research/replay-recorder-v4/src/config.rs b/rust/ai-research/replay-recorder-v4/src/config.rs
index e7e90f5926fd..1d2978f24b7b 100644
--- a/rust/ai-research/replay-recorder-v4/src/config.rs
+++ b/rust/ai-research/replay-recorder-v4/src/config.rs
@@ -127,6 +127,10 @@ pub struct Config {
)]
pub image_fetch_topic: String,
+ /// The fetcher's crawl history table. Empty turns off the read, so every new reference of the day goes out.
+ #[envconfig(from = "AI_RESEARCH_V4_IMAGE_FETCH_HISTORY_TABLE", default = "")]
+ pub image_fetch_history_table: String,
+
/// Where the block and image index rows go for the Parquet sink.
#[envconfig(from = "AI_RESEARCH_V4_INDEX_TOPIC", default = "ai_research_v4_index")]
pub index_topic: String,
diff --git a/rust/ai-research/replay-recorder-v4/src/flush.rs b/rust/ai-research/replay-recorder-v4/src/flush.rs
index 535dc2826de0..00b88b3e61b6 100644
--- a/rust/ai-research/replay-recorder-v4/src/flush.rs
+++ b/rust/ai-research/replay-recorder-v4/src/flush.rs
@@ -23,9 +23,10 @@ use replay_core_v4::keys::table::{AwsKeyTable, CachedKeyTable};
use tokio::sync::Semaphore;
use crate::batch::Block;
+use crate::history::{mark_fresh, FetchHistory};
use crate::index::{Index, IndexMessage};
use crate::store::BlobStore;
-use crate::urls::{fetch_jobs, records};
+use crate::urls::{fetch_jobs, records, FlushImages};
use crate::write::{
block_index_rows, compress_block, new_images, object_key, seal_blocking, seal_compressed,
session_keyed_frames, write_blocks, write_images, CompressedBlock, FlushAttempt, WriteError,
@@ -34,6 +35,9 @@ use crate::write::{
/// How long a fetch job may wait for room in the producer queue.
const ENQUEUE_TIMEOUT: Duration = Duration::from_secs(10);
+/// The read runs while the flush seals its blocks, so a slow read rarely makes the flush longer.
+const HISTORY_READ_BUDGET: Duration = Duration::from_secs(5);
+
/// Fetch job records sent at once, so a slow broker slows the flush instead of filling the producer queue.
const SENDS_IN_FLIGHT: usize = 64;
@@ -50,9 +54,11 @@ pub struct Writer {
/// The flush does not wait for index rows, so they get their own producer with a delivery timeout of minutes.
pub index_producer: FutureProducer<KafkaContext>,
pub index_topic: String,
- /// Remote image references this process sent to the fetcher, by UTC day. A reference goes out again each day, so
- /// the fetcher sees it within every refresh interval and keeps the image as long as the blocks that reference it.
+ /// Remote image references this process sent to the fetcher, or found fresh in its history, by UTC day. A
+ /// reference goes out again each day unless its stored image needs no refresh, so the fetcher refreshes it within
+ /// every refresh interval and keeps the image as long as the blocks that reference it.
pub sent_urls: moka::sync::Cache<(String, String), ()>,
+ pub fetch_history: Option<Arc<dyn FetchHistory>>,
/// Seals running at once, over all attempts: the flush concurrency.
pub seal_permits: Arc<Semaphore>,
}
@@ -164,6 +170,17 @@ pub async fn write_flush(
drop(resolved);
let month = key_month(&day);
let attempt = FlushAttempt::new(now);
+ let already_sent = |reference: &str| {
+ writer
+ .sent_urls
+ .contains_key(&(day.clone(), reference.to_owned()))
+ };
+ let images = FlushImages::new(
+ blocks.iter().map(|block| &**block),
+ &keys,
+ month,
+ &already_sent,
+ );
let object_key = object_key(&day, now);
let seals: Vec<_> = blocks
@@ -191,10 +208,12 @@ pub async fn write_flush(
})
.collect();
let mut done = Vec::with_capacity(seals.len());
- let results: Vec<_> = stream::iter(seals)
- .buffer_unordered(concurrency.max(1))
- .collect()
- .await;
+ let (results, ()) = tokio::join!(
+ stream::iter(seals)
+ .buffer_unordered(concurrency.max(1))
+ .collect::<Vec<_>>(),
+ mark_fresh_images(writer, &images, &day),
+ );
for result in results {
match result {
Ok(sealed) => done.push(sealed),
@@ -236,21 +255,17 @@ pub async fn write_flush(
kept[index] = true;
}
let jobs = fetch_jobs(
+ &images,
blocks
.iter()
- .map(|b| &**b)
.zip(stats.iter_mut())
- .zip(kept)
- .filter_map(|(pair, kept)| kept.then_some(pair)),
- &keys,
- month,
+ .enumerate()
+ .filter(|(index, _)| kept[*index])
+ .map(|(index, (block, stats))| (index, &**block, stats)),
now.timestamp_millis(),
- &|reference| {
- writer
- .sent_urls
- .contains_key(&(day.clone(), reference.to_owned()))
- },
+ &already_sent,
);
+ drop(images);
let session_frames = session_keyed_frames(&sealed);
let image_rows = write_images(
@@ -297,6 +312,21 @@ pub async fn write_flush(
Ok(())
}
+async fn mark_fresh_images(writer: &Writer, images: &FlushImages, day: &str) {
+ let Some(history) = &writer.fetch_history else {
+ return;
+ };
+ let deadline = tokio::time::Instant::now() + HISTORY_READ_BUDGET;
+ mark_fresh(
+ history.as_ref(),
+ &writer.sent_urls,
+ images.unsent_references(),
+ day,
+ deadline,
+ )
+ .await;
+}
+
/// Sends every fetch job of the flush, then waits for all deliveries before the flush commits, as the v3 mirror does.
/// A send that fails does not fail the flush, because its blocks are stored by then. The references it did not send
/// stay unmarked, so a later flush that sees them sends them again.
diff --git a/rust/ai-research/replay-recorder-v4/src/history.rs b/rust/ai-research/replay-recorder-v4/src/history.rs
new file mode 100644
index 000000000000..e36ac3d7db0d
--- /dev/null
+++ b/rust/ai-research/replay-recorder-v4/src/history.rs
@@ -0,0 +1,415 @@
+use std::collections::{HashMap, HashSet};
+use std::time::{Duration, Instant};
+
+use async_trait::async_trait;
+use aws_sdk_dynamodb::error::DisplayErrorContext;
+use aws_sdk_dynamodb::types::{AttributeValue, KeysAndAttributes};
+use chrono::NaiveDate;
+use futures::stream::{self, StreamExt};
+use metrics::{counter, histogram};
+use rand::Rng;
+use replay_core_v4::fetch_history::{UrlHistory, UrlJob};
+
+pub const DYNAMODB_BATCH_GET_KEY_LIMIT: usize = 100;
+const READS_IN_FLIGHT: usize = 16;
+const MAX_CALLS_WITHOUT_PROGRESS: u32 = 5;
+
+#[derive(Debug, Default, PartialEq)]
+pub struct HistoryRead {
+ pub entries: HashMap<String, UrlHistory>,
+ pub invalid_rows: usize,
+ pub unread_references: usize,
+ pub error: Option<String>,
+}
+
+impl HistoryRead {
+ fn unread(references: usize, error: impl Into<String>) -> Self {
+ Self {
+ unread_references: references,
+ error: Some(error.into()),
+ ..Self::default()
+ }
+ }
+}
+
+#[async_trait]
+pub trait FetchHistory: Send + Sync {
+ /// Reads at most [`DYNAMODB_BATCH_GET_KEY_LIMIT`] references, and returns what it read by `deadline`.
+ async fn read(&self, references: &[String], deadline: tokio::time::Instant) -> HistoryRead;
+}
+
+pub struct AwsFetchHistory {
+ client: aws_sdk_dynamodb::Client,
+ table: String,
+}
+
+impl AwsFetchHistory {
+ pub fn new(client: aws_sdk_dynamodb::Client, table: String) -> Self {
+ Self { client, table }
+ }
+}
+
+#[async_trait]
+impl FetchHistory for AwsFetchHistory {
+ async fn read(&self, references: &[String], deadline: tokio::time::Instant) -> HistoryRead {
+ let mut pending: Vec<HashMap<String, AttributeValue>> = references
+ .iter()
+ .map(|reference| {
+ HashMap::from([("key".to_owned(), AttributeValue::S(reference.clone()))])
+ })
+ .collect();
+ let mut read = HistoryRead::default();
+ let mut calls_without_progress = 0;
+ loop {
+ // `key` and `value` are DynamoDB reserved words.
+ let request = KeysAndAttributes::builder()
+ .set_keys(Some(pending.clone()))
+ .projection_expression("#k, #e, #v")
+ .expression_attribute_names("#k", "key")
+ .expression_attribute_names("#e", "expires_at")
+ .expression_attribute_names("#v", "value")
+ .build();
+ let request = match request {
+ Ok(request) => request,
+ Err(error) => return HistoryRead::unread(pending.len(), error.to_string()),
+ };
+ let sent = self
+ .client
+ .batch_get_item()
+ .request_items(&self.table, request)
+ .send();
+ let output = match tokio::time::timeout_at(deadline, sent).await {
+ Err(_) => {
+ read.error = Some("the read ran past its deadline".to_owned());
+ break;
+ }
+ Ok(Err(error)) => {
+ read.error = Some(DisplayErrorContext(&error).to_string());
+ break;
+ }
+ Ok(Ok(output)) => output,
+ };
+ for item in output
+ .responses()
+ .and_then(|r| r.get(&self.table))
+ .into_iter()
+ .flatten()
+ {
+ match url_history(item) {
+ Some((reference, entry)) => {
+ read.entries.insert(reference, entry);
+ }
+ None => read.invalid_rows += 1,
+ }
+ }
+ let unprocessed = output
+ .unprocessed_keys()
+ .and_then(|u| u.get(&self.table))
+ .map(|u| u.keys().to_vec())
+ .unwrap_or_default();
+ calls_without_progress = if unprocessed.len() < pending.len() {
+ 0
+ } else {
+ calls_without_progress + 1
+ };
+ pending = unprocessed;
+ if pending.is_empty() {
+ break;
+ }
+ let wake = tokio::time::Instant::now() + backoff(calls_without_progress);
+ if calls_without_progress == MAX_CALLS_WITHOUT_PROGRESS || wake >= deadline {
+ read.error = Some("DynamoDB left keys unprocessed".to_owned());
+ break;
+ }
+ tokio::time::sleep_until(wake).await;
+ }
+ read.unread_references = pending.len();
+ read
+ }
+}
+
+/// The fetcher reads a row with no finite `expires_at` as a miss, though no rule here uses the expiry.
+fn url_history(item: &HashMap<String, AttributeValue>) -> Option<(String, UrlHistory)> {
+ let reference = item.get("key")?.as_s().ok()?;
+ let expires_at: f64 = item.get("expires_at")?.as_n().ok()?.parse().ok()?;
+ if !expires_at.is_finite() {
+ return None;
+ }
+ let entry = UrlHistory::parse(reference, item.get("value")?.as_s().ok()?)?;
+ Some((reference.to_owned(), entry))
+}
+
+fn day_end_ms(day: &str) -> Option<f64> {
+ let next_day = NaiveDate::parse_from_str(day, "%Y-%m-%d")
+ .ok()?
+ .succ_opt()?;
+ Some(next_day.and_hms_opt(0, 0, 0)?.and_utc().timestamp_millis() as f64)
+}
+
+fn backoff(attempt: u32) -> Duration {
+ let ceiling = 3_000u64.min(50u64.saturating_mul(1 << attempt.min(16)));
+ Duration::from_millis(rand::thread_rng().gen_range(ceiling / 2..=ceiling))
+}
+
+/// Reads each reference that `sent_urls` does not hold for `day` once, and marks the ones the fetcher would still drop
+/// when `day` ends. So each day that a block references an image, either a job goes out or the stored copy is younger
+/// than the refresh interval, which the `images-v4` lifecycle allows for.
+pub async fn mark_fresh<'a>(
+ history: &dyn FetchHistory,
+ sent_urls: &moka::sync::Cache<(String, String), ()>,
+ references: impl IntoIterator<Item = &'a String>,
+ day: &str,
+ deadline: tokio::time::Instant,
+) {
+ let Some(day_end_ms) = day_end_ms(day) else {
+ return;
+ };
+ let unsent: Vec<String> = {
+ let mut seen = HashSet::new();
+ references
+ .into_iter()
+ .filter(|reference| {
+ seen.insert(*reference)
+ && !sent_urls.contains_key(&(day.to_owned(), (*reference).clone()))
+ })
+ .cloned()
+ .collect()
+ };
+ for reference in fresh_references(history, &unsent, day_end_ms, deadline).await {
+ sent_urls.insert((day.to_owned(), reference), ());
+ }
+}
+
+/// A reference that a read does not return by `deadline` is not fresh, so the fetcher decides its job.
+async fn fresh_references(
+ history: &dyn FetchHistory,
+ references: &[String],
+ drops_at_ms: f64,
+ deadline: tokio::time::Instant,
+) -> HashSet<String> {
+ if references.is_empty() {
+ return HashSet::new();
+ }
+ let started = Instant::now();
+ let past_deadline =
+ |references: usize| HistoryRead::unread(references, "the read ran past its deadline");
+ let reads: Vec<_> = references
+ .chunks(DYNAMODB_BATCH_GET_KEY_LIMIT)
+ .map(|chunk| async move {
+ if tokio::time::Instant::now() >= deadline {
+ return past_deadline(chunk.len());
+ }
+ tokio::time::timeout_at(deadline, history.read(chunk, deadline))
+ .await
+ .unwrap_or_else(|_| past_deadline(chunk.len()))
+ })
+ .collect();
+ let reads: Vec<HistoryRead> = stream::iter(reads)
+ .buffer_unordered(READS_IN_FLIGHT)
+ .collect()
+ .await;
+ histogram!("ai_research_v4_recorder_image_fetch_history_duration_seconds")
+ .record(started.elapsed().as_secs_f64());
+
+ let mut fresh = HashSet::new();
+ let (mut due, mut invalid, mut unread) = (0, 0, 0);
+ let mut first_error = None;
+ for read in reads {
+ invalid += read.invalid_rows;
+ unread += read.unread_references;
+ first_error = first_error.or(read.error);
+ for (reference, entry) in read.entries {
+ if entry.job_at(drops_at_ms) == UrlJob::Drop {
+ fresh.insert(reference);
+ } else {
+ due += 1;
+ }
+ }
+ }
+ if let Some(error) = first_error {
+ tracing::warn!(%error, references = unread, "image fetch history read failed: those jobs go out unchecked");
+ }
+
+ let absent = references
+ .len()
+ .saturating_sub(fresh.len() + due + invalid + unread);
+ for (outcome, count) in [
+ ("fresh", fresh.len()),
+ ("due", due),
+ ("absent", absent),
+ ("invalid", invalid),
+ ("error", unread),
+ ] {
+ counter!("ai_research_v4_recorder_image_fetch_history_total", "outcome" => outcome)
+ .increment(count as u64);
+ }
+ fresh
+}
+
+#[cfg(test)]
+mod tests {
+ use replay_core_v4::fetch_history::REFRESH_INTERVAL_MS;
+ use serde_json::json;
+
+ use super::*;
+
+ const DAY: &str = "2026-10-26";
+ const HOUR_MS: f64 = 3600.0 * 1000.0;
+
+ struct MemoryHistory {
+ entries: HashMap<String, UrlHistory>,
+ read: std::sync::Mutex<Vec<String>>,
+ }
+
+ #[async_trait]
+ impl FetchHistory for MemoryHistory {
+ async fn read(&self, references: &[String], _: tokio::time::Instant) -> HistoryRead {
+ self.read.lock().unwrap().extend_from_slice(references);
+ HistoryRead {
+ entries: references
+ .iter()
+ .filter_map(|r| self.entries.get(r).map(|entry| (r.clone(), *entry)))
+ .collect(),
+ ..HistoryRead::default()
+ }
+ }
+ }
+
+ enum Broken {
+ Failing,
+ Stalled,
+ }
+
+ #[async_trait]
+ impl FetchHistory for Broken {
+ async fn read(&self, references: &[String], _: tokio::time::Instant) -> HistoryRead {
+ match self {
+ Broken::Failing => HistoryRead::unread(references.len(), "AccessDeniedException"),
+ Broken::Stalled => std::future::pending().await,
+ }
+ }
+ }
+
+ fn marked(sent_urls: &moka::sync::Cache<(String, String), ()>, reference: &str) -> bool {
+ sent_urls.contains_key(&(DAY.to_owned(), reference.to_owned()))
+ }
+
+ #[tokio::test]
+ async fn reads_each_cache_miss_once_and_marks_what_the_fetcher_drops_until_the_day_ends() {
+ let day_end_ms = day_end_ms(DAY).unwrap();
+ let entry = |next_fetch_in_ms: f64, stored_ago_ms: f64| UrlHistory {
+ next_fetch_at_ms: day_end_ms + next_fetch_in_ms,
+ stored_at_ms: Some(day_end_ms - stored_ago_ms),
+ };
+ let history = MemoryHistory {
+ entries: HashMap::from([
+ ("fresh".to_owned(), entry(30.0 * 24.0 * HOUR_MS, HOUR_MS)),
+ ("refetch_today".to_owned(), entry(-HOUR_MS, HOUR_MS)),
+ (
+ "refresh_today".to_owned(),
+ entry(30.0 * 24.0 * HOUR_MS, REFRESH_INTERVAL_MS),
+ ),
+ ]),
+ read: Default::default(),
+ };
+ let sent_urls = crate::urls::sent_urls(100);
+ sent_urls.insert((DAY.to_owned(), "sent".to_owned()), ());
+ let references = [
+ "sent",
+ "fresh",
+ "fresh",
+ "refetch_today",
+ "refresh_today",
+ "unknown",
+ ]
+ .map(str::to_owned);
+
+ mark_fresh(
+ &history,
+ &sent_urls,
+ &references,
+ DAY,
+ tokio::time::Instant::now() + Duration::from_secs(5),
+ )
+ .await;
+
+ let mut read = history.read.into_inner().unwrap();
+ read.sort();
+ assert_eq!(read, ["fresh", "refetch_today", "refresh_today", "unknown"]);
+ assert!(marked(&sent_urls, "fresh"));
+ for reference in ["refetch_today", "refresh_today", "unknown"] {
+ assert!(!marked(&sent_urls, reference), "{reference}");
+ }
+ }
+
+ #[tokio::test(start_paused = true)]
+ async fn marks_nothing_when_the_read_fails_or_stalls() {
+ for history in [Broken::Failing, Broken::Stalled] {
+ let references: Vec<String> = (0..2 * DYNAMODB_BATCH_GET_KEY_LIMIT)
+ .map(|i| format!("r{i:03}"))
+ .collect();
+ let sent_urls = crate::urls::sent_urls(1_000);
+ mark_fresh(
+ &history,
+ &sent_urls,
+ &references,
+ DAY,
+ tokio::time::Instant::now() + Duration::from_secs(5),
+ )
+ .await;
+ assert!(!references.iter().any(|r| marked(&sent_urls, r)));
+ }
+ }
+
+ #[tokio::test]
+ async fn sends_no_read_once_the_deadline_has_passed() {
+ let history = MemoryHistory {
+ entries: HashMap::new(),
+ read: Default::default(),
+ };
+ let references = ["r".to_owned()];
+ let sent_urls = crate::urls::sent_urls(100);
+ mark_fresh(
+ &history,
+ &sent_urls,
+ &references,
+ DAY,
+ tokio::time::Instant::now(),
+ )
+ .await;
+ assert!(history.read.into_inner().unwrap().is_empty());
+ }
+
+ #[test]
+ fn reads_a_row_only_with_a_numeric_expiry() {
+ let reference = "imageurl:v4:7:2026-10:AAAAAAAAAAAAAAAAAAAAAA";
+ let value = json!({
+ "kind": "url",
+ "key": reference,
+ "nextFetchAtMs": 1_795_000_000_000i64,
+ "storageExpiresAtMs": 1_794_000_000_000i64,
+ "outcome": "ok",
+ });
+ let row = |expires_at: Option<&str>| {
+ let mut row = HashMap::from([
+ ("key".to_owned(), AttributeValue::S(reference.to_owned())),
+ ("value".to_owned(), AttributeValue::S(value.to_string())),
+ ]);
+ if let Some(expires_at) = expires_at {
+ row.insert(
+ "expires_at".to_owned(),
+ AttributeValue::N(expires_at.to_owned()),
+ );
+ }
+ row
+ };
+
+ assert_eq!(
+ url_history(&row(Some("1794000000")))
+ .map(|(key, history)| (key, history.next_fetch_at_ms)),
+ Some((reference.to_owned(), 1_795_000_000_000.0))
+ );
+ assert_eq!(url_history(&row(None)), None);
+ assert_eq!(url_history(&row(Some("x"))), None);
+ }
+}
diff --git a/rust/ai-research/replay-recorder-v4/src/lib.rs b/rust/ai-research/replay-recorder-v4/src/lib.rs
index d50a0f954add..13f32d19bf56 100644
--- a/rust/ai-research/replay-recorder-v4/src/lib.rs
+++ b/rust/ai-research/replay-recorder-v4/src/lib.rs
@@ -5,6 +5,7 @@ pub mod batch;
pub mod config;
pub mod consumer;
pub mod flush;
+pub mod history;
pub mod index;
pub mod limits;
pub mod process;
diff --git a/rust/ai-research/replay-recorder-v4/src/main.rs b/rust/ai-research/replay-recorder-v4/src/main.rs
index 01cf5b1a287f..8f3c337a8576 100644
--- a/rust/ai-research/replay-recorder-v4/src/main.rs
+++ b/rust/ai-research/replay-recorder-v4/src/main.rs
@@ -19,6 +19,7 @@ use replay_core_v4::keys::table::{AwsKeyTable, CachedKeyTable};
use replay_recorder_v4::config::Config;
use replay_recorder_v4::consumer::{run, RecorderConsumer, Settings, Sources};
use replay_recorder_v4::flush::{Writer, PRODUCER_FLUSH_ON_SHUTDOWN};
+use replay_recorder_v4::history::{AwsFetchHistory, FetchHistory};
use replay_recorder_v4::limits::{Restrictions, SessionLimiter};
use replay_recorder_v4::store::S3Store;
use replay_recorder_v4::teams::TeamCache;
@@ -246,6 +247,21 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
.await?,
index_topic: config.index_topic.clone(),
sent_urls: replay_recorder_v4::urls::sent_urls(500_000),
+ fetch_history: (!config.image_fetch_history_table.is_empty()).then(
+ || -> Arc<dyn FetchHistory> {
+ // A failed read only sends its jobs unchecked, so SDK retries would only use up the retry quota
+ // that this process's key table reads share.
+ let client = aws_sdk_dynamodb::Client::from_conf(
+ aws_sdk_dynamodb::config::Builder::from(&aws)
+ .retry_config(aws_sdk_dynamodb::config::retry::RetryConfig::disabled())
+ .build(),
+ );
+ Arc::new(AwsFetchHistory::new(
+ client,
+ config.image_fetch_history_table.clone(),
+ ))
+ },
+ ),
seal_permits: Arc::new(tokio::sync::Semaphore::new(settings.flush_concurrency)),
})
} else {
diff --git a/rust/ai-research/replay-recorder-v4/src/urls.rs b/rust/ai-research/replay-recorder-v4/src/urls.rs
index d10f266f3bcc..94e01032560b 100644
--- a/rust/ai-research/replay-recorder-v4/src/urls.rs
+++ b/rust/ai-research/replay-recorder-v4/src/urls.rs
@@ -1,6 +1,6 @@
//! Fetch jobs for the remote images a flush's blocks referenced, in the v3 fetch topic's job shape.
-use std::collections::{BTreeMap, HashSet};
+use std::collections::{BTreeMap, HashMap};
use std::time::Duration;
use metrics::counter;
@@ -23,39 +23,94 @@ const MAX_RECORD_BYTES: usize = 512 * 1024;
/// How many redirects a job may follow, as in v3.
const REMAINING_HOPS: u32 = 10;
-/// Builds one job per new remote image URL, grouped by registrable domain, the topic's key. Counts each block's URLs
-/// in the stats paired with it. `already_sent` names references this process sent today.
+/// The remote image URLs of a flush's blocks that the URL policy accepts. The flush holds these until it builds its
+/// jobs, so a reference is held once, and only while this process had not sent it today when the flush started.
+pub struct FlushImages {
+ unsent: Vec<(CanonicalUrl, String)>,
+ blocks: Vec<BlockImages>,
+}
+
+#[derive(Default)]
+struct BlockImages {
+ accepted: u32,
+ unsent: Vec<usize>,
+}
+
+impl FlushImages {
+ pub fn new<'b>(
+ blocks: impl IntoIterator<Item = &'b Block>,
+ keys: &FlushKeys,
+ key_month: &str,
+ already_sent: &dyn Fn(&str) -> bool,
+ ) -> Self {
+ let mut unsent = Vec::new();
+ let mut positions: HashMap<String, usize> = HashMap::new();
+ let blocks = blocks
+ .into_iter()
+ .map(|block| {
+ let mut images = BlockImages::default();
+ for (url, reference) in remote_images(block, keys, key_month) {
+ images.accepted += 1;
+ if already_sent(&reference) {
+ continue;
+ }
+ let position = *positions.entry(reference.clone()).or_insert_with(|| {
+ unsent.push((url, reference));
+ unsent.len() - 1
+ });
+ images.unsent.push(position);
+ }
+ images
+ })
+ .collect();
+ Self { unsent, blocks }
+ }
+
+ pub fn unsent_references(&self) -> impl Iterator<Item = &String> {
+ self.unsent.iter().map(|(_, reference)| reference)
+ }
+}
+
+/// Empty when the flush has no keys for the block's session.
+fn remote_images(block: &Block, keys: &FlushKeys, key_month: &str) -> Vec<(CanonicalUrl, String)> {
+ let id = SessionId {
+ team_id: block.session.team_id,
+ session_id: block.session.session_id.clone(),
+ };
+ let (Some(_), Some(team_key)) = (
+ keys.sessions.get(&id),
+ keys.team_months.get(&block.session.team_id),
+ ) else {
+ return Vec::new();
+ };
+ let url_key = image_url_key(team_key);
+ block
+ .image_urls
+ .iter()
+ .filter_map(|raw| url_reference(raw, block.session.team_id, key_month, &url_key).ok())
+ .collect()
+}
+
+/// Builds one job per new remote image URL, grouped by registrable domain, the topic's key. Counts the URLs of each
+/// block in the stats paired with it. `index` is the block's position in the blocks that `images` was built from.
+/// `already_sent` names references this process sent or found fresh today.
pub fn fetch_jobs<'b>(
- blocks: impl IntoIterator<Item = (&'b Block, &'b mut BlockStats)>,
- keys: &FlushKeys,
- key_month: &str,
+ images: &FlushImages,
+ blocks: impl IntoIterator<Item = (usize, &'b Block, &'b mut BlockStats)>,
now_ms: i64,
already_sent: &dyn Fn(&str) -> bool,
) -> BTreeMap<String, Vec<Value>> {
let mut jobs: BTreeMap<String, Vec<Value>> = BTreeMap::new();
- let mut seen = HashSet::new();
- for (block, stats) in blocks {
- let id = SessionId {
- team_id: block.session.team_id,
- session_id: block.session.session_id.clone(),
- };
- let (Some(_), Some(team_key)) = (
- keys.sessions.get(&id),
- keys.team_months.get(&block.session.team_id),
- ) else {
- continue;
- };
- let url_key = image_url_key(team_key);
- for raw in &block.image_urls {
- let Ok((url, reference)) =
- url_reference(raw, block.session.team_id, key_month, &url_key)
- else {
- continue;
- };
- stats.image_urls += 1;
- if already_sent(&reference) || !seen.insert(reference.clone()) {
+ let mut seen = vec![false; images.unsent.len()];
+ for (index, block, stats) in blocks {
+ let block_images = &images.blocks[index];
+ stats.image_urls += block_images.accepted;
+ for &position in &block_images.unsent {
+ let (url, reference) = &images.unsent[position];
+ if seen[position] || already_sent(reference) {
continue;
}
+ seen[position] = true;
jobs.entry(url.domain.clone()).or_default().push(json!({
"sessionId": block.session.session_id,
"originalRef": reference,
@@ -178,20 +233,11 @@ mod tests {
assert!((0..100).all(|i| sent.contains_key(&mark(1, i))));
}
- #[test]
- fn sends_each_new_public_image_url_once_by_domain() {
- let urls = [
- "https://cdn.example.com/a.png",
- "https://cdn.example.com/a.png",
- "https://img2.cdn.example.com/b.png",
- "https://cdn.example.com/sent.png",
- "http://localhost/private.png",
- "https://bat.bing.com/action/0",
- ];
- let block = Block {
+ fn block(session_id: &str, urls: &[&str]) -> Block {
+ Block {
session: SessionKey {
team_id: 7,
- session_id: "s".to_owned(),
+ session_id: session_id.to_owned(),
},
origin: Origin {
tp: Tp {
@@ -212,20 +258,41 @@ mod tests {
css_rewrites: Vec::new(),
visited_urls: Vec::new(),
captured_at_ms: None,
- };
- let mut stats = BlockStats::default();
+ }
+ }
+
+ fn keys(session_ids: &[&str]) -> FlushKeys {
let mut keys = FlushKeys::default();
- keys.sessions.insert(
- SessionId {
- team_id: 7,
- session_id: "s".to_owned(),
- },
- Key {
- key: [1; 32],
- key_month: "2026-10".to_owned(),
- },
- );
+ for session_id in session_ids {
+ keys.sessions.insert(
+ SessionId {
+ team_id: 7,
+ session_id: (*session_id).to_owned(),
+ },
+ Key {
+ key: [1; 32],
+ key_month: "2026-10".to_owned(),
+ },
+ );
+ }
keys.team_months.insert(7, [2; 32]);
+ keys
+ }
+
+ #[test]
+ fn sends_each_new_public_image_url_once_by_domain() {
+ let block = block(
+ "s",
+ &[
+ "https://cdn.example.com/a.png",
+ "https://cdn.example.com/a.png",
+ "https://img2.cdn.example.com/b.png",
+ "https://cdn.example.com/sent.png",
+ "http://localhost/private.png",
+ "https://bat.bing.com/action/0",
+ ],
+ );
+ let mut stats = BlockStats::default();
let sent_hash = reference_hash(
&image_url_key(&[2; 32]),
try_canonicalize("https://cdn.example.com/sent.png")
@@ -234,14 +301,10 @@ mod tests {
.as_bytes(),
);
let sent = image_url_ref(7, "2026-10", &sent_hash);
+ let already_sent = |reference: &str| reference == sent;
- let jobs = fetch_jobs(
- [(&block, &mut stats)],
- &keys,
- "2026-10",
- 1_000,
- &|reference| reference == sent,
- );
+ let images = FlushImages::new([&block], &keys(&["s"]), "2026-10", &already_sent);
+ let jobs = fetch_jobs(&images, [(0, &block, &mut stats)], 1_000, &already_sent);
let domains: Vec<_> = jobs.keys().cloned().collect();
assert_eq!(domains, ["example.com"]);
let fetched: Vec<_> = jobs["example.com"]
@@ -268,6 +331,41 @@ mod tests {
assert_eq!(stats.image_urls, 4);
}
+ #[test]
+ fn holds_each_unsent_reference_once_and_sends_it_for_the_first_kept_block() {
+ let shared = "https://cdn.example.com/shared.png";
+ let blocks = [
+ block("dropped", &[shared]),
+ block(
+ "first",
+ &[shared, shared, "https://cdn.example.com/own.png"],
+ ),
+ block("second", &[shared]),
+ ];
+ let images = FlushImages::new(
+ &blocks,
+ &keys(&["dropped", "first", "second"]),
+ "2026-10",
+ &|_| false,
+ );
+ assert_eq!(images.unsent_references().count(), 2);
+
+ let mut stats = [BlockStats::default(), BlockStats::default()];
+ let [first_stats, second_stats] = &mut stats;
+ let jobs = fetch_jobs(
+ &images,
+ [(1, &blocks[1], first_stats), (2, &blocks[2], second_stats)],
+ 1_000,
+ &|_| false,
+ );
+ let sessions: Vec<_> = jobs["example.com"]
+ .iter()
+ .map(|job| job["sessionId"].as_str().unwrap())
+ .collect();
+ assert_eq!(sessions, ["first", "first"]);
+ assert_eq!((stats[0].image_urls, stats[1].image_urls), (3, 1));
+ }
+
#[test]
fn splits_a_domains_jobs_at_the_fetchers_record_limit() {
let jobs = BTreeMap::from([(
diff --git a/rust/ai-research/replay-recorder-v4/src/write.rs b/rust/ai-research/replay-recorder-v4/src/write.rs
index 84880cddfc2d..1f2288912ce2 100644
--- a/rust/ai-research/replay-recorder-v4/src/write.rs
+++ b/rust/ai-research/replay-recorder-v4/src/write.rs
@@ -1258,7 +1258,8 @@ mod tests {
keys.sessions.insert(id, key);
keys.team_months.insert(7, [6; 32]);
let mut stats = BlockStats::default();
- let jobs = crate::urls::fetch_jobs([(&block, &mut stats)], &keys, "2026-10", 1, &|_| false);
+ let images = crate::urls::FlushImages::new([&block], &keys, "2026-10", &|_| false);
+ let jobs = crate::urls::fetch_jobs(&images, [(0, &block, &mut stats)], 1, &|_| false);
let (lines, _, _) = write_block(&block, &[6; 32], "2026-10", || Ok(Vec::new())).unwrap();
let lines = String::from_utf8(lines).unwrap();