Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(ai-research): run the image fetcher as a v4 lane (#117984)

Nicholas Waltz86a59204160b
diff --git a/nodejs/src/capabilities.ts b/nodejs/src/capabilities.ts
index 3cb41d31f8b7..829e7e223c50 100644
--- a/nodejs/src/capabilities.ts
+++ b/nodejs/src/capabilities.ts
@@ -211,6 +211,8 @@ export function getPluginServerCapabilities(
case PluginServerMode.recordings_blob_ingestion_v2_ml_image_scrub_dlq_replay:
case PluginServerMode.recordings_blob_ingestion_v2_ml_image_fetch:
case PluginServerMode.recordings_blob_ingestion_v2_ml_image_fetch_retry:
+ case PluginServerMode.ai_research_v4_image_fetch:
+ case PluginServerMode.ai_research_v4_image_fetch_retry:
case PluginServerMode.recording_api:
throw new Error(`Mode ${mode} is handled by IngestionSessionReplayServer, not PluginServer`)
}
diff --git a/nodejs/src/common/config.ts b/nodejs/src/common/config.ts
index 8a6d7df6f711..d718de4234b6 100644
--- a/nodejs/src/common/config.ts
+++ b/nodejs/src/common/config.ts
@@ -42,6 +42,8 @@ export enum PluginServerMode {
recordings_blob_ingestion_v2_ml_image_scrub_dlq_replay = 'recordings-blob-ingestion-v2-ml-image-scrub-dlq-replay',
recordings_blob_ingestion_v2_ml_image_fetch = 'recordings-blob-ingestion-v2-ml-image-fetch',
recordings_blob_ingestion_v2_ml_image_fetch_retry = 'recordings-blob-ingestion-v2-ml-image-fetch-retry',
+ ai_research_v4_image_fetch = 'ai-research-v4-image-fetch',
+ ai_research_v4_image_fetch_retry = 'ai-research-v4-image-fetch-retry',
cdp_processed_events = 'cdp-processed-events',
cdp_person_updates = 'cdp-person-updates',
cdp_data_warehouse_events = 'cdp-data-warehouse-events',
diff --git a/nodejs/src/common/config/kafka-topics.ts b/nodejs/src/common/config/kafka-topics.ts
index 12475ea56851..61c8aa6299a0 100644
--- a/nodejs/src/common/config/kafka-topics.ts
+++ b/nodejs/src/common/config/kafka-topics.ts
@@ -55,6 +55,11 @@ export const KAFKA_SESSION_REPLAY_IMAGE_FETCH = `${prefix}session_replay_image_f
export const KAFKA_SESSION_REPLAY_IMAGE_FETCH_RETRY_1M = `${prefix}ai_research_session_replay_image_fetch_retry_1m${suffix}`
export const KAFKA_SESSION_REPLAY_IMAGE_FETCH_RETRY_10M = `${prefix}ai_research_session_replay_image_fetch_retry_10m${suffix}`
export const KAFKA_SESSION_REPLAY_IMAGE_FETCH_RETRY_1H = `${prefix}ai_research_session_replay_image_fetch_retry_1h${suffix}`
+// The v4 image fetch lane: remote image URLs from the v4 recorder, and their delay tiers.
+export const KAFKA_AI_RESEARCH_V4_IMAGE_FETCH = `${prefix}ai_research_v4_image_fetch${suffix}`
+export const KAFKA_AI_RESEARCH_V4_IMAGE_FETCH_RETRY_1M = `${prefix}ai_research_v4_image_fetch_retry_1m${suffix}`
+export const KAFKA_AI_RESEARCH_V4_IMAGE_FETCH_RETRY_10M = `${prefix}ai_research_v4_image_fetch_retry_10m${suffix}`
+export const KAFKA_AI_RESEARCH_V4_IMAGE_FETCH_RETRY_1H = `${prefix}ai_research_v4_image_fetch_retry_1h${suffix}`
// images the scrub sidecar cannot process, parked so they stop holding the head of their partition.
// The original bytes are kept: unscrubbed content must never reach the ML bucket, but it must not be
diff --git a/nodejs/src/index.ts b/nodejs/src/index.ts
index 03d6ba6766cb..8eb63a6f4773 100644
--- a/nodejs/src/index.ts
+++ b/nodejs/src/index.ts
@@ -6,6 +6,10 @@ import { initTracing } from '~/common/tracing/otel'
import { initSuperProperties } from '~/common/utils/posthog'
import { getDefaultIngestionConsumerConfig } from '~/ingestion/config'
import { PluginServer } from '~/server'
+import {
+ AiResearchV4ImageFetchRetryServer,
+ AiResearchV4ImageFetchServer,
+} from '~/servers/ai-research-v4-image-fetch-server'
import { NodeServer } from '~/servers/base-server'
import { ErrorTrackingServer } from '~/servers/error-tracking-server'
import { IngestionApiServer } from '~/servers/ingestion-api-server'
@@ -57,6 +61,12 @@ function createServer(): NodeServer {
case PluginServerMode.recordings_blob_ingestion_v2_ml_image_fetch_retry:
return new IngestionSessionReplayMlImageFetchRetryServer()
+ case PluginServerMode.ai_research_v4_image_fetch:
+ return new AiResearchV4ImageFetchServer()
+
+ case PluginServerMode.ai_research_v4_image_fetch_retry:
+ return new AiResearchV4ImageFetchRetryServer()
+
case PluginServerMode.recording_api:
return new RecordingApiServer()
diff --git a/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/config.ts b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/config.ts
new file mode 100644
index 000000000000..c4741bf8c4b9
--- /dev/null
+++ b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/config.ts
@@ -0,0 +1,208 @@
+/** Settings of the v4 AI research lanes. */
+import { defaultConfig, overrideConfigWithEnv } from '~/common/config/config'
+import { KAFKA_CLICKHOUSE_TOPHOG } from '~/common/config/kafka-topics'
+import {
+ INGESTION_DOWNSTREAM_PRODUCER,
+ getDefaultKafkaDownstreamProducerEnvConfig,
+} from '~/ingestion/common/outputs/producers'
+import { getDefaultIngestionConsumerConfig } from '~/ingestion/config'
+import {
+ getDefaultSessionRecordingApiConfig,
+ getDefaultSessionRecordingConfig,
+ getDefaultSessionReplayOutputsConfig,
+} from '~/ingestion/pipelines/sessionreplay/config'
+import { getDefaultKafkaSessionreplayProducerEnvConfig } from '~/ingestion/pipelines/sessionreplay/shared/outputs/producer-config'
+import type { IngestionSessionReplayServerConfig } from '~/servers/ingestion-session-replay-server'
+
+import type { AiResearchV4ImageFetchProducerName } from './image-fetch-v4/producer'
+
+export type AiResearchV4Config = {
+ /** The v4 image fetch lane: the team month key table, its KMS key, and the bucket fetched images go to. */
+ AI_RESEARCH_V4_KEY_TABLE: string
+ AI_RESEARCH_V4_KMS_KEY_ARN: string
+ AI_RESEARCH_V4_IMAGES_BUCKET: string
+ /** KMS decrypt requests per second one pod may send for team month keys. */
+ AI_RESEARCH_V4_KMS_REQUESTS_PER_SECOND: number
+ /** The v4 lane's consumer group. Its own setting, so a v4 pod never joins a v3 group. Retry tiers add a suffix. */
+ AI_RESEARCH_V4_IMAGE_FETCH_GROUP_ID: string
+ /** How long all key lookups of one batch may take. A job whose key is not read by then goes to a delay tier. */
+ AI_RESEARCH_V4_IMAGE_FETCH_KEY_BUDGET_MS: number
+ /**
+ * The share of the per registrable domain limits below that the v4 lane uses. The v3 lane's pods hold their own
+ * budget, so a host sees both lanes.
+ */
+ AI_RESEARCH_V4_IMAGE_FETCH_HOST_BUDGET_SHARE: number
+
+ /** US-only, PEM, multiple keys allowed (comma-separated). */
+ AI_RESEARCH_V4_IMAGE_FETCH_WEB_BOT_AUTH_PRIVATE_KEYS: string
+ /** While true, the fetch lane parses input but sends no request and writes no crawl history. */
+ AI_RESEARCH_V4_IMAGE_FETCH_DRY_RUN: boolean
+ /** Optional topic for rejected frontier records. Empty commits and drops rejected input without quarantine. */
+ AI_RESEARCH_V4_IMAGE_FETCH_DLQ_TOPIC: string
+ AI_RESEARCH_V4_IMAGE_FETCH_BATCH_SIZE: number
+ /** Kafka group members per image-fetch worker. */
+ AI_RESEARCH_V4_IMAGE_FETCH_TARGET_PARTITIONS_PER_BATCH: number
+ /** While true, the batches of all group members in one worker join into one fetch pass. While false, each batch starts its own pass. */
+ AI_RESEARCH_V4_IMAGE_FETCH_JOIN_MEMBER_BATCHES: boolean
+ /**
+ * While true, every batch adds its URLs to one pod-wide candidate pool and finishes on its own,
+ * and AI_RESEARCH_V4_IMAGE_FETCH_JOIN_MEMBER_BATCHES has no effect.
+ */
+ AI_RESEARCH_V4_IMAGE_FETCH_CONTINUOUS_POOL: boolean
+ /** A group member reads its next batch once fewer of its queued URLs than this can run. */
+ AI_RESEARCH_V4_IMAGE_FETCH_POOL_REFILL_RUNNABLE_URLS: number
+ /** A group member also waits while this many of its URLs are queued. The wait stops after 30 s to keep the consumer loop healthy, so a member can pass this by about one batch until pass deadlines remove its queued URLs. */
+ AI_RESEARCH_V4_IMAGE_FETCH_POOL_MAX_QUEUED_URLS_PER_MEMBER: number
+ /** The lane's own crawl history table: the URL history of v4 references, and robots.txt and tdmrep by origin. */
+ AI_RESEARCH_V4_IMAGE_FETCH_HISTORY_TABLE: string
+ /** Bounds one DynamoDB request so an unavailable store cannot hold the poll loop. */
+ AI_RESEARCH_V4_IMAGE_FETCH_DYNAMODB_TIMEOUT_MS: number
+ /** The shortest time before the lane fetches a URL again. */
+ AI_RESEARCH_V4_IMAGE_FETCH_CRAWL_HISTORY_TTL_SECONDS: number
+
+ /** Optional steady request rate for one registrable domain. Zero uses the concurrency limit only. */
+ AI_RESEARCH_V4_IMAGE_FETCH_REGISTRABLE_DOMAIN_REQUESTS_PER_SECOND: number
+ /** Tokens one idle registrable domain can retain when the steady request rate is enabled. */
+ AI_RESEARCH_V4_IMAGE_FETCH_REGISTRABLE_DOMAIN_BURST: number
+ /** Also the worker count for one registrable domain or origin. */
+ AI_RESEARCH_V4_IMAGE_FETCH_MAX_CONCURRENT_PER_REGISTRABLE_DOMAIN: number
+ /**
+ * Requests this pod holds open across every registrable domain at once.
+ *
+ * The topic key sends one registrable domain to one partition and one pod. Its rate limit lives
+ * on that pod. The state limits prevent unbounded in-memory maps.
+ *
+ * This bounds the pod instead. A body reads into a buffer, so the resident peak is roughly this
+ * times AI_RESEARCH_V4_IMAGE_FETCH_MAX_IMAGE_BYTES, and the sockets and the DNS lookups
+ * scale with it too. Raise the pod's memory before you raise this.
+ */
+ AI_RESEARCH_V4_IMAGE_FETCH_MAX_IN_FLIGHT_REQUESTS: number
+ /** Image writes to S3, and republished frontier records, in flight at once. */
+ AI_RESEARCH_V4_IMAGE_FETCH_MAX_PENDING_PUBLISHES: number
+ /**
+ * Wall time the fetch pass of one poll batch may take.
+ *
+ * A batch can hold more URLs for one registrable domain than a polite rate carries in this
+ * time. What the pass does not reach goes to a delay topic without spending a hop. The value
+ * sits well inside Kafka's max.poll.interval.ms of 300s, which the crawl history round trips
+ * share.
+ */
+ AI_RESEARCH_V4_IMAGE_FETCH_REQUEST_BUDGET_MS: number
+ /** Refused before the body when the response declares more, and abandoned mid-body when it declares nothing. */
+ AI_RESEARCH_V4_IMAGE_FETCH_MAX_IMAGE_BYTES: number
+ /** Covers one URL including its redirects, separate from the connect timeout of the shared request layer. */
+ AI_RESEARCH_V4_IMAGE_FETCH_REQUEST_TIMEOUT_MS: number
+ AI_RESEARCH_V4_IMAGE_FETCH_MAX_REDIRECTS: number
+ /** Consecutive failures before this pod stops sending to one registrable domain. */
+ AI_RESEARCH_V4_IMAGE_FETCH_REGISTRABLE_DOMAIN_BREAKER_FAILURES: number
+ AI_RESEARCH_V4_IMAGE_FETCH_REGISTRABLE_DOMAIN_BREAKER_COOLDOWN_MS: number
+ /** The longest calculated breaker back-off. A longer valid `Retry-After` remains authoritative. */
+ AI_RESEARCH_V4_IMAGE_FETCH_REGISTRABLE_DOMAIN_BREAKER_MAX_COOLDOWN_MS: number
+ /** Registrable domains one pod holds request-control state for. */
+ AI_RESEARCH_V4_IMAGE_FETCH_MAX_TRACKED_REGISTRABLE_DOMAINS: number
+ /** Origins one pod holds configuration and crawl-delay state for. */
+ AI_RESEARCH_V4_IMAGE_FETCH_MAX_TRACKED_ORIGINS: number
+
+ /**
+ * The delay topic one retry pod drains, and the period every record in it waits.
+ *
+ * One server runs every tier, so three deployments of one image cover 1 minute, 10 minutes, and
+ * one hour. The pod's `max.poll.interval.ms` must exceed the period, or the broker evicts the
+ * consumer while it sleeps.
+ */
+ AI_RESEARCH_V4_IMAGE_FETCH_RETRY_TOPIC: string
+ AI_RESEARCH_V4_IMAGE_FETCH_RETRY_DELAY_MS: number
+ /** Records released together after one wait based on the latest broker append timestamp. */
+ AI_RESEARCH_V4_IMAGE_FETCH_RETRY_BATCH_SIZE: number
+
+ /** Where the lane's TopHog rows go. */
+ AI_RESEARCH_V4_IMAGE_FETCH_TOPHOG_TOPIC: string
+ AI_RESEARCH_V4_IMAGE_FETCH_TOPHOG_PRODUCER: AiResearchV4ImageFetchProducerName
+
+ /**
+ * The lane's own producer on the replay cluster, for frontier and delay tier records. Its broker and security
+ * settings are those of the replay producer.
+ */
+ KAFKA_AI_RESEARCH_V4_IMAGE_FETCH_PRODUCER_COMPRESSION_CODEC: string
+ KAFKA_AI_RESEARCH_V4_IMAGE_FETCH_PRODUCER_LINGER_MS: string
+ KAFKA_AI_RESEARCH_V4_IMAGE_FETCH_PRODUCER_QUEUE_BUFFERING_MAX_MESSAGES: string
+ KAFKA_AI_RESEARCH_V4_IMAGE_FETCH_PRODUCER_QUEUE_BUFFERING_MAX_KBYTES: string
+ KAFKA_AI_RESEARCH_V4_IMAGE_FETCH_PRODUCER_MESSAGE_MAX_BYTES: string
+ /** A short local delivery timeout, so a slow topic fails a batch fast instead of holding its offsets. */
+ KAFKA_AI_RESEARCH_V4_IMAGE_FETCH_PRODUCER_MESSAGE_TIMEOUT_MS: string
+}
+
+export function getDefaultAiResearchV4Config(): AiResearchV4Config {
+ return {
+ AI_RESEARCH_V4_KEY_TABLE: '',
+ AI_RESEARCH_V4_KMS_KEY_ARN: '',
+ AI_RESEARCH_V4_IMAGES_BUCKET: '',
+ AI_RESEARCH_V4_KMS_REQUESTS_PER_SECOND: 150,
+ AI_RESEARCH_V4_IMAGE_FETCH_GROUP_ID: 'ai-research-v4-image-fetch',
+ AI_RESEARCH_V4_IMAGE_FETCH_KEY_BUDGET_MS: 60_000,
+ AI_RESEARCH_V4_IMAGE_FETCH_HOST_BUDGET_SHARE: 0.25,
+
+ AI_RESEARCH_V4_IMAGE_FETCH_WEB_BOT_AUTH_PRIVATE_KEYS: '',
+ AI_RESEARCH_V4_IMAGE_FETCH_DRY_RUN: true,
+ AI_RESEARCH_V4_IMAGE_FETCH_DLQ_TOPIC: '',
+ AI_RESEARCH_V4_IMAGE_FETCH_BATCH_SIZE: 500,
+ AI_RESEARCH_V4_IMAGE_FETCH_TARGET_PARTITIONS_PER_BATCH: 2,
+ AI_RESEARCH_V4_IMAGE_FETCH_JOIN_MEMBER_BATCHES: true,
+ AI_RESEARCH_V4_IMAGE_FETCH_CONTINUOUS_POOL: false,
+ AI_RESEARCH_V4_IMAGE_FETCH_POOL_REFILL_RUNNABLE_URLS: 100,
+ AI_RESEARCH_V4_IMAGE_FETCH_POOL_MAX_QUEUED_URLS_PER_MEMBER: 2_000,
+ AI_RESEARCH_V4_IMAGE_FETCH_HISTORY_TABLE: '',
+ AI_RESEARCH_V4_IMAGE_FETCH_DYNAMODB_TIMEOUT_MS: 5_000,
+ AI_RESEARCH_V4_IMAGE_FETCH_CRAWL_HISTORY_TTL_SECONDS: 30 * 24 * 60 * 60,
+ AI_RESEARCH_V4_IMAGE_FETCH_REGISTRABLE_DOMAIN_REQUESTS_PER_SECOND: 0,
+ AI_RESEARCH_V4_IMAGE_FETCH_REGISTRABLE_DOMAIN_BURST: 6,
+ AI_RESEARCH_V4_IMAGE_FETCH_MAX_CONCURRENT_PER_REGISTRABLE_DOMAIN: 6,
+ AI_RESEARCH_V4_IMAGE_FETCH_MAX_IN_FLIGHT_REQUESTS: 300,
+ AI_RESEARCH_V4_IMAGE_FETCH_MAX_PENDING_PUBLISHES: 100,
+ AI_RESEARCH_V4_IMAGE_FETCH_REQUEST_BUDGET_MS: 40_000,
+ AI_RESEARCH_V4_IMAGE_FETCH_MAX_IMAGE_BYTES: 20 * 1024 * 1024,
+ AI_RESEARCH_V4_IMAGE_FETCH_REQUEST_TIMEOUT_MS: 10_000,
+ AI_RESEARCH_V4_IMAGE_FETCH_MAX_REDIRECTS: 3,
+ AI_RESEARCH_V4_IMAGE_FETCH_REGISTRABLE_DOMAIN_BREAKER_FAILURES: 5,
+ AI_RESEARCH_V4_IMAGE_FETCH_REGISTRABLE_DOMAIN_BREAKER_COOLDOWN_MS: 60_000,
+ AI_RESEARCH_V4_IMAGE_FETCH_REGISTRABLE_DOMAIN_BREAKER_MAX_COOLDOWN_MS: 60 * 60 * 1000,
+ AI_RESEARCH_V4_IMAGE_FETCH_MAX_TRACKED_REGISTRABLE_DOMAINS: 20_000,
+ AI_RESEARCH_V4_IMAGE_FETCH_MAX_TRACKED_ORIGINS: 20_000,
+ AI_RESEARCH_V4_IMAGE_FETCH_RETRY_TOPIC: '',
+ AI_RESEARCH_V4_IMAGE_FETCH_RETRY_DELAY_MS: 60_000,
+ AI_RESEARCH_V4_IMAGE_FETCH_RETRY_BATCH_SIZE: 500,
+
+ AI_RESEARCH_V4_IMAGE_FETCH_TOPHOG_TOPIC: KAFKA_CLICKHOUSE_TOPHOG,
+ AI_RESEARCH_V4_IMAGE_FETCH_TOPHOG_PRODUCER: INGESTION_DOWNSTREAM_PRODUCER,
+
+ KAFKA_AI_RESEARCH_V4_IMAGE_FETCH_PRODUCER_COMPRESSION_CODEC: 'snappy',
+ KAFKA_AI_RESEARCH_V4_IMAGE_FETCH_PRODUCER_LINGER_MS: '',
+ KAFKA_AI_RESEARCH_V4_IMAGE_FETCH_PRODUCER_QUEUE_BUFFERING_MAX_MESSAGES: '50000',
+ KAFKA_AI_RESEARCH_V4_IMAGE_FETCH_PRODUCER_QUEUE_BUFFERING_MAX_KBYTES: '32768',
+ KAFKA_AI_RESEARCH_V4_IMAGE_FETCH_PRODUCER_MESSAGE_MAX_BYTES: `${40 * 1024 * 1024 + 64 * 1024}`,
+ KAFKA_AI_RESEARCH_V4_IMAGE_FETCH_PRODUCER_MESSAGE_TIMEOUT_MS: '20000',
+ }
+}
+
+export function getAiResearchV4Config(env: Record<string, string | undefined> = process.env): AiResearchV4Config {
+ return overrideConfigWithEnv(getDefaultAiResearchV4Config(), env)
+}
+
+/** Full config of a v4 server: the replay settings it shares with the primary ingester, plus the v4 settings. */
+export type AiResearchV4ServerConfig = IngestionSessionReplayServerConfig & AiResearchV4Config
+
+export function buildAiResearchV4ServerConfig(
+ config: Partial<AiResearchV4ServerConfig> = {}
+): AiResearchV4ServerConfig {
+ return {
+ ...defaultConfig,
+ ...overrideConfigWithEnv(getDefaultIngestionConsumerConfig()),
+ ...overrideConfigWithEnv(getDefaultKafkaDownstreamProducerEnvConfig()),
+ ...overrideConfigWithEnv(getDefaultKafkaSessionreplayProducerEnvConfig()),
+ ...overrideConfigWithEnv(getDefaultSessionRecordingConfig()),
+ ...overrideConfigWithEnv(getDefaultSessionRecordingApiConfig()),
+ ...overrideConfigWithEnv(getDefaultSessionReplayOutputsConfig()),
+ ...getAiResearchV4Config(),
+ ...config,
+ }
+}
diff --git a/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/block-reason.ts b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/block-reason.ts
new file mode 100644
index 000000000000..3fa500061de1
--- /dev/null
+++ b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/block-reason.ts
@@ -0,0 +1,38 @@
+export type ImageFetchBlockReason =
+ | 'domain_concurrency'
+ | 'request_capacity'
+ | 'origin_crawl_delay'
+ | 'registrable_domain_rate'
+ | 'retry_after'
+ | 'transient_backoff'
+ | 'unknown_backoff'
+ | 'configuration_unreachable'
+ | 'configuration_deferred'
+ | 'breaker_open'
+ | 'connection_limit'
+ | 'origin_map_full'
+ | 'registrable_domain_map_full'
+ | 'pass_deadline'
+ | 'request_deadline'
+
+const IMAGE_FETCH_BLOCK_REASONS = new Set<ImageFetchBlockReason>([
+ 'domain_concurrency',
+ 'request_capacity',
+ 'origin_crawl_delay',
+ 'registrable_domain_rate',
+ 'retry_after',
+ 'transient_backoff',
+ 'unknown_backoff',
+ 'configuration_unreachable',
+ 'configuration_deferred',
+ 'breaker_open',
+ 'connection_limit',
+ 'origin_map_full',
+ 'registrable_domain_map_full',
+ 'pass_deadline',
+ 'request_deadline',
+])
+
+export function isImageFetchBlockReason(value: unknown): value is ImageFetchBlockReason {
+ return typeof value === 'string' && IMAGE_FETCH_BLOCK_REASONS.has(value as ImageFetchBlockReason)
+}
diff --git a/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/collected-urls-record.test.ts b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/collected-urls-record.test.ts
new file mode 100644
index 000000000000..cd7fbca54980
--- /dev/null
+++ b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/collected-urls-record.test.ts
@@ -0,0 +1,162 @@
+import {
+ FetchCandidate,
+ MAX_HOPS,
+ MAX_JOBS_PER_RECORD,
+ MAX_RECORD_BYTES,
+ parseCollectedUrlsRecord,
+ serializeFrontierRecord,
+} from './collected-urls-record'
+
+const REF = `imageurl:v4:7:2026-10:${'a'.repeat(22)}`
+
+function candidate(overrides: Partial<FetchCandidate> = {}): FetchCandidate {
+ return {
+ originalRef: REF,
+ currentUrl: 'https://cdn.example.com/a.png',
+ host: 'cdn.example.com',
+ origin: 'https://cdn.example.com',
+ registrableDomain: 'example.com',
+ remainingHops: MAX_HOPS,
+ notBeforeMs: 0,
+ firstSeenAtMs: 1_700_000_000_000,
+ fetchCount: 0,
+ republishCount: 0,
+ lastRepublishReason: null,
+ ...overrides,
+ }
+}
+
+function record(overrides: Record<string, unknown> = {}): Buffer {
+ return Buffer.from(
+ JSON.stringify({
+ v: 2,
+ jobs: [
+ {
+ originalRef: REF,
+ currentUrl: 'https://cdn.example.com/a.png',
+ remainingHops: MAX_HOPS,
+ notBeforeMs: 0,
+ firstSeenAtMs: 1_700_000_000_000,
+ fetchCount: 0,
+ republishCount: 0,
+ lastRepublishReason: null,
+ ...overrides,
+ },
+ ],
+ })
+ )
+}
+
+describe('frontier record', () => {
+ it('round trips the durable job state', () => {
+ const durableCandidate = candidate({ lastBlockReason: 'configuration_unreachable' })
+ const parsed = parseCollectedUrlsRecord(serializeFrontierRecord([durableCandidate]), 'example.com')
+
+ expect(parsed).toEqual({ ok: true, candidates: [durableCandidate], urlCount: 1, rejected: [], skipped: [] })
+ })
+
+ it('does not persist source partition attribution', () => {
+ const parsed = parseCollectedUrlsRecord(
+ serializeFrontierRecord([candidate({ sourcePartitions: [7] })]),
+ 'example.com'
+ )
+
+ expect(parsed).toEqual({ ok: true, candidates: [candidate()], urlCount: 1, rejected: [], skipped: [] })
+ })
+
+ it('accepts and removes the legacy low-origin-diversity marker', () => {
+ expect(parseCollectedUrlsRecord(record({ lowOriginDiversityDeferred: true }), 'example.com')).toEqual({
+ ok: true,
+ candidates: [candidate()],
+ urlCount: 1,
+ rejected: [],
+ skipped: [],
+ })
+ })
+
+ it('drops a job with a non-boolean low-origin-diversity marker', () => {
+ expect(parseCollectedUrlsRecord(record({ lowOriginDiversityDeferred: 'true' }), 'example.com')).toEqual({
+ ok: false,
+ reason: 'bad_url',
+ })
+ })
+
+ it.each([
+ ['missing value', null, 'example.com', 'malformed'],
+ ['missing key', record(), null, 'malformed'],
+ ['invalid JSON', Buffer.from('{'), 'example.com', 'malformed'],
+ ['unknown version', Buffer.from('{"v":3,"jobs":[]}'), 'example.com', 'unsupported_version'],
+ ['a v3 legacy record', Buffer.from('{"v":1,"urls":[]}'), 'example.com', 'unsupported_version'],
+ ['oversized bytes', Buffer.alloc(MAX_RECORD_BYTES + 1), 'example.com', 'oversized_record'],
+ [
+ 'too many jobs',
+ Buffer.from(JSON.stringify({ v: 2, jobs: Array(MAX_JOBS_PER_RECORD + 1).fill({}) })),
+ 'example.com',
+ 'oversized_record',
+ ],
+ ])('refuses %s', (_name, value, key, reason) => {
+ expect(parseCollectedUrlsRecord(value, key)).toEqual({ ok: false, reason })
+ })
+
+ it.each([
+ ['global ref', `imageurl:${'a'.repeat(22)}`, 'bad_ref'],
+ ['v3 ref', `imageurl:v3:7:2026-10:${'a'.repeat(22)}`, 'bad_ref'],
+ ['byte ref', `image:${'b'.repeat(32)}:${'a'.repeat(22)}`, 'bad_ref'],
+ ['credentials', 'https://user:[email protected]/a.png', 'bad_url'],
+ ['HTTP', 'http://cdn.example.com/a.png', 'bad_url'],
+ ['non-default port', 'https://cdn.example.com:444/a.png', 'bad_url'],
+ ['private address', 'https://127.0.0.1/a.png', 'bad_url'],
+ ['non-canonical URL', 'https://CDN.EXAMPLE.COM/a.png', 'bad_url'],
+ ])('drops a job with %s', (_name, value, reason) => {
+ const overrides = value.startsWith('image') ? { originalRef: value } : { currentUrl: value }
+ const parsed = parseCollectedUrlsRecord(record(overrides), 'example.com')
+
+ expect(parsed).toEqual({ ok: false, reason })
+ })
+
+ it('skips a tracking beacon job and keeps the rest of the record', () => {
+ const beacon = candidate({
+ originalRef: `imageurl:v4:7:2026-10:${'b'.repeat(22)}`,
+ currentUrl: 'https://analytics.twitter.com/i/adsct?txn_id=abc&p_id=Twitter',
+ host: 'analytics.twitter.com',
+ origin: 'https://analytics.twitter.com',
+ registrableDomain: 'twitter.com',
+ })
+ const image = candidate({
+ currentUrl: 'https://cdn.twitter.com/logo.png',
+ host: 'cdn.twitter.com',
+ origin: 'https://cdn.twitter.com',
+ registrableDomain: 'twitter.com',
+ })
+
+ const parsed = parseCollectedUrlsRecord(serializeFrontierRecord([beacon, image]), 'twitter.com')
+
+ expect(parsed).toEqual({
+ ok: true,
+ candidates: [image],
+ urlCount: 2,
+ rejected: [],
+ skipped: [{ reason: 'tracking_beacon' }],
+ })
+ })
+
+ it('drops a job placed on another registrable-domain partition', () => {
+ const parsed = parseCollectedUrlsRecord(record(), 'other.net')
+
+ expect(parsed).toEqual({ ok: false, reason: 'foreign_domain' })
+ })
+
+ it.each([
+ ['remainingHops', MAX_HOPS + 1],
+ ['notBeforeMs', -1],
+ ['firstSeenAtMs', Number.POSITIVE_INFINITY],
+ ['fetchCount', 1.5],
+ ['republishCount', Number.MAX_SAFE_INTEGER + 1],
+ ['lastRepublishReason', 'unknown'],
+ ['lastBlockReason', 'unknown'],
+ ])('drops an invalid %s', (field, value) => {
+ const parsed = parseCollectedUrlsRecord(record({ [field]: value }), 'example.com')
+
+ expect(parsed).toEqual({ ok: false, reason: 'bad_url' })
+ })
+})
diff --git a/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/collected-urls-record.ts b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/collected-urls-record.ts
new file mode 100644
index 000000000000..18a8eb77ed63
--- /dev/null
+++ b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/collected-urls-record.ts
@@ -0,0 +1,239 @@
+import type { UrlPolicyDecline } from '@posthog/ai-research-v4-native'
+
+import { parseJSON } from '~/common/utils/json-parse'
+
+import { ImageFetchBlockReason, isImageFetchBlockReason } from './block-reason'
+import { tryCanonicalizeUrl } from './politeness-key'
+import { parseV4UrlRef } from './v4-url-ref'
+
+export const MAX_HOPS = 10
+export const MAX_JOBS_PER_RECORD = 1_000
+export const MAX_RECORD_BYTES = 512 * 1024
+
+export type UrlDropReason =
+ | 'malformed'
+ | 'unsupported_version'
+ | 'bad_ref'
+ | 'bad_url'
+ | 'foreign_domain'
+ | 'oversized_record'
+/** Why the parser dropped a job on its own, without rejecting the record that carries it. */
+export type UrlSkipReason = UrlPolicyDecline
+export type StoredRepublishReason =
+ | 'redirect'
+ | 'retry'
+ | 'not_ready'
+ | 'pass_deadline'
+ | 'origin_map_full'
+ | 'registrable_domain_map_full'
+export type RepublishReason = StoredRepublishReason
+
+export interface FetchCandidate {
+ /** Raw UUIDv7 session ID of the session that collected the URL, carried through republishes. */
+ sessionId?: string
+ originalRef: string
+ currentUrl: string
+ host: string
+ origin: string
+ registrableDomain: string
+ remainingHops: number
+ notBeforeMs: number
+ firstSeenAtMs: number
+ fetchCount: number
+ republishCount: number
+ lastRepublishReason: StoredRepublishReason | null
+ lastBlockReason?: ImageFetchBlockReason
+ sourcePartitions?: readonly number[]
+}
+
+export interface FrontierRecord {
+ v: 2
+ jobs: Array<
+ Pick<
+ FetchCandidate,
+ | 'sessionId'
+ | 'originalRef'
+ | 'currentUrl'
+ | 'remainingHops'
+ | 'notBeforeMs'
+ | 'firstSeenAtMs'
+ | 'fetchCount'
+ | 'republishCount'
+ | 'lastRepublishReason'
+ | 'lastBlockReason'
+ >
+ >
+}
+
+export type RecordParse =
+ | {
+ ok: true
+ candidates: FetchCandidate[]
+ urlCount: number
+ rejected: { reason: UrlDropReason }[]
+ skipped: { reason: UrlSkipReason }[]
+ }
+ | {
+ ok: false
+ reason: Extract<
+ UrlDropReason,
+ 'malformed' | 'unsupported_version' | 'oversized_record' | 'bad_ref' | 'bad_url' | 'foreign_domain'
+ >
+ }
+
+function isRecord(value: unknown): value is Record<string, unknown> {
+ return typeof value === 'object' && value !== null && !Array.isArray(value)
+}
+
+function isNonNegativeSafeInteger(value: unknown): value is number {
+ return Number.isSafeInteger(value) && (value as number) >= 0
+}
+
+function isStoredRepublishReason(value: unknown): value is StoredRepublishReason | null {
+ return (
+ value === null ||
+ value === 'redirect' ||
+ value === 'retry' ||
+ value === 'not_ready' ||
+ value === 'pass_deadline' ||
+ value === 'origin_map_full' ||
+ value === 'registrable_domain_map_full'
+ )
+}
+
+export function parseCollectedUrlsRecord(value: Buffer | null, key: string | null): RecordParse {
+ if (!value || !key) {
+ return { ok: false, reason: 'malformed' }
+ }
+ if (value.length > MAX_RECORD_BYTES) {
+ return { ok: false, reason: 'oversized_record' }
+ }
+
+ let parsed: unknown
+ try {
+ parsed = parseJSON(value.toString())
+ } catch {
+ return { ok: false, reason: 'malformed' }
+ }
+ if (!isRecord(parsed)) {
+ return { ok: false, reason: 'malformed' }
+ }
+ if (parsed.v !== 2 && !(parsed.v === 1 && Array.isArray(parsed.jobs))) {
+ return { ok: false, reason: 'unsupported_version' }
+ }
+ if (!Array.isArray(parsed.jobs) || parsed.jobs.length === 0) {
+ return { ok: false, reason: 'malformed' }
+ }
+ if (parsed.jobs.length > MAX_JOBS_PER_RECORD) {
+ return { ok: false, reason: 'oversized_record' }
+ }
+
+ const candidates: FetchCandidate[] = []
+ const rejected: { reason: UrlDropReason }[] = []
+ const skipped: { reason: UrlSkipReason }[] = []
+ for (const job of parsed.jobs) {
+ const parsedJob = parseJob(job, key)
+ if (parsedJob.kind === 'rejected') {
+ return { ok: false, reason: parsedJob.reason }
+ }
+ if (parsedJob.kind === 'skipped') {
+ skipped.push({ reason: parsedJob.reason })
+ continue
+ }
+ candidates.push(parsedJob.candidate)
+ }
+ return { ok: true, candidates, urlCount: parsed.jobs.length, rejected, skipped }
+}
+
+type ParsedJob =
+ | { kind: 'candidate'; candidate: FetchCandidate }
+ | { kind: 'rejected'; reason: Extract<UrlDropReason, 'bad_ref' | 'bad_url' | 'foreign_domain'> }
+ | { kind: 'skipped'; reason: UrlSkipReason }
+
+function parseJob(job: unknown, kafkaKey: string): ParsedJob {
+ if (!isRecord(job)) {
+ return { kind: 'rejected', reason: 'bad_url' }
+ }
+ const {
+ sessionId,
+ originalRef,
+ currentUrl,
+ remainingHops,
+ notBeforeMs,
+ firstSeenAtMs,
+ fetchCount,
+ republishCount,
+ lastRepublishReason,
+ lastBlockReason,
+ lowOriginDiversityDeferred,
+ } = job
+ if (
+ (sessionId !== undefined && typeof sessionId !== 'string') ||
+ typeof originalRef !== 'string' ||
+ typeof currentUrl !== 'string' ||
+ !isNonNegativeSafeInteger(remainingHops) ||
+ remainingHops > MAX_HOPS ||
+ !isNonNegativeSafeInteger(notBeforeMs) ||
+ !isNonNegativeSafeInteger(firstSeenAtMs) ||
+ !isNonNegativeSafeInteger(fetchCount) ||
+ !isNonNegativeSafeInteger(republishCount) ||
+ !isStoredRepublishReason(lastRepublishReason) ||
+ (lastBlockReason !== undefined && !isImageFetchBlockReason(lastBlockReason)) ||
+ (lowOriginDiversityDeferred !== undefined && typeof lowOriginDiversityDeferred !== 'boolean')
+ ) {
+ return { kind: 'rejected', reason: 'bad_url' }
+ }
+ if (!parseV4UrlRef(originalRef)) {
+ return { kind: 'rejected', reason: 'bad_ref' }
+ }
+ const verdict = tryCanonicalizeUrl(currentUrl)
+ if (!verdict.ok) {
+ // A rejected job sends its whole record to the dead-letter topic, and a record can hold an
+ // unwanted URL next to real images, so an unwanted URL is skipped on its own instead.
+ return verdict.unwanted ? { kind: 'skipped', reason: verdict.decline } : { kind: 'rejected', reason: 'bad_url' }
+ }
+ const canonical = verdict.url
+ if (canonical.fetch !== currentUrl) {
+ return { kind: 'rejected', reason: 'bad_url' }
+ }
+ if (canonical.domain !== kafkaKey) {
+ return { kind: 'rejected', reason: 'foreign_domain' }
+ }
+ return {
+ kind: 'candidate',
+ candidate: {
+ ...(sessionId ? { sessionId } : {}),
+ originalRef,
+ currentUrl,
+ host: canonical.host,
+ origin: new URL(currentUrl).origin,
+ registrableDomain: canonical.domain,
+ remainingHops,
+ notBeforeMs,
+ firstSeenAtMs,
+ fetchCount,
+ republishCount,
+ lastRepublishReason,
+ lastBlockReason,
+ },
+ }
+}
+
+export function serializeFrontierRecord(candidates: FetchCandidate[]): Buffer {
+ const record: FrontierRecord = {
+ v: 2,
+ jobs: candidates.map((candidate) => ({
+ ...(candidate.sessionId ? { sessionId: candidate.sessionId } : {}),
+ originalRef: candidate.originalRef,
+ currentUrl: candidate.currentUrl,
+ remainingHops: candidate.remainingHops,
+ notBeforeMs: candidate.notBeforeMs,
+ firstSeenAtMs: candidate.firstSeenAtMs,
+ fetchCount: candidate.fetchCount,
+ republishCount: candidate.republishCount,
+ lastRepublishReason: candidate.lastRepublishReason,
+ lastBlockReason: candidate.lastBlockReason,
+ })),
+ }
+ return Buffer.from(JSON.stringify(record))
+}
diff --git a/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/configuration-policy.test.ts b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/configuration-policy.test.ts
new file mode 100644
index 000000000000..d3656d8c5eb9
--- /dev/null
+++ b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/configuration-policy.test.ts
@@ -0,0 +1,699 @@
+import { register } from 'prom-client'
+
+import { StreamedResponse, fetchStreamed } from '~/common/utils/request'
+
+import {
+ ConfigurationFetchResult,
+ ConfigurationPolicyService,
+ HttpConfigurationFetcher,
+ explicitFreshnessLifetimeMs,
+ parseRobotsPolicy,
+ responseOptOutReason,
+} from './configuration-policy'
+import { ConfigurationCacheItem, ConfigurationFile, configurationCacheKey } from './crawl-history'
+
+jest.mock('~/common/utils/request', () => ({
+ ...jest.requireActual('~/common/utils/request'),
+ fetchStreamed: jest.fn(),
+}))
+
+const fetchStreamedMock = fetchStreamed as jest.MockedFunction<typeof fetchStreamed>
+
+const NOW_MS = Date.parse('2026-08-22T12:00:00Z')
+const ORIGIN = 'https://example.com'
+
+function cached(
+ file: ConfigurationFile,
+ status: ConfigurationCacheItem['status'],
+ body?: string
+): ConfigurationCacheItem {
+ return {
+ kind: file,
+ key: configurationCacheKey(ORIGIN, file),
+ origin: ORIGIN,
+ status,
+ body,
+ fetchedAtMs: NOW_MS,
+ refreshAtMs: NOW_MS + 23 * 60 * 60 * 1000,
+ freshUntilMs: NOW_MS + 24 * 60 * 60 * 1000,
+ retryAtMs: NOW_MS,
+ storageExpiresAtMs: NOW_MS + 30 * 24 * 60 * 60 * 1000,
+ }
+}
+
+function service(results: Partial<Record<ConfigurationFile, ConfigurationFetchResult>> = {}): {
+ policy: ConfigurationPolicyService
+ fetch: jest.Mock
+} {
+ const fetch = jest.fn((_: string, file: ConfigurationFile) =>
+ Promise.resolve(results[file] ?? ({ outcome: 'absent' } as const))
+ )
+ return {
+ policy: new ConfigurationPolicyService({ fetch } as unknown as HttpConfigurationFetcher),
+ fetch,
+ }
+}
+
+function response(
+ status: number,
+ headerLines: Array<{ name: string; value: string }> = [],
+ body: { bytes: Buffer; overLimit: boolean } = { bytes: Buffer.alloc(0), overLimit: false }
+): StreamedResponse {
+ return {
+ status,
+ headers: Object.fromEntries(headerLines.map(({ name, value }) => [name, value])),
+ headerLines,
+ read: jest.fn().mockResolvedValue(body),
+ discard: jest.fn(),
+ }
+}
+
+function httpFetcher(headersForGet = jest.fn().mockReturnValue({})): HttpConfigurationFetcher {
+ return new HttpConfigurationFetcher(
+ { headersForGet },
+ {
+ run: async (_url, _deadlineMs, request) => ({ ran: true as const, value: await request() }),
+ },
+ 5_000
+ )
+}
+
+describe('robots policy', () => {
+ it('uses the RFC 9309 group and path matcher', async () => {
+ const body = [
+ 'User-agent: *',
+ 'Disallow: /wildcard',
+ 'User-agent: PostHogImageFetcherBot',
+ 'Disallow: /private',
+ 'Allow: /private/public.png',
+ ].join('\n')
+
+ await expect(parseRobotsPolicy(body, `${ORIGIN}/private/image.png`)).resolves.toMatchObject({
+ allowed: false,
+ reason: 'robots_disallow',
+ })
+ await expect(parseRobotsPolicy(body, `${ORIGIN}/private/public.png`)).resolves.toMatchObject({
+ allowed: true,
+ crawlDelayMs: 0,
+ })
+ await expect(parseRobotsPolicy(body, `${ORIGIN}/wildcard/image.png`)).resolves.toMatchObject({
+ allowed: true,
+ crawlDelayMs: 0,
+ })
+ })
+
+ it('uses the greatest valid crawl delay in every selected field line', async () => {
+ const body = [
+ 'User-agent: PostHogImageFetcherBot',
+ 'Crawl-delay: 0.25',
+ 'Crawl-delay: invalid',
+ 'Crawl-delay: 0x10',
+ 'Crawl-delay: 1e3',
+ 'Crawl-delay: 2.5',
+ // 16.1 * 1000 is 16100.000000000002, which is not a safe integer.
+ 'Crawl-delay: 16.1',
+ 'Allow: /',
+ ].join('\n')
+
+ const result = await parseRobotsPolicy(body, `${ORIGIN}/image.png`)
+ expect(result).toMatchObject({
+ allowed: true,
+ crawlDelayMs: 16_100,
+ })
+ })
+
+ it.each([
+ ['a token refusal', 'train-ai=n', false],
+ ['a later allowance', 'train-ai=n, train-ai=y', true],
+ ['a quoted value', 'train-ai="n"', true],
+ ['a boolean value', 'train-ai=?0', true],
+ ['an invalid dictionary', 'train-ai=n, INVALID=y', true],
+ ])('applies Content-Usage with %s', async (_name, value, allowed) => {
+ const body = `User-agent: *\nContent-Usage: ${value}\nAllow: /`
+
+ await expect(parseRobotsPolicy(body, `${ORIGIN}/image.png`)).resolves.toMatchObject({ allowed })
+ })
+
+ it('applies a path-scoped Content-Signal only to a matching URL', async () => {
+ const body = 'User-agent: *\nContent-Signal: /private/ ai-train=no, search=yes\nAllow: /'
+
+ await expect(parseRobotsPolicy(body, `${ORIGIN}/private/image.png`)).resolves.toMatchObject({
+ allowed: false,
+ reason: 'content_signal',
+ })
+ await expect(parseRobotsPolicy(body, `${ORIGIN}/public/image.png`)).resolves.toMatchObject({ allowed: true })
+ })
+})
+
+describe('response opt-out policy', () => {
+ it('uses the last duplicate Content-Usage dictionary member', () => {
+ expect(
+ responseOptOutReason(
+ [
+ { name: 'content-usage', value: 'train-ai=n' },
+ { name: 'content-usage', value: 'train-ai=y' },
+ ],
+ false
+ )
+ ).toBeUndefined()
+ expect(
+ responseOptOutReason(
+ [
+ { name: 'content-usage', value: 'train-ai=y' },
+ { name: 'content-usage', value: 'train-ai=n' },
+ ],
+ false
+ )
+ ).toBe('content_usage')
+ })
+
+ it('lets the response TDM value override the tdmrep file', () => {
+ expect(responseOptOutReason([{ name: 'tdm-reservation', value: '0' }], true)).toBeUndefined()
+ expect(responseOptOutReason([{ name: 'tdm-reservation', value: '1' }], false)).toBe('tdm_reservation')
+ })
+
+ it.each([
+ ['a scope for this bot', 'PostHogImageFetcherBot: noindex, noimageai', 'x_robots_tag'],
+ ['a scope for another bot', 'OtherBot: noai', undefined],
+ [
+ 'no scope beside a colon-bearing directive',
+ 'noai, unavailable_after: 25 Jun 2026 15:00:00 PST',
+ 'x_robots_tag',
+ ],
+ [
+ 'a scope for another bot beside a colon-bearing directive',
+ 'OtherBot: noai, unavailable_after: 25 Jun 2026 15:00:00 PST',
+ undefined,
+ ],
+ [
+ 'no scope after a leading valued directive',
+ 'unavailable_after: 25 Jun 2026 15:00:00 PST, noai',
+ 'x_robots_tag',
+ ],
+ ['no scope after a leading max-image-preview', 'max-image-preview: large, noai', 'x_robots_tag'],
+ [
+ 'a scope for another bot before a valued directive',
+ 'OtherBot: unavailable_after: 25 Jun 2026 15:00:00 PST, noai',
+ undefined,
+ ],
+ ])('applies an X-Robots-Tag value with %s', (_name, value, expected) => {
+ expect(responseOptOutReason([{ name: 'x-robots-tag', value }], false)).toBe(expected)
+ })
+})
+
+describe('ConfigurationPolicyService', () => {
+ beforeEach(() => register.resetMetrics())
+ it('checks both files, one request at a time, before it allows an image request', async () => {
+ const { policy, fetch } = service()
+ let active = 0
+ let mostActive = 0
+ fetch.mockImplementation(async () => {
+ mostActive = Math.max(mostActive, ++active)
+ await new Promise((resolve) => setImmediate(resolve))
+ active -= 1
+ return { outcome: 'absent' }
+ })
+
+ await expect(policy.check(`${ORIGIN}/image.png`, new Map(), NOW_MS)).resolves.toMatchObject({ allowed: true })
+ expect(fetch.mock.calls).toEqual([
+ [ORIGIN, 'robots'],
+ [ORIGIN, 'tdmrep'],
+ ])
+ // The v4 lane allows one connection per registrable domain by default, so a second request would be refused.
+ expect(mostActive).toBe(1)
+ })
+
+ it('uses a cached configuration without a new request', async () => {
+ const { policy, fetch } = service()
+ const cache = new Map([
+ [configurationCacheKey(ORIGIN, 'robots'), cached('robots', 'absent')],
+ [configurationCacheKey(ORIGIN, 'tdmrep'), cached('tdmrep', 'absent')],
+ ])
+
+ await expect(policy.check(`${ORIGIN}/image.png`, cache, NOW_MS)).resolves.toMatchObject({ allowed: true })
+ expect(fetch).not.toHaveBeenCalled()
+ const lookups = await register.getSingleMetric('ai_research_v4_image_fetch_configuration_lookups_total')!.get()
+ expect(lookups.values).toEqual(
+ expect.arrayContaining([
+ expect.objectContaining({ labels: { file: 'robots', source: 'cache', outcome: 'absent' }, value: 1 }),
+ expect.objectContaining({ labels: { file: 'tdmrep', source: 'cache', outcome: 'absent' }, value: 1 }),
+ ])
+ )
+ })
+
+ it('counts shared configuration requests separately from new requests', async () => {
+ let resolve!: (result: ConfigurationFetchResult) => void
+ const pending = new Promise<ConfigurationFetchResult>((done) => {
+ resolve = done
+ })
+ const fetch = jest.fn(() => pending)
+ const policy = new ConfigurationPolicyService({ fetch } as unknown as HttpConfigurationFetcher)
+ const first = policy.check(`${ORIGIN}/first.png`, new Map(), NOW_MS)
+ const second = policy.check(`${ORIGIN}/second.png`, new Map(), NOW_MS)
+ expect(fetch).toHaveBeenCalledTimes(1)
+ resolve({ outcome: 'unreachable' })
+ await Promise.all([first, second])
+ expect(fetch).toHaveBeenCalledTimes(2)
+ const lookups = await register.getSingleMetric('ai_research_v4_image_fetch_configuration_lookups_total')!.get()
+ for (const file of ['robots', 'tdmrep']) {
+ expect(lookups.values).toEqual(
+ expect.arrayContaining([
+ expect.objectContaining({ labels: { file, source: 'network', outcome: 'unreachable' }, value: 1 }),
+ expect.objectContaining({ labels: { file, source: 'shared', outcome: 'unreachable' }, value: 1 }),
+ ])
+ )
+ }
+ })
+
+ it('parses each cached policy revision once in a fetch pass', async () => {
+ const { ParsedRobots } = await import('@trybyte/robotstxt-parser')
+ const parseRobots = jest.spyOn(ParsedRobots, 'parse')
+ const parseJson = jest.spyOn(JSON, 'parse')
+ const { policy } = service()
+ const robotsBody = 'User-agent: *\nAllow: /'
+ const tdmrepBody = JSON.stringify([{ location: '/', 'tdm-reservation': 0 }])
+ const cache = new Map([
+ [configurationCacheKey(ORIGIN, 'robots'), cached('robots', 'available', robotsBody)],
+ [configurationCacheKey(ORIGIN, 'tdmrep'), cached('tdmrep', 'available', tdmrepBody)],
+ ])
+
+ try {
+ const pass = policy.createPass()
+ await pass.check(`${ORIGIN}/first.png`, cache, NOW_MS)
+ await pass.check(`${ORIGIN}/second.png`, cache, NOW_MS)
+
+ expect(parseRobots.mock.calls.filter(([body]) => body === robotsBody)).toHaveLength(1)
+ expect(parseJson.mock.calls.filter(([body]) => body === tdmrepBody)).toHaveLength(1)
+
+ const revisedCache = new Map(
+ [...cache].map(([key, item]) => [key, { ...item, fetchedAtMs: item.fetchedAtMs + 1 }] as const)
+ )
+ await pass.check(`${ORIGIN}/third.png`, revisedCache, NOW_MS)
+
+ expect(parseRobots.mock.calls.filter(([body]) => body === robotsBody)).toHaveLength(2)
+ expect(parseJson.mock.calls.filter(([body]) => body === tdmrepBody)).toHaveLength(2)
+
+ await policy.createPass().check(`${ORIGIN}/fourth.png`, cache, NOW_MS)
+
+ expect(parseRobots.mock.calls.filter(([body]) => body === robotsBody)).toHaveLength(3)
+ expect(parseJson.mock.calls.filter(([body]) => body === tdmrepBody)).toHaveLength(3)
+ } finally {
+ parseRobots.mockRestore()
+ parseJson.mockRestore()
+ }
+ })
+
+ it('uses the first matching TDMRep rule and honors the end anchor', async () => {
+ const { policy } = service()
+ const cache = new Map([
+ [configurationCacheKey(ORIGIN, 'robots'), cached('robots', 'absent')],
+ [
+ configurationCacheKey(ORIGIN, 'tdmrep'),
+ cached(
+ 'tdmrep',
+ 'available',
+ JSON.stringify([
+ { location: '/private/exact$', 'tdm-reservation': 0 },
+ { location: '/private/*', 'tdm-reservation': 1 },
+ ])
+ ),
+ ],
+ ])
+
+ await expect(policy.check(`${ORIGIN}/private/exact`, cache, NOW_MS)).resolves.toMatchObject({ allowed: true })
+ await expect(policy.check(`${ORIGIN}/private/exact-more`, cache, NOW_MS)).resolves.toMatchObject({
+ allowed: true,
+ tdmrepReservation: true,
+ })
+ })
+
+ it.each([
+ ['1', true],
+ [true, true],
+ [0, false],
+ ])('reads a TDMRep reservation of %p as %p', async (reservation, tdmrepReservation) => {
+ const { policy } = service()
+ const cache = new Map([
+ [configurationCacheKey(ORIGIN, 'robots'), cached('robots', 'absent')],
+ [
+ configurationCacheKey(ORIGIN, 'tdmrep'),
+ cached('tdmrep', 'available', JSON.stringify([{ location: '/', 'tdm-reservation': reservation }])),
+ ],
+ ])
+
+ await expect(policy.check(`${ORIGIN}/image.png`, cache, NOW_MS)).resolves.toMatchObject({
+ allowed: true,
+ tdmrepReservation,
+ })
+ })
+
+ it.each([
+ {
+ description: 'keeps a cached TDMRep file',
+ cachedTdmrep: [
+ [
+ configurationCacheKey(ORIGIN, 'tdmrep'),
+ {
+ ...cached('tdmrep', 'available', JSON.stringify([{ location: '/', 'tdm-reservation': 1 }])),
+ refreshAtMs: NOW_MS - 1,
+ },
+ ] as const,
+ ],
+ status: 'available',
+ tdmrepReservation: true,
+ },
+ { description: 'records an absent TDMRep file', cachedTdmrep: [], status: 'absent', tdmrepReservation: false },
+ ])('$description when a refresh returns a non-array body', async ({ cachedTdmrep, status, tdmrepReservation }) => {
+ const { policy } = service({ tdmrep: { outcome: 'absent', nonArrayBody: true } })
+ const cache = new Map([[configurationCacheKey(ORIGIN, 'robots'), cached('robots', 'absent')], ...cachedTdmrep])
+
+ await expect(policy.check(`${ORIGIN}/image.png`, cache, NOW_MS)).resolves.toMatchObject({
+ allowed: true,
+ tdmrepReservation,
+ updates: [expect.objectContaining({ kind: 'tdmrep', status })],
+ })
+ })
+
+ it('keeps a previous usable file when its refresh is unreachable', async () => {
+ const { policy } = service({ robots: { outcome: 'unreachable' } })
+ const previous = { ...cached('robots', 'absent'), refreshAtMs: NOW_MS - 1 }
+ const cache = new Map([
+ [configurationCacheKey(ORIGIN, 'robots'), previous],
+ [configurationCacheKey(ORIGIN, 'tdmrep'), cached('tdmrep', 'absent')],
+ ])
+
+ await expect(policy.check(`${ORIGIN}/image.png`, cache, NOW_MS)).resolves.toMatchObject({
+ allowed: true,
+ updates: [expect.objectContaining({ kind: 'robots', status: 'absent', retryAtMs: NOW_MS + 3_600_000 })],
+ })
+ })
+
+ it('returns a transient refusal when no configuration file is reachable', async () => {
+ const { policy } = service({ robots: { outcome: 'unreachable' } })
+
+ await expect(policy.check(`${ORIGIN}/image.png`, new Map(), NOW_MS)).resolves.toMatchObject({
+ allowed: false,
+ transient: true,
+ reason: 'configuration_unreachable',
+ })
+ })
+
+ it('preserves a registrable-domain state deferral', async () => {
+ const { policy } = service({
+ robots: { outcome: 'deferred', reason: 'registrable_domain_map_full' },
+ })
+
+ await expect(policy.check(`${ORIGIN}/image.png`, new Map(), NOW_MS)).resolves.toMatchObject({
+ allowed: false,
+ transient: true,
+ reason: 'registrable_domain_map_full',
+ })
+ })
+
+ it('keeps a terminal refusal when the other configuration file is unreachable', async () => {
+ const { policy } = service({
+ robots: {
+ outcome: 'available',
+ body: 'User-agent: *\nDisallow: /private',
+ cache: { requestTimeMs: NOW_MS, responseTimeMs: NOW_MS },
+ },
+ tdmrep: { outcome: 'unreachable' },
+ })
+
+ await expect(policy.check(`${ORIGIN}/private/image.png`, new Map(), NOW_MS)).resolves.toMatchObject({
+ allowed: false,
+ transient: false,
+ reason: 'robots_disallow',
+ })
+ })
+})
+
+describe('HttpConfigurationFetcher', () => {
+ beforeEach(() => {
+ fetchStreamedMock.mockReset()
+ register.resetMetrics()
+ })
+
+ it.each([
+ [404, 'absent', 'absent'],
+ [410, 'absent', 'absent'],
+ [401, 'refused', 'http_401'],
+ [403, 'refused', 'http_403'],
+ [429, 'unreachable', 'http_429'],
+ [500, 'unreachable', 'http_5xx'],
+ [204, 'unreachable', 'unexpected_status'],
+ ])('maps HTTP %s to %s', async (status, outcome, reason) => {
+ fetchStreamedMock.mockResolvedValue(response(status))
+
+ await expect(httpFetcher().fetch(ORIGIN, 'robots')).resolves.toMatchObject({ outcome })
+ const fetches = await register.getSingleMetric('ai_research_v4_image_fetch_configuration_fetches_total')!.get()
+ expect(fetches.values).toEqual([
+ expect.objectContaining({ labels: { file: 'robots', outcome, reason }, value: 1 }),
+ ])
+ })
+
+ it('uses the retained prefix when robots.txt exceeds its byte limit', async () => {
+ fetchStreamedMock.mockResolvedValue(
+ response(200, [], {
+ bytes: Buffer.concat([
+ Buffer.from('User-agent: *\nDisallow: /private'),
+ Buffer.from([0xf0, 0x9f, 0x98]),
+ ]),
+ overLimit: true,
+ })
+ )
+
+ await expect(httpFetcher().fetch(ORIGIN, 'robots')).resolves.toMatchObject({
+ outcome: 'available',
+ body: 'User-agent: *\nDisallow: /private',
+ })
+ })
+
+ it('treats invalid UTF-8 configuration text as unreachable', async () => {
+ fetchStreamedMock.mockResolvedValue(
+ response(200, [], { bytes: Buffer.from([0x75, 0x73, 0x65, 0x72, 0xff]), overLimit: false })
+ )
+
+ await expect(httpFetcher().fetch(ORIGIN, 'robots')).resolves.toMatchObject({ outcome: 'unreachable' })
+ })
+
+ it.each([
+ {
+ description: 'a string reservation',
+ bytes: Buffer.from('[{"location": "/", "tdm-reservation": "1"}]'),
+ overLimit: false,
+ result: { outcome: 'available' },
+ reason: 'available',
+ },
+ {
+ description: 'a byte order mark and leading whitespace',
+ bytes: Buffer.from('\uFEFF\n []'),
+ overLimit: false,
+ result: { outcome: 'available' },
+ reason: 'available',
+ },
+ {
+ description: 'a damaged array',
+ bytes: Buffer.from('[{"location": "/",'),
+ overLimit: false,
+ result: { outcome: 'unreachable' },
+ reason: 'invalid_document',
+ },
+ {
+ description: 'an oversized array',
+ bytes: Buffer.from('[]'),
+ overLimit: true,
+ result: { outcome: 'unreachable' },
+ reason: 'body_limit',
+ },
+ {
+ description: 'an oversized whitespace prefix',
+ bytes: Buffer.from('\uFEFF \n\t '),
+ overLimit: true,
+ result: { outcome: 'unreachable' },
+ reason: 'body_limit',
+ },
+ {
+ description: 'an empty body',
+ bytes: Buffer.alloc(0),
+ overLimit: false,
+ result: { outcome: 'absent', nonArrayBody: true },
+ reason: 'not_json_array',
+ },
+ {
+ description: 'an HTML page',
+ bytes: Buffer.from('<!doctype html><title>Home</title>'),
+ overLimit: false,
+ result: { outcome: 'absent', nonArrayBody: true },
+ reason: 'not_json_array',
+ },
+ {
+ description: 'a JSON object',
+ bytes: Buffer.from('{"error": "not found"}'),
+ overLimit: false,
+ result: { outcome: 'absent', nonArrayBody: true },
+ reason: 'not_json_array',
+ },
+ {
+ description: 'an oversized HTML page',
+ bytes: Buffer.from('<!doctype html>'),
+ overLimit: true,
+ result: { outcome: 'absent', nonArrayBody: true },
+ reason: 'not_json_array',
+ },
+ {
+ description: 'a Latin-1 HTML page',
+ bytes: Buffer.from('<p>caf\u00e9</p>', 'latin1'),
+ overLimit: false,
+ result: { outcome: 'absent', nonArrayBody: true },
+ reason: 'not_json_array',
+ },
+ ])('maps a TDMRep body with $description to $result.outcome', async ({ bytes, overLimit, result, reason }) => {
+ fetchStreamedMock.mockResolvedValue(response(200, [], { bytes, overLimit }))
+
+ await expect(httpFetcher().fetch(ORIGIN, 'tdmrep')).resolves.toEqual(expect.objectContaining(result))
+ const fetches = await register.getSingleMetric('ai_research_v4_image_fetch_configuration_fetches_total')!.get()
+ expect(fetches.values).toEqual([
+ expect.objectContaining({ labels: { file: 'tdmrep', outcome: result.outcome, reason }, value: 1 }),
+ ])
+ })
+
+ it('treats repeated Location field lines as unreachable', async () => {
+ fetchStreamedMock.mockResolvedValue(
+ response(302, [
+ { name: 'location', value: '/first' },
+ { name: 'location', value: '/second' },
+ ])
+ )
+
+ await expect(httpFetcher().fetch(ORIGIN, 'robots')).resolves.toMatchObject({ outcome: 'unreachable' })
+ expect(fetchStreamedMock).toHaveBeenCalledTimes(1)
+ })
+
+ it('follows, signs and identifies each configuration redirect target', async () => {
+ fetchStreamedMock
+ .mockResolvedValueOnce(response(302, [{ name: 'location', value: 'https://cdn.example.com/robots' }]))
+ .mockResolvedValueOnce(
+ response(200, [], { bytes: Buffer.from('User-agent: *\nAllow: /'), overLimit: false })
+ )
+ const headersForGet = jest
+ .fn()
+ .mockReturnValueOnce({ signature: 'first' })
+ .mockReturnValueOnce({ signature: 'second' })
+
+ await expect(httpFetcher(headersForGet).fetch(ORIGIN, 'robots')).resolves.toMatchObject({
+ outcome: 'available',
+ })
+ expect(headersForGet.mock.calls).toEqual([
+ ['https://example.com/robots.txt'],
+ ['https://cdn.example.com/robots'],
+ ])
+ expect(fetchStreamedMock.mock.calls.map(([, options]) => options.allowH2)).toEqual([true, true])
+ expect(fetchStreamedMock.mock.calls.map(([, options]) => options.headers)).toEqual([
+ expect.objectContaining({ referer: 'https://us.posthog.com/' }),
+ expect.objectContaining({ referer: 'https://us.posthog.com/' }),
+ ])
+ })
+
+ it('does not follow a configuration redirect to another registrable domain', async () => {
+ fetchStreamedMock.mockResolvedValueOnce(
+ response(302, [{ name: 'location', value: 'https://cdn.example.net/robots' }])
+ )
+ const headersForGet = jest.fn().mockReturnValue({ signature: 'value' })
+
+ await expect(httpFetcher(headersForGet).fetch(ORIGIN, 'robots')).resolves.toMatchObject({
+ outcome: 'unreachable',
+ })
+ expect(fetchStreamedMock).toHaveBeenCalledTimes(1)
+ expect(headersForGet).toHaveBeenCalledTimes(1)
+ })
+})
+
+describe('explicitFreshnessLifetimeMs', () => {
+ it('uses s-maxage and subtracts the corrected current age', () => {
+ expect(
+ explicitFreshnessLifetimeMs(
+ {
+ requestTimeMs: NOW_MS - 2_000,
+ responseTimeMs: NOW_MS,
+ date: new Date(NOW_MS - 1_000).toUTCString(),
+ age: '10',
+ cacheControl: 'max-age=100, s-maxage=200',
+ },
+ NOW_MS
+ )
+ ).toBe(188_000)
+ })
+
+ it.each(['no-cache', 'no-store', 'private', 'must-revalidate'])('does not extend freshness for %s', (directive) => {
+ expect(
+ explicitFreshnessLifetimeMs(
+ {
+ requestTimeMs: NOW_MS,
+ responseTimeMs: NOW_MS,
+ cacheControl: `${directive}, max-age=999999`,
+ },
+ NOW_MS
+ )
+ ).toBe(0)
+ })
+
+ it.each([
+ ['duplicate max-age', 'max-age=60, max-age=120'],
+ ['decimal max-age', 'max-age=1.5'],
+ ['exponent max-age', 'max-age=1e3'],
+ ['unsafe max-age', 'max-age=9007199254740992'],
+ ['invalid s-maxage before a valid max-age', 's-maxage=invalid, max-age=3600'],
+ ])('does not extend freshness for %s', (_name, cacheControl) => {
+ expect(
+ explicitFreshnessLifetimeMs(
+ {
+ requestTimeMs: NOW_MS,
+ responseTimeMs: NOW_MS,
+ cacheControl,
+ },
+ NOW_MS
+ )
+ ).toBe(0)
+ })
+
+ it('uses every combined Cache-Control field line', () => {
+ expect(
+ explicitFreshnessLifetimeMs(
+ {
+ requestTimeMs: NOW_MS,
+ responseTimeMs: NOW_MS,
+ cacheControl: 'public, max-age=86400, no-store',
+ },
+ NOW_MS
+ )
+ ).toBe(0)
+ })
+
+ it('ignores a decimal Age value', () => {
+ expect(
+ explicitFreshnessLifetimeMs(
+ {
+ requestTimeMs: NOW_MS,
+ responseTimeMs: NOW_MS,
+ age: '1.5',
+ cacheControl: 'max-age=10',
+ },
+ NOW_MS
+ )
+ ).toBe(10_000)
+ })
+
+ it('subtracts time spent resident after the response arrived', () => {
+ expect(
+ explicitFreshnessLifetimeMs(
+ {
+ requestTimeMs: NOW_MS,
+ responseTimeMs: NOW_MS,
+ cacheControl: 'max-age=10',
+ },
+ NOW_MS + 4_000
+ )
+ ).toBe(6_000)
+ })
+})
diff --git a/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/configuration-policy.ts b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/configuration-policy.ts
new file mode 100644
index 000000000000..d3f4ba19b2b9
--- /dev/null
+++ b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/configuration-policy.ts
@@ -0,0 +1,806 @@
+import type { ParsedRobots, RobotsParseHandler } from '@trybyte/robotstxt-parser'
+import { Token, parseDictionary } from 'structured-headers'
+
+import { parseJSON } from '~/common/utils/json-parse'
+import { fetchStreamed } from '~/common/utils/request'
+
+import { ConfigurationCacheItem, ConfigurationFile, HttpCacheMetadata, configurationCacheKey } from './crawl-history'
+import { ImageFetchRequestMetrics } from './metrics'
+import { canonicalizeUrl, politenessKey } from './politeness-key'
+import { ConfigurationFetchReason, ImageFetchProcessingMetrics } from './processing-metrics'
+import { BOT_NAME, REQUEST_IDENTITY_HEADERS } from './request-identity'
+import { WebBotAuthRequestSigner } from './web-bot-auth'
+import { wildcardPatternMatchesPathname } from './wildcard-pattern'
+
+const CONFIG_BODY_LIMIT = 500 * 1024
+const CONFIG_REDIRECT_LIMIT = 5
+const CONFIG_FRESH_MS = 24 * 60 * 60 * 1000
+const CONFIG_REFRESH_MS = 23 * 60 * 60 * 1000
+const CONFIG_RETRY_MS = 60 * 60 * 1000
+const CONFIG_STORAGE_MS = 30 * 24 * 60 * 60 * 1000
+const REDIRECT_STATUSES = new Set([301, 302, 303, 307, 308])
+const UTF8_BYTE_ORDER_MARK = [0xef, 0xbb, 0xbf]
+const JSON_WHITESPACE_BYTES = new Set([' ', '\t', '\n', '\r'].map((character) => character.charCodeAt(0)))
+const JSON_ARRAY_START_BYTE = '['.charCodeAt(0)
+
+export type ConfigurationFetchResult =
+ | { outcome: 'available'; body: string; cache: HttpCacheMetadata }
+ | { outcome: 'absent'; cache?: HttpCacheMetadata; nonArrayBody?: boolean }
+ | { outcome: 'refused' | 'unreachable'; cache?: HttpCacheMetadata }
+ | { outcome: 'deferred'; reason: ConfigurationRequestBlockReason }
+
+export type ConfigurationRequestBlockReason =
+ | 'breaker_open'
+ | 'backoff'
+ | 'deadline'
+ | 'origin_map_full'
+ | 'registrable_domain_map_full'
+ | 'connection_limit'
+
+export type RobotsPolicyRefusalReason = 'robots_disallow' | 'content_usage' | 'content_signal'
+export type OriginPolicyReason =
+ | RobotsPolicyRefusalReason
+ | 'robots_refused'
+ | 'tdmrep_refused'
+ | 'origin_map_full'
+ | 'registrable_domain_map_full'
+ | 'configuration_deferred'
+ | 'configuration_unreachable'
+export type ResponseOptOutReason = 'x_robots_tag' | 'content_usage' | 'tdm_reservation'
+
+export interface ConfigurationRequestScheduler {
+ run<T>(
+ url: URL,
+ deadlineMs: number,
+ request: () => Promise<T>
+ ): Promise<{ ran: true; value: T } | { ran: false; reason: ConfigurationRequestBlockReason }>
+}
+
+type ConfigurationHop =
+ | { kind: 'redirect'; location: string; cache: HttpCacheMetadata }
+ | { kind: 'done'; result: ConfigurationFetchResult; reason?: ConfigurationFetchReason }
+
+export class HttpConfigurationFetcher {
+ constructor(
+ private readonly signer: WebBotAuthRequestSigner,
+ private readonly scheduler: ConfigurationRequestScheduler,
+ private readonly timeoutMs: number
+ ) {}
+
+ public async fetch(origin: string, file: ConfigurationFile): Promise<ConfigurationFetchResult> {
+ return await ImageFetchProcessingMetrics.measure('configuration_fetch', () =>
+ this.fetchConfiguration(origin, file)
+ )
+ }
+
+ private async fetchConfiguration(origin: string, file: ConfigurationFile): Promise<ConfigurationFetchResult> {
+ const complete = (
+ result: ConfigurationFetchResult,
+ reason: ConfigurationFetchReason = result.outcome
+ ): ConfigurationFetchResult => ImageFetchProcessingMetrics.observeConfigurationFetch(file, result, reason)
+ const deadlineMs = Date.now() + this.timeoutMs
+ let target = new URL(file === 'robots' ? '/robots.txt' : '/.well-known/tdmrep.json', origin)
+ const registrableDomain = politenessKey(target.hostname)
+ for (let redirects = 0; ; redirects++) {
+ const canonical = canonicalizeUrl(target.toString())
+ if (!canonical) {
+ return complete({ outcome: 'refused' }, 'invalid_url')
+ }
+ target = new URL(canonical.fetch)
+ let scheduled:
+ | { ran: true; value: ConfigurationHop }
+ | { ran: false; reason: ConfigurationRequestBlockReason }
+ try {
+ scheduled = await this.scheduler.run(target, deadlineMs, () => this.hop(target, file, deadlineMs))
+ } catch {
+ return complete({ outcome: 'unreachable' }, Date.now() >= deadlineMs ? 'timeout' : 'request_error')
+ }
+ if (!scheduled.ran) {
+ return complete({ outcome: 'deferred', reason: scheduled.reason })
+ }
+ const hop = scheduled.value
+ if (hop.kind === 'done') {
+ return complete(hop.result, hop.reason)
+ }
+ if (redirects >= CONFIG_REDIRECT_LIMIT) {
+ return complete({ outcome: 'unreachable', cache: hop.cache }, 'redirect_limit')
+ }
+ try {
+ const redirectTarget = new URL(hop.location, target)
+ if (politenessKey(redirectTarget.hostname) !== registrableDomain) {
+ return complete({ outcome: 'unreachable', cache: hop.cache }, 'cross_domain_redirect')
+ }
+ target = redirectTarget
+ } catch {
+ return complete({ outcome: 'unreachable', cache: hop.cache }, 'invalid_redirect')
+ }
+ }
+ }
+
+ private async hop(target: URL, file: ConfigurationFile, deadlineMs: number): Promise<ConfigurationHop> {
+ const requestTimeMs = Date.now()
+ const canonical = canonicalizeUrl(target.toString())
+ let response: Awaited<ReturnType<typeof fetchStreamed>>
+ try {
+ response = await fetchStreamed(target.toString(), {
+ timeoutMs: Math.max(1, deadlineMs - Date.now()),
+ allowH2: true,
+ headers: {
+ ...REQUEST_IDENTITY_HEADERS,
+ accept: file === 'robots' ? 'text/plain,*/*;q=0.1' : 'application/json,*/*;q=0.1',
+ 'accept-encoding': 'identity',
+ ...this.signer.headersForGet(target.toString()),
+ },
+ })
+ } catch (error) {
+ if (canonical) {
+ ImageFetchRequestMetrics.observeRequest('network_error', (Date.now() - requestTimeMs) / 1000)
+ }
+ throw error
+ }
+ const complete = (result: ConfigurationHop): ConfigurationHop => {
+ if (canonical) {
+ ImageFetchRequestMetrics.observeRequest(
+ ImageFetchRequestMetrics.outcomeForHttpStatus(response.status),
+ (Date.now() - requestTimeMs) / 1000
+ )
+ }
+ return result
+ }
+ const cache = cacheMetadata(requestTimeMs, Date.now(), response.headerLines)
+ if (REDIRECT_STATUSES.has(response.status)) {
+ response.discard()
+ const locations = headerValues(response.headerLines, 'location')
+ const location = locations.length === 1 ? locations[0] : undefined
+ return complete(
+ location
+ ? { kind: 'redirect', location, cache }
+ : { kind: 'done', result: { outcome: 'unreachable', cache }, reason: 'missing_location' }
+ )
+ }
+ if (response.status === 404 || response.status === 410) {
+ response.discard()
+ return complete({ kind: 'done', result: { outcome: 'absent', cache } })
+ }
+ if (response.status === 429 || response.status >= 500) {
+ response.discard()
+ return complete({
+ kind: 'done',
+ result: { outcome: 'unreachable', cache },
+ reason: response.status === 429 ? 'http_429' : 'http_5xx',
+ })
+ }
+ if (response.status >= 400 && response.status < 500) {
+ response.discard()
+ return complete({ kind: 'done', result: { outcome: 'refused', cache }, reason: `http_${response.status}` })
+ }
+ if (response.status !== 200) {
+ response.discard()
+ return complete({ kind: 'done', result: { outcome: 'unreachable', cache }, reason: 'unexpected_status' })
+ }
+ const body = await response.read(CONFIG_BODY_LIMIT)
+ // This check comes before the size and UTF-8 checks, so that an oversized or non-UTF-8 HTML page also means that the origin has no tdmrep.json (README 3.14).
+ if (file === 'tdmrep' && cannotBeJsonArray(body)) {
+ return complete({
+ kind: 'done',
+ result: { outcome: 'absent', cache, nonArrayBody: true },
+ reason: 'not_json_array',
+ })
+ }
+ if (body.overLimit && file === 'tdmrep') {
+ return complete({ kind: 'done', result: { outcome: 'unreachable', cache }, reason: 'body_limit' })
+ }
+ let text: string
+ try {
+ text = new TextDecoder('utf-8', { fatal: true }).decode(body.bytes, { stream: body.overLimit })
+ } catch {
+ return complete({ kind: 'done', result: { outcome: 'unreachable', cache }, reason: 'invalid_utf8' })
+ }
+ if (file === 'tdmrep' && !Array.isArray(parseTdmrepDocument(text))) {
+ return complete({ kind: 'done', result: { outcome: 'unreachable', cache }, reason: 'invalid_document' })
+ }
+ return complete({ kind: 'done', result: { outcome: 'available', body: text, cache } })
+ }
+}
+
+export interface OriginPolicyDecision {
+ allowed: boolean
+ transient: boolean
+ reason?: OriginPolicyReason
+ crawlDelayMs: number
+ tdmrepReservation: boolean
+ updates: ConfigurationCacheItem[]
+}
+
+export interface ConfigurationPolicyPass {
+ check(url: string, cached: Map<string, ConfigurationCacheItem>, nowMs: number): Promise<OriginPolicyDecision>
+}
+
+interface ParsedRobotsConfiguration {
+ matcher: ParsedRobots
+ fields: Array<{ name: string; value: string }>
+}
+
+interface ParsedConfigurationCache {
+ robots: Map<string, Promise<ParsedRobotsConfiguration>>
+ tdmrep: Map<string, unknown>
+}
+
+export class ConfigurationPolicyService {
+ private readonly inFlight = new Map<string, Promise<ConfigurationFetchResult>>()
+
+ constructor(private readonly fetcher: HttpConfigurationFetcher) {}
+
+ public createPass(): ConfigurationPolicyPass {
+ const parsed: ParsedConfigurationCache = {
+ robots: new Map(),
+ tdmrep: new Map(),
+ }
+ return {
+ check: (url, cached, nowMs) => this.checkWithParsedCache(url, cached, nowMs, parsed),
+ }
+ }
+
+ public async check(
+ url: string,
+ cached: Map<string, ConfigurationCacheItem>,
+ nowMs: number
+ ): Promise<OriginPolicyDecision> {
+ return await this.createPass().check(url, cached, nowMs)
+ }
+
+ private async checkWithParsedCache(
+ url: string,
+ cached: Map<string, ConfigurationCacheItem>,
+ nowMs: number,
+ parsed: ParsedConfigurationCache
+ ): Promise<OriginPolicyDecision> {
+ const origin = new URL(url).origin
+ // One after the other: a host limit of one connection would refuse the second of two concurrent requests.
+ const robots = await this.load(origin, 'robots', cached.get(configurationCacheKey(origin, 'robots')), nowMs)
+ const tdmrep = await this.load(origin, 'tdmrep', cached.get(configurationCacheKey(origin, 'tdmrep')), nowMs)
+ const updates = [...robots.updates, ...tdmrep.updates]
+ if (robots.item.status === 'refused') {
+ return {
+ allowed: false,
+ transient: false,
+ reason: 'robots_refused',
+ crawlDelayMs: 0,
+ tdmrepReservation: false,
+ updates,
+ }
+ }
+ if (tdmrep.item.status === 'refused') {
+ return {
+ allowed: false,
+ transient: false,
+ reason: 'tdmrep_refused',
+ crawlDelayMs: 0,
+ tdmrepReservation: false,
+ updates,
+ }
+ }
+
+ const robotsPolicy = robots.item.body
+ ? evaluateRobotsPolicy(await parsedRobotsConfiguration(robots.item, parsed), url)
+ : defaultRobotsPolicy()
+ if (!robotsPolicy.allowed) {
+ return {
+ allowed: false,
+ transient: false,
+ reason: robotsPolicy.reason,
+ crawlDelayMs: robotsPolicy.crawlDelayMs,
+ tdmrepReservation: false,
+ updates,
+ }
+ }
+ const tdmrepReservation = Boolean(
+ tdmrep.item.body && tdmrepRefuses(parsedTdmrepDocument(tdmrep.item, parsed), new URL(url))
+ )
+ const deferredReason = requestControlDeferralReason(robots.deferredReason, tdmrep.deferredReason)
+ if (deferredReason) {
+ return {
+ allowed: false,
+ transient: true,
+ reason: deferredReason,
+ crawlDelayMs: 0,
+ tdmrepReservation: false,
+ updates,
+ }
+ }
+ if (robots.item.status === 'unreachable' || tdmrep.item.status === 'unreachable') {
+ return {
+ allowed: false,
+ transient: true,
+ reason: 'configuration_unreachable',
+ crawlDelayMs: 0,
+ tdmrepReservation: false,
+ updates,
+ }
+ }
+ return {
+ allowed: true,
+ transient: false,
+ crawlDelayMs: robotsPolicy.crawlDelayMs,
+ tdmrepReservation,
+ updates,
+ }
+ }
+
+ private async load(
+ origin: string,
+ file: ConfigurationFile,
+ previous: ConfigurationCacheItem | undefined,
+ nowMs: number
+ ): Promise<{
+ item: ConfigurationCacheItem
+ updates: ConfigurationCacheItem[]
+ deferredReason?: ConfigurationRequestBlockReason
+ }> {
+ if (previous && previous.storageExpiresAtMs <= nowMs) {
+ previous = undefined
+ }
+ if (previous && previous.refreshAtMs > nowMs) {
+ ImageFetchProcessingMetrics.observeConfigurationLookup(file, 'cache', previous.status)
+ return { item: previous, updates: [] }
+ }
+ if (previous?.status === 'unreachable' && previous.retryAtMs > nowMs) {
+ ImageFetchProcessingMetrics.observeConfigurationLookup(file, 'cache', previous.status)
+ return { item: previous, updates: [] }
+ }
+ const key = configurationCacheKey(origin, file)
+ let request = this.inFlight.get(key)
+ const source = request ? 'shared' : 'network'
+ if (!request) {
+ request = this.fetcher.fetch(origin, file).finally(() => this.inFlight.delete(key))
+ this.inFlight.set(key, request)
+ }
+ const fetched = await request.catch((error) => {
+ ImageFetchProcessingMetrics.observeConfigurationLookup(file, source, 'error')
+ throw error
+ })
+ ImageFetchProcessingMetrics.observeConfigurationLookup(file, source, fetched.outcome)
+ if (fetched.outcome === 'deferred') {
+ return {
+ item: previous ?? unreachableItem(origin, file, nowMs),
+ updates: [],
+ deferredReason: fetched.reason,
+ }
+ }
+ const nonArrayBodyReplacesFile =
+ fetched.outcome === 'absent' && fetched.nonArrayBody === true && previous?.status === 'available'
+ if (
+ (fetched.outcome === 'unreachable' || nonArrayBodyReplacesFile) &&
+ previous &&
+ previous.status !== 'unreachable'
+ ) {
+ const retained = {
+ ...previous,
+ refreshAtMs: nowMs + CONFIG_RETRY_MS,
+ retryAtMs: nowMs + CONFIG_RETRY_MS,
+ storageExpiresAtMs: Math.max(previous.storageExpiresAtMs, nowMs + CONFIG_STORAGE_MS),
+ }
+ return { item: retained, updates: [retained] }
+ }
+ const explicitFreshMs = fetched.cache ? explicitFreshnessLifetimeMs(fetched.cache) : 0
+ const freshForMs =
+ fetched.outcome === 'unreachable' ? CONFIG_RETRY_MS : Math.max(CONFIG_FRESH_MS, explicitFreshMs)
+ const item: ConfigurationCacheItem = {
+ kind: file,
+ key,
+ origin,
+ status: fetched.outcome,
+ body: fetched.outcome === 'available' ? fetched.body : undefined,
+ fetchedAtMs: nowMs,
+ refreshAtMs: nowMs + (fetched.outcome === 'unreachable' ? CONFIG_RETRY_MS : CONFIG_REFRESH_MS),
+ freshUntilMs: nowMs + freshForMs,
+ retryAtMs: nowMs + (fetched.outcome === 'unreachable' ? CONFIG_RETRY_MS : 0),
+ storageExpiresAtMs: nowMs + Math.max(CONFIG_STORAGE_MS, freshForMs),
+ }
+ return { item, updates: [item] }
+ }
+}
+
+function requestControlDeferralReason(
+ ...reasons: Array<ConfigurationRequestBlockReason | undefined>
+):
+ | Extract<OriginPolicyReason, 'origin_map_full' | 'registrable_domain_map_full' | 'configuration_deferred'>
+ | undefined {
+ if (reasons.includes('registrable_domain_map_full')) {
+ return 'registrable_domain_map_full'
+ }
+ if (reasons.includes('origin_map_full')) {
+ return 'origin_map_full'
+ }
+ return reasons.some(Boolean) ? 'configuration_deferred' : undefined
+}
+
+function unreachableItem(origin: string, file: ConfigurationFile, nowMs: number): ConfigurationCacheItem {
+ return {
+ kind: file,
+ key: configurationCacheKey(origin, file),
+ origin,
+ status: 'unreachable',
+ fetchedAtMs: nowMs,
+ refreshAtMs: nowMs + CONFIG_RETRY_MS,
+ freshUntilMs: nowMs + CONFIG_RETRY_MS,
+ retryAtMs: nowMs + CONFIG_RETRY_MS,
+ storageExpiresAtMs: nowMs + CONFIG_STORAGE_MS,
+ }
+}
+
+interface ExtensionGroup {
+ userAgents: string[]
+ fields: Array<{ name: string; value: string }>
+}
+
+class ExtensionCollector implements RobotsParseHandler {
+ public readonly groups: ExtensionGroup[] = []
+ private current: ExtensionGroup | undefined
+ private hasRules = false
+
+ handleRobotsStart(): void {}
+ handleRobotsEnd(): void {}
+ reportLineMetadata(): void {}
+
+ handleUserAgent(_lineNum: number, value: string): void {
+ if (!this.current || this.hasRules) {
+ this.current = { userAgents: [], fields: [] }
+ this.groups.push(this.current)
+ this.hasRules = false
+ }
+ this.current.userAgents.push(value.trim().toLowerCase())
+ }
+
+ handleAllow(): void {
+ this.hasRules = true
+ }
+
+ handleDisallow(): void {
+ this.hasRules = true
+ }
+
+ handleSitemap(): void {
+ this.hasRules = true
+ }
+
+ handleUnknownAction(_lineNum: number, action: string, value: string): void {
+ if (!this.current) {
+ return
+ }
+ this.hasRules = true
+ this.current.fields.push({ name: action.trim().toLowerCase(), value: value.trim() })
+ }
+}
+
+function selectedExtensionFields(
+ body: string,
+ parseRobotsTxt: typeof import('@trybyte/robotstxt-parser').parseRobotsTxt
+): Array<{ name: string; value: string }> {
+ const collector = new ExtensionCollector()
+ parseRobotsTxt(body, collector)
+ const bot = BOT_NAME.toLowerCase()
+ const matches = collector.groups.map((group) => ({
+ group,
+ specificity: Math.max(
+ ...group.userAgents.map((agent) => (agent !== '*' && bot.startsWith(agent) ? agent.length : 0)),
+ 0
+ ),
+ wildcard: group.userAgents.includes('*'),
+ }))
+ const specificity = Math.max(...matches.map((match) => match.specificity), 0)
+ const selected =
+ specificity > 0
+ ? matches.filter((match) => match.specificity === specificity)
+ : matches.filter((match) => match.wildcard)
+ return selected.flatMap((match) => match.group.fields)
+}
+
+function defaultRobotsPolicy(): { allowed: true; crawlDelayMs: number } {
+ return { allowed: true, crawlDelayMs: 0 }
+}
+
+export async function parseRobotsPolicy(
+ body: string,
+ url: string
+): Promise<{ allowed: boolean; crawlDelayMs: number; reason?: RobotsPolicyRefusalReason }> {
+ return evaluateRobotsPolicy(await parseRobotsConfiguration(body), url)
+}
+
+async function parseRobotsConfiguration(body: string): Promise<ParsedRobotsConfiguration> {
+ const robotsParser = await import('@trybyte/robotstxt-parser')
+ const { ParsedRobots, parseRobotsTxt } = robotsParser
+ return {
+ matcher: ParsedRobots.parse(body),
+ fields: selectedExtensionFields(body, parseRobotsTxt),
+ }
+}
+
+function evaluateRobotsPolicy(
+ parsed: ParsedRobotsConfiguration,
+ url: string
+): { allowed: boolean; crawlDelayMs: number; reason?: RobotsPolicyRefusalReason } {
+ if (!parsed.matcher.checkUrl(BOT_NAME, url).allowed) {
+ return { allowed: false, crawlDelayMs: 0, reason: 'robots_disallow' }
+ }
+ const crawlDelayMs = Math.max(
+ 0,
+ ...parsed.fields
+ .filter((field) => field.name === 'crawl-delay')
+ .flatMap((field) => {
+ const value = field.value.trim()
+ if (!/^\d+(?:\.\d+)?$/.test(value)) {
+ return []
+ }
+ // A decimal multiplied by 1000 is not always exact. 16.1 * 1000 is 16100.000000000002. The safe-integer guard below rejects that value, but README 7.9 accepts the delay.
+ const milliseconds = Math.round(Number(value) * 1000)
+ return Number.isSafeInteger(milliseconds) ? [milliseconds] : []
+ })
+ )
+ if (
+ contentUsageRefuses(parsed.fields.filter((field) => field.name === 'content-usage').map((field) => field.value))
+ ) {
+ return { allowed: false, crawlDelayMs, reason: 'content_usage' }
+ }
+ if (parsed.fields.some((field) => field.name === 'content-signal' && contentSignalRefuses(field.value, url))) {
+ return { allowed: false, crawlDelayMs, reason: 'content_signal' }
+ }
+ return { allowed: true, crawlDelayMs }
+}
+
+function parseTdmrepDocument(body: string): unknown {
+ try {
+ return parseJSON(body)
+ } catch {
+ return undefined
+ }
+}
+
+function tdmrepRefuses(parsed: unknown, url: URL): boolean {
+ if (!Array.isArray(parsed)) {
+ return false
+ }
+ for (const rule of parsed) {
+ if (!isObject(rule) || typeof rule.location !== 'string') {
+ continue
+ }
+ if (wildcardPatternMatchesPathname(rule.location, url.pathname)) {
+ return isTdmReservation(rule['tdm-reservation'])
+ }
+ }
+ return false
+}
+
+function isTdmReservation(value: unknown): boolean {
+ // TDMRep reads "1" and true as protocol errors, which mean no reservation. README 3.13 reads them as a reservation, because a site that writes them intends to reserve.
+ return value === 1 || value === '1' || value === true
+}
+
+function cannotBeJsonArray(body: { bytes: Uint8Array; overLimit: boolean }): boolean {
+ const firstByte = firstByteAfterJsonWhitespace(body.bytes)
+ if (firstByte === undefined) {
+ return !body.overLimit
+ }
+ return firstByte !== JSON_ARRAY_START_BYTE
+}
+
+function firstByteAfterJsonWhitespace(bytes: Uint8Array): number | undefined {
+ let index = hasUtf8ByteOrderMark(bytes) ? UTF8_BYTE_ORDER_MARK.length : 0
+ while (index < bytes.length && JSON_WHITESPACE_BYTES.has(bytes[index])) {
+ index++
+ }
+ return index < bytes.length ? bytes[index] : undefined
+}
+
+function hasUtf8ByteOrderMark(bytes: Uint8Array): boolean {
+ return UTF8_BYTE_ORDER_MARK.every((byte, index) => bytes[index] === byte)
+}
+
+async function parsedRobotsConfiguration(
+ item: ConfigurationCacheItem,
+ parsed: ParsedConfigurationCache
+): Promise<ParsedRobotsConfiguration> {
+ const revision = configurationRevision(item)
+ let configuration = parsed.robots.get(revision)
+ if (!configuration) {
+ configuration = parseRobotsConfiguration(item.body ?? '')
+ parsed.robots.set(revision, configuration)
+ }
+ return await configuration
+}
+
+function parsedTdmrepDocument(item: ConfigurationCacheItem, parsed: ParsedConfigurationCache): unknown {
+ const revision = configurationRevision(item)
+ if (!parsed.tdmrep.has(revision)) {
+ parsed.tdmrep.set(revision, parseTdmrepDocument(item.body ?? ''))
+ }
+ return parsed.tdmrep.get(revision)
+}
+
+function configurationRevision(item: ConfigurationCacheItem): string {
+ return `${item.key}\0${item.fetchedAtMs}`
+}
+
+function isObject(value: unknown): value is Record<string, unknown> {
+ return typeof value === 'object' && value !== null && !Array.isArray(value)
+}
+
+function contentUsageRefuses(values: string[]): boolean {
+ if (values.length === 0) {
+ return false
+ }
+ try {
+ const trainAi = parseDictionary(values.join(', ')).get('train-ai')
+ return Boolean(
+ trainAi && !Array.isArray(trainAi[0]) && trainAi[0] instanceof Token && trainAi[0].toString() === 'n'
+ )
+ } catch {
+ return false
+ }
+}
+
+function contentSignalRefuses(value: string, url: string): boolean {
+ const trimmed = value.trim()
+ const pathEnd = trimmed.startsWith('/') ? trimmed.search(/\s/) : -1
+ if (pathEnd > 0 && !wildcardPatternMatchesPathname(trimmed.slice(0, pathEnd), new URL(url).pathname)) {
+ return false
+ }
+ const preferences = (pathEnd > 0 ? trimmed.slice(pathEnd) : trimmed)
+ .split(',')
+ .map((part) => part.trim().split('=', 2))
+ const aiTrain = preferences.filter(([name]) => name === 'ai-train').at(-1)
+ return aiTrain?.[1] === 'no'
+}
+
+export function responseOptOutReason(
+ headerLines: Array<{ name: string; value: string }>,
+ tdmrepReservation: boolean
+): ResponseOptOutReason | undefined {
+ if (headerLines.some((line) => line.name === 'x-robots-tag' && xRobotsTagRefuses(line.value))) {
+ return 'x_robots_tag'
+ }
+ if (contentUsageRefuses(headerLines.filter((line) => line.name === 'content-usage').map((line) => line.value))) {
+ return 'content_usage'
+ }
+ const headerReservation = tdmReservationHeader(headerLines)
+ if (headerReservation ?? tdmrepReservation) {
+ return 'tdm_reservation'
+ }
+ return undefined
+}
+
+// X-Robots-Tag directives that carry a value, from https://developers.google.com/search/docs/crawling-indexing/robots-meta-tag. A colon inside one of these does not start a bot scope.
+const VALUED_X_ROBOTS_DIRECTIVES = ['unavailable_after', 'max-snippet', 'max-image-preview', 'max-video-preview']
+
+function xRobotsTagRefuses(value: string): boolean {
+ const lower = value.toLowerCase()
+ const colon = lower.indexOf(':')
+ // A bot scope is one token before the first colon. A valued directive looks the same, so the lane must tell the two apart. Otherwise it reads `unavailable_after: <date>, noai` as another bot's scope and ignores an opt-out that README 2.6 requires.
+ // A prefix that names no listed directive reads as a bot scope. This keeps another bot's scope intact. A new valued directive therefore needs an entry in the list above, because until then the lane skips a `noai` that follows it.
+ const prefix = colon >= 0 ? lower.slice(0, colon).trim() : ''
+ const scoped = colon >= 0 && !prefix.includes(',') && !VALUED_X_ROBOTS_DIRECTIVES.includes(prefix)
+ if (scoped && prefix !== BOT_NAME.toLowerCase()) {
+ return false
+ }
+ const directives = scoped ? lower.slice(colon + 1) : lower
+ return directives.split(',').some((directive) => ['noai', 'noimageai'].includes(directive.trim()))
+}
+
+function tdmReservationHeader(headerLines: Array<{ name: string; value: string }>): boolean | undefined {
+ const values = headerLines
+ .filter((line) => line.name === 'tdm-reservation')
+ .flatMap((line) => line.value.split(','))
+ .map((value) => value.trim())
+ if (values.includes('1')) {
+ return true
+ }
+ if (values.includes('0')) {
+ return false
+ }
+ return undefined
+}
+
+function headerValues(headerLines: Array<{ name: string; value: string }>, name: string): string[] {
+ return headerLines.filter((line) => line.name === name).map((line) => line.value)
+}
+
+function cacheMetadata(
+ requestTimeMs: number,
+ responseTimeMs: number,
+ headerLines: Array<{ name: string; value: string }>
+): HttpCacheMetadata {
+ return {
+ requestTimeMs,
+ responseTimeMs,
+ etag: singletonHeader(headerLines, 'etag'),
+ lastModified: singletonHeader(headerLines, 'last-modified'),
+ date: singletonHeader(headerLines, 'date'),
+ age: singletonHeader(headerLines, 'age'),
+ cacheControl: combinedHeader(headerLines, 'cache-control'),
+ expires: singletonHeader(headerLines, 'expires'),
+ }
+}
+
+function singletonHeader(headerLines: Array<{ name: string; value: string }>, name: string): string | undefined {
+ const values = headerValues(headerLines, name)
+ return values.length === 1 ? values[0] : undefined
+}
+
+function combinedHeader(headerLines: Array<{ name: string; value: string }>, name: string): string | undefined {
+ const values = headerValues(headerLines, name)
+ return values.length > 0 ? values.join(', ') : undefined
+}
+
+export function explicitFreshnessLifetimeMs(cache: HttpCacheMetadata, nowMs = Date.now()): number {
+ const directives = cacheControlDirectives(cache.cacheControl)
+ if (['no-cache', 'no-store', 'private', 'must-revalidate'].some((directive) => directives.has(directive))) {
+ return 0
+ }
+ let lifetimeMs = 0
+ if (directives.has('s-maxage')) {
+ lifetimeMs = singletonDeltaSeconds(directives.get('s-maxage')) * 1000
+ } else if (directives.has('max-age')) {
+ lifetimeMs = singletonDeltaSeconds(directives.get('max-age')) * 1000
+ } else if (cache.expires) {
+ const expiresMs = Date.parse(cache.expires)
+ const dateMs = cache.date ? Date.parse(cache.date) : cache.responseTimeMs
+ if (Number.isFinite(expiresMs) && Number.isFinite(dateMs)) {
+ lifetimeMs = Math.max(0, expiresMs - dateMs)
+ }
+ }
+ const dateMs = cache.date ? Date.parse(cache.date) : cache.responseTimeMs
+ const apparentAgeMs = Number.isFinite(dateMs) ? Math.max(0, cache.responseTimeMs - dateMs) : 0
+ const ageValueMs = nonNegativeInteger(cache.age) * 1000
+ const correctedAgeMs = ageValueMs + Math.max(0, cache.responseTimeMs - cache.requestTimeMs)
+ const currentAgeMs = Math.max(apparentAgeMs, correctedAgeMs) + Math.max(0, nowMs - cache.responseTimeMs)
+ return Math.max(0, lifetimeMs - currentAgeMs)
+}
+
+function cacheControlDirectives(cacheControl: string | undefined): Map<string, Array<string | undefined>> {
+ const directives = new Map<string, Array<string | undefined>>()
+ for (const directive of splitCommaSeparatedValues(cacheControl ?? '')) {
+ const [rawName, ...rawValueParts] = directive.trim().split('=')
+ const name = rawName.toLowerCase()
+ if (!name) {
+ continue
+ }
+ const rawValue = rawValueParts.length > 0 ? rawValueParts.join('=').trim() : undefined
+ const value = rawValue?.startsWith('"') && rawValue.endsWith('"') ? rawValue.slice(1, -1) : rawValue
+ directives.set(name, [...(directives.get(name) ?? []), value])
+ }
+ return directives
+}
+
+function splitCommaSeparatedValues(value: string): string[] {
+ const values: string[] = []
+ let start = 0
+ let quoted = false
+ let escaped = false
+ for (let index = 0; index < value.length; index++) {
+ const character = value[index]
+ if (escaped) {
+ escaped = false
+ } else if (character === '\\' && quoted) {
+ escaped = true
+ } else if (character === '"') {
+ quoted = !quoted
+ } else if (character === ',' && !quoted) {
+ values.push(value.slice(start, index))
+ start = index + 1
+ }
+ }
+ values.push(value.slice(start))
+ return values
+}
+
+function singletonDeltaSeconds(values: Array<string | undefined> | undefined): number {
+ return values?.length === 1 ? nonNegativeInteger(values[0]) : 0
+}
+
+function nonNegativeInteger(value: string | undefined): number {
+ if (!value || !/^\d+$/.test(value.trim())) {
+ return 0
+ }
+ const parsed = Number(value)
+ return Number.isSafeInteger(parsed) ? parsed : 0
+}
diff --git a/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/fetch-candidate-pool.test.ts b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/fetch-candidate-pool.test.ts
new file mode 100644
index 000000000000..39e3ea3a0006
--- /dev/null
+++ b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/fetch-candidate-pool.test.ts
@@ -0,0 +1,249 @@
+import { FetchCandidate, MAX_HOPS } from './collected-urls-record'
+import { FetchCandidatePool, FetchCandidatePoolLease, OriginPacing } from './fetch-candidate-pool'
+
+const NOW_MS = 1_700_000_000_000
+
+function candidate(url: string, registrableDomain: string, firstSeenAtMs = NOW_MS): FetchCandidate {
+ const parsed = new URL(url)
+ return {
+ originalRef: `imageurl:${url}`,
+ currentUrl: url,
+ host: parsed.host,
+ origin: parsed.origin,
+ registrableDomain,
+ remainingHops: MAX_HOPS,
+ notBeforeMs: 0,
+ firstSeenAtMs,
+ fetchCount: 0,
+ republishCount: 0,
+ lastRepublishReason: null,
+ }
+}
+
+function buildPool(
+ crawlDelayMsByOrigin: Record<string, number> = {},
+ limits: { maxConcurrentPerRegistrableDomain?: number; refillRunnableUrls?: number } = {}
+): FetchCandidatePool<string> {
+ const pacing: OriginPacing = {
+ originCrawlDelayMs: (origin) => crawlDelayMsByOrigin[origin] ?? 0,
+ originNextImageStartAtMs: () => 0,
+ }
+ return new FetchCandidatePool<string>(
+ {
+ maxConcurrentPerRegistrableDomain: limits.maxConcurrentPerRegistrableDomain ?? 2,
+ refillRunnableUrls: limits.refillRunnableUrls ?? 3,
+ maxQueuedUrlsPerOwner: 100,
+ },
+ pacing
+ )
+}
+
+async function takeReady(pool: FetchCandidatePool<string>): Promise<FetchCandidatePoolLease<string> | undefined> {
+ let lease: FetchCandidatePoolLease<string> | undefined
+ let settled = false
+ void pool.take().then((taken) => {
+ lease = taken
+ settled = true
+ })
+ await jest.advanceTimersByTimeAsync(0)
+ if (!settled) {
+ pool.close()
+ await jest.advanceTimersByTimeAsync(0)
+ }
+ return lease
+}
+
+describe('FetchCandidatePool', () => {
+ beforeEach(() => jest.useFakeTimers().setSystemTime(NOW_MS))
+ afterEach(() => jest.useRealTimers())
+
+ it('takes the earliest captured candidates of a domain under its limit, across batches from different partitions', async () => {
+ const pool = buildPool()
+ pool.add(
+ [
+ candidate('https://cdn.hot.com/1.png', 'hot.com'),
+ candidate('https://cdn.hot.com/2.png', 'hot.com'),
+ candidate('https://cdn.hot.com/deferred.png', 'hot.com', NOW_MS - 86_400_000),
+ ],
+ 'partition-1',
+ NOW_MS + 40_000
+ )
+ jest.advanceTimersByTime(10)
+ pool.add([candidate('https://cdn.quiet.com/1.png', 'quiet.com')], 'partition-2', NOW_MS + 40_000)
+
+ const leases = [await takeReady(pool), await takeReady(pool), await takeReady(pool)]
+
+ expect(leases.map((lease) => lease?.candidate.currentUrl)).toEqual([
+ 'https://cdn.hot.com/deferred.png',
+ 'https://cdn.hot.com/1.png',
+ 'https://cdn.quiet.com/1.png',
+ ])
+ expect(leases[2]?.context).toBe('partition-2')
+ expect(await takeReady(pool)).toBeUndefined()
+ })
+
+ it('frees the domain slot on release, not before', async () => {
+ const pool = buildPool({}, { maxConcurrentPerRegistrableDomain: 1 })
+ pool.add(
+ [candidate('https://cdn.hot.com/1.png', 'hot.com'), candidate('https://img.hot.com/2.png', 'hot.com')],
+ 'partition-1',
+ NOW_MS + 40_000
+ )
+
+ const first = await takeReady(pool)
+ const waiting = pool.take()
+ await jest.advanceTimersByTimeAsync(0)
+ first?.release()
+
+ expect((await waiting)?.candidate.currentUrl).toBe('https://img.hot.com/2.png')
+ })
+
+ it('holds an origin until its crawl delay passes while other origins keep running', async () => {
+ const pool = buildPool({ 'https://slow.example.com': 5_000 })
+ pool.add(
+ [
+ candidate('https://slow.example.com/1.png', 'example.com'),
+ candidate('https://slow.example.com/2.png', 'example.com'),
+ candidate('https://fast.other.net/1.png', 'other.net'),
+ ],
+ 'partition-1',
+ NOW_MS + 40_000
+ )
+
+ const first = await takeReady(pool)
+ const second = await takeReady(pool)
+ const delayed = pool.take()
+ await jest.advanceTimersByTimeAsync(4_999)
+ let delayedTaken = false
+ void delayed.then(() => {
+ delayedTaken = true
+ })
+ await jest.advanceTimersByTimeAsync(0)
+ expect(delayedTaken).toBe(false)
+ await jest.advanceTimersByTimeAsync(1)
+
+ expect([first, second, await delayed].map((lease) => lease?.candidate.currentUrl)).toEqual([
+ 'https://slow.example.com/1.png',
+ 'https://fast.other.net/1.png',
+ 'https://slow.example.com/2.png',
+ ])
+ })
+
+ it('holds an origin for a crawl delay longer than the longest Node timer with no timer loop', async () => {
+ const pool = buildPool({ 'https://slow.example.com': 3_000_000_000 })
+ pool.add(
+ [
+ candidate('https://slow.example.com/1.png', 'example.com'),
+ candidate('https://slow.example.com/2.png', 'example.com'),
+ ],
+ 'partition-1',
+ NOW_MS + 40_000
+ )
+ await takeReady(pool)
+ let delayedTaken = false
+ void pool.take().then(() => {
+ delayedTaken = true
+ })
+ const timers = jest.spyOn(global, 'setTimeout')
+
+ await jest.advanceTimersByTimeAsync(100)
+
+ expect(delayedTaken).toBe(false)
+ expect(timers).not.toHaveBeenCalled()
+ timers.mockRestore()
+ pool.close()
+ })
+
+ it('hands out a batch past its deadline without a domain slot', async () => {
+ const pool = buildPool({}, { maxConcurrentPerRegistrableDomain: 1 })
+ pool.add(
+ [candidate('https://cdn.hot.com/1.png', 'hot.com'), candidate('https://cdn.hot.com/2.png', 'hot.com')],
+ 'partition-1',
+ NOW_MS + 1_000
+ )
+ const held = await takeReady(pool)
+ await jest.advanceTimersByTimeAsync(1_001)
+
+ expect(held?.candidate.currentUrl).toBe('https://cdn.hot.com/1.png')
+ expect((await takeReady(pool))?.candidate.currentUrl).toBe('https://cdn.hot.com/2.png')
+ expect(pool.candidateCount).toBe(0)
+ })
+
+ it('withdraws the queued candidates of a failed batch only', async () => {
+ const pool = buildPool()
+ const failed = pool.add(
+ [
+ candidate('https://a.example.com/1.png', 'example.com'),
+ candidate('https://b.example.com/2.png', 'example.com'),
+ ],
+ 'failed',
+ NOW_MS + 40_000
+ )
+ pool.add([candidate('https://c.other.net/1.png', 'other.net')], 'healthy', NOW_MS + 40_000)
+
+ await takeReady(pool)
+
+ expect(pool.withdraw(failed)).toBe(1)
+ expect((await takeReady(pool))?.context).toBe('healthy')
+ expect(await takeReady(pool)).toBeUndefined()
+ })
+
+ it.each([
+ ['counts a hot domain only up to its limit, so the owner still has room', 10, 'hot.com', true],
+ ['has no room once enough distinct domains can run', 1, 'domain', false],
+ ])('%s', (_name, urlsPerDomain, domainPrefix, expectedRoom) => {
+ const pool = buildPool({}, { maxConcurrentPerRegistrableDomain: 2, refillRunnableUrls: 3 })
+ const owner = pool.createOwner()
+ const candidates = Array.from({ length: 10 }, (_unused, index) => {
+ const domain = urlsPerDomain === 10 ? domainPrefix : `${domainPrefix}${index}.com`
+ return candidate(`https://cdn.${domain}/${index}.png`, domain)
+ })
+
+ pool.add(candidates, 'partition-1', NOW_MS + 40_000, owner)
+
+ expect(owner.queuedUrls).toBe(10)
+ expect(owner.hasRoom()).toBe(expectedRoom)
+ })
+
+ it('keeps an owner without room while a batch waits for admission', async () => {
+ const pool = buildPool()
+ const owner = pool.createOwner()
+ const admission = owner.beginAdmission()
+ let hadRoom: boolean | undefined
+ void owner.waitForRoom(30_000).then((room) => {
+ hadRoom = room
+ })
+ await jest.advanceTimersByTimeAsync(0)
+ expect(hadRoom).toBeUndefined()
+
+ admission.admitted()
+ await jest.advanceTimersByTimeAsync(0)
+
+ expect(hadRoom).toBe(true)
+ })
+
+ it('gives an owner room when a fetch worker finds nothing eligible, even with runnable URLs queued', async () => {
+ const pool = new FetchCandidatePool<string>(
+ { maxConcurrentPerRegistrableDomain: 2, refillRunnableUrls: 3, maxQueuedUrlsPerOwner: 100 },
+ { originCrawlDelayMs: () => 5_000, originNextImageStartAtMs: () => NOW_MS + 5_000 }
+ )
+ const owner = pool.createOwner()
+ const delayedCandidates = Array.from({ length: 5 }, (_unused, index) =>
+ candidate(`https://cdn.site${index}.com/a.png`, `site${index}.com`)
+ )
+ pool.add(delayedCandidates, 'partition-1', NOW_MS + 40_000, owner)
+ let hadRoom: boolean | undefined
+ void owner.waitForRoom(30_000).then((room) => {
+ hadRoom = room
+ })
+ await jest.advanceTimersByTimeAsync(0)
+ expect(owner.hasRoom()).toBe(false)
+ expect(hadRoom).toBeUndefined()
+
+ void pool.take()
+ await jest.advanceTimersByTimeAsync(0)
+
+ expect(hadRoom).toBe(true)
+ pool.close()
+ })
+})
diff --git a/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/fetch-candidate-pool.ts b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/fetch-candidate-pool.ts
new file mode 100644
index 000000000000..8ede9411785e
--- /dev/null
+++ b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/fetch-candidate-pool.ts
@@ -0,0 +1,611 @@
+import { FetchCandidate } from './collected-urls-record'
+import { IndexedPriorityQueue } from './fetch-candidate-queue'
+
+export interface FetchCandidatePoolLimits {
+ refillRunnableUrls: number
+ maxQueuedUrlsPerOwner: number
+}
+
+export interface FetchCandidatePoolOptions extends FetchCandidatePoolLimits {
+ maxConcurrentPerRegistrableDomain: number
+}
+
+export interface OriginPacing {
+ originCrawlDelayMs(origin: string): number
+ originNextImageStartAtMs(origin: string): number
+}
+
+export interface FetchCandidatePoolLease<T> {
+ candidate: FetchCandidate
+ context: T
+ waitedMs: number
+ release(): void
+}
+
+export interface FetchCandidatePoolAdmission {
+ owner: FetchCandidatePoolOwner
+ admitted(): void
+}
+
+type EntryState = 'queued' | 'expired' | 'taken' | 'withdrawn'
+
+interface EntryPayload<T> {
+ candidate: FetchCandidate
+ context: T
+ batch: PoolBatch<T>
+}
+
+interface PoolEntry<T> {
+ /** Cleared once the entry leaves the pool, because a spent entry can stay in its origin array until compaction. */
+ payload: EntryPayload<T> | undefined
+ sequence: number
+ enqueuedAtMs: number
+ state: EntryState
+ origin: OriginQueue<T>
+}
+
+interface OriginQueue<T> {
+ origin: string
+ domain: DomainQueue<T>
+ entries: PoolEntry<T>[]
+ head: number
+ queued: number
+ active: number
+ pacedUntilMs: number
+ delayed: DelayedOrigin<T> | undefined
+ heapIndex: number
+}
+
+interface DomainQueue<T> {
+ registrableDomain: string
+ origins: Map<string, OriginQueue<T>>
+ eligibleOrigins: IndexedPriorityQueue<OriginQueue<T>>
+ active: number
+ heapIndex: number
+}
+
+interface DelayedOrigin<T> {
+ origin: OriginQueue<T>
+ readyAtMs: number
+ heapIndex: number
+}
+
+export interface PoolBatch<T> {
+ readonly owner: FetchCandidatePoolOwner | undefined
+ readonly entries: PoolEntry<T>[]
+ queued: number
+ expiryTimer: NodeJS.Timeout | undefined
+}
+
+const COMPACT_AFTER_ENTRIES = 64
+const MAX_TIMER_DELAY_MS = 2 ** 31 - 1
+
+/**
+ * One queue of fetch candidates for the whole pod.
+ *
+ * Every consumer's batch adds its candidates here, and the fetch workers take them in the order they
+ * entered the pool, among the origins that are under their limits and past their crawl delay. The
+ * order uses pool entry and not record age, because the pass deadline and the batch timeout count
+ * from when the pod received the batch. Inside one batch, the candidates enter in capture order. A
+ * pass-deadline deferral goes back to the end of its partition, behind URLs captured later. In input
+ * order, a domain that cannot serve its whole batch would serve those later URLs on every pass and
+ * never reach the deferred one. The limits count across the pod, so two batches from one partition
+ * cannot both claim a domain's slots.
+ */
+export class FetchCandidatePool<T> {
+ private readonly domains = new Map<string, DomainQueue<T>>()
+ private readonly eligibleDomains = new IndexedPriorityQueue<DomainQueue<T>>(domainHasOlderHead)
+ private readonly delayedOrigins = new IndexedPriorityQueue<DelayedOrigin<T>>(
+ (left, right) => left.readyAtMs < right.readyAtMs
+ )
+ private delayTimer: NodeJS.Timeout | undefined
+ private delayTimerAtMs = Number.POSITIVE_INFINITY
+ private expired: PoolEntry<T>[] = []
+ private expiredHead = 0
+ private waiters: (() => void)[] = []
+ private readonly owners: FetchCandidatePoolOwner[] = []
+ private sequence = 0
+ private queuedCount = 0
+ private closed = false
+
+ constructor(
+ private readonly options: FetchCandidatePoolOptions,
+ private readonly pacing: OriginPacing
+ ) {
+ for (const [name, value] of Object.entries(options)) {
+ if (!Number.isSafeInteger(value) || value <= 0) {
+ throw new Error(`the image fetch candidate pool ${name} must be a positive integer, got ${value}`)
+ }
+ }
+ }
+
+ public get candidateCount(): number {
+ return this.queuedCount + (this.expired.length - this.expiredHead)
+ }
+
+ public createOwner(): FetchCandidatePoolOwner {
+ const owner = new FetchCandidatePoolOwner(this.options)
+ this.owners.push(owner)
+ return owner
+ }
+
+ public add(
+ candidates: FetchCandidate[],
+ context: T,
+ deadlineMs: number,
+ owner?: FetchCandidatePoolOwner
+ ): PoolBatch<T> {
+ if (this.closed) {
+ throw new Error('the image fetch candidate pool is closed, so no worker would take these candidates')
+ }
+ const nowMs = Date.now()
+ const batch: PoolBatch<T> = { owner, entries: [], queued: 0, expiryTimer: undefined }
+ const touched = new Set<OriginQueue<T>>()
+ for (const candidate of [...candidates].sort((left, right) => left.firstSeenAtMs - right.firstSeenAtMs)) {
+ const origin = this.originQueueFor(candidate)
+ const entry: PoolEntry<T> = {
+ payload: { candidate, context, batch },
+ sequence: this.sequence++,
+ enqueuedAtMs: nowMs,
+ state: 'queued',
+ origin,
+ }
+ origin.entries.push(entry)
+ origin.queued += 1
+ batch.entries.push(entry)
+ batch.queued += 1
+ this.queuedCount += 1
+ owner?.recordQueued(candidate.registrableDomain)
+ touched.add(origin)
+ }
+ for (const origin of touched) {
+ this.refreshOrigin(origin, nowMs)
+ }
+ if (batch.queued > 0) {
+ // The fetch path treats a candidate as late only once the clock is past the deadline.
+ batch.expiryTimer = setTimeout(() => this.expire(batch), Math.max(0, deadlineMs - nowMs) + 1)
+ batch.expiryTimer.unref()
+ this.wake(1)
+ }
+ return batch
+ }
+
+ public async take(): Promise<FetchCandidatePoolLease<T> | undefined> {
+ for (;;) {
+ if (this.closed) {
+ return undefined
+ }
+ const lease = this.tryTake()
+ if (lease) {
+ if (this.hasSelectableWork()) {
+ this.wake(1)
+ }
+ return lease
+ }
+ for (const owner of this.owners) {
+ owner.recordIdleWorker()
+ }
+ await new Promise<void>((resolve) => this.waiters.push(resolve))
+ }
+ }
+
+ /** Removes the batch's candidates that no worker has taken, and returns how many it removed. */
+ public withdraw(batch: PoolBatch<T>): number {
+ const touched = new Set<OriginQueue<T>>()
+ let withdrawn = 0
+ for (const entry of batch.entries) {
+ if (entry.state === 'queued') {
+ this.leaveOriginQueue(entry)
+ touched.add(entry.origin)
+ } else if (entry.state !== 'expired') {
+ continue
+ }
+ entry.state = 'withdrawn'
+ entry.payload = undefined
+ withdrawn += 1
+ }
+ this.clearExpiry(batch)
+ const nowMs = Date.now()
+ for (const origin of touched) {
+ this.refreshOrigin(origin, nowMs)
+ }
+ return withdrawn
+ }
+
+ public close(): void {
+ this.closed = true
+ if (this.delayTimer) {
+ clearTimeout(this.delayTimer)
+ this.delayTimer = undefined
+ }
+ this.wake(this.waiters.length)
+ }
+
+ private tryTake(): FetchCandidatePoolLease<T> | undefined {
+ const expiredEntry = this.nextExpired()
+ if (expiredEntry) {
+ expiredEntry.state = 'taken'
+ return this.lease(expiredEntry, Date.now(), () => undefined)
+ }
+ const domain = this.eligibleDomains.peek()
+ if (!domain) {
+ return undefined
+ }
+ const origin = domain.eligibleOrigins.peek()!
+ const entry = origin.entries[origin.head]
+ const nowMs = Date.now()
+ this.leaveOriginQueue(entry)
+ entry.state = 'taken'
+ origin.active += 1
+ domain.active += 1
+ const crawlDelayMs = this.pacing.originCrawlDelayMs(origin.origin)
+ if (crawlDelayMs > 0) {
+ origin.pacedUntilMs = Math.max(nowMs, origin.pacedUntilMs) + crawlDelayMs
+ }
+ this.refreshOrigin(origin, nowMs)
+ let released = false
+ return this.lease(entry, nowMs, () => {
+ if (released) {
+ return
+ }
+ released = true
+ origin.active -= 1
+ domain.active -= 1
+ this.refreshOrigin(origin, Date.now())
+ this.wake(1)
+ })
+ }
+
+ private lease(entry: PoolEntry<T>, nowMs: number, release: () => void): FetchCandidatePoolLease<T> {
+ const { candidate, context } = entry.payload!
+ entry.payload = undefined
+ return {
+ candidate,
+ context,
+ waitedMs: Math.max(0, nowMs - entry.enqueuedAtMs),
+ release,
+ }
+ }
+
+ private leaveOriginQueue(entry: PoolEntry<T>): void {
+ const { batch, candidate } = entry.payload!
+ entry.origin.queued -= 1
+ batch.queued -= 1
+ this.queuedCount -= 1
+ batch.owner?.recordLeft(candidate.registrableDomain)
+ if (batch.queued === 0) {
+ this.clearExpiry(batch)
+ }
+ }
+
+ private expire(batch: PoolBatch<T>): void {
+ batch.expiryTimer = undefined
+ const touched = new Set<OriginQueue<T>>()
+ for (const entry of batch.entries) {
+ if (entry.state !== 'queued') {
+ continue
+ }
+ this.leaveOriginQueue(entry)
+ entry.state = 'expired'
+ this.expired.push(entry)
+ touched.add(entry.origin)
+ }
+ const nowMs = Date.now()
+ for (const origin of touched) {
+ this.refreshOrigin(origin, nowMs)
+ }
+ this.wake(1)
+ }
+
+ private clearExpiry(batch: PoolBatch<T>): void {
+ if (batch.expiryTimer) {
+ clearTimeout(batch.expiryTimer)
+ batch.expiryTimer = undefined
+ }
+ }
+
+ private nextExpired(): PoolEntry<T> | undefined {
+ while (this.expiredHead < this.expired.length) {
+ const entry = this.expired[this.expiredHead++]
+ if (entry.state === 'expired') {
+ this.compactExpired()
+ return entry
+ }
+ }
+ this.compactExpired()
+ return undefined
+ }
+
+ private compactExpired(): void {
+ if (this.expiredHead === this.expired.length) {
+ this.expired = []
+ this.expiredHead = 0
+ } else if (this.expiredHead > COMPACT_AFTER_ENTRIES && this.expiredHead * 2 > this.expired.length) {
+ this.expired = this.expired.slice(this.expiredHead)
+ this.expiredHead = 0
+ }
+ }
+
+ private hasSelectableWork(): boolean {
+ for (let index = this.expiredHead; index < this.expired.length; index++) {
+ if (this.expired[index].state === 'expired') {
+ return true
+ }
+ }
+ return this.eligibleDomains.size > 0
+ }
+
+ private wake(count: number): void {
+ for (let woken = 0; woken < count && this.waiters.length > 0; woken++) {
+ this.waiters.shift()!()
+ }
+ }
+
+ private originQueueFor(candidate: FetchCandidate): OriginQueue<T> {
+ let domain = this.domains.get(candidate.registrableDomain)
+ if (!domain) {
+ domain = {
+ registrableDomain: candidate.registrableDomain,
+ origins: new Map(),
+ eligibleOrigins: new IndexedPriorityQueue<OriginQueue<T>>(originHasOlderHead),
+ active: 0,
+ heapIndex: -1,
+ }
+ this.domains.set(candidate.registrableDomain, domain)
+ }
+ let origin = domain.origins.get(candidate.origin)
+ if (!origin) {
+ origin = {
+ origin: candidate.origin,
+ domain,
+ entries: [],
+ head: 0,
+ queued: 0,
+ active: 0,
+ pacedUntilMs: 0,
+ delayed: undefined,
+ heapIndex: -1,
+ }
+ domain.origins.set(candidate.origin, origin)
+ }
+ return origin
+ }
+
+ private refreshOrigin(origin: OriginQueue<T>, nowMs: number): void {
+ this.advanceHead(origin)
+ const domain = origin.domain
+ if (origin.queued === 0 || origin.active >= this.options.maxConcurrentPerRegistrableDomain) {
+ domain.eligibleOrigins.remove(origin)
+ this.undelay(origin)
+ if (origin.queued === 0 && origin.active === 0) {
+ domain.origins.delete(origin.origin)
+ }
+ } else {
+ const readyAtMs = Math.max(origin.pacedUntilMs, this.pacing.originNextImageStartAtMs(origin.origin))
+ if (readyAtMs > nowMs) {
+ domain.eligibleOrigins.remove(origin)
+ this.delay(origin, readyAtMs)
+ } else {
+ this.undelay(origin)
+ domain.eligibleOrigins.addOrUpdate(origin)
+ }
+ }
+ this.refreshDomain(domain)
+ }
+
+ private refreshDomain(domain: DomainQueue<T>): void {
+ if (domain.origins.size === 0 && domain.active === 0) {
+ this.eligibleDomains.remove(domain)
+ if (this.domains.get(domain.registrableDomain) === domain) {
+ this.domains.delete(domain.registrableDomain)
+ }
+ return
+ }
+ if (domain.active < this.options.maxConcurrentPerRegistrableDomain && domain.eligibleOrigins.size > 0) {
+ this.eligibleDomains.addOrUpdate(domain)
+ } else {
+ this.eligibleDomains.remove(domain)
+ }
+ }
+
+ private advanceHead(origin: OriginQueue<T>): void {
+ while (origin.head < origin.entries.length && origin.entries[origin.head].state !== 'queued') {
+ origin.head += 1
+ }
+ if (origin.head === origin.entries.length) {
+ origin.entries = []
+ origin.head = 0
+ } else if (origin.head > COMPACT_AFTER_ENTRIES && origin.head * 2 > origin.entries.length) {
+ origin.entries = origin.entries.slice(origin.head)
+ origin.head = 0
+ }
+ }
+
+ private delay(origin: OriginQueue<T>, readyAtMs: number): void {
+ if (origin.delayed) {
+ origin.delayed.readyAtMs = readyAtMs
+ this.delayedOrigins.addOrUpdate(origin.delayed)
+ } else {
+ origin.delayed = { origin, readyAtMs, heapIndex: -1 }
+ this.delayedOrigins.addOrUpdate(origin.delayed)
+ }
+ this.armDelayTimer()
+ }
+
+ private undelay(origin: OriginQueue<T>): void {
+ if (!origin.delayed) {
+ return
+ }
+ this.delayedOrigins.remove(origin.delayed)
+ origin.delayed = undefined
+ this.armDelayTimer()
+ }
+
+ private armDelayTimer(): void {
+ const next = this.delayedOrigins.peek()
+ const nextAtMs = next?.readyAtMs ?? Number.POSITIVE_INFINITY
+ if (nextAtMs === this.delayTimerAtMs || this.closed) {
+ return
+ }
+ if (this.delayTimer) {
+ clearTimeout(this.delayTimer)
+ this.delayTimer = undefined
+ }
+ this.delayTimerAtMs = nextAtMs
+ if (next) {
+ // Node runs a longer timer after 1 ms, so a long crawl delay would rearm it in a tight loop.
+ const waitMs = Math.min(MAX_TIMER_DELAY_MS, Math.max(0, nextAtMs - Date.now()))
+ this.delayTimer = setTimeout(() => this.releaseDelayedOrigins(), waitMs)
+ this.delayTimer.unref()
+ }
+ }
+
+ private releaseDelayedOrigins(): void {
+ this.delayTimer = undefined
+ this.delayTimerAtMs = Number.POSITIVE_INFINITY
+ const nowMs = Date.now()
+ for (
+ let next = this.delayedOrigins.peek();
+ next && next.readyAtMs <= nowMs;
+ next = this.delayedOrigins.peek()
+ ) {
+ this.delayedOrigins.remove(next)
+ next.origin.delayed = undefined
+ this.refreshOrigin(next.origin, nowMs)
+ }
+ this.armDelayTimer()
+ if (this.eligibleDomains.size > 0) {
+ this.wake(1)
+ }
+ }
+}
+
+/**
+ * The window of one consumer: how much of its work waits in the pool.
+ *
+ * A registrable domain counts as runnable only up to its concurrency limit, because the pool never
+ * runs more than that at once. A window full of one hot domain therefore still lets the consumer
+ * read more, so its other domains do not wait in Kafka while candidate slots are free.
+ */
+export class FetchCandidatePoolOwner {
+ private queued = 0
+ private runnable = 0
+ private pendingAdmissions = 0
+ private readonly queuedByDomain = new Map<string, number>()
+ private readonly roomWaiters = new Set<() => void>()
+
+ constructor(private readonly options: FetchCandidatePoolOptions) {}
+
+ public get queuedUrls(): number {
+ return this.queued
+ }
+
+ public get runnableUrls(): number {
+ return this.runnable
+ }
+
+ public beginAdmission(): FetchCandidatePoolAdmission {
+ this.pendingAdmissions += 1
+ let done = false
+ return {
+ owner: this,
+ admitted: () => {
+ if (done) {
+ return
+ }
+ done = true
+ this.pendingAdmissions -= 1
+ this.notifyIfRoom()
+ },
+ }
+ }
+
+ public hasRoom(): boolean {
+ return (
+ this.pendingAdmissions === 0 &&
+ this.runnable < this.options.refillRunnableUrls &&
+ this.queued < this.options.maxQueuedUrlsPerOwner
+ )
+ }
+
+ /** Resolves true when the owner has room for another batch, or false when the wait runs out. */
+ public waitForRoom(maxWaitMs: number): Promise<boolean> {
+ if (this.hasRoom()) {
+ return Promise.resolve(true)
+ }
+ return new Promise((resolve) => {
+ const waiter = (): void => {
+ clearTimeout(timer)
+ resolve(true)
+ }
+ const timer = setTimeout(() => {
+ this.roomWaiters.delete(waiter)
+ resolve(false)
+ }, maxWaitMs)
+ this.roomWaiters.add(waiter)
+ })
+ }
+
+ public recordQueued(registrableDomain: string): void {
+ this.queued += 1
+ this.changeDomainCount(registrableDomain, 1)
+ }
+
+ public recordLeft(registrableDomain: string): void {
+ this.queued -= 1
+ this.changeDomainCount(registrableDomain, -1)
+ this.notifyIfRoom()
+ }
+
+ /**
+ * A fetch worker found no eligible candidate. Queued URLs that count as runnable can still be
+ * unable to run, for example behind a crawl delay or behind another member's hold on a domain,
+ * so the member reads more while it is under the queue limit.
+ */
+ public recordIdleWorker(): void {
+ if (this.pendingAdmissions === 0 && this.queued < this.options.maxQueuedUrlsPerOwner) {
+ this.releaseRoomWaiters()
+ }
+ }
+
+ private changeDomainCount(registrableDomain: string, delta: number): void {
+ const limit = this.options.maxConcurrentPerRegistrableDomain
+ const before = this.queuedByDomain.get(registrableDomain) ?? 0
+ const after = before + delta
+ if (after === 0) {
+ this.queuedByDomain.delete(registrableDomain)
+ } else {
+ this.queuedByDomain.set(registrableDomain, after)
+ }
+ this.runnable += Math.min(after, limit) - Math.min(before, limit)
+ }
+
+ private notifyIfRoom(): void {
+ if (this.hasRoom()) {
+ this.releaseRoomWaiters()
+ }
+ }
+
+ private releaseRoomWaiters(): void {
+ if (this.roomWaiters.size === 0) {
+ return
+ }
+ const waiters = [...this.roomWaiters]
+ this.roomWaiters.clear()
+ for (const waiter of waiters) {
+ waiter()
+ }
+ }
+}
+
+function originHeadSequence<T>(origin: OriginQueue<T>): number {
+ return origin.entries[origin.head].sequence
+}
+
+function originHasOlderHead<T>(left: OriginQueue<T>, right: OriginQueue<T>): boolean {
+ return originHeadSequence(left) < originHeadSequence(right)
+}
+
+function domainHasOlderHead<T>(left: DomainQueue<T>, right: DomainQueue<T>): boolean {
+ return originHeadSequence(left.eligibleOrigins.peek()!) < originHeadSequence(right.eligibleOrigins.peek()!)
+}
diff --git a/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/fetch-candidate-queue.test.ts b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/fetch-candidate-queue.test.ts
new file mode 100644
index 000000000000..e6565e585d38
--- /dev/null
+++ b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/fetch-candidate-queue.test.ts
@@ -0,0 +1,133 @@
+import { FetchCandidate, MAX_HOPS } from './collected-urls-record'
+import { FetchCandidateQueue } from './fetch-candidate-queue'
+
+const OPTIONS = {
+ maxConcurrentPerRegistrableDomain: 6,
+ maxInFlightRequests: 300,
+}
+
+function candidate(index: number): FetchCandidate {
+ return {
+ originalRef: `ref-${index}`,
+ currentUrl: `https://origin-${index}.example.com/image.png`,
+ host: `origin-${index}.example.com`,
+ origin: `https://origin-${index}.example.com`,
+ registrableDomain: 'example.com',
+ remainingHops: MAX_HOPS,
+ notBeforeMs: 0,
+ firstSeenAtMs: 1_700_000_000_000,
+ fetchCount: 0,
+ republishCount: 0,
+ lastRepublishReason: null,
+ }
+}
+
+function dominantOriginCandidate(index: number): FetchCandidate {
+ return {
+ ...candidate(index),
+ currentUrl: `https://dominant.example.com/${index}.png`,
+ host: 'dominant.example.com',
+ origin: 'https://dominant.example.com',
+ }
+}
+
+function domainCandidate(domainIndex: number, candidateIndex: number): FetchCandidate {
+ const index = domainIndex * 1_000 + candidateIndex
+ const registrableDomain = `domain-${domainIndex}.example`
+ return {
+ ...candidate(index),
+ currentUrl: `https://cdn.${registrableDomain}/${candidateIndex}.png`,
+ host: `cdn.${registrableDomain}`,
+ origin: `https://cdn.${registrableDomain}`,
+ registrableDomain,
+ }
+}
+
+function concurrentDomainCounts(queue: FetchCandidateQueue, count: number): Map<string, number> {
+ const counts = new Map<string, number>()
+ for (let index = 0; index < count; index++) {
+ const lease = queue.take()
+ expect(lease).toBeDefined()
+ if (lease) {
+ counts.set(lease.candidate.registrableDomain, (counts.get(lease.candidate.registrableDomain) ?? 0) + 1)
+ }
+ }
+ return counts
+}
+
+describe('FetchCandidateQueue', () => {
+ it('drains a maximum-sized diverse poll batch without scanning every remaining origin', () => {
+ const candidateCount = 50_000
+ const queue = new FetchCandidateQueue(
+ Array.from({ length: candidateCount }, (_, index) => candidate(index)),
+ OPTIONS
+ )
+ let drained = 0
+
+ for (;;) {
+ const lease = queue.take()
+ if (!lease) {
+ break
+ }
+ drained += 1
+ lease.release()
+ }
+
+ expect(drained).toBe(candidateCount)
+ })
+
+ it('keeps one selectable domain entry during a long dominant-origin run', () => {
+ const dominantCandidateCount = 50_000
+ const queue = new FetchCandidateQueue(
+ [
+ ...Array.from({ length: dominantCandidateCount }, (_, index) => dominantOriginCandidate(index)),
+ candidate(dominantCandidateCount),
+ ],
+ {
+ ...OPTIONS,
+ maxConcurrentPerRegistrableDomain: 1,
+ }
+ )
+ let drained = 0
+ let maximumSelectableDomains = 0
+
+ for (;;) {
+ const lease = queue.take()
+ if (!lease) {
+ break
+ }
+ drained += 1
+ lease.release()
+ maximumSelectableDomains = Math.max(maximumSelectableDomains, queue.selectableRegistrableDomainCount)
+ }
+
+ expect(drained).toBe(dominantCandidateCount + 1)
+ expect(maximumSelectableDomains).toBe(1)
+ })
+
+ it.each([
+ {
+ name: 'gives 150 equal domains two requests each',
+ domainSizes: Array.from({ length: 150 }, () => 6),
+ expectedCounts: Array.from({ length: 150 }, () => 2),
+ },
+ {
+ name: 'gives smaller domains their proportional request share',
+ domainSizes: [...Array.from({ length: 50 }, () => 100), ...Array.from({ length: 50 }, () => 10)],
+ expectedCounts: [...Array.from({ length: 50 }, () => 5), ...Array.from({ length: 50 }, () => 1)],
+ },
+ ])('$name', ({ domainSizes, expectedCounts }) => {
+ const queue = new FetchCandidateQueue(
+ domainSizes.flatMap((size, domainIndex) =>
+ Array.from({ length: size }, (_, candidateIndex) => domainCandidate(domainIndex, candidateIndex))
+ ),
+ OPTIONS
+ )
+
+ const counts = concurrentDomainCounts(queue, OPTIONS.maxInFlightRequests)
+
+ expect(
+ Array.from({ length: domainSizes.length }, (_, index) => counts.get(`domain-${index}.example`) ?? 0)
+ ).toEqual(expectedCounts)
+ })
+})
diff --git a/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/fetch-candidate-queue.ts b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/fetch-candidate-queue.ts
new file mode 100644
index 000000000000..d6578c07558e
--- /dev/null
+++ b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/fetch-candidate-queue.ts
@@ -0,0 +1,420 @@
+import { FetchCandidate } from './collected-urls-record'
+
+export interface FetchCandidateLease {
+ candidate: FetchCandidate
+ release(): void
+}
+
+interface OriginQueue {
+ origin: string
+ candidates: FetchCandidate[]
+ head: number
+ sequence: number
+ active: number
+ initialCandidateCount: number
+ targetConcurrent: number
+ heapIndex: number
+}
+
+interface RegistrableDomainQueue {
+ registrableDomain: string
+ origins: Map<string, OriginQueue>
+ availableOrigins: IndexedPriorityQueue<OriginQueue>
+ initialCandidateCount: number
+ waitingCandidateCount: number
+ targetConcurrent: number
+ sequence: number
+ active: number
+ heapIndex: number
+}
+
+export interface IndexedPriorityQueueItem {
+ heapIndex: number
+}
+
+export class IndexedPriorityQueue<T extends IndexedPriorityQueueItem> {
+ private readonly heap: T[] = []
+
+ constructor(private readonly hasPriority: (left: T, right: T) => boolean) {}
+
+ public get size(): number {
+ return this.heap.length
+ }
+
+ public addOrUpdate(item: T): void {
+ if (item.heapIndex < 0) {
+ item.heapIndex = this.heap.length
+ this.heap.push(item)
+ this.siftUp(item.heapIndex)
+ return
+ }
+ this.rebalance(item.heapIndex)
+ }
+
+ public remove(item: T): void {
+ const index = item.heapIndex
+ if (index < 0) {
+ return
+ }
+ const last = this.heap.pop()!
+ item.heapIndex = -1
+ if (index === this.heap.length) {
+ return
+ }
+ this.heap[index] = last
+ last.heapIndex = index
+ this.rebalance(index)
+ }
+
+ public peek(): T | undefined {
+ return this.heap[0]
+ }
+
+ public poll(): T | undefined {
+ const item = this.heap[0]
+ if (item) {
+ this.remove(item)
+ }
+ return item
+ }
+
+ private rebalance(index: number): void {
+ const parentIndex = Math.floor((index - 1) / 2)
+ if (index > 0 && this.hasPriority(this.heap[index], this.heap[parentIndex])) {
+ this.siftUp(index)
+ } else {
+ this.siftDown(index)
+ }
+ }
+
+ private siftUp(startIndex: number): void {
+ let index = startIndex
+ while (index > 0) {
+ const parentIndex = Math.floor((index - 1) / 2)
+ if (!this.hasPriority(this.heap[index], this.heap[parentIndex])) {
+ return
+ }
+ this.swap(index, parentIndex)
+ index = parentIndex
+ }
+ }
+
+ private siftDown(startIndex: number): void {
+ let index = startIndex
+ for (;;) {
+ const leftIndex = index * 2 + 1
+ if (leftIndex >= this.heap.length) {
+ return
+ }
+ const rightIndex = leftIndex + 1
+ const childIndex =
+ rightIndex < this.heap.length && this.hasPriority(this.heap[rightIndex], this.heap[leftIndex])
+ ? rightIndex
+ : leftIndex
+ if (!this.hasPriority(this.heap[childIndex], this.heap[index])) {
+ return
+ }
+ this.swap(index, childIndex)
+ index = childIndex
+ }
+ }
+
+ private swap(leftIndex: number, rightIndex: number): void {
+ const left = this.heap[leftIndex]
+ const right = this.heap[rightIndex]
+ this.heap[leftIndex] = right
+ this.heap[rightIndex] = left
+ left.heapIndex = rightIndex
+ right.heapIndex = leftIndex
+ }
+}
+
+export interface FetchCandidateQueueOptions {
+ maxConcurrentPerRegistrableDomain: number
+ maxInFlightRequests: number
+}
+
+export interface DeduplicatedFetchCandidates {
+ candidates: FetchCandidate[]
+ duplicateCount: number
+}
+
+export function deduplicateFetchCandidates(candidates: FetchCandidate[]): DeduplicatedFetchCandidates {
+ const candidatesByCanonicalRef = new Map<string, FetchCandidate>()
+ let duplicateCount = 0
+ for (const candidate of candidates) {
+ const existing = candidatesByCanonicalRef.get(candidate.originalRef)
+ if (existing) {
+ duplicateCount += 1
+ candidatesByCanonicalRef.set(candidate.originalRef, mergeDuplicateFetchCandidates(existing, candidate))
+ } else {
+ candidatesByCanonicalRef.set(candidate.originalRef, candidate)
+ }
+ }
+ return { candidates: [...candidatesByCanonicalRef.values()], duplicateCount }
+}
+
+export class FetchCandidateQueue {
+ private readonly domains = new Map<string, RegistrableDomainQueue>()
+ private readonly availableDomains = new IndexedPriorityQueue<RegistrableDomainQueue>(domainQueueHasPriority)
+ private waitingCandidates = 0
+ private initialSchedulableSlots = 0
+ private aborted = false
+
+ constructor(
+ candidates: FetchCandidate[],
+ private readonly options: FetchCandidateQueueOptions
+ ) {
+ const deduplicated = deduplicateFetchCandidates(candidates)
+ let domainSequence = 0
+ let originSequence = 0
+ for (const candidate of deduplicated.candidates) {
+ let domain = this.domains.get(candidate.registrableDomain)
+ if (!domain) {
+ domain = {
+ registrableDomain: candidate.registrableDomain,
+ origins: new Map(),
+ availableOrigins: new IndexedPriorityQueue<OriginQueue>(originQueueHasPriority),
+ initialCandidateCount: 0,
+ waitingCandidateCount: 0,
+ targetConcurrent: 0,
+ sequence: domainSequence++,
+ active: 0,
+ heapIndex: -1,
+ }
+ this.domains.set(candidate.registrableDomain, domain)
+ }
+ let originQueue = domain.origins.get(candidate.origin)
+ if (!originQueue) {
+ originQueue = {
+ origin: candidate.origin,
+ candidates: [],
+ head: 0,
+ sequence: originSequence++,
+ active: 0,
+ initialCandidateCount: 0,
+ targetConcurrent: 0,
+ heapIndex: -1,
+ }
+ domain.origins.set(candidate.origin, originQueue)
+ }
+ originQueue.candidates.push(candidate)
+ originQueue.initialCandidateCount += 1
+ domain.initialCandidateCount += 1
+ domain.waitingCandidateCount += 1
+ this.waitingCandidates += 1
+ }
+ allocateConcurrentTargets(
+ [...this.domains.values()],
+ this.options.maxInFlightRequests,
+ this.options.maxConcurrentPerRegistrableDomain
+ )
+ for (const domain of this.domains.values()) {
+ allocateConcurrentTargets(
+ [...domain.origins.values()],
+ domain.targetConcurrent,
+ this.options.maxConcurrentPerRegistrableDomain
+ )
+ for (const origin of domain.origins.values()) {
+ this.refreshOriginSelection(domain, origin)
+ }
+ this.initialSchedulableSlots += Math.min(
+ domain.initialCandidateCount,
+ this.options.maxConcurrentPerRegistrableDomain
+ )
+ this.refreshDomainSelection(domain)
+ }
+ }
+
+ public get candidateCount(): number {
+ return this.waitingCandidates
+ }
+
+ public get selectableRegistrableDomainCount(): number {
+ return this.availableDomains.size
+ }
+
+ public get schedulableSlotsAtStart(): number {
+ return this.initialSchedulableSlots
+ }
+
+ public availableRequestSlotsAtStart(availableConnections: (registrableDomain: string) => number): number {
+ let slots = 0
+ for (const domain of this.domains.values()) {
+ slots += Math.min(domain.initialCandidateCount, availableConnections(domain.registrableDomain))
+ }
+ return slots
+ }
+
+ public abort(): void {
+ this.aborted = true
+ }
+
+ public take(): FetchCandidateLease | undefined {
+ if (this.aborted) {
+ return undefined
+ }
+ const selected = this.takeHighestPriorityOrigin()
+ if (!selected) {
+ return undefined
+ }
+ const candidate = selected.origin.candidates[selected.origin.head++]!
+ selected.domain.waitingCandidateCount -= 1
+ this.refreshOriginSelection(selected.domain, selected.origin)
+ this.refreshDomainSelection(selected.domain)
+
+ this.waitingCandidates -= 1
+ let released = false
+ return {
+ candidate,
+ release: () => {
+ if (released) {
+ return
+ }
+ released = true
+ selected.domain.active = Math.max(0, selected.domain.active - 1)
+ selected.origin.active = Math.max(0, selected.origin.active - 1)
+ if (originCandidateCount(selected.origin) === 0 && selected.origin.active === 0) {
+ selected.domain.origins.delete(selected.origin.origin)
+ } else {
+ this.refreshOriginSelection(selected.domain, selected.origin)
+ }
+ if (selected.domain.origins.size === 0) {
+ this.availableDomains.remove(selected.domain)
+ this.domains.delete(selected.domain.registrableDomain)
+ } else {
+ this.refreshDomainSelection(selected.domain)
+ }
+ },
+ }
+ }
+
+ private takeHighestPriorityOrigin(): { domain: RegistrableDomainQueue; origin: OriginQueue } | undefined {
+ for (;;) {
+ const domain = this.availableDomains.poll()
+ if (!domain) {
+ return undefined
+ }
+ if (domain.active >= this.options.maxConcurrentPerRegistrableDomain) {
+ continue
+ }
+ const origin = domain.availableOrigins.poll()
+ if (!origin) {
+ continue
+ }
+ domain.active += 1
+ origin.active += 1
+ return { domain, origin }
+ }
+ }
+
+ private refreshOriginSelection(domain: RegistrableDomainQueue, origin: OriginQueue): void {
+ if (originCandidateCount(origin) === 0 || origin.active >= this.options.maxConcurrentPerRegistrableDomain) {
+ domain.availableOrigins.remove(origin)
+ return
+ }
+ domain.availableOrigins.addOrUpdate(origin)
+ }
+
+ private refreshDomainSelection(domain: RegistrableDomainQueue): void {
+ const origin = domain.availableOrigins.peek()
+ if (!origin || domain.active >= this.options.maxConcurrentPerRegistrableDomain) {
+ this.availableDomains.remove(domain)
+ return
+ }
+ this.availableDomains.addOrUpdate(domain)
+ }
+}
+
+function originQueueHasPriority(left: OriginQueue, right: OriginQueue): boolean {
+ const leftDistance = left.targetConcurrent - left.active
+ const rightDistance = right.targetConcurrent - right.active
+ return (
+ leftDistance > rightDistance ||
+ (leftDistance === rightDistance && originCandidateCount(left) > originCandidateCount(right)) ||
+ (leftDistance === rightDistance &&
+ originCandidateCount(left) === originCandidateCount(right) &&
+ left.sequence < right.sequence)
+ )
+}
+
+function originCandidateCount(origin: OriginQueue): number {
+ return origin.candidates.length - origin.head
+}
+
+function domainQueueHasPriority(left: RegistrableDomainQueue, right: RegistrableDomainQueue): boolean {
+ const leftDistance = left.targetConcurrent - left.active
+ const rightDistance = right.targetConcurrent - right.active
+ return (
+ leftDistance > rightDistance ||
+ (leftDistance === rightDistance && left.waitingCandidateCount > right.waitingCandidateCount) ||
+ (leftDistance === rightDistance &&
+ left.waitingCandidateCount === right.waitingCandidateCount &&
+ left.sequence < right.sequence)
+ )
+}
+
+interface ConcurrentTarget {
+ initialCandidateCount: number
+ targetConcurrent: number
+}
+
+function allocateConcurrentTargets<T extends ConcurrentTarget>(
+ queues: T[],
+ totalCapacity: number,
+ maximumPerQueue: number
+): void {
+ let remainingCapacity = Math.min(
+ totalCapacity,
+ queues.reduce((sum, queue) => sum + queue.initialCandidateCount, 0)
+ )
+ let remainingWeight = queues.reduce((sum, queue) => sum + queue.initialCandidateCount, 0)
+ let remainingQueues = queues
+
+ while (remainingQueues.length > 0 && remainingCapacity > 0 && remainingWeight > 0) {
+ const queuesAtLimit = remainingQueues.filter((queue) => {
+ const proportionalTarget = (remainingCapacity * queue.initialCandidateCount) / remainingWeight
+ return proportionalTarget >= Math.min(maximumPerQueue, queue.initialCandidateCount)
+ })
+ if (queuesAtLimit.length === 0) {
+ for (const queue of remainingQueues) {
+ queue.targetConcurrent = (remainingCapacity * queue.initialCandidateCount) / remainingWeight
+ }
+ return
+ }
+ const queuesAtLimitSet = new Set(queuesAtLimit)
+ for (const queue of queuesAtLimit) {
+ queue.targetConcurrent = Math.min(maximumPerQueue, queue.initialCandidateCount)
+ remainingCapacity -= queue.targetConcurrent
+ remainingWeight -= queue.initialCandidateCount
+ }
+ remainingQueues = remainingQueues.filter((queue) => !queuesAtLimitSet.has(queue))
+ }
+}
+
+export function mergeDuplicateFetchCandidates(left: FetchCandidate, right: FetchCandidate): FetchCandidate {
+ let preferredRoute = left
+ if (
+ right.remainingHops < left.remainingHops ||
+ (right.remainingHops === left.remainingHops && right.republishCount > left.republishCount) ||
+ (right.remainingHops === left.remainingHops &&
+ right.republishCount === left.republishCount &&
+ right.fetchCount > left.fetchCount)
+ ) {
+ preferredRoute = right
+ }
+ const latestState = left.republishCount >= right.republishCount ? left : right
+ const sourcePartitions = [...new Set([...(left.sourcePartitions ?? []), ...(right.sourcePartitions ?? [])])].sort(
+ (leftPartition, rightPartition) => leftPartition - rightPartition
+ )
+ return {
+ ...preferredRoute,
+ remainingHops: Math.min(left.remainingHops, right.remainingHops),
+ notBeforeMs: Math.max(left.notBeforeMs, right.notBeforeMs),
+ firstSeenAtMs: Math.min(left.firstSeenAtMs, right.firstSeenAtMs),
+ fetchCount: Math.max(left.fetchCount, right.fetchCount),
+ republishCount: Math.max(left.republishCount, right.republishCount),
+ lastRepublishReason: latestState.lastRepublishReason,
+ sourcePartitions: sourcePartitions.length > 0 ? sourcePartitions : undefined,
+ }
+}
diff --git a/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/fetch-runner.test.ts b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/fetch-runner.test.ts
new file mode 100644
index 000000000000..1e27daa0cbb1
--- /dev/null
+++ b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/fetch-runner.test.ts
@@ -0,0 +1,919 @@
+import { S3ServiceException } from '@aws-sdk/client-s3'
+import { createHash } from 'node:crypto'
+
+import { logger } from '~/common/utils/logger'
+import { RecordedTopHogMetric, createRecordingTopHog } from '~/tests/helpers/tophog'
+
+import { FetchCandidate, MAX_HOPS } from './collected-urls-record'
+import { ConfigurationPolicyService, OriginPolicyDecision } from './configuration-policy'
+import { ConfigurationCacheItem, CrawlHistoryItem, HttpCacheMetadata } from './crawl-history'
+import { DELAY_TOO_LONG, FetchRunner, FetchRunnerOptions, HOPS_EXHAUSTED } from './fetch-runner'
+import { FrontierPublisher, RepublishResult } from './frontier-publisher'
+import { HostBudget } from './host-budget'
+import { ImageFetchOptions, ImageFetchResult, ImageFetcher } from './image-fetcher'
+import { ImageFetchRequestMetrics } from './metrics'
+import { OriginRequestScheduler } from './origin-request-scheduler'
+import { ResponseUrlBinding } from './response-url-binding'
+import { ImageFetchTopHogMetrics } from './tophog-metrics'
+
+const NOW_MS = 1_700_000_000_000
+const OPTIONS: FetchRunnerOptions = {
+ maxConcurrentPerRegistrableDomain: 2,
+ maxInFlightRequests: 50,
+ batchBudgetMs: 20_000,
+ maxBytes: 20 * 1024 * 1024,
+ requestTimeoutMs: 10_000,
+ maxRedirects: 3,
+ seenTtlSeconds: 30 * 24 * 60 * 60,
+}
+const POOL_OPTIONS: FetchRunnerOptions = {
+ ...OPTIONS,
+ continuousPool: { refillRunnableUrls: 100, maxQueuedUrlsPerOwner: 1_000 },
+}
+const builtRunners: FetchRunner[] = []
+const bind: ResponseUrlBinding = (url) => createHash('sha256').update(url).digest('base64url').slice(0, 22)
+
+function candidate(overrides: Partial<FetchCandidate> = {}): FetchCandidate {
+ return {
+ originalRef: `imageurl:${'a'.repeat(22)}`,
+ currentUrl: 'https://cdn.example.com/a.png',
+ host: 'cdn.example.com',
+ origin: 'https://cdn.example.com',
+ registrableDomain: 'example.com',
+ remainingHops: MAX_HOPS,
+ notBeforeMs: 0,
+ firstSeenAtMs: NOW_MS - 1_000,
+ fetchCount: 0,
+ republishCount: 0,
+ lastRepublishReason: null,
+ ...overrides,
+ }
+}
+
+interface Harness {
+ runner: FetchRunner
+ budget: HostBudget
+ fetch: jest.Mock<Promise<ImageFetchResult>, [string, ImageFetchOptions]>
+ check: jest.Mock<Promise<OriginPolicyDecision>, [string, Map<string, ConfigurationCacheItem>, number]>
+ createPass: jest.Mock
+ republish: jest.Mock<Promise<RepublishResult>, any[]>
+ publishImage: jest.Mock<Promise<void>, any[]>
+ topHogRecords: Map<string, RecordedTopHogMetric[]>
+}
+
+function build(
+ result: Partial<ImageFetchResult> = {},
+ policy: Partial<OriginPolicyDecision> = {},
+ republishResult: RepublishResult = 'queued',
+ options: FetchRunnerOptions = OPTIONS
+): Harness {
+ const fetch = jest.fn((url: string, _options: ImageFetchOptions) =>
+ Promise.resolve({ outcome: 'ok', redirects: 0, currentUrl: url, ...result } as ImageFetchResult)
+ )
+ const check = jest.fn((_url: string, _cached: Map<string, ConfigurationCacheItem>, _nowMs: number) =>
+ Promise.resolve({
+ allowed: true,
+ transient: false,
+ crawlDelayMs: 1_000,
+ tdmrepReservation: false,
+ updates: [],
+ ...policy,
+ } as OriginPolicyDecision)
+ )
+ const createPass = jest.fn(() => ({ check }))
+ const republish = jest.fn(() => Promise.resolve(republishResult))
+ const createRepublishBatch = jest.fn(() => ({ republish, flush: () => Promise.resolve({ failedUrls: 0 }) }))
+ const publishImage = jest.fn(() => Promise.resolve())
+ const scheduler = {
+ runImage: async (_url: URL, _deadlineMs: number, request: () => Promise<ImageFetchResult>) => ({
+ ran: true as const,
+ value: await request(),
+ }),
+ authorizeImageRedirect: () => Promise.resolve(true),
+ } as unknown as OriginRequestScheduler
+ const budget = new HostBudget({
+ requestsPerSecond: 1,
+ burst: 5,
+ maxConcurrent: 6,
+ breakerFailures: 5,
+ breakerCooldownMs: 60_000,
+ breakerMaxCooldownMs: 3_600_000,
+ maxTrackedRegistrableDomains: 20_000,
+ maxTrackedOrigins: 20_000,
+ random: () => 0,
+ })
+ const recordingTopHog = createRecordingTopHog()
+ const runner = new FetchRunner(
+ { fetch } as ImageFetcher,
+ budget,
+ scheduler,
+ { createPass } as unknown as ConfigurationPolicyService,
+ options,
+ { createRepublishBatch } as unknown as FrontierPublisher,
+ { refreshIntervalMs: 7 * 24 * 60 * 60 * 1000, publishImage, refreshImage: () => Promise.resolve(true) },
+ { bindingFor: () => Promise.resolve(bind) },
+ new ImageFetchTopHogMetrics(recordingTopHog.registry)
+ )
+ builtRunners.push(runner)
+ return { runner, budget, fetch, check, createPass, republish, publishImage, topHogRecords: recordingTopHog.records }
+}
+
+describe('FetchRunner', () => {
+ beforeEach(() => jest.useFakeTimers().setSystemTime(NOW_MS))
+ afterEach(async () => {
+ await Promise.all(builtRunners.splice(0).map((runner) => runner.close()))
+ jest.useRealTimers()
+ jest.restoreAllMocks()
+ })
+
+ it('passes cached validators and TDM state to the image request', async () => {
+ const harness = build({}, { tdmrepReservation: true })
+ const cache: HttpCacheMetadata = {
+ requestTimeMs: NOW_MS - 10,
+ responseTimeMs: NOW_MS,
+ responseUrlHmac: bind(candidate().currentUrl),
+ etag: '"image-v1"',
+ }
+ const stored = new Map<string, CrawlHistoryItem>([
+ [
+ candidate().originalRef,
+ {
+ kind: 'url',
+ key: candidate().originalRef,
+ nextFetchAtMs: NOW_MS,
+ storageExpiresAtMs: NOW_MS,
+ outcome: 'ok',
+ cache,
+ },
+ ],
+ ])
+
+ await harness.runner.run([candidate({ sourcePartitions: [7, 42] })], stored)
+
+ expect(harness.fetch.mock.calls[0][1]).toMatchObject({
+ maxBytes: OPTIONS.maxBytes,
+ maxRedirects: OPTIONS.maxRedirects,
+ sourcePartitions: [7, 42],
+ cache,
+ tdmrepReservation: true,
+ })
+ })
+
+ it.each([
+ ['another URL', bind('https://other.example.net/a.png')],
+ ['no URL', undefined],
+ ])('sends no validator that %s issued, and binds the response to the URL that sent it', async (_name, binding) => {
+ const harness = build({ cache: { requestTimeMs: NOW_MS - 10, responseTimeMs: NOW_MS, etag: '"image-v2"' } })
+ const stored = new Map<string, CrawlHistoryItem>([
+ [
+ candidate().originalRef,
+ {
+ kind: 'url',
+ key: candidate().originalRef,
+ nextFetchAtMs: NOW_MS,
+ storageExpiresAtMs: NOW_MS,
+ outcome: 'ok',
+ cache: {
+ requestTimeMs: NOW_MS - 10,
+ responseTimeMs: NOW_MS,
+ responseUrlHmac: binding,
+ etag: '"v1"',
+ },
+ },
+ ],
+ ])
+
+ const [attempt] = await harness.runner.run([candidate()], stored)
+
+ expect(harness.fetch.mock.calls[0][1].cache).toBeUndefined()
+ expect(attempt.history?.cache).toMatchObject({
+ responseUrlHmac: bind(candidate().currentUrl),
+ etag: '"image-v2"',
+ })
+ })
+
+ it.each([
+ ['a fetch pass', OPTIONS],
+ ['the pod candidate pool', POOL_OPTIONS],
+ ])('publishes an accepted image and records a terminal URL result through %s', async (_mode, options) => {
+ const bytes = Buffer.from('image')
+ const harness = build({ bytes, contentType: 'image/png' }, {}, 'queued', options)
+
+ const [attempt] = await harness.runner.run([candidate()], new Map())
+
+ expect(harness.publishImage).toHaveBeenCalledWith(
+ expect.objectContaining({ originalRef: candidate().originalRef }),
+ expect.objectContaining({ bytes })
+ )
+ expect(attempt).toMatchObject({
+ outcome: 'ok',
+ finished: true,
+ lost: false,
+ history: { kind: 'url', key: candidate().originalRef, nextFetchAtMs: NOW_MS + 30 * 24 * 60 * 60 * 1000 },
+ })
+ })
+
+ it('uses one worker limit across sibling origins', async () => {
+ const harness = build({}, {}, 'queued', { ...OPTIONS, maxConcurrentPerRegistrableDomain: 1 })
+ let releaseFirst: (() => void) | undefined
+ harness.fetch.mockImplementationOnce(
+ (url: string) =>
+ new Promise<ImageFetchResult>((resolve) => {
+ releaseFirst = () => resolve({ outcome: 'ok', redirects: 0, currentUrl: url })
+ })
+ )
+ const first = candidate({ sourcePartitions: [110] })
+ const sibling = candidate({
+ originalRef: `imageurl:${'b'.repeat(22)}`,
+ currentUrl: 'https://images.example.com/b.png',
+ host: 'images.example.com',
+ origin: 'https://images.example.com',
+ sourcePartitions: [110],
+ })
+
+ const run = harness.runner.run([first, sibling], new Map())
+ await jest.advanceTimersByTimeAsync(0)
+
+ expect(harness.fetch).toHaveBeenCalledTimes(1)
+ expect(releaseFirst).toBeDefined()
+ releaseFirst?.()
+ await run
+ expect(harness.fetch).toHaveBeenCalledTimes(2)
+ expect(harness.topHogRecords.get('ai_research_v4_image_fetch_block_events_by_registrable_domain')).toEqual([
+ {
+ key: { registrable_domain: 'example.com', reason: 'domain_concurrency', partition: '110' },
+ value: 1,
+ },
+ ])
+ })
+
+ it('starts the next request for a registrable domain while an earlier image publish waits', async () => {
+ const harness = build({ bytes: Buffer.from('image'), contentType: 'image/png' }, {}, 'queued', {
+ ...OPTIONS,
+ maxConcurrentPerRegistrableDomain: 1,
+ })
+ let releasePublish: () => void = () => undefined
+ harness.publishImage.mockImplementationOnce(
+ () =>
+ new Promise<void>((resolve) => {
+ releasePublish = resolve
+ })
+ )
+ const second = candidate({
+ originalRef: `imageurl:${'b'.repeat(22)}`,
+ currentUrl: 'https://cdn.example.com/b.png',
+ })
+
+ let passFinished = false
+ const run = harness.runner.run([candidate(), second], new Map()).finally(() => {
+ passFinished = true
+ })
+ await jest.advanceTimersByTimeAsync(0)
+
+ expect(harness.fetch.mock.calls.map(([url]) => url)).toEqual([candidate().currentUrl, second.currentUrl])
+ expect(passFinished).toBe(false)
+ releasePublish()
+ const attempts = await run
+ expect(attempts.map((attempt) => [attempt.candidate.originalRef, attempt.outcome, attempt.lost])).toEqual(
+ expect.arrayContaining([
+ [candidate().originalRef, 'ok', false],
+ [second.originalRef, 'ok', false],
+ ])
+ )
+ expect(attempts).toHaveLength(2)
+ })
+
+ it('allocates sibling-origin workers by queue share', async () => {
+ const harness = build({}, {}, 'queued', { ...OPTIONS, maxConcurrentPerRegistrableDomain: 2 })
+ let releaseFirst: (() => void) | undefined
+ let releaseSecond: (() => void) | undefined
+ harness.fetch
+ .mockImplementationOnce(
+ (url: string) =>
+ new Promise<ImageFetchResult>((resolve) => {
+ releaseFirst = () => resolve({ outcome: 'ok', redirects: 0, currentUrl: url })
+ })
+ )
+ .mockImplementationOnce(
+ (url: string) =>
+ new Promise<ImageFetchResult>((resolve) => {
+ releaseSecond = () => resolve({ outcome: 'ok', redirects: 0, currentUrl: url })
+ })
+ )
+ const sameOrigin = candidate({ originalRef: `imageurl:${'b'.repeat(22)}` })
+ const siblingOrigin = candidate({
+ originalRef: `imageurl:${'c'.repeat(22)}`,
+ currentUrl: 'https://images.example.com/c.png',
+ host: 'images.example.com',
+ origin: 'https://images.example.com',
+ })
+
+ const run = harness.runner.run([candidate(), sameOrigin, siblingOrigin], new Map())
+ await jest.advanceTimersByTimeAsync(0)
+
+ expect(harness.fetch.mock.calls.map(([url]) => url)).toEqual([candidate().currentUrl, siblingOrigin.currentUrl])
+ releaseFirst?.()
+ releaseSecond?.()
+ await run
+ expect(harness.fetch).toHaveBeenCalledTimes(3)
+ })
+
+ it('records initial capacity after live pod and registrable-domain limits', async () => {
+ const observeCapacity = jest
+ .spyOn(ImageFetchRequestMetrics, 'observeBatchSchedulableCapacity')
+ .mockImplementation()
+ const harness = build({}, {}, 'queued', {
+ ...OPTIONS,
+ maxConcurrentPerRegistrableDomain: 2,
+ maxInFlightRequests: 3,
+ })
+ const candidates = [
+ candidate({ sourcePartitions: [110] }),
+ ...Array.from({ length: 2 }, (_, index) =>
+ candidate({
+ originalRef: `imageurl:${String(index + 1).repeat(22)}`,
+ currentUrl: `https://cdn.example.com/image-${index}.png`,
+ sourcePartitions: [110],
+ })
+ ),
+ ...Array.from({ length: 2 }, (_, index) =>
+ candidate({
+ originalRef: `imageurl:${String(index + 3).repeat(22)}`,
+ currentUrl: `https://origin-${index}.other.net/image.png`,
+ host: `origin-${index}.other.net`,
+ origin: `https://origin-${index}.other.net`,
+ registrableDomain: 'other.net',
+ sourcePartitions: [42],
+ })
+ ),
+ ]
+ const grant = harness.budget.take('example.com', 'https://cdn.example.com', NOW_MS, NOW_MS + 10_000, true)
+ expect(grant).toMatchObject({ granted: true, waitMs: 0 })
+ expect(harness.budget.acquireConnection('example.com', 'https://cdn.example.com')).toBe(true)
+ harness.budget.markRequestStarted(
+ 'example.com',
+ 'https://cdn.example.com',
+ NOW_MS,
+ grant.granted ? grant.reservedStartAtMs : null,
+ 'configuration'
+ )
+
+ await harness.runner.run(candidates, new Map())
+
+ expect(observeCapacity).toHaveBeenCalledWith(3, 3)
+ expect(harness.topHogRecords.get('ai_research_v4_image_fetch_block_events_by_registrable_domain')).toEqual([
+ {
+ key: { registrable_domain: 'example.com', reason: 'domain_concurrency', partition: '110' },
+ value: 2,
+ },
+ ])
+ harness.budget.releaseConnection('example.com', 'https://cdn.example.com')
+ })
+
+ it('keeps one request slot on the largest origin queue', async () => {
+ const harness = build({}, {}, 'queued', {
+ ...OPTIONS,
+ maxConcurrentPerRegistrableDomain: 1,
+ maxInFlightRequests: 1,
+ })
+ const small = candidate({
+ currentUrl: 'https://small.example.com/a.png',
+ host: 'small.example.com',
+ origin: 'https://small.example.com',
+ })
+ const largeFirst = candidate({
+ originalRef: `imageurl:${'b'.repeat(22)}`,
+ currentUrl: 'https://large.example.com/a.png',
+ host: 'large.example.com',
+ origin: 'https://large.example.com',
+ })
+ const largeSecond = candidate({
+ originalRef: `imageurl:${'c'.repeat(22)}`,
+ currentUrl: 'https://large.example.com/b.png',
+ host: 'large.example.com',
+ origin: 'https://large.example.com',
+ })
+
+ await harness.runner.run([small, largeFirst, largeSecond], new Map())
+
+ expect(harness.fetch.mock.calls.map(([url]) => url)).toEqual([
+ largeFirst.currentUrl,
+ largeSecond.currentUrl,
+ small.currentUrl,
+ ])
+ })
+
+ it('processes a low-capacity tail instead of republishing it for more diversity', async () => {
+ const harness = build({}, {}, 'queued', {
+ ...OPTIONS,
+ maxInFlightRequests: 1,
+ })
+ const candidates = Array.from({ length: 4 }, (_, index) => {
+ const origin = `https://cdn-${index}.example.com`
+ return candidate({
+ originalRef: `imageurl:${index.toString().padStart(22, '0')}`,
+ currentUrl: `${origin}/${index}.png`,
+ host: `cdn-${index}.example.com`,
+ origin,
+ })
+ })
+
+ const attempts = await harness.runner.run(candidates, new Map())
+
+ expect(harness.fetch).toHaveBeenCalledTimes(4)
+ expect(harness.republish).not.toHaveBeenCalled()
+ expect(attempts).toHaveLength(4)
+ })
+
+ it('keeps the pod request limit across overlapping passes', async () => {
+ const observeCapacity = jest
+ .spyOn(ImageFetchRequestMetrics, 'observeBatchSchedulableCapacity')
+ .mockImplementation()
+ const harness = build({}, {}, 'queued', { ...OPTIONS, maxInFlightRequests: 1 })
+ const releases: Array<() => void> = []
+ let signalSecondFetchStarted: () => void = () => undefined
+ const secondFetchStarted = new Promise<void>((resolve) => {
+ signalSecondFetchStarted = resolve
+ })
+ harness.fetch.mockImplementation(
+ (url: string) =>
+ new Promise<ImageFetchResult>((resolve) => {
+ releases.push(() => resolve({ outcome: 'ok', redirects: 0, currentUrl: url }))
+ if (releases.length === 2) {
+ signalSecondFetchStarted()
+ }
+ })
+ )
+ const otherDomain = candidate({
+ originalRef: `imageurl:${'b'.repeat(22)}`,
+ currentUrl: 'https://cdn.other.net/b.png',
+ host: 'cdn.other.net',
+ origin: 'https://cdn.other.net',
+ registrableDomain: 'other.net',
+ })
+
+ const firstRun = harness.runner.run([candidate()], new Map())
+ const secondRun = harness.runner.run([otherDomain], new Map())
+ await jest.advanceTimersByTimeAsync(0)
+
+ expect(harness.fetch).toHaveBeenCalledTimes(1)
+ expect(observeCapacity.mock.calls).toEqual([
+ [1, 1],
+ [0, 1],
+ ])
+ releases[0]()
+ await secondFetchStarted
+ expect(harness.fetch).toHaveBeenCalledTimes(2)
+ releases[1]()
+ await Promise.all([firstRun, secondRun])
+ })
+
+ it('finishes a pooled batch with queued candidates before close stops the pool', async () => {
+ const harness = build({}, {}, 'queued', { ...POOL_OPTIONS, maxInFlightRequests: 1 })
+ let releaseFirst: () => void = () => undefined
+ harness.fetch.mockImplementationOnce(
+ (url: string) =>
+ new Promise<ImageFetchResult>((resolve) => {
+ releaseFirst = () => resolve({ outcome: 'ok', redirects: 0, currentUrl: url })
+ })
+ )
+ const queued = candidate({
+ originalRef: `imageurl:${'b'.repeat(22)}`,
+ currentUrl: 'https://cdn.other.net/b.png',
+ host: 'cdn.other.net',
+ origin: 'https://cdn.other.net',
+ registrableDomain: 'other.net',
+ })
+
+ const run = harness.runner.run([candidate(), queued], new Map())
+ await jest.advanceTimersByTimeAsync(0)
+ let closed = false
+ const close = harness.runner.close().then(() => {
+ closed = true
+ })
+ await jest.advanceTimersByTimeAsync(0)
+ expect(closed).toBe(false)
+ releaseFirst()
+ await jest.advanceTimersByTimeAsync(0)
+
+ expect((await run).map((attempt) => attempt.candidate.currentUrl)).toEqual([
+ candidate().currentUrl,
+ queued.currentUrl,
+ ])
+ await close
+ })
+
+ it('shares one registrable domain limit between batches in the pod candidate pool', async () => {
+ const harness = build({}, {}, 'queued', POOL_OPTIONS)
+ const releases: Array<() => void> = []
+ harness.fetch.mockImplementation(
+ (url: string) =>
+ new Promise<ImageFetchResult>((resolve) => {
+ releases.push(() => resolve({ outcome: 'ok', redirects: 0, currentUrl: url }))
+ })
+ )
+ const onOrigin = (label: string): FetchCandidate =>
+ candidate({
+ originalRef: `imageurl:${label.repeat(22)}`,
+ currentUrl: `https://${label}.example.com/a.png`,
+ host: `${label}.example.com`,
+ origin: `https://${label}.example.com`,
+ })
+
+ const firstBatch = harness.runner.run([onOrigin('a'), onOrigin('b')], new Map())
+ const secondBatch = harness.runner.run([onOrigin('c'), onOrigin('d')], new Map())
+ await jest.advanceTimersByTimeAsync(0)
+
+ expect(harness.fetch).toHaveBeenCalledTimes(2)
+ releases.splice(0).forEach((release) => release())
+ await jest.advanceTimersByTimeAsync(0)
+ expect(harness.fetch).toHaveBeenCalledTimes(4)
+ releases.splice(0).forEach((release) => release())
+ await Promise.all([firstBatch, secondBatch])
+ })
+
+ it('does not start a queued request after another worker aborts its pass', async () => {
+ const harness = build({}, {}, 'queued', { ...OPTIONS, maxInFlightRequests: 2 })
+ let releaseBlocker: () => void = () => undefined
+ let signalBlockerStarted: () => void = () => undefined
+ const blockerStarted = new Promise<void>((resolve) => {
+ signalBlockerStarted = resolve
+ })
+ harness.fetch.mockImplementation((url: string) => {
+ if (url.includes('blocker')) {
+ signalBlockerStarted()
+ return new Promise<ImageFetchResult>((resolve) => {
+ releaseBlocker = () => resolve({ outcome: 'ok', redirects: 0, currentUrl: url })
+ })
+ }
+ if (url.includes('failure')) {
+ return Promise.reject(new Error('dependency unavailable'))
+ }
+ return Promise.resolve({ outcome: 'ok', redirects: 0, currentUrl: url })
+ })
+ const blocker = candidate({ currentUrl: 'https://blocker.example.com/a.png' })
+ const failure = candidate({
+ originalRef: `imageurl:${'b'.repeat(22)}`,
+ currentUrl: 'https://failure.example.com/b.png',
+ host: 'failure.example.com',
+ origin: 'https://failure.example.com',
+ })
+ const mustNotStart = candidate({
+ originalRef: `imageurl:${'c'.repeat(22)}`,
+ currentUrl: 'https://queued.example.com/c.png',
+ host: 'queued.example.com',
+ origin: 'https://queued.example.com',
+ })
+
+ const blockingRun = harness.runner.run([blocker], new Map())
+ await blockerStarted
+ const failingRun = harness.runner.run([failure, mustNotStart], new Map())
+
+ await expect(failingRun).rejects.toThrow('dependency unavailable')
+ expect(harness.fetch.mock.calls.map(([url]) => url)).toEqual([blocker.currentUrl, failure.currentUrl])
+ releaseBlocker()
+ await blockingRun
+ })
+
+ it.each([
+ ['a fetch pass', OPTIONS],
+ ['the pod candidate pool', POOL_OPTIONS],
+ ])('stops taking queue work after a fatal candidate error in %s', async (_mode, options) => {
+ const harness = build({}, {}, 'queued', { ...options, maxInFlightRequests: 1 })
+ harness.fetch.mockRejectedValueOnce(new Error('dependency unavailable'))
+ const second = candidate({ originalRef: `imageurl:${'b'.repeat(22)}` })
+
+ await expect(harness.runner.run([candidate(), second], new Map())).rejects.toThrow('dependency unavailable')
+
+ expect(harness.fetch).toHaveBeenCalledTimes(1)
+ })
+
+ it('deduplicates canonical refs before queue scheduling', async () => {
+ const harness = build()
+ const duplicate = candidate({ republishCount: 1, lastRepublishReason: 'retry' })
+
+ const attempts = await harness.runner.run([candidate(), duplicate], new Map())
+
+ expect(harness.fetch).toHaveBeenCalledTimes(1)
+ expect(attempts).toHaveLength(1)
+ expect(attempts[0].candidate).toMatchObject({ republishCount: 1, lastRepublishReason: 'retry' })
+ })
+
+ it.each([
+ ['a fetch pass', OPTIONS],
+ ['the pod candidate pool', POOL_OPTIONS],
+ ])('marks an image publish failure as lost after all candidate work settles in %s', async (_mode, options) => {
+ const harness = build({ bytes: Buffer.from('image'), contentType: 'image/png' }, {}, 'queued', options)
+ harness.publishImage.mockRejectedValue(
+ new S3ServiceException({
+ name: 'SlowDown',
+ $fault: 'server',
+ $metadata: { httpStatusCode: 503 },
+ message: 'images/url/month=2026-10/7/secret-key',
+ })
+ )
+ const warn = jest.spyOn(logger, 'warn')
+
+ const attempts = await harness.runner.run(
+ [candidate(), candidate({ originalRef: `imageurl:${'b'.repeat(22)}` })],
+ new Map()
+ )
+
+ expect(attempts).toEqual([
+ expect.objectContaining({ outcome: 'publish_failed', finished: false, lost: true }),
+ expect.objectContaining({ outcome: 'publish_failed', finished: false, lost: true }),
+ ])
+ // One log for the burst, with the error name and status and nothing that names the image.
+ const logged = warn.mock.calls.filter(([, event]) => event === 'ai_research_v4_image_fetch_storage_failed')
+ expect(logged).toEqual([
+ [
+ '🌐',
+ 'ai_research_v4_image_fetch_storage_failed',
+ { operation: 'publish', error: 'SlowDown', code: undefined, httpStatusCode: 503, failures: 1 },
+ ],
+ ])
+ warn.mockRestore()
+ })
+
+ it('writes a terminal policy refusal without opening an image socket', async () => {
+ const harness = build({}, { allowed: false, transient: false, reason: 'robots_disallow' })
+
+ const [attempt] = await harness.runner.run([candidate()], new Map())
+
+ expect(harness.fetch).not.toHaveBeenCalled()
+ expect(attempt).toMatchObject({ outcome: 'robots_disallow', finished: true })
+ })
+
+ it('delays a transient configuration failure without writing URL history', async () => {
+ const harness = build({}, { allowed: false, transient: true, reason: 'configuration_unreachable' })
+
+ const [attempt] = await harness.runner.run([candidate()], new Map())
+
+ expect(harness.republish).toHaveBeenCalledWith(
+ expect.objectContaining({
+ originalRef: candidate().originalRef,
+ lastBlockReason: 'configuration_unreachable',
+ }),
+ {
+ currentUrl: candidate().currentUrl,
+ host: candidate().host,
+ origin: candidate().origin,
+ registrableDomain: candidate().registrableDomain,
+ },
+ 'not_ready',
+ 3_600_000
+ )
+ expect(attempt).toMatchObject({
+ outcome: 'backoff',
+ finished: false,
+ lost: false,
+ block: { reason: 'configuration_unreachable', waitMs: 3_600_000 },
+ })
+ expect(attempt.history).toBeUndefined()
+ })
+
+ it('keeps the effective URL and charges every request before retrying', async () => {
+ const harness = build({
+ outcome: 'server_error',
+ status: 503,
+ redirects: 2,
+ currentUrl: 'https://cdn.example.com/final.png',
+ retryAfterMs: 120_000,
+ })
+ const [attempt] = await harness.runner.run([candidate()], new Map())
+
+ expect(harness.republish).toHaveBeenCalledWith(
+ expect.objectContaining({
+ currentUrl: 'https://cdn.example.com/final.png',
+ remainingHops: MAX_HOPS - 2,
+ fetchCount: 3,
+ }),
+ expect.objectContaining({ currentUrl: 'https://cdn.example.com/final.png' }),
+ 'retry',
+ 120_000
+ )
+ expect(attempt).toMatchObject({
+ outcome: 'server_error',
+ finished: false,
+ block: { reason: 'retry_after', waitMs: 120_000 },
+ })
+ })
+
+ it('republishes an unfollowed redirect target with the original ref', async () => {
+ const harness = build({
+ outcome: 'redirect_offsite',
+ status: 302,
+ currentUrl: candidate().currentUrl,
+ redirectTarget: { url: 'https://img.other.net/a.png', host: 'img.other.net' },
+ })
+
+ const [attempt] = await harness.runner.run([candidate()], new Map())
+
+ expect(harness.republish).toHaveBeenCalledWith(
+ expect.objectContaining({ originalRef: candidate().originalRef, fetchCount: 1 }),
+ {
+ currentUrl: 'https://img.other.net/a.png',
+ host: 'img.other.net',
+ origin: 'https://img.other.net',
+ registrableDomain: 'other.net',
+ },
+ 'redirect',
+ 0
+ )
+ expect(attempt).toMatchObject({ outcome: 'redirect_offsite', finished: false })
+ })
+
+ it('records a terminal refusal from a same-origin redirect policy check', async () => {
+ const harness = build({
+ outcome: 'redirect_policy_refused',
+ redirects: 1,
+ currentUrl: 'https://cdn.example.com/private/a.png',
+ refusalReason: 'robots_disallow',
+ policyTransient: false,
+ })
+
+ const [attempt] = await harness.runner.run([candidate()], new Map())
+
+ expect(harness.republish).not.toHaveBeenCalled()
+ expect(attempt).toMatchObject({
+ outcome: 'robots_disallow',
+ finished: true,
+ candidate: { remainingHops: MAX_HOPS - 1, fetchCount: 1 },
+ })
+ })
+
+ it('delays a transient same-origin redirect policy failure without spending another hop', async () => {
+ const harness = build({
+ outcome: 'redirect_policy_refused',
+ redirects: 1,
+ currentUrl: 'https://cdn.example.com/next/a.png',
+ refusalReason: 'configuration_unreachable',
+ policyTransient: true,
+ })
+
+ await harness.runner.run([candidate()], new Map())
+
+ expect(harness.republish).toHaveBeenCalledWith(
+ expect.objectContaining({
+ currentUrl: 'https://cdn.example.com/next/a.png',
+ remainingHops: MAX_HOPS - 1,
+ fetchCount: 1,
+ }),
+ expect.any(Object),
+ 'not_ready',
+ 3_600_000
+ )
+ })
+
+ it('persists a crawl wait that extends beyond the current pass', async () => {
+ const harness = build({
+ outcome: 'request_deferred',
+ schedulingReason: 'deadline',
+ schedulingWaitMs: 600_000,
+ })
+
+ await harness.runner.run([candidate()], new Map())
+
+ expect(harness.republish).toHaveBeenCalledWith(expect.any(Object), expect.any(Object), 'not_ready', 600_000)
+ })
+
+ it.each([
+ ['a fetch pass', OPTIONS],
+ ['the pod candidate pool', POOL_OPTIONS],
+ ])('returns pass-deadline work directly to the frontier from %s', async (_mode, options) => {
+ const harness = build({}, {}, 'queued', { ...options, batchBudgetMs: -1 })
+
+ await harness.runner.run([candidate()], new Map())
+
+ expect(harness.republish).toHaveBeenCalledWith(expect.any(Object), expect.any(Object), 'pass_deadline', 0)
+ })
+
+ it('finishes a retry when its last hop was the failed request', async () => {
+ const harness = build({ outcome: 'timeout' })
+
+ const [attempt] = await harness.runner.run([candidate({ remainingHops: 1 })], new Map())
+
+ expect(harness.republish).not.toHaveBeenCalled()
+ expect(attempt).toMatchObject({ outcome: HOPS_EXHAUSTED, finished: true })
+ })
+
+ it('does not start policy or image requests when the hop budget is empty', async () => {
+ const harness = build()
+
+ const [attempt] = await harness.runner.run([candidate({ remainingHops: 0 })], new Map())
+
+ expect(harness.check).not.toHaveBeenCalled()
+ expect(harness.fetch).not.toHaveBeenCalled()
+ expect(attempt).toMatchObject({ outcome: HOPS_EXHAUSTED, finished: true })
+ })
+
+ it('records a terminal refusal when no delay topic can hold the wait', async () => {
+ const harness = build({ outcome: 'timeout' }, {}, 'refused_delay')
+ const [attempt] = await harness.runner.run([candidate()], new Map())
+
+ expect(attempt).toMatchObject({ outcome: DELAY_TOO_LONG, finished: true })
+ })
+
+ it('merges a 304 response into the previous cache metadata', async () => {
+ const previousCache: HttpCacheMetadata = {
+ requestTimeMs: NOW_MS - 2_000,
+ responseTimeMs: NOW_MS - 1_000,
+ etag: '"v1"',
+ lastModified: 'yesterday',
+ }
+ const harness = build({
+ outcome: 'not_modified',
+ status: 304,
+ cache: { requestTimeMs: NOW_MS - 10, responseTimeMs: NOW_MS, etag: '"v2"' },
+ })
+ const stored = new Map<string, CrawlHistoryItem>([
+ [
+ candidate().originalRef,
+ {
+ kind: 'url',
+ key: candidate().originalRef,
+ nextFetchAtMs: NOW_MS,
+ storageExpiresAtMs: NOW_MS,
+ outcome: 'ok',
+ cache: previousCache,
+ },
+ ],
+ ])
+
+ const [attempt] = await harness.runner.run([candidate()], stored)
+
+ expect(attempt.history?.cache).toMatchObject({ etag: '"v2"', lastModified: 'yesterday' })
+ })
+
+ it('retains validators that a 304 response omits', async () => {
+ const previousCache: HttpCacheMetadata = {
+ requestTimeMs: NOW_MS - 2_000,
+ responseTimeMs: NOW_MS - 1_000,
+ etag: '"v1"',
+ lastModified: 'yesterday',
+ }
+ const harness = build({
+ outcome: 'not_modified',
+ status: 304,
+ cache: { requestTimeMs: NOW_MS - 10, responseTimeMs: NOW_MS },
+ })
+ const stored = new Map<string, CrawlHistoryItem>([
+ [
+ candidate().originalRef,
+ {
+ kind: 'url',
+ key: candidate().originalRef,
+ nextFetchAtMs: NOW_MS,
+ storageExpiresAtMs: NOW_MS,
+ outcome: 'ok',
+ cache: previousCache,
+ },
+ ],
+ ])
+
+ const [attempt] = await harness.runner.run([candidate()], stored)
+
+ expect(attempt.history?.cache).toMatchObject({
+ requestTimeMs: NOW_MS - 10,
+ responseTimeMs: NOW_MS,
+ etag: '"v1"',
+ lastModified: 'yesterday',
+ })
+ })
+
+ it('does not store a redirect target validator under the original URL ref', async () => {
+ const harness = build({
+ redirects: 1,
+ currentUrl: 'https://cdn.example.com/moved.png',
+ cache: {
+ requestTimeMs: NOW_MS - 10,
+ responseTimeMs: NOW_MS,
+ etag: '"redirect-target"',
+ lastModified: 'today',
+ cacheControl: 'max-age=60',
+ },
+ })
+
+ const [attempt] = await harness.runner.run([candidate()], new Map())
+
+ expect(attempt.history?.cache).toMatchObject({ cacheControl: 'max-age=60' })
+ expect(attempt.history?.cache?.etag).toBeUndefined()
+ expect(attempt.history?.cache?.lastModified).toBeUndefined()
+ })
+
+ it('extends URL history to the end of explicit freshness', async () => {
+ const fortyDaysMs = 40 * 24 * 60 * 60 * 1000
+ const harness = build({
+ cache: {
+ requestTimeMs: NOW_MS,
+ responseTimeMs: NOW_MS,
+ cacheControl: `s-maxage=${fortyDaysMs / 1000}`,
+ },
+ })
+
+ const [attempt] = await harness.runner.run([candidate()], new Map())
+
+ expect(attempt.history?.nextFetchAtMs).toBe(NOW_MS + fortyDaysMs)
+ expect(attempt.history?.storageExpiresAtMs).toBe(NOW_MS + fortyDaysMs)
+ })
+})
diff --git a/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/fetch-runner.ts b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/fetch-runner.ts
new file mode 100644
index 000000000000..90d631cfa175
--- /dev/null
+++ b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/fetch-runner.ts
@@ -0,0 +1,971 @@
+import { ConcurrencyController } from '~/common/utils/concurrencyController'
+import { logger } from '~/common/utils/logger'
+
+import type { ImageFetchBlockReason } from './block-reason'
+import { FetchCandidate, MAX_HOPS, RepublishReason } from './collected-urls-record'
+import { ConfigurationPolicyPass, ConfigurationPolicyService } from './configuration-policy'
+import { ConfigurationCacheItem, CrawlHistoryItem, HttpCacheMetadata, UrlCrawlHistoryItem } from './crawl-history'
+import {
+ FetchCandidatePool,
+ FetchCandidatePoolAdmission,
+ FetchCandidatePoolLimits,
+ PoolBatch,
+} from './fetch-candidate-pool'
+import { FetchCandidateLease, FetchCandidateQueue, deduplicateFetchCandidates } from './fetch-candidate-queue'
+import { FrontierPublisher, RepublishBatch, RepublishResult } from './frontier-publisher'
+import { HostBudget } from './host-budget'
+import {
+ FetchOutcome,
+ FetchRefusalReason,
+ ImageFetchResult,
+ ImageFetcher,
+ RequestScheduleBlockReason,
+ TransientFetchOutcome,
+} from './image-fetcher'
+import type { RefreshingImageSink } from './image-sink'
+import { ImageFetchPoolMetrics, ImageFetchRequestMetrics } from './metrics'
+import { OriginRequestScheduler } from './origin-request-scheduler'
+import { canonicalizeUrl } from './politeness-key'
+import { ImageFetchProcessingMetrics } from './processing-metrics'
+import type { ResponseUrlBinder } from './response-url-binding'
+import { ImageFetchTopHogMetrics } from './tophog-metrics'
+import { urlHistoryExpiresAtMs, urlHistoryStorageExpiresAtMs } from './url-history-expiry'
+
+export type ShedReason =
+ | 'breaker_open'
+ | 'backoff'
+ | 'deadline'
+ | 'connection_limit'
+ | 'origin_map_full'
+ | 'registrable_domain_map_full'
+export const HOPS_EXHAUSTED = 'hops_exhausted'
+export const DELAY_TOO_LONG = 'delay_too_long'
+export type AttemptOutcome =
+ | FetchOutcome
+ | FetchRefusalReason
+ | ShedReason
+ | 'publish_failed'
+ | typeof HOPS_EXHAUSTED
+ | typeof DELAY_TOO_LONG
+
+export interface FetchAttempt {
+ candidate: FetchCandidate
+ outcome: AttemptOutcome
+ finished: boolean
+ lost: boolean
+ block?: { reason: ImageFetchBlockReason; waitMs: number }
+ history?: UrlCrawlHistoryItem
+ configurationUpdates: ConfigurationCacheItem[]
+}
+
+export interface FetchRunnerOptions {
+ maxConcurrentPerRegistrableDomain: number
+ maxInFlightRequests: number
+ batchBudgetMs: number
+ maxBytes: number
+ requestTimeoutMs: number
+ maxRedirects: number
+ seenTtlSeconds: number
+ /** Set to run every batch through one pod-wide candidate pool instead of a fetch pass of its own. */
+ continuousPool?: FetchCandidatePoolLimits
+}
+
+const TRANSIENT_OUTCOMES = new Set<TransientFetchOutcome>(['timeout', 'error', 'rate_limited', 'server_error'])
+const ONE_MINUTE_MS = 60_000
+const CONFIGURATION_RETRY_MS = 60 * 60 * 1000
+const STORAGE_FAILURE_LOG_INTERVAL_MS = 10_000
+
+function requirePositive(name: string, value: number): void {
+ if (!Number.isFinite(value) || value <= 0) {
+ throw new Error(`${name} must be a positive number, got ${value}`)
+ }
+}
+
+function requirePositiveSafeInteger(name: string, value: number): void {
+ if (!Number.isSafeInteger(value) || value <= 0) {
+ throw new Error(`${name} must be a positive safe integer, got ${value}`)
+ }
+}
+
+export function isTransientOutcome(outcome: AttemptOutcome): outcome is TransientFetchOutcome {
+ return TRANSIENT_OUTCOMES.has(outcome as TransientFetchOutcome)
+}
+
+function isRequestStateFull(
+ reason: string | undefined
+): reason is Extract<RequestScheduleBlockReason, 'origin_map_full' | 'registrable_domain_map_full'> {
+ return reason === 'origin_map_full' || reason === 'registrable_domain_map_full'
+}
+
+export interface FetchPass {
+ run(
+ candidates: FetchCandidate[],
+ stored: Map<string, CrawlHistoryItem>,
+ republishBatch?: RepublishBatch,
+ admission?: FetchCandidatePoolAdmission
+ ): Promise<FetchAttempt[]>
+}
+
+interface FetchPassState {
+ failure?: { error: unknown }
+}
+
+interface PooledBatch {
+ stored: Map<string, CrawlHistoryItem>
+ configurationItems: Map<string, ConfigurationCacheItem>
+ configurationPolicy: ConfigurationPolicyPass
+ deadlineMs: number
+ republishBatch: RepublishBatch
+ state: FetchPassState
+ attempts: FetchAttempt[]
+ unsettled: number
+ poolBatch?: PoolBatch<PooledBatch>
+ settle: () => void
+}
+
+export class FetchRunner implements FetchPass {
+ private readonly candidateWork: ConcurrencyController
+ private readonly pool?: FetchCandidatePool<PooledBatch>
+ private poolWorkers?: Promise<void>[]
+ private readonly activePooledBatches = new Set<Promise<FetchAttempt[]>>()
+ private storageFailureLoggedAtMs = Number.NEGATIVE_INFINITY
+ private storageFailuresSinceLog = 0
+
+ constructor(
+ private readonly fetcher: ImageFetcher,
+ private readonly budget: HostBudget,
+ private readonly scheduler: OriginRequestScheduler,
+ private readonly configurationPolicy: ConfigurationPolicyService,
+ private readonly options: FetchRunnerOptions,
+ private readonly publisher: FrontierPublisher,
+ private readonly imageSink: RefreshingImageSink,
+ private readonly responseUrlBinder: ResponseUrlBinder,
+ private readonly topHogMetrics?: ImageFetchTopHogMetrics
+ ) {
+ requirePositiveSafeInteger(
+ 'AI_RESEARCH_V4_IMAGE_FETCH_MAX_CONCURRENT_PER_REGISTRABLE_DOMAIN',
+ options.maxConcurrentPerRegistrableDomain
+ )
+ requirePositiveSafeInteger('AI_RESEARCH_V4_IMAGE_FETCH_MAX_IN_FLIGHT_REQUESTS', options.maxInFlightRequests)
+ requirePositive('AI_RESEARCH_V4_IMAGE_FETCH_MAX_IMAGE_BYTES', options.maxBytes)
+ requirePositive('AI_RESEARCH_V4_IMAGE_FETCH_REQUEST_TIMEOUT_MS', options.requestTimeoutMs)
+ requirePositive('AI_RESEARCH_V4_IMAGE_FETCH_CRAWL_HISTORY_TTL_SECONDS', options.seenTtlSeconds)
+ if (!Number.isFinite(options.batchBudgetMs) || !Number.isFinite(options.maxRedirects)) {
+ throw new Error('image fetch pass limits must be finite numbers')
+ }
+ this.candidateWork = new ConcurrencyController(options.maxInFlightRequests)
+ ImageFetchRequestMetrics.trackBudget(budget, scheduler)
+ if (options.continuousPool) {
+ this.pool = new FetchCandidatePool<PooledBatch>(
+ {
+ ...options.continuousPool,
+ maxConcurrentPerRegistrableDomain: options.maxConcurrentPerRegistrableDomain,
+ },
+ budget
+ )
+ ImageFetchProcessingMetrics.trackQueue(this.pool)
+ }
+ }
+
+ public get candidatePool(): FetchCandidatePool<PooledBatch> | undefined {
+ return this.pool
+ }
+
+ public async run(
+ candidates: FetchCandidate[],
+ stored: Map<string, CrawlHistoryItem>,
+ republishBatch?: RepublishBatch,
+ admission?: FetchCandidatePoolAdmission
+ ): Promise<FetchAttempt[]> {
+ if (this.pool) {
+ const pooledBatch = this.runPooled(this.pool, candidates, stored, republishBatch, admission)
+ this.activePooledBatches.add(pooledBatch)
+ try {
+ return await pooledBatch
+ } finally {
+ this.activePooledBatches.delete(pooledBatch)
+ }
+ }
+ admission?.admitted()
+ return await this.runPass(candidates, stored, republishBatch)
+ }
+
+ /**
+ * Waits for every pooled batch before it stops the workers. The Kafka consumer drain on shutdown
+ * has a timeout, and a batch that outlives it still needs the workers to take its queued candidates.
+ */
+ public async close(): Promise<void> {
+ await Promise.allSettled(this.activePooledBatches)
+ this.pool?.close()
+ await Promise.all(this.poolWorkers ?? [])
+ }
+
+ private async runPass(
+ candidates: FetchCandidate[],
+ stored: Map<string, CrawlHistoryItem>,
+ republishBatch?: RepublishBatch
+ ): Promise<FetchAttempt[]> {
+ const activeRepublishBatch = republishBatch ?? this.publisher.createRepublishBatch()
+ const deadlineMs = Date.now() + this.options.batchBudgetMs
+ const configurationPolicy = this.configurationPolicy.createPass()
+ const passState: FetchPassState = {}
+ const configurationItems = configurationItemsFrom(stored)
+ const queue = new FetchCandidateQueue(candidates, this.options)
+ this.topHogMetrics?.recordConcurrencyLimitedUrls(candidates, (registrableDomain) =>
+ this.budget.availableConnections(registrableDomain, this.options.maxConcurrentPerRegistrableDomain)
+ )
+ const podRequestSlots = Math.max(0, this.options.maxInFlightRequests - this.candidateWork.running)
+ ImageFetchRequestMetrics.observeBatchSchedulableCapacity(
+ Math.min(
+ podRequestSlots,
+ queue.availableRequestSlotsAtStart((registrableDomain) =>
+ this.budget.availableConnections(registrableDomain, this.options.maxConcurrentPerRegistrableDomain)
+ )
+ ),
+ this.options.maxInFlightRequests
+ )
+ const attempts: FetchAttempt[] = []
+ const stopTrackingQueue = ImageFetchProcessingMetrics.trackQueue(queue)
+ const workers = Array.from(
+ { length: Math.min(this.options.maxInFlightRequests, queue.schedulableSlotsAtStart) },
+ () =>
+ this.runQueueWorker(
+ queue,
+ stored,
+ configurationItems,
+ configurationPolicy,
+ deadlineMs,
+ attempts,
+ activeRepublishBatch,
+ passState
+ )
+ )
+ const settledWorkers = await Promise.allSettled(workers).finally(stopTrackingQueue)
+ if (passState.failure) {
+ throw passState.failure.error
+ }
+ const failedWorker = settledWorkers.find(
+ (settled): settled is PromiseRejectedResult => settled.status === 'rejected'
+ )
+ if (failedWorker) {
+ throw failedWorker.reason
+ }
+ return await this.finishPass(attempts, activeRepublishBatch, !republishBatch)
+ }
+
+ private async runPooled(
+ pool: FetchCandidatePool<PooledBatch>,
+ candidates: FetchCandidate[],
+ stored: Map<string, CrawlHistoryItem>,
+ republishBatch: RepublishBatch | undefined,
+ admission: FetchCandidatePoolAdmission | undefined
+ ): Promise<FetchAttempt[]> {
+ const activeRepublishBatch = republishBatch ?? this.publisher.createRepublishBatch()
+ const { candidates: unique } = deduplicateFetchCandidates(candidates)
+ this.topHogMetrics?.recordConcurrencyLimitedUrls(unique, (registrableDomain) =>
+ this.budget.availableConnections(registrableDomain, this.options.maxConcurrentPerRegistrableDomain)
+ )
+ let settle: () => void = () => undefined
+ const settled = new Promise<void>((resolve) => {
+ settle = resolve
+ })
+ const batch: PooledBatch = {
+ stored,
+ configurationItems: configurationItemsFrom(stored),
+ configurationPolicy: this.configurationPolicy.createPass(),
+ deadlineMs: Date.now() + this.options.batchBudgetMs,
+ republishBatch: activeRepublishBatch,
+ state: {},
+ attempts: [],
+ unsettled: unique.length,
+ settle,
+ }
+ this.startPoolWorkers(pool)
+ try {
+ batch.poolBatch = pool.add(unique, batch, batch.deadlineMs, admission?.owner)
+ } finally {
+ admission?.admitted()
+ }
+ if (batch.unsettled === 0) {
+ settle()
+ }
+ await settled
+ if (batch.state.failure) {
+ throw batch.state.failure.error
+ }
+ return await this.finishPass(batch.attempts, activeRepublishBatch, !republishBatch)
+ }
+
+ private startPoolWorkers(pool: FetchCandidatePool<PooledBatch>): void {
+ this.poolWorkers ??= Array.from({ length: this.options.maxInFlightRequests }, () => this.runPoolWorker(pool))
+ }
+
+ private async runPoolWorker(pool: FetchCandidatePool<PooledBatch>): Promise<void> {
+ for (;;) {
+ const lease = await pool.take()
+ if (!lease) {
+ return
+ }
+ ImageFetchPoolMetrics.observeWait(lease.waitedMs / 1000)
+ let signalSlotReleased: () => void = () => undefined
+ const slotReleased = new Promise<void>((resolve) => {
+ signalSlotReleased = resolve
+ })
+ const releaseRegistrableDomainSlot = (): void => {
+ lease.release()
+ signalSlotReleased()
+ }
+ const work = this.processPooledLease(pool, lease, releaseRegistrableDomainSlot)
+ await Promise.race([slotReleased, work])
+ }
+ }
+
+ private async processPooledLease(
+ pool: FetchCandidatePool<PooledBatch>,
+ lease: FetchCandidateLease & { context: PooledBatch },
+ releaseRegistrableDomainSlot: () => void
+ ): Promise<void> {
+ const batch = lease.context
+ try {
+ batch.attempts.push(
+ await this.processLease(
+ lease,
+ batch.stored,
+ batch.configurationItems,
+ batch.configurationPolicy,
+ batch.deadlineMs,
+ batch.republishBatch,
+ batch.state,
+ releaseRegistrableDomainSlot
+ )
+ )
+ } catch (error) {
+ batch.state.failure ??= { error }
+ if (batch.poolBatch) {
+ batch.unsettled -= pool.withdraw(batch.poolBatch)
+ }
+ } finally {
+ releaseRegistrableDomainSlot()
+ batch.unsettled -= 1
+ if (batch.unsettled <= 0) {
+ batch.settle()
+ }
+ }
+ }
+
+ private async finishPass(
+ attempts: FetchAttempt[],
+ republishBatch: RepublishBatch,
+ flushRepublishBatch: boolean
+ ): Promise<FetchAttempt[]> {
+ if (flushRepublishBatch) {
+ const result = await republishBatch.flush()
+ if (result.failedUrls > 0) {
+ throw new Error(`the image fetch lane could not account for ${result.failedUrls} URLs`)
+ }
+ }
+ this.logFailures(attempts)
+ return attempts
+ }
+
+ private async runQueueWorker(
+ queue: FetchCandidateQueue,
+ stored: Map<string, CrawlHistoryItem>,
+ configurationItems: Map<string, ConfigurationCacheItem>,
+ configurationPolicy: ConfigurationPolicyPass,
+ deadlineMs: number,
+ attempts: FetchAttempt[],
+ republishBatch: RepublishBatch,
+ passState: FetchPassState
+ ): Promise<void> {
+ const leaseWork: Promise<void>[] = []
+ for (;;) {
+ const lease = queue.take()
+ if (!lease) {
+ break
+ }
+ let signalSlotReleased: () => void = () => undefined
+ const slotReleased = new Promise<void>((resolve) => {
+ signalSlotReleased = resolve
+ })
+ const releaseRegistrableDomainSlot = (): void => {
+ lease.release()
+ signalSlotReleased()
+ }
+ const work = (async (): Promise<void> => {
+ try {
+ attempts.push(
+ await this.processLease(
+ lease,
+ stored,
+ configurationItems,
+ configurationPolicy,
+ deadlineMs,
+ republishBatch,
+ passState,
+ releaseRegistrableDomainSlot
+ )
+ )
+ } catch (error) {
+ passState.failure ??= { error }
+ queue.abort()
+ throw error
+ } finally {
+ releaseRegistrableDomainSlot()
+ }
+ })()
+ leaseWork.push(work)
+ await Promise.race([slotReleased, work.catch(() => undefined)])
+ }
+ const failedLease = (await Promise.allSettled(leaseWork)).find(
+ (settled): settled is PromiseRejectedResult => settled.status === 'rejected'
+ )
+ if (failedLease) {
+ throw failedLease.reason
+ }
+ }
+
+ private async processLease(
+ lease: FetchCandidateLease,
+ stored: Map<string, CrawlHistoryItem>,
+ configurationItems: Map<string, ConfigurationCacheItem>,
+ configurationPolicy: ConfigurationPolicyPass,
+ deadlineMs: number,
+ republishBatch: RepublishBatch,
+ passState: FetchPassState,
+ releaseRegistrableDomainSlot: () => void
+ ): Promise<FetchAttempt> {
+ const candidate = lease.candidate
+ if (candidate.remainingHops === 0) {
+ return this.terminal(candidate, HOPS_EXHAUSTED, undefined, [])
+ }
+ if (Date.now() > deadlineMs) {
+ return await this.republish(republishBatch, candidate, 'deadline', 'pass_deadline', 0, [], 'pass_deadline')
+ }
+ return await ImageFetchProcessingMetrics.runLimited(
+ this.candidateWork,
+ 'candidate_admission',
+ 'candidate_work',
+ {
+ debugTag: candidate.registrableDomain,
+ fn: async () => {
+ if (passState.failure) {
+ throw passState.failure.error
+ }
+ try {
+ if (Date.now() > deadlineMs) {
+ return await this.republish(
+ republishBatch,
+ candidate,
+ 'deadline',
+ 'pass_deadline',
+ 0,
+ [],
+ 'pass_deadline'
+ )
+ }
+ return await this.fetchOne(
+ candidate,
+ stored,
+ configurationItems,
+ configurationPolicy,
+ deadlineMs,
+ republishBatch,
+ releaseRegistrableDomainSlot
+ )
+ } catch (error) {
+ passState.failure ??= { error }
+ throw error
+ }
+ },
+ }
+ )
+ }
+
+ private async fetchOne(
+ candidate: FetchCandidate,
+ stored: Map<string, CrawlHistoryItem>,
+ configurationItems: Map<string, ConfigurationCacheItem>,
+ configurationPolicy: ConfigurationPolicyPass,
+ deadlineMs: number,
+ republishBatch: RepublishBatch,
+ releaseRegistrableDomainSlot: () => void
+ ): Promise<FetchAttempt> {
+ if (candidate.remainingHops === 0) {
+ return this.terminal(candidate, HOPS_EXHAUSTED, undefined, [])
+ }
+ const policy = await ImageFetchProcessingMetrics.measure('candidate_policy', () =>
+ configurationPolicy.check(candidate.currentUrl, configurationItems, Date.now())
+ )
+ const configurationUpdates = [...policy.updates]
+ for (const update of policy.updates) {
+ configurationItems.set(update.key, update)
+ }
+ if (!policy.allowed) {
+ ImageFetchRequestMetrics.observePolicyAndBudgetDecision(true, policy.reason ?? 'configuration_refused')
+ if (policy.transient) {
+ if (isRequestStateFull(policy.reason)) {
+ return await this.republish(
+ republishBatch,
+ candidate,
+ policy.reason,
+ policy.reason,
+ ONE_MINUTE_MS,
+ configurationUpdates,
+ policy.reason
+ )
+ }
+ const waitMs = policy.reason === 'configuration_unreachable' ? CONFIGURATION_RETRY_MS : ONE_MINUTE_MS
+ return await this.republish(
+ republishBatch,
+ candidate,
+ 'backoff',
+ 'not_ready',
+ waitMs,
+ configurationUpdates,
+ policy.reason === 'configuration_unreachable'
+ ? 'configuration_unreachable'
+ : 'configuration_deferred'
+ )
+ }
+ return this.terminal(
+ candidate,
+ policy.reason ?? 'configuration_refused',
+ undefined,
+ configurationUpdates,
+ policy.reason ?? 'configuration_refused'
+ )
+ }
+ if (!this.budget.setCrawlDelay(candidate.origin, policy.crawlDelayMs, Date.now())) {
+ ImageFetchRequestMetrics.observePolicyAndBudgetDecision(true, 'origin_map_full')
+ return await this.republish(
+ republishBatch,
+ candidate,
+ 'origin_map_full',
+ 'origin_map_full',
+ ONE_MINUTE_MS,
+ configurationUpdates,
+ 'origin_map_full'
+ )
+ }
+
+ const previous = stored.get(candidate.originalRef)
+ const previousUrl = previous?.kind === 'url' ? previous : undefined
+ const bindResponseUrl = await this.responseUrlBinder.bindingFor(
+ candidate.originalRef,
+ AbortSignal.timeout(Math.max(1, deadlineMs - Date.now()))
+ )
+ const previousBinding = previousUrl?.cache?.responseUrlHmac
+ // Another URL would learn what the issuing URL sent, and its 304 would not say this image is current.
+ const validators =
+ previousBinding !== undefined && previousBinding === bindResponseUrl?.(candidate.currentUrl)
+ ? previousUrl?.cache
+ : undefined
+ const fetched = await ImageFetchProcessingMetrics.measure('candidate_fetch', () =>
+ this.fetcher.fetch(candidate.currentUrl, {
+ sourcePartitions: candidate.sourcePartitions,
+ maxBytes: this.options.maxBytes,
+ timeoutMs: this.options.requestTimeoutMs,
+ maxRedirects: Math.min(this.options.maxRedirects, candidate.remainingHops),
+ cache: validators,
+ tdmrepReservation: policy.tdmrepReservation,
+ onRedirectResponse: () => this.budget.recordCompletedResponse(candidate.registrableDomain, Date.now()),
+ isDifferentOrigin: (url) => url.origin !== candidate.origin,
+ scheduleRequest: (url, requestDeadlineMs, request) =>
+ this.scheduler.runImage(
+ url,
+ Math.min(deadlineMs, requestDeadlineMs),
+ request,
+ candidate.sourcePartitions
+ ),
+ checkRedirectPolicy: async (url) => {
+ const redirectPolicy = await ImageFetchProcessingMetrics.measure('candidate_policy', () =>
+ configurationPolicy.check(url, configurationItems, Date.now())
+ )
+ for (const update of redirectPolicy.updates) {
+ configurationItems.set(update.key, update)
+ configurationUpdates.push(update)
+ }
+ if (!redirectPolicy.allowed) {
+ return {
+ allowed: false,
+ transient: redirectPolicy.transient,
+ reason: redirectPolicy.reason ?? 'configuration_refused',
+ }
+ }
+ const origin = new URL(url).origin
+ if (!this.budget.setCrawlDelay(origin, redirectPolicy.crawlDelayMs, Date.now())) {
+ return { allowed: false, transient: true, reason: 'origin_map_full' }
+ }
+ return { allowed: true, tdmrepReservation: redirectPolicy.tdmrepReservation }
+ },
+ })
+ )
+ // One fetch can serve the jobs of several teams, so each job binds the response under its own key.
+ const result: ImageFetchResult = fetched.cache
+ ? { ...fetched, cache: { ...fetched.cache, responseUrlHmac: bindResponseUrl?.(fetched.currentUrl) } }
+ : fetched
+ ImageFetchRequestMetrics.observeRedirectCount(result.redirects)
+ if (result.bytes) {
+ ImageFetchRequestMetrics.observeBytes(result.bytes.length)
+ }
+
+ const effectiveUrl = canonicalizeUrl(result.currentUrl)
+ if (!effectiveUrl) {
+ return this.terminal(candidate, 'bad_redirect', result.cache, configurationUpdates)
+ }
+ const attemptedCandidate: FetchCandidate = {
+ ...candidate,
+ currentUrl: effectiveUrl.fetch,
+ host: effectiveUrl.host,
+ origin: new URL(effectiveUrl.fetch).origin,
+ registrableDomain: effectiveUrl.domain,
+ remainingHops: candidate.remainingHops - result.redirects,
+ fetchCount:
+ candidate.fetchCount +
+ result.redirects +
+ (result.outcome === 'request_deferred' || result.outcome === 'redirect_policy_refused' ? 0 : 1),
+ }
+
+ if (result.outcome === 'redirect_policy_refused') {
+ const reason = result.refusalReason ?? 'configuration_refused'
+ ImageFetchRequestMetrics.observePolicyAndBudgetDecision(true, reason)
+ if (result.policyTransient) {
+ const stateFull = isRequestStateFull(reason)
+ const republishReason: RepublishReason = stateFull ? reason : 'not_ready'
+ const waitMs = reason === 'configuration_unreachable' ? CONFIGURATION_RETRY_MS : ONE_MINUTE_MS
+ return await this.republish(
+ republishBatch,
+ attemptedCandidate,
+ reason,
+ republishReason,
+ waitMs,
+ configurationUpdates,
+ stateFull
+ ? reason
+ : reason === 'configuration_unreachable'
+ ? 'configuration_unreachable'
+ : 'configuration_deferred'
+ )
+ }
+ return this.terminal(attemptedCandidate, reason, undefined, configurationUpdates, reason)
+ }
+
+ if (result.outcome === 'request_deferred') {
+ const reason = result.schedulingReason ?? 'deadline'
+ ImageFetchRequestMetrics.observePolicyAndBudgetDecision(true, reason)
+ const republishReason: RepublishReason = isRequestStateFull(reason)
+ ? reason
+ : reason === 'deadline' && Date.now() >= deadlineMs
+ ? 'pass_deadline'
+ : 'not_ready'
+ const waitMs =
+ republishReason === 'pass_deadline'
+ ? 0
+ : Math.max(
+ ONE_MINUTE_MS,
+ result.schedulingWaitMs ??
+ this.budget.blockedForMs(attemptedCandidate.registrableDomain, Date.now())
+ )
+ return await this.republish(
+ republishBatch,
+ attemptedCandidate,
+ reason,
+ republishReason,
+ waitMs,
+ configurationUpdates,
+ result.schedulingBlockingReason ??
+ (republishReason === 'pass_deadline'
+ ? 'pass_deadline'
+ : reason === 'backoff'
+ ? 'unknown_backoff'
+ : reason === 'deadline'
+ ? 'request_deadline'
+ : reason)
+ )
+ }
+
+ ImageFetchRequestMetrics.observePolicyAndBudgetDecision(false)
+
+ if (isTransientOutcome(result.outcome)) {
+ const delayMs = this.budget.recordTransientFailure(
+ candidate.registrableDomain,
+ Date.now(),
+ result.retryAfterMs
+ )
+ const attempt = await this.republish(
+ republishBatch,
+ attemptedCandidate,
+ result.outcome,
+ 'retry',
+ delayMs,
+ configurationUpdates,
+ result.retryAfterMs === undefined ? 'transient_backoff' : 'retry_after'
+ )
+ return attempt
+ }
+
+ this.budget.recordCompletedResponse(candidate.registrableDomain, Date.now())
+ if (
+ (result.outcome === 'redirect_offsite' || result.outcome === 'redirect_continuation') &&
+ result.redirectTarget
+ ) {
+ const canonical = canonicalizeUrl(result.redirectTarget.url)
+ if (!canonical) {
+ return this.terminal(attemptedCandidate, 'bad_redirect', result.cache, configurationUpdates)
+ }
+ return await this.republishToTarget(
+ republishBatch,
+ attemptedCandidate,
+ result.outcome,
+ {
+ currentUrl: canonical.fetch,
+ host: canonical.host,
+ origin: new URL(canonical.fetch).origin,
+ registrableDomain: canonical.domain,
+ },
+ 'redirect',
+ 0,
+ configurationUpdates
+ )
+ }
+ // A missing or broken response leaves the stored image in place, so the lane keeps it alive. An opt-out does not.
+ let storedAtMs = result.outcome === 'opt_out' ? undefined : previousUrl?.storedAtMs
+ if (result.outcome === 'ok' || result.outcome === 'not_modified') {
+ releaseRegistrableDomainSlot()
+ let stored = true
+ try {
+ if (result.outcome === 'ok') {
+ await this.imageSink.publishImage(attemptedCandidate, result)
+ } else {
+ // A 304 says the stored image is current, so the sink refreshes it in place.
+ stored = await this.imageSink.refreshImage(attemptedCandidate)
+ }
+ } catch (error) {
+ this.logStorageFailure(result.outcome === 'ok' ? 'publish' : 'refresh', error)
+ return {
+ candidate: attemptedCandidate,
+ outcome: 'publish_failed',
+ finished: false,
+ lost: true,
+ configurationUpdates,
+ }
+ }
+ if (!stored) {
+ return await this.refetchWithoutValidators(
+ republishBatch,
+ attemptedCandidate,
+ previousUrl,
+ configurationUpdates
+ )
+ }
+ storedAtMs = Date.now()
+ }
+ const responseCache = result.redirects > 0 ? withoutValidators(result.cache) : result.cache
+ const mergedCache =
+ result.outcome === 'not_modified' ? mergeCache(previousUrl?.cache, responseCache) : responseCache
+ return this.terminal(
+ attemptedCandidate,
+ result.outcome,
+ mergedCache,
+ configurationUpdates,
+ result.refusalReason,
+ storedAtMs
+ )
+ }
+
+ /**
+ * A 304 for an image S3 no longer holds, for example after the lifecycle expired it. The history drops the
+ * validators before the job comes back, so the next request asks for the whole image.
+ */
+ private async refetchWithoutValidators(
+ republishBatch: RepublishBatch,
+ candidate: FetchCandidate,
+ previousUrl: UrlCrawlHistoryItem | undefined,
+ configurationUpdates: ConfigurationCacheItem[]
+ ): Promise<FetchAttempt> {
+ const attempt = await this.republishToTarget(
+ republishBatch,
+ candidate,
+ 'not_modified',
+ candidate,
+ 'retry',
+ 0,
+ configurationUpdates
+ )
+ if (attempt.finished) {
+ return attempt
+ }
+ const nowMs = Date.now()
+ return {
+ ...attempt,
+ history: {
+ kind: 'url',
+ key: candidate.originalRef,
+ nextFetchAtMs: nowMs,
+ storageExpiresAtMs: urlHistoryStorageExpiresAtMs(candidate.originalRef, nowMs),
+ outcome: 'not_stored',
+ cache: withoutValidators(previousUrl?.cache),
+ },
+ }
+ }
+
+ private async republish(
+ republishBatch: RepublishBatch,
+ candidate: FetchCandidate,
+ outcome: AttemptOutcome,
+ reason: RepublishReason,
+ waitMs: number,
+ configurationUpdates: ConfigurationCacheItem[],
+ blockingReason: ImageFetchBlockReason
+ ): Promise<FetchAttempt> {
+ return await this.republishToTarget(
+ republishBatch,
+ candidate,
+ outcome,
+ {
+ currentUrl: candidate.currentUrl,
+ host: candidate.host,
+ origin: candidate.origin,
+ registrableDomain: candidate.registrableDomain,
+ },
+ reason,
+ waitMs,
+ configurationUpdates,
+ { reason: blockingReason, waitMs }
+ )
+ }
+
+ private async republishToTarget(
+ republishBatch: RepublishBatch,
+ candidate: FetchCandidate,
+ outcome: AttemptOutcome,
+ target: Pick<FetchCandidate, 'currentUrl' | 'host' | 'origin' | 'registrableDomain'>,
+ reason: RepublishReason,
+ waitMs: number,
+ configurationUpdates: ConfigurationCacheItem[],
+ block?: { reason: ImageFetchBlockReason; waitMs: number }
+ ): Promise<FetchAttempt> {
+ if ((reason === 'redirect' || reason === 'retry') && candidate.remainingHops <= 1) {
+ return this.terminal(candidate, HOPS_EXHAUSTED, undefined, configurationUpdates)
+ }
+ const republishedCandidate: FetchCandidate = {
+ ...candidate,
+ lastBlockReason: block?.reason,
+ }
+ const result: RepublishResult = await republishBatch.republish(republishedCandidate, target, reason, waitMs)
+ if (result === 'refused_delay') {
+ return this.terminal(candidate, DELAY_TOO_LONG, undefined, configurationUpdates)
+ }
+ return {
+ candidate: republishedCandidate,
+ outcome,
+ finished: false,
+ lost: false,
+ block,
+ configurationUpdates,
+ }
+ }
+
+ private terminal(
+ candidate: FetchCandidate,
+ outcome: AttemptOutcome,
+ cache: HttpCacheMetadata | undefined,
+ configurationUpdates: ConfigurationCacheItem[],
+ refusalReason: FetchRefusalReason | 'none' = 'none',
+ storedAtMs?: number
+ ): FetchAttempt {
+ const nowMs = Date.now()
+ const nextFetchAtMs = urlHistoryExpiresAtMs(nowMs, this.options.seenTtlSeconds, cache)
+ ImageFetchRequestMetrics.observeCompletedUrl(
+ outcome,
+ refusalReason,
+ Math.max(0, nowMs - candidate.firstSeenAtMs) / 1000,
+ candidate.fetchCount,
+ candidate.republishCount
+ )
+ ImageFetchRequestMetrics.observeHops(MAX_HOPS - candidate.remainingHops)
+ return {
+ candidate,
+ outcome,
+ finished: true,
+ lost: false,
+ history: {
+ kind: 'url',
+ key: candidate.originalRef,
+ nextFetchAtMs,
+ storageExpiresAtMs: urlHistoryStorageExpiresAtMs(candidate.originalRef, nextFetchAtMs),
+ outcome: String(outcome),
+ cache,
+ ...(storedAtMs === undefined ? {} : { storedAtMs }),
+ },
+ configurationUpdates,
+ }
+ }
+
+ /**
+ * S3 and KMS fail in bursts, so this logs at most one failure per interval, with the count since the last one. The
+ * error message can name the object key, so only the error name and status are logged, never the URL or the key.
+ */
+ private logStorageFailure(operation: 'publish' | 'refresh', error: unknown): void {
+ this.storageFailuresSinceLog += 1
+ const nowMs = Date.now()
+ if (nowMs - this.storageFailureLoggedAtMs < STORAGE_FAILURE_LOG_INTERVAL_MS) {
+ return
+ }
+ const { name, Code, $metadata } = (typeof error === 'object' && error !== null ? error : {}) as {
+ name?: unknown
+ Code?: unknown
+ $metadata?: { httpStatusCode?: unknown }
+ }
+ logger.warn('🌐', 'ai_research_v4_image_fetch_storage_failed', {
+ operation,
+ error: typeof name === 'string' ? name : 'unknown',
+ code: typeof Code === 'string' ? Code : undefined,
+ httpStatusCode: typeof $metadata?.httpStatusCode === 'number' ? $metadata.httpStatusCode : undefined,
+ failures: this.storageFailuresSinceLog,
+ })
+ this.storageFailureLoggedAtMs = nowMs
+ this.storageFailuresSinceLog = 0
+ }
+
+ private logFailures(attempts: FetchAttempt[]): void {
+ const failures = attempts.filter((attempt) => attempt.outcome !== 'ok' && attempt.outcome !== 'not_modified')
+ if (failures.length === 0) {
+ return
+ }
+ logger.warn('🌐', 'ai_research_v4_image_fetch_batch_failures', {
+ count: failures.length,
+ outcomes: [...new Set(failures.map((attempt) => String(attempt.outcome)))].slice(0, 10),
+ hosts: [...new Set(failures.map((attempt) => attempt.candidate.host))].slice(0, 5),
+ })
+ }
+}
+
+function mergeCache(
+ previous: HttpCacheMetadata | undefined,
+ update: HttpCacheMetadata | undefined
+): HttpCacheMetadata | undefined {
+ if (!previous) {
+ return update
+ }
+ if (!update) {
+ return previous
+ }
+ const definedUpdate = Object.fromEntries(Object.entries(update).filter(([, value]) => value !== undefined))
+ return { ...previous, ...definedUpdate }
+}
+
+function withoutValidators(cache: HttpCacheMetadata | undefined): HttpCacheMetadata | undefined {
+ return cache ? { ...cache, etag: undefined, lastModified: undefined } : undefined
+}
+
+function configurationItemsFrom(stored: Map<string, CrawlHistoryItem>): Map<string, ConfigurationCacheItem> {
+ const configurationItems = new Map<string, ConfigurationCacheItem>()
+ for (const [key, item] of stored) {
+ if (item.kind === 'robots' || item.kind === 'tdmrep') {
+ configurationItems.set(key, item)
+ }
+ }
+ return configurationItems
+}
diff --git a/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/frontier-dead-letter-sink.test.ts b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/frontier-dead-letter-sink.test.ts
new file mode 100644
index 000000000000..77e9048b5dc0
--- /dev/null
+++ b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/frontier-dead-letter-sink.test.ts
@@ -0,0 +1,94 @@
+import { Message } from 'node-rdkafka'
+
+import { KafkaProducerWrapper } from '~/common/kafka/producer'
+
+import { MAX_RECORD_BYTES } from './collected-urls-record'
+import { KafkaFrontierDeadLetterSink } from './frontier-dead-letter-sink'
+
+describe('KafkaFrontierDeadLetterSink', () => {
+ it('preserves the source key and value with bounded diagnostic headers', async () => {
+ const produce = jest.fn(() => Promise.resolve())
+ const sink = new KafkaFrontierDeadLetterSink({ produce } as unknown as KafkaProducerWrapper, 'frontier-dlq', [
+ 'frontier',
+ ])
+ const source = {
+ topic: 'frontier',
+ partition: 7,
+ offset: 42,
+ key: Buffer.from('example.com'),
+ value: Buffer.from('{"v":2,"jobs":[]}'),
+ } as Message
+
+ await sink.park(source, 'bad_url')
+
+ expect(produce).toHaveBeenCalledWith({
+ topic: 'frontier-dlq',
+ key: source.key,
+ value: source.value,
+ headers: {
+ 'dlq-reason': 'bad_url',
+ 'frontier-record-version': '2',
+ 'source-topic': 'frontier',
+ 'source-partition': '7',
+ 'source-offset': '42',
+ },
+ })
+ })
+
+ it('does not copy an unsupported version into a header', async () => {
+ const produce = jest.fn(() => Promise.resolve())
+ const sink = new KafkaFrontierDeadLetterSink({ produce } as unknown as KafkaProducerWrapper, 'frontier-dlq', [
+ 'frontier',
+ ])
+
+ await sink.park(
+ {
+ topic: 'frontier',
+ partition: 0,
+ offset: 1,
+ key: null,
+ value: Buffer.from('{"v":"private input"}'),
+ } as Message,
+ 'unsupported_version'
+ )
+
+ expect(produce).toHaveBeenCalledWith(
+ expect.objectContaining({
+ headers: expect.objectContaining({ 'frontier-record-version': 'unsupported' }),
+ })
+ )
+ })
+
+ it('does not parse an oversized rejected value to classify its version', async () => {
+ const produce = jest.fn(() => Promise.resolve())
+ const sink = new KafkaFrontierDeadLetterSink({ produce } as unknown as KafkaProducerWrapper, 'frontier-dlq', [
+ 'frontier',
+ ])
+
+ await sink.park(
+ {
+ topic: 'frontier',
+ partition: 0,
+ offset: 1,
+ key: null,
+ value: Buffer.alloc(MAX_RECORD_BYTES + 1, '{'),
+ } as Message,
+ 'oversized_record'
+ )
+
+ expect(produce).toHaveBeenCalledWith(
+ expect.objectContaining({
+ headers: expect.objectContaining({ 'frontier-record-version': 'unknown' }),
+ })
+ )
+ })
+
+ it('refuses a destination used by the image-fetch pipeline', () => {
+ expect(
+ () =>
+ new KafkaFrontierDeadLetterSink({ produce: jest.fn() } as unknown as KafkaProducerWrapper, 'frontier', [
+ 'frontier',
+ ])
+ ).toThrow('AI_RESEARCH_V4_IMAGE_FETCH_DLQ_TOPIC')
+ })
+})
diff --git a/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/frontier-dead-letter-sink.ts b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/frontier-dead-letter-sink.ts
new file mode 100644
index 000000000000..df5407481538
--- /dev/null
+++ b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/frontier-dead-letter-sink.ts
@@ -0,0 +1,63 @@
+import { Message } from 'node-rdkafka'
+
+import { parseKafkaHeaders } from '~/common/kafka/consumer/consumer-v1'
+import { KafkaProducer } from '~/common/kafka/producer'
+import { parseJSON } from '~/common/utils/json-parse'
+
+import { MAX_RECORD_BYTES, UrlDropReason } from './collected-urls-record'
+
+export type FrontierDeadLetterReason = UrlDropReason | 'multiple'
+type FrontierRecordVersion = '1' | '2' | 'unsupported' | 'unknown'
+
+export interface FrontierDeadLetterSink {
+ park(message: Message, reason: FrontierDeadLetterReason): Promise<void>
+}
+
+export class KafkaFrontierDeadLetterSink implements FrontierDeadLetterSink {
+ constructor(
+ private readonly producer: KafkaProducer,
+ private readonly topic: string,
+ blockedDestinations: readonly string[]
+ ) {
+ if (blockedDestinations.includes(topic)) {
+ throw new Error(`AI_RESEARCH_V4_IMAGE_FETCH_DLQ_TOPIC cannot use image-fetch topic ${topic}`)
+ }
+ }
+
+ public async park(message: Message, reason: FrontierDeadLetterReason): Promise<void> {
+ await this.producer.produce({
+ topic: this.topic,
+ key: message.key ?? null,
+ value: message.value,
+ headers: {
+ ...parseKafkaHeaders(message.headers),
+ 'dlq-reason': reason,
+ 'frontier-record-version': classifyFrontierRecordVersion(message.value),
+ 'source-topic': message.topic,
+ 'source-partition': String(message.partition),
+ 'source-offset': String(message.offset),
+ },
+ })
+ }
+}
+
+function classifyFrontierRecordVersion(value: Buffer | null): FrontierRecordVersion {
+ if (!value || value.length > MAX_RECORD_BYTES) {
+ return 'unknown'
+ }
+ try {
+ const parsed: unknown = parseJSON(value.toString())
+ if (typeof parsed !== 'object' || parsed === null || Array.isArray(parsed) || !('v' in parsed)) {
+ return 'unknown'
+ }
+ if (parsed.v === 1) {
+ return '1'
+ }
+ if (parsed.v === 2) {
+ return '2'
+ }
+ return 'unsupported'
+ } catch {
+ return 'unknown'
+ }
+}
diff --git a/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/frontier-publisher.test.ts b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/frontier-publisher.test.ts
new file mode 100644
index 000000000000..e2b60454d201
--- /dev/null
+++ b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/frontier-publisher.test.ts
@@ -0,0 +1,405 @@
+import { KafkaProducerWrapper } from '~/common/kafka/producer'
+import { parseJSON } from '~/common/utils/json-parse'
+import { CAPTURE_TIMESTAMP_HEADER } from '~/ingestion/pipelines/sessionreplay/shared/capture-watermark'
+
+import {
+ FetchCandidate,
+ MAX_HOPS,
+ MAX_JOBS_PER_RECORD,
+ MAX_RECORD_BYTES,
+ parseCollectedUrlsRecord,
+} from './collected-urls-record'
+import { FrontierPublisher, RepublishBatch } from './frontier-publisher'
+import { ImageFetchRequestMetrics } from './metrics'
+
+const FRONTIER = 'session_replay_image_fetch'
+const TIERS = [
+ { topic: 'retry_1m', delayMs: 60_000, metricTopic: 'retry_1m' as const },
+ { topic: 'retry_10m', delayMs: 600_000, metricTopic: 'retry_10m' as const },
+ { topic: 'retry_1h', delayMs: 3_600_000, metricTopic: 'retry_1h' as const },
+]
+
+function candidate(overrides: Partial<FetchCandidate> = {}): FetchCandidate {
+ return {
+ originalRef: `imageurl:v4:7:2026-10:${'a'.repeat(22)}`,
+ currentUrl: 'https://cdn.example.com/a.png',
+ host: 'cdn.example.com',
+ origin: 'https://cdn.example.com',
+ registrableDomain: 'example.com',
+ remainingHops: MAX_HOPS,
+ notBeforeMs: 0,
+ firstSeenAtMs: 1_700_000_000_000,
+ fetchCount: 0,
+ republishCount: 0,
+ lastRepublishReason: null,
+ ...overrides,
+ }
+}
+
+interface SentMessage {
+ topic: string
+ key: string
+ value: Buffer
+ headers?: Record<string, string>
+}
+
+function build(): {
+ publisher: FrontierPublisher
+ batch: RepublishBatch
+ sent: SentMessage[]
+} {
+ const sent: SentMessage[] = []
+ const producer = {
+ produce: (message: { topic: string; key: Buffer; value: Buffer; headers?: Record<string, string> }) => {
+ sent.push({ ...message, key: message.key.toString() })
+ return Promise.resolve()
+ },
+ } as unknown as KafkaProducerWrapper
+ const publisher = new FrontierPublisher(producer, {
+ frontierTopic: FRONTIER,
+ delayTiers: TIERS,
+ maxConcurrentRepublishes: 2,
+ })
+ return { publisher, batch: publisher.createRepublishBatch(), sent }
+}
+
+describe('FrontierPublisher', () => {
+ beforeEach(() => jest.useFakeTimers().setSystemTime(1_700_000_000_000))
+ afterEach(() => {
+ jest.useRealTimers()
+ jest.restoreAllMocks()
+ })
+
+ it('groups frontier redirects by target registrable domain', async () => {
+ const { batch, sent } = build()
+ const republished = jest.spyOn(ImageFetchRequestMetrics, 'incRepublished').mockImplementation()
+ const target = {
+ currentUrl: 'https://img.other.net/a.png',
+ host: 'img.other.net',
+ origin: 'https://img.other.net',
+ registrableDomain: 'other.net',
+ }
+
+ expect(await batch.republish(candidate(), target, 'redirect')).toBe('queued')
+ expect(
+ await batch.republish(
+ candidate({ originalRef: `imageurl:v4:7:2026-10:${'b'.repeat(22)}`, firstSeenAtMs: 1_699_999_000_000 }),
+ target,
+ 'redirect'
+ )
+ ).toBe('queued')
+ expect(sent).toEqual([])
+ await expect(batch.flush()).resolves.toEqual({ failedUrls: 0 })
+
+ expect(republished).toHaveBeenCalledTimes(2)
+ expect(sent).toHaveLength(1)
+ expect(sent[0]).toMatchObject({
+ topic: FRONTIER,
+ key: 'other.net',
+ headers: { [CAPTURE_TIMESTAMP_HEADER]: '1699999000000' },
+ })
+ expect(parseCollectedUrlsRecord(sent[0].value, 'other.net')).toMatchObject({
+ ok: true,
+ candidates: [
+ {
+ originalRef: `imageurl:v4:7:2026-10:${'a'.repeat(22)}`,
+ currentUrl: 'https://img.other.net/a.png',
+ remainingHops: MAX_HOPS - 1,
+ republishCount: 1,
+ lastRepublishReason: 'redirect',
+ },
+ { originalRef: `imageurl:v4:7:2026-10:${'b'.repeat(22)}` },
+ ],
+ })
+ })
+
+ it.each([
+ ['an unspecified retry', 0, 'retry_1m', 60_000],
+ ['a short retry', 30_000, 'retry_1m', 60_000],
+ ['a medium retry', 120_000, 'retry_10m', 600_000],
+ ['a long supported retry', 3_600_000, 'retry_1h', 3_600_000],
+ ])('parks %s once in the smallest sufficient delay topic', async (_name, waitMs, topic, delayMs) => {
+ const { batch, sent } = build()
+ const republished = jest.spyOn(ImageFetchRequestMetrics, 'incRepublished').mockImplementation()
+
+ expect(await batch.republish(candidate(), candidate(), 'retry', waitMs)).toBe('queued')
+ await batch.flush()
+
+ expect(republished).toHaveBeenCalledWith('retry', 'delay')
+ expect(sent[0].topic).toBe(topic)
+ expect(parseCollectedUrlsRecord(sent[0].value, 'example.com')).toMatchObject({
+ ok: true,
+ candidates: [{ notBeforeMs: 1_700_000_000_000 + delayMs }],
+ })
+ })
+
+ it('groups delayed jobs by destination topic and registrable domain', async () => {
+ const { batch, sent } = build()
+ const second = candidate({ originalRef: `imageurl:v4:7:2026-10:${'b'.repeat(22)}` })
+ const other = candidate({
+ originalRef: `imageurl:v4:7:2026-10:${'c'.repeat(22)}`,
+ currentUrl: 'https://img.other.net/c.png',
+ host: 'img.other.net',
+ origin: 'https://img.other.net',
+ registrableDomain: 'other.net',
+ })
+
+ await batch.republish(candidate(), candidate(), 'retry', 30_000)
+ await batch.republish(second, second, 'not_ready', 30_000)
+ await batch.republish(other, other, 'retry', 30_000)
+ await batch.flush()
+
+ expect(sent).toHaveLength(2)
+ const example = sent.find((item) => item.key === 'example.com')!
+ expect(example.topic).toBe('retry_1m')
+ expect(parseCollectedUrlsRecord(example.value, 'example.com')).toMatchObject({
+ ok: true,
+ candidates: [{ originalRef: candidate().originalRef }, { originalRef: second.originalRef }],
+ })
+ })
+
+ it('keeps configured destination topics separate when they share a metric class', async () => {
+ const sent: SentMessage[] = []
+ const producer = {
+ produce: (message: { topic: string; key: Buffer; value: Buffer }) => {
+ sent.push({ ...message, key: message.key.toString() })
+ return Promise.resolve()
+ },
+ } as unknown as KafkaProducerWrapper
+ const publisher = new FrontierPublisher(producer, {
+ frontierTopic: FRONTIER,
+ delayTiers: [
+ { topic: 'short', delayMs: 60_000, metricTopic: 'retry_1m' },
+ { topic: 'long', delayMs: 120_000, metricTopic: 'retry_1m' },
+ ],
+ maxConcurrentRepublishes: 2,
+ })
+ const batch = publisher.createRepublishBatch()
+
+ await batch.republish(candidate(), candidate(), 'not_ready', 30_000)
+ await batch.republish(
+ candidate({ originalRef: `imageurl:v4:7:2026-10:${'b'.repeat(22)}` }),
+ candidate(),
+ 'not_ready',
+ 90_000
+ )
+ await batch.flush()
+
+ expect(sent.map((message) => message.topic).sort()).toEqual(['long', 'short'])
+ })
+
+ it('records messages, registrable domains, and delivery time for each destination topic', async () => {
+ const { batch } = build()
+ const observeRepublishBatch = jest.spyOn(ImageFetchRequestMetrics, 'observeRepublishBatch').mockImplementation()
+ const observeRepublishFlush = jest.spyOn(ImageFetchRequestMetrics, 'observeRepublishFlush').mockImplementation()
+ const other = candidate({
+ originalRef: `imageurl:v4:7:2026-10:${'b'.repeat(22)}`,
+ currentUrl: 'https://img.other.net/b.png',
+ host: 'img.other.net',
+ origin: 'https://img.other.net',
+ registrableDomain: 'other.net',
+ })
+
+ await batch.republish(candidate(), candidate(), 'pass_deadline')
+ await batch.republish(other, other, 'retry', 30_000)
+ await batch.flush()
+
+ expect(observeRepublishBatch).toHaveBeenCalledWith('frontier', 1, 1, expect.any(Number))
+ expect(observeRepublishBatch).toHaveBeenCalledWith('retry_1m', 1, 1, expect.any(Number))
+ expect(observeRepublishFlush).toHaveBeenCalledWith(expect.any(Number))
+ })
+
+ it('splits one registrable domain at the job-count limit', async () => {
+ const { batch, sent } = build()
+ const candidates = Array.from({ length: MAX_JOBS_PER_RECORD + 1 }, (_, index) =>
+ candidate({ originalRef: `imageurl:v4:7:2026-10:${index.toString().padStart(22, '0')}` })
+ )
+
+ await Promise.all(candidates.map((item) => batch.republish(item, item, 'pass_deadline')))
+ await batch.flush()
+
+ expect(sent).toHaveLength(2)
+ expect(sent.map((item) => (parseJSON(item.value.toString()) as { jobs: unknown[] }).jobs.length)).toEqual([
+ MAX_JOBS_PER_RECORD,
+ 1,
+ ])
+ })
+
+ it('splits one registrable domain at the serialized-byte limit', async () => {
+ const { batch, sent } = build()
+ const longPath = 'a'.repeat(Math.floor(MAX_RECORD_BYTES / 2))
+ const first = candidate({ currentUrl: `https://cdn.example.com/${longPath}/1.png` })
+ const second = candidate({
+ originalRef: `imageurl:v4:7:2026-10:${'b'.repeat(22)}`,
+ currentUrl: `https://cdn.example.com/${longPath}/2.png`,
+ })
+
+ await batch.republish(first, first, 'pass_deadline')
+ await batch.republish(second, second, 'pass_deadline')
+ await batch.flush()
+
+ expect(sent).toHaveLength(2)
+ expect(sent.every((item) => item.value.length <= MAX_RECORD_BYTES)).toBe(true)
+ })
+
+ it('refuses a delay longer than the largest deployed tier', async () => {
+ const { batch, sent } = build()
+
+ expect(await batch.republish(candidate(), candidate(), 'retry', 3_600_001)).toBe('refused_delay')
+ await batch.flush()
+ expect(sent).toEqual([])
+ })
+
+ it('refuses a redirect that has no hop left after republishing', async () => {
+ const { batch, sent } = build()
+
+ expect(await batch.republish(candidate({ remainingHops: 1 }), candidate(), 'redirect')).toBe('refused_delay')
+ await batch.flush()
+ expect(sent).toEqual([])
+ })
+
+ it('publishes every target-domain group from redirect fanout to the frontier', async () => {
+ const { batch, sent } = build()
+ const other = candidate({
+ originalRef: `imageurl:v4:7:2026-10:${'b'.repeat(22)}`,
+ currentUrl: 'https://img.other.net/b.png',
+ host: 'img.other.net',
+ origin: 'https://img.other.net',
+ registrableDomain: 'other.net',
+ })
+
+ await batch.republish(candidate(), candidate(), 'pass_deadline')
+ await batch.republish(other, other, 'redirect')
+
+ await expect(batch.flush()).resolves.toEqual({ failedUrls: 0 })
+ expect(sent).toHaveLength(2)
+ expect(sent.every((item) => item.topic === FRONTIER)).toBe(true)
+ const redirected = sent.find((item) => item.key === 'other.net')!
+ expect(parseCollectedUrlsRecord(redirected.value, 'other.net')).toMatchObject({
+ ok: true,
+ candidates: [{ lastRepublishReason: 'redirect', notBeforeMs: 0 }],
+ })
+ })
+
+ it('bounds concurrent Kafka republish deliveries', async () => {
+ let active = 0
+ let peak = 0
+ const releases: Array<() => void> = []
+ const producer = {
+ produce: () =>
+ new Promise<void>((resolve) => {
+ active++
+ peak = Math.max(peak, active)
+ releases.push(() => {
+ active--
+ resolve()
+ })
+ }),
+ } as unknown as KafkaProducerWrapper
+ const publisher = new FrontierPublisher(producer, {
+ frontierTopic: FRONTIER,
+ delayTiers: TIERS,
+ maxConcurrentRepublishes: 2,
+ })
+ const batch = publisher.createRepublishBatch()
+
+ for (const domain of ['one.example', 'two.example', 'three.example']) {
+ const item = candidate({
+ currentUrl: `https://${domain}/a.png`,
+ host: domain,
+ origin: `https://${domain}`,
+ registrableDomain: domain,
+ })
+ await batch.republish(item, item, 'pass_deadline')
+ }
+
+ const flushing = batch.flush()
+ expect(active).toBe(2)
+ expect(peak).toBe(2)
+
+ const firstTwo = releases.splice(0, 2)
+ firstTwo[0]()
+ for (let index = 0; index < 10; index++) {
+ await Promise.resolve()
+ }
+ expect(active).toBe(2)
+ expect(peak).toBe(2)
+
+ firstTwo[1]()
+ releases[0]()
+ await expect(flushing).resolves.toEqual({ failedUrls: 0 })
+ })
+
+ it('reports every URL in a failed grouped message', async () => {
+ const producer = { produce: () => Promise.reject(new Error('broker down')) } as unknown as KafkaProducerWrapper
+ const publisher = new FrontierPublisher(producer, {
+ frontierTopic: FRONTIER,
+ delayTiers: TIERS,
+ maxConcurrentRepublishes: 2,
+ })
+ const batch = publisher.createRepublishBatch()
+ const failed = jest.spyOn(ImageFetchRequestMetrics, 'incRepublishFailed').mockImplementation()
+
+ await batch.republish(candidate(), candidate(), 'pass_deadline')
+ await batch.republish(
+ candidate({ originalRef: `imageurl:v4:7:2026-10:${'b'.repeat(22)}` }),
+ candidate(),
+ 'pass_deadline'
+ )
+
+ await expect(batch.flush()).resolves.toEqual({ failedUrls: 2 })
+ expect(failed).toHaveBeenCalledTimes(2)
+ })
+
+ it('stops starting republish deliveries after the first failure', async () => {
+ const produce = jest.fn(() => Promise.reject(new Error('broker down')))
+ const producer = { produce } as unknown as KafkaProducerWrapper
+ const publisher = new FrontierPublisher(producer, {
+ frontierTopic: FRONTIER,
+ delayTiers: TIERS,
+ maxConcurrentRepublishes: 1,
+ })
+ const batch = publisher.createRepublishBatch()
+
+ for (const domain of ['one.example', 'two.example', 'three.example']) {
+ const item = candidate({
+ currentUrl: `https://${domain}/a.png`,
+ host: domain,
+ origin: `https://${domain}`,
+ registrableDomain: domain,
+ })
+ await batch.republish(item, item, 'pass_deadline')
+ }
+
+ await expect(batch.flush()).resolves.toEqual({ failedUrls: 3 })
+ expect(produce).toHaveBeenCalledTimes(1)
+ })
+
+ it('does not start republish deliveries after the finalization deadline', async () => {
+ const produce = jest.fn(() => Promise.resolve())
+ const producer = { produce } as unknown as KafkaProducerWrapper
+ const publisher = new FrontierPublisher(producer, {
+ frontierTopic: FRONTIER,
+ delayTiers: TIERS,
+ maxConcurrentRepublishes: 2,
+ })
+ const batch = publisher.createRepublishBatch(performance.now())
+ const deadlineExceeded = jest
+ .spyOn(ImageFetchRequestMetrics, 'incRepublishFlushDeadlineExceeded')
+ .mockImplementation()
+
+ for (const domain of ['one.example', 'two.example', 'three.example']) {
+ const item = candidate({
+ currentUrl: `https://${domain}/a.png`,
+ host: domain,
+ origin: `https://${domain}`,
+ registrableDomain: domain,
+ })
+ await batch.republish(item, item, 'pass_deadline')
+ }
+
+ await expect(batch.flush()).resolves.toEqual({ failedUrls: 3 })
+ expect(produce).not.toHaveBeenCalled()
+ expect(deadlineExceeded).toHaveBeenCalledTimes(1)
+ })
+})
diff --git a/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/frontier-publisher.ts b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/frontier-publisher.ts
new file mode 100644
index 000000000000..a3e7354efd5d
--- /dev/null
+++ b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/frontier-publisher.ts
@@ -0,0 +1,331 @@
+import { KafkaProducer } from '~/common/kafka/producer'
+import { ConcurrencyController } from '~/common/utils/concurrencyController'
+import { logger } from '~/common/utils/logger'
+import { CAPTURE_TIMESTAMP_HEADER } from '~/ingestion/pipelines/sessionreplay/shared/capture-watermark'
+
+import {
+ FetchCandidate,
+ MAX_JOBS_PER_RECORD,
+ MAX_RECORD_BYTES,
+ RepublishReason,
+ serializeFrontierRecord,
+} from './collected-urls-record'
+import { ImageFetchRequestMetrics, RepublishTopic } from './metrics'
+
+export interface DelayTier {
+ topic: string
+ delayMs: number
+ metricTopic: Exclude<RepublishTopic, 'frontier'>
+}
+
+export interface FrontierPublisherOptions {
+ frontierTopic: string
+ delayTiers: DelayTier[]
+ maxConcurrentRepublishes: number
+}
+
+export type RepublishResult = 'queued' | 'refused_delay'
+
+export interface RepublishFlushResult {
+ failedUrls: number
+}
+
+interface RepublishDestination {
+ topic: string
+ metricTopic: RepublishTopic
+ delayMs: number
+}
+
+interface PendingRepublish {
+ candidate: FetchCandidate
+ reason: RepublishReason
+ destination: RepublishDestination
+}
+
+interface PlannedRepublishMessage {
+ topic: string
+ metricTopic: RepublishTopic
+ registrableDomain: string
+ candidates: FetchCandidate[]
+ reasons: RepublishReason[]
+}
+
+interface RepublishDeliveryState {
+ failed: boolean
+ deadlineExceeded: boolean
+}
+
+type RepublishDeliveryResult = 'published' | 'failed' | 'skipped'
+
+const EMPTY_FRONTIER_RECORD_BYTES = serializeFrontierRecord([]).length
+
+export interface RepublishBatch {
+ republish(
+ candidate: FetchCandidate,
+ target: Pick<FetchCandidate, 'currentUrl' | 'host' | 'origin' | 'registrableDomain'>,
+ reason: RepublishReason,
+ waitMs?: number
+ ): Promise<RepublishResult>
+ flush(): Promise<RepublishFlushResult>
+}
+
+/** Sends each URL that is not done to the frontier or a delay tier. */
+export class FrontierPublisher {
+ private readonly delayTiers: DelayTier[]
+ private readonly republishes: ConcurrencyController
+
+ constructor(
+ private readonly producer: KafkaProducer,
+ private readonly options: FrontierPublisherOptions
+ ) {
+ this.delayTiers = [...options.delayTiers].sort((left, right) => left.delayMs - right.delayMs)
+ if (this.delayTiers.length === 0) {
+ throw new Error('the image fetch lane needs at least one delay tier')
+ }
+ if (!Number.isInteger(options.maxConcurrentRepublishes) || options.maxConcurrentRepublishes < 1) {
+ throw new Error('the image fetch lane needs a positive republish limit')
+ }
+ this.republishes = new ConcurrencyController(options.maxConcurrentRepublishes)
+ }
+
+ public createRepublishBatch(deadlineAtMonotonicMs = Number.POSITIVE_INFINITY): RepublishBatch {
+ return new BufferedRepublishBatch(
+ this.producer,
+ this.options.frontierTopic,
+ this.delayTiers,
+ this.republishes,
+ deadlineAtMonotonicMs
+ )
+ }
+}
+
+class BufferedRepublishBatch implements RepublishBatch {
+ private readonly pending: PendingRepublish[] = []
+ private flushed = false
+
+ constructor(
+ private readonly producer: KafkaProducer,
+ private readonly frontierTopic: string,
+ private readonly delayTiers: DelayTier[],
+ private readonly republishes: ConcurrencyController,
+ private readonly deadlineAtMonotonicMs: number
+ ) {}
+
+ public republish(
+ candidate: FetchCandidate,
+ target: Pick<FetchCandidate, 'currentUrl' | 'host' | 'origin' | 'registrableDomain'>,
+ reason: RepublishReason,
+ waitMs = 0
+ ): Promise<RepublishResult> {
+ if (this.flushed) {
+ return Promise.reject(new Error('an image fetch republish batch cannot accept work after its flush'))
+ }
+ const spendsHop = reason === 'redirect' || reason === 'retry'
+ const remainingHops = candidate.remainingHops - (spendsHop ? 1 : 0)
+ if (remainingHops <= 0) {
+ return Promise.resolve('refused_delay')
+ }
+
+ const effectiveWaitMs = reason === 'retry' ? Math.max(waitMs, this.delayTiers[0].delayMs) : waitMs
+ const destination = this.destination(effectiveWaitMs)
+ if (!destination) {
+ return Promise.resolve('refused_delay')
+ }
+ this.pending.push({
+ candidate: {
+ ...candidate,
+ ...target,
+ remainingHops,
+ notBeforeMs: destination.delayMs > 0 ? Date.now() + destination.delayMs : 0,
+ republishCount: candidate.republishCount + 1,
+ lastRepublishReason: reason,
+ },
+ reason,
+ destination,
+ })
+ return Promise.resolve('queued')
+ }
+
+ public async flush(): Promise<RepublishFlushResult> {
+ if (this.flushed) {
+ throw new Error('an image fetch republish batch cannot flush more than once')
+ }
+ this.flushed = true
+ if (this.pending.length === 0) {
+ return { failedUrls: 0 }
+ }
+ const startedAt = process.hrtime.bigint()
+ try {
+ const plans = this.planMessages()
+ const deliveryState: RepublishDeliveryState = { failed: false, deadlineExceeded: false }
+ const plansByTopic = new Map<RepublishTopic, PlannedRepublishMessage[]>()
+ for (const plan of plans) {
+ const topicPlans = plansByTopic.get(plan.metricTopic)
+ if (topicPlans) {
+ topicPlans.push(plan)
+ } else {
+ plansByTopic.set(plan.metricTopic, [plan])
+ }
+ }
+ const topicResults = await Promise.all(
+ [...plansByTopic.entries()].map(([metricTopic, topicPlans]) =>
+ this.publishTopic(metricTopic, topicPlans, deliveryState)
+ )
+ )
+ if (deliveryState.deadlineExceeded) {
+ ImageFetchRequestMetrics.incRepublishFlushDeadlineExceeded()
+ }
+ return { failedUrls: topicResults.reduce((sum, result) => sum + result.failedUrls, 0) }
+ } finally {
+ ImageFetchRequestMetrics.observeRepublishFlush(Number(process.hrtime.bigint() - startedAt) / 1e9)
+ }
+ }
+
+ private destination(waitMs: number): RepublishDestination | undefined {
+ if (waitMs <= 0) {
+ return { topic: this.frontierTopic, metricTopic: 'frontier', delayMs: 0 }
+ }
+ const tier = this.delayTiers.find((candidateTier) => candidateTier.delayMs >= waitMs)
+ return tier ? { topic: tier.topic, metricTopic: tier.metricTopic, delayMs: tier.delayMs } : undefined
+ }
+
+ private planMessages(): PlannedRepublishMessage[] {
+ const groups = new Map<string, PendingRepublish[]>()
+ for (const item of this.pending) {
+ const key = `${item.destination.topic}\0${item.candidate.registrableDomain}`
+ const group = groups.get(key)
+ if (group) {
+ group.push(item)
+ } else {
+ groups.set(key, [item])
+ }
+ }
+ return [...groups.values()].flatMap((items) => this.packGroup(items))
+ }
+
+ private packGroup(items: PendingRepublish[]): PlannedRepublishMessage[] {
+ const plans: PlannedRepublishMessage[] = []
+ let candidates: FetchCandidate[] = []
+ let reasons: RepublishReason[] = []
+ let recordBytes = EMPTY_FRONTIER_RECORD_BYTES
+ const finishPlan = (): void => {
+ if (candidates.length === 0) {
+ return
+ }
+ plans.push({
+ topic: items[0].destination.topic,
+ metricTopic: items[0].destination.metricTopic,
+ registrableDomain: items[0].candidate.registrableDomain,
+ candidates,
+ reasons,
+ })
+ candidates = []
+ reasons = []
+ recordBytes = EMPTY_FRONTIER_RECORD_BYTES
+ }
+ for (const item of items) {
+ const candidateBytes = serializeFrontierRecord([item.candidate]).length - EMPTY_FRONTIER_RECORD_BYTES
+ const nextRecordBytes = recordBytes + candidateBytes + (candidates.length > 0 ? 1 : 0)
+ if (
+ candidates.length > 0 &&
+ (candidates.length === MAX_JOBS_PER_RECORD || nextRecordBytes > MAX_RECORD_BYTES)
+ ) {
+ finishPlan()
+ }
+ recordBytes += candidateBytes + (candidates.length > 0 ? 1 : 0)
+ candidates.push(item.candidate)
+ reasons.push(item.reason)
+ if (recordBytes > MAX_RECORD_BYTES) {
+ throw new Error('one image fetch republish job exceeds the Kafka record limit')
+ }
+ }
+ finishPlan()
+ return plans
+ }
+
+ private async publishTopic(
+ metricTopic: RepublishTopic,
+ plans: PlannedRepublishMessage[],
+ deliveryState: RepublishDeliveryState
+ ): Promise<RepublishFlushResult> {
+ const startedAt = process.hrtime.bigint()
+ const results = await Promise.all(
+ plans.map((plan) =>
+ this.republishes.run({
+ debugTag: plan.metricTopic,
+ fn: async (): Promise<RepublishDeliveryResult> => {
+ if (deliveryState.failed) {
+ return 'skipped'
+ }
+ if (performance.now() >= this.deadlineAtMonotonicMs) {
+ deliveryState.failed = true
+ deliveryState.deadlineExceeded = true
+ return 'skipped'
+ }
+ try {
+ await this.producer.produce({
+ topic: plan.topic,
+ key: Buffer.from(plan.registrableDomain),
+ value: serializeFrontierRecord(plan.candidates),
+ headers: {
+ [CAPTURE_TIMESTAMP_HEADER]: String(earliestFirstSeenAtMs(plan.candidates)),
+ },
+ })
+ return 'published'
+ } catch {
+ deliveryState.failed = true
+ return 'failed'
+ }
+ },
+ })
+ )
+ )
+ let failedUrls = 0
+ let attemptedMessages = 0
+ const attemptedRegistrableDomains = new Set<string>()
+ for (let index = 0; index < plans.length; index++) {
+ const plan = plans[index]
+ if (results[index] === 'skipped') {
+ failedUrls += plan.candidates.length
+ continue
+ }
+ attemptedMessages++
+ attemptedRegistrableDomains.add(plan.registrableDomain)
+ if (results[index] === 'failed') {
+ failedUrls += plan.candidates.length
+ this.recordFailedPlan(plan)
+ continue
+ }
+ for (const reason of plan.reasons) {
+ ImageFetchRequestMetrics.incRepublished(reason, metricTopic === 'frontier' ? 'frontier' : 'delay')
+ }
+ }
+ if (attemptedMessages > 0) {
+ ImageFetchRequestMetrics.observeRepublishBatch(
+ metricTopic,
+ attemptedMessages,
+ attemptedRegistrableDomains.size,
+ Number(process.hrtime.bigint() - startedAt) / 1e9
+ )
+ }
+ return { failedUrls }
+ }
+
+ private recordFailedPlan(plan: PlannedRepublishMessage): void {
+ logger.warn('🌐', 'ai_research_v4_image_fetch_republish_failed', {
+ topic: plan.topic,
+ urls: plan.candidates.length,
+ })
+ for (const reason of plan.reasons) {
+ ImageFetchRequestMetrics.incRepublishFailed(reason)
+ }
+ }
+}
+
+function earliestFirstSeenAtMs(candidates: FetchCandidate[]): number {
+ let earliest = candidates[0].firstSeenAtMs
+ for (const candidate of candidates) {
+ earliest = Math.min(earliest, candidate.firstSeenAtMs)
+ }
+ return earliest
+}
diff --git a/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/host-budget.test.ts b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/host-budget.test.ts
new file mode 100644
index 000000000000..75ef862d109b
--- /dev/null
+++ b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/host-budget.test.ts
@@ -0,0 +1,262 @@
+import { HostBudget, HostBudgetOptions } from './host-budget'
+
+const OPTIONS: HostBudgetOptions = {
+ requestsPerSecond: 1,
+ burst: 2,
+ maxConcurrent: 2,
+ breakerFailures: 3,
+ breakerCooldownMs: 60_000,
+ breakerMaxCooldownMs: 600_000,
+ maxTrackedRegistrableDomains: 3,
+ maxTrackedOrigins: 3,
+ random: () => 1,
+}
+const DEADLINE_MS = 1_000_000
+const REGISTRABLE_DOMAIN = 'example.com'
+const ORIGIN = 'https://cdn.example.com'
+
+function budget(overrides: Partial<HostBudgetOptions> = {}): HostBudget {
+ return new HostBudget({ ...OPTIONS, ...overrides })
+}
+
+describe('HostBudget', () => {
+ it('does not reserve a token before a crawl-delay wait finishes', () => {
+ const host = budget({ burst: 5 })
+ host.setCrawlDelay(ORIGIN, 1_000, 1_000)
+
+ const first = host.take(REGISTRABLE_DOMAIN, ORIGIN, 1_000, DEADLINE_MS)
+ expect(first).toEqual({
+ granted: true,
+ waitMs: 0,
+ waitScope: null,
+ halfOpenProbe: false,
+ reservedStartAtMs: 1_000,
+ })
+ host.markRequestStarted(
+ REGISTRABLE_DOMAIN,
+ ORIGIN,
+ 1_000,
+ first.granted ? first.reservedStartAtMs : null,
+ 'image'
+ )
+ expect(host.take(REGISTRABLE_DOMAIN, ORIGIN, 1_000, DEADLINE_MS)).toEqual({
+ granted: true,
+ waitMs: 1_000,
+ waitScope: 'origin_crawl_delay',
+ halfOpenProbe: false,
+ reservedStartAtMs: null,
+ })
+ expect(host.take(REGISTRABLE_DOMAIN, ORIGIN, 1_000, DEADLINE_MS)).toEqual({
+ granted: true,
+ waitMs: 1_000,
+ waitScope: 'origin_crawl_delay',
+ halfOpenProbe: false,
+ reservedStartAtMs: null,
+ })
+ })
+
+ it('uses the larger robots crawl delay', () => {
+ const host = budget({ burst: 5 })
+ host.setCrawlDelay(ORIGIN, 4_000, 1_000)
+
+ const first = host.take(REGISTRABLE_DOMAIN, ORIGIN, 1_000, DEADLINE_MS)
+ host.markRequestStarted(
+ REGISTRABLE_DOMAIN,
+ ORIGIN,
+ 1_000,
+ first.granted ? first.reservedStartAtMs : null,
+ 'image'
+ )
+
+ expect(host.take(REGISTRABLE_DOMAIN, ORIGIN, 1_000, DEADLINE_MS)).toMatchObject({ waitMs: 4_000 })
+ })
+
+ it('keeps crawl delay separate for sibling origins', () => {
+ const host = budget({ burst: 5 })
+ host.setCrawlDelay('https://a.example.com', 4_000, 1_000)
+
+ host.take(REGISTRABLE_DOMAIN, 'https://a.example.com', 1_000, DEADLINE_MS)
+
+ expect(host.take(REGISTRABLE_DOMAIN, 'https://b.example.com', 1_000, DEADLINE_MS)).toMatchObject({
+ granted: true,
+ waitMs: 0,
+ })
+ })
+
+ it('shares token capacity across sibling origins', () => {
+ const host = budget({ burst: 1 })
+
+ expect(host.take(REGISTRABLE_DOMAIN, 'https://a.example.com', 1_000, DEADLINE_MS)).toMatchObject({
+ granted: true,
+ waitMs: 0,
+ })
+ expect(host.take(REGISTRABLE_DOMAIN, 'https://b.example.com', 1_000, DEADLINE_MS)).toMatchObject({
+ granted: true,
+ waitMs: 1_000,
+ })
+ })
+
+ it('returns a token when a reserved request cannot run', () => {
+ const host = budget({ burst: 1 })
+ const grant = host.take(REGISTRABLE_DOMAIN, ORIGIN, 1_000, DEADLINE_MS)
+ expect(grant.granted).toBe(true)
+ host.returnGrant(REGISTRABLE_DOMAIN, ORIGIN, 1_000, grant.granted ? grant.reservedStartAtMs : null)
+
+ expect(host.take(REGISTRABLE_DOMAIN, ORIGIN, 1_000, DEADLINE_MS)).toMatchObject({ granted: true })
+ })
+
+ it('applies half-to-full jitter exponential backoff and Retry-After', () => {
+ const host = budget()
+
+ expect(host.recordTransientFailure(REGISTRABLE_DOMAIN, 1_000)).toBe(60_000)
+ expect(host.recordTransientFailure(REGISTRABLE_DOMAIN, 1_000, 180_000)).toBe(180_000)
+ expect(host.take(REGISTRABLE_DOMAIN, ORIGIN, 180_999, DEADLINE_MS)).toEqual({
+ granted: false,
+ reason: 'backoff',
+ backoffReason: 'retry_after',
+ waitMs: 1,
+ })
+ })
+
+ it('does not shorten a future block when a concurrent request succeeds', () => {
+ const host = budget()
+ host.recordTransientFailure(REGISTRABLE_DOMAIN, 1_000, 180_000)
+ host.recordCompletedResponse(REGISTRABLE_DOMAIN, 2_000)
+
+ expect(host.take(REGISTRABLE_DOMAIN, ORIGIN, 2_000, DEADLINE_MS)).toEqual({
+ granted: false,
+ reason: 'backoff',
+ backoffReason: 'retry_after',
+ waitMs: 179_000,
+ })
+ })
+
+ it('opens the breaker after three consecutive transient failures', () => {
+ const host = budget()
+ for (let failure = 0; failure < OPTIONS.breakerFailures; failure++) {
+ host.recordTransientFailure(REGISTRABLE_DOMAIN, 1_000)
+ }
+
+ expect(host.take(REGISTRABLE_DOMAIN, ORIGIN, 1_000, DEADLINE_MS)).toEqual({
+ granted: false,
+ reason: 'breaker_open',
+ waitMs: 240_000,
+ })
+ expect(host.blockedRegistrableDomains(1_000)).toBe(1)
+ })
+
+ it('applies a transient failure to sibling origins', () => {
+ const host = budget()
+ host.recordTransientFailure(REGISTRABLE_DOMAIN, 1_000)
+
+ expect(host.take(REGISTRABLE_DOMAIN, 'https://other.example.com', 1_000, DEADLINE_MS)).toMatchObject({
+ granted: false,
+ reason: 'backoff',
+ })
+ })
+
+ it('grants one half-open probe after the breaker cooldown', () => {
+ const host = budget()
+ for (let failure = 0; failure < OPTIONS.breakerFailures; failure++) {
+ host.recordTransientFailure(REGISTRABLE_DOMAIN, 1_000)
+ }
+ const afterCooldownMs = 1_000 + 240_000
+
+ expect(host.take(REGISTRABLE_DOMAIN, ORIGIN, afterCooldownMs, DEADLINE_MS)).toEqual({
+ granted: true,
+ waitMs: 0,
+ waitScope: null,
+ halfOpenProbe: true,
+ reservedStartAtMs: afterCooldownMs,
+ })
+ expect(host.take(REGISTRABLE_DOMAIN, 'https://other.example.com', afterCooldownMs, DEADLINE_MS)).toEqual({
+ granted: false,
+ reason: 'breaker_open',
+ waitMs: OPTIONS.breakerCooldownMs,
+ })
+ })
+
+ it('closes the breaker after the half-open probe gets a response', () => {
+ const host = budget()
+ for (let failure = 0; failure < OPTIONS.breakerFailures; failure++) {
+ host.recordTransientFailure(REGISTRABLE_DOMAIN, 1_000)
+ }
+ const afterCooldownMs = 1_000 + 240_000
+ host.take(REGISTRABLE_DOMAIN, ORIGIN, afterCooldownMs, DEADLINE_MS)
+ host.recordCompletedResponse(REGISTRABLE_DOMAIN, afterCooldownMs)
+
+ expect(host.take(REGISTRABLE_DOMAIN, ORIGIN, afterCooldownMs, DEADLINE_MS)).toMatchObject({ granted: true })
+ })
+
+ it('limits concurrent connections across sibling origins', () => {
+ const host = budget()
+ const origins = ['https://a.example.com', 'https://b.example.com', 'https://c.example.com']
+ for (const origin of origins) {
+ host.take(REGISTRABLE_DOMAIN, origin, 1_000, DEADLINE_MS)
+ }
+
+ expect(host.acquireConnection(REGISTRABLE_DOMAIN, origins[0])).toBe(true)
+ expect(host.acquireConnection(REGISTRABLE_DOMAIN, origins[1])).toBe(true)
+ expect(host.acquireConnection(REGISTRABLE_DOMAIN, origins[2])).toBe(false)
+ host.releaseConnection(REGISTRABLE_DOMAIN, origins[0])
+ expect(host.acquireConnection(REGISTRABLE_DOMAIN, origins[2])).toBe(true)
+ })
+
+ it('does not evict an origin with a connection or scheduled request', () => {
+ const host = budget({ maxTrackedRegistrableDomains: 2, maxTrackedOrigins: 2, maxConcurrent: 1 })
+ host.take('busy.example', 'https://busy.example', 1_000, DEADLINE_MS)
+ host.acquireConnection('busy.example', 'https://busy.example')
+ host.requestScheduled('https://config.example', 1_000)
+ host.take('config.example', 'https://config.example', 1_000, DEADLINE_MS, true)
+
+ expect(host.take('new.example', 'https://new.example', 1_000, DEADLINE_MS)).toEqual({
+ granted: false,
+ reason: 'registrable_domain_map_full',
+ waitMs: 0,
+ })
+ expect(host.acquireConnection('busy.example', 'https://busy.example')).toBe(false)
+ })
+
+ it('evicts an idle full-budget origin', () => {
+ const host = budget({ maxTrackedRegistrableDomains: 1, maxTrackedOrigins: 1 })
+ const grant = host.take('idle.example', 'https://idle.example', 1_000, DEADLINE_MS)
+ expect(grant.granted).toBe(true)
+ host.returnGrant('idle.example', 'https://idle.example', 1_000, grant.granted ? grant.reservedStartAtMs : null)
+
+ expect(host.take('new.example', 'https://new.example', 1_000, DEADLINE_MS)).toMatchObject({ granted: true })
+ expect(host.trackedRegistrableDomains).toBe(1)
+ expect(host.trackedOrigins).toBe(1)
+ })
+
+ it.each([
+ ['during its cooldown', 1_000 + 239_999, { granted: false, reason: 'registrable_domain_map_full' }],
+ ['after its cooldown', 1_000 + 240_000, { granted: true }],
+ ])('evicts an idle domain whose breaker is open only %s', (_name, nowMs, expected) => {
+ const host = budget({ maxTrackedRegistrableDomains: 1 })
+ for (let failure = 0; failure < OPTIONS.breakerFailures; failure++) {
+ host.recordTransientFailure('failed.example', 1_000)
+ }
+
+ expect(host.take('new.example', 'https://new.example', nowMs, DEADLINE_MS)).toMatchObject(expected)
+ })
+
+ it('returns a crawl wait that extends beyond the pass deadline', () => {
+ const host = budget({ burst: 5 })
+ host.setCrawlDelay(ORIGIN, 600_000, 1_000)
+ const first = host.take(REGISTRABLE_DOMAIN, ORIGIN, 1_000, DEADLINE_MS)
+ host.markRequestStarted(
+ REGISTRABLE_DOMAIN,
+ ORIGIN,
+ 1_000,
+ first.granted ? first.reservedStartAtMs : null,
+ 'image'
+ )
+
+ expect(host.take(REGISTRABLE_DOMAIN, ORIGIN, 1_000, 20_000)).toEqual({
+ granted: false,
+ reason: 'deadline',
+ waitMs: 600_000,
+ waitScope: 'origin_crawl_delay',
+ })
+ })
+})
diff --git a/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/host-budget.ts b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/host-budget.ts
new file mode 100644
index 000000000000..c56722fa846b
--- /dev/null
+++ b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/host-budget.ts
@@ -0,0 +1,454 @@
+import type { ImageFetchBlockReason } from './block-reason'
+
+export interface HostBudgetOptions {
+ requestsPerSecond: number
+ burst: number
+ maxConcurrent: number
+ breakerFailures: number
+ breakerCooldownMs: number
+ breakerMaxCooldownMs: number
+ maxTrackedRegistrableDomains: number
+ maxTrackedOrigins: number
+ random?: () => number
+}
+
+export type BudgetBlockReason =
+ | 'breaker_open'
+ | 'backoff'
+ | 'deadline'
+ | 'origin_map_full'
+ | 'registrable_domain_map_full'
+export type BudgetWaitScope = 'origin_crawl_delay' | 'registrable_domain_rate'
+export type HostBackoffReason = Extract<ImageFetchBlockReason, 'retry_after' | 'transient_backoff'>
+export type BudgetGrant =
+ | {
+ granted: true
+ waitMs: number
+ waitScope: BudgetWaitScope | null
+ halfOpenProbe: boolean
+ reservedStartAtMs: number | null
+ }
+ | {
+ granted: false
+ reason: BudgetBlockReason
+ waitMs: number
+ backoffReason?: HostBackoffReason
+ waitScope?: BudgetWaitScope
+ }
+
+const EVICTION_SCAN_LIMIT = 64
+
+interface RegistrableDomainState {
+ inFlight: number
+ pendingGrants: number
+ tokens: number
+ lastRefillMs: number
+ consecutiveTransientFailures: number
+ blockedUntilMs: number
+ backoffReason: HostBackoffReason | null
+ breakerOpen: boolean
+ halfOpenProbeInFlight: boolean
+}
+
+interface OriginState {
+ inFlight: number
+ pendingRequests: number
+ lastRequestStartedAtMs: number | null
+ reservedStartTimesMs: number[]
+ crawlDelayMs: number
+}
+
+export class HostBudget {
+ private readonly registrableDomains = new Map<string, RegistrableDomainState>()
+ private readonly origins = new Map<string, OriginState>()
+ private readonly random: () => number
+
+ constructor(private readonly options: HostBudgetOptions) {
+ for (const [name, value] of Object.entries(options)) {
+ if (name === 'random') {
+ continue
+ }
+ const number = value as number
+ const invalid = !Number.isFinite(number) || (name === 'requestsPerSecond' ? number < 0 : number <= 0)
+ if (invalid) {
+ const validRange =
+ name === 'requestsPerSecond'
+ ? 'a finite number greater than or equal to zero'
+ : 'a finite number greater than zero'
+ throw new Error(`the image fetch request budget ${name} must be ${validRange}, got ${value}`)
+ }
+ }
+ this.random = options.random ?? Math.random
+ }
+
+ public take(
+ registrableDomain: string,
+ origin: string,
+ nowMs: number,
+ deadlineMs: number,
+ ignoreImageDelay = false
+ ): BudgetGrant {
+ const registrableDomainState = this.registrableDomainStateFor(registrableDomain, nowMs)
+ if (!registrableDomainState) {
+ return { granted: false, reason: 'registrable_domain_map_full', waitMs: 0 }
+ }
+ const originState = this.originStateFor(origin, nowMs)
+ if (!originState) {
+ return { granted: false, reason: 'origin_map_full', waitMs: 0 }
+ }
+ let halfOpenProbe = false
+ if (!ignoreImageDelay) {
+ if (registrableDomainState.blockedUntilMs > nowMs) {
+ if (registrableDomainState.breakerOpen) {
+ return {
+ granted: false,
+ reason: 'breaker_open',
+ waitMs: registrableDomainState.blockedUntilMs - nowMs,
+ }
+ }
+ return {
+ granted: false,
+ reason: 'backoff',
+ waitMs: registrableDomainState.blockedUntilMs - nowMs,
+ backoffReason: registrableDomainState.backoffReason ?? undefined,
+ }
+ }
+ if (registrableDomainState.breakerOpen) {
+ if (registrableDomainState.halfOpenProbeInFlight) {
+ return { granted: false, reason: 'breaker_open', waitMs: this.options.breakerCooldownMs }
+ }
+ halfOpenProbe = true
+ }
+ }
+ this.refill(registrableDomainState, nowMs)
+ const tokenWaitMs =
+ this.options.requestsPerSecond === 0 || registrableDomainState.tokens >= 1
+ ? 0
+ : Math.ceil(((1 - registrableDomainState.tokens) / this.options.requestsPerSecond) * 1000)
+ const previousStartMs = originState.lastRequestStartedAtMs ?? Number.NEGATIVE_INFINITY
+ const crawlWaitMs = ignoreImageDelay ? 0 : Math.max(0, previousStartMs + originState.crawlDelayMs - nowMs)
+ const waitMs = Math.max(tokenWaitMs, crawlWaitMs)
+ const waitScope: BudgetWaitScope | null =
+ waitMs === 0 ? null : crawlWaitMs > tokenWaitMs ? 'origin_crawl_delay' : 'registrable_domain_rate'
+ if (nowMs + waitMs > deadlineMs) {
+ return { granted: false, reason: 'deadline', waitMs, waitScope: waitScope ?? undefined }
+ }
+ if (waitMs > 0) {
+ return {
+ granted: true,
+ waitMs,
+ waitScope,
+ halfOpenProbe: false,
+ reservedStartAtMs: null,
+ }
+ }
+ if (halfOpenProbe) {
+ registrableDomainState.halfOpenProbeInFlight = true
+ }
+ if (this.options.requestsPerSecond > 0) {
+ registrableDomainState.tokens -= 1
+ }
+ registrableDomainState.pendingGrants += 1
+ const reservedStartAtMs = ignoreImageDelay ? null : nowMs
+ if (reservedStartAtMs !== null) {
+ originState.reservedStartTimesMs.push(reservedStartAtMs)
+ }
+ return { granted: true, waitMs, waitScope, halfOpenProbe, reservedStartAtMs }
+ }
+
+ public markRequestStarted(
+ registrableDomain: string,
+ origin: string,
+ nowMs: number,
+ reservedStartAtMs: number | null,
+ requestKind: 'configuration' | 'image'
+ ): void {
+ const registrableDomainState = this.registrableDomains.get(registrableDomain)
+ if (registrableDomainState) {
+ registrableDomainState.pendingGrants = Math.max(0, registrableDomainState.pendingGrants - 1)
+ }
+ const originState = this.origins.get(origin)
+ if (originState && requestKind === 'image') {
+ this.removeReservation(originState, reservedStartAtMs)
+ originState.lastRequestStartedAtMs = nowMs
+ }
+ }
+
+ public blockedForMs(registrableDomain: string, nowMs: number): number {
+ const state = this.registrableDomains.get(registrableDomain)
+ if (!state) {
+ return 0
+ }
+ if (state.breakerOpen && state.halfOpenProbeInFlight && state.blockedUntilMs <= nowMs) {
+ return this.options.breakerCooldownMs
+ }
+ return Math.max(0, state.blockedUntilMs - nowMs)
+ }
+
+ public returnGrant(
+ registrableDomain: string,
+ origin: string,
+ nowMs: number,
+ reservedStartAtMs: number | null,
+ halfOpenProbe = false
+ ): void {
+ const originState = this.origins.get(origin)
+ if (originState) {
+ this.removeReservation(originState, reservedStartAtMs)
+ }
+ const registrableDomainState = this.registrableDomains.get(registrableDomain)
+ if (!registrableDomainState) {
+ return
+ }
+ registrableDomainState.pendingGrants = Math.max(0, registrableDomainState.pendingGrants - 1)
+ if (this.options.requestsPerSecond > 0) {
+ this.refill(registrableDomainState, nowMs)
+ registrableDomainState.tokens = Math.min(this.options.burst, registrableDomainState.tokens + 1)
+ }
+ if (halfOpenProbe && registrableDomainState.breakerOpen) {
+ registrableDomainState.halfOpenProbeInFlight = false
+ }
+ }
+
+ public acquireConnection(registrableDomain: string, origin: string): boolean {
+ const registrableDomainState = this.registrableDomains.get(registrableDomain)
+ const originState = this.origins.get(origin)
+ if (!registrableDomainState || !originState || registrableDomainState.inFlight >= this.options.maxConcurrent) {
+ return false
+ }
+ registrableDomainState.inFlight += 1
+ originState.inFlight += 1
+ return true
+ }
+
+ public availableConnections(registrableDomain: string, connectionLimit = this.options.maxConcurrent): number {
+ const inFlight = this.registrableDomains.get(registrableDomain)?.inFlight ?? 0
+ return Math.max(0, Math.min(this.options.maxConcurrent, connectionLimit) - inFlight)
+ }
+
+ public releaseConnection(registrableDomain: string, origin: string): void {
+ const registrableDomainState = this.registrableDomains.get(registrableDomain)
+ if (registrableDomainState) {
+ registrableDomainState.inFlight = Math.max(0, registrableDomainState.inFlight - 1)
+ }
+ const originState = this.origins.get(origin)
+ if (originState) {
+ originState.inFlight = Math.max(0, originState.inFlight - 1)
+ }
+ }
+
+ public requestScheduled(origin: string, nowMs: number): boolean {
+ const state = this.originStateFor(origin, nowMs)
+ if (!state) {
+ return false
+ }
+ state.pendingRequests += 1
+ return true
+ }
+
+ public requestFinished(origin: string): void {
+ const state = this.origins.get(origin)
+ if (state) {
+ state.pendingRequests = Math.max(0, state.pendingRequests - 1)
+ }
+ }
+
+ public setCrawlDelay(origin: string, crawlDelayMs: number, nowMs: number): boolean {
+ const state = this.originStateFor(origin, nowMs)
+ if (!state) {
+ return false
+ }
+ state.crawlDelayMs = Math.max(0, crawlDelayMs)
+ return true
+ }
+
+ public originCrawlDelayMs(origin: string): number {
+ return this.origins.get(origin)?.crawlDelayMs ?? 0
+ }
+
+ public originNextImageStartAtMs(origin: string): number {
+ const state = this.origins.get(origin)
+ if (!state || state.lastRequestStartedAtMs === null) {
+ return 0
+ }
+ return state.lastRequestStartedAtMs + state.crawlDelayMs
+ }
+
+ public recordTransientFailure(registrableDomain: string, nowMs: number, retryAfterMs?: number): number {
+ const state = this.registrableDomainStateFor(registrableDomain, nowMs)
+ if (!state) {
+ return this.options.breakerCooldownMs
+ }
+ state.consecutiveTransientFailures += 1
+ const maximumDelayMs = Math.min(
+ this.options.breakerCooldownMs * 2 ** (state.consecutiveTransientFailures - 1),
+ this.options.breakerMaxCooldownMs
+ )
+ const minimumDelayMs = Math.ceil(maximumDelayMs / 2)
+ const jitteredDelayMs = Math.min(
+ maximumDelayMs,
+ minimumDelayMs + Math.floor(this.random() * (maximumDelayMs - minimumDelayMs + 1))
+ )
+ const delayMs = Math.max(jitteredDelayMs, retryAfterMs ?? 0)
+ const blockedUntilMs = nowMs + delayMs
+ if (blockedUntilMs >= state.blockedUntilMs) {
+ state.backoffReason = retryAfterMs === undefined ? 'transient_backoff' : 'retry_after'
+ }
+ state.blockedUntilMs = Math.max(state.blockedUntilMs, blockedUntilMs)
+ if (state.consecutiveTransientFailures >= this.options.breakerFailures) {
+ state.breakerOpen = true
+ state.halfOpenProbeInFlight = false
+ }
+ return delayMs
+ }
+
+ public recordCompletedResponse(registrableDomain: string, nowMs: number): void {
+ const state = this.registrableDomains.get(registrableDomain)
+ if (!state) {
+ return
+ }
+ state.consecutiveTransientFailures = 0
+ state.blockedUntilMs = Math.max(state.blockedUntilMs, nowMs)
+ if (state.blockedUntilMs <= nowMs) {
+ state.backoffReason = null
+ }
+ state.breakerOpen = false
+ state.halfOpenProbeInFlight = false
+ }
+
+ public get trackedRegistrableDomains(): number {
+ return this.registrableDomains.size
+ }
+
+ public get trackedOrigins(): number {
+ return this.origins.size
+ }
+
+ public evictedWhileBlocked = 0
+
+ public blockedRegistrableDomains(nowMs: number): number {
+ let blocked = 0
+ for (const state of this.registrableDomains.values()) {
+ if (state.blockedUntilMs > nowMs || state.breakerOpen) {
+ blocked += 1
+ }
+ }
+ return blocked
+ }
+
+ private refill(state: RegistrableDomainState, nowMs: number): void {
+ const elapsedMs = Math.max(0, nowMs - state.lastRefillMs)
+ state.lastRefillMs = nowMs
+ state.tokens = Math.min(this.options.burst, state.tokens + (elapsedMs / 1000) * this.options.requestsPerSecond)
+ }
+
+ private registrableDomainStateFor(registrableDomain: string, nowMs: number): RegistrableDomainState | undefined {
+ const existing = this.registrableDomains.get(registrableDomain)
+ if (existing) {
+ this.registrableDomains.delete(registrableDomain)
+ this.registrableDomains.set(registrableDomain, existing)
+ return existing
+ }
+ if (!this.evictRegistrableDomainIfFull(nowMs)) {
+ return undefined
+ }
+ const state: RegistrableDomainState = {
+ inFlight: 0,
+ pendingGrants: 0,
+ tokens: this.options.burst,
+ lastRefillMs: nowMs,
+ consecutiveTransientFailures: 0,
+ blockedUntilMs: 0,
+ backoffReason: null,
+ breakerOpen: false,
+ halfOpenProbeInFlight: false,
+ }
+ this.registrableDomains.set(registrableDomain, state)
+ return state
+ }
+
+ private originStateFor(origin: string, nowMs: number): OriginState | undefined {
+ const existing = this.origins.get(origin)
+ if (existing) {
+ this.origins.delete(origin)
+ this.origins.set(origin, existing)
+ return existing
+ }
+ if (!this.evictOriginIfFull(nowMs)) {
+ return undefined
+ }
+ const state: OriginState = {
+ inFlight: 0,
+ pendingRequests: 0,
+ lastRequestStartedAtMs: null,
+ reservedStartTimesMs: [],
+ crawlDelayMs: 0,
+ }
+ this.origins.set(origin, state)
+ return state
+ }
+
+ private evictRegistrableDomainIfFull(nowMs: number): boolean {
+ if (this.registrableDomains.size < this.options.maxTrackedRegistrableDomains) {
+ return true
+ }
+ for (let scanned = 0; scanned < EVICTION_SCAN_LIMIT; scanned++) {
+ const oldest = this.registrableDomains.entries().next().value as
+ | [string, RegistrableDomainState]
+ | undefined
+ if (!oldest) {
+ return true
+ }
+ const [registrableDomain, state] = oldest
+ this.refill(state, nowMs)
+ const eligible =
+ state.inFlight === 0 &&
+ state.pendingGrants === 0 &&
+ state.blockedUntilMs <= nowMs &&
+ !state.halfOpenProbeInFlight &&
+ state.tokens >= this.options.burst
+ if (eligible) {
+ this.registrableDomains.delete(registrableDomain)
+ return true
+ }
+ this.registrableDomains.delete(registrableDomain)
+ this.registrableDomains.set(registrableDomain, state)
+ }
+ return false
+ }
+
+ private evictOriginIfFull(nowMs: number): boolean {
+ if (this.origins.size < this.options.maxTrackedOrigins) {
+ return true
+ }
+ for (let scanned = 0; scanned < EVICTION_SCAN_LIMIT; scanned++) {
+ const oldest = this.origins.entries().next().value as [string, OriginState] | undefined
+ if (!oldest) {
+ return true
+ }
+ const [origin, state] = oldest
+ const eligible =
+ state.inFlight === 0 &&
+ state.pendingRequests === 0 &&
+ state.reservedStartTimesMs.length === 0 &&
+ (state.lastRequestStartedAtMs === null || state.lastRequestStartedAtMs + state.crawlDelayMs <= nowMs)
+ if (eligible) {
+ this.origins.delete(origin)
+ return true
+ }
+ this.origins.delete(origin)
+ this.origins.set(origin, state)
+ }
+ return false
+ }
+
+ private removeReservation(state: OriginState, reservedStartAtMs: number | null): void {
+ if (reservedStartAtMs === null) {
+ return
+ }
+ const index = state.reservedStartTimesMs.indexOf(reservedStartAtMs)
+ if (index >= 0) {
+ state.reservedStartTimesMs.splice(index, 1)
+ }
+ }
+}
diff --git a/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/image-fetch-batch-joiner.test.ts b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/image-fetch-batch-joiner.test.ts
new file mode 100644
index 000000000000..afd17d066a46
--- /dev/null
+++ b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/image-fetch-batch-joiner.test.ts
@@ -0,0 +1,171 @@
+import { Message } from 'node-rdkafka'
+import { register } from 'prom-client'
+
+import { IMAGE_FETCH_BATCH_JOIN_TIMEOUT_MS, ImageFetchBatchJoiner } from './image-fetch-batch-joiner'
+
+function message(partition: number): Message {
+ return {
+ topic: 'session_replay_image_fetch',
+ partition,
+ offset: 1,
+ value: Buffer.from(`partition-${partition}`),
+ size: 11,
+ }
+}
+
+function deferred(): { promise: Promise<void>; resolve: () => void; reject: (error: unknown) => void } {
+ let resolve!: () => void
+ let reject!: (error: unknown) => void
+ const promise = new Promise<void>((done, fail) => {
+ resolve = done
+ reject = fail
+ })
+ return { promise, resolve, reject }
+}
+
+describe('ImageFetchBatchJoiner', () => {
+ beforeEach(() => {
+ jest.useFakeTimers()
+ register.resetMetrics()
+ })
+ afterEach(() => jest.useRealTimers())
+
+ it.each([1, 2, 16])('joins batches from %i consumers before processing', async (consumerCount) => {
+ const pass = deferred()
+ const processBatch = jest.fn(() => pass.promise)
+ const joiner = new ImageFetchBatchJoiner(consumerCount, processBatch)
+
+ const messages = Array.from({ length: consumerCount }, (_, partition) => message(partition))
+ const pending = messages.slice(0, -1).map((item) => joiner.handleBatch([item]))
+ expect(processBatch).not.toHaveBeenCalled()
+ pending.push(joiner.handleBatch([messages[consumerCount - 1]]))
+
+ const batches = await Promise.all(pending)
+ expect(processBatch).toHaveBeenCalledTimes(1)
+ expect(processBatch).toHaveBeenCalledWith(messages)
+ expect(batches.every((batch) => batch?.backgroundTask === batches[0]?.backgroundTask)).toBe(true)
+ const completed = jest.fn()
+ const completion = Promise.all(batches.map((batch) => batch?.backgroundTask ?? Promise.resolve())).then(
+ completed
+ )
+ await jest.advanceTimersByTimeAsync(0)
+ expect(completed).not.toHaveBeenCalled()
+ pass.resolve()
+ await completion
+ })
+
+ it.each([2, 16])(
+ 'processes a partial batch with a target of %i when the join window ends',
+ async (consumerCount) => {
+ const processBatch = jest.fn(() => Promise.resolve())
+ const joiner = new ImageFetchBatchJoiner(consumerCount, processBatch)
+
+ const pending = joiner.handleBatch([message(0)])
+ await jest.advanceTimersByTimeAsync(IMAGE_FETCH_BATCH_JOIN_TIMEOUT_MS - 1)
+ expect(processBatch).not.toHaveBeenCalled()
+ await jest.advanceTimersByTimeAsync(1)
+
+ await (
+ await pending
+ )?.backgroundTask
+ expect(processBatch).toHaveBeenCalledWith([message(0)])
+ }
+ )
+
+ it('rejects active and pending consumers when processing fails', async () => {
+ const error = new Error('fetch pass failed')
+ const pass = deferred()
+ const joiner = new ImageFetchBatchJoiner(2, () => pass.promise)
+
+ const first = joiner.handleBatch([message(0)])
+ await jest.advanceTimersByTimeAsync(IMAGE_FETCH_BATCH_JOIN_TIMEOUT_MS)
+ const firstCompletion = (await first)?.backgroundTask
+ const second = joiner.handleBatch([message(1)])
+ const active = register.getSingleMetric('ai_research_v4_image_fetch_stage_active')!
+ expect((await active.get()).values).toEqual(
+ expect.arrayContaining([
+ expect.objectContaining({ labels: { stage: 'consumer_join' }, value: 1 }),
+ expect.objectContaining({ labels: { stage: 'consumer_process' }, value: 1 }),
+ ])
+ )
+ const rejected = Promise.all([expect(firstCompletion).rejects.toBe(error), expect(second).rejects.toBe(error)])
+ pass.reject(error)
+ await rejected
+ await expect(joiner.handleBatch([message(2)])).rejects.toBe(error)
+ expect((await active.get()).values).toEqual(
+ expect.arrayContaining([
+ expect.objectContaining({ labels: { stage: 'consumer_join' }, value: 0 }),
+ expect.objectContaining({ labels: { stage: 'consumer_process' }, value: 0 }),
+ ])
+ )
+ })
+
+ it('does not hold a later partial batch behind active processing', async () => {
+ const firstPass = deferred()
+ const secondPass = deferred()
+ const processBatch = jest
+ .fn()
+ .mockImplementationOnce(() => firstPass.promise)
+ .mockImplementationOnce(() => secondPass.promise)
+ const joiner = new ImageFetchBatchJoiner(2, processBatch)
+
+ const first = joiner.handleBatch([message(0)])
+ await jest.advanceTimersByTimeAsync(IMAGE_FETCH_BATCH_JOIN_TIMEOUT_MS)
+ const second = joiner.handleBatch([message(1)])
+ await jest.advanceTimersByTimeAsync(IMAGE_FETCH_BATCH_JOIN_TIMEOUT_MS)
+ expect(processBatch).toHaveBeenCalledTimes(2)
+
+ secondPass.resolve()
+ await (
+ await second
+ )?.backgroundTask
+ firstPass.resolve()
+ await (
+ await first
+ )?.backgroundTask
+ expect(processBatch).toHaveBeenLastCalledWith([message(1)])
+ })
+
+ it('keeps a fast consumer out of the same group twice while overlapping processing', async () => {
+ const firstPass = deferred()
+ const secondPass = deferred()
+ const processBatch = jest
+ .fn()
+ .mockImplementationOnce(() => firstPass.promise)
+ .mockImplementationOnce(() => secondPass.promise)
+ const joiner = new ImageFetchBatchJoiner(2, processBatch)
+ const secondFromFastConsumer = jest.fn(() => joiner.handleBatch([message(0)]))
+
+ const firstFast = joiner.handleBatch([message(0)])
+ const secondFast = firstFast.then(secondFromFastConsumer)
+ await jest.advanceTimersByTimeAsync(0)
+ expect(secondFromFastConsumer).not.toHaveBeenCalled()
+
+ const firstSlow = joiner.handleBatch([message(1)])
+ await firstSlow
+ await jest.advanceTimersByTimeAsync(0)
+ expect(secondFromFastConsumer).toHaveBeenCalledTimes(1)
+ expect(processBatch).toHaveBeenCalledTimes(1)
+
+ const secondSlow = joiner.handleBatch([message(1)])
+ const secondBatches = await Promise.all([secondFast, secondSlow])
+ expect(processBatch.mock.calls).toEqual([[[message(0), message(1)]], [[message(0), message(1)]]])
+ const active = register.getSingleMetric('ai_research_v4_image_fetch_stage_active')!
+ expect((await active.get()).values).toEqual(
+ expect.arrayContaining([expect.objectContaining({ labels: { stage: 'consumer_process' }, value: 4 })])
+ )
+
+ secondPass.resolve()
+ await Promise.all(secondBatches.map((batch) => batch?.backgroundTask ?? Promise.resolve()))
+ expect((await active.get()).values).toEqual(
+ expect.arrayContaining([expect.objectContaining({ labels: { stage: 'consumer_process' }, value: 2 })])
+ )
+ firstPass.resolve()
+ await (
+ await firstFast
+ )?.backgroundTask
+ expect((await active.get()).values).toEqual(
+ expect.arrayContaining([expect.objectContaining({ labels: { stage: 'consumer_process' }, value: 0 })])
+ )
+ })
+})
diff --git a/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/image-fetch-batch-joiner.ts b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/image-fetch-batch-joiner.ts
new file mode 100644
index 000000000000..7da871ec9539
--- /dev/null
+++ b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/image-fetch-batch-joiner.ts
@@ -0,0 +1,128 @@
+import { Message } from 'node-rdkafka'
+
+import { ImageFetchProcessingMetrics, ProcessingStageTimer } from './processing-metrics'
+
+export const IMAGE_FETCH_BATCH_JOIN_TIMEOUT_MS = 500
+export const MAX_IMAGE_FETCH_BATCHES_PER_PASS = 16
+
+type BatchProcessor = (messages: Message[]) => Promise<void>
+
+type DispatchedBatch = { backgroundTask: Promise<void> }
+
+type BatchWaiter = {
+ resolve: (batch: DispatchedBatch) => void
+ reject: (error: unknown) => void
+ timer: ProcessingStageTimer
+}
+
+type PendingBatchGroup = {
+ batches: Message[][]
+ waiters: BatchWaiter[]
+ timeout?: NodeJS.Timeout
+}
+
+export function assertImageFetchBatchTarget(targetBatchCount: number): void {
+ if (
+ !Number.isSafeInteger(targetBatchCount) ||
+ targetBatchCount < 1 ||
+ targetBatchCount > MAX_IMAGE_FETCH_BATCHES_PER_PASS
+ ) {
+ throw new Error(`image fetch batch target must be an integer between 1 and ${MAX_IMAGE_FETCH_BATCHES_PER_PASS}`)
+ }
+}
+
+export class ImageFetchBatchJoiner {
+ private pendingGroup?: PendingBatchGroup
+ private readonly activeProcessing = new Set<Promise<void>>()
+ private failed = false
+ private failure: unknown
+
+ constructor(
+ private readonly targetBatchCount: number,
+ private readonly processBatch: BatchProcessor
+ ) {
+ assertImageFetchBatchTarget(targetBatchCount)
+ }
+
+ public handleBatch(messages: Message[]): Promise<DispatchedBatch | void> {
+ if (messages.length === 0) {
+ return Promise.resolve()
+ }
+ if (this.failed) {
+ return Promise.reject(this.failure)
+ }
+
+ return new Promise<DispatchedBatch>((resolve, reject) => {
+ const group = this.pendingGroup ?? this.createPendingGroup()
+ group.batches.push(messages)
+ group.waiters.push({ resolve, reject, timer: ImageFetchProcessingMetrics.start('consumer_join') })
+ if (group.batches.length >= this.targetBatchCount) {
+ this.dispatch(group)
+ }
+ })
+ }
+
+ public async waitForProcessing(): Promise<void> {
+ await Promise.allSettled(this.activeProcessing)
+ }
+
+ private createPendingGroup(): PendingBatchGroup {
+ const group: PendingBatchGroup = { batches: [], waiters: [] }
+ group.timeout = setTimeout(() => this.dispatch(group), IMAGE_FETCH_BATCH_JOIN_TIMEOUT_MS)
+ this.pendingGroup = group
+ return group
+ }
+
+ private dispatch(group: PendingBatchGroup): void {
+ if (this.pendingGroup !== group) {
+ return
+ }
+ this.pendingGroup = undefined
+ if (group.timeout) {
+ clearTimeout(group.timeout)
+ }
+
+ ImageFetchProcessingMetrics.joinedBatches.observe(group.batches.length)
+ for (const { timer } of group.waiters) {
+ timer.move('consumer_process')
+ }
+ const processing = (async () => {
+ if (this.failed) {
+ throw this.failure
+ }
+ try {
+ await this.processBatch(group.batches.flat())
+ } catch (error) {
+ this.fail(error)
+ throw error
+ }
+ })().finally(() => {
+ group.waiters.forEach(({ timer }) => timer.finish())
+ this.activeProcessing.delete(processing)
+ })
+ this.activeProcessing.add(processing)
+ for (const { resolve } of group.waiters) {
+ resolve({ backgroundTask: processing })
+ }
+ }
+
+ private fail(error: unknown): void {
+ if (!this.failed) {
+ this.failed = true
+ this.failure = error
+ }
+
+ const pendingGroup = this.pendingGroup
+ if (!pendingGroup) {
+ return
+ }
+ this.pendingGroup = undefined
+ if (pendingGroup.timeout) {
+ clearTimeout(pendingGroup.timeout)
+ }
+ for (const { reject, timer } of pendingGroup.waiters) {
+ timer.finish()
+ reject(this.failure)
+ }
+ }
+}
diff --git a/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/image-fetcher.test.ts b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/image-fetcher.test.ts
new file mode 100644
index 000000000000..aac181b2bc1f
--- /dev/null
+++ b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/image-fetcher.test.ts
@@ -0,0 +1,619 @@
+import { ResolutionError, SecureRequestError, StreamedResponse, fetchStreamed } from '~/common/utils/request'
+
+import { HttpImageFetcher, ImageFetchOptions, RedirectPolicy } from './image-fetcher'
+import { ImageFetchRequestMetrics } from './metrics'
+import { WebBotAuthRequestSigner } from './web-bot-auth'
+
+jest.mock('~/common/utils/request', () => ({
+ ...jest.requireActual('~/common/utils/request'),
+ fetchStreamed: jest.fn(),
+}))
+
+const fetchStreamedMock = fetchStreamed as jest.MockedFunction<typeof fetchStreamed>
+
+const PNG = Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a, 0x00])
+const GIF = Buffer.from('GIF89a-and-then-some')
+
+const OPTIONS: ImageFetchOptions = {
+ maxBytes: 1000,
+ timeoutMs: 5000,
+ maxRedirects: 3,
+ isDifferentOrigin: () => false,
+ scheduleRequest: async (_url, _deadlineMs, request) => ({ ran: true as const, value: await request() }),
+ checkRedirectPolicy: () => Promise.resolve({ allowed: true as const, tdmrepReservation: false }),
+ tdmrepReservation: false,
+}
+
+function respond(
+ status: number,
+ headers: Record<string, string>,
+ body: { bytes: Buffer; overLimit: boolean } = { bytes: Buffer.alloc(0), overLimit: false }
+): StreamedResponse {
+ return {
+ status,
+ headers,
+ headerLines: Object.entries(headers).map(([name, value]) => ({ name, value })),
+ read: jest.fn().mockResolvedValue(body),
+ discard: jest.fn(),
+ }
+}
+
+function image(bytes: Buffer, contentType: string, extraHeaders: Record<string, string> = {}): StreamedResponse {
+ return respond(200, { 'content-type': contentType, ...extraHeaders }, { bytes, overLimit: false })
+}
+
+const NOOP_SIGNER: WebBotAuthRequestSigner = { headersForGet: () => ({}) }
+/** Everything public by default, so a test that cares about the policy says so. */
+const fetcher = (
+ policy: Partial<RedirectPolicy> = {},
+ webBotAuthSigner: WebBotAuthRequestSigner = NOOP_SIGNER
+): HttpImageFetcher =>
+ new HttpImageFetcher({ maxUrlLength: 2048, isPublicHost: () => true, ...policy }, webBotAuthSigner)
+
+describe('HttpImageFetcher', () => {
+ beforeEach(() => {
+ fetchStreamedMock.mockReset()
+ })
+ afterEach(() => jest.restoreAllMocks())
+
+ it('opts every image request into HTTP/2', async () => {
+ fetchStreamedMock.mockResolvedValue(image(PNG, 'image/png'))
+
+ await fetcher().fetch('https://cdn.example.com/a.png', OPTIONS)
+
+ expect(fetchStreamedMock).toHaveBeenCalledWith(
+ 'https://cdn.example.com/a.png',
+ expect.objectContaining({ allowH2: true })
+ )
+ })
+
+ it('identifies every request as PostHogImageFetcherBot referred by us.posthog.com', async () => {
+ fetchStreamedMock.mockResolvedValue(image(PNG, 'image/png'))
+
+ await fetcher().fetch('https://cdn.example.com/a.png', OPTIONS)
+
+ expect(fetchStreamedMock).toHaveBeenCalledWith(
+ 'https://cdn.example.com/a.png',
+ expect.objectContaining({
+ headers: expect.objectContaining({
+ 'user-agent':
+ 'PostHogImageFetcherBot/1.0 (+https://posthog.com/docs/ai-research/image-fetcher-bot)',
+ referer: 'https://us.posthog.com/',
+ }),
+ })
+ )
+ })
+
+ it('attributes image requests to the source partition', async () => {
+ const observeRequest = jest.spyOn(ImageFetchRequestMetrics, 'observeRequest')
+ fetchStreamedMock.mockResolvedValue(image(PNG, 'image/png'))
+
+ await fetcher().fetch('https://cdn.example.com/a.png', { ...OPTIONS, sourcePartitions: [7, 42] })
+
+ expect(observeRequest).toHaveBeenCalledWith('2xx', expect.any(Number), [7, 42])
+ })
+
+ it('returns the bytes of an image whose payload matches its declared type', async () => {
+ fetchStreamedMock.mockResolvedValue(image(PNG, 'image/png; charset=binary'))
+
+ const result = await fetcher().fetch('https://cdn.example.com/a.png', OPTIONS)
+
+ expect(result).toMatchObject({
+ outcome: 'ok',
+ status: 200,
+ bytes: PNG,
+ contentType: 'image/png',
+ redirects: 0,
+ currentUrl: 'https://cdn.example.com/a.png',
+ })
+ })
+
+ it('uses an ETag validator and accepts the resulting 304', async () => {
+ fetchStreamedMock.mockResolvedValue(respond(304, { etag: '"v1"' }))
+
+ const result = await fetcher().fetch('https://cdn.example.com/a.png', {
+ ...OPTIONS,
+ cache: {
+ requestTimeMs: Date.now() - 1_000,
+ responseTimeMs: Date.now() - 500,
+ etag: '"v1"',
+ lastModified: 'yesterday',
+ },
+ })
+
+ expect(fetchStreamedMock.mock.calls[0][1].headers).toMatchObject({ 'if-none-match': '"v1"' })
+ expect(fetchStreamedMock.mock.calls[0][1].headers).not.toHaveProperty('if-modified-since')
+ expect(result).toMatchObject({ outcome: 'not_modified', status: 304 })
+ })
+
+ it('does not accept an unsolicited 304', async () => {
+ fetchStreamedMock.mockResolvedValue(respond(304, {}))
+
+ const result = await fetcher().fetch('https://cdn.example.com/a.png', OPTIONS)
+
+ expect(result).toMatchObject({ outcome: 'unexpected_status', status: 304 })
+ })
+
+ it('does not send an original URL validator to a redirect target', async () => {
+ fetchStreamedMock
+ .mockResolvedValueOnce(respond(302, { location: '/moved.png' }))
+ .mockResolvedValueOnce(image(PNG, 'image/png'))
+
+ await fetcher().fetch('https://cdn.example.com/a.png', {
+ ...OPTIONS,
+ cache: {
+ requestTimeMs: Date.now() - 1_000,
+ responseTimeMs: Date.now() - 500,
+ etag: '"v1"',
+ },
+ })
+
+ expect(fetchStreamedMock.mock.calls[0][1].headers).toMatchObject({ 'if-none-match': '"v1"' })
+ expect(fetchStreamedMock.mock.calls[1][1].headers).not.toHaveProperty('if-none-match')
+ })
+
+ it.each([
+ ['a type that is not an image', 'text/html', PNG, 'not_image'],
+ ['a type outside the raster set', 'image/svg+xml', PNG, 'not_image'],
+ ['the BMP format', 'image/bmp', Buffer.from('BM'), 'not_image'],
+ ['the AVIF format', 'image/avif', Buffer.from('\x00\x00\x00\x18ftypavif', 'binary'), 'not_image'],
+ ['a payload that is not the declared format', 'image/gif', PNG, 'ok'],
+ ['a payload of the wrong raster format', 'image/png', GIF, 'ok'],
+ ])('handles %s', async (_name, contentType, bytes, outcome) => {
+ fetchStreamedMock.mockResolvedValue(image(bytes, contentType))
+
+ const result = await fetcher().fetch('https://cdn.example.com/a', OPTIONS)
+
+ expect(result).toMatchObject({ outcome })
+ })
+
+ it('refuses a declared size over the limit without reading the body', async () => {
+ const response = image(PNG, 'image/png', { 'content-length': '99999' })
+ fetchStreamedMock.mockResolvedValue(response)
+
+ const result = await fetcher().fetch('https://cdn.example.com/a.png', OPTIONS)
+
+ expect(result).toMatchObject({ outcome: 'too_large' })
+ expect(response.read).not.toHaveBeenCalled()
+ expect(response.discard).toHaveBeenCalled()
+ })
+
+ it('refuses a body that passes the limit while it is being read', async () => {
+ // The case an origin reaches when it declares no length. No check before the body catches it.
+ fetchStreamedMock.mockResolvedValue(
+ respond(200, { 'content-type': 'image/png' }, { bytes: Buffer.alloc(0), overLimit: true })
+ )
+
+ const result = await fetcher().fetch('https://cdn.example.com/a.png', OPTIONS)
+
+ expect(result).toMatchObject({ outcome: 'too_large' })
+ })
+
+ it.each([
+ [404, 'not_found'],
+ [410, 'not_found'],
+ [403, 'forbidden'],
+ [429, 'rate_limited'],
+ [503, 'server_error'],
+ [500, 'server_error'],
+ [418, 'unexpected_status'],
+ ])('maps status %s to %s', async (status, outcome) => {
+ fetchStreamedMock.mockResolvedValue(respond(status, {}))
+
+ const result = await fetcher().fetch('https://cdn.example.com/a.png', OPTIONS)
+
+ expect(result).toMatchObject({ outcome, status })
+ })
+
+ it.each([
+ ['a count of seconds', '120', 120_000],
+ ['an HTTP date', new Date(Date.now() + 90_000).toUTCString(), 90_000],
+ ])('reads a Retry-After header given as %s', async (_name, header, expectedMs) => {
+ fetchStreamedMock.mockResolvedValue(respond(429, { 'retry-after': header }))
+
+ const result = await fetcher().fetch('https://cdn.example.com/a.png', OPTIONS)
+
+ // The parser resolves a date against the clock, so the range allows for the test's own runtime.
+ expect(result.retryAfterMs).toBeGreaterThan(expectedMs - 2000)
+ expect(result.retryAfterMs).toBeLessThanOrEqual(expectedMs)
+ })
+
+ it('uses the longest valid value from repeated Retry-After field lines', async () => {
+ const response = respond(429, {})
+ response.headerLines = [
+ { name: 'retry-after', value: '10' },
+ { name: 'retry-after', value: 'invalid' },
+ { name: 'retry-after', value: '30' },
+ ]
+ fetchStreamedMock.mockResolvedValue(response)
+
+ const result = await fetcher().fetch('https://cdn.example.com/a.png', OPTIONS)
+
+ expect(result.retryAfterMs).toBe(30_000)
+ })
+
+ it.each([408, 425])('ignores Retry-After on status %s', async (status) => {
+ fetchStreamedMock.mockResolvedValue(respond(status, { 'retry-after': '120' }))
+
+ const result = await fetcher().fetch('https://cdn.example.com/a.png', OPTIONS)
+
+ expect(result).toMatchObject({ outcome: 'rate_limited', status })
+ expect(result.retryAfterMs).toBeUndefined()
+ })
+
+ it('follows a redirect and reports how many hops it took', async () => {
+ fetchStreamedMock
+ .mockResolvedValueOnce(respond(302, { location: '/moved.png' }))
+ .mockResolvedValueOnce(image(PNG, 'image/png'))
+
+ const result = await fetcher().fetch('https://cdn.example.com/a.png', OPTIONS)
+
+ expect(result).toMatchObject({ outcome: 'ok', redirects: 1 })
+ expect(fetchStreamedMock.mock.calls[1][0]).toBe('https://cdn.example.com/moved.png')
+ })
+
+ it('refuses a redirect with repeated Location field lines', async () => {
+ const response = respond(302, {})
+ response.headerLines = [
+ { name: 'location', value: '/first.png' },
+ { name: 'location', value: '/second.png' },
+ ]
+ fetchStreamedMock.mockResolvedValue(response)
+
+ const result = await fetcher().fetch('https://cdn.example.com/a.png', OPTIONS)
+
+ expect(result).toMatchObject({ outcome: 'bad_redirect' })
+ expect(fetchStreamedMock).toHaveBeenCalledTimes(1)
+ })
+
+ it('signs each redirect hop for its target URL', async () => {
+ fetchStreamedMock
+ .mockResolvedValueOnce(respond(302, { location: 'https://images.example.net/moved.png' }))
+ .mockResolvedValueOnce(image(PNG, 'image/png'))
+ const headersForGet = jest
+ .fn()
+ .mockReturnValueOnce({ signature: 'first-hop-signature' })
+ .mockReturnValueOnce({ signature: 'second-hop-signature' })
+
+ await fetcher({}, { headersForGet }).fetch('https://cdn.example.com/a.png', OPTIONS)
+
+ expect(headersForGet.mock.calls).toEqual([
+ ['https://cdn.example.com/a.png'],
+ ['https://images.example.net/moved.png'],
+ ])
+ expect(fetchStreamedMock.mock.calls[0][1].headers).toMatchObject({ signature: 'first-hop-signature' })
+ expect(fetchStreamedMock.mock.calls[1][1].headers).toMatchObject({ signature: 'second-hop-signature' })
+ })
+
+ it.each([
+ ['it leaves HTTP for another scheme', 'javascript:alert(1)'],
+ ['it carries credentials for the next hop', 'https://user:[email protected]/a.png'],
+ ['it downgrades from HTTPS to plain HTTP', 'http://other.example/a.png'],
+ ])('refuses a redirect when %s', async (_name, location) => {
+ fetchStreamedMock.mockResolvedValue(respond(302, { location }))
+
+ const result = await fetcher().fetch('https://cdn.example.com/a.png', OPTIONS)
+
+ expect(result).toMatchObject({ outcome: 'bad_redirect' })
+ expect(fetchStreamedMock).toHaveBeenCalledTimes(1)
+ })
+
+ it('refuses a downgrade when the first hop named its scheme in capitals (requirement 9)', async () => {
+ // A record can carry `HTTPS://`, because the parser validates a parsed copy and stores the
+ // string it was given. A guard that reads the raw string misses the capitals and follows
+ // the hop in clear text. `URL.protocol` is lower case whatever the URL used.
+ fetchStreamedMock.mockResolvedValue(respond(302, { location: 'http://cdn.example.com/b.png' }))
+
+ const result = await fetcher().fetch('HTTPS://cdn.example.com/a.png', OPTIONS)
+
+ expect(result).toMatchObject({ outcome: 'bad_redirect' })
+ expect(fetchStreamedMock).toHaveBeenCalledTimes(1)
+ })
+
+ it.each([
+ ['a host the collector would have refused', 'https://internal.corp/a.png', { isPublicHost: () => false }],
+ ['a target past the length limit', `https://cdn.example.com/${'a'.repeat(300)}.png`, { maxUrlLength: 100 }],
+ ['a port the scheme does not own', 'https://cdn.example.com:11211/a.png', {}],
+ ])('refuses a redirect to %s (requirement 8)', async (_name, location, policy) => {
+ // The collector applied these checks to the first candidate. A redirect target has passed
+ // none of them, so a hop could otherwise reach a name that resolves only inside a network.
+ fetchStreamedMock.mockResolvedValue(respond(302, { location }))
+
+ const result = await fetcher(policy).fetch('https://cdn.example.com/a.png', OPTIONS)
+
+ expect(result).toMatchObject({ outcome: 'bad_redirect' })
+ expect(fetchStreamedMock).toHaveBeenCalledTimes(1)
+ })
+
+ it('hands off an offsite target that arrives at the redirect limit (requirement 7)', async () => {
+ // The limit bounds the hops this request follows itself. Nobody here follows a target for
+ // another operator, so it goes back to Kafka and costs one hop rather than being written off
+ // with hops left.
+ fetchStreamedMock.mockResolvedValue(respond(302, { location: 'https://img.other.net/a.png' }))
+
+ const result = await fetcher().fetch('https://cdn.example.com/a.png', {
+ ...OPTIONS,
+ maxRedirects: 0,
+ isDifferentOrigin: () => true,
+ })
+
+ expect(result).toMatchObject({
+ outcome: 'redirect_offsite',
+ redirectTarget: { url: 'https://img.other.net/a.png', host: 'img.other.net' },
+ })
+ })
+
+ it('stops at the redirect limit for a target on the same origin', async () => {
+ fetchStreamedMock.mockResolvedValue(respond(302, { location: 'https://cdn.example.com/b.png' }))
+
+ const result = await fetcher().fetch('https://cdn.example.com/a.png', { ...OPTIONS, maxRedirects: 0 })
+
+ expect(result).toMatchObject({
+ outcome: 'redirect_continuation',
+ redirectTarget: { url: 'https://cdn.example.com/b.png', host: 'cdn.example.com' },
+ })
+ })
+
+ it('defers a redirect whose target has no budget left, so nothing is written to the crawl history for it', async () => {
+ // `bad_redirect` is terminal. A CDN in cooldown would otherwise suppress every image behind
+ // it for the crawl history TTL, because the lane read a momentary budget as a property of
+ // the URL.
+ fetchStreamedMock.mockResolvedValue(respond(302, { location: 'https://cdn.example.net/a.png' }))
+
+ let requestNumber = 0
+ const result = await fetcher().fetch('https://cdn.example.com/a.png', {
+ ...OPTIONS,
+ scheduleRequest: async (_url, _deadlineMs, request) => {
+ requestNumber += 1
+ return requestNumber === 1
+ ? { ran: true as const, value: await request() }
+ : {
+ ran: false as const,
+ reason: 'backoff' as const,
+ blockingReason: 'retry_after' as const,
+ waitMs: 90_000,
+ }
+ },
+ })
+
+ expect(result).toMatchObject({
+ outcome: 'request_deferred',
+ redirects: 1,
+ currentUrl: 'https://cdn.example.net/a.png',
+ schedulingReason: 'backoff',
+ schedulingBlockingReason: 'retry_after',
+ schedulingWaitMs: 90_000,
+ })
+ expect(fetchStreamedMock).toHaveBeenCalledTimes(1)
+ })
+
+ it('keeps an accepted response coding for the image store', async () => {
+ fetchStreamedMock.mockResolvedValue(image(PNG, 'image/png', { 'content-encoding': 'gzip' }))
+
+ const result = await fetcher().fetch('https://cdn.example.com/a.png', OPTIONS)
+
+ expect(result).toMatchObject({ outcome: 'ok', contentEncoding: 'gzip' })
+ })
+
+ it('keeps repeated content codings in received order', async () => {
+ const response = image(PNG, 'image/png')
+ response.headerLines = [
+ { name: 'content-type', value: 'image/png' },
+ { name: 'content-encoding', value: 'gzip' },
+ { name: 'content-encoding', value: 'br, zstd' },
+ ]
+ fetchStreamedMock.mockResolvedValue(response)
+
+ const result = await fetcher().fetch('https://cdn.example.com/a.png', OPTIONS)
+
+ expect(result).toMatchObject({ outcome: 'ok', contentEncoding: 'gzip, br, zstd' })
+ })
+
+ it('refuses conflicting repeated Content-Type field lines', async () => {
+ const response = image(PNG, 'image/png')
+ response.headerLines = [
+ { name: 'content-type', value: 'image/png' },
+ { name: 'content-type', value: 'image/jpeg' },
+ ]
+ fetchStreamedMock.mockResolvedValue(response)
+
+ const result = await fetcher().fetch('https://cdn.example.com/a.png', OPTIONS)
+
+ expect(result).toMatchObject({ outcome: 'not_image' })
+ })
+
+ it('combines repeated Cache-Control field lines in received order', async () => {
+ const response = image(PNG, 'image/png')
+ response.headerLines.push(
+ { name: 'cache-control', value: 'public, max-age=86400' },
+ { name: 'cache-control', value: 'no-store' }
+ )
+ fetchStreamedMock.mockResolvedValue(response)
+
+ const result = await fetcher().fetch('https://cdn.example.com/a.png', OPTIONS)
+
+ expect(result.cache?.cacheControl).toBe('public, max-age=86400, no-store')
+ })
+
+ it('refuses an unsupported response coding', async () => {
+ fetchStreamedMock.mockResolvedValue(image(PNG, 'image/png', { 'content-encoding': 'compress' }))
+
+ const result = await fetcher().fetch('https://cdn.example.com/a.png', OPTIONS)
+
+ expect(result).toMatchObject({ outcome: 'unsupported_encoding' })
+ })
+
+ it.each(['gzip,', 'gzip, br, deflate, zstd, identity'])(
+ 'refuses a malformed or excessive response coding list: %s',
+ async (contentEncoding) => {
+ fetchStreamedMock.mockResolvedValue(image(PNG, 'image/png', { 'content-encoding': contentEncoding }))
+
+ const result = await fetcher().fetch('https://cdn.example.com/a.png', OPTIONS)
+
+ expect(result).toMatchObject({ outcome: 'unsupported_encoding' })
+ }
+ )
+
+ it('applies a refusal from any repeated opt-out field line', async () => {
+ const response = image(PNG, 'image/png')
+ response.headerLines = [
+ { name: 'content-type', value: 'image/png' },
+ { name: 'x-robots-tag', value: 'all' },
+ { name: 'x-robots-tag', value: 'PostHogImageFetcherBot: noimageai' },
+ ]
+ fetchStreamedMock.mockResolvedValue(response)
+
+ const result = await fetcher().fetch('https://cdn.example.com/a.png', OPTIONS)
+
+ expect(result).toMatchObject({ outcome: 'opt_out', refusalReason: 'x_robots_tag' })
+ })
+
+ it('lets a response header override a TDMRep reservation', async () => {
+ fetchStreamedMock.mockResolvedValue(image(PNG, 'image/png', { 'tdm-reservation': '0' }))
+
+ const result = await fetcher().fetch('https://cdn.example.com/a.png', {
+ ...OPTIONS,
+ tdmrepReservation: true,
+ })
+
+ expect(result).toMatchObject({ outcome: 'ok' })
+ })
+
+ it('applies a TDMRep reservation when the response does not override it', async () => {
+ fetchStreamedMock.mockResolvedValue(image(PNG, 'image/png'))
+
+ const result = await fetcher().fetch('https://cdn.example.com/a.png', {
+ ...OPTIONS,
+ tdmrepReservation: true,
+ })
+
+ expect(result).toMatchObject({ outcome: 'opt_out', refusalReason: 'tdm_reservation' })
+ })
+
+ it('checks path policy again before a same-origin redirect target is requested', async () => {
+ fetchStreamedMock.mockResolvedValue(respond(302, { location: '/private/image.png' }))
+
+ const result = await fetcher().fetch('https://cdn.example.com/public/image.png', {
+ ...OPTIONS,
+ checkRedirectPolicy: () =>
+ Promise.resolve({ allowed: false as const, transient: false, reason: 'robots_disallow' }),
+ })
+
+ expect(result).toMatchObject({
+ outcome: 'redirect_policy_refused',
+ redirects: 1,
+ currentUrl: 'https://cdn.example.com/private/image.png',
+ refusalReason: 'robots_disallow',
+ policyTransient: false,
+ })
+ expect(fetchStreamedMock).toHaveBeenCalledTimes(1)
+ })
+
+ it('applies the redirected path TDM policy to the next response', async () => {
+ fetchStreamedMock
+ .mockResolvedValueOnce(respond(302, { location: '/reserved/image.png' }))
+ .mockResolvedValueOnce(image(PNG, 'image/png'))
+
+ const result = await fetcher().fetch('https://cdn.example.com/public/image.png', {
+ ...OPTIONS,
+ checkRedirectPolicy: () => Promise.resolve({ allowed: true as const, tdmrepReservation: true }),
+ })
+
+ expect(result).toMatchObject({ outcome: 'opt_out', refusalReason: 'tdm_reservation', redirects: 1 })
+ })
+
+ it('uses one request deadline for every redirect hop', async () => {
+ fetchStreamedMock
+ .mockResolvedValueOnce(respond(302, { location: '/moved.png' }))
+ .mockResolvedValueOnce(image(PNG, 'image/png'))
+ const deadlines: number[] = []
+
+ await fetcher().fetch('https://cdn.example.com/a.png', {
+ ...OPTIONS,
+ scheduleRequest: async (_url, deadlineMs, request) => {
+ deadlines.push(deadlineMs)
+ return { ran: true as const, value: await request() }
+ },
+ })
+
+ expect(deadlines).toHaveLength(2)
+ expect(new Set(deadlines).size).toBe(1)
+ expect(deadlines[0] - Date.now()).toBeGreaterThan(0)
+ expect(deadlines[0] - Date.now()).toBeLessThanOrEqual(OPTIONS.timeoutMs)
+ })
+
+ it('gives up on a redirect chain longer than the limit', async () => {
+ fetchStreamedMock.mockResolvedValue(respond(302, { location: '/again.png' }))
+ let scheduledRequests = 0
+
+ const result = await fetcher().fetch('https://cdn.example.com/a.png', {
+ ...OPTIONS,
+ maxRedirects: 2,
+ scheduleRequest: async (_url, _deadlineMs, request) => {
+ scheduledRequests += 1
+ return { ran: true as const, value: await request() }
+ },
+ })
+
+ expect(result).toMatchObject({
+ outcome: 'redirect_continuation',
+ redirects: 2,
+ status: 302,
+ redirectTarget: { url: 'https://cdn.example.com/again.png' },
+ })
+ // Three requests carry two redirects. The third response is read and refused, and the
+ // budget of whatever it points at is never asked for a token.
+ expect(fetchStreamedMock).toHaveBeenCalledTimes(3)
+ expect(scheduledRequests).toBe(3)
+ })
+
+ it('reports a request scheduler that threw rather than letting it escape', async () => {
+ const result = await fetcher().fetch('https://cdn.example.com/a.png', {
+ ...OPTIONS,
+ scheduleRequest: () => Promise.reject(new Error('budget exploded')),
+ })
+
+ expect(result).toEqual({
+ outcome: 'error',
+ redirects: 0,
+ currentUrl: 'https://cdn.example.com/a.png',
+ })
+ })
+
+ it('accepts a Retry-After period of zero as a non-negative integer', async () => {
+ fetchStreamedMock.mockResolvedValue(respond(429, { 'retry-after': '0' }))
+
+ const result = await fetcher().fetch('https://cdn.example.com/a.png', OPTIONS)
+
+ expect(result.retryAfterMs).toBe(0)
+ })
+
+ it.each([
+ ['a negative period', '-5'],
+ ['a decimal period', '1.5'],
+ ['an exponent', '1e3'],
+ ['a signed period', '+5'],
+ ['an unsafe period', '9007199254740991'],
+ ['whitespace', ' '],
+ ['a date already past', new Date(Date.now() - 60_000).toUTCString()],
+ ])('reports no Retry-After period for %s, so exponential backoff has no minimum', async (_name, header) => {
+ fetchStreamedMock.mockResolvedValue(respond(429, { 'retry-after': header }))
+
+ const result = await fetcher().fetch('https://cdn.example.com/a.png', OPTIONS)
+
+ expect(result.retryAfterMs).toBeUndefined()
+ })
+
+ it.each([
+ ['an address the SSRF check refuses', new SecureRequestError('Hostname is not allowed'), 'blocked'],
+ ['a name that did not resolve', new ResolutionError('Invalid hostname'), 'error'],
+ ['a request that timed out', Object.assign(new Error('timeout'), { name: 'TimeoutError' }), 'timeout'],
+ ['a connection that failed', new Error('socket hang up'), 'error'],
+ ])('reports %s as an outcome rather than throwing', async (_name, error, outcome) => {
+ fetchStreamedMock.mockRejectedValue(error)
+
+ const result = await fetcher().fetch('https://cdn.example.com/a.png', OPTIONS)
+
+ expect(result).toMatchObject({ outcome, redirects: 0, currentUrl: 'https://cdn.example.com/a.png' })
+ })
+})
diff --git a/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/image-fetcher.ts b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/image-fetcher.ts
new file mode 100644
index 000000000000..573b510d783c
--- /dev/null
+++ b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/image-fetcher.ts
@@ -0,0 +1,477 @@
+import { InvalidRequestError, ResolutionError, SecureRequestError, fetchStreamed } from '~/common/utils/request'
+
+import type { ImageFetchBlockReason } from './block-reason'
+import { OriginPolicyReason, ResponseOptOutReason, responseOptOutReason } from './configuration-policy'
+import { HttpCacheMetadata } from './crawl-history'
+import { ImageFetchRequestMetrics } from './metrics'
+import { canonicalizeUrl } from './politeness-key'
+import { REQUEST_IDENTITY_HEADERS } from './request-identity'
+import { WebBotAuthRequestSigner } from './web-bot-auth'
+
+/**
+ * The raster formats that the fetch lane accepts.
+ *
+ * SVG is absent on purpose. It is a text format that can carry the page's own data, so its redaction
+ * belongs on the inline path rather than on an image model.
+ *
+ * AVIF is absent because the image store validates only PNG, JPEG, GIF and WebP bytes
+ * (`image-transport.ts`), so it would reject every AVIF image that this lane fetched.
+ */
+const ALLOWED_CONTENT_TYPES = ['image/png', 'image/jpeg', 'image/gif', 'image/webp'] as const
+
+export type ImageContentType = (typeof ALLOWED_CONTENT_TYPES)[number]
+
+export type FetchOutcome =
+ | 'ok'
+ | 'not_found'
+ | 'forbidden'
+ | 'rate_limited'
+ | 'server_error'
+ | 'unexpected_status'
+ | 'bad_redirect'
+ | 'request_deferred'
+ | 'redirect_policy_refused'
+ | 'redirect_offsite'
+ | 'redirect_continuation'
+ | 'too_large'
+ | 'not_image'
+ | 'unsupported_encoding'
+ | 'blocked'
+ | 'opt_out'
+ | 'not_modified'
+ | 'timeout'
+ | 'error'
+
+export type TransientFetchOutcome = Extract<FetchOutcome, 'timeout' | 'error' | 'rate_limited' | 'server_error'>
+export type FetchRefusalReason = OriginPolicyReason | ResponseOptOutReason | 'configuration_refused'
+
+export interface ImageFetchResult {
+ outcome: FetchOutcome
+ redirects: number
+ currentUrl: string
+ status?: number
+ bytes?: Buffer
+ contentType?: ImageContentType
+ contentEncoding?: string
+ cache?: HttpCacheMetadata
+ refusalReason?: FetchRefusalReason
+ /** Set by a 429 or a 503 that named a period. The caller holds the registrable domain for that period. */
+ retryAfterMs?: number
+ schedulingReason?: RequestScheduleBlockReason
+ schedulingBlockingReason?: ImageFetchBlockReason
+ schedulingWaitMs?: number
+ policyTransient?: boolean
+ /** Where a redirect this lane did not follow points. The caller republishes it rather than fetching it. */
+ redirectTarget?: { url: string; host: string }
+}
+
+export interface ImageFetchOptions {
+ sourcePartitions?: readonly number[]
+ maxBytes: number
+ /** Covers the redirect chain as a whole, so a chain of slow hops cannot outlive one hop's budget. */
+ timeoutMs: number
+ maxRedirects: number
+ scheduleRequest: <T>(
+ url: URL,
+ deadlineMs: number,
+ request: () => Promise<T>
+ ) => Promise<
+ | { ran: true; value: T }
+ | {
+ ran: false
+ reason: RequestScheduleBlockReason
+ blockingReason: ImageFetchBlockReason
+ waitMs: number
+ }
+ >
+ checkRedirectPolicy: (url: string) => Promise<RedirectTargetPolicy>
+ isDifferentOrigin: (url: URL) => boolean
+ /** Validators that `url` issued. The caller checks that, and no redirect target gets them. */
+ cache?: HttpCacheMetadata
+ tdmrepReservation: boolean
+ onRedirectResponse?: () => void
+}
+
+export type RequestScheduleBlockReason =
+ | 'breaker_open'
+ | 'backoff'
+ | 'deadline'
+ | 'origin_map_full'
+ | 'registrable_domain_map_full'
+ | 'connection_limit'
+
+export type RedirectTargetPolicy =
+ | { allowed: true; tdmrepReservation: boolean }
+ | { allowed: false; transient: boolean; reason: OriginPolicyReason | 'configuration_refused' }
+
+export interface ImageFetcher {
+ fetch(url: string, options: ImageFetchOptions): Promise<ImageFetchResult>
+}
+
+const REQUEST_HEADERS: Record<string, string> = {
+ ...REQUEST_IDENTITY_HEADERS,
+ accept: 'image/*',
+ 'accept-encoding': 'gzip, deflate, br, zstd',
+}
+
+const REDIRECT_STATUSES = new Set([301, 302, 303, 307, 308])
+
+/**
+ * This sends no user credential: no cookie and no `Authorization`, because this lane must not reach an
+ * image behind a login. The `Referer` names PostHog and never the page that showed the image, because
+ * that page would tell the origin where on the customer's site the image sat. Web Bot Auth identifies
+ * PostHog as the operator of the request.
+ *
+ * Every refusal is an outcome rather than a throw. This runs inside a Kafka batch, one URL at a
+ * time, and a throw would abandon the URLs after it in the same batch.
+ */
+export class HttpImageFetcher implements ImageFetcher {
+ constructor(
+ private readonly policy: RedirectPolicy,
+ private readonly webBotAuthSigner: WebBotAuthRequestSigner
+ ) {}
+
+ public async fetch(url: string, options: ImageFetchOptions): Promise<ImageFetchResult> {
+ const deadlineMs = Date.now() + options.timeoutMs
+ let target = url
+ let currentCache = options.cache
+ let tdmrepReservation = options.tdmrepReservation
+ for (let redirects = 0; ; redirects++) {
+ const remainingMs = deadlineMs - Date.now()
+ if (remainingMs <= 0) {
+ return { outcome: 'timeout', redirects, currentUrl: target }
+ }
+ let scheduled:
+ | { ran: true; value: HopResult }
+ | {
+ ran: false
+ reason: RequestScheduleBlockReason
+ blockingReason: ImageFetchBlockReason
+ waitMs: number
+ }
+ try {
+ scheduled = await options.scheduleRequest(new URL(target), deadlineMs, () =>
+ this.hop(
+ target,
+ Math.max(1, deadlineMs - Date.now()),
+ options.maxBytes,
+ currentCache,
+ tdmrepReservation,
+ options.sourcePartitions
+ )
+ )
+ } catch (error) {
+ return { outcome: classifyError(error), redirects, currentUrl: target }
+ }
+ if (!scheduled.ran) {
+ return {
+ outcome: 'request_deferred',
+ redirects,
+ currentUrl: target,
+ schedulingReason: scheduled.reason,
+ schedulingBlockingReason: scheduled.blockingReason,
+ schedulingWaitMs: scheduled.waitMs,
+ }
+ }
+ const hop = scheduled.value
+ if (hop.kind !== 'redirect') {
+ return { ...hop.result, redirects, currentUrl: target }
+ }
+ const next = resolveRedirect(target, hop.location, this.policy)
+ if (!next) {
+ return { outcome: 'bad_redirect', redirects, currentUrl: target, status: hop.status }
+ }
+ options.onRedirectResponse?.()
+ if (options.isDifferentOrigin(next)) {
+ return {
+ outcome: 'redirect_offsite',
+ redirects,
+ currentUrl: target,
+ status: hop.status,
+ redirectTarget: { url: next.toString(), host: next.hostname },
+ }
+ }
+ if (redirects >= options.maxRedirects) {
+ return {
+ outcome: 'redirect_continuation',
+ redirects,
+ currentUrl: target,
+ status: hop.status,
+ redirectTarget: { url: next.toString(), host: next.hostname },
+ cache: hop.cache,
+ }
+ }
+ const redirectPolicy = await options.checkRedirectPolicy(next.toString())
+ if (!redirectPolicy.allowed) {
+ return {
+ outcome: 'redirect_policy_refused',
+ redirects: redirects + 1,
+ currentUrl: next.toString(),
+ status: hop.status,
+ refusalReason: redirectPolicy.reason,
+ policyTransient: redirectPolicy.transient,
+ cache: hop.cache,
+ }
+ }
+ target = next.toString()
+ currentCache = undefined
+ tdmrepReservation = redirectPolicy.tdmrepReservation
+ }
+ }
+
+ private async hop(
+ url: string,
+ timeoutMs: number,
+ maxBytes: number,
+ previousCache: HttpCacheMetadata | undefined,
+ tdmrepReservation: boolean,
+ sourcePartitions: readonly number[] | undefined
+ ): Promise<HopResult> {
+ const requestTimeMs = Date.now()
+ const canonical = canonicalizeUrl(url)
+ let requestOutcome = ImageFetchRequestMetrics.outcomeForHttpStatus()
+ const headers: Record<string, string> = { ...REQUEST_HEADERS, ...this.webBotAuthSigner.headersForGet(url) }
+ if (previousCache?.etag) {
+ headers['if-none-match'] = previousCache.etag
+ } else if (previousCache?.lastModified) {
+ headers['if-modified-since'] = previousCache.lastModified
+ }
+ try {
+ const response = await fetchStreamed(url, { headers, timeoutMs, allowH2: true })
+ const status = response.status
+ requestOutcome = ImageFetchRequestMetrics.outcomeForHttpStatus(status)
+ const cache = cacheMetadata(requestTimeMs, Date.now(), response.headerLines)
+ const optOut = responseOptOutReason(response.headerLines, tdmrepReservation)
+ if (optOut) {
+ response.discard()
+ return { kind: 'done', result: { outcome: 'opt_out', status, refusalReason: optOut, cache } }
+ }
+
+ if (REDIRECT_STATUSES.has(status)) {
+ response.discard()
+ const locations = headerValues(response.headerLines, 'location')
+ const location = locations.length === 1 ? locations[0] : undefined
+ return location
+ ? { kind: 'redirect', status, location, cache }
+ : { kind: 'done', result: { outcome: 'bad_redirect', status, cache } }
+ }
+ if (status === 304) {
+ response.discard()
+ const outcome =
+ previousCache?.etag || previousCache?.lastModified ? 'not_modified' : 'unexpected_status'
+ return { kind: 'done', result: { outcome, status, cache } }
+ }
+ if (status !== 200) {
+ response.discard()
+ return {
+ kind: 'done',
+ result: { ...statusResult(status, headerValues(response.headerLines, 'retry-after')), cache },
+ }
+ }
+
+ const contentTypes = headerValues(response.headerLines, 'content-type').map(normalizeContentType)
+ const contentType = contentTypes[0]
+ if (!contentType || contentTypes.some((candidate) => candidate !== contentType)) {
+ response.discard()
+ return { kind: 'done', result: { outcome: 'not_image', status, cache } }
+ }
+ const contentEncoding = normalizeContentEncoding(headerValues(response.headerLines, 'content-encoding'))
+ if (contentEncoding === null) {
+ response.discard()
+ return { kind: 'done', result: { outcome: 'unsupported_encoding', status, cache } }
+ }
+ const declaredTooLarge = headerValues(response.headerLines, 'content-length').some((value) => {
+ const declaredBytes = Number(value)
+ return Number.isFinite(declaredBytes) && declaredBytes > maxBytes
+ })
+ if (declaredTooLarge) {
+ response.discard()
+ return { kind: 'done', result: { outcome: 'too_large', status, cache } }
+ }
+
+ const { bytes, overLimit } = await response.read(maxBytes, false)
+ if (overLimit) {
+ return { kind: 'done', result: { outcome: 'too_large', status, cache } }
+ }
+ return { kind: 'done', result: { outcome: 'ok', status, bytes, contentType, contentEncoding, cache } }
+ } finally {
+ if (canonical) {
+ ImageFetchRequestMetrics.observeRequest(
+ requestOutcome,
+ (Date.now() - requestTimeMs) / 1000,
+ sourcePartitions
+ )
+ }
+ }
+ }
+}
+
+type HopResult =
+ | { kind: 'redirect'; status: number; location: string; cache: HttpCacheMetadata }
+ | { kind: 'done'; result: Omit<ImageFetchResult, 'redirects' | 'currentUrl'> }
+
+function statusResult(status: number, retryAfterValues: string[]): Omit<ImageFetchResult, 'redirects' | 'currentUrl'> {
+ const retryAfterMs = maximumRetryAfterMs(retryAfterValues)
+ if (status === 404 || status === 410) {
+ return { outcome: 'not_found', status }
+ }
+ if (status === 401 || status === 403) {
+ return { outcome: 'forbidden', status }
+ }
+ if (status === 408 || status === 425) {
+ return { outcome: 'rate_limited', status }
+ }
+ if (status === 429) {
+ return { outcome: 'rate_limited', status, retryAfterMs }
+ }
+ if (status === 503) {
+ return { outcome: 'server_error', status, retryAfterMs }
+ }
+ if (status >= 500) {
+ return { outcome: 'server_error', status }
+ }
+ return { outcome: 'unexpected_status', status }
+}
+
+function maximumRetryAfterMs(values: string[]): number | undefined {
+ const parsed = values.flatMap((value) => {
+ const retryAfterMs = parseRetryAfterMs(value)
+ return retryAfterMs === undefined ? [] : [retryAfterMs]
+ })
+ return parsed.length > 0 ? Math.max(...parsed) : undefined
+}
+
+/** RFC 9110 allows a count of seconds or an HTTP date. Real responses use both. */
+export function parseRetryAfterMs(value: string | undefined): number | undefined {
+ if (!value) {
+ return undefined
+ }
+ const trimmed = value.trim()
+ if (/^\d+$/.test(trimmed)) {
+ const seconds = Number(trimmed)
+ return Number.isSafeInteger(seconds) && seconds <= Number.MAX_SAFE_INTEGER / 1000 ? seconds * 1000 : undefined
+ }
+ const dateMs = Date.parse(trimmed)
+ if (Number.isNaN(dateMs)) {
+ return undefined
+ }
+ const heldMs = dateMs - Date.now()
+ return heldMs > 0 ? heldMs : undefined
+}
+
+/**
+ * The rules a redirect target must pass, beyond the SSRF checks every hop re-enters.
+ *
+ * The collector applied all of these to the first candidate before it reached the topic. A redirect
+ * target has passed none of them, so a hop could otherwise reach a host the collector would have
+ * refused, such as a single-label name, or a name under a suffix that resolves only inside a
+ * network. Requirement 8.
+ */
+export interface RedirectPolicy {
+ maxUrlLength: number
+ isPublicHost: (host: string) => boolean
+}
+
+function resolveRedirect(from: string, location: string, policy: RedirectPolicy): URL | null {
+ let next: URL
+ try {
+ next = new URL(location, from)
+ } catch {
+ return null
+ }
+ // Requirement 9: no downgrade to plain HTTP. `URL.protocol` is lower case already, which a
+ // comparison against the raw location string is not.
+ if (next.protocol !== 'https:') {
+ return null
+ }
+ // This lane sends no credentials, and a userinfo part would put some back on the next hop.
+ if (next.username || next.password) {
+ return null
+ }
+ // A port other than the scheme's own would turn this lane into a port prober. A page names any
+ // host and port, the connection leaves from our egress addresses, and the outcome metric reports
+ // what answered. Few images are served on another port, so this refuses the whole case.
+ if (next.port !== '') {
+ return null
+ }
+ const canonical = canonicalizeUrl(next.toString())
+ if (!canonical || canonical.fetch.length > policy.maxUrlLength) {
+ return null
+ }
+ if (!policy.isPublicHost(canonical.host)) {
+ return null
+ }
+ return new URL(canonical.fetch)
+}
+
+function normalizeContentType(header: string | undefined): ImageContentType | undefined {
+ if (!header) {
+ return undefined
+ }
+ const type = header.split(';')[0].trim().toLowerCase()
+ return ALLOWED_CONTENT_TYPES.find((allowed) => allowed === type)
+}
+
+const ALLOWED_CONTENT_ENCODINGS = new Set(['gzip', 'deflate', 'br', 'zstd'])
+const MAX_CONTENT_ENCODING_LAYERS = 4
+
+function normalizeContentEncoding(values: string[]): string | undefined | null {
+ const declaredCodings = values.flatMap((value) => value.split(',')).map((coding) => coding.trim().toLowerCase())
+ if (declaredCodings.length > MAX_CONTENT_ENCODING_LAYERS || declaredCodings.some((coding) => coding === '')) {
+ return null
+ }
+ const codings = declaredCodings.filter((coding) => coding !== 'identity')
+ if (codings.some((coding) => !ALLOWED_CONTENT_ENCODINGS.has(coding))) {
+ return null
+ }
+ return codings.length > 0 ? codings.join(', ') : undefined
+}
+
+function headerValues(headerLines: Array<{ name: string; value: string }>, name: string): string[] {
+ return headerLines.filter((line) => line.name === name).map((line) => line.value)
+}
+
+function cacheMetadata(
+ requestTimeMs: number,
+ responseTimeMs: number,
+ headerLines: Array<{ name: string; value: string }>
+): HttpCacheMetadata {
+ return {
+ requestTimeMs,
+ responseTimeMs,
+ etag: singletonHeader(headerLines, 'etag'),
+ lastModified: singletonHeader(headerLines, 'last-modified'),
+ date: singletonHeader(headerLines, 'date'),
+ age: singletonHeader(headerLines, 'age'),
+ cacheControl: combinedHeader(headerLines, 'cache-control'),
+ expires: singletonHeader(headerLines, 'expires'),
+ }
+}
+
+function singletonHeader(headerLines: Array<{ name: string; value: string }>, name: string): string | undefined {
+ const values = headerValues(headerLines, name)
+ return values.length === 1 ? values[0] : undefined
+}
+
+function combinedHeader(headerLines: Array<{ name: string; value: string }>, name: string): string | undefined {
+ const values = headerValues(headerLines, name)
+ return values.length > 0 ? values.join(', ') : undefined
+}
+
+export function classifyError(error: unknown): FetchOutcome {
+ // `blocked` is terminal, so it must only cover a permanent property of the URL. A refused
+ // address and an unparsable URL are permanent. A name that did not resolve is not, because the
+ // shared lookup raises the same error for a resolver timeout as for a name that does not exist.
+ if (error instanceof SecureRequestError || error instanceof InvalidRequestError) {
+ return 'blocked'
+ }
+ if (error instanceof ResolutionError) {
+ return 'error'
+ }
+ const name = error instanceof Error ? error.name : ''
+ const code = typeof error === 'object' && error !== null ? String((error as { code?: unknown }).code ?? '') : ''
+ if (name === 'TimeoutError' || name === 'AbortError' || code.includes('TIMEOUT')) {
+ return 'timeout'
+ }
+ return 'error'
+}
diff --git a/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/image-sink.ts b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/image-sink.ts
index 1922ad21b318..d1692c79214e 100644
--- a/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/image-sink.ts
+++ b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/image-sink.ts
@@ -1,5 +1,5 @@
-import type { FetchCandidate } from '~/ingestion/pipelines/sessionreplay/ml-mirror-image-fetch/collected-urls-record'
-import type { ImageFetchResult } from '~/ingestion/pipelines/sessionreplay/ml-mirror-image-fetch/image-fetcher'
+import type { FetchCandidate } from './collected-urls-record'
+import type { ImageFetchResult } from './image-fetcher'
/**
* Where the lane stores each fetched image. The image counts as handled only once the sink resolves: a rejection keeps
diff --git a/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/job-admission.ts b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/job-admission.ts
new file mode 100644
index 000000000000..9ca1793b2379
--- /dev/null
+++ b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/job-admission.ts
@@ -0,0 +1,145 @@
+import pLimit from 'p-limit'
+import { Counter } from 'prom-client'
+
+import {
+ type TeamMonthKey,
+ type TeamMonthKeys,
+ teamMonthKeyExpiresAtMs,
+ untilAborted,
+} from '~/ingestion/pipelines/sessionreplay/ai-research/v4-keys/team-month-keys'
+
+import type { FetchCandidate } from './collected-urls-record'
+import { parseV4UrlRef } from './v4-url-ref'
+
+/** Each lookup's read budget starts when it starts, so a batch of many new team months must not start them all at once. */
+const KEY_LOOKUP_CONCURRENCY = 16
+/**
+ * Bounds all key lookups of one batch together, so a batch of many new team months still leaves the fetch, history and
+ * republish phases their time inside the consumer's 240s batch timeout.
+ */
+export const DEFAULT_KEY_BUDGET_MS = 60_000
+/** A job whose key was not read in time waits in the first delay tier, so it does not come straight back to a busy key store. */
+const DEFERRED_JOB_DELAY_MS = 60_000
+
+const droppedJobs = new Counter({
+ name: 'ai_research_v4_image_fetch_dropped_jobs_total',
+ help: 'v4 fetch jobs dropped before any request, by reason.',
+ labelNames: ['reason'],
+})
+
+const deferredJobs = new Counter({
+ name: 'ai_research_v4_image_fetch_deferred_jobs_total',
+ help: 'v4 fetch jobs sent to a delay tier because their team month key was not read within the batch key budget.',
+})
+
+const DEFERRED = Symbol('deferred')
+
+/**
+ * Drops a v4 job before any request when nothing could store its image: its team month has no live key, because the
+ * key row expired, is missing or is a tombstone. The recorder stores the row before it sends a URL, so a missing row
+ * means it expired.
+ */
+export class V4JobAdmission {
+ private readonly keyLookups = pLimit(KEY_LOOKUP_CONCURRENCY)
+
+ constructor(
+ private readonly keys: Pick<TeamMonthKeys, 'get'>,
+ private readonly keyBudgetMs = DEFAULT_KEY_BUDGET_MS
+ ) {
+ if (!Number.isSafeInteger(keyBudgetMs) || keyBudgetMs <= 0 || keyBudgetMs > 120_000) {
+ throw new Error(
+ `AI_RESEARCH_V4_IMAGE_FETCH_KEY_BUDGET_MS must be a whole number in (0, 120000], got ${keyBudgetMs}`
+ )
+ }
+ }
+
+ /**
+ * Resolves to the jobs to fetch. A job whose key was not read within the budget stays in the batch with a later
+ * `notBeforeMs`, so the consumer sends it to a delay tier, as the fetch pass does with a job its deadline passed.
+ * A key read that fails before the budget runs out still fails the batch.
+ */
+ public async admit(candidates: FetchCandidate[], nowMs: number): Promise<FetchCandidate[]> {
+ const budget = new AbortController()
+ const timer = setTimeout(() => budget.abort(new Error('the v4 key admission budget ran out')), this.keyBudgetMs)
+ try {
+ return await this.admitWithin(candidates, nowMs, budget.signal)
+ } finally {
+ clearTimeout(timer)
+ // Stops the lookups this batch still waits on when one of them failed.
+ budget.abort()
+ }
+ }
+
+ private async admitWithin(
+ candidates: FetchCandidate[],
+ nowMs: number,
+ budget: AbortSignal
+ ): Promise<FetchCandidate[]> {
+ const keys = new Map<string, Promise<TeamMonthKey | typeof DEFERRED>>()
+ const live: { candidate: FetchCandidate; teamMonth: string }[] = []
+ for (const candidate of candidates) {
+ const ref = parseV4UrlRef(candidate.originalRef)
+ if (!ref) {
+ droppedJobs.labels('foreign_ref').inc()
+ continue
+ }
+ // DynamoDB removes an expired row some time after `expires_at`, so the row may still be there.
+ if (teamMonthKeyExpiresAtMs(ref.keyMonth) <= nowMs) {
+ droppedJobs.labels('expired').inc()
+ continue
+ }
+ const teamMonth = `${ref.teamId}:${ref.keyMonth}`
+ if (!keys.has(teamMonth)) {
+ keys.set(
+ teamMonth,
+ this.lookUp(budget, () => this.keys.get(ref.teamId, ref.keyMonth, budget))
+ )
+ }
+ live.push({ candidate, teamMonth })
+ }
+ const resolved = new Map(
+ await Promise.all([...keys].map(async ([teamMonth, key]) => [teamMonth, await key] as const))
+ )
+ const admitted: FetchCandidate[] = []
+ let deferred = 0
+ for (const { candidate, teamMonth } of live) {
+ const key = resolved.get(teamMonth)!
+ if (key === DEFERRED) {
+ deferred++
+ admitted.push({
+ ...candidate,
+ notBeforeMs: Math.max(candidate.notBeforeMs, nowMs + DEFERRED_JOB_DELAY_MS),
+ lastBlockReason: 'pass_deadline',
+ })
+ } else if (key.status === 'ok') {
+ admitted.push(candidate)
+ } else {
+ droppedJobs.labels(key.status === 'deleted' ? 'deleted' : 'missing_key').inc()
+ }
+ }
+ if (deferred > 0) {
+ deferredJobs.inc(deferred)
+ }
+ return admitted
+ }
+
+ private async lookUp(
+ budget: AbortSignal,
+ get: () => Promise<TeamMonthKey>
+ ): Promise<TeamMonthKey | typeof DEFERRED> {
+ try {
+ return await untilAborted(
+ this.keyLookups(() => {
+ budget.throwIfAborted()
+ return get()
+ }),
+ budget
+ )
+ } catch (error) {
+ if (budget.aborted) {
+ return DEFERRED
+ }
+ throw error
+ }
+ }
+}
diff --git a/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/lane.ts b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/lane.ts
new file mode 100644
index 000000000000..524fc9d5f742
--- /dev/null
+++ b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/lane.ts
@@ -0,0 +1,54 @@
+import {
+ KAFKA_AI_RESEARCH_V4_IMAGE_FETCH,
+ KAFKA_AI_RESEARCH_V4_IMAGE_FETCH_RETRY_1H,
+ KAFKA_AI_RESEARCH_V4_IMAGE_FETCH_RETRY_1M,
+ KAFKA_AI_RESEARCH_V4_IMAGE_FETCH_RETRY_10M,
+} from '~/common/config/kafka-topics'
+
+/** The v4 lane has its own frontier, delay tiers and consumer groups, so its jobs never mix with the v3 lane's. */
+export const V4_IMAGE_FETCH_LANE = {
+ frontier: KAFKA_AI_RESEARCH_V4_IMAGE_FETCH,
+ retry1m: KAFKA_AI_RESEARCH_V4_IMAGE_FETCH_RETRY_1M,
+ retry10m: KAFKA_AI_RESEARCH_V4_IMAGE_FETCH_RETRY_10M,
+ retry1h: KAFKA_AI_RESEARCH_V4_IMAGE_FETCH_RETRY_1H,
+} as const
+
+/** The delay tiers the publisher writes to. A retry pod drains one of these and nothing else. */
+export const V4_IMAGE_FETCH_DELAY_TOPICS: readonly string[] = [
+ V4_IMAGE_FETCH_LANE.retry1m,
+ V4_IMAGE_FETCH_LANE.retry10m,
+ V4_IMAGE_FETCH_LANE.retry1h,
+]
+
+/** A dead letter must never land on a topic the lane consumes, which would loop it back. */
+export const BLOCKED_IMAGE_FETCH_DEAD_LETTER_TOPICS: readonly string[] = [
+ V4_IMAGE_FETCH_LANE.frontier,
+ ...V4_IMAGE_FETCH_DELAY_TOPICS,
+]
+
+/** The v3 lane's default group. A v4 pod in it would take v3 partitions and read none of their topics. */
+const V3_IMAGE_FETCH_GROUP_ID = 'session-replay-ml-image-fetch'
+
+export function assertOwnImageFetchGroup(group: string): void {
+ if (!group || group === V3_IMAGE_FETCH_GROUP_ID) {
+ throw new Error('AI_RESEARCH_V4_IMAGE_FETCH_GROUP_ID must be set and differ from the v3 lane group')
+ }
+}
+
+export interface HostLimits {
+ requestsPerSecond: number
+ burst: number
+ maxConcurrent: number
+}
+
+/** The v4 per registrable domain limits: a share of the configured limits, at least one request at a time. */
+export function v4HostLimits(limits: HostLimits, share: number): HostLimits {
+ if (!(share > 0 && share <= 1)) {
+ throw new Error(`AI_RESEARCH_V4_IMAGE_FETCH_HOST_BUDGET_SHARE must be in (0, 1], got ${share}`)
+ }
+ return {
+ requestsPerSecond: limits.requestsPerSecond * share,
+ burst: Math.max(1, Math.floor(limits.burst * share)),
+ maxConcurrent: Math.max(1, Math.floor(limits.maxConcurrent * share)),
+ }
+}
diff --git a/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/metrics.test.ts b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/metrics.test.ts
new file mode 100644
index 000000000000..ac28b12e9313
--- /dev/null
+++ b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/metrics.test.ts
@@ -0,0 +1,186 @@
+import { register } from 'prom-client'
+
+import { ImageFetchConsumerMetrics, ImageFetchRequestMetrics } from './metrics'
+
+describe('image fetch metrics', () => {
+ beforeEach(() => register.resetMetrics())
+ afterEach(() => jest.restoreAllMocks())
+
+ test.each([
+ [undefined, 'network_error'],
+ [199, 'other'],
+ [200, '2xx'],
+ [299, '2xx'],
+ [300, '3xx'],
+ [399, '3xx'],
+ [400, '4xx'],
+ [499, '4xx'],
+ [500, '5xx'],
+ [599, '5xx'],
+ [600, 'other'],
+ ] as const)('maps HTTP status %s to the bounded %s outcome', (status, expected) => {
+ expect(ImageFetchRequestMetrics.outcomeForHttpStatus(status)).toBe(expected)
+ })
+
+ it('records retry causes with fixed semantic labels', async () => {
+ ImageFetchRequestMetrics.incRetryCause('rate_limited')
+ ImageFetchRequestMetrics.incRetryCause('server_error')
+ ImageFetchRequestMetrics.incRetryCause('timeout')
+ ImageFetchRequestMetrics.incRetryCause('error')
+
+ const metric = (await register.getMetricsAsJSON()).find(
+ (candidate) => candidate.name === 'ai_research_v4_image_fetch_retry_causes_total'
+ )
+ const causes = metric?.values.map((value) => value.labels.cause)
+
+ expect(causes).toEqual(['rate_limited', 'server_error', 'timeout', 'error'])
+ })
+
+ it('records bounded concentration and effective-count distributions', async () => {
+ ImageFetchConsumerMetrics.observeBatchDiversity([6, 3, 1], [9, 1])
+
+ const metrics = await register.getMetricsAsJSON()
+ const topShare = metrics.find((metric) => metric.name === 'ai_research_v4_image_fetch_batch_top_share')
+ const effectiveCount = metrics.find(
+ (metric) => metric.name === 'ai_research_v4_image_fetch_batch_effective_count'
+ )
+ const topShareSums = topShare?.values.filter(
+ (value) => 'metricName' in value && String(value.metricName).endsWith('_sum')
+ )
+ const effectiveCountSums = effectiveCount?.values.filter(
+ (value) => 'metricName' in value && String(value.metricName).endsWith('_sum')
+ )
+
+ expect(topShareSums).toEqual([
+ expect.objectContaining({ labels: { scope: 'origin', top_n: '1' }, value: 0.6 }),
+ expect.objectContaining({ labels: { scope: 'origin', top_n: '5' }, value: 1 }),
+ expect.objectContaining({ labels: { scope: 'origin', top_n: '10' }, value: 1 }),
+ expect.objectContaining({ labels: { scope: 'registrable_domain', top_n: '1' }, value: 0.9 }),
+ expect.objectContaining({ labels: { scope: 'registrable_domain', top_n: '5' }, value: 1 }),
+ expect.objectContaining({ labels: { scope: 'registrable_domain', top_n: '10' }, value: 1 }),
+ ])
+ expect(effectiveCountSums).toEqual([
+ expect.objectContaining({ labels: { scope: 'origin' }, value: 100 / 46 }),
+ expect.objectContaining({ labels: { scope: 'registrable_domain' }, value: 100 / 82 }),
+ ])
+ })
+
+ it('records partition flow, outcomes, waits, and concentration with bounded labels', async () => {
+ ImageFetchConsumerMetrics.observePartitionRecord(7, 10, 9)
+ ImageFetchConsumerMetrics.incPartitionUrls(7, 'unique', 8)
+ ImageFetchConsumerMetrics.observePartitionBatchDiversity(7, [6, 2], [7, 1])
+ ImageFetchRequestMetrics.observeRequest('2xx', 0.25, [7, 42])
+ ImageFetchRequestMetrics.observeSchedulerWait('origin_crawl_delay', 1.5, [7, 42])
+ ImageFetchRequestMetrics.incPartitionAttempt(7, 'completed', 'ok')
+ ImageFetchRequestMetrics.incPartitionAttempt(7, 'republished', 'backoff')
+
+ const metrics = await register.getMetricsAsJSON()
+ const records = metrics.find((metric) => metric.name === 'ai_research_v4_image_fetch_partition_records_total')
+ const urls = metrics.find((metric) => metric.name === 'ai_research_v4_image_fetch_partition_urls_total')
+ const requests = metrics.find((metric) => metric.name === 'ai_research_v4_image_fetch_partition_requests_total')
+ const waits = metrics.find(
+ (metric) => metric.name === 'ai_research_v4_image_fetch_partition_scheduler_wait_seconds'
+ )
+ const attempts = metrics.find((metric) => metric.name === 'ai_research_v4_image_fetch_partition_attempts_total')
+ const topShare = metrics.find((metric) => metric.name === 'ai_research_v4_image_fetch_partition_top_share')
+ const effectiveCount = metrics.find(
+ (metric) => metric.name === 'ai_research_v4_image_fetch_partition_effective_count'
+ )
+
+ expect(records?.values).toContainEqual(expect.objectContaining({ labels: { partition: '7' }, value: 1 }))
+ expect(urls?.values).toEqual(
+ expect.arrayContaining([
+ expect.objectContaining({ labels: { partition: '7', stage: 'parsed' }, value: 10 }),
+ expect.objectContaining({ labels: { partition: '7', stage: 'accepted' }, value: 9 }),
+ expect.objectContaining({ labels: { partition: '7', stage: 'unique' }, value: 8 }),
+ ])
+ )
+ expect(requests?.values).toEqual(
+ expect.arrayContaining([
+ expect.objectContaining({ labels: { partition: '7', outcome: '2xx' }, value: 1 }),
+ expect.objectContaining({ labels: { partition: '42', outcome: '2xx' }, value: 1 }),
+ ])
+ )
+ expect(waits?.values).toEqual(
+ expect.arrayContaining([
+ expect.objectContaining({ labels: { partition: '7', scope: 'origin_crawl_delay' }, value: 1 }),
+ expect.objectContaining({ labels: { partition: '42', scope: 'origin_crawl_delay' }, value: 1 }),
+ ])
+ )
+ expect(attempts?.values).toEqual(
+ expect.arrayContaining([
+ expect.objectContaining({
+ labels: { partition: '7', disposition: 'completed', outcome: 'ok' },
+ value: 1,
+ }),
+ expect.objectContaining({
+ labels: { partition: '7', disposition: 'republished', outcome: 'backoff' },
+ value: 1,
+ }),
+ ])
+ )
+ expect(topShare?.values).toEqual(
+ expect.arrayContaining([
+ expect.objectContaining({
+ metricName: expect.stringMatching(/_sum$/),
+ labels: { partition: '7', scope: 'origin' },
+ value: 0.75,
+ }),
+ expect.objectContaining({
+ metricName: expect.stringMatching(/_sum$/),
+ labels: { partition: '7', scope: 'registrable_domain' },
+ value: 0.875,
+ }),
+ ])
+ )
+ expect(effectiveCount?.values).toEqual(
+ expect.arrayContaining([
+ expect.objectContaining({
+ metricName: expect.stringMatching(/_sum$/),
+ labels: { partition: '7', scope: 'origin' },
+ value: 64 / 40,
+ }),
+ expect.objectContaining({
+ metricName: expect.stringMatching(/_sum$/),
+ labels: { partition: '7', scope: 'registrable_domain' },
+ value: 64 / 50,
+ }),
+ ])
+ )
+ })
+
+ it('records initial schedulable slots against the pod request limit', async () => {
+ ImageFetchRequestMetrics.observeBatchSchedulableCapacity(7, 10)
+
+ const metrics = await register.getMetricsAsJSON()
+ const slots = metrics.find((metric) => metric.name === 'ai_research_v4_image_fetch_batch_schedulable_slots')
+ const ratio = metrics.find(
+ (metric) => metric.name === 'ai_research_v4_image_fetch_batch_schedulable_capacity_ratio'
+ )
+
+ expect(slots?.values).toContainEqual(
+ expect.objectContaining({ metricName: expect.stringMatching(/_sum$/), value: 7 })
+ )
+ expect(ratio?.values).toContainEqual(
+ expect.objectContaining({ metricName: expect.stringMatching(/_sum$/), value: 0.7 })
+ )
+ })
+
+ it('keeps the oldest active batch age when another batch finishes', async () => {
+ jest.spyOn(performance, 'now').mockReturnValue(3_000)
+ const firstBatchId = ImageFetchConsumerMetrics.startBatch(1_000)
+ const secondBatchId = ImageFetchConsumerMetrics.startBatch(2_000)
+
+ ImageFetchConsumerMetrics.finishBatch(secondBatchId)
+ const activeMetric = (await register.getMetricsAsJSON()).find(
+ (metric) => metric.name === 'ai_research_v4_image_fetch_consumer_active_batch_elapsed_seconds'
+ )
+ expect(activeMetric?.values[0].value).toBe(2)
+
+ ImageFetchConsumerMetrics.finishBatch(firstBatchId)
+ const idleMetric = (await register.getMetricsAsJSON()).find(
+ (metric) => metric.name === 'ai_research_v4_image_fetch_consumer_active_batch_elapsed_seconds'
+ )
+ expect(idleMetric?.values[0].value).toBe(0)
+ })
+})
diff --git a/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/metrics.ts b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/metrics.ts
new file mode 100644
index 000000000000..f73ffe51e9e1
--- /dev/null
+++ b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/metrics.ts
@@ -0,0 +1,663 @@
+import { Counter, Gauge, Histogram } from 'prom-client'
+
+import type { RepublishReason, UrlDropReason, UrlSkipReason } from './collected-urls-record'
+import type { AttemptOutcome } from './fetch-runner'
+import type { FrontierDeadLetterReason } from './frontier-dead-letter-sink'
+import type { FetchRefusalReason, RequestScheduleBlockReason, TransientFetchOutcome } from './image-fetcher'
+
+/** What the in-flight gauge reads. Narrower than `ConcurrencyController`, so the metrics do not depend on the whole of it. */
+export interface InFlightCount {
+ readonly running: number
+}
+
+/** What the budget gauges read. Narrower than `HostBudget`, so the metrics do not depend on the whole of it. */
+export interface BudgetCounts {
+ readonly trackedRegistrableDomains: number
+ readonly trackedOrigins: number
+ readonly evictedWhileBlocked: number
+ blockedRegistrableDomains(nowMs: number): number
+}
+
+export type DedupScope = 'batch' | 'store'
+export type SchedulerWaitScope = 'origin_crawl_delay' | 'registrable_domain_rate' | 'request_capacity'
+export type HttpRequestOutcome = '2xx' | '3xx' | '4xx' | '5xx' | 'other' | 'network_error'
+export type RepublishDestination = 'frontier' | 'delay'
+export type RepublishTopic = 'frontier' | 'retry_1m' | 'retry_10m' | 'retry_1h'
+type BatchDiversityScope = 'origin' | 'registrable_domain'
+type PartitionUrlStage = 'parsed' | 'accepted' | 'unique' | 'fetchable' | 'not_ready' | 'store_deduped'
+type PartitionAttemptDisposition = 'completed' | 'republished'
+type PolicyAndBudgetReason = FetchRefusalReason | RequestScheduleBlockReason | 'none'
+
+const BATCH_DIVERSITY_TOP_COUNTS = [1, 5, 10] as const
+
+export class ImageFetchConsumerMetrics {
+ private static readonly fetchable = new Counter({
+ name: 'ai_research_v4_image_fetch_consumer_fetchable_total',
+ help: 'URLs that passed every check and would have been fetched. In dry run no request is sent, so this is the offered rate rather than the sent rate',
+ })
+ /**
+ * `batch` costs only memory, while `store` costs a shared-store read.
+ */
+ private static readonly deduped = new Counter({
+ name: 'ai_research_v4_image_fetch_consumer_deduped_total',
+ help: 'URLs skipped as already known, by the layer that caught them: "batch" (another copy in the same poll batch) or "store" (a durable crawl-history result)',
+ labelNames: ['scope'],
+ })
+ /**
+ * A sustained rate means the producer and consumer disagree about the wire format.
+ */
+ private static readonly dropped = new Counter({
+ name: 'ai_research_v4_image_fetch_consumer_dropped_total',
+ help: 'URLs refused before dedup because the versioned record, URL, ref, or registrable-domain key was invalid. When a dead-letter topic is configured, its Kafka acknowledgement precedes this increment and the source commit',
+ labelNames: ['reason'],
+ })
+ /**
+ * A sustained rate here is expected after a beacon-list change, while the fetcher drains the
+ * queued beacons at parse speed. It is not the invalid-input signal that `dropped` carries.
+ */
+ private static readonly skipped = new Counter({
+ name: 'ai_research_v4_image_fetch_consumer_skipped_total',
+ help: 'Frontier jobs the parser dropped on their own because the URL policy refuses the URL as unwanted, by decline reason. The record and its other jobs proceed and no dead-letter record is written. The count lands after the batch finishes its durable work, so a failed batch is not counted twice; a crash before the offset store can count it once more, like every counter in this consumer',
+ labelNames: ['reason'],
+ })
+ private static readonly deadLettered = new Counter({
+ name: 'ai_research_v4_image_fetch_consumer_dead_lettered_total',
+ help: 'Rejected frontier records acknowledged by the dead-letter topic before the source offset can advance',
+ labelNames: ['reason'],
+ })
+ private static readonly deadLetterFailed = new Counter({
+ name: 'ai_research_v4_image_fetch_consumer_dead_letter_failed_total',
+ help: 'Rejected frontier records that could not reach the dead-letter topic. The source batch fails so its offsets remain uncommitted',
+ labelNames: ['reason'],
+ })
+ /**
+ * This distribution shows how much independent origin state one poll can activate.
+ */
+ private static readonly originsPerBatch = new Histogram({
+ name: 'ai_research_v4_image_fetch_consumer_origins_per_batch',
+ help: 'Distinct origins in one poll batch',
+ buckets: [1, 2, 4, 8, 16, 32, 64],
+ })
+ private static readonly distinctOriginsPerBatch = new Histogram({
+ name: 'ai_research_v4_image_fetch_consumer_distinct_origins_per_batch',
+ help: 'Distinct origins in one poll batch, including empty batches and the full configured batch range',
+ buckets: [0, 1, 2, 4, 8, 16, 32, 64, 128, 256, 512, 1024, 2048, 4096, 8192, 16_384],
+ })
+ private static readonly registrableDomainsPerBatch = new Histogram({
+ name: 'ai_research_v4_image_fetch_consumer_registrable_domains_per_batch',
+ help: 'Distinct registrable domains in one poll batch',
+ buckets: [0, 1, 2, 4, 8, 16, 32, 64, 128, 256, 512, 1024, 2048, 4096, 8192, 16_384],
+ })
+ private static readonly batchTopShare = new Histogram({
+ name: 'ai_research_v4_image_fetch_batch_top_share',
+ help: 'Share of deduplicated canonical URL jobs held by the largest fixed number of origins or registrable domains in one poll batch',
+ labelNames: ['scope', 'top_n'],
+ buckets: [0.01, 0.05, 0.1, 0.25, 0.5, 0.75, 0.9, 0.95, 1],
+ })
+ private static readonly batchEffectiveCount = new Histogram({
+ name: 'ai_research_v4_image_fetch_batch_effective_count',
+ help: 'Inverse Simpson effective count of origins or registrable domains among deduplicated canonical URL jobs in one poll batch',
+ labelNames: ['scope'],
+ buckets: [1, 2, 4, 8, 16, 32, 64, 128, 256, 512, 1024, 2048, 4096, 8192, 16_384, 32_768, 65_536],
+ })
+ private static readonly partitionRecords = new Counter({
+ name: 'ai_research_v4_image_fetch_partition_records_total',
+ help: 'Valid Kafka records handled by source partition',
+ labelNames: ['partition'],
+ })
+ private static readonly partitionUrls = new Counter({
+ name: 'ai_research_v4_image_fetch_partition_urls_total',
+ help: 'URL jobs attributed to a source partition and bounded processing stage. A shared deduplicated job counts once for each contributing partition',
+ labelNames: ['partition', 'stage'],
+ })
+ private static readonly partitionTopShare = new Histogram({
+ name: 'ai_research_v4_image_fetch_partition_top_share',
+ help: 'Share of deduplicated URL jobs in one source partition held by its largest origin or registrable domain in a joined batch',
+ labelNames: ['partition', 'scope'],
+ buckets: [0.01, 0.05, 0.1, 0.25, 0.5, 0.75, 0.9, 0.95, 1],
+ })
+ private static readonly partitionEffectiveCount = new Histogram({
+ name: 'ai_research_v4_image_fetch_partition_effective_count',
+ help: 'Inverse Simpson effective count of origins or registrable domains for one source partition in a joined batch',
+ labelNames: ['partition', 'scope'],
+ buckets: [1, 2, 4, 8, 16, 32, 64, 128, 256, 512, 1024, 2048, 4096, 8192, 16_384],
+ })
+ private static readonly urlsPerRecord = new Histogram({
+ name: 'ai_research_v4_image_fetch_consumer_urls_per_record',
+ help: 'URLs carried by one Kafka record. The producer packs a record to a byte budget, so a distribution pinned at the top means the count bound rather than the byte budget is what split it',
+ buckets: [1, 8, 32, 64, 128, 256, 512],
+ })
+ private static readonly storeErrors = new Counter({
+ name: 'ai_research_v4_image_fetch_consumer_store_errors_total',
+ help: 'Crawl-history keys in a failed read or write. Either failure stops the batch before Kafka offsets advance',
+ labelNames: ['operation'],
+ })
+ private static readonly storeDuration = new Histogram({
+ name: 'ai_research_v4_image_fetch_consumer_store_duration_seconds',
+ help: 'Crawl-history operation wall time by bounded operation and outcome',
+ labelNames: ['operation', 'outcome'],
+ buckets: [0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10, 30, 60, 120],
+ })
+ private static readonly batchDuration = new Histogram({
+ name: 'ai_research_v4_image_fetch_consumer_batch_duration_seconds',
+ help: 'Wall time per completed poll batch. Read against Kafka max.poll.interval.ms (300s)',
+ buckets: [0.01, 0.05, 0.1, 0.5, 1, 5, 15, 30, 60, 120, 240, 300, 600],
+ })
+ private static readonly activeBatchStartedAtMs = new Map<symbol, number>()
+ private static readonly activeBatchElapsed = new Gauge({
+ name: 'ai_research_v4_image_fetch_consumer_active_batch_elapsed_seconds',
+ help: 'Elapsed wall time of the active non-empty poll batch, or zero between batches. This exposes a stuck batch before Kafka max.poll.interval.ms revokes it',
+ collect() {
+ const startedAtMs =
+ ImageFetchConsumerMetrics.activeBatchStartedAtMs.size > 0
+ ? Math.min(...ImageFetchConsumerMetrics.activeBatchStartedAtMs.values())
+ : undefined
+ this.set(startedAtMs === undefined ? 0 : Math.max(0, performance.now() - startedAtMs) / 1000)
+ },
+ })
+ private static readonly ageSeconds = new Histogram({
+ name: 'ai_research_v4_image_fetch_consumer_url_age_seconds',
+ help: 'Age of a URL when this lane reached it, measured from capture rather than from produce. The tail shows the age of the backlog',
+ buckets: [1, 10, 60, 300, 900, 3600, 6 * 3600, 24 * 3600],
+ })
+ private static readonly dryRun = new Gauge({
+ name: 'ai_research_v4_image_fetch_consumer_dry_run',
+ help: '1 while the lane sends no outbound request, 0 once fetching is enabled. Every other metric of this lane means something different either side of this value',
+ })
+
+ public static setDryRun(enabled: boolean): void {
+ this.dryRun.set(enabled ? 1 : 0)
+ }
+ public static incFetchable(count: number): void {
+ this.fetchable.inc(count)
+ }
+ public static incDeduped(scope: DedupScope, count: number): void {
+ this.deduped.labels(scope).inc(count)
+ }
+ public static incDropped(reason: UrlDropReason, count: number): void {
+ this.dropped.labels(reason).inc(count)
+ }
+ public static incSkipped(reason: UrlSkipReason, count: number): void {
+ this.skipped.labels(reason).inc(count)
+ }
+ public static incDeadLettered(reason: FrontierDeadLetterReason): void {
+ this.deadLettered.labels(reason).inc()
+ }
+ public static incDeadLetterFailed(reason: FrontierDeadLetterReason): void {
+ this.deadLetterFailed.labels(reason).inc()
+ }
+ public static incStoreError(operation: 'read' | 'write', count: number): void {
+ this.storeErrors.labels(operation).inc(count)
+ }
+ public static observeStoreDuration(
+ operation: 'read' | 'write',
+ outcome: 'success' | 'error',
+ durationSeconds: number
+ ): void {
+ this.storeDuration.labels(operation, outcome).observe(durationSeconds)
+ }
+ public static observeBatch(origins: number, registrableDomains: number, durationSeconds: number): void {
+ this.originsPerBatch.observe(origins)
+ this.distinctOriginsPerBatch.observe(origins)
+ this.registrableDomainsPerBatch.observe(registrableDomains)
+ this.batchDuration.observe(durationSeconds)
+ }
+ public static observeBatchDiversity(originCounts: number[], registrableDomainCounts: number[]): void {
+ this.observeBatchDistribution('origin', originCounts)
+ this.observeBatchDistribution('registrable_domain', registrableDomainCounts)
+ }
+ public static startBatch(nowMs = performance.now()): symbol {
+ const batchId = Symbol()
+ this.activeBatchStartedAtMs.set(batchId, nowMs)
+ return batchId
+ }
+ public static finishBatch(batchId: symbol): void {
+ this.activeBatchStartedAtMs.delete(batchId)
+ }
+ public static observeRecord(urls: number): void {
+ this.urlsPerRecord.observe(urls)
+ }
+ public static observePartitionRecord(partition: number, urls: number, acceptedUrls: number): void {
+ const partitionLabel = String(partition)
+ this.partitionRecords.labels(partitionLabel).inc()
+ this.partitionUrls.labels(partitionLabel, 'parsed').inc(urls)
+ this.partitionUrls.labels(partitionLabel, 'accepted').inc(acceptedUrls)
+ }
+ public static incPartitionUrls(partition: number, stage: PartitionUrlStage, count: number): void {
+ this.partitionUrls.labels(String(partition), stage).inc(count)
+ }
+ public static observePartitionBatchDiversity(
+ partition: number,
+ originCounts: number[],
+ registrableDomainCounts: number[]
+ ): void {
+ this.observePartitionDistribution(partition, 'origin', originCounts)
+ this.observePartitionDistribution(partition, 'registrable_domain', registrableDomainCounts)
+ }
+ public static observeAge(ageSeconds: number): void {
+ this.ageSeconds.observe(ageSeconds)
+ }
+
+ private static observeBatchDistribution(scope: BatchDiversityScope, counts: number[]): void {
+ const summary = summarizeBatchDistribution(counts)
+ if (!summary) {
+ return
+ }
+ for (const topCount of BATCH_DIVERSITY_TOP_COUNTS) {
+ const topTotal = summary.largestCounts.slice(0, topCount).reduce((total, count) => total + count, 0)
+ this.batchTopShare.labels(scope, String(topCount)).observe(topTotal / summary.total)
+ }
+ this.batchEffectiveCount.labels(scope).observe((summary.total * summary.total) / summary.sumOfSquares)
+ }
+
+ private static observePartitionDistribution(partition: number, scope: BatchDiversityScope, counts: number[]): void {
+ const summary = summarizeBatchDistribution(counts)
+ if (!summary) {
+ return
+ }
+ const partitionLabel = String(partition)
+ this.partitionTopShare.labels(partitionLabel, scope).observe(summary.largestCounts[0] / summary.total)
+ this.partitionEffectiveCount
+ .labels(partitionLabel, scope)
+ .observe((summary.total * summary.total) / summary.sumOfSquares)
+ }
+}
+
+function summarizeBatchDistribution(
+ counts: number[]
+): { total: number; sumOfSquares: number; largestCounts: number[] } | undefined {
+ let total = 0
+ let sumOfSquares = 0
+ const largestCounts: number[] = []
+ for (const count of counts) {
+ total += count
+ sumOfSquares += count * count
+ const insertionIndex = largestCounts.findIndex((existing) => count > existing)
+ if (insertionIndex >= 0) {
+ largestCounts.splice(insertionIndex, 0, count)
+ } else if (largestCounts.length < BATCH_DIVERSITY_TOP_COUNTS.at(-1)!) {
+ largestCounts.push(count)
+ }
+ if (largestCounts.length > BATCH_DIVERSITY_TOP_COUNTS.at(-1)!) {
+ largestCounts.pop()
+ }
+ }
+ return total > 0 ? { total, sumOfSquares, largestCounts } : undefined
+}
+
+/**
+ * The request path.
+ *
+ * No metric here carries a host or a URL. The host set is unbounded, so it lives in the structured
+ * logs of the runner. A URL is page content and belongs in no metric, log, or trace.
+ */
+export class ImageFetchRequestMetrics {
+ private static readonly completedUrls = new Counter({
+ name: 'ai_research_v4_image_fetch_completed_urls_total',
+ help: 'Completed URLs by final outcome and refusal reason',
+ labelNames: ['outcome', 'refusal_reason'],
+ })
+ private static readonly completedUrlSystemTime = new Histogram({
+ name: 'ai_research_v4_image_fetch_completed_url_system_time_seconds',
+ help: 'Time from first collection until the URL reaches a terminal outcome',
+ buckets: [1, 60, 600, 3600, 6 * 3600, 24 * 3600, 7 * 24 * 3600, 30 * 24 * 3600],
+ })
+ private static readonly completedUrlFetches = new Histogram({
+ name: 'ai_research_v4_image_fetch_completed_url_fetches',
+ help: 'Image HTTP requests made before a URL reaches a terminal outcome',
+ buckets: [0, 1, 2, 3, 5, 10],
+ })
+ private static readonly completedUrlRepublishes = new Histogram({
+ name: 'ai_research_v4_image_fetch_completed_url_republishes',
+ help: 'Frontier and delay-topic republishes before a URL reaches a terminal outcome',
+ buckets: [0, 1, 2, 3, 5, 10],
+ })
+ private static readonly policyAndBudgetDecisions = new Counter({
+ name: 'ai_research_v4_image_fetch_policy_and_budget_decisions_total',
+ help: 'Origin-policy and registrable-domain request-control decisions after block state is known',
+ labelNames: ['blocked', 'reason'],
+ })
+ private static readonly batchSchedulableSlots = new Histogram({
+ name: 'ai_research_v4_image_fetch_batch_schedulable_slots',
+ help: 'Request slots that the initial fetch queue can use after live pod and registrable-domain concurrency limits',
+ buckets: [1, 2, 4, 8, 16, 32, 64, 128, 256, 300, 512, 1024],
+ })
+ private static readonly batchSchedulableCapacityRatio = new Histogram({
+ name: 'ai_research_v4_image_fetch_batch_schedulable_capacity_ratio',
+ help: 'Share of the pod request limit that the initial fetch queue can use after live concurrency limits',
+ buckets: [0.01, 0.05, 0.1, 0.25, 0.5, 0.75, 0.9, 0.95, 1],
+ })
+ /**
+ * `ok` against the sum is the yield of the lane.
+ */
+ private static readonly outcomes = new Counter({
+ name: 'ai_research_v4_image_fetch_requests_total',
+ help: 'Completed external HTTP requests by status class or network failure',
+ labelNames: ['outcome'],
+ })
+ private static readonly duration = new Histogram({
+ name: 'ai_research_v4_image_fetch_request_duration_seconds',
+ help: 'Time for one completed external HTTP request, including its response body',
+ labelNames: ['outcome'],
+ buckets: [0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10, 30],
+ })
+ private static readonly partitionOutcomes = new Counter({
+ name: 'ai_research_v4_image_fetch_partition_requests_total',
+ help: 'Completed image HTTP requests attributed to each source partition and status class or network failure',
+ labelNames: ['partition', 'outcome'],
+ })
+ private static readonly partitionDuration = new Histogram({
+ name: 'ai_research_v4_image_fetch_partition_request_duration_seconds',
+ help: 'Time for one completed image HTTP request attributed to each source partition, including its response body',
+ labelNames: ['partition', 'outcome'],
+ buckets: [0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10, 30],
+ })
+ private static readonly partitionAttempts = new Counter({
+ name: 'ai_research_v4_image_fetch_partition_attempts_total',
+ help: 'URL jobs completed or republished after successful delivery and URL accounting, attributed to each source partition and outcome',
+ labelNames: ['partition', 'disposition', 'outcome'],
+ })
+ private static readonly retryCauses = new Counter({
+ name: 'ai_research_v4_image_fetch_retry_causes_total',
+ help: 'Transient fetch outcomes that caused the URL to be scheduled for another attempt',
+ labelNames: ['cause'],
+ })
+ /**
+ * Read a rising tail with the `deadline` outcome to distinguish headroom from load that an
+ * origin's allowed rate cannot carry within one pass.
+ */
+ private static readonly schedulerWait = new Histogram({
+ name: 'ai_research_v4_image_fetch_scheduler_wait_seconds',
+ help: 'Time a request waited for an origin crawl delay, registrable-domain rate, or pod request slot',
+ labelNames: ['scope'],
+ buckets: [0, 0.1, 0.5, 1, 2, 5, 10, 20],
+ })
+ private static readonly partitionSchedulerWait = new Histogram({
+ name: 'ai_research_v4_image_fetch_partition_scheduler_wait_seconds',
+ help: 'Time an image request waited, attributed to each source partition and politeness or capacity scope',
+ labelNames: ['partition', 'scope'],
+ buckets: [0, 0.1, 0.5, 1, 2, 5, 10, 20],
+ })
+ private static readonly responseBytes = new Histogram({
+ name: 'ai_research_v4_image_fetch_response_bytes',
+ help: 'Size of a downloaded image. The tail against the configured byte limit says how much of the catalog the limit refuses',
+ buckets: [
+ 1024,
+ 8 * 1024,
+ 32 * 1024,
+ 128 * 1024,
+ 512 * 1024,
+ 1024 * 1024,
+ 2 * 1024 * 1024,
+ 10 * 1024 * 1024,
+ 20 * 1024 * 1024,
+ ],
+ })
+ private static readonly redirects = new Histogram({
+ name: 'ai_research_v4_image_fetch_redirects',
+ help: 'Redirects followed for one URL. Each hop uses its registrable-domain budget and origin crawl delay',
+ buckets: [0, 1, 2, 3],
+ })
+ /**
+ * Both gauges read the budget at scrape time. A hold expires by the clock, so a count taken at
+ * the end of a batch would report blocked origins until the next batch arrives.
+ */
+ private static readonly trackedRegistrableDomains = new Gauge({
+ name: 'ai_research_v4_image_fetch_tracked_registrable_domains',
+ help: 'Registrable domains this pod holds request-control state for',
+ collect() {
+ this.set(ImageFetchRequestMetrics.budget?.trackedRegistrableDomains ?? 0)
+ },
+ })
+ private static readonly trackedOrigins = new Gauge({
+ name: 'ai_research_v4_image_fetch_tracked_origins',
+ help: 'Origins this pod holds configuration and crawl-delay state for',
+ collect() {
+ this.set(ImageFetchRequestMetrics.budget?.trackedOrigins ?? 0)
+ },
+ })
+ private static readonly blockedRegistrableDomains = new Gauge({
+ name: 'ai_research_v4_image_fetch_blocked_registrable_domains',
+ help: 'Registrable domains held by transient back-off, Retry-After, or an open circuit breaker',
+ collect() {
+ this.set(ImageFetchRequestMetrics.budget?.blockedRegistrableDomains(Date.now()) ?? 0)
+ },
+ })
+
+ public static outcomeForHttpStatus(status?: number): HttpRequestOutcome {
+ if (status === undefined) {
+ return 'network_error'
+ }
+ if (status >= 200 && status < 300) {
+ return '2xx'
+ }
+ if (status >= 300 && status < 400) {
+ return '3xx'
+ }
+ if (status >= 400 && status < 500) {
+ return '4xx'
+ }
+ if (status >= 500 && status < 600) {
+ return '5xx'
+ }
+ return 'other'
+ }
+ public static observeRequest(
+ outcome: HttpRequestOutcome,
+ durationSeconds: number,
+ sourcePartitions?: readonly number[]
+ ): void {
+ this.outcomes.labels(outcome).inc()
+ this.duration.labels(outcome).observe(durationSeconds)
+ for (const sourcePartition of new Set(sourcePartitions ?? [])) {
+ const partitionLabel = String(sourcePartition)
+ this.partitionOutcomes.labels(partitionLabel, outcome).inc()
+ this.partitionDuration.labels(partitionLabel, outcome).observe(durationSeconds)
+ }
+ }
+ public static incRetryCause(cause: TransientFetchOutcome): void {
+ this.retryCauses.labels(cause).inc()
+ }
+ public static observeRedirectCount(redirects: number): void {
+ this.redirects.observe(redirects)
+ }
+ public static observeCompletedUrl(
+ outcome: AttemptOutcome,
+ refusalReason: FetchRefusalReason | 'none',
+ systemTimeSeconds: number,
+ fetches: number,
+ republishes: number
+ ): void {
+ this.completedUrls.labels(outcome, refusalReason).inc()
+ this.completedUrlSystemTime.observe(systemTimeSeconds)
+ this.completedUrlFetches.observe(fetches)
+ this.completedUrlRepublishes.observe(republishes)
+ }
+ public static observePolicyAndBudgetDecision(blocked: boolean, reason: PolicyAndBudgetReason = 'none'): void {
+ this.policyAndBudgetDecisions.labels(blocked ? 'true' : 'false', reason).inc()
+ }
+ public static observeBatchSchedulableCapacity(slots: number, podRequestLimit: number): void {
+ const boundedSlots = Math.min(slots, podRequestLimit)
+ this.batchSchedulableSlots.observe(boundedSlots)
+ this.batchSchedulableCapacityRatio.observe(boundedSlots / podRequestLimit)
+ }
+ public static observeSchedulerWait(
+ scope: SchedulerWaitScope,
+ waitSeconds: number,
+ sourcePartitions?: readonly number[]
+ ): void {
+ this.schedulerWait.labels(scope).observe(waitSeconds)
+ for (const sourcePartition of new Set(sourcePartitions ?? [])) {
+ this.partitionSchedulerWait.labels(String(sourcePartition), scope).observe(waitSeconds)
+ }
+ }
+ public static incPartitionAttempt(
+ partition: number,
+ disposition: PartitionAttemptDisposition,
+ outcome: AttemptOutcome
+ ): void {
+ this.partitionAttempts.labels(String(partition), disposition, outcome).inc()
+ }
+ public static observeBytes(bytes: number): void {
+ this.responseBytes.observe(bytes)
+ }
+ private static readonly evictedWhileBlocked = new Gauge({
+ name: 'ai_research_v4_image_fetch_registrable_domains_evicted_while_blocked',
+ help: 'Registrable domains removed from request-control state while still blocked. This must stay zero',
+ collect() {
+ this.set(ImageFetchRequestMetrics.budget?.evictedWhileBlocked ?? 0)
+ },
+ })
+
+ /**
+ * Requests holding a socket right now, against the pod limit.
+ *
+ * A request waiting for a politeness token has not reached this yet, so a value at the limit
+ * means the pod is the bottleneck rather than the sites.
+ */
+ private static readonly inFlight = new Gauge({
+ name: 'ai_research_v4_image_fetch_requests_in_flight',
+ help: 'External requests this pod holds open. A URL waiting for its registrable-domain rate limit holds no socket and is not counted',
+ collect() {
+ this.set(ImageFetchRequestMetrics.requests?.running ?? 0)
+ },
+ })
+
+ private static budget: BudgetCounts | undefined
+ private static requests: InFlightCount | undefined
+
+ /** The runner owns both. */
+ public static trackBudget(budget: BudgetCounts, requests: InFlightCount): void {
+ this.budget = budget
+ this.requests = requests
+ }
+
+ /**
+ * Read against `ai_research_v4_image_fetch_requests_total` this is the amplification factor, because the lane
+ * reads every republished message again. A rate that approaches the request rate means most work
+ * goes around rather than completes.
+ */
+ private static readonly republished = new Counter({
+ name: 'ai_research_v4_image_fetch_republished_total',
+ help: 'URLs published back to Kafka by bounded reason and destination class. "redirect" left the origin. "retry" hit a transient failure and waits in a delay topic. "not_ready" arrived before its wait ended',
+ labelNames: ['reason', 'topic'],
+ })
+ private static readonly republishFailed = new Counter({
+ name: 'ai_research_v4_image_fetch_republish_failed_total',
+ help: 'URLs in a Kafka message whose republish delivery failed. The input batch throws and leaves its offsets uncommitted',
+ labelNames: ['reason'],
+ })
+ private static readonly republishMessagesPerBatch = new Histogram({
+ name: 'ai_research_v4_image_fetch_republish_messages_per_batch',
+ help: 'Kafka record delivery attempts by one fetch batch for each destination topic class',
+ labelNames: ['topic'],
+ buckets: [1, 2, 4, 8, 16, 32, 64, 128, 256, 512, 1024, 2048, 4096, 8192, 16_384],
+ })
+ private static readonly republishRegistrableDomainsPerBatch = new Histogram({
+ name: 'ai_research_v4_image_fetch_republish_registrable_domains_per_batch',
+ help: 'Distinct registrable-domain keys in delivery attempts by one fetch batch for each destination topic class',
+ labelNames: ['topic'],
+ buckets: [1, 2, 4, 8, 16, 32, 64, 128, 256, 512, 1024, 2048, 4096, 8192, 16_384],
+ })
+ private static readonly republishDuration = new Histogram({
+ name: 'ai_research_v4_image_fetch_republish_duration_seconds',
+ help: 'Wall time from scheduling one destination topic class until all its started Kafka delivery attempts settle',
+ labelNames: ['topic'],
+ buckets: [0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10, 30, 60, 120, 240, 300],
+ })
+ private static readonly republishFlushDuration = new Histogram({
+ name: 'ai_research_v4_image_fetch_republish_flush_duration_seconds',
+ help: 'Total wall time to group, serialize, and wait for every started republish delivery in one fetch batch',
+ buckets: [0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10, 30, 60, 120, 240, 300],
+ })
+ private static readonly republishFlushDeadlineExceeded = new Counter({
+ name: 'ai_research_v4_image_fetch_republish_flush_deadline_exceeded_total',
+ help: 'Fetch batches that stopped starting Kafka republish records to preserve final history-write and poll-interval margin',
+ })
+ /**
+ * Zero is the common case. The tail shows redirect chains and retries.
+ */
+ private static readonly hopsUsed = new Histogram({
+ name: 'ai_research_v4_image_fetch_hops_used',
+ help: 'Redirects and retries spent from one URL job before the lane finished it',
+ buckets: [0, 1, 2, 3, 5, 10],
+ })
+
+ public static incRepublished(reason: RepublishReason, destination: RepublishDestination): void {
+ this.republished.labels(reason, destination).inc()
+ }
+ public static incRepublishFailed(reason: RepublishReason): void {
+ this.republishFailed.labels(reason).inc()
+ }
+ public static observeRepublishBatch(
+ topic: RepublishTopic,
+ messages: number,
+ registrableDomains: number,
+ durationSeconds: number
+ ): void {
+ this.republishMessagesPerBatch.labels(topic).observe(messages)
+ this.republishRegistrableDomainsPerBatch.labels(topic).observe(registrableDomains)
+ this.republishDuration.labels(topic).observe(durationSeconds)
+ }
+ public static observeRepublishFlush(durationSeconds: number): void {
+ this.republishFlushDuration.observe(durationSeconds)
+ }
+ public static incRepublishFlushDeadlineExceeded(): void {
+ this.republishFlushDeadlineExceeded.inc()
+ }
+ public static observeHops(hops: number): void {
+ this.hopsUsed.observe(hops)
+ }
+}
+
+/**
+ * The consumer of one delay topic.
+ *
+ * Lag on these topics is the design at work rather than a fault, because a record that waits out its
+ * period counts as lag. Read `ai_research_v4_image_fetch_retry_wait_seconds` against the period of the topic
+ * instead. A wait far short of the period means records arrive with their wait already spent, which
+ * means the tier below is too small.
+ */
+export class RetryDelayMetrics {
+ private static readonly released = new Counter({
+ name: 'ai_research_v4_image_fetch_retry_released_total',
+ help: 'Records this delay consumer handled by outcome. A malformed record is dropped, while an invalid timestamp or failed publish leaves the offset uncommitted',
+ labelNames: ['outcome'],
+ })
+ private static readonly waitSeconds = new Histogram({
+ name: 'ai_research_v4_image_fetch_retry_wait_seconds',
+ help: 'Time a record still had to wait when this consumer read it. Compare with the period of the topic: a much shorter wait means the records arrived late, and the consumer is behind',
+ buckets: [1, 10, 60, 300, 600, 1800, 3600],
+ })
+
+ public static incReleased(
+ outcome: 'released' | 'failed' | 'malformed' | 'invalid_timestamp' | 'abandoned',
+ count = 1
+ ): void {
+ this.released.labels(outcome).inc(count)
+ }
+ public static observeWait(waitSeconds: number): void {
+ this.waitSeconds.observe(waitSeconds)
+ }
+}
+
+export class ImageFetchPoolMetrics {
+ private static readonly waitSeconds = new Histogram({
+ name: 'ai_research_v4_image_fetch_pool_wait_seconds',
+ help: 'Time a candidate waited in the pod candidate pool before a fetch worker took it',
+ buckets: [0, 0.1, 0.5, 1, 2, 5, 10, 20, 40],
+ })
+ private static readonly refillWaitSeconds = new Histogram({
+ name: 'ai_research_v4_image_fetch_pool_refill_wait_seconds',
+ help: 'Time a consumer waited for room in its pool window before it read its next batch. outcome=timeout means the wait reached its cap',
+ labelNames: ['outcome'],
+ buckets: [0, 0.1, 0.5, 1, 2, 5, 10, 30],
+ })
+
+ public static observeWait(waitSeconds: number): void {
+ this.waitSeconds.observe(waitSeconds)
+ }
+ public static observeRefillWait(outcome: 'room' | 'timeout', waitSeconds: number): void {
+ this.refillWaitSeconds.labels(outcome).observe(waitSeconds)
+ }
+}
diff --git a/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/origin-request-scheduler.test.ts b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/origin-request-scheduler.test.ts
new file mode 100644
index 000000000000..3e610226aaff
--- /dev/null
+++ b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/origin-request-scheduler.test.ts
@@ -0,0 +1,307 @@
+import { register } from 'prom-client'
+
+import { createRecordingTopHog } from '~/tests/helpers/tophog'
+
+import { HostBudget, HostBudgetOptions } from './host-budget'
+import { ImageFetchRequestMetrics } from './metrics'
+import { OriginRequestScheduler } from './origin-request-scheduler'
+import { ImageFetchTopHogMetrics } from './tophog-metrics'
+
+const OPTIONS: HostBudgetOptions = {
+ requestsPerSecond: 1,
+ burst: 5,
+ maxConcurrent: 6,
+ breakerFailures: 2,
+ breakerCooldownMs: 60_000,
+ breakerMaxCooldownMs: 3_600_000,
+ maxTrackedRegistrableDomains: 20_000,
+ maxTrackedOrigins: 20_000,
+ random: () => 1,
+}
+const ORIGIN = new URL('https://example.com')
+const REGISTRABLE_DOMAIN = 'example.com'
+
+describe('OriginRequestScheduler', () => {
+ beforeEach(() => jest.useFakeTimers().setSystemTime(1_700_000_000_000))
+ afterEach(() => {
+ jest.restoreAllMocks()
+ jest.useRealTimers()
+ })
+
+ it('keeps concurrent request start times at least one second apart', async () => {
+ register.resetMetrics()
+ const budget = new HostBudget(OPTIONS)
+ budget.setCrawlDelay(ORIGIN.origin, 1_000, Date.now())
+ const recordingTopHog = createRecordingTopHog()
+ const scheduler = new OriginRequestScheduler(budget, 300, new ImageFetchTopHogMetrics(recordingTopHog.registry))
+ const observeSchedulerWait = jest.spyOn(ImageFetchRequestMetrics, 'observeSchedulerWait')
+ const startedAtMs: number[] = []
+ const deadlineMs = Date.now() + 10_000
+
+ const requests = [0, 1, 2].map(() =>
+ scheduler.runImage(
+ ORIGIN,
+ deadlineMs,
+ () => {
+ startedAtMs.push(Date.now())
+ return Promise.resolve()
+ },
+ [7, 42]
+ )
+ )
+ await jest.advanceTimersByTimeAsync(0)
+ const active = register.getSingleMetric('ai_research_v4_image_fetch_stage_active')!
+ expect((await active.get()).values).toEqual(
+ expect.arrayContaining([
+ expect.objectContaining({ labels: { stage: 'image_origin_crawl_delay' }, value: 2 }),
+ expect.objectContaining({ labels: { stage: 'image_http' }, value: 0 }),
+ ])
+ )
+ await jest.runAllTimersAsync()
+ expect((await active.get()).values.every(({ value }) => value === 0)).toBe(true)
+ const durations = await register.getSingleMetric('ai_research_v4_image_fetch_stage_duration_seconds')!.get()
+ expect(durations.values).toEqual(
+ expect.arrayContaining([
+ expect.objectContaining({
+ metricName: 'ai_research_v4_image_fetch_stage_duration_seconds_sum',
+ labels: { stage: 'image_origin_crawl_delay' },
+ value: 3,
+ }),
+ ])
+ )
+
+ expect(await Promise.all(requests)).toEqual([
+ { ran: true, value: undefined },
+ { ran: true, value: undefined },
+ { ran: true, value: undefined },
+ ])
+ expect(startedAtMs).toEqual([1_700_000_000_000, 1_700_000_001_000, 1_700_000_002_000])
+ expect(observeSchedulerWait).toHaveBeenCalledWith('origin_crawl_delay', 1, [7, 42])
+ expect(observeSchedulerWait).toHaveBeenCalledWith('request_capacity', 0, [7, 42])
+ expect(recordingTopHog.records.get('ai_research_v4_image_fetch_block_events_by_registrable_domain')).toEqual(
+ expect.arrayContaining([
+ {
+ key: {
+ registrable_domain: REGISTRABLE_DOMAIN,
+ reason: 'origin_crawl_delay',
+ partition: '7',
+ },
+ value: 1,
+ },
+ ])
+ )
+ expect(recordingTopHog.records.get('ai_research_v4_image_fetch_blocked_ms_by_registrable_domain')).toEqual(
+ expect.arrayContaining([
+ {
+ key: {
+ registrable_domain: REGISTRABLE_DOMAIN,
+ reason: 'origin_crawl_delay',
+ partition: '7',
+ },
+ value: 1_000,
+ },
+ ])
+ )
+ expect(
+ recordingTopHog.records
+ .get('ai_research_v4_image_fetch_block_events_by_registrable_domain')
+ ?.some(({ key }) => key.reason === 'request_capacity')
+ ).toBe(false)
+ })
+
+ it('allows six concurrent same-origin requests when the request rate and crawl delay are disabled', async () => {
+ const scheduler = new OriginRequestScheduler(
+ new HostBudget({ ...OPTIONS, requestsPerSecond: 0, burst: 6, maxConcurrent: 6 }),
+ 300
+ )
+ const releases: Array<() => void> = []
+ const requests = Array.from({ length: 7 }, () =>
+ scheduler.runImage(
+ ORIGIN,
+ Date.now() + 10_000,
+ () =>
+ new Promise<void>((resolve) => {
+ releases.push(resolve)
+ })
+ )
+ )
+
+ await expect(requests[6]).resolves.toEqual({
+ ran: false,
+ reason: 'connection_limit',
+ blockingReason: 'connection_limit',
+ waitMs: 0,
+ })
+ expect(releases).toHaveLength(6)
+ releases.forEach((release) => release())
+ await expect(Promise.all(requests.slice(0, 6))).resolves.toEqual(
+ Array.from({ length: 6 }, () => ({ ran: true, value: undefined }))
+ )
+ })
+
+ it('runs one half-open probe while later requests remain blocked', async () => {
+ const budget = new HostBudget(OPTIONS)
+ budget.recordTransientFailure(REGISTRABLE_DOMAIN, Date.now())
+ budget.recordTransientFailure(REGISTRABLE_DOMAIN, Date.now())
+ jest.setSystemTime(Date.now() + 120_001)
+ const scheduler = new OriginRequestScheduler(budget, 300)
+ let finishProbe: (() => void) | undefined
+ const probe = scheduler.runImage(
+ ORIGIN,
+ Date.now() + 10_000,
+ () =>
+ new Promise<void>((resolve) => {
+ finishProbe = resolve
+ })
+ )
+
+ await Promise.resolve()
+ await expect(scheduler.runImage(ORIGIN, Date.now() + 10_000, () => Promise.resolve())).resolves.toEqual({
+ ran: false,
+ reason: 'breaker_open',
+ blockingReason: 'breaker_open',
+ waitMs: OPTIONS.breakerCooldownMs,
+ })
+ expect(finishProbe).toBeDefined()
+ finishProbe?.()
+ await expect(probe).resolves.toEqual({ ran: true, value: undefined })
+ })
+
+ it('preserves the response backoff reason', async () => {
+ const budget = new HostBudget(OPTIONS)
+ budget.recordTransientFailure(REGISTRABLE_DOMAIN, Date.now(), 120_000)
+ const scheduler = new OriginRequestScheduler(budget, 300)
+
+ await expect(scheduler.runImage(ORIGIN, Date.now() + 10_000, () => Promise.resolve())).resolves.toEqual({
+ ran: false,
+ reason: 'backoff',
+ blockingReason: 'retry_after',
+ waitMs: 120_000,
+ })
+ })
+
+ it('returns a queued request to the caller when its pass deadline expires', async () => {
+ const scheduler = new OriginRequestScheduler(new HostBudget(OPTIONS), 1)
+ let releaseFirst: (() => void) | undefined
+ const first = scheduler.runImage(
+ new URL('https://first.example'),
+ Date.now() + 10_000,
+ () =>
+ new Promise<void>((resolve) => {
+ releaseFirst = resolve
+ })
+ )
+ const second = scheduler.runImage(new URL('https://second.example'), Date.now() + 500, () => Promise.resolve())
+
+ await Promise.resolve()
+ jest.advanceTimersByTime(501)
+ releaseFirst?.()
+
+ await expect(first).resolves.toEqual({ ran: true, value: undefined })
+ await expect(second).resolves.toEqual({
+ ran: false,
+ reason: 'deadline',
+ blockingReason: 'request_deadline',
+ waitMs: 0,
+ })
+ })
+
+ it('checks the registrable-domain token bucket after a pod-capacity wait', async () => {
+ const scheduler = new OriginRequestScheduler(new HostBudget({ ...OPTIONS, burst: 2 }), 1)
+ const startedAtMs: number[] = []
+ let releaseFirst: (() => void) | undefined
+ const startRequest = (origin: string, hold = false): Promise<unknown> =>
+ scheduler.runImage(new URL(origin), Date.now() + 30_000, () => {
+ startedAtMs.push(Date.now())
+ if (!hold) {
+ return Promise.resolve()
+ }
+ return new Promise<void>((resolve) => {
+ releaseFirst = resolve
+ })
+ })
+
+ const requests = [
+ startRequest('https://a.example.com/image.png', true),
+ startRequest('https://b.example.com/image.png'),
+ startRequest('https://c.example.com/image.png'),
+ startRequest('https://d.example.com/image.png'),
+ ]
+ await Promise.resolve()
+ jest.advanceTimersByTime(10_000)
+ releaseFirst?.()
+ await jest.runAllTimersAsync()
+ await Promise.all(requests)
+
+ expect(startedAtMs).toEqual([1_700_000_000_000, 1_700_000_010_000, 1_700_000_010_000, 1_700_000_011_000])
+ })
+
+ it('returns a crawl wait that is longer than the pass deadline', async () => {
+ const budget = new HostBudget(OPTIONS)
+ budget.setCrawlDelay(ORIGIN.origin, 600_000, Date.now())
+ const scheduler = new OriginRequestScheduler(budget, 300)
+ await expect(scheduler.runImage(ORIGIN, Date.now() + 10_000, () => Promise.resolve())).resolves.toMatchObject({
+ ran: true,
+ })
+
+ await expect(scheduler.runImage(ORIGIN, Date.now() + 20_000, () => Promise.resolve())).resolves.toEqual({
+ ran: false,
+ reason: 'deadline',
+ blockingReason: 'origin_crawl_delay',
+ waitMs: 600_000,
+ })
+ })
+
+ it('shares the connection limit across sibling origins', async () => {
+ const scheduler = new OriginRequestScheduler(new HostBudget({ ...OPTIONS, maxConcurrent: 1 }), 300)
+ let releaseFirst: (() => void) | undefined
+ const first = scheduler.runImage(
+ new URL('https://a.example.com/image.png'),
+ Date.now() + 10_000,
+ () =>
+ new Promise<void>((resolve) => {
+ releaseFirst = resolve
+ })
+ )
+
+ await Promise.resolve()
+ await expect(
+ scheduler.runImage(new URL('https://b.example.com/image.png'), Date.now() + 10_000, () => Promise.resolve())
+ ).resolves.toEqual({
+ ran: false,
+ reason: 'connection_limit',
+ blockingReason: 'connection_limit',
+ waitMs: 0,
+ })
+
+ releaseFirst?.()
+ await expect(first).resolves.toEqual({ ran: true, value: undefined })
+ })
+
+ it('counts a configuration request against the registrable-domain connection limit', async () => {
+ const scheduler = new OriginRequestScheduler(new HostBudget({ ...OPTIONS, maxConcurrent: 1 }), 300)
+ let releaseConfiguration: (() => void) | undefined
+ const configuration = scheduler.run(
+ new URL('https://a.example.com/robots.txt'),
+ Date.now() + 10_000,
+ () =>
+ new Promise<void>((resolve) => {
+ releaseConfiguration = resolve
+ })
+ )
+
+ await Promise.resolve()
+ await expect(
+ scheduler.runImage(new URL('https://b.example.com/image.png'), Date.now() + 10_000, () => Promise.resolve())
+ ).resolves.toEqual({
+ ran: false,
+ reason: 'connection_limit',
+ blockingReason: 'connection_limit',
+ waitMs: 0,
+ })
+
+ expect(releaseConfiguration).toBeDefined()
+ releaseConfiguration?.()
+ await expect(configuration).resolves.toEqual({ ran: true, value: undefined })
+ })
+})
diff --git a/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/origin-request-scheduler.ts b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/origin-request-scheduler.ts
new file mode 100644
index 000000000000..d7be78697dee
--- /dev/null
+++ b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/origin-request-scheduler.ts
@@ -0,0 +1,192 @@
+import { ConcurrencyController } from '~/common/utils/concurrencyController'
+import { delay } from '~/common/utils/utils'
+
+import type { ImageFetchBlockReason } from './block-reason'
+import { ConfigurationRequestScheduler } from './configuration-policy'
+import { BudgetBlockReason, BudgetGrant, HostBudget } from './host-budget'
+import { ImageFetchRequestMetrics, SchedulerWaitScope } from './metrics'
+import { politenessKey } from './politeness-key'
+import { ImageFetchProcessingMetrics } from './processing-metrics'
+import { ImageFetchTopHogMetrics } from './tophog-metrics'
+
+export type ScheduledRequest<T> =
+ | { ran: true; value: T }
+ | {
+ ran: false
+ reason: BudgetBlockReason | 'connection_limit'
+ blockingReason: ImageFetchBlockReason
+ waitMs: number
+ }
+
+function blockingReasonForStoppedGrant(grant: Extract<BudgetGrant, { granted: false }>): ImageFetchBlockReason {
+ if (grant.reason === 'backoff') {
+ return grant.backoffReason ?? 'unknown_backoff'
+ }
+ if (grant.reason === 'deadline') {
+ return grant.waitScope ?? 'request_deadline'
+ }
+ return grant.reason
+}
+
+export class OriginRequestScheduler implements ConfigurationRequestScheduler {
+ private readonly inFlight: ConcurrencyController
+
+ constructor(
+ private readonly budget: HostBudget,
+ maxInFlightRequests: number,
+ private readonly topHogMetrics?: ImageFetchTopHogMetrics
+ ) {
+ this.inFlight = new ConcurrencyController(maxInFlightRequests)
+ }
+
+ public run<T>(url: URL, deadlineMs: number, request: () => Promise<T>): Promise<ScheduledRequest<T>> {
+ return this.runScheduled(url, deadlineMs, true, undefined, request)
+ }
+
+ public runImage<T>(
+ url: URL,
+ deadlineMs: number,
+ request: () => Promise<T>,
+ sourcePartitions?: readonly number[]
+ ): Promise<ScheduledRequest<T>> {
+ return this.runScheduled(url, deadlineMs, false, sourcePartitions, request)
+ }
+
+ public get running(): number {
+ return this.inFlight.running
+ }
+
+ private async runScheduled<T>(
+ url: URL,
+ deadlineMs: number,
+ configurationRequest: boolean,
+ sourcePartitions: readonly number[] | undefined,
+ request: () => Promise<T>
+ ): Promise<ScheduledRequest<T>> {
+ const requestKind = configurationRequest ? 'configuration' : 'image'
+ const origin = url.origin
+ const registrableDomain = politenessKey(url.hostname)
+ const nowMs = Date.now()
+ if (!this.budget.requestScheduled(origin, nowMs)) {
+ return { ran: false, reason: 'origin_map_full', blockingReason: 'origin_map_full', waitMs: 0 }
+ }
+ try {
+ for (;;) {
+ const capacityWaitStartedAtMs = Date.now()
+ const capacityWait = ImageFetchProcessingMetrics.start(`${requestKind}_request_capacity`)
+ const scheduled = await this.inFlight
+ .run({
+ debugTag: registrableDomain,
+ fn: async () => {
+ capacityWait.finish()
+ const checkedAtMs = Date.now()
+ this.recordSchedulerWait(
+ 'request_capacity',
+ Math.max(0, checkedAtMs - capacityWaitStartedAtMs),
+ registrableDomain,
+ configurationRequest,
+ sourcePartitions
+ )
+ const grant = this.budget.take(
+ registrableDomain,
+ origin,
+ checkedAtMs,
+ deadlineMs,
+ configurationRequest
+ )
+ if (!grant.granted) {
+ return {
+ kind: 'stopped',
+ reason: grant.reason,
+ blockingReason: blockingReasonForStoppedGrant(grant),
+ waitMs: grant.waitMs,
+ } as const
+ }
+ if (grant.waitMs > 0) {
+ return {
+ kind: 'wait',
+ waitMs: grant.waitMs,
+ waitScope: grant.waitScope ?? 'registrable_domain_rate',
+ } as const
+ }
+ if (!this.budget.acquireConnection(registrableDomain, origin)) {
+ this.budget.returnGrant(
+ registrableDomain,
+ origin,
+ checkedAtMs,
+ grant.reservedStartAtMs,
+ grant.halfOpenProbe
+ )
+ return {
+ kind: 'stopped',
+ reason: 'connection_limit' as const,
+ blockingReason: 'connection_limit' as const,
+ waitMs: 0,
+ } as const
+ }
+ try {
+ this.budget.markRequestStarted(
+ registrableDomain,
+ origin,
+ Date.now(),
+ grant.reservedStartAtMs,
+ configurationRequest ? 'configuration' : 'image'
+ )
+ return {
+ kind: 'ran',
+ value: await ImageFetchProcessingMetrics.measure(`${requestKind}_http`, request),
+ } as const
+ } finally {
+ this.budget.releaseConnection(registrableDomain, origin)
+ }
+ },
+ })
+ .finally(() => capacityWait.finish())
+ if (scheduled.kind === 'ran') {
+ return { ran: true, value: scheduled.value }
+ }
+ if (scheduled.kind === 'stopped') {
+ return {
+ ran: false,
+ reason: scheduled.reason,
+ blockingReason: scheduled.blockingReason,
+ waitMs: scheduled.waitMs,
+ }
+ }
+ if (Date.now() + scheduled.waitMs > deadlineMs) {
+ return {
+ ran: false,
+ reason: 'deadline',
+ blockingReason: scheduled.waitScope,
+ waitMs: scheduled.waitMs,
+ }
+ }
+ this.recordSchedulerWait(
+ scheduled.waitScope,
+ scheduled.waitMs,
+ registrableDomain,
+ configurationRequest,
+ sourcePartitions
+ )
+ await ImageFetchProcessingMetrics.measure(`${requestKind}_${scheduled.waitScope}`, () =>
+ delay(scheduled.waitMs)
+ )
+ }
+ } finally {
+ this.budget.requestFinished(origin)
+ }
+ }
+
+ private recordSchedulerWait(
+ scope: SchedulerWaitScope,
+ waitMs: number,
+ registrableDomain: string,
+ configurationRequest: boolean,
+ sourcePartitions: readonly number[] | undefined
+ ): void {
+ ImageFetchRequestMetrics.observeSchedulerWait(scope, waitMs / 1000, sourcePartitions)
+ if (!configurationRequest) {
+ this.topHogMetrics?.recordSchedulerWait(registrableDomain, sourcePartitions, scope, waitMs)
+ }
+ }
+}
diff --git a/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/politeness-key.test.ts b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/politeness-key.test.ts
new file mode 100644
index 000000000000..3e6348fce64b
--- /dev/null
+++ b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/politeness-key.test.ts
@@ -0,0 +1,50 @@
+import { isPublicHost, politenessKey, tryCanonicalizeUrl } from '@posthog/ai-research-v4-native'
+
+/**
+ * The crate owns the rule and tests it. This file tests only that the fetch lane reaches that
+ * function across the FFI boundary, so every case here asserts behaviour the crate already had.
+ */
+describe('the v4 url policy addon', () => {
+ it('returns the canonical forms of a url and refuses a private host', () => {
+ // Values from the url_policy tests in replay-core-v4: the dedup form drops the volatile
+ // parameters, and the fetch form keeps every byte.
+ const raw = 'https://cdn.example.com/a.png?w=200&%63b=first&CB=second&no%63ache=third&RND=fourth'
+ expect(tryCanonicalizeUrl(raw)).toEqual({
+ ok: true,
+ url: {
+ fetch: raw,
+ dedup: 'https://cdn.example.com/a.png?w=200',
+ host: 'cdn.example.com',
+ domain: 'example.com',
+ },
+ })
+ expect(tryCanonicalizeUrl('https://169.254.169.254/latest/meta-data/iam/security-credentials/')).toEqual({
+ ok: false,
+ decline: 'non_public_host',
+ unwanted: false,
+ })
+ expect(isPublicHost('169.254.169.254')).toBe(false)
+ })
+
+ describe('politenessKey', () => {
+ it.each([['d111.cloudfront.net'], ['bucket.s3.amazonaws.com'], ['user.github.io'], ['myapp.vercel.app']])(
+ 'keeps %s as its own operator',
+ (host) => {
+ // Each provider sits in the private section of the public suffix list. A list read
+ // without that section folds every tenant of one provider into a single rate budget.
+ expect(politenessKey(host)).toBe(host)
+ }
+ )
+
+ it.each([
+ ['img1.cdn.example.com', 'example.com'],
+ ['a.b.example.co.uk', 'example.co.uk'],
+ ])('folds %s onto %s', (host, expected) => {
+ expect(politenessKey(host)).toBe(expected)
+ })
+
+ it.each([['203.0.113.4'], ['::1']])('returns the address %s unchanged', (host) => {
+ expect(politenessKey(host)).toBe(host)
+ })
+ })
+})
diff --git a/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/politeness-key.ts b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/politeness-key.ts
new file mode 100644
index 000000000000..c3b30169c8dc
--- /dev/null
+++ b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/politeness-key.ts
@@ -0,0 +1,59 @@
+/**
+ * The URL policy of the collector, reached through the v4 native addon rather than reimplemented
+ * here. One public suffix list keeps producer and consumer canonicalization aligned.
+ *
+ * The addon is a native library, and `index.ts` imports every server whatever mode the pod runs.
+ * This loads the addon on first use rather than at import, so only a pod in this lane pays for it.
+ * The fetch server defers it the same way.
+ */
+let cached: typeof import('@posthog/ai-research-v4-native') | undefined
+
+function addon(): typeof import('@posthog/ai-research-v4-native') {
+ if (!cached) {
+ const loaded = require('@posthog/ai-research-v4-native') as typeof import('@posthog/ai-research-v4-native')
+ if (
+ typeof loaded.politenessKey !== 'function' ||
+ typeof loaded.isPublicHost !== 'function' ||
+ typeof loaded.tryCanonicalizeUrl !== 'function'
+ ) {
+ throw new Error('the ai-research-v4-native addon is missing the url policy: rebuild index.node')
+ }
+ cached = loaded
+ }
+ return cached
+}
+
+/**
+ * Load the addon now and run one URL through it, so a stale `index.node` stops the pod at startup.
+ *
+ * The parser calls into it for every URL. A stale addon must stop the pod before it reads a batch.
+ */
+export function assertUrlPolicyLoaded(): void {
+ const verdict = addon().tryCanonicalizeUrl('https://example.com/probe.png')
+ if (!verdict.ok) {
+ throw new Error(`the ai-research-v4-native addon refused a plain URL (${verdict.decline}): rebuild index.node`)
+ }
+}
+
+export function politenessKey(host: string): string {
+ return addon().politenessKey(host)
+}
+
+/**
+ * False for a host the collector would decline, including private and reserved names.
+ *
+ * The connection layer refuses a private address, so this is not the only guard against one. It is
+ * the only guard against a name that looks internal and whose DNS answer is public, because no
+ * address check can refuse that.
+ */
+export function isPublicHost(host: string): boolean {
+ return addon().isPublicHost(host)
+}
+
+export function canonicalizeUrl(url: string): import('@posthog/ai-research-v4-native').CanonicalUrl | null {
+ return addon().canonicalizeUrl(url)
+}
+
+export function tryCanonicalizeUrl(url: string): import('@posthog/ai-research-v4-native').UrlPolicyVerdict {
+ return addon().tryCanonicalizeUrl(url)
+}
diff --git a/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/pool-window-batch-handler.test.ts b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/pool-window-batch-handler.test.ts
new file mode 100644
index 000000000000..500ddf7fef84
--- /dev/null
+++ b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/pool-window-batch-handler.test.ts
@@ -0,0 +1,77 @@
+import { Message } from 'node-rdkafka'
+
+import { MAX_HOPS } from './collected-urls-record'
+import { FetchCandidatePool, FetchCandidatePoolAdmission } from './fetch-candidate-pool'
+import { createPoolWindowBatchHandler } from './pool-window-batch-handler'
+
+const NOW_MS = 1_700_000_000_000
+const MESSAGE: Message = {
+ topic: 'session_replay_image_fetch',
+ partition: 3,
+ offset: 1,
+ value: Buffer.from('x'),
+ size: 1,
+}
+
+function distinctDomainCandidates(count: number): Parameters<FetchCandidatePool<string>['add']>[0] {
+ return Array.from({ length: count }, (_unused, index) => ({
+ originalRef: `imageurl:${index}`,
+ currentUrl: `https://cdn${index}.example${index}.com/a.png`,
+ host: `cdn${index}.example${index}.com`,
+ origin: `https://cdn${index}.example${index}.com`,
+ registrableDomain: `example${index}.com`,
+ remainingHops: MAX_HOPS,
+ notBeforeMs: 0,
+ firstSeenAtMs: NOW_MS,
+ fetchCount: 0,
+ republishCount: 0,
+ lastRepublishReason: null,
+ }))
+}
+
+describe('createPoolWindowBatchHandler', () => {
+ beforeEach(() => jest.useFakeTimers().setSystemTime(NOW_MS))
+ afterEach(() => jest.useRealTimers())
+
+ it.each([
+ ['returns once the batch enters the pool and the window has room', 0, 500],
+ ['returns at the wait cap while the window stays full', 20, 30_000],
+ ])('%s', async (_name, queuedAfterAdmission, expectedReturnMs) => {
+ const pool = new FetchCandidatePool<string>(
+ { maxConcurrentPerRegistrableDomain: 6, refillRunnableUrls: 10, maxQueuedUrlsPerOwner: 1_000 },
+ { originCrawlDelayMs: () => 0, originNextImageStartAtMs: () => 0 }
+ )
+ const owner = pool.createOwner()
+ let finishBatch: () => void = () => undefined
+ const processBatch = jest.fn(
+ (_messages: Message[], admission: FetchCandidatePoolAdmission) =>
+ new Promise<void>((resolve) => {
+ setTimeout(() => {
+ pool.add(
+ distinctDomainCandidates(queuedAfterAdmission),
+ 'batch',
+ NOW_MS + 40_000,
+ admission.owner
+ )
+ admission.admitted()
+ }, 500)
+ finishBatch = resolve
+ })
+ )
+ const handler = createPoolWindowBatchHandler(owner, processBatch)
+
+ let returnedAtMs: number | undefined
+ const returned = handler([MESSAGE]).then((result) => {
+ returnedAtMs = Date.now()
+ return result
+ })
+ await jest.advanceTimersByTimeAsync(30_000)
+ const result = await returned
+ finishBatch()
+
+ expect(processBatch).toHaveBeenCalledTimes(1)
+ expect(returnedAtMs! - NOW_MS).toBe(expectedReturnMs)
+ await expect(result?.backgroundTask).resolves.toBeUndefined()
+ pool.close()
+ })
+})
diff --git a/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/pool-window-batch-handler.ts b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/pool-window-batch-handler.ts
new file mode 100644
index 000000000000..c0e7b7994316
--- /dev/null
+++ b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/pool-window-batch-handler.ts
@@ -0,0 +1,38 @@
+import { Message } from 'node-rdkafka'
+
+import { FetchCandidatePoolAdmission, FetchCandidatePoolOwner } from './fetch-candidate-pool'
+import { ImageFetchPoolMetrics } from './metrics'
+
+/** consumer-v2 reports a stalled loop after 60 s without a tick, and this wait holds the loop. */
+export const POOL_ROOM_WAIT_CAP_MS = 30_000
+
+type BatchProcessor = (messages: Message[], admission: FetchCandidatePoolAdmission) => Promise<void>
+
+/**
+ * The batch handler of one consumer in continuous-pool mode.
+ *
+ * It starts the batch at once and returns only when the consumer's pool window has room, so the
+ * consumer reads its next batch while the pool still holds work for it.
+ */
+export function createPoolWindowBatchHandler(
+ owner: FetchCandidatePoolOwner,
+ processBatch: BatchProcessor,
+ roomWaitCapMs = POOL_ROOM_WAIT_CAP_MS
+): (messages: Message[]) => Promise<{ backgroundTask: Promise<void> } | void> {
+ return async (messages) => {
+ if (messages.length === 0) {
+ return
+ }
+ const admission = owner.beginAdmission()
+ const backgroundTask = processBatch(messages, admission).finally(() => admission.admitted())
+ // consumer-v2 attaches its own handler after this returns, so a failure during the wait must not count as unhandled.
+ void backgroundTask.catch(() => undefined)
+ const waitStartedAtMs = performance.now()
+ const hadRoom = await owner.waitForRoom(roomWaitCapMs)
+ ImageFetchPoolMetrics.observeRefillWait(
+ hadRoom ? 'room' : 'timeout',
+ (performance.now() - waitStartedAtMs) / 1000
+ )
+ return { backgroundTask }
+ }
+}
diff --git a/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/processing-metrics.test.ts b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/processing-metrics.test.ts
new file mode 100644
index 000000000000..5dcaf781d61a
--- /dev/null
+++ b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/processing-metrics.test.ts
@@ -0,0 +1,89 @@
+import { register } from 'prom-client'
+
+import { ConcurrencyController } from '~/common/utils/concurrencyController'
+
+import { ImageFetchProcessingMetrics } from './processing-metrics'
+
+async function sample(name: string, stage: string): Promise<number | undefined> {
+ const metric = await register.getSingleMetric(name)!.get()
+ return metric.values.find((value) => value.labels.stage === stage)?.value
+}
+
+describe('ImageFetchProcessingMetrics', () => {
+ beforeEach(() => register.resetMetrics())
+ afterEach(() => jest.restoreAllMocks())
+
+ it('keeps the oldest overlapping operation visible and removes finished stages', async () => {
+ const clock = jest.spyOn(performance, 'now').mockReturnValue(1000)
+ const first = ImageFetchProcessingMetrics.start('batch_fetch')
+ clock.mockReturnValue(2000)
+ const second = ImageFetchProcessingMetrics.start('batch_fetch')
+ clock.mockReturnValue(5000)
+ expect(await sample('ai_research_v4_image_fetch_stage_active', 'batch_fetch')).toBe(2)
+ expect(await sample('ai_research_v4_image_fetch_stage_oldest_seconds', 'batch_fetch')).toBe(4)
+ first.move('batch_history_write')
+ expect(await sample('ai_research_v4_image_fetch_stage_oldest_seconds', 'batch_fetch')).toBe(3)
+ second.finish()
+ second.finish()
+ first.finish()
+ expect(await sample('ai_research_v4_image_fetch_stage_active', 'batch_fetch')).toBe(0)
+ expect(await sample('ai_research_v4_image_fetch_stage_oldest_seconds', 'batch_fetch')).toBe(0)
+ const duration = await register.getSingleMetric('ai_research_v4_image_fetch_stage_duration_seconds')!.get()
+ expect(duration.values).toEqual(
+ expect.arrayContaining([
+ expect.objectContaining({
+ metricName: 'ai_research_v4_image_fetch_stage_duration_seconds_count',
+ labels: { stage: 'batch_fetch' },
+ value: 2,
+ }),
+ expect.objectContaining({
+ metricName: 'ai_research_v4_image_fetch_stage_duration_seconds_sum',
+ labels: { stage: 'batch_fetch' },
+ value: 7,
+ }),
+ ])
+ )
+ })
+
+ it('separates admission from running work and cleans up a rejected operation', async () => {
+ const controller = new ConcurrencyController(1)
+ let release!: () => void
+ const blocked = new Promise<void>((resolve) => {
+ release = resolve
+ })
+ const first = ImageFetchProcessingMetrics.runLimited(controller, 'candidate_admission', 'candidate_work', {
+ debugTag: 'example.com',
+ fn: () => blocked,
+ })
+ const error = new Error('synthetic failure')
+ const second = ImageFetchProcessingMetrics.runLimited(controller, 'candidate_admission', 'candidate_work', {
+ debugTag: 'example.org',
+ fn: () => Promise.reject(error),
+ })
+ const rejected = expect(second).rejects.toBe(error)
+ expect(await sample('ai_research_v4_image_fetch_stage_active', 'candidate_admission')).toBe(1)
+ expect(await sample('ai_research_v4_image_fetch_stage_active', 'candidate_work')).toBe(1)
+ release()
+ await first
+ await rejected
+ expect(await sample('ai_research_v4_image_fetch_stage_active', 'candidate_admission')).toBe(0)
+ expect(await sample('ai_research_v4_image_fetch_stage_active', 'candidate_work')).toBe(0)
+ await expect(ImageFetchProcessingMetrics.measure('candidate_policy', () => Promise.reject(error))).rejects.toBe(
+ error
+ )
+ expect(await sample('ai_research_v4_image_fetch_stage_active', 'candidate_policy')).toBe(0)
+ })
+
+ it('collects live queue depths across passes and stops retaining finished queues', async () => {
+ const queue = { candidateCount: 4 }
+ const stopFirst = ImageFetchProcessingMetrics.trackQueue(queue)
+ const stopSecond = ImageFetchProcessingMetrics.trackQueue({ candidateCount: 2 })
+ const metric = register.getSingleMetric('ai_research_v4_image_fetch_candidates_queued')!
+ expect((await metric.get()).values[0].value).toBe(6)
+ queue.candidateCount = 1
+ expect((await metric.get()).values[0].value).toBe(3)
+ stopFirst()
+ stopSecond()
+ expect((await metric.get()).values[0].value).toBe(0)
+ })
+})
diff --git a/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/processing-metrics.ts b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/processing-metrics.ts
new file mode 100644
index 000000000000..3c5492d171d1
--- /dev/null
+++ b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/processing-metrics.ts
@@ -0,0 +1,209 @@
+import { Counter, Gauge, Histogram } from 'prom-client'
+
+import { ConcurrencyController } from '~/common/utils/concurrencyController'
+
+import type { ConfigurationFetchResult } from './configuration-policy'
+import type { ConfigurationFile } from './crawl-history'
+
+export type ProcessingStage =
+ | 'batch_parse'
+ | 'batch_filter'
+ | 'batch_history_read'
+ | 'batch_fetch'
+ | 'batch_prepare_republish'
+ | 'batch_history_write'
+ | 'batch_republish_flush'
+ | 'batch_finalize'
+ | 'batch_dead_letter'
+ | 'consumer_join'
+ | 'consumer_process'
+ | 'candidate_admission'
+ | 'candidate_work'
+ | 'candidate_policy'
+ | 'candidate_fetch'
+ | 'image_publish_admission'
+ | 'image_publish_delivery'
+ | 'configuration_fetch'
+ | `configuration_${'request_capacity' | 'origin_crawl_delay' | 'registrable_domain_rate' | 'http'}`
+ | `image_${'request_capacity' | 'origin_crawl_delay' | 'registrable_domain_rate' | 'http'}`
+
+export type ConfigurationFetchReason =
+ | ConfigurationFetchResult['outcome']
+ | 'invalid_url'
+ | 'request_error'
+ | 'timeout'
+ | 'redirect_limit'
+ | 'cross_domain_redirect'
+ | 'invalid_redirect'
+ | 'missing_location'
+ | `http_${number}`
+ | 'http_5xx'
+ | 'unexpected_status'
+ | 'body_limit'
+ | 'invalid_utf8'
+ | 'invalid_document'
+ | 'not_json_array'
+
+export class ImageFetchProcessingMetrics {
+ private static readonly active = new Map<ProcessingStage, Map<symbol, number>>()
+ private static readonly queues = new Set<{ readonly candidateCount: number }>()
+
+ private static readonly activeGauge = new Gauge({
+ name: 'ai_research_v4_image_fetch_stage_active',
+ help: 'Active operations by stage; nested stages overlap and must not be summed together',
+ labelNames: ['stage'],
+ collect() {
+ for (const [stage, operations] of ImageFetchProcessingMetrics.active) {
+ this.set({ stage }, operations.size)
+ }
+ },
+ })
+
+ private static readonly oldestGauge = new Gauge({
+ name: 'ai_research_v4_image_fetch_stage_oldest_seconds',
+ help: 'Age of the oldest active operation in each stage, or zero when idle',
+ labelNames: ['stage'],
+ collect() {
+ const now = performance.now()
+ for (const [stage, operations] of ImageFetchProcessingMetrics.active) {
+ const first = operations.values().next().value
+ this.set({ stage }, first === undefined ? 0 : Math.max(0, now - first) / 1000)
+ }
+ },
+ })
+
+ private static readonly queuedCandidates = new Gauge({
+ name: 'ai_research_v4_image_fetch_candidates_queued',
+ help: 'Candidates waiting for selection from all active passes, before candidate admission',
+ collect() {
+ let count = 0
+ for (const queue of ImageFetchProcessingMetrics.queues) {
+ count += queue.candidateCount
+ }
+ this.set(count)
+ },
+ })
+
+ private static readonly duration = new Histogram({
+ name: 'ai_research_v4_image_fetch_stage_duration_seconds',
+ help: 'Elapsed time of finished stage visits, including failed operations; nested stages overlap',
+ labelNames: ['stage'],
+ buckets: [0.001, 0.01, 0.1, 0.5, 1, 5, 10, 30, 60, 120, 240, 600],
+ })
+
+ public static readonly joinedBatches = new Histogram({
+ name: 'ai_research_v4_image_fetch_joined_batches',
+ help: 'Consumer batches in each dispatched fetch group',
+ buckets: [1, 2, 4, 8, 12, 16],
+ })
+
+ private static readonly configurationLookups = new Counter({
+ name: 'ai_research_v4_image_fetch_configuration_lookups_total',
+ help: 'Configuration lookups by source and observed status; shared requests count once per caller',
+ labelNames: ['file', 'source', 'outcome'],
+ })
+
+ private static readonly configurationFetches = new Counter({
+ name: 'ai_research_v4_image_fetch_configuration_fetches_total',
+ help: 'Completed configuration fetches, once per redirect chain, by final outcome and reason',
+ labelNames: ['file', 'outcome', 'reason'],
+ })
+
+ public static start(stage: ProcessingStage): ProcessingStageTimer {
+ return new ProcessingStageTimer(stage)
+ }
+
+ public static enter(stage: ProcessingStage): symbol {
+ let operations = this.active.get(stage)
+ if (!operations) {
+ operations = new Map()
+ this.active.set(stage, operations)
+ }
+ const id = Symbol()
+ operations.set(id, performance.now())
+ return id
+ }
+
+ public static leave(stage: ProcessingStage, id: symbol): void {
+ const operations = this.active.get(stage)
+ const started = operations?.get(id)
+ if (started !== undefined) {
+ operations!.delete(id)
+ this.duration.observe({ stage }, Math.max(0, performance.now() - started) / 1000)
+ }
+ }
+
+ public static async measure<T>(stage: ProcessingStage, operation: () => Promise<T>): Promise<T> {
+ const timer = this.start(stage)
+ try {
+ return await operation()
+ } finally {
+ timer.finish()
+ }
+ }
+
+ public static async runLimited<T>(
+ controller: ConcurrencyController,
+ waitingStage: ProcessingStage,
+ runningStage: ProcessingStage,
+ options: { debugTag: string; fn: () => Promise<T> }
+ ): Promise<T> {
+ const timer = this.start(waitingStage)
+ try {
+ return await controller.run({
+ ...options,
+ fn: () => {
+ timer.move(runningStage)
+ return options.fn()
+ },
+ })
+ } finally {
+ timer.finish()
+ }
+ }
+
+ public static trackQueue(queue: { readonly candidateCount: number }): () => void {
+ this.queues.add(queue)
+ return () => {
+ this.queues.delete(queue)
+ }
+ }
+
+ public static observeConfigurationLookup(
+ file: ConfigurationFile,
+ source: 'cache' | 'shared' | 'network',
+ outcome: ConfigurationFetchResult['outcome'] | 'error'
+ ): void {
+ this.configurationLookups.inc({ file, source, outcome })
+ }
+
+ public static observeConfigurationFetch(
+ file: ConfigurationFile,
+ result: ConfigurationFetchResult,
+ reason: ConfigurationFetchReason
+ ): ConfigurationFetchResult {
+ this.configurationFetches.inc({ file, outcome: result.outcome, reason })
+ return result
+ }
+}
+
+export class ProcessingStageTimer {
+ private id: symbol | undefined
+
+ constructor(private stage: ProcessingStage) {
+ this.id = ImageFetchProcessingMetrics.enter(stage)
+ }
+
+ public move(stage: ProcessingStage): void {
+ this.finish()
+ this.stage = stage
+ this.id = ImageFetchProcessingMetrics.enter(stage)
+ }
+
+ public finish(): void {
+ if (this.id !== undefined) {
+ ImageFetchProcessingMetrics.leave(this.stage, this.id)
+ this.id = undefined
+ }
+ }
+}
diff --git a/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/producer.ts b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/producer.ts
new file mode 100644
index 000000000000..4216c093f8bc
--- /dev/null
+++ b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/producer.ts
@@ -0,0 +1,55 @@
+import { TOPHOG_OUTPUT, TophogOutput } from '~/common/outputs'
+import { IngestionOutputsBuilder } from '~/common/outputs/ingestion-outputs-builder'
+import type { AllowedConfigKey } from '~/common/outputs/kafka-producer-config'
+import { KafkaProducerRegistryBuilder } from '~/common/outputs/kafka-producer-registry-builder'
+import {
+ INGESTION_DOWNSTREAM_PRODUCER,
+ INGESTION_DOWNSTREAM_PRODUCER_CONFIG_MAP,
+ IngestionDownstreamProducer,
+} from '~/ingestion/common/outputs/producers'
+
+/**
+ * The v4 fetch lane's own producer on the replay cluster. It sends frontier records to the frontier and the delay tiers
+ * in a queue of its own. Broker and security config are those of the replay producer.
+ */
+export const AI_RESEARCH_V4_IMAGE_FETCH_PRODUCER = 'AI_RESEARCH_V4_IMAGE_FETCH' as const
+
+export const AI_RESEARCH_V4_IMAGE_FETCH_PRODUCER_CONFIG_MAP = {
+ 'client.id': 'KAFKA_INGESTION_SESSIONREPLAY_PRODUCER_CLIENT_ID',
+ 'metadata.broker.list': 'KAFKA_INGESTION_SESSIONREPLAY_PRODUCER_METADATA_BROKER_LIST',
+ 'security.protocol': 'KAFKA_INGESTION_SESSIONREPLAY_PRODUCER_SECURITY_PROTOCOL',
+ 'compression.codec': 'KAFKA_AI_RESEARCH_V4_IMAGE_FETCH_PRODUCER_COMPRESSION_CODEC',
+ 'linger.ms': 'KAFKA_AI_RESEARCH_V4_IMAGE_FETCH_PRODUCER_LINGER_MS',
+ 'queue.buffering.max.messages': 'KAFKA_AI_RESEARCH_V4_IMAGE_FETCH_PRODUCER_QUEUE_BUFFERING_MAX_MESSAGES',
+ 'queue.buffering.max.kbytes': 'KAFKA_AI_RESEARCH_V4_IMAGE_FETCH_PRODUCER_QUEUE_BUFFERING_MAX_KBYTES',
+ 'message.max.bytes': 'KAFKA_AI_RESEARCH_V4_IMAGE_FETCH_PRODUCER_MESSAGE_MAX_BYTES',
+ 'message.timeout.ms': 'KAFKA_AI_RESEARCH_V4_IMAGE_FETCH_PRODUCER_MESSAGE_TIMEOUT_MS',
+} as const satisfies Partial<Record<AllowedConfigKey, string>>
+
+/** DOWNSTREAM carries the TopHog rows to ClickHouse. */
+export type AiResearchV4ImageFetchProducerName =
+ | IngestionDownstreamProducer
+ | typeof AI_RESEARCH_V4_IMAGE_FETCH_PRODUCER
+
+type ProducerConfigKey =
+ | (typeof INGESTION_DOWNSTREAM_PRODUCER_CONFIG_MAP)[keyof typeof INGESTION_DOWNSTREAM_PRODUCER_CONFIG_MAP]
+ | (typeof AI_RESEARCH_V4_IMAGE_FETCH_PRODUCER_CONFIG_MAP)[keyof typeof AI_RESEARCH_V4_IMAGE_FETCH_PRODUCER_CONFIG_MAP]
+
+export function createV4ImageFetchProducerRegistry(
+ kafkaClientRack: string | undefined
+): KafkaProducerRegistryBuilder<AiResearchV4ImageFetchProducerName, ProducerConfigKey> {
+ return new KafkaProducerRegistryBuilder(kafkaClientRack)
+ .register(INGESTION_DOWNSTREAM_PRODUCER, INGESTION_DOWNSTREAM_PRODUCER_CONFIG_MAP)
+ .register(AI_RESEARCH_V4_IMAGE_FETCH_PRODUCER, AI_RESEARCH_V4_IMAGE_FETCH_PRODUCER_CONFIG_MAP)
+}
+
+export function createV4ImageFetchOutputs(): IngestionOutputsBuilder<
+ TophogOutput,
+ 'AI_RESEARCH_V4_IMAGE_FETCH_TOPHOG_TOPIC',
+ 'AI_RESEARCH_V4_IMAGE_FETCH_TOPHOG_PRODUCER'
+> {
+ return new IngestionOutputsBuilder().register(TOPHOG_OUTPUT, {
+ topicKey: 'AI_RESEARCH_V4_IMAGE_FETCH_TOPHOG_TOPIC',
+ producerKey: 'AI_RESEARCH_V4_IMAGE_FETCH_TOPHOG_PRODUCER',
+ })
+}
diff --git a/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/raw-image-store.test.ts b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/raw-image-store.test.ts
index df7eaff8910c..5a576ce88eac 100644
--- a/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/raw-image-store.test.ts
+++ b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/raw-image-store.test.ts
@@ -3,9 +3,9 @@ import { createDecipheriv } from 'node:crypto'
import { parseJSON } from '~/common/utils/json-parse'
import type { TeamMonthKey } from '~/ingestion/pipelines/sessionreplay/ai-research/v4-keys/team-month-keys'
-import type { FetchCandidate } from '~/ingestion/pipelines/sessionreplay/ml-mirror-image-fetch/collected-urls-record'
-import type { ImageFetchResult } from '~/ingestion/pipelines/sessionreplay/ml-mirror-image-fetch/image-fetcher'
+import type { FetchCandidate } from './collected-urls-record'
+import type { ImageFetchResult } from './image-fetcher'
import { RawImageStore } from './raw-image-store'
const PNG = Buffer.concat([Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]), Buffer.from('png body')])
diff --git a/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/raw-image-store.ts b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/raw-image-store.ts
index d5d3cd670f89..2b54b528cdf5 100644
--- a/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/raw-image-store.ts
+++ b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/raw-image-store.ts
@@ -4,12 +4,12 @@ import { Counter } from 'prom-client'
import { ConcurrencyController } from '~/common/utils/concurrencyController'
import { sealV4Frame } from '~/ingestion/pipelines/sessionreplay/ai-research/v4-keys/frame'
import type { TeamMonthKeys } from '~/ingestion/pipelines/sessionreplay/ai-research/v4-keys/team-month-keys'
-import type { FetchCandidate } from '~/ingestion/pipelines/sessionreplay/ml-mirror-image-fetch/collected-urls-record'
-import type { ImageFetchResult } from '~/ingestion/pipelines/sessionreplay/ml-mirror-image-fetch/image-fetcher'
-import { ImageFetchProcessingMetrics } from '~/ingestion/pipelines/sessionreplay/ml-mirror-image-fetch/processing-metrics'
+import type { FetchCandidate } from './collected-urls-record'
+import type { ImageFetchResult } from './image-fetcher'
import type { RefreshingImageSink } from './image-sink'
import { InvalidImageTransportError, prepareFetchedImage } from './image-transport'
+import { ImageFetchProcessingMetrics } from './processing-metrics'
import { parseV4UrlRef } from './v4-url-ref'
const S3_WRITE_TIMEOUT_MS = 30_000
diff --git a/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/request-identity.ts b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/request-identity.ts
new file mode 100644
index 000000000000..d93e1aaaa1d1
--- /dev/null
+++ b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/request-identity.ts
@@ -0,0 +1,7 @@
+export const BOT_NAME = 'PostHogImageFetcherBot'
+
+export const REQUEST_IDENTITY_HEADERS: Readonly<Record<string, string>> = {
+ 'user-agent': `${BOT_NAME}/1.0 (+https://posthog.com/docs/ai-research/image-fetcher-bot)`,
+ // Replay customers already allow requests from this host so that the replay viewer works. Change it before this lane runs outside prod-us.
+ referer: 'https://us.posthog.com/',
+}
diff --git a/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/response-url-binding.ts b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/response-url-binding.ts
new file mode 100644
index 000000000000..684805f851f6
--- /dev/null
+++ b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/response-url-binding.ts
@@ -0,0 +1,48 @@
+import { createHmac, hkdfSync } from 'node:crypto'
+
+import type {
+ TeamMonthKey,
+ TeamMonthKeys,
+} from '~/ingestion/pipelines/sessionreplay/ai-research/v4-keys/team-month-keys'
+
+import { parseV4UrlRef } from './v4-url-ref'
+
+/** The HKDF info of `replay_core_v4::seal::image_url_key`, the key behind the hash of an `imageurl:v4` reference. */
+const IMAGE_URL_KEY_INFO = 'ml-image-url-key'
+/** The length `replay_core_v4::refs::reference_hash` truncates a hash to. */
+const HASH_CHARS = 22
+
+/** The keyed hash that the crawl history stores in place of a response URL. */
+export type ResponseUrlBinding = (url: string) => string
+
+export interface ResponseUrlBinder {
+ bindingFor(originalRef: string, signal: AbortSignal): Promise<ResponseUrlBinding | undefined>
+}
+
+/**
+ * Binds cached validators to the URL that issued them without storing that URL, which can hold personal data. The
+ * binding is the HMAC of the URL under the image URL key of the reference's team month, so it compares only within
+ * that team month. A reference with no live key has no binding, and its fetch sends no validators.
+ */
+export class TeamMonthResponseUrlBinder implements ResponseUrlBinder {
+ constructor(private readonly keys: Pick<TeamMonthKeys, 'get'>) {}
+
+ public async bindingFor(originalRef: string, signal: AbortSignal): Promise<ResponseUrlBinding | undefined> {
+ const ref = parseV4UrlRef(originalRef)
+ if (!ref) {
+ return undefined
+ }
+ let key: TeamMonthKey
+ try {
+ key = await this.keys.get(ref.teamId, ref.keyMonth, signal)
+ } catch {
+ // Admission read this key moments ago, so a failure here costs one revalidation and not the fetch.
+ return undefined
+ }
+ if (key.status !== 'ok') {
+ return undefined
+ }
+ const urlKey = Buffer.from(hkdfSync('sha256', key.key, Buffer.alloc(0), IMAGE_URL_KEY_INFO, 32))
+ return (url) => createHmac('sha256', urlKey).update(url).digest('base64url').slice(0, HASH_CHARS)
+ }
+}
diff --git a/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/retry-delay-consumer.test.ts b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/retry-delay-consumer.test.ts
new file mode 100644
index 000000000000..335a7d2f235c
--- /dev/null
+++ b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/retry-delay-consumer.test.ts
@@ -0,0 +1,215 @@
+import { Message } from 'node-rdkafka'
+
+import { KafkaProducerWrapper } from '~/common/kafka/producer'
+
+import { RetryDelayMetrics } from './metrics'
+import { RetryDelayConsumer } from './retry-delay-consumer'
+
+const NOW_MS = 1_700_000_000_000
+const DELAY_MS = 60_000
+const FRONTIER = 'session_replay_image_fetch'
+
+function message(timestamp: number | undefined, overrides: Partial<Message> = {}): Message {
+ return {
+ value: Buffer.from('{"v":1}'),
+ key: Buffer.from('example.com'),
+ size: 7,
+ topic: 'ai_research_session_replay_image_fetch_retry_1m',
+ partition: 0,
+ offset: 0,
+ timestamp,
+ ...overrides,
+ }
+}
+
+interface Harness {
+ consumer: RetryDelayConsumer
+ produce: jest.Mock<Promise<void>, any[]>
+ storeOffsets: jest.Mock<void, [Message[]]>
+ heartbeat: jest.Mock<void, []>
+}
+
+function build(overrides: { isStopping?: () => boolean; delayMs?: number; releaseGroupSize?: number } = {}): Harness {
+ const produce = jest.fn(() => Promise.resolve())
+ const storeOffsets = jest.fn()
+ const heartbeat = jest.fn()
+ const consumer = new RetryDelayConsumer({ produce } as unknown as KafkaProducerWrapper, {
+ frontierTopic: FRONTIER,
+ delayMs: overrides.delayMs ?? DELAY_MS,
+ heartbeat,
+ heartbeatIntervalMs: 10_000,
+ isStopping: overrides.isStopping,
+ storeOffsets,
+ releaseGroupSize: overrides.releaseGroupSize,
+ })
+ return { consumer, produce, storeOffsets, heartbeat }
+}
+
+describe('RetryDelayConsumer', () => {
+ beforeEach(() => jest.useFakeTimers().setSystemTime(NOW_MS))
+ afterEach(() => {
+ jest.restoreAllMocks()
+ jest.useRealTimers()
+ })
+
+ it.each([
+ [{ delayMs: 0 }, 'positive delay'],
+ [{ delayMs: -1 }, 'positive delay'],
+ [{ delayMs: Number.NaN }, 'positive delay'],
+ [{ releaseGroupSize: 0 }, 'positive whole release group size'],
+ [{ releaseGroupSize: 1.5 }, 'positive whole release group size'],
+ ])('refuses invalid options %p', (overrides, error) => {
+ expect(() => build(overrides)).toThrow(error)
+ })
+
+ it('does nothing for an empty batch', async () => {
+ const harness = build()
+
+ await harness.consumer.handleBatch([])
+
+ expect(harness.produce).not.toHaveBeenCalled()
+ expect(harness.storeOffsets).not.toHaveBeenCalled()
+ })
+
+ it('publishes a ripe batch and stores its offsets once', async () => {
+ const harness = build()
+ const messages = [message(NOW_MS - DELAY_MS, { offset: 1 }), message(NOW_MS - DELAY_MS, { offset: 2 })]
+
+ await harness.consumer.handleBatch(messages)
+
+ expect(harness.produce).toHaveBeenCalledTimes(2)
+ expect(harness.produce.mock.calls[0][0]).toMatchObject({
+ topic: FRONTIER,
+ key: messages[0].key,
+ value: messages[0].value,
+ })
+ expect(harness.storeOffsets).toHaveBeenCalledTimes(1)
+ expect(harness.storeOffsets).toHaveBeenCalledWith(messages)
+ })
+
+ it('waits once for the newest append timestamp in the batch', async () => {
+ const harness = build()
+ const older = message(NOW_MS - 30_000, { offset: 1 })
+ const newer = message(NOW_MS - 10_000, { offset: 2 })
+
+ const handled = harness.consumer.handleBatch([older, newer])
+ await jest.advanceTimersByTimeAsync(30_000)
+ expect(harness.produce).not.toHaveBeenCalled()
+ await jest.advanceTimersByTimeAsync(20_000)
+ await handled
+
+ expect(harness.produce).toHaveBeenCalledTimes(2)
+ expect(harness.storeOffsets).toHaveBeenCalledTimes(1)
+ })
+
+ it('reports health while a batch is deliberately waiting', async () => {
+ const harness = build()
+
+ const handled = harness.consumer.handleBatch([message(NOW_MS)])
+ await jest.advanceTimersByTimeAsync(DELAY_MS)
+ await handled
+
+ expect(harness.heartbeat).toHaveBeenCalledTimes(8)
+ })
+
+ it.each([undefined, Number.NaN, 0, -1])(
+ 'throws for an invalid broker append timestamp of %p',
+ async (timestamp) => {
+ const harness = build()
+ const incReleased = jest.spyOn(RetryDelayMetrics, 'incReleased')
+
+ await expect(harness.consumer.handleBatch([message(timestamp)])).rejects.toThrow(
+ 'requires broker append timestamps'
+ )
+ expect(incReleased).toHaveBeenCalledWith('invalid_timestamp', 1)
+ expect(harness.produce).not.toHaveBeenCalled()
+ expect(harness.storeOffsets).not.toHaveBeenCalled()
+ }
+ )
+
+ it('throws and stores no offset when any frontier publish fails', async () => {
+ const harness = build()
+ harness.produce.mockRejectedValueOnce(new Error('broker unavailable'))
+
+ await expect(harness.consumer.handleBatch([message(NOW_MS - DELAY_MS)])).rejects.toThrow(
+ 'could not publish a record back to the frontier'
+ )
+ expect(harness.storeOffsets).not.toHaveBeenCalled()
+ })
+
+ it('publishes each release group concurrently, and reports health and starts the next group after it settles', async () => {
+ const harness = build({ releaseGroupSize: 2 })
+ const acknowledgements: (() => void)[] = []
+ harness.produce.mockImplementation(() => new Promise<void>((resolve) => acknowledgements.push(resolve)))
+ const messages = [
+ message(NOW_MS - DELAY_MS, { offset: 1 }),
+ message(NOW_MS - DELAY_MS, { offset: 2 }),
+ message(NOW_MS - DELAY_MS, { offset: 3 }),
+ ]
+
+ const handled = harness.consumer.handleBatch(messages)
+ await jest.advanceTimersByTimeAsync(0)
+ expect(harness.produce).toHaveBeenCalledTimes(2)
+ expect(harness.heartbeat).not.toHaveBeenCalled()
+
+ acknowledgements.splice(0).forEach((acknowledge) => acknowledge())
+ await jest.advanceTimersByTimeAsync(0)
+ expect(harness.produce).toHaveBeenCalledTimes(3)
+ expect(harness.heartbeat).toHaveBeenCalledTimes(1)
+ expect(harness.storeOffsets).not.toHaveBeenCalled()
+
+ acknowledgements.splice(0).forEach((acknowledge) => acknowledge())
+ await handled
+ expect(harness.storeOffsets).toHaveBeenCalledWith(messages)
+ })
+
+ it('stops publishing the batch after the release group with the first frontier failure', async () => {
+ const harness = build({ releaseGroupSize: 2 })
+ harness.produce.mockResolvedValueOnce().mockRejectedValueOnce(new Error('broker unavailable'))
+ const messages = [
+ message(NOW_MS - DELAY_MS, { offset: 1 }),
+ message(NOW_MS - DELAY_MS, { offset: 2 }),
+ message(NOW_MS - DELAY_MS, { offset: 3 }),
+ ]
+
+ await expect(harness.consumer.handleBatch(messages)).rejects.toThrow(
+ 'could not publish a record back to the frontier'
+ )
+
+ expect(harness.produce).toHaveBeenCalledTimes(2)
+ expect(harness.storeOffsets).not.toHaveBeenCalled()
+ })
+
+ it('stores the batch offset after it skips a malformed record', async () => {
+ const harness = build()
+ const malformed = message(NOW_MS - DELAY_MS, { value: null })
+
+ await harness.consumer.handleBatch([malformed])
+
+ expect(harness.produce).not.toHaveBeenCalled()
+ expect(harness.storeOffsets).toHaveBeenCalledWith([malformed])
+ })
+
+ it('abandons the batch without storing offsets when shutdown has started', async () => {
+ const harness = build({ isStopping: () => true })
+
+ await harness.consumer.handleBatch([message(NOW_MS)])
+
+ expect(harness.produce).not.toHaveBeenCalled()
+ expect(harness.storeOffsets).not.toHaveBeenCalled()
+ })
+
+ it('abandons an active wait when shutdown starts', async () => {
+ let stopping = false
+ const harness = build({ isStopping: () => stopping })
+ const handled = harness.consumer.handleBatch([message(NOW_MS)])
+
+ await jest.advanceTimersByTimeAsync(10_000)
+ stopping = true
+ await jest.advanceTimersByTimeAsync(10_000)
+ await handled
+
+ expect(harness.produce).not.toHaveBeenCalled()
+ expect(harness.storeOffsets).not.toHaveBeenCalled()
+ })
+})
diff --git a/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/retry-delay-consumer.ts b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/retry-delay-consumer.ts
new file mode 100644
index 000000000000..e7b73564c330
--- /dev/null
+++ b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/retry-delay-consumer.ts
@@ -0,0 +1,153 @@
+import { Message } from 'node-rdkafka'
+
+import { parseKafkaHeaders } from '~/common/kafka/consumer/consumer-v1'
+import { KafkaProducer } from '~/common/kafka/producer'
+import { logger } from '~/common/utils/logger'
+import { delay } from '~/common/utils/utils'
+
+import { RetryDelayMetrics } from './metrics'
+
+export interface RetryDelayConsumerOptions {
+ frontierTopic: string
+ /** The period every record in this topic waits. The period belongs to the topic, not to the record. */
+ delayMs: number
+ /** Called while the consumer sleeps or releases, so the health check does not restart a pod that is working as designed. */
+ heartbeat: () => void
+ /** The default sits well inside CONSUMER_MAX_HEARTBEAT_INTERVAL_MS (30s), which binds before the loop stall threshold. */
+ heartbeatIntervalMs?: number
+ /** True once the pod is shutting down. A wait of an hour would otherwise hold the rolling deploy until Kubernetes killed it. */
+ isStopping?: () => boolean
+ storeOffsets: (messages: Message[]) => void
+ releaseGroupSize?: number
+}
+
+const DEFAULT_HEARTBEAT_INTERVAL_MS = 10_000
+/**
+ * Each release waits for one broker acknowledgement, so releasing one record at a time limits a tier to one record per
+ * produce round trip. A failed group throws and the consumer reads the whole batch again, so the group size also limits
+ * how many records one failure publishes twice.
+ */
+const DEFAULT_RELEASE_GROUP_SIZE = 50
+
+/**
+ * The consumer of one delay topic.
+ *
+ * Kafka has no delayed delivery, so a retry waits in a topic whose period is fixed and a consumer
+ * holds it there. Every record in one topic waits the same period, so the records leave in the
+ * order they become ready.
+ *
+ * The record already carries its hop budget and its earliest fetch time, so nothing here parses it.
+ *
+ * Two properties of this lane live outside this file, and both are in the README. Its
+ * `max.poll.interval.ms` must exceed the period of its topic, or the broker evicts it mid-sleep. It
+ * runs one pod and no more, because adding consumers cannot make a record become ready sooner.
+ */
+export class RetryDelayConsumer {
+ constructor(
+ private readonly producer: KafkaProducer,
+ private readonly options: RetryDelayConsumerOptions
+ ) {
+ if (!Number.isFinite(options.delayMs) || options.delayMs <= 0) {
+ throw new Error(`the retry delay consumer needs a positive delay, got ${options.delayMs}`)
+ }
+ const releaseGroupSize = options.releaseGroupSize ?? DEFAULT_RELEASE_GROUP_SIZE
+ if (!Number.isInteger(releaseGroupSize) || releaseGroupSize <= 0) {
+ throw new Error(
+ `the retry delay consumer needs a positive whole release group size, got ${releaseGroupSize}`
+ )
+ }
+ }
+
+ public async handleBatch(messages: Message[]): Promise<void> {
+ if (messages.length === 0) {
+ return
+ }
+ if (this.stopping()) {
+ RetryDelayMetrics.incReleased('abandoned', messages.length)
+ return
+ }
+ const invalidTimestampCount = messages.filter(
+ (message) =>
+ message.timestamp === undefined || !Number.isFinite(message.timestamp) || message.timestamp <= 0
+ ).length
+ if (invalidTimestampCount > 0) {
+ RetryDelayMetrics.incReleased('invalid_timestamp', invalidTimestampCount)
+ throw new Error('the image fetch retry lane requires broker append timestamps')
+ }
+ const waitMs = this.remainingBatchWaitMs(messages)
+ if (waitMs > 0) {
+ RetryDelayMetrics.observeWait(waitMs / 1000)
+ if (!(await this.sleepWithHeartbeat(waitMs))) {
+ RetryDelayMetrics.incReleased('abandoned', messages.length)
+ return
+ }
+ }
+ const releaseGroupSize = this.options.releaseGroupSize ?? DEFAULT_RELEASE_GROUP_SIZE
+ for (let start = 0; start < messages.length; start += releaseGroupSize) {
+ const group = messages.slice(start, start + releaseGroupSize)
+ await Promise.all(group.map((message) => this.release(message)))
+ // A slow broker can make the releases of one batch outlast the health check, as a long wait would.
+ this.options.heartbeat()
+ }
+ this.options.storeOffsets(messages)
+ }
+
+ private stopping(): boolean {
+ return this.options.isStopping?.() ?? false
+ }
+
+ /**
+ * Measured from when the record was written rather than from when it was read, so a consumer
+ * that restarts does not begin the wait again. librdkafka reports an absent timestamp as -1,
+ * which is not nullish, so a plain `??` would make the whole period look already spent.
+ */
+ private remainingBatchWaitMs(messages: Message[]): number {
+ const latestReadyAtMs = Math.max(...messages.map((message) => message.timestamp! + this.options.delayMs))
+ return Math.max(0, latestReadyAtMs - Date.now())
+ }
+
+ /** False when the pod started shutting down during the wait, so the caller abandons the record. */
+ private async sleepWithHeartbeat(waitMs: number): Promise<boolean> {
+ const interval = this.options.heartbeatIntervalMs ?? DEFAULT_HEARTBEAT_INTERVAL_MS
+ const deadline = Date.now() + waitMs
+ while (Date.now() < deadline) {
+ if (this.stopping()) {
+ return false
+ }
+ this.options.heartbeat()
+ await delay(Math.min(interval, deadline - Date.now()))
+ }
+ this.options.heartbeat()
+ return !this.stopping()
+ }
+
+ /**
+ * True when this consumer is finished with the record, whether it went out or can never go out.
+ *
+ * A failed produce returns false, because reading the record again is the only way it comes
+ * back. Nothing else holds it.
+ */
+ private async release(message: Message): Promise<void> {
+ if (!message.value || !message.key) {
+ RetryDelayMetrics.incReleased('malformed')
+ return
+ }
+ try {
+ await this.producer.produce({
+ topic: this.options.frontierTopic,
+ key: message.key,
+ value: message.value,
+ headers: parseKafkaHeaders(message.headers),
+ })
+ } catch (error) {
+ logger.warn('🌐', 'ai_research_v4_image_fetch_retry_release_failed', {
+ error: error instanceof Error ? error.name : 'unknown',
+ })
+ RetryDelayMetrics.incReleased('failed')
+ throw new Error('the image fetch retry lane could not publish a record back to the frontier', {
+ cause: error,
+ })
+ }
+ RetryDelayMetrics.incReleased('released')
+ }
+}
diff --git a/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/shared-image-fetcher.test.ts b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/shared-image-fetcher.test.ts
new file mode 100644
index 000000000000..37baab86bd02
--- /dev/null
+++ b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/shared-image-fetcher.test.ts
@@ -0,0 +1,51 @@
+import type { ImageFetchOptions, ImageFetchResult } from './image-fetcher'
+import { SharedImageFetcher } from './shared-image-fetcher'
+
+const IMAGE_URL = 'https://example.com/a.png'
+const OK: ImageFetchResult = {
+ outcome: 'ok',
+ redirects: 0,
+ currentUrl: IMAGE_URL,
+ bytes: Buffer.from('png'),
+ contentType: 'image/png',
+}
+const options = (tdmrepReservation: boolean) => ({ tdmrepReservation }) as ImageFetchOptions
+
+describe('SharedImageFetcher', () => {
+ it.each([
+ ['one after the other', false],
+ ['at the same time', true],
+ ])('fetches again for a job whose TDM reservation differs, %s', async (_name, concurrent) => {
+ const inner = { fetch: jest.fn((_url: string, _options: ImageFetchOptions) => Promise.resolve(OK)) }
+ const fetcher = new SharedImageFetcher(inner)
+
+ if (concurrent) {
+ await Promise.all([fetcher.fetch(IMAGE_URL, options(false)), fetcher.fetch(IMAGE_URL, options(true))])
+ } else {
+ await fetcher.fetch(IMAGE_URL, options(false))
+ await fetcher.fetch(IMAGE_URL, options(true))
+ }
+ await fetcher.fetch(IMAGE_URL, options(true))
+
+ expect(inner.fetch.mock.calls.map(([, used]) => used.tdmrepReservation)).toEqual([false, true])
+ })
+
+ it.each([
+ ['more entries than its cap', Buffer.from('png'), 64 * 1024 * 1024, 2],
+ ['empty images whose URLs and headers outgrow its size', Buffer.alloc(0), 3_000, 100],
+ ])('evicts the oldest image when it holds %s', async (_name, bytes, maxBytes, maxEntries) => {
+ const inner = {
+ fetch: jest.fn((url: string, _options: ImageFetchOptions) =>
+ Promise.resolve({ ...OK, currentUrl: url, bytes })
+ ),
+ }
+ const fetcher = new SharedImageFetcher(inner, maxBytes, 60_000, maxEntries)
+ const urls = ['a', 'b', 'c', 'a'].map((name) => `https://example.com/${name}.png`)
+
+ for (const url of urls) {
+ await fetcher.fetch(url, options(false))
+ }
+
+ expect(inner.fetch.mock.calls.map(([url]) => url)).toEqual(urls)
+ })
+})
diff --git a/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/shared-image-fetcher.ts b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/shared-image-fetcher.ts
new file mode 100644
index 000000000000..b1a64ad52458
--- /dev/null
+++ b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/shared-image-fetcher.ts
@@ -0,0 +1,76 @@
+import { LRUCache } from 'lru-cache'
+import { Counter } from 'prom-client'
+
+import type { ImageFetchOptions, ImageFetchResult, ImageFetcher } from './image-fetcher'
+
+const sharedFetches = new Counter({
+ name: 'ai_research_v4_image_fetch_shared_fetches_total',
+ help: 'v4 image fetches served by a recent fetch of the same URL in this process, with no request.',
+})
+
+const ENTRY_OVERHEAD_BYTES = 1024
+
+/**
+ * Serves one fetch of a URL to every job that asks for it within a short time. The v4 recorder sends one job per team
+ * for the same URL and month, so without this each team costs the host a request. Only the plain bytes are shared, in
+ * this process: the sink still seals each team's copy under that team's key.
+ */
+export class SharedImageFetcher implements ImageFetcher {
+ private readonly recent: LRUCache<string, ImageFetchResult>
+ private readonly inFlight = new Map<string, Promise<ImageFetchResult>>()
+
+ constructor(
+ private readonly inner: ImageFetcher,
+ maxBytes = 64 * 1024 * 1024,
+ lifetimeMs = 60_000,
+ maxEntries = 4096
+ ) {
+ this.recent = new LRUCache({
+ max: maxEntries,
+ maxSize: maxBytes,
+ // An origin can answer many URLs with empty images, so each entry also pays for its strings and headers.
+ sizeCalculation: (result, id) =>
+ (result.bytes?.length ?? 0) + 2 * (id.length + result.currentUrl.length) + ENTRY_OVERHEAD_BYTES,
+ ttl: lifetimeMs,
+ ttlAutopurge: true,
+ })
+ }
+
+ public async fetch(url: string, options: ImageFetchOptions): Promise<ImageFetchResult> {
+ // The TDM reservation decides if a response is refused, so only jobs with the same reservation share a response.
+ const id = `${options.tdmrepReservation ? 'reserved' : 'open'} ${url}`
+ const recent = this.recent.get(id)
+ if (recent) {
+ sharedFetches.inc()
+ return recent
+ }
+ const pending = this.inFlight.get(id)
+ if (pending) {
+ const result = await pending
+ if (isShareable(result)) {
+ sharedFetches.inc()
+ return result
+ }
+ }
+ const own = this.inner.fetch(url, options)
+ if (!this.inFlight.has(id)) {
+ this.inFlight.set(id, own)
+ const forget = (): void => {
+ if (this.inFlight.get(id) === own) {
+ this.inFlight.delete(id)
+ }
+ }
+ own.then(forget, forget)
+ }
+ const result = await own
+ if (isShareable(result)) {
+ this.recent.set(id, result)
+ }
+ return result
+ }
+}
+
+/** A redirect depends on the hops a job has left, so only a direct image is shared. */
+function isShareable(result: ImageFetchResult): boolean {
+ return result.outcome === 'ok' && result.redirects === 0 && !!result.bytes
+}
diff --git a/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/tophog-metrics.ts b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/tophog-metrics.ts
new file mode 100644
index 000000000000..981abe6029da
--- /dev/null
+++ b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/tophog-metrics.ts
@@ -0,0 +1,126 @@
+import type { TopHogRegistry } from '~/ingestion/framework/extensions/tophog'
+
+import type { ImageFetchBlockReason } from './block-reason'
+import type { FetchCandidate } from './collected-urls-record'
+import { deduplicateFetchCandidates } from './fetch-candidate-queue'
+import type { FetchAttempt } from './fetch-runner'
+import type { SchedulerWaitScope } from './metrics'
+
+const TOP_DOMAIN_FACTORS = 20
+const MAX_TRACKED_DOMAIN_FACTORS = 2_000
+
+type Recorder = { record(key: Record<string, string>, value: number): void }
+
+export class ImageFetchTopHogMetrics {
+ private readonly attempts: Recorder
+ private readonly blockEvents: Recorder
+ private readonly blockedMs: Recorder
+
+ constructor(topHog: TopHogRegistry) {
+ this.attempts = topHog.registerSum('ai_research_v4_image_fetch_attempts_by_registrable_domain', {
+ topN: TOP_DOMAIN_FACTORS,
+ maxKeys: MAX_TRACKED_DOMAIN_FACTORS,
+ })
+ this.blockEvents = topHog.registerSum('ai_research_v4_image_fetch_block_events_by_registrable_domain', {
+ topN: TOP_DOMAIN_FACTORS,
+ maxKeys: MAX_TRACKED_DOMAIN_FACTORS,
+ })
+ this.blockedMs = topHog.registerSum('ai_research_v4_image_fetch_blocked_ms_by_registrable_domain', {
+ topN: TOP_DOMAIN_FACTORS,
+ maxKeys: MAX_TRACKED_DOMAIN_FACTORS,
+ })
+ }
+
+ public recordAttempt(attempt: FetchAttempt): void {
+ this.recordForSourcePartitions(
+ this.attempts,
+ attempt.candidate.sourcePartitions,
+ {
+ registrable_domain: attempt.candidate.registrableDomain,
+ disposition: attempt.finished ? 'completed' : 'republished',
+ outcome: attempt.outcome,
+ },
+ 1
+ )
+ if (attempt.block) {
+ this.recordBlock(
+ attempt.candidate.registrableDomain,
+ attempt.candidate.sourcePartitions,
+ attempt.block.reason,
+ 1,
+ attempt.block.waitMs
+ )
+ }
+ }
+
+ public recordConcurrencyLimitedUrls(
+ candidates: FetchCandidate[],
+ availableConnections: (registrableDomain: string) => number
+ ): void {
+ const candidatesByRegistrableDomain = new Map<
+ string,
+ { candidateCount: number; sourcePartitions: Set<number> }
+ >()
+ for (const candidate of deduplicateFetchCandidates(candidates).candidates) {
+ const domainCandidates = candidatesByRegistrableDomain.get(candidate.registrableDomain) ?? {
+ candidateCount: 0,
+ sourcePartitions: new Set<number>(),
+ }
+ domainCandidates.candidateCount += 1
+ for (const sourcePartition of candidate.sourcePartitions ?? []) {
+ domainCandidates.sourcePartitions.add(sourcePartition)
+ }
+ candidatesByRegistrableDomain.set(candidate.registrableDomain, domainCandidates)
+ }
+ for (const [registrableDomain, domainCandidates] of candidatesByRegistrableDomain) {
+ const limitedUrls = Math.max(0, domainCandidates.candidateCount - availableConnections(registrableDomain))
+ if (limitedUrls > 0) {
+ this.recordBlock(
+ registrableDomain,
+ [...domainCandidates.sourcePartitions],
+ 'domain_concurrency',
+ limitedUrls,
+ 0
+ )
+ }
+ }
+ }
+
+ public recordSchedulerWait(
+ registrableDomain: string,
+ sourcePartitions: readonly number[] | undefined,
+ scope: SchedulerWaitScope,
+ waitMs: number
+ ): void {
+ if (waitMs <= 0) {
+ return
+ }
+ this.recordBlock(registrableDomain, sourcePartitions, scope, 1, waitMs)
+ }
+
+ private recordBlock(
+ registrableDomain: string,
+ sourcePartitions: readonly number[] | undefined,
+ reason: ImageFetchBlockReason,
+ eventCount: number,
+ waitMs: number
+ ): void {
+ const key = { registrable_domain: registrableDomain, reason }
+ this.recordForSourcePartitions(this.blockEvents, sourcePartitions, key, eventCount)
+ if (waitMs > 0) {
+ this.recordForSourcePartitions(this.blockedMs, sourcePartitions, key, waitMs)
+ }
+ }
+
+ private recordForSourcePartitions(
+ recorder: Recorder,
+ sourcePartitions: readonly number[] | undefined,
+ key: Record<string, string>,
+ value: number
+ ): void {
+ const partitions = sourcePartitions?.length ? new Set(sourcePartitions) : new Set(['unattributed'])
+ for (const partition of partitions) {
+ recorder.record({ ...key, partition: String(partition) }, value)
+ }
+ }
+}
diff --git a/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/url-fetch-consumer.test.ts b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/url-fetch-consumer.test.ts
new file mode 100644
index 000000000000..9ef0db1aba4b
--- /dev/null
+++ b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/url-fetch-consumer.test.ts
@@ -0,0 +1,747 @@
+import { Message } from 'node-rdkafka'
+import { register } from 'prom-client'
+
+import { RecordedTopHogMetric, createRecordingTopHog } from '~/tests/helpers/tophog'
+
+import { FetchCandidate, MAX_HOPS, serializeFrontierRecord } from './collected-urls-record'
+import { CrawlHistoryItem, CrawlHistoryStore, configurationCacheKey } from './crawl-history'
+import { AttemptOutcome, DELAY_TOO_LONG, FetchAttempt, FetchPass, HOPS_EXHAUSTED } from './fetch-runner'
+import { FrontierDeadLetterSink } from './frontier-dead-letter-sink'
+import { FrontierPublisher, RepublishFlushResult, RepublishResult } from './frontier-publisher'
+import { ImageFetchConsumerMetrics, ImageFetchRequestMetrics } from './metrics'
+import { ImageFetchTopHogMetrics } from './tophog-metrics'
+import { UrlFetchConsumer } from './url-fetch-consumer'
+
+const NOW_MS = 1_700_000_000_000
+
+function candidate(name: string, overrides: Partial<FetchCandidate> = {}): FetchCandidate {
+ const hash = name.padEnd(22, '0')
+ return {
+ originalRef: `imageurl:v4:7:2026-10:${hash}`,
+ currentUrl: `https://cdn.example.com/${name}.png`,
+ host: 'cdn.example.com',
+ origin: 'https://cdn.example.com',
+ registrableDomain: 'example.com',
+ remainingHops: MAX_HOPS,
+ notBeforeMs: 0,
+ firstSeenAtMs: NOW_MS,
+ fetchCount: 0,
+ republishCount: 0,
+ lastRepublishReason: null,
+ ...overrides,
+ }
+}
+
+function message(candidates: FetchCandidate[], key = 'example.com', partition = 0): Message {
+ const value = serializeFrontierRecord(candidates)
+ return {
+ value,
+ key: Buffer.from(key),
+ size: value.length,
+ topic: 'session_replay_image_fetch',
+ partition,
+ offset: 0,
+ }
+}
+
+class FakeCrawlHistory implements CrawlHistoryStore {
+ public readonly items = new Map<string, CrawlHistoryItem>()
+ public readKeys: string[][] = []
+ public writes: CrawlHistoryItem[][] = []
+ public readError: Error | undefined
+ public writeError: Error | undefined
+
+ read(keys: string[]): Promise<Map<string, CrawlHistoryItem>> {
+ this.readKeys.push(keys)
+ if (this.readError) {
+ return Promise.reject(this.readError)
+ }
+ return Promise.resolve(
+ new Map(keys.flatMap((key) => (this.items.has(key) ? [[key, this.items.get(key)!] as const] : [])))
+ )
+ }
+
+ write(items: CrawlHistoryItem[]): Promise<void> {
+ this.writes.push(items)
+ if (this.writeError) {
+ return Promise.reject(this.writeError)
+ }
+ for (const item of items) {
+ this.items.set(item.key, item)
+ }
+ return Promise.resolve()
+ }
+}
+
+function terminal(candidate: FetchCandidate, outcome: AttemptOutcome = 'ok'): FetchAttempt {
+ return {
+ candidate,
+ outcome,
+ finished: true,
+ lost: false,
+ history: {
+ kind: 'url',
+ key: candidate.originalRef,
+ nextFetchAtMs: NOW_MS + 30 * 24 * 60 * 60 * 1000,
+ storageExpiresAtMs: NOW_MS + 30 * 24 * 60 * 60 * 1000,
+ outcome,
+ },
+ configurationUpdates: [],
+ }
+}
+
+interface Harness {
+ consumer: UrlFetchConsumer
+ history: FakeCrawlHistory
+ run: jest.Mock<Promise<FetchAttempt[]>, [FetchCandidate[], Map<string, CrawlHistoryItem>]>
+ republish: jest.Mock<Promise<RepublishResult>, any[]>
+ flush: jest.Mock<Promise<RepublishFlushResult>, []>
+ park: jest.Mock<Promise<void>, any[]>
+ topHogRecords: Map<string, RecordedTopHogMetric[]>
+}
+
+function build(dryRun = false, deadLettersEnabled = true): Harness {
+ const history = new FakeCrawlHistory()
+ const run = jest.fn((candidates: FetchCandidate[], _stored: Map<string, CrawlHistoryItem>) =>
+ Promise.resolve(candidates.map((item) => terminal(item)))
+ )
+ const republish = jest.fn(() => Promise.resolve('queued' as const))
+ const flush = jest.fn(() => Promise.resolve({ failedUrls: 0 }))
+ const park = jest.fn(() => Promise.resolve())
+ const recordingTopHog = createRecordingTopHog()
+ const topHogMetrics = new ImageFetchTopHogMetrics(recordingTopHog.registry)
+ const consumer = new UrlFetchConsumer(
+ history,
+ { createRepublishBatch: () => ({ republish, flush }) } as unknown as FrontierPublisher,
+ { seenTtlSeconds: 30 * 24 * 60 * 60, dryRun },
+ dryRun ? undefined : ({ run } as FetchPass),
+ deadLettersEnabled ? ({ park } as FrontierDeadLetterSink) : null,
+ topHogMetrics
+ )
+ return { consumer, history, run, republish, flush, park, topHogRecords: recordingTopHog.records }
+}
+
+describe('UrlFetchConsumer', () => {
+ afterEach(() => jest.restoreAllMocks())
+
+ it.each([Number.NaN, 0, 3_599, 3_600.5])('refuses an invalid crawl-history TTL of %p', (seenTtlSeconds) => {
+ expect(
+ () =>
+ new UrlFetchConsumer(new FakeCrawlHistory(), {} as FrontierPublisher, { seenTtlSeconds, dryRun: true })
+ ).toThrow('AI_RESEARCH_V4_IMAGE_FETCH_CRAWL_HISTORY_TTL_SECONDS')
+ })
+
+ it('refuses active mode without a fetch pass', () => {
+ expect(
+ () =>
+ new UrlFetchConsumer(new FakeCrawlHistory(), {} as FrontierPublisher, {
+ seenTtlSeconds: 3_600,
+ dryRun: false,
+ })
+ ).toThrow('fetch runner')
+ })
+
+ it('parses dry-run traffic without reading or writing shared state', async () => {
+ const harness = build(true)
+
+ await harness.consumer.handleBatch([message([candidate('a')])], NOW_MS)
+
+ expect(harness.history.readKeys).toEqual([])
+ expect(harness.history.writes).toEqual([])
+ })
+
+ it('bulk reads URL and per-origin configuration keys', async () => {
+ const harness = build()
+
+ await harness.consumer.handleBatch([message([candidate('a'), candidate('b')])], NOW_MS)
+
+ expect(harness.history.readKeys).toEqual([
+ [
+ candidate('a').originalRef,
+ candidate('b').originalRef,
+ configurationCacheKey(candidate('a').origin, 'robots'),
+ configurationCacheKey(candidate('a').origin, 'tdmrep'),
+ ],
+ ])
+ expect(harness.run.mock.calls[0][0]).toEqual([
+ candidate('a', { sourcePartitions: [0] }),
+ candidate('b', { sourcePartitions: [0] }),
+ ])
+ expect(harness.history.writes[0]).toHaveLength(2)
+ })
+
+ it('records distinct origins and registrable domains for the poll batch', async () => {
+ const harness = build()
+ const observeBatch = jest.spyOn(ImageFetchConsumerMetrics, 'observeBatch')
+ const observeBatchDiversity = jest.spyOn(ImageFetchConsumerMetrics, 'observeBatchDiversity')
+ const otherExampleOrigin = candidate('b', {
+ currentUrl: 'https://img.example.com/b.png',
+ host: 'img.example.com',
+ origin: 'https://img.example.com',
+ })
+ const otherRegistrableDomain = candidate('c', {
+ currentUrl: 'https://img.other.net/c.png',
+ host: 'img.other.net',
+ origin: 'https://img.other.net',
+ registrableDomain: 'other.net',
+ })
+
+ await harness.consumer.handleBatch(
+ [message([candidate('a'), otherExampleOrigin]), message([otherRegistrableDomain], 'other.net')],
+ NOW_MS
+ )
+
+ expect(observeBatch).toHaveBeenCalledWith(3, 2, expect.any(Number))
+ expect(observeBatchDiversity).toHaveBeenCalledWith([1, 1, 1], [2, 1])
+ })
+
+ it('attributes joined batch work and outcomes to each source partition', async () => {
+ const harness = build()
+ const observePartitionRecord = jest.spyOn(ImageFetchConsumerMetrics, 'observePartitionRecord')
+ const incPartitionUrls = jest.spyOn(ImageFetchConsumerMetrics, 'incPartitionUrls')
+ const observePartitionBatchDiversity = jest.spyOn(ImageFetchConsumerMetrics, 'observePartitionBatchDiversity')
+ const incPartitionAttempt = jest.spyOn(ImageFetchRequestMetrics, 'incPartitionAttempt')
+ const fromPartition7 = candidate('a')
+ const fromPartition42 = candidate('b', {
+ currentUrl: 'https://cdn.other.net/b.png',
+ host: 'cdn.other.net',
+ origin: 'https://cdn.other.net',
+ registrableDomain: 'other.net',
+ })
+
+ await harness.consumer.handleBatch(
+ [message([fromPartition7], 'example.com', 7), message([fromPartition42], 'other.net', 42)],
+ NOW_MS
+ )
+
+ expect(observePartitionRecord).toHaveBeenCalledWith(7, 1, 1)
+ expect(observePartitionRecord).toHaveBeenCalledWith(42, 1, 1)
+ expect(incPartitionUrls).toHaveBeenCalledWith(7, 'unique', 1)
+ expect(incPartitionUrls).toHaveBeenCalledWith(7, 'fetchable', 1)
+ expect(incPartitionUrls).toHaveBeenCalledWith(42, 'unique', 1)
+ expect(incPartitionUrls).toHaveBeenCalledWith(42, 'fetchable', 1)
+ expect(observePartitionBatchDiversity).toHaveBeenCalledWith(7, [1], [1])
+ expect(observePartitionBatchDiversity).toHaveBeenCalledWith(42, [1], [1])
+ expect(harness.run.mock.calls[0][0]).toEqual([
+ { ...fromPartition7, sourcePartitions: [7] },
+ { ...fromPartition42, sourcePartitions: [42] },
+ ])
+ expect(harness.topHogRecords.get('ai_research_v4_image_fetch_attempts_by_registrable_domain')).toEqual([
+ {
+ key: {
+ registrable_domain: 'example.com',
+ disposition: 'completed',
+ outcome: 'ok',
+ partition: '7',
+ },
+ value: 1,
+ },
+ {
+ key: {
+ registrable_domain: 'other.net',
+ disposition: 'completed',
+ outcome: 'ok',
+ partition: '42',
+ },
+ value: 1,
+ },
+ ])
+ expect(incPartitionAttempt).toHaveBeenCalledWith(7, 'completed', 'ok')
+ expect(incPartitionAttempt).toHaveBeenCalledWith(42, 'completed', 'ok')
+ })
+
+ it('keeps attribution when one ref arrives from different partitions', async () => {
+ const harness = build()
+ const incPartitionUrls = jest.spyOn(ImageFetchConsumerMetrics, 'incPartitionUrls')
+ const observePartitionBatchDiversity = jest.spyOn(ImageFetchConsumerMetrics, 'observePartitionBatchDiversity')
+ const incPartitionAttempt = jest.spyOn(ImageFetchRequestMetrics, 'incPartitionAttempt')
+ const fromPartition7 = candidate('a')
+ const fromPartition42 = candidate('a', {
+ currentUrl: 'https://cdn.other.net/a.png',
+ host: 'cdn.other.net',
+ origin: 'https://cdn.other.net',
+ registrableDomain: 'other.net',
+ remainingHops: 9,
+ republishCount: 1,
+ lastRepublishReason: 'redirect',
+ })
+
+ await harness.consumer.handleBatch(
+ [message([fromPartition7], 'example.com', 7), message([fromPartition42], 'other.net', 42)],
+ NOW_MS
+ )
+
+ expect(harness.run.mock.calls[0][0]).toEqual([{ ...fromPartition42, sourcePartitions: [7, 42] }])
+ for (const partition of [7, 42]) {
+ expect(incPartitionUrls).toHaveBeenCalledWith(partition, 'unique', 1)
+ expect(incPartitionUrls).toHaveBeenCalledWith(partition, 'fetchable', 1)
+ expect(observePartitionBatchDiversity).toHaveBeenCalledWith(partition, [1], [1])
+ expect(incPartitionAttempt).toHaveBeenCalledWith(partition, 'completed', 'ok')
+ }
+ })
+
+ it.each([
+ ['a ref', candidate('a'), candidate('a')],
+ [
+ 'a ref from two sessions',
+ candidate('a', { sessionId: '01a0c669-8800-7000-8000-000000000001' }),
+ candidate('a', { sessionId: '01a0c669-8800-7000-8000-000000000002' }),
+ ],
+ ])('deduplicates %s within the batch', async (_name, first, second) => {
+ const harness = build()
+
+ await harness.consumer.handleBatch([message([first]), message([second])], NOW_MS)
+
+ expect(harness.run.mock.calls[0][0]).toEqual([{ ...first, sourcePartitions: [0] }])
+ })
+
+ it('keeps the most conservative durable state from duplicate jobs', async () => {
+ const harness = build()
+ const stale = candidate('a')
+ const advanced = candidate('a', {
+ currentUrl: 'https://cdn.example.com/moved.png',
+ remainingHops: 7,
+ firstSeenAtMs: NOW_MS - 5_000,
+ fetchCount: 4,
+ republishCount: 3,
+ lastRepublishReason: 'retry',
+ })
+
+ await harness.consumer.handleBatch([message([stale]), message([advanced])], NOW_MS)
+
+ expect(harness.run.mock.calls[0][0]).toEqual([{ ...advanced, sourcePartitions: [0] }])
+ })
+
+ it('keeps the latest not-before time from duplicate jobs', async () => {
+ const harness = build()
+ const stale = candidate('a')
+ const delayed = candidate('a', {
+ remainingHops: 9,
+ notBeforeMs: NOW_MS + 30_000,
+ republishCount: 1,
+ lastRepublishReason: 'retry',
+ })
+
+ await harness.consumer.handleBatch([message([stale]), message([delayed])], NOW_MS)
+
+ expect(harness.run.mock.calls[0][0]).toEqual([])
+ expect(harness.republish).toHaveBeenCalledWith(
+ expect.objectContaining({ remainingHops: 9, notBeforeMs: NOW_MS + 30_000 }),
+ expect.any(Object),
+ 'not_ready',
+ 30_000
+ )
+ })
+
+ it('skips a ref whose crawl-history interval has not ended', async () => {
+ const [stored, fresh] = [candidate('a'), candidate('b')]
+ const harness = build()
+ harness.history.items.set(stored.originalRef, {
+ kind: 'url',
+ key: stored.originalRef,
+ nextFetchAtMs: NOW_MS + 1,
+ storageExpiresAtMs: NOW_MS + 1,
+ outcome: 'ok',
+ })
+
+ await harness.consumer.handleBatch([message([stored, fresh])], NOW_MS)
+
+ expect(harness.run.mock.calls[0][0]).toEqual([{ ...fresh, sourcePartitions: [0] }])
+ })
+
+ it('republishes a job that arrives before its durable not-before time', async () => {
+ const harness = build()
+ const early = candidate('a', {
+ notBeforeMs: NOW_MS + 30_000,
+ lastBlockReason: 'configuration_unreachable',
+ })
+
+ await harness.consumer.handleBatch([message([early])], NOW_MS)
+
+ expect(harness.run.mock.calls[0][0]).toEqual([])
+ expect(harness.republish).toHaveBeenCalledWith(
+ { ...early, sourcePartitions: [0], lastBlockReason: 'configuration_unreachable' },
+ {
+ currentUrl: early.currentUrl,
+ host: early.host,
+ origin: early.origin,
+ registrableDomain: early.registrableDomain,
+ },
+ 'not_ready',
+ 30_000
+ )
+ expect(harness.history.writes).toEqual([])
+ expect(harness.topHogRecords.get('ai_research_v4_image_fetch_attempts_by_registrable_domain')).toEqual([
+ {
+ key: {
+ registrable_domain: 'example.com',
+ disposition: 'republished',
+ outcome: 'backoff',
+ partition: '0',
+ },
+ value: 1,
+ },
+ ])
+ expect(harness.topHogRecords.get('ai_research_v4_image_fetch_block_events_by_registrable_domain')).toEqual([
+ {
+ key: {
+ registrable_domain: 'example.com',
+ reason: 'configuration_unreachable',
+ partition: '0',
+ },
+ value: 1,
+ },
+ ])
+ expect(harness.topHogRecords.get('ai_research_v4_image_fetch_blocked_ms_by_registrable_domain')).toEqual([
+ {
+ key: {
+ registrable_domain: 'example.com',
+ reason: 'configuration_unreachable',
+ partition: '0',
+ },
+ value: 30_000,
+ },
+ ])
+ })
+
+ it('records a terminal refusal when the remaining delay is over one hour', async () => {
+ const harness = build()
+ harness.republish.mockResolvedValue('refused_delay')
+ const early = candidate('a', { notBeforeMs: NOW_MS + 3_600_001 })
+
+ await harness.consumer.handleBatch([message([early])], NOW_MS)
+
+ expect(harness.history.writes[0]).toEqual([
+ expect.objectContaining({ kind: 'url', key: early.originalRef, outcome: DELAY_TOO_LONG }),
+ ])
+ })
+
+ it('records a terminal refusal when a delayed job has no remaining hops', async () => {
+ const harness = build()
+ const early = candidate('a', { notBeforeMs: NOW_MS + 30_000, remainingHops: 0 })
+
+ await harness.consumer.handleBatch([message([early])], NOW_MS)
+
+ expect(harness.republish).not.toHaveBeenCalled()
+ expect(harness.history.writes[0]).toEqual([
+ expect.objectContaining({ kind: 'url', key: early.originalRef, outcome: HOPS_EXHAUSTED }),
+ ])
+ })
+
+ it('throws when a not-ready republish delivery fails', async () => {
+ const harness = build()
+ harness.flush.mockResolvedValue({ failedUrls: 1 })
+ const early = candidate('a', { notBeforeMs: NOW_MS + 30_000 })
+
+ await expect(harness.consumer.handleBatch([message([early])], NOW_MS)).rejects.toThrow('account for 1 URLs')
+ expect(harness.history.writes).toEqual([])
+ })
+
+ it('records a retry cause after the republish batch is durable', async () => {
+ const harness = build()
+ const retryCause = jest.spyOn(ImageFetchRequestMetrics, 'incRetryCause').mockImplementation()
+ harness.run.mockImplementation((candidates) =>
+ Promise.resolve(
+ candidates.map((item) => ({
+ candidate: item,
+ outcome: 'server_error',
+ finished: false,
+ lost: false,
+ configurationUpdates: [],
+ }))
+ )
+ )
+
+ await harness.consumer.handleBatch([message([candidate('a')])], NOW_MS)
+
+ expect(retryCause).toHaveBeenCalledWith('server_error')
+ })
+
+ it('does not record a retry cause when the republish batch fails', async () => {
+ const harness = build()
+ const retryCause = jest.spyOn(ImageFetchRequestMetrics, 'incRetryCause').mockImplementation()
+ harness.flush.mockResolvedValue({ failedUrls: 1 })
+ harness.run.mockImplementation((candidates) =>
+ Promise.resolve(
+ candidates.map((item) => ({
+ candidate: item,
+ outcome: 'server_error',
+ finished: false,
+ lost: false,
+ configurationUpdates: [],
+ }))
+ )
+ )
+
+ await expect(harness.consumer.handleBatch([message([candidate('a')])], NOW_MS)).rejects.toThrow(
+ 'account for 1 URLs'
+ )
+ expect(retryCause).not.toHaveBeenCalled()
+ })
+
+ it('quarantines a malformed record before completing the source batch', async () => {
+ const harness = build()
+ const invalid = message([candidate('a')])
+ invalid.value = Buffer.from('{')
+ let releasePark: () => void = () => undefined
+ harness.park.mockImplementation(
+ () =>
+ new Promise<void>((resolve) => {
+ releasePark = resolve
+ })
+ )
+ const completed = jest.fn()
+
+ const sourceBatch = harness.consumer.handleBatch([invalid], NOW_MS)
+ void sourceBatch.then(completed)
+ await new Promise((resolve) => setImmediate(resolve))
+
+ expect(harness.park).toHaveBeenCalledWith(invalid, 'malformed')
+ expect(completed).not.toHaveBeenCalled()
+ expect(harness.run).not.toHaveBeenCalled()
+
+ releasePark()
+ await expect(sourceBatch).resolves.toBeUndefined()
+ expect(completed).toHaveBeenCalledTimes(1)
+ })
+
+ it('quarantines rejected records concurrently', async () => {
+ const harness = build()
+ const first = message([candidate('a')])
+ first.value = Buffer.from('{')
+ const second = message([candidate('b')])
+ second.value = Buffer.from('{')
+ second.offset = 1
+ const releases: Array<() => void> = []
+ harness.park.mockImplementation(
+ () =>
+ new Promise<void>((resolve) => {
+ releases.push(resolve)
+ })
+ )
+
+ const sourceBatch = harness.consumer.handleBatch([first, second], NOW_MS)
+ await new Promise((resolve) => setImmediate(resolve))
+
+ expect(harness.park).toHaveBeenCalledTimes(2)
+
+ for (const release of releases) {
+ release()
+ }
+ await expect(sourceBatch).resolves.toBeUndefined()
+ })
+
+ it('does not quarantine rejected records before other batch work succeeds', async () => {
+ const harness = build()
+ const invalid = message([candidate('a')])
+ invalid.value = Buffer.from('{')
+ harness.history.readError = new Error('read failed')
+
+ await expect(harness.consumer.handleBatch([invalid, message([candidate('b')])], NOW_MS)).rejects.toThrow(
+ 'read failed'
+ )
+
+ expect(harness.park).not.toHaveBeenCalled()
+ })
+
+ it('keeps the source batch uncommitted when quarantine publication fails', async () => {
+ const harness = build()
+ const invalid = message([candidate('a')])
+ invalid.value = Buffer.from('{')
+ harness.park.mockRejectedValue(new Error('DLQ unavailable'))
+ const deadLetterFailed = jest.spyOn(ImageFetchConsumerMetrics, 'incDeadLetterFailed')
+ const dropped = jest.spyOn(ImageFetchConsumerMetrics, 'incDropped')
+
+ await expect(harness.consumer.handleBatch([invalid], NOW_MS)).rejects.toThrow('DLQ unavailable')
+
+ expect(deadLetterFailed).toHaveBeenCalledWith('malformed')
+ expect(dropped).not.toHaveBeenCalled()
+ expect(harness.run).not.toHaveBeenCalled()
+ expect(harness.history.readKeys).toEqual([])
+ })
+
+ it('commits and drops rejected input when quarantine is disabled', async () => {
+ const harness = build(false, false)
+ const invalid = message([candidate('a')])
+ invalid.value = Buffer.from('{')
+
+ await expect(harness.consumer.handleBatch([invalid], NOW_MS)).resolves.toBeUndefined()
+
+ expect(harness.park).not.toHaveBeenCalled()
+ expect(harness.run).not.toHaveBeenCalled()
+ })
+
+ it('drops a tracking beacon job without quarantining its record', async () => {
+ const harness = build()
+ const beacon = candidate('beacon', {
+ currentUrl: 'https://analytics.twitter.com/i/adsct?txn_id=abc&p_id=Twitter',
+ host: 'analytics.twitter.com',
+ origin: 'https://analytics.twitter.com',
+ registrableDomain: 'twitter.com',
+ })
+ const image = candidate('logo', {
+ currentUrl: 'https://cdn.twitter.com/logo.png',
+ host: 'cdn.twitter.com',
+ origin: 'https://cdn.twitter.com',
+ registrableDomain: 'twitter.com',
+ })
+ const dropped = jest.spyOn(ImageFetchConsumerMetrics, 'incDropped')
+ const skipped = jest.spyOn(ImageFetchConsumerMetrics, 'incSkipped')
+
+ await expect(
+ harness.consumer.handleBatch([message([beacon, image], 'twitter.com')], NOW_MS)
+ ).resolves.toBeUndefined()
+
+ expect(harness.park).not.toHaveBeenCalled()
+ expect(harness.run).toHaveBeenCalledTimes(1)
+ expect(harness.run.mock.calls[0][0].map((fetched) => fetched.originalRef)).toEqual([image.originalRef])
+ expect(skipped).toHaveBeenCalledWith('tracking_beacon', 1)
+ expect(dropped).not.toHaveBeenCalled()
+ })
+
+ it('does not count a skipped beacon when the batch fails before its commit', async () => {
+ const harness = build()
+ const beacon = candidate('beacon', {
+ currentUrl: 'https://analytics.twitter.com/i/adsct?txn_id=abc&p_id=Twitter',
+ host: 'analytics.twitter.com',
+ origin: 'https://analytics.twitter.com',
+ registrableDomain: 'twitter.com',
+ })
+ const image = candidate('logo', {
+ currentUrl: 'https://cdn.twitter.com/logo.png',
+ host: 'cdn.twitter.com',
+ origin: 'https://cdn.twitter.com',
+ registrableDomain: 'twitter.com',
+ })
+ harness.history.readError = new Error('read failed')
+ const skipped = jest.spyOn(ImageFetchConsumerMetrics, 'incSkipped')
+
+ await expect(harness.consumer.handleBatch([message([beacon, image], 'twitter.com')], NOW_MS)).rejects.toThrow(
+ 'read failed'
+ )
+
+ expect(skipped).not.toHaveBeenCalled()
+ })
+
+ it('rejects a whole multi-job record when one job belongs to another partition', async () => {
+ const harness = build()
+ const foreign = candidate('foreign', {
+ currentUrl: 'https://img.other.net/foreign.png',
+ host: 'img.other.net',
+ origin: 'https://img.other.net',
+ registrableDomain: 'other.net',
+ })
+
+ await harness.consumer.handleBatch([message([candidate('a'), foreign])], NOW_MS)
+
+ expect(harness.run).not.toHaveBeenCalled()
+ expect(harness.history.readKeys).toEqual([])
+ })
+
+ it('throws when the bulk read fails', async () => {
+ register.resetMetrics()
+ const harness = build()
+ harness.history.readError = new Error('read failed')
+ const observeBatch = jest.spyOn(ImageFetchConsumerMetrics, 'observeBatch')
+ const observeStoreDuration = jest.spyOn(ImageFetchConsumerMetrics, 'observeStoreDuration')
+ const startBatch = jest.spyOn(ImageFetchConsumerMetrics, 'startBatch')
+ const finishBatch = jest.spyOn(ImageFetchConsumerMetrics, 'finishBatch')
+
+ await expect(harness.consumer.handleBatch([message([candidate('a')])], NOW_MS)).rejects.toThrow('read failed')
+ expect(observeBatch).toHaveBeenCalledWith(1, 1, expect.any(Number))
+ expect(observeStoreDuration).toHaveBeenCalledWith('read', 'error', expect.any(Number))
+ expect(startBatch).toHaveBeenCalledTimes(1)
+ expect(finishBatch).toHaveBeenCalledTimes(1)
+ const active = await register.getSingleMetric('ai_research_v4_image_fetch_stage_active')!.get()
+ expect(active.values.every(({ value }) => value === 0)).toBe(true)
+ const durations = await register.getSingleMetric('ai_research_v4_image_fetch_stage_duration_seconds')!.get()
+ expect(durations.values).toEqual(
+ expect.arrayContaining([
+ expect.objectContaining({
+ metricName: 'ai_research_v4_image_fetch_stage_duration_seconds_count',
+ labels: { stage: 'batch_history_read' },
+ value: 1,
+ }),
+ ])
+ )
+ })
+
+ it('throws when the final bulk write fails', async () => {
+ const harness = build()
+ harness.history.writeError = new Error('write failed')
+
+ await expect(harness.consumer.handleBatch([message([candidate('a')])], NOW_MS)).rejects.toThrow('write failed')
+ expect(harness.flush).not.toHaveBeenCalled()
+ })
+
+ it('writes durable state before it flushes buffered republishes', async () => {
+ register.resetMetrics()
+ const harness = build()
+ const order: string[] = []
+ harness.run.mockImplementation((candidates) => {
+ order.push('published')
+ return Promise.resolve(candidates.map((item) => terminal(item)))
+ })
+ const write = harness.history.write.bind(harness.history)
+ harness.history.write = async (items) => {
+ order.push('history')
+ await write(items)
+ }
+ harness.flush.mockImplementation(() => {
+ order.push('republished')
+ return Promise.resolve({ failedUrls: 0 })
+ })
+
+ await harness.consumer.handleBatch([message([candidate('a')])], NOW_MS)
+
+ expect(order).toEqual(['published', 'history', 'republished'])
+ const durations = await register.getSingleMetric('ai_research_v4_image_fetch_stage_duration_seconds')!.get()
+ const completedStages = durations.values.map(({ labels }) => labels.stage)
+ expect(completedStages).toEqual(
+ expect.arrayContaining([
+ 'batch_parse',
+ 'batch_history_read',
+ 'batch_filter',
+ 'batch_fetch',
+ 'batch_prepare_republish',
+ 'batch_history_write',
+ 'batch_republish_flush',
+ 'batch_finalize',
+ 'batch_dead_letter',
+ ])
+ )
+ const active = await register.getSingleMetric('ai_research_v4_image_fetch_stage_active')!.get()
+ expect(active.values.every(({ value }) => value === 0)).toBe(true)
+ })
+
+ it('throws when the fetch pass reports a lost URL', async () => {
+ const harness = build()
+ harness.run.mockImplementation((candidates) =>
+ Promise.resolve([
+ {
+ candidate: candidates[0],
+ outcome: 'timeout',
+ finished: false,
+ lost: true,
+ configurationUpdates: [],
+ },
+ terminal(candidates[1]),
+ ])
+ )
+
+ await expect(harness.consumer.handleBatch([message([candidate('a'), candidate('b')])], NOW_MS)).rejects.toThrow(
+ 'account for 1 URLs'
+ )
+ expect(harness.topHogRecords.get('ai_research_v4_image_fetch_attempts_by_registrable_domain')).toEqual([
+ {
+ key: {
+ registrable_domain: 'example.com',
+ disposition: 'completed',
+ outcome: 'ok',
+ partition: '0',
+ },
+ value: 1,
+ },
+ ])
+ })
+})
diff --git a/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/url-fetch-consumer.ts b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/url-fetch-consumer.ts
new file mode 100644
index 000000000000..5a6a5bb1cc9c
--- /dev/null
+++ b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/url-fetch-consumer.ts
@@ -0,0 +1,514 @@
+import { Message } from 'node-rdkafka'
+import pLimit from 'p-limit'
+
+import { logger } from '~/common/utils/logger'
+
+import {
+ FetchCandidate,
+ MAX_HOPS,
+ UrlDropReason,
+ UrlSkipReason,
+ parseCollectedUrlsRecord,
+} from './collected-urls-record'
+import { CrawlHistoryItem, CrawlHistoryStore, UrlCrawlHistoryItem, configurationCacheKey } from './crawl-history'
+import { FetchCandidatePoolAdmission } from './fetch-candidate-pool'
+import { mergeDuplicateFetchCandidates } from './fetch-candidate-queue'
+import {
+ AttemptOutcome,
+ DELAY_TOO_LONG,
+ FetchAttempt,
+ FetchPass,
+ HOPS_EXHAUSTED,
+ isTransientOutcome,
+} from './fetch-runner'
+import { FrontierDeadLetterReason, FrontierDeadLetterSink } from './frontier-dead-letter-sink'
+import { FrontierPublisher, RepublishBatch } from './frontier-publisher'
+import type { RefreshingImageSink } from './image-sink'
+import { ImageFetchConsumerMetrics, ImageFetchRequestMetrics } from './metrics'
+import { ImageFetchProcessingMetrics } from './processing-metrics'
+import { ImageFetchTopHogMetrics } from './tophog-metrics'
+import { urlHistoryExpiresAtMs, urlHistoryStorageExpiresAtMs } from './url-history-expiry'
+
+const ONE_HOUR_MS = 60 * 60 * 1000
+const REPUBLISH_DEADLINE_FROM_BATCH_START_MS = 200_000
+const DEAD_LETTER_BATCH_BUDGET_MS = 50_000
+const DEAD_LETTER_PUBLISH_CONCURRENCY = 8
+
+type RejectedFrontierRecord = {
+ message: Message
+ reasons: UrlDropReason[]
+}
+
+export interface UrlFetchConsumerOptions {
+ seenTtlSeconds: number
+ dryRun: boolean
+ /** Drops jobs before any request for them, and resolves to the jobs that stay. The lane counts what it drops. */
+ admitCandidates?: (candidates: FetchCandidate[], nowMs: number) => Promise<FetchCandidate[]>
+ /** The lane's sink. A job not due for a fetch refreshes its stored image when the image is stale. */
+ refreshingSink?: RefreshingImageSink
+}
+
+export class UrlFetchConsumer {
+ constructor(
+ private readonly crawlHistory: CrawlHistoryStore,
+ private readonly publisher: FrontierPublisher,
+ private readonly options: UrlFetchConsumerOptions,
+ private readonly runner?: FetchPass,
+ private readonly deadLetters: FrontierDeadLetterSink | null = null,
+ private readonly topHogMetrics?: ImageFetchTopHogMetrics
+ ) {
+ if (!Number.isInteger(options.seenTtlSeconds) || options.seenTtlSeconds < 60 * 60) {
+ throw new Error('AI_RESEARCH_V4_IMAGE_FETCH_CRAWL_HISTORY_TTL_SECONDS must be at least 3600')
+ }
+ if (!options.dryRun && !runner) {
+ throw new Error('the image fetch lane needs a fetch runner once dry run is cleared')
+ }
+ ImageFetchConsumerMetrics.setDryRun(options.dryRun)
+ }
+
+ public async handleBatch(
+ messages: Message[],
+ nowMs: number,
+ admission?: FetchCandidatePoolAdmission
+ ): Promise<void> {
+ const startedAt = process.hrtime.bigint()
+ const republishDeadlineAtMonotonicMs = performance.now() + REPUBLISH_DEADLINE_FROM_BATCH_START_MS
+ const drops = new Map<UrlDropReason, number>()
+ const skips = new Map<UrlSkipReason, number>()
+ const rejectedRecords: RejectedFrontierRecord[] = []
+ const candidatesByRef = new Map<string, FetchCandidate>()
+ let dedupedInBatch = 0
+ let originCount = 0
+ let registrableDomainCount = 0
+ let originCandidateCounts: number[] = []
+ let registrableDomainCandidateCounts: number[] = []
+ const activeBatchId = ImageFetchConsumerMetrics.startBatch()
+ const stage = ImageFetchProcessingMetrics.start('batch_parse')
+
+ try {
+ for (const message of messages) {
+ const parsed = this.parse(message)
+ if (!parsed.ok) {
+ rejectedRecords.push({ message, reasons: [parsed.reason] })
+ continue
+ }
+ ImageFetchConsumerMetrics.observeRecord(parsed.urlCount)
+ ImageFetchConsumerMetrics.observePartitionRecord(
+ message.partition,
+ parsed.urlCount,
+ parsed.candidates.length
+ )
+ if (parsed.rejected.length > 0) {
+ rejectedRecords.push({
+ message,
+ reasons: parsed.rejected.map((rejected) => rejected.reason),
+ })
+ }
+ for (const { reason } of parsed.skipped) {
+ skips.set(reason, (skips.get(reason) ?? 0) + 1)
+ }
+ for (const candidate of parsed.candidates) {
+ const partitionCandidate = { ...candidate, sourcePartitions: [message.partition] }
+ const existing = candidatesByRef.get(partitionCandidate.originalRef)
+ if (existing) {
+ dedupedInBatch += 1
+ candidatesByRef.set(
+ partitionCandidate.originalRef,
+ mergeDuplicateFetchCandidates(existing, partitionCandidate)
+ )
+ } else {
+ candidatesByRef.set(partitionCandidate.originalRef, partitionCandidate)
+ }
+ }
+ }
+ const parsedCandidates = [...candidatesByRef.values()]
+ candidatesByRef.clear()
+ const candidates =
+ this.options.admitCandidates && !this.options.dryRun && parsedCandidates.length > 0
+ ? await this.options.admitCandidates(parsedCandidates, nowMs)
+ : parsedCandidates
+ const origins = new Map<string, number>()
+ const registrableDomains = new Map<string, number>()
+ const partitionOrigins = new Map<number, Map<string, number>>()
+ const partitionRegistrableDomains = new Map<number, Map<string, number>>()
+ for (const candidate of candidates) {
+ ImageFetchConsumerMetrics.observeAge(Math.max(0, nowMs - candidate.firstSeenAtMs) / 1000)
+ origins.set(candidate.origin, (origins.get(candidate.origin) ?? 0) + 1)
+ registrableDomains.set(
+ candidate.registrableDomain,
+ (registrableDomains.get(candidate.registrableDomain) ?? 0) + 1
+ )
+ for (const sourcePartition of candidate.sourcePartitions ?? []) {
+ ImageFetchConsumerMetrics.incPartitionUrls(sourcePartition, 'unique', 1)
+ incrementNestedCount(partitionOrigins, sourcePartition, candidate.origin)
+ incrementNestedCount(partitionRegistrableDomains, sourcePartition, candidate.registrableDomain)
+ }
+ }
+ originCount = origins.size
+ registrableDomainCount = registrableDomains.size
+ originCandidateCounts = [...origins.values()]
+ registrableDomainCandidateCounts = [...registrableDomains.values()]
+ for (const [partition, partitionOriginCounts] of partitionOrigins) {
+ ImageFetchConsumerMetrics.observePartitionBatchDiversity(
+ partition,
+ [...partitionOriginCounts.values()],
+ [...(partitionRegistrableDomains.get(partition)?.values() ?? [])]
+ )
+ }
+
+ if (this.options.dryRun || candidates.length === 0) {
+ stage.move('batch_dead_letter')
+ await this.parkRejectedRecords(rejectedRecords, drops)
+ this.countSkippedJobs(skips)
+ return
+ }
+
+ const keys = [
+ ...candidates.map((candidate) => candidate.originalRef),
+ ...[...origins.keys()].flatMap((origin) => [
+ configurationCacheKey(origin, 'robots'),
+ configurationCacheKey(origin, 'tdmrep'),
+ ]),
+ ]
+ stage.move('batch_history_read')
+ const stored = await this.runCrawlHistoryOperation('read', keys.length, () => this.crawlHistory.read(keys))
+
+ stage.move('batch_filter')
+ const fetchable: FetchCandidate[] = []
+ const notReady: FetchCandidate[] = []
+ const refreshDue: { candidate: FetchCandidate; history: UrlCrawlHistoryItem }[] = []
+ for (const candidate of candidates) {
+ const history = stored.get(candidate.originalRef)
+ if (history?.kind === 'url' && history.nextFetchAtMs > nowMs) {
+ if (this.isRefreshDue(history, nowMs)) {
+ refreshDue.push({ candidate, history })
+ continue
+ }
+ ImageFetchConsumerMetrics.incDeduped('store', 1)
+ for (const sourcePartition of candidate.sourcePartitions ?? []) {
+ ImageFetchConsumerMetrics.incPartitionUrls(sourcePartition, 'store_deduped', 1)
+ }
+ continue
+ }
+ if (candidate.notBeforeMs > nowMs) {
+ notReady.push(candidate)
+ for (const sourcePartition of candidate.sourcePartitions ?? []) {
+ ImageFetchConsumerMetrics.incPartitionUrls(sourcePartition, 'not_ready', 1)
+ }
+ } else {
+ fetchable.push(candidate)
+ for (const sourcePartition of candidate.sourcePartitions ?? []) {
+ ImageFetchConsumerMetrics.incPartitionUrls(sourcePartition, 'fetchable', 1)
+ }
+ }
+ }
+
+ const refreshed = await Promise.all(
+ refreshDue.map(async ({ candidate, history }): Promise<UrlCrawlHistoryItem> => {
+ if (await this.options.refreshingSink!.refreshImage(candidate)) {
+ return { ...history, storedAtMs: nowMs }
+ }
+ // The stored image is gone, for example because S3 expired it. Admission already dropped a job
+ // whose key is deleted, and a refused image has no stored time, so this one is fetched again now,
+ // with no validators: a 304 cannot bring back bytes that are no longer stored.
+ const missing: UrlCrawlHistoryItem = {
+ ...history,
+ storedAtMs: undefined,
+ nextFetchAtMs: nowMs,
+ cache: undefined,
+ }
+ stored.set(candidate.originalRef, missing)
+ if (candidate.notBeforeMs > nowMs) {
+ notReady.push(candidate)
+ } else {
+ fetchable.push(candidate)
+ }
+ return missing
+ })
+ )
+ ImageFetchConsumerMetrics.incFetchable(fetchable.length)
+
+ const republishBatch = this.publisher.createRepublishBatch(republishDeadlineAtMonotonicMs)
+ stage.move('batch_fetch')
+ const attempts = await this.runner!.run(fetchable, stored, republishBatch, admission)
+ stage.move('batch_prepare_republish')
+ attempts.push(
+ ...(await Promise.all(
+ notReady.map((candidate) => this.republishNotReady(republishBatch, candidate, nowMs))
+ ))
+ )
+ const updates: CrawlHistoryItem[] = [...refreshed]
+ for (const attempt of attempts) {
+ updates.push(...attempt.configurationUpdates)
+ if (attempt.history) {
+ updates.push(attempt.history)
+ }
+ }
+ if (updates.length > 0) {
+ stage.move('batch_history_write')
+ await this.runCrawlHistoryOperation('write', updates.length, () => this.crawlHistory.write(updates))
+ }
+ stage.move('batch_republish_flush')
+ const republishResult = await republishBatch.flush()
+ stage.move('batch_finalize')
+ for (const attempt of attempts) {
+ if (attempt.lost || (!attempt.finished && republishResult.failedUrls > 0)) {
+ continue
+ }
+ for (const sourcePartition of attempt.candidate.sourcePartitions ?? []) {
+ ImageFetchRequestMetrics.incPartitionAttempt(
+ sourcePartition,
+ attempt.finished ? 'completed' : 'republished',
+ attempt.outcome
+ )
+ }
+ this.topHogMetrics?.recordAttempt(attempt)
+ if (!attempt.finished && isTransientOutcome(attempt.outcome)) {
+ ImageFetchRequestMetrics.incRetryCause(attempt.outcome)
+ }
+ }
+ const lost = attempts.filter((attempt) => attempt.lost).length + republishResult.failedUrls
+ if (lost > 0) {
+ throw new Error(`the image fetch lane could not account for ${lost} URLs`)
+ }
+ stage.move('batch_dead_letter')
+ await this.parkRejectedRecords(rejectedRecords, drops)
+ this.countSkippedJobs(skips)
+ } finally {
+ stage.finish()
+ ImageFetchConsumerMetrics.finishBatch(activeBatchId)
+ this.recordMetrics(
+ drops,
+ dedupedInBatch,
+ originCount,
+ registrableDomainCount,
+ originCandidateCounts,
+ registrableDomainCandidateCounts,
+ startedAt
+ )
+ }
+ }
+
+ private parse(message: Message): ReturnType<typeof parseCollectedUrlsRecord> {
+ try {
+ return parseCollectedUrlsRecord(message.value, message.key?.toString() ?? null)
+ } catch (error) {
+ logger.error('🌐', 'ai_research_v4_image_fetch_record_parse_threw', {
+ error: error instanceof Error ? error.name : 'unknown',
+ })
+ return { ok: false, reason: 'malformed' }
+ }
+ }
+
+ /** Runs after the batch's durable work, so a batch that fails and is redelivered counts its skips once. */
+ private countSkippedJobs(skips: Map<UrlSkipReason, number>): void {
+ for (const [reason, count] of skips) {
+ ImageFetchConsumerMetrics.incSkipped(reason, count)
+ }
+ }
+
+ private async parkRejectedRecords(
+ records: RejectedFrontierRecord[],
+ drops: Map<UrlDropReason, number>
+ ): Promise<void> {
+ const deadlineAtMonotonicMs = performance.now() + DEAD_LETTER_BATCH_BUDGET_MS
+ const limit = pLimit(DEAD_LETTER_PUBLISH_CONCURRENCY)
+ let firstFailure: unknown
+ const outcomes = await Promise.allSettled(
+ records.map(({ message, reasons }) =>
+ limit(async () => {
+ if (firstFailure) {
+ throw firstFailure
+ }
+ try {
+ await this.parkRejectedRecord(message, reasons, deadlineAtMonotonicMs)
+ } catch (error) {
+ firstFailure = error
+ throw error
+ }
+ })
+ )
+ )
+ for (const outcome of outcomes) {
+ if (outcome.status === 'rejected') {
+ throw outcome.reason
+ }
+ }
+ for (const { reasons } of records) {
+ for (const reason of reasons) {
+ drops.set(reason, (drops.get(reason) ?? 0) + 1)
+ }
+ }
+ }
+
+ private async parkRejectedRecord(
+ message: Message,
+ reasons: UrlDropReason[],
+ deadlineAtMonotonicMs: number
+ ): Promise<void> {
+ if (!this.deadLetters) {
+ return
+ }
+ const reason = this.deadLetterReason(reasons)
+ const remainingMs = deadlineAtMonotonicMs - performance.now()
+ if (remainingMs <= 0) {
+ const error = new Error('image-fetch dead-letter batch exceeded its publish budget')
+ ImageFetchConsumerMetrics.incDeadLetterFailed(reason)
+ logger.error('🌐', 'ai_research_v4_image_fetch_dead_letter_publish_failed', { reason, error: error.name })
+ throw error
+ }
+ let timeout: NodeJS.Timeout | undefined
+ try {
+ await Promise.race([
+ this.deadLetters.park(message, reason),
+ new Promise<never>((_, reject) => {
+ timeout = setTimeout(
+ () => reject(new Error('image-fetch dead-letter batch exceeded its publish budget')),
+ remainingMs
+ )
+ }),
+ ])
+ } catch (error) {
+ ImageFetchConsumerMetrics.incDeadLetterFailed(reason)
+ logger.error('🌐', 'ai_research_v4_image_fetch_dead_letter_publish_failed', {
+ reason,
+ error: error instanceof Error ? error.name : 'unknown',
+ })
+ throw error
+ } finally {
+ if (timeout) {
+ clearTimeout(timeout)
+ }
+ }
+ ImageFetchConsumerMetrics.incDeadLettered(reason)
+ }
+
+ private deadLetterReason(reasons: UrlDropReason[]): FrontierDeadLetterReason {
+ const distinctReasons = new Set(reasons)
+ return distinctReasons.size === 1 ? reasons[0] : 'multiple'
+ }
+
+ private async runCrawlHistoryOperation<T>(
+ operation: 'read' | 'write',
+ affectedKeys: number,
+ run: () => Promise<T>
+ ): Promise<T> {
+ const startedAt = process.hrtime.bigint()
+ let outcome: 'success' | 'error' = 'success'
+ try {
+ return await run()
+ } catch (error) {
+ outcome = 'error'
+ ImageFetchConsumerMetrics.incStoreError(operation, affectedKeys)
+ throw error
+ } finally {
+ ImageFetchConsumerMetrics.observeStoreDuration(
+ operation,
+ outcome,
+ Number(process.hrtime.bigint() - startedAt) / 1e9
+ )
+ }
+ }
+
+ private isRefreshDue(history: UrlCrawlHistoryItem, nowMs: number): boolean {
+ const sink = this.options.refreshingSink
+ return !!sink && history.storedAtMs !== undefined && nowMs - history.storedAtMs >= sink.refreshIntervalMs
+ }
+
+ private async republishNotReady(
+ republishBatch: RepublishBatch,
+ candidate: FetchCandidate,
+ nowMs: number
+ ): Promise<FetchAttempt> {
+ if (candidate.remainingHops === 0) {
+ return this.terminalAttempt(candidate, HOPS_EXHAUSTED, nowMs)
+ }
+ const waitMs = candidate.notBeforeMs - nowMs
+ const blockingReason = candidate.lastBlockReason ?? 'unknown_backoff'
+ const republishedCandidate: FetchCandidate = { ...candidate, lastBlockReason: blockingReason }
+ const result = await republishBatch.republish(
+ republishedCandidate,
+ {
+ currentUrl: candidate.currentUrl,
+ host: candidate.host,
+ origin: candidate.origin,
+ registrableDomain: candidate.registrableDomain,
+ },
+ 'not_ready',
+ waitMs
+ )
+ if (result === 'refused_delay' || waitMs > ONE_HOUR_MS) {
+ return this.terminalAttempt(candidate, DELAY_TOO_LONG, nowMs)
+ }
+ return {
+ candidate: republishedCandidate,
+ outcome: 'backoff',
+ finished: false,
+ lost: false,
+ block: { reason: blockingReason, waitMs },
+ configurationUpdates: [],
+ }
+ }
+
+ private terminalAttempt(candidate: FetchCandidate, outcome: AttemptOutcome, nowMs: number): FetchAttempt {
+ ImageFetchRequestMetrics.observeCompletedUrl(
+ outcome,
+ 'none',
+ Math.max(0, nowMs - candidate.firstSeenAtMs) / 1000,
+ candidate.fetchCount,
+ candidate.republishCount
+ )
+ ImageFetchRequestMetrics.observeHops(MAX_HOPS - candidate.remainingHops)
+ return {
+ candidate,
+ outcome,
+ finished: true,
+ lost: false,
+ history: this.terminalHistory(candidate, outcome, nowMs),
+ configurationUpdates: [],
+ }
+ }
+
+ private terminalHistory(candidate: FetchCandidate, outcome: AttemptOutcome, nowMs: number): UrlCrawlHistoryItem {
+ const nextFetchAtMs = urlHistoryExpiresAtMs(nowMs, this.options.seenTtlSeconds)
+ return {
+ kind: 'url',
+ key: candidate.originalRef,
+ nextFetchAtMs,
+ storageExpiresAtMs: urlHistoryStorageExpiresAtMs(candidate.originalRef, nextFetchAtMs),
+ outcome,
+ }
+ }
+
+ private recordMetrics(
+ drops: Map<UrlDropReason, number>,
+ dedupedInBatch: number,
+ origins: number,
+ registrableDomains: number,
+ originCandidateCounts: number[],
+ registrableDomainCandidateCounts: number[],
+ startedAt: bigint
+ ): void {
+ if (dedupedInBatch > 0) {
+ ImageFetchConsumerMetrics.incDeduped('batch', dedupedInBatch)
+ }
+ for (const [reason, count] of drops) {
+ ImageFetchConsumerMetrics.incDropped(reason, count)
+ }
+ ImageFetchConsumerMetrics.observeBatchDiversity(originCandidateCounts, registrableDomainCandidateCounts)
+ ImageFetchConsumerMetrics.observeBatch(
+ origins,
+ registrableDomains,
+ Number(process.hrtime.bigint() - startedAt) / 1e9
+ )
+ }
+}
+
+function incrementNestedCount(counts: Map<number, Map<string, number>>, partition: number, key: string): void {
+ let partitionCounts = counts.get(partition)
+ if (!partitionCounts) {
+ partitionCounts = new Map<string, number>()
+ counts.set(partition, partitionCounts)
+ }
+ partitionCounts.set(key, (partitionCounts.get(key) ?? 0) + 1)
+}
diff --git a/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/url-history-expiry.ts b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/url-history-expiry.ts
index a882606482f9..ab3a6b3d186c 100644
--- a/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/url-history-expiry.ts
+++ b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/url-history-expiry.ts
@@ -1,5 +1,4 @@
-import { explicitFreshnessLifetimeMs } from '~/ingestion/pipelines/sessionreplay/ml-mirror-image-fetch/configuration-policy'
-
+import { explicitFreshnessLifetimeMs } from './configuration-policy'
import { HttpCacheMetadata } from './crawl-history'
import { parseV4UrlRef } from './v4-url-ref'
diff --git a/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/v4-lane.test.ts b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/v4-lane.test.ts
new file mode 100644
index 000000000000..bac29f17da60
--- /dev/null
+++ b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/v4-lane.test.ts
@@ -0,0 +1,530 @@
+import { GetItemCommand } from '@aws-sdk/client-dynamodb'
+import { CopyObjectCommand, PutObjectCommand, S3ServiceException } from '@aws-sdk/client-s3'
+import { Message } from 'node-rdkafka'
+import { createDecipheriv, generateKeyPairSync } from 'node:crypto'
+
+import { KAFKA_AI_RESEARCH_V4_IMAGE_FETCH_RETRY_1M } from '~/common/config/kafka-topics'
+import { KafkaProducer } from '~/common/kafka/producer'
+import { parseJSON } from '~/common/utils/json-parse'
+import { StreamedResponse, fetchStreamed } from '~/common/utils/request'
+import { buildAiResearchV4ServerConfig } from '~/ingestion/pipelines/sessionreplay/ai-research/config'
+import vector from '~/ingestion/pipelines/sessionreplay/ai-research/v4-keys/encryption-vector-v4.json'
+import { TeamMonthKeys } from '~/ingestion/pipelines/sessionreplay/ai-research/v4-keys/team-month-keys'
+import { buildFetchRunner, buildFrontierPublisher } from '~/servers/ai-research-v4-image-fetch-server'
+import { createRecordingTopHog } from '~/tests/helpers/tophog'
+
+import type { FetchCandidate } from './collected-urls-record'
+import { CrawlHistoryItem, CrawlHistoryStore, UrlCrawlHistoryItem, configurationCacheKey } from './crawl-history'
+import { FetchRunner } from './fetch-runner'
+import { V4JobAdmission } from './job-admission'
+import { RawImageStore } from './raw-image-store'
+import { ImageFetchTopHogMetrics } from './tophog-metrics'
+import { UrlFetchConsumer } from './url-fetch-consumer'
+
+jest.mock('~/common/utils/request', () => ({
+ ...jest.requireActual('~/common/utils/request'),
+ fetchStreamed: jest.fn(),
+}))
+
+const fetchStreamedMock = fetchStreamed as jest.MockedFunction<typeof fetchStreamed>
+
+const NOW_MS = Date.parse('2026-10-15T12:00:00Z')
+const PNG = Buffer.concat([Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]), Buffer.from('png body')])
+const TEAM_KEYS: Record<string, Buffer> = {
+ 'team:7': Buffer.from(vector.teamMonthKey, 'base64'),
+ 'team:8': Buffer.alloc(32, 8),
+}
+const OTHER_HASH = 'B'.repeat(22)
+const DAY_MS = 24 * 60 * 60 * 1000
+const IMAGE_OBJECT_KEY = `images/url/month=${vector.identity.keyMonth}/7/${vector.url.ref.split(':').pop()}`
+
+function respond(status: number, headers: Record<string, string> = {}, bytes = Buffer.alloc(0)): StreamedResponse {
+ return {
+ status,
+ headers,
+ headerLines: Object.entries(headers).map(([name, value]) => ({ name, value })),
+ read: jest.fn().mockResolvedValue({ bytes, overLimit: false }),
+ discard: jest.fn(),
+ }
+}
+
+/** A record in the shape `replay-recorder-v4/src/urls.rs` produces: keyed by registrable domain, with no headers. */
+function recorderRecord(jobs: { ref: string; url: string }[]): Message {
+ const value = Buffer.from(
+ JSON.stringify({
+ v: 2,
+ jobs: jobs.map(({ ref, url }) => ({
+ sessionId: vector.identity.sessionId,
+ originalRef: ref,
+ currentUrl: url,
+ remainingHops: 10,
+ notBeforeMs: 0,
+ firstSeenAtMs: NOW_MS,
+ fetchCount: 0,
+ republishCount: 0,
+ lastRepublishReason: null,
+ })),
+ })
+ )
+ return { value, key: Buffer.from('example.com'), size: value.length, topic: 'v4', partition: 0, offset: 0 }
+}
+
+function openFrame(key: Buffer, frame: Buffer): { context: Record<string, unknown>; body: Buffer } {
+ const length = frame.readUInt16BE(6)
+ const aad = frame.subarray(8, 8 + length)
+ const nonce = frame.subarray(8 + length, 20 + length)
+ const sealed = frame.subarray(20 + length)
+ const decipher = createDecipheriv('aes-256-gcm', key, nonce, { authTagLength: 16 })
+ decipher.setAAD(aad)
+ decipher.setAuthTag(sealed.subarray(sealed.length - 16))
+ const body = Buffer.concat([decipher.update(sealed.subarray(0, sealed.length - 16)), decipher.final()])
+ return { context: parseJSON(aad.toString()).context, body }
+}
+
+/** The origin's robots.txt and tdmrep.json are known for a year, so the jobs do not wait on one configuration fetch. */
+class MemoryCrawlHistory implements CrawlHistoryStore {
+ public readonly items = new Map<string, CrawlHistoryItem>(
+ (['robots', 'tdmrep'] as const).map((kind) => {
+ const origin = new URL(vector.url.url).origin
+ const nowMs = Date.now()
+ const item: CrawlHistoryItem = {
+ kind,
+ key: configurationCacheKey(origin, kind),
+ origin,
+ status: 'absent',
+ fetchedAtMs: nowMs,
+ refreshAtMs: nowMs + 365 * DAY_MS,
+ freshUntilMs: nowMs + 365 * DAY_MS,
+ retryAtMs: nowMs,
+ storageExpiresAtMs: nowMs + 365 * DAY_MS,
+ }
+ return [item.key, item]
+ })
+ )
+ read(keys: string[]): Promise<Map<string, CrawlHistoryItem>> {
+ return Promise.resolve(
+ new Map(keys.flatMap((key) => (this.items.has(key) ? [[key, this.items.get(key)!]] : [])))
+ )
+ }
+ write(items: CrawlHistoryItem[]): Promise<void> {
+ items.forEach((item) => this.items.set(item.key, item))
+ return Promise.resolve()
+ }
+}
+
+/** `keyReadsHang` makes every key row read wait until its caller gives up on it. */
+function lane(rows: Record<string, Record<string, any> | undefined>, keyReadsHang = false, keyBudgetMs?: number) {
+ const config = buildAiResearchV4ServerConfig({
+ AI_RESEARCH_V4_IMAGE_FETCH_WEB_BOT_AUTH_PRIVATE_KEYS: generateKeyPairSync('ed25519')
+ .privateKey.export({ type: 'pkcs8', format: 'pem' })
+ .toString(),
+ })
+ const producer = { produce: jest.fn(() => Promise.resolve()) } as unknown as KafkaProducer
+ const publisher = buildFrontierPublisher(producer, 10)
+ const dynamo = {
+ send: jest.fn((command: GetItemCommand, options?: any) => {
+ if (keyReadsHang) {
+ return new Promise<never>((_resolve, reject) =>
+ options?.abortSignal?.addEventListener('abort', () => reject(options.abortSignal.reason))
+ )
+ }
+ const id = `${command.input.Key!.pk.S}/${command.input.Key!.sk.S}`
+ return Promise.resolve({ Item: rows[id] })
+ }),
+ }
+ const kms = {
+ send: jest.fn((command: any) =>
+ Promise.resolve({ Plaintext: TEAM_KEYS[`team:${command.input.EncryptionContext.team_id}`] })
+ ),
+ }
+ const keys = new TeamMonthKeys(dynamo, kms, 'keys', 'arn')
+ const puts: PutObjectCommand[] = []
+ const copies: CopyObjectCommand[] = []
+ const s3 = {
+ send: jest.fn((command: PutObjectCommand | CopyObjectCommand) => {
+ if (command instanceof CopyObjectCommand) {
+ copies.push(command)
+ } else {
+ puts.push(command)
+ }
+ return Promise.resolve({})
+ }),
+ }
+ const store = new RawImageStore(keys, s3 as any, 'images-v4', 10)
+ const topHog = new ImageFetchTopHogMetrics(createRecordingTopHog().registry)
+ const runner = buildFetchRunner(config, publisher, topHog, store, keys)
+ const admission = new V4JobAdmission(keys, keyBudgetMs)
+ const deadLetters = { park: jest.fn(() => Promise.resolve()) }
+ const history = new MemoryCrawlHistory()
+ const consumer = new UrlFetchConsumer(
+ history,
+ publisher,
+ {
+ seenTtlSeconds: config.AI_RESEARCH_V4_IMAGE_FETCH_CRAWL_HISTORY_TTL_SECONDS,
+ dryRun: false,
+ admitCandidates: (candidates, nowMs) => admission.admit(candidates, nowMs),
+ refreshingSink: store,
+ },
+ runner,
+ deadLetters,
+ topHog
+ )
+ return { consumer, runner, puts, copies, history, deadLetters, producer, dynamo, s3 }
+}
+
+const row = (team: number, month = vector.identity.keyMonth, extra: Record<string, any> = {}) => ({
+ [`team:${team}/month:${month}`]: {
+ pk: { S: `team:${team}` },
+ sk: { S: `month:${month}` },
+ wrapped_key: { B: Buffer.from('w') },
+ ...extra,
+ },
+})
+
+describe('v4 image fetch lane', () => {
+ const runners: FetchRunner[] = []
+
+ beforeEach(() => {
+ fetchStreamedMock.mockReset()
+ fetchStreamedMock.mockImplementation((url) =>
+ Promise.resolve(
+ new URL(url).pathname.endsWith('.png')
+ ? respond(200, { 'content-type': 'image/png' }, PNG)
+ : respond(404)
+ )
+ )
+ })
+
+ afterEach(async () => {
+ await Promise.all(runners.splice(0).map((runner) => runner.close()))
+ })
+
+ it("stores a recorder record's image under each team's month key, with one request for a URL two teams share", async () => {
+ const harness = lane({ ...row(7), ...row(8) })
+ runners.push(harness.runner)
+ const otherTeamRef = `imageurl:v4:8:${vector.identity.keyMonth}:${OTHER_HASH}`
+
+ await harness.consumer.handleBatch(
+ [
+ recorderRecord([
+ { ref: vector.url.ref, url: vector.url.url },
+ { ref: otherTeamRef, url: vector.url.url },
+ ]),
+ ],
+ NOW_MS
+ )
+
+ expect(harness.deadLetters.park).not.toHaveBeenCalled()
+ expect(fetchStreamedMock.mock.calls.filter(([url]) => url === vector.url.url)).toHaveLength(1)
+ const stored = new Map(harness.puts.map((put) => [put.input.Key, put.input.Body as Buffer]))
+ expect([...stored.keys()].sort()).toEqual([
+ IMAGE_OBJECT_KEY,
+ `images/url/month=${vector.identity.keyMonth}/8/${OTHER_HASH}`,
+ ])
+ for (const [team, ref] of [
+ [7, vector.url.ref],
+ [8, otherTeamRef],
+ ] as const) {
+ const objectKey = `images/url/month=${vector.identity.keyMonth}/${team}/${ref.split(':').pop()}`
+ const { context, body } = openFrame(TEAM_KEYS[`team:${team}`], stored.get(objectKey)!)
+ expect(body).toEqual(PNG)
+ expect(context).toEqual({
+ teamId: team,
+ keyMonth: vector.identity.keyMonth,
+ kind: 'image-url',
+ codec: 'none',
+ ref,
+ })
+ }
+ // Each team binds the shared response under its own month key, as the reference hash does, and stores no URL.
+ const urlHistory = [vector.url.ref, otherTeamRef].map(
+ (ref) => harness.history.items.get(ref) as UrlCrawlHistoryItem
+ )
+ expect(urlHistory.map((item) => item.cache?.responseUrlHmac)).toEqual([
+ vector.url.ref.split(':').pop(),
+ expect.not.stringMatching(vector.url.ref.split(':').pop()!),
+ ])
+ expect(JSON.stringify(urlHistory)).not.toContain(new URL(vector.url.url).host)
+ })
+
+ it.each([
+ // The row of 2026-05 expired on 2026-10-01, and DynamoDB may not have removed it yet.
+ ['a month whose key expired', row(7, '2026-05'), `imageurl:v4:7:2026-05:${OTHER_HASH}`],
+ [
+ 'a tombstoned team month',
+ row(7, vector.identity.keyMonth, { deleted: { BOOL: true }, wrapped_key: undefined }),
+ vector.url.ref,
+ ],
+ ['a team month without a key row', {}, vector.url.ref],
+ ])('drops a job for %s before any request', async (_name, rows, ref) => {
+ const harness = lane(rows)
+ runners.push(harness.runner)
+
+ await harness.consumer.handleBatch([recorderRecord([{ ref, url: vector.url.url }])], NOW_MS)
+
+ expect(fetchStreamedMock).not.toHaveBeenCalled()
+ expect(harness.puts).toHaveLength(0)
+ expect(harness.deadLetters.park).not.toHaveBeenCalled()
+ })
+
+ it('sends a job whose key was not read within the batch key budget to the first delay tier, with no request', async () => {
+ const harness = lane(row(7), true, 50)
+ runners.push(harness.runner)
+
+ await harness.consumer.handleBatch([recorderRecord([{ ref: vector.url.ref, url: vector.url.url }])], NOW_MS)
+
+ expect(fetchStreamedMock).not.toHaveBeenCalled()
+ expect(harness.deadLetters.park).not.toHaveBeenCalled()
+ const produced = (harness.producer.produce as jest.Mock).mock.calls.map(([record]) => record.topic)
+ expect(produced).toEqual([KAFKA_AI_RESEARCH_V4_IMAGE_FETCH_RETRY_1M])
+ // The abandoned read was stopped, not left to run for its own budget.
+ const [, readOptions] = harness.dynamo.send.mock.calls[0] as unknown as [unknown, { abortSignal: AbortSignal }]
+ expect(readOptions.abortSignal.aborted).toBe(true)
+ })
+
+ describe('with a reference seen again later in its month', () => {
+ const imageRequests = (): number =>
+ fetchStreamedMock.mock.calls.filter(([url]) => url === vector.url.url).length
+ const urlHistory = (harness: ReturnType<typeof lane>): UrlCrawlHistoryItem =>
+ harness.history.items.get(vector.url.ref) as UrlCrawlHistoryItem
+ const seeAt = async (harness: ReturnType<typeof lane>, nowMs: number): Promise<void> => {
+ jest.setSystemTime(nowMs)
+ await harness.consumer.handleBatch([recorderRecord([{ ref: vector.url.ref, url: vector.url.url }])], nowMs)
+ }
+
+ let harness: ReturnType<typeof lane>
+
+ beforeEach(async () => {
+ // Only the clock is fake: the fetch pass still runs on real timers.
+ jest.useFakeTimers({
+ now: NOW_MS,
+ doNotFake: [
+ 'nextTick',
+ 'queueMicrotask',
+ 'setImmediate',
+ 'setTimeout',
+ 'clearTimeout',
+ 'setInterval',
+ 'clearInterval',
+ 'performance',
+ 'hrtime',
+ ],
+ })
+ fetchStreamedMock.mockImplementation((url, options) =>
+ Promise.resolve(
+ url !== vector.url.url
+ ? respond(404)
+ : (options?.headers as Record<string, string> | undefined)?.['if-none-match'] === '"v1"'
+ ? respond(304, { etag: '"v1"' })
+ : respond(200, { 'content-type': 'image/png', etag: '"v1"' }, PNG)
+ )
+ )
+ harness = lane(row(7))
+ runners.push(harness.runner)
+ await seeAt(harness, NOW_MS)
+ expect([imageRequests(), harness.puts.length]).toEqual([1, 1])
+ })
+
+ afterEach(() => {
+ jest.useRealTimers()
+ })
+
+ it('keeps its history until 8 days after the end of its month', () => {
+ expect(urlHistory(harness)).toMatchObject({
+ storedAtMs: NOW_MS,
+ storageExpiresAtMs: Date.parse('2026-11-09T00:00:00Z'),
+ })
+ })
+
+ it('drops a job inside the refresh interval with no request and no write', async () => {
+ await seeAt(harness, NOW_MS + 6 * DAY_MS)
+
+ expect([imageRequests(), harness.puts.length, harness.copies.length]).toEqual([1, 1, 0])
+ })
+
+ it('refreshes the stored image by a copy onto itself after 7 days, with no origin request', async () => {
+ await seeAt(harness, NOW_MS + 7 * DAY_MS)
+
+ expect([imageRequests(), harness.puts.length]).toEqual([1, 1])
+ expect(harness.copies.map((copy) => copy.input.Key)).toEqual([IMAGE_OBJECT_KEY])
+ expect(urlHistory(harness).storedAtMs).toBe(NOW_MS + 7 * DAY_MS)
+
+ await seeAt(harness, NOW_MS + 8 * DAY_MS)
+ expect(harness.copies).toHaveLength(1)
+ })
+
+ it('fetches the whole image again in the same batch when its refresh copy finds it gone', async () => {
+ harness.s3.send.mockImplementation((command: PutObjectCommand | CopyObjectCommand) => {
+ if (command instanceof CopyObjectCommand) {
+ harness.copies.push(command)
+ return Promise.reject(
+ new S3ServiceException({
+ name: 'NoSuchKey',
+ $fault: 'client',
+ $metadata: { httpStatusCode: 404 },
+ })
+ )
+ }
+ harness.puts.push(command)
+ return Promise.resolve({})
+ })
+
+ await seeAt(harness, NOW_MS + 7 * DAY_MS)
+
+ const requests = fetchStreamedMock.mock.calls.filter(([url]) => url === vector.url.url)
+ expect(
+ requests.map(([, options]) => (options?.headers as Record<string, string>)['if-none-match'])
+ ).toEqual([undefined, undefined])
+ expect([harness.copies.length, harness.puts.length]).toEqual([1, 2])
+ expect(urlHistory(harness).storedAtMs).toBe(NOW_MS + 7 * DAY_MS)
+ })
+
+ it.each([
+ ['refreshes the image by a copy when the origin answers 304', 304, 0, 1],
+ ['overwrites the image when the origin answers 200', 200, 1, 0],
+ ])('revalidates after the refetch interval and %s', async (_name, status, puts, copies) => {
+ if (status === 200) {
+ fetchStreamedMock.mockImplementation(() =>
+ Promise.resolve(respond(200, { 'content-type': 'image/png', etag: '"v2"' }, PNG))
+ )
+ }
+ const refetchAtMs = urlHistory(harness).nextFetchAtMs
+ expect(refetchAtMs).toBe(NOW_MS + 30 * DAY_MS)
+ // A fresh lane reads the history, so no fetch the first lane shared in memory serves this job.
+ const later = lane(row(7))
+ runners.push(later.runner)
+ later.history.items.set(vector.url.ref, urlHistory(harness))
+
+ await seeAt(later, refetchAtMs)
+
+ const requests = fetchStreamedMock.mock.calls.filter(([url]) => url === vector.url.url)
+ expect(requests).toHaveLength(2)
+ expect((requests[1][1]?.headers as Record<string, string>)['if-none-match']).toBe('"v1"')
+ expect([later.puts.length, later.copies.length]).toEqual([puts, copies])
+ expect([...later.puts, ...later.copies].map((write) => write.input.Key)).toEqual([IMAGE_OBJECT_KEY])
+ expect(urlHistory(later)).toMatchObject({
+ storedAtMs: refetchAtMs,
+ nextFetchAtMs: refetchAtMs + 30 * DAY_MS,
+ })
+ })
+
+ it.each([
+ ['a 404', respond(404)],
+ ['a page that is not an image', respond(200, { 'content-type': 'text/html' }, Buffer.from('<html>'))],
+ ])('keeps refreshing the stored image after a refetch finds %s', async (_name, response) => {
+ fetchStreamedMock.mockImplementation(() => Promise.resolve(response))
+ const refetchAtMs = urlHistory(harness).nextFetchAtMs
+ const later = lane(row(7))
+ runners.push(later.runner)
+ later.history.items.set(vector.url.ref, urlHistory(harness))
+
+ await seeAt(later, refetchAtMs)
+ expect(urlHistory(later).storedAtMs).toBe(NOW_MS)
+ await seeAt(later, refetchAtMs + DAY_MS)
+
+ expect(later.puts).toHaveLength(0)
+ expect(later.copies.map((copy) => copy.input.Key)).toEqual([IMAGE_OBJECT_KEY])
+ })
+
+ it('fetches the whole image again when the origin answers 304 for an image S3 no longer holds', async () => {
+ const refetchAtMs = urlHistory(harness).nextFetchAtMs
+ const later = lane(row(7))
+ runners.push(later.runner)
+ later.history.items.set(vector.url.ref, urlHistory(harness))
+ later.s3.send.mockImplementation((command: PutObjectCommand | CopyObjectCommand) => {
+ if (command instanceof CopyObjectCommand) {
+ return Promise.reject(
+ new S3ServiceException({
+ name: 'NoSuchKey',
+ $fault: 'client',
+ $metadata: { httpStatusCode: 404 },
+ })
+ )
+ }
+ later.puts.push(command)
+ return Promise.resolve({})
+ })
+
+ await seeAt(later, refetchAtMs)
+ const [retry] = (later.producer.produce as jest.Mock).mock.calls.map(([record]) => record)
+ expect(retry.topic).toBe(KAFKA_AI_RESEARCH_V4_IMAGE_FETCH_RETRY_1M)
+ const retryAtMs = refetchAtMs + 60_000
+ jest.setSystemTime(retryAtMs)
+ await later.consumer.handleBatch(
+ [
+ {
+ value: retry.value,
+ key: retry.key,
+ size: retry.value.length,
+ topic: 'v4',
+ partition: 0,
+ offset: 1,
+ },
+ ],
+ retryAtMs
+ )
+
+ const requests = fetchStreamedMock.mock.calls.filter(([url]) => url === vector.url.url)
+ expect(
+ requests.map(([, options]) => (options?.headers as Record<string, string>)['if-none-match'])
+ ).toEqual([undefined, '"v1"', undefined])
+ expect(later.puts.map((put) => put.input.Key)).toEqual([IMAGE_OBJECT_KEY])
+ expect(urlHistory(later).storedAtMs).toBe(retryAtMs)
+ })
+ })
+
+ describe('with many slow key lookups', () => {
+ beforeEach(() => {
+ jest.useFakeTimers({ now: NOW_MS })
+ })
+
+ afterEach(() => {
+ jest.useRealTimers()
+ })
+
+ it('finishes admission within the batch key budget, runs lookups bounded, and defers the jobs it did not reach', async () => {
+ const LOOKUP_MS = 16_000
+ let running = 0
+ let mostRunning = 0
+ let started = 0
+ const keys = {
+ get: async () => {
+ started++
+ running++
+ mostRunning = Math.max(mostRunning, running)
+ await new Promise((resolve) => setTimeout(resolve, LOOKUP_MS))
+ running--
+ return { status: 'ok' as const, key: Buffer.alloc(32) }
+ },
+ }
+ const candidates = Array.from(
+ { length: 256 },
+ (_, team) =>
+ ({
+ originalRef: `imageurl:v4:${team + 1}:${vector.identity.keyMonth}:${OTHER_HASH}`,
+ notBeforeMs: 0,
+ }) as FetchCandidate
+ )
+ let admitted: FetchCandidate[] | undefined
+ void new V4JobAdmission(keys, 60_000).admit(candidates, NOW_MS).then((result) => {
+ admitted = result
+ })
+
+ await jest.advanceTimersByTimeAsync(60_000)
+
+ expect(admitted).toHaveLength(256)
+ expect(mostRunning).toBeLessThanOrEqual(16)
+ // Lookups start at 0s, 16s, 32s and 48s, and only the first three rounds finish within 60s.
+ const ready = admitted!.filter((candidate) => candidate.notBeforeMs === 0)
+ expect(ready).toHaveLength(48)
+ const deferred = admitted!.filter((candidate) => candidate.notBeforeMs > 0)
+ expect(deferred).toHaveLength(208)
+ expect(deferred.every((candidate) => candidate.notBeforeMs === NOW_MS + 60_000)).toBe(true)
+ expect(deferred.every((candidate) => candidate.lastBlockReason === 'pass_deadline')).toBe(true)
+
+ // A lookup still queued at the deadline never starts.
+ await jest.advanceTimersByTimeAsync(60_000)
+ expect(started).toBe(64)
+ })
+ })
+})
diff --git a/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/web-bot-auth.test.ts b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/web-bot-auth.test.ts
new file mode 100644
index 000000000000..e9d212950ce2
--- /dev/null
+++ b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/web-bot-auth.test.ts
@@ -0,0 +1,129 @@
+import { KeyObject, createHash, generateKeyPairSync, verify as verifySignature } from 'node:crypto'
+
+import { createWebBotAuthRequestSigner } from './web-bot-auth'
+
+const SIGNATURE_AGENT = '"https://us.posthog.com/.well-known/http-message-signatures-directory"'
+
+type TestKeyPair = {
+ privateKeyPem: string
+ publicKey: KeyObject
+}
+
+function generateEd25519KeyPair(): TestKeyPair {
+ const { privateKey, publicKey } = generateKeyPairSync('ed25519')
+ return {
+ privateKeyPem: privateKey.export({ format: 'pem', type: 'pkcs8' }).toString(),
+ publicKey,
+ }
+}
+
+function keyId(publicKey: KeyObject): string {
+ const jwk = publicKey.export({ format: 'jwk' })
+ const canonicalJwk = JSON.stringify({ crv: jwk.crv, kty: jwk.kty, x: jwk.x })
+ return createHash('sha256').update(canonicalJwk).digest('base64url')
+}
+
+function expectSignedHeadersToVerify(
+ method: string,
+ url: string,
+ headers: Record<string, string>,
+ publicKey: KeyObject,
+ label: string,
+ expectedResult = true
+): void {
+ const signatureInput = headers['signature-input'].split(', ').find((member) => member.startsWith(`${label}=`))
+ const signature = headers.signature.split(', ').find((member) => member.startsWith(`${label}=`))
+ if (!signatureInput || !signature) {
+ throw new Error(`Missing ${label}`)
+ }
+
+ const parameters = signatureInput.slice(label.length + 1)
+ const parametersMatch = parameters.match(
+ /^\("@method" "@authority" "@target-uri" "signature-agent"\);created=(\d+);keyid="([^"]+)";alg="ed25519";expires=(\d+);nonce="([^"]+)";tag="web-bot-auth"$/
+ )
+ if (!parametersMatch) {
+ throw new Error(`Invalid ${label} parameters`)
+ }
+ const [, created, actualKeyId, expires, nonce] = parametersMatch
+ expect(actualKeyId).toBe(keyId(publicKey))
+ expect(Number(expires) - Number(created)).toBe(60)
+ expect(Buffer.from(nonce, 'base64url')).toHaveLength(64)
+
+ const targetUrl = new URL(url)
+ targetUrl.hash = ''
+ const signatureBase =
+ `"@method": ${method}\n` +
+ `"@authority": ${targetUrl.host}\n` +
+ `"@target-uri": ${targetUrl.toString()}\n` +
+ `"signature-agent": ${SIGNATURE_AGENT}\n` +
+ `"@signature-params": ${parameters}`
+ const signatureBytes = Buffer.from(signature.slice(`${label}=:`.length, -1), 'base64')
+ expect(verifySignature(null, Buffer.from(signatureBase), publicKey, signatureBytes)).toBe(expectedResult)
+}
+
+describe('Web Bot Auth request signing', () => {
+ beforeEach(() => {
+ jest.useFakeTimers().setSystemTime(new Date('2026-08-19T08:00:00Z'))
+ })
+
+ afterEach(() => {
+ jest.useRealTimers()
+ })
+
+ it('creates a Cloudflare-compatible signature with current protocol parameters', () => {
+ const keyPair = generateEd25519KeyPair()
+ const signer = createWebBotAuthRequestSigner(keyPair.privateKeyPem)
+ const url = 'https://cdn.example.com/image.png?size=large'
+
+ const headers = signer.headersForGet(url)
+
+ expect(headers['signature-agent']).toBe(SIGNATURE_AGENT)
+ expectSignedHeadersToVerify('GET', url, headers, keyPair.publicKey, 'sig1')
+ })
+
+ it.each([
+ ['a different method', 'POST', 'https://cdn.example.com/image.png?size=large'],
+ ['a different path', 'GET', 'https://cdn.example.com/other.png?size=large'],
+ ['a different query', 'GET', 'https://cdn.example.com/image.png?size=small'],
+ ])('does not verify for %s', (_name, method, replayUrl) => {
+ const keyPair = generateEd25519KeyPair()
+ const signer = createWebBotAuthRequestSigner(keyPair.privateKeyPem)
+ const headers = signer.headersForGet('https://cdn.example.com/image.png?size=large')
+
+ expectSignedHeadersToVerify(method, replayUrl, headers, keyPair.publicKey, 'sig1', false)
+ })
+
+ it('signs with the first configured key during rotation', () => {
+ const firstKeyPair = generateEd25519KeyPair()
+ const secondKeyPair = generateEd25519KeyPair()
+ const signer = createWebBotAuthRequestSigner(
+ `${firstKeyPair.privateKeyPem},${secondKeyPair.privateKeyPem.replaceAll('\n', '\\n')}`
+ )
+ const url = 'https://cdn.example.com/image.png'
+
+ const headers = signer.headersForGet(url)
+
+ expectSignedHeadersToVerify('GET', url, headers, firstKeyPair.publicKey, 'sig1')
+ expect(headers['signature-input']).not.toContain(keyId(secondKeyPair.publicKey))
+ expect(headers.signature).not.toContain('sig2=')
+ })
+
+ it.each([
+ ['an empty value', '', 'must contain at least one'],
+ ['a malformed key', 'not a PEM', 'entry 1 could not be loaded'],
+ [
+ 'a malformed non-active key',
+ `${generateEd25519KeyPair().privateKeyPem},not a PEM`,
+ 'entry 2 could not be loaded',
+ ],
+ [
+ 'a non-Ed25519 key',
+ generateKeyPairSync('rsa', { modulusLength: 2048 })
+ .privateKey.export({ format: 'pem', type: 'pkcs8' })
+ .toString(),
+ 'entry 1 is not an Ed25519 private key',
+ ],
+ ])('rejects %s', (_name, privateKeyPems, expectedError) => {
+ expect(() => createWebBotAuthRequestSigner(privateKeyPems)).toThrow(expectedError)
+ })
+})
diff --git a/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/web-bot-auth.ts b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/web-bot-auth.ts
new file mode 100644
index 000000000000..b0c75b4128ed
--- /dev/null
+++ b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/web-bot-auth.ts
@@ -0,0 +1,75 @@
+import { KeyObject, createHash, createPrivateKey, createPublicKey, randomBytes, sign } from 'node:crypto'
+
+const SIGNATURE_AGENT_URL = 'https://us.posthog.com/.well-known/http-message-signatures-directory'
+const SIGNATURE_AGENT_HEADER = JSON.stringify(SIGNATURE_AGENT_URL)
+const SIGNATURE_LIFETIME_SECONDS = 60
+const REQUEST_METHOD = 'GET'
+
+type SigningKey = {
+ privateKey: KeyObject
+ keyId: string
+}
+
+export interface WebBotAuthRequestSigner {
+ headersForGet(url: string): Record<string, string>
+}
+
+export function createWebBotAuthRequestSigner(commaSeparatedPrivateKeyPems: string): WebBotAuthRequestSigner {
+ const privateKeyPems = commaSeparatedPrivateKeyPems.split(',')
+ if (privateKeyPems.length === 1 && privateKeyPems[0].trim() === '') {
+ throw new Error('WEB_BOT_AUTH_PRIVATE_KEYS must contain at least one Ed25519 private key')
+ }
+
+ const signingKeys = privateKeyPems.map((privateKeyPem, index) => loadSigningKey(privateKeyPem, index + 1))
+ return new Ed25519WebBotAuthRequestSigner(signingKeys[0])
+}
+
+class Ed25519WebBotAuthRequestSigner implements WebBotAuthRequestSigner {
+ constructor(private readonly signingKey: SigningKey) {}
+
+ public headersForGet(url: string): Record<string, string> {
+ const targetUrl = new URL(url)
+ targetUrl.hash = ''
+ const authority = targetUrl.host
+ const targetUri = targetUrl.toString()
+ const created = Math.floor(Date.now() / 1000)
+ const expires = created + SIGNATURE_LIFETIME_SECONDS
+ const nonce = randomBytes(64).toString('base64url')
+ const parameters =
+ `("@method" "@authority" "@target-uri" "signature-agent");created=${created};keyid="${this.signingKey.keyId}";` +
+ `alg="ed25519";expires=${expires};nonce="${nonce}";tag="web-bot-auth"`
+ const signatureBase =
+ `"@method": ${REQUEST_METHOD}\n` +
+ `"@authority": ${authority}\n` +
+ `"@target-uri": ${targetUri}\n` +
+ `"signature-agent": ${SIGNATURE_AGENT_HEADER}\n` +
+ `"@signature-params": ${parameters}`
+
+ return {
+ // Cloudflare rejects the dictionary form from newer drafts, so use the compatible structured string.
+ 'signature-agent': SIGNATURE_AGENT_HEADER,
+ 'signature-input': `sig1=${parameters}`,
+ signature: `sig1=:${sign(null, Buffer.from(signatureBase), this.signingKey.privateKey).toString('base64')}:`,
+ }
+ }
+}
+
+function loadSigningKey(privateKeyPem: string, keyIndex: number): SigningKey {
+ let privateKey: KeyObject
+ try {
+ privateKey = createPrivateKey(privateKeyPem.trim().replaceAll('\\n', '\n'))
+ } catch {
+ throw new Error(`WEB_BOT_AUTH_PRIVATE_KEYS entry ${keyIndex} could not be loaded`)
+ }
+ if (privateKey.asymmetricKeyType !== 'ed25519') {
+ throw new Error(`WEB_BOT_AUTH_PRIVATE_KEYS entry ${keyIndex} is not an Ed25519 private key`)
+ }
+
+ const publicJwk = createPublicKey(privateKey).export({ format: 'jwk' })
+ if (publicJwk.kty !== 'OKP' || publicJwk.crv !== 'Ed25519' || !publicJwk.x) {
+ throw new Error(`WEB_BOT_AUTH_PRIVATE_KEYS entry ${keyIndex} has an invalid public key`)
+ }
+ const canonicalJwk = JSON.stringify({ crv: publicJwk.crv, kty: publicJwk.kty, x: publicJwk.x })
+ const keyId = createHash('sha256').update(canonicalJwk).digest('base64url')
+ return { privateKey, keyId }
+}
diff --git a/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/wildcard-pattern.test.ts b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/wildcard-pattern.test.ts
new file mode 100644
index 000000000000..f33e8423d5e7
--- /dev/null
+++ b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/wildcard-pattern.test.ts
@@ -0,0 +1,28 @@
+import { wildcardPatternMatchesPathname } from './wildcard-pattern'
+
+describe('wildcardPatternMatchesPathname', () => {
+ it.each([
+ ['a prefix', '/images/', '/images/photo.png', true],
+ ['an exact path', '/images/photo.png$', '/images/photo.png', true],
+ ['an exact path mismatch', '/images/photo.png$', '/images/photo.png/preview', false],
+ ['a middle wildcard', '/images/*/preview$', '/images/v1/preview', true],
+ ['an anchored suffix mismatch', '/images/*/preview$', '/images/v1/preview/more', false],
+ ['a leading wildcard', '*/private/*', '/v1/private/image.png', true],
+ ['literal regular expression characters', '/literal.+(item)$', '/literal.+(item)', true],
+ ['an unreserved percent encoding', '/asset%7Ename$', '/asset~name', true],
+ ['consecutive wildcards', '/asset/**/preview$', '/asset//preview', true],
+ ['overlapping anchored segments', '*a*a$', '/aaa', true],
+ ])('matches %s', (_name, pattern, pathname, expected) => {
+ expect(wildcardPatternMatchesPathname(pattern, pathname)).toBe(expected)
+ })
+
+ it('does not compile wildcard patterns as regular expressions', () => {
+ const regexpConstructor = jest.spyOn(globalThis, 'RegExp').mockReturnValue(/$a/)
+ try {
+ expect(wildcardPatternMatchesPathname(`${'*a'.repeat(64)}*b$`, `/${'a'.repeat(128)}c`)).toBe(false)
+ expect(regexpConstructor).not.toHaveBeenCalled()
+ } finally {
+ regexpConstructor.mockRestore()
+ }
+ })
+})
diff --git a/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/wildcard-pattern.ts b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/wildcard-pattern.ts
new file mode 100644
index 000000000000..460f5d984695
--- /dev/null
+++ b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/wildcard-pattern.ts
@@ -0,0 +1,86 @@
+export function wildcardPatternMatchesPathname(pattern: string, pathname: string): boolean {
+ const decodedPattern = decodeUnreserved(pattern)
+ const endAnchored = decodedPattern.endsWith('$')
+ const matchPattern = endAnchored ? decodedPattern.slice(0, -1) : decodedPattern
+ const decodedPathname = decodeUnreserved(pathname)
+
+ if (!matchPattern.includes('*')) {
+ return endAnchored ? decodedPathname === matchPattern : decodedPathname.startsWith(matchPattern)
+ }
+
+ const literalSegments = matchPattern.split('*').filter(Boolean)
+ if (literalSegments.length === 0) {
+ return true
+ }
+
+ let nextSearchIndex = 0
+ let nextSegmentIndex = 0
+ if (!matchPattern.startsWith('*')) {
+ const firstSegment = literalSegments[0]
+ if (!decodedPathname.startsWith(firstSegment)) {
+ return false
+ }
+ nextSearchIndex = firstSegment.length
+ nextSegmentIndex = 1
+ }
+
+ let segmentSearchEnd = literalSegments.length
+ let pathnameSearchEnd = decodedPathname.length
+ if (endAnchored && !matchPattern.endsWith('*')) {
+ const finalSegmentIndex = literalSegments.length - 1
+ const finalSegment = literalSegments[finalSegmentIndex]
+ if (!decodedPathname.endsWith(finalSegment)) {
+ return false
+ }
+ segmentSearchEnd = finalSegmentIndex
+ pathnameSearchEnd = decodedPathname.length - finalSegment.length
+ }
+
+ for (; nextSegmentIndex < segmentSearchEnd; nextSegmentIndex++) {
+ const segment = literalSegments[nextSegmentIndex]
+ const matchIndex = findLiteralSegment(decodedPathname, segment, nextSearchIndex, pathnameSearchEnd)
+ if (matchIndex === -1) {
+ return false
+ }
+ nextSearchIndex = matchIndex + segment.length
+ }
+
+ return nextSearchIndex <= pathnameSearchEnd
+}
+
+function findLiteralSegment(value: string, segment: string, startIndex: number, endIndex: number): number {
+ const prefixLengths = buildPrefixLengths(segment)
+ for (let index = startIndex, matchedLength = 0; index < endIndex; index++) {
+ while (matchedLength > 0 && value[index] !== segment[matchedLength]) {
+ matchedLength = prefixLengths[matchedLength - 1]
+ }
+ if (value[index] === segment[matchedLength]) {
+ matchedLength++
+ }
+ if (matchedLength === segment.length) {
+ return index - segment.length + 1
+ }
+ }
+ return -1
+}
+
+function buildPrefixLengths(value: string): number[] {
+ const prefixLengths = Array<number>(value.length).fill(0)
+ for (let index = 1, matchedLength = 0; index < value.length; index++) {
+ while (matchedLength > 0 && value[index] !== value[matchedLength]) {
+ matchedLength = prefixLengths[matchedLength - 1]
+ }
+ if (value[index] === value[matchedLength]) {
+ matchedLength++
+ }
+ prefixLengths[index] = matchedLength
+ }
+ return prefixLengths
+}
+
+function decodeUnreserved(value: string): string {
+ return value.replace(/%[0-9A-Fa-f]{2}/g, (encoded) => {
+ const character = String.fromCharCode(Number.parseInt(encoded.slice(1), 16))
+ return /^[A-Za-z0-9._~-]$/.test(character) ? character : encoded.toUpperCase()
+ })
+}
diff --git a/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/server/v4-consumer-server.ts b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/server/v4-consumer-server.ts
new file mode 100644
index 000000000000..61ad4ce34d79
--- /dev/null
+++ b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/server/v4-consumer-server.ts
@@ -0,0 +1,34 @@
+import { initializePrometheusLabels } from '~/common/api/router'
+import {
+ AiResearchV4ServerConfig,
+ buildAiResearchV4ServerConfig,
+} from '~/ingestion/pipelines/sessionreplay/ai-research/config'
+import { CleanupResources, NodeServer, ServerLifecycle } from '~/servers/base-server'
+
+/** The lifecycle every v4 Kafka consumer server shares: the HTTP server, metric labels, start and stop. */
+export abstract class AiResearchV4ConsumerServer implements NodeServer {
+ readonly lifecycle: ServerLifecycle
+ protected readonly config: AiResearchV4ServerConfig
+
+ constructor(config: Partial<AiResearchV4ServerConfig> = {}) {
+ this.config = buildAiResearchV4ServerConfig(config)
+ this.lifecycle = new ServerLifecycle(this.config)
+ }
+
+ async start(): Promise<void> {
+ return this.lifecycle.start(
+ () => {
+ initializePrometheusLabels(this.config.INGESTION_PIPELINE, this.config.INGESTION_LANE)
+ return this.startServices()
+ },
+ () => this.getCleanupResources()
+ )
+ }
+
+ async stop(error?: Error): Promise<void> {
+ return this.lifecycle.stop(() => this.getCleanupResources(), error)
+ }
+
+ protected abstract startServices(): Promise<void>
+ protected abstract getCleanupResources(): CleanupResources
+}
diff --git a/nodejs/src/servers/ai-research-v4-image-fetch-server.test.ts b/nodejs/src/servers/ai-research-v4-image-fetch-server.test.ts
new file mode 100644
index 000000000000..5c90f1852082
--- /dev/null
+++ b/nodejs/src/servers/ai-research-v4-image-fetch-server.test.ts
@@ -0,0 +1,234 @@
+import { Message } from 'node-rdkafka'
+
+import { KAFKA_AI_RESEARCH_V4_IMAGE_FETCH } from '~/common/config/kafka-topics'
+import { KafkaProducer } from '~/common/kafka/producer'
+import { buildAiResearchV4ServerConfig } from '~/ingestion/pipelines/sessionreplay/ai-research/config'
+import {
+ FetchCandidatePool,
+ FetchCandidatePoolAdmission,
+} from '~/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/fetch-candidate-pool'
+import { KafkaFrontierDeadLetterSink } from '~/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/frontier-dead-letter-sink'
+import { ImageFetchBatchJoiner } from '~/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/image-fetch-batch-joiner'
+import {
+ BLOCKED_IMAGE_FETCH_DEAD_LETTER_TOPICS,
+ V4_IMAGE_FETCH_LANE,
+} from '~/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/lane'
+
+import {
+ buildImageFetchBatchHandlers,
+ buildImageFetchConsumerConfigs,
+ buildImageFetchConsumerOverrides,
+ imageFetchBatchesPerPass,
+ shutdownImageFetchConsumers,
+} from './ai-research-v4-image-fetch-server'
+
+describe('v4 image fetch consumer wiring', () => {
+ it('consumes only the v4 frontier, in its own group', () => {
+ // A v4 pod on the v3 frontier would fetch v3 URLs and store their images unscrubbed.
+ const consumerConfigs = buildImageFetchConsumerConfigs(buildAiResearchV4ServerConfig({}))
+ expect(new Set(consumerConfigs.map((config) => config.topic))).toEqual(
+ new Set([KAFKA_AI_RESEARCH_V4_IMAGE_FETCH])
+ )
+ expect(new Set(consumerConfigs.map((config) => config.groupId))).toEqual(
+ new Set(['ai-research-v4-image-fetch'])
+ )
+ })
+
+ it.each(['', 'session-replay-ml-image-fetch'])('refuses the group %p', (group) => {
+ const config = buildAiResearchV4ServerConfig({ AI_RESEARCH_V4_IMAGE_FETCH_GROUP_ID: group })
+ expect(() => buildImageFetchConsumerConfigs(config)).toThrow('AI_RESEARCH_V4_IMAGE_FETCH_GROUP_ID')
+ })
+
+ it.each(Object.values(V4_IMAGE_FETCH_LANE))('refuses lane topic %s as the dead-letter topic', (topic) => {
+ expect(
+ () => new KafkaFrontierDeadLetterSink({} as KafkaProducer, topic, BLOCKED_IMAGE_FETCH_DEAD_LETTER_TOPICS)
+ ).toThrow()
+ })
+
+ it.each([
+ ['the default', {}, 2, 2, 2, 102_400],
+ ['four consumers', { AI_RESEARCH_V4_IMAGE_FETCH_TARGET_PARTITIONS_PER_BATCH: 4 }, 4, 4, 2, 164_096],
+ ['eight consumers', { AI_RESEARCH_V4_IMAGE_FETCH_TARGET_PARTITIONS_PER_BATCH: 8 }, 8, 8, 2, 328_192],
+ [
+ 'eight unjoined consumers',
+ {
+ AI_RESEARCH_V4_IMAGE_FETCH_TARGET_PARTITIONS_PER_BATCH: 8,
+ AI_RESEARCH_V4_IMAGE_FETCH_JOIN_MEMBER_BATCHES: false,
+ },
+ 8,
+ 1,
+ 2,
+ 328_192,
+ ],
+ [
+ 'eight continuous-pool consumers',
+ {
+ AI_RESEARCH_V4_IMAGE_FETCH_TARGET_PARTITIONS_PER_BATCH: 8,
+ AI_RESEARCH_V4_IMAGE_FETCH_CONTINUOUS_POOL: true,
+ },
+ 8,
+ 8,
+ 4,
+ 328_192,
+ ],
+ ['sixteen consumers', { AI_RESEARCH_V4_IMAGE_FETCH_TARGET_PARTITIONS_PER_BATCH: 16 }, 16, 16, 2, 656_384],
+ ])(
+ 'creates %s number of Kafka group members',
+ (
+ _name,
+ overrides,
+ expectedConsumers,
+ expectedBatchesPerPass,
+ expectedBatchesInFlightPerMember,
+ expectedQueueBudget
+ ) => {
+ const serverConfig = buildAiResearchV4ServerConfig(overrides)
+ const consumerConfigs = buildImageFetchConsumerConfigs(serverConfig)
+ const consumerOverrides = buildImageFetchConsumerOverrides(serverConfig, consumerConfigs.length)
+
+ expect(consumerConfigs).toHaveLength(expectedConsumers)
+ expect(imageFetchBatchesPerPass(serverConfig, consumerConfigs.length)).toBe(expectedBatchesPerPass)
+ expect(consumerConfigs.map((config) => config.maxBackgroundTasks)).toEqual(
+ Array(expectedConsumers).fill(expectedBatchesInFlightPerMember)
+ )
+ expect(consumerConfigs.every((config) => config.backgroundTaskTimeoutMs === 240_000)).toBe(true)
+ expect(consumerConfigs.map((config) => config.groupId)).toEqual(
+ Array(expectedConsumers).fill(serverConfig.AI_RESEARCH_V4_IMAGE_FETCH_GROUP_ID)
+ )
+ expect(Number(consumerOverrides['queued.max.messages.kbytes']) * expectedConsumers).toBeLessThanOrEqual(
+ expectedQueueBudget
+ )
+ expect(Number(consumerOverrides['queued.max.messages.kbytes']) * expectedConsumers).toBeGreaterThanOrEqual(
+ expectedQueueBudget - expectedConsumers
+ )
+ expect(Number(consumerOverrides['queued.max.messages.kbytes']) * 1024).toBeGreaterThanOrEqual(
+ serverConfig.AI_RESEARCH_V4_IMAGE_FETCH_MAX_IMAGE_BYTES * 2 + 64 * 1024
+ )
+ }
+ )
+
+ it.each([
+ ['the joiner without a pool', false],
+ ['its own pool window with a pool', true],
+ ])('routes each member batch through %s', async (_name, withPool) => {
+ const pool = withPool
+ ? new FetchCandidatePool<unknown>(
+ { maxConcurrentPerRegistrableDomain: 6, refillRunnableUrls: 100, maxQueuedUrlsPerOwner: 1_000 },
+ { originCrawlDelayMs: () => 0, originNextImageStartAtMs: () => 0 }
+ )
+ : undefined
+ const fetchConsumer = { handleBatch: jest.fn().mockResolvedValue(undefined) }
+ const joiner = {
+ handleBatch: jest.fn().mockResolvedValue(undefined),
+ waitForProcessing: jest.fn().mockResolvedValue(undefined),
+ }
+ const message: Message = { topic: 'test-topic', partition: 0, offset: 1, value: Buffer.from('x'), size: 1 }
+
+ const { handlers } = buildImageFetchBatchHandlers(2, fetchConsumer, { candidatePool: pool }, joiner)
+ await Promise.all(handlers.map((handler) => handler([message])))
+
+ expect(handlers).toHaveLength(2)
+ expect(joiner.handleBatch).toHaveBeenCalledTimes(withPool ? 0 : 2)
+ expect(fetchConsumer.handleBatch).toHaveBeenCalledTimes(withPool ? 2 : 0)
+ if (withPool) {
+ const [firstAdmission, secondAdmission] = fetchConsumer.handleBatch.mock.calls.map((call) => call[2])
+ expect(firstAdmission.owner).not.toBe(secondAdmission.owner)
+ }
+ pool?.close()
+ })
+
+ it('waits for a pooled member batch that is still preparing before it closes the fetch runner', async () => {
+ const pool = new FetchCandidatePool<unknown>(
+ { maxConcurrentPerRegistrableDomain: 6, refillRunnableUrls: 100, maxQueuedUrlsPerOwner: 1_000 },
+ { originCrawlDelayMs: () => 0, originNextImageStartAtMs: () => 0 }
+ )
+ let finishBatch: () => void = () => undefined
+ const fetchConsumer = {
+ handleBatch: jest.fn(
+ (_messages: Message[], _nowMs: number, admission?: FetchCandidatePoolAdmission) =>
+ new Promise<void>((resolve) => {
+ admission?.admitted()
+ finishBatch = resolve
+ })
+ ),
+ }
+ const joiner = { handleBatch: jest.fn(), waitForProcessing: jest.fn().mockResolvedValue(undefined) }
+ const startedBatches = buildImageFetchBatchHandlers(1, fetchConsumer, { candidatePool: pool }, joiner)
+ await startedBatches.handlers[0]([
+ { topic: 'test-topic', partition: 0, offset: 1, value: Buffer.from('x'), size: 1 },
+ ])
+ const consumers = [{ stopConsuming: jest.fn().mockResolvedValue(undefined), disconnect: jest.fn() }]
+ const fetchRunner = { close: jest.fn().mockResolvedValue(undefined) }
+
+ const shutdown = shutdownImageFetchConsumers(consumers, startedBatches, fetchRunner)
+ await new Promise((resolve) => setImmediate(resolve))
+ expect(fetchRunner.close).not.toHaveBeenCalled()
+ finishBatch()
+ await shutdown
+
+ expect(fetchRunner.close).toHaveBeenCalledTimes(1)
+ pool.close()
+ })
+
+ it.each([false, true])('waits for joined work before disconnecting and cleaning up (failure=%s)', async (fails) => {
+ jest.useFakeTimers()
+ try {
+ const processBatch = jest.fn(
+ (messages: Message[]) =>
+ new Promise<void>((resolve, reject) => {
+ setTimeout(
+ () => {
+ if (fails && messages[0].offset === 1) {
+ reject(new Error('fetch failed'))
+ } else {
+ resolve()
+ }
+ },
+ messages[0].offset === 1 ? 30_000 : 50_000
+ )
+ })
+ )
+ const joiner = new ImageFetchBatchJoiner(1, processBatch)
+ const completions = await Promise.all(
+ [1, 2].map((offset) =>
+ joiner.handleBatch([
+ { topic: 'test-topic', partition: 0, offset, value: Buffer.from('test'), size: 4 },
+ ])
+ )
+ )
+ const work = Promise.allSettled(completions.map((batch) => batch?.backgroundTask ?? Promise.resolve()))
+ const consumers = [0, 1].map(() => ({
+ stopConsuming: jest.fn(() => new Promise<void>((resolve) => setTimeout(resolve, 20_000))),
+ disconnect: jest.fn().mockResolvedValue(undefined),
+ }))
+ const cleanup = jest.fn()
+ const shutdown = shutdownImageFetchConsumers(consumers, joiner).then(cleanup)
+
+ expect(consumers.every((consumer) => consumer.stopConsuming.mock.calls.length === 1)).toBe(true)
+ await jest.advanceTimersByTimeAsync(30_000)
+ expect(consumers.every((consumer) => consumer.disconnect.mock.calls.length === 0)).toBe(true)
+ expect(cleanup).not.toHaveBeenCalled()
+
+ await jest.advanceTimersByTimeAsync(20_000)
+ await shutdown
+ await work
+ expect(consumers.every((consumer) => consumer.disconnect.mock.calls.length === 1)).toBe(true)
+ expect(cleanup).toHaveBeenCalledTimes(1)
+ } finally {
+ jest.useRealTimers()
+ }
+ })
+
+ it.each([0, -1, 1.5, 17, Number.NaN, Number.MAX_SAFE_INTEGER + 1])(
+ 'refuses invalid target partition count %p',
+ (targetPartitionsPerBatch) => {
+ const serverConfig = buildAiResearchV4ServerConfig({
+ AI_RESEARCH_V4_IMAGE_FETCH_TARGET_PARTITIONS_PER_BATCH: targetPartitionsPerBatch,
+ })
+
+ expect(() => buildImageFetchConsumerConfigs(serverConfig)).toThrow(
+ 'image fetch batch target must be an integer between 1 and 16'
+ )
+ }
+ )
+})
diff --git a/nodejs/src/servers/ai-research-v4-image-fetch-server.ts b/nodejs/src/servers/ai-research-v4-image-fetch-server.ts
new file mode 100644
index 000000000000..9ef9b22ec8f6
--- /dev/null
+++ b/nodejs/src/servers/ai-research-v4-image-fetch-server.ts
@@ -0,0 +1,562 @@
+import { DynamoDBClient } from '@aws-sdk/client-dynamodb'
+import { KMSClient } from '@aws-sdk/client-kms'
+import { NodeHttpHandler } from '@smithy/node-http-handler'
+import { LibrdKafkaError } from 'node-rdkafka'
+
+import { KafkaConsumer, KafkaConsumerConfig, findOffsetsToCommit } from '~/common/kafka/consumer/consumer-v1'
+import {
+ EachBatch,
+ KafkaConsumerV2,
+ KafkaConsumerV2Config,
+ RdKafkaConsumerOverrides,
+} from '~/common/kafka/consumer/consumer-v2'
+import { KafkaProducer } from '~/common/kafka/producer'
+import { KafkaProducerRegistry } from '~/common/outputs/kafka-producer-registry'
+import { logger } from '~/common/utils/logger'
+import { TopHog } from '~/ingestion/framework/tophog/tophog'
+import { AiResearchV4ServerConfig } from '~/ingestion/pipelines/sessionreplay/ai-research/config'
+import {
+ ConfigurationPolicyService,
+ HttpConfigurationFetcher,
+} from '~/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/configuration-policy'
+import { DynamoDBCrawlHistory } from '~/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/dynamodb-crawl-history'
+import { FetchCandidatePool } from '~/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/fetch-candidate-pool'
+import { FetchRunner } from '~/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/fetch-runner'
+import { KafkaFrontierDeadLetterSink } from '~/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/frontier-dead-letter-sink'
+import { FrontierPublisher } from '~/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/frontier-publisher'
+import { HostBudget } from '~/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/host-budget'
+import {
+ ImageFetchBatchJoiner,
+ assertImageFetchBatchTarget,
+} from '~/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/image-fetch-batch-joiner'
+import { HttpImageFetcher } from '~/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/image-fetcher'
+import type { RefreshingImageSink } from '~/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/image-sink'
+import { V4JobAdmission } from '~/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/job-admission'
+import {
+ BLOCKED_IMAGE_FETCH_DEAD_LETTER_TOPICS,
+ HostLimits,
+ V4_IMAGE_FETCH_DELAY_TOPICS,
+ V4_IMAGE_FETCH_LANE,
+ assertOwnImageFetchGroup,
+ v4HostLimits,
+} from '~/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/lane'
+import { OriginRequestScheduler } from '~/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/origin-request-scheduler'
+import { assertUrlPolicyLoaded } from '~/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/politeness-key'
+import { createPoolWindowBatchHandler } from '~/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/pool-window-batch-handler'
+import {
+ AI_RESEARCH_V4_IMAGE_FETCH_PRODUCER,
+ AiResearchV4ImageFetchProducerName,
+ createV4ImageFetchOutputs,
+ createV4ImageFetchProducerRegistry,
+} from '~/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/producer'
+import { RawImageStore } from '~/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/raw-image-store'
+import { TeamMonthResponseUrlBinder } from '~/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/response-url-binding'
+import { RetryDelayConsumer } from '~/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/retry-delay-consumer'
+import { SharedImageFetcher } from '~/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/shared-image-fetcher'
+import { ImageFetchTopHogMetrics } from '~/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/tophog-metrics'
+import { UrlFetchConsumer } from '~/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/url-fetch-consumer'
+import { createWebBotAuthRequestSigner } from '~/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/web-bot-auth'
+import { AiResearchV4ConsumerServer } from '~/ingestion/pipelines/sessionreplay/ai-research/server/v4-consumer-server'
+import { TeamMonthKeys } from '~/ingestion/pipelines/sessionreplay/ai-research/v4-keys/team-month-keys'
+import { CaptureWatermark, capturedRecords } from '~/ingestion/pipelines/sessionreplay/shared/capture-watermark'
+import { buildSessionRecordingS3Client } from '~/ingestion/pipelines/sessionreplay/shared/s3-client'
+import { HealthCheckResultOk } from '~/types'
+
+import { CleanupResources } from './base-server'
+
+/**
+ * How long the store may spend on one batch, well inside Kafka's max.poll.interval.ms of 300s.
+ *
+ * The poll interval binds rather than the health check, because the heartbeat above keeps the health
+ * check satisfied through a long batch. A batch that passes the poll interval loses the partition
+ * mid-batch to a pod that will be just as slow, so the lane sheds the rest of the batch instead.
+ */
+const STORE_BATCH_BUDGET_MS = 50_000
+const IMAGE_FETCH_KAFKA_QUEUE_BUDGET_KBYTES = 102_400
+const IMAGE_FETCH_MIN_CONSUMER_QUEUE_KBYTES = 25_600
+/**
+ * The pool window paces a member in continuous-pool mode. This limit stays above that window at typical
+ * batch sizes, and it is the hard bound on a member's resident URLs and crash replay when batches are large.
+ */
+const POOL_MAX_BATCHES_IN_FLIGHT_PER_MEMBER = 4
+
+/** Matches MAX_URL_LEN in the crate, which is what the collector applied to the first candidate. */
+const MAX_REDIRECT_URL_LENGTH = 2048
+
+/**
+ * Headroom on top of the sleeping a batch can do, for the publishing between the sleeps.
+ *
+ * librdkafka rejects a `max.poll.interval.ms` above one day, so a tier and a batch size whose
+ * product passes that is a configuration error rather than something to clamp silently.
+ */
+const POLL_INTERVAL_MARGIN_MS = 60_000
+const MAX_POLL_INTERVAL_MS = 86_400_000
+const REVOKED_PARTITION_CODES = new Set([-172, -190, -142, -144])
+
+/** The addon is a native library and `index.ts` imports every server, so it loads on first use. */
+let urlPolicy: typeof import('@posthog/ai-research-v4-native') | undefined
+function getUrlPolicy(): typeof import('@posthog/ai-research-v4-native') {
+ if (!urlPolicy) {
+ urlPolicy = require('@posthog/ai-research-v4-native') as typeof import('@posthog/ai-research-v4-native')
+ if (typeof urlPolicy.isPublicHost !== 'function') {
+ throw new Error('the ai-research-v4-native addon has no isPublicHost: rebuild index.node')
+ }
+ }
+ return urlPolicy
+}
+
+export function buildFrontierPublisher(producer: KafkaProducer, maxConcurrentRepublishes: number): FrontierPublisher {
+ return new FrontierPublisher(producer, {
+ frontierTopic: V4_IMAGE_FETCH_LANE.frontier,
+ delayTiers: [
+ { topic: V4_IMAGE_FETCH_LANE.retry1m, delayMs: 60_000, metricTopic: 'retry_1m' },
+ { topic: V4_IMAGE_FETCH_LANE.retry10m, delayMs: 600_000, metricTopic: 'retry_10m' },
+ { topic: V4_IMAGE_FETCH_LANE.retry1h, delayMs: 3_600_000, metricTopic: 'retry_1h' },
+ ],
+ maxConcurrentRepublishes,
+ })
+}
+
+/** The lane's per registrable domain limits: the configured share of the configured limits. */
+export function imageFetchHostLimits(config: AiResearchV4ServerConfig): HostLimits {
+ return v4HostLimits(
+ {
+ requestsPerSecond: config.AI_RESEARCH_V4_IMAGE_FETCH_REGISTRABLE_DOMAIN_REQUESTS_PER_SECOND,
+ burst: config.AI_RESEARCH_V4_IMAGE_FETCH_REGISTRABLE_DOMAIN_BURST,
+ maxConcurrent: config.AI_RESEARCH_V4_IMAGE_FETCH_MAX_CONCURRENT_PER_REGISTRABLE_DOMAIN,
+ },
+ config.AI_RESEARCH_V4_IMAGE_FETCH_HOST_BUDGET_SHARE
+ )
+}
+
+export function buildFetchRunner(
+ config: AiResearchV4ServerConfig,
+ publisher: FrontierPublisher,
+ topHogMetrics: ImageFetchTopHogMetrics,
+ imageSink: RefreshingImageSink,
+ keys: Pick<TeamMonthKeys, 'get'>,
+ hostLimits: HostLimits = imageFetchHostLimits(config)
+): FetchRunner {
+ const webBotAuthSigner = createWebBotAuthRequestSigner(config.AI_RESEARCH_V4_IMAGE_FETCH_WEB_BOT_AUTH_PRIVATE_KEYS)
+ const budget = new HostBudget({
+ ...hostLimits,
+ breakerFailures: config.AI_RESEARCH_V4_IMAGE_FETCH_REGISTRABLE_DOMAIN_BREAKER_FAILURES,
+ breakerCooldownMs: config.AI_RESEARCH_V4_IMAGE_FETCH_REGISTRABLE_DOMAIN_BREAKER_COOLDOWN_MS,
+ breakerMaxCooldownMs: config.AI_RESEARCH_V4_IMAGE_FETCH_REGISTRABLE_DOMAIN_BREAKER_MAX_COOLDOWN_MS,
+ maxTrackedRegistrableDomains: config.AI_RESEARCH_V4_IMAGE_FETCH_MAX_TRACKED_REGISTRABLE_DOMAINS,
+ maxTrackedOrigins: config.AI_RESEARCH_V4_IMAGE_FETCH_MAX_TRACKED_ORIGINS,
+ })
+ const scheduler = new OriginRequestScheduler(
+ budget,
+ config.AI_RESEARCH_V4_IMAGE_FETCH_MAX_IN_FLIGHT_REQUESTS,
+ topHogMetrics
+ )
+ const configurationPolicy = new ConfigurationPolicyService(
+ new HttpConfigurationFetcher(webBotAuthSigner, scheduler, config.AI_RESEARCH_V4_IMAGE_FETCH_REQUEST_TIMEOUT_MS)
+ )
+ return new FetchRunner(
+ // The recorder sends one job per team for the same URL and month, so one fetch serves them all.
+ new SharedImageFetcher(
+ new HttpImageFetcher(
+ {
+ maxUrlLength: MAX_REDIRECT_URL_LENGTH,
+ // The crate's rule, so a redirect target meets the check the collector already applied
+ // to the first candidate rather than a second answer to the same question.
+ isPublicHost: (host) => getUrlPolicy().isPublicHost(host),
+ },
+ webBotAuthSigner
+ )
+ ),
+ budget,
+ scheduler,
+ configurationPolicy,
+ {
+ maxConcurrentPerRegistrableDomain: hostLimits.maxConcurrent,
+ maxInFlightRequests: config.AI_RESEARCH_V4_IMAGE_FETCH_MAX_IN_FLIGHT_REQUESTS,
+ batchBudgetMs: config.AI_RESEARCH_V4_IMAGE_FETCH_REQUEST_BUDGET_MS,
+ maxBytes: config.AI_RESEARCH_V4_IMAGE_FETCH_MAX_IMAGE_BYTES,
+ requestTimeoutMs: config.AI_RESEARCH_V4_IMAGE_FETCH_REQUEST_TIMEOUT_MS,
+ maxRedirects: config.AI_RESEARCH_V4_IMAGE_FETCH_MAX_REDIRECTS,
+ seenTtlSeconds: config.AI_RESEARCH_V4_IMAGE_FETCH_CRAWL_HISTORY_TTL_SECONDS,
+ continuousPool: config.AI_RESEARCH_V4_IMAGE_FETCH_CONTINUOUS_POOL
+ ? {
+ refillRunnableUrls: config.AI_RESEARCH_V4_IMAGE_FETCH_POOL_REFILL_RUNNABLE_URLS,
+ maxQueuedUrlsPerOwner: config.AI_RESEARCH_V4_IMAGE_FETCH_POOL_MAX_QUEUED_URLS_PER_MEMBER,
+ }
+ : undefined,
+ },
+ publisher,
+ imageSink,
+ new TeamMonthResponseUrlBinder(keys),
+ topHogMetrics
+ )
+}
+
+export function buildImageFetchConsumerConfigs(config: AiResearchV4ServerConfig): KafkaConsumerV2Config[] {
+ assertOwnImageFetchGroup(config.AI_RESEARCH_V4_IMAGE_FETCH_GROUP_ID)
+ const targetBatchCount = config.AI_RESEARCH_V4_IMAGE_FETCH_TARGET_PARTITIONS_PER_BATCH
+ assertImageFetchBatchTarget(targetBatchCount)
+ return Array.from({ length: targetBatchCount }, () => ({
+ topic: V4_IMAGE_FETCH_LANE.frontier,
+ groupId: config.AI_RESEARCH_V4_IMAGE_FETCH_GROUP_ID,
+ autoCommit: true,
+ autoOffsetStore: true,
+ fetchBatchSize: config.AI_RESEARCH_V4_IMAGE_FETCH_BATCH_SIZE,
+ maxBackgroundTasks: config.AI_RESEARCH_V4_IMAGE_FETCH_CONTINUOUS_POOL
+ ? POOL_MAX_BATCHES_IN_FLIGHT_PER_MEMBER
+ : 2,
+ backgroundTaskTimeoutMs: 240_000,
+ }))
+}
+
+export function imageFetchBatchesPerPass(config: AiResearchV4ServerConfig, consumerCount: number): number {
+ return config.AI_RESEARCH_V4_IMAGE_FETCH_JOIN_MEMBER_BATCHES ? consumerCount : 1
+}
+
+export function buildImageFetchConsumerOverrides(
+ config: AiResearchV4ServerConfig,
+ consumerCount: number
+): RdKafkaConsumerOverrides {
+ assertImageFetchBatchTarget(consumerCount)
+ const maximumRecordBytes = config.AI_RESEARCH_V4_IMAGE_FETCH_MAX_IMAGE_BYTES * 2 + 64 * 1024
+ return {
+ 'fetch.message.max.bytes': maximumRecordBytes,
+ 'max.partition.fetch.bytes': maximumRecordBytes,
+ 'queued.max.messages.kbytes': Math.max(
+ IMAGE_FETCH_MIN_CONSUMER_QUEUE_KBYTES,
+ Math.ceil(maximumRecordBytes / 1024),
+ Math.floor(IMAGE_FETCH_KAFKA_QUEUE_BUDGET_KBYTES / consumerCount)
+ ),
+ }
+}
+
+export interface ImageFetchBatchHandlers {
+ handlers: EachBatch[]
+ /** Resolves once every batch started so far has settled, including pooled batches that the Kafka drain gave up on. */
+ waitForProcessing(): Promise<void>
+}
+
+export function buildImageFetchBatchHandlers(
+ consumerCount: number,
+ fetchConsumer: Pick<UrlFetchConsumer, 'handleBatch'>,
+ fetchRunner: { candidatePool?: Pick<FetchCandidatePool<unknown>, 'createOwner'> },
+ batchJoiner: Pick<ImageFetchBatchJoiner, 'handleBatch' | 'waitForProcessing'>
+): ImageFetchBatchHandlers {
+ const pool = fetchRunner.candidatePool
+ const pooledBatches = new Set<Promise<void>>()
+ const trackPooledBatch = (batch: Promise<void>): Promise<void> => {
+ pooledBatches.add(batch)
+ const forget = (): void => {
+ pooledBatches.delete(batch)
+ }
+ batch.then(forget, forget)
+ return batch
+ }
+ const handlers = Array.from({ length: consumerCount }, (): EachBatch => {
+ if (!pool) {
+ return (messages) => batchJoiner.handleBatch(messages)
+ }
+ return createPoolWindowBatchHandler(pool.createOwner(), (messages, admission) =>
+ trackPooledBatch(fetchConsumer.handleBatch(messages, Date.now(), admission))
+ )
+ })
+ return {
+ handlers,
+ waitForProcessing: async () => {
+ await batchJoiner.waitForProcessing()
+ await Promise.allSettled(pooledBatches)
+ },
+ }
+}
+
+export async function shutdownImageFetchConsumers(
+ consumers: Pick<KafkaConsumerV2, 'stopConsuming' | 'disconnect'>[],
+ startedBatches: Pick<ImageFetchBatchHandlers, 'waitForProcessing'>,
+ fetchRunner?: Pick<FetchRunner, 'close'>
+): Promise<void> {
+ await Promise.allSettled(consumers.map((consumer) => consumer.stopConsuming()))
+ await startedBatches.waitForProcessing()
+ await Promise.allSettled(consumers.map((consumer) => consumer.disconnect()))
+ await fetchRunner?.close()
+}
+
+/**
+ * The v4 image fetch lane: the fetch policy of the v3 lane over the v4 recorder's URLs, storing each image sealed and
+ * unscrubbed in `images-v4`. It reads the team month keys and never writes them, and it uses a share of the configured
+ * per-host limits.
+ */
+export class AiResearchV4ImageFetchServer extends AiResearchV4ConsumerServer {
+ private crawlHistoryClient?: DynamoDBClient
+ private producerRegistry?: KafkaProducerRegistry<AiResearchV4ImageFetchProducerName>
+ private topHog?: TopHog
+
+ protected async startServices(): Promise<void> {
+ // Here rather than on the first record. The parser calls into this addon for every URL, and
+ // it must answer with a reason rather than raise, so a build that shipped without the addon
+ // has to stop the pod at startup instead.
+ assertUrlPolicyLoaded()
+
+ const consumerConfigs = buildImageFetchConsumerConfigs(this.config)
+ const dryRun = this.config.AI_RESEARCH_V4_IMAGE_FETCH_DRY_RUN
+ const { AI_RESEARCH_V4_KEY_TABLE, AI_RESEARCH_V4_KMS_KEY_ARN, AI_RESEARCH_V4_IMAGES_BUCKET } = this.config
+ if (!AI_RESEARCH_V4_KEY_TABLE || !AI_RESEARCH_V4_KMS_KEY_ARN || !AI_RESEARCH_V4_IMAGES_BUCKET) {
+ throw new Error(
+ 'the v4 image fetch lane needs AI_RESEARCH_V4_KEY_TABLE, AI_RESEARCH_V4_KMS_KEY_ARN and AI_RESEARCH_V4_IMAGES_BUCKET'
+ )
+ }
+ const historyTable = this.config.AI_RESEARCH_V4_IMAGE_FETCH_HISTORY_TABLE
+ if (!historyTable) {
+ throw new Error('AI_RESEARCH_V4_IMAGE_FETCH_HISTORY_TABLE must be set for the v4 image fetch lane')
+ }
+ const dynamoDBTimeoutMs = this.config.AI_RESEARCH_V4_IMAGE_FETCH_DYNAMODB_TIMEOUT_MS
+ if (!Number.isFinite(dynamoDBTimeoutMs) || dynamoDBTimeoutMs <= 0) {
+ throw new Error(
+ `AI_RESEARCH_V4_IMAGE_FETCH_DYNAMODB_TIMEOUT_MS must be a positive number, got ${dynamoDBTimeoutMs}`
+ )
+ }
+ const s3 = buildSessionRecordingS3Client(this.config)
+ if (!s3) {
+ throw new Error('SESSION_RECORDING_V2_S3_* must be configured for the v4 image fetch lane')
+ }
+ const region = this.config.SESSION_RECORDING_V2_S3_REGION || 'us-east-1'
+ this.crawlHistoryClient = new DynamoDBClient({
+ region,
+ endpoint: this.config.SESSION_RECORDING_DYNAMODB_ENDPOINT,
+ maxAttempts: 5,
+ requestHandler: new NodeHttpHandler(),
+ })
+ const crawlHistory = new DynamoDBCrawlHistory(
+ this.crawlHistoryClient,
+ historyTable,
+ dynamoDBTimeoutMs,
+ STORE_BATCH_BUDGET_MS
+ )
+ await crawlHistory.validateAccess(Date.now())
+
+ const keys = new TeamMonthKeys(
+ new DynamoDBClient({ region, endpoint: this.config.SESSION_RECORDING_DYNAMODB_ENDPOINT, maxAttempts: 5 }),
+ new KMSClient({ region, maxAttempts: 3 }),
+ AI_RESEARCH_V4_KEY_TABLE,
+ AI_RESEARCH_V4_KMS_KEY_ARN,
+ this.config.AI_RESEARCH_V4_KMS_REQUESTS_PER_SECOND
+ )
+ const admission = new V4JobAdmission(keys, this.config.AI_RESEARCH_V4_IMAGE_FETCH_KEY_BUDGET_MS)
+ const imageSink = new RawImageStore(
+ keys,
+ s3,
+ AI_RESEARCH_V4_IMAGES_BUCKET,
+ this.config.AI_RESEARCH_V4_IMAGE_FETCH_MAX_PENDING_PUBLISHES
+ )
+
+ // Built even in dry run, so the wiring is exercised by every start rather than only by the
+ // one that clears the flag.
+ this.producerRegistry = await createV4ImageFetchProducerRegistry(this.config.KAFKA_CLIENT_RACK).build(
+ this.config
+ )
+ const outputs = createV4ImageFetchOutputs().build(this.producerRegistry, this.config)
+ this.topHog = new TopHog({
+ outputs,
+ pipeline: this.config.INGESTION_PIPELINE ?? 'unknown',
+ lane: this.config.INGESTION_LANE ?? 'unknown',
+ })
+ this.topHog.start()
+ const topHogMetrics = new ImageFetchTopHogMetrics(this.topHog)
+ const producer = this.producerRegistry.getProducer(AI_RESEARCH_V4_IMAGE_FETCH_PRODUCER)
+ const publisher = buildFrontierPublisher(producer, this.config.AI_RESEARCH_V4_IMAGE_FETCH_MAX_PENDING_PUBLISHES)
+ const deadLetterTopic = this.config.AI_RESEARCH_V4_IMAGE_FETCH_DLQ_TOPIC
+ const deadLetters = deadLetterTopic
+ ? new KafkaFrontierDeadLetterSink(producer, deadLetterTopic, BLOCKED_IMAGE_FETCH_DEAD_LETTER_TOPICS)
+ : null
+
+ const fetchRunner = buildFetchRunner(this.config, publisher, topHogMetrics, imageSink, keys)
+ const fetchConsumer = new UrlFetchConsumer(
+ crawlHistory,
+ publisher,
+ {
+ seenTtlSeconds: this.config.AI_RESEARCH_V4_IMAGE_FETCH_CRAWL_HISTORY_TTL_SECONDS,
+ dryRun,
+ admitCandidates: (candidates, nowMs) => admission.admit(candidates, nowMs),
+ refreshingSink: imageSink,
+ },
+ fetchRunner,
+ deadLetters,
+ topHogMetrics
+ )
+ logger.info('🌐', 'ai_research_v4_image_fetch_started', {
+ dryRun,
+ continuousPool: this.config.AI_RESEARCH_V4_IMAGE_FETCH_CONTINUOUS_POOL,
+ })
+
+ const batchJoiner = new ImageFetchBatchJoiner(
+ imageFetchBatchesPerPass(this.config, consumerConfigs.length),
+ (messages) => fetchConsumer.handleBatch(messages, Date.now())
+ )
+ const batchHandlers = buildImageFetchBatchHandlers(
+ consumerConfigs.length,
+ fetchConsumer,
+ fetchRunner,
+ batchJoiner
+ )
+ const consumerOverrides = buildImageFetchConsumerOverrides(this.config, consumerConfigs.length)
+ const consumers = consumerConfigs.map(
+ (consumerConfig) => new KafkaConsumerV2(consumerConfig, consumerOverrides)
+ )
+
+ this.lifecycle.services.push({
+ id: 'ai-research-v4-image-fetch',
+ onShutdown: () => shutdownImageFetchConsumers(consumers, batchHandlers, fetchRunner),
+ healthcheck: () => {
+ for (const consumer of consumers) {
+ const health = consumer.isHealthy()
+ if (health.isError()) {
+ return health
+ }
+ }
+ return new HealthCheckResultOk()
+ },
+ })
+ const watermark = new CaptureWatermark('ai_research_v4_image_fetch')
+ await Promise.all(
+ consumers.map((consumer, index) =>
+ consumer.connect(
+ (messages) => {
+ watermark.hold(capturedRecords(messages, () => 'image_urls'))
+ return batchHandlers.handlers[index](messages)
+ },
+ (partitions) => {
+ watermark.forget(partitions)
+ return Promise.resolve()
+ },
+ (offsets) => watermark.release(offsets)
+ )
+ )
+ )
+ }
+
+ protected getCleanupResources(): CleanupResources {
+ return {
+ kafkaProducers: [],
+ redisPools: [],
+ additionalCleanup: async () => {
+ this.crawlHistoryClient?.destroy()
+ try {
+ await this.topHog?.stop()
+ } finally {
+ await this.producerRegistry?.disconnectAll()
+ }
+ },
+ }
+ }
+}
+
+/** Relays the v4 lane's delay tiers back to its frontier. */
+export class AiResearchV4ImageFetchRetryServer extends AiResearchV4ConsumerServer {
+ private producerRegistry?: KafkaProducerRegistry<AiResearchV4ImageFetchProducerName>
+
+ protected async startServices(): Promise<void> {
+ assertOwnImageFetchGroup(this.config.AI_RESEARCH_V4_IMAGE_FETCH_GROUP_ID)
+ const topic = this.config.AI_RESEARCH_V4_IMAGE_FETCH_RETRY_TOPIC
+ const delayMs = this.config.AI_RESEARCH_V4_IMAGE_FETCH_RETRY_DELAY_MS
+ const batchSize = this.config.AI_RESEARCH_V4_IMAGE_FETCH_RETRY_BATCH_SIZE
+ const producerDeliveryTimeoutMs = Number(
+ this.config.KAFKA_AI_RESEARCH_V4_IMAGE_FETCH_PRODUCER_MESSAGE_TIMEOUT_MS
+ )
+ // A typo here reads as an empty topic name, and librdkafka would subscribe to it and sit
+ // idle while the tier filled. Naming the three the publisher writes to also stops a pod
+ // draining the frontier itself, which has no period and would publish every record straight
+ // back to it.
+ if (!V4_IMAGE_FETCH_DELAY_TOPICS.includes(topic)) {
+ throw new Error(
+ `AI_RESEARCH_V4_IMAGE_FETCH_RETRY_TOPIC must name one of ${V4_IMAGE_FETCH_DELAY_TOPICS.join(', ')}, got "${topic}"`
+ )
+ }
+ if (!Number.isInteger(batchSize) || batchSize < 1) {
+ throw new Error(`AI_RESEARCH_V4_IMAGE_FETCH_RETRY_BATCH_SIZE must be positive, got ${batchSize}`)
+ }
+ if (!Number.isFinite(producerDeliveryTimeoutMs) || producerDeliveryTimeoutMs <= 0) {
+ throw new Error(`the image fetch retry producer needs a positive delivery timeout`)
+ }
+
+ this.producerRegistry = await createV4ImageFetchProducerRegistry(this.config.KAFKA_CLIENT_RACK).build(
+ this.config
+ )
+ const producer = this.producerRegistry.getProducer(AI_RESEARCH_V4_IMAGE_FETCH_PRODUCER)
+
+ const consumerConfig: KafkaConsumerConfig = {
+ topic,
+ groupId: `${this.config.AI_RESEARCH_V4_IMAGE_FETCH_GROUP_ID}-retry-${delayMs}`,
+ autoCommit: true,
+ // Stored after every publish in the batch is acknowledged.
+ autoOffsetStore: false,
+ fetchBatchSize: batchSize,
+ }
+ // Set here rather than left to the deployment. This consumer sleeps for the period of its
+ // topic inside one batch, and the shared default of 300s would evict the 10m and 1h tiers
+ // mid-sleep, then replay the partition into a consumer that sleeps just as long.
+ //
+ // The batch waits once for its latest broker append timestamp.
+ const pollIntervalMs = delayMs + batchSize * producerDeliveryTimeoutMs + POLL_INTERVAL_MARGIN_MS
+ if (pollIntervalMs > MAX_POLL_INTERVAL_MS) {
+ throw new Error(
+ `a delay of ${delayMs}ms and batch of ${batchSize} need a poll interval of ${pollIntervalMs}ms, which is past what Kafka allows`
+ )
+ }
+ const consumer = new KafkaConsumer(consumerConfig, { 'max.poll.interval.ms': pollIntervalMs })
+ // The shutdown handler sets this before it disconnects the consumer, so a stopping pod
+ // abandons the wait it holds rather than making the rolling deploy wait out a tier period.
+ let stopping = false
+ const watermark = new CaptureWatermark(`ai_research_v4_image_fetch_retry_${delayMs}`)
+ const delayConsumer = new RetryDelayConsumer(producer, {
+ isStopping: () => stopping,
+ storeOffsets: (messages) => {
+ try {
+ const offsets = findOffsetsToCommit(messages)
+ consumer.offsetsStore(offsets)
+ watermark.release(offsets)
+ } catch (error) {
+ const code = (error as LibrdKafkaError | undefined)?.code
+ if (!stopping && !REVOKED_PARTITION_CODES.has(code ?? 0)) {
+ throw error
+ }
+ logger.warn('🌐', 'ai_research_v4_image_fetch_retry_offset_store_failed', {
+ partitions: messages.map((message) => message.partition),
+ code,
+ error: error instanceof Error ? error.name : 'unknown',
+ })
+ }
+ },
+ frontierTopic: V4_IMAGE_FETCH_LANE.frontier,
+ delayMs,
+ heartbeat: () => consumer.reportDeliberateWait(),
+ })
+ logger.info('🌐', 'ai_research_v4_image_fetch_retry_started', { topic, delayMs, pollIntervalMs })
+
+ await consumer.connect(
+ (messages) => {
+ watermark.hold(capturedRecords(messages, () => 'image_urls'))
+ return delayConsumer.handleBatch(messages)
+ },
+ (partitions) => {
+ watermark.forget(partitions)
+ return Promise.resolve()
+ }
+ )
+
+ this.lifecycle.services.push({
+ id: 'ai-research-v4-image-fetch-retry',
+ onShutdown: () => {
+ stopping = true
+ return consumer.disconnect()
+ },
+ healthcheck: () => consumer.isHealthy(),
+ })
+ }
+
+ protected getCleanupResources(): CleanupResources {
+ return {
+ kafkaProducers: [],
+ redisPools: [],
+ additionalCleanup: () => this.producerRegistry?.disconnectAll(),
+ }
+ }
+}