:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(ai-research): run the image fetcher as a v4 lane (#117984)
Nicholas Waltz86a59204160b
diff --git a/nodejs/src/capabilities.ts b/nodejs/src/capabilities.tsindex 3cb41d31f8b7..829e7e223c50 100644--- a/nodejs/src/capabilities.ts+++ b/nodejs/src/capabilities.ts@@ -211,6 +211,8 @@ export function getPluginServerCapabilities(case PluginServerMode.recordings_blob_ingestion_v2_ml_image_scrub_dlq_replay:case PluginServerMode.recordings_blob_ingestion_v2_ml_image_fetch:case PluginServerMode.recordings_blob_ingestion_v2_ml_image_fetch_retry:+ case PluginServerMode.ai_research_v4_image_fetch:+ case PluginServerMode.ai_research_v4_image_fetch_retry:case PluginServerMode.recording_api:throw new Error(`Mode ${mode} is handled by IngestionSessionReplayServer, not PluginServer`)}diff --git a/nodejs/src/common/config.ts b/nodejs/src/common/config.tsindex 8a6d7df6f711..d718de4234b6 100644--- a/nodejs/src/common/config.ts+++ b/nodejs/src/common/config.ts@@ -42,6 +42,8 @@ export enum PluginServerMode {recordings_blob_ingestion_v2_ml_image_scrub_dlq_replay = 'recordings-blob-ingestion-v2-ml-image-scrub-dlq-replay',recordings_blob_ingestion_v2_ml_image_fetch = 'recordings-blob-ingestion-v2-ml-image-fetch',recordings_blob_ingestion_v2_ml_image_fetch_retry = 'recordings-blob-ingestion-v2-ml-image-fetch-retry',+ ai_research_v4_image_fetch = 'ai-research-v4-image-fetch',+ ai_research_v4_image_fetch_retry = 'ai-research-v4-image-fetch-retry',cdp_processed_events = 'cdp-processed-events',cdp_person_updates = 'cdp-person-updates',cdp_data_warehouse_events = 'cdp-data-warehouse-events',diff --git a/nodejs/src/common/config/kafka-topics.ts b/nodejs/src/common/config/kafka-topics.tsindex 12475ea56851..61c8aa6299a0 100644--- a/nodejs/src/common/config/kafka-topics.ts+++ b/nodejs/src/common/config/kafka-topics.ts@@ -55,6 +55,11 @@ export const KAFKA_SESSION_REPLAY_IMAGE_FETCH = `${prefix}session_replay_image_fexport const KAFKA_SESSION_REPLAY_IMAGE_FETCH_RETRY_1M = `${prefix}ai_research_session_replay_image_fetch_retry_1m${suffix}`export const KAFKA_SESSION_REPLAY_IMAGE_FETCH_RETRY_10M = `${prefix}ai_research_session_replay_image_fetch_retry_10m${suffix}`export const KAFKA_SESSION_REPLAY_IMAGE_FETCH_RETRY_1H = `${prefix}ai_research_session_replay_image_fetch_retry_1h${suffix}`+// The v4 image fetch lane: remote image URLs from the v4 recorder, and their delay tiers.+export const KAFKA_AI_RESEARCH_V4_IMAGE_FETCH = `${prefix}ai_research_v4_image_fetch${suffix}`+export const KAFKA_AI_RESEARCH_V4_IMAGE_FETCH_RETRY_1M = `${prefix}ai_research_v4_image_fetch_retry_1m${suffix}`+export const KAFKA_AI_RESEARCH_V4_IMAGE_FETCH_RETRY_10M = `${prefix}ai_research_v4_image_fetch_retry_10m${suffix}`+export const KAFKA_AI_RESEARCH_V4_IMAGE_FETCH_RETRY_1H = `${prefix}ai_research_v4_image_fetch_retry_1h${suffix}`// images the scrub sidecar cannot process, parked so they stop holding the head of their partition.// The original bytes are kept: unscrubbed content must never reach the ML bucket, but it must not bediff --git a/nodejs/src/index.ts b/nodejs/src/index.tsindex 03d6ba6766cb..8eb63a6f4773 100644--- a/nodejs/src/index.ts+++ b/nodejs/src/index.ts@@ -6,6 +6,10 @@ import { initTracing } from '~/common/tracing/otel'import { initSuperProperties } from '~/common/utils/posthog'import { getDefaultIngestionConsumerConfig } from '~/ingestion/config'import { PluginServer } from '~/server'+import {+ AiResearchV4ImageFetchRetryServer,+ AiResearchV4ImageFetchServer,+} from '~/servers/ai-research-v4-image-fetch-server'import { NodeServer } from '~/servers/base-server'import { ErrorTrackingServer } from '~/servers/error-tracking-server'import { IngestionApiServer } from '~/servers/ingestion-api-server'@@ -57,6 +61,12 @@ function createServer(): NodeServer {case PluginServerMode.recordings_blob_ingestion_v2_ml_image_fetch_retry:return new IngestionSessionReplayMlImageFetchRetryServer()+ case PluginServerMode.ai_research_v4_image_fetch:+ return new AiResearchV4ImageFetchServer()++ case PluginServerMode.ai_research_v4_image_fetch_retry:+ return new AiResearchV4ImageFetchRetryServer()+case PluginServerMode.recording_api:return new RecordingApiServer()diff --git a/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/config.ts b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/config.tsnew file mode 100644index 000000000000..c4741bf8c4b9--- /dev/null+++ b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/config.ts@@ -0,0 +1,208 @@+/** Settings of the v4 AI research lanes. */+import { defaultConfig, overrideConfigWithEnv } from '~/common/config/config'+import { KAFKA_CLICKHOUSE_TOPHOG } from '~/common/config/kafka-topics'+import {+ INGESTION_DOWNSTREAM_PRODUCER,+ getDefaultKafkaDownstreamProducerEnvConfig,+} from '~/ingestion/common/outputs/producers'+import { getDefaultIngestionConsumerConfig } from '~/ingestion/config'+import {+ getDefaultSessionRecordingApiConfig,+ getDefaultSessionRecordingConfig,+ getDefaultSessionReplayOutputsConfig,+} from '~/ingestion/pipelines/sessionreplay/config'+import { getDefaultKafkaSessionreplayProducerEnvConfig } from '~/ingestion/pipelines/sessionreplay/shared/outputs/producer-config'+import type { IngestionSessionReplayServerConfig } from '~/servers/ingestion-session-replay-server'++import type { AiResearchV4ImageFetchProducerName } from './image-fetch-v4/producer'++export type AiResearchV4Config = {+ /** The v4 image fetch lane: the team month key table, its KMS key, and the bucket fetched images go to. */+ AI_RESEARCH_V4_KEY_TABLE: string+ AI_RESEARCH_V4_KMS_KEY_ARN: string+ AI_RESEARCH_V4_IMAGES_BUCKET: string+ /** KMS decrypt requests per second one pod may send for team month keys. */+ AI_RESEARCH_V4_KMS_REQUESTS_PER_SECOND: number+ /** The v4 lane's consumer group. Its own setting, so a v4 pod never joins a v3 group. Retry tiers add a suffix. */+ AI_RESEARCH_V4_IMAGE_FETCH_GROUP_ID: string+ /** How long all key lookups of one batch may take. A job whose key is not read by then goes to a delay tier. */+ AI_RESEARCH_V4_IMAGE_FETCH_KEY_BUDGET_MS: number+ /**+ * The share of the per registrable domain limits below that the v4 lane uses. The v3 lane's pods hold their own+ * budget, so a host sees both lanes.+ */+ AI_RESEARCH_V4_IMAGE_FETCH_HOST_BUDGET_SHARE: number++ /** US-only, PEM, multiple keys allowed (comma-separated). */+ AI_RESEARCH_V4_IMAGE_FETCH_WEB_BOT_AUTH_PRIVATE_KEYS: string+ /** While true, the fetch lane parses input but sends no request and writes no crawl history. */+ AI_RESEARCH_V4_IMAGE_FETCH_DRY_RUN: boolean+ /** Optional topic for rejected frontier records. Empty commits and drops rejected input without quarantine. */+ AI_RESEARCH_V4_IMAGE_FETCH_DLQ_TOPIC: string+ AI_RESEARCH_V4_IMAGE_FETCH_BATCH_SIZE: number+ /** Kafka group members per image-fetch worker. */+ AI_RESEARCH_V4_IMAGE_FETCH_TARGET_PARTITIONS_PER_BATCH: number+ /** While true, the batches of all group members in one worker join into one fetch pass. While false, each batch starts its own pass. */+ AI_RESEARCH_V4_IMAGE_FETCH_JOIN_MEMBER_BATCHES: boolean+ /**+ * While true, every batch adds its URLs to one pod-wide candidate pool and finishes on its own,+ * and AI_RESEARCH_V4_IMAGE_FETCH_JOIN_MEMBER_BATCHES has no effect.+ */+ AI_RESEARCH_V4_IMAGE_FETCH_CONTINUOUS_POOL: boolean+ /** A group member reads its next batch once fewer of its queued URLs than this can run. */+ AI_RESEARCH_V4_IMAGE_FETCH_POOL_REFILL_RUNNABLE_URLS: number+ /** A group member also waits while this many of its URLs are queued. The wait stops after 30 s to keep the consumer loop healthy, so a member can pass this by about one batch until pass deadlines remove its queued URLs. */+ AI_RESEARCH_V4_IMAGE_FETCH_POOL_MAX_QUEUED_URLS_PER_MEMBER: number+ /** The lane's own crawl history table: the URL history of v4 references, and robots.txt and tdmrep by origin. */+ AI_RESEARCH_V4_IMAGE_FETCH_HISTORY_TABLE: string+ /** Bounds one DynamoDB request so an unavailable store cannot hold the poll loop. */+ AI_RESEARCH_V4_IMAGE_FETCH_DYNAMODB_TIMEOUT_MS: number+ /** The shortest time before the lane fetches a URL again. */+ AI_RESEARCH_V4_IMAGE_FETCH_CRAWL_HISTORY_TTL_SECONDS: number++ /** Optional steady request rate for one registrable domain. Zero uses the concurrency limit only. */+ AI_RESEARCH_V4_IMAGE_FETCH_REGISTRABLE_DOMAIN_REQUESTS_PER_SECOND: number+ /** Tokens one idle registrable domain can retain when the steady request rate is enabled. */+ AI_RESEARCH_V4_IMAGE_FETCH_REGISTRABLE_DOMAIN_BURST: number+ /** Also the worker count for one registrable domain or origin. */+ AI_RESEARCH_V4_IMAGE_FETCH_MAX_CONCURRENT_PER_REGISTRABLE_DOMAIN: number+ /**+ * Requests this pod holds open across every registrable domain at once.+ *+ * The topic key sends one registrable domain to one partition and one pod. Its rate limit lives+ * on that pod. The state limits prevent unbounded in-memory maps.+ *+ * This bounds the pod instead. A body reads into a buffer, so the resident peak is roughly this+ * times AI_RESEARCH_V4_IMAGE_FETCH_MAX_IMAGE_BYTES, and the sockets and the DNS lookups+ * scale with it too. Raise the pod's memory before you raise this.+ */+ AI_RESEARCH_V4_IMAGE_FETCH_MAX_IN_FLIGHT_REQUESTS: number+ /** Image writes to S3, and republished frontier records, in flight at once. */+ AI_RESEARCH_V4_IMAGE_FETCH_MAX_PENDING_PUBLISHES: number+ /**+ * Wall time the fetch pass of one poll batch may take.+ *+ * A batch can hold more URLs for one registrable domain than a polite rate carries in this+ * time. What the pass does not reach goes to a delay topic without spending a hop. The value+ * sits well inside Kafka's max.poll.interval.ms of 300s, which the crawl history round trips+ * share.+ */+ AI_RESEARCH_V4_IMAGE_FETCH_REQUEST_BUDGET_MS: number+ /** Refused before the body when the response declares more, and abandoned mid-body when it declares nothing. */+ AI_RESEARCH_V4_IMAGE_FETCH_MAX_IMAGE_BYTES: number+ /** Covers one URL including its redirects, separate from the connect timeout of the shared request layer. */+ AI_RESEARCH_V4_IMAGE_FETCH_REQUEST_TIMEOUT_MS: number+ AI_RESEARCH_V4_IMAGE_FETCH_MAX_REDIRECTS: number+ /** Consecutive failures before this pod stops sending to one registrable domain. */+ AI_RESEARCH_V4_IMAGE_FETCH_REGISTRABLE_DOMAIN_BREAKER_FAILURES: number+ AI_RESEARCH_V4_IMAGE_FETCH_REGISTRABLE_DOMAIN_BREAKER_COOLDOWN_MS: number+ /** The longest calculated breaker back-off. A longer valid `Retry-After` remains authoritative. */+ AI_RESEARCH_V4_IMAGE_FETCH_REGISTRABLE_DOMAIN_BREAKER_MAX_COOLDOWN_MS: number+ /** Registrable domains one pod holds request-control state for. */+ AI_RESEARCH_V4_IMAGE_FETCH_MAX_TRACKED_REGISTRABLE_DOMAINS: number+ /** Origins one pod holds configuration and crawl-delay state for. */+ AI_RESEARCH_V4_IMAGE_FETCH_MAX_TRACKED_ORIGINS: number++ /**+ * The delay topic one retry pod drains, and the period every record in it waits.+ *+ * One server runs every tier, so three deployments of one image cover 1 minute, 10 minutes, and+ * one hour. The pod's `max.poll.interval.ms` must exceed the period, or the broker evicts the+ * consumer while it sleeps.+ */+ AI_RESEARCH_V4_IMAGE_FETCH_RETRY_TOPIC: string+ AI_RESEARCH_V4_IMAGE_FETCH_RETRY_DELAY_MS: number+ /** Records released together after one wait based on the latest broker append timestamp. */+ AI_RESEARCH_V4_IMAGE_FETCH_RETRY_BATCH_SIZE: number++ /** Where the lane's TopHog rows go. */+ AI_RESEARCH_V4_IMAGE_FETCH_TOPHOG_TOPIC: string+ AI_RESEARCH_V4_IMAGE_FETCH_TOPHOG_PRODUCER: AiResearchV4ImageFetchProducerName++ /**+ * The lane's own producer on the replay cluster, for frontier and delay tier records. Its broker and security+ * settings are those of the replay producer.+ */+ KAFKA_AI_RESEARCH_V4_IMAGE_FETCH_PRODUCER_COMPRESSION_CODEC: string+ KAFKA_AI_RESEARCH_V4_IMAGE_FETCH_PRODUCER_LINGER_MS: string+ KAFKA_AI_RESEARCH_V4_IMAGE_FETCH_PRODUCER_QUEUE_BUFFERING_MAX_MESSAGES: string+ KAFKA_AI_RESEARCH_V4_IMAGE_FETCH_PRODUCER_QUEUE_BUFFERING_MAX_KBYTES: string+ KAFKA_AI_RESEARCH_V4_IMAGE_FETCH_PRODUCER_MESSAGE_MAX_BYTES: string+ /** A short local delivery timeout, so a slow topic fails a batch fast instead of holding its offsets. */+ KAFKA_AI_RESEARCH_V4_IMAGE_FETCH_PRODUCER_MESSAGE_TIMEOUT_MS: string+}++export function getDefaultAiResearchV4Config(): AiResearchV4Config {+ return {+ AI_RESEARCH_V4_KEY_TABLE: '',+ AI_RESEARCH_V4_KMS_KEY_ARN: '',+ AI_RESEARCH_V4_IMAGES_BUCKET: '',+ AI_RESEARCH_V4_KMS_REQUESTS_PER_SECOND: 150,+ AI_RESEARCH_V4_IMAGE_FETCH_GROUP_ID: 'ai-research-v4-image-fetch',+ AI_RESEARCH_V4_IMAGE_FETCH_KEY_BUDGET_MS: 60_000,+ AI_RESEARCH_V4_IMAGE_FETCH_HOST_BUDGET_SHARE: 0.25,++ AI_RESEARCH_V4_IMAGE_FETCH_WEB_BOT_AUTH_PRIVATE_KEYS: '',+ AI_RESEARCH_V4_IMAGE_FETCH_DRY_RUN: true,+ AI_RESEARCH_V4_IMAGE_FETCH_DLQ_TOPIC: '',+ AI_RESEARCH_V4_IMAGE_FETCH_BATCH_SIZE: 500,+ AI_RESEARCH_V4_IMAGE_FETCH_TARGET_PARTITIONS_PER_BATCH: 2,+ AI_RESEARCH_V4_IMAGE_FETCH_JOIN_MEMBER_BATCHES: true,+ AI_RESEARCH_V4_IMAGE_FETCH_CONTINUOUS_POOL: false,+ AI_RESEARCH_V4_IMAGE_FETCH_POOL_REFILL_RUNNABLE_URLS: 100,+ AI_RESEARCH_V4_IMAGE_FETCH_POOL_MAX_QUEUED_URLS_PER_MEMBER: 2_000,+ AI_RESEARCH_V4_IMAGE_FETCH_HISTORY_TABLE: '',+ AI_RESEARCH_V4_IMAGE_FETCH_DYNAMODB_TIMEOUT_MS: 5_000,+ AI_RESEARCH_V4_IMAGE_FETCH_CRAWL_HISTORY_TTL_SECONDS: 30 * 24 * 60 * 60,+ AI_RESEARCH_V4_IMAGE_FETCH_REGISTRABLE_DOMAIN_REQUESTS_PER_SECOND: 0,+ AI_RESEARCH_V4_IMAGE_FETCH_REGISTRABLE_DOMAIN_BURST: 6,+ AI_RESEARCH_V4_IMAGE_FETCH_MAX_CONCURRENT_PER_REGISTRABLE_DOMAIN: 6,+ AI_RESEARCH_V4_IMAGE_FETCH_MAX_IN_FLIGHT_REQUESTS: 300,+ AI_RESEARCH_V4_IMAGE_FETCH_MAX_PENDING_PUBLISHES: 100,+ AI_RESEARCH_V4_IMAGE_FETCH_REQUEST_BUDGET_MS: 40_000,+ AI_RESEARCH_V4_IMAGE_FETCH_MAX_IMAGE_BYTES: 20 * 1024 * 1024,+ AI_RESEARCH_V4_IMAGE_FETCH_REQUEST_TIMEOUT_MS: 10_000,+ AI_RESEARCH_V4_IMAGE_FETCH_MAX_REDIRECTS: 3,+ AI_RESEARCH_V4_IMAGE_FETCH_REGISTRABLE_DOMAIN_BREAKER_FAILURES: 5,+ AI_RESEARCH_V4_IMAGE_FETCH_REGISTRABLE_DOMAIN_BREAKER_COOLDOWN_MS: 60_000,+ AI_RESEARCH_V4_IMAGE_FETCH_REGISTRABLE_DOMAIN_BREAKER_MAX_COOLDOWN_MS: 60 * 60 * 1000,+ AI_RESEARCH_V4_IMAGE_FETCH_MAX_TRACKED_REGISTRABLE_DOMAINS: 20_000,+ AI_RESEARCH_V4_IMAGE_FETCH_MAX_TRACKED_ORIGINS: 20_000,+ AI_RESEARCH_V4_IMAGE_FETCH_RETRY_TOPIC: '',+ AI_RESEARCH_V4_IMAGE_FETCH_RETRY_DELAY_MS: 60_000,+ AI_RESEARCH_V4_IMAGE_FETCH_RETRY_BATCH_SIZE: 500,++ AI_RESEARCH_V4_IMAGE_FETCH_TOPHOG_TOPIC: KAFKA_CLICKHOUSE_TOPHOG,+ AI_RESEARCH_V4_IMAGE_FETCH_TOPHOG_PRODUCER: INGESTION_DOWNSTREAM_PRODUCER,++ KAFKA_AI_RESEARCH_V4_IMAGE_FETCH_PRODUCER_COMPRESSION_CODEC: 'snappy',+ KAFKA_AI_RESEARCH_V4_IMAGE_FETCH_PRODUCER_LINGER_MS: '',+ KAFKA_AI_RESEARCH_V4_IMAGE_FETCH_PRODUCER_QUEUE_BUFFERING_MAX_MESSAGES: '50000',+ KAFKA_AI_RESEARCH_V4_IMAGE_FETCH_PRODUCER_QUEUE_BUFFERING_MAX_KBYTES: '32768',+ KAFKA_AI_RESEARCH_V4_IMAGE_FETCH_PRODUCER_MESSAGE_MAX_BYTES: `${40 * 1024 * 1024 + 64 * 1024}`,+ KAFKA_AI_RESEARCH_V4_IMAGE_FETCH_PRODUCER_MESSAGE_TIMEOUT_MS: '20000',+ }+}++export function getAiResearchV4Config(env: Record<string, string | undefined> = process.env): AiResearchV4Config {+ return overrideConfigWithEnv(getDefaultAiResearchV4Config(), env)+}++/** Full config of a v4 server: the replay settings it shares with the primary ingester, plus the v4 settings. */+export type AiResearchV4ServerConfig = IngestionSessionReplayServerConfig & AiResearchV4Config++export function buildAiResearchV4ServerConfig(+ config: Partial<AiResearchV4ServerConfig> = {}+): AiResearchV4ServerConfig {+ return {+ ...defaultConfig,+ ...overrideConfigWithEnv(getDefaultIngestionConsumerConfig()),+ ...overrideConfigWithEnv(getDefaultKafkaDownstreamProducerEnvConfig()),+ ...overrideConfigWithEnv(getDefaultKafkaSessionreplayProducerEnvConfig()),+ ...overrideConfigWithEnv(getDefaultSessionRecordingConfig()),+ ...overrideConfigWithEnv(getDefaultSessionRecordingApiConfig()),+ ...overrideConfigWithEnv(getDefaultSessionReplayOutputsConfig()),+ ...getAiResearchV4Config(),+ ...config,+ }+}diff --git a/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/block-reason.ts b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/block-reason.tsnew file mode 100644index 000000000000..3fa500061de1--- /dev/null+++ b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/block-reason.ts@@ -0,0 +1,38 @@+export type ImageFetchBlockReason =+ | 'domain_concurrency'+ | 'request_capacity'+ | 'origin_crawl_delay'+ | 'registrable_domain_rate'+ | 'retry_after'+ | 'transient_backoff'+ | 'unknown_backoff'+ | 'configuration_unreachable'+ | 'configuration_deferred'+ | 'breaker_open'+ | 'connection_limit'+ | 'origin_map_full'+ | 'registrable_domain_map_full'+ | 'pass_deadline'+ | 'request_deadline'++const IMAGE_FETCH_BLOCK_REASONS = new Set<ImageFetchBlockReason>([+ 'domain_concurrency',+ 'request_capacity',+ 'origin_crawl_delay',+ 'registrable_domain_rate',+ 'retry_after',+ 'transient_backoff',+ 'unknown_backoff',+ 'configuration_unreachable',+ 'configuration_deferred',+ 'breaker_open',+ 'connection_limit',+ 'origin_map_full',+ 'registrable_domain_map_full',+ 'pass_deadline',+ 'request_deadline',+])++export function isImageFetchBlockReason(value: unknown): value is ImageFetchBlockReason {+ return typeof value === 'string' && IMAGE_FETCH_BLOCK_REASONS.has(value as ImageFetchBlockReason)+}diff --git a/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/collected-urls-record.test.ts b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/collected-urls-record.test.tsnew file mode 100644index 000000000000..cd7fbca54980--- /dev/null+++ b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/collected-urls-record.test.ts@@ -0,0 +1,162 @@+import {+ FetchCandidate,+ MAX_HOPS,+ MAX_JOBS_PER_RECORD,+ MAX_RECORD_BYTES,+ parseCollectedUrlsRecord,+ serializeFrontierRecord,+} from './collected-urls-record'++const REF = `imageurl:v4:7:2026-10:${'a'.repeat(22)}`++function candidate(overrides: Partial<FetchCandidate> = {}): FetchCandidate {+ return {+ originalRef: REF,+ currentUrl: 'https://cdn.example.com/a.png',+ host: 'cdn.example.com',+ origin: 'https://cdn.example.com',+ registrableDomain: 'example.com',+ remainingHops: MAX_HOPS,+ notBeforeMs: 0,+ firstSeenAtMs: 1_700_000_000_000,+ fetchCount: 0,+ republishCount: 0,+ lastRepublishReason: null,+ ...overrides,+ }+}++function record(overrides: Record<string, unknown> = {}): Buffer {+ return Buffer.from(+ JSON.stringify({+ v: 2,+ jobs: [+ {+ originalRef: REF,+ currentUrl: 'https://cdn.example.com/a.png',+ remainingHops: MAX_HOPS,+ notBeforeMs: 0,+ firstSeenAtMs: 1_700_000_000_000,+ fetchCount: 0,+ republishCount: 0,+ lastRepublishReason: null,+ ...overrides,+ },+ ],+ })+ )+}++describe('frontier record', () => {+ it('round trips the durable job state', () => {+ const durableCandidate = candidate({ lastBlockReason: 'configuration_unreachable' })+ const parsed = parseCollectedUrlsRecord(serializeFrontierRecord([durableCandidate]), 'example.com')++ expect(parsed).toEqual({ ok: true, candidates: [durableCandidate], urlCount: 1, rejected: [], skipped: [] })+ })++ it('does not persist source partition attribution', () => {+ const parsed = parseCollectedUrlsRecord(+ serializeFrontierRecord([candidate({ sourcePartitions: [7] })]),+ 'example.com'+ )++ expect(parsed).toEqual({ ok: true, candidates: [candidate()], urlCount: 1, rejected: [], skipped: [] })+ })++ it('accepts and removes the legacy low-origin-diversity marker', () => {+ expect(parseCollectedUrlsRecord(record({ lowOriginDiversityDeferred: true }), 'example.com')).toEqual({+ ok: true,+ candidates: [candidate()],+ urlCount: 1,+ rejected: [],+ skipped: [],+ })+ })++ it('drops a job with a non-boolean low-origin-diversity marker', () => {+ expect(parseCollectedUrlsRecord(record({ lowOriginDiversityDeferred: 'true' }), 'example.com')).toEqual({+ ok: false,+ reason: 'bad_url',+ })+ })++ it.each([+ ['missing value', null, 'example.com', 'malformed'],+ ['missing key', record(), null, 'malformed'],+ ['invalid JSON', Buffer.from('{'), 'example.com', 'malformed'],+ ['unknown version', Buffer.from('{"v":3,"jobs":[]}'), 'example.com', 'unsupported_version'],+ ['a v3 legacy record', Buffer.from('{"v":1,"urls":[]}'), 'example.com', 'unsupported_version'],+ ['oversized bytes', Buffer.alloc(MAX_RECORD_BYTES + 1), 'example.com', 'oversized_record'],+ [+ 'too many jobs',+ Buffer.from(JSON.stringify({ v: 2, jobs: Array(MAX_JOBS_PER_RECORD + 1).fill({}) })),+ 'example.com',+ 'oversized_record',+ ],+ ])('refuses %s', (_name, value, key, reason) => {+ expect(parseCollectedUrlsRecord(value, key)).toEqual({ ok: false, reason })+ })++ it.each([+ ['global ref', `imageurl:${'a'.repeat(22)}`, 'bad_ref'],+ ['v3 ref', `imageurl:v3:7:2026-10:${'a'.repeat(22)}`, 'bad_ref'],+ ['byte ref', `image:${'b'.repeat(32)}:${'a'.repeat(22)}`, 'bad_ref'],+ ['credentials', 'https://user:[email protected]/a.png', 'bad_url'],+ ['HTTP', 'http://cdn.example.com/a.png', 'bad_url'],+ ['non-default port', 'https://cdn.example.com:444/a.png', 'bad_url'],+ ['private address', 'https://127.0.0.1/a.png', 'bad_url'],+ ['non-canonical URL', 'https://CDN.EXAMPLE.COM/a.png', 'bad_url'],+ ])('drops a job with %s', (_name, value, reason) => {+ const overrides = value.startsWith('image') ? { originalRef: value } : { currentUrl: value }+ const parsed = parseCollectedUrlsRecord(record(overrides), 'example.com')++ expect(parsed).toEqual({ ok: false, reason })+ })++ it('skips a tracking beacon job and keeps the rest of the record', () => {+ const beacon = candidate({+ originalRef: `imageurl:v4:7:2026-10:${'b'.repeat(22)}`,+ currentUrl: 'https://analytics.twitter.com/i/adsct?txn_id=abc&p_id=Twitter',+ host: 'analytics.twitter.com',+ origin: 'https://analytics.twitter.com',+ registrableDomain: 'twitter.com',+ })+ const image = candidate({+ currentUrl: 'https://cdn.twitter.com/logo.png',+ host: 'cdn.twitter.com',+ origin: 'https://cdn.twitter.com',+ registrableDomain: 'twitter.com',+ })++ const parsed = parseCollectedUrlsRecord(serializeFrontierRecord([beacon, image]), 'twitter.com')++ expect(parsed).toEqual({+ ok: true,+ candidates: [image],+ urlCount: 2,+ rejected: [],+ skipped: [{ reason: 'tracking_beacon' }],+ })+ })++ it('drops a job placed on another registrable-domain partition', () => {+ const parsed = parseCollectedUrlsRecord(record(), 'other.net')++ expect(parsed).toEqual({ ok: false, reason: 'foreign_domain' })+ })++ it.each([+ ['remainingHops', MAX_HOPS + 1],+ ['notBeforeMs', -1],+ ['firstSeenAtMs', Number.POSITIVE_INFINITY],+ ['fetchCount', 1.5],+ ['republishCount', Number.MAX_SAFE_INTEGER + 1],+ ['lastRepublishReason', 'unknown'],+ ['lastBlockReason', 'unknown'],+ ])('drops an invalid %s', (field, value) => {+ const parsed = parseCollectedUrlsRecord(record({ [field]: value }), 'example.com')++ expect(parsed).toEqual({ ok: false, reason: 'bad_url' })+ })+})diff --git a/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/collected-urls-record.ts b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/collected-urls-record.tsnew file mode 100644index 000000000000..18a8eb77ed63--- /dev/null+++ b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/collected-urls-record.ts@@ -0,0 +1,239 @@+import type { UrlPolicyDecline } from '@posthog/ai-research-v4-native'++import { parseJSON } from '~/common/utils/json-parse'++import { ImageFetchBlockReason, isImageFetchBlockReason } from './block-reason'+import { tryCanonicalizeUrl } from './politeness-key'+import { parseV4UrlRef } from './v4-url-ref'++export const MAX_HOPS = 10+export const MAX_JOBS_PER_RECORD = 1_000+export const MAX_RECORD_BYTES = 512 * 1024++export type UrlDropReason =+ | 'malformed'+ | 'unsupported_version'+ | 'bad_ref'+ | 'bad_url'+ | 'foreign_domain'+ | 'oversized_record'+/** Why the parser dropped a job on its own, without rejecting the record that carries it. */+export type UrlSkipReason = UrlPolicyDecline+export type StoredRepublishReason =+ | 'redirect'+ | 'retry'+ | 'not_ready'+ | 'pass_deadline'+ | 'origin_map_full'+ | 'registrable_domain_map_full'+export type RepublishReason = StoredRepublishReason++export interface FetchCandidate {+ /** Raw UUIDv7 session ID of the session that collected the URL, carried through republishes. */+ sessionId?: string+ originalRef: string+ currentUrl: string+ host: string+ origin: string+ registrableDomain: string+ remainingHops: number+ notBeforeMs: number+ firstSeenAtMs: number+ fetchCount: number+ republishCount: number+ lastRepublishReason: StoredRepublishReason | null+ lastBlockReason?: ImageFetchBlockReason+ sourcePartitions?: readonly number[]+}++export interface FrontierRecord {+ v: 2+ jobs: Array<+ Pick<+ FetchCandidate,+ | 'sessionId'+ | 'originalRef'+ | 'currentUrl'+ | 'remainingHops'+ | 'notBeforeMs'+ | 'firstSeenAtMs'+ | 'fetchCount'+ | 'republishCount'+ | 'lastRepublishReason'+ | 'lastBlockReason'+ >+ >+}++export type RecordParse =+ | {+ ok: true+ candidates: FetchCandidate[]+ urlCount: number+ rejected: { reason: UrlDropReason }[]+ skipped: { reason: UrlSkipReason }[]+ }+ | {+ ok: false+ reason: Extract<+ UrlDropReason,+ 'malformed' | 'unsupported_version' | 'oversized_record' | 'bad_ref' | 'bad_url' | 'foreign_domain'+ >+ }++function isRecord(value: unknown): value is Record<string, unknown> {+ return typeof value === 'object' && value !== null && !Array.isArray(value)+}++function isNonNegativeSafeInteger(value: unknown): value is number {+ return Number.isSafeInteger(value) && (value as number) >= 0+}++function isStoredRepublishReason(value: unknown): value is StoredRepublishReason | null {+ return (+ value === null ||+ value === 'redirect' ||+ value === 'retry' ||+ value === 'not_ready' ||+ value === 'pass_deadline' ||+ value === 'origin_map_full' ||+ value === 'registrable_domain_map_full'+ )+}++export function parseCollectedUrlsRecord(value: Buffer | null, key: string | null): RecordParse {+ if (!value || !key) {+ return { ok: false, reason: 'malformed' }+ }+ if (value.length > MAX_RECORD_BYTES) {+ return { ok: false, reason: 'oversized_record' }+ }++ let parsed: unknown+ try {+ parsed = parseJSON(value.toString())+ } catch {+ return { ok: false, reason: 'malformed' }+ }+ if (!isRecord(parsed)) {+ return { ok: false, reason: 'malformed' }+ }+ if (parsed.v !== 2 && !(parsed.v === 1 && Array.isArray(parsed.jobs))) {+ return { ok: false, reason: 'unsupported_version' }+ }+ if (!Array.isArray(parsed.jobs) || parsed.jobs.length === 0) {+ return { ok: false, reason: 'malformed' }+ }+ if (parsed.jobs.length > MAX_JOBS_PER_RECORD) {+ return { ok: false, reason: 'oversized_record' }+ }++ const candidates: FetchCandidate[] = []+ const rejected: { reason: UrlDropReason }[] = []+ const skipped: { reason: UrlSkipReason }[] = []+ for (const job of parsed.jobs) {+ const parsedJob = parseJob(job, key)+ if (parsedJob.kind === 'rejected') {+ return { ok: false, reason: parsedJob.reason }+ }+ if (parsedJob.kind === 'skipped') {+ skipped.push({ reason: parsedJob.reason })+ continue+ }+ candidates.push(parsedJob.candidate)+ }+ return { ok: true, candidates, urlCount: parsed.jobs.length, rejected, skipped }+}++type ParsedJob =+ | { kind: 'candidate'; candidate: FetchCandidate }+ | { kind: 'rejected'; reason: Extract<UrlDropReason, 'bad_ref' | 'bad_url' | 'foreign_domain'> }+ | { kind: 'skipped'; reason: UrlSkipReason }++function parseJob(job: unknown, kafkaKey: string): ParsedJob {+ if (!isRecord(job)) {+ return { kind: 'rejected', reason: 'bad_url' }+ }+ const {+ sessionId,+ originalRef,+ currentUrl,+ remainingHops,+ notBeforeMs,+ firstSeenAtMs,+ fetchCount,+ republishCount,+ lastRepublishReason,+ lastBlockReason,+ lowOriginDiversityDeferred,+ } = job+ if (+ (sessionId !== undefined && typeof sessionId !== 'string') ||+ typeof originalRef !== 'string' ||+ typeof currentUrl !== 'string' ||+ !isNonNegativeSafeInteger(remainingHops) ||+ remainingHops > MAX_HOPS ||+ !isNonNegativeSafeInteger(notBeforeMs) ||+ !isNonNegativeSafeInteger(firstSeenAtMs) ||+ !isNonNegativeSafeInteger(fetchCount) ||+ !isNonNegativeSafeInteger(republishCount) ||+ !isStoredRepublishReason(lastRepublishReason) ||+ (lastBlockReason !== undefined && !isImageFetchBlockReason(lastBlockReason)) ||+ (lowOriginDiversityDeferred !== undefined && typeof lowOriginDiversityDeferred !== 'boolean')+ ) {+ return { kind: 'rejected', reason: 'bad_url' }+ }+ if (!parseV4UrlRef(originalRef)) {+ return { kind: 'rejected', reason: 'bad_ref' }+ }+ const verdict = tryCanonicalizeUrl(currentUrl)+ if (!verdict.ok) {+ // A rejected job sends its whole record to the dead-letter topic, and a record can hold an+ // unwanted URL next to real images, so an unwanted URL is skipped on its own instead.+ return verdict.unwanted ? { kind: 'skipped', reason: verdict.decline } : { kind: 'rejected', reason: 'bad_url' }+ }+ const canonical = verdict.url+ if (canonical.fetch !== currentUrl) {+ return { kind: 'rejected', reason: 'bad_url' }+ }+ if (canonical.domain !== kafkaKey) {+ return { kind: 'rejected', reason: 'foreign_domain' }+ }+ return {+ kind: 'candidate',+ candidate: {+ ...(sessionId ? { sessionId } : {}),+ originalRef,+ currentUrl,+ host: canonical.host,+ origin: new URL(currentUrl).origin,+ registrableDomain: canonical.domain,+ remainingHops,+ notBeforeMs,+ firstSeenAtMs,+ fetchCount,+ republishCount,+ lastRepublishReason,+ lastBlockReason,+ },+ }+}++export function serializeFrontierRecord(candidates: FetchCandidate[]): Buffer {+ const record: FrontierRecord = {+ v: 2,+ jobs: candidates.map((candidate) => ({+ ...(candidate.sessionId ? { sessionId: candidate.sessionId } : {}),+ originalRef: candidate.originalRef,+ currentUrl: candidate.currentUrl,+ remainingHops: candidate.remainingHops,+ notBeforeMs: candidate.notBeforeMs,+ firstSeenAtMs: candidate.firstSeenAtMs,+ fetchCount: candidate.fetchCount,+ republishCount: candidate.republishCount,+ lastRepublishReason: candidate.lastRepublishReason,+ lastBlockReason: candidate.lastBlockReason,+ })),+ }+ return Buffer.from(JSON.stringify(record))+}diff --git a/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/configuration-policy.test.ts b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/configuration-policy.test.tsnew file mode 100644index 000000000000..d3656d8c5eb9--- /dev/null+++ b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/configuration-policy.test.ts@@ -0,0 +1,699 @@+import { register } from 'prom-client'++import { StreamedResponse, fetchStreamed } from '~/common/utils/request'++import {+ ConfigurationFetchResult,+ ConfigurationPolicyService,+ HttpConfigurationFetcher,+ explicitFreshnessLifetimeMs,+ parseRobotsPolicy,+ responseOptOutReason,+} from './configuration-policy'+import { ConfigurationCacheItem, ConfigurationFile, configurationCacheKey } from './crawl-history'++jest.mock('~/common/utils/request', () => ({+ ...jest.requireActual('~/common/utils/request'),+ fetchStreamed: jest.fn(),+}))++const fetchStreamedMock = fetchStreamed as jest.MockedFunction<typeof fetchStreamed>++const NOW_MS = Date.parse('2026-08-22T12:00:00Z')+const ORIGIN = 'https://example.com'++function cached(+ file: ConfigurationFile,+ status: ConfigurationCacheItem['status'],+ body?: string+): ConfigurationCacheItem {+ return {+ kind: file,+ key: configurationCacheKey(ORIGIN, file),+ origin: ORIGIN,+ status,+ body,+ fetchedAtMs: NOW_MS,+ refreshAtMs: NOW_MS + 23 * 60 * 60 * 1000,+ freshUntilMs: NOW_MS + 24 * 60 * 60 * 1000,+ retryAtMs: NOW_MS,+ storageExpiresAtMs: NOW_MS + 30 * 24 * 60 * 60 * 1000,+ }+}++function service(results: Partial<Record<ConfigurationFile, ConfigurationFetchResult>> = {}): {+ policy: ConfigurationPolicyService+ fetch: jest.Mock+} {+ const fetch = jest.fn((_: string, file: ConfigurationFile) =>+ Promise.resolve(results[file] ?? ({ outcome: 'absent' } as const))+ )+ return {+ policy: new ConfigurationPolicyService({ fetch } as unknown as HttpConfigurationFetcher),+ fetch,+ }+}++function response(+ status: number,+ headerLines: Array<{ name: string; value: string }> = [],+ body: { bytes: Buffer; overLimit: boolean } = { bytes: Buffer.alloc(0), overLimit: false }+): StreamedResponse {+ return {+ status,+ headers: Object.fromEntries(headerLines.map(({ name, value }) => [name, value])),+ headerLines,+ read: jest.fn().mockResolvedValue(body),+ discard: jest.fn(),+ }+}++function httpFetcher(headersForGet = jest.fn().mockReturnValue({})): HttpConfigurationFetcher {+ return new HttpConfigurationFetcher(+ { headersForGet },+ {+ run: async (_url, _deadlineMs, request) => ({ ran: true as const, value: await request() }),+ },+ 5_000+ )+}++describe('robots policy', () => {+ it('uses the RFC 9309 group and path matcher', async () => {+ const body = [+ 'User-agent: *',+ 'Disallow: /wildcard',+ 'User-agent: PostHogImageFetcherBot',+ 'Disallow: /private',+ 'Allow: /private/public.png',+ ].join('\n')++ await expect(parseRobotsPolicy(body, `${ORIGIN}/private/image.png`)).resolves.toMatchObject({+ allowed: false,+ reason: 'robots_disallow',+ })+ await expect(parseRobotsPolicy(body, `${ORIGIN}/private/public.png`)).resolves.toMatchObject({+ allowed: true,+ crawlDelayMs: 0,+ })+ await expect(parseRobotsPolicy(body, `${ORIGIN}/wildcard/image.png`)).resolves.toMatchObject({+ allowed: true,+ crawlDelayMs: 0,+ })+ })++ it('uses the greatest valid crawl delay in every selected field line', async () => {+ const body = [+ 'User-agent: PostHogImageFetcherBot',+ 'Crawl-delay: 0.25',+ 'Crawl-delay: invalid',+ 'Crawl-delay: 0x10',+ 'Crawl-delay: 1e3',+ 'Crawl-delay: 2.5',+ // 16.1 * 1000 is 16100.000000000002, which is not a safe integer.+ 'Crawl-delay: 16.1',+ 'Allow: /',+ ].join('\n')++ const result = await parseRobotsPolicy(body, `${ORIGIN}/image.png`)+ expect(result).toMatchObject({+ allowed: true,+ crawlDelayMs: 16_100,+ })+ })++ it.each([+ ['a token refusal', 'train-ai=n', false],+ ['a later allowance', 'train-ai=n, train-ai=y', true],+ ['a quoted value', 'train-ai="n"', true],+ ['a boolean value', 'train-ai=?0', true],+ ['an invalid dictionary', 'train-ai=n, INVALID=y', true],+ ])('applies Content-Usage with %s', async (_name, value, allowed) => {+ const body = `User-agent: *\nContent-Usage: ${value}\nAllow: /`++ await expect(parseRobotsPolicy(body, `${ORIGIN}/image.png`)).resolves.toMatchObject({ allowed })+ })++ it('applies a path-scoped Content-Signal only to a matching URL', async () => {+ const body = 'User-agent: *\nContent-Signal: /private/ ai-train=no, search=yes\nAllow: /'++ await expect(parseRobotsPolicy(body, `${ORIGIN}/private/image.png`)).resolves.toMatchObject({+ allowed: false,+ reason: 'content_signal',+ })+ await expect(parseRobotsPolicy(body, `${ORIGIN}/public/image.png`)).resolves.toMatchObject({ allowed: true })+ })+})++describe('response opt-out policy', () => {+ it('uses the last duplicate Content-Usage dictionary member', () => {+ expect(+ responseOptOutReason(+ [+ { name: 'content-usage', value: 'train-ai=n' },+ { name: 'content-usage', value: 'train-ai=y' },+ ],+ false+ )+ ).toBeUndefined()+ expect(+ responseOptOutReason(+ [+ { name: 'content-usage', value: 'train-ai=y' },+ { name: 'content-usage', value: 'train-ai=n' },+ ],+ false+ )+ ).toBe('content_usage')+ })++ it('lets the response TDM value override the tdmrep file', () => {+ expect(responseOptOutReason([{ name: 'tdm-reservation', value: '0' }], true)).toBeUndefined()+ expect(responseOptOutReason([{ name: 'tdm-reservation', value: '1' }], false)).toBe('tdm_reservation')+ })++ it.each([+ ['a scope for this bot', 'PostHogImageFetcherBot: noindex, noimageai', 'x_robots_tag'],+ ['a scope for another bot', 'OtherBot: noai', undefined],+ [+ 'no scope beside a colon-bearing directive',+ 'noai, unavailable_after: 25 Jun 2026 15:00:00 PST',+ 'x_robots_tag',+ ],+ [+ 'a scope for another bot beside a colon-bearing directive',+ 'OtherBot: noai, unavailable_after: 25 Jun 2026 15:00:00 PST',+ undefined,+ ],+ [+ 'no scope after a leading valued directive',+ 'unavailable_after: 25 Jun 2026 15:00:00 PST, noai',+ 'x_robots_tag',+ ],+ ['no scope after a leading max-image-preview', 'max-image-preview: large, noai', 'x_robots_tag'],+ [+ 'a scope for another bot before a valued directive',+ 'OtherBot: unavailable_after: 25 Jun 2026 15:00:00 PST, noai',+ undefined,+ ],+ ])('applies an X-Robots-Tag value with %s', (_name, value, expected) => {+ expect(responseOptOutReason([{ name: 'x-robots-tag', value }], false)).toBe(expected)+ })+})++describe('ConfigurationPolicyService', () => {+ beforeEach(() => register.resetMetrics())+ it('checks both files, one request at a time, before it allows an image request', async () => {+ const { policy, fetch } = service()+ let active = 0+ let mostActive = 0+ fetch.mockImplementation(async () => {+ mostActive = Math.max(mostActive, ++active)+ await new Promise((resolve) => setImmediate(resolve))+ active -= 1+ return { outcome: 'absent' }+ })++ await expect(policy.check(`${ORIGIN}/image.png`, new Map(), NOW_MS)).resolves.toMatchObject({ allowed: true })+ expect(fetch.mock.calls).toEqual([+ [ORIGIN, 'robots'],+ [ORIGIN, 'tdmrep'],+ ])+ // The v4 lane allows one connection per registrable domain by default, so a second request would be refused.+ expect(mostActive).toBe(1)+ })++ it('uses a cached configuration without a new request', async () => {+ const { policy, fetch } = service()+ const cache = new Map([+ [configurationCacheKey(ORIGIN, 'robots'), cached('robots', 'absent')],+ [configurationCacheKey(ORIGIN, 'tdmrep'), cached('tdmrep', 'absent')],+ ])++ await expect(policy.check(`${ORIGIN}/image.png`, cache, NOW_MS)).resolves.toMatchObject({ allowed: true })+ expect(fetch).not.toHaveBeenCalled()+ const lookups = await register.getSingleMetric('ai_research_v4_image_fetch_configuration_lookups_total')!.get()+ expect(lookups.values).toEqual(+ expect.arrayContaining([+ expect.objectContaining({ labels: { file: 'robots', source: 'cache', outcome: 'absent' }, value: 1 }),+ expect.objectContaining({ labels: { file: 'tdmrep', source: 'cache', outcome: 'absent' }, value: 1 }),+ ])+ )+ })++ it('counts shared configuration requests separately from new requests', async () => {+ let resolve!: (result: ConfigurationFetchResult) => void+ const pending = new Promise<ConfigurationFetchResult>((done) => {+ resolve = done+ })+ const fetch = jest.fn(() => pending)+ const policy = new ConfigurationPolicyService({ fetch } as unknown as HttpConfigurationFetcher)+ const first = policy.check(`${ORIGIN}/first.png`, new Map(), NOW_MS)+ const second = policy.check(`${ORIGIN}/second.png`, new Map(), NOW_MS)+ expect(fetch).toHaveBeenCalledTimes(1)+ resolve({ outcome: 'unreachable' })+ await Promise.all([first, second])+ expect(fetch).toHaveBeenCalledTimes(2)+ const lookups = await register.getSingleMetric('ai_research_v4_image_fetch_configuration_lookups_total')!.get()+ for (const file of ['robots', 'tdmrep']) {+ expect(lookups.values).toEqual(+ expect.arrayContaining([+ expect.objectContaining({ labels: { file, source: 'network', outcome: 'unreachable' }, value: 1 }),+ expect.objectContaining({ labels: { file, source: 'shared', outcome: 'unreachable' }, value: 1 }),+ ])+ )+ }+ })++ it('parses each cached policy revision once in a fetch pass', async () => {+ const { ParsedRobots } = await import('@trybyte/robotstxt-parser')+ const parseRobots = jest.spyOn(ParsedRobots, 'parse')+ const parseJson = jest.spyOn(JSON, 'parse')+ const { policy } = service()+ const robotsBody = 'User-agent: *\nAllow: /'+ const tdmrepBody = JSON.stringify([{ location: '/', 'tdm-reservation': 0 }])+ const cache = new Map([+ [configurationCacheKey(ORIGIN, 'robots'), cached('robots', 'available', robotsBody)],+ [configurationCacheKey(ORIGIN, 'tdmrep'), cached('tdmrep', 'available', tdmrepBody)],+ ])++ try {+ const pass = policy.createPass()+ await pass.check(`${ORIGIN}/first.png`, cache, NOW_MS)+ await pass.check(`${ORIGIN}/second.png`, cache, NOW_MS)++ expect(parseRobots.mock.calls.filter(([body]) => body === robotsBody)).toHaveLength(1)+ expect(parseJson.mock.calls.filter(([body]) => body === tdmrepBody)).toHaveLength(1)++ const revisedCache = new Map(+ [...cache].map(([key, item]) => [key, { ...item, fetchedAtMs: item.fetchedAtMs + 1 }] as const)+ )+ await pass.check(`${ORIGIN}/third.png`, revisedCache, NOW_MS)++ expect(parseRobots.mock.calls.filter(([body]) => body === robotsBody)).toHaveLength(2)+ expect(parseJson.mock.calls.filter(([body]) => body === tdmrepBody)).toHaveLength(2)++ await policy.createPass().check(`${ORIGIN}/fourth.png`, cache, NOW_MS)++ expect(parseRobots.mock.calls.filter(([body]) => body === robotsBody)).toHaveLength(3)+ expect(parseJson.mock.calls.filter(([body]) => body === tdmrepBody)).toHaveLength(3)+ } finally {+ parseRobots.mockRestore()+ parseJson.mockRestore()+ }+ })++ it('uses the first matching TDMRep rule and honors the end anchor', async () => {+ const { policy } = service()+ const cache = new Map([+ [configurationCacheKey(ORIGIN, 'robots'), cached('robots', 'absent')],+ [+ configurationCacheKey(ORIGIN, 'tdmrep'),+ cached(+ 'tdmrep',+ 'available',+ JSON.stringify([+ { location: '/private/exact$', 'tdm-reservation': 0 },+ { location: '/private/*', 'tdm-reservation': 1 },+ ])+ ),+ ],+ ])++ await expect(policy.check(`${ORIGIN}/private/exact`, cache, NOW_MS)).resolves.toMatchObject({ allowed: true })+ await expect(policy.check(`${ORIGIN}/private/exact-more`, cache, NOW_MS)).resolves.toMatchObject({+ allowed: true,+ tdmrepReservation: true,+ })+ })++ it.each([+ ['1', true],+ [true, true],+ [0, false],+ ])('reads a TDMRep reservation of %p as %p', async (reservation, tdmrepReservation) => {+ const { policy } = service()+ const cache = new Map([+ [configurationCacheKey(ORIGIN, 'robots'), cached('robots', 'absent')],+ [+ configurationCacheKey(ORIGIN, 'tdmrep'),+ cached('tdmrep', 'available', JSON.stringify([{ location: '/', 'tdm-reservation': reservation }])),+ ],+ ])++ await expect(policy.check(`${ORIGIN}/image.png`, cache, NOW_MS)).resolves.toMatchObject({+ allowed: true,+ tdmrepReservation,+ })+ })++ it.each([+ {+ description: 'keeps a cached TDMRep file',+ cachedTdmrep: [+ [+ configurationCacheKey(ORIGIN, 'tdmrep'),+ {+ ...cached('tdmrep', 'available', JSON.stringify([{ location: '/', 'tdm-reservation': 1 }])),+ refreshAtMs: NOW_MS - 1,+ },+ ] as const,+ ],+ status: 'available',+ tdmrepReservation: true,+ },+ { description: 'records an absent TDMRep file', cachedTdmrep: [], status: 'absent', tdmrepReservation: false },+ ])('$description when a refresh returns a non-array body', async ({ cachedTdmrep, status, tdmrepReservation }) => {+ const { policy } = service({ tdmrep: { outcome: 'absent', nonArrayBody: true } })+ const cache = new Map([[configurationCacheKey(ORIGIN, 'robots'), cached('robots', 'absent')], ...cachedTdmrep])++ await expect(policy.check(`${ORIGIN}/image.png`, cache, NOW_MS)).resolves.toMatchObject({+ allowed: true,+ tdmrepReservation,+ updates: [expect.objectContaining({ kind: 'tdmrep', status })],+ })+ })++ it('keeps a previous usable file when its refresh is unreachable', async () => {+ const { policy } = service({ robots: { outcome: 'unreachable' } })+ const previous = { ...cached('robots', 'absent'), refreshAtMs: NOW_MS - 1 }+ const cache = new Map([+ [configurationCacheKey(ORIGIN, 'robots'), previous],+ [configurationCacheKey(ORIGIN, 'tdmrep'), cached('tdmrep', 'absent')],+ ])++ await expect(policy.check(`${ORIGIN}/image.png`, cache, NOW_MS)).resolves.toMatchObject({+ allowed: true,+ updates: [expect.objectContaining({ kind: 'robots', status: 'absent', retryAtMs: NOW_MS + 3_600_000 })],+ })+ })++ it('returns a transient refusal when no configuration file is reachable', async () => {+ const { policy } = service({ robots: { outcome: 'unreachable' } })++ await expect(policy.check(`${ORIGIN}/image.png`, new Map(), NOW_MS)).resolves.toMatchObject({+ allowed: false,+ transient: true,+ reason: 'configuration_unreachable',+ })+ })++ it('preserves a registrable-domain state deferral', async () => {+ const { policy } = service({+ robots: { outcome: 'deferred', reason: 'registrable_domain_map_full' },+ })++ await expect(policy.check(`${ORIGIN}/image.png`, new Map(), NOW_MS)).resolves.toMatchObject({+ allowed: false,+ transient: true,+ reason: 'registrable_domain_map_full',+ })+ })++ it('keeps a terminal refusal when the other configuration file is unreachable', async () => {+ const { policy } = service({+ robots: {+ outcome: 'available',+ body: 'User-agent: *\nDisallow: /private',+ cache: { requestTimeMs: NOW_MS, responseTimeMs: NOW_MS },+ },+ tdmrep: { outcome: 'unreachable' },+ })++ await expect(policy.check(`${ORIGIN}/private/image.png`, new Map(), NOW_MS)).resolves.toMatchObject({+ allowed: false,+ transient: false,+ reason: 'robots_disallow',+ })+ })+})++describe('HttpConfigurationFetcher', () => {+ beforeEach(() => {+ fetchStreamedMock.mockReset()+ register.resetMetrics()+ })++ it.each([+ [404, 'absent', 'absent'],+ [410, 'absent', 'absent'],+ [401, 'refused', 'http_401'],+ [403, 'refused', 'http_403'],+ [429, 'unreachable', 'http_429'],+ [500, 'unreachable', 'http_5xx'],+ [204, 'unreachable', 'unexpected_status'],+ ])('maps HTTP %s to %s', async (status, outcome, reason) => {+ fetchStreamedMock.mockResolvedValue(response(status))++ await expect(httpFetcher().fetch(ORIGIN, 'robots')).resolves.toMatchObject({ outcome })+ const fetches = await register.getSingleMetric('ai_research_v4_image_fetch_configuration_fetches_total')!.get()+ expect(fetches.values).toEqual([+ expect.objectContaining({ labels: { file: 'robots', outcome, reason }, value: 1 }),+ ])+ })++ it('uses the retained prefix when robots.txt exceeds its byte limit', async () => {+ fetchStreamedMock.mockResolvedValue(+ response(200, [], {+ bytes: Buffer.concat([+ Buffer.from('User-agent: *\nDisallow: /private'),+ Buffer.from([0xf0, 0x9f, 0x98]),+ ]),+ overLimit: true,+ })+ )++ await expect(httpFetcher().fetch(ORIGIN, 'robots')).resolves.toMatchObject({+ outcome: 'available',+ body: 'User-agent: *\nDisallow: /private',+ })+ })++ it('treats invalid UTF-8 configuration text as unreachable', async () => {+ fetchStreamedMock.mockResolvedValue(+ response(200, [], { bytes: Buffer.from([0x75, 0x73, 0x65, 0x72, 0xff]), overLimit: false })+ )++ await expect(httpFetcher().fetch(ORIGIN, 'robots')).resolves.toMatchObject({ outcome: 'unreachable' })+ })++ it.each([+ {+ description: 'a string reservation',+ bytes: Buffer.from('[{"location": "/", "tdm-reservation": "1"}]'),+ overLimit: false,+ result: { outcome: 'available' },+ reason: 'available',+ },+ {+ description: 'a byte order mark and leading whitespace',+ bytes: Buffer.from('\uFEFF\n []'),+ overLimit: false,+ result: { outcome: 'available' },+ reason: 'available',+ },+ {+ description: 'a damaged array',+ bytes: Buffer.from('[{"location": "/",'),+ overLimit: false,+ result: { outcome: 'unreachable' },+ reason: 'invalid_document',+ },+ {+ description: 'an oversized array',+ bytes: Buffer.from('[]'),+ overLimit: true,+ result: { outcome: 'unreachable' },+ reason: 'body_limit',+ },+ {+ description: 'an oversized whitespace prefix',+ bytes: Buffer.from('\uFEFF \n\t '),+ overLimit: true,+ result: { outcome: 'unreachable' },+ reason: 'body_limit',+ },+ {+ description: 'an empty body',+ bytes: Buffer.alloc(0),+ overLimit: false,+ result: { outcome: 'absent', nonArrayBody: true },+ reason: 'not_json_array',+ },+ {+ description: 'an HTML page',+ bytes: Buffer.from('<!doctype html><title>Home</title>'),+ overLimit: false,+ result: { outcome: 'absent', nonArrayBody: true },+ reason: 'not_json_array',+ },+ {+ description: 'a JSON object',+ bytes: Buffer.from('{"error": "not found"}'),+ overLimit: false,+ result: { outcome: 'absent', nonArrayBody: true },+ reason: 'not_json_array',+ },+ {+ description: 'an oversized HTML page',+ bytes: Buffer.from('<!doctype html>'),+ overLimit: true,+ result: { outcome: 'absent', nonArrayBody: true },+ reason: 'not_json_array',+ },+ {+ description: 'a Latin-1 HTML page',+ bytes: Buffer.from('<p>caf\u00e9</p>', 'latin1'),+ overLimit: false,+ result: { outcome: 'absent', nonArrayBody: true },+ reason: 'not_json_array',+ },+ ])('maps a TDMRep body with $description to $result.outcome', async ({ bytes, overLimit, result, reason }) => {+ fetchStreamedMock.mockResolvedValue(response(200, [], { bytes, overLimit }))++ await expect(httpFetcher().fetch(ORIGIN, 'tdmrep')).resolves.toEqual(expect.objectContaining(result))+ const fetches = await register.getSingleMetric('ai_research_v4_image_fetch_configuration_fetches_total')!.get()+ expect(fetches.values).toEqual([+ expect.objectContaining({ labels: { file: 'tdmrep', outcome: result.outcome, reason }, value: 1 }),+ ])+ })++ it('treats repeated Location field lines as unreachable', async () => {+ fetchStreamedMock.mockResolvedValue(+ response(302, [+ { name: 'location', value: '/first' },+ { name: 'location', value: '/second' },+ ])+ )++ await expect(httpFetcher().fetch(ORIGIN, 'robots')).resolves.toMatchObject({ outcome: 'unreachable' })+ expect(fetchStreamedMock).toHaveBeenCalledTimes(1)+ })++ it('follows, signs and identifies each configuration redirect target', async () => {+ fetchStreamedMock+ .mockResolvedValueOnce(response(302, [{ name: 'location', value: 'https://cdn.example.com/robots' }]))+ .mockResolvedValueOnce(+ response(200, [], { bytes: Buffer.from('User-agent: *\nAllow: /'), overLimit: false })+ )+ const headersForGet = jest+ .fn()+ .mockReturnValueOnce({ signature: 'first' })+ .mockReturnValueOnce({ signature: 'second' })++ await expect(httpFetcher(headersForGet).fetch(ORIGIN, 'robots')).resolves.toMatchObject({+ outcome: 'available',+ })+ expect(headersForGet.mock.calls).toEqual([+ ['https://example.com/robots.txt'],+ ['https://cdn.example.com/robots'],+ ])+ expect(fetchStreamedMock.mock.calls.map(([, options]) => options.allowH2)).toEqual([true, true])+ expect(fetchStreamedMock.mock.calls.map(([, options]) => options.headers)).toEqual([+ expect.objectContaining({ referer: 'https://us.posthog.com/' }),+ expect.objectContaining({ referer: 'https://us.posthog.com/' }),+ ])+ })++ it('does not follow a configuration redirect to another registrable domain', async () => {+ fetchStreamedMock.mockResolvedValueOnce(+ response(302, [{ name: 'location', value: 'https://cdn.example.net/robots' }])+ )+ const headersForGet = jest.fn().mockReturnValue({ signature: 'value' })++ await expect(httpFetcher(headersForGet).fetch(ORIGIN, 'robots')).resolves.toMatchObject({+ outcome: 'unreachable',+ })+ expect(fetchStreamedMock).toHaveBeenCalledTimes(1)+ expect(headersForGet).toHaveBeenCalledTimes(1)+ })+})++describe('explicitFreshnessLifetimeMs', () => {+ it('uses s-maxage and subtracts the corrected current age', () => {+ expect(+ explicitFreshnessLifetimeMs(+ {+ requestTimeMs: NOW_MS - 2_000,+ responseTimeMs: NOW_MS,+ date: new Date(NOW_MS - 1_000).toUTCString(),+ age: '10',+ cacheControl: 'max-age=100, s-maxage=200',+ },+ NOW_MS+ )+ ).toBe(188_000)+ })++ it.each(['no-cache', 'no-store', 'private', 'must-revalidate'])('does not extend freshness for %s', (directive) => {+ expect(+ explicitFreshnessLifetimeMs(+ {+ requestTimeMs: NOW_MS,+ responseTimeMs: NOW_MS,+ cacheControl: `${directive}, max-age=999999`,+ },+ NOW_MS+ )+ ).toBe(0)+ })++ it.each([+ ['duplicate max-age', 'max-age=60, max-age=120'],+ ['decimal max-age', 'max-age=1.5'],+ ['exponent max-age', 'max-age=1e3'],+ ['unsafe max-age', 'max-age=9007199254740992'],+ ['invalid s-maxage before a valid max-age', 's-maxage=invalid, max-age=3600'],+ ])('does not extend freshness for %s', (_name, cacheControl) => {+ expect(+ explicitFreshnessLifetimeMs(+ {+ requestTimeMs: NOW_MS,+ responseTimeMs: NOW_MS,+ cacheControl,+ },+ NOW_MS+ )+ ).toBe(0)+ })++ it('uses every combined Cache-Control field line', () => {+ expect(+ explicitFreshnessLifetimeMs(+ {+ requestTimeMs: NOW_MS,+ responseTimeMs: NOW_MS,+ cacheControl: 'public, max-age=86400, no-store',+ },+ NOW_MS+ )+ ).toBe(0)+ })++ it('ignores a decimal Age value', () => {+ expect(+ explicitFreshnessLifetimeMs(+ {+ requestTimeMs: NOW_MS,+ responseTimeMs: NOW_MS,+ age: '1.5',+ cacheControl: 'max-age=10',+ },+ NOW_MS+ )+ ).toBe(10_000)+ })++ it('subtracts time spent resident after the response arrived', () => {+ expect(+ explicitFreshnessLifetimeMs(+ {+ requestTimeMs: NOW_MS,+ responseTimeMs: NOW_MS,+ cacheControl: 'max-age=10',+ },+ NOW_MS + 4_000+ )+ ).toBe(6_000)+ })+})diff --git a/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/configuration-policy.ts b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/configuration-policy.tsnew file mode 100644index 000000000000..d3f4ba19b2b9--- /dev/null+++ b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/configuration-policy.ts@@ -0,0 +1,806 @@+import type { ParsedRobots, RobotsParseHandler } from '@trybyte/robotstxt-parser'+import { Token, parseDictionary } from 'structured-headers'++import { parseJSON } from '~/common/utils/json-parse'+import { fetchStreamed } from '~/common/utils/request'++import { ConfigurationCacheItem, ConfigurationFile, HttpCacheMetadata, configurationCacheKey } from './crawl-history'+import { ImageFetchRequestMetrics } from './metrics'+import { canonicalizeUrl, politenessKey } from './politeness-key'+import { ConfigurationFetchReason, ImageFetchProcessingMetrics } from './processing-metrics'+import { BOT_NAME, REQUEST_IDENTITY_HEADERS } from './request-identity'+import { WebBotAuthRequestSigner } from './web-bot-auth'+import { wildcardPatternMatchesPathname } from './wildcard-pattern'++const CONFIG_BODY_LIMIT = 500 * 1024+const CONFIG_REDIRECT_LIMIT = 5+const CONFIG_FRESH_MS = 24 * 60 * 60 * 1000+const CONFIG_REFRESH_MS = 23 * 60 * 60 * 1000+const CONFIG_RETRY_MS = 60 * 60 * 1000+const CONFIG_STORAGE_MS = 30 * 24 * 60 * 60 * 1000+const REDIRECT_STATUSES = new Set([301, 302, 303, 307, 308])+const UTF8_BYTE_ORDER_MARK = [0xef, 0xbb, 0xbf]+const JSON_WHITESPACE_BYTES = new Set([' ', '\t', '\n', '\r'].map((character) => character.charCodeAt(0)))+const JSON_ARRAY_START_BYTE = '['.charCodeAt(0)++export type ConfigurationFetchResult =+ | { outcome: 'available'; body: string; cache: HttpCacheMetadata }+ | { outcome: 'absent'; cache?: HttpCacheMetadata; nonArrayBody?: boolean }+ | { outcome: 'refused' | 'unreachable'; cache?: HttpCacheMetadata }+ | { outcome: 'deferred'; reason: ConfigurationRequestBlockReason }++export type ConfigurationRequestBlockReason =+ | 'breaker_open'+ | 'backoff'+ | 'deadline'+ | 'origin_map_full'+ | 'registrable_domain_map_full'+ | 'connection_limit'++export type RobotsPolicyRefusalReason = 'robots_disallow' | 'content_usage' | 'content_signal'+export type OriginPolicyReason =+ | RobotsPolicyRefusalReason+ | 'robots_refused'+ | 'tdmrep_refused'+ | 'origin_map_full'+ | 'registrable_domain_map_full'+ | 'configuration_deferred'+ | 'configuration_unreachable'+export type ResponseOptOutReason = 'x_robots_tag' | 'content_usage' | 'tdm_reservation'++export interface ConfigurationRequestScheduler {+ run<T>(+ url: URL,+ deadlineMs: number,+ request: () => Promise<T>+ ): Promise<{ ran: true; value: T } | { ran: false; reason: ConfigurationRequestBlockReason }>+}++type ConfigurationHop =+ | { kind: 'redirect'; location: string; cache: HttpCacheMetadata }+ | { kind: 'done'; result: ConfigurationFetchResult; reason?: ConfigurationFetchReason }++export class HttpConfigurationFetcher {+ constructor(+ private readonly signer: WebBotAuthRequestSigner,+ private readonly scheduler: ConfigurationRequestScheduler,+ private readonly timeoutMs: number+ ) {}++ public async fetch(origin: string, file: ConfigurationFile): Promise<ConfigurationFetchResult> {+ return await ImageFetchProcessingMetrics.measure('configuration_fetch', () =>+ this.fetchConfiguration(origin, file)+ )+ }++ private async fetchConfiguration(origin: string, file: ConfigurationFile): Promise<ConfigurationFetchResult> {+ const complete = (+ result: ConfigurationFetchResult,+ reason: ConfigurationFetchReason = result.outcome+ ): ConfigurationFetchResult => ImageFetchProcessingMetrics.observeConfigurationFetch(file, result, reason)+ const deadlineMs = Date.now() + this.timeoutMs+ let target = new URL(file === 'robots' ? '/robots.txt' : '/.well-known/tdmrep.json', origin)+ const registrableDomain = politenessKey(target.hostname)+ for (let redirects = 0; ; redirects++) {+ const canonical = canonicalizeUrl(target.toString())+ if (!canonical) {+ return complete({ outcome: 'refused' }, 'invalid_url')+ }+ target = new URL(canonical.fetch)+ let scheduled:+ | { ran: true; value: ConfigurationHop }+ | { ran: false; reason: ConfigurationRequestBlockReason }+ try {+ scheduled = await this.scheduler.run(target, deadlineMs, () => this.hop(target, file, deadlineMs))+ } catch {+ return complete({ outcome: 'unreachable' }, Date.now() >= deadlineMs ? 'timeout' : 'request_error')+ }+ if (!scheduled.ran) {+ return complete({ outcome: 'deferred', reason: scheduled.reason })+ }+ const hop = scheduled.value+ if (hop.kind === 'done') {+ return complete(hop.result, hop.reason)+ }+ if (redirects >= CONFIG_REDIRECT_LIMIT) {+ return complete({ outcome: 'unreachable', cache: hop.cache }, 'redirect_limit')+ }+ try {+ const redirectTarget = new URL(hop.location, target)+ if (politenessKey(redirectTarget.hostname) !== registrableDomain) {+ return complete({ outcome: 'unreachable', cache: hop.cache }, 'cross_domain_redirect')+ }+ target = redirectTarget+ } catch {+ return complete({ outcome: 'unreachable', cache: hop.cache }, 'invalid_redirect')+ }+ }+ }++ private async hop(target: URL, file: ConfigurationFile, deadlineMs: number): Promise<ConfigurationHop> {+ const requestTimeMs = Date.now()+ const canonical = canonicalizeUrl(target.toString())+ let response: Awaited<ReturnType<typeof fetchStreamed>>+ try {+ response = await fetchStreamed(target.toString(), {+ timeoutMs: Math.max(1, deadlineMs - Date.now()),+ allowH2: true,+ headers: {+ ...REQUEST_IDENTITY_HEADERS,+ accept: file === 'robots' ? 'text/plain,*/*;q=0.1' : 'application/json,*/*;q=0.1',+ 'accept-encoding': 'identity',+ ...this.signer.headersForGet(target.toString()),+ },+ })+ } catch (error) {+ if (canonical) {+ ImageFetchRequestMetrics.observeRequest('network_error', (Date.now() - requestTimeMs) / 1000)+ }+ throw error+ }+ const complete = (result: ConfigurationHop): ConfigurationHop => {+ if (canonical) {+ ImageFetchRequestMetrics.observeRequest(+ ImageFetchRequestMetrics.outcomeForHttpStatus(response.status),+ (Date.now() - requestTimeMs) / 1000+ )+ }+ return result+ }+ const cache = cacheMetadata(requestTimeMs, Date.now(), response.headerLines)+ if (REDIRECT_STATUSES.has(response.status)) {+ response.discard()+ const locations = headerValues(response.headerLines, 'location')+ const location = locations.length === 1 ? locations[0] : undefined+ return complete(+ location+ ? { kind: 'redirect', location, cache }+ : { kind: 'done', result: { outcome: 'unreachable', cache }, reason: 'missing_location' }+ )+ }+ if (response.status === 404 || response.status === 410) {+ response.discard()+ return complete({ kind: 'done', result: { outcome: 'absent', cache } })+ }+ if (response.status === 429 || response.status >= 500) {+ response.discard()+ return complete({+ kind: 'done',+ result: { outcome: 'unreachable', cache },+ reason: response.status === 429 ? 'http_429' : 'http_5xx',+ })+ }+ if (response.status >= 400 && response.status < 500) {+ response.discard()+ return complete({ kind: 'done', result: { outcome: 'refused', cache }, reason: `http_${response.status}` })+ }+ if (response.status !== 200) {+ response.discard()+ return complete({ kind: 'done', result: { outcome: 'unreachable', cache }, reason: 'unexpected_status' })+ }+ const body = await response.read(CONFIG_BODY_LIMIT)+ // This check comes before the size and UTF-8 checks, so that an oversized or non-UTF-8 HTML page also means that the origin has no tdmrep.json (README 3.14).+ if (file === 'tdmrep' && cannotBeJsonArray(body)) {+ return complete({+ kind: 'done',+ result: { outcome: 'absent', cache, nonArrayBody: true },+ reason: 'not_json_array',+ })+ }+ if (body.overLimit && file === 'tdmrep') {+ return complete({ kind: 'done', result: { outcome: 'unreachable', cache }, reason: 'body_limit' })+ }+ let text: string+ try {+ text = new TextDecoder('utf-8', { fatal: true }).decode(body.bytes, { stream: body.overLimit })+ } catch {+ return complete({ kind: 'done', result: { outcome: 'unreachable', cache }, reason: 'invalid_utf8' })+ }+ if (file === 'tdmrep' && !Array.isArray(parseTdmrepDocument(text))) {+ return complete({ kind: 'done', result: { outcome: 'unreachable', cache }, reason: 'invalid_document' })+ }+ return complete({ kind: 'done', result: { outcome: 'available', body: text, cache } })+ }+}++export interface OriginPolicyDecision {+ allowed: boolean+ transient: boolean+ reason?: OriginPolicyReason+ crawlDelayMs: number+ tdmrepReservation: boolean+ updates: ConfigurationCacheItem[]+}++export interface ConfigurationPolicyPass {+ check(url: string, cached: Map<string, ConfigurationCacheItem>, nowMs: number): Promise<OriginPolicyDecision>+}++interface ParsedRobotsConfiguration {+ matcher: ParsedRobots+ fields: Array<{ name: string; value: string }>+}++interface ParsedConfigurationCache {+ robots: Map<string, Promise<ParsedRobotsConfiguration>>+ tdmrep: Map<string, unknown>+}++export class ConfigurationPolicyService {+ private readonly inFlight = new Map<string, Promise<ConfigurationFetchResult>>()++ constructor(private readonly fetcher: HttpConfigurationFetcher) {}++ public createPass(): ConfigurationPolicyPass {+ const parsed: ParsedConfigurationCache = {+ robots: new Map(),+ tdmrep: new Map(),+ }+ return {+ check: (url, cached, nowMs) => this.checkWithParsedCache(url, cached, nowMs, parsed),+ }+ }++ public async check(+ url: string,+ cached: Map<string, ConfigurationCacheItem>,+ nowMs: number+ ): Promise<OriginPolicyDecision> {+ return await this.createPass().check(url, cached, nowMs)+ }++ private async checkWithParsedCache(+ url: string,+ cached: Map<string, ConfigurationCacheItem>,+ nowMs: number,+ parsed: ParsedConfigurationCache+ ): Promise<OriginPolicyDecision> {+ const origin = new URL(url).origin+ // One after the other: a host limit of one connection would refuse the second of two concurrent requests.+ const robots = await this.load(origin, 'robots', cached.get(configurationCacheKey(origin, 'robots')), nowMs)+ const tdmrep = await this.load(origin, 'tdmrep', cached.get(configurationCacheKey(origin, 'tdmrep')), nowMs)+ const updates = [...robots.updates, ...tdmrep.updates]+ if (robots.item.status === 'refused') {+ return {+ allowed: false,+ transient: false,+ reason: 'robots_refused',+ crawlDelayMs: 0,+ tdmrepReservation: false,+ updates,+ }+ }+ if (tdmrep.item.status === 'refused') {+ return {+ allowed: false,+ transient: false,+ reason: 'tdmrep_refused',+ crawlDelayMs: 0,+ tdmrepReservation: false,+ updates,+ }+ }++ const robotsPolicy = robots.item.body+ ? evaluateRobotsPolicy(await parsedRobotsConfiguration(robots.item, parsed), url)+ : defaultRobotsPolicy()+ if (!robotsPolicy.allowed) {+ return {+ allowed: false,+ transient: false,+ reason: robotsPolicy.reason,+ crawlDelayMs: robotsPolicy.crawlDelayMs,+ tdmrepReservation: false,+ updates,+ }+ }+ const tdmrepReservation = Boolean(+ tdmrep.item.body && tdmrepRefuses(parsedTdmrepDocument(tdmrep.item, parsed), new URL(url))+ )+ const deferredReason = requestControlDeferralReason(robots.deferredReason, tdmrep.deferredReason)+ if (deferredReason) {+ return {+ allowed: false,+ transient: true,+ reason: deferredReason,+ crawlDelayMs: 0,+ tdmrepReservation: false,+ updates,+ }+ }+ if (robots.item.status === 'unreachable' || tdmrep.item.status === 'unreachable') {+ return {+ allowed: false,+ transient: true,+ reason: 'configuration_unreachable',+ crawlDelayMs: 0,+ tdmrepReservation: false,+ updates,+ }+ }+ return {+ allowed: true,+ transient: false,+ crawlDelayMs: robotsPolicy.crawlDelayMs,+ tdmrepReservation,+ updates,+ }+ }++ private async load(+ origin: string,+ file: ConfigurationFile,+ previous: ConfigurationCacheItem | undefined,+ nowMs: number+ ): Promise<{+ item: ConfigurationCacheItem+ updates: ConfigurationCacheItem[]+ deferredReason?: ConfigurationRequestBlockReason+ }> {+ if (previous && previous.storageExpiresAtMs <= nowMs) {+ previous = undefined+ }+ if (previous && previous.refreshAtMs > nowMs) {+ ImageFetchProcessingMetrics.observeConfigurationLookup(file, 'cache', previous.status)+ return { item: previous, updates: [] }+ }+ if (previous?.status === 'unreachable' && previous.retryAtMs > nowMs) {+ ImageFetchProcessingMetrics.observeConfigurationLookup(file, 'cache', previous.status)+ return { item: previous, updates: [] }+ }+ const key = configurationCacheKey(origin, file)+ let request = this.inFlight.get(key)+ const source = request ? 'shared' : 'network'+ if (!request) {+ request = this.fetcher.fetch(origin, file).finally(() => this.inFlight.delete(key))+ this.inFlight.set(key, request)+ }+ const fetched = await request.catch((error) => {+ ImageFetchProcessingMetrics.observeConfigurationLookup(file, source, 'error')+ throw error+ })+ ImageFetchProcessingMetrics.observeConfigurationLookup(file, source, fetched.outcome)+ if (fetched.outcome === 'deferred') {+ return {+ item: previous ?? unreachableItem(origin, file, nowMs),+ updates: [],+ deferredReason: fetched.reason,+ }+ }+ const nonArrayBodyReplacesFile =+ fetched.outcome === 'absent' && fetched.nonArrayBody === true && previous?.status === 'available'+ if (+ (fetched.outcome === 'unreachable' || nonArrayBodyReplacesFile) &&+ previous &&+ previous.status !== 'unreachable'+ ) {+ const retained = {+ ...previous,+ refreshAtMs: nowMs + CONFIG_RETRY_MS,+ retryAtMs: nowMs + CONFIG_RETRY_MS,+ storageExpiresAtMs: Math.max(previous.storageExpiresAtMs, nowMs + CONFIG_STORAGE_MS),+ }+ return { item: retained, updates: [retained] }+ }+ const explicitFreshMs = fetched.cache ? explicitFreshnessLifetimeMs(fetched.cache) : 0+ const freshForMs =+ fetched.outcome === 'unreachable' ? CONFIG_RETRY_MS : Math.max(CONFIG_FRESH_MS, explicitFreshMs)+ const item: ConfigurationCacheItem = {+ kind: file,+ key,+ origin,+ status: fetched.outcome,+ body: fetched.outcome === 'available' ? fetched.body : undefined,+ fetchedAtMs: nowMs,+ refreshAtMs: nowMs + (fetched.outcome === 'unreachable' ? CONFIG_RETRY_MS : CONFIG_REFRESH_MS),+ freshUntilMs: nowMs + freshForMs,+ retryAtMs: nowMs + (fetched.outcome === 'unreachable' ? CONFIG_RETRY_MS : 0),+ storageExpiresAtMs: nowMs + Math.max(CONFIG_STORAGE_MS, freshForMs),+ }+ return { item, updates: [item] }+ }+}++function requestControlDeferralReason(+ ...reasons: Array<ConfigurationRequestBlockReason | undefined>+):+ | Extract<OriginPolicyReason, 'origin_map_full' | 'registrable_domain_map_full' | 'configuration_deferred'>+ | undefined {+ if (reasons.includes('registrable_domain_map_full')) {+ return 'registrable_domain_map_full'+ }+ if (reasons.includes('origin_map_full')) {+ return 'origin_map_full'+ }+ return reasons.some(Boolean) ? 'configuration_deferred' : undefined+}++function unreachableItem(origin: string, file: ConfigurationFile, nowMs: number): ConfigurationCacheItem {+ return {+ kind: file,+ key: configurationCacheKey(origin, file),+ origin,+ status: 'unreachable',+ fetchedAtMs: nowMs,+ refreshAtMs: nowMs + CONFIG_RETRY_MS,+ freshUntilMs: nowMs + CONFIG_RETRY_MS,+ retryAtMs: nowMs + CONFIG_RETRY_MS,+ storageExpiresAtMs: nowMs + CONFIG_STORAGE_MS,+ }+}++interface ExtensionGroup {+ userAgents: string[]+ fields: Array<{ name: string; value: string }>+}++class ExtensionCollector implements RobotsParseHandler {+ public readonly groups: ExtensionGroup[] = []+ private current: ExtensionGroup | undefined+ private hasRules = false++ handleRobotsStart(): void {}+ handleRobotsEnd(): void {}+ reportLineMetadata(): void {}++ handleUserAgent(_lineNum: number, value: string): void {+ if (!this.current || this.hasRules) {+ this.current = { userAgents: [], fields: [] }+ this.groups.push(this.current)+ this.hasRules = false+ }+ this.current.userAgents.push(value.trim().toLowerCase())+ }++ handleAllow(): void {+ this.hasRules = true+ }++ handleDisallow(): void {+ this.hasRules = true+ }++ handleSitemap(): void {+ this.hasRules = true+ }++ handleUnknownAction(_lineNum: number, action: string, value: string): void {+ if (!this.current) {+ return+ }+ this.hasRules = true+ this.current.fields.push({ name: action.trim().toLowerCase(), value: value.trim() })+ }+}++function selectedExtensionFields(+ body: string,+ parseRobotsTxt: typeof import('@trybyte/robotstxt-parser').parseRobotsTxt+): Array<{ name: string; value: string }> {+ const collector = new ExtensionCollector()+ parseRobotsTxt(body, collector)+ const bot = BOT_NAME.toLowerCase()+ const matches = collector.groups.map((group) => ({+ group,+ specificity: Math.max(+ ...group.userAgents.map((agent) => (agent !== '*' && bot.startsWith(agent) ? agent.length : 0)),+ 0+ ),+ wildcard: group.userAgents.includes('*'),+ }))+ const specificity = Math.max(...matches.map((match) => match.specificity), 0)+ const selected =+ specificity > 0+ ? matches.filter((match) => match.specificity === specificity)+ : matches.filter((match) => match.wildcard)+ return selected.flatMap((match) => match.group.fields)+}++function defaultRobotsPolicy(): { allowed: true; crawlDelayMs: number } {+ return { allowed: true, crawlDelayMs: 0 }+}++export async function parseRobotsPolicy(+ body: string,+ url: string+): Promise<{ allowed: boolean; crawlDelayMs: number; reason?: RobotsPolicyRefusalReason }> {+ return evaluateRobotsPolicy(await parseRobotsConfiguration(body), url)+}++async function parseRobotsConfiguration(body: string): Promise<ParsedRobotsConfiguration> {+ const robotsParser = await import('@trybyte/robotstxt-parser')+ const { ParsedRobots, parseRobotsTxt } = robotsParser+ return {+ matcher: ParsedRobots.parse(body),+ fields: selectedExtensionFields(body, parseRobotsTxt),+ }+}++function evaluateRobotsPolicy(+ parsed: ParsedRobotsConfiguration,+ url: string+): { allowed: boolean; crawlDelayMs: number; reason?: RobotsPolicyRefusalReason } {+ if (!parsed.matcher.checkUrl(BOT_NAME, url).allowed) {+ return { allowed: false, crawlDelayMs: 0, reason: 'robots_disallow' }+ }+ const crawlDelayMs = Math.max(+ 0,+ ...parsed.fields+ .filter((field) => field.name === 'crawl-delay')+ .flatMap((field) => {+ const value = field.value.trim()+ if (!/^\d+(?:\.\d+)?$/.test(value)) {+ return []+ }+ // A decimal multiplied by 1000 is not always exact. 16.1 * 1000 is 16100.000000000002. The safe-integer guard below rejects that value, but README 7.9 accepts the delay.+ const milliseconds = Math.round(Number(value) * 1000)+ return Number.isSafeInteger(milliseconds) ? [milliseconds] : []+ })+ )+ if (+ contentUsageRefuses(parsed.fields.filter((field) => field.name === 'content-usage').map((field) => field.value))+ ) {+ return { allowed: false, crawlDelayMs, reason: 'content_usage' }+ }+ if (parsed.fields.some((field) => field.name === 'content-signal' && contentSignalRefuses(field.value, url))) {+ return { allowed: false, crawlDelayMs, reason: 'content_signal' }+ }+ return { allowed: true, crawlDelayMs }+}++function parseTdmrepDocument(body: string): unknown {+ try {+ return parseJSON(body)+ } catch {+ return undefined+ }+}++function tdmrepRefuses(parsed: unknown, url: URL): boolean {+ if (!Array.isArray(parsed)) {+ return false+ }+ for (const rule of parsed) {+ if (!isObject(rule) || typeof rule.location !== 'string') {+ continue+ }+ if (wildcardPatternMatchesPathname(rule.location, url.pathname)) {+ return isTdmReservation(rule['tdm-reservation'])+ }+ }+ return false+}++function isTdmReservation(value: unknown): boolean {+ // TDMRep reads "1" and true as protocol errors, which mean no reservation. README 3.13 reads them as a reservation, because a site that writes them intends to reserve.+ return value === 1 || value === '1' || value === true+}++function cannotBeJsonArray(body: { bytes: Uint8Array; overLimit: boolean }): boolean {+ const firstByte = firstByteAfterJsonWhitespace(body.bytes)+ if (firstByte === undefined) {+ return !body.overLimit+ }+ return firstByte !== JSON_ARRAY_START_BYTE+}++function firstByteAfterJsonWhitespace(bytes: Uint8Array): number | undefined {+ let index = hasUtf8ByteOrderMark(bytes) ? UTF8_BYTE_ORDER_MARK.length : 0+ while (index < bytes.length && JSON_WHITESPACE_BYTES.has(bytes[index])) {+ index+++ }+ return index < bytes.length ? bytes[index] : undefined+}++function hasUtf8ByteOrderMark(bytes: Uint8Array): boolean {+ return UTF8_BYTE_ORDER_MARK.every((byte, index) => bytes[index] === byte)+}++async function parsedRobotsConfiguration(+ item: ConfigurationCacheItem,+ parsed: ParsedConfigurationCache+): Promise<ParsedRobotsConfiguration> {+ const revision = configurationRevision(item)+ let configuration = parsed.robots.get(revision)+ if (!configuration) {+ configuration = parseRobotsConfiguration(item.body ?? '')+ parsed.robots.set(revision, configuration)+ }+ return await configuration+}++function parsedTdmrepDocument(item: ConfigurationCacheItem, parsed: ParsedConfigurationCache): unknown {+ const revision = configurationRevision(item)+ if (!parsed.tdmrep.has(revision)) {+ parsed.tdmrep.set(revision, parseTdmrepDocument(item.body ?? ''))+ }+ return parsed.tdmrep.get(revision)+}++function configurationRevision(item: ConfigurationCacheItem): string {+ return `${item.key}\0${item.fetchedAtMs}`+}++function isObject(value: unknown): value is Record<string, unknown> {+ return typeof value === 'object' && value !== null && !Array.isArray(value)+}++function contentUsageRefuses(values: string[]): boolean {+ if (values.length === 0) {+ return false+ }+ try {+ const trainAi = parseDictionary(values.join(', ')).get('train-ai')+ return Boolean(+ trainAi && !Array.isArray(trainAi[0]) && trainAi[0] instanceof Token && trainAi[0].toString() === 'n'+ )+ } catch {+ return false+ }+}++function contentSignalRefuses(value: string, url: string): boolean {+ const trimmed = value.trim()+ const pathEnd = trimmed.startsWith('/') ? trimmed.search(/\s/) : -1+ if (pathEnd > 0 && !wildcardPatternMatchesPathname(trimmed.slice(0, pathEnd), new URL(url).pathname)) {+ return false+ }+ const preferences = (pathEnd > 0 ? trimmed.slice(pathEnd) : trimmed)+ .split(',')+ .map((part) => part.trim().split('=', 2))+ const aiTrain = preferences.filter(([name]) => name === 'ai-train').at(-1)+ return aiTrain?.[1] === 'no'+}++export function responseOptOutReason(+ headerLines: Array<{ name: string; value: string }>,+ tdmrepReservation: boolean+): ResponseOptOutReason | undefined {+ if (headerLines.some((line) => line.name === 'x-robots-tag' && xRobotsTagRefuses(line.value))) {+ return 'x_robots_tag'+ }+ if (contentUsageRefuses(headerLines.filter((line) => line.name === 'content-usage').map((line) => line.value))) {+ return 'content_usage'+ }+ const headerReservation = tdmReservationHeader(headerLines)+ if (headerReservation ?? tdmrepReservation) {+ return 'tdm_reservation'+ }+ return undefined+}++// X-Robots-Tag directives that carry a value, from https://developers.google.com/search/docs/crawling-indexing/robots-meta-tag. A colon inside one of these does not start a bot scope.+const VALUED_X_ROBOTS_DIRECTIVES = ['unavailable_after', 'max-snippet', 'max-image-preview', 'max-video-preview']++function xRobotsTagRefuses(value: string): boolean {+ const lower = value.toLowerCase()+ const colon = lower.indexOf(':')+ // A bot scope is one token before the first colon. A valued directive looks the same, so the lane must tell the two apart. Otherwise it reads `unavailable_after: <date>, noai` as another bot's scope and ignores an opt-out that README 2.6 requires.+ // A prefix that names no listed directive reads as a bot scope. This keeps another bot's scope intact. A new valued directive therefore needs an entry in the list above, because until then the lane skips a `noai` that follows it.+ const prefix = colon >= 0 ? lower.slice(0, colon).trim() : ''+ const scoped = colon >= 0 && !prefix.includes(',') && !VALUED_X_ROBOTS_DIRECTIVES.includes(prefix)+ if (scoped && prefix !== BOT_NAME.toLowerCase()) {+ return false+ }+ const directives = scoped ? lower.slice(colon + 1) : lower+ return directives.split(',').some((directive) => ['noai', 'noimageai'].includes(directive.trim()))+}++function tdmReservationHeader(headerLines: Array<{ name: string; value: string }>): boolean | undefined {+ const values = headerLines+ .filter((line) => line.name === 'tdm-reservation')+ .flatMap((line) => line.value.split(','))+ .map((value) => value.trim())+ if (values.includes('1')) {+ return true+ }+ if (values.includes('0')) {+ return false+ }+ return undefined+}++function headerValues(headerLines: Array<{ name: string; value: string }>, name: string): string[] {+ return headerLines.filter((line) => line.name === name).map((line) => line.value)+}++function cacheMetadata(+ requestTimeMs: number,+ responseTimeMs: number,+ headerLines: Array<{ name: string; value: string }>+): HttpCacheMetadata {+ return {+ requestTimeMs,+ responseTimeMs,+ etag: singletonHeader(headerLines, 'etag'),+ lastModified: singletonHeader(headerLines, 'last-modified'),+ date: singletonHeader(headerLines, 'date'),+ age: singletonHeader(headerLines, 'age'),+ cacheControl: combinedHeader(headerLines, 'cache-control'),+ expires: singletonHeader(headerLines, 'expires'),+ }+}++function singletonHeader(headerLines: Array<{ name: string; value: string }>, name: string): string | undefined {+ const values = headerValues(headerLines, name)+ return values.length === 1 ? values[0] : undefined+}++function combinedHeader(headerLines: Array<{ name: string; value: string }>, name: string): string | undefined {+ const values = headerValues(headerLines, name)+ return values.length > 0 ? values.join(', ') : undefined+}++export function explicitFreshnessLifetimeMs(cache: HttpCacheMetadata, nowMs = Date.now()): number {+ const directives = cacheControlDirectives(cache.cacheControl)+ if (['no-cache', 'no-store', 'private', 'must-revalidate'].some((directive) => directives.has(directive))) {+ return 0+ }+ let lifetimeMs = 0+ if (directives.has('s-maxage')) {+ lifetimeMs = singletonDeltaSeconds(directives.get('s-maxage')) * 1000+ } else if (directives.has('max-age')) {+ lifetimeMs = singletonDeltaSeconds(directives.get('max-age')) * 1000+ } else if (cache.expires) {+ const expiresMs = Date.parse(cache.expires)+ const dateMs = cache.date ? Date.parse(cache.date) : cache.responseTimeMs+ if (Number.isFinite(expiresMs) && Number.isFinite(dateMs)) {+ lifetimeMs = Math.max(0, expiresMs - dateMs)+ }+ }+ const dateMs = cache.date ? Date.parse(cache.date) : cache.responseTimeMs+ const apparentAgeMs = Number.isFinite(dateMs) ? Math.max(0, cache.responseTimeMs - dateMs) : 0+ const ageValueMs = nonNegativeInteger(cache.age) * 1000+ const correctedAgeMs = ageValueMs + Math.max(0, cache.responseTimeMs - cache.requestTimeMs)+ const currentAgeMs = Math.max(apparentAgeMs, correctedAgeMs) + Math.max(0, nowMs - cache.responseTimeMs)+ return Math.max(0, lifetimeMs - currentAgeMs)+}++function cacheControlDirectives(cacheControl: string | undefined): Map<string, Array<string | undefined>> {+ const directives = new Map<string, Array<string | undefined>>()+ for (const directive of splitCommaSeparatedValues(cacheControl ?? '')) {+ const [rawName, ...rawValueParts] = directive.trim().split('=')+ const name = rawName.toLowerCase()+ if (!name) {+ continue+ }+ const rawValue = rawValueParts.length > 0 ? rawValueParts.join('=').trim() : undefined+ const value = rawValue?.startsWith('"') && rawValue.endsWith('"') ? rawValue.slice(1, -1) : rawValue+ directives.set(name, [...(directives.get(name) ?? []), value])+ }+ return directives+}++function splitCommaSeparatedValues(value: string): string[] {+ const values: string[] = []+ let start = 0+ let quoted = false+ let escaped = false+ for (let index = 0; index < value.length; index++) {+ const character = value[index]+ if (escaped) {+ escaped = false+ } else if (character === '\\' && quoted) {+ escaped = true+ } else if (character === '"') {+ quoted = !quoted+ } else if (character === ',' && !quoted) {+ values.push(value.slice(start, index))+ start = index + 1+ }+ }+ values.push(value.slice(start))+ return values+}++function singletonDeltaSeconds(values: Array<string | undefined> | undefined): number {+ return values?.length === 1 ? nonNegativeInteger(values[0]) : 0+}++function nonNegativeInteger(value: string | undefined): number {+ if (!value || !/^\d+$/.test(value.trim())) {+ return 0+ }+ const parsed = Number(value)+ return Number.isSafeInteger(parsed) ? parsed : 0+}diff --git a/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/fetch-candidate-pool.test.ts b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/fetch-candidate-pool.test.tsnew file mode 100644index 000000000000..39e3ea3a0006--- /dev/null+++ b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/fetch-candidate-pool.test.ts@@ -0,0 +1,249 @@+import { FetchCandidate, MAX_HOPS } from './collected-urls-record'+import { FetchCandidatePool, FetchCandidatePoolLease, OriginPacing } from './fetch-candidate-pool'++const NOW_MS = 1_700_000_000_000++function candidate(url: string, registrableDomain: string, firstSeenAtMs = NOW_MS): FetchCandidate {+ const parsed = new URL(url)+ return {+ originalRef: `imageurl:${url}`,+ currentUrl: url,+ host: parsed.host,+ origin: parsed.origin,+ registrableDomain,+ remainingHops: MAX_HOPS,+ notBeforeMs: 0,+ firstSeenAtMs,+ fetchCount: 0,+ republishCount: 0,+ lastRepublishReason: null,+ }+}++function buildPool(+ crawlDelayMsByOrigin: Record<string, number> = {},+ limits: { maxConcurrentPerRegistrableDomain?: number; refillRunnableUrls?: number } = {}+): FetchCandidatePool<string> {+ const pacing: OriginPacing = {+ originCrawlDelayMs: (origin) => crawlDelayMsByOrigin[origin] ?? 0,+ originNextImageStartAtMs: () => 0,+ }+ return new FetchCandidatePool<string>(+ {+ maxConcurrentPerRegistrableDomain: limits.maxConcurrentPerRegistrableDomain ?? 2,+ refillRunnableUrls: limits.refillRunnableUrls ?? 3,+ maxQueuedUrlsPerOwner: 100,+ },+ pacing+ )+}++async function takeReady(pool: FetchCandidatePool<string>): Promise<FetchCandidatePoolLease<string> | undefined> {+ let lease: FetchCandidatePoolLease<string> | undefined+ let settled = false+ void pool.take().then((taken) => {+ lease = taken+ settled = true+ })+ await jest.advanceTimersByTimeAsync(0)+ if (!settled) {+ pool.close()+ await jest.advanceTimersByTimeAsync(0)+ }+ return lease+}++describe('FetchCandidatePool', () => {+ beforeEach(() => jest.useFakeTimers().setSystemTime(NOW_MS))+ afterEach(() => jest.useRealTimers())++ it('takes the earliest captured candidates of a domain under its limit, across batches from different partitions', async () => {+ const pool = buildPool()+ pool.add(+ [+ candidate('https://cdn.hot.com/1.png', 'hot.com'),+ candidate('https://cdn.hot.com/2.png', 'hot.com'),+ candidate('https://cdn.hot.com/deferred.png', 'hot.com', NOW_MS - 86_400_000),+ ],+ 'partition-1',+ NOW_MS + 40_000+ )+ jest.advanceTimersByTime(10)+ pool.add([candidate('https://cdn.quiet.com/1.png', 'quiet.com')], 'partition-2', NOW_MS + 40_000)++ const leases = [await takeReady(pool), await takeReady(pool), await takeReady(pool)]++ expect(leases.map((lease) => lease?.candidate.currentUrl)).toEqual([+ 'https://cdn.hot.com/deferred.png',+ 'https://cdn.hot.com/1.png',+ 'https://cdn.quiet.com/1.png',+ ])+ expect(leases[2]?.context).toBe('partition-2')+ expect(await takeReady(pool)).toBeUndefined()+ })++ it('frees the domain slot on release, not before', async () => {+ const pool = buildPool({}, { maxConcurrentPerRegistrableDomain: 1 })+ pool.add(+ [candidate('https://cdn.hot.com/1.png', 'hot.com'), candidate('https://img.hot.com/2.png', 'hot.com')],+ 'partition-1',+ NOW_MS + 40_000+ )++ const first = await takeReady(pool)+ const waiting = pool.take()+ await jest.advanceTimersByTimeAsync(0)+ first?.release()++ expect((await waiting)?.candidate.currentUrl).toBe('https://img.hot.com/2.png')+ })++ it('holds an origin until its crawl delay passes while other origins keep running', async () => {+ const pool = buildPool({ 'https://slow.example.com': 5_000 })+ pool.add(+ [+ candidate('https://slow.example.com/1.png', 'example.com'),+ candidate('https://slow.example.com/2.png', 'example.com'),+ candidate('https://fast.other.net/1.png', 'other.net'),+ ],+ 'partition-1',+ NOW_MS + 40_000+ )++ const first = await takeReady(pool)+ const second = await takeReady(pool)+ const delayed = pool.take()+ await jest.advanceTimersByTimeAsync(4_999)+ let delayedTaken = false+ void delayed.then(() => {+ delayedTaken = true+ })+ await jest.advanceTimersByTimeAsync(0)+ expect(delayedTaken).toBe(false)+ await jest.advanceTimersByTimeAsync(1)++ expect([first, second, await delayed].map((lease) => lease?.candidate.currentUrl)).toEqual([+ 'https://slow.example.com/1.png',+ 'https://fast.other.net/1.png',+ 'https://slow.example.com/2.png',+ ])+ })++ it('holds an origin for a crawl delay longer than the longest Node timer with no timer loop', async () => {+ const pool = buildPool({ 'https://slow.example.com': 3_000_000_000 })+ pool.add(+ [+ candidate('https://slow.example.com/1.png', 'example.com'),+ candidate('https://slow.example.com/2.png', 'example.com'),+ ],+ 'partition-1',+ NOW_MS + 40_000+ )+ await takeReady(pool)+ let delayedTaken = false+ void pool.take().then(() => {+ delayedTaken = true+ })+ const timers = jest.spyOn(global, 'setTimeout')++ await jest.advanceTimersByTimeAsync(100)++ expect(delayedTaken).toBe(false)+ expect(timers).not.toHaveBeenCalled()+ timers.mockRestore()+ pool.close()+ })++ it('hands out a batch past its deadline without a domain slot', async () => {+ const pool = buildPool({}, { maxConcurrentPerRegistrableDomain: 1 })+ pool.add(+ [candidate('https://cdn.hot.com/1.png', 'hot.com'), candidate('https://cdn.hot.com/2.png', 'hot.com')],+ 'partition-1',+ NOW_MS + 1_000+ )+ const held = await takeReady(pool)+ await jest.advanceTimersByTimeAsync(1_001)++ expect(held?.candidate.currentUrl).toBe('https://cdn.hot.com/1.png')+ expect((await takeReady(pool))?.candidate.currentUrl).toBe('https://cdn.hot.com/2.png')+ expect(pool.candidateCount).toBe(0)+ })++ it('withdraws the queued candidates of a failed batch only', async () => {+ const pool = buildPool()+ const failed = pool.add(+ [+ candidate('https://a.example.com/1.png', 'example.com'),+ candidate('https://b.example.com/2.png', 'example.com'),+ ],+ 'failed',+ NOW_MS + 40_000+ )+ pool.add([candidate('https://c.other.net/1.png', 'other.net')], 'healthy', NOW_MS + 40_000)++ await takeReady(pool)++ expect(pool.withdraw(failed)).toBe(1)+ expect((await takeReady(pool))?.context).toBe('healthy')+ expect(await takeReady(pool)).toBeUndefined()+ })++ it.each([+ ['counts a hot domain only up to its limit, so the owner still has room', 10, 'hot.com', true],+ ['has no room once enough distinct domains can run', 1, 'domain', false],+ ])('%s', (_name, urlsPerDomain, domainPrefix, expectedRoom) => {+ const pool = buildPool({}, { maxConcurrentPerRegistrableDomain: 2, refillRunnableUrls: 3 })+ const owner = pool.createOwner()+ const candidates = Array.from({ length: 10 }, (_unused, index) => {+ const domain = urlsPerDomain === 10 ? domainPrefix : `${domainPrefix}${index}.com`+ return candidate(`https://cdn.${domain}/${index}.png`, domain)+ })++ pool.add(candidates, 'partition-1', NOW_MS + 40_000, owner)++ expect(owner.queuedUrls).toBe(10)+ expect(owner.hasRoom()).toBe(expectedRoom)+ })++ it('keeps an owner without room while a batch waits for admission', async () => {+ const pool = buildPool()+ const owner = pool.createOwner()+ const admission = owner.beginAdmission()+ let hadRoom: boolean | undefined+ void owner.waitForRoom(30_000).then((room) => {+ hadRoom = room+ })+ await jest.advanceTimersByTimeAsync(0)+ expect(hadRoom).toBeUndefined()++ admission.admitted()+ await jest.advanceTimersByTimeAsync(0)++ expect(hadRoom).toBe(true)+ })++ it('gives an owner room when a fetch worker finds nothing eligible, even with runnable URLs queued', async () => {+ const pool = new FetchCandidatePool<string>(+ { maxConcurrentPerRegistrableDomain: 2, refillRunnableUrls: 3, maxQueuedUrlsPerOwner: 100 },+ { originCrawlDelayMs: () => 5_000, originNextImageStartAtMs: () => NOW_MS + 5_000 }+ )+ const owner = pool.createOwner()+ const delayedCandidates = Array.from({ length: 5 }, (_unused, index) =>+ candidate(`https://cdn.site${index}.com/a.png`, `site${index}.com`)+ )+ pool.add(delayedCandidates, 'partition-1', NOW_MS + 40_000, owner)+ let hadRoom: boolean | undefined+ void owner.waitForRoom(30_000).then((room) => {+ hadRoom = room+ })+ await jest.advanceTimersByTimeAsync(0)+ expect(owner.hasRoom()).toBe(false)+ expect(hadRoom).toBeUndefined()++ void pool.take()+ await jest.advanceTimersByTimeAsync(0)++ expect(hadRoom).toBe(true)+ pool.close()+ })+})diff --git a/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/fetch-candidate-pool.ts b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/fetch-candidate-pool.tsnew file mode 100644index 000000000000..8ede9411785e--- /dev/null+++ b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/fetch-candidate-pool.ts@@ -0,0 +1,611 @@+import { FetchCandidate } from './collected-urls-record'+import { IndexedPriorityQueue } from './fetch-candidate-queue'++export interface FetchCandidatePoolLimits {+ refillRunnableUrls: number+ maxQueuedUrlsPerOwner: number+}++export interface FetchCandidatePoolOptions extends FetchCandidatePoolLimits {+ maxConcurrentPerRegistrableDomain: number+}++export interface OriginPacing {+ originCrawlDelayMs(origin: string): number+ originNextImageStartAtMs(origin: string): number+}++export interface FetchCandidatePoolLease<T> {+ candidate: FetchCandidate+ context: T+ waitedMs: number+ release(): void+}++export interface FetchCandidatePoolAdmission {+ owner: FetchCandidatePoolOwner+ admitted(): void+}++type EntryState = 'queued' | 'expired' | 'taken' | 'withdrawn'++interface EntryPayload<T> {+ candidate: FetchCandidate+ context: T+ batch: PoolBatch<T>+}++interface PoolEntry<T> {+ /** Cleared once the entry leaves the pool, because a spent entry can stay in its origin array until compaction. */+ payload: EntryPayload<T> | undefined+ sequence: number+ enqueuedAtMs: number+ state: EntryState+ origin: OriginQueue<T>+}++interface OriginQueue<T> {+ origin: string+ domain: DomainQueue<T>+ entries: PoolEntry<T>[]+ head: number+ queued: number+ active: number+ pacedUntilMs: number+ delayed: DelayedOrigin<T> | undefined+ heapIndex: number+}++interface DomainQueue<T> {+ registrableDomain: string+ origins: Map<string, OriginQueue<T>>+ eligibleOrigins: IndexedPriorityQueue<OriginQueue<T>>+ active: number+ heapIndex: number+}++interface DelayedOrigin<T> {+ origin: OriginQueue<T>+ readyAtMs: number+ heapIndex: number+}++export interface PoolBatch<T> {+ readonly owner: FetchCandidatePoolOwner | undefined+ readonly entries: PoolEntry<T>[]+ queued: number+ expiryTimer: NodeJS.Timeout | undefined+}++const COMPACT_AFTER_ENTRIES = 64+const MAX_TIMER_DELAY_MS = 2 ** 31 - 1++/**+ * One queue of fetch candidates for the whole pod.+ *+ * Every consumer's batch adds its candidates here, and the fetch workers take them in the order they+ * entered the pool, among the origins that are under their limits and past their crawl delay. The+ * order uses pool entry and not record age, because the pass deadline and the batch timeout count+ * from when the pod received the batch. Inside one batch, the candidates enter in capture order. A+ * pass-deadline deferral goes back to the end of its partition, behind URLs captured later. In input+ * order, a domain that cannot serve its whole batch would serve those later URLs on every pass and+ * never reach the deferred one. The limits count across the pod, so two batches from one partition+ * cannot both claim a domain's slots.+ */+export class FetchCandidatePool<T> {+ private readonly domains = new Map<string, DomainQueue<T>>()+ private readonly eligibleDomains = new IndexedPriorityQueue<DomainQueue<T>>(domainHasOlderHead)+ private readonly delayedOrigins = new IndexedPriorityQueue<DelayedOrigin<T>>(+ (left, right) => left.readyAtMs < right.readyAtMs+ )+ private delayTimer: NodeJS.Timeout | undefined+ private delayTimerAtMs = Number.POSITIVE_INFINITY+ private expired: PoolEntry<T>[] = []+ private expiredHead = 0+ private waiters: (() => void)[] = []+ private readonly owners: FetchCandidatePoolOwner[] = []+ private sequence = 0+ private queuedCount = 0+ private closed = false++ constructor(+ private readonly options: FetchCandidatePoolOptions,+ private readonly pacing: OriginPacing+ ) {+ for (const [name, value] of Object.entries(options)) {+ if (!Number.isSafeInteger(value) || value <= 0) {+ throw new Error(`the image fetch candidate pool ${name} must be a positive integer, got ${value}`)+ }+ }+ }++ public get candidateCount(): number {+ return this.queuedCount + (this.expired.length - this.expiredHead)+ }++ public createOwner(): FetchCandidatePoolOwner {+ const owner = new FetchCandidatePoolOwner(this.options)+ this.owners.push(owner)+ return owner+ }++ public add(+ candidates: FetchCandidate[],+ context: T,+ deadlineMs: number,+ owner?: FetchCandidatePoolOwner+ ): PoolBatch<T> {+ if (this.closed) {+ throw new Error('the image fetch candidate pool is closed, so no worker would take these candidates')+ }+ const nowMs = Date.now()+ const batch: PoolBatch<T> = { owner, entries: [], queued: 0, expiryTimer: undefined }+ const touched = new Set<OriginQueue<T>>()+ for (const candidate of [...candidates].sort((left, right) => left.firstSeenAtMs - right.firstSeenAtMs)) {+ const origin = this.originQueueFor(candidate)+ const entry: PoolEntry<T> = {+ payload: { candidate, context, batch },+ sequence: this.sequence++,+ enqueuedAtMs: nowMs,+ state: 'queued',+ origin,+ }+ origin.entries.push(entry)+ origin.queued += 1+ batch.entries.push(entry)+ batch.queued += 1+ this.queuedCount += 1+ owner?.recordQueued(candidate.registrableDomain)+ touched.add(origin)+ }+ for (const origin of touched) {+ this.refreshOrigin(origin, nowMs)+ }+ if (batch.queued > 0) {+ // The fetch path treats a candidate as late only once the clock is past the deadline.+ batch.expiryTimer = setTimeout(() => this.expire(batch), Math.max(0, deadlineMs - nowMs) + 1)+ batch.expiryTimer.unref()+ this.wake(1)+ }+ return batch+ }++ public async take(): Promise<FetchCandidatePoolLease<T> | undefined> {+ for (;;) {+ if (this.closed) {+ return undefined+ }+ const lease = this.tryTake()+ if (lease) {+ if (this.hasSelectableWork()) {+ this.wake(1)+ }+ return lease+ }+ for (const owner of this.owners) {+ owner.recordIdleWorker()+ }+ await new Promise<void>((resolve) => this.waiters.push(resolve))+ }+ }++ /** Removes the batch's candidates that no worker has taken, and returns how many it removed. */+ public withdraw(batch: PoolBatch<T>): number {+ const touched = new Set<OriginQueue<T>>()+ let withdrawn = 0+ for (const entry of batch.entries) {+ if (entry.state === 'queued') {+ this.leaveOriginQueue(entry)+ touched.add(entry.origin)+ } else if (entry.state !== 'expired') {+ continue+ }+ entry.state = 'withdrawn'+ entry.payload = undefined+ withdrawn += 1+ }+ this.clearExpiry(batch)+ const nowMs = Date.now()+ for (const origin of touched) {+ this.refreshOrigin(origin, nowMs)+ }+ return withdrawn+ }++ public close(): void {+ this.closed = true+ if (this.delayTimer) {+ clearTimeout(this.delayTimer)+ this.delayTimer = undefined+ }+ this.wake(this.waiters.length)+ }++ private tryTake(): FetchCandidatePoolLease<T> | undefined {+ const expiredEntry = this.nextExpired()+ if (expiredEntry) {+ expiredEntry.state = 'taken'+ return this.lease(expiredEntry, Date.now(), () => undefined)+ }+ const domain = this.eligibleDomains.peek()+ if (!domain) {+ return undefined+ }+ const origin = domain.eligibleOrigins.peek()!+ const entry = origin.entries[origin.head]+ const nowMs = Date.now()+ this.leaveOriginQueue(entry)+ entry.state = 'taken'+ origin.active += 1+ domain.active += 1+ const crawlDelayMs = this.pacing.originCrawlDelayMs(origin.origin)+ if (crawlDelayMs > 0) {+ origin.pacedUntilMs = Math.max(nowMs, origin.pacedUntilMs) + crawlDelayMs+ }+ this.refreshOrigin(origin, nowMs)+ let released = false+ return this.lease(entry, nowMs, () => {+ if (released) {+ return+ }+ released = true+ origin.active -= 1+ domain.active -= 1+ this.refreshOrigin(origin, Date.now())+ this.wake(1)+ })+ }++ private lease(entry: PoolEntry<T>, nowMs: number, release: () => void): FetchCandidatePoolLease<T> {+ const { candidate, context } = entry.payload!+ entry.payload = undefined+ return {+ candidate,+ context,+ waitedMs: Math.max(0, nowMs - entry.enqueuedAtMs),+ release,+ }+ }++ private leaveOriginQueue(entry: PoolEntry<T>): void {+ const { batch, candidate } = entry.payload!+ entry.origin.queued -= 1+ batch.queued -= 1+ this.queuedCount -= 1+ batch.owner?.recordLeft(candidate.registrableDomain)+ if (batch.queued === 0) {+ this.clearExpiry(batch)+ }+ }++ private expire(batch: PoolBatch<T>): void {+ batch.expiryTimer = undefined+ const touched = new Set<OriginQueue<T>>()+ for (const entry of batch.entries) {+ if (entry.state !== 'queued') {+ continue+ }+ this.leaveOriginQueue(entry)+ entry.state = 'expired'+ this.expired.push(entry)+ touched.add(entry.origin)+ }+ const nowMs = Date.now()+ for (const origin of touched) {+ this.refreshOrigin(origin, nowMs)+ }+ this.wake(1)+ }++ private clearExpiry(batch: PoolBatch<T>): void {+ if (batch.expiryTimer) {+ clearTimeout(batch.expiryTimer)+ batch.expiryTimer = undefined+ }+ }++ private nextExpired(): PoolEntry<T> | undefined {+ while (this.expiredHead < this.expired.length) {+ const entry = this.expired[this.expiredHead++]+ if (entry.state === 'expired') {+ this.compactExpired()+ return entry+ }+ }+ this.compactExpired()+ return undefined+ }++ private compactExpired(): void {+ if (this.expiredHead === this.expired.length) {+ this.expired = []+ this.expiredHead = 0+ } else if (this.expiredHead > COMPACT_AFTER_ENTRIES && this.expiredHead * 2 > this.expired.length) {+ this.expired = this.expired.slice(this.expiredHead)+ this.expiredHead = 0+ }+ }++ private hasSelectableWork(): boolean {+ for (let index = this.expiredHead; index < this.expired.length; index++) {+ if (this.expired[index].state === 'expired') {+ return true+ }+ }+ return this.eligibleDomains.size > 0+ }++ private wake(count: number): void {+ for (let woken = 0; woken < count && this.waiters.length > 0; woken++) {+ this.waiters.shift()!()+ }+ }++ private originQueueFor(candidate: FetchCandidate): OriginQueue<T> {+ let domain = this.domains.get(candidate.registrableDomain)+ if (!domain) {+ domain = {+ registrableDomain: candidate.registrableDomain,+ origins: new Map(),+ eligibleOrigins: new IndexedPriorityQueue<OriginQueue<T>>(originHasOlderHead),+ active: 0,+ heapIndex: -1,+ }+ this.domains.set(candidate.registrableDomain, domain)+ }+ let origin = domain.origins.get(candidate.origin)+ if (!origin) {+ origin = {+ origin: candidate.origin,+ domain,+ entries: [],+ head: 0,+ queued: 0,+ active: 0,+ pacedUntilMs: 0,+ delayed: undefined,+ heapIndex: -1,+ }+ domain.origins.set(candidate.origin, origin)+ }+ return origin+ }++ private refreshOrigin(origin: OriginQueue<T>, nowMs: number): void {+ this.advanceHead(origin)+ const domain = origin.domain+ if (origin.queued === 0 || origin.active >= this.options.maxConcurrentPerRegistrableDomain) {+ domain.eligibleOrigins.remove(origin)+ this.undelay(origin)+ if (origin.queued === 0 && origin.active === 0) {+ domain.origins.delete(origin.origin)+ }+ } else {+ const readyAtMs = Math.max(origin.pacedUntilMs, this.pacing.originNextImageStartAtMs(origin.origin))+ if (readyAtMs > nowMs) {+ domain.eligibleOrigins.remove(origin)+ this.delay(origin, readyAtMs)+ } else {+ this.undelay(origin)+ domain.eligibleOrigins.addOrUpdate(origin)+ }+ }+ this.refreshDomain(domain)+ }++ private refreshDomain(domain: DomainQueue<T>): void {+ if (domain.origins.size === 0 && domain.active === 0) {+ this.eligibleDomains.remove(domain)+ if (this.domains.get(domain.registrableDomain) === domain) {+ this.domains.delete(domain.registrableDomain)+ }+ return+ }+ if (domain.active < this.options.maxConcurrentPerRegistrableDomain && domain.eligibleOrigins.size > 0) {+ this.eligibleDomains.addOrUpdate(domain)+ } else {+ this.eligibleDomains.remove(domain)+ }+ }++ private advanceHead(origin: OriginQueue<T>): void {+ while (origin.head < origin.entries.length && origin.entries[origin.head].state !== 'queued') {+ origin.head += 1+ }+ if (origin.head === origin.entries.length) {+ origin.entries = []+ origin.head = 0+ } else if (origin.head > COMPACT_AFTER_ENTRIES && origin.head * 2 > origin.entries.length) {+ origin.entries = origin.entries.slice(origin.head)+ origin.head = 0+ }+ }++ private delay(origin: OriginQueue<T>, readyAtMs: number): void {+ if (origin.delayed) {+ origin.delayed.readyAtMs = readyAtMs+ this.delayedOrigins.addOrUpdate(origin.delayed)+ } else {+ origin.delayed = { origin, readyAtMs, heapIndex: -1 }+ this.delayedOrigins.addOrUpdate(origin.delayed)+ }+ this.armDelayTimer()+ }++ private undelay(origin: OriginQueue<T>): void {+ if (!origin.delayed) {+ return+ }+ this.delayedOrigins.remove(origin.delayed)+ origin.delayed = undefined+ this.armDelayTimer()+ }++ private armDelayTimer(): void {+ const next = this.delayedOrigins.peek()+ const nextAtMs = next?.readyAtMs ?? Number.POSITIVE_INFINITY+ if (nextAtMs === this.delayTimerAtMs || this.closed) {+ return+ }+ if (this.delayTimer) {+ clearTimeout(this.delayTimer)+ this.delayTimer = undefined+ }+ this.delayTimerAtMs = nextAtMs+ if (next) {+ // Node runs a longer timer after 1 ms, so a long crawl delay would rearm it in a tight loop.+ const waitMs = Math.min(MAX_TIMER_DELAY_MS, Math.max(0, nextAtMs - Date.now()))+ this.delayTimer = setTimeout(() => this.releaseDelayedOrigins(), waitMs)+ this.delayTimer.unref()+ }+ }++ private releaseDelayedOrigins(): void {+ this.delayTimer = undefined+ this.delayTimerAtMs = Number.POSITIVE_INFINITY+ const nowMs = Date.now()+ for (+ let next = this.delayedOrigins.peek();+ next && next.readyAtMs <= nowMs;+ next = this.delayedOrigins.peek()+ ) {+ this.delayedOrigins.remove(next)+ next.origin.delayed = undefined+ this.refreshOrigin(next.origin, nowMs)+ }+ this.armDelayTimer()+ if (this.eligibleDomains.size > 0) {+ this.wake(1)+ }+ }+}++/**+ * The window of one consumer: how much of its work waits in the pool.+ *+ * A registrable domain counts as runnable only up to its concurrency limit, because the pool never+ * runs more than that at once. A window full of one hot domain therefore still lets the consumer+ * read more, so its other domains do not wait in Kafka while candidate slots are free.+ */+export class FetchCandidatePoolOwner {+ private queued = 0+ private runnable = 0+ private pendingAdmissions = 0+ private readonly queuedByDomain = new Map<string, number>()+ private readonly roomWaiters = new Set<() => void>()++ constructor(private readonly options: FetchCandidatePoolOptions) {}++ public get queuedUrls(): number {+ return this.queued+ }++ public get runnableUrls(): number {+ return this.runnable+ }++ public beginAdmission(): FetchCandidatePoolAdmission {+ this.pendingAdmissions += 1+ let done = false+ return {+ owner: this,+ admitted: () => {+ if (done) {+ return+ }+ done = true+ this.pendingAdmissions -= 1+ this.notifyIfRoom()+ },+ }+ }++ public hasRoom(): boolean {+ return (+ this.pendingAdmissions === 0 &&+ this.runnable < this.options.refillRunnableUrls &&+ this.queued < this.options.maxQueuedUrlsPerOwner+ )+ }++ /** Resolves true when the owner has room for another batch, or false when the wait runs out. */+ public waitForRoom(maxWaitMs: number): Promise<boolean> {+ if (this.hasRoom()) {+ return Promise.resolve(true)+ }+ return new Promise((resolve) => {+ const waiter = (): void => {+ clearTimeout(timer)+ resolve(true)+ }+ const timer = setTimeout(() => {+ this.roomWaiters.delete(waiter)+ resolve(false)+ }, maxWaitMs)+ this.roomWaiters.add(waiter)+ })+ }++ public recordQueued(registrableDomain: string): void {+ this.queued += 1+ this.changeDomainCount(registrableDomain, 1)+ }++ public recordLeft(registrableDomain: string): void {+ this.queued -= 1+ this.changeDomainCount(registrableDomain, -1)+ this.notifyIfRoom()+ }++ /**+ * A fetch worker found no eligible candidate. Queued URLs that count as runnable can still be+ * unable to run, for example behind a crawl delay or behind another member's hold on a domain,+ * so the member reads more while it is under the queue limit.+ */+ public recordIdleWorker(): void {+ if (this.pendingAdmissions === 0 && this.queued < this.options.maxQueuedUrlsPerOwner) {+ this.releaseRoomWaiters()+ }+ }++ private changeDomainCount(registrableDomain: string, delta: number): void {+ const limit = this.options.maxConcurrentPerRegistrableDomain+ const before = this.queuedByDomain.get(registrableDomain) ?? 0+ const after = before + delta+ if (after === 0) {+ this.queuedByDomain.delete(registrableDomain)+ } else {+ this.queuedByDomain.set(registrableDomain, after)+ }+ this.runnable += Math.min(after, limit) - Math.min(before, limit)+ }++ private notifyIfRoom(): void {+ if (this.hasRoom()) {+ this.releaseRoomWaiters()+ }+ }++ private releaseRoomWaiters(): void {+ if (this.roomWaiters.size === 0) {+ return+ }+ const waiters = [...this.roomWaiters]+ this.roomWaiters.clear()+ for (const waiter of waiters) {+ waiter()+ }+ }+}++function originHeadSequence<T>(origin: OriginQueue<T>): number {+ return origin.entries[origin.head].sequence+}++function originHasOlderHead<T>(left: OriginQueue<T>, right: OriginQueue<T>): boolean {+ return originHeadSequence(left) < originHeadSequence(right)+}++function domainHasOlderHead<T>(left: DomainQueue<T>, right: DomainQueue<T>): boolean {+ return originHeadSequence(left.eligibleOrigins.peek()!) < originHeadSequence(right.eligibleOrigins.peek()!)+}diff --git a/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/fetch-candidate-queue.test.ts b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/fetch-candidate-queue.test.tsnew file mode 100644index 000000000000..e6565e585d38--- /dev/null+++ b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/fetch-candidate-queue.test.ts@@ -0,0 +1,133 @@+import { FetchCandidate, MAX_HOPS } from './collected-urls-record'+import { FetchCandidateQueue } from './fetch-candidate-queue'++const OPTIONS = {+ maxConcurrentPerRegistrableDomain: 6,+ maxInFlightRequests: 300,+}++function candidate(index: number): FetchCandidate {+ return {+ originalRef: `ref-${index}`,+ currentUrl: `https://origin-${index}.example.com/image.png`,+ host: `origin-${index}.example.com`,+ origin: `https://origin-${index}.example.com`,+ registrableDomain: 'example.com',+ remainingHops: MAX_HOPS,+ notBeforeMs: 0,+ firstSeenAtMs: 1_700_000_000_000,+ fetchCount: 0,+ republishCount: 0,+ lastRepublishReason: null,+ }+}++function dominantOriginCandidate(index: number): FetchCandidate {+ return {+ ...candidate(index),+ currentUrl: `https://dominant.example.com/${index}.png`,+ host: 'dominant.example.com',+ origin: 'https://dominant.example.com',+ }+}++function domainCandidate(domainIndex: number, candidateIndex: number): FetchCandidate {+ const index = domainIndex * 1_000 + candidateIndex+ const registrableDomain = `domain-${domainIndex}.example`+ return {+ ...candidate(index),+ currentUrl: `https://cdn.${registrableDomain}/${candidateIndex}.png`,+ host: `cdn.${registrableDomain}`,+ origin: `https://cdn.${registrableDomain}`,+ registrableDomain,+ }+}++function concurrentDomainCounts(queue: FetchCandidateQueue, count: number): Map<string, number> {+ const counts = new Map<string, number>()+ for (let index = 0; index < count; index++) {+ const lease = queue.take()+ expect(lease).toBeDefined()+ if (lease) {+ counts.set(lease.candidate.registrableDomain, (counts.get(lease.candidate.registrableDomain) ?? 0) + 1)+ }+ }+ return counts+}++describe('FetchCandidateQueue', () => {+ it('drains a maximum-sized diverse poll batch without scanning every remaining origin', () => {+ const candidateCount = 50_000+ const queue = new FetchCandidateQueue(+ Array.from({ length: candidateCount }, (_, index) => candidate(index)),+ OPTIONS+ )+ let drained = 0++ for (;;) {+ const lease = queue.take()+ if (!lease) {+ break+ }+ drained += 1+ lease.release()+ }++ expect(drained).toBe(candidateCount)+ })++ it('keeps one selectable domain entry during a long dominant-origin run', () => {+ const dominantCandidateCount = 50_000+ const queue = new FetchCandidateQueue(+ [+ ...Array.from({ length: dominantCandidateCount }, (_, index) => dominantOriginCandidate(index)),+ candidate(dominantCandidateCount),+ ],+ {+ ...OPTIONS,+ maxConcurrentPerRegistrableDomain: 1,+ }+ )+ let drained = 0+ let maximumSelectableDomains = 0++ for (;;) {+ const lease = queue.take()+ if (!lease) {+ break+ }+ drained += 1+ lease.release()+ maximumSelectableDomains = Math.max(maximumSelectableDomains, queue.selectableRegistrableDomainCount)+ }++ expect(drained).toBe(dominantCandidateCount + 1)+ expect(maximumSelectableDomains).toBe(1)+ })++ it.each([+ {+ name: 'gives 150 equal domains two requests each',+ domainSizes: Array.from({ length: 150 }, () => 6),+ expectedCounts: Array.from({ length: 150 }, () => 2),+ },+ {+ name: 'gives smaller domains their proportional request share',+ domainSizes: [...Array.from({ length: 50 }, () => 100), ...Array.from({ length: 50 }, () => 10)],+ expectedCounts: [...Array.from({ length: 50 }, () => 5), ...Array.from({ length: 50 }, () => 1)],+ },+ ])('$name', ({ domainSizes, expectedCounts }) => {+ const queue = new FetchCandidateQueue(+ domainSizes.flatMap((size, domainIndex) =>+ Array.from({ length: size }, (_, candidateIndex) => domainCandidate(domainIndex, candidateIndex))+ ),+ OPTIONS+ )++ const counts = concurrentDomainCounts(queue, OPTIONS.maxInFlightRequests)++ expect(+ Array.from({ length: domainSizes.length }, (_, index) => counts.get(`domain-${index}.example`) ?? 0)+ ).toEqual(expectedCounts)+ })+})diff --git a/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/fetch-candidate-queue.ts b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/fetch-candidate-queue.tsnew file mode 100644index 000000000000..d6578c07558e--- /dev/null+++ b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/fetch-candidate-queue.ts@@ -0,0 +1,420 @@+import { FetchCandidate } from './collected-urls-record'++export interface FetchCandidateLease {+ candidate: FetchCandidate+ release(): void+}++interface OriginQueue {+ origin: string+ candidates: FetchCandidate[]+ head: number+ sequence: number+ active: number+ initialCandidateCount: number+ targetConcurrent: number+ heapIndex: number+}++interface RegistrableDomainQueue {+ registrableDomain: string+ origins: Map<string, OriginQueue>+ availableOrigins: IndexedPriorityQueue<OriginQueue>+ initialCandidateCount: number+ waitingCandidateCount: number+ targetConcurrent: number+ sequence: number+ active: number+ heapIndex: number+}++export interface IndexedPriorityQueueItem {+ heapIndex: number+}++export class IndexedPriorityQueue<T extends IndexedPriorityQueueItem> {+ private readonly heap: T[] = []++ constructor(private readonly hasPriority: (left: T, right: T) => boolean) {}++ public get size(): number {+ return this.heap.length+ }++ public addOrUpdate(item: T): void {+ if (item.heapIndex < 0) {+ item.heapIndex = this.heap.length+ this.heap.push(item)+ this.siftUp(item.heapIndex)+ return+ }+ this.rebalance(item.heapIndex)+ }++ public remove(item: T): void {+ const index = item.heapIndex+ if (index < 0) {+ return+ }+ const last = this.heap.pop()!+ item.heapIndex = -1+ if (index === this.heap.length) {+ return+ }+ this.heap[index] = last+ last.heapIndex = index+ this.rebalance(index)+ }++ public peek(): T | undefined {+ return this.heap[0]+ }++ public poll(): T | undefined {+ const item = this.heap[0]+ if (item) {+ this.remove(item)+ }+ return item+ }++ private rebalance(index: number): void {+ const parentIndex = Math.floor((index - 1) / 2)+ if (index > 0 && this.hasPriority(this.heap[index], this.heap[parentIndex])) {+ this.siftUp(index)+ } else {+ this.siftDown(index)+ }+ }++ private siftUp(startIndex: number): void {+ let index = startIndex+ while (index > 0) {+ const parentIndex = Math.floor((index - 1) / 2)+ if (!this.hasPriority(this.heap[index], this.heap[parentIndex])) {+ return+ }+ this.swap(index, parentIndex)+ index = parentIndex+ }+ }++ private siftDown(startIndex: number): void {+ let index = startIndex+ for (;;) {+ const leftIndex = index * 2 + 1+ if (leftIndex >= this.heap.length) {+ return+ }+ const rightIndex = leftIndex + 1+ const childIndex =+ rightIndex < this.heap.length && this.hasPriority(this.heap[rightIndex], this.heap[leftIndex])+ ? rightIndex+ : leftIndex+ if (!this.hasPriority(this.heap[childIndex], this.heap[index])) {+ return+ }+ this.swap(index, childIndex)+ index = childIndex+ }+ }++ private swap(leftIndex: number, rightIndex: number): void {+ const left = this.heap[leftIndex]+ const right = this.heap[rightIndex]+ this.heap[leftIndex] = right+ this.heap[rightIndex] = left+ left.heapIndex = rightIndex+ right.heapIndex = leftIndex+ }+}++export interface FetchCandidateQueueOptions {+ maxConcurrentPerRegistrableDomain: number+ maxInFlightRequests: number+}++export interface DeduplicatedFetchCandidates {+ candidates: FetchCandidate[]+ duplicateCount: number+}++export function deduplicateFetchCandidates(candidates: FetchCandidate[]): DeduplicatedFetchCandidates {+ const candidatesByCanonicalRef = new Map<string, FetchCandidate>()+ let duplicateCount = 0+ for (const candidate of candidates) {+ const existing = candidatesByCanonicalRef.get(candidate.originalRef)+ if (existing) {+ duplicateCount += 1+ candidatesByCanonicalRef.set(candidate.originalRef, mergeDuplicateFetchCandidates(existing, candidate))+ } else {+ candidatesByCanonicalRef.set(candidate.originalRef, candidate)+ }+ }+ return { candidates: [...candidatesByCanonicalRef.values()], duplicateCount }+}++export class FetchCandidateQueue {+ private readonly domains = new Map<string, RegistrableDomainQueue>()+ private readonly availableDomains = new IndexedPriorityQueue<RegistrableDomainQueue>(domainQueueHasPriority)+ private waitingCandidates = 0+ private initialSchedulableSlots = 0+ private aborted = false++ constructor(+ candidates: FetchCandidate[],+ private readonly options: FetchCandidateQueueOptions+ ) {+ const deduplicated = deduplicateFetchCandidates(candidates)+ let domainSequence = 0+ let originSequence = 0+ for (const candidate of deduplicated.candidates) {+ let domain = this.domains.get(candidate.registrableDomain)+ if (!domain) {+ domain = {+ registrableDomain: candidate.registrableDomain,+ origins: new Map(),+ availableOrigins: new IndexedPriorityQueue<OriginQueue>(originQueueHasPriority),+ initialCandidateCount: 0,+ waitingCandidateCount: 0,+ targetConcurrent: 0,+ sequence: domainSequence++,+ active: 0,+ heapIndex: -1,+ }+ this.domains.set(candidate.registrableDomain, domain)+ }+ let originQueue = domain.origins.get(candidate.origin)+ if (!originQueue) {+ originQueue = {+ origin: candidate.origin,+ candidates: [],+ head: 0,+ sequence: originSequence++,+ active: 0,+ initialCandidateCount: 0,+ targetConcurrent: 0,+ heapIndex: -1,+ }+ domain.origins.set(candidate.origin, originQueue)+ }+ originQueue.candidates.push(candidate)+ originQueue.initialCandidateCount += 1+ domain.initialCandidateCount += 1+ domain.waitingCandidateCount += 1+ this.waitingCandidates += 1+ }+ allocateConcurrentTargets(+ [...this.domains.values()],+ this.options.maxInFlightRequests,+ this.options.maxConcurrentPerRegistrableDomain+ )+ for (const domain of this.domains.values()) {+ allocateConcurrentTargets(+ [...domain.origins.values()],+ domain.targetConcurrent,+ this.options.maxConcurrentPerRegistrableDomain+ )+ for (const origin of domain.origins.values()) {+ this.refreshOriginSelection(domain, origin)+ }+ this.initialSchedulableSlots += Math.min(+ domain.initialCandidateCount,+ this.options.maxConcurrentPerRegistrableDomain+ )+ this.refreshDomainSelection(domain)+ }+ }++ public get candidateCount(): number {+ return this.waitingCandidates+ }++ public get selectableRegistrableDomainCount(): number {+ return this.availableDomains.size+ }++ public get schedulableSlotsAtStart(): number {+ return this.initialSchedulableSlots+ }++ public availableRequestSlotsAtStart(availableConnections: (registrableDomain: string) => number): number {+ let slots = 0+ for (const domain of this.domains.values()) {+ slots += Math.min(domain.initialCandidateCount, availableConnections(domain.registrableDomain))+ }+ return slots+ }++ public abort(): void {+ this.aborted = true+ }++ public take(): FetchCandidateLease | undefined {+ if (this.aborted) {+ return undefined+ }+ const selected = this.takeHighestPriorityOrigin()+ if (!selected) {+ return undefined+ }+ const candidate = selected.origin.candidates[selected.origin.head++]!+ selected.domain.waitingCandidateCount -= 1+ this.refreshOriginSelection(selected.domain, selected.origin)+ this.refreshDomainSelection(selected.domain)++ this.waitingCandidates -= 1+ let released = false+ return {+ candidate,+ release: () => {+ if (released) {+ return+ }+ released = true+ selected.domain.active = Math.max(0, selected.domain.active - 1)+ selected.origin.active = Math.max(0, selected.origin.active - 1)+ if (originCandidateCount(selected.origin) === 0 && selected.origin.active === 0) {+ selected.domain.origins.delete(selected.origin.origin)+ } else {+ this.refreshOriginSelection(selected.domain, selected.origin)+ }+ if (selected.domain.origins.size === 0) {+ this.availableDomains.remove(selected.domain)+ this.domains.delete(selected.domain.registrableDomain)+ } else {+ this.refreshDomainSelection(selected.domain)+ }+ },+ }+ }++ private takeHighestPriorityOrigin(): { domain: RegistrableDomainQueue; origin: OriginQueue } | undefined {+ for (;;) {+ const domain = this.availableDomains.poll()+ if (!domain) {+ return undefined+ }+ if (domain.active >= this.options.maxConcurrentPerRegistrableDomain) {+ continue+ }+ const origin = domain.availableOrigins.poll()+ if (!origin) {+ continue+ }+ domain.active += 1+ origin.active += 1+ return { domain, origin }+ }+ }++ private refreshOriginSelection(domain: RegistrableDomainQueue, origin: OriginQueue): void {+ if (originCandidateCount(origin) === 0 || origin.active >= this.options.maxConcurrentPerRegistrableDomain) {+ domain.availableOrigins.remove(origin)+ return+ }+ domain.availableOrigins.addOrUpdate(origin)+ }++ private refreshDomainSelection(domain: RegistrableDomainQueue): void {+ const origin = domain.availableOrigins.peek()+ if (!origin || domain.active >= this.options.maxConcurrentPerRegistrableDomain) {+ this.availableDomains.remove(domain)+ return+ }+ this.availableDomains.addOrUpdate(domain)+ }+}++function originQueueHasPriority(left: OriginQueue, right: OriginQueue): boolean {+ const leftDistance = left.targetConcurrent - left.active+ const rightDistance = right.targetConcurrent - right.active+ return (+ leftDistance > rightDistance ||+ (leftDistance === rightDistance && originCandidateCount(left) > originCandidateCount(right)) ||+ (leftDistance === rightDistance &&+ originCandidateCount(left) === originCandidateCount(right) &&+ left.sequence < right.sequence)+ )+}++function originCandidateCount(origin: OriginQueue): number {+ return origin.candidates.length - origin.head+}++function domainQueueHasPriority(left: RegistrableDomainQueue, right: RegistrableDomainQueue): boolean {+ const leftDistance = left.targetConcurrent - left.active+ const rightDistance = right.targetConcurrent - right.active+ return (+ leftDistance > rightDistance ||+ (leftDistance === rightDistance && left.waitingCandidateCount > right.waitingCandidateCount) ||+ (leftDistance === rightDistance &&+ left.waitingCandidateCount === right.waitingCandidateCount &&+ left.sequence < right.sequence)+ )+}++interface ConcurrentTarget {+ initialCandidateCount: number+ targetConcurrent: number+}++function allocateConcurrentTargets<T extends ConcurrentTarget>(+ queues: T[],+ totalCapacity: number,+ maximumPerQueue: number+): void {+ let remainingCapacity = Math.min(+ totalCapacity,+ queues.reduce((sum, queue) => sum + queue.initialCandidateCount, 0)+ )+ let remainingWeight = queues.reduce((sum, queue) => sum + queue.initialCandidateCount, 0)+ let remainingQueues = queues++ while (remainingQueues.length > 0 && remainingCapacity > 0 && remainingWeight > 0) {+ const queuesAtLimit = remainingQueues.filter((queue) => {+ const proportionalTarget = (remainingCapacity * queue.initialCandidateCount) / remainingWeight+ return proportionalTarget >= Math.min(maximumPerQueue, queue.initialCandidateCount)+ })+ if (queuesAtLimit.length === 0) {+ for (const queue of remainingQueues) {+ queue.targetConcurrent = (remainingCapacity * queue.initialCandidateCount) / remainingWeight+ }+ return+ }+ const queuesAtLimitSet = new Set(queuesAtLimit)+ for (const queue of queuesAtLimit) {+ queue.targetConcurrent = Math.min(maximumPerQueue, queue.initialCandidateCount)+ remainingCapacity -= queue.targetConcurrent+ remainingWeight -= queue.initialCandidateCount+ }+ remainingQueues = remainingQueues.filter((queue) => !queuesAtLimitSet.has(queue))+ }+}++export function mergeDuplicateFetchCandidates(left: FetchCandidate, right: FetchCandidate): FetchCandidate {+ let preferredRoute = left+ if (+ right.remainingHops < left.remainingHops ||+ (right.remainingHops === left.remainingHops && right.republishCount > left.republishCount) ||+ (right.remainingHops === left.remainingHops &&+ right.republishCount === left.republishCount &&+ right.fetchCount > left.fetchCount)+ ) {+ preferredRoute = right+ }+ const latestState = left.republishCount >= right.republishCount ? left : right+ const sourcePartitions = [...new Set([...(left.sourcePartitions ?? []), ...(right.sourcePartitions ?? [])])].sort(+ (leftPartition, rightPartition) => leftPartition - rightPartition+ )+ return {+ ...preferredRoute,+ remainingHops: Math.min(left.remainingHops, right.remainingHops),+ notBeforeMs: Math.max(left.notBeforeMs, right.notBeforeMs),+ firstSeenAtMs: Math.min(left.firstSeenAtMs, right.firstSeenAtMs),+ fetchCount: Math.max(left.fetchCount, right.fetchCount),+ republishCount: Math.max(left.republishCount, right.republishCount),+ lastRepublishReason: latestState.lastRepublishReason,+ sourcePartitions: sourcePartitions.length > 0 ? sourcePartitions : undefined,+ }+}diff --git a/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/fetch-runner.test.ts b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/fetch-runner.test.tsnew file mode 100644index 000000000000..1e27daa0cbb1--- /dev/null+++ b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/fetch-runner.test.ts@@ -0,0 +1,919 @@+import { S3ServiceException } from '@aws-sdk/client-s3'+import { createHash } from 'node:crypto'++import { logger } from '~/common/utils/logger'+import { RecordedTopHogMetric, createRecordingTopHog } from '~/tests/helpers/tophog'++import { FetchCandidate, MAX_HOPS } from './collected-urls-record'+import { ConfigurationPolicyService, OriginPolicyDecision } from './configuration-policy'+import { ConfigurationCacheItem, CrawlHistoryItem, HttpCacheMetadata } from './crawl-history'+import { DELAY_TOO_LONG, FetchRunner, FetchRunnerOptions, HOPS_EXHAUSTED } from './fetch-runner'+import { FrontierPublisher, RepublishResult } from './frontier-publisher'+import { HostBudget } from './host-budget'+import { ImageFetchOptions, ImageFetchResult, ImageFetcher } from './image-fetcher'+import { ImageFetchRequestMetrics } from './metrics'+import { OriginRequestScheduler } from './origin-request-scheduler'+import { ResponseUrlBinding } from './response-url-binding'+import { ImageFetchTopHogMetrics } from './tophog-metrics'++const NOW_MS = 1_700_000_000_000+const OPTIONS: FetchRunnerOptions = {+ maxConcurrentPerRegistrableDomain: 2,+ maxInFlightRequests: 50,+ batchBudgetMs: 20_000,+ maxBytes: 20 * 1024 * 1024,+ requestTimeoutMs: 10_000,+ maxRedirects: 3,+ seenTtlSeconds: 30 * 24 * 60 * 60,+}+const POOL_OPTIONS: FetchRunnerOptions = {+ ...OPTIONS,+ continuousPool: { refillRunnableUrls: 100, maxQueuedUrlsPerOwner: 1_000 },+}+const builtRunners: FetchRunner[] = []+const bind: ResponseUrlBinding = (url) => createHash('sha256').update(url).digest('base64url').slice(0, 22)++function candidate(overrides: Partial<FetchCandidate> = {}): FetchCandidate {+ return {+ originalRef: `imageurl:${'a'.repeat(22)}`,+ currentUrl: 'https://cdn.example.com/a.png',+ host: 'cdn.example.com',+ origin: 'https://cdn.example.com',+ registrableDomain: 'example.com',+ remainingHops: MAX_HOPS,+ notBeforeMs: 0,+ firstSeenAtMs: NOW_MS - 1_000,+ fetchCount: 0,+ republishCount: 0,+ lastRepublishReason: null,+ ...overrides,+ }+}++interface Harness {+ runner: FetchRunner+ budget: HostBudget+ fetch: jest.Mock<Promise<ImageFetchResult>, [string, ImageFetchOptions]>+ check: jest.Mock<Promise<OriginPolicyDecision>, [string, Map<string, ConfigurationCacheItem>, number]>+ createPass: jest.Mock+ republish: jest.Mock<Promise<RepublishResult>, any[]>+ publishImage: jest.Mock<Promise<void>, any[]>+ topHogRecords: Map<string, RecordedTopHogMetric[]>+}++function build(+ result: Partial<ImageFetchResult> = {},+ policy: Partial<OriginPolicyDecision> = {},+ republishResult: RepublishResult = 'queued',+ options: FetchRunnerOptions = OPTIONS+): Harness {+ const fetch = jest.fn((url: string, _options: ImageFetchOptions) =>+ Promise.resolve({ outcome: 'ok', redirects: 0, currentUrl: url, ...result } as ImageFetchResult)+ )+ const check = jest.fn((_url: string, _cached: Map<string, ConfigurationCacheItem>, _nowMs: number) =>+ Promise.resolve({+ allowed: true,+ transient: false,+ crawlDelayMs: 1_000,+ tdmrepReservation: false,+ updates: [],+ ...policy,+ } as OriginPolicyDecision)+ )+ const createPass = jest.fn(() => ({ check }))+ const republish = jest.fn(() => Promise.resolve(republishResult))+ const createRepublishBatch = jest.fn(() => ({ republish, flush: () => Promise.resolve({ failedUrls: 0 }) }))+ const publishImage = jest.fn(() => Promise.resolve())+ const scheduler = {+ runImage: async (_url: URL, _deadlineMs: number, request: () => Promise<ImageFetchResult>) => ({+ ran: true as const,+ value: await request(),+ }),+ authorizeImageRedirect: () => Promise.resolve(true),+ } as unknown as OriginRequestScheduler+ const budget = new HostBudget({+ requestsPerSecond: 1,+ burst: 5,+ maxConcurrent: 6,+ breakerFailures: 5,+ breakerCooldownMs: 60_000,+ breakerMaxCooldownMs: 3_600_000,+ maxTrackedRegistrableDomains: 20_000,+ maxTrackedOrigins: 20_000,+ random: () => 0,+ })+ const recordingTopHog = createRecordingTopHog()+ const runner = new FetchRunner(+ { fetch } as ImageFetcher,+ budget,+ scheduler,+ { createPass } as unknown as ConfigurationPolicyService,+ options,+ { createRepublishBatch } as unknown as FrontierPublisher,+ { refreshIntervalMs: 7 * 24 * 60 * 60 * 1000, publishImage, refreshImage: () => Promise.resolve(true) },+ { bindingFor: () => Promise.resolve(bind) },+ new ImageFetchTopHogMetrics(recordingTopHog.registry)+ )+ builtRunners.push(runner)+ return { runner, budget, fetch, check, createPass, republish, publishImage, topHogRecords: recordingTopHog.records }+}++describe('FetchRunner', () => {+ beforeEach(() => jest.useFakeTimers().setSystemTime(NOW_MS))+ afterEach(async () => {+ await Promise.all(builtRunners.splice(0).map((runner) => runner.close()))+ jest.useRealTimers()+ jest.restoreAllMocks()+ })++ it('passes cached validators and TDM state to the image request', async () => {+ const harness = build({}, { tdmrepReservation: true })+ const cache: HttpCacheMetadata = {+ requestTimeMs: NOW_MS - 10,+ responseTimeMs: NOW_MS,+ responseUrlHmac: bind(candidate().currentUrl),+ etag: '"image-v1"',+ }+ const stored = new Map<string, CrawlHistoryItem>([+ [+ candidate().originalRef,+ {+ kind: 'url',+ key: candidate().originalRef,+ nextFetchAtMs: NOW_MS,+ storageExpiresAtMs: NOW_MS,+ outcome: 'ok',+ cache,+ },+ ],+ ])++ await harness.runner.run([candidate({ sourcePartitions: [7, 42] })], stored)++ expect(harness.fetch.mock.calls[0][1]).toMatchObject({+ maxBytes: OPTIONS.maxBytes,+ maxRedirects: OPTIONS.maxRedirects,+ sourcePartitions: [7, 42],+ cache,+ tdmrepReservation: true,+ })+ })++ it.each([+ ['another URL', bind('https://other.example.net/a.png')],+ ['no URL', undefined],+ ])('sends no validator that %s issued, and binds the response to the URL that sent it', async (_name, binding) => {+ const harness = build({ cache: { requestTimeMs: NOW_MS - 10, responseTimeMs: NOW_MS, etag: '"image-v2"' } })+ const stored = new Map<string, CrawlHistoryItem>([+ [+ candidate().originalRef,+ {+ kind: 'url',+ key: candidate().originalRef,+ nextFetchAtMs: NOW_MS,+ storageExpiresAtMs: NOW_MS,+ outcome: 'ok',+ cache: {+ requestTimeMs: NOW_MS - 10,+ responseTimeMs: NOW_MS,+ responseUrlHmac: binding,+ etag: '"v1"',+ },+ },+ ],+ ])++ const [attempt] = await harness.runner.run([candidate()], stored)++ expect(harness.fetch.mock.calls[0][1].cache).toBeUndefined()+ expect(attempt.history?.cache).toMatchObject({+ responseUrlHmac: bind(candidate().currentUrl),+ etag: '"image-v2"',+ })+ })++ it.each([+ ['a fetch pass', OPTIONS],+ ['the pod candidate pool', POOL_OPTIONS],+ ])('publishes an accepted image and records a terminal URL result through %s', async (_mode, options) => {+ const bytes = Buffer.from('image')+ const harness = build({ bytes, contentType: 'image/png' }, {}, 'queued', options)++ const [attempt] = await harness.runner.run([candidate()], new Map())++ expect(harness.publishImage).toHaveBeenCalledWith(+ expect.objectContaining({ originalRef: candidate().originalRef }),+ expect.objectContaining({ bytes })+ )+ expect(attempt).toMatchObject({+ outcome: 'ok',+ finished: true,+ lost: false,+ history: { kind: 'url', key: candidate().originalRef, nextFetchAtMs: NOW_MS + 30 * 24 * 60 * 60 * 1000 },+ })+ })++ it('uses one worker limit across sibling origins', async () => {+ const harness = build({}, {}, 'queued', { ...OPTIONS, maxConcurrentPerRegistrableDomain: 1 })+ let releaseFirst: (() => void) | undefined+ harness.fetch.mockImplementationOnce(+ (url: string) =>+ new Promise<ImageFetchResult>((resolve) => {+ releaseFirst = () => resolve({ outcome: 'ok', redirects: 0, currentUrl: url })+ })+ )+ const first = candidate({ sourcePartitions: [110] })+ const sibling = candidate({+ originalRef: `imageurl:${'b'.repeat(22)}`,+ currentUrl: 'https://images.example.com/b.png',+ host: 'images.example.com',+ origin: 'https://images.example.com',+ sourcePartitions: [110],+ })++ const run = harness.runner.run([first, sibling], new Map())+ await jest.advanceTimersByTimeAsync(0)++ expect(harness.fetch).toHaveBeenCalledTimes(1)+ expect(releaseFirst).toBeDefined()+ releaseFirst?.()+ await run+ expect(harness.fetch).toHaveBeenCalledTimes(2)+ expect(harness.topHogRecords.get('ai_research_v4_image_fetch_block_events_by_registrable_domain')).toEqual([+ {+ key: { registrable_domain: 'example.com', reason: 'domain_concurrency', partition: '110' },+ value: 1,+ },+ ])+ })++ it('starts the next request for a registrable domain while an earlier image publish waits', async () => {+ const harness = build({ bytes: Buffer.from('image'), contentType: 'image/png' }, {}, 'queued', {+ ...OPTIONS,+ maxConcurrentPerRegistrableDomain: 1,+ })+ let releasePublish: () => void = () => undefined+ harness.publishImage.mockImplementationOnce(+ () =>+ new Promise<void>((resolve) => {+ releasePublish = resolve+ })+ )+ const second = candidate({+ originalRef: `imageurl:${'b'.repeat(22)}`,+ currentUrl: 'https://cdn.example.com/b.png',+ })++ let passFinished = false+ const run = harness.runner.run([candidate(), second], new Map()).finally(() => {+ passFinished = true+ })+ await jest.advanceTimersByTimeAsync(0)++ expect(harness.fetch.mock.calls.map(([url]) => url)).toEqual([candidate().currentUrl, second.currentUrl])+ expect(passFinished).toBe(false)+ releasePublish()+ const attempts = await run+ expect(attempts.map((attempt) => [attempt.candidate.originalRef, attempt.outcome, attempt.lost])).toEqual(+ expect.arrayContaining([+ [candidate().originalRef, 'ok', false],+ [second.originalRef, 'ok', false],+ ])+ )+ expect(attempts).toHaveLength(2)+ })++ it('allocates sibling-origin workers by queue share', async () => {+ const harness = build({}, {}, 'queued', { ...OPTIONS, maxConcurrentPerRegistrableDomain: 2 })+ let releaseFirst: (() => void) | undefined+ let releaseSecond: (() => void) | undefined+ harness.fetch+ .mockImplementationOnce(+ (url: string) =>+ new Promise<ImageFetchResult>((resolve) => {+ releaseFirst = () => resolve({ outcome: 'ok', redirects: 0, currentUrl: url })+ })+ )+ .mockImplementationOnce(+ (url: string) =>+ new Promise<ImageFetchResult>((resolve) => {+ releaseSecond = () => resolve({ outcome: 'ok', redirects: 0, currentUrl: url })+ })+ )+ const sameOrigin = candidate({ originalRef: `imageurl:${'b'.repeat(22)}` })+ const siblingOrigin = candidate({+ originalRef: `imageurl:${'c'.repeat(22)}`,+ currentUrl: 'https://images.example.com/c.png',+ host: 'images.example.com',+ origin: 'https://images.example.com',+ })++ const run = harness.runner.run([candidate(), sameOrigin, siblingOrigin], new Map())+ await jest.advanceTimersByTimeAsync(0)++ expect(harness.fetch.mock.calls.map(([url]) => url)).toEqual([candidate().currentUrl, siblingOrigin.currentUrl])+ releaseFirst?.()+ releaseSecond?.()+ await run+ expect(harness.fetch).toHaveBeenCalledTimes(3)+ })++ it('records initial capacity after live pod and registrable-domain limits', async () => {+ const observeCapacity = jest+ .spyOn(ImageFetchRequestMetrics, 'observeBatchSchedulableCapacity')+ .mockImplementation()+ const harness = build({}, {}, 'queued', {+ ...OPTIONS,+ maxConcurrentPerRegistrableDomain: 2,+ maxInFlightRequests: 3,+ })+ const candidates = [+ candidate({ sourcePartitions: [110] }),+ ...Array.from({ length: 2 }, (_, index) =>+ candidate({+ originalRef: `imageurl:${String(index + 1).repeat(22)}`,+ currentUrl: `https://cdn.example.com/image-${index}.png`,+ sourcePartitions: [110],+ })+ ),+ ...Array.from({ length: 2 }, (_, index) =>+ candidate({+ originalRef: `imageurl:${String(index + 3).repeat(22)}`,+ currentUrl: `https://origin-${index}.other.net/image.png`,+ host: `origin-${index}.other.net`,+ origin: `https://origin-${index}.other.net`,+ registrableDomain: 'other.net',+ sourcePartitions: [42],+ })+ ),+ ]+ const grant = harness.budget.take('example.com', 'https://cdn.example.com', NOW_MS, NOW_MS + 10_000, true)+ expect(grant).toMatchObject({ granted: true, waitMs: 0 })+ expect(harness.budget.acquireConnection('example.com', 'https://cdn.example.com')).toBe(true)+ harness.budget.markRequestStarted(+ 'example.com',+ 'https://cdn.example.com',+ NOW_MS,+ grant.granted ? grant.reservedStartAtMs : null,+ 'configuration'+ )++ await harness.runner.run(candidates, new Map())++ expect(observeCapacity).toHaveBeenCalledWith(3, 3)+ expect(harness.topHogRecords.get('ai_research_v4_image_fetch_block_events_by_registrable_domain')).toEqual([+ {+ key: { registrable_domain: 'example.com', reason: 'domain_concurrency', partition: '110' },+ value: 2,+ },+ ])+ harness.budget.releaseConnection('example.com', 'https://cdn.example.com')+ })++ it('keeps one request slot on the largest origin queue', async () => {+ const harness = build({}, {}, 'queued', {+ ...OPTIONS,+ maxConcurrentPerRegistrableDomain: 1,+ maxInFlightRequests: 1,+ })+ const small = candidate({+ currentUrl: 'https://small.example.com/a.png',+ host: 'small.example.com',+ origin: 'https://small.example.com',+ })+ const largeFirst = candidate({+ originalRef: `imageurl:${'b'.repeat(22)}`,+ currentUrl: 'https://large.example.com/a.png',+ host: 'large.example.com',+ origin: 'https://large.example.com',+ })+ const largeSecond = candidate({+ originalRef: `imageurl:${'c'.repeat(22)}`,+ currentUrl: 'https://large.example.com/b.png',+ host: 'large.example.com',+ origin: 'https://large.example.com',+ })++ await harness.runner.run([small, largeFirst, largeSecond], new Map())++ expect(harness.fetch.mock.calls.map(([url]) => url)).toEqual([+ largeFirst.currentUrl,+ largeSecond.currentUrl,+ small.currentUrl,+ ])+ })++ it('processes a low-capacity tail instead of republishing it for more diversity', async () => {+ const harness = build({}, {}, 'queued', {+ ...OPTIONS,+ maxInFlightRequests: 1,+ })+ const candidates = Array.from({ length: 4 }, (_, index) => {+ const origin = `https://cdn-${index}.example.com`+ return candidate({+ originalRef: `imageurl:${index.toString().padStart(22, '0')}`,+ currentUrl: `${origin}/${index}.png`,+ host: `cdn-${index}.example.com`,+ origin,+ })+ })++ const attempts = await harness.runner.run(candidates, new Map())++ expect(harness.fetch).toHaveBeenCalledTimes(4)+ expect(harness.republish).not.toHaveBeenCalled()+ expect(attempts).toHaveLength(4)+ })++ it('keeps the pod request limit across overlapping passes', async () => {+ const observeCapacity = jest+ .spyOn(ImageFetchRequestMetrics, 'observeBatchSchedulableCapacity')+ .mockImplementation()+ const harness = build({}, {}, 'queued', { ...OPTIONS, maxInFlightRequests: 1 })+ const releases: Array<() => void> = []+ let signalSecondFetchStarted: () => void = () => undefined+ const secondFetchStarted = new Promise<void>((resolve) => {+ signalSecondFetchStarted = resolve+ })+ harness.fetch.mockImplementation(+ (url: string) =>+ new Promise<ImageFetchResult>((resolve) => {+ releases.push(() => resolve({ outcome: 'ok', redirects: 0, currentUrl: url }))+ if (releases.length === 2) {+ signalSecondFetchStarted()+ }+ })+ )+ const otherDomain = candidate({+ originalRef: `imageurl:${'b'.repeat(22)}`,+ currentUrl: 'https://cdn.other.net/b.png',+ host: 'cdn.other.net',+ origin: 'https://cdn.other.net',+ registrableDomain: 'other.net',+ })++ const firstRun = harness.runner.run([candidate()], new Map())+ const secondRun = harness.runner.run([otherDomain], new Map())+ await jest.advanceTimersByTimeAsync(0)++ expect(harness.fetch).toHaveBeenCalledTimes(1)+ expect(observeCapacity.mock.calls).toEqual([+ [1, 1],+ [0, 1],+ ])+ releases[0]()+ await secondFetchStarted+ expect(harness.fetch).toHaveBeenCalledTimes(2)+ releases[1]()+ await Promise.all([firstRun, secondRun])+ })++ it('finishes a pooled batch with queued candidates before close stops the pool', async () => {+ const harness = build({}, {}, 'queued', { ...POOL_OPTIONS, maxInFlightRequests: 1 })+ let releaseFirst: () => void = () => undefined+ harness.fetch.mockImplementationOnce(+ (url: string) =>+ new Promise<ImageFetchResult>((resolve) => {+ releaseFirst = () => resolve({ outcome: 'ok', redirects: 0, currentUrl: url })+ })+ )+ const queued = candidate({+ originalRef: `imageurl:${'b'.repeat(22)}`,+ currentUrl: 'https://cdn.other.net/b.png',+ host: 'cdn.other.net',+ origin: 'https://cdn.other.net',+ registrableDomain: 'other.net',+ })++ const run = harness.runner.run([candidate(), queued], new Map())+ await jest.advanceTimersByTimeAsync(0)+ let closed = false+ const close = harness.runner.close().then(() => {+ closed = true+ })+ await jest.advanceTimersByTimeAsync(0)+ expect(closed).toBe(false)+ releaseFirst()+ await jest.advanceTimersByTimeAsync(0)++ expect((await run).map((attempt) => attempt.candidate.currentUrl)).toEqual([+ candidate().currentUrl,+ queued.currentUrl,+ ])+ await close+ })++ it('shares one registrable domain limit between batches in the pod candidate pool', async () => {+ const harness = build({}, {}, 'queued', POOL_OPTIONS)+ const releases: Array<() => void> = []+ harness.fetch.mockImplementation(+ (url: string) =>+ new Promise<ImageFetchResult>((resolve) => {+ releases.push(() => resolve({ outcome: 'ok', redirects: 0, currentUrl: url }))+ })+ )+ const onOrigin = (label: string): FetchCandidate =>+ candidate({+ originalRef: `imageurl:${label.repeat(22)}`,+ currentUrl: `https://${label}.example.com/a.png`,+ host: `${label}.example.com`,+ origin: `https://${label}.example.com`,+ })++ const firstBatch = harness.runner.run([onOrigin('a'), onOrigin('b')], new Map())+ const secondBatch = harness.runner.run([onOrigin('c'), onOrigin('d')], new Map())+ await jest.advanceTimersByTimeAsync(0)++ expect(harness.fetch).toHaveBeenCalledTimes(2)+ releases.splice(0).forEach((release) => release())+ await jest.advanceTimersByTimeAsync(0)+ expect(harness.fetch).toHaveBeenCalledTimes(4)+ releases.splice(0).forEach((release) => release())+ await Promise.all([firstBatch, secondBatch])+ })++ it('does not start a queued request after another worker aborts its pass', async () => {+ const harness = build({}, {}, 'queued', { ...OPTIONS, maxInFlightRequests: 2 })+ let releaseBlocker: () => void = () => undefined+ let signalBlockerStarted: () => void = () => undefined+ const blockerStarted = new Promise<void>((resolve) => {+ signalBlockerStarted = resolve+ })+ harness.fetch.mockImplementation((url: string) => {+ if (url.includes('blocker')) {+ signalBlockerStarted()+ return new Promise<ImageFetchResult>((resolve) => {+ releaseBlocker = () => resolve({ outcome: 'ok', redirects: 0, currentUrl: url })+ })+ }+ if (url.includes('failure')) {+ return Promise.reject(new Error('dependency unavailable'))+ }+ return Promise.resolve({ outcome: 'ok', redirects: 0, currentUrl: url })+ })+ const blocker = candidate({ currentUrl: 'https://blocker.example.com/a.png' })+ const failure = candidate({+ originalRef: `imageurl:${'b'.repeat(22)}`,+ currentUrl: 'https://failure.example.com/b.png',+ host: 'failure.example.com',+ origin: 'https://failure.example.com',+ })+ const mustNotStart = candidate({+ originalRef: `imageurl:${'c'.repeat(22)}`,+ currentUrl: 'https://queued.example.com/c.png',+ host: 'queued.example.com',+ origin: 'https://queued.example.com',+ })++ const blockingRun = harness.runner.run([blocker], new Map())+ await blockerStarted+ const failingRun = harness.runner.run([failure, mustNotStart], new Map())++ await expect(failingRun).rejects.toThrow('dependency unavailable')+ expect(harness.fetch.mock.calls.map(([url]) => url)).toEqual([blocker.currentUrl, failure.currentUrl])+ releaseBlocker()+ await blockingRun+ })++ it.each([+ ['a fetch pass', OPTIONS],+ ['the pod candidate pool', POOL_OPTIONS],+ ])('stops taking queue work after a fatal candidate error in %s', async (_mode, options) => {+ const harness = build({}, {}, 'queued', { ...options, maxInFlightRequests: 1 })+ harness.fetch.mockRejectedValueOnce(new Error('dependency unavailable'))+ const second = candidate({ originalRef: `imageurl:${'b'.repeat(22)}` })++ await expect(harness.runner.run([candidate(), second], new Map())).rejects.toThrow('dependency unavailable')++ expect(harness.fetch).toHaveBeenCalledTimes(1)+ })++ it('deduplicates canonical refs before queue scheduling', async () => {+ const harness = build()+ const duplicate = candidate({ republishCount: 1, lastRepublishReason: 'retry' })++ const attempts = await harness.runner.run([candidate(), duplicate], new Map())++ expect(harness.fetch).toHaveBeenCalledTimes(1)+ expect(attempts).toHaveLength(1)+ expect(attempts[0].candidate).toMatchObject({ republishCount: 1, lastRepublishReason: 'retry' })+ })++ it.each([+ ['a fetch pass', OPTIONS],+ ['the pod candidate pool', POOL_OPTIONS],+ ])('marks an image publish failure as lost after all candidate work settles in %s', async (_mode, options) => {+ const harness = build({ bytes: Buffer.from('image'), contentType: 'image/png' }, {}, 'queued', options)+ harness.publishImage.mockRejectedValue(+ new S3ServiceException({+ name: 'SlowDown',+ $fault: 'server',+ $metadata: { httpStatusCode: 503 },+ message: 'images/url/month=2026-10/7/secret-key',+ })+ )+ const warn = jest.spyOn(logger, 'warn')++ const attempts = await harness.runner.run(+ [candidate(), candidate({ originalRef: `imageurl:${'b'.repeat(22)}` })],+ new Map()+ )++ expect(attempts).toEqual([+ expect.objectContaining({ outcome: 'publish_failed', finished: false, lost: true }),+ expect.objectContaining({ outcome: 'publish_failed', finished: false, lost: true }),+ ])+ // One log for the burst, with the error name and status and nothing that names the image.+ const logged = warn.mock.calls.filter(([, event]) => event === 'ai_research_v4_image_fetch_storage_failed')+ expect(logged).toEqual([+ [+ '🌐',+ 'ai_research_v4_image_fetch_storage_failed',+ { operation: 'publish', error: 'SlowDown', code: undefined, httpStatusCode: 503, failures: 1 },+ ],+ ])+ warn.mockRestore()+ })++ it('writes a terminal policy refusal without opening an image socket', async () => {+ const harness = build({}, { allowed: false, transient: false, reason: 'robots_disallow' })++ const [attempt] = await harness.runner.run([candidate()], new Map())++ expect(harness.fetch).not.toHaveBeenCalled()+ expect(attempt).toMatchObject({ outcome: 'robots_disallow', finished: true })+ })++ it('delays a transient configuration failure without writing URL history', async () => {+ const harness = build({}, { allowed: false, transient: true, reason: 'configuration_unreachable' })++ const [attempt] = await harness.runner.run([candidate()], new Map())++ expect(harness.republish).toHaveBeenCalledWith(+ expect.objectContaining({+ originalRef: candidate().originalRef,+ lastBlockReason: 'configuration_unreachable',+ }),+ {+ currentUrl: candidate().currentUrl,+ host: candidate().host,+ origin: candidate().origin,+ registrableDomain: candidate().registrableDomain,+ },+ 'not_ready',+ 3_600_000+ )+ expect(attempt).toMatchObject({+ outcome: 'backoff',+ finished: false,+ lost: false,+ block: { reason: 'configuration_unreachable', waitMs: 3_600_000 },+ })+ expect(attempt.history).toBeUndefined()+ })++ it('keeps the effective URL and charges every request before retrying', async () => {+ const harness = build({+ outcome: 'server_error',+ status: 503,+ redirects: 2,+ currentUrl: 'https://cdn.example.com/final.png',+ retryAfterMs: 120_000,+ })+ const [attempt] = await harness.runner.run([candidate()], new Map())++ expect(harness.republish).toHaveBeenCalledWith(+ expect.objectContaining({+ currentUrl: 'https://cdn.example.com/final.png',+ remainingHops: MAX_HOPS - 2,+ fetchCount: 3,+ }),+ expect.objectContaining({ currentUrl: 'https://cdn.example.com/final.png' }),+ 'retry',+ 120_000+ )+ expect(attempt).toMatchObject({+ outcome: 'server_error',+ finished: false,+ block: { reason: 'retry_after', waitMs: 120_000 },+ })+ })++ it('republishes an unfollowed redirect target with the original ref', async () => {+ const harness = build({+ outcome: 'redirect_offsite',+ status: 302,+ currentUrl: candidate().currentUrl,+ redirectTarget: { url: 'https://img.other.net/a.png', host: 'img.other.net' },+ })++ const [attempt] = await harness.runner.run([candidate()], new Map())++ expect(harness.republish).toHaveBeenCalledWith(+ expect.objectContaining({ originalRef: candidate().originalRef, fetchCount: 1 }),+ {+ currentUrl: 'https://img.other.net/a.png',+ host: 'img.other.net',+ origin: 'https://img.other.net',+ registrableDomain: 'other.net',+ },+ 'redirect',+ 0+ )+ expect(attempt).toMatchObject({ outcome: 'redirect_offsite', finished: false })+ })++ it('records a terminal refusal from a same-origin redirect policy check', async () => {+ const harness = build({+ outcome: 'redirect_policy_refused',+ redirects: 1,+ currentUrl: 'https://cdn.example.com/private/a.png',+ refusalReason: 'robots_disallow',+ policyTransient: false,+ })++ const [attempt] = await harness.runner.run([candidate()], new Map())++ expect(harness.republish).not.toHaveBeenCalled()+ expect(attempt).toMatchObject({+ outcome: 'robots_disallow',+ finished: true,+ candidate: { remainingHops: MAX_HOPS - 1, fetchCount: 1 },+ })+ })++ it('delays a transient same-origin redirect policy failure without spending another hop', async () => {+ const harness = build({+ outcome: 'redirect_policy_refused',+ redirects: 1,+ currentUrl: 'https://cdn.example.com/next/a.png',+ refusalReason: 'configuration_unreachable',+ policyTransient: true,+ })++ await harness.runner.run([candidate()], new Map())++ expect(harness.republish).toHaveBeenCalledWith(+ expect.objectContaining({+ currentUrl: 'https://cdn.example.com/next/a.png',+ remainingHops: MAX_HOPS - 1,+ fetchCount: 1,+ }),+ expect.any(Object),+ 'not_ready',+ 3_600_000+ )+ })++ it('persists a crawl wait that extends beyond the current pass', async () => {+ const harness = build({+ outcome: 'request_deferred',+ schedulingReason: 'deadline',+ schedulingWaitMs: 600_000,+ })++ await harness.runner.run([candidate()], new Map())++ expect(harness.republish).toHaveBeenCalledWith(expect.any(Object), expect.any(Object), 'not_ready', 600_000)+ })++ it.each([+ ['a fetch pass', OPTIONS],+ ['the pod candidate pool', POOL_OPTIONS],+ ])('returns pass-deadline work directly to the frontier from %s', async (_mode, options) => {+ const harness = build({}, {}, 'queued', { ...options, batchBudgetMs: -1 })++ await harness.runner.run([candidate()], new Map())++ expect(harness.republish).toHaveBeenCalledWith(expect.any(Object), expect.any(Object), 'pass_deadline', 0)+ })++ it('finishes a retry when its last hop was the failed request', async () => {+ const harness = build({ outcome: 'timeout' })++ const [attempt] = await harness.runner.run([candidate({ remainingHops: 1 })], new Map())++ expect(harness.republish).not.toHaveBeenCalled()+ expect(attempt).toMatchObject({ outcome: HOPS_EXHAUSTED, finished: true })+ })++ it('does not start policy or image requests when the hop budget is empty', async () => {+ const harness = build()++ const [attempt] = await harness.runner.run([candidate({ remainingHops: 0 })], new Map())++ expect(harness.check).not.toHaveBeenCalled()+ expect(harness.fetch).not.toHaveBeenCalled()+ expect(attempt).toMatchObject({ outcome: HOPS_EXHAUSTED, finished: true })+ })++ it('records a terminal refusal when no delay topic can hold the wait', async () => {+ const harness = build({ outcome: 'timeout' }, {}, 'refused_delay')+ const [attempt] = await harness.runner.run([candidate()], new Map())++ expect(attempt).toMatchObject({ outcome: DELAY_TOO_LONG, finished: true })+ })++ it('merges a 304 response into the previous cache metadata', async () => {+ const previousCache: HttpCacheMetadata = {+ requestTimeMs: NOW_MS - 2_000,+ responseTimeMs: NOW_MS - 1_000,+ etag: '"v1"',+ lastModified: 'yesterday',+ }+ const harness = build({+ outcome: 'not_modified',+ status: 304,+ cache: { requestTimeMs: NOW_MS - 10, responseTimeMs: NOW_MS, etag: '"v2"' },+ })+ const stored = new Map<string, CrawlHistoryItem>([+ [+ candidate().originalRef,+ {+ kind: 'url',+ key: candidate().originalRef,+ nextFetchAtMs: NOW_MS,+ storageExpiresAtMs: NOW_MS,+ outcome: 'ok',+ cache: previousCache,+ },+ ],+ ])++ const [attempt] = await harness.runner.run([candidate()], stored)++ expect(attempt.history?.cache).toMatchObject({ etag: '"v2"', lastModified: 'yesterday' })+ })++ it('retains validators that a 304 response omits', async () => {+ const previousCache: HttpCacheMetadata = {+ requestTimeMs: NOW_MS - 2_000,+ responseTimeMs: NOW_MS - 1_000,+ etag: '"v1"',+ lastModified: 'yesterday',+ }+ const harness = build({+ outcome: 'not_modified',+ status: 304,+ cache: { requestTimeMs: NOW_MS - 10, responseTimeMs: NOW_MS },+ })+ const stored = new Map<string, CrawlHistoryItem>([+ [+ candidate().originalRef,+ {+ kind: 'url',+ key: candidate().originalRef,+ nextFetchAtMs: NOW_MS,+ storageExpiresAtMs: NOW_MS,+ outcome: 'ok',+ cache: previousCache,+ },+ ],+ ])++ const [attempt] = await harness.runner.run([candidate()], stored)++ expect(attempt.history?.cache).toMatchObject({+ requestTimeMs: NOW_MS - 10,+ responseTimeMs: NOW_MS,+ etag: '"v1"',+ lastModified: 'yesterday',+ })+ })++ it('does not store a redirect target validator under the original URL ref', async () => {+ const harness = build({+ redirects: 1,+ currentUrl: 'https://cdn.example.com/moved.png',+ cache: {+ requestTimeMs: NOW_MS - 10,+ responseTimeMs: NOW_MS,+ etag: '"redirect-target"',+ lastModified: 'today',+ cacheControl: 'max-age=60',+ },+ })++ const [attempt] = await harness.runner.run([candidate()], new Map())++ expect(attempt.history?.cache).toMatchObject({ cacheControl: 'max-age=60' })+ expect(attempt.history?.cache?.etag).toBeUndefined()+ expect(attempt.history?.cache?.lastModified).toBeUndefined()+ })++ it('extends URL history to the end of explicit freshness', async () => {+ const fortyDaysMs = 40 * 24 * 60 * 60 * 1000+ const harness = build({+ cache: {+ requestTimeMs: NOW_MS,+ responseTimeMs: NOW_MS,+ cacheControl: `s-maxage=${fortyDaysMs / 1000}`,+ },+ })++ const [attempt] = await harness.runner.run([candidate()], new Map())++ expect(attempt.history?.nextFetchAtMs).toBe(NOW_MS + fortyDaysMs)+ expect(attempt.history?.storageExpiresAtMs).toBe(NOW_MS + fortyDaysMs)+ })+})diff --git a/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/fetch-runner.ts b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/fetch-runner.tsnew file mode 100644index 000000000000..90d631cfa175--- /dev/null+++ b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/fetch-runner.ts@@ -0,0 +1,971 @@+import { ConcurrencyController } from '~/common/utils/concurrencyController'+import { logger } from '~/common/utils/logger'++import type { ImageFetchBlockReason } from './block-reason'+import { FetchCandidate, MAX_HOPS, RepublishReason } from './collected-urls-record'+import { ConfigurationPolicyPass, ConfigurationPolicyService } from './configuration-policy'+import { ConfigurationCacheItem, CrawlHistoryItem, HttpCacheMetadata, UrlCrawlHistoryItem } from './crawl-history'+import {+ FetchCandidatePool,+ FetchCandidatePoolAdmission,+ FetchCandidatePoolLimits,+ PoolBatch,+} from './fetch-candidate-pool'+import { FetchCandidateLease, FetchCandidateQueue, deduplicateFetchCandidates } from './fetch-candidate-queue'+import { FrontierPublisher, RepublishBatch, RepublishResult } from './frontier-publisher'+import { HostBudget } from './host-budget'+import {+ FetchOutcome,+ FetchRefusalReason,+ ImageFetchResult,+ ImageFetcher,+ RequestScheduleBlockReason,+ TransientFetchOutcome,+} from './image-fetcher'+import type { RefreshingImageSink } from './image-sink'+import { ImageFetchPoolMetrics, ImageFetchRequestMetrics } from './metrics'+import { OriginRequestScheduler } from './origin-request-scheduler'+import { canonicalizeUrl } from './politeness-key'+import { ImageFetchProcessingMetrics } from './processing-metrics'+import type { ResponseUrlBinder } from './response-url-binding'+import { ImageFetchTopHogMetrics } from './tophog-metrics'+import { urlHistoryExpiresAtMs, urlHistoryStorageExpiresAtMs } from './url-history-expiry'++export type ShedReason =+ | 'breaker_open'+ | 'backoff'+ | 'deadline'+ | 'connection_limit'+ | 'origin_map_full'+ | 'registrable_domain_map_full'+export const HOPS_EXHAUSTED = 'hops_exhausted'+export const DELAY_TOO_LONG = 'delay_too_long'+export type AttemptOutcome =+ | FetchOutcome+ | FetchRefusalReason+ | ShedReason+ | 'publish_failed'+ | typeof HOPS_EXHAUSTED+ | typeof DELAY_TOO_LONG++export interface FetchAttempt {+ candidate: FetchCandidate+ outcome: AttemptOutcome+ finished: boolean+ lost: boolean+ block?: { reason: ImageFetchBlockReason; waitMs: number }+ history?: UrlCrawlHistoryItem+ configurationUpdates: ConfigurationCacheItem[]+}++export interface FetchRunnerOptions {+ maxConcurrentPerRegistrableDomain: number+ maxInFlightRequests: number+ batchBudgetMs: number+ maxBytes: number+ requestTimeoutMs: number+ maxRedirects: number+ seenTtlSeconds: number+ /** Set to run every batch through one pod-wide candidate pool instead of a fetch pass of its own. */+ continuousPool?: FetchCandidatePoolLimits+}++const TRANSIENT_OUTCOMES = new Set<TransientFetchOutcome>(['timeout', 'error', 'rate_limited', 'server_error'])+const ONE_MINUTE_MS = 60_000+const CONFIGURATION_RETRY_MS = 60 * 60 * 1000+const STORAGE_FAILURE_LOG_INTERVAL_MS = 10_000++function requirePositive(name: string, value: number): void {+ if (!Number.isFinite(value) || value <= 0) {+ throw new Error(`${name} must be a positive number, got ${value}`)+ }+}++function requirePositiveSafeInteger(name: string, value: number): void {+ if (!Number.isSafeInteger(value) || value <= 0) {+ throw new Error(`${name} must be a positive safe integer, got ${value}`)+ }+}++export function isTransientOutcome(outcome: AttemptOutcome): outcome is TransientFetchOutcome {+ return TRANSIENT_OUTCOMES.has(outcome as TransientFetchOutcome)+}++function isRequestStateFull(+ reason: string | undefined+): reason is Extract<RequestScheduleBlockReason, 'origin_map_full' | 'registrable_domain_map_full'> {+ return reason === 'origin_map_full' || reason === 'registrable_domain_map_full'+}++export interface FetchPass {+ run(+ candidates: FetchCandidate[],+ stored: Map<string, CrawlHistoryItem>,+ republishBatch?: RepublishBatch,+ admission?: FetchCandidatePoolAdmission+ ): Promise<FetchAttempt[]>+}++interface FetchPassState {+ failure?: { error: unknown }+}++interface PooledBatch {+ stored: Map<string, CrawlHistoryItem>+ configurationItems: Map<string, ConfigurationCacheItem>+ configurationPolicy: ConfigurationPolicyPass+ deadlineMs: number+ republishBatch: RepublishBatch+ state: FetchPassState+ attempts: FetchAttempt[]+ unsettled: number+ poolBatch?: PoolBatch<PooledBatch>+ settle: () => void+}++export class FetchRunner implements FetchPass {+ private readonly candidateWork: ConcurrencyController+ private readonly pool?: FetchCandidatePool<PooledBatch>+ private poolWorkers?: Promise<void>[]+ private readonly activePooledBatches = new Set<Promise<FetchAttempt[]>>()+ private storageFailureLoggedAtMs = Number.NEGATIVE_INFINITY+ private storageFailuresSinceLog = 0++ constructor(+ private readonly fetcher: ImageFetcher,+ private readonly budget: HostBudget,+ private readonly scheduler: OriginRequestScheduler,+ private readonly configurationPolicy: ConfigurationPolicyService,+ private readonly options: FetchRunnerOptions,+ private readonly publisher: FrontierPublisher,+ private readonly imageSink: RefreshingImageSink,+ private readonly responseUrlBinder: ResponseUrlBinder,+ private readonly topHogMetrics?: ImageFetchTopHogMetrics+ ) {+ requirePositiveSafeInteger(+ 'AI_RESEARCH_V4_IMAGE_FETCH_MAX_CONCURRENT_PER_REGISTRABLE_DOMAIN',+ options.maxConcurrentPerRegistrableDomain+ )+ requirePositiveSafeInteger('AI_RESEARCH_V4_IMAGE_FETCH_MAX_IN_FLIGHT_REQUESTS', options.maxInFlightRequests)+ requirePositive('AI_RESEARCH_V4_IMAGE_FETCH_MAX_IMAGE_BYTES', options.maxBytes)+ requirePositive('AI_RESEARCH_V4_IMAGE_FETCH_REQUEST_TIMEOUT_MS', options.requestTimeoutMs)+ requirePositive('AI_RESEARCH_V4_IMAGE_FETCH_CRAWL_HISTORY_TTL_SECONDS', options.seenTtlSeconds)+ if (!Number.isFinite(options.batchBudgetMs) || !Number.isFinite(options.maxRedirects)) {+ throw new Error('image fetch pass limits must be finite numbers')+ }+ this.candidateWork = new ConcurrencyController(options.maxInFlightRequests)+ ImageFetchRequestMetrics.trackBudget(budget, scheduler)+ if (options.continuousPool) {+ this.pool = new FetchCandidatePool<PooledBatch>(+ {+ ...options.continuousPool,+ maxConcurrentPerRegistrableDomain: options.maxConcurrentPerRegistrableDomain,+ },+ budget+ )+ ImageFetchProcessingMetrics.trackQueue(this.pool)+ }+ }++ public get candidatePool(): FetchCandidatePool<PooledBatch> | undefined {+ return this.pool+ }++ public async run(+ candidates: FetchCandidate[],+ stored: Map<string, CrawlHistoryItem>,+ republishBatch?: RepublishBatch,+ admission?: FetchCandidatePoolAdmission+ ): Promise<FetchAttempt[]> {+ if (this.pool) {+ const pooledBatch = this.runPooled(this.pool, candidates, stored, republishBatch, admission)+ this.activePooledBatches.add(pooledBatch)+ try {+ return await pooledBatch+ } finally {+ this.activePooledBatches.delete(pooledBatch)+ }+ }+ admission?.admitted()+ return await this.runPass(candidates, stored, republishBatch)+ }++ /**+ * Waits for every pooled batch before it stops the workers. The Kafka consumer drain on shutdown+ * has a timeout, and a batch that outlives it still needs the workers to take its queued candidates.+ */+ public async close(): Promise<void> {+ await Promise.allSettled(this.activePooledBatches)+ this.pool?.close()+ await Promise.all(this.poolWorkers ?? [])+ }++ private async runPass(+ candidates: FetchCandidate[],+ stored: Map<string, CrawlHistoryItem>,+ republishBatch?: RepublishBatch+ ): Promise<FetchAttempt[]> {+ const activeRepublishBatch = republishBatch ?? this.publisher.createRepublishBatch()+ const deadlineMs = Date.now() + this.options.batchBudgetMs+ const configurationPolicy = this.configurationPolicy.createPass()+ const passState: FetchPassState = {}+ const configurationItems = configurationItemsFrom(stored)+ const queue = new FetchCandidateQueue(candidates, this.options)+ this.topHogMetrics?.recordConcurrencyLimitedUrls(candidates, (registrableDomain) =>+ this.budget.availableConnections(registrableDomain, this.options.maxConcurrentPerRegistrableDomain)+ )+ const podRequestSlots = Math.max(0, this.options.maxInFlightRequests - this.candidateWork.running)+ ImageFetchRequestMetrics.observeBatchSchedulableCapacity(+ Math.min(+ podRequestSlots,+ queue.availableRequestSlotsAtStart((registrableDomain) =>+ this.budget.availableConnections(registrableDomain, this.options.maxConcurrentPerRegistrableDomain)+ )+ ),+ this.options.maxInFlightRequests+ )+ const attempts: FetchAttempt[] = []+ const stopTrackingQueue = ImageFetchProcessingMetrics.trackQueue(queue)+ const workers = Array.from(+ { length: Math.min(this.options.maxInFlightRequests, queue.schedulableSlotsAtStart) },+ () =>+ this.runQueueWorker(+ queue,+ stored,+ configurationItems,+ configurationPolicy,+ deadlineMs,+ attempts,+ activeRepublishBatch,+ passState+ )+ )+ const settledWorkers = await Promise.allSettled(workers).finally(stopTrackingQueue)+ if (passState.failure) {+ throw passState.failure.error+ }+ const failedWorker = settledWorkers.find(+ (settled): settled is PromiseRejectedResult => settled.status === 'rejected'+ )+ if (failedWorker) {+ throw failedWorker.reason+ }+ return await this.finishPass(attempts, activeRepublishBatch, !republishBatch)+ }++ private async runPooled(+ pool: FetchCandidatePool<PooledBatch>,+ candidates: FetchCandidate[],+ stored: Map<string, CrawlHistoryItem>,+ republishBatch: RepublishBatch | undefined,+ admission: FetchCandidatePoolAdmission | undefined+ ): Promise<FetchAttempt[]> {+ const activeRepublishBatch = republishBatch ?? this.publisher.createRepublishBatch()+ const { candidates: unique } = deduplicateFetchCandidates(candidates)+ this.topHogMetrics?.recordConcurrencyLimitedUrls(unique, (registrableDomain) =>+ this.budget.availableConnections(registrableDomain, this.options.maxConcurrentPerRegistrableDomain)+ )+ let settle: () => void = () => undefined+ const settled = new Promise<void>((resolve) => {+ settle = resolve+ })+ const batch: PooledBatch = {+ stored,+ configurationItems: configurationItemsFrom(stored),+ configurationPolicy: this.configurationPolicy.createPass(),+ deadlineMs: Date.now() + this.options.batchBudgetMs,+ republishBatch: activeRepublishBatch,+ state: {},+ attempts: [],+ unsettled: unique.length,+ settle,+ }+ this.startPoolWorkers(pool)+ try {+ batch.poolBatch = pool.add(unique, batch, batch.deadlineMs, admission?.owner)+ } finally {+ admission?.admitted()+ }+ if (batch.unsettled === 0) {+ settle()+ }+ await settled+ if (batch.state.failure) {+ throw batch.state.failure.error+ }+ return await this.finishPass(batch.attempts, activeRepublishBatch, !republishBatch)+ }++ private startPoolWorkers(pool: FetchCandidatePool<PooledBatch>): void {+ this.poolWorkers ??= Array.from({ length: this.options.maxInFlightRequests }, () => this.runPoolWorker(pool))+ }++ private async runPoolWorker(pool: FetchCandidatePool<PooledBatch>): Promise<void> {+ for (;;) {+ const lease = await pool.take()+ if (!lease) {+ return+ }+ ImageFetchPoolMetrics.observeWait(lease.waitedMs / 1000)+ let signalSlotReleased: () => void = () => undefined+ const slotReleased = new Promise<void>((resolve) => {+ signalSlotReleased = resolve+ })+ const releaseRegistrableDomainSlot = (): void => {+ lease.release()+ signalSlotReleased()+ }+ const work = this.processPooledLease(pool, lease, releaseRegistrableDomainSlot)+ await Promise.race([slotReleased, work])+ }+ }++ private async processPooledLease(+ pool: FetchCandidatePool<PooledBatch>,+ lease: FetchCandidateLease & { context: PooledBatch },+ releaseRegistrableDomainSlot: () => void+ ): Promise<void> {+ const batch = lease.context+ try {+ batch.attempts.push(+ await this.processLease(+ lease,+ batch.stored,+ batch.configurationItems,+ batch.configurationPolicy,+ batch.deadlineMs,+ batch.republishBatch,+ batch.state,+ releaseRegistrableDomainSlot+ )+ )+ } catch (error) {+ batch.state.failure ??= { error }+ if (batch.poolBatch) {+ batch.unsettled -= pool.withdraw(batch.poolBatch)+ }+ } finally {+ releaseRegistrableDomainSlot()+ batch.unsettled -= 1+ if (batch.unsettled <= 0) {+ batch.settle()+ }+ }+ }++ private async finishPass(+ attempts: FetchAttempt[],+ republishBatch: RepublishBatch,+ flushRepublishBatch: boolean+ ): Promise<FetchAttempt[]> {+ if (flushRepublishBatch) {+ const result = await republishBatch.flush()+ if (result.failedUrls > 0) {+ throw new Error(`the image fetch lane could not account for ${result.failedUrls} URLs`)+ }+ }+ this.logFailures(attempts)+ return attempts+ }++ private async runQueueWorker(+ queue: FetchCandidateQueue,+ stored: Map<string, CrawlHistoryItem>,+ configurationItems: Map<string, ConfigurationCacheItem>,+ configurationPolicy: ConfigurationPolicyPass,+ deadlineMs: number,+ attempts: FetchAttempt[],+ republishBatch: RepublishBatch,+ passState: FetchPassState+ ): Promise<void> {+ const leaseWork: Promise<void>[] = []+ for (;;) {+ const lease = queue.take()+ if (!lease) {+ break+ }+ let signalSlotReleased: () => void = () => undefined+ const slotReleased = new Promise<void>((resolve) => {+ signalSlotReleased = resolve+ })+ const releaseRegistrableDomainSlot = (): void => {+ lease.release()+ signalSlotReleased()+ }+ const work = (async (): Promise<void> => {+ try {+ attempts.push(+ await this.processLease(+ lease,+ stored,+ configurationItems,+ configurationPolicy,+ deadlineMs,+ republishBatch,+ passState,+ releaseRegistrableDomainSlot+ )+ )+ } catch (error) {+ passState.failure ??= { error }+ queue.abort()+ throw error+ } finally {+ releaseRegistrableDomainSlot()+ }+ })()+ leaseWork.push(work)+ await Promise.race([slotReleased, work.catch(() => undefined)])+ }+ const failedLease = (await Promise.allSettled(leaseWork)).find(+ (settled): settled is PromiseRejectedResult => settled.status === 'rejected'+ )+ if (failedLease) {+ throw failedLease.reason+ }+ }++ private async processLease(+ lease: FetchCandidateLease,+ stored: Map<string, CrawlHistoryItem>,+ configurationItems: Map<string, ConfigurationCacheItem>,+ configurationPolicy: ConfigurationPolicyPass,+ deadlineMs: number,+ republishBatch: RepublishBatch,+ passState: FetchPassState,+ releaseRegistrableDomainSlot: () => void+ ): Promise<FetchAttempt> {+ const candidate = lease.candidate+ if (candidate.remainingHops === 0) {+ return this.terminal(candidate, HOPS_EXHAUSTED, undefined, [])+ }+ if (Date.now() > deadlineMs) {+ return await this.republish(republishBatch, candidate, 'deadline', 'pass_deadline', 0, [], 'pass_deadline')+ }+ return await ImageFetchProcessingMetrics.runLimited(+ this.candidateWork,+ 'candidate_admission',+ 'candidate_work',+ {+ debugTag: candidate.registrableDomain,+ fn: async () => {+ if (passState.failure) {+ throw passState.failure.error+ }+ try {+ if (Date.now() > deadlineMs) {+ return await this.republish(+ republishBatch,+ candidate,+ 'deadline',+ 'pass_deadline',+ 0,+ [],+ 'pass_deadline'+ )+ }+ return await this.fetchOne(+ candidate,+ stored,+ configurationItems,+ configurationPolicy,+ deadlineMs,+ republishBatch,+ releaseRegistrableDomainSlot+ )+ } catch (error) {+ passState.failure ??= { error }+ throw error+ }+ },+ }+ )+ }++ private async fetchOne(+ candidate: FetchCandidate,+ stored: Map<string, CrawlHistoryItem>,+ configurationItems: Map<string, ConfigurationCacheItem>,+ configurationPolicy: ConfigurationPolicyPass,+ deadlineMs: number,+ republishBatch: RepublishBatch,+ releaseRegistrableDomainSlot: () => void+ ): Promise<FetchAttempt> {+ if (candidate.remainingHops === 0) {+ return this.terminal(candidate, HOPS_EXHAUSTED, undefined, [])+ }+ const policy = await ImageFetchProcessingMetrics.measure('candidate_policy', () =>+ configurationPolicy.check(candidate.currentUrl, configurationItems, Date.now())+ )+ const configurationUpdates = [...policy.updates]+ for (const update of policy.updates) {+ configurationItems.set(update.key, update)+ }+ if (!policy.allowed) {+ ImageFetchRequestMetrics.observePolicyAndBudgetDecision(true, policy.reason ?? 'configuration_refused')+ if (policy.transient) {+ if (isRequestStateFull(policy.reason)) {+ return await this.republish(+ republishBatch,+ candidate,+ policy.reason,+ policy.reason,+ ONE_MINUTE_MS,+ configurationUpdates,+ policy.reason+ )+ }+ const waitMs = policy.reason === 'configuration_unreachable' ? CONFIGURATION_RETRY_MS : ONE_MINUTE_MS+ return await this.republish(+ republishBatch,+ candidate,+ 'backoff',+ 'not_ready',+ waitMs,+ configurationUpdates,+ policy.reason === 'configuration_unreachable'+ ? 'configuration_unreachable'+ : 'configuration_deferred'+ )+ }+ return this.terminal(+ candidate,+ policy.reason ?? 'configuration_refused',+ undefined,+ configurationUpdates,+ policy.reason ?? 'configuration_refused'+ )+ }+ if (!this.budget.setCrawlDelay(candidate.origin, policy.crawlDelayMs, Date.now())) {+ ImageFetchRequestMetrics.observePolicyAndBudgetDecision(true, 'origin_map_full')+ return await this.republish(+ republishBatch,+ candidate,+ 'origin_map_full',+ 'origin_map_full',+ ONE_MINUTE_MS,+ configurationUpdates,+ 'origin_map_full'+ )+ }++ const previous = stored.get(candidate.originalRef)+ const previousUrl = previous?.kind === 'url' ? previous : undefined+ const bindResponseUrl = await this.responseUrlBinder.bindingFor(+ candidate.originalRef,+ AbortSignal.timeout(Math.max(1, deadlineMs - Date.now()))+ )+ const previousBinding = previousUrl?.cache?.responseUrlHmac+ // Another URL would learn what the issuing URL sent, and its 304 would not say this image is current.+ const validators =+ previousBinding !== undefined && previousBinding === bindResponseUrl?.(candidate.currentUrl)+ ? previousUrl?.cache+ : undefined+ const fetched = await ImageFetchProcessingMetrics.measure('candidate_fetch', () =>+ this.fetcher.fetch(candidate.currentUrl, {+ sourcePartitions: candidate.sourcePartitions,+ maxBytes: this.options.maxBytes,+ timeoutMs: this.options.requestTimeoutMs,+ maxRedirects: Math.min(this.options.maxRedirects, candidate.remainingHops),+ cache: validators,+ tdmrepReservation: policy.tdmrepReservation,+ onRedirectResponse: () => this.budget.recordCompletedResponse(candidate.registrableDomain, Date.now()),+ isDifferentOrigin: (url) => url.origin !== candidate.origin,+ scheduleRequest: (url, requestDeadlineMs, request) =>+ this.scheduler.runImage(+ url,+ Math.min(deadlineMs, requestDeadlineMs),+ request,+ candidate.sourcePartitions+ ),+ checkRedirectPolicy: async (url) => {+ const redirectPolicy = await ImageFetchProcessingMetrics.measure('candidate_policy', () =>+ configurationPolicy.check(url, configurationItems, Date.now())+ )+ for (const update of redirectPolicy.updates) {+ configurationItems.set(update.key, update)+ configurationUpdates.push(update)+ }+ if (!redirectPolicy.allowed) {+ return {+ allowed: false,+ transient: redirectPolicy.transient,+ reason: redirectPolicy.reason ?? 'configuration_refused',+ }+ }+ const origin = new URL(url).origin+ if (!this.budget.setCrawlDelay(origin, redirectPolicy.crawlDelayMs, Date.now())) {+ return { allowed: false, transient: true, reason: 'origin_map_full' }+ }+ return { allowed: true, tdmrepReservation: redirectPolicy.tdmrepReservation }+ },+ })+ )+ // One fetch can serve the jobs of several teams, so each job binds the response under its own key.+ const result: ImageFetchResult = fetched.cache+ ? { ...fetched, cache: { ...fetched.cache, responseUrlHmac: bindResponseUrl?.(fetched.currentUrl) } }+ : fetched+ ImageFetchRequestMetrics.observeRedirectCount(result.redirects)+ if (result.bytes) {+ ImageFetchRequestMetrics.observeBytes(result.bytes.length)+ }++ const effectiveUrl = canonicalizeUrl(result.currentUrl)+ if (!effectiveUrl) {+ return this.terminal(candidate, 'bad_redirect', result.cache, configurationUpdates)+ }+ const attemptedCandidate: FetchCandidate = {+ ...candidate,+ currentUrl: effectiveUrl.fetch,+ host: effectiveUrl.host,+ origin: new URL(effectiveUrl.fetch).origin,+ registrableDomain: effectiveUrl.domain,+ remainingHops: candidate.remainingHops - result.redirects,+ fetchCount:+ candidate.fetchCount ++ result.redirects ++ (result.outcome === 'request_deferred' || result.outcome === 'redirect_policy_refused' ? 0 : 1),+ }++ if (result.outcome === 'redirect_policy_refused') {+ const reason = result.refusalReason ?? 'configuration_refused'+ ImageFetchRequestMetrics.observePolicyAndBudgetDecision(true, reason)+ if (result.policyTransient) {+ const stateFull = isRequestStateFull(reason)+ const republishReason: RepublishReason = stateFull ? reason : 'not_ready'+ const waitMs = reason === 'configuration_unreachable' ? CONFIGURATION_RETRY_MS : ONE_MINUTE_MS+ return await this.republish(+ republishBatch,+ attemptedCandidate,+ reason,+ republishReason,+ waitMs,+ configurationUpdates,+ stateFull+ ? reason+ : reason === 'configuration_unreachable'+ ? 'configuration_unreachable'+ : 'configuration_deferred'+ )+ }+ return this.terminal(attemptedCandidate, reason, undefined, configurationUpdates, reason)+ }++ if (result.outcome === 'request_deferred') {+ const reason = result.schedulingReason ?? 'deadline'+ ImageFetchRequestMetrics.observePolicyAndBudgetDecision(true, reason)+ const republishReason: RepublishReason = isRequestStateFull(reason)+ ? reason+ : reason === 'deadline' && Date.now() >= deadlineMs+ ? 'pass_deadline'+ : 'not_ready'+ const waitMs =+ republishReason === 'pass_deadline'+ ? 0+ : Math.max(+ ONE_MINUTE_MS,+ result.schedulingWaitMs ??+ this.budget.blockedForMs(attemptedCandidate.registrableDomain, Date.now())+ )+ return await this.republish(+ republishBatch,+ attemptedCandidate,+ reason,+ republishReason,+ waitMs,+ configurationUpdates,+ result.schedulingBlockingReason ??+ (republishReason === 'pass_deadline'+ ? 'pass_deadline'+ : reason === 'backoff'+ ? 'unknown_backoff'+ : reason === 'deadline'+ ? 'request_deadline'+ : reason)+ )+ }++ ImageFetchRequestMetrics.observePolicyAndBudgetDecision(false)++ if (isTransientOutcome(result.outcome)) {+ const delayMs = this.budget.recordTransientFailure(+ candidate.registrableDomain,+ Date.now(),+ result.retryAfterMs+ )+ const attempt = await this.republish(+ republishBatch,+ attemptedCandidate,+ result.outcome,+ 'retry',+ delayMs,+ configurationUpdates,+ result.retryAfterMs === undefined ? 'transient_backoff' : 'retry_after'+ )+ return attempt+ }++ this.budget.recordCompletedResponse(candidate.registrableDomain, Date.now())+ if (+ (result.outcome === 'redirect_offsite' || result.outcome === 'redirect_continuation') &&+ result.redirectTarget+ ) {+ const canonical = canonicalizeUrl(result.redirectTarget.url)+ if (!canonical) {+ return this.terminal(attemptedCandidate, 'bad_redirect', result.cache, configurationUpdates)+ }+ return await this.republishToTarget(+ republishBatch,+ attemptedCandidate,+ result.outcome,+ {+ currentUrl: canonical.fetch,+ host: canonical.host,+ origin: new URL(canonical.fetch).origin,+ registrableDomain: canonical.domain,+ },+ 'redirect',+ 0,+ configurationUpdates+ )+ }+ // A missing or broken response leaves the stored image in place, so the lane keeps it alive. An opt-out does not.+ let storedAtMs = result.outcome === 'opt_out' ? undefined : previousUrl?.storedAtMs+ if (result.outcome === 'ok' || result.outcome === 'not_modified') {+ releaseRegistrableDomainSlot()+ let stored = true+ try {+ if (result.outcome === 'ok') {+ await this.imageSink.publishImage(attemptedCandidate, result)+ } else {+ // A 304 says the stored image is current, so the sink refreshes it in place.+ stored = await this.imageSink.refreshImage(attemptedCandidate)+ }+ } catch (error) {+ this.logStorageFailure(result.outcome === 'ok' ? 'publish' : 'refresh', error)+ return {+ candidate: attemptedCandidate,+ outcome: 'publish_failed',+ finished: false,+ lost: true,+ configurationUpdates,+ }+ }+ if (!stored) {+ return await this.refetchWithoutValidators(+ republishBatch,+ attemptedCandidate,+ previousUrl,+ configurationUpdates+ )+ }+ storedAtMs = Date.now()+ }+ const responseCache = result.redirects > 0 ? withoutValidators(result.cache) : result.cache+ const mergedCache =+ result.outcome === 'not_modified' ? mergeCache(previousUrl?.cache, responseCache) : responseCache+ return this.terminal(+ attemptedCandidate,+ result.outcome,+ mergedCache,+ configurationUpdates,+ result.refusalReason,+ storedAtMs+ )+ }++ /**+ * A 304 for an image S3 no longer holds, for example after the lifecycle expired it. The history drops the+ * validators before the job comes back, so the next request asks for the whole image.+ */+ private async refetchWithoutValidators(+ republishBatch: RepublishBatch,+ candidate: FetchCandidate,+ previousUrl: UrlCrawlHistoryItem | undefined,+ configurationUpdates: ConfigurationCacheItem[]+ ): Promise<FetchAttempt> {+ const attempt = await this.republishToTarget(+ republishBatch,+ candidate,+ 'not_modified',+ candidate,+ 'retry',+ 0,+ configurationUpdates+ )+ if (attempt.finished) {+ return attempt+ }+ const nowMs = Date.now()+ return {+ ...attempt,+ history: {+ kind: 'url',+ key: candidate.originalRef,+ nextFetchAtMs: nowMs,+ storageExpiresAtMs: urlHistoryStorageExpiresAtMs(candidate.originalRef, nowMs),+ outcome: 'not_stored',+ cache: withoutValidators(previousUrl?.cache),+ },+ }+ }++ private async republish(+ republishBatch: RepublishBatch,+ candidate: FetchCandidate,+ outcome: AttemptOutcome,+ reason: RepublishReason,+ waitMs: number,+ configurationUpdates: ConfigurationCacheItem[],+ blockingReason: ImageFetchBlockReason+ ): Promise<FetchAttempt> {+ return await this.republishToTarget(+ republishBatch,+ candidate,+ outcome,+ {+ currentUrl: candidate.currentUrl,+ host: candidate.host,+ origin: candidate.origin,+ registrableDomain: candidate.registrableDomain,+ },+ reason,+ waitMs,+ configurationUpdates,+ { reason: blockingReason, waitMs }+ )+ }++ private async republishToTarget(+ republishBatch: RepublishBatch,+ candidate: FetchCandidate,+ outcome: AttemptOutcome,+ target: Pick<FetchCandidate, 'currentUrl' | 'host' | 'origin' | 'registrableDomain'>,+ reason: RepublishReason,+ waitMs: number,+ configurationUpdates: ConfigurationCacheItem[],+ block?: { reason: ImageFetchBlockReason; waitMs: number }+ ): Promise<FetchAttempt> {+ if ((reason === 'redirect' || reason === 'retry') && candidate.remainingHops <= 1) {+ return this.terminal(candidate, HOPS_EXHAUSTED, undefined, configurationUpdates)+ }+ const republishedCandidate: FetchCandidate = {+ ...candidate,+ lastBlockReason: block?.reason,+ }+ const result: RepublishResult = await republishBatch.republish(republishedCandidate, target, reason, waitMs)+ if (result === 'refused_delay') {+ return this.terminal(candidate, DELAY_TOO_LONG, undefined, configurationUpdates)+ }+ return {+ candidate: republishedCandidate,+ outcome,+ finished: false,+ lost: false,+ block,+ configurationUpdates,+ }+ }++ private terminal(+ candidate: FetchCandidate,+ outcome: AttemptOutcome,+ cache: HttpCacheMetadata | undefined,+ configurationUpdates: ConfigurationCacheItem[],+ refusalReason: FetchRefusalReason | 'none' = 'none',+ storedAtMs?: number+ ): FetchAttempt {+ const nowMs = Date.now()+ const nextFetchAtMs = urlHistoryExpiresAtMs(nowMs, this.options.seenTtlSeconds, cache)+ ImageFetchRequestMetrics.observeCompletedUrl(+ outcome,+ refusalReason,+ Math.max(0, nowMs - candidate.firstSeenAtMs) / 1000,+ candidate.fetchCount,+ candidate.republishCount+ )+ ImageFetchRequestMetrics.observeHops(MAX_HOPS - candidate.remainingHops)+ return {+ candidate,+ outcome,+ finished: true,+ lost: false,+ history: {+ kind: 'url',+ key: candidate.originalRef,+ nextFetchAtMs,+ storageExpiresAtMs: urlHistoryStorageExpiresAtMs(candidate.originalRef, nextFetchAtMs),+ outcome: String(outcome),+ cache,+ ...(storedAtMs === undefined ? {} : { storedAtMs }),+ },+ configurationUpdates,+ }+ }++ /**+ * S3 and KMS fail in bursts, so this logs at most one failure per interval, with the count since the last one. The+ * error message can name the object key, so only the error name and status are logged, never the URL or the key.+ */+ private logStorageFailure(operation: 'publish' | 'refresh', error: unknown): void {+ this.storageFailuresSinceLog += 1+ const nowMs = Date.now()+ if (nowMs - this.storageFailureLoggedAtMs < STORAGE_FAILURE_LOG_INTERVAL_MS) {+ return+ }+ const { name, Code, $metadata } = (typeof error === 'object' && error !== null ? error : {}) as {+ name?: unknown+ Code?: unknown+ $metadata?: { httpStatusCode?: unknown }+ }+ logger.warn('🌐', 'ai_research_v4_image_fetch_storage_failed', {+ operation,+ error: typeof name === 'string' ? name : 'unknown',+ code: typeof Code === 'string' ? Code : undefined,+ httpStatusCode: typeof $metadata?.httpStatusCode === 'number' ? $metadata.httpStatusCode : undefined,+ failures: this.storageFailuresSinceLog,+ })+ this.storageFailureLoggedAtMs = nowMs+ this.storageFailuresSinceLog = 0+ }++ private logFailures(attempts: FetchAttempt[]): void {+ const failures = attempts.filter((attempt) => attempt.outcome !== 'ok' && attempt.outcome !== 'not_modified')+ if (failures.length === 0) {+ return+ }+ logger.warn('🌐', 'ai_research_v4_image_fetch_batch_failures', {+ count: failures.length,+ outcomes: [...new Set(failures.map((attempt) => String(attempt.outcome)))].slice(0, 10),+ hosts: [...new Set(failures.map((attempt) => attempt.candidate.host))].slice(0, 5),+ })+ }+}++function mergeCache(+ previous: HttpCacheMetadata | undefined,+ update: HttpCacheMetadata | undefined+): HttpCacheMetadata | undefined {+ if (!previous) {+ return update+ }+ if (!update) {+ return previous+ }+ const definedUpdate = Object.fromEntries(Object.entries(update).filter(([, value]) => value !== undefined))+ return { ...previous, ...definedUpdate }+}++function withoutValidators(cache: HttpCacheMetadata | undefined): HttpCacheMetadata | undefined {+ return cache ? { ...cache, etag: undefined, lastModified: undefined } : undefined+}++function configurationItemsFrom(stored: Map<string, CrawlHistoryItem>): Map<string, ConfigurationCacheItem> {+ const configurationItems = new Map<string, ConfigurationCacheItem>()+ for (const [key, item] of stored) {+ if (item.kind === 'robots' || item.kind === 'tdmrep') {+ configurationItems.set(key, item)+ }+ }+ return configurationItems+}diff --git a/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/frontier-dead-letter-sink.test.ts b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/frontier-dead-letter-sink.test.tsnew file mode 100644index 000000000000..77e9048b5dc0--- /dev/null+++ b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/frontier-dead-letter-sink.test.ts@@ -0,0 +1,94 @@+import { Message } from 'node-rdkafka'++import { KafkaProducerWrapper } from '~/common/kafka/producer'++import { MAX_RECORD_BYTES } from './collected-urls-record'+import { KafkaFrontierDeadLetterSink } from './frontier-dead-letter-sink'++describe('KafkaFrontierDeadLetterSink', () => {+ it('preserves the source key and value with bounded diagnostic headers', async () => {+ const produce = jest.fn(() => Promise.resolve())+ const sink = new KafkaFrontierDeadLetterSink({ produce } as unknown as KafkaProducerWrapper, 'frontier-dlq', [+ 'frontier',+ ])+ const source = {+ topic: 'frontier',+ partition: 7,+ offset: 42,+ key: Buffer.from('example.com'),+ value: Buffer.from('{"v":2,"jobs":[]}'),+ } as Message++ await sink.park(source, 'bad_url')++ expect(produce).toHaveBeenCalledWith({+ topic: 'frontier-dlq',+ key: source.key,+ value: source.value,+ headers: {+ 'dlq-reason': 'bad_url',+ 'frontier-record-version': '2',+ 'source-topic': 'frontier',+ 'source-partition': '7',+ 'source-offset': '42',+ },+ })+ })++ it('does not copy an unsupported version into a header', async () => {+ const produce = jest.fn(() => Promise.resolve())+ const sink = new KafkaFrontierDeadLetterSink({ produce } as unknown as KafkaProducerWrapper, 'frontier-dlq', [+ 'frontier',+ ])++ await sink.park(+ {+ topic: 'frontier',+ partition: 0,+ offset: 1,+ key: null,+ value: Buffer.from('{"v":"private input"}'),+ } as Message,+ 'unsupported_version'+ )++ expect(produce).toHaveBeenCalledWith(+ expect.objectContaining({+ headers: expect.objectContaining({ 'frontier-record-version': 'unsupported' }),+ })+ )+ })++ it('does not parse an oversized rejected value to classify its version', async () => {+ const produce = jest.fn(() => Promise.resolve())+ const sink = new KafkaFrontierDeadLetterSink({ produce } as unknown as KafkaProducerWrapper, 'frontier-dlq', [+ 'frontier',+ ])++ await sink.park(+ {+ topic: 'frontier',+ partition: 0,+ offset: 1,+ key: null,+ value: Buffer.alloc(MAX_RECORD_BYTES + 1, '{'),+ } as Message,+ 'oversized_record'+ )++ expect(produce).toHaveBeenCalledWith(+ expect.objectContaining({+ headers: expect.objectContaining({ 'frontier-record-version': 'unknown' }),+ })+ )+ })++ it('refuses a destination used by the image-fetch pipeline', () => {+ expect(+ () =>+ new KafkaFrontierDeadLetterSink({ produce: jest.fn() } as unknown as KafkaProducerWrapper, 'frontier', [+ 'frontier',+ ])+ ).toThrow('AI_RESEARCH_V4_IMAGE_FETCH_DLQ_TOPIC')+ })+})diff --git a/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/frontier-dead-letter-sink.ts b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/frontier-dead-letter-sink.tsnew file mode 100644index 000000000000..df5407481538--- /dev/null+++ b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/frontier-dead-letter-sink.ts@@ -0,0 +1,63 @@+import { Message } from 'node-rdkafka'++import { parseKafkaHeaders } from '~/common/kafka/consumer/consumer-v1'+import { KafkaProducer } from '~/common/kafka/producer'+import { parseJSON } from '~/common/utils/json-parse'++import { MAX_RECORD_BYTES, UrlDropReason } from './collected-urls-record'++export type FrontierDeadLetterReason = UrlDropReason | 'multiple'+type FrontierRecordVersion = '1' | '2' | 'unsupported' | 'unknown'++export interface FrontierDeadLetterSink {+ park(message: Message, reason: FrontierDeadLetterReason): Promise<void>+}++export class KafkaFrontierDeadLetterSink implements FrontierDeadLetterSink {+ constructor(+ private readonly producer: KafkaProducer,+ private readonly topic: string,+ blockedDestinations: readonly string[]+ ) {+ if (blockedDestinations.includes(topic)) {+ throw new Error(`AI_RESEARCH_V4_IMAGE_FETCH_DLQ_TOPIC cannot use image-fetch topic ${topic}`)+ }+ }++ public async park(message: Message, reason: FrontierDeadLetterReason): Promise<void> {+ await this.producer.produce({+ topic: this.topic,+ key: message.key ?? null,+ value: message.value,+ headers: {+ ...parseKafkaHeaders(message.headers),+ 'dlq-reason': reason,+ 'frontier-record-version': classifyFrontierRecordVersion(message.value),+ 'source-topic': message.topic,+ 'source-partition': String(message.partition),+ 'source-offset': String(message.offset),+ },+ })+ }+}++function classifyFrontierRecordVersion(value: Buffer | null): FrontierRecordVersion {+ if (!value || value.length > MAX_RECORD_BYTES) {+ return 'unknown'+ }+ try {+ const parsed: unknown = parseJSON(value.toString())+ if (typeof parsed !== 'object' || parsed === null || Array.isArray(parsed) || !('v' in parsed)) {+ return 'unknown'+ }+ if (parsed.v === 1) {+ return '1'+ }+ if (parsed.v === 2) {+ return '2'+ }+ return 'unsupported'+ } catch {+ return 'unknown'+ }+}diff --git a/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/frontier-publisher.test.ts b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/frontier-publisher.test.tsnew file mode 100644index 000000000000..e2b60454d201--- /dev/null+++ b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/frontier-publisher.test.ts@@ -0,0 +1,405 @@+import { KafkaProducerWrapper } from '~/common/kafka/producer'+import { parseJSON } from '~/common/utils/json-parse'+import { CAPTURE_TIMESTAMP_HEADER } from '~/ingestion/pipelines/sessionreplay/shared/capture-watermark'++import {+ FetchCandidate,+ MAX_HOPS,+ MAX_JOBS_PER_RECORD,+ MAX_RECORD_BYTES,+ parseCollectedUrlsRecord,+} from './collected-urls-record'+import { FrontierPublisher, RepublishBatch } from './frontier-publisher'+import { ImageFetchRequestMetrics } from './metrics'++const FRONTIER = 'session_replay_image_fetch'+const TIERS = [+ { topic: 'retry_1m', delayMs: 60_000, metricTopic: 'retry_1m' as const },+ { topic: 'retry_10m', delayMs: 600_000, metricTopic: 'retry_10m' as const },+ { topic: 'retry_1h', delayMs: 3_600_000, metricTopic: 'retry_1h' as const },+]++function candidate(overrides: Partial<FetchCandidate> = {}): FetchCandidate {+ return {+ originalRef: `imageurl:v4:7:2026-10:${'a'.repeat(22)}`,+ currentUrl: 'https://cdn.example.com/a.png',+ host: 'cdn.example.com',+ origin: 'https://cdn.example.com',+ registrableDomain: 'example.com',+ remainingHops: MAX_HOPS,+ notBeforeMs: 0,+ firstSeenAtMs: 1_700_000_000_000,+ fetchCount: 0,+ republishCount: 0,+ lastRepublishReason: null,+ ...overrides,+ }+}++interface SentMessage {+ topic: string+ key: string+ value: Buffer+ headers?: Record<string, string>+}++function build(): {+ publisher: FrontierPublisher+ batch: RepublishBatch+ sent: SentMessage[]+} {+ const sent: SentMessage[] = []+ const producer = {+ produce: (message: { topic: string; key: Buffer; value: Buffer; headers?: Record<string, string> }) => {+ sent.push({ ...message, key: message.key.toString() })+ return Promise.resolve()+ },+ } as unknown as KafkaProducerWrapper+ const publisher = new FrontierPublisher(producer, {+ frontierTopic: FRONTIER,+ delayTiers: TIERS,+ maxConcurrentRepublishes: 2,+ })+ return { publisher, batch: publisher.createRepublishBatch(), sent }+}++describe('FrontierPublisher', () => {+ beforeEach(() => jest.useFakeTimers().setSystemTime(1_700_000_000_000))+ afterEach(() => {+ jest.useRealTimers()+ jest.restoreAllMocks()+ })++ it('groups frontier redirects by target registrable domain', async () => {+ const { batch, sent } = build()+ const republished = jest.spyOn(ImageFetchRequestMetrics, 'incRepublished').mockImplementation()+ const target = {+ currentUrl: 'https://img.other.net/a.png',+ host: 'img.other.net',+ origin: 'https://img.other.net',+ registrableDomain: 'other.net',+ }++ expect(await batch.republish(candidate(), target, 'redirect')).toBe('queued')+ expect(+ await batch.republish(+ candidate({ originalRef: `imageurl:v4:7:2026-10:${'b'.repeat(22)}`, firstSeenAtMs: 1_699_999_000_000 }),+ target,+ 'redirect'+ )+ ).toBe('queued')+ expect(sent).toEqual([])+ await expect(batch.flush()).resolves.toEqual({ failedUrls: 0 })++ expect(republished).toHaveBeenCalledTimes(2)+ expect(sent).toHaveLength(1)+ expect(sent[0]).toMatchObject({+ topic: FRONTIER,+ key: 'other.net',+ headers: { [CAPTURE_TIMESTAMP_HEADER]: '1699999000000' },+ })+ expect(parseCollectedUrlsRecord(sent[0].value, 'other.net')).toMatchObject({+ ok: true,+ candidates: [+ {+ originalRef: `imageurl:v4:7:2026-10:${'a'.repeat(22)}`,+ currentUrl: 'https://img.other.net/a.png',+ remainingHops: MAX_HOPS - 1,+ republishCount: 1,+ lastRepublishReason: 'redirect',+ },+ { originalRef: `imageurl:v4:7:2026-10:${'b'.repeat(22)}` },+ ],+ })+ })++ it.each([+ ['an unspecified retry', 0, 'retry_1m', 60_000],+ ['a short retry', 30_000, 'retry_1m', 60_000],+ ['a medium retry', 120_000, 'retry_10m', 600_000],+ ['a long supported retry', 3_600_000, 'retry_1h', 3_600_000],+ ])('parks %s once in the smallest sufficient delay topic', async (_name, waitMs, topic, delayMs) => {+ const { batch, sent } = build()+ const republished = jest.spyOn(ImageFetchRequestMetrics, 'incRepublished').mockImplementation()++ expect(await batch.republish(candidate(), candidate(), 'retry', waitMs)).toBe('queued')+ await batch.flush()++ expect(republished).toHaveBeenCalledWith('retry', 'delay')+ expect(sent[0].topic).toBe(topic)+ expect(parseCollectedUrlsRecord(sent[0].value, 'example.com')).toMatchObject({+ ok: true,+ candidates: [{ notBeforeMs: 1_700_000_000_000 + delayMs }],+ })+ })++ it('groups delayed jobs by destination topic and registrable domain', async () => {+ const { batch, sent } = build()+ const second = candidate({ originalRef: `imageurl:v4:7:2026-10:${'b'.repeat(22)}` })+ const other = candidate({+ originalRef: `imageurl:v4:7:2026-10:${'c'.repeat(22)}`,+ currentUrl: 'https://img.other.net/c.png',+ host: 'img.other.net',+ origin: 'https://img.other.net',+ registrableDomain: 'other.net',+ })++ await batch.republish(candidate(), candidate(), 'retry', 30_000)+ await batch.republish(second, second, 'not_ready', 30_000)+ await batch.republish(other, other, 'retry', 30_000)+ await batch.flush()++ expect(sent).toHaveLength(2)+ const example = sent.find((item) => item.key === 'example.com')!+ expect(example.topic).toBe('retry_1m')+ expect(parseCollectedUrlsRecord(example.value, 'example.com')).toMatchObject({+ ok: true,+ candidates: [{ originalRef: candidate().originalRef }, { originalRef: second.originalRef }],+ })+ })++ it('keeps configured destination topics separate when they share a metric class', async () => {+ const sent: SentMessage[] = []+ const producer = {+ produce: (message: { topic: string; key: Buffer; value: Buffer }) => {+ sent.push({ ...message, key: message.key.toString() })+ return Promise.resolve()+ },+ } as unknown as KafkaProducerWrapper+ const publisher = new FrontierPublisher(producer, {+ frontierTopic: FRONTIER,+ delayTiers: [+ { topic: 'short', delayMs: 60_000, metricTopic: 'retry_1m' },+ { topic: 'long', delayMs: 120_000, metricTopic: 'retry_1m' },+ ],+ maxConcurrentRepublishes: 2,+ })+ const batch = publisher.createRepublishBatch()++ await batch.republish(candidate(), candidate(), 'not_ready', 30_000)+ await batch.republish(+ candidate({ originalRef: `imageurl:v4:7:2026-10:${'b'.repeat(22)}` }),+ candidate(),+ 'not_ready',+ 90_000+ )+ await batch.flush()++ expect(sent.map((message) => message.topic).sort()).toEqual(['long', 'short'])+ })++ it('records messages, registrable domains, and delivery time for each destination topic', async () => {+ const { batch } = build()+ const observeRepublishBatch = jest.spyOn(ImageFetchRequestMetrics, 'observeRepublishBatch').mockImplementation()+ const observeRepublishFlush = jest.spyOn(ImageFetchRequestMetrics, 'observeRepublishFlush').mockImplementation()+ const other = candidate({+ originalRef: `imageurl:v4:7:2026-10:${'b'.repeat(22)}`,+ currentUrl: 'https://img.other.net/b.png',+ host: 'img.other.net',+ origin: 'https://img.other.net',+ registrableDomain: 'other.net',+ })++ await batch.republish(candidate(), candidate(), 'pass_deadline')+ await batch.republish(other, other, 'retry', 30_000)+ await batch.flush()++ expect(observeRepublishBatch).toHaveBeenCalledWith('frontier', 1, 1, expect.any(Number))+ expect(observeRepublishBatch).toHaveBeenCalledWith('retry_1m', 1, 1, expect.any(Number))+ expect(observeRepublishFlush).toHaveBeenCalledWith(expect.any(Number))+ })++ it('splits one registrable domain at the job-count limit', async () => {+ const { batch, sent } = build()+ const candidates = Array.from({ length: MAX_JOBS_PER_RECORD + 1 }, (_, index) =>+ candidate({ originalRef: `imageurl:v4:7:2026-10:${index.toString().padStart(22, '0')}` })+ )++ await Promise.all(candidates.map((item) => batch.republish(item, item, 'pass_deadline')))+ await batch.flush()++ expect(sent).toHaveLength(2)+ expect(sent.map((item) => (parseJSON(item.value.toString()) as { jobs: unknown[] }).jobs.length)).toEqual([+ MAX_JOBS_PER_RECORD,+ 1,+ ])+ })++ it('splits one registrable domain at the serialized-byte limit', async () => {+ const { batch, sent } = build()+ const longPath = 'a'.repeat(Math.floor(MAX_RECORD_BYTES / 2))+ const first = candidate({ currentUrl: `https://cdn.example.com/${longPath}/1.png` })+ const second = candidate({+ originalRef: `imageurl:v4:7:2026-10:${'b'.repeat(22)}`,+ currentUrl: `https://cdn.example.com/${longPath}/2.png`,+ })++ await batch.republish(first, first, 'pass_deadline')+ await batch.republish(second, second, 'pass_deadline')+ await batch.flush()++ expect(sent).toHaveLength(2)+ expect(sent.every((item) => item.value.length <= MAX_RECORD_BYTES)).toBe(true)+ })++ it('refuses a delay longer than the largest deployed tier', async () => {+ const { batch, sent } = build()++ expect(await batch.republish(candidate(), candidate(), 'retry', 3_600_001)).toBe('refused_delay')+ await batch.flush()+ expect(sent).toEqual([])+ })++ it('refuses a redirect that has no hop left after republishing', async () => {+ const { batch, sent } = build()++ expect(await batch.republish(candidate({ remainingHops: 1 }), candidate(), 'redirect')).toBe('refused_delay')+ await batch.flush()+ expect(sent).toEqual([])+ })++ it('publishes every target-domain group from redirect fanout to the frontier', async () => {+ const { batch, sent } = build()+ const other = candidate({+ originalRef: `imageurl:v4:7:2026-10:${'b'.repeat(22)}`,+ currentUrl: 'https://img.other.net/b.png',+ host: 'img.other.net',+ origin: 'https://img.other.net',+ registrableDomain: 'other.net',+ })++ await batch.republish(candidate(), candidate(), 'pass_deadline')+ await batch.republish(other, other, 'redirect')++ await expect(batch.flush()).resolves.toEqual({ failedUrls: 0 })+ expect(sent).toHaveLength(2)+ expect(sent.every((item) => item.topic === FRONTIER)).toBe(true)+ const redirected = sent.find((item) => item.key === 'other.net')!+ expect(parseCollectedUrlsRecord(redirected.value, 'other.net')).toMatchObject({+ ok: true,+ candidates: [{ lastRepublishReason: 'redirect', notBeforeMs: 0 }],+ })+ })++ it('bounds concurrent Kafka republish deliveries', async () => {+ let active = 0+ let peak = 0+ const releases: Array<() => void> = []+ const producer = {+ produce: () =>+ new Promise<void>((resolve) => {+ active+++ peak = Math.max(peak, active)+ releases.push(() => {+ active--+ resolve()+ })+ }),+ } as unknown as KafkaProducerWrapper+ const publisher = new FrontierPublisher(producer, {+ frontierTopic: FRONTIER,+ delayTiers: TIERS,+ maxConcurrentRepublishes: 2,+ })+ const batch = publisher.createRepublishBatch()++ for (const domain of ['one.example', 'two.example', 'three.example']) {+ const item = candidate({+ currentUrl: `https://${domain}/a.png`,+ host: domain,+ origin: `https://${domain}`,+ registrableDomain: domain,+ })+ await batch.republish(item, item, 'pass_deadline')+ }++ const flushing = batch.flush()+ expect(active).toBe(2)+ expect(peak).toBe(2)++ const firstTwo = releases.splice(0, 2)+ firstTwo[0]()+ for (let index = 0; index < 10; index++) {+ await Promise.resolve()+ }+ expect(active).toBe(2)+ expect(peak).toBe(2)++ firstTwo[1]()+ releases[0]()+ await expect(flushing).resolves.toEqual({ failedUrls: 0 })+ })++ it('reports every URL in a failed grouped message', async () => {+ const producer = { produce: () => Promise.reject(new Error('broker down')) } as unknown as KafkaProducerWrapper+ const publisher = new FrontierPublisher(producer, {+ frontierTopic: FRONTIER,+ delayTiers: TIERS,+ maxConcurrentRepublishes: 2,+ })+ const batch = publisher.createRepublishBatch()+ const failed = jest.spyOn(ImageFetchRequestMetrics, 'incRepublishFailed').mockImplementation()++ await batch.republish(candidate(), candidate(), 'pass_deadline')+ await batch.republish(+ candidate({ originalRef: `imageurl:v4:7:2026-10:${'b'.repeat(22)}` }),+ candidate(),+ 'pass_deadline'+ )++ await expect(batch.flush()).resolves.toEqual({ failedUrls: 2 })+ expect(failed).toHaveBeenCalledTimes(2)+ })++ it('stops starting republish deliveries after the first failure', async () => {+ const produce = jest.fn(() => Promise.reject(new Error('broker down')))+ const producer = { produce } as unknown as KafkaProducerWrapper+ const publisher = new FrontierPublisher(producer, {+ frontierTopic: FRONTIER,+ delayTiers: TIERS,+ maxConcurrentRepublishes: 1,+ })+ const batch = publisher.createRepublishBatch()++ for (const domain of ['one.example', 'two.example', 'three.example']) {+ const item = candidate({+ currentUrl: `https://${domain}/a.png`,+ host: domain,+ origin: `https://${domain}`,+ registrableDomain: domain,+ })+ await batch.republish(item, item, 'pass_deadline')+ }++ await expect(batch.flush()).resolves.toEqual({ failedUrls: 3 })+ expect(produce).toHaveBeenCalledTimes(1)+ })++ it('does not start republish deliveries after the finalization deadline', async () => {+ const produce = jest.fn(() => Promise.resolve())+ const producer = { produce } as unknown as KafkaProducerWrapper+ const publisher = new FrontierPublisher(producer, {+ frontierTopic: FRONTIER,+ delayTiers: TIERS,+ maxConcurrentRepublishes: 2,+ })+ const batch = publisher.createRepublishBatch(performance.now())+ const deadlineExceeded = jest+ .spyOn(ImageFetchRequestMetrics, 'incRepublishFlushDeadlineExceeded')+ .mockImplementation()++ for (const domain of ['one.example', 'two.example', 'three.example']) {+ const item = candidate({+ currentUrl: `https://${domain}/a.png`,+ host: domain,+ origin: `https://${domain}`,+ registrableDomain: domain,+ })+ await batch.republish(item, item, 'pass_deadline')+ }++ await expect(batch.flush()).resolves.toEqual({ failedUrls: 3 })+ expect(produce).not.toHaveBeenCalled()+ expect(deadlineExceeded).toHaveBeenCalledTimes(1)+ })+})diff --git a/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/frontier-publisher.ts b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/frontier-publisher.tsnew file mode 100644index 000000000000..a3e7354efd5d--- /dev/null+++ b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/frontier-publisher.ts@@ -0,0 +1,331 @@+import { KafkaProducer } from '~/common/kafka/producer'+import { ConcurrencyController } from '~/common/utils/concurrencyController'+import { logger } from '~/common/utils/logger'+import { CAPTURE_TIMESTAMP_HEADER } from '~/ingestion/pipelines/sessionreplay/shared/capture-watermark'++import {+ FetchCandidate,+ MAX_JOBS_PER_RECORD,+ MAX_RECORD_BYTES,+ RepublishReason,+ serializeFrontierRecord,+} from './collected-urls-record'+import { ImageFetchRequestMetrics, RepublishTopic } from './metrics'++export interface DelayTier {+ topic: string+ delayMs: number+ metricTopic: Exclude<RepublishTopic, 'frontier'>+}++export interface FrontierPublisherOptions {+ frontierTopic: string+ delayTiers: DelayTier[]+ maxConcurrentRepublishes: number+}++export type RepublishResult = 'queued' | 'refused_delay'++export interface RepublishFlushResult {+ failedUrls: number+}++interface RepublishDestination {+ topic: string+ metricTopic: RepublishTopic+ delayMs: number+}++interface PendingRepublish {+ candidate: FetchCandidate+ reason: RepublishReason+ destination: RepublishDestination+}++interface PlannedRepublishMessage {+ topic: string+ metricTopic: RepublishTopic+ registrableDomain: string+ candidates: FetchCandidate[]+ reasons: RepublishReason[]+}++interface RepublishDeliveryState {+ failed: boolean+ deadlineExceeded: boolean+}++type RepublishDeliveryResult = 'published' | 'failed' | 'skipped'++const EMPTY_FRONTIER_RECORD_BYTES = serializeFrontierRecord([]).length++export interface RepublishBatch {+ republish(+ candidate: FetchCandidate,+ target: Pick<FetchCandidate, 'currentUrl' | 'host' | 'origin' | 'registrableDomain'>,+ reason: RepublishReason,+ waitMs?: number+ ): Promise<RepublishResult>+ flush(): Promise<RepublishFlushResult>+}++/** Sends each URL that is not done to the frontier or a delay tier. */+export class FrontierPublisher {+ private readonly delayTiers: DelayTier[]+ private readonly republishes: ConcurrencyController++ constructor(+ private readonly producer: KafkaProducer,+ private readonly options: FrontierPublisherOptions+ ) {+ this.delayTiers = [...options.delayTiers].sort((left, right) => left.delayMs - right.delayMs)+ if (this.delayTiers.length === 0) {+ throw new Error('the image fetch lane needs at least one delay tier')+ }+ if (!Number.isInteger(options.maxConcurrentRepublishes) || options.maxConcurrentRepublishes < 1) {+ throw new Error('the image fetch lane needs a positive republish limit')+ }+ this.republishes = new ConcurrencyController(options.maxConcurrentRepublishes)+ }++ public createRepublishBatch(deadlineAtMonotonicMs = Number.POSITIVE_INFINITY): RepublishBatch {+ return new BufferedRepublishBatch(+ this.producer,+ this.options.frontierTopic,+ this.delayTiers,+ this.republishes,+ deadlineAtMonotonicMs+ )+ }+}++class BufferedRepublishBatch implements RepublishBatch {+ private readonly pending: PendingRepublish[] = []+ private flushed = false++ constructor(+ private readonly producer: KafkaProducer,+ private readonly frontierTopic: string,+ private readonly delayTiers: DelayTier[],+ private readonly republishes: ConcurrencyController,+ private readonly deadlineAtMonotonicMs: number+ ) {}++ public republish(+ candidate: FetchCandidate,+ target: Pick<FetchCandidate, 'currentUrl' | 'host' | 'origin' | 'registrableDomain'>,+ reason: RepublishReason,+ waitMs = 0+ ): Promise<RepublishResult> {+ if (this.flushed) {+ return Promise.reject(new Error('an image fetch republish batch cannot accept work after its flush'))+ }+ const spendsHop = reason === 'redirect' || reason === 'retry'+ const remainingHops = candidate.remainingHops - (spendsHop ? 1 : 0)+ if (remainingHops <= 0) {+ return Promise.resolve('refused_delay')+ }++ const effectiveWaitMs = reason === 'retry' ? Math.max(waitMs, this.delayTiers[0].delayMs) : waitMs+ const destination = this.destination(effectiveWaitMs)+ if (!destination) {+ return Promise.resolve('refused_delay')+ }+ this.pending.push({+ candidate: {+ ...candidate,+ ...target,+ remainingHops,+ notBeforeMs: destination.delayMs > 0 ? Date.now() + destination.delayMs : 0,+ republishCount: candidate.republishCount + 1,+ lastRepublishReason: reason,+ },+ reason,+ destination,+ })+ return Promise.resolve('queued')+ }++ public async flush(): Promise<RepublishFlushResult> {+ if (this.flushed) {+ throw new Error('an image fetch republish batch cannot flush more than once')+ }+ this.flushed = true+ if (this.pending.length === 0) {+ return { failedUrls: 0 }+ }+ const startedAt = process.hrtime.bigint()+ try {+ const plans = this.planMessages()+ const deliveryState: RepublishDeliveryState = { failed: false, deadlineExceeded: false }+ const plansByTopic = new Map<RepublishTopic, PlannedRepublishMessage[]>()+ for (const plan of plans) {+ const topicPlans = plansByTopic.get(plan.metricTopic)+ if (topicPlans) {+ topicPlans.push(plan)+ } else {+ plansByTopic.set(plan.metricTopic, [plan])+ }+ }+ const topicResults = await Promise.all(+ [...plansByTopic.entries()].map(([metricTopic, topicPlans]) =>+ this.publishTopic(metricTopic, topicPlans, deliveryState)+ )+ )+ if (deliveryState.deadlineExceeded) {+ ImageFetchRequestMetrics.incRepublishFlushDeadlineExceeded()+ }+ return { failedUrls: topicResults.reduce((sum, result) => sum + result.failedUrls, 0) }+ } finally {+ ImageFetchRequestMetrics.observeRepublishFlush(Number(process.hrtime.bigint() - startedAt) / 1e9)+ }+ }++ private destination(waitMs: number): RepublishDestination | undefined {+ if (waitMs <= 0) {+ return { topic: this.frontierTopic, metricTopic: 'frontier', delayMs: 0 }+ }+ const tier = this.delayTiers.find((candidateTier) => candidateTier.delayMs >= waitMs)+ return tier ? { topic: tier.topic, metricTopic: tier.metricTopic, delayMs: tier.delayMs } : undefined+ }++ private planMessages(): PlannedRepublishMessage[] {+ const groups = new Map<string, PendingRepublish[]>()+ for (const item of this.pending) {+ const key = `${item.destination.topic}\0${item.candidate.registrableDomain}`+ const group = groups.get(key)+ if (group) {+ group.push(item)+ } else {+ groups.set(key, [item])+ }+ }+ return [...groups.values()].flatMap((items) => this.packGroup(items))+ }++ private packGroup(items: PendingRepublish[]): PlannedRepublishMessage[] {+ const plans: PlannedRepublishMessage[] = []+ let candidates: FetchCandidate[] = []+ let reasons: RepublishReason[] = []+ let recordBytes = EMPTY_FRONTIER_RECORD_BYTES+ const finishPlan = (): void => {+ if (candidates.length === 0) {+ return+ }+ plans.push({+ topic: items[0].destination.topic,+ metricTopic: items[0].destination.metricTopic,+ registrableDomain: items[0].candidate.registrableDomain,+ candidates,+ reasons,+ })+ candidates = []+ reasons = []+ recordBytes = EMPTY_FRONTIER_RECORD_BYTES+ }+ for (const item of items) {+ const candidateBytes = serializeFrontierRecord([item.candidate]).length - EMPTY_FRONTIER_RECORD_BYTES+ const nextRecordBytes = recordBytes + candidateBytes + (candidates.length > 0 ? 1 : 0)+ if (+ candidates.length > 0 &&+ (candidates.length === MAX_JOBS_PER_RECORD || nextRecordBytes > MAX_RECORD_BYTES)+ ) {+ finishPlan()+ }+ recordBytes += candidateBytes + (candidates.length > 0 ? 1 : 0)+ candidates.push(item.candidate)+ reasons.push(item.reason)+ if (recordBytes > MAX_RECORD_BYTES) {+ throw new Error('one image fetch republish job exceeds the Kafka record limit')+ }+ }+ finishPlan()+ return plans+ }++ private async publishTopic(+ metricTopic: RepublishTopic,+ plans: PlannedRepublishMessage[],+ deliveryState: RepublishDeliveryState+ ): Promise<RepublishFlushResult> {+ const startedAt = process.hrtime.bigint()+ const results = await Promise.all(+ plans.map((plan) =>+ this.republishes.run({+ debugTag: plan.metricTopic,+ fn: async (): Promise<RepublishDeliveryResult> => {+ if (deliveryState.failed) {+ return 'skipped'+ }+ if (performance.now() >= this.deadlineAtMonotonicMs) {+ deliveryState.failed = true+ deliveryState.deadlineExceeded = true+ return 'skipped'+ }+ try {+ await this.producer.produce({+ topic: plan.topic,+ key: Buffer.from(plan.registrableDomain),+ value: serializeFrontierRecord(plan.candidates),+ headers: {+ [CAPTURE_TIMESTAMP_HEADER]: String(earliestFirstSeenAtMs(plan.candidates)),+ },+ })+ return 'published'+ } catch {+ deliveryState.failed = true+ return 'failed'+ }+ },+ })+ )+ )+ let failedUrls = 0+ let attemptedMessages = 0+ const attemptedRegistrableDomains = new Set<string>()+ for (let index = 0; index < plans.length; index++) {+ const plan = plans[index]+ if (results[index] === 'skipped') {+ failedUrls += plan.candidates.length+ continue+ }+ attemptedMessages+++ attemptedRegistrableDomains.add(plan.registrableDomain)+ if (results[index] === 'failed') {+ failedUrls += plan.candidates.length+ this.recordFailedPlan(plan)+ continue+ }+ for (const reason of plan.reasons) {+ ImageFetchRequestMetrics.incRepublished(reason, metricTopic === 'frontier' ? 'frontier' : 'delay')+ }+ }+ if (attemptedMessages > 0) {+ ImageFetchRequestMetrics.observeRepublishBatch(+ metricTopic,+ attemptedMessages,+ attemptedRegistrableDomains.size,+ Number(process.hrtime.bigint() - startedAt) / 1e9+ )+ }+ return { failedUrls }+ }++ private recordFailedPlan(plan: PlannedRepublishMessage): void {+ logger.warn('🌐', 'ai_research_v4_image_fetch_republish_failed', {+ topic: plan.topic,+ urls: plan.candidates.length,+ })+ for (const reason of plan.reasons) {+ ImageFetchRequestMetrics.incRepublishFailed(reason)+ }+ }+}++function earliestFirstSeenAtMs(candidates: FetchCandidate[]): number {+ let earliest = candidates[0].firstSeenAtMs+ for (const candidate of candidates) {+ earliest = Math.min(earliest, candidate.firstSeenAtMs)+ }+ return earliest+}diff --git a/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/host-budget.test.ts b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/host-budget.test.tsnew file mode 100644index 000000000000..75ef862d109b--- /dev/null+++ b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/host-budget.test.ts@@ -0,0 +1,262 @@+import { HostBudget, HostBudgetOptions } from './host-budget'++const OPTIONS: HostBudgetOptions = {+ requestsPerSecond: 1,+ burst: 2,+ maxConcurrent: 2,+ breakerFailures: 3,+ breakerCooldownMs: 60_000,+ breakerMaxCooldownMs: 600_000,+ maxTrackedRegistrableDomains: 3,+ maxTrackedOrigins: 3,+ random: () => 1,+}+const DEADLINE_MS = 1_000_000+const REGISTRABLE_DOMAIN = 'example.com'+const ORIGIN = 'https://cdn.example.com'++function budget(overrides: Partial<HostBudgetOptions> = {}): HostBudget {+ return new HostBudget({ ...OPTIONS, ...overrides })+}++describe('HostBudget', () => {+ it('does not reserve a token before a crawl-delay wait finishes', () => {+ const host = budget({ burst: 5 })+ host.setCrawlDelay(ORIGIN, 1_000, 1_000)++ const first = host.take(REGISTRABLE_DOMAIN, ORIGIN, 1_000, DEADLINE_MS)+ expect(first).toEqual({+ granted: true,+ waitMs: 0,+ waitScope: null,+ halfOpenProbe: false,+ reservedStartAtMs: 1_000,+ })+ host.markRequestStarted(+ REGISTRABLE_DOMAIN,+ ORIGIN,+ 1_000,+ first.granted ? first.reservedStartAtMs : null,+ 'image'+ )+ expect(host.take(REGISTRABLE_DOMAIN, ORIGIN, 1_000, DEADLINE_MS)).toEqual({+ granted: true,+ waitMs: 1_000,+ waitScope: 'origin_crawl_delay',+ halfOpenProbe: false,+ reservedStartAtMs: null,+ })+ expect(host.take(REGISTRABLE_DOMAIN, ORIGIN, 1_000, DEADLINE_MS)).toEqual({+ granted: true,+ waitMs: 1_000,+ waitScope: 'origin_crawl_delay',+ halfOpenProbe: false,+ reservedStartAtMs: null,+ })+ })++ it('uses the larger robots crawl delay', () => {+ const host = budget({ burst: 5 })+ host.setCrawlDelay(ORIGIN, 4_000, 1_000)++ const first = host.take(REGISTRABLE_DOMAIN, ORIGIN, 1_000, DEADLINE_MS)+ host.markRequestStarted(+ REGISTRABLE_DOMAIN,+ ORIGIN,+ 1_000,+ first.granted ? first.reservedStartAtMs : null,+ 'image'+ )++ expect(host.take(REGISTRABLE_DOMAIN, ORIGIN, 1_000, DEADLINE_MS)).toMatchObject({ waitMs: 4_000 })+ })++ it('keeps crawl delay separate for sibling origins', () => {+ const host = budget({ burst: 5 })+ host.setCrawlDelay('https://a.example.com', 4_000, 1_000)++ host.take(REGISTRABLE_DOMAIN, 'https://a.example.com', 1_000, DEADLINE_MS)++ expect(host.take(REGISTRABLE_DOMAIN, 'https://b.example.com', 1_000, DEADLINE_MS)).toMatchObject({+ granted: true,+ waitMs: 0,+ })+ })++ it('shares token capacity across sibling origins', () => {+ const host = budget({ burst: 1 })++ expect(host.take(REGISTRABLE_DOMAIN, 'https://a.example.com', 1_000, DEADLINE_MS)).toMatchObject({+ granted: true,+ waitMs: 0,+ })+ expect(host.take(REGISTRABLE_DOMAIN, 'https://b.example.com', 1_000, DEADLINE_MS)).toMatchObject({+ granted: true,+ waitMs: 1_000,+ })+ })++ it('returns a token when a reserved request cannot run', () => {+ const host = budget({ burst: 1 })+ const grant = host.take(REGISTRABLE_DOMAIN, ORIGIN, 1_000, DEADLINE_MS)+ expect(grant.granted).toBe(true)+ host.returnGrant(REGISTRABLE_DOMAIN, ORIGIN, 1_000, grant.granted ? grant.reservedStartAtMs : null)++ expect(host.take(REGISTRABLE_DOMAIN, ORIGIN, 1_000, DEADLINE_MS)).toMatchObject({ granted: true })+ })++ it('applies half-to-full jitter exponential backoff and Retry-After', () => {+ const host = budget()++ expect(host.recordTransientFailure(REGISTRABLE_DOMAIN, 1_000)).toBe(60_000)+ expect(host.recordTransientFailure(REGISTRABLE_DOMAIN, 1_000, 180_000)).toBe(180_000)+ expect(host.take(REGISTRABLE_DOMAIN, ORIGIN, 180_999, DEADLINE_MS)).toEqual({+ granted: false,+ reason: 'backoff',+ backoffReason: 'retry_after',+ waitMs: 1,+ })+ })++ it('does not shorten a future block when a concurrent request succeeds', () => {+ const host = budget()+ host.recordTransientFailure(REGISTRABLE_DOMAIN, 1_000, 180_000)+ host.recordCompletedResponse(REGISTRABLE_DOMAIN, 2_000)++ expect(host.take(REGISTRABLE_DOMAIN, ORIGIN, 2_000, DEADLINE_MS)).toEqual({+ granted: false,+ reason: 'backoff',+ backoffReason: 'retry_after',+ waitMs: 179_000,+ })+ })++ it('opens the breaker after three consecutive transient failures', () => {+ const host = budget()+ for (let failure = 0; failure < OPTIONS.breakerFailures; failure++) {+ host.recordTransientFailure(REGISTRABLE_DOMAIN, 1_000)+ }++ expect(host.take(REGISTRABLE_DOMAIN, ORIGIN, 1_000, DEADLINE_MS)).toEqual({+ granted: false,+ reason: 'breaker_open',+ waitMs: 240_000,+ })+ expect(host.blockedRegistrableDomains(1_000)).toBe(1)+ })++ it('applies a transient failure to sibling origins', () => {+ const host = budget()+ host.recordTransientFailure(REGISTRABLE_DOMAIN, 1_000)++ expect(host.take(REGISTRABLE_DOMAIN, 'https://other.example.com', 1_000, DEADLINE_MS)).toMatchObject({+ granted: false,+ reason: 'backoff',+ })+ })++ it('grants one half-open probe after the breaker cooldown', () => {+ const host = budget()+ for (let failure = 0; failure < OPTIONS.breakerFailures; failure++) {+ host.recordTransientFailure(REGISTRABLE_DOMAIN, 1_000)+ }+ const afterCooldownMs = 1_000 + 240_000++ expect(host.take(REGISTRABLE_DOMAIN, ORIGIN, afterCooldownMs, DEADLINE_MS)).toEqual({+ granted: true,+ waitMs: 0,+ waitScope: null,+ halfOpenProbe: true,+ reservedStartAtMs: afterCooldownMs,+ })+ expect(host.take(REGISTRABLE_DOMAIN, 'https://other.example.com', afterCooldownMs, DEADLINE_MS)).toEqual({+ granted: false,+ reason: 'breaker_open',+ waitMs: OPTIONS.breakerCooldownMs,+ })+ })++ it('closes the breaker after the half-open probe gets a response', () => {+ const host = budget()+ for (let failure = 0; failure < OPTIONS.breakerFailures; failure++) {+ host.recordTransientFailure(REGISTRABLE_DOMAIN, 1_000)+ }+ const afterCooldownMs = 1_000 + 240_000+ host.take(REGISTRABLE_DOMAIN, ORIGIN, afterCooldownMs, DEADLINE_MS)+ host.recordCompletedResponse(REGISTRABLE_DOMAIN, afterCooldownMs)++ expect(host.take(REGISTRABLE_DOMAIN, ORIGIN, afterCooldownMs, DEADLINE_MS)).toMatchObject({ granted: true })+ })++ it('limits concurrent connections across sibling origins', () => {+ const host = budget()+ const origins = ['https://a.example.com', 'https://b.example.com', 'https://c.example.com']+ for (const origin of origins) {+ host.take(REGISTRABLE_DOMAIN, origin, 1_000, DEADLINE_MS)+ }++ expect(host.acquireConnection(REGISTRABLE_DOMAIN, origins[0])).toBe(true)+ expect(host.acquireConnection(REGISTRABLE_DOMAIN, origins[1])).toBe(true)+ expect(host.acquireConnection(REGISTRABLE_DOMAIN, origins[2])).toBe(false)+ host.releaseConnection(REGISTRABLE_DOMAIN, origins[0])+ expect(host.acquireConnection(REGISTRABLE_DOMAIN, origins[2])).toBe(true)+ })++ it('does not evict an origin with a connection or scheduled request', () => {+ const host = budget({ maxTrackedRegistrableDomains: 2, maxTrackedOrigins: 2, maxConcurrent: 1 })+ host.take('busy.example', 'https://busy.example', 1_000, DEADLINE_MS)+ host.acquireConnection('busy.example', 'https://busy.example')+ host.requestScheduled('https://config.example', 1_000)+ host.take('config.example', 'https://config.example', 1_000, DEADLINE_MS, true)++ expect(host.take('new.example', 'https://new.example', 1_000, DEADLINE_MS)).toEqual({+ granted: false,+ reason: 'registrable_domain_map_full',+ waitMs: 0,+ })+ expect(host.acquireConnection('busy.example', 'https://busy.example')).toBe(false)+ })++ it('evicts an idle full-budget origin', () => {+ const host = budget({ maxTrackedRegistrableDomains: 1, maxTrackedOrigins: 1 })+ const grant = host.take('idle.example', 'https://idle.example', 1_000, DEADLINE_MS)+ expect(grant.granted).toBe(true)+ host.returnGrant('idle.example', 'https://idle.example', 1_000, grant.granted ? grant.reservedStartAtMs : null)++ expect(host.take('new.example', 'https://new.example', 1_000, DEADLINE_MS)).toMatchObject({ granted: true })+ expect(host.trackedRegistrableDomains).toBe(1)+ expect(host.trackedOrigins).toBe(1)+ })++ it.each([+ ['during its cooldown', 1_000 + 239_999, { granted: false, reason: 'registrable_domain_map_full' }],+ ['after its cooldown', 1_000 + 240_000, { granted: true }],+ ])('evicts an idle domain whose breaker is open only %s', (_name, nowMs, expected) => {+ const host = budget({ maxTrackedRegistrableDomains: 1 })+ for (let failure = 0; failure < OPTIONS.breakerFailures; failure++) {+ host.recordTransientFailure('failed.example', 1_000)+ }++ expect(host.take('new.example', 'https://new.example', nowMs, DEADLINE_MS)).toMatchObject(expected)+ })++ it('returns a crawl wait that extends beyond the pass deadline', () => {+ const host = budget({ burst: 5 })+ host.setCrawlDelay(ORIGIN, 600_000, 1_000)+ const first = host.take(REGISTRABLE_DOMAIN, ORIGIN, 1_000, DEADLINE_MS)+ host.markRequestStarted(+ REGISTRABLE_DOMAIN,+ ORIGIN,+ 1_000,+ first.granted ? first.reservedStartAtMs : null,+ 'image'+ )++ expect(host.take(REGISTRABLE_DOMAIN, ORIGIN, 1_000, 20_000)).toEqual({+ granted: false,+ reason: 'deadline',+ waitMs: 600_000,+ waitScope: 'origin_crawl_delay',+ })+ })+})diff --git a/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/host-budget.ts b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/host-budget.tsnew file mode 100644index 000000000000..c56722fa846b--- /dev/null+++ b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/host-budget.ts@@ -0,0 +1,454 @@+import type { ImageFetchBlockReason } from './block-reason'++export interface HostBudgetOptions {+ requestsPerSecond: number+ burst: number+ maxConcurrent: number+ breakerFailures: number+ breakerCooldownMs: number+ breakerMaxCooldownMs: number+ maxTrackedRegistrableDomains: number+ maxTrackedOrigins: number+ random?: () => number+}++export type BudgetBlockReason =+ | 'breaker_open'+ | 'backoff'+ | 'deadline'+ | 'origin_map_full'+ | 'registrable_domain_map_full'+export type BudgetWaitScope = 'origin_crawl_delay' | 'registrable_domain_rate'+export type HostBackoffReason = Extract<ImageFetchBlockReason, 'retry_after' | 'transient_backoff'>+export type BudgetGrant =+ | {+ granted: true+ waitMs: number+ waitScope: BudgetWaitScope | null+ halfOpenProbe: boolean+ reservedStartAtMs: number | null+ }+ | {+ granted: false+ reason: BudgetBlockReason+ waitMs: number+ backoffReason?: HostBackoffReason+ waitScope?: BudgetWaitScope+ }++const EVICTION_SCAN_LIMIT = 64++interface RegistrableDomainState {+ inFlight: number+ pendingGrants: number+ tokens: number+ lastRefillMs: number+ consecutiveTransientFailures: number+ blockedUntilMs: number+ backoffReason: HostBackoffReason | null+ breakerOpen: boolean+ halfOpenProbeInFlight: boolean+}++interface OriginState {+ inFlight: number+ pendingRequests: number+ lastRequestStartedAtMs: number | null+ reservedStartTimesMs: number[]+ crawlDelayMs: number+}++export class HostBudget {+ private readonly registrableDomains = new Map<string, RegistrableDomainState>()+ private readonly origins = new Map<string, OriginState>()+ private readonly random: () => number++ constructor(private readonly options: HostBudgetOptions) {+ for (const [name, value] of Object.entries(options)) {+ if (name === 'random') {+ continue+ }+ const number = value as number+ const invalid = !Number.isFinite(number) || (name === 'requestsPerSecond' ? number < 0 : number <= 0)+ if (invalid) {+ const validRange =+ name === 'requestsPerSecond'+ ? 'a finite number greater than or equal to zero'+ : 'a finite number greater than zero'+ throw new Error(`the image fetch request budget ${name} must be ${validRange}, got ${value}`)+ }+ }+ this.random = options.random ?? Math.random+ }++ public take(+ registrableDomain: string,+ origin: string,+ nowMs: number,+ deadlineMs: number,+ ignoreImageDelay = false+ ): BudgetGrant {+ const registrableDomainState = this.registrableDomainStateFor(registrableDomain, nowMs)+ if (!registrableDomainState) {+ return { granted: false, reason: 'registrable_domain_map_full', waitMs: 0 }+ }+ const originState = this.originStateFor(origin, nowMs)+ if (!originState) {+ return { granted: false, reason: 'origin_map_full', waitMs: 0 }+ }+ let halfOpenProbe = false+ if (!ignoreImageDelay) {+ if (registrableDomainState.blockedUntilMs > nowMs) {+ if (registrableDomainState.breakerOpen) {+ return {+ granted: false,+ reason: 'breaker_open',+ waitMs: registrableDomainState.blockedUntilMs - nowMs,+ }+ }+ return {+ granted: false,+ reason: 'backoff',+ waitMs: registrableDomainState.blockedUntilMs - nowMs,+ backoffReason: registrableDomainState.backoffReason ?? undefined,+ }+ }+ if (registrableDomainState.breakerOpen) {+ if (registrableDomainState.halfOpenProbeInFlight) {+ return { granted: false, reason: 'breaker_open', waitMs: this.options.breakerCooldownMs }+ }+ halfOpenProbe = true+ }+ }+ this.refill(registrableDomainState, nowMs)+ const tokenWaitMs =+ this.options.requestsPerSecond === 0 || registrableDomainState.tokens >= 1+ ? 0+ : Math.ceil(((1 - registrableDomainState.tokens) / this.options.requestsPerSecond) * 1000)+ const previousStartMs = originState.lastRequestStartedAtMs ?? Number.NEGATIVE_INFINITY+ const crawlWaitMs = ignoreImageDelay ? 0 : Math.max(0, previousStartMs + originState.crawlDelayMs - nowMs)+ const waitMs = Math.max(tokenWaitMs, crawlWaitMs)+ const waitScope: BudgetWaitScope | null =+ waitMs === 0 ? null : crawlWaitMs > tokenWaitMs ? 'origin_crawl_delay' : 'registrable_domain_rate'+ if (nowMs + waitMs > deadlineMs) {+ return { granted: false, reason: 'deadline', waitMs, waitScope: waitScope ?? undefined }+ }+ if (waitMs > 0) {+ return {+ granted: true,+ waitMs,+ waitScope,+ halfOpenProbe: false,+ reservedStartAtMs: null,+ }+ }+ if (halfOpenProbe) {+ registrableDomainState.halfOpenProbeInFlight = true+ }+ if (this.options.requestsPerSecond > 0) {+ registrableDomainState.tokens -= 1+ }+ registrableDomainState.pendingGrants += 1+ const reservedStartAtMs = ignoreImageDelay ? null : nowMs+ if (reservedStartAtMs !== null) {+ originState.reservedStartTimesMs.push(reservedStartAtMs)+ }+ return { granted: true, waitMs, waitScope, halfOpenProbe, reservedStartAtMs }+ }++ public markRequestStarted(+ registrableDomain: string,+ origin: string,+ nowMs: number,+ reservedStartAtMs: number | null,+ requestKind: 'configuration' | 'image'+ ): void {+ const registrableDomainState = this.registrableDomains.get(registrableDomain)+ if (registrableDomainState) {+ registrableDomainState.pendingGrants = Math.max(0, registrableDomainState.pendingGrants - 1)+ }+ const originState = this.origins.get(origin)+ if (originState && requestKind === 'image') {+ this.removeReservation(originState, reservedStartAtMs)+ originState.lastRequestStartedAtMs = nowMs+ }+ }++ public blockedForMs(registrableDomain: string, nowMs: number): number {+ const state = this.registrableDomains.get(registrableDomain)+ if (!state) {+ return 0+ }+ if (state.breakerOpen && state.halfOpenProbeInFlight && state.blockedUntilMs <= nowMs) {+ return this.options.breakerCooldownMs+ }+ return Math.max(0, state.blockedUntilMs - nowMs)+ }++ public returnGrant(+ registrableDomain: string,+ origin: string,+ nowMs: number,+ reservedStartAtMs: number | null,+ halfOpenProbe = false+ ): void {+ const originState = this.origins.get(origin)+ if (originState) {+ this.removeReservation(originState, reservedStartAtMs)+ }+ const registrableDomainState = this.registrableDomains.get(registrableDomain)+ if (!registrableDomainState) {+ return+ }+ registrableDomainState.pendingGrants = Math.max(0, registrableDomainState.pendingGrants - 1)+ if (this.options.requestsPerSecond > 0) {+ this.refill(registrableDomainState, nowMs)+ registrableDomainState.tokens = Math.min(this.options.burst, registrableDomainState.tokens + 1)+ }+ if (halfOpenProbe && registrableDomainState.breakerOpen) {+ registrableDomainState.halfOpenProbeInFlight = false+ }+ }++ public acquireConnection(registrableDomain: string, origin: string): boolean {+ const registrableDomainState = this.registrableDomains.get(registrableDomain)+ const originState = this.origins.get(origin)+ if (!registrableDomainState || !originState || registrableDomainState.inFlight >= this.options.maxConcurrent) {+ return false+ }+ registrableDomainState.inFlight += 1+ originState.inFlight += 1+ return true+ }++ public availableConnections(registrableDomain: string, connectionLimit = this.options.maxConcurrent): number {+ const inFlight = this.registrableDomains.get(registrableDomain)?.inFlight ?? 0+ return Math.max(0, Math.min(this.options.maxConcurrent, connectionLimit) - inFlight)+ }++ public releaseConnection(registrableDomain: string, origin: string): void {+ const registrableDomainState = this.registrableDomains.get(registrableDomain)+ if (registrableDomainState) {+ registrableDomainState.inFlight = Math.max(0, registrableDomainState.inFlight - 1)+ }+ const originState = this.origins.get(origin)+ if (originState) {+ originState.inFlight = Math.max(0, originState.inFlight - 1)+ }+ }++ public requestScheduled(origin: string, nowMs: number): boolean {+ const state = this.originStateFor(origin, nowMs)+ if (!state) {+ return false+ }+ state.pendingRequests += 1+ return true+ }++ public requestFinished(origin: string): void {+ const state = this.origins.get(origin)+ if (state) {+ state.pendingRequests = Math.max(0, state.pendingRequests - 1)+ }+ }++ public setCrawlDelay(origin: string, crawlDelayMs: number, nowMs: number): boolean {+ const state = this.originStateFor(origin, nowMs)+ if (!state) {+ return false+ }+ state.crawlDelayMs = Math.max(0, crawlDelayMs)+ return true+ }++ public originCrawlDelayMs(origin: string): number {+ return this.origins.get(origin)?.crawlDelayMs ?? 0+ }++ public originNextImageStartAtMs(origin: string): number {+ const state = this.origins.get(origin)+ if (!state || state.lastRequestStartedAtMs === null) {+ return 0+ }+ return state.lastRequestStartedAtMs + state.crawlDelayMs+ }++ public recordTransientFailure(registrableDomain: string, nowMs: number, retryAfterMs?: number): number {+ const state = this.registrableDomainStateFor(registrableDomain, nowMs)+ if (!state) {+ return this.options.breakerCooldownMs+ }+ state.consecutiveTransientFailures += 1+ const maximumDelayMs = Math.min(+ this.options.breakerCooldownMs * 2 ** (state.consecutiveTransientFailures - 1),+ this.options.breakerMaxCooldownMs+ )+ const minimumDelayMs = Math.ceil(maximumDelayMs / 2)+ const jitteredDelayMs = Math.min(+ maximumDelayMs,+ minimumDelayMs + Math.floor(this.random() * (maximumDelayMs - minimumDelayMs + 1))+ )+ const delayMs = Math.max(jitteredDelayMs, retryAfterMs ?? 0)+ const blockedUntilMs = nowMs + delayMs+ if (blockedUntilMs >= state.blockedUntilMs) {+ state.backoffReason = retryAfterMs === undefined ? 'transient_backoff' : 'retry_after'+ }+ state.blockedUntilMs = Math.max(state.blockedUntilMs, blockedUntilMs)+ if (state.consecutiveTransientFailures >= this.options.breakerFailures) {+ state.breakerOpen = true+ state.halfOpenProbeInFlight = false+ }+ return delayMs+ }++ public recordCompletedResponse(registrableDomain: string, nowMs: number): void {+ const state = this.registrableDomains.get(registrableDomain)+ if (!state) {+ return+ }+ state.consecutiveTransientFailures = 0+ state.blockedUntilMs = Math.max(state.blockedUntilMs, nowMs)+ if (state.blockedUntilMs <= nowMs) {+ state.backoffReason = null+ }+ state.breakerOpen = false+ state.halfOpenProbeInFlight = false+ }++ public get trackedRegistrableDomains(): number {+ return this.registrableDomains.size+ }++ public get trackedOrigins(): number {+ return this.origins.size+ }++ public evictedWhileBlocked = 0++ public blockedRegistrableDomains(nowMs: number): number {+ let blocked = 0+ for (const state of this.registrableDomains.values()) {+ if (state.blockedUntilMs > nowMs || state.breakerOpen) {+ blocked += 1+ }+ }+ return blocked+ }++ private refill(state: RegistrableDomainState, nowMs: number): void {+ const elapsedMs = Math.max(0, nowMs - state.lastRefillMs)+ state.lastRefillMs = nowMs+ state.tokens = Math.min(this.options.burst, state.tokens + (elapsedMs / 1000) * this.options.requestsPerSecond)+ }++ private registrableDomainStateFor(registrableDomain: string, nowMs: number): RegistrableDomainState | undefined {+ const existing = this.registrableDomains.get(registrableDomain)+ if (existing) {+ this.registrableDomains.delete(registrableDomain)+ this.registrableDomains.set(registrableDomain, existing)+ return existing+ }+ if (!this.evictRegistrableDomainIfFull(nowMs)) {+ return undefined+ }+ const state: RegistrableDomainState = {+ inFlight: 0,+ pendingGrants: 0,+ tokens: this.options.burst,+ lastRefillMs: nowMs,+ consecutiveTransientFailures: 0,+ blockedUntilMs: 0,+ backoffReason: null,+ breakerOpen: false,+ halfOpenProbeInFlight: false,+ }+ this.registrableDomains.set(registrableDomain, state)+ return state+ }++ private originStateFor(origin: string, nowMs: number): OriginState | undefined {+ const existing = this.origins.get(origin)+ if (existing) {+ this.origins.delete(origin)+ this.origins.set(origin, existing)+ return existing+ }+ if (!this.evictOriginIfFull(nowMs)) {+ return undefined+ }+ const state: OriginState = {+ inFlight: 0,+ pendingRequests: 0,+ lastRequestStartedAtMs: null,+ reservedStartTimesMs: [],+ crawlDelayMs: 0,+ }+ this.origins.set(origin, state)+ return state+ }++ private evictRegistrableDomainIfFull(nowMs: number): boolean {+ if (this.registrableDomains.size < this.options.maxTrackedRegistrableDomains) {+ return true+ }+ for (let scanned = 0; scanned < EVICTION_SCAN_LIMIT; scanned++) {+ const oldest = this.registrableDomains.entries().next().value as+ | [string, RegistrableDomainState]+ | undefined+ if (!oldest) {+ return true+ }+ const [registrableDomain, state] = oldest+ this.refill(state, nowMs)+ const eligible =+ state.inFlight === 0 &&+ state.pendingGrants === 0 &&+ state.blockedUntilMs <= nowMs &&+ !state.halfOpenProbeInFlight &&+ state.tokens >= this.options.burst+ if (eligible) {+ this.registrableDomains.delete(registrableDomain)+ return true+ }+ this.registrableDomains.delete(registrableDomain)+ this.registrableDomains.set(registrableDomain, state)+ }+ return false+ }++ private evictOriginIfFull(nowMs: number): boolean {+ if (this.origins.size < this.options.maxTrackedOrigins) {+ return true+ }+ for (let scanned = 0; scanned < EVICTION_SCAN_LIMIT; scanned++) {+ const oldest = this.origins.entries().next().value as [string, OriginState] | undefined+ if (!oldest) {+ return true+ }+ const [origin, state] = oldest+ const eligible =+ state.inFlight === 0 &&+ state.pendingRequests === 0 &&+ state.reservedStartTimesMs.length === 0 &&+ (state.lastRequestStartedAtMs === null || state.lastRequestStartedAtMs + state.crawlDelayMs <= nowMs)+ if (eligible) {+ this.origins.delete(origin)+ return true+ }+ this.origins.delete(origin)+ this.origins.set(origin, state)+ }+ return false+ }++ private removeReservation(state: OriginState, reservedStartAtMs: number | null): void {+ if (reservedStartAtMs === null) {+ return+ }+ const index = state.reservedStartTimesMs.indexOf(reservedStartAtMs)+ if (index >= 0) {+ state.reservedStartTimesMs.splice(index, 1)+ }+ }+}diff --git a/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/image-fetch-batch-joiner.test.ts b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/image-fetch-batch-joiner.test.tsnew file mode 100644index 000000000000..afd17d066a46--- /dev/null+++ b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/image-fetch-batch-joiner.test.ts@@ -0,0 +1,171 @@+import { Message } from 'node-rdkafka'+import { register } from 'prom-client'++import { IMAGE_FETCH_BATCH_JOIN_TIMEOUT_MS, ImageFetchBatchJoiner } from './image-fetch-batch-joiner'++function message(partition: number): Message {+ return {+ topic: 'session_replay_image_fetch',+ partition,+ offset: 1,+ value: Buffer.from(`partition-${partition}`),+ size: 11,+ }+}++function deferred(): { promise: Promise<void>; resolve: () => void; reject: (error: unknown) => void } {+ let resolve!: () => void+ let reject!: (error: unknown) => void+ const promise = new Promise<void>((done, fail) => {+ resolve = done+ reject = fail+ })+ return { promise, resolve, reject }+}++describe('ImageFetchBatchJoiner', () => {+ beforeEach(() => {+ jest.useFakeTimers()+ register.resetMetrics()+ })+ afterEach(() => jest.useRealTimers())++ it.each([1, 2, 16])('joins batches from %i consumers before processing', async (consumerCount) => {+ const pass = deferred()+ const processBatch = jest.fn(() => pass.promise)+ const joiner = new ImageFetchBatchJoiner(consumerCount, processBatch)++ const messages = Array.from({ length: consumerCount }, (_, partition) => message(partition))+ const pending = messages.slice(0, -1).map((item) => joiner.handleBatch([item]))+ expect(processBatch).not.toHaveBeenCalled()+ pending.push(joiner.handleBatch([messages[consumerCount - 1]]))++ const batches = await Promise.all(pending)+ expect(processBatch).toHaveBeenCalledTimes(1)+ expect(processBatch).toHaveBeenCalledWith(messages)+ expect(batches.every((batch) => batch?.backgroundTask === batches[0]?.backgroundTask)).toBe(true)+ const completed = jest.fn()+ const completion = Promise.all(batches.map((batch) => batch?.backgroundTask ?? Promise.resolve())).then(+ completed+ )+ await jest.advanceTimersByTimeAsync(0)+ expect(completed).not.toHaveBeenCalled()+ pass.resolve()+ await completion+ })++ it.each([2, 16])(+ 'processes a partial batch with a target of %i when the join window ends',+ async (consumerCount) => {+ const processBatch = jest.fn(() => Promise.resolve())+ const joiner = new ImageFetchBatchJoiner(consumerCount, processBatch)++ const pending = joiner.handleBatch([message(0)])+ await jest.advanceTimersByTimeAsync(IMAGE_FETCH_BATCH_JOIN_TIMEOUT_MS - 1)+ expect(processBatch).not.toHaveBeenCalled()+ await jest.advanceTimersByTimeAsync(1)++ await (+ await pending+ )?.backgroundTask+ expect(processBatch).toHaveBeenCalledWith([message(0)])+ }+ )++ it('rejects active and pending consumers when processing fails', async () => {+ const error = new Error('fetch pass failed')+ const pass = deferred()+ const joiner = new ImageFetchBatchJoiner(2, () => pass.promise)++ const first = joiner.handleBatch([message(0)])+ await jest.advanceTimersByTimeAsync(IMAGE_FETCH_BATCH_JOIN_TIMEOUT_MS)+ const firstCompletion = (await first)?.backgroundTask+ const second = joiner.handleBatch([message(1)])+ const active = register.getSingleMetric('ai_research_v4_image_fetch_stage_active')!+ expect((await active.get()).values).toEqual(+ expect.arrayContaining([+ expect.objectContaining({ labels: { stage: 'consumer_join' }, value: 1 }),+ expect.objectContaining({ labels: { stage: 'consumer_process' }, value: 1 }),+ ])+ )+ const rejected = Promise.all([expect(firstCompletion).rejects.toBe(error), expect(second).rejects.toBe(error)])+ pass.reject(error)+ await rejected+ await expect(joiner.handleBatch([message(2)])).rejects.toBe(error)+ expect((await active.get()).values).toEqual(+ expect.arrayContaining([+ expect.objectContaining({ labels: { stage: 'consumer_join' }, value: 0 }),+ expect.objectContaining({ labels: { stage: 'consumer_process' }, value: 0 }),+ ])+ )+ })++ it('does not hold a later partial batch behind active processing', async () => {+ const firstPass = deferred()+ const secondPass = deferred()+ const processBatch = jest+ .fn()+ .mockImplementationOnce(() => firstPass.promise)+ .mockImplementationOnce(() => secondPass.promise)+ const joiner = new ImageFetchBatchJoiner(2, processBatch)++ const first = joiner.handleBatch([message(0)])+ await jest.advanceTimersByTimeAsync(IMAGE_FETCH_BATCH_JOIN_TIMEOUT_MS)+ const second = joiner.handleBatch([message(1)])+ await jest.advanceTimersByTimeAsync(IMAGE_FETCH_BATCH_JOIN_TIMEOUT_MS)+ expect(processBatch).toHaveBeenCalledTimes(2)++ secondPass.resolve()+ await (+ await second+ )?.backgroundTask+ firstPass.resolve()+ await (+ await first+ )?.backgroundTask+ expect(processBatch).toHaveBeenLastCalledWith([message(1)])+ })++ it('keeps a fast consumer out of the same group twice while overlapping processing', async () => {+ const firstPass = deferred()+ const secondPass = deferred()+ const processBatch = jest+ .fn()+ .mockImplementationOnce(() => firstPass.promise)+ .mockImplementationOnce(() => secondPass.promise)+ const joiner = new ImageFetchBatchJoiner(2, processBatch)+ const secondFromFastConsumer = jest.fn(() => joiner.handleBatch([message(0)]))++ const firstFast = joiner.handleBatch([message(0)])+ const secondFast = firstFast.then(secondFromFastConsumer)+ await jest.advanceTimersByTimeAsync(0)+ expect(secondFromFastConsumer).not.toHaveBeenCalled()++ const firstSlow = joiner.handleBatch([message(1)])+ await firstSlow+ await jest.advanceTimersByTimeAsync(0)+ expect(secondFromFastConsumer).toHaveBeenCalledTimes(1)+ expect(processBatch).toHaveBeenCalledTimes(1)++ const secondSlow = joiner.handleBatch([message(1)])+ const secondBatches = await Promise.all([secondFast, secondSlow])+ expect(processBatch.mock.calls).toEqual([[[message(0), message(1)]], [[message(0), message(1)]]])+ const active = register.getSingleMetric('ai_research_v4_image_fetch_stage_active')!+ expect((await active.get()).values).toEqual(+ expect.arrayContaining([expect.objectContaining({ labels: { stage: 'consumer_process' }, value: 4 })])+ )++ secondPass.resolve()+ await Promise.all(secondBatches.map((batch) => batch?.backgroundTask ?? Promise.resolve()))+ expect((await active.get()).values).toEqual(+ expect.arrayContaining([expect.objectContaining({ labels: { stage: 'consumer_process' }, value: 2 })])+ )+ firstPass.resolve()+ await (+ await firstFast+ )?.backgroundTask+ expect((await active.get()).values).toEqual(+ expect.arrayContaining([expect.objectContaining({ labels: { stage: 'consumer_process' }, value: 0 })])+ )+ })+})diff --git a/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/image-fetch-batch-joiner.ts b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/image-fetch-batch-joiner.tsnew file mode 100644index 000000000000..7da871ec9539--- /dev/null+++ b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/image-fetch-batch-joiner.ts@@ -0,0 +1,128 @@+import { Message } from 'node-rdkafka'++import { ImageFetchProcessingMetrics, ProcessingStageTimer } from './processing-metrics'++export const IMAGE_FETCH_BATCH_JOIN_TIMEOUT_MS = 500+export const MAX_IMAGE_FETCH_BATCHES_PER_PASS = 16++type BatchProcessor = (messages: Message[]) => Promise<void>++type DispatchedBatch = { backgroundTask: Promise<void> }++type BatchWaiter = {+ resolve: (batch: DispatchedBatch) => void+ reject: (error: unknown) => void+ timer: ProcessingStageTimer+}++type PendingBatchGroup = {+ batches: Message[][]+ waiters: BatchWaiter[]+ timeout?: NodeJS.Timeout+}++export function assertImageFetchBatchTarget(targetBatchCount: number): void {+ if (+ !Number.isSafeInteger(targetBatchCount) ||+ targetBatchCount < 1 ||+ targetBatchCount > MAX_IMAGE_FETCH_BATCHES_PER_PASS+ ) {+ throw new Error(`image fetch batch target must be an integer between 1 and ${MAX_IMAGE_FETCH_BATCHES_PER_PASS}`)+ }+}++export class ImageFetchBatchJoiner {+ private pendingGroup?: PendingBatchGroup+ private readonly activeProcessing = new Set<Promise<void>>()+ private failed = false+ private failure: unknown++ constructor(+ private readonly targetBatchCount: number,+ private readonly processBatch: BatchProcessor+ ) {+ assertImageFetchBatchTarget(targetBatchCount)+ }++ public handleBatch(messages: Message[]): Promise<DispatchedBatch | void> {+ if (messages.length === 0) {+ return Promise.resolve()+ }+ if (this.failed) {+ return Promise.reject(this.failure)+ }++ return new Promise<DispatchedBatch>((resolve, reject) => {+ const group = this.pendingGroup ?? this.createPendingGroup()+ group.batches.push(messages)+ group.waiters.push({ resolve, reject, timer: ImageFetchProcessingMetrics.start('consumer_join') })+ if (group.batches.length >= this.targetBatchCount) {+ this.dispatch(group)+ }+ })+ }++ public async waitForProcessing(): Promise<void> {+ await Promise.allSettled(this.activeProcessing)+ }++ private createPendingGroup(): PendingBatchGroup {+ const group: PendingBatchGroup = { batches: [], waiters: [] }+ group.timeout = setTimeout(() => this.dispatch(group), IMAGE_FETCH_BATCH_JOIN_TIMEOUT_MS)+ this.pendingGroup = group+ return group+ }++ private dispatch(group: PendingBatchGroup): void {+ if (this.pendingGroup !== group) {+ return+ }+ this.pendingGroup = undefined+ if (group.timeout) {+ clearTimeout(group.timeout)+ }++ ImageFetchProcessingMetrics.joinedBatches.observe(group.batches.length)+ for (const { timer } of group.waiters) {+ timer.move('consumer_process')+ }+ const processing = (async () => {+ if (this.failed) {+ throw this.failure+ }+ try {+ await this.processBatch(group.batches.flat())+ } catch (error) {+ this.fail(error)+ throw error+ }+ })().finally(() => {+ group.waiters.forEach(({ timer }) => timer.finish())+ this.activeProcessing.delete(processing)+ })+ this.activeProcessing.add(processing)+ for (const { resolve } of group.waiters) {+ resolve({ backgroundTask: processing })+ }+ }++ private fail(error: unknown): void {+ if (!this.failed) {+ this.failed = true+ this.failure = error+ }++ const pendingGroup = this.pendingGroup+ if (!pendingGroup) {+ return+ }+ this.pendingGroup = undefined+ if (pendingGroup.timeout) {+ clearTimeout(pendingGroup.timeout)+ }+ for (const { reject, timer } of pendingGroup.waiters) {+ timer.finish()+ reject(this.failure)+ }+ }+}diff --git a/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/image-fetcher.test.ts b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/image-fetcher.test.tsnew file mode 100644index 000000000000..aac181b2bc1f--- /dev/null+++ b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/image-fetcher.test.ts@@ -0,0 +1,619 @@+import { ResolutionError, SecureRequestError, StreamedResponse, fetchStreamed } from '~/common/utils/request'++import { HttpImageFetcher, ImageFetchOptions, RedirectPolicy } from './image-fetcher'+import { ImageFetchRequestMetrics } from './metrics'+import { WebBotAuthRequestSigner } from './web-bot-auth'++jest.mock('~/common/utils/request', () => ({+ ...jest.requireActual('~/common/utils/request'),+ fetchStreamed: jest.fn(),+}))++const fetchStreamedMock = fetchStreamed as jest.MockedFunction<typeof fetchStreamed>++const PNG = Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a, 0x00])+const GIF = Buffer.from('GIF89a-and-then-some')++const OPTIONS: ImageFetchOptions = {+ maxBytes: 1000,+ timeoutMs: 5000,+ maxRedirects: 3,+ isDifferentOrigin: () => false,+ scheduleRequest: async (_url, _deadlineMs, request) => ({ ran: true as const, value: await request() }),+ checkRedirectPolicy: () => Promise.resolve({ allowed: true as const, tdmrepReservation: false }),+ tdmrepReservation: false,+}++function respond(+ status: number,+ headers: Record<string, string>,+ body: { bytes: Buffer; overLimit: boolean } = { bytes: Buffer.alloc(0), overLimit: false }+): StreamedResponse {+ return {+ status,+ headers,+ headerLines: Object.entries(headers).map(([name, value]) => ({ name, value })),+ read: jest.fn().mockResolvedValue(body),+ discard: jest.fn(),+ }+}++function image(bytes: Buffer, contentType: string, extraHeaders: Record<string, string> = {}): StreamedResponse {+ return respond(200, { 'content-type': contentType, ...extraHeaders }, { bytes, overLimit: false })+}++const NOOP_SIGNER: WebBotAuthRequestSigner = { headersForGet: () => ({}) }+/** Everything public by default, so a test that cares about the policy says so. */+const fetcher = (+ policy: Partial<RedirectPolicy> = {},+ webBotAuthSigner: WebBotAuthRequestSigner = NOOP_SIGNER+): HttpImageFetcher =>+ new HttpImageFetcher({ maxUrlLength: 2048, isPublicHost: () => true, ...policy }, webBotAuthSigner)++describe('HttpImageFetcher', () => {+ beforeEach(() => {+ fetchStreamedMock.mockReset()+ })+ afterEach(() => jest.restoreAllMocks())++ it('opts every image request into HTTP/2', async () => {+ fetchStreamedMock.mockResolvedValue(image(PNG, 'image/png'))++ await fetcher().fetch('https://cdn.example.com/a.png', OPTIONS)++ expect(fetchStreamedMock).toHaveBeenCalledWith(+ 'https://cdn.example.com/a.png',+ expect.objectContaining({ allowH2: true })+ )+ })++ it('identifies every request as PostHogImageFetcherBot referred by us.posthog.com', async () => {+ fetchStreamedMock.mockResolvedValue(image(PNG, 'image/png'))++ await fetcher().fetch('https://cdn.example.com/a.png', OPTIONS)++ expect(fetchStreamedMock).toHaveBeenCalledWith(+ 'https://cdn.example.com/a.png',+ expect.objectContaining({+ headers: expect.objectContaining({+ 'user-agent':+ 'PostHogImageFetcherBot/1.0 (+https://posthog.com/docs/ai-research/image-fetcher-bot)',+ referer: 'https://us.posthog.com/',+ }),+ })+ )+ })++ it('attributes image requests to the source partition', async () => {+ const observeRequest = jest.spyOn(ImageFetchRequestMetrics, 'observeRequest')+ fetchStreamedMock.mockResolvedValue(image(PNG, 'image/png'))++ await fetcher().fetch('https://cdn.example.com/a.png', { ...OPTIONS, sourcePartitions: [7, 42] })++ expect(observeRequest).toHaveBeenCalledWith('2xx', expect.any(Number), [7, 42])+ })++ it('returns the bytes of an image whose payload matches its declared type', async () => {+ fetchStreamedMock.mockResolvedValue(image(PNG, 'image/png; charset=binary'))++ const result = await fetcher().fetch('https://cdn.example.com/a.png', OPTIONS)++ expect(result).toMatchObject({+ outcome: 'ok',+ status: 200,+ bytes: PNG,+ contentType: 'image/png',+ redirects: 0,+ currentUrl: 'https://cdn.example.com/a.png',+ })+ })++ it('uses an ETag validator and accepts the resulting 304', async () => {+ fetchStreamedMock.mockResolvedValue(respond(304, { etag: '"v1"' }))++ const result = await fetcher().fetch('https://cdn.example.com/a.png', {+ ...OPTIONS,+ cache: {+ requestTimeMs: Date.now() - 1_000,+ responseTimeMs: Date.now() - 500,+ etag: '"v1"',+ lastModified: 'yesterday',+ },+ })++ expect(fetchStreamedMock.mock.calls[0][1].headers).toMatchObject({ 'if-none-match': '"v1"' })+ expect(fetchStreamedMock.mock.calls[0][1].headers).not.toHaveProperty('if-modified-since')+ expect(result).toMatchObject({ outcome: 'not_modified', status: 304 })+ })++ it('does not accept an unsolicited 304', async () => {+ fetchStreamedMock.mockResolvedValue(respond(304, {}))++ const result = await fetcher().fetch('https://cdn.example.com/a.png', OPTIONS)++ expect(result).toMatchObject({ outcome: 'unexpected_status', status: 304 })+ })++ it('does not send an original URL validator to a redirect target', async () => {+ fetchStreamedMock+ .mockResolvedValueOnce(respond(302, { location: '/moved.png' }))+ .mockResolvedValueOnce(image(PNG, 'image/png'))++ await fetcher().fetch('https://cdn.example.com/a.png', {+ ...OPTIONS,+ cache: {+ requestTimeMs: Date.now() - 1_000,+ responseTimeMs: Date.now() - 500,+ etag: '"v1"',+ },+ })++ expect(fetchStreamedMock.mock.calls[0][1].headers).toMatchObject({ 'if-none-match': '"v1"' })+ expect(fetchStreamedMock.mock.calls[1][1].headers).not.toHaveProperty('if-none-match')+ })++ it.each([+ ['a type that is not an image', 'text/html', PNG, 'not_image'],+ ['a type outside the raster set', 'image/svg+xml', PNG, 'not_image'],+ ['the BMP format', 'image/bmp', Buffer.from('BM'), 'not_image'],+ ['the AVIF format', 'image/avif', Buffer.from('\x00\x00\x00\x18ftypavif', 'binary'), 'not_image'],+ ['a payload that is not the declared format', 'image/gif', PNG, 'ok'],+ ['a payload of the wrong raster format', 'image/png', GIF, 'ok'],+ ])('handles %s', async (_name, contentType, bytes, outcome) => {+ fetchStreamedMock.mockResolvedValue(image(bytes, contentType))++ const result = await fetcher().fetch('https://cdn.example.com/a', OPTIONS)++ expect(result).toMatchObject({ outcome })+ })++ it('refuses a declared size over the limit without reading the body', async () => {+ const response = image(PNG, 'image/png', { 'content-length': '99999' })+ fetchStreamedMock.mockResolvedValue(response)++ const result = await fetcher().fetch('https://cdn.example.com/a.png', OPTIONS)++ expect(result).toMatchObject({ outcome: 'too_large' })+ expect(response.read).not.toHaveBeenCalled()+ expect(response.discard).toHaveBeenCalled()+ })++ it('refuses a body that passes the limit while it is being read', async () => {+ // The case an origin reaches when it declares no length. No check before the body catches it.+ fetchStreamedMock.mockResolvedValue(+ respond(200, { 'content-type': 'image/png' }, { bytes: Buffer.alloc(0), overLimit: true })+ )++ const result = await fetcher().fetch('https://cdn.example.com/a.png', OPTIONS)++ expect(result).toMatchObject({ outcome: 'too_large' })+ })++ it.each([+ [404, 'not_found'],+ [410, 'not_found'],+ [403, 'forbidden'],+ [429, 'rate_limited'],+ [503, 'server_error'],+ [500, 'server_error'],+ [418, 'unexpected_status'],+ ])('maps status %s to %s', async (status, outcome) => {+ fetchStreamedMock.mockResolvedValue(respond(status, {}))++ const result = await fetcher().fetch('https://cdn.example.com/a.png', OPTIONS)++ expect(result).toMatchObject({ outcome, status })+ })++ it.each([+ ['a count of seconds', '120', 120_000],+ ['an HTTP date', new Date(Date.now() + 90_000).toUTCString(), 90_000],+ ])('reads a Retry-After header given as %s', async (_name, header, expectedMs) => {+ fetchStreamedMock.mockResolvedValue(respond(429, { 'retry-after': header }))++ const result = await fetcher().fetch('https://cdn.example.com/a.png', OPTIONS)++ // The parser resolves a date against the clock, so the range allows for the test's own runtime.+ expect(result.retryAfterMs).toBeGreaterThan(expectedMs - 2000)+ expect(result.retryAfterMs).toBeLessThanOrEqual(expectedMs)+ })++ it('uses the longest valid value from repeated Retry-After field lines', async () => {+ const response = respond(429, {})+ response.headerLines = [+ { name: 'retry-after', value: '10' },+ { name: 'retry-after', value: 'invalid' },+ { name: 'retry-after', value: '30' },+ ]+ fetchStreamedMock.mockResolvedValue(response)++ const result = await fetcher().fetch('https://cdn.example.com/a.png', OPTIONS)++ expect(result.retryAfterMs).toBe(30_000)+ })++ it.each([408, 425])('ignores Retry-After on status %s', async (status) => {+ fetchStreamedMock.mockResolvedValue(respond(status, { 'retry-after': '120' }))++ const result = await fetcher().fetch('https://cdn.example.com/a.png', OPTIONS)++ expect(result).toMatchObject({ outcome: 'rate_limited', status })+ expect(result.retryAfterMs).toBeUndefined()+ })++ it('follows a redirect and reports how many hops it took', async () => {+ fetchStreamedMock+ .mockResolvedValueOnce(respond(302, { location: '/moved.png' }))+ .mockResolvedValueOnce(image(PNG, 'image/png'))++ const result = await fetcher().fetch('https://cdn.example.com/a.png', OPTIONS)++ expect(result).toMatchObject({ outcome: 'ok', redirects: 1 })+ expect(fetchStreamedMock.mock.calls[1][0]).toBe('https://cdn.example.com/moved.png')+ })++ it('refuses a redirect with repeated Location field lines', async () => {+ const response = respond(302, {})+ response.headerLines = [+ { name: 'location', value: '/first.png' },+ { name: 'location', value: '/second.png' },+ ]+ fetchStreamedMock.mockResolvedValue(response)++ const result = await fetcher().fetch('https://cdn.example.com/a.png', OPTIONS)++ expect(result).toMatchObject({ outcome: 'bad_redirect' })+ expect(fetchStreamedMock).toHaveBeenCalledTimes(1)+ })++ it('signs each redirect hop for its target URL', async () => {+ fetchStreamedMock+ .mockResolvedValueOnce(respond(302, { location: 'https://images.example.net/moved.png' }))+ .mockResolvedValueOnce(image(PNG, 'image/png'))+ const headersForGet = jest+ .fn()+ .mockReturnValueOnce({ signature: 'first-hop-signature' })+ .mockReturnValueOnce({ signature: 'second-hop-signature' })++ await fetcher({}, { headersForGet }).fetch('https://cdn.example.com/a.png', OPTIONS)++ expect(headersForGet.mock.calls).toEqual([+ ['https://cdn.example.com/a.png'],+ ['https://images.example.net/moved.png'],+ ])+ expect(fetchStreamedMock.mock.calls[0][1].headers).toMatchObject({ signature: 'first-hop-signature' })+ expect(fetchStreamedMock.mock.calls[1][1].headers).toMatchObject({ signature: 'second-hop-signature' })+ })++ it.each([+ ['it leaves HTTP for another scheme', 'javascript:alert(1)'],+ ['it carries credentials for the next hop', 'https://user:[email protected]/a.png'],+ ['it downgrades from HTTPS to plain HTTP', 'http://other.example/a.png'],+ ])('refuses a redirect when %s', async (_name, location) => {+ fetchStreamedMock.mockResolvedValue(respond(302, { location }))++ const result = await fetcher().fetch('https://cdn.example.com/a.png', OPTIONS)++ expect(result).toMatchObject({ outcome: 'bad_redirect' })+ expect(fetchStreamedMock).toHaveBeenCalledTimes(1)+ })++ it('refuses a downgrade when the first hop named its scheme in capitals (requirement 9)', async () => {+ // A record can carry `HTTPS://`, because the parser validates a parsed copy and stores the+ // string it was given. A guard that reads the raw string misses the capitals and follows+ // the hop in clear text. `URL.protocol` is lower case whatever the URL used.+ fetchStreamedMock.mockResolvedValue(respond(302, { location: 'http://cdn.example.com/b.png' }))++ const result = await fetcher().fetch('HTTPS://cdn.example.com/a.png', OPTIONS)++ expect(result).toMatchObject({ outcome: 'bad_redirect' })+ expect(fetchStreamedMock).toHaveBeenCalledTimes(1)+ })++ it.each([+ ['a host the collector would have refused', 'https://internal.corp/a.png', { isPublicHost: () => false }],+ ['a target past the length limit', `https://cdn.example.com/${'a'.repeat(300)}.png`, { maxUrlLength: 100 }],+ ['a port the scheme does not own', 'https://cdn.example.com:11211/a.png', {}],+ ])('refuses a redirect to %s (requirement 8)', async (_name, location, policy) => {+ // The collector applied these checks to the first candidate. A redirect target has passed+ // none of them, so a hop could otherwise reach a name that resolves only inside a network.+ fetchStreamedMock.mockResolvedValue(respond(302, { location }))++ const result = await fetcher(policy).fetch('https://cdn.example.com/a.png', OPTIONS)++ expect(result).toMatchObject({ outcome: 'bad_redirect' })+ expect(fetchStreamedMock).toHaveBeenCalledTimes(1)+ })++ it('hands off an offsite target that arrives at the redirect limit (requirement 7)', async () => {+ // The limit bounds the hops this request follows itself. Nobody here follows a target for+ // another operator, so it goes back to Kafka and costs one hop rather than being written off+ // with hops left.+ fetchStreamedMock.mockResolvedValue(respond(302, { location: 'https://img.other.net/a.png' }))++ const result = await fetcher().fetch('https://cdn.example.com/a.png', {+ ...OPTIONS,+ maxRedirects: 0,+ isDifferentOrigin: () => true,+ })++ expect(result).toMatchObject({+ outcome: 'redirect_offsite',+ redirectTarget: { url: 'https://img.other.net/a.png', host: 'img.other.net' },+ })+ })++ it('stops at the redirect limit for a target on the same origin', async () => {+ fetchStreamedMock.mockResolvedValue(respond(302, { location: 'https://cdn.example.com/b.png' }))++ const result = await fetcher().fetch('https://cdn.example.com/a.png', { ...OPTIONS, maxRedirects: 0 })++ expect(result).toMatchObject({+ outcome: 'redirect_continuation',+ redirectTarget: { url: 'https://cdn.example.com/b.png', host: 'cdn.example.com' },+ })+ })++ it('defers a redirect whose target has no budget left, so nothing is written to the crawl history for it', async () => {+ // `bad_redirect` is terminal. A CDN in cooldown would otherwise suppress every image behind+ // it for the crawl history TTL, because the lane read a momentary budget as a property of+ // the URL.+ fetchStreamedMock.mockResolvedValue(respond(302, { location: 'https://cdn.example.net/a.png' }))++ let requestNumber = 0+ const result = await fetcher().fetch('https://cdn.example.com/a.png', {+ ...OPTIONS,+ scheduleRequest: async (_url, _deadlineMs, request) => {+ requestNumber += 1+ return requestNumber === 1+ ? { ran: true as const, value: await request() }+ : {+ ran: false as const,+ reason: 'backoff' as const,+ blockingReason: 'retry_after' as const,+ waitMs: 90_000,+ }+ },+ })++ expect(result).toMatchObject({+ outcome: 'request_deferred',+ redirects: 1,+ currentUrl: 'https://cdn.example.net/a.png',+ schedulingReason: 'backoff',+ schedulingBlockingReason: 'retry_after',+ schedulingWaitMs: 90_000,+ })+ expect(fetchStreamedMock).toHaveBeenCalledTimes(1)+ })++ it('keeps an accepted response coding for the image store', async () => {+ fetchStreamedMock.mockResolvedValue(image(PNG, 'image/png', { 'content-encoding': 'gzip' }))++ const result = await fetcher().fetch('https://cdn.example.com/a.png', OPTIONS)++ expect(result).toMatchObject({ outcome: 'ok', contentEncoding: 'gzip' })+ })++ it('keeps repeated content codings in received order', async () => {+ const response = image(PNG, 'image/png')+ response.headerLines = [+ { name: 'content-type', value: 'image/png' },+ { name: 'content-encoding', value: 'gzip' },+ { name: 'content-encoding', value: 'br, zstd' },+ ]+ fetchStreamedMock.mockResolvedValue(response)++ const result = await fetcher().fetch('https://cdn.example.com/a.png', OPTIONS)++ expect(result).toMatchObject({ outcome: 'ok', contentEncoding: 'gzip, br, zstd' })+ })++ it('refuses conflicting repeated Content-Type field lines', async () => {+ const response = image(PNG, 'image/png')+ response.headerLines = [+ { name: 'content-type', value: 'image/png' },+ { name: 'content-type', value: 'image/jpeg' },+ ]+ fetchStreamedMock.mockResolvedValue(response)++ const result = await fetcher().fetch('https://cdn.example.com/a.png', OPTIONS)++ expect(result).toMatchObject({ outcome: 'not_image' })+ })++ it('combines repeated Cache-Control field lines in received order', async () => {+ const response = image(PNG, 'image/png')+ response.headerLines.push(+ { name: 'cache-control', value: 'public, max-age=86400' },+ { name: 'cache-control', value: 'no-store' }+ )+ fetchStreamedMock.mockResolvedValue(response)++ const result = await fetcher().fetch('https://cdn.example.com/a.png', OPTIONS)++ expect(result.cache?.cacheControl).toBe('public, max-age=86400, no-store')+ })++ it('refuses an unsupported response coding', async () => {+ fetchStreamedMock.mockResolvedValue(image(PNG, 'image/png', { 'content-encoding': 'compress' }))++ const result = await fetcher().fetch('https://cdn.example.com/a.png', OPTIONS)++ expect(result).toMatchObject({ outcome: 'unsupported_encoding' })+ })++ it.each(['gzip,', 'gzip, br, deflate, zstd, identity'])(+ 'refuses a malformed or excessive response coding list: %s',+ async (contentEncoding) => {+ fetchStreamedMock.mockResolvedValue(image(PNG, 'image/png', { 'content-encoding': contentEncoding }))++ const result = await fetcher().fetch('https://cdn.example.com/a.png', OPTIONS)++ expect(result).toMatchObject({ outcome: 'unsupported_encoding' })+ }+ )++ it('applies a refusal from any repeated opt-out field line', async () => {+ const response = image(PNG, 'image/png')+ response.headerLines = [+ { name: 'content-type', value: 'image/png' },+ { name: 'x-robots-tag', value: 'all' },+ { name: 'x-robots-tag', value: 'PostHogImageFetcherBot: noimageai' },+ ]+ fetchStreamedMock.mockResolvedValue(response)++ const result = await fetcher().fetch('https://cdn.example.com/a.png', OPTIONS)++ expect(result).toMatchObject({ outcome: 'opt_out', refusalReason: 'x_robots_tag' })+ })++ it('lets a response header override a TDMRep reservation', async () => {+ fetchStreamedMock.mockResolvedValue(image(PNG, 'image/png', { 'tdm-reservation': '0' }))++ const result = await fetcher().fetch('https://cdn.example.com/a.png', {+ ...OPTIONS,+ tdmrepReservation: true,+ })++ expect(result).toMatchObject({ outcome: 'ok' })+ })++ it('applies a TDMRep reservation when the response does not override it', async () => {+ fetchStreamedMock.mockResolvedValue(image(PNG, 'image/png'))++ const result = await fetcher().fetch('https://cdn.example.com/a.png', {+ ...OPTIONS,+ tdmrepReservation: true,+ })++ expect(result).toMatchObject({ outcome: 'opt_out', refusalReason: 'tdm_reservation' })+ })++ it('checks path policy again before a same-origin redirect target is requested', async () => {+ fetchStreamedMock.mockResolvedValue(respond(302, { location: '/private/image.png' }))++ const result = await fetcher().fetch('https://cdn.example.com/public/image.png', {+ ...OPTIONS,+ checkRedirectPolicy: () =>+ Promise.resolve({ allowed: false as const, transient: false, reason: 'robots_disallow' }),+ })++ expect(result).toMatchObject({+ outcome: 'redirect_policy_refused',+ redirects: 1,+ currentUrl: 'https://cdn.example.com/private/image.png',+ refusalReason: 'robots_disallow',+ policyTransient: false,+ })+ expect(fetchStreamedMock).toHaveBeenCalledTimes(1)+ })++ it('applies the redirected path TDM policy to the next response', async () => {+ fetchStreamedMock+ .mockResolvedValueOnce(respond(302, { location: '/reserved/image.png' }))+ .mockResolvedValueOnce(image(PNG, 'image/png'))++ const result = await fetcher().fetch('https://cdn.example.com/public/image.png', {+ ...OPTIONS,+ checkRedirectPolicy: () => Promise.resolve({ allowed: true as const, tdmrepReservation: true }),+ })++ expect(result).toMatchObject({ outcome: 'opt_out', refusalReason: 'tdm_reservation', redirects: 1 })+ })++ it('uses one request deadline for every redirect hop', async () => {+ fetchStreamedMock+ .mockResolvedValueOnce(respond(302, { location: '/moved.png' }))+ .mockResolvedValueOnce(image(PNG, 'image/png'))+ const deadlines: number[] = []++ await fetcher().fetch('https://cdn.example.com/a.png', {+ ...OPTIONS,+ scheduleRequest: async (_url, deadlineMs, request) => {+ deadlines.push(deadlineMs)+ return { ran: true as const, value: await request() }+ },+ })++ expect(deadlines).toHaveLength(2)+ expect(new Set(deadlines).size).toBe(1)+ expect(deadlines[0] - Date.now()).toBeGreaterThan(0)+ expect(deadlines[0] - Date.now()).toBeLessThanOrEqual(OPTIONS.timeoutMs)+ })++ it('gives up on a redirect chain longer than the limit', async () => {+ fetchStreamedMock.mockResolvedValue(respond(302, { location: '/again.png' }))+ let scheduledRequests = 0++ const result = await fetcher().fetch('https://cdn.example.com/a.png', {+ ...OPTIONS,+ maxRedirects: 2,+ scheduleRequest: async (_url, _deadlineMs, request) => {+ scheduledRequests += 1+ return { ran: true as const, value: await request() }+ },+ })++ expect(result).toMatchObject({+ outcome: 'redirect_continuation',+ redirects: 2,+ status: 302,+ redirectTarget: { url: 'https://cdn.example.com/again.png' },+ })+ // Three requests carry two redirects. The third response is read and refused, and the+ // budget of whatever it points at is never asked for a token.+ expect(fetchStreamedMock).toHaveBeenCalledTimes(3)+ expect(scheduledRequests).toBe(3)+ })++ it('reports a request scheduler that threw rather than letting it escape', async () => {+ const result = await fetcher().fetch('https://cdn.example.com/a.png', {+ ...OPTIONS,+ scheduleRequest: () => Promise.reject(new Error('budget exploded')),+ })++ expect(result).toEqual({+ outcome: 'error',+ redirects: 0,+ currentUrl: 'https://cdn.example.com/a.png',+ })+ })++ it('accepts a Retry-After period of zero as a non-negative integer', async () => {+ fetchStreamedMock.mockResolvedValue(respond(429, { 'retry-after': '0' }))++ const result = await fetcher().fetch('https://cdn.example.com/a.png', OPTIONS)++ expect(result.retryAfterMs).toBe(0)+ })++ it.each([+ ['a negative period', '-5'],+ ['a decimal period', '1.5'],+ ['an exponent', '1e3'],+ ['a signed period', '+5'],+ ['an unsafe period', '9007199254740991'],+ ['whitespace', ' '],+ ['a date already past', new Date(Date.now() - 60_000).toUTCString()],+ ])('reports no Retry-After period for %s, so exponential backoff has no minimum', async (_name, header) => {+ fetchStreamedMock.mockResolvedValue(respond(429, { 'retry-after': header }))++ const result = await fetcher().fetch('https://cdn.example.com/a.png', OPTIONS)++ expect(result.retryAfterMs).toBeUndefined()+ })++ it.each([+ ['an address the SSRF check refuses', new SecureRequestError('Hostname is not allowed'), 'blocked'],+ ['a name that did not resolve', new ResolutionError('Invalid hostname'), 'error'],+ ['a request that timed out', Object.assign(new Error('timeout'), { name: 'TimeoutError' }), 'timeout'],+ ['a connection that failed', new Error('socket hang up'), 'error'],+ ])('reports %s as an outcome rather than throwing', async (_name, error, outcome) => {+ fetchStreamedMock.mockRejectedValue(error)++ const result = await fetcher().fetch('https://cdn.example.com/a.png', OPTIONS)++ expect(result).toMatchObject({ outcome, redirects: 0, currentUrl: 'https://cdn.example.com/a.png' })+ })+})diff --git a/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/image-fetcher.ts b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/image-fetcher.tsnew file mode 100644index 000000000000..573b510d783c--- /dev/null+++ b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/image-fetcher.ts@@ -0,0 +1,477 @@+import { InvalidRequestError, ResolutionError, SecureRequestError, fetchStreamed } from '~/common/utils/request'++import type { ImageFetchBlockReason } from './block-reason'+import { OriginPolicyReason, ResponseOptOutReason, responseOptOutReason } from './configuration-policy'+import { HttpCacheMetadata } from './crawl-history'+import { ImageFetchRequestMetrics } from './metrics'+import { canonicalizeUrl } from './politeness-key'+import { REQUEST_IDENTITY_HEADERS } from './request-identity'+import { WebBotAuthRequestSigner } from './web-bot-auth'++/**+ * The raster formats that the fetch lane accepts.+ *+ * SVG is absent on purpose. It is a text format that can carry the page's own data, so its redaction+ * belongs on the inline path rather than on an image model.+ *+ * AVIF is absent because the image store validates only PNG, JPEG, GIF and WebP bytes+ * (`image-transport.ts`), so it would reject every AVIF image that this lane fetched.+ */+const ALLOWED_CONTENT_TYPES = ['image/png', 'image/jpeg', 'image/gif', 'image/webp'] as const++export type ImageContentType = (typeof ALLOWED_CONTENT_TYPES)[number]++export type FetchOutcome =+ | 'ok'+ | 'not_found'+ | 'forbidden'+ | 'rate_limited'+ | 'server_error'+ | 'unexpected_status'+ | 'bad_redirect'+ | 'request_deferred'+ | 'redirect_policy_refused'+ | 'redirect_offsite'+ | 'redirect_continuation'+ | 'too_large'+ | 'not_image'+ | 'unsupported_encoding'+ | 'blocked'+ | 'opt_out'+ | 'not_modified'+ | 'timeout'+ | 'error'++export type TransientFetchOutcome = Extract<FetchOutcome, 'timeout' | 'error' | 'rate_limited' | 'server_error'>+export type FetchRefusalReason = OriginPolicyReason | ResponseOptOutReason | 'configuration_refused'++export interface ImageFetchResult {+ outcome: FetchOutcome+ redirects: number+ currentUrl: string+ status?: number+ bytes?: Buffer+ contentType?: ImageContentType+ contentEncoding?: string+ cache?: HttpCacheMetadata+ refusalReason?: FetchRefusalReason+ /** Set by a 429 or a 503 that named a period. The caller holds the registrable domain for that period. */+ retryAfterMs?: number+ schedulingReason?: RequestScheduleBlockReason+ schedulingBlockingReason?: ImageFetchBlockReason+ schedulingWaitMs?: number+ policyTransient?: boolean+ /** Where a redirect this lane did not follow points. The caller republishes it rather than fetching it. */+ redirectTarget?: { url: string; host: string }+}++export interface ImageFetchOptions {+ sourcePartitions?: readonly number[]+ maxBytes: number+ /** Covers the redirect chain as a whole, so a chain of slow hops cannot outlive one hop's budget. */+ timeoutMs: number+ maxRedirects: number+ scheduleRequest: <T>(+ url: URL,+ deadlineMs: number,+ request: () => Promise<T>+ ) => Promise<+ | { ran: true; value: T }+ | {+ ran: false+ reason: RequestScheduleBlockReason+ blockingReason: ImageFetchBlockReason+ waitMs: number+ }+ >+ checkRedirectPolicy: (url: string) => Promise<RedirectTargetPolicy>+ isDifferentOrigin: (url: URL) => boolean+ /** Validators that `url` issued. The caller checks that, and no redirect target gets them. */+ cache?: HttpCacheMetadata+ tdmrepReservation: boolean+ onRedirectResponse?: () => void+}++export type RequestScheduleBlockReason =+ | 'breaker_open'+ | 'backoff'+ | 'deadline'+ | 'origin_map_full'+ | 'registrable_domain_map_full'+ | 'connection_limit'++export type RedirectTargetPolicy =+ | { allowed: true; tdmrepReservation: boolean }+ | { allowed: false; transient: boolean; reason: OriginPolicyReason | 'configuration_refused' }++export interface ImageFetcher {+ fetch(url: string, options: ImageFetchOptions): Promise<ImageFetchResult>+}++const REQUEST_HEADERS: Record<string, string> = {+ ...REQUEST_IDENTITY_HEADERS,+ accept: 'image/*',+ 'accept-encoding': 'gzip, deflate, br, zstd',+}++const REDIRECT_STATUSES = new Set([301, 302, 303, 307, 308])++/**+ * This sends no user credential: no cookie and no `Authorization`, because this lane must not reach an+ * image behind a login. The `Referer` names PostHog and never the page that showed the image, because+ * that page would tell the origin where on the customer's site the image sat. Web Bot Auth identifies+ * PostHog as the operator of the request.+ *+ * Every refusal is an outcome rather than a throw. This runs inside a Kafka batch, one URL at a+ * time, and a throw would abandon the URLs after it in the same batch.+ */+export class HttpImageFetcher implements ImageFetcher {+ constructor(+ private readonly policy: RedirectPolicy,+ private readonly webBotAuthSigner: WebBotAuthRequestSigner+ ) {}++ public async fetch(url: string, options: ImageFetchOptions): Promise<ImageFetchResult> {+ const deadlineMs = Date.now() + options.timeoutMs+ let target = url+ let currentCache = options.cache+ let tdmrepReservation = options.tdmrepReservation+ for (let redirects = 0; ; redirects++) {+ const remainingMs = deadlineMs - Date.now()+ if (remainingMs <= 0) {+ return { outcome: 'timeout', redirects, currentUrl: target }+ }+ let scheduled:+ | { ran: true; value: HopResult }+ | {+ ran: false+ reason: RequestScheduleBlockReason+ blockingReason: ImageFetchBlockReason+ waitMs: number+ }+ try {+ scheduled = await options.scheduleRequest(new URL(target), deadlineMs, () =>+ this.hop(+ target,+ Math.max(1, deadlineMs - Date.now()),+ options.maxBytes,+ currentCache,+ tdmrepReservation,+ options.sourcePartitions+ )+ )+ } catch (error) {+ return { outcome: classifyError(error), redirects, currentUrl: target }+ }+ if (!scheduled.ran) {+ return {+ outcome: 'request_deferred',+ redirects,+ currentUrl: target,+ schedulingReason: scheduled.reason,+ schedulingBlockingReason: scheduled.blockingReason,+ schedulingWaitMs: scheduled.waitMs,+ }+ }+ const hop = scheduled.value+ if (hop.kind !== 'redirect') {+ return { ...hop.result, redirects, currentUrl: target }+ }+ const next = resolveRedirect(target, hop.location, this.policy)+ if (!next) {+ return { outcome: 'bad_redirect', redirects, currentUrl: target, status: hop.status }+ }+ options.onRedirectResponse?.()+ if (options.isDifferentOrigin(next)) {+ return {+ outcome: 'redirect_offsite',+ redirects,+ currentUrl: target,+ status: hop.status,+ redirectTarget: { url: next.toString(), host: next.hostname },+ }+ }+ if (redirects >= options.maxRedirects) {+ return {+ outcome: 'redirect_continuation',+ redirects,+ currentUrl: target,+ status: hop.status,+ redirectTarget: { url: next.toString(), host: next.hostname },+ cache: hop.cache,+ }+ }+ const redirectPolicy = await options.checkRedirectPolicy(next.toString())+ if (!redirectPolicy.allowed) {+ return {+ outcome: 'redirect_policy_refused',+ redirects: redirects + 1,+ currentUrl: next.toString(),+ status: hop.status,+ refusalReason: redirectPolicy.reason,+ policyTransient: redirectPolicy.transient,+ cache: hop.cache,+ }+ }+ target = next.toString()+ currentCache = undefined+ tdmrepReservation = redirectPolicy.tdmrepReservation+ }+ }++ private async hop(+ url: string,+ timeoutMs: number,+ maxBytes: number,+ previousCache: HttpCacheMetadata | undefined,+ tdmrepReservation: boolean,+ sourcePartitions: readonly number[] | undefined+ ): Promise<HopResult> {+ const requestTimeMs = Date.now()+ const canonical = canonicalizeUrl(url)+ let requestOutcome = ImageFetchRequestMetrics.outcomeForHttpStatus()+ const headers: Record<string, string> = { ...REQUEST_HEADERS, ...this.webBotAuthSigner.headersForGet(url) }+ if (previousCache?.etag) {+ headers['if-none-match'] = previousCache.etag+ } else if (previousCache?.lastModified) {+ headers['if-modified-since'] = previousCache.lastModified+ }+ try {+ const response = await fetchStreamed(url, { headers, timeoutMs, allowH2: true })+ const status = response.status+ requestOutcome = ImageFetchRequestMetrics.outcomeForHttpStatus(status)+ const cache = cacheMetadata(requestTimeMs, Date.now(), response.headerLines)+ const optOut = responseOptOutReason(response.headerLines, tdmrepReservation)+ if (optOut) {+ response.discard()+ return { kind: 'done', result: { outcome: 'opt_out', status, refusalReason: optOut, cache } }+ }++ if (REDIRECT_STATUSES.has(status)) {+ response.discard()+ const locations = headerValues(response.headerLines, 'location')+ const location = locations.length === 1 ? locations[0] : undefined+ return location+ ? { kind: 'redirect', status, location, cache }+ : { kind: 'done', result: { outcome: 'bad_redirect', status, cache } }+ }+ if (status === 304) {+ response.discard()+ const outcome =+ previousCache?.etag || previousCache?.lastModified ? 'not_modified' : 'unexpected_status'+ return { kind: 'done', result: { outcome, status, cache } }+ }+ if (status !== 200) {+ response.discard()+ return {+ kind: 'done',+ result: { ...statusResult(status, headerValues(response.headerLines, 'retry-after')), cache },+ }+ }++ const contentTypes = headerValues(response.headerLines, 'content-type').map(normalizeContentType)+ const contentType = contentTypes[0]+ if (!contentType || contentTypes.some((candidate) => candidate !== contentType)) {+ response.discard()+ return { kind: 'done', result: { outcome: 'not_image', status, cache } }+ }+ const contentEncoding = normalizeContentEncoding(headerValues(response.headerLines, 'content-encoding'))+ if (contentEncoding === null) {+ response.discard()+ return { kind: 'done', result: { outcome: 'unsupported_encoding', status, cache } }+ }+ const declaredTooLarge = headerValues(response.headerLines, 'content-length').some((value) => {+ const declaredBytes = Number(value)+ return Number.isFinite(declaredBytes) && declaredBytes > maxBytes+ })+ if (declaredTooLarge) {+ response.discard()+ return { kind: 'done', result: { outcome: 'too_large', status, cache } }+ }++ const { bytes, overLimit } = await response.read(maxBytes, false)+ if (overLimit) {+ return { kind: 'done', result: { outcome: 'too_large', status, cache } }+ }+ return { kind: 'done', result: { outcome: 'ok', status, bytes, contentType, contentEncoding, cache } }+ } finally {+ if (canonical) {+ ImageFetchRequestMetrics.observeRequest(+ requestOutcome,+ (Date.now() - requestTimeMs) / 1000,+ sourcePartitions+ )+ }+ }+ }+}++type HopResult =+ | { kind: 'redirect'; status: number; location: string; cache: HttpCacheMetadata }+ | { kind: 'done'; result: Omit<ImageFetchResult, 'redirects' | 'currentUrl'> }++function statusResult(status: number, retryAfterValues: string[]): Omit<ImageFetchResult, 'redirects' | 'currentUrl'> {+ const retryAfterMs = maximumRetryAfterMs(retryAfterValues)+ if (status === 404 || status === 410) {+ return { outcome: 'not_found', status }+ }+ if (status === 401 || status === 403) {+ return { outcome: 'forbidden', status }+ }+ if (status === 408 || status === 425) {+ return { outcome: 'rate_limited', status }+ }+ if (status === 429) {+ return { outcome: 'rate_limited', status, retryAfterMs }+ }+ if (status === 503) {+ return { outcome: 'server_error', status, retryAfterMs }+ }+ if (status >= 500) {+ return { outcome: 'server_error', status }+ }+ return { outcome: 'unexpected_status', status }+}++function maximumRetryAfterMs(values: string[]): number | undefined {+ const parsed = values.flatMap((value) => {+ const retryAfterMs = parseRetryAfterMs(value)+ return retryAfterMs === undefined ? [] : [retryAfterMs]+ })+ return parsed.length > 0 ? Math.max(...parsed) : undefined+}++/** RFC 9110 allows a count of seconds or an HTTP date. Real responses use both. */+export function parseRetryAfterMs(value: string | undefined): number | undefined {+ if (!value) {+ return undefined+ }+ const trimmed = value.trim()+ if (/^\d+$/.test(trimmed)) {+ const seconds = Number(trimmed)+ return Number.isSafeInteger(seconds) && seconds <= Number.MAX_SAFE_INTEGER / 1000 ? seconds * 1000 : undefined+ }+ const dateMs = Date.parse(trimmed)+ if (Number.isNaN(dateMs)) {+ return undefined+ }+ const heldMs = dateMs - Date.now()+ return heldMs > 0 ? heldMs : undefined+}++/**+ * The rules a redirect target must pass, beyond the SSRF checks every hop re-enters.+ *+ * The collector applied all of these to the first candidate before it reached the topic. A redirect+ * target has passed none of them, so a hop could otherwise reach a host the collector would have+ * refused, such as a single-label name, or a name under a suffix that resolves only inside a+ * network. Requirement 8.+ */+export interface RedirectPolicy {+ maxUrlLength: number+ isPublicHost: (host: string) => boolean+}++function resolveRedirect(from: string, location: string, policy: RedirectPolicy): URL | null {+ let next: URL+ try {+ next = new URL(location, from)+ } catch {+ return null+ }+ // Requirement 9: no downgrade to plain HTTP. `URL.protocol` is lower case already, which a+ // comparison against the raw location string is not.+ if (next.protocol !== 'https:') {+ return null+ }+ // This lane sends no credentials, and a userinfo part would put some back on the next hop.+ if (next.username || next.password) {+ return null+ }+ // A port other than the scheme's own would turn this lane into a port prober. A page names any+ // host and port, the connection leaves from our egress addresses, and the outcome metric reports+ // what answered. Few images are served on another port, so this refuses the whole case.+ if (next.port !== '') {+ return null+ }+ const canonical = canonicalizeUrl(next.toString())+ if (!canonical || canonical.fetch.length > policy.maxUrlLength) {+ return null+ }+ if (!policy.isPublicHost(canonical.host)) {+ return null+ }+ return new URL(canonical.fetch)+}++function normalizeContentType(header: string | undefined): ImageContentType | undefined {+ if (!header) {+ return undefined+ }+ const type = header.split(';')[0].trim().toLowerCase()+ return ALLOWED_CONTENT_TYPES.find((allowed) => allowed === type)+}++const ALLOWED_CONTENT_ENCODINGS = new Set(['gzip', 'deflate', 'br', 'zstd'])+const MAX_CONTENT_ENCODING_LAYERS = 4++function normalizeContentEncoding(values: string[]): string | undefined | null {+ const declaredCodings = values.flatMap((value) => value.split(',')).map((coding) => coding.trim().toLowerCase())+ if (declaredCodings.length > MAX_CONTENT_ENCODING_LAYERS || declaredCodings.some((coding) => coding === '')) {+ return null+ }+ const codings = declaredCodings.filter((coding) => coding !== 'identity')+ if (codings.some((coding) => !ALLOWED_CONTENT_ENCODINGS.has(coding))) {+ return null+ }+ return codings.length > 0 ? codings.join(', ') : undefined+}++function headerValues(headerLines: Array<{ name: string; value: string }>, name: string): string[] {+ return headerLines.filter((line) => line.name === name).map((line) => line.value)+}++function cacheMetadata(+ requestTimeMs: number,+ responseTimeMs: number,+ headerLines: Array<{ name: string; value: string }>+): HttpCacheMetadata {+ return {+ requestTimeMs,+ responseTimeMs,+ etag: singletonHeader(headerLines, 'etag'),+ lastModified: singletonHeader(headerLines, 'last-modified'),+ date: singletonHeader(headerLines, 'date'),+ age: singletonHeader(headerLines, 'age'),+ cacheControl: combinedHeader(headerLines, 'cache-control'),+ expires: singletonHeader(headerLines, 'expires'),+ }+}++function singletonHeader(headerLines: Array<{ name: string; value: string }>, name: string): string | undefined {+ const values = headerValues(headerLines, name)+ return values.length === 1 ? values[0] : undefined+}++function combinedHeader(headerLines: Array<{ name: string; value: string }>, name: string): string | undefined {+ const values = headerValues(headerLines, name)+ return values.length > 0 ? values.join(', ') : undefined+}++export function classifyError(error: unknown): FetchOutcome {+ // `blocked` is terminal, so it must only cover a permanent property of the URL. A refused+ // address and an unparsable URL are permanent. A name that did not resolve is not, because the+ // shared lookup raises the same error for a resolver timeout as for a name that does not exist.+ if (error instanceof SecureRequestError || error instanceof InvalidRequestError) {+ return 'blocked'+ }+ if (error instanceof ResolutionError) {+ return 'error'+ }+ const name = error instanceof Error ? error.name : ''+ const code = typeof error === 'object' && error !== null ? String((error as { code?: unknown }).code ?? '') : ''+ if (name === 'TimeoutError' || name === 'AbortError' || code.includes('TIMEOUT')) {+ return 'timeout'+ }+ return 'error'+}diff --git a/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/image-sink.ts b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/image-sink.tsindex 1922ad21b318..d1692c79214e 100644--- a/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/image-sink.ts+++ b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/image-sink.ts@@ -1,5 +1,5 @@-import type { FetchCandidate } from '~/ingestion/pipelines/sessionreplay/ml-mirror-image-fetch/collected-urls-record'-import type { ImageFetchResult } from '~/ingestion/pipelines/sessionreplay/ml-mirror-image-fetch/image-fetcher'+import type { FetchCandidate } from './collected-urls-record'+import type { ImageFetchResult } from './image-fetcher'/*** Where the lane stores each fetched image. The image counts as handled only once the sink resolves: a rejection keepsdiff --git a/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/job-admission.ts b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/job-admission.tsnew file mode 100644index 000000000000..9ca1793b2379--- /dev/null+++ b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/job-admission.ts@@ -0,0 +1,145 @@+import pLimit from 'p-limit'+import { Counter } from 'prom-client'++import {+ type TeamMonthKey,+ type TeamMonthKeys,+ teamMonthKeyExpiresAtMs,+ untilAborted,+} from '~/ingestion/pipelines/sessionreplay/ai-research/v4-keys/team-month-keys'++import type { FetchCandidate } from './collected-urls-record'+import { parseV4UrlRef } from './v4-url-ref'++/** Each lookup's read budget starts when it starts, so a batch of many new team months must not start them all at once. */+const KEY_LOOKUP_CONCURRENCY = 16+/**+ * Bounds all key lookups of one batch together, so a batch of many new team months still leaves the fetch, history and+ * republish phases their time inside the consumer's 240s batch timeout.+ */+export const DEFAULT_KEY_BUDGET_MS = 60_000+/** A job whose key was not read in time waits in the first delay tier, so it does not come straight back to a busy key store. */+const DEFERRED_JOB_DELAY_MS = 60_000++const droppedJobs = new Counter({+ name: 'ai_research_v4_image_fetch_dropped_jobs_total',+ help: 'v4 fetch jobs dropped before any request, by reason.',+ labelNames: ['reason'],+})++const deferredJobs = new Counter({+ name: 'ai_research_v4_image_fetch_deferred_jobs_total',+ help: 'v4 fetch jobs sent to a delay tier because their team month key was not read within the batch key budget.',+})++const DEFERRED = Symbol('deferred')++/**+ * Drops a v4 job before any request when nothing could store its image: its team month has no live key, because the+ * key row expired, is missing or is a tombstone. The recorder stores the row before it sends a URL, so a missing row+ * means it expired.+ */+export class V4JobAdmission {+ private readonly keyLookups = pLimit(KEY_LOOKUP_CONCURRENCY)++ constructor(+ private readonly keys: Pick<TeamMonthKeys, 'get'>,+ private readonly keyBudgetMs = DEFAULT_KEY_BUDGET_MS+ ) {+ if (!Number.isSafeInteger(keyBudgetMs) || keyBudgetMs <= 0 || keyBudgetMs > 120_000) {+ throw new Error(+ `AI_RESEARCH_V4_IMAGE_FETCH_KEY_BUDGET_MS must be a whole number in (0, 120000], got ${keyBudgetMs}`+ )+ }+ }++ /**+ * Resolves to the jobs to fetch. A job whose key was not read within the budget stays in the batch with a later+ * `notBeforeMs`, so the consumer sends it to a delay tier, as the fetch pass does with a job its deadline passed.+ * A key read that fails before the budget runs out still fails the batch.+ */+ public async admit(candidates: FetchCandidate[], nowMs: number): Promise<FetchCandidate[]> {+ const budget = new AbortController()+ const timer = setTimeout(() => budget.abort(new Error('the v4 key admission budget ran out')), this.keyBudgetMs)+ try {+ return await this.admitWithin(candidates, nowMs, budget.signal)+ } finally {+ clearTimeout(timer)+ // Stops the lookups this batch still waits on when one of them failed.+ budget.abort()+ }+ }++ private async admitWithin(+ candidates: FetchCandidate[],+ nowMs: number,+ budget: AbortSignal+ ): Promise<FetchCandidate[]> {+ const keys = new Map<string, Promise<TeamMonthKey | typeof DEFERRED>>()+ const live: { candidate: FetchCandidate; teamMonth: string }[] = []+ for (const candidate of candidates) {+ const ref = parseV4UrlRef(candidate.originalRef)+ if (!ref) {+ droppedJobs.labels('foreign_ref').inc()+ continue+ }+ // DynamoDB removes an expired row some time after `expires_at`, so the row may still be there.+ if (teamMonthKeyExpiresAtMs(ref.keyMonth) <= nowMs) {+ droppedJobs.labels('expired').inc()+ continue+ }+ const teamMonth = `${ref.teamId}:${ref.keyMonth}`+ if (!keys.has(teamMonth)) {+ keys.set(+ teamMonth,+ this.lookUp(budget, () => this.keys.get(ref.teamId, ref.keyMonth, budget))+ )+ }+ live.push({ candidate, teamMonth })+ }+ const resolved = new Map(+ await Promise.all([...keys].map(async ([teamMonth, key]) => [teamMonth, await key] as const))+ )+ const admitted: FetchCandidate[] = []+ let deferred = 0+ for (const { candidate, teamMonth } of live) {+ const key = resolved.get(teamMonth)!+ if (key === DEFERRED) {+ deferred+++ admitted.push({+ ...candidate,+ notBeforeMs: Math.max(candidate.notBeforeMs, nowMs + DEFERRED_JOB_DELAY_MS),+ lastBlockReason: 'pass_deadline',+ })+ } else if (key.status === 'ok') {+ admitted.push(candidate)+ } else {+ droppedJobs.labels(key.status === 'deleted' ? 'deleted' : 'missing_key').inc()+ }+ }+ if (deferred > 0) {+ deferredJobs.inc(deferred)+ }+ return admitted+ }++ private async lookUp(+ budget: AbortSignal,+ get: () => Promise<TeamMonthKey>+ ): Promise<TeamMonthKey | typeof DEFERRED> {+ try {+ return await untilAborted(+ this.keyLookups(() => {+ budget.throwIfAborted()+ return get()+ }),+ budget+ )+ } catch (error) {+ if (budget.aborted) {+ return DEFERRED+ }+ throw error+ }+ }+}diff --git a/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/lane.ts b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/lane.tsnew file mode 100644index 000000000000..524fc9d5f742--- /dev/null+++ b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/lane.ts@@ -0,0 +1,54 @@+import {+ KAFKA_AI_RESEARCH_V4_IMAGE_FETCH,+ KAFKA_AI_RESEARCH_V4_IMAGE_FETCH_RETRY_1H,+ KAFKA_AI_RESEARCH_V4_IMAGE_FETCH_RETRY_1M,+ KAFKA_AI_RESEARCH_V4_IMAGE_FETCH_RETRY_10M,+} from '~/common/config/kafka-topics'++/** The v4 lane has its own frontier, delay tiers and consumer groups, so its jobs never mix with the v3 lane's. */+export const V4_IMAGE_FETCH_LANE = {+ frontier: KAFKA_AI_RESEARCH_V4_IMAGE_FETCH,+ retry1m: KAFKA_AI_RESEARCH_V4_IMAGE_FETCH_RETRY_1M,+ retry10m: KAFKA_AI_RESEARCH_V4_IMAGE_FETCH_RETRY_10M,+ retry1h: KAFKA_AI_RESEARCH_V4_IMAGE_FETCH_RETRY_1H,+} as const++/** The delay tiers the publisher writes to. A retry pod drains one of these and nothing else. */+export const V4_IMAGE_FETCH_DELAY_TOPICS: readonly string[] = [+ V4_IMAGE_FETCH_LANE.retry1m,+ V4_IMAGE_FETCH_LANE.retry10m,+ V4_IMAGE_FETCH_LANE.retry1h,+]++/** A dead letter must never land on a topic the lane consumes, which would loop it back. */+export const BLOCKED_IMAGE_FETCH_DEAD_LETTER_TOPICS: readonly string[] = [+ V4_IMAGE_FETCH_LANE.frontier,+ ...V4_IMAGE_FETCH_DELAY_TOPICS,+]++/** The v3 lane's default group. A v4 pod in it would take v3 partitions and read none of their topics. */+const V3_IMAGE_FETCH_GROUP_ID = 'session-replay-ml-image-fetch'++export function assertOwnImageFetchGroup(group: string): void {+ if (!group || group === V3_IMAGE_FETCH_GROUP_ID) {+ throw new Error('AI_RESEARCH_V4_IMAGE_FETCH_GROUP_ID must be set and differ from the v3 lane group')+ }+}++export interface HostLimits {+ requestsPerSecond: number+ burst: number+ maxConcurrent: number+}++/** The v4 per registrable domain limits: a share of the configured limits, at least one request at a time. */+export function v4HostLimits(limits: HostLimits, share: number): HostLimits {+ if (!(share > 0 && share <= 1)) {+ throw new Error(`AI_RESEARCH_V4_IMAGE_FETCH_HOST_BUDGET_SHARE must be in (0, 1], got ${share}`)+ }+ return {+ requestsPerSecond: limits.requestsPerSecond * share,+ burst: Math.max(1, Math.floor(limits.burst * share)),+ maxConcurrent: Math.max(1, Math.floor(limits.maxConcurrent * share)),+ }+}diff --git a/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/metrics.test.ts b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/metrics.test.tsnew file mode 100644index 000000000000..ac28b12e9313--- /dev/null+++ b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/metrics.test.ts@@ -0,0 +1,186 @@+import { register } from 'prom-client'++import { ImageFetchConsumerMetrics, ImageFetchRequestMetrics } from './metrics'++describe('image fetch metrics', () => {+ beforeEach(() => register.resetMetrics())+ afterEach(() => jest.restoreAllMocks())++ test.each([+ [undefined, 'network_error'],+ [199, 'other'],+ [200, '2xx'],+ [299, '2xx'],+ [300, '3xx'],+ [399, '3xx'],+ [400, '4xx'],+ [499, '4xx'],+ [500, '5xx'],+ [599, '5xx'],+ [600, 'other'],+ ] as const)('maps HTTP status %s to the bounded %s outcome', (status, expected) => {+ expect(ImageFetchRequestMetrics.outcomeForHttpStatus(status)).toBe(expected)+ })++ it('records retry causes with fixed semantic labels', async () => {+ ImageFetchRequestMetrics.incRetryCause('rate_limited')+ ImageFetchRequestMetrics.incRetryCause('server_error')+ ImageFetchRequestMetrics.incRetryCause('timeout')+ ImageFetchRequestMetrics.incRetryCause('error')++ const metric = (await register.getMetricsAsJSON()).find(+ (candidate) => candidate.name === 'ai_research_v4_image_fetch_retry_causes_total'+ )+ const causes = metric?.values.map((value) => value.labels.cause)++ expect(causes).toEqual(['rate_limited', 'server_error', 'timeout', 'error'])+ })++ it('records bounded concentration and effective-count distributions', async () => {+ ImageFetchConsumerMetrics.observeBatchDiversity([6, 3, 1], [9, 1])++ const metrics = await register.getMetricsAsJSON()+ const topShare = metrics.find((metric) => metric.name === 'ai_research_v4_image_fetch_batch_top_share')+ const effectiveCount = metrics.find(+ (metric) => metric.name === 'ai_research_v4_image_fetch_batch_effective_count'+ )+ const topShareSums = topShare?.values.filter(+ (value) => 'metricName' in value && String(value.metricName).endsWith('_sum')+ )+ const effectiveCountSums = effectiveCount?.values.filter(+ (value) => 'metricName' in value && String(value.metricName).endsWith('_sum')+ )++ expect(topShareSums).toEqual([+ expect.objectContaining({ labels: { scope: 'origin', top_n: '1' }, value: 0.6 }),+ expect.objectContaining({ labels: { scope: 'origin', top_n: '5' }, value: 1 }),+ expect.objectContaining({ labels: { scope: 'origin', top_n: '10' }, value: 1 }),+ expect.objectContaining({ labels: { scope: 'registrable_domain', top_n: '1' }, value: 0.9 }),+ expect.objectContaining({ labels: { scope: 'registrable_domain', top_n: '5' }, value: 1 }),+ expect.objectContaining({ labels: { scope: 'registrable_domain', top_n: '10' }, value: 1 }),+ ])+ expect(effectiveCountSums).toEqual([+ expect.objectContaining({ labels: { scope: 'origin' }, value: 100 / 46 }),+ expect.objectContaining({ labels: { scope: 'registrable_domain' }, value: 100 / 82 }),+ ])+ })++ it('records partition flow, outcomes, waits, and concentration with bounded labels', async () => {+ ImageFetchConsumerMetrics.observePartitionRecord(7, 10, 9)+ ImageFetchConsumerMetrics.incPartitionUrls(7, 'unique', 8)+ ImageFetchConsumerMetrics.observePartitionBatchDiversity(7, [6, 2], [7, 1])+ ImageFetchRequestMetrics.observeRequest('2xx', 0.25, [7, 42])+ ImageFetchRequestMetrics.observeSchedulerWait('origin_crawl_delay', 1.5, [7, 42])+ ImageFetchRequestMetrics.incPartitionAttempt(7, 'completed', 'ok')+ ImageFetchRequestMetrics.incPartitionAttempt(7, 'republished', 'backoff')++ const metrics = await register.getMetricsAsJSON()+ const records = metrics.find((metric) => metric.name === 'ai_research_v4_image_fetch_partition_records_total')+ const urls = metrics.find((metric) => metric.name === 'ai_research_v4_image_fetch_partition_urls_total')+ const requests = metrics.find((metric) => metric.name === 'ai_research_v4_image_fetch_partition_requests_total')+ const waits = metrics.find(+ (metric) => metric.name === 'ai_research_v4_image_fetch_partition_scheduler_wait_seconds'+ )+ const attempts = metrics.find((metric) => metric.name === 'ai_research_v4_image_fetch_partition_attempts_total')+ const topShare = metrics.find((metric) => metric.name === 'ai_research_v4_image_fetch_partition_top_share')+ const effectiveCount = metrics.find(+ (metric) => metric.name === 'ai_research_v4_image_fetch_partition_effective_count'+ )++ expect(records?.values).toContainEqual(expect.objectContaining({ labels: { partition: '7' }, value: 1 }))+ expect(urls?.values).toEqual(+ expect.arrayContaining([+ expect.objectContaining({ labels: { partition: '7', stage: 'parsed' }, value: 10 }),+ expect.objectContaining({ labels: { partition: '7', stage: 'accepted' }, value: 9 }),+ expect.objectContaining({ labels: { partition: '7', stage: 'unique' }, value: 8 }),+ ])+ )+ expect(requests?.values).toEqual(+ expect.arrayContaining([+ expect.objectContaining({ labels: { partition: '7', outcome: '2xx' }, value: 1 }),+ expect.objectContaining({ labels: { partition: '42', outcome: '2xx' }, value: 1 }),+ ])+ )+ expect(waits?.values).toEqual(+ expect.arrayContaining([+ expect.objectContaining({ labels: { partition: '7', scope: 'origin_crawl_delay' }, value: 1 }),+ expect.objectContaining({ labels: { partition: '42', scope: 'origin_crawl_delay' }, value: 1 }),+ ])+ )+ expect(attempts?.values).toEqual(+ expect.arrayContaining([+ expect.objectContaining({+ labels: { partition: '7', disposition: 'completed', outcome: 'ok' },+ value: 1,+ }),+ expect.objectContaining({+ labels: { partition: '7', disposition: 'republished', outcome: 'backoff' },+ value: 1,+ }),+ ])+ )+ expect(topShare?.values).toEqual(+ expect.arrayContaining([+ expect.objectContaining({+ metricName: expect.stringMatching(/_sum$/),+ labels: { partition: '7', scope: 'origin' },+ value: 0.75,+ }),+ expect.objectContaining({+ metricName: expect.stringMatching(/_sum$/),+ labels: { partition: '7', scope: 'registrable_domain' },+ value: 0.875,+ }),+ ])+ )+ expect(effectiveCount?.values).toEqual(+ expect.arrayContaining([+ expect.objectContaining({+ metricName: expect.stringMatching(/_sum$/),+ labels: { partition: '7', scope: 'origin' },+ value: 64 / 40,+ }),+ expect.objectContaining({+ metricName: expect.stringMatching(/_sum$/),+ labels: { partition: '7', scope: 'registrable_domain' },+ value: 64 / 50,+ }),+ ])+ )+ })++ it('records initial schedulable slots against the pod request limit', async () => {+ ImageFetchRequestMetrics.observeBatchSchedulableCapacity(7, 10)++ const metrics = await register.getMetricsAsJSON()+ const slots = metrics.find((metric) => metric.name === 'ai_research_v4_image_fetch_batch_schedulable_slots')+ const ratio = metrics.find(+ (metric) => metric.name === 'ai_research_v4_image_fetch_batch_schedulable_capacity_ratio'+ )++ expect(slots?.values).toContainEqual(+ expect.objectContaining({ metricName: expect.stringMatching(/_sum$/), value: 7 })+ )+ expect(ratio?.values).toContainEqual(+ expect.objectContaining({ metricName: expect.stringMatching(/_sum$/), value: 0.7 })+ )+ })++ it('keeps the oldest active batch age when another batch finishes', async () => {+ jest.spyOn(performance, 'now').mockReturnValue(3_000)+ const firstBatchId = ImageFetchConsumerMetrics.startBatch(1_000)+ const secondBatchId = ImageFetchConsumerMetrics.startBatch(2_000)++ ImageFetchConsumerMetrics.finishBatch(secondBatchId)+ const activeMetric = (await register.getMetricsAsJSON()).find(+ (metric) => metric.name === 'ai_research_v4_image_fetch_consumer_active_batch_elapsed_seconds'+ )+ expect(activeMetric?.values[0].value).toBe(2)++ ImageFetchConsumerMetrics.finishBatch(firstBatchId)+ const idleMetric = (await register.getMetricsAsJSON()).find(+ (metric) => metric.name === 'ai_research_v4_image_fetch_consumer_active_batch_elapsed_seconds'+ )+ expect(idleMetric?.values[0].value).toBe(0)+ })+})diff --git a/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/metrics.ts b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/metrics.tsnew file mode 100644index 000000000000..f73ffe51e9e1--- /dev/null+++ b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/metrics.ts@@ -0,0 +1,663 @@+import { Counter, Gauge, Histogram } from 'prom-client'++import type { RepublishReason, UrlDropReason, UrlSkipReason } from './collected-urls-record'+import type { AttemptOutcome } from './fetch-runner'+import type { FrontierDeadLetterReason } from './frontier-dead-letter-sink'+import type { FetchRefusalReason, RequestScheduleBlockReason, TransientFetchOutcome } from './image-fetcher'++/** What the in-flight gauge reads. Narrower than `ConcurrencyController`, so the metrics do not depend on the whole of it. */+export interface InFlightCount {+ readonly running: number+}++/** What the budget gauges read. Narrower than `HostBudget`, so the metrics do not depend on the whole of it. */+export interface BudgetCounts {+ readonly trackedRegistrableDomains: number+ readonly trackedOrigins: number+ readonly evictedWhileBlocked: number+ blockedRegistrableDomains(nowMs: number): number+}++export type DedupScope = 'batch' | 'store'+export type SchedulerWaitScope = 'origin_crawl_delay' | 'registrable_domain_rate' | 'request_capacity'+export type HttpRequestOutcome = '2xx' | '3xx' | '4xx' | '5xx' | 'other' | 'network_error'+export type RepublishDestination = 'frontier' | 'delay'+export type RepublishTopic = 'frontier' | 'retry_1m' | 'retry_10m' | 'retry_1h'+type BatchDiversityScope = 'origin' | 'registrable_domain'+type PartitionUrlStage = 'parsed' | 'accepted' | 'unique' | 'fetchable' | 'not_ready' | 'store_deduped'+type PartitionAttemptDisposition = 'completed' | 'republished'+type PolicyAndBudgetReason = FetchRefusalReason | RequestScheduleBlockReason | 'none'++const BATCH_DIVERSITY_TOP_COUNTS = [1, 5, 10] as const++export class ImageFetchConsumerMetrics {+ private static readonly fetchable = new Counter({+ name: 'ai_research_v4_image_fetch_consumer_fetchable_total',+ help: 'URLs that passed every check and would have been fetched. In dry run no request is sent, so this is the offered rate rather than the sent rate',+ })+ /**+ * `batch` costs only memory, while `store` costs a shared-store read.+ */+ private static readonly deduped = new Counter({+ name: 'ai_research_v4_image_fetch_consumer_deduped_total',+ help: 'URLs skipped as already known, by the layer that caught them: "batch" (another copy in the same poll batch) or "store" (a durable crawl-history result)',+ labelNames: ['scope'],+ })+ /**+ * A sustained rate means the producer and consumer disagree about the wire format.+ */+ private static readonly dropped = new Counter({+ name: 'ai_research_v4_image_fetch_consumer_dropped_total',+ help: 'URLs refused before dedup because the versioned record, URL, ref, or registrable-domain key was invalid. When a dead-letter topic is configured, its Kafka acknowledgement precedes this increment and the source commit',+ labelNames: ['reason'],+ })+ /**+ * A sustained rate here is expected after a beacon-list change, while the fetcher drains the+ * queued beacons at parse speed. It is not the invalid-input signal that `dropped` carries.+ */+ private static readonly skipped = new Counter({+ name: 'ai_research_v4_image_fetch_consumer_skipped_total',+ help: 'Frontier jobs the parser dropped on their own because the URL policy refuses the URL as unwanted, by decline reason. The record and its other jobs proceed and no dead-letter record is written. The count lands after the batch finishes its durable work, so a failed batch is not counted twice; a crash before the offset store can count it once more, like every counter in this consumer',+ labelNames: ['reason'],+ })+ private static readonly deadLettered = new Counter({+ name: 'ai_research_v4_image_fetch_consumer_dead_lettered_total',+ help: 'Rejected frontier records acknowledged by the dead-letter topic before the source offset can advance',+ labelNames: ['reason'],+ })+ private static readonly deadLetterFailed = new Counter({+ name: 'ai_research_v4_image_fetch_consumer_dead_letter_failed_total',+ help: 'Rejected frontier records that could not reach the dead-letter topic. The source batch fails so its offsets remain uncommitted',+ labelNames: ['reason'],+ })+ /**+ * This distribution shows how much independent origin state one poll can activate.+ */+ private static readonly originsPerBatch = new Histogram({+ name: 'ai_research_v4_image_fetch_consumer_origins_per_batch',+ help: 'Distinct origins in one poll batch',+ buckets: [1, 2, 4, 8, 16, 32, 64],+ })+ private static readonly distinctOriginsPerBatch = new Histogram({+ name: 'ai_research_v4_image_fetch_consumer_distinct_origins_per_batch',+ help: 'Distinct origins in one poll batch, including empty batches and the full configured batch range',+ buckets: [0, 1, 2, 4, 8, 16, 32, 64, 128, 256, 512, 1024, 2048, 4096, 8192, 16_384],+ })+ private static readonly registrableDomainsPerBatch = new Histogram({+ name: 'ai_research_v4_image_fetch_consumer_registrable_domains_per_batch',+ help: 'Distinct registrable domains in one poll batch',+ buckets: [0, 1, 2, 4, 8, 16, 32, 64, 128, 256, 512, 1024, 2048, 4096, 8192, 16_384],+ })+ private static readonly batchTopShare = new Histogram({+ name: 'ai_research_v4_image_fetch_batch_top_share',+ help: 'Share of deduplicated canonical URL jobs held by the largest fixed number of origins or registrable domains in one poll batch',+ labelNames: ['scope', 'top_n'],+ buckets: [0.01, 0.05, 0.1, 0.25, 0.5, 0.75, 0.9, 0.95, 1],+ })+ private static readonly batchEffectiveCount = new Histogram({+ name: 'ai_research_v4_image_fetch_batch_effective_count',+ help: 'Inverse Simpson effective count of origins or registrable domains among deduplicated canonical URL jobs in one poll batch',+ labelNames: ['scope'],+ buckets: [1, 2, 4, 8, 16, 32, 64, 128, 256, 512, 1024, 2048, 4096, 8192, 16_384, 32_768, 65_536],+ })+ private static readonly partitionRecords = new Counter({+ name: 'ai_research_v4_image_fetch_partition_records_total',+ help: 'Valid Kafka records handled by source partition',+ labelNames: ['partition'],+ })+ private static readonly partitionUrls = new Counter({+ name: 'ai_research_v4_image_fetch_partition_urls_total',+ help: 'URL jobs attributed to a source partition and bounded processing stage. A shared deduplicated job counts once for each contributing partition',+ labelNames: ['partition', 'stage'],+ })+ private static readonly partitionTopShare = new Histogram({+ name: 'ai_research_v4_image_fetch_partition_top_share',+ help: 'Share of deduplicated URL jobs in one source partition held by its largest origin or registrable domain in a joined batch',+ labelNames: ['partition', 'scope'],+ buckets: [0.01, 0.05, 0.1, 0.25, 0.5, 0.75, 0.9, 0.95, 1],+ })+ private static readonly partitionEffectiveCount = new Histogram({+ name: 'ai_research_v4_image_fetch_partition_effective_count',+ help: 'Inverse Simpson effective count of origins or registrable domains for one source partition in a joined batch',+ labelNames: ['partition', 'scope'],+ buckets: [1, 2, 4, 8, 16, 32, 64, 128, 256, 512, 1024, 2048, 4096, 8192, 16_384],+ })+ private static readonly urlsPerRecord = new Histogram({+ name: 'ai_research_v4_image_fetch_consumer_urls_per_record',+ help: 'URLs carried by one Kafka record. The producer packs a record to a byte budget, so a distribution pinned at the top means the count bound rather than the byte budget is what split it',+ buckets: [1, 8, 32, 64, 128, 256, 512],+ })+ private static readonly storeErrors = new Counter({+ name: 'ai_research_v4_image_fetch_consumer_store_errors_total',+ help: 'Crawl-history keys in a failed read or write. Either failure stops the batch before Kafka offsets advance',+ labelNames: ['operation'],+ })+ private static readonly storeDuration = new Histogram({+ name: 'ai_research_v4_image_fetch_consumer_store_duration_seconds',+ help: 'Crawl-history operation wall time by bounded operation and outcome',+ labelNames: ['operation', 'outcome'],+ buckets: [0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10, 30, 60, 120],+ })+ private static readonly batchDuration = new Histogram({+ name: 'ai_research_v4_image_fetch_consumer_batch_duration_seconds',+ help: 'Wall time per completed poll batch. Read against Kafka max.poll.interval.ms (300s)',+ buckets: [0.01, 0.05, 0.1, 0.5, 1, 5, 15, 30, 60, 120, 240, 300, 600],+ })+ private static readonly activeBatchStartedAtMs = new Map<symbol, number>()+ private static readonly activeBatchElapsed = new Gauge({+ name: 'ai_research_v4_image_fetch_consumer_active_batch_elapsed_seconds',+ help: 'Elapsed wall time of the active non-empty poll batch, or zero between batches. This exposes a stuck batch before Kafka max.poll.interval.ms revokes it',+ collect() {+ const startedAtMs =+ ImageFetchConsumerMetrics.activeBatchStartedAtMs.size > 0+ ? Math.min(...ImageFetchConsumerMetrics.activeBatchStartedAtMs.values())+ : undefined+ this.set(startedAtMs === undefined ? 0 : Math.max(0, performance.now() - startedAtMs) / 1000)+ },+ })+ private static readonly ageSeconds = new Histogram({+ name: 'ai_research_v4_image_fetch_consumer_url_age_seconds',+ help: 'Age of a URL when this lane reached it, measured from capture rather than from produce. The tail shows the age of the backlog',+ buckets: [1, 10, 60, 300, 900, 3600, 6 * 3600, 24 * 3600],+ })+ private static readonly dryRun = new Gauge({+ name: 'ai_research_v4_image_fetch_consumer_dry_run',+ help: '1 while the lane sends no outbound request, 0 once fetching is enabled. Every other metric of this lane means something different either side of this value',+ })++ public static setDryRun(enabled: boolean): void {+ this.dryRun.set(enabled ? 1 : 0)+ }+ public static incFetchable(count: number): void {+ this.fetchable.inc(count)+ }+ public static incDeduped(scope: DedupScope, count: number): void {+ this.deduped.labels(scope).inc(count)+ }+ public static incDropped(reason: UrlDropReason, count: number): void {+ this.dropped.labels(reason).inc(count)+ }+ public static incSkipped(reason: UrlSkipReason, count: number): void {+ this.skipped.labels(reason).inc(count)+ }+ public static incDeadLettered(reason: FrontierDeadLetterReason): void {+ this.deadLettered.labels(reason).inc()+ }+ public static incDeadLetterFailed(reason: FrontierDeadLetterReason): void {+ this.deadLetterFailed.labels(reason).inc()+ }+ public static incStoreError(operation: 'read' | 'write', count: number): void {+ this.storeErrors.labels(operation).inc(count)+ }+ public static observeStoreDuration(+ operation: 'read' | 'write',+ outcome: 'success' | 'error',+ durationSeconds: number+ ): void {+ this.storeDuration.labels(operation, outcome).observe(durationSeconds)+ }+ public static observeBatch(origins: number, registrableDomains: number, durationSeconds: number): void {+ this.originsPerBatch.observe(origins)+ this.distinctOriginsPerBatch.observe(origins)+ this.registrableDomainsPerBatch.observe(registrableDomains)+ this.batchDuration.observe(durationSeconds)+ }+ public static observeBatchDiversity(originCounts: number[], registrableDomainCounts: number[]): void {+ this.observeBatchDistribution('origin', originCounts)+ this.observeBatchDistribution('registrable_domain', registrableDomainCounts)+ }+ public static startBatch(nowMs = performance.now()): symbol {+ const batchId = Symbol()+ this.activeBatchStartedAtMs.set(batchId, nowMs)+ return batchId+ }+ public static finishBatch(batchId: symbol): void {+ this.activeBatchStartedAtMs.delete(batchId)+ }+ public static observeRecord(urls: number): void {+ this.urlsPerRecord.observe(urls)+ }+ public static observePartitionRecord(partition: number, urls: number, acceptedUrls: number): void {+ const partitionLabel = String(partition)+ this.partitionRecords.labels(partitionLabel).inc()+ this.partitionUrls.labels(partitionLabel, 'parsed').inc(urls)+ this.partitionUrls.labels(partitionLabel, 'accepted').inc(acceptedUrls)+ }+ public static incPartitionUrls(partition: number, stage: PartitionUrlStage, count: number): void {+ this.partitionUrls.labels(String(partition), stage).inc(count)+ }+ public static observePartitionBatchDiversity(+ partition: number,+ originCounts: number[],+ registrableDomainCounts: number[]+ ): void {+ this.observePartitionDistribution(partition, 'origin', originCounts)+ this.observePartitionDistribution(partition, 'registrable_domain', registrableDomainCounts)+ }+ public static observeAge(ageSeconds: number): void {+ this.ageSeconds.observe(ageSeconds)+ }++ private static observeBatchDistribution(scope: BatchDiversityScope, counts: number[]): void {+ const summary = summarizeBatchDistribution(counts)+ if (!summary) {+ return+ }+ for (const topCount of BATCH_DIVERSITY_TOP_COUNTS) {+ const topTotal = summary.largestCounts.slice(0, topCount).reduce((total, count) => total + count, 0)+ this.batchTopShare.labels(scope, String(topCount)).observe(topTotal / summary.total)+ }+ this.batchEffectiveCount.labels(scope).observe((summary.total * summary.total) / summary.sumOfSquares)+ }++ private static observePartitionDistribution(partition: number, scope: BatchDiversityScope, counts: number[]): void {+ const summary = summarizeBatchDistribution(counts)+ if (!summary) {+ return+ }+ const partitionLabel = String(partition)+ this.partitionTopShare.labels(partitionLabel, scope).observe(summary.largestCounts[0] / summary.total)+ this.partitionEffectiveCount+ .labels(partitionLabel, scope)+ .observe((summary.total * summary.total) / summary.sumOfSquares)+ }+}++function summarizeBatchDistribution(+ counts: number[]+): { total: number; sumOfSquares: number; largestCounts: number[] } | undefined {+ let total = 0+ let sumOfSquares = 0+ const largestCounts: number[] = []+ for (const count of counts) {+ total += count+ sumOfSquares += count * count+ const insertionIndex = largestCounts.findIndex((existing) => count > existing)+ if (insertionIndex >= 0) {+ largestCounts.splice(insertionIndex, 0, count)+ } else if (largestCounts.length < BATCH_DIVERSITY_TOP_COUNTS.at(-1)!) {+ largestCounts.push(count)+ }+ if (largestCounts.length > BATCH_DIVERSITY_TOP_COUNTS.at(-1)!) {+ largestCounts.pop()+ }+ }+ return total > 0 ? { total, sumOfSquares, largestCounts } : undefined+}++/**+ * The request path.+ *+ * No metric here carries a host or a URL. The host set is unbounded, so it lives in the structured+ * logs of the runner. A URL is page content and belongs in no metric, log, or trace.+ */+export class ImageFetchRequestMetrics {+ private static readonly completedUrls = new Counter({+ name: 'ai_research_v4_image_fetch_completed_urls_total',+ help: 'Completed URLs by final outcome and refusal reason',+ labelNames: ['outcome', 'refusal_reason'],+ })+ private static readonly completedUrlSystemTime = new Histogram({+ name: 'ai_research_v4_image_fetch_completed_url_system_time_seconds',+ help: 'Time from first collection until the URL reaches a terminal outcome',+ buckets: [1, 60, 600, 3600, 6 * 3600, 24 * 3600, 7 * 24 * 3600, 30 * 24 * 3600],+ })+ private static readonly completedUrlFetches = new Histogram({+ name: 'ai_research_v4_image_fetch_completed_url_fetches',+ help: 'Image HTTP requests made before a URL reaches a terminal outcome',+ buckets: [0, 1, 2, 3, 5, 10],+ })+ private static readonly completedUrlRepublishes = new Histogram({+ name: 'ai_research_v4_image_fetch_completed_url_republishes',+ help: 'Frontier and delay-topic republishes before a URL reaches a terminal outcome',+ buckets: [0, 1, 2, 3, 5, 10],+ })+ private static readonly policyAndBudgetDecisions = new Counter({+ name: 'ai_research_v4_image_fetch_policy_and_budget_decisions_total',+ help: 'Origin-policy and registrable-domain request-control decisions after block state is known',+ labelNames: ['blocked', 'reason'],+ })+ private static readonly batchSchedulableSlots = new Histogram({+ name: 'ai_research_v4_image_fetch_batch_schedulable_slots',+ help: 'Request slots that the initial fetch queue can use after live pod and registrable-domain concurrency limits',+ buckets: [1, 2, 4, 8, 16, 32, 64, 128, 256, 300, 512, 1024],+ })+ private static readonly batchSchedulableCapacityRatio = new Histogram({+ name: 'ai_research_v4_image_fetch_batch_schedulable_capacity_ratio',+ help: 'Share of the pod request limit that the initial fetch queue can use after live concurrency limits',+ buckets: [0.01, 0.05, 0.1, 0.25, 0.5, 0.75, 0.9, 0.95, 1],+ })+ /**+ * `ok` against the sum is the yield of the lane.+ */+ private static readonly outcomes = new Counter({+ name: 'ai_research_v4_image_fetch_requests_total',+ help: 'Completed external HTTP requests by status class or network failure',+ labelNames: ['outcome'],+ })+ private static readonly duration = new Histogram({+ name: 'ai_research_v4_image_fetch_request_duration_seconds',+ help: 'Time for one completed external HTTP request, including its response body',+ labelNames: ['outcome'],+ buckets: [0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10, 30],+ })+ private static readonly partitionOutcomes = new Counter({+ name: 'ai_research_v4_image_fetch_partition_requests_total',+ help: 'Completed image HTTP requests attributed to each source partition and status class or network failure',+ labelNames: ['partition', 'outcome'],+ })+ private static readonly partitionDuration = new Histogram({+ name: 'ai_research_v4_image_fetch_partition_request_duration_seconds',+ help: 'Time for one completed image HTTP request attributed to each source partition, including its response body',+ labelNames: ['partition', 'outcome'],+ buckets: [0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10, 30],+ })+ private static readonly partitionAttempts = new Counter({+ name: 'ai_research_v4_image_fetch_partition_attempts_total',+ help: 'URL jobs completed or republished after successful delivery and URL accounting, attributed to each source partition and outcome',+ labelNames: ['partition', 'disposition', 'outcome'],+ })+ private static readonly retryCauses = new Counter({+ name: 'ai_research_v4_image_fetch_retry_causes_total',+ help: 'Transient fetch outcomes that caused the URL to be scheduled for another attempt',+ labelNames: ['cause'],+ })+ /**+ * Read a rising tail with the `deadline` outcome to distinguish headroom from load that an+ * origin's allowed rate cannot carry within one pass.+ */+ private static readonly schedulerWait = new Histogram({+ name: 'ai_research_v4_image_fetch_scheduler_wait_seconds',+ help: 'Time a request waited for an origin crawl delay, registrable-domain rate, or pod request slot',+ labelNames: ['scope'],+ buckets: [0, 0.1, 0.5, 1, 2, 5, 10, 20],+ })+ private static readonly partitionSchedulerWait = new Histogram({+ name: 'ai_research_v4_image_fetch_partition_scheduler_wait_seconds',+ help: 'Time an image request waited, attributed to each source partition and politeness or capacity scope',+ labelNames: ['partition', 'scope'],+ buckets: [0, 0.1, 0.5, 1, 2, 5, 10, 20],+ })+ private static readonly responseBytes = new Histogram({+ name: 'ai_research_v4_image_fetch_response_bytes',+ help: 'Size of a downloaded image. The tail against the configured byte limit says how much of the catalog the limit refuses',+ buckets: [+ 1024,+ 8 * 1024,+ 32 * 1024,+ 128 * 1024,+ 512 * 1024,+ 1024 * 1024,+ 2 * 1024 * 1024,+ 10 * 1024 * 1024,+ 20 * 1024 * 1024,+ ],+ })+ private static readonly redirects = new Histogram({+ name: 'ai_research_v4_image_fetch_redirects',+ help: 'Redirects followed for one URL. Each hop uses its registrable-domain budget and origin crawl delay',+ buckets: [0, 1, 2, 3],+ })+ /**+ * Both gauges read the budget at scrape time. A hold expires by the clock, so a count taken at+ * the end of a batch would report blocked origins until the next batch arrives.+ */+ private static readonly trackedRegistrableDomains = new Gauge({+ name: 'ai_research_v4_image_fetch_tracked_registrable_domains',+ help: 'Registrable domains this pod holds request-control state for',+ collect() {+ this.set(ImageFetchRequestMetrics.budget?.trackedRegistrableDomains ?? 0)+ },+ })+ private static readonly trackedOrigins = new Gauge({+ name: 'ai_research_v4_image_fetch_tracked_origins',+ help: 'Origins this pod holds configuration and crawl-delay state for',+ collect() {+ this.set(ImageFetchRequestMetrics.budget?.trackedOrigins ?? 0)+ },+ })+ private static readonly blockedRegistrableDomains = new Gauge({+ name: 'ai_research_v4_image_fetch_blocked_registrable_domains',+ help: 'Registrable domains held by transient back-off, Retry-After, or an open circuit breaker',+ collect() {+ this.set(ImageFetchRequestMetrics.budget?.blockedRegistrableDomains(Date.now()) ?? 0)+ },+ })++ public static outcomeForHttpStatus(status?: number): HttpRequestOutcome {+ if (status === undefined) {+ return 'network_error'+ }+ if (status >= 200 && status < 300) {+ return '2xx'+ }+ if (status >= 300 && status < 400) {+ return '3xx'+ }+ if (status >= 400 && status < 500) {+ return '4xx'+ }+ if (status >= 500 && status < 600) {+ return '5xx'+ }+ return 'other'+ }+ public static observeRequest(+ outcome: HttpRequestOutcome,+ durationSeconds: number,+ sourcePartitions?: readonly number[]+ ): void {+ this.outcomes.labels(outcome).inc()+ this.duration.labels(outcome).observe(durationSeconds)+ for (const sourcePartition of new Set(sourcePartitions ?? [])) {+ const partitionLabel = String(sourcePartition)+ this.partitionOutcomes.labels(partitionLabel, outcome).inc()+ this.partitionDuration.labels(partitionLabel, outcome).observe(durationSeconds)+ }+ }+ public static incRetryCause(cause: TransientFetchOutcome): void {+ this.retryCauses.labels(cause).inc()+ }+ public static observeRedirectCount(redirects: number): void {+ this.redirects.observe(redirects)+ }+ public static observeCompletedUrl(+ outcome: AttemptOutcome,+ refusalReason: FetchRefusalReason | 'none',+ systemTimeSeconds: number,+ fetches: number,+ republishes: number+ ): void {+ this.completedUrls.labels(outcome, refusalReason).inc()+ this.completedUrlSystemTime.observe(systemTimeSeconds)+ this.completedUrlFetches.observe(fetches)+ this.completedUrlRepublishes.observe(republishes)+ }+ public static observePolicyAndBudgetDecision(blocked: boolean, reason: PolicyAndBudgetReason = 'none'): void {+ this.policyAndBudgetDecisions.labels(blocked ? 'true' : 'false', reason).inc()+ }+ public static observeBatchSchedulableCapacity(slots: number, podRequestLimit: number): void {+ const boundedSlots = Math.min(slots, podRequestLimit)+ this.batchSchedulableSlots.observe(boundedSlots)+ this.batchSchedulableCapacityRatio.observe(boundedSlots / podRequestLimit)+ }+ public static observeSchedulerWait(+ scope: SchedulerWaitScope,+ waitSeconds: number,+ sourcePartitions?: readonly number[]+ ): void {+ this.schedulerWait.labels(scope).observe(waitSeconds)+ for (const sourcePartition of new Set(sourcePartitions ?? [])) {+ this.partitionSchedulerWait.labels(String(sourcePartition), scope).observe(waitSeconds)+ }+ }+ public static incPartitionAttempt(+ partition: number,+ disposition: PartitionAttemptDisposition,+ outcome: AttemptOutcome+ ): void {+ this.partitionAttempts.labels(String(partition), disposition, outcome).inc()+ }+ public static observeBytes(bytes: number): void {+ this.responseBytes.observe(bytes)+ }+ private static readonly evictedWhileBlocked = new Gauge({+ name: 'ai_research_v4_image_fetch_registrable_domains_evicted_while_blocked',+ help: 'Registrable domains removed from request-control state while still blocked. This must stay zero',+ collect() {+ this.set(ImageFetchRequestMetrics.budget?.evictedWhileBlocked ?? 0)+ },+ })++ /**+ * Requests holding a socket right now, against the pod limit.+ *+ * A request waiting for a politeness token has not reached this yet, so a value at the limit+ * means the pod is the bottleneck rather than the sites.+ */+ private static readonly inFlight = new Gauge({+ name: 'ai_research_v4_image_fetch_requests_in_flight',+ help: 'External requests this pod holds open. A URL waiting for its registrable-domain rate limit holds no socket and is not counted',+ collect() {+ this.set(ImageFetchRequestMetrics.requests?.running ?? 0)+ },+ })++ private static budget: BudgetCounts | undefined+ private static requests: InFlightCount | undefined++ /** The runner owns both. */+ public static trackBudget(budget: BudgetCounts, requests: InFlightCount): void {+ this.budget = budget+ this.requests = requests+ }++ /**+ * Read against `ai_research_v4_image_fetch_requests_total` this is the amplification factor, because the lane+ * reads every republished message again. A rate that approaches the request rate means most work+ * goes around rather than completes.+ */+ private static readonly republished = new Counter({+ name: 'ai_research_v4_image_fetch_republished_total',+ help: 'URLs published back to Kafka by bounded reason and destination class. "redirect" left the origin. "retry" hit a transient failure and waits in a delay topic. "not_ready" arrived before its wait ended',+ labelNames: ['reason', 'topic'],+ })+ private static readonly republishFailed = new Counter({+ name: 'ai_research_v4_image_fetch_republish_failed_total',+ help: 'URLs in a Kafka message whose republish delivery failed. The input batch throws and leaves its offsets uncommitted',+ labelNames: ['reason'],+ })+ private static readonly republishMessagesPerBatch = new Histogram({+ name: 'ai_research_v4_image_fetch_republish_messages_per_batch',+ help: 'Kafka record delivery attempts by one fetch batch for each destination topic class',+ labelNames: ['topic'],+ buckets: [1, 2, 4, 8, 16, 32, 64, 128, 256, 512, 1024, 2048, 4096, 8192, 16_384],+ })+ private static readonly republishRegistrableDomainsPerBatch = new Histogram({+ name: 'ai_research_v4_image_fetch_republish_registrable_domains_per_batch',+ help: 'Distinct registrable-domain keys in delivery attempts by one fetch batch for each destination topic class',+ labelNames: ['topic'],+ buckets: [1, 2, 4, 8, 16, 32, 64, 128, 256, 512, 1024, 2048, 4096, 8192, 16_384],+ })+ private static readonly republishDuration = new Histogram({+ name: 'ai_research_v4_image_fetch_republish_duration_seconds',+ help: 'Wall time from scheduling one destination topic class until all its started Kafka delivery attempts settle',+ labelNames: ['topic'],+ buckets: [0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10, 30, 60, 120, 240, 300],+ })+ private static readonly republishFlushDuration = new Histogram({+ name: 'ai_research_v4_image_fetch_republish_flush_duration_seconds',+ help: 'Total wall time to group, serialize, and wait for every started republish delivery in one fetch batch',+ buckets: [0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10, 30, 60, 120, 240, 300],+ })+ private static readonly republishFlushDeadlineExceeded = new Counter({+ name: 'ai_research_v4_image_fetch_republish_flush_deadline_exceeded_total',+ help: 'Fetch batches that stopped starting Kafka republish records to preserve final history-write and poll-interval margin',+ })+ /**+ * Zero is the common case. The tail shows redirect chains and retries.+ */+ private static readonly hopsUsed = new Histogram({+ name: 'ai_research_v4_image_fetch_hops_used',+ help: 'Redirects and retries spent from one URL job before the lane finished it',+ buckets: [0, 1, 2, 3, 5, 10],+ })++ public static incRepublished(reason: RepublishReason, destination: RepublishDestination): void {+ this.republished.labels(reason, destination).inc()+ }+ public static incRepublishFailed(reason: RepublishReason): void {+ this.republishFailed.labels(reason).inc()+ }+ public static observeRepublishBatch(+ topic: RepublishTopic,+ messages: number,+ registrableDomains: number,+ durationSeconds: number+ ): void {+ this.republishMessagesPerBatch.labels(topic).observe(messages)+ this.republishRegistrableDomainsPerBatch.labels(topic).observe(registrableDomains)+ this.republishDuration.labels(topic).observe(durationSeconds)+ }+ public static observeRepublishFlush(durationSeconds: number): void {+ this.republishFlushDuration.observe(durationSeconds)+ }+ public static incRepublishFlushDeadlineExceeded(): void {+ this.republishFlushDeadlineExceeded.inc()+ }+ public static observeHops(hops: number): void {+ this.hopsUsed.observe(hops)+ }+}++/**+ * The consumer of one delay topic.+ *+ * Lag on these topics is the design at work rather than a fault, because a record that waits out its+ * period counts as lag. Read `ai_research_v4_image_fetch_retry_wait_seconds` against the period of the topic+ * instead. A wait far short of the period means records arrive with their wait already spent, which+ * means the tier below is too small.+ */+export class RetryDelayMetrics {+ private static readonly released = new Counter({+ name: 'ai_research_v4_image_fetch_retry_released_total',+ help: 'Records this delay consumer handled by outcome. A malformed record is dropped, while an invalid timestamp or failed publish leaves the offset uncommitted',+ labelNames: ['outcome'],+ })+ private static readonly waitSeconds = new Histogram({+ name: 'ai_research_v4_image_fetch_retry_wait_seconds',+ help: 'Time a record still had to wait when this consumer read it. Compare with the period of the topic: a much shorter wait means the records arrived late, and the consumer is behind',+ buckets: [1, 10, 60, 300, 600, 1800, 3600],+ })++ public static incReleased(+ outcome: 'released' | 'failed' | 'malformed' | 'invalid_timestamp' | 'abandoned',+ count = 1+ ): void {+ this.released.labels(outcome).inc(count)+ }+ public static observeWait(waitSeconds: number): void {+ this.waitSeconds.observe(waitSeconds)+ }+}++export class ImageFetchPoolMetrics {+ private static readonly waitSeconds = new Histogram({+ name: 'ai_research_v4_image_fetch_pool_wait_seconds',+ help: 'Time a candidate waited in the pod candidate pool before a fetch worker took it',+ buckets: [0, 0.1, 0.5, 1, 2, 5, 10, 20, 40],+ })+ private static readonly refillWaitSeconds = new Histogram({+ name: 'ai_research_v4_image_fetch_pool_refill_wait_seconds',+ help: 'Time a consumer waited for room in its pool window before it read its next batch. outcome=timeout means the wait reached its cap',+ labelNames: ['outcome'],+ buckets: [0, 0.1, 0.5, 1, 2, 5, 10, 30],+ })++ public static observeWait(waitSeconds: number): void {+ this.waitSeconds.observe(waitSeconds)+ }+ public static observeRefillWait(outcome: 'room' | 'timeout', waitSeconds: number): void {+ this.refillWaitSeconds.labels(outcome).observe(waitSeconds)+ }+}diff --git a/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/origin-request-scheduler.test.ts b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/origin-request-scheduler.test.tsnew file mode 100644index 000000000000..3e610226aaff--- /dev/null+++ b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/origin-request-scheduler.test.ts@@ -0,0 +1,307 @@+import { register } from 'prom-client'++import { createRecordingTopHog } from '~/tests/helpers/tophog'++import { HostBudget, HostBudgetOptions } from './host-budget'+import { ImageFetchRequestMetrics } from './metrics'+import { OriginRequestScheduler } from './origin-request-scheduler'+import { ImageFetchTopHogMetrics } from './tophog-metrics'++const OPTIONS: HostBudgetOptions = {+ requestsPerSecond: 1,+ burst: 5,+ maxConcurrent: 6,+ breakerFailures: 2,+ breakerCooldownMs: 60_000,+ breakerMaxCooldownMs: 3_600_000,+ maxTrackedRegistrableDomains: 20_000,+ maxTrackedOrigins: 20_000,+ random: () => 1,+}+const ORIGIN = new URL('https://example.com')+const REGISTRABLE_DOMAIN = 'example.com'++describe('OriginRequestScheduler', () => {+ beforeEach(() => jest.useFakeTimers().setSystemTime(1_700_000_000_000))+ afterEach(() => {+ jest.restoreAllMocks()+ jest.useRealTimers()+ })++ it('keeps concurrent request start times at least one second apart', async () => {+ register.resetMetrics()+ const budget = new HostBudget(OPTIONS)+ budget.setCrawlDelay(ORIGIN.origin, 1_000, Date.now())+ const recordingTopHog = createRecordingTopHog()+ const scheduler = new OriginRequestScheduler(budget, 300, new ImageFetchTopHogMetrics(recordingTopHog.registry))+ const observeSchedulerWait = jest.spyOn(ImageFetchRequestMetrics, 'observeSchedulerWait')+ const startedAtMs: number[] = []+ const deadlineMs = Date.now() + 10_000++ const requests = [0, 1, 2].map(() =>+ scheduler.runImage(+ ORIGIN,+ deadlineMs,+ () => {+ startedAtMs.push(Date.now())+ return Promise.resolve()+ },+ [7, 42]+ )+ )+ await jest.advanceTimersByTimeAsync(0)+ const active = register.getSingleMetric('ai_research_v4_image_fetch_stage_active')!+ expect((await active.get()).values).toEqual(+ expect.arrayContaining([+ expect.objectContaining({ labels: { stage: 'image_origin_crawl_delay' }, value: 2 }),+ expect.objectContaining({ labels: { stage: 'image_http' }, value: 0 }),+ ])+ )+ await jest.runAllTimersAsync()+ expect((await active.get()).values.every(({ value }) => value === 0)).toBe(true)+ const durations = await register.getSingleMetric('ai_research_v4_image_fetch_stage_duration_seconds')!.get()+ expect(durations.values).toEqual(+ expect.arrayContaining([+ expect.objectContaining({+ metricName: 'ai_research_v4_image_fetch_stage_duration_seconds_sum',+ labels: { stage: 'image_origin_crawl_delay' },+ value: 3,+ }),+ ])+ )++ expect(await Promise.all(requests)).toEqual([+ { ran: true, value: undefined },+ { ran: true, value: undefined },+ { ran: true, value: undefined },+ ])+ expect(startedAtMs).toEqual([1_700_000_000_000, 1_700_000_001_000, 1_700_000_002_000])+ expect(observeSchedulerWait).toHaveBeenCalledWith('origin_crawl_delay', 1, [7, 42])+ expect(observeSchedulerWait).toHaveBeenCalledWith('request_capacity', 0, [7, 42])+ expect(recordingTopHog.records.get('ai_research_v4_image_fetch_block_events_by_registrable_domain')).toEqual(+ expect.arrayContaining([+ {+ key: {+ registrable_domain: REGISTRABLE_DOMAIN,+ reason: 'origin_crawl_delay',+ partition: '7',+ },+ value: 1,+ },+ ])+ )+ expect(recordingTopHog.records.get('ai_research_v4_image_fetch_blocked_ms_by_registrable_domain')).toEqual(+ expect.arrayContaining([+ {+ key: {+ registrable_domain: REGISTRABLE_DOMAIN,+ reason: 'origin_crawl_delay',+ partition: '7',+ },+ value: 1_000,+ },+ ])+ )+ expect(+ recordingTopHog.records+ .get('ai_research_v4_image_fetch_block_events_by_registrable_domain')+ ?.some(({ key }) => key.reason === 'request_capacity')+ ).toBe(false)+ })++ it('allows six concurrent same-origin requests when the request rate and crawl delay are disabled', async () => {+ const scheduler = new OriginRequestScheduler(+ new HostBudget({ ...OPTIONS, requestsPerSecond: 0, burst: 6, maxConcurrent: 6 }),+ 300+ )+ const releases: Array<() => void> = []+ const requests = Array.from({ length: 7 }, () =>+ scheduler.runImage(+ ORIGIN,+ Date.now() + 10_000,+ () =>+ new Promise<void>((resolve) => {+ releases.push(resolve)+ })+ )+ )++ await expect(requests[6]).resolves.toEqual({+ ran: false,+ reason: 'connection_limit',+ blockingReason: 'connection_limit',+ waitMs: 0,+ })+ expect(releases).toHaveLength(6)+ releases.forEach((release) => release())+ await expect(Promise.all(requests.slice(0, 6))).resolves.toEqual(+ Array.from({ length: 6 }, () => ({ ran: true, value: undefined }))+ )+ })++ it('runs one half-open probe while later requests remain blocked', async () => {+ const budget = new HostBudget(OPTIONS)+ budget.recordTransientFailure(REGISTRABLE_DOMAIN, Date.now())+ budget.recordTransientFailure(REGISTRABLE_DOMAIN, Date.now())+ jest.setSystemTime(Date.now() + 120_001)+ const scheduler = new OriginRequestScheduler(budget, 300)+ let finishProbe: (() => void) | undefined+ const probe = scheduler.runImage(+ ORIGIN,+ Date.now() + 10_000,+ () =>+ new Promise<void>((resolve) => {+ finishProbe = resolve+ })+ )++ await Promise.resolve()+ await expect(scheduler.runImage(ORIGIN, Date.now() + 10_000, () => Promise.resolve())).resolves.toEqual({+ ran: false,+ reason: 'breaker_open',+ blockingReason: 'breaker_open',+ waitMs: OPTIONS.breakerCooldownMs,+ })+ expect(finishProbe).toBeDefined()+ finishProbe?.()+ await expect(probe).resolves.toEqual({ ran: true, value: undefined })+ })++ it('preserves the response backoff reason', async () => {+ const budget = new HostBudget(OPTIONS)+ budget.recordTransientFailure(REGISTRABLE_DOMAIN, Date.now(), 120_000)+ const scheduler = new OriginRequestScheduler(budget, 300)++ await expect(scheduler.runImage(ORIGIN, Date.now() + 10_000, () => Promise.resolve())).resolves.toEqual({+ ran: false,+ reason: 'backoff',+ blockingReason: 'retry_after',+ waitMs: 120_000,+ })+ })++ it('returns a queued request to the caller when its pass deadline expires', async () => {+ const scheduler = new OriginRequestScheduler(new HostBudget(OPTIONS), 1)+ let releaseFirst: (() => void) | undefined+ const first = scheduler.runImage(+ new URL('https://first.example'),+ Date.now() + 10_000,+ () =>+ new Promise<void>((resolve) => {+ releaseFirst = resolve+ })+ )+ const second = scheduler.runImage(new URL('https://second.example'), Date.now() + 500, () => Promise.resolve())++ await Promise.resolve()+ jest.advanceTimersByTime(501)+ releaseFirst?.()++ await expect(first).resolves.toEqual({ ran: true, value: undefined })+ await expect(second).resolves.toEqual({+ ran: false,+ reason: 'deadline',+ blockingReason: 'request_deadline',+ waitMs: 0,+ })+ })++ it('checks the registrable-domain token bucket after a pod-capacity wait', async () => {+ const scheduler = new OriginRequestScheduler(new HostBudget({ ...OPTIONS, burst: 2 }), 1)+ const startedAtMs: number[] = []+ let releaseFirst: (() => void) | undefined+ const startRequest = (origin: string, hold = false): Promise<unknown> =>+ scheduler.runImage(new URL(origin), Date.now() + 30_000, () => {+ startedAtMs.push(Date.now())+ if (!hold) {+ return Promise.resolve()+ }+ return new Promise<void>((resolve) => {+ releaseFirst = resolve+ })+ })++ const requests = [+ startRequest('https://a.example.com/image.png', true),+ startRequest('https://b.example.com/image.png'),+ startRequest('https://c.example.com/image.png'),+ startRequest('https://d.example.com/image.png'),+ ]+ await Promise.resolve()+ jest.advanceTimersByTime(10_000)+ releaseFirst?.()+ await jest.runAllTimersAsync()+ await Promise.all(requests)++ expect(startedAtMs).toEqual([1_700_000_000_000, 1_700_000_010_000, 1_700_000_010_000, 1_700_000_011_000])+ })++ it('returns a crawl wait that is longer than the pass deadline', async () => {+ const budget = new HostBudget(OPTIONS)+ budget.setCrawlDelay(ORIGIN.origin, 600_000, Date.now())+ const scheduler = new OriginRequestScheduler(budget, 300)+ await expect(scheduler.runImage(ORIGIN, Date.now() + 10_000, () => Promise.resolve())).resolves.toMatchObject({+ ran: true,+ })++ await expect(scheduler.runImage(ORIGIN, Date.now() + 20_000, () => Promise.resolve())).resolves.toEqual({+ ran: false,+ reason: 'deadline',+ blockingReason: 'origin_crawl_delay',+ waitMs: 600_000,+ })+ })++ it('shares the connection limit across sibling origins', async () => {+ const scheduler = new OriginRequestScheduler(new HostBudget({ ...OPTIONS, maxConcurrent: 1 }), 300)+ let releaseFirst: (() => void) | undefined+ const first = scheduler.runImage(+ new URL('https://a.example.com/image.png'),+ Date.now() + 10_000,+ () =>+ new Promise<void>((resolve) => {+ releaseFirst = resolve+ })+ )++ await Promise.resolve()+ await expect(+ scheduler.runImage(new URL('https://b.example.com/image.png'), Date.now() + 10_000, () => Promise.resolve())+ ).resolves.toEqual({+ ran: false,+ reason: 'connection_limit',+ blockingReason: 'connection_limit',+ waitMs: 0,+ })++ releaseFirst?.()+ await expect(first).resolves.toEqual({ ran: true, value: undefined })+ })++ it('counts a configuration request against the registrable-domain connection limit', async () => {+ const scheduler = new OriginRequestScheduler(new HostBudget({ ...OPTIONS, maxConcurrent: 1 }), 300)+ let releaseConfiguration: (() => void) | undefined+ const configuration = scheduler.run(+ new URL('https://a.example.com/robots.txt'),+ Date.now() + 10_000,+ () =>+ new Promise<void>((resolve) => {+ releaseConfiguration = resolve+ })+ )++ await Promise.resolve()+ await expect(+ scheduler.runImage(new URL('https://b.example.com/image.png'), Date.now() + 10_000, () => Promise.resolve())+ ).resolves.toEqual({+ ran: false,+ reason: 'connection_limit',+ blockingReason: 'connection_limit',+ waitMs: 0,+ })++ expect(releaseConfiguration).toBeDefined()+ releaseConfiguration?.()+ await expect(configuration).resolves.toEqual({ ran: true, value: undefined })+ })+})diff --git a/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/origin-request-scheduler.ts b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/origin-request-scheduler.tsnew file mode 100644index 000000000000..d7be78697dee--- /dev/null+++ b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/origin-request-scheduler.ts@@ -0,0 +1,192 @@+import { ConcurrencyController } from '~/common/utils/concurrencyController'+import { delay } from '~/common/utils/utils'++import type { ImageFetchBlockReason } from './block-reason'+import { ConfigurationRequestScheduler } from './configuration-policy'+import { BudgetBlockReason, BudgetGrant, HostBudget } from './host-budget'+import { ImageFetchRequestMetrics, SchedulerWaitScope } from './metrics'+import { politenessKey } from './politeness-key'+import { ImageFetchProcessingMetrics } from './processing-metrics'+import { ImageFetchTopHogMetrics } from './tophog-metrics'++export type ScheduledRequest<T> =+ | { ran: true; value: T }+ | {+ ran: false+ reason: BudgetBlockReason | 'connection_limit'+ blockingReason: ImageFetchBlockReason+ waitMs: number+ }++function blockingReasonForStoppedGrant(grant: Extract<BudgetGrant, { granted: false }>): ImageFetchBlockReason {+ if (grant.reason === 'backoff') {+ return grant.backoffReason ?? 'unknown_backoff'+ }+ if (grant.reason === 'deadline') {+ return grant.waitScope ?? 'request_deadline'+ }+ return grant.reason+}++export class OriginRequestScheduler implements ConfigurationRequestScheduler {+ private readonly inFlight: ConcurrencyController++ constructor(+ private readonly budget: HostBudget,+ maxInFlightRequests: number,+ private readonly topHogMetrics?: ImageFetchTopHogMetrics+ ) {+ this.inFlight = new ConcurrencyController(maxInFlightRequests)+ }++ public run<T>(url: URL, deadlineMs: number, request: () => Promise<T>): Promise<ScheduledRequest<T>> {+ return this.runScheduled(url, deadlineMs, true, undefined, request)+ }++ public runImage<T>(+ url: URL,+ deadlineMs: number,+ request: () => Promise<T>,+ sourcePartitions?: readonly number[]+ ): Promise<ScheduledRequest<T>> {+ return this.runScheduled(url, deadlineMs, false, sourcePartitions, request)+ }++ public get running(): number {+ return this.inFlight.running+ }++ private async runScheduled<T>(+ url: URL,+ deadlineMs: number,+ configurationRequest: boolean,+ sourcePartitions: readonly number[] | undefined,+ request: () => Promise<T>+ ): Promise<ScheduledRequest<T>> {+ const requestKind = configurationRequest ? 'configuration' : 'image'+ const origin = url.origin+ const registrableDomain = politenessKey(url.hostname)+ const nowMs = Date.now()+ if (!this.budget.requestScheduled(origin, nowMs)) {+ return { ran: false, reason: 'origin_map_full', blockingReason: 'origin_map_full', waitMs: 0 }+ }+ try {+ for (;;) {+ const capacityWaitStartedAtMs = Date.now()+ const capacityWait = ImageFetchProcessingMetrics.start(`${requestKind}_request_capacity`)+ const scheduled = await this.inFlight+ .run({+ debugTag: registrableDomain,+ fn: async () => {+ capacityWait.finish()+ const checkedAtMs = Date.now()+ this.recordSchedulerWait(+ 'request_capacity',+ Math.max(0, checkedAtMs - capacityWaitStartedAtMs),+ registrableDomain,+ configurationRequest,+ sourcePartitions+ )+ const grant = this.budget.take(+ registrableDomain,+ origin,+ checkedAtMs,+ deadlineMs,+ configurationRequest+ )+ if (!grant.granted) {+ return {+ kind: 'stopped',+ reason: grant.reason,+ blockingReason: blockingReasonForStoppedGrant(grant),+ waitMs: grant.waitMs,+ } as const+ }+ if (grant.waitMs > 0) {+ return {+ kind: 'wait',+ waitMs: grant.waitMs,+ waitScope: grant.waitScope ?? 'registrable_domain_rate',+ } as const+ }+ if (!this.budget.acquireConnection(registrableDomain, origin)) {+ this.budget.returnGrant(+ registrableDomain,+ origin,+ checkedAtMs,+ grant.reservedStartAtMs,+ grant.halfOpenProbe+ )+ return {+ kind: 'stopped',+ reason: 'connection_limit' as const,+ blockingReason: 'connection_limit' as const,+ waitMs: 0,+ } as const+ }+ try {+ this.budget.markRequestStarted(+ registrableDomain,+ origin,+ Date.now(),+ grant.reservedStartAtMs,+ configurationRequest ? 'configuration' : 'image'+ )+ return {+ kind: 'ran',+ value: await ImageFetchProcessingMetrics.measure(`${requestKind}_http`, request),+ } as const+ } finally {+ this.budget.releaseConnection(registrableDomain, origin)+ }+ },+ })+ .finally(() => capacityWait.finish())+ if (scheduled.kind === 'ran') {+ return { ran: true, value: scheduled.value }+ }+ if (scheduled.kind === 'stopped') {+ return {+ ran: false,+ reason: scheduled.reason,+ blockingReason: scheduled.blockingReason,+ waitMs: scheduled.waitMs,+ }+ }+ if (Date.now() + scheduled.waitMs > deadlineMs) {+ return {+ ran: false,+ reason: 'deadline',+ blockingReason: scheduled.waitScope,+ waitMs: scheduled.waitMs,+ }+ }+ this.recordSchedulerWait(+ scheduled.waitScope,+ scheduled.waitMs,+ registrableDomain,+ configurationRequest,+ sourcePartitions+ )+ await ImageFetchProcessingMetrics.measure(`${requestKind}_${scheduled.waitScope}`, () =>+ delay(scheduled.waitMs)+ )+ }+ } finally {+ this.budget.requestFinished(origin)+ }+ }++ private recordSchedulerWait(+ scope: SchedulerWaitScope,+ waitMs: number,+ registrableDomain: string,+ configurationRequest: boolean,+ sourcePartitions: readonly number[] | undefined+ ): void {+ ImageFetchRequestMetrics.observeSchedulerWait(scope, waitMs / 1000, sourcePartitions)+ if (!configurationRequest) {+ this.topHogMetrics?.recordSchedulerWait(registrableDomain, sourcePartitions, scope, waitMs)+ }+ }+}diff --git a/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/politeness-key.test.ts b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/politeness-key.test.tsnew file mode 100644index 000000000000..3e6348fce64b--- /dev/null+++ b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/politeness-key.test.ts@@ -0,0 +1,50 @@+import { isPublicHost, politenessKey, tryCanonicalizeUrl } from '@posthog/ai-research-v4-native'++/**+ * The crate owns the rule and tests it. This file tests only that the fetch lane reaches that+ * function across the FFI boundary, so every case here asserts behaviour the crate already had.+ */+describe('the v4 url policy addon', () => {+ it('returns the canonical forms of a url and refuses a private host', () => {+ // Values from the url_policy tests in replay-core-v4: the dedup form drops the volatile+ // parameters, and the fetch form keeps every byte.+ const raw = 'https://cdn.example.com/a.png?w=200&%63b=first&CB=second&no%63ache=third&RND=fourth'+ expect(tryCanonicalizeUrl(raw)).toEqual({+ ok: true,+ url: {+ fetch: raw,+ dedup: 'https://cdn.example.com/a.png?w=200',+ host: 'cdn.example.com',+ domain: 'example.com',+ },+ })+ expect(tryCanonicalizeUrl('https://169.254.169.254/latest/meta-data/iam/security-credentials/')).toEqual({+ ok: false,+ decline: 'non_public_host',+ unwanted: false,+ })+ expect(isPublicHost('169.254.169.254')).toBe(false)+ })++ describe('politenessKey', () => {+ it.each([['d111.cloudfront.net'], ['bucket.s3.amazonaws.com'], ['user.github.io'], ['myapp.vercel.app']])(+ 'keeps %s as its own operator',+ (host) => {+ // Each provider sits in the private section of the public suffix list. A list read+ // without that section folds every tenant of one provider into a single rate budget.+ expect(politenessKey(host)).toBe(host)+ }+ )++ it.each([+ ['img1.cdn.example.com', 'example.com'],+ ['a.b.example.co.uk', 'example.co.uk'],+ ])('folds %s onto %s', (host, expected) => {+ expect(politenessKey(host)).toBe(expected)+ })++ it.each([['203.0.113.4'], ['::1']])('returns the address %s unchanged', (host) => {+ expect(politenessKey(host)).toBe(host)+ })+ })+})diff --git a/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/politeness-key.ts b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/politeness-key.tsnew file mode 100644index 000000000000..c3b30169c8dc--- /dev/null+++ b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/politeness-key.ts@@ -0,0 +1,59 @@+/**+ * The URL policy of the collector, reached through the v4 native addon rather than reimplemented+ * here. One public suffix list keeps producer and consumer canonicalization aligned.+ *+ * The addon is a native library, and `index.ts` imports every server whatever mode the pod runs.+ * This loads the addon on first use rather than at import, so only a pod in this lane pays for it.+ * The fetch server defers it the same way.+ */+let cached: typeof import('@posthog/ai-research-v4-native') | undefined++function addon(): typeof import('@posthog/ai-research-v4-native') {+ if (!cached) {+ const loaded = require('@posthog/ai-research-v4-native') as typeof import('@posthog/ai-research-v4-native')+ if (+ typeof loaded.politenessKey !== 'function' ||+ typeof loaded.isPublicHost !== 'function' ||+ typeof loaded.tryCanonicalizeUrl !== 'function'+ ) {+ throw new Error('the ai-research-v4-native addon is missing the url policy: rebuild index.node')+ }+ cached = loaded+ }+ return cached+}++/**+ * Load the addon now and run one URL through it, so a stale `index.node` stops the pod at startup.+ *+ * The parser calls into it for every URL. A stale addon must stop the pod before it reads a batch.+ */+export function assertUrlPolicyLoaded(): void {+ const verdict = addon().tryCanonicalizeUrl('https://example.com/probe.png')+ if (!verdict.ok) {+ throw new Error(`the ai-research-v4-native addon refused a plain URL (${verdict.decline}): rebuild index.node`)+ }+}++export function politenessKey(host: string): string {+ return addon().politenessKey(host)+}++/**+ * False for a host the collector would decline, including private and reserved names.+ *+ * The connection layer refuses a private address, so this is not the only guard against one. It is+ * the only guard against a name that looks internal and whose DNS answer is public, because no+ * address check can refuse that.+ */+export function isPublicHost(host: string): boolean {+ return addon().isPublicHost(host)+}++export function canonicalizeUrl(url: string): import('@posthog/ai-research-v4-native').CanonicalUrl | null {+ return addon().canonicalizeUrl(url)+}++export function tryCanonicalizeUrl(url: string): import('@posthog/ai-research-v4-native').UrlPolicyVerdict {+ return addon().tryCanonicalizeUrl(url)+}diff --git a/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/pool-window-batch-handler.test.ts b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/pool-window-batch-handler.test.tsnew file mode 100644index 000000000000..500ddf7fef84--- /dev/null+++ b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/pool-window-batch-handler.test.ts@@ -0,0 +1,77 @@+import { Message } from 'node-rdkafka'++import { MAX_HOPS } from './collected-urls-record'+import { FetchCandidatePool, FetchCandidatePoolAdmission } from './fetch-candidate-pool'+import { createPoolWindowBatchHandler } from './pool-window-batch-handler'++const NOW_MS = 1_700_000_000_000+const MESSAGE: Message = {+ topic: 'session_replay_image_fetch',+ partition: 3,+ offset: 1,+ value: Buffer.from('x'),+ size: 1,+}++function distinctDomainCandidates(count: number): Parameters<FetchCandidatePool<string>['add']>[0] {+ return Array.from({ length: count }, (_unused, index) => ({+ originalRef: `imageurl:${index}`,+ currentUrl: `https://cdn${index}.example${index}.com/a.png`,+ host: `cdn${index}.example${index}.com`,+ origin: `https://cdn${index}.example${index}.com`,+ registrableDomain: `example${index}.com`,+ remainingHops: MAX_HOPS,+ notBeforeMs: 0,+ firstSeenAtMs: NOW_MS,+ fetchCount: 0,+ republishCount: 0,+ lastRepublishReason: null,+ }))+}++describe('createPoolWindowBatchHandler', () => {+ beforeEach(() => jest.useFakeTimers().setSystemTime(NOW_MS))+ afterEach(() => jest.useRealTimers())++ it.each([+ ['returns once the batch enters the pool and the window has room', 0, 500],+ ['returns at the wait cap while the window stays full', 20, 30_000],+ ])('%s', async (_name, queuedAfterAdmission, expectedReturnMs) => {+ const pool = new FetchCandidatePool<string>(+ { maxConcurrentPerRegistrableDomain: 6, refillRunnableUrls: 10, maxQueuedUrlsPerOwner: 1_000 },+ { originCrawlDelayMs: () => 0, originNextImageStartAtMs: () => 0 }+ )+ const owner = pool.createOwner()+ let finishBatch: () => void = () => undefined+ const processBatch = jest.fn(+ (_messages: Message[], admission: FetchCandidatePoolAdmission) =>+ new Promise<void>((resolve) => {+ setTimeout(() => {+ pool.add(+ distinctDomainCandidates(queuedAfterAdmission),+ 'batch',+ NOW_MS + 40_000,+ admission.owner+ )+ admission.admitted()+ }, 500)+ finishBatch = resolve+ })+ )+ const handler = createPoolWindowBatchHandler(owner, processBatch)++ let returnedAtMs: number | undefined+ const returned = handler([MESSAGE]).then((result) => {+ returnedAtMs = Date.now()+ return result+ })+ await jest.advanceTimersByTimeAsync(30_000)+ const result = await returned+ finishBatch()++ expect(processBatch).toHaveBeenCalledTimes(1)+ expect(returnedAtMs! - NOW_MS).toBe(expectedReturnMs)+ await expect(result?.backgroundTask).resolves.toBeUndefined()+ pool.close()+ })+})diff --git a/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/pool-window-batch-handler.ts b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/pool-window-batch-handler.tsnew file mode 100644index 000000000000..c0e7b7994316--- /dev/null+++ b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/pool-window-batch-handler.ts@@ -0,0 +1,38 @@+import { Message } from 'node-rdkafka'++import { FetchCandidatePoolAdmission, FetchCandidatePoolOwner } from './fetch-candidate-pool'+import { ImageFetchPoolMetrics } from './metrics'++/** consumer-v2 reports a stalled loop after 60 s without a tick, and this wait holds the loop. */+export const POOL_ROOM_WAIT_CAP_MS = 30_000++type BatchProcessor = (messages: Message[], admission: FetchCandidatePoolAdmission) => Promise<void>++/**+ * The batch handler of one consumer in continuous-pool mode.+ *+ * It starts the batch at once and returns only when the consumer's pool window has room, so the+ * consumer reads its next batch while the pool still holds work for it.+ */+export function createPoolWindowBatchHandler(+ owner: FetchCandidatePoolOwner,+ processBatch: BatchProcessor,+ roomWaitCapMs = POOL_ROOM_WAIT_CAP_MS+): (messages: Message[]) => Promise<{ backgroundTask: Promise<void> } | void> {+ return async (messages) => {+ if (messages.length === 0) {+ return+ }+ const admission = owner.beginAdmission()+ const backgroundTask = processBatch(messages, admission).finally(() => admission.admitted())+ // consumer-v2 attaches its own handler after this returns, so a failure during the wait must not count as unhandled.+ void backgroundTask.catch(() => undefined)+ const waitStartedAtMs = performance.now()+ const hadRoom = await owner.waitForRoom(roomWaitCapMs)+ ImageFetchPoolMetrics.observeRefillWait(+ hadRoom ? 'room' : 'timeout',+ (performance.now() - waitStartedAtMs) / 1000+ )+ return { backgroundTask }+ }+}diff --git a/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/processing-metrics.test.ts b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/processing-metrics.test.tsnew file mode 100644index 000000000000..5dcaf781d61a--- /dev/null+++ b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/processing-metrics.test.ts@@ -0,0 +1,89 @@+import { register } from 'prom-client'++import { ConcurrencyController } from '~/common/utils/concurrencyController'++import { ImageFetchProcessingMetrics } from './processing-metrics'++async function sample(name: string, stage: string): Promise<number | undefined> {+ const metric = await register.getSingleMetric(name)!.get()+ return metric.values.find((value) => value.labels.stage === stage)?.value+}++describe('ImageFetchProcessingMetrics', () => {+ beforeEach(() => register.resetMetrics())+ afterEach(() => jest.restoreAllMocks())++ it('keeps the oldest overlapping operation visible and removes finished stages', async () => {+ const clock = jest.spyOn(performance, 'now').mockReturnValue(1000)+ const first = ImageFetchProcessingMetrics.start('batch_fetch')+ clock.mockReturnValue(2000)+ const second = ImageFetchProcessingMetrics.start('batch_fetch')+ clock.mockReturnValue(5000)+ expect(await sample('ai_research_v4_image_fetch_stage_active', 'batch_fetch')).toBe(2)+ expect(await sample('ai_research_v4_image_fetch_stage_oldest_seconds', 'batch_fetch')).toBe(4)+ first.move('batch_history_write')+ expect(await sample('ai_research_v4_image_fetch_stage_oldest_seconds', 'batch_fetch')).toBe(3)+ second.finish()+ second.finish()+ first.finish()+ expect(await sample('ai_research_v4_image_fetch_stage_active', 'batch_fetch')).toBe(0)+ expect(await sample('ai_research_v4_image_fetch_stage_oldest_seconds', 'batch_fetch')).toBe(0)+ const duration = await register.getSingleMetric('ai_research_v4_image_fetch_stage_duration_seconds')!.get()+ expect(duration.values).toEqual(+ expect.arrayContaining([+ expect.objectContaining({+ metricName: 'ai_research_v4_image_fetch_stage_duration_seconds_count',+ labels: { stage: 'batch_fetch' },+ value: 2,+ }),+ expect.objectContaining({+ metricName: 'ai_research_v4_image_fetch_stage_duration_seconds_sum',+ labels: { stage: 'batch_fetch' },+ value: 7,+ }),+ ])+ )+ })++ it('separates admission from running work and cleans up a rejected operation', async () => {+ const controller = new ConcurrencyController(1)+ let release!: () => void+ const blocked = new Promise<void>((resolve) => {+ release = resolve+ })+ const first = ImageFetchProcessingMetrics.runLimited(controller, 'candidate_admission', 'candidate_work', {+ debugTag: 'example.com',+ fn: () => blocked,+ })+ const error = new Error('synthetic failure')+ const second = ImageFetchProcessingMetrics.runLimited(controller, 'candidate_admission', 'candidate_work', {+ debugTag: 'example.org',+ fn: () => Promise.reject(error),+ })+ const rejected = expect(second).rejects.toBe(error)+ expect(await sample('ai_research_v4_image_fetch_stage_active', 'candidate_admission')).toBe(1)+ expect(await sample('ai_research_v4_image_fetch_stage_active', 'candidate_work')).toBe(1)+ release()+ await first+ await rejected+ expect(await sample('ai_research_v4_image_fetch_stage_active', 'candidate_admission')).toBe(0)+ expect(await sample('ai_research_v4_image_fetch_stage_active', 'candidate_work')).toBe(0)+ await expect(ImageFetchProcessingMetrics.measure('candidate_policy', () => Promise.reject(error))).rejects.toBe(+ error+ )+ expect(await sample('ai_research_v4_image_fetch_stage_active', 'candidate_policy')).toBe(0)+ })++ it('collects live queue depths across passes and stops retaining finished queues', async () => {+ const queue = { candidateCount: 4 }+ const stopFirst = ImageFetchProcessingMetrics.trackQueue(queue)+ const stopSecond = ImageFetchProcessingMetrics.trackQueue({ candidateCount: 2 })+ const metric = register.getSingleMetric('ai_research_v4_image_fetch_candidates_queued')!+ expect((await metric.get()).values[0].value).toBe(6)+ queue.candidateCount = 1+ expect((await metric.get()).values[0].value).toBe(3)+ stopFirst()+ stopSecond()+ expect((await metric.get()).values[0].value).toBe(0)+ })+})diff --git a/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/processing-metrics.ts b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/processing-metrics.tsnew file mode 100644index 000000000000..3c5492d171d1--- /dev/null+++ b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/processing-metrics.ts@@ -0,0 +1,209 @@+import { Counter, Gauge, Histogram } from 'prom-client'++import { ConcurrencyController } from '~/common/utils/concurrencyController'++import type { ConfigurationFetchResult } from './configuration-policy'+import type { ConfigurationFile } from './crawl-history'++export type ProcessingStage =+ | 'batch_parse'+ | 'batch_filter'+ | 'batch_history_read'+ | 'batch_fetch'+ | 'batch_prepare_republish'+ | 'batch_history_write'+ | 'batch_republish_flush'+ | 'batch_finalize'+ | 'batch_dead_letter'+ | 'consumer_join'+ | 'consumer_process'+ | 'candidate_admission'+ | 'candidate_work'+ | 'candidate_policy'+ | 'candidate_fetch'+ | 'image_publish_admission'+ | 'image_publish_delivery'+ | 'configuration_fetch'+ | `configuration_${'request_capacity' | 'origin_crawl_delay' | 'registrable_domain_rate' | 'http'}`+ | `image_${'request_capacity' | 'origin_crawl_delay' | 'registrable_domain_rate' | 'http'}`++export type ConfigurationFetchReason =+ | ConfigurationFetchResult['outcome']+ | 'invalid_url'+ | 'request_error'+ | 'timeout'+ | 'redirect_limit'+ | 'cross_domain_redirect'+ | 'invalid_redirect'+ | 'missing_location'+ | `http_${number}`+ | 'http_5xx'+ | 'unexpected_status'+ | 'body_limit'+ | 'invalid_utf8'+ | 'invalid_document'+ | 'not_json_array'++export class ImageFetchProcessingMetrics {+ private static readonly active = new Map<ProcessingStage, Map<symbol, number>>()+ private static readonly queues = new Set<{ readonly candidateCount: number }>()++ private static readonly activeGauge = new Gauge({+ name: 'ai_research_v4_image_fetch_stage_active',+ help: 'Active operations by stage; nested stages overlap and must not be summed together',+ labelNames: ['stage'],+ collect() {+ for (const [stage, operations] of ImageFetchProcessingMetrics.active) {+ this.set({ stage }, operations.size)+ }+ },+ })++ private static readonly oldestGauge = new Gauge({+ name: 'ai_research_v4_image_fetch_stage_oldest_seconds',+ help: 'Age of the oldest active operation in each stage, or zero when idle',+ labelNames: ['stage'],+ collect() {+ const now = performance.now()+ for (const [stage, operations] of ImageFetchProcessingMetrics.active) {+ const first = operations.values().next().value+ this.set({ stage }, first === undefined ? 0 : Math.max(0, now - first) / 1000)+ }+ },+ })++ private static readonly queuedCandidates = new Gauge({+ name: 'ai_research_v4_image_fetch_candidates_queued',+ help: 'Candidates waiting for selection from all active passes, before candidate admission',+ collect() {+ let count = 0+ for (const queue of ImageFetchProcessingMetrics.queues) {+ count += queue.candidateCount+ }+ this.set(count)+ },+ })++ private static readonly duration = new Histogram({+ name: 'ai_research_v4_image_fetch_stage_duration_seconds',+ help: 'Elapsed time of finished stage visits, including failed operations; nested stages overlap',+ labelNames: ['stage'],+ buckets: [0.001, 0.01, 0.1, 0.5, 1, 5, 10, 30, 60, 120, 240, 600],+ })++ public static readonly joinedBatches = new Histogram({+ name: 'ai_research_v4_image_fetch_joined_batches',+ help: 'Consumer batches in each dispatched fetch group',+ buckets: [1, 2, 4, 8, 12, 16],+ })++ private static readonly configurationLookups = new Counter({+ name: 'ai_research_v4_image_fetch_configuration_lookups_total',+ help: 'Configuration lookups by source and observed status; shared requests count once per caller',+ labelNames: ['file', 'source', 'outcome'],+ })++ private static readonly configurationFetches = new Counter({+ name: 'ai_research_v4_image_fetch_configuration_fetches_total',+ help: 'Completed configuration fetches, once per redirect chain, by final outcome and reason',+ labelNames: ['file', 'outcome', 'reason'],+ })++ public static start(stage: ProcessingStage): ProcessingStageTimer {+ return new ProcessingStageTimer(stage)+ }++ public static enter(stage: ProcessingStage): symbol {+ let operations = this.active.get(stage)+ if (!operations) {+ operations = new Map()+ this.active.set(stage, operations)+ }+ const id = Symbol()+ operations.set(id, performance.now())+ return id+ }++ public static leave(stage: ProcessingStage, id: symbol): void {+ const operations = this.active.get(stage)+ const started = operations?.get(id)+ if (started !== undefined) {+ operations!.delete(id)+ this.duration.observe({ stage }, Math.max(0, performance.now() - started) / 1000)+ }+ }++ public static async measure<T>(stage: ProcessingStage, operation: () => Promise<T>): Promise<T> {+ const timer = this.start(stage)+ try {+ return await operation()+ } finally {+ timer.finish()+ }+ }++ public static async runLimited<T>(+ controller: ConcurrencyController,+ waitingStage: ProcessingStage,+ runningStage: ProcessingStage,+ options: { debugTag: string; fn: () => Promise<T> }+ ): Promise<T> {+ const timer = this.start(waitingStage)+ try {+ return await controller.run({+ ...options,+ fn: () => {+ timer.move(runningStage)+ return options.fn()+ },+ })+ } finally {+ timer.finish()+ }+ }++ public static trackQueue(queue: { readonly candidateCount: number }): () => void {+ this.queues.add(queue)+ return () => {+ this.queues.delete(queue)+ }+ }++ public static observeConfigurationLookup(+ file: ConfigurationFile,+ source: 'cache' | 'shared' | 'network',+ outcome: ConfigurationFetchResult['outcome'] | 'error'+ ): void {+ this.configurationLookups.inc({ file, source, outcome })+ }++ public static observeConfigurationFetch(+ file: ConfigurationFile,+ result: ConfigurationFetchResult,+ reason: ConfigurationFetchReason+ ): ConfigurationFetchResult {+ this.configurationFetches.inc({ file, outcome: result.outcome, reason })+ return result+ }+}++export class ProcessingStageTimer {+ private id: symbol | undefined++ constructor(private stage: ProcessingStage) {+ this.id = ImageFetchProcessingMetrics.enter(stage)+ }++ public move(stage: ProcessingStage): void {+ this.finish()+ this.stage = stage+ this.id = ImageFetchProcessingMetrics.enter(stage)+ }++ public finish(): void {+ if (this.id !== undefined) {+ ImageFetchProcessingMetrics.leave(this.stage, this.id)+ this.id = undefined+ }+ }+}diff --git a/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/producer.ts b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/producer.tsnew file mode 100644index 000000000000..4216c093f8bc--- /dev/null+++ b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/producer.ts@@ -0,0 +1,55 @@+import { TOPHOG_OUTPUT, TophogOutput } from '~/common/outputs'+import { IngestionOutputsBuilder } from '~/common/outputs/ingestion-outputs-builder'+import type { AllowedConfigKey } from '~/common/outputs/kafka-producer-config'+import { KafkaProducerRegistryBuilder } from '~/common/outputs/kafka-producer-registry-builder'+import {+ INGESTION_DOWNSTREAM_PRODUCER,+ INGESTION_DOWNSTREAM_PRODUCER_CONFIG_MAP,+ IngestionDownstreamProducer,+} from '~/ingestion/common/outputs/producers'++/**+ * The v4 fetch lane's own producer on the replay cluster. It sends frontier records to the frontier and the delay tiers+ * in a queue of its own. Broker and security config are those of the replay producer.+ */+export const AI_RESEARCH_V4_IMAGE_FETCH_PRODUCER = 'AI_RESEARCH_V4_IMAGE_FETCH' as const++export const AI_RESEARCH_V4_IMAGE_FETCH_PRODUCER_CONFIG_MAP = {+ 'client.id': 'KAFKA_INGESTION_SESSIONREPLAY_PRODUCER_CLIENT_ID',+ 'metadata.broker.list': 'KAFKA_INGESTION_SESSIONREPLAY_PRODUCER_METADATA_BROKER_LIST',+ 'security.protocol': 'KAFKA_INGESTION_SESSIONREPLAY_PRODUCER_SECURITY_PROTOCOL',+ 'compression.codec': 'KAFKA_AI_RESEARCH_V4_IMAGE_FETCH_PRODUCER_COMPRESSION_CODEC',+ 'linger.ms': 'KAFKA_AI_RESEARCH_V4_IMAGE_FETCH_PRODUCER_LINGER_MS',+ 'queue.buffering.max.messages': 'KAFKA_AI_RESEARCH_V4_IMAGE_FETCH_PRODUCER_QUEUE_BUFFERING_MAX_MESSAGES',+ 'queue.buffering.max.kbytes': 'KAFKA_AI_RESEARCH_V4_IMAGE_FETCH_PRODUCER_QUEUE_BUFFERING_MAX_KBYTES',+ 'message.max.bytes': 'KAFKA_AI_RESEARCH_V4_IMAGE_FETCH_PRODUCER_MESSAGE_MAX_BYTES',+ 'message.timeout.ms': 'KAFKA_AI_RESEARCH_V4_IMAGE_FETCH_PRODUCER_MESSAGE_TIMEOUT_MS',+} as const satisfies Partial<Record<AllowedConfigKey, string>>++/** DOWNSTREAM carries the TopHog rows to ClickHouse. */+export type AiResearchV4ImageFetchProducerName =+ | IngestionDownstreamProducer+ | typeof AI_RESEARCH_V4_IMAGE_FETCH_PRODUCER++type ProducerConfigKey =+ | (typeof INGESTION_DOWNSTREAM_PRODUCER_CONFIG_MAP)[keyof typeof INGESTION_DOWNSTREAM_PRODUCER_CONFIG_MAP]+ | (typeof AI_RESEARCH_V4_IMAGE_FETCH_PRODUCER_CONFIG_MAP)[keyof typeof AI_RESEARCH_V4_IMAGE_FETCH_PRODUCER_CONFIG_MAP]++export function createV4ImageFetchProducerRegistry(+ kafkaClientRack: string | undefined+): KafkaProducerRegistryBuilder<AiResearchV4ImageFetchProducerName, ProducerConfigKey> {+ return new KafkaProducerRegistryBuilder(kafkaClientRack)+ .register(INGESTION_DOWNSTREAM_PRODUCER, INGESTION_DOWNSTREAM_PRODUCER_CONFIG_MAP)+ .register(AI_RESEARCH_V4_IMAGE_FETCH_PRODUCER, AI_RESEARCH_V4_IMAGE_FETCH_PRODUCER_CONFIG_MAP)+}++export function createV4ImageFetchOutputs(): IngestionOutputsBuilder<+ TophogOutput,+ 'AI_RESEARCH_V4_IMAGE_FETCH_TOPHOG_TOPIC',+ 'AI_RESEARCH_V4_IMAGE_FETCH_TOPHOG_PRODUCER'+> {+ return new IngestionOutputsBuilder().register(TOPHOG_OUTPUT, {+ topicKey: 'AI_RESEARCH_V4_IMAGE_FETCH_TOPHOG_TOPIC',+ producerKey: 'AI_RESEARCH_V4_IMAGE_FETCH_TOPHOG_PRODUCER',+ })+}diff --git a/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/raw-image-store.test.ts b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/raw-image-store.test.tsindex df7eaff8910c..5a576ce88eac 100644--- a/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/raw-image-store.test.ts+++ b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/raw-image-store.test.ts@@ -3,9 +3,9 @@ import { createDecipheriv } from 'node:crypto'import { parseJSON } from '~/common/utils/json-parse'import type { TeamMonthKey } from '~/ingestion/pipelines/sessionreplay/ai-research/v4-keys/team-month-keys'-import type { FetchCandidate } from '~/ingestion/pipelines/sessionreplay/ml-mirror-image-fetch/collected-urls-record'-import type { ImageFetchResult } from '~/ingestion/pipelines/sessionreplay/ml-mirror-image-fetch/image-fetcher'+import type { FetchCandidate } from './collected-urls-record'+import type { ImageFetchResult } from './image-fetcher'import { RawImageStore } from './raw-image-store'const PNG = Buffer.concat([Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]), Buffer.from('png body')])diff --git a/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/raw-image-store.ts b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/raw-image-store.tsindex d5d3cd670f89..2b54b528cdf5 100644--- a/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/raw-image-store.ts+++ b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/raw-image-store.ts@@ -4,12 +4,12 @@ import { Counter } from 'prom-client'import { ConcurrencyController } from '~/common/utils/concurrencyController'import { sealV4Frame } from '~/ingestion/pipelines/sessionreplay/ai-research/v4-keys/frame'import type { TeamMonthKeys } from '~/ingestion/pipelines/sessionreplay/ai-research/v4-keys/team-month-keys'-import type { FetchCandidate } from '~/ingestion/pipelines/sessionreplay/ml-mirror-image-fetch/collected-urls-record'-import type { ImageFetchResult } from '~/ingestion/pipelines/sessionreplay/ml-mirror-image-fetch/image-fetcher'-import { ImageFetchProcessingMetrics } from '~/ingestion/pipelines/sessionreplay/ml-mirror-image-fetch/processing-metrics'+import type { FetchCandidate } from './collected-urls-record'+import type { ImageFetchResult } from './image-fetcher'import type { RefreshingImageSink } from './image-sink'import { InvalidImageTransportError, prepareFetchedImage } from './image-transport'+import { ImageFetchProcessingMetrics } from './processing-metrics'import { parseV4UrlRef } from './v4-url-ref'const S3_WRITE_TIMEOUT_MS = 30_000diff --git a/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/request-identity.ts b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/request-identity.tsnew file mode 100644index 000000000000..d93e1aaaa1d1--- /dev/null+++ b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/request-identity.ts@@ -0,0 +1,7 @@+export const BOT_NAME = 'PostHogImageFetcherBot'++export const REQUEST_IDENTITY_HEADERS: Readonly<Record<string, string>> = {+ 'user-agent': `${BOT_NAME}/1.0 (+https://posthog.com/docs/ai-research/image-fetcher-bot)`,+ // Replay customers already allow requests from this host so that the replay viewer works. Change it before this lane runs outside prod-us.+ referer: 'https://us.posthog.com/',+}diff --git a/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/response-url-binding.ts b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/response-url-binding.tsnew file mode 100644index 000000000000..684805f851f6--- /dev/null+++ b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/response-url-binding.ts@@ -0,0 +1,48 @@+import { createHmac, hkdfSync } from 'node:crypto'++import type {+ TeamMonthKey,+ TeamMonthKeys,+} from '~/ingestion/pipelines/sessionreplay/ai-research/v4-keys/team-month-keys'++import { parseV4UrlRef } from './v4-url-ref'++/** The HKDF info of `replay_core_v4::seal::image_url_key`, the key behind the hash of an `imageurl:v4` reference. */+const IMAGE_URL_KEY_INFO = 'ml-image-url-key'+/** The length `replay_core_v4::refs::reference_hash` truncates a hash to. */+const HASH_CHARS = 22++/** The keyed hash that the crawl history stores in place of a response URL. */+export type ResponseUrlBinding = (url: string) => string++export interface ResponseUrlBinder {+ bindingFor(originalRef: string, signal: AbortSignal): Promise<ResponseUrlBinding | undefined>+}++/**+ * Binds cached validators to the URL that issued them without storing that URL, which can hold personal data. The+ * binding is the HMAC of the URL under the image URL key of the reference's team month, so it compares only within+ * that team month. A reference with no live key has no binding, and its fetch sends no validators.+ */+export class TeamMonthResponseUrlBinder implements ResponseUrlBinder {+ constructor(private readonly keys: Pick<TeamMonthKeys, 'get'>) {}++ public async bindingFor(originalRef: string, signal: AbortSignal): Promise<ResponseUrlBinding | undefined> {+ const ref = parseV4UrlRef(originalRef)+ if (!ref) {+ return undefined+ }+ let key: TeamMonthKey+ try {+ key = await this.keys.get(ref.teamId, ref.keyMonth, signal)+ } catch {+ // Admission read this key moments ago, so a failure here costs one revalidation and not the fetch.+ return undefined+ }+ if (key.status !== 'ok') {+ return undefined+ }+ const urlKey = Buffer.from(hkdfSync('sha256', key.key, Buffer.alloc(0), IMAGE_URL_KEY_INFO, 32))+ return (url) => createHmac('sha256', urlKey).update(url).digest('base64url').slice(0, HASH_CHARS)+ }+}diff --git a/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/retry-delay-consumer.test.ts b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/retry-delay-consumer.test.tsnew file mode 100644index 000000000000..335a7d2f235c--- /dev/null+++ b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/retry-delay-consumer.test.ts@@ -0,0 +1,215 @@+import { Message } from 'node-rdkafka'++import { KafkaProducerWrapper } from '~/common/kafka/producer'++import { RetryDelayMetrics } from './metrics'+import { RetryDelayConsumer } from './retry-delay-consumer'++const NOW_MS = 1_700_000_000_000+const DELAY_MS = 60_000+const FRONTIER = 'session_replay_image_fetch'++function message(timestamp: number | undefined, overrides: Partial<Message> = {}): Message {+ return {+ value: Buffer.from('{"v":1}'),+ key: Buffer.from('example.com'),+ size: 7,+ topic: 'ai_research_session_replay_image_fetch_retry_1m',+ partition: 0,+ offset: 0,+ timestamp,+ ...overrides,+ }+}++interface Harness {+ consumer: RetryDelayConsumer+ produce: jest.Mock<Promise<void>, any[]>+ storeOffsets: jest.Mock<void, [Message[]]>+ heartbeat: jest.Mock<void, []>+}++function build(overrides: { isStopping?: () => boolean; delayMs?: number; releaseGroupSize?: number } = {}): Harness {+ const produce = jest.fn(() => Promise.resolve())+ const storeOffsets = jest.fn()+ const heartbeat = jest.fn()+ const consumer = new RetryDelayConsumer({ produce } as unknown as KafkaProducerWrapper, {+ frontierTopic: FRONTIER,+ delayMs: overrides.delayMs ?? DELAY_MS,+ heartbeat,+ heartbeatIntervalMs: 10_000,+ isStopping: overrides.isStopping,+ storeOffsets,+ releaseGroupSize: overrides.releaseGroupSize,+ })+ return { consumer, produce, storeOffsets, heartbeat }+}++describe('RetryDelayConsumer', () => {+ beforeEach(() => jest.useFakeTimers().setSystemTime(NOW_MS))+ afterEach(() => {+ jest.restoreAllMocks()+ jest.useRealTimers()+ })++ it.each([+ [{ delayMs: 0 }, 'positive delay'],+ [{ delayMs: -1 }, 'positive delay'],+ [{ delayMs: Number.NaN }, 'positive delay'],+ [{ releaseGroupSize: 0 }, 'positive whole release group size'],+ [{ releaseGroupSize: 1.5 }, 'positive whole release group size'],+ ])('refuses invalid options %p', (overrides, error) => {+ expect(() => build(overrides)).toThrow(error)+ })++ it('does nothing for an empty batch', async () => {+ const harness = build()++ await harness.consumer.handleBatch([])++ expect(harness.produce).not.toHaveBeenCalled()+ expect(harness.storeOffsets).not.toHaveBeenCalled()+ })++ it('publishes a ripe batch and stores its offsets once', async () => {+ const harness = build()+ const messages = [message(NOW_MS - DELAY_MS, { offset: 1 }), message(NOW_MS - DELAY_MS, { offset: 2 })]++ await harness.consumer.handleBatch(messages)++ expect(harness.produce).toHaveBeenCalledTimes(2)+ expect(harness.produce.mock.calls[0][0]).toMatchObject({+ topic: FRONTIER,+ key: messages[0].key,+ value: messages[0].value,+ })+ expect(harness.storeOffsets).toHaveBeenCalledTimes(1)+ expect(harness.storeOffsets).toHaveBeenCalledWith(messages)+ })++ it('waits once for the newest append timestamp in the batch', async () => {+ const harness = build()+ const older = message(NOW_MS - 30_000, { offset: 1 })+ const newer = message(NOW_MS - 10_000, { offset: 2 })++ const handled = harness.consumer.handleBatch([older, newer])+ await jest.advanceTimersByTimeAsync(30_000)+ expect(harness.produce).not.toHaveBeenCalled()+ await jest.advanceTimersByTimeAsync(20_000)+ await handled++ expect(harness.produce).toHaveBeenCalledTimes(2)+ expect(harness.storeOffsets).toHaveBeenCalledTimes(1)+ })++ it('reports health while a batch is deliberately waiting', async () => {+ const harness = build()++ const handled = harness.consumer.handleBatch([message(NOW_MS)])+ await jest.advanceTimersByTimeAsync(DELAY_MS)+ await handled++ expect(harness.heartbeat).toHaveBeenCalledTimes(8)+ })++ it.each([undefined, Number.NaN, 0, -1])(+ 'throws for an invalid broker append timestamp of %p',+ async (timestamp) => {+ const harness = build()+ const incReleased = jest.spyOn(RetryDelayMetrics, 'incReleased')++ await expect(harness.consumer.handleBatch([message(timestamp)])).rejects.toThrow(+ 'requires broker append timestamps'+ )+ expect(incReleased).toHaveBeenCalledWith('invalid_timestamp', 1)+ expect(harness.produce).not.toHaveBeenCalled()+ expect(harness.storeOffsets).not.toHaveBeenCalled()+ }+ )++ it('throws and stores no offset when any frontier publish fails', async () => {+ const harness = build()+ harness.produce.mockRejectedValueOnce(new Error('broker unavailable'))++ await expect(harness.consumer.handleBatch([message(NOW_MS - DELAY_MS)])).rejects.toThrow(+ 'could not publish a record back to the frontier'+ )+ expect(harness.storeOffsets).not.toHaveBeenCalled()+ })++ it('publishes each release group concurrently, and reports health and starts the next group after it settles', async () => {+ const harness = build({ releaseGroupSize: 2 })+ const acknowledgements: (() => void)[] = []+ harness.produce.mockImplementation(() => new Promise<void>((resolve) => acknowledgements.push(resolve)))+ const messages = [+ message(NOW_MS - DELAY_MS, { offset: 1 }),+ message(NOW_MS - DELAY_MS, { offset: 2 }),+ message(NOW_MS - DELAY_MS, { offset: 3 }),+ ]++ const handled = harness.consumer.handleBatch(messages)+ await jest.advanceTimersByTimeAsync(0)+ expect(harness.produce).toHaveBeenCalledTimes(2)+ expect(harness.heartbeat).not.toHaveBeenCalled()++ acknowledgements.splice(0).forEach((acknowledge) => acknowledge())+ await jest.advanceTimersByTimeAsync(0)+ expect(harness.produce).toHaveBeenCalledTimes(3)+ expect(harness.heartbeat).toHaveBeenCalledTimes(1)+ expect(harness.storeOffsets).not.toHaveBeenCalled()++ acknowledgements.splice(0).forEach((acknowledge) => acknowledge())+ await handled+ expect(harness.storeOffsets).toHaveBeenCalledWith(messages)+ })++ it('stops publishing the batch after the release group with the first frontier failure', async () => {+ const harness = build({ releaseGroupSize: 2 })+ harness.produce.mockResolvedValueOnce().mockRejectedValueOnce(new Error('broker unavailable'))+ const messages = [+ message(NOW_MS - DELAY_MS, { offset: 1 }),+ message(NOW_MS - DELAY_MS, { offset: 2 }),+ message(NOW_MS - DELAY_MS, { offset: 3 }),+ ]++ await expect(harness.consumer.handleBatch(messages)).rejects.toThrow(+ 'could not publish a record back to the frontier'+ )++ expect(harness.produce).toHaveBeenCalledTimes(2)+ expect(harness.storeOffsets).not.toHaveBeenCalled()+ })++ it('stores the batch offset after it skips a malformed record', async () => {+ const harness = build()+ const malformed = message(NOW_MS - DELAY_MS, { value: null })++ await harness.consumer.handleBatch([malformed])++ expect(harness.produce).not.toHaveBeenCalled()+ expect(harness.storeOffsets).toHaveBeenCalledWith([malformed])+ })++ it('abandons the batch without storing offsets when shutdown has started', async () => {+ const harness = build({ isStopping: () => true })++ await harness.consumer.handleBatch([message(NOW_MS)])++ expect(harness.produce).not.toHaveBeenCalled()+ expect(harness.storeOffsets).not.toHaveBeenCalled()+ })++ it('abandons an active wait when shutdown starts', async () => {+ let stopping = false+ const harness = build({ isStopping: () => stopping })+ const handled = harness.consumer.handleBatch([message(NOW_MS)])++ await jest.advanceTimersByTimeAsync(10_000)+ stopping = true+ await jest.advanceTimersByTimeAsync(10_000)+ await handled++ expect(harness.produce).not.toHaveBeenCalled()+ expect(harness.storeOffsets).not.toHaveBeenCalled()+ })+})diff --git a/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/retry-delay-consumer.ts b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/retry-delay-consumer.tsnew file mode 100644index 000000000000..e7b73564c330--- /dev/null+++ b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/retry-delay-consumer.ts@@ -0,0 +1,153 @@+import { Message } from 'node-rdkafka'++import { parseKafkaHeaders } from '~/common/kafka/consumer/consumer-v1'+import { KafkaProducer } from '~/common/kafka/producer'+import { logger } from '~/common/utils/logger'+import { delay } from '~/common/utils/utils'++import { RetryDelayMetrics } from './metrics'++export interface RetryDelayConsumerOptions {+ frontierTopic: string+ /** The period every record in this topic waits. The period belongs to the topic, not to the record. */+ delayMs: number+ /** Called while the consumer sleeps or releases, so the health check does not restart a pod that is working as designed. */+ heartbeat: () => void+ /** The default sits well inside CONSUMER_MAX_HEARTBEAT_INTERVAL_MS (30s), which binds before the loop stall threshold. */+ heartbeatIntervalMs?: number+ /** True once the pod is shutting down. A wait of an hour would otherwise hold the rolling deploy until Kubernetes killed it. */+ isStopping?: () => boolean+ storeOffsets: (messages: Message[]) => void+ releaseGroupSize?: number+}++const DEFAULT_HEARTBEAT_INTERVAL_MS = 10_000+/**+ * Each release waits for one broker acknowledgement, so releasing one record at a time limits a tier to one record per+ * produce round trip. A failed group throws and the consumer reads the whole batch again, so the group size also limits+ * how many records one failure publishes twice.+ */+const DEFAULT_RELEASE_GROUP_SIZE = 50++/**+ * The consumer of one delay topic.+ *+ * Kafka has no delayed delivery, so a retry waits in a topic whose period is fixed and a consumer+ * holds it there. Every record in one topic waits the same period, so the records leave in the+ * order they become ready.+ *+ * The record already carries its hop budget and its earliest fetch time, so nothing here parses it.+ *+ * Two properties of this lane live outside this file, and both are in the README. Its+ * `max.poll.interval.ms` must exceed the period of its topic, or the broker evicts it mid-sleep. It+ * runs one pod and no more, because adding consumers cannot make a record become ready sooner.+ */+export class RetryDelayConsumer {+ constructor(+ private readonly producer: KafkaProducer,+ private readonly options: RetryDelayConsumerOptions+ ) {+ if (!Number.isFinite(options.delayMs) || options.delayMs <= 0) {+ throw new Error(`the retry delay consumer needs a positive delay, got ${options.delayMs}`)+ }+ const releaseGroupSize = options.releaseGroupSize ?? DEFAULT_RELEASE_GROUP_SIZE+ if (!Number.isInteger(releaseGroupSize) || releaseGroupSize <= 0) {+ throw new Error(+ `the retry delay consumer needs a positive whole release group size, got ${releaseGroupSize}`+ )+ }+ }++ public async handleBatch(messages: Message[]): Promise<void> {+ if (messages.length === 0) {+ return+ }+ if (this.stopping()) {+ RetryDelayMetrics.incReleased('abandoned', messages.length)+ return+ }+ const invalidTimestampCount = messages.filter(+ (message) =>+ message.timestamp === undefined || !Number.isFinite(message.timestamp) || message.timestamp <= 0+ ).length+ if (invalidTimestampCount > 0) {+ RetryDelayMetrics.incReleased('invalid_timestamp', invalidTimestampCount)+ throw new Error('the image fetch retry lane requires broker append timestamps')+ }+ const waitMs = this.remainingBatchWaitMs(messages)+ if (waitMs > 0) {+ RetryDelayMetrics.observeWait(waitMs / 1000)+ if (!(await this.sleepWithHeartbeat(waitMs))) {+ RetryDelayMetrics.incReleased('abandoned', messages.length)+ return+ }+ }+ const releaseGroupSize = this.options.releaseGroupSize ?? DEFAULT_RELEASE_GROUP_SIZE+ for (let start = 0; start < messages.length; start += releaseGroupSize) {+ const group = messages.slice(start, start + releaseGroupSize)+ await Promise.all(group.map((message) => this.release(message)))+ // A slow broker can make the releases of one batch outlast the health check, as a long wait would.+ this.options.heartbeat()+ }+ this.options.storeOffsets(messages)+ }++ private stopping(): boolean {+ return this.options.isStopping?.() ?? false+ }++ /**+ * Measured from when the record was written rather than from when it was read, so a consumer+ * that restarts does not begin the wait again. librdkafka reports an absent timestamp as -1,+ * which is not nullish, so a plain `??` would make the whole period look already spent.+ */+ private remainingBatchWaitMs(messages: Message[]): number {+ const latestReadyAtMs = Math.max(...messages.map((message) => message.timestamp! + this.options.delayMs))+ return Math.max(0, latestReadyAtMs - Date.now())+ }++ /** False when the pod started shutting down during the wait, so the caller abandons the record. */+ private async sleepWithHeartbeat(waitMs: number): Promise<boolean> {+ const interval = this.options.heartbeatIntervalMs ?? DEFAULT_HEARTBEAT_INTERVAL_MS+ const deadline = Date.now() + waitMs+ while (Date.now() < deadline) {+ if (this.stopping()) {+ return false+ }+ this.options.heartbeat()+ await delay(Math.min(interval, deadline - Date.now()))+ }+ this.options.heartbeat()+ return !this.stopping()+ }++ /**+ * True when this consumer is finished with the record, whether it went out or can never go out.+ *+ * A failed produce returns false, because reading the record again is the only way it comes+ * back. Nothing else holds it.+ */+ private async release(message: Message): Promise<void> {+ if (!message.value || !message.key) {+ RetryDelayMetrics.incReleased('malformed')+ return+ }+ try {+ await this.producer.produce({+ topic: this.options.frontierTopic,+ key: message.key,+ value: message.value,+ headers: parseKafkaHeaders(message.headers),+ })+ } catch (error) {+ logger.warn('🌐', 'ai_research_v4_image_fetch_retry_release_failed', {+ error: error instanceof Error ? error.name : 'unknown',+ })+ RetryDelayMetrics.incReleased('failed')+ throw new Error('the image fetch retry lane could not publish a record back to the frontier', {+ cause: error,+ })+ }+ RetryDelayMetrics.incReleased('released')+ }+}diff --git a/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/shared-image-fetcher.test.ts b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/shared-image-fetcher.test.tsnew file mode 100644index 000000000000..37baab86bd02--- /dev/null+++ b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/shared-image-fetcher.test.ts@@ -0,0 +1,51 @@+import type { ImageFetchOptions, ImageFetchResult } from './image-fetcher'+import { SharedImageFetcher } from './shared-image-fetcher'++const IMAGE_URL = 'https://example.com/a.png'+const OK: ImageFetchResult = {+ outcome: 'ok',+ redirects: 0,+ currentUrl: IMAGE_URL,+ bytes: Buffer.from('png'),+ contentType: 'image/png',+}+const options = (tdmrepReservation: boolean) => ({ tdmrepReservation }) as ImageFetchOptions++describe('SharedImageFetcher', () => {+ it.each([+ ['one after the other', false],+ ['at the same time', true],+ ])('fetches again for a job whose TDM reservation differs, %s', async (_name, concurrent) => {+ const inner = { fetch: jest.fn((_url: string, _options: ImageFetchOptions) => Promise.resolve(OK)) }+ const fetcher = new SharedImageFetcher(inner)++ if (concurrent) {+ await Promise.all([fetcher.fetch(IMAGE_URL, options(false)), fetcher.fetch(IMAGE_URL, options(true))])+ } else {+ await fetcher.fetch(IMAGE_URL, options(false))+ await fetcher.fetch(IMAGE_URL, options(true))+ }+ await fetcher.fetch(IMAGE_URL, options(true))++ expect(inner.fetch.mock.calls.map(([, used]) => used.tdmrepReservation)).toEqual([false, true])+ })++ it.each([+ ['more entries than its cap', Buffer.from('png'), 64 * 1024 * 1024, 2],+ ['empty images whose URLs and headers outgrow its size', Buffer.alloc(0), 3_000, 100],+ ])('evicts the oldest image when it holds %s', async (_name, bytes, maxBytes, maxEntries) => {+ const inner = {+ fetch: jest.fn((url: string, _options: ImageFetchOptions) =>+ Promise.resolve({ ...OK, currentUrl: url, bytes })+ ),+ }+ const fetcher = new SharedImageFetcher(inner, maxBytes, 60_000, maxEntries)+ const urls = ['a', 'b', 'c', 'a'].map((name) => `https://example.com/${name}.png`)++ for (const url of urls) {+ await fetcher.fetch(url, options(false))+ }++ expect(inner.fetch.mock.calls.map(([url]) => url)).toEqual(urls)+ })+})diff --git a/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/shared-image-fetcher.ts b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/shared-image-fetcher.tsnew file mode 100644index 000000000000..b1a64ad52458--- /dev/null+++ b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/shared-image-fetcher.ts@@ -0,0 +1,76 @@+import { LRUCache } from 'lru-cache'+import { Counter } from 'prom-client'++import type { ImageFetchOptions, ImageFetchResult, ImageFetcher } from './image-fetcher'++const sharedFetches = new Counter({+ name: 'ai_research_v4_image_fetch_shared_fetches_total',+ help: 'v4 image fetches served by a recent fetch of the same URL in this process, with no request.',+})++const ENTRY_OVERHEAD_BYTES = 1024++/**+ * Serves one fetch of a URL to every job that asks for it within a short time. The v4 recorder sends one job per team+ * for the same URL and month, so without this each team costs the host a request. Only the plain bytes are shared, in+ * this process: the sink still seals each team's copy under that team's key.+ */+export class SharedImageFetcher implements ImageFetcher {+ private readonly recent: LRUCache<string, ImageFetchResult>+ private readonly inFlight = new Map<string, Promise<ImageFetchResult>>()++ constructor(+ private readonly inner: ImageFetcher,+ maxBytes = 64 * 1024 * 1024,+ lifetimeMs = 60_000,+ maxEntries = 4096+ ) {+ this.recent = new LRUCache({+ max: maxEntries,+ maxSize: maxBytes,+ // An origin can answer many URLs with empty images, so each entry also pays for its strings and headers.+ sizeCalculation: (result, id) =>+ (result.bytes?.length ?? 0) + 2 * (id.length + result.currentUrl.length) + ENTRY_OVERHEAD_BYTES,+ ttl: lifetimeMs,+ ttlAutopurge: true,+ })+ }++ public async fetch(url: string, options: ImageFetchOptions): Promise<ImageFetchResult> {+ // The TDM reservation decides if a response is refused, so only jobs with the same reservation share a response.+ const id = `${options.tdmrepReservation ? 'reserved' : 'open'} ${url}`+ const recent = this.recent.get(id)+ if (recent) {+ sharedFetches.inc()+ return recent+ }+ const pending = this.inFlight.get(id)+ if (pending) {+ const result = await pending+ if (isShareable(result)) {+ sharedFetches.inc()+ return result+ }+ }+ const own = this.inner.fetch(url, options)+ if (!this.inFlight.has(id)) {+ this.inFlight.set(id, own)+ const forget = (): void => {+ if (this.inFlight.get(id) === own) {+ this.inFlight.delete(id)+ }+ }+ own.then(forget, forget)+ }+ const result = await own+ if (isShareable(result)) {+ this.recent.set(id, result)+ }+ return result+ }+}++/** A redirect depends on the hops a job has left, so only a direct image is shared. */+function isShareable(result: ImageFetchResult): boolean {+ return result.outcome === 'ok' && result.redirects === 0 && !!result.bytes+}diff --git a/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/tophog-metrics.ts b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/tophog-metrics.tsnew file mode 100644index 000000000000..981abe6029da--- /dev/null+++ b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/tophog-metrics.ts@@ -0,0 +1,126 @@+import type { TopHogRegistry } from '~/ingestion/framework/extensions/tophog'++import type { ImageFetchBlockReason } from './block-reason'+import type { FetchCandidate } from './collected-urls-record'+import { deduplicateFetchCandidates } from './fetch-candidate-queue'+import type { FetchAttempt } from './fetch-runner'+import type { SchedulerWaitScope } from './metrics'++const TOP_DOMAIN_FACTORS = 20+const MAX_TRACKED_DOMAIN_FACTORS = 2_000++type Recorder = { record(key: Record<string, string>, value: number): void }++export class ImageFetchTopHogMetrics {+ private readonly attempts: Recorder+ private readonly blockEvents: Recorder+ private readonly blockedMs: Recorder++ constructor(topHog: TopHogRegistry) {+ this.attempts = topHog.registerSum('ai_research_v4_image_fetch_attempts_by_registrable_domain', {+ topN: TOP_DOMAIN_FACTORS,+ maxKeys: MAX_TRACKED_DOMAIN_FACTORS,+ })+ this.blockEvents = topHog.registerSum('ai_research_v4_image_fetch_block_events_by_registrable_domain', {+ topN: TOP_DOMAIN_FACTORS,+ maxKeys: MAX_TRACKED_DOMAIN_FACTORS,+ })+ this.blockedMs = topHog.registerSum('ai_research_v4_image_fetch_blocked_ms_by_registrable_domain', {+ topN: TOP_DOMAIN_FACTORS,+ maxKeys: MAX_TRACKED_DOMAIN_FACTORS,+ })+ }++ public recordAttempt(attempt: FetchAttempt): void {+ this.recordForSourcePartitions(+ this.attempts,+ attempt.candidate.sourcePartitions,+ {+ registrable_domain: attempt.candidate.registrableDomain,+ disposition: attempt.finished ? 'completed' : 'republished',+ outcome: attempt.outcome,+ },+ 1+ )+ if (attempt.block) {+ this.recordBlock(+ attempt.candidate.registrableDomain,+ attempt.candidate.sourcePartitions,+ attempt.block.reason,+ 1,+ attempt.block.waitMs+ )+ }+ }++ public recordConcurrencyLimitedUrls(+ candidates: FetchCandidate[],+ availableConnections: (registrableDomain: string) => number+ ): void {+ const candidatesByRegistrableDomain = new Map<+ string,+ { candidateCount: number; sourcePartitions: Set<number> }+ >()+ for (const candidate of deduplicateFetchCandidates(candidates).candidates) {+ const domainCandidates = candidatesByRegistrableDomain.get(candidate.registrableDomain) ?? {+ candidateCount: 0,+ sourcePartitions: new Set<number>(),+ }+ domainCandidates.candidateCount += 1+ for (const sourcePartition of candidate.sourcePartitions ?? []) {+ domainCandidates.sourcePartitions.add(sourcePartition)+ }+ candidatesByRegistrableDomain.set(candidate.registrableDomain, domainCandidates)+ }+ for (const [registrableDomain, domainCandidates] of candidatesByRegistrableDomain) {+ const limitedUrls = Math.max(0, domainCandidates.candidateCount - availableConnections(registrableDomain))+ if (limitedUrls > 0) {+ this.recordBlock(+ registrableDomain,+ [...domainCandidates.sourcePartitions],+ 'domain_concurrency',+ limitedUrls,+ 0+ )+ }+ }+ }++ public recordSchedulerWait(+ registrableDomain: string,+ sourcePartitions: readonly number[] | undefined,+ scope: SchedulerWaitScope,+ waitMs: number+ ): void {+ if (waitMs <= 0) {+ return+ }+ this.recordBlock(registrableDomain, sourcePartitions, scope, 1, waitMs)+ }++ private recordBlock(+ registrableDomain: string,+ sourcePartitions: readonly number[] | undefined,+ reason: ImageFetchBlockReason,+ eventCount: number,+ waitMs: number+ ): void {+ const key = { registrable_domain: registrableDomain, reason }+ this.recordForSourcePartitions(this.blockEvents, sourcePartitions, key, eventCount)+ if (waitMs > 0) {+ this.recordForSourcePartitions(this.blockedMs, sourcePartitions, key, waitMs)+ }+ }++ private recordForSourcePartitions(+ recorder: Recorder,+ sourcePartitions: readonly number[] | undefined,+ key: Record<string, string>,+ value: number+ ): void {+ const partitions = sourcePartitions?.length ? new Set(sourcePartitions) : new Set(['unattributed'])+ for (const partition of partitions) {+ recorder.record({ ...key, partition: String(partition) }, value)+ }+ }+}diff --git a/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/url-fetch-consumer.test.ts b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/url-fetch-consumer.test.tsnew file mode 100644index 000000000000..9ef0db1aba4b--- /dev/null+++ b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/url-fetch-consumer.test.ts@@ -0,0 +1,747 @@+import { Message } from 'node-rdkafka'+import { register } from 'prom-client'++import { RecordedTopHogMetric, createRecordingTopHog } from '~/tests/helpers/tophog'++import { FetchCandidate, MAX_HOPS, serializeFrontierRecord } from './collected-urls-record'+import { CrawlHistoryItem, CrawlHistoryStore, configurationCacheKey } from './crawl-history'+import { AttemptOutcome, DELAY_TOO_LONG, FetchAttempt, FetchPass, HOPS_EXHAUSTED } from './fetch-runner'+import { FrontierDeadLetterSink } from './frontier-dead-letter-sink'+import { FrontierPublisher, RepublishFlushResult, RepublishResult } from './frontier-publisher'+import { ImageFetchConsumerMetrics, ImageFetchRequestMetrics } from './metrics'+import { ImageFetchTopHogMetrics } from './tophog-metrics'+import { UrlFetchConsumer } from './url-fetch-consumer'++const NOW_MS = 1_700_000_000_000++function candidate(name: string, overrides: Partial<FetchCandidate> = {}): FetchCandidate {+ const hash = name.padEnd(22, '0')+ return {+ originalRef: `imageurl:v4:7:2026-10:${hash}`,+ currentUrl: `https://cdn.example.com/${name}.png`,+ host: 'cdn.example.com',+ origin: 'https://cdn.example.com',+ registrableDomain: 'example.com',+ remainingHops: MAX_HOPS,+ notBeforeMs: 0,+ firstSeenAtMs: NOW_MS,+ fetchCount: 0,+ republishCount: 0,+ lastRepublishReason: null,+ ...overrides,+ }+}++function message(candidates: FetchCandidate[], key = 'example.com', partition = 0): Message {+ const value = serializeFrontierRecord(candidates)+ return {+ value,+ key: Buffer.from(key),+ size: value.length,+ topic: 'session_replay_image_fetch',+ partition,+ offset: 0,+ }+}++class FakeCrawlHistory implements CrawlHistoryStore {+ public readonly items = new Map<string, CrawlHistoryItem>()+ public readKeys: string[][] = []+ public writes: CrawlHistoryItem[][] = []+ public readError: Error | undefined+ public writeError: Error | undefined++ read(keys: string[]): Promise<Map<string, CrawlHistoryItem>> {+ this.readKeys.push(keys)+ if (this.readError) {+ return Promise.reject(this.readError)+ }+ return Promise.resolve(+ new Map(keys.flatMap((key) => (this.items.has(key) ? [[key, this.items.get(key)!] as const] : [])))+ )+ }++ write(items: CrawlHistoryItem[]): Promise<void> {+ this.writes.push(items)+ if (this.writeError) {+ return Promise.reject(this.writeError)+ }+ for (const item of items) {+ this.items.set(item.key, item)+ }+ return Promise.resolve()+ }+}++function terminal(candidate: FetchCandidate, outcome: AttemptOutcome = 'ok'): FetchAttempt {+ return {+ candidate,+ outcome,+ finished: true,+ lost: false,+ history: {+ kind: 'url',+ key: candidate.originalRef,+ nextFetchAtMs: NOW_MS + 30 * 24 * 60 * 60 * 1000,+ storageExpiresAtMs: NOW_MS + 30 * 24 * 60 * 60 * 1000,+ outcome,+ },+ configurationUpdates: [],+ }+}++interface Harness {+ consumer: UrlFetchConsumer+ history: FakeCrawlHistory+ run: jest.Mock<Promise<FetchAttempt[]>, [FetchCandidate[], Map<string, CrawlHistoryItem>]>+ republish: jest.Mock<Promise<RepublishResult>, any[]>+ flush: jest.Mock<Promise<RepublishFlushResult>, []>+ park: jest.Mock<Promise<void>, any[]>+ topHogRecords: Map<string, RecordedTopHogMetric[]>+}++function build(dryRun = false, deadLettersEnabled = true): Harness {+ const history = new FakeCrawlHistory()+ const run = jest.fn((candidates: FetchCandidate[], _stored: Map<string, CrawlHistoryItem>) =>+ Promise.resolve(candidates.map((item) => terminal(item)))+ )+ const republish = jest.fn(() => Promise.resolve('queued' as const))+ const flush = jest.fn(() => Promise.resolve({ failedUrls: 0 }))+ const park = jest.fn(() => Promise.resolve())+ const recordingTopHog = createRecordingTopHog()+ const topHogMetrics = new ImageFetchTopHogMetrics(recordingTopHog.registry)+ const consumer = new UrlFetchConsumer(+ history,+ { createRepublishBatch: () => ({ republish, flush }) } as unknown as FrontierPublisher,+ { seenTtlSeconds: 30 * 24 * 60 * 60, dryRun },+ dryRun ? undefined : ({ run } as FetchPass),+ deadLettersEnabled ? ({ park } as FrontierDeadLetterSink) : null,+ topHogMetrics+ )+ return { consumer, history, run, republish, flush, park, topHogRecords: recordingTopHog.records }+}++describe('UrlFetchConsumer', () => {+ afterEach(() => jest.restoreAllMocks())++ it.each([Number.NaN, 0, 3_599, 3_600.5])('refuses an invalid crawl-history TTL of %p', (seenTtlSeconds) => {+ expect(+ () =>+ new UrlFetchConsumer(new FakeCrawlHistory(), {} as FrontierPublisher, { seenTtlSeconds, dryRun: true })+ ).toThrow('AI_RESEARCH_V4_IMAGE_FETCH_CRAWL_HISTORY_TTL_SECONDS')+ })++ it('refuses active mode without a fetch pass', () => {+ expect(+ () =>+ new UrlFetchConsumer(new FakeCrawlHistory(), {} as FrontierPublisher, {+ seenTtlSeconds: 3_600,+ dryRun: false,+ })+ ).toThrow('fetch runner')+ })++ it('parses dry-run traffic without reading or writing shared state', async () => {+ const harness = build(true)++ await harness.consumer.handleBatch([message([candidate('a')])], NOW_MS)++ expect(harness.history.readKeys).toEqual([])+ expect(harness.history.writes).toEqual([])+ })++ it('bulk reads URL and per-origin configuration keys', async () => {+ const harness = build()++ await harness.consumer.handleBatch([message([candidate('a'), candidate('b')])], NOW_MS)++ expect(harness.history.readKeys).toEqual([+ [+ candidate('a').originalRef,+ candidate('b').originalRef,+ configurationCacheKey(candidate('a').origin, 'robots'),+ configurationCacheKey(candidate('a').origin, 'tdmrep'),+ ],+ ])+ expect(harness.run.mock.calls[0][0]).toEqual([+ candidate('a', { sourcePartitions: [0] }),+ candidate('b', { sourcePartitions: [0] }),+ ])+ expect(harness.history.writes[0]).toHaveLength(2)+ })++ it('records distinct origins and registrable domains for the poll batch', async () => {+ const harness = build()+ const observeBatch = jest.spyOn(ImageFetchConsumerMetrics, 'observeBatch')+ const observeBatchDiversity = jest.spyOn(ImageFetchConsumerMetrics, 'observeBatchDiversity')+ const otherExampleOrigin = candidate('b', {+ currentUrl: 'https://img.example.com/b.png',+ host: 'img.example.com',+ origin: 'https://img.example.com',+ })+ const otherRegistrableDomain = candidate('c', {+ currentUrl: 'https://img.other.net/c.png',+ host: 'img.other.net',+ origin: 'https://img.other.net',+ registrableDomain: 'other.net',+ })++ await harness.consumer.handleBatch(+ [message([candidate('a'), otherExampleOrigin]), message([otherRegistrableDomain], 'other.net')],+ NOW_MS+ )++ expect(observeBatch).toHaveBeenCalledWith(3, 2, expect.any(Number))+ expect(observeBatchDiversity).toHaveBeenCalledWith([1, 1, 1], [2, 1])+ })++ it('attributes joined batch work and outcomes to each source partition', async () => {+ const harness = build()+ const observePartitionRecord = jest.spyOn(ImageFetchConsumerMetrics, 'observePartitionRecord')+ const incPartitionUrls = jest.spyOn(ImageFetchConsumerMetrics, 'incPartitionUrls')+ const observePartitionBatchDiversity = jest.spyOn(ImageFetchConsumerMetrics, 'observePartitionBatchDiversity')+ const incPartitionAttempt = jest.spyOn(ImageFetchRequestMetrics, 'incPartitionAttempt')+ const fromPartition7 = candidate('a')+ const fromPartition42 = candidate('b', {+ currentUrl: 'https://cdn.other.net/b.png',+ host: 'cdn.other.net',+ origin: 'https://cdn.other.net',+ registrableDomain: 'other.net',+ })++ await harness.consumer.handleBatch(+ [message([fromPartition7], 'example.com', 7), message([fromPartition42], 'other.net', 42)],+ NOW_MS+ )++ expect(observePartitionRecord).toHaveBeenCalledWith(7, 1, 1)+ expect(observePartitionRecord).toHaveBeenCalledWith(42, 1, 1)+ expect(incPartitionUrls).toHaveBeenCalledWith(7, 'unique', 1)+ expect(incPartitionUrls).toHaveBeenCalledWith(7, 'fetchable', 1)+ expect(incPartitionUrls).toHaveBeenCalledWith(42, 'unique', 1)+ expect(incPartitionUrls).toHaveBeenCalledWith(42, 'fetchable', 1)+ expect(observePartitionBatchDiversity).toHaveBeenCalledWith(7, [1], [1])+ expect(observePartitionBatchDiversity).toHaveBeenCalledWith(42, [1], [1])+ expect(harness.run.mock.calls[0][0]).toEqual([+ { ...fromPartition7, sourcePartitions: [7] },+ { ...fromPartition42, sourcePartitions: [42] },+ ])+ expect(harness.topHogRecords.get('ai_research_v4_image_fetch_attempts_by_registrable_domain')).toEqual([+ {+ key: {+ registrable_domain: 'example.com',+ disposition: 'completed',+ outcome: 'ok',+ partition: '7',+ },+ value: 1,+ },+ {+ key: {+ registrable_domain: 'other.net',+ disposition: 'completed',+ outcome: 'ok',+ partition: '42',+ },+ value: 1,+ },+ ])+ expect(incPartitionAttempt).toHaveBeenCalledWith(7, 'completed', 'ok')+ expect(incPartitionAttempt).toHaveBeenCalledWith(42, 'completed', 'ok')+ })++ it('keeps attribution when one ref arrives from different partitions', async () => {+ const harness = build()+ const incPartitionUrls = jest.spyOn(ImageFetchConsumerMetrics, 'incPartitionUrls')+ const observePartitionBatchDiversity = jest.spyOn(ImageFetchConsumerMetrics, 'observePartitionBatchDiversity')+ const incPartitionAttempt = jest.spyOn(ImageFetchRequestMetrics, 'incPartitionAttempt')+ const fromPartition7 = candidate('a')+ const fromPartition42 = candidate('a', {+ currentUrl: 'https://cdn.other.net/a.png',+ host: 'cdn.other.net',+ origin: 'https://cdn.other.net',+ registrableDomain: 'other.net',+ remainingHops: 9,+ republishCount: 1,+ lastRepublishReason: 'redirect',+ })++ await harness.consumer.handleBatch(+ [message([fromPartition7], 'example.com', 7), message([fromPartition42], 'other.net', 42)],+ NOW_MS+ )++ expect(harness.run.mock.calls[0][0]).toEqual([{ ...fromPartition42, sourcePartitions: [7, 42] }])+ for (const partition of [7, 42]) {+ expect(incPartitionUrls).toHaveBeenCalledWith(partition, 'unique', 1)+ expect(incPartitionUrls).toHaveBeenCalledWith(partition, 'fetchable', 1)+ expect(observePartitionBatchDiversity).toHaveBeenCalledWith(partition, [1], [1])+ expect(incPartitionAttempt).toHaveBeenCalledWith(partition, 'completed', 'ok')+ }+ })++ it.each([+ ['a ref', candidate('a'), candidate('a')],+ [+ 'a ref from two sessions',+ candidate('a', { sessionId: '01a0c669-8800-7000-8000-000000000001' }),+ candidate('a', { sessionId: '01a0c669-8800-7000-8000-000000000002' }),+ ],+ ])('deduplicates %s within the batch', async (_name, first, second) => {+ const harness = build()++ await harness.consumer.handleBatch([message([first]), message([second])], NOW_MS)++ expect(harness.run.mock.calls[0][0]).toEqual([{ ...first, sourcePartitions: [0] }])+ })++ it('keeps the most conservative durable state from duplicate jobs', async () => {+ const harness = build()+ const stale = candidate('a')+ const advanced = candidate('a', {+ currentUrl: 'https://cdn.example.com/moved.png',+ remainingHops: 7,+ firstSeenAtMs: NOW_MS - 5_000,+ fetchCount: 4,+ republishCount: 3,+ lastRepublishReason: 'retry',+ })++ await harness.consumer.handleBatch([message([stale]), message([advanced])], NOW_MS)++ expect(harness.run.mock.calls[0][0]).toEqual([{ ...advanced, sourcePartitions: [0] }])+ })++ it('keeps the latest not-before time from duplicate jobs', async () => {+ const harness = build()+ const stale = candidate('a')+ const delayed = candidate('a', {+ remainingHops: 9,+ notBeforeMs: NOW_MS + 30_000,+ republishCount: 1,+ lastRepublishReason: 'retry',+ })++ await harness.consumer.handleBatch([message([stale]), message([delayed])], NOW_MS)++ expect(harness.run.mock.calls[0][0]).toEqual([])+ expect(harness.republish).toHaveBeenCalledWith(+ expect.objectContaining({ remainingHops: 9, notBeforeMs: NOW_MS + 30_000 }),+ expect.any(Object),+ 'not_ready',+ 30_000+ )+ })++ it('skips a ref whose crawl-history interval has not ended', async () => {+ const [stored, fresh] = [candidate('a'), candidate('b')]+ const harness = build()+ harness.history.items.set(stored.originalRef, {+ kind: 'url',+ key: stored.originalRef,+ nextFetchAtMs: NOW_MS + 1,+ storageExpiresAtMs: NOW_MS + 1,+ outcome: 'ok',+ })++ await harness.consumer.handleBatch([message([stored, fresh])], NOW_MS)++ expect(harness.run.mock.calls[0][0]).toEqual([{ ...fresh, sourcePartitions: [0] }])+ })++ it('republishes a job that arrives before its durable not-before time', async () => {+ const harness = build()+ const early = candidate('a', {+ notBeforeMs: NOW_MS + 30_000,+ lastBlockReason: 'configuration_unreachable',+ })++ await harness.consumer.handleBatch([message([early])], NOW_MS)++ expect(harness.run.mock.calls[0][0]).toEqual([])+ expect(harness.republish).toHaveBeenCalledWith(+ { ...early, sourcePartitions: [0], lastBlockReason: 'configuration_unreachable' },+ {+ currentUrl: early.currentUrl,+ host: early.host,+ origin: early.origin,+ registrableDomain: early.registrableDomain,+ },+ 'not_ready',+ 30_000+ )+ expect(harness.history.writes).toEqual([])+ expect(harness.topHogRecords.get('ai_research_v4_image_fetch_attempts_by_registrable_domain')).toEqual([+ {+ key: {+ registrable_domain: 'example.com',+ disposition: 'republished',+ outcome: 'backoff',+ partition: '0',+ },+ value: 1,+ },+ ])+ expect(harness.topHogRecords.get('ai_research_v4_image_fetch_block_events_by_registrable_domain')).toEqual([+ {+ key: {+ registrable_domain: 'example.com',+ reason: 'configuration_unreachable',+ partition: '0',+ },+ value: 1,+ },+ ])+ expect(harness.topHogRecords.get('ai_research_v4_image_fetch_blocked_ms_by_registrable_domain')).toEqual([+ {+ key: {+ registrable_domain: 'example.com',+ reason: 'configuration_unreachable',+ partition: '0',+ },+ value: 30_000,+ },+ ])+ })++ it('records a terminal refusal when the remaining delay is over one hour', async () => {+ const harness = build()+ harness.republish.mockResolvedValue('refused_delay')+ const early = candidate('a', { notBeforeMs: NOW_MS + 3_600_001 })++ await harness.consumer.handleBatch([message([early])], NOW_MS)++ expect(harness.history.writes[0]).toEqual([+ expect.objectContaining({ kind: 'url', key: early.originalRef, outcome: DELAY_TOO_LONG }),+ ])+ })++ it('records a terminal refusal when a delayed job has no remaining hops', async () => {+ const harness = build()+ const early = candidate('a', { notBeforeMs: NOW_MS + 30_000, remainingHops: 0 })++ await harness.consumer.handleBatch([message([early])], NOW_MS)++ expect(harness.republish).not.toHaveBeenCalled()+ expect(harness.history.writes[0]).toEqual([+ expect.objectContaining({ kind: 'url', key: early.originalRef, outcome: HOPS_EXHAUSTED }),+ ])+ })++ it('throws when a not-ready republish delivery fails', async () => {+ const harness = build()+ harness.flush.mockResolvedValue({ failedUrls: 1 })+ const early = candidate('a', { notBeforeMs: NOW_MS + 30_000 })++ await expect(harness.consumer.handleBatch([message([early])], NOW_MS)).rejects.toThrow('account for 1 URLs')+ expect(harness.history.writes).toEqual([])+ })++ it('records a retry cause after the republish batch is durable', async () => {+ const harness = build()+ const retryCause = jest.spyOn(ImageFetchRequestMetrics, 'incRetryCause').mockImplementation()+ harness.run.mockImplementation((candidates) =>+ Promise.resolve(+ candidates.map((item) => ({+ candidate: item,+ outcome: 'server_error',+ finished: false,+ lost: false,+ configurationUpdates: [],+ }))+ )+ )++ await harness.consumer.handleBatch([message([candidate('a')])], NOW_MS)++ expect(retryCause).toHaveBeenCalledWith('server_error')+ })++ it('does not record a retry cause when the republish batch fails', async () => {+ const harness = build()+ const retryCause = jest.spyOn(ImageFetchRequestMetrics, 'incRetryCause').mockImplementation()+ harness.flush.mockResolvedValue({ failedUrls: 1 })+ harness.run.mockImplementation((candidates) =>+ Promise.resolve(+ candidates.map((item) => ({+ candidate: item,+ outcome: 'server_error',+ finished: false,+ lost: false,+ configurationUpdates: [],+ }))+ )+ )++ await expect(harness.consumer.handleBatch([message([candidate('a')])], NOW_MS)).rejects.toThrow(+ 'account for 1 URLs'+ )+ expect(retryCause).not.toHaveBeenCalled()+ })++ it('quarantines a malformed record before completing the source batch', async () => {+ const harness = build()+ const invalid = message([candidate('a')])+ invalid.value = Buffer.from('{')+ let releasePark: () => void = () => undefined+ harness.park.mockImplementation(+ () =>+ new Promise<void>((resolve) => {+ releasePark = resolve+ })+ )+ const completed = jest.fn()++ const sourceBatch = harness.consumer.handleBatch([invalid], NOW_MS)+ void sourceBatch.then(completed)+ await new Promise((resolve) => setImmediate(resolve))++ expect(harness.park).toHaveBeenCalledWith(invalid, 'malformed')+ expect(completed).not.toHaveBeenCalled()+ expect(harness.run).not.toHaveBeenCalled()++ releasePark()+ await expect(sourceBatch).resolves.toBeUndefined()+ expect(completed).toHaveBeenCalledTimes(1)+ })++ it('quarantines rejected records concurrently', async () => {+ const harness = build()+ const first = message([candidate('a')])+ first.value = Buffer.from('{')+ const second = message([candidate('b')])+ second.value = Buffer.from('{')+ second.offset = 1+ const releases: Array<() => void> = []+ harness.park.mockImplementation(+ () =>+ new Promise<void>((resolve) => {+ releases.push(resolve)+ })+ )++ const sourceBatch = harness.consumer.handleBatch([first, second], NOW_MS)+ await new Promise((resolve) => setImmediate(resolve))++ expect(harness.park).toHaveBeenCalledTimes(2)++ for (const release of releases) {+ release()+ }+ await expect(sourceBatch).resolves.toBeUndefined()+ })++ it('does not quarantine rejected records before other batch work succeeds', async () => {+ const harness = build()+ const invalid = message([candidate('a')])+ invalid.value = Buffer.from('{')+ harness.history.readError = new Error('read failed')++ await expect(harness.consumer.handleBatch([invalid, message([candidate('b')])], NOW_MS)).rejects.toThrow(+ 'read failed'+ )++ expect(harness.park).not.toHaveBeenCalled()+ })++ it('keeps the source batch uncommitted when quarantine publication fails', async () => {+ const harness = build()+ const invalid = message([candidate('a')])+ invalid.value = Buffer.from('{')+ harness.park.mockRejectedValue(new Error('DLQ unavailable'))+ const deadLetterFailed = jest.spyOn(ImageFetchConsumerMetrics, 'incDeadLetterFailed')+ const dropped = jest.spyOn(ImageFetchConsumerMetrics, 'incDropped')++ await expect(harness.consumer.handleBatch([invalid], NOW_MS)).rejects.toThrow('DLQ unavailable')++ expect(deadLetterFailed).toHaveBeenCalledWith('malformed')+ expect(dropped).not.toHaveBeenCalled()+ expect(harness.run).not.toHaveBeenCalled()+ expect(harness.history.readKeys).toEqual([])+ })++ it('commits and drops rejected input when quarantine is disabled', async () => {+ const harness = build(false, false)+ const invalid = message([candidate('a')])+ invalid.value = Buffer.from('{')++ await expect(harness.consumer.handleBatch([invalid], NOW_MS)).resolves.toBeUndefined()++ expect(harness.park).not.toHaveBeenCalled()+ expect(harness.run).not.toHaveBeenCalled()+ })++ it('drops a tracking beacon job without quarantining its record', async () => {+ const harness = build()+ const beacon = candidate('beacon', {+ currentUrl: 'https://analytics.twitter.com/i/adsct?txn_id=abc&p_id=Twitter',+ host: 'analytics.twitter.com',+ origin: 'https://analytics.twitter.com',+ registrableDomain: 'twitter.com',+ })+ const image = candidate('logo', {+ currentUrl: 'https://cdn.twitter.com/logo.png',+ host: 'cdn.twitter.com',+ origin: 'https://cdn.twitter.com',+ registrableDomain: 'twitter.com',+ })+ const dropped = jest.spyOn(ImageFetchConsumerMetrics, 'incDropped')+ const skipped = jest.spyOn(ImageFetchConsumerMetrics, 'incSkipped')++ await expect(+ harness.consumer.handleBatch([message([beacon, image], 'twitter.com')], NOW_MS)+ ).resolves.toBeUndefined()++ expect(harness.park).not.toHaveBeenCalled()+ expect(harness.run).toHaveBeenCalledTimes(1)+ expect(harness.run.mock.calls[0][0].map((fetched) => fetched.originalRef)).toEqual([image.originalRef])+ expect(skipped).toHaveBeenCalledWith('tracking_beacon', 1)+ expect(dropped).not.toHaveBeenCalled()+ })++ it('does not count a skipped beacon when the batch fails before its commit', async () => {+ const harness = build()+ const beacon = candidate('beacon', {+ currentUrl: 'https://analytics.twitter.com/i/adsct?txn_id=abc&p_id=Twitter',+ host: 'analytics.twitter.com',+ origin: 'https://analytics.twitter.com',+ registrableDomain: 'twitter.com',+ })+ const image = candidate('logo', {+ currentUrl: 'https://cdn.twitter.com/logo.png',+ host: 'cdn.twitter.com',+ origin: 'https://cdn.twitter.com',+ registrableDomain: 'twitter.com',+ })+ harness.history.readError = new Error('read failed')+ const skipped = jest.spyOn(ImageFetchConsumerMetrics, 'incSkipped')++ await expect(harness.consumer.handleBatch([message([beacon, image], 'twitter.com')], NOW_MS)).rejects.toThrow(+ 'read failed'+ )++ expect(skipped).not.toHaveBeenCalled()+ })++ it('rejects a whole multi-job record when one job belongs to another partition', async () => {+ const harness = build()+ const foreign = candidate('foreign', {+ currentUrl: 'https://img.other.net/foreign.png',+ host: 'img.other.net',+ origin: 'https://img.other.net',+ registrableDomain: 'other.net',+ })++ await harness.consumer.handleBatch([message([candidate('a'), foreign])], NOW_MS)++ expect(harness.run).not.toHaveBeenCalled()+ expect(harness.history.readKeys).toEqual([])+ })++ it('throws when the bulk read fails', async () => {+ register.resetMetrics()+ const harness = build()+ harness.history.readError = new Error('read failed')+ const observeBatch = jest.spyOn(ImageFetchConsumerMetrics, 'observeBatch')+ const observeStoreDuration = jest.spyOn(ImageFetchConsumerMetrics, 'observeStoreDuration')+ const startBatch = jest.spyOn(ImageFetchConsumerMetrics, 'startBatch')+ const finishBatch = jest.spyOn(ImageFetchConsumerMetrics, 'finishBatch')++ await expect(harness.consumer.handleBatch([message([candidate('a')])], NOW_MS)).rejects.toThrow('read failed')+ expect(observeBatch).toHaveBeenCalledWith(1, 1, expect.any(Number))+ expect(observeStoreDuration).toHaveBeenCalledWith('read', 'error', expect.any(Number))+ expect(startBatch).toHaveBeenCalledTimes(1)+ expect(finishBatch).toHaveBeenCalledTimes(1)+ const active = await register.getSingleMetric('ai_research_v4_image_fetch_stage_active')!.get()+ expect(active.values.every(({ value }) => value === 0)).toBe(true)+ const durations = await register.getSingleMetric('ai_research_v4_image_fetch_stage_duration_seconds')!.get()+ expect(durations.values).toEqual(+ expect.arrayContaining([+ expect.objectContaining({+ metricName: 'ai_research_v4_image_fetch_stage_duration_seconds_count',+ labels: { stage: 'batch_history_read' },+ value: 1,+ }),+ ])+ )+ })++ it('throws when the final bulk write fails', async () => {+ const harness = build()+ harness.history.writeError = new Error('write failed')++ await expect(harness.consumer.handleBatch([message([candidate('a')])], NOW_MS)).rejects.toThrow('write failed')+ expect(harness.flush).not.toHaveBeenCalled()+ })++ it('writes durable state before it flushes buffered republishes', async () => {+ register.resetMetrics()+ const harness = build()+ const order: string[] = []+ harness.run.mockImplementation((candidates) => {+ order.push('published')+ return Promise.resolve(candidates.map((item) => terminal(item)))+ })+ const write = harness.history.write.bind(harness.history)+ harness.history.write = async (items) => {+ order.push('history')+ await write(items)+ }+ harness.flush.mockImplementation(() => {+ order.push('republished')+ return Promise.resolve({ failedUrls: 0 })+ })++ await harness.consumer.handleBatch([message([candidate('a')])], NOW_MS)++ expect(order).toEqual(['published', 'history', 'republished'])+ const durations = await register.getSingleMetric('ai_research_v4_image_fetch_stage_duration_seconds')!.get()+ const completedStages = durations.values.map(({ labels }) => labels.stage)+ expect(completedStages).toEqual(+ expect.arrayContaining([+ 'batch_parse',+ 'batch_history_read',+ 'batch_filter',+ 'batch_fetch',+ 'batch_prepare_republish',+ 'batch_history_write',+ 'batch_republish_flush',+ 'batch_finalize',+ 'batch_dead_letter',+ ])+ )+ const active = await register.getSingleMetric('ai_research_v4_image_fetch_stage_active')!.get()+ expect(active.values.every(({ value }) => value === 0)).toBe(true)+ })++ it('throws when the fetch pass reports a lost URL', async () => {+ const harness = build()+ harness.run.mockImplementation((candidates) =>+ Promise.resolve([+ {+ candidate: candidates[0],+ outcome: 'timeout',+ finished: false,+ lost: true,+ configurationUpdates: [],+ },+ terminal(candidates[1]),+ ])+ )++ await expect(harness.consumer.handleBatch([message([candidate('a'), candidate('b')])], NOW_MS)).rejects.toThrow(+ 'account for 1 URLs'+ )+ expect(harness.topHogRecords.get('ai_research_v4_image_fetch_attempts_by_registrable_domain')).toEqual([+ {+ key: {+ registrable_domain: 'example.com',+ disposition: 'completed',+ outcome: 'ok',+ partition: '0',+ },+ value: 1,+ },+ ])+ })+})diff --git a/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/url-fetch-consumer.ts b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/url-fetch-consumer.tsnew file mode 100644index 000000000000..5a6a5bb1cc9c--- /dev/null+++ b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/url-fetch-consumer.ts@@ -0,0 +1,514 @@+import { Message } from 'node-rdkafka'+import pLimit from 'p-limit'++import { logger } from '~/common/utils/logger'++import {+ FetchCandidate,+ MAX_HOPS,+ UrlDropReason,+ UrlSkipReason,+ parseCollectedUrlsRecord,+} from './collected-urls-record'+import { CrawlHistoryItem, CrawlHistoryStore, UrlCrawlHistoryItem, configurationCacheKey } from './crawl-history'+import { FetchCandidatePoolAdmission } from './fetch-candidate-pool'+import { mergeDuplicateFetchCandidates } from './fetch-candidate-queue'+import {+ AttemptOutcome,+ DELAY_TOO_LONG,+ FetchAttempt,+ FetchPass,+ HOPS_EXHAUSTED,+ isTransientOutcome,+} from './fetch-runner'+import { FrontierDeadLetterReason, FrontierDeadLetterSink } from './frontier-dead-letter-sink'+import { FrontierPublisher, RepublishBatch } from './frontier-publisher'+import type { RefreshingImageSink } from './image-sink'+import { ImageFetchConsumerMetrics, ImageFetchRequestMetrics } from './metrics'+import { ImageFetchProcessingMetrics } from './processing-metrics'+import { ImageFetchTopHogMetrics } from './tophog-metrics'+import { urlHistoryExpiresAtMs, urlHistoryStorageExpiresAtMs } from './url-history-expiry'++const ONE_HOUR_MS = 60 * 60 * 1000+const REPUBLISH_DEADLINE_FROM_BATCH_START_MS = 200_000+const DEAD_LETTER_BATCH_BUDGET_MS = 50_000+const DEAD_LETTER_PUBLISH_CONCURRENCY = 8++type RejectedFrontierRecord = {+ message: Message+ reasons: UrlDropReason[]+}++export interface UrlFetchConsumerOptions {+ seenTtlSeconds: number+ dryRun: boolean+ /** Drops jobs before any request for them, and resolves to the jobs that stay. The lane counts what it drops. */+ admitCandidates?: (candidates: FetchCandidate[], nowMs: number) => Promise<FetchCandidate[]>+ /** The lane's sink. A job not due for a fetch refreshes its stored image when the image is stale. */+ refreshingSink?: RefreshingImageSink+}++export class UrlFetchConsumer {+ constructor(+ private readonly crawlHistory: CrawlHistoryStore,+ private readonly publisher: FrontierPublisher,+ private readonly options: UrlFetchConsumerOptions,+ private readonly runner?: FetchPass,+ private readonly deadLetters: FrontierDeadLetterSink | null = null,+ private readonly topHogMetrics?: ImageFetchTopHogMetrics+ ) {+ if (!Number.isInteger(options.seenTtlSeconds) || options.seenTtlSeconds < 60 * 60) {+ throw new Error('AI_RESEARCH_V4_IMAGE_FETCH_CRAWL_HISTORY_TTL_SECONDS must be at least 3600')+ }+ if (!options.dryRun && !runner) {+ throw new Error('the image fetch lane needs a fetch runner once dry run is cleared')+ }+ ImageFetchConsumerMetrics.setDryRun(options.dryRun)+ }++ public async handleBatch(+ messages: Message[],+ nowMs: number,+ admission?: FetchCandidatePoolAdmission+ ): Promise<void> {+ const startedAt = process.hrtime.bigint()+ const republishDeadlineAtMonotonicMs = performance.now() + REPUBLISH_DEADLINE_FROM_BATCH_START_MS+ const drops = new Map<UrlDropReason, number>()+ const skips = new Map<UrlSkipReason, number>()+ const rejectedRecords: RejectedFrontierRecord[] = []+ const candidatesByRef = new Map<string, FetchCandidate>()+ let dedupedInBatch = 0+ let originCount = 0+ let registrableDomainCount = 0+ let originCandidateCounts: number[] = []+ let registrableDomainCandidateCounts: number[] = []+ const activeBatchId = ImageFetchConsumerMetrics.startBatch()+ const stage = ImageFetchProcessingMetrics.start('batch_parse')++ try {+ for (const message of messages) {+ const parsed = this.parse(message)+ if (!parsed.ok) {+ rejectedRecords.push({ message, reasons: [parsed.reason] })+ continue+ }+ ImageFetchConsumerMetrics.observeRecord(parsed.urlCount)+ ImageFetchConsumerMetrics.observePartitionRecord(+ message.partition,+ parsed.urlCount,+ parsed.candidates.length+ )+ if (parsed.rejected.length > 0) {+ rejectedRecords.push({+ message,+ reasons: parsed.rejected.map((rejected) => rejected.reason),+ })+ }+ for (const { reason } of parsed.skipped) {+ skips.set(reason, (skips.get(reason) ?? 0) + 1)+ }+ for (const candidate of parsed.candidates) {+ const partitionCandidate = { ...candidate, sourcePartitions: [message.partition] }+ const existing = candidatesByRef.get(partitionCandidate.originalRef)+ if (existing) {+ dedupedInBatch += 1+ candidatesByRef.set(+ partitionCandidate.originalRef,+ mergeDuplicateFetchCandidates(existing, partitionCandidate)+ )+ } else {+ candidatesByRef.set(partitionCandidate.originalRef, partitionCandidate)+ }+ }+ }+ const parsedCandidates = [...candidatesByRef.values()]+ candidatesByRef.clear()+ const candidates =+ this.options.admitCandidates && !this.options.dryRun && parsedCandidates.length > 0+ ? await this.options.admitCandidates(parsedCandidates, nowMs)+ : parsedCandidates+ const origins = new Map<string, number>()+ const registrableDomains = new Map<string, number>()+ const partitionOrigins = new Map<number, Map<string, number>>()+ const partitionRegistrableDomains = new Map<number, Map<string, number>>()+ for (const candidate of candidates) {+ ImageFetchConsumerMetrics.observeAge(Math.max(0, nowMs - candidate.firstSeenAtMs) / 1000)+ origins.set(candidate.origin, (origins.get(candidate.origin) ?? 0) + 1)+ registrableDomains.set(+ candidate.registrableDomain,+ (registrableDomains.get(candidate.registrableDomain) ?? 0) + 1+ )+ for (const sourcePartition of candidate.sourcePartitions ?? []) {+ ImageFetchConsumerMetrics.incPartitionUrls(sourcePartition, 'unique', 1)+ incrementNestedCount(partitionOrigins, sourcePartition, candidate.origin)+ incrementNestedCount(partitionRegistrableDomains, sourcePartition, candidate.registrableDomain)+ }+ }+ originCount = origins.size+ registrableDomainCount = registrableDomains.size+ originCandidateCounts = [...origins.values()]+ registrableDomainCandidateCounts = [...registrableDomains.values()]+ for (const [partition, partitionOriginCounts] of partitionOrigins) {+ ImageFetchConsumerMetrics.observePartitionBatchDiversity(+ partition,+ [...partitionOriginCounts.values()],+ [...(partitionRegistrableDomains.get(partition)?.values() ?? [])]+ )+ }++ if (this.options.dryRun || candidates.length === 0) {+ stage.move('batch_dead_letter')+ await this.parkRejectedRecords(rejectedRecords, drops)+ this.countSkippedJobs(skips)+ return+ }++ const keys = [+ ...candidates.map((candidate) => candidate.originalRef),+ ...[...origins.keys()].flatMap((origin) => [+ configurationCacheKey(origin, 'robots'),+ configurationCacheKey(origin, 'tdmrep'),+ ]),+ ]+ stage.move('batch_history_read')+ const stored = await this.runCrawlHistoryOperation('read', keys.length, () => this.crawlHistory.read(keys))++ stage.move('batch_filter')+ const fetchable: FetchCandidate[] = []+ const notReady: FetchCandidate[] = []+ const refreshDue: { candidate: FetchCandidate; history: UrlCrawlHistoryItem }[] = []+ for (const candidate of candidates) {+ const history = stored.get(candidate.originalRef)+ if (history?.kind === 'url' && history.nextFetchAtMs > nowMs) {+ if (this.isRefreshDue(history, nowMs)) {+ refreshDue.push({ candidate, history })+ continue+ }+ ImageFetchConsumerMetrics.incDeduped('store', 1)+ for (const sourcePartition of candidate.sourcePartitions ?? []) {+ ImageFetchConsumerMetrics.incPartitionUrls(sourcePartition, 'store_deduped', 1)+ }+ continue+ }+ if (candidate.notBeforeMs > nowMs) {+ notReady.push(candidate)+ for (const sourcePartition of candidate.sourcePartitions ?? []) {+ ImageFetchConsumerMetrics.incPartitionUrls(sourcePartition, 'not_ready', 1)+ }+ } else {+ fetchable.push(candidate)+ for (const sourcePartition of candidate.sourcePartitions ?? []) {+ ImageFetchConsumerMetrics.incPartitionUrls(sourcePartition, 'fetchable', 1)+ }+ }+ }++ const refreshed = await Promise.all(+ refreshDue.map(async ({ candidate, history }): Promise<UrlCrawlHistoryItem> => {+ if (await this.options.refreshingSink!.refreshImage(candidate)) {+ return { ...history, storedAtMs: nowMs }+ }+ // The stored image is gone, for example because S3 expired it. Admission already dropped a job+ // whose key is deleted, and a refused image has no stored time, so this one is fetched again now,+ // with no validators: a 304 cannot bring back bytes that are no longer stored.+ const missing: UrlCrawlHistoryItem = {+ ...history,+ storedAtMs: undefined,+ nextFetchAtMs: nowMs,+ cache: undefined,+ }+ stored.set(candidate.originalRef, missing)+ if (candidate.notBeforeMs > nowMs) {+ notReady.push(candidate)+ } else {+ fetchable.push(candidate)+ }+ return missing+ })+ )+ ImageFetchConsumerMetrics.incFetchable(fetchable.length)++ const republishBatch = this.publisher.createRepublishBatch(republishDeadlineAtMonotonicMs)+ stage.move('batch_fetch')+ const attempts = await this.runner!.run(fetchable, stored, republishBatch, admission)+ stage.move('batch_prepare_republish')+ attempts.push(+ ...(await Promise.all(+ notReady.map((candidate) => this.republishNotReady(republishBatch, candidate, nowMs))+ ))+ )+ const updates: CrawlHistoryItem[] = [...refreshed]+ for (const attempt of attempts) {+ updates.push(...attempt.configurationUpdates)+ if (attempt.history) {+ updates.push(attempt.history)+ }+ }+ if (updates.length > 0) {+ stage.move('batch_history_write')+ await this.runCrawlHistoryOperation('write', updates.length, () => this.crawlHistory.write(updates))+ }+ stage.move('batch_republish_flush')+ const republishResult = await republishBatch.flush()+ stage.move('batch_finalize')+ for (const attempt of attempts) {+ if (attempt.lost || (!attempt.finished && republishResult.failedUrls > 0)) {+ continue+ }+ for (const sourcePartition of attempt.candidate.sourcePartitions ?? []) {+ ImageFetchRequestMetrics.incPartitionAttempt(+ sourcePartition,+ attempt.finished ? 'completed' : 'republished',+ attempt.outcome+ )+ }+ this.topHogMetrics?.recordAttempt(attempt)+ if (!attempt.finished && isTransientOutcome(attempt.outcome)) {+ ImageFetchRequestMetrics.incRetryCause(attempt.outcome)+ }+ }+ const lost = attempts.filter((attempt) => attempt.lost).length + republishResult.failedUrls+ if (lost > 0) {+ throw new Error(`the image fetch lane could not account for ${lost} URLs`)+ }+ stage.move('batch_dead_letter')+ await this.parkRejectedRecords(rejectedRecords, drops)+ this.countSkippedJobs(skips)+ } finally {+ stage.finish()+ ImageFetchConsumerMetrics.finishBatch(activeBatchId)+ this.recordMetrics(+ drops,+ dedupedInBatch,+ originCount,+ registrableDomainCount,+ originCandidateCounts,+ registrableDomainCandidateCounts,+ startedAt+ )+ }+ }++ private parse(message: Message): ReturnType<typeof parseCollectedUrlsRecord> {+ try {+ return parseCollectedUrlsRecord(message.value, message.key?.toString() ?? null)+ } catch (error) {+ logger.error('🌐', 'ai_research_v4_image_fetch_record_parse_threw', {+ error: error instanceof Error ? error.name : 'unknown',+ })+ return { ok: false, reason: 'malformed' }+ }+ }++ /** Runs after the batch's durable work, so a batch that fails and is redelivered counts its skips once. */+ private countSkippedJobs(skips: Map<UrlSkipReason, number>): void {+ for (const [reason, count] of skips) {+ ImageFetchConsumerMetrics.incSkipped(reason, count)+ }+ }++ private async parkRejectedRecords(+ records: RejectedFrontierRecord[],+ drops: Map<UrlDropReason, number>+ ): Promise<void> {+ const deadlineAtMonotonicMs = performance.now() + DEAD_LETTER_BATCH_BUDGET_MS+ const limit = pLimit(DEAD_LETTER_PUBLISH_CONCURRENCY)+ let firstFailure: unknown+ const outcomes = await Promise.allSettled(+ records.map(({ message, reasons }) =>+ limit(async () => {+ if (firstFailure) {+ throw firstFailure+ }+ try {+ await this.parkRejectedRecord(message, reasons, deadlineAtMonotonicMs)+ } catch (error) {+ firstFailure = error+ throw error+ }+ })+ )+ )+ for (const outcome of outcomes) {+ if (outcome.status === 'rejected') {+ throw outcome.reason+ }+ }+ for (const { reasons } of records) {+ for (const reason of reasons) {+ drops.set(reason, (drops.get(reason) ?? 0) + 1)+ }+ }+ }++ private async parkRejectedRecord(+ message: Message,+ reasons: UrlDropReason[],+ deadlineAtMonotonicMs: number+ ): Promise<void> {+ if (!this.deadLetters) {+ return+ }+ const reason = this.deadLetterReason(reasons)+ const remainingMs = deadlineAtMonotonicMs - performance.now()+ if (remainingMs <= 0) {+ const error = new Error('image-fetch dead-letter batch exceeded its publish budget')+ ImageFetchConsumerMetrics.incDeadLetterFailed(reason)+ logger.error('🌐', 'ai_research_v4_image_fetch_dead_letter_publish_failed', { reason, error: error.name })+ throw error+ }+ let timeout: NodeJS.Timeout | undefined+ try {+ await Promise.race([+ this.deadLetters.park(message, reason),+ new Promise<never>((_, reject) => {+ timeout = setTimeout(+ () => reject(new Error('image-fetch dead-letter batch exceeded its publish budget')),+ remainingMs+ )+ }),+ ])+ } catch (error) {+ ImageFetchConsumerMetrics.incDeadLetterFailed(reason)+ logger.error('🌐', 'ai_research_v4_image_fetch_dead_letter_publish_failed', {+ reason,+ error: error instanceof Error ? error.name : 'unknown',+ })+ throw error+ } finally {+ if (timeout) {+ clearTimeout(timeout)+ }+ }+ ImageFetchConsumerMetrics.incDeadLettered(reason)+ }++ private deadLetterReason(reasons: UrlDropReason[]): FrontierDeadLetterReason {+ const distinctReasons = new Set(reasons)+ return distinctReasons.size === 1 ? reasons[0] : 'multiple'+ }++ private async runCrawlHistoryOperation<T>(+ operation: 'read' | 'write',+ affectedKeys: number,+ run: () => Promise<T>+ ): Promise<T> {+ const startedAt = process.hrtime.bigint()+ let outcome: 'success' | 'error' = 'success'+ try {+ return await run()+ } catch (error) {+ outcome = 'error'+ ImageFetchConsumerMetrics.incStoreError(operation, affectedKeys)+ throw error+ } finally {+ ImageFetchConsumerMetrics.observeStoreDuration(+ operation,+ outcome,+ Number(process.hrtime.bigint() - startedAt) / 1e9+ )+ }+ }++ private isRefreshDue(history: UrlCrawlHistoryItem, nowMs: number): boolean {+ const sink = this.options.refreshingSink+ return !!sink && history.storedAtMs !== undefined && nowMs - history.storedAtMs >= sink.refreshIntervalMs+ }++ private async republishNotReady(+ republishBatch: RepublishBatch,+ candidate: FetchCandidate,+ nowMs: number+ ): Promise<FetchAttempt> {+ if (candidate.remainingHops === 0) {+ return this.terminalAttempt(candidate, HOPS_EXHAUSTED, nowMs)+ }+ const waitMs = candidate.notBeforeMs - nowMs+ const blockingReason = candidate.lastBlockReason ?? 'unknown_backoff'+ const republishedCandidate: FetchCandidate = { ...candidate, lastBlockReason: blockingReason }+ const result = await republishBatch.republish(+ republishedCandidate,+ {+ currentUrl: candidate.currentUrl,+ host: candidate.host,+ origin: candidate.origin,+ registrableDomain: candidate.registrableDomain,+ },+ 'not_ready',+ waitMs+ )+ if (result === 'refused_delay' || waitMs > ONE_HOUR_MS) {+ return this.terminalAttempt(candidate, DELAY_TOO_LONG, nowMs)+ }+ return {+ candidate: republishedCandidate,+ outcome: 'backoff',+ finished: false,+ lost: false,+ block: { reason: blockingReason, waitMs },+ configurationUpdates: [],+ }+ }++ private terminalAttempt(candidate: FetchCandidate, outcome: AttemptOutcome, nowMs: number): FetchAttempt {+ ImageFetchRequestMetrics.observeCompletedUrl(+ outcome,+ 'none',+ Math.max(0, nowMs - candidate.firstSeenAtMs) / 1000,+ candidate.fetchCount,+ candidate.republishCount+ )+ ImageFetchRequestMetrics.observeHops(MAX_HOPS - candidate.remainingHops)+ return {+ candidate,+ outcome,+ finished: true,+ lost: false,+ history: this.terminalHistory(candidate, outcome, nowMs),+ configurationUpdates: [],+ }+ }++ private terminalHistory(candidate: FetchCandidate, outcome: AttemptOutcome, nowMs: number): UrlCrawlHistoryItem {+ const nextFetchAtMs = urlHistoryExpiresAtMs(nowMs, this.options.seenTtlSeconds)+ return {+ kind: 'url',+ key: candidate.originalRef,+ nextFetchAtMs,+ storageExpiresAtMs: urlHistoryStorageExpiresAtMs(candidate.originalRef, nextFetchAtMs),+ outcome,+ }+ }++ private recordMetrics(+ drops: Map<UrlDropReason, number>,+ dedupedInBatch: number,+ origins: number,+ registrableDomains: number,+ originCandidateCounts: number[],+ registrableDomainCandidateCounts: number[],+ startedAt: bigint+ ): void {+ if (dedupedInBatch > 0) {+ ImageFetchConsumerMetrics.incDeduped('batch', dedupedInBatch)+ }+ for (const [reason, count] of drops) {+ ImageFetchConsumerMetrics.incDropped(reason, count)+ }+ ImageFetchConsumerMetrics.observeBatchDiversity(originCandidateCounts, registrableDomainCandidateCounts)+ ImageFetchConsumerMetrics.observeBatch(+ origins,+ registrableDomains,+ Number(process.hrtime.bigint() - startedAt) / 1e9+ )+ }+}++function incrementNestedCount(counts: Map<number, Map<string, number>>, partition: number, key: string): void {+ let partitionCounts = counts.get(partition)+ if (!partitionCounts) {+ partitionCounts = new Map<string, number>()+ counts.set(partition, partitionCounts)+ }+ partitionCounts.set(key, (partitionCounts.get(key) ?? 0) + 1)+}diff --git a/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/url-history-expiry.ts b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/url-history-expiry.tsindex a882606482f9..ab3a6b3d186c 100644--- a/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/url-history-expiry.ts+++ b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/url-history-expiry.ts@@ -1,5 +1,4 @@-import { explicitFreshnessLifetimeMs } from '~/ingestion/pipelines/sessionreplay/ml-mirror-image-fetch/configuration-policy'-+import { explicitFreshnessLifetimeMs } from './configuration-policy'import { HttpCacheMetadata } from './crawl-history'import { parseV4UrlRef } from './v4-url-ref'diff --git a/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/v4-lane.test.ts b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/v4-lane.test.tsnew file mode 100644index 000000000000..bac29f17da60--- /dev/null+++ b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/v4-lane.test.ts@@ -0,0 +1,530 @@+import { GetItemCommand } from '@aws-sdk/client-dynamodb'+import { CopyObjectCommand, PutObjectCommand, S3ServiceException } from '@aws-sdk/client-s3'+import { Message } from 'node-rdkafka'+import { createDecipheriv, generateKeyPairSync } from 'node:crypto'++import { KAFKA_AI_RESEARCH_V4_IMAGE_FETCH_RETRY_1M } from '~/common/config/kafka-topics'+import { KafkaProducer } from '~/common/kafka/producer'+import { parseJSON } from '~/common/utils/json-parse'+import { StreamedResponse, fetchStreamed } from '~/common/utils/request'+import { buildAiResearchV4ServerConfig } from '~/ingestion/pipelines/sessionreplay/ai-research/config'+import vector from '~/ingestion/pipelines/sessionreplay/ai-research/v4-keys/encryption-vector-v4.json'+import { TeamMonthKeys } from '~/ingestion/pipelines/sessionreplay/ai-research/v4-keys/team-month-keys'+import { buildFetchRunner, buildFrontierPublisher } from '~/servers/ai-research-v4-image-fetch-server'+import { createRecordingTopHog } from '~/tests/helpers/tophog'++import type { FetchCandidate } from './collected-urls-record'+import { CrawlHistoryItem, CrawlHistoryStore, UrlCrawlHistoryItem, configurationCacheKey } from './crawl-history'+import { FetchRunner } from './fetch-runner'+import { V4JobAdmission } from './job-admission'+import { RawImageStore } from './raw-image-store'+import { ImageFetchTopHogMetrics } from './tophog-metrics'+import { UrlFetchConsumer } from './url-fetch-consumer'++jest.mock('~/common/utils/request', () => ({+ ...jest.requireActual('~/common/utils/request'),+ fetchStreamed: jest.fn(),+}))++const fetchStreamedMock = fetchStreamed as jest.MockedFunction<typeof fetchStreamed>++const NOW_MS = Date.parse('2026-10-15T12:00:00Z')+const PNG = Buffer.concat([Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]), Buffer.from('png body')])+const TEAM_KEYS: Record<string, Buffer> = {+ 'team:7': Buffer.from(vector.teamMonthKey, 'base64'),+ 'team:8': Buffer.alloc(32, 8),+}+const OTHER_HASH = 'B'.repeat(22)+const DAY_MS = 24 * 60 * 60 * 1000+const IMAGE_OBJECT_KEY = `images/url/month=${vector.identity.keyMonth}/7/${vector.url.ref.split(':').pop()}`++function respond(status: number, headers: Record<string, string> = {}, bytes = Buffer.alloc(0)): StreamedResponse {+ return {+ status,+ headers,+ headerLines: Object.entries(headers).map(([name, value]) => ({ name, value })),+ read: jest.fn().mockResolvedValue({ bytes, overLimit: false }),+ discard: jest.fn(),+ }+}++/** A record in the shape `replay-recorder-v4/src/urls.rs` produces: keyed by registrable domain, with no headers. */+function recorderRecord(jobs: { ref: string; url: string }[]): Message {+ const value = Buffer.from(+ JSON.stringify({+ v: 2,+ jobs: jobs.map(({ ref, url }) => ({+ sessionId: vector.identity.sessionId,+ originalRef: ref,+ currentUrl: url,+ remainingHops: 10,+ notBeforeMs: 0,+ firstSeenAtMs: NOW_MS,+ fetchCount: 0,+ republishCount: 0,+ lastRepublishReason: null,+ })),+ })+ )+ return { value, key: Buffer.from('example.com'), size: value.length, topic: 'v4', partition: 0, offset: 0 }+}++function openFrame(key: Buffer, frame: Buffer): { context: Record<string, unknown>; body: Buffer } {+ const length = frame.readUInt16BE(6)+ const aad = frame.subarray(8, 8 + length)+ const nonce = frame.subarray(8 + length, 20 + length)+ const sealed = frame.subarray(20 + length)+ const decipher = createDecipheriv('aes-256-gcm', key, nonce, { authTagLength: 16 })+ decipher.setAAD(aad)+ decipher.setAuthTag(sealed.subarray(sealed.length - 16))+ const body = Buffer.concat([decipher.update(sealed.subarray(0, sealed.length - 16)), decipher.final()])+ return { context: parseJSON(aad.toString()).context, body }+}++/** The origin's robots.txt and tdmrep.json are known for a year, so the jobs do not wait on one configuration fetch. */+class MemoryCrawlHistory implements CrawlHistoryStore {+ public readonly items = new Map<string, CrawlHistoryItem>(+ (['robots', 'tdmrep'] as const).map((kind) => {+ const origin = new URL(vector.url.url).origin+ const nowMs = Date.now()+ const item: CrawlHistoryItem = {+ kind,+ key: configurationCacheKey(origin, kind),+ origin,+ status: 'absent',+ fetchedAtMs: nowMs,+ refreshAtMs: nowMs + 365 * DAY_MS,+ freshUntilMs: nowMs + 365 * DAY_MS,+ retryAtMs: nowMs,+ storageExpiresAtMs: nowMs + 365 * DAY_MS,+ }+ return [item.key, item]+ })+ )+ read(keys: string[]): Promise<Map<string, CrawlHistoryItem>> {+ return Promise.resolve(+ new Map(keys.flatMap((key) => (this.items.has(key) ? [[key, this.items.get(key)!]] : [])))+ )+ }+ write(items: CrawlHistoryItem[]): Promise<void> {+ items.forEach((item) => this.items.set(item.key, item))+ return Promise.resolve()+ }+}++/** `keyReadsHang` makes every key row read wait until its caller gives up on it. */+function lane(rows: Record<string, Record<string, any> | undefined>, keyReadsHang = false, keyBudgetMs?: number) {+ const config = buildAiResearchV4ServerConfig({+ AI_RESEARCH_V4_IMAGE_FETCH_WEB_BOT_AUTH_PRIVATE_KEYS: generateKeyPairSync('ed25519')+ .privateKey.export({ type: 'pkcs8', format: 'pem' })+ .toString(),+ })+ const producer = { produce: jest.fn(() => Promise.resolve()) } as unknown as KafkaProducer+ const publisher = buildFrontierPublisher(producer, 10)+ const dynamo = {+ send: jest.fn((command: GetItemCommand, options?: any) => {+ if (keyReadsHang) {+ return new Promise<never>((_resolve, reject) =>+ options?.abortSignal?.addEventListener('abort', () => reject(options.abortSignal.reason))+ )+ }+ const id = `${command.input.Key!.pk.S}/${command.input.Key!.sk.S}`+ return Promise.resolve({ Item: rows[id] })+ }),+ }+ const kms = {+ send: jest.fn((command: any) =>+ Promise.resolve({ Plaintext: TEAM_KEYS[`team:${command.input.EncryptionContext.team_id}`] })+ ),+ }+ const keys = new TeamMonthKeys(dynamo, kms, 'keys', 'arn')+ const puts: PutObjectCommand[] = []+ const copies: CopyObjectCommand[] = []+ const s3 = {+ send: jest.fn((command: PutObjectCommand | CopyObjectCommand) => {+ if (command instanceof CopyObjectCommand) {+ copies.push(command)+ } else {+ puts.push(command)+ }+ return Promise.resolve({})+ }),+ }+ const store = new RawImageStore(keys, s3 as any, 'images-v4', 10)+ const topHog = new ImageFetchTopHogMetrics(createRecordingTopHog().registry)+ const runner = buildFetchRunner(config, publisher, topHog, store, keys)+ const admission = new V4JobAdmission(keys, keyBudgetMs)+ const deadLetters = { park: jest.fn(() => Promise.resolve()) }+ const history = new MemoryCrawlHistory()+ const consumer = new UrlFetchConsumer(+ history,+ publisher,+ {+ seenTtlSeconds: config.AI_RESEARCH_V4_IMAGE_FETCH_CRAWL_HISTORY_TTL_SECONDS,+ dryRun: false,+ admitCandidates: (candidates, nowMs) => admission.admit(candidates, nowMs),+ refreshingSink: store,+ },+ runner,+ deadLetters,+ topHog+ )+ return { consumer, runner, puts, copies, history, deadLetters, producer, dynamo, s3 }+}++const row = (team: number, month = vector.identity.keyMonth, extra: Record<string, any> = {}) => ({+ [`team:${team}/month:${month}`]: {+ pk: { S: `team:${team}` },+ sk: { S: `month:${month}` },+ wrapped_key: { B: Buffer.from('w') },+ ...extra,+ },+})++describe('v4 image fetch lane', () => {+ const runners: FetchRunner[] = []++ beforeEach(() => {+ fetchStreamedMock.mockReset()+ fetchStreamedMock.mockImplementation((url) =>+ Promise.resolve(+ new URL(url).pathname.endsWith('.png')+ ? respond(200, { 'content-type': 'image/png' }, PNG)+ : respond(404)+ )+ )+ })++ afterEach(async () => {+ await Promise.all(runners.splice(0).map((runner) => runner.close()))+ })++ it("stores a recorder record's image under each team's month key, with one request for a URL two teams share", async () => {+ const harness = lane({ ...row(7), ...row(8) })+ runners.push(harness.runner)+ const otherTeamRef = `imageurl:v4:8:${vector.identity.keyMonth}:${OTHER_HASH}`++ await harness.consumer.handleBatch(+ [+ recorderRecord([+ { ref: vector.url.ref, url: vector.url.url },+ { ref: otherTeamRef, url: vector.url.url },+ ]),+ ],+ NOW_MS+ )++ expect(harness.deadLetters.park).not.toHaveBeenCalled()+ expect(fetchStreamedMock.mock.calls.filter(([url]) => url === vector.url.url)).toHaveLength(1)+ const stored = new Map(harness.puts.map((put) => [put.input.Key, put.input.Body as Buffer]))+ expect([...stored.keys()].sort()).toEqual([+ IMAGE_OBJECT_KEY,+ `images/url/month=${vector.identity.keyMonth}/8/${OTHER_HASH}`,+ ])+ for (const [team, ref] of [+ [7, vector.url.ref],+ [8, otherTeamRef],+ ] as const) {+ const objectKey = `images/url/month=${vector.identity.keyMonth}/${team}/${ref.split(':').pop()}`+ const { context, body } = openFrame(TEAM_KEYS[`team:${team}`], stored.get(objectKey)!)+ expect(body).toEqual(PNG)+ expect(context).toEqual({+ teamId: team,+ keyMonth: vector.identity.keyMonth,+ kind: 'image-url',+ codec: 'none',+ ref,+ })+ }+ // Each team binds the shared response under its own month key, as the reference hash does, and stores no URL.+ const urlHistory = [vector.url.ref, otherTeamRef].map(+ (ref) => harness.history.items.get(ref) as UrlCrawlHistoryItem+ )+ expect(urlHistory.map((item) => item.cache?.responseUrlHmac)).toEqual([+ vector.url.ref.split(':').pop(),+ expect.not.stringMatching(vector.url.ref.split(':').pop()!),+ ])+ expect(JSON.stringify(urlHistory)).not.toContain(new URL(vector.url.url).host)+ })++ it.each([+ // The row of 2026-05 expired on 2026-10-01, and DynamoDB may not have removed it yet.+ ['a month whose key expired', row(7, '2026-05'), `imageurl:v4:7:2026-05:${OTHER_HASH}`],+ [+ 'a tombstoned team month',+ row(7, vector.identity.keyMonth, { deleted: { BOOL: true }, wrapped_key: undefined }),+ vector.url.ref,+ ],+ ['a team month without a key row', {}, vector.url.ref],+ ])('drops a job for %s before any request', async (_name, rows, ref) => {+ const harness = lane(rows)+ runners.push(harness.runner)++ await harness.consumer.handleBatch([recorderRecord([{ ref, url: vector.url.url }])], NOW_MS)++ expect(fetchStreamedMock).not.toHaveBeenCalled()+ expect(harness.puts).toHaveLength(0)+ expect(harness.deadLetters.park).not.toHaveBeenCalled()+ })++ it('sends a job whose key was not read within the batch key budget to the first delay tier, with no request', async () => {+ const harness = lane(row(7), true, 50)+ runners.push(harness.runner)++ await harness.consumer.handleBatch([recorderRecord([{ ref: vector.url.ref, url: vector.url.url }])], NOW_MS)++ expect(fetchStreamedMock).not.toHaveBeenCalled()+ expect(harness.deadLetters.park).not.toHaveBeenCalled()+ const produced = (harness.producer.produce as jest.Mock).mock.calls.map(([record]) => record.topic)+ expect(produced).toEqual([KAFKA_AI_RESEARCH_V4_IMAGE_FETCH_RETRY_1M])+ // The abandoned read was stopped, not left to run for its own budget.+ const [, readOptions] = harness.dynamo.send.mock.calls[0] as unknown as [unknown, { abortSignal: AbortSignal }]+ expect(readOptions.abortSignal.aborted).toBe(true)+ })++ describe('with a reference seen again later in its month', () => {+ const imageRequests = (): number =>+ fetchStreamedMock.mock.calls.filter(([url]) => url === vector.url.url).length+ const urlHistory = (harness: ReturnType<typeof lane>): UrlCrawlHistoryItem =>+ harness.history.items.get(vector.url.ref) as UrlCrawlHistoryItem+ const seeAt = async (harness: ReturnType<typeof lane>, nowMs: number): Promise<void> => {+ jest.setSystemTime(nowMs)+ await harness.consumer.handleBatch([recorderRecord([{ ref: vector.url.ref, url: vector.url.url }])], nowMs)+ }++ let harness: ReturnType<typeof lane>++ beforeEach(async () => {+ // Only the clock is fake: the fetch pass still runs on real timers.+ jest.useFakeTimers({+ now: NOW_MS,+ doNotFake: [+ 'nextTick',+ 'queueMicrotask',+ 'setImmediate',+ 'setTimeout',+ 'clearTimeout',+ 'setInterval',+ 'clearInterval',+ 'performance',+ 'hrtime',+ ],+ })+ fetchStreamedMock.mockImplementation((url, options) =>+ Promise.resolve(+ url !== vector.url.url+ ? respond(404)+ : (options?.headers as Record<string, string> | undefined)?.['if-none-match'] === '"v1"'+ ? respond(304, { etag: '"v1"' })+ : respond(200, { 'content-type': 'image/png', etag: '"v1"' }, PNG)+ )+ )+ harness = lane(row(7))+ runners.push(harness.runner)+ await seeAt(harness, NOW_MS)+ expect([imageRequests(), harness.puts.length]).toEqual([1, 1])+ })++ afterEach(() => {+ jest.useRealTimers()+ })++ it('keeps its history until 8 days after the end of its month', () => {+ expect(urlHistory(harness)).toMatchObject({+ storedAtMs: NOW_MS,+ storageExpiresAtMs: Date.parse('2026-11-09T00:00:00Z'),+ })+ })++ it('drops a job inside the refresh interval with no request and no write', async () => {+ await seeAt(harness, NOW_MS + 6 * DAY_MS)++ expect([imageRequests(), harness.puts.length, harness.copies.length]).toEqual([1, 1, 0])+ })++ it('refreshes the stored image by a copy onto itself after 7 days, with no origin request', async () => {+ await seeAt(harness, NOW_MS + 7 * DAY_MS)++ expect([imageRequests(), harness.puts.length]).toEqual([1, 1])+ expect(harness.copies.map((copy) => copy.input.Key)).toEqual([IMAGE_OBJECT_KEY])+ expect(urlHistory(harness).storedAtMs).toBe(NOW_MS + 7 * DAY_MS)++ await seeAt(harness, NOW_MS + 8 * DAY_MS)+ expect(harness.copies).toHaveLength(1)+ })++ it('fetches the whole image again in the same batch when its refresh copy finds it gone', async () => {+ harness.s3.send.mockImplementation((command: PutObjectCommand | CopyObjectCommand) => {+ if (command instanceof CopyObjectCommand) {+ harness.copies.push(command)+ return Promise.reject(+ new S3ServiceException({+ name: 'NoSuchKey',+ $fault: 'client',+ $metadata: { httpStatusCode: 404 },+ })+ )+ }+ harness.puts.push(command)+ return Promise.resolve({})+ })++ await seeAt(harness, NOW_MS + 7 * DAY_MS)++ const requests = fetchStreamedMock.mock.calls.filter(([url]) => url === vector.url.url)+ expect(+ requests.map(([, options]) => (options?.headers as Record<string, string>)['if-none-match'])+ ).toEqual([undefined, undefined])+ expect([harness.copies.length, harness.puts.length]).toEqual([1, 2])+ expect(urlHistory(harness).storedAtMs).toBe(NOW_MS + 7 * DAY_MS)+ })++ it.each([+ ['refreshes the image by a copy when the origin answers 304', 304, 0, 1],+ ['overwrites the image when the origin answers 200', 200, 1, 0],+ ])('revalidates after the refetch interval and %s', async (_name, status, puts, copies) => {+ if (status === 200) {+ fetchStreamedMock.mockImplementation(() =>+ Promise.resolve(respond(200, { 'content-type': 'image/png', etag: '"v2"' }, PNG))+ )+ }+ const refetchAtMs = urlHistory(harness).nextFetchAtMs+ expect(refetchAtMs).toBe(NOW_MS + 30 * DAY_MS)+ // A fresh lane reads the history, so no fetch the first lane shared in memory serves this job.+ const later = lane(row(7))+ runners.push(later.runner)+ later.history.items.set(vector.url.ref, urlHistory(harness))++ await seeAt(later, refetchAtMs)++ const requests = fetchStreamedMock.mock.calls.filter(([url]) => url === vector.url.url)+ expect(requests).toHaveLength(2)+ expect((requests[1][1]?.headers as Record<string, string>)['if-none-match']).toBe('"v1"')+ expect([later.puts.length, later.copies.length]).toEqual([puts, copies])+ expect([...later.puts, ...later.copies].map((write) => write.input.Key)).toEqual([IMAGE_OBJECT_KEY])+ expect(urlHistory(later)).toMatchObject({+ storedAtMs: refetchAtMs,+ nextFetchAtMs: refetchAtMs + 30 * DAY_MS,+ })+ })++ it.each([+ ['a 404', respond(404)],+ ['a page that is not an image', respond(200, { 'content-type': 'text/html' }, Buffer.from('<html>'))],+ ])('keeps refreshing the stored image after a refetch finds %s', async (_name, response) => {+ fetchStreamedMock.mockImplementation(() => Promise.resolve(response))+ const refetchAtMs = urlHistory(harness).nextFetchAtMs+ const later = lane(row(7))+ runners.push(later.runner)+ later.history.items.set(vector.url.ref, urlHistory(harness))++ await seeAt(later, refetchAtMs)+ expect(urlHistory(later).storedAtMs).toBe(NOW_MS)+ await seeAt(later, refetchAtMs + DAY_MS)++ expect(later.puts).toHaveLength(0)+ expect(later.copies.map((copy) => copy.input.Key)).toEqual([IMAGE_OBJECT_KEY])+ })++ it('fetches the whole image again when the origin answers 304 for an image S3 no longer holds', async () => {+ const refetchAtMs = urlHistory(harness).nextFetchAtMs+ const later = lane(row(7))+ runners.push(later.runner)+ later.history.items.set(vector.url.ref, urlHistory(harness))+ later.s3.send.mockImplementation((command: PutObjectCommand | CopyObjectCommand) => {+ if (command instanceof CopyObjectCommand) {+ return Promise.reject(+ new S3ServiceException({+ name: 'NoSuchKey',+ $fault: 'client',+ $metadata: { httpStatusCode: 404 },+ })+ )+ }+ later.puts.push(command)+ return Promise.resolve({})+ })++ await seeAt(later, refetchAtMs)+ const [retry] = (later.producer.produce as jest.Mock).mock.calls.map(([record]) => record)+ expect(retry.topic).toBe(KAFKA_AI_RESEARCH_V4_IMAGE_FETCH_RETRY_1M)+ const retryAtMs = refetchAtMs + 60_000+ jest.setSystemTime(retryAtMs)+ await later.consumer.handleBatch(+ [+ {+ value: retry.value,+ key: retry.key,+ size: retry.value.length,+ topic: 'v4',+ partition: 0,+ offset: 1,+ },+ ],+ retryAtMs+ )++ const requests = fetchStreamedMock.mock.calls.filter(([url]) => url === vector.url.url)+ expect(+ requests.map(([, options]) => (options?.headers as Record<string, string>)['if-none-match'])+ ).toEqual([undefined, '"v1"', undefined])+ expect(later.puts.map((put) => put.input.Key)).toEqual([IMAGE_OBJECT_KEY])+ expect(urlHistory(later).storedAtMs).toBe(retryAtMs)+ })+ })++ describe('with many slow key lookups', () => {+ beforeEach(() => {+ jest.useFakeTimers({ now: NOW_MS })+ })++ afterEach(() => {+ jest.useRealTimers()+ })++ it('finishes admission within the batch key budget, runs lookups bounded, and defers the jobs it did not reach', async () => {+ const LOOKUP_MS = 16_000+ let running = 0+ let mostRunning = 0+ let started = 0+ const keys = {+ get: async () => {+ started+++ running+++ mostRunning = Math.max(mostRunning, running)+ await new Promise((resolve) => setTimeout(resolve, LOOKUP_MS))+ running--+ return { status: 'ok' as const, key: Buffer.alloc(32) }+ },+ }+ const candidates = Array.from(+ { length: 256 },+ (_, team) =>+ ({+ originalRef: `imageurl:v4:${team + 1}:${vector.identity.keyMonth}:${OTHER_HASH}`,+ notBeforeMs: 0,+ }) as FetchCandidate+ )+ let admitted: FetchCandidate[] | undefined+ void new V4JobAdmission(keys, 60_000).admit(candidates, NOW_MS).then((result) => {+ admitted = result+ })++ await jest.advanceTimersByTimeAsync(60_000)++ expect(admitted).toHaveLength(256)+ expect(mostRunning).toBeLessThanOrEqual(16)+ // Lookups start at 0s, 16s, 32s and 48s, and only the first three rounds finish within 60s.+ const ready = admitted!.filter((candidate) => candidate.notBeforeMs === 0)+ expect(ready).toHaveLength(48)+ const deferred = admitted!.filter((candidate) => candidate.notBeforeMs > 0)+ expect(deferred).toHaveLength(208)+ expect(deferred.every((candidate) => candidate.notBeforeMs === NOW_MS + 60_000)).toBe(true)+ expect(deferred.every((candidate) => candidate.lastBlockReason === 'pass_deadline')).toBe(true)++ // A lookup still queued at the deadline never starts.+ await jest.advanceTimersByTimeAsync(60_000)+ expect(started).toBe(64)+ })+ })+})diff --git a/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/web-bot-auth.test.ts b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/web-bot-auth.test.tsnew file mode 100644index 000000000000..e9d212950ce2--- /dev/null+++ b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/web-bot-auth.test.ts@@ -0,0 +1,129 @@+import { KeyObject, createHash, generateKeyPairSync, verify as verifySignature } from 'node:crypto'++import { createWebBotAuthRequestSigner } from './web-bot-auth'++const SIGNATURE_AGENT = '"https://us.posthog.com/.well-known/http-message-signatures-directory"'++type TestKeyPair = {+ privateKeyPem: string+ publicKey: KeyObject+}++function generateEd25519KeyPair(): TestKeyPair {+ const { privateKey, publicKey } = generateKeyPairSync('ed25519')+ return {+ privateKeyPem: privateKey.export({ format: 'pem', type: 'pkcs8' }).toString(),+ publicKey,+ }+}++function keyId(publicKey: KeyObject): string {+ const jwk = publicKey.export({ format: 'jwk' })+ const canonicalJwk = JSON.stringify({ crv: jwk.crv, kty: jwk.kty, x: jwk.x })+ return createHash('sha256').update(canonicalJwk).digest('base64url')+}++function expectSignedHeadersToVerify(+ method: string,+ url: string,+ headers: Record<string, string>,+ publicKey: KeyObject,+ label: string,+ expectedResult = true+): void {+ const signatureInput = headers['signature-input'].split(', ').find((member) => member.startsWith(`${label}=`))+ const signature = headers.signature.split(', ').find((member) => member.startsWith(`${label}=`))+ if (!signatureInput || !signature) {+ throw new Error(`Missing ${label}`)+ }++ const parameters = signatureInput.slice(label.length + 1)+ const parametersMatch = parameters.match(+ /^\("@method" "@authority" "@target-uri" "signature-agent"\);created=(\d+);keyid="([^"]+)";alg="ed25519";expires=(\d+);nonce="([^"]+)";tag="web-bot-auth"$/+ )+ if (!parametersMatch) {+ throw new Error(`Invalid ${label} parameters`)+ }+ const [, created, actualKeyId, expires, nonce] = parametersMatch+ expect(actualKeyId).toBe(keyId(publicKey))+ expect(Number(expires) - Number(created)).toBe(60)+ expect(Buffer.from(nonce, 'base64url')).toHaveLength(64)++ const targetUrl = new URL(url)+ targetUrl.hash = ''+ const signatureBase =+ `"@method": ${method}\n` ++ `"@authority": ${targetUrl.host}\n` ++ `"@target-uri": ${targetUrl.toString()}\n` ++ `"signature-agent": ${SIGNATURE_AGENT}\n` ++ `"@signature-params": ${parameters}`+ const signatureBytes = Buffer.from(signature.slice(`${label}=:`.length, -1), 'base64')+ expect(verifySignature(null, Buffer.from(signatureBase), publicKey, signatureBytes)).toBe(expectedResult)+}++describe('Web Bot Auth request signing', () => {+ beforeEach(() => {+ jest.useFakeTimers().setSystemTime(new Date('2026-08-19T08:00:00Z'))+ })++ afterEach(() => {+ jest.useRealTimers()+ })++ it('creates a Cloudflare-compatible signature with current protocol parameters', () => {+ const keyPair = generateEd25519KeyPair()+ const signer = createWebBotAuthRequestSigner(keyPair.privateKeyPem)+ const url = 'https://cdn.example.com/image.png?size=large'++ const headers = signer.headersForGet(url)++ expect(headers['signature-agent']).toBe(SIGNATURE_AGENT)+ expectSignedHeadersToVerify('GET', url, headers, keyPair.publicKey, 'sig1')+ })++ it.each([+ ['a different method', 'POST', 'https://cdn.example.com/image.png?size=large'],+ ['a different path', 'GET', 'https://cdn.example.com/other.png?size=large'],+ ['a different query', 'GET', 'https://cdn.example.com/image.png?size=small'],+ ])('does not verify for %s', (_name, method, replayUrl) => {+ const keyPair = generateEd25519KeyPair()+ const signer = createWebBotAuthRequestSigner(keyPair.privateKeyPem)+ const headers = signer.headersForGet('https://cdn.example.com/image.png?size=large')++ expectSignedHeadersToVerify(method, replayUrl, headers, keyPair.publicKey, 'sig1', false)+ })++ it('signs with the first configured key during rotation', () => {+ const firstKeyPair = generateEd25519KeyPair()+ const secondKeyPair = generateEd25519KeyPair()+ const signer = createWebBotAuthRequestSigner(+ `${firstKeyPair.privateKeyPem},${secondKeyPair.privateKeyPem.replaceAll('\n', '\\n')}`+ )+ const url = 'https://cdn.example.com/image.png'++ const headers = signer.headersForGet(url)++ expectSignedHeadersToVerify('GET', url, headers, firstKeyPair.publicKey, 'sig1')+ expect(headers['signature-input']).not.toContain(keyId(secondKeyPair.publicKey))+ expect(headers.signature).not.toContain('sig2=')+ })++ it.each([+ ['an empty value', '', 'must contain at least one'],+ ['a malformed key', 'not a PEM', 'entry 1 could not be loaded'],+ [+ 'a malformed non-active key',+ `${generateEd25519KeyPair().privateKeyPem},not a PEM`,+ 'entry 2 could not be loaded',+ ],+ [+ 'a non-Ed25519 key',+ generateKeyPairSync('rsa', { modulusLength: 2048 })+ .privateKey.export({ format: 'pem', type: 'pkcs8' })+ .toString(),+ 'entry 1 is not an Ed25519 private key',+ ],+ ])('rejects %s', (_name, privateKeyPems, expectedError) => {+ expect(() => createWebBotAuthRequestSigner(privateKeyPems)).toThrow(expectedError)+ })+})diff --git a/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/web-bot-auth.ts b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/web-bot-auth.tsnew file mode 100644index 000000000000..b0c75b4128ed--- /dev/null+++ b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/web-bot-auth.ts@@ -0,0 +1,75 @@+import { KeyObject, createHash, createPrivateKey, createPublicKey, randomBytes, sign } from 'node:crypto'++const SIGNATURE_AGENT_URL = 'https://us.posthog.com/.well-known/http-message-signatures-directory'+const SIGNATURE_AGENT_HEADER = JSON.stringify(SIGNATURE_AGENT_URL)+const SIGNATURE_LIFETIME_SECONDS = 60+const REQUEST_METHOD = 'GET'++type SigningKey = {+ privateKey: KeyObject+ keyId: string+}++export interface WebBotAuthRequestSigner {+ headersForGet(url: string): Record<string, string>+}++export function createWebBotAuthRequestSigner(commaSeparatedPrivateKeyPems: string): WebBotAuthRequestSigner {+ const privateKeyPems = commaSeparatedPrivateKeyPems.split(',')+ if (privateKeyPems.length === 1 && privateKeyPems[0].trim() === '') {+ throw new Error('WEB_BOT_AUTH_PRIVATE_KEYS must contain at least one Ed25519 private key')+ }++ const signingKeys = privateKeyPems.map((privateKeyPem, index) => loadSigningKey(privateKeyPem, index + 1))+ return new Ed25519WebBotAuthRequestSigner(signingKeys[0])+}++class Ed25519WebBotAuthRequestSigner implements WebBotAuthRequestSigner {+ constructor(private readonly signingKey: SigningKey) {}++ public headersForGet(url: string): Record<string, string> {+ const targetUrl = new URL(url)+ targetUrl.hash = ''+ const authority = targetUrl.host+ const targetUri = targetUrl.toString()+ const created = Math.floor(Date.now() / 1000)+ const expires = created + SIGNATURE_LIFETIME_SECONDS+ const nonce = randomBytes(64).toString('base64url')+ const parameters =+ `("@method" "@authority" "@target-uri" "signature-agent");created=${created};keyid="${this.signingKey.keyId}";` ++ `alg="ed25519";expires=${expires};nonce="${nonce}";tag="web-bot-auth"`+ const signatureBase =+ `"@method": ${REQUEST_METHOD}\n` ++ `"@authority": ${authority}\n` ++ `"@target-uri": ${targetUri}\n` ++ `"signature-agent": ${SIGNATURE_AGENT_HEADER}\n` ++ `"@signature-params": ${parameters}`++ return {+ // Cloudflare rejects the dictionary form from newer drafts, so use the compatible structured string.+ 'signature-agent': SIGNATURE_AGENT_HEADER,+ 'signature-input': `sig1=${parameters}`,+ signature: `sig1=:${sign(null, Buffer.from(signatureBase), this.signingKey.privateKey).toString('base64')}:`,+ }+ }+}++function loadSigningKey(privateKeyPem: string, keyIndex: number): SigningKey {+ let privateKey: KeyObject+ try {+ privateKey = createPrivateKey(privateKeyPem.trim().replaceAll('\\n', '\n'))+ } catch {+ throw new Error(`WEB_BOT_AUTH_PRIVATE_KEYS entry ${keyIndex} could not be loaded`)+ }+ if (privateKey.asymmetricKeyType !== 'ed25519') {+ throw new Error(`WEB_BOT_AUTH_PRIVATE_KEYS entry ${keyIndex} is not an Ed25519 private key`)+ }++ const publicJwk = createPublicKey(privateKey).export({ format: 'jwk' })+ if (publicJwk.kty !== 'OKP' || publicJwk.crv !== 'Ed25519' || !publicJwk.x) {+ throw new Error(`WEB_BOT_AUTH_PRIVATE_KEYS entry ${keyIndex} has an invalid public key`)+ }+ const canonicalJwk = JSON.stringify({ crv: publicJwk.crv, kty: publicJwk.kty, x: publicJwk.x })+ const keyId = createHash('sha256').update(canonicalJwk).digest('base64url')+ return { privateKey, keyId }+}diff --git a/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/wildcard-pattern.test.ts b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/wildcard-pattern.test.tsnew file mode 100644index 000000000000..f33e8423d5e7--- /dev/null+++ b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/wildcard-pattern.test.ts@@ -0,0 +1,28 @@+import { wildcardPatternMatchesPathname } from './wildcard-pattern'++describe('wildcardPatternMatchesPathname', () => {+ it.each([+ ['a prefix', '/images/', '/images/photo.png', true],+ ['an exact path', '/images/photo.png$', '/images/photo.png', true],+ ['an exact path mismatch', '/images/photo.png$', '/images/photo.png/preview', false],+ ['a middle wildcard', '/images/*/preview$', '/images/v1/preview', true],+ ['an anchored suffix mismatch', '/images/*/preview$', '/images/v1/preview/more', false],+ ['a leading wildcard', '*/private/*', '/v1/private/image.png', true],+ ['literal regular expression characters', '/literal.+(item)$', '/literal.+(item)', true],+ ['an unreserved percent encoding', '/asset%7Ename$', '/asset~name', true],+ ['consecutive wildcards', '/asset/**/preview$', '/asset//preview', true],+ ['overlapping anchored segments', '*a*a$', '/aaa', true],+ ])('matches %s', (_name, pattern, pathname, expected) => {+ expect(wildcardPatternMatchesPathname(pattern, pathname)).toBe(expected)+ })++ it('does not compile wildcard patterns as regular expressions', () => {+ const regexpConstructor = jest.spyOn(globalThis, 'RegExp').mockReturnValue(/$a/)+ try {+ expect(wildcardPatternMatchesPathname(`${'*a'.repeat(64)}*b$`, `/${'a'.repeat(128)}c`)).toBe(false)+ expect(regexpConstructor).not.toHaveBeenCalled()+ } finally {+ regexpConstructor.mockRestore()+ }+ })+})diff --git a/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/wildcard-pattern.ts b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/wildcard-pattern.tsnew file mode 100644index 000000000000..460f5d984695--- /dev/null+++ b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/wildcard-pattern.ts@@ -0,0 +1,86 @@+export function wildcardPatternMatchesPathname(pattern: string, pathname: string): boolean {+ const decodedPattern = decodeUnreserved(pattern)+ const endAnchored = decodedPattern.endsWith('$')+ const matchPattern = endAnchored ? decodedPattern.slice(0, -1) : decodedPattern+ const decodedPathname = decodeUnreserved(pathname)++ if (!matchPattern.includes('*')) {+ return endAnchored ? decodedPathname === matchPattern : decodedPathname.startsWith(matchPattern)+ }++ const literalSegments = matchPattern.split('*').filter(Boolean)+ if (literalSegments.length === 0) {+ return true+ }++ let nextSearchIndex = 0+ let nextSegmentIndex = 0+ if (!matchPattern.startsWith('*')) {+ const firstSegment = literalSegments[0]+ if (!decodedPathname.startsWith(firstSegment)) {+ return false+ }+ nextSearchIndex = firstSegment.length+ nextSegmentIndex = 1+ }++ let segmentSearchEnd = literalSegments.length+ let pathnameSearchEnd = decodedPathname.length+ if (endAnchored && !matchPattern.endsWith('*')) {+ const finalSegmentIndex = literalSegments.length - 1+ const finalSegment = literalSegments[finalSegmentIndex]+ if (!decodedPathname.endsWith(finalSegment)) {+ return false+ }+ segmentSearchEnd = finalSegmentIndex+ pathnameSearchEnd = decodedPathname.length - finalSegment.length+ }++ for (; nextSegmentIndex < segmentSearchEnd; nextSegmentIndex++) {+ const segment = literalSegments[nextSegmentIndex]+ const matchIndex = findLiteralSegment(decodedPathname, segment, nextSearchIndex, pathnameSearchEnd)+ if (matchIndex === -1) {+ return false+ }+ nextSearchIndex = matchIndex + segment.length+ }++ return nextSearchIndex <= pathnameSearchEnd+}++function findLiteralSegment(value: string, segment: string, startIndex: number, endIndex: number): number {+ const prefixLengths = buildPrefixLengths(segment)+ for (let index = startIndex, matchedLength = 0; index < endIndex; index++) {+ while (matchedLength > 0 && value[index] !== segment[matchedLength]) {+ matchedLength = prefixLengths[matchedLength - 1]+ }+ if (value[index] === segment[matchedLength]) {+ matchedLength+++ }+ if (matchedLength === segment.length) {+ return index - segment.length + 1+ }+ }+ return -1+}++function buildPrefixLengths(value: string): number[] {+ const prefixLengths = Array<number>(value.length).fill(0)+ for (let index = 1, matchedLength = 0; index < value.length; index++) {+ while (matchedLength > 0 && value[index] !== value[matchedLength]) {+ matchedLength = prefixLengths[matchedLength - 1]+ }+ if (value[index] === value[matchedLength]) {+ matchedLength+++ }+ prefixLengths[index] = matchedLength+ }+ return prefixLengths+}++function decodeUnreserved(value: string): string {+ return value.replace(/%[0-9A-Fa-f]{2}/g, (encoded) => {+ const character = String.fromCharCode(Number.parseInt(encoded.slice(1), 16))+ return /^[A-Za-z0-9._~-]$/.test(character) ? character : encoded.toUpperCase()+ })+}diff --git a/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/server/v4-consumer-server.ts b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/server/v4-consumer-server.tsnew file mode 100644index 000000000000..61ad4ce34d79--- /dev/null+++ b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/server/v4-consumer-server.ts@@ -0,0 +1,34 @@+import { initializePrometheusLabels } from '~/common/api/router'+import {+ AiResearchV4ServerConfig,+ buildAiResearchV4ServerConfig,+} from '~/ingestion/pipelines/sessionreplay/ai-research/config'+import { CleanupResources, NodeServer, ServerLifecycle } from '~/servers/base-server'++/** The lifecycle every v4 Kafka consumer server shares: the HTTP server, metric labels, start and stop. */+export abstract class AiResearchV4ConsumerServer implements NodeServer {+ readonly lifecycle: ServerLifecycle+ protected readonly config: AiResearchV4ServerConfig++ constructor(config: Partial<AiResearchV4ServerConfig> = {}) {+ this.config = buildAiResearchV4ServerConfig(config)+ this.lifecycle = new ServerLifecycle(this.config)+ }++ async start(): Promise<void> {+ return this.lifecycle.start(+ () => {+ initializePrometheusLabels(this.config.INGESTION_PIPELINE, this.config.INGESTION_LANE)+ return this.startServices()+ },+ () => this.getCleanupResources()+ )+ }++ async stop(error?: Error): Promise<void> {+ return this.lifecycle.stop(() => this.getCleanupResources(), error)+ }++ protected abstract startServices(): Promise<void>+ protected abstract getCleanupResources(): CleanupResources+}diff --git a/nodejs/src/servers/ai-research-v4-image-fetch-server.test.ts b/nodejs/src/servers/ai-research-v4-image-fetch-server.test.tsnew file mode 100644index 000000000000..5c90f1852082--- /dev/null+++ b/nodejs/src/servers/ai-research-v4-image-fetch-server.test.ts@@ -0,0 +1,234 @@+import { Message } from 'node-rdkafka'++import { KAFKA_AI_RESEARCH_V4_IMAGE_FETCH } from '~/common/config/kafka-topics'+import { KafkaProducer } from '~/common/kafka/producer'+import { buildAiResearchV4ServerConfig } from '~/ingestion/pipelines/sessionreplay/ai-research/config'+import {+ FetchCandidatePool,+ FetchCandidatePoolAdmission,+} from '~/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/fetch-candidate-pool'+import { KafkaFrontierDeadLetterSink } from '~/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/frontier-dead-letter-sink'+import { ImageFetchBatchJoiner } from '~/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/image-fetch-batch-joiner'+import {+ BLOCKED_IMAGE_FETCH_DEAD_LETTER_TOPICS,+ V4_IMAGE_FETCH_LANE,+} from '~/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/lane'++import {+ buildImageFetchBatchHandlers,+ buildImageFetchConsumerConfigs,+ buildImageFetchConsumerOverrides,+ imageFetchBatchesPerPass,+ shutdownImageFetchConsumers,+} from './ai-research-v4-image-fetch-server'++describe('v4 image fetch consumer wiring', () => {+ it('consumes only the v4 frontier, in its own group', () => {+ // A v4 pod on the v3 frontier would fetch v3 URLs and store their images unscrubbed.+ const consumerConfigs = buildImageFetchConsumerConfigs(buildAiResearchV4ServerConfig({}))+ expect(new Set(consumerConfigs.map((config) => config.topic))).toEqual(+ new Set([KAFKA_AI_RESEARCH_V4_IMAGE_FETCH])+ )+ expect(new Set(consumerConfigs.map((config) => config.groupId))).toEqual(+ new Set(['ai-research-v4-image-fetch'])+ )+ })++ it.each(['', 'session-replay-ml-image-fetch'])('refuses the group %p', (group) => {+ const config = buildAiResearchV4ServerConfig({ AI_RESEARCH_V4_IMAGE_FETCH_GROUP_ID: group })+ expect(() => buildImageFetchConsumerConfigs(config)).toThrow('AI_RESEARCH_V4_IMAGE_FETCH_GROUP_ID')+ })++ it.each(Object.values(V4_IMAGE_FETCH_LANE))('refuses lane topic %s as the dead-letter topic', (topic) => {+ expect(+ () => new KafkaFrontierDeadLetterSink({} as KafkaProducer, topic, BLOCKED_IMAGE_FETCH_DEAD_LETTER_TOPICS)+ ).toThrow()+ })++ it.each([+ ['the default', {}, 2, 2, 2, 102_400],+ ['four consumers', { AI_RESEARCH_V4_IMAGE_FETCH_TARGET_PARTITIONS_PER_BATCH: 4 }, 4, 4, 2, 164_096],+ ['eight consumers', { AI_RESEARCH_V4_IMAGE_FETCH_TARGET_PARTITIONS_PER_BATCH: 8 }, 8, 8, 2, 328_192],+ [+ 'eight unjoined consumers',+ {+ AI_RESEARCH_V4_IMAGE_FETCH_TARGET_PARTITIONS_PER_BATCH: 8,+ AI_RESEARCH_V4_IMAGE_FETCH_JOIN_MEMBER_BATCHES: false,+ },+ 8,+ 1,+ 2,+ 328_192,+ ],+ [+ 'eight continuous-pool consumers',+ {+ AI_RESEARCH_V4_IMAGE_FETCH_TARGET_PARTITIONS_PER_BATCH: 8,+ AI_RESEARCH_V4_IMAGE_FETCH_CONTINUOUS_POOL: true,+ },+ 8,+ 8,+ 4,+ 328_192,+ ],+ ['sixteen consumers', { AI_RESEARCH_V4_IMAGE_FETCH_TARGET_PARTITIONS_PER_BATCH: 16 }, 16, 16, 2, 656_384],+ ])(+ 'creates %s number of Kafka group members',+ (+ _name,+ overrides,+ expectedConsumers,+ expectedBatchesPerPass,+ expectedBatchesInFlightPerMember,+ expectedQueueBudget+ ) => {+ const serverConfig = buildAiResearchV4ServerConfig(overrides)+ const consumerConfigs = buildImageFetchConsumerConfigs(serverConfig)+ const consumerOverrides = buildImageFetchConsumerOverrides(serverConfig, consumerConfigs.length)++ expect(consumerConfigs).toHaveLength(expectedConsumers)+ expect(imageFetchBatchesPerPass(serverConfig, consumerConfigs.length)).toBe(expectedBatchesPerPass)+ expect(consumerConfigs.map((config) => config.maxBackgroundTasks)).toEqual(+ Array(expectedConsumers).fill(expectedBatchesInFlightPerMember)+ )+ expect(consumerConfigs.every((config) => config.backgroundTaskTimeoutMs === 240_000)).toBe(true)+ expect(consumerConfigs.map((config) => config.groupId)).toEqual(+ Array(expectedConsumers).fill(serverConfig.AI_RESEARCH_V4_IMAGE_FETCH_GROUP_ID)+ )+ expect(Number(consumerOverrides['queued.max.messages.kbytes']) * expectedConsumers).toBeLessThanOrEqual(+ expectedQueueBudget+ )+ expect(Number(consumerOverrides['queued.max.messages.kbytes']) * expectedConsumers).toBeGreaterThanOrEqual(+ expectedQueueBudget - expectedConsumers+ )+ expect(Number(consumerOverrides['queued.max.messages.kbytes']) * 1024).toBeGreaterThanOrEqual(+ serverConfig.AI_RESEARCH_V4_IMAGE_FETCH_MAX_IMAGE_BYTES * 2 + 64 * 1024+ )+ }+ )++ it.each([+ ['the joiner without a pool', false],+ ['its own pool window with a pool', true],+ ])('routes each member batch through %s', async (_name, withPool) => {+ const pool = withPool+ ? new FetchCandidatePool<unknown>(+ { maxConcurrentPerRegistrableDomain: 6, refillRunnableUrls: 100, maxQueuedUrlsPerOwner: 1_000 },+ { originCrawlDelayMs: () => 0, originNextImageStartAtMs: () => 0 }+ )+ : undefined+ const fetchConsumer = { handleBatch: jest.fn().mockResolvedValue(undefined) }+ const joiner = {+ handleBatch: jest.fn().mockResolvedValue(undefined),+ waitForProcessing: jest.fn().mockResolvedValue(undefined),+ }+ const message: Message = { topic: 'test-topic', partition: 0, offset: 1, value: Buffer.from('x'), size: 1 }++ const { handlers } = buildImageFetchBatchHandlers(2, fetchConsumer, { candidatePool: pool }, joiner)+ await Promise.all(handlers.map((handler) => handler([message])))++ expect(handlers).toHaveLength(2)+ expect(joiner.handleBatch).toHaveBeenCalledTimes(withPool ? 0 : 2)+ expect(fetchConsumer.handleBatch).toHaveBeenCalledTimes(withPool ? 2 : 0)+ if (withPool) {+ const [firstAdmission, secondAdmission] = fetchConsumer.handleBatch.mock.calls.map((call) => call[2])+ expect(firstAdmission.owner).not.toBe(secondAdmission.owner)+ }+ pool?.close()+ })++ it('waits for a pooled member batch that is still preparing before it closes the fetch runner', async () => {+ const pool = new FetchCandidatePool<unknown>(+ { maxConcurrentPerRegistrableDomain: 6, refillRunnableUrls: 100, maxQueuedUrlsPerOwner: 1_000 },+ { originCrawlDelayMs: () => 0, originNextImageStartAtMs: () => 0 }+ )+ let finishBatch: () => void = () => undefined+ const fetchConsumer = {+ handleBatch: jest.fn(+ (_messages: Message[], _nowMs: number, admission?: FetchCandidatePoolAdmission) =>+ new Promise<void>((resolve) => {+ admission?.admitted()+ finishBatch = resolve+ })+ ),+ }+ const joiner = { handleBatch: jest.fn(), waitForProcessing: jest.fn().mockResolvedValue(undefined) }+ const startedBatches = buildImageFetchBatchHandlers(1, fetchConsumer, { candidatePool: pool }, joiner)+ await startedBatches.handlers[0]([+ { topic: 'test-topic', partition: 0, offset: 1, value: Buffer.from('x'), size: 1 },+ ])+ const consumers = [{ stopConsuming: jest.fn().mockResolvedValue(undefined), disconnect: jest.fn() }]+ const fetchRunner = { close: jest.fn().mockResolvedValue(undefined) }++ const shutdown = shutdownImageFetchConsumers(consumers, startedBatches, fetchRunner)+ await new Promise((resolve) => setImmediate(resolve))+ expect(fetchRunner.close).not.toHaveBeenCalled()+ finishBatch()+ await shutdown++ expect(fetchRunner.close).toHaveBeenCalledTimes(1)+ pool.close()+ })++ it.each([false, true])('waits for joined work before disconnecting and cleaning up (failure=%s)', async (fails) => {+ jest.useFakeTimers()+ try {+ const processBatch = jest.fn(+ (messages: Message[]) =>+ new Promise<void>((resolve, reject) => {+ setTimeout(+ () => {+ if (fails && messages[0].offset === 1) {+ reject(new Error('fetch failed'))+ } else {+ resolve()+ }+ },+ messages[0].offset === 1 ? 30_000 : 50_000+ )+ })+ )+ const joiner = new ImageFetchBatchJoiner(1, processBatch)+ const completions = await Promise.all(+ [1, 2].map((offset) =>+ joiner.handleBatch([+ { topic: 'test-topic', partition: 0, offset, value: Buffer.from('test'), size: 4 },+ ])+ )+ )+ const work = Promise.allSettled(completions.map((batch) => batch?.backgroundTask ?? Promise.resolve()))+ const consumers = [0, 1].map(() => ({+ stopConsuming: jest.fn(() => new Promise<void>((resolve) => setTimeout(resolve, 20_000))),+ disconnect: jest.fn().mockResolvedValue(undefined),+ }))+ const cleanup = jest.fn()+ const shutdown = shutdownImageFetchConsumers(consumers, joiner).then(cleanup)++ expect(consumers.every((consumer) => consumer.stopConsuming.mock.calls.length === 1)).toBe(true)+ await jest.advanceTimersByTimeAsync(30_000)+ expect(consumers.every((consumer) => consumer.disconnect.mock.calls.length === 0)).toBe(true)+ expect(cleanup).not.toHaveBeenCalled()++ await jest.advanceTimersByTimeAsync(20_000)+ await shutdown+ await work+ expect(consumers.every((consumer) => consumer.disconnect.mock.calls.length === 1)).toBe(true)+ expect(cleanup).toHaveBeenCalledTimes(1)+ } finally {+ jest.useRealTimers()+ }+ })++ it.each([0, -1, 1.5, 17, Number.NaN, Number.MAX_SAFE_INTEGER + 1])(+ 'refuses invalid target partition count %p',+ (targetPartitionsPerBatch) => {+ const serverConfig = buildAiResearchV4ServerConfig({+ AI_RESEARCH_V4_IMAGE_FETCH_TARGET_PARTITIONS_PER_BATCH: targetPartitionsPerBatch,+ })++ expect(() => buildImageFetchConsumerConfigs(serverConfig)).toThrow(+ 'image fetch batch target must be an integer between 1 and 16'+ )+ }+ )+})diff --git a/nodejs/src/servers/ai-research-v4-image-fetch-server.ts b/nodejs/src/servers/ai-research-v4-image-fetch-server.tsnew file mode 100644index 000000000000..9ef9b22ec8f6--- /dev/null+++ b/nodejs/src/servers/ai-research-v4-image-fetch-server.ts@@ -0,0 +1,562 @@+import { DynamoDBClient } from '@aws-sdk/client-dynamodb'+import { KMSClient } from '@aws-sdk/client-kms'+import { NodeHttpHandler } from '@smithy/node-http-handler'+import { LibrdKafkaError } from 'node-rdkafka'++import { KafkaConsumer, KafkaConsumerConfig, findOffsetsToCommit } from '~/common/kafka/consumer/consumer-v1'+import {+ EachBatch,+ KafkaConsumerV2,+ KafkaConsumerV2Config,+ RdKafkaConsumerOverrides,+} from '~/common/kafka/consumer/consumer-v2'+import { KafkaProducer } from '~/common/kafka/producer'+import { KafkaProducerRegistry } from '~/common/outputs/kafka-producer-registry'+import { logger } from '~/common/utils/logger'+import { TopHog } from '~/ingestion/framework/tophog/tophog'+import { AiResearchV4ServerConfig } from '~/ingestion/pipelines/sessionreplay/ai-research/config'+import {+ ConfigurationPolicyService,+ HttpConfigurationFetcher,+} from '~/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/configuration-policy'+import { DynamoDBCrawlHistory } from '~/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/dynamodb-crawl-history'+import { FetchCandidatePool } from '~/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/fetch-candidate-pool'+import { FetchRunner } from '~/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/fetch-runner'+import { KafkaFrontierDeadLetterSink } from '~/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/frontier-dead-letter-sink'+import { FrontierPublisher } from '~/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/frontier-publisher'+import { HostBudget } from '~/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/host-budget'+import {+ ImageFetchBatchJoiner,+ assertImageFetchBatchTarget,+} from '~/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/image-fetch-batch-joiner'+import { HttpImageFetcher } from '~/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/image-fetcher'+import type { RefreshingImageSink } from '~/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/image-sink'+import { V4JobAdmission } from '~/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/job-admission'+import {+ BLOCKED_IMAGE_FETCH_DEAD_LETTER_TOPICS,+ HostLimits,+ V4_IMAGE_FETCH_DELAY_TOPICS,+ V4_IMAGE_FETCH_LANE,+ assertOwnImageFetchGroup,+ v4HostLimits,+} from '~/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/lane'+import { OriginRequestScheduler } from '~/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/origin-request-scheduler'+import { assertUrlPolicyLoaded } from '~/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/politeness-key'+import { createPoolWindowBatchHandler } from '~/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/pool-window-batch-handler'+import {+ AI_RESEARCH_V4_IMAGE_FETCH_PRODUCER,+ AiResearchV4ImageFetchProducerName,+ createV4ImageFetchOutputs,+ createV4ImageFetchProducerRegistry,+} from '~/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/producer'+import { RawImageStore } from '~/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/raw-image-store'+import { TeamMonthResponseUrlBinder } from '~/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/response-url-binding'+import { RetryDelayConsumer } from '~/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/retry-delay-consumer'+import { SharedImageFetcher } from '~/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/shared-image-fetcher'+import { ImageFetchTopHogMetrics } from '~/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/tophog-metrics'+import { UrlFetchConsumer } from '~/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/url-fetch-consumer'+import { createWebBotAuthRequestSigner } from '~/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/web-bot-auth'+import { AiResearchV4ConsumerServer } from '~/ingestion/pipelines/sessionreplay/ai-research/server/v4-consumer-server'+import { TeamMonthKeys } from '~/ingestion/pipelines/sessionreplay/ai-research/v4-keys/team-month-keys'+import { CaptureWatermark, capturedRecords } from '~/ingestion/pipelines/sessionreplay/shared/capture-watermark'+import { buildSessionRecordingS3Client } from '~/ingestion/pipelines/sessionreplay/shared/s3-client'+import { HealthCheckResultOk } from '~/types'++import { CleanupResources } from './base-server'++/**+ * How long the store may spend on one batch, well inside Kafka's max.poll.interval.ms of 300s.+ *+ * The poll interval binds rather than the health check, because the heartbeat above keeps the health+ * check satisfied through a long batch. A batch that passes the poll interval loses the partition+ * mid-batch to a pod that will be just as slow, so the lane sheds the rest of the batch instead.+ */+const STORE_BATCH_BUDGET_MS = 50_000+const IMAGE_FETCH_KAFKA_QUEUE_BUDGET_KBYTES = 102_400+const IMAGE_FETCH_MIN_CONSUMER_QUEUE_KBYTES = 25_600+/**+ * The pool window paces a member in continuous-pool mode. This limit stays above that window at typical+ * batch sizes, and it is the hard bound on a member's resident URLs and crash replay when batches are large.+ */+const POOL_MAX_BATCHES_IN_FLIGHT_PER_MEMBER = 4++/** Matches MAX_URL_LEN in the crate, which is what the collector applied to the first candidate. */+const MAX_REDIRECT_URL_LENGTH = 2048++/**+ * Headroom on top of the sleeping a batch can do, for the publishing between the sleeps.+ *+ * librdkafka rejects a `max.poll.interval.ms` above one day, so a tier and a batch size whose+ * product passes that is a configuration error rather than something to clamp silently.+ */+const POLL_INTERVAL_MARGIN_MS = 60_000+const MAX_POLL_INTERVAL_MS = 86_400_000+const REVOKED_PARTITION_CODES = new Set([-172, -190, -142, -144])++/** The addon is a native library and `index.ts` imports every server, so it loads on first use. */+let urlPolicy: typeof import('@posthog/ai-research-v4-native') | undefined+function getUrlPolicy(): typeof import('@posthog/ai-research-v4-native') {+ if (!urlPolicy) {+ urlPolicy = require('@posthog/ai-research-v4-native') as typeof import('@posthog/ai-research-v4-native')+ if (typeof urlPolicy.isPublicHost !== 'function') {+ throw new Error('the ai-research-v4-native addon has no isPublicHost: rebuild index.node')+ }+ }+ return urlPolicy+}++export function buildFrontierPublisher(producer: KafkaProducer, maxConcurrentRepublishes: number): FrontierPublisher {+ return new FrontierPublisher(producer, {+ frontierTopic: V4_IMAGE_FETCH_LANE.frontier,+ delayTiers: [+ { topic: V4_IMAGE_FETCH_LANE.retry1m, delayMs: 60_000, metricTopic: 'retry_1m' },+ { topic: V4_IMAGE_FETCH_LANE.retry10m, delayMs: 600_000, metricTopic: 'retry_10m' },+ { topic: V4_IMAGE_FETCH_LANE.retry1h, delayMs: 3_600_000, metricTopic: 'retry_1h' },+ ],+ maxConcurrentRepublishes,+ })+}++/** The lane's per registrable domain limits: the configured share of the configured limits. */+export function imageFetchHostLimits(config: AiResearchV4ServerConfig): HostLimits {+ return v4HostLimits(+ {+ requestsPerSecond: config.AI_RESEARCH_V4_IMAGE_FETCH_REGISTRABLE_DOMAIN_REQUESTS_PER_SECOND,+ burst: config.AI_RESEARCH_V4_IMAGE_FETCH_REGISTRABLE_DOMAIN_BURST,+ maxConcurrent: config.AI_RESEARCH_V4_IMAGE_FETCH_MAX_CONCURRENT_PER_REGISTRABLE_DOMAIN,+ },+ config.AI_RESEARCH_V4_IMAGE_FETCH_HOST_BUDGET_SHARE+ )+}++export function buildFetchRunner(+ config: AiResearchV4ServerConfig,+ publisher: FrontierPublisher,+ topHogMetrics: ImageFetchTopHogMetrics,+ imageSink: RefreshingImageSink,+ keys: Pick<TeamMonthKeys, 'get'>,+ hostLimits: HostLimits = imageFetchHostLimits(config)+): FetchRunner {+ const webBotAuthSigner = createWebBotAuthRequestSigner(config.AI_RESEARCH_V4_IMAGE_FETCH_WEB_BOT_AUTH_PRIVATE_KEYS)+ const budget = new HostBudget({+ ...hostLimits,+ breakerFailures: config.AI_RESEARCH_V4_IMAGE_FETCH_REGISTRABLE_DOMAIN_BREAKER_FAILURES,+ breakerCooldownMs: config.AI_RESEARCH_V4_IMAGE_FETCH_REGISTRABLE_DOMAIN_BREAKER_COOLDOWN_MS,+ breakerMaxCooldownMs: config.AI_RESEARCH_V4_IMAGE_FETCH_REGISTRABLE_DOMAIN_BREAKER_MAX_COOLDOWN_MS,+ maxTrackedRegistrableDomains: config.AI_RESEARCH_V4_IMAGE_FETCH_MAX_TRACKED_REGISTRABLE_DOMAINS,+ maxTrackedOrigins: config.AI_RESEARCH_V4_IMAGE_FETCH_MAX_TRACKED_ORIGINS,+ })+ const scheduler = new OriginRequestScheduler(+ budget,+ config.AI_RESEARCH_V4_IMAGE_FETCH_MAX_IN_FLIGHT_REQUESTS,+ topHogMetrics+ )+ const configurationPolicy = new ConfigurationPolicyService(+ new HttpConfigurationFetcher(webBotAuthSigner, scheduler, config.AI_RESEARCH_V4_IMAGE_FETCH_REQUEST_TIMEOUT_MS)+ )+ return new FetchRunner(+ // The recorder sends one job per team for the same URL and month, so one fetch serves them all.+ new SharedImageFetcher(+ new HttpImageFetcher(+ {+ maxUrlLength: MAX_REDIRECT_URL_LENGTH,+ // The crate's rule, so a redirect target meets the check the collector already applied+ // to the first candidate rather than a second answer to the same question.+ isPublicHost: (host) => getUrlPolicy().isPublicHost(host),+ },+ webBotAuthSigner+ )+ ),+ budget,+ scheduler,+ configurationPolicy,+ {+ maxConcurrentPerRegistrableDomain: hostLimits.maxConcurrent,+ maxInFlightRequests: config.AI_RESEARCH_V4_IMAGE_FETCH_MAX_IN_FLIGHT_REQUESTS,+ batchBudgetMs: config.AI_RESEARCH_V4_IMAGE_FETCH_REQUEST_BUDGET_MS,+ maxBytes: config.AI_RESEARCH_V4_IMAGE_FETCH_MAX_IMAGE_BYTES,+ requestTimeoutMs: config.AI_RESEARCH_V4_IMAGE_FETCH_REQUEST_TIMEOUT_MS,+ maxRedirects: config.AI_RESEARCH_V4_IMAGE_FETCH_MAX_REDIRECTS,+ seenTtlSeconds: config.AI_RESEARCH_V4_IMAGE_FETCH_CRAWL_HISTORY_TTL_SECONDS,+ continuousPool: config.AI_RESEARCH_V4_IMAGE_FETCH_CONTINUOUS_POOL+ ? {+ refillRunnableUrls: config.AI_RESEARCH_V4_IMAGE_FETCH_POOL_REFILL_RUNNABLE_URLS,+ maxQueuedUrlsPerOwner: config.AI_RESEARCH_V4_IMAGE_FETCH_POOL_MAX_QUEUED_URLS_PER_MEMBER,+ }+ : undefined,+ },+ publisher,+ imageSink,+ new TeamMonthResponseUrlBinder(keys),+ topHogMetrics+ )+}++export function buildImageFetchConsumerConfigs(config: AiResearchV4ServerConfig): KafkaConsumerV2Config[] {+ assertOwnImageFetchGroup(config.AI_RESEARCH_V4_IMAGE_FETCH_GROUP_ID)+ const targetBatchCount = config.AI_RESEARCH_V4_IMAGE_FETCH_TARGET_PARTITIONS_PER_BATCH+ assertImageFetchBatchTarget(targetBatchCount)+ return Array.from({ length: targetBatchCount }, () => ({+ topic: V4_IMAGE_FETCH_LANE.frontier,+ groupId: config.AI_RESEARCH_V4_IMAGE_FETCH_GROUP_ID,+ autoCommit: true,+ autoOffsetStore: true,+ fetchBatchSize: config.AI_RESEARCH_V4_IMAGE_FETCH_BATCH_SIZE,+ maxBackgroundTasks: config.AI_RESEARCH_V4_IMAGE_FETCH_CONTINUOUS_POOL+ ? POOL_MAX_BATCHES_IN_FLIGHT_PER_MEMBER+ : 2,+ backgroundTaskTimeoutMs: 240_000,+ }))+}++export function imageFetchBatchesPerPass(config: AiResearchV4ServerConfig, consumerCount: number): number {+ return config.AI_RESEARCH_V4_IMAGE_FETCH_JOIN_MEMBER_BATCHES ? consumerCount : 1+}++export function buildImageFetchConsumerOverrides(+ config: AiResearchV4ServerConfig,+ consumerCount: number+): RdKafkaConsumerOverrides {+ assertImageFetchBatchTarget(consumerCount)+ const maximumRecordBytes = config.AI_RESEARCH_V4_IMAGE_FETCH_MAX_IMAGE_BYTES * 2 + 64 * 1024+ return {+ 'fetch.message.max.bytes': maximumRecordBytes,+ 'max.partition.fetch.bytes': maximumRecordBytes,+ 'queued.max.messages.kbytes': Math.max(+ IMAGE_FETCH_MIN_CONSUMER_QUEUE_KBYTES,+ Math.ceil(maximumRecordBytes / 1024),+ Math.floor(IMAGE_FETCH_KAFKA_QUEUE_BUDGET_KBYTES / consumerCount)+ ),+ }+}++export interface ImageFetchBatchHandlers {+ handlers: EachBatch[]+ /** Resolves once every batch started so far has settled, including pooled batches that the Kafka drain gave up on. */+ waitForProcessing(): Promise<void>+}++export function buildImageFetchBatchHandlers(+ consumerCount: number,+ fetchConsumer: Pick<UrlFetchConsumer, 'handleBatch'>,+ fetchRunner: { candidatePool?: Pick<FetchCandidatePool<unknown>, 'createOwner'> },+ batchJoiner: Pick<ImageFetchBatchJoiner, 'handleBatch' | 'waitForProcessing'>+): ImageFetchBatchHandlers {+ const pool = fetchRunner.candidatePool+ const pooledBatches = new Set<Promise<void>>()+ const trackPooledBatch = (batch: Promise<void>): Promise<void> => {+ pooledBatches.add(batch)+ const forget = (): void => {+ pooledBatches.delete(batch)+ }+ batch.then(forget, forget)+ return batch+ }+ const handlers = Array.from({ length: consumerCount }, (): EachBatch => {+ if (!pool) {+ return (messages) => batchJoiner.handleBatch(messages)+ }+ return createPoolWindowBatchHandler(pool.createOwner(), (messages, admission) =>+ trackPooledBatch(fetchConsumer.handleBatch(messages, Date.now(), admission))+ )+ })+ return {+ handlers,+ waitForProcessing: async () => {+ await batchJoiner.waitForProcessing()+ await Promise.allSettled(pooledBatches)+ },+ }+}++export async function shutdownImageFetchConsumers(+ consumers: Pick<KafkaConsumerV2, 'stopConsuming' | 'disconnect'>[],+ startedBatches: Pick<ImageFetchBatchHandlers, 'waitForProcessing'>,+ fetchRunner?: Pick<FetchRunner, 'close'>+): Promise<void> {+ await Promise.allSettled(consumers.map((consumer) => consumer.stopConsuming()))+ await startedBatches.waitForProcessing()+ await Promise.allSettled(consumers.map((consumer) => consumer.disconnect()))+ await fetchRunner?.close()+}++/**+ * The v4 image fetch lane: the fetch policy of the v3 lane over the v4 recorder's URLs, storing each image sealed and+ * unscrubbed in `images-v4`. It reads the team month keys and never writes them, and it uses a share of the configured+ * per-host limits.+ */+export class AiResearchV4ImageFetchServer extends AiResearchV4ConsumerServer {+ private crawlHistoryClient?: DynamoDBClient+ private producerRegistry?: KafkaProducerRegistry<AiResearchV4ImageFetchProducerName>+ private topHog?: TopHog++ protected async startServices(): Promise<void> {+ // Here rather than on the first record. The parser calls into this addon for every URL, and+ // it must answer with a reason rather than raise, so a build that shipped without the addon+ // has to stop the pod at startup instead.+ assertUrlPolicyLoaded()++ const consumerConfigs = buildImageFetchConsumerConfigs(this.config)+ const dryRun = this.config.AI_RESEARCH_V4_IMAGE_FETCH_DRY_RUN+ const { AI_RESEARCH_V4_KEY_TABLE, AI_RESEARCH_V4_KMS_KEY_ARN, AI_RESEARCH_V4_IMAGES_BUCKET } = this.config+ if (!AI_RESEARCH_V4_KEY_TABLE || !AI_RESEARCH_V4_KMS_KEY_ARN || !AI_RESEARCH_V4_IMAGES_BUCKET) {+ throw new Error(+ 'the v4 image fetch lane needs AI_RESEARCH_V4_KEY_TABLE, AI_RESEARCH_V4_KMS_KEY_ARN and AI_RESEARCH_V4_IMAGES_BUCKET'+ )+ }+ const historyTable = this.config.AI_RESEARCH_V4_IMAGE_FETCH_HISTORY_TABLE+ if (!historyTable) {+ throw new Error('AI_RESEARCH_V4_IMAGE_FETCH_HISTORY_TABLE must be set for the v4 image fetch lane')+ }+ const dynamoDBTimeoutMs = this.config.AI_RESEARCH_V4_IMAGE_FETCH_DYNAMODB_TIMEOUT_MS+ if (!Number.isFinite(dynamoDBTimeoutMs) || dynamoDBTimeoutMs <= 0) {+ throw new Error(+ `AI_RESEARCH_V4_IMAGE_FETCH_DYNAMODB_TIMEOUT_MS must be a positive number, got ${dynamoDBTimeoutMs}`+ )+ }+ const s3 = buildSessionRecordingS3Client(this.config)+ if (!s3) {+ throw new Error('SESSION_RECORDING_V2_S3_* must be configured for the v4 image fetch lane')+ }+ const region = this.config.SESSION_RECORDING_V2_S3_REGION || 'us-east-1'+ this.crawlHistoryClient = new DynamoDBClient({+ region,+ endpoint: this.config.SESSION_RECORDING_DYNAMODB_ENDPOINT,+ maxAttempts: 5,+ requestHandler: new NodeHttpHandler(),+ })+ const crawlHistory = new DynamoDBCrawlHistory(+ this.crawlHistoryClient,+ historyTable,+ dynamoDBTimeoutMs,+ STORE_BATCH_BUDGET_MS+ )+ await crawlHistory.validateAccess(Date.now())++ const keys = new TeamMonthKeys(+ new DynamoDBClient({ region, endpoint: this.config.SESSION_RECORDING_DYNAMODB_ENDPOINT, maxAttempts: 5 }),+ new KMSClient({ region, maxAttempts: 3 }),+ AI_RESEARCH_V4_KEY_TABLE,+ AI_RESEARCH_V4_KMS_KEY_ARN,+ this.config.AI_RESEARCH_V4_KMS_REQUESTS_PER_SECOND+ )+ const admission = new V4JobAdmission(keys, this.config.AI_RESEARCH_V4_IMAGE_FETCH_KEY_BUDGET_MS)+ const imageSink = new RawImageStore(+ keys,+ s3,+ AI_RESEARCH_V4_IMAGES_BUCKET,+ this.config.AI_RESEARCH_V4_IMAGE_FETCH_MAX_PENDING_PUBLISHES+ )++ // Built even in dry run, so the wiring is exercised by every start rather than only by the+ // one that clears the flag.+ this.producerRegistry = await createV4ImageFetchProducerRegistry(this.config.KAFKA_CLIENT_RACK).build(+ this.config+ )+ const outputs = createV4ImageFetchOutputs().build(this.producerRegistry, this.config)+ this.topHog = new TopHog({+ outputs,+ pipeline: this.config.INGESTION_PIPELINE ?? 'unknown',+ lane: this.config.INGESTION_LANE ?? 'unknown',+ })+ this.topHog.start()+ const topHogMetrics = new ImageFetchTopHogMetrics(this.topHog)+ const producer = this.producerRegistry.getProducer(AI_RESEARCH_V4_IMAGE_FETCH_PRODUCER)+ const publisher = buildFrontierPublisher(producer, this.config.AI_RESEARCH_V4_IMAGE_FETCH_MAX_PENDING_PUBLISHES)+ const deadLetterTopic = this.config.AI_RESEARCH_V4_IMAGE_FETCH_DLQ_TOPIC+ const deadLetters = deadLetterTopic+ ? new KafkaFrontierDeadLetterSink(producer, deadLetterTopic, BLOCKED_IMAGE_FETCH_DEAD_LETTER_TOPICS)+ : null++ const fetchRunner = buildFetchRunner(this.config, publisher, topHogMetrics, imageSink, keys)+ const fetchConsumer = new UrlFetchConsumer(+ crawlHistory,+ publisher,+ {+ seenTtlSeconds: this.config.AI_RESEARCH_V4_IMAGE_FETCH_CRAWL_HISTORY_TTL_SECONDS,+ dryRun,+ admitCandidates: (candidates, nowMs) => admission.admit(candidates, nowMs),+ refreshingSink: imageSink,+ },+ fetchRunner,+ deadLetters,+ topHogMetrics+ )+ logger.info('🌐', 'ai_research_v4_image_fetch_started', {+ dryRun,+ continuousPool: this.config.AI_RESEARCH_V4_IMAGE_FETCH_CONTINUOUS_POOL,+ })++ const batchJoiner = new ImageFetchBatchJoiner(+ imageFetchBatchesPerPass(this.config, consumerConfigs.length),+ (messages) => fetchConsumer.handleBatch(messages, Date.now())+ )+ const batchHandlers = buildImageFetchBatchHandlers(+ consumerConfigs.length,+ fetchConsumer,+ fetchRunner,+ batchJoiner+ )+ const consumerOverrides = buildImageFetchConsumerOverrides(this.config, consumerConfigs.length)+ const consumers = consumerConfigs.map(+ (consumerConfig) => new KafkaConsumerV2(consumerConfig, consumerOverrides)+ )++ this.lifecycle.services.push({+ id: 'ai-research-v4-image-fetch',+ onShutdown: () => shutdownImageFetchConsumers(consumers, batchHandlers, fetchRunner),+ healthcheck: () => {+ for (const consumer of consumers) {+ const health = consumer.isHealthy()+ if (health.isError()) {+ return health+ }+ }+ return new HealthCheckResultOk()+ },+ })+ const watermark = new CaptureWatermark('ai_research_v4_image_fetch')+ await Promise.all(+ consumers.map((consumer, index) =>+ consumer.connect(+ (messages) => {+ watermark.hold(capturedRecords(messages, () => 'image_urls'))+ return batchHandlers.handlers[index](messages)+ },+ (partitions) => {+ watermark.forget(partitions)+ return Promise.resolve()+ },+ (offsets) => watermark.release(offsets)+ )+ )+ )+ }++ protected getCleanupResources(): CleanupResources {+ return {+ kafkaProducers: [],+ redisPools: [],+ additionalCleanup: async () => {+ this.crawlHistoryClient?.destroy()+ try {+ await this.topHog?.stop()+ } finally {+ await this.producerRegistry?.disconnectAll()+ }+ },+ }+ }+}++/** Relays the v4 lane's delay tiers back to its frontier. */+export class AiResearchV4ImageFetchRetryServer extends AiResearchV4ConsumerServer {+ private producerRegistry?: KafkaProducerRegistry<AiResearchV4ImageFetchProducerName>++ protected async startServices(): Promise<void> {+ assertOwnImageFetchGroup(this.config.AI_RESEARCH_V4_IMAGE_FETCH_GROUP_ID)+ const topic = this.config.AI_RESEARCH_V4_IMAGE_FETCH_RETRY_TOPIC+ const delayMs = this.config.AI_RESEARCH_V4_IMAGE_FETCH_RETRY_DELAY_MS+ const batchSize = this.config.AI_RESEARCH_V4_IMAGE_FETCH_RETRY_BATCH_SIZE+ const producerDeliveryTimeoutMs = Number(+ this.config.KAFKA_AI_RESEARCH_V4_IMAGE_FETCH_PRODUCER_MESSAGE_TIMEOUT_MS+ )+ // A typo here reads as an empty topic name, and librdkafka would subscribe to it and sit+ // idle while the tier filled. Naming the three the publisher writes to also stops a pod+ // draining the frontier itself, which has no period and would publish every record straight+ // back to it.+ if (!V4_IMAGE_FETCH_DELAY_TOPICS.includes(topic)) {+ throw new Error(+ `AI_RESEARCH_V4_IMAGE_FETCH_RETRY_TOPIC must name one of ${V4_IMAGE_FETCH_DELAY_TOPICS.join(', ')}, got "${topic}"`+ )+ }+ if (!Number.isInteger(batchSize) || batchSize < 1) {+ throw new Error(`AI_RESEARCH_V4_IMAGE_FETCH_RETRY_BATCH_SIZE must be positive, got ${batchSize}`)+ }+ if (!Number.isFinite(producerDeliveryTimeoutMs) || producerDeliveryTimeoutMs <= 0) {+ throw new Error(`the image fetch retry producer needs a positive delivery timeout`)+ }++ this.producerRegistry = await createV4ImageFetchProducerRegistry(this.config.KAFKA_CLIENT_RACK).build(+ this.config+ )+ const producer = this.producerRegistry.getProducer(AI_RESEARCH_V4_IMAGE_FETCH_PRODUCER)++ const consumerConfig: KafkaConsumerConfig = {+ topic,+ groupId: `${this.config.AI_RESEARCH_V4_IMAGE_FETCH_GROUP_ID}-retry-${delayMs}`,+ autoCommit: true,+ // Stored after every publish in the batch is acknowledged.+ autoOffsetStore: false,+ fetchBatchSize: batchSize,+ }+ // Set here rather than left to the deployment. This consumer sleeps for the period of its+ // topic inside one batch, and the shared default of 300s would evict the 10m and 1h tiers+ // mid-sleep, then replay the partition into a consumer that sleeps just as long.+ //+ // The batch waits once for its latest broker append timestamp.+ const pollIntervalMs = delayMs + batchSize * producerDeliveryTimeoutMs + POLL_INTERVAL_MARGIN_MS+ if (pollIntervalMs > MAX_POLL_INTERVAL_MS) {+ throw new Error(+ `a delay of ${delayMs}ms and batch of ${batchSize} need a poll interval of ${pollIntervalMs}ms, which is past what Kafka allows`+ )+ }+ const consumer = new KafkaConsumer(consumerConfig, { 'max.poll.interval.ms': pollIntervalMs })+ // The shutdown handler sets this before it disconnects the consumer, so a stopping pod+ // abandons the wait it holds rather than making the rolling deploy wait out a tier period.+ let stopping = false+ const watermark = new CaptureWatermark(`ai_research_v4_image_fetch_retry_${delayMs}`)+ const delayConsumer = new RetryDelayConsumer(producer, {+ isStopping: () => stopping,+ storeOffsets: (messages) => {+ try {+ const offsets = findOffsetsToCommit(messages)+ consumer.offsetsStore(offsets)+ watermark.release(offsets)+ } catch (error) {+ const code = (error as LibrdKafkaError | undefined)?.code+ if (!stopping && !REVOKED_PARTITION_CODES.has(code ?? 0)) {+ throw error+ }+ logger.warn('🌐', 'ai_research_v4_image_fetch_retry_offset_store_failed', {+ partitions: messages.map((message) => message.partition),+ code,+ error: error instanceof Error ? error.name : 'unknown',+ })+ }+ },+ frontierTopic: V4_IMAGE_FETCH_LANE.frontier,+ delayMs,+ heartbeat: () => consumer.reportDeliberateWait(),+ })+ logger.info('🌐', 'ai_research_v4_image_fetch_retry_started', { topic, delayMs, pollIntervalMs })++ await consumer.connect(+ (messages) => {+ watermark.hold(capturedRecords(messages, () => 'image_urls'))+ return delayConsumer.handleBatch(messages)+ },+ (partitions) => {+ watermark.forget(partitions)+ return Promise.resolve()+ }+ )++ this.lifecycle.services.push({+ id: 'ai-research-v4-image-fetch-retry',+ onShutdown: () => {+ stopping = true+ return consumer.disconnect()+ },+ healthcheck: () => consumer.isHealthy(),+ })+ }++ protected getCleanupResources(): CleanupResources {+ return {+ kafkaProducers: [],+ redisPools: [],+ additionalCleanup: () => this.producerRegistry?.disconnectAll(),+ }+ }+}