:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
fix(ai-research-v4): finish image urls older than a capture age limit (#125342)
Nicholas Waltzbb8ff5951305
diff --git a/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/config.ts b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/config.tsindex 8acfd7e052e6..99767edd114c 100644--- a/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/config.ts+++ b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/config.ts@@ -88,6 +88,21 @@ export type AiResearchV4Config = {* max.poll.interval.ms of 300s, which the crawl history round trips share.*/AI_RESEARCH_V4_IMAGE_FETCH_REQUEST_BUDGET_MS: number+ /**+ * Time since capture after which a URL that is not done finishes as `url_too_old`.+ *+ * A republish keeps the capture time, and the capture watermark is the oldest capture time the+ * lane holds. A URL that a domain refuses for days would hold the watermark back for all of them.+ */+ AI_RESEARCH_V4_IMAGE_FETCH_MAX_URL_AGE_MS: number+ /**+ * Epoch milliseconds. The age limit applies only to URLs captured at or after this time.+ *+ * URLs captured before it keep republishing without the limit. Without this, every URL older+ * than the limit would finish in the first minutes after a deploy and write its crawl history+ * at once. Set it to the deploy time or later.+ */+ AI_RESEARCH_V4_IMAGE_FETCH_MAX_URL_AGE_FROM_MS: number/** Refused before the body when the response declares more, and abandoned mid-body when it declares nothing. */AI_RESEARCH_V4_IMAGE_FETCH_MAX_IMAGE_BYTES: number/** Covers one URL including its redirects, separate from the connect timeout of the shared request layer. */@@ -174,6 +189,8 @@ export function getDefaultAiResearchV4Config(): AiResearchV4Config {AI_RESEARCH_V4_IMAGE_FETCH_MAX_IN_FLIGHT_REQUESTS: 300,AI_RESEARCH_V4_IMAGE_FETCH_MAX_PENDING_PUBLISHES: 100,AI_RESEARCH_V4_IMAGE_FETCH_REQUEST_BUDGET_MS: 40_000,+ AI_RESEARCH_V4_IMAGE_FETCH_MAX_URL_AGE_MS: 24 * 60 * 60 * 1000,+ AI_RESEARCH_V4_IMAGE_FETCH_MAX_URL_AGE_FROM_MS: Date.parse('2026-11-02T00:00:00Z'),AI_RESEARCH_V4_IMAGE_FETCH_MAX_IMAGE_BYTES: 20 * 1024 * 1024,AI_RESEARCH_V4_IMAGE_FETCH_REQUEST_TIMEOUT_MS: 10_000,AI_RESEARCH_V4_IMAGE_FETCH_MAX_REDIRECTS: 3,diff --git a/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/fetch-runner.test.ts b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/fetch-runner.test.tsindex beadc89a5182..324fa4e4cbca 100644--- a/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/fetch-runner.test.ts+++ b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/fetch-runner.test.ts@@ -7,7 +7,7 @@ import { RecordedTopHogMetric, createRecordingTopHog } from '~/tests/helpers/topimport { FetchCandidate, MAX_HOPS } from './collected-urls-record'import { ConfigurationPolicyService, OriginPolicyDecision } from './configuration-policy'import { ConfigurationCacheItem, CrawlHistoryItem, HttpCacheMetadata } from './crawl-history'-import { DELAY_TOO_LONG, FetchRunner, FetchRunnerOptions, HOPS_EXHAUSTED } from './fetch-runner'+import { DELAY_TOO_LONG, FetchRunner, FetchRunnerOptions, HOPS_EXHAUSTED, URL_TOO_OLD } from './fetch-runner'import { FrontierPublisher, RepublishResult } from './frontier-publisher'import { HostBudget } from './host-budget'import { ImageFetchOptions, ImageFetchResult, ImageFetcher } from './image-fetcher'@@ -849,11 +849,14 @@ describe('FetchRunner', () => {expect(attempt).toMatchObject({ outcome: HOPS_EXHAUSTED, finished: true })})- it('records a terminal refusal when no delay topic can hold the wait', async () => {- const harness = build({ outcome: 'timeout' }, {}, 'refused_delay')+ it.each([+ ['no delay topic can hold the wait', 'refused_delay', DELAY_TOO_LONG],+ ['the URL is past the age limit', 'refused_age', URL_TOO_OLD],+ ] as const)('records a terminal refusal when %s', async (_name, republishResult, outcome) => {+ const harness = build({ outcome: 'timeout' }, {}, republishResult)const [attempt] = await harness.runner.run([candidate()], new Map())- expect(attempt).toMatchObject({ outcome: DELAY_TOO_LONG, finished: true })+ expect(attempt).toMatchObject({ outcome, finished: true, history: { outcome } })})it('merges a 304 response into the previous cache metadata', async () => {diff --git a/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/fetch-runner.ts b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/fetch-runner.tsindex 85a1994b262f..8b5a750a912c 100644--- a/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/fetch-runner.ts+++ b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/fetch-runner.ts@@ -40,6 +40,7 @@ export type ShedReason =| 'registrable_domain_map_full'export const HOPS_EXHAUSTED = 'hops_exhausted'export const DELAY_TOO_LONG = 'delay_too_long'+export const URL_TOO_OLD = 'url_too_old'export type AttemptOutcome =| FetchOutcome| FetchRefusalReason@@ -47,6 +48,7 @@ export type AttemptOutcome =| 'publish_failed'| typeof HOPS_EXHAUSTED| typeof DELAY_TOO_LONG+ | typeof URL_TOO_OLDexport interface FetchAttempt {candidate: FetchCandidate@@ -877,6 +879,9 @@ export class FetchRunner implements FetchPass {if (result === 'refused_delay') {return this.terminal(candidate, DELAY_TOO_LONG, undefined, configurationUpdates)}+ if (result === 'refused_age') {+ return this.terminal(candidate, URL_TOO_OLD, undefined, configurationUpdates)+ }return {candidate: republishedCandidate,outcome,diff --git a/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/frontier-publisher.test.ts b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/frontier-publisher.test.tsindex e2b60454d201..9964ed2e1933 100644--- a/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/frontier-publisher.test.ts+++ b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/frontier-publisher.test.ts@@ -18,6 +18,7 @@ const TIERS = [{ topic: 'retry_10m', delayMs: 600_000, metricTopic: 'retry_10m' as const },{ topic: 'retry_1h', delayMs: 3_600_000, metricTopic: 'retry_1h' as const },]+const MAX_URL_AGE_MS = 24 * 60 * 60 * 1000function candidate(overrides: Partial<FetchCandidate> = {}): FetchCandidate {return {@@ -43,7 +44,7 @@ interface SentMessage {headers?: Record<string, string>}-function build(): {+function build(maxUrlAgeFromMs = 0): {publisher: FrontierPublisherbatch: RepublishBatchsent: SentMessage[]@@ -59,6 +60,8 @@ function build(): {frontierTopic: FRONTIER,delayTiers: TIERS,maxConcurrentRepublishes: 2,+ maxUrlAgeMs: MAX_URL_AGE_MS,+ maxUrlAgeFromMs,})return { publisher, batch: publisher.createRepublishBatch(), sent }}@@ -242,20 +245,39 @@ describe('FrontierPublisher', () => {expect(sent.every((item) => item.value.length <= MAX_RECORD_BYTES)).toBe(true)})- it('refuses a delay longer than the largest deployed tier', async () => {+ it.each([+ ['a delay longer than the largest deployed tier', candidate(), 'retry', 3_600_001, 'refused_delay'],+ [+ 'a redirect that has no hop left after republishing',+ candidate({ remainingHops: 1 }),+ 'redirect',+ 0,+ 'refused_delay',+ ],+ [+ 'a URL captured longer ago than the age limit',+ candidate({ firstSeenAtMs: 1_700_000_000_000 - MAX_URL_AGE_MS }),+ 'pass_deadline',+ 0,+ 'refused_age',+ ],+ ] as const)('refuses %s', async (_name, job, reason, waitMs, expected) => {const { batch, sent } = build()- expect(await batch.republish(candidate(), candidate(), 'retry', 3_600_001)).toBe('refused_delay')+ expect(await batch.republish(job, candidate(), reason, waitMs)).toBe(expected)await batch.flush()expect(sent).toEqual([])})- it('refuses a redirect that has no hop left after republishing', async () => {- const { batch, sent } = build()+ it('republishes a URL past the age limit that was captured before the limit applies', async () => {+ const capturedAtMs = 1_700_000_000_000 - MAX_URL_AGE_MS+ const { batch, sent } = build(capturedAtMs + 1)- expect(await batch.republish(candidate({ remainingHops: 1 }), candidate(), 'redirect')).toBe('refused_delay')+ expect(await batch.republish(candidate({ firstSeenAtMs: capturedAtMs }), candidate(), 'pass_deadline')).toBe(+ 'queued'+ )await batch.flush()- expect(sent).toEqual([])+ expect(sent).toHaveLength(1)})it('publishes every target-domain group from redirect fanout to the frontier', async () => {diff --git a/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/frontier-publisher.ts b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/frontier-publisher.tsindex a3e7354efd5d..511876b551c1 100644--- a/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/frontier-publisher.ts+++ b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/frontier-publisher.ts@@ -22,9 +22,11 @@ export interface FrontierPublisherOptions {frontierTopic: stringdelayTiers: DelayTier[]maxConcurrentRepublishes: number+ maxUrlAgeMs?: number+ maxUrlAgeFromMs?: number}-export type RepublishResult = 'queued' | 'refused_delay'+export type RepublishResult = 'queued' | 'refused_delay' | 'refused_age'export interface RepublishFlushResult {failedUrls: number@@ -85,6 +87,9 @@ export class FrontierPublisher {if (!Number.isInteger(options.maxConcurrentRepublishes) || options.maxConcurrentRepublishes < 1) {throw new Error('the image fetch lane needs a positive republish limit')}+ if (options.maxUrlAgeMs !== undefined && !(options.maxUrlAgeMs > 0)) {+ throw new Error('the image fetch lane needs a positive URL age limit')+ }this.republishes = new ConcurrencyController(options.maxConcurrentRepublishes)}@@ -94,7 +99,9 @@ export class FrontierPublisher {this.options.frontierTopic,this.delayTiers,this.republishes,- deadlineAtMonotonicMs+ deadlineAtMonotonicMs,+ this.options.maxUrlAgeMs ?? Number.POSITIVE_INFINITY,+ this.options.maxUrlAgeFromMs ?? 0)}}@@ -108,7 +115,9 @@ class BufferedRepublishBatch implements RepublishBatch {private readonly frontierTopic: string,private readonly delayTiers: DelayTier[],private readonly republishes: ConcurrencyController,- private readonly deadlineAtMonotonicMs: number+ private readonly deadlineAtMonotonicMs: number,+ private readonly maxUrlAgeMs: number,+ private readonly maxUrlAgeFromMs: number) {}public republish(@@ -120,6 +129,12 @@ class BufferedRepublishBatch implements RepublishBatch {if (this.flushed) {return Promise.reject(new Error('an image fetch republish batch cannot accept work after its flush'))}+ if (+ candidate.firstSeenAtMs >= this.maxUrlAgeFromMs &&+ Date.now() - candidate.firstSeenAtMs >= this.maxUrlAgeMs+ ) {+ return Promise.resolve('refused_age')+ }const spendsHop = reason === 'redirect' || reason === 'retry'const remainingHops = candidate.remainingHops - (spendsHop ? 1 : 0)if (remainingHops <= 0) {diff --git a/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/url-fetch-consumer.test.ts b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/url-fetch-consumer.test.tsindex 9ef0db1aba4b..f08fb5efb6e8 100644--- a/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/url-fetch-consumer.test.ts+++ b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/url-fetch-consumer.test.ts@@ -5,7 +5,7 @@ import { RecordedTopHogMetric, createRecordingTopHog } from '~/tests/helpers/topimport { FetchCandidate, MAX_HOPS, serializeFrontierRecord } from './collected-urls-record'import { CrawlHistoryItem, CrawlHistoryStore, configurationCacheKey } from './crawl-history'-import { AttemptOutcome, DELAY_TOO_LONG, FetchAttempt, FetchPass, HOPS_EXHAUSTED } from './fetch-runner'+import { AttemptOutcome, DELAY_TOO_LONG, FetchAttempt, FetchPass, HOPS_EXHAUSTED, URL_TOO_OLD } from './fetch-runner'import { FrontierDeadLetterSink } from './frontier-dead-letter-sink'import { FrontierPublisher, RepublishFlushResult, RepublishResult } from './frontier-publisher'import { ImageFetchConsumerMetrics, ImageFetchRequestMetrics } from './metrics'@@ -404,15 +404,18 @@ describe('UrlFetchConsumer', () => {])})- it('records a terminal refusal when the remaining delay is over one hour', async () => {+ it.each([+ ['the remaining delay is over one hour', 'refused_delay', DELAY_TOO_LONG],+ ['a delayed job is past the age limit', 'refused_age', URL_TOO_OLD],+ ] as const)('records a terminal refusal when %s', async (_name, republishResult, outcome) => {const harness = build()- harness.republish.mockResolvedValue('refused_delay')+ harness.republish.mockResolvedValue(republishResult)const early = candidate('a', { notBeforeMs: NOW_MS + 3_600_001 })await harness.consumer.handleBatch([message([early])], NOW_MS)expect(harness.history.writes[0]).toEqual([- expect.objectContaining({ kind: 'url', key: early.originalRef, outcome: DELAY_TOO_LONG }),+ expect.objectContaining({ kind: 'url', key: early.originalRef, outcome }),])})diff --git a/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/url-fetch-consumer.ts b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/url-fetch-consumer.tsindex 5a6a5bb1cc9c..76f0df1cd571 100644--- a/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/url-fetch-consumer.ts+++ b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/url-fetch-consumer.ts@@ -19,6 +19,7 @@ import {FetchAttempt,FetchPass,HOPS_EXHAUSTED,+ URL_TOO_OLD,isTransientOutcome,} from './fetch-runner'import { FrontierDeadLetterReason, FrontierDeadLetterSink } from './frontier-dead-letter-sink'@@ -437,6 +438,9 @@ export class UrlFetchConsumer {'not_ready',waitMs)+ if (result === 'refused_age') {+ return this.terminalAttempt(candidate, URL_TOO_OLD, nowMs)+ }if (result === 'refused_delay' || waitMs > ONE_HOUR_MS) {return this.terminalAttempt(candidate, DELAY_TOO_LONG, nowMs)}diff --git a/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/v4-lane.test.ts b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/v4-lane.test.tsindex 278e5f1e0580..4f5dde7d842f 100644--- a/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/v4-lane.test.ts+++ b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/v4-lane.test.ts@@ -59,7 +59,7 @@ function recorderRecord(jobs: { ref: string; url: string }[]): Message {currentUrl: url,remainingHops: 10,notBeforeMs: 0,- firstSeenAtMs: NOW_MS,+ firstSeenAtMs: Date.now(),fetchCount: 0,republishCount: 0,lastRepublishReason: null,@@ -124,7 +124,12 @@ function lane(.toString(),})const producer = { produce: jest.fn(() => Promise.resolve()) } as unknown as KafkaProducer- const publisher = buildFrontierPublisher(producer, 10)+ const publisher = buildFrontierPublisher(+ producer,+ 10,+ config.AI_RESEARCH_V4_IMAGE_FETCH_MAX_URL_AGE_MS,+ config.AI_RESEARCH_V4_IMAGE_FETCH_MAX_URL_AGE_FROM_MS+ )const dynamo = {send: jest.fn((command: GetItemCommand, options?: any) => {if (keyReads === 'hang') {diff --git a/nodejs/src/servers/ai-research-v4-image-fetch-server.ts b/nodejs/src/servers/ai-research-v4-image-fetch-server.tsindex 9ef9b22ec8f6..c141317059c7 100644--- a/nodejs/src/servers/ai-research-v4-image-fetch-server.ts+++ b/nodejs/src/servers/ai-research-v4-image-fetch-server.ts@@ -105,7 +105,12 @@ function getUrlPolicy(): typeof import('@posthog/ai-research-v4-native') {return urlPolicy}-export function buildFrontierPublisher(producer: KafkaProducer, maxConcurrentRepublishes: number): FrontierPublisher {+export function buildFrontierPublisher(+ producer: KafkaProducer,+ maxConcurrentRepublishes: number,+ maxUrlAgeMs: number,+ maxUrlAgeFromMs: number+): FrontierPublisher {return new FrontierPublisher(producer, {frontierTopic: V4_IMAGE_FETCH_LANE.frontier,delayTiers: [@@ -114,6 +119,8 @@ export function buildFrontierPublisher(producer: KafkaProducer, maxConcurrentRep{ topic: V4_IMAGE_FETCH_LANE.retry1h, delayMs: 3_600_000, metricTopic: 'retry_1h' },],maxConcurrentRepublishes,+ maxUrlAgeMs,+ maxUrlAgeFromMs,})}@@ -362,7 +369,12 @@ export class AiResearchV4ImageFetchServer extends AiResearchV4ConsumerServer {this.topHog.start()const topHogMetrics = new ImageFetchTopHogMetrics(this.topHog)const producer = this.producerRegistry.getProducer(AI_RESEARCH_V4_IMAGE_FETCH_PRODUCER)- const publisher = buildFrontierPublisher(producer, this.config.AI_RESEARCH_V4_IMAGE_FETCH_MAX_PENDING_PUBLISHES)+ const publisher = buildFrontierPublisher(+ producer,+ this.config.AI_RESEARCH_V4_IMAGE_FETCH_MAX_PENDING_PUBLISHES,+ this.config.AI_RESEARCH_V4_IMAGE_FETCH_MAX_URL_AGE_MS,+ this.config.AI_RESEARCH_V4_IMAGE_FETCH_MAX_URL_AGE_FROM_MS+ )const deadLetterTopic = this.config.AI_RESEARCH_V4_IMAGE_FETCH_DLQ_TOPICconst deadLetters = deadLetterTopic? new KafkaFrontierDeadLetterSink(producer, deadLetterTopic, BLOCKED_IMAGE_FETCH_DEAD_LETTER_TOPICS)