Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(ai-research-v4): finish image urls older than a capture age limit (#125342)

Nicholas Waltzbb8ff5951305
diff --git a/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/config.ts b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/config.ts
index 8acfd7e052e6..99767edd114c 100644
--- a/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/config.ts
+++ b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/config.ts
@@ -88,6 +88,21 @@ export type AiResearchV4Config = {
* max.poll.interval.ms of 300s, which the crawl history round trips share.
*/
AI_RESEARCH_V4_IMAGE_FETCH_REQUEST_BUDGET_MS: number
+ /**
+ * Time since capture after which a URL that is not done finishes as `url_too_old`.
+ *
+ * A republish keeps the capture time, and the capture watermark is the oldest capture time the
+ * lane holds. A URL that a domain refuses for days would hold the watermark back for all of them.
+ */
+ AI_RESEARCH_V4_IMAGE_FETCH_MAX_URL_AGE_MS: number
+ /**
+ * Epoch milliseconds. The age limit applies only to URLs captured at or after this time.
+ *
+ * URLs captured before it keep republishing without the limit. Without this, every URL older
+ * than the limit would finish in the first minutes after a deploy and write its crawl history
+ * at once. Set it to the deploy time or later.
+ */
+ AI_RESEARCH_V4_IMAGE_FETCH_MAX_URL_AGE_FROM_MS: number
/** Refused before the body when the response declares more, and abandoned mid-body when it declares nothing. */
AI_RESEARCH_V4_IMAGE_FETCH_MAX_IMAGE_BYTES: number
/** Covers one URL including its redirects, separate from the connect timeout of the shared request layer. */
@@ -174,6 +189,8 @@ export function getDefaultAiResearchV4Config(): AiResearchV4Config {
AI_RESEARCH_V4_IMAGE_FETCH_MAX_IN_FLIGHT_REQUESTS: 300,
AI_RESEARCH_V4_IMAGE_FETCH_MAX_PENDING_PUBLISHES: 100,
AI_RESEARCH_V4_IMAGE_FETCH_REQUEST_BUDGET_MS: 40_000,
+ AI_RESEARCH_V4_IMAGE_FETCH_MAX_URL_AGE_MS: 24 * 60 * 60 * 1000,
+ AI_RESEARCH_V4_IMAGE_FETCH_MAX_URL_AGE_FROM_MS: Date.parse('2026-11-02T00:00:00Z'),
AI_RESEARCH_V4_IMAGE_FETCH_MAX_IMAGE_BYTES: 20 * 1024 * 1024,
AI_RESEARCH_V4_IMAGE_FETCH_REQUEST_TIMEOUT_MS: 10_000,
AI_RESEARCH_V4_IMAGE_FETCH_MAX_REDIRECTS: 3,
diff --git a/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/fetch-runner.test.ts b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/fetch-runner.test.ts
index beadc89a5182..324fa4e4cbca 100644
--- a/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/fetch-runner.test.ts
+++ b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/fetch-runner.test.ts
@@ -7,7 +7,7 @@ import { RecordedTopHogMetric, createRecordingTopHog } from '~/tests/helpers/top
import { FetchCandidate, MAX_HOPS } from './collected-urls-record'
import { ConfigurationPolicyService, OriginPolicyDecision } from './configuration-policy'
import { ConfigurationCacheItem, CrawlHistoryItem, HttpCacheMetadata } from './crawl-history'
-import { DELAY_TOO_LONG, FetchRunner, FetchRunnerOptions, HOPS_EXHAUSTED } from './fetch-runner'
+import { DELAY_TOO_LONG, FetchRunner, FetchRunnerOptions, HOPS_EXHAUSTED, URL_TOO_OLD } from './fetch-runner'
import { FrontierPublisher, RepublishResult } from './frontier-publisher'
import { HostBudget } from './host-budget'
import { ImageFetchOptions, ImageFetchResult, ImageFetcher } from './image-fetcher'
@@ -849,11 +849,14 @@ describe('FetchRunner', () => {
expect(attempt).toMatchObject({ outcome: HOPS_EXHAUSTED, finished: true })
})
- it('records a terminal refusal when no delay topic can hold the wait', async () => {
- const harness = build({ outcome: 'timeout' }, {}, 'refused_delay')
+ it.each([
+ ['no delay topic can hold the wait', 'refused_delay', DELAY_TOO_LONG],
+ ['the URL is past the age limit', 'refused_age', URL_TOO_OLD],
+ ] as const)('records a terminal refusal when %s', async (_name, republishResult, outcome) => {
+ const harness = build({ outcome: 'timeout' }, {}, republishResult)
const [attempt] = await harness.runner.run([candidate()], new Map())
- expect(attempt).toMatchObject({ outcome: DELAY_TOO_LONG, finished: true })
+ expect(attempt).toMatchObject({ outcome, finished: true, history: { outcome } })
})
it('merges a 304 response into the previous cache metadata', async () => {
diff --git a/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/fetch-runner.ts b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/fetch-runner.ts
index 85a1994b262f..8b5a750a912c 100644
--- a/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/fetch-runner.ts
+++ b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/fetch-runner.ts
@@ -40,6 +40,7 @@ export type ShedReason =
| 'registrable_domain_map_full'
export const HOPS_EXHAUSTED = 'hops_exhausted'
export const DELAY_TOO_LONG = 'delay_too_long'
+export const URL_TOO_OLD = 'url_too_old'
export type AttemptOutcome =
| FetchOutcome
| FetchRefusalReason
@@ -47,6 +48,7 @@ export type AttemptOutcome =
| 'publish_failed'
| typeof HOPS_EXHAUSTED
| typeof DELAY_TOO_LONG
+ | typeof URL_TOO_OLD
export interface FetchAttempt {
candidate: FetchCandidate
@@ -877,6 +879,9 @@ export class FetchRunner implements FetchPass {
if (result === 'refused_delay') {
return this.terminal(candidate, DELAY_TOO_LONG, undefined, configurationUpdates)
}
+ if (result === 'refused_age') {
+ return this.terminal(candidate, URL_TOO_OLD, undefined, configurationUpdates)
+ }
return {
candidate: republishedCandidate,
outcome,
diff --git a/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/frontier-publisher.test.ts b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/frontier-publisher.test.ts
index e2b60454d201..9964ed2e1933 100644
--- a/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/frontier-publisher.test.ts
+++ b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/frontier-publisher.test.ts
@@ -18,6 +18,7 @@ const TIERS = [
{ topic: 'retry_10m', delayMs: 600_000, metricTopic: 'retry_10m' as const },
{ topic: 'retry_1h', delayMs: 3_600_000, metricTopic: 'retry_1h' as const },
]
+const MAX_URL_AGE_MS = 24 * 60 * 60 * 1000
function candidate(overrides: Partial<FetchCandidate> = {}): FetchCandidate {
return {
@@ -43,7 +44,7 @@ interface SentMessage {
headers?: Record<string, string>
}
-function build(): {
+function build(maxUrlAgeFromMs = 0): {
publisher: FrontierPublisher
batch: RepublishBatch
sent: SentMessage[]
@@ -59,6 +60,8 @@ function build(): {
frontierTopic: FRONTIER,
delayTiers: TIERS,
maxConcurrentRepublishes: 2,
+ maxUrlAgeMs: MAX_URL_AGE_MS,
+ maxUrlAgeFromMs,
})
return { publisher, batch: publisher.createRepublishBatch(), sent }
}
@@ -242,20 +245,39 @@ describe('FrontierPublisher', () => {
expect(sent.every((item) => item.value.length <= MAX_RECORD_BYTES)).toBe(true)
})
- it('refuses a delay longer than the largest deployed tier', async () => {
+ it.each([
+ ['a delay longer than the largest deployed tier', candidate(), 'retry', 3_600_001, 'refused_delay'],
+ [
+ 'a redirect that has no hop left after republishing',
+ candidate({ remainingHops: 1 }),
+ 'redirect',
+ 0,
+ 'refused_delay',
+ ],
+ [
+ 'a URL captured longer ago than the age limit',
+ candidate({ firstSeenAtMs: 1_700_000_000_000 - MAX_URL_AGE_MS }),
+ 'pass_deadline',
+ 0,
+ 'refused_age',
+ ],
+ ] as const)('refuses %s', async (_name, job, reason, waitMs, expected) => {
const { batch, sent } = build()
- expect(await batch.republish(candidate(), candidate(), 'retry', 3_600_001)).toBe('refused_delay')
+ expect(await batch.republish(job, candidate(), reason, waitMs)).toBe(expected)
await batch.flush()
expect(sent).toEqual([])
})
- it('refuses a redirect that has no hop left after republishing', async () => {
- const { batch, sent } = build()
+ it('republishes a URL past the age limit that was captured before the limit applies', async () => {
+ const capturedAtMs = 1_700_000_000_000 - MAX_URL_AGE_MS
+ const { batch, sent } = build(capturedAtMs + 1)
- expect(await batch.republish(candidate({ remainingHops: 1 }), candidate(), 'redirect')).toBe('refused_delay')
+ expect(await batch.republish(candidate({ firstSeenAtMs: capturedAtMs }), candidate(), 'pass_deadline')).toBe(
+ 'queued'
+ )
await batch.flush()
- expect(sent).toEqual([])
+ expect(sent).toHaveLength(1)
})
it('publishes every target-domain group from redirect fanout to the frontier', async () => {
diff --git a/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/frontier-publisher.ts b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/frontier-publisher.ts
index a3e7354efd5d..511876b551c1 100644
--- a/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/frontier-publisher.ts
+++ b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/frontier-publisher.ts
@@ -22,9 +22,11 @@ export interface FrontierPublisherOptions {
frontierTopic: string
delayTiers: DelayTier[]
maxConcurrentRepublishes: number
+ maxUrlAgeMs?: number
+ maxUrlAgeFromMs?: number
}
-export type RepublishResult = 'queued' | 'refused_delay'
+export type RepublishResult = 'queued' | 'refused_delay' | 'refused_age'
export interface RepublishFlushResult {
failedUrls: number
@@ -85,6 +87,9 @@ export class FrontierPublisher {
if (!Number.isInteger(options.maxConcurrentRepublishes) || options.maxConcurrentRepublishes < 1) {
throw new Error('the image fetch lane needs a positive republish limit')
}
+ if (options.maxUrlAgeMs !== undefined && !(options.maxUrlAgeMs > 0)) {
+ throw new Error('the image fetch lane needs a positive URL age limit')
+ }
this.republishes = new ConcurrencyController(options.maxConcurrentRepublishes)
}
@@ -94,7 +99,9 @@ export class FrontierPublisher {
this.options.frontierTopic,
this.delayTiers,
this.republishes,
- deadlineAtMonotonicMs
+ deadlineAtMonotonicMs,
+ this.options.maxUrlAgeMs ?? Number.POSITIVE_INFINITY,
+ this.options.maxUrlAgeFromMs ?? 0
)
}
}
@@ -108,7 +115,9 @@ class BufferedRepublishBatch implements RepublishBatch {
private readonly frontierTopic: string,
private readonly delayTiers: DelayTier[],
private readonly republishes: ConcurrencyController,
- private readonly deadlineAtMonotonicMs: number
+ private readonly deadlineAtMonotonicMs: number,
+ private readonly maxUrlAgeMs: number,
+ private readonly maxUrlAgeFromMs: number
) {}
public republish(
@@ -120,6 +129,12 @@ class BufferedRepublishBatch implements RepublishBatch {
if (this.flushed) {
return Promise.reject(new Error('an image fetch republish batch cannot accept work after its flush'))
}
+ if (
+ candidate.firstSeenAtMs >= this.maxUrlAgeFromMs &&
+ Date.now() - candidate.firstSeenAtMs >= this.maxUrlAgeMs
+ ) {
+ return Promise.resolve('refused_age')
+ }
const spendsHop = reason === 'redirect' || reason === 'retry'
const remainingHops = candidate.remainingHops - (spendsHop ? 1 : 0)
if (remainingHops <= 0) {
diff --git a/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/url-fetch-consumer.test.ts b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/url-fetch-consumer.test.ts
index 9ef0db1aba4b..f08fb5efb6e8 100644
--- a/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/url-fetch-consumer.test.ts
+++ b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/url-fetch-consumer.test.ts
@@ -5,7 +5,7 @@ import { RecordedTopHogMetric, createRecordingTopHog } from '~/tests/helpers/top
import { FetchCandidate, MAX_HOPS, serializeFrontierRecord } from './collected-urls-record'
import { CrawlHistoryItem, CrawlHistoryStore, configurationCacheKey } from './crawl-history'
-import { AttemptOutcome, DELAY_TOO_LONG, FetchAttempt, FetchPass, HOPS_EXHAUSTED } from './fetch-runner'
+import { AttemptOutcome, DELAY_TOO_LONG, FetchAttempt, FetchPass, HOPS_EXHAUSTED, URL_TOO_OLD } from './fetch-runner'
import { FrontierDeadLetterSink } from './frontier-dead-letter-sink'
import { FrontierPublisher, RepublishFlushResult, RepublishResult } from './frontier-publisher'
import { ImageFetchConsumerMetrics, ImageFetchRequestMetrics } from './metrics'
@@ -404,15 +404,18 @@ describe('UrlFetchConsumer', () => {
])
})
- it('records a terminal refusal when the remaining delay is over one hour', async () => {
+ it.each([
+ ['the remaining delay is over one hour', 'refused_delay', DELAY_TOO_LONG],
+ ['a delayed job is past the age limit', 'refused_age', URL_TOO_OLD],
+ ] as const)('records a terminal refusal when %s', async (_name, republishResult, outcome) => {
const harness = build()
- harness.republish.mockResolvedValue('refused_delay')
+ harness.republish.mockResolvedValue(republishResult)
const early = candidate('a', { notBeforeMs: NOW_MS + 3_600_001 })
await harness.consumer.handleBatch([message([early])], NOW_MS)
expect(harness.history.writes[0]).toEqual([
- expect.objectContaining({ kind: 'url', key: early.originalRef, outcome: DELAY_TOO_LONG }),
+ expect.objectContaining({ kind: 'url', key: early.originalRef, outcome }),
])
})
diff --git a/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/url-fetch-consumer.ts b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/url-fetch-consumer.ts
index 5a6a5bb1cc9c..76f0df1cd571 100644
--- a/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/url-fetch-consumer.ts
+++ b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/url-fetch-consumer.ts
@@ -19,6 +19,7 @@ import {
FetchAttempt,
FetchPass,
HOPS_EXHAUSTED,
+ URL_TOO_OLD,
isTransientOutcome,
} from './fetch-runner'
import { FrontierDeadLetterReason, FrontierDeadLetterSink } from './frontier-dead-letter-sink'
@@ -437,6 +438,9 @@ export class UrlFetchConsumer {
'not_ready',
waitMs
)
+ if (result === 'refused_age') {
+ return this.terminalAttempt(candidate, URL_TOO_OLD, nowMs)
+ }
if (result === 'refused_delay' || waitMs > ONE_HOUR_MS) {
return this.terminalAttempt(candidate, DELAY_TOO_LONG, nowMs)
}
diff --git a/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/v4-lane.test.ts b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/v4-lane.test.ts
index 278e5f1e0580..4f5dde7d842f 100644
--- a/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/v4-lane.test.ts
+++ b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/v4-lane.test.ts
@@ -59,7 +59,7 @@ function recorderRecord(jobs: { ref: string; url: string }[]): Message {
currentUrl: url,
remainingHops: 10,
notBeforeMs: 0,
- firstSeenAtMs: NOW_MS,
+ firstSeenAtMs: Date.now(),
fetchCount: 0,
republishCount: 0,
lastRepublishReason: null,
@@ -124,7 +124,12 @@ function lane(
.toString(),
})
const producer = { produce: jest.fn(() => Promise.resolve()) } as unknown as KafkaProducer
- const publisher = buildFrontierPublisher(producer, 10)
+ const publisher = buildFrontierPublisher(
+ producer,
+ 10,
+ config.AI_RESEARCH_V4_IMAGE_FETCH_MAX_URL_AGE_MS,
+ config.AI_RESEARCH_V4_IMAGE_FETCH_MAX_URL_AGE_FROM_MS
+ )
const dynamo = {
send: jest.fn((command: GetItemCommand, options?: any) => {
if (keyReads === 'hang') {
diff --git a/nodejs/src/servers/ai-research-v4-image-fetch-server.ts b/nodejs/src/servers/ai-research-v4-image-fetch-server.ts
index 9ef9b22ec8f6..c141317059c7 100644
--- a/nodejs/src/servers/ai-research-v4-image-fetch-server.ts
+++ b/nodejs/src/servers/ai-research-v4-image-fetch-server.ts
@@ -105,7 +105,12 @@ function getUrlPolicy(): typeof import('@posthog/ai-research-v4-native') {
return urlPolicy
}
-export function buildFrontierPublisher(producer: KafkaProducer, maxConcurrentRepublishes: number): FrontierPublisher {
+export function buildFrontierPublisher(
+ producer: KafkaProducer,
+ maxConcurrentRepublishes: number,
+ maxUrlAgeMs: number,
+ maxUrlAgeFromMs: number
+): FrontierPublisher {
return new FrontierPublisher(producer, {
frontierTopic: V4_IMAGE_FETCH_LANE.frontier,
delayTiers: [
@@ -114,6 +119,8 @@ export function buildFrontierPublisher(producer: KafkaProducer, maxConcurrentRep
{ topic: V4_IMAGE_FETCH_LANE.retry1h, delayMs: 3_600_000, metricTopic: 'retry_1h' },
],
maxConcurrentRepublishes,
+ maxUrlAgeMs,
+ maxUrlAgeFromMs,
})
}
@@ -362,7 +369,12 @@ export class AiResearchV4ImageFetchServer extends AiResearchV4ConsumerServer {
this.topHog.start()
const topHogMetrics = new ImageFetchTopHogMetrics(this.topHog)
const producer = this.producerRegistry.getProducer(AI_RESEARCH_V4_IMAGE_FETCH_PRODUCER)
- const publisher = buildFrontierPublisher(producer, this.config.AI_RESEARCH_V4_IMAGE_FETCH_MAX_PENDING_PUBLISHES)
+ const publisher = buildFrontierPublisher(
+ producer,
+ this.config.AI_RESEARCH_V4_IMAGE_FETCH_MAX_PENDING_PUBLISHES,
+ this.config.AI_RESEARCH_V4_IMAGE_FETCH_MAX_URL_AGE_MS,
+ this.config.AI_RESEARCH_V4_IMAGE_FETCH_MAX_URL_AGE_FROM_MS
+ )
const deadLetterTopic = this.config.AI_RESEARCH_V4_IMAGE_FETCH_DLQ_TOPIC
const deadLetters = deadLetterTopic
? new KafkaFrontierDeadLetterSink(producer, deadLetterTopic, BLOCKED_IMAGE_FETCH_DEAD_LETTER_TOPICS)