Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(ai-research-v4): delay pass-deadline urls while their domain backs off (#125341)

Nicholas Waltz2ccbea26fefc
diff --git a/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/config.ts b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/config.ts
index 17ba5c8d7422..8acfd7e052e6 100644
--- a/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/config.ts
+++ b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/config.ts
@@ -83,9 +83,9 @@ export type AiResearchV4Config = {
* Wall time the fetch pass of one poll batch may take.
*
* A batch can hold more URLs for one registrable domain than a polite rate carries in this
- * time. What the pass does not reach goes to a delay topic without spending a hop. The value
- * sits well inside Kafka's max.poll.interval.ms of 300s, which the crawl history round trips
- * share.
+ * time. What the pass does not reach goes back to the frontier without spending a hop, or to a
+ * delay topic while its registrable domain backs off. The value sits well inside Kafka's
+ * max.poll.interval.ms of 300s, which the crawl history round trips share.
*/
AI_RESEARCH_V4_IMAGE_FETCH_REQUEST_BUDGET_MS: number
/** Refused before the body when the response declares more, and abandoned mid-body when it declares nothing. */
diff --git a/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/fetch-runner.test.ts b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/fetch-runner.test.ts
index d28d2385e5ad..beadc89a5182 100644
--- a/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/fetch-runner.test.ts
+++ b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/fetch-runner.test.ts
@@ -784,15 +784,50 @@ describe('FetchRunner', () => {
expect(harness.republish).toHaveBeenCalledWith(expect.any(Object), expect.any(Object), 'not_ready', 600_000)
})
- it.each([
- ['a fetch pass', OPTIONS],
- ['the pod candidate pool', POOL_OPTIONS],
- ])('returns pass-deadline work directly to the frontier from %s', async (_mode, options) => {
+ it.each<[string, FetchRunnerOptions, 'none' | 'transient' | 'retry_after_1h', string, number]>([
+ ['returns pass-deadline work directly to the frontier from a fetch pass', OPTIONS, 'none', 'pass_deadline', 0],
+ [
+ 'returns pass-deadline work directly to the frontier from the pod pool',
+ POOL_OPTIONS,
+ 'none',
+ 'pass_deadline',
+ 0,
+ ],
+ [
+ 'delays pass-deadline work of a backed-off domain from a fetch pass',
+ OPTIONS,
+ 'transient',
+ 'not_ready',
+ 60_000,
+ ],
+ [
+ 'delays pass-deadline work of a backed-off domain from the pod pool',
+ POOL_OPTIONS,
+ 'transient',
+ 'not_ready',
+ 60_000,
+ ],
+ [
+ 'holds pass-deadline work of a long back-off in the ten-minute tier',
+ OPTIONS,
+ 'retry_after_1h',
+ 'not_ready',
+ 600_000,
+ ],
+ ])('%s', async (_name, options, backOff, reason, waitMs) => {
const harness = build({}, {}, 'queued', { ...options, batchBudgetMs: -1 })
+ if (backOff !== 'none') {
+ harness.budget.recordTransientFailure(
+ 'example.com',
+ NOW_MS,
+ backOff === 'retry_after_1h' ? 3_600_000 : undefined
+ )
+ }
await harness.runner.run([candidate()], new Map())
- expect(harness.republish).toHaveBeenCalledWith(expect.any(Object), expect.any(Object), 'pass_deadline', 0)
+ expect(harness.fetch).not.toHaveBeenCalled()
+ expect(harness.republish).toHaveBeenCalledWith(expect.any(Object), expect.any(Object), reason, waitMs)
})
it('finishes a retry when its last hop was the failed request', async () => {
diff --git a/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/fetch-runner.ts b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/fetch-runner.ts
index bcfd4908c5f8..85a1994b262f 100644
--- a/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/fetch-runner.ts
+++ b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/fetch-runner.ts
@@ -72,6 +72,7 @@ export interface FetchRunnerOptions {
const TRANSIENT_OUTCOMES = new Set<TransientFetchOutcome>(['timeout', 'error', 'rate_limited', 'server_error'])
const ONE_MINUTE_MS = 60_000
+const TEN_MINUTES_MS = 10 * ONE_MINUTE_MS
const CONFIGURATION_RETRY_MS = 60 * 60 * 1000
const STORAGE_FAILURE_LOG_INTERVAL_MS = 10_000
@@ -440,7 +441,7 @@ export class FetchRunner implements FetchPass {
return this.terminal(candidate, HOPS_EXHAUSTED, undefined, [])
}
if (Date.now() > deadlineMs) {
- return await this.republish(republishBatch, candidate, 'deadline', 'pass_deadline', 0, [], 'pass_deadline')
+ return await this.deferPastDeadline(republishBatch, candidate)
}
return await ImageFetchProcessingMetrics.runLimited(
this.candidateWork,
@@ -454,15 +455,7 @@ export class FetchRunner implements FetchPass {
}
try {
if (Date.now() > deadlineMs) {
- return await this.republish(
- republishBatch,
- candidate,
- 'deadline',
- 'pass_deadline',
- 0,
- [],
- 'pass_deadline'
- )
+ return await this.deferPastDeadline(republishBatch, candidate)
}
return await this.fetchOne(
candidate,
@@ -815,6 +808,28 @@ export class FetchRunner implements FetchPass {
}
}
+ /**
+ * A URL that the pass did not reach goes back to the frontier, so the next pass can try it at once. A URL whose
+ * registrable domain is backing off would only wait in the pool until that pass ends too, and loop through the
+ * frontier until the back-off ends. It waits out the back-off in a delay tier instead. The wait stops at the
+ * 10-minute tier because the one-hour tier runs one relay pod, and a back-off can outlast every tier.
+ */
+ private async deferPastDeadline(republishBatch: RepublishBatch, candidate: FetchCandidate): Promise<FetchAttempt> {
+ const blockedMs = this.budget.blockedForMs(candidate.registrableDomain, Date.now())
+ if (blockedMs > 0) {
+ return await this.republish(
+ republishBatch,
+ candidate,
+ 'backoff',
+ 'not_ready',
+ Math.min(Math.max(ONE_MINUTE_MS, blockedMs), TEN_MINUTES_MS),
+ [],
+ 'unknown_backoff'
+ )
+ }
+ return await this.republish(republishBatch, candidate, 'deadline', 'pass_deadline', 0, [], 'pass_deadline')
+ }
+
private async republish(
republishBatch: RepublishBatch,
candidate: FetchCandidate,