:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
fix(ai-research-v4): delay pass-deadline urls while their domain backs off (#125341)
Nicholas Waltz2ccbea26fefc
diff --git a/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/config.ts b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/config.tsindex 17ba5c8d7422..8acfd7e052e6 100644--- a/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/config.ts+++ b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/config.ts@@ -83,9 +83,9 @@ export type AiResearchV4Config = {* Wall time the fetch pass of one poll batch may take.** A batch can hold more URLs for one registrable domain than a polite rate carries in this- * time. What the pass does not reach goes to a delay topic without spending a hop. The value- * sits well inside Kafka's max.poll.interval.ms of 300s, which the crawl history round trips- * share.+ * time. What the pass does not reach goes back to the frontier without spending a hop, or to a+ * delay topic while its registrable domain backs off. The value sits well inside Kafka's+ * max.poll.interval.ms of 300s, which the crawl history round trips share.*/AI_RESEARCH_V4_IMAGE_FETCH_REQUEST_BUDGET_MS: number/** Refused before the body when the response declares more, and abandoned mid-body when it declares nothing. */diff --git a/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/fetch-runner.test.ts b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/fetch-runner.test.tsindex d28d2385e5ad..beadc89a5182 100644--- a/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/fetch-runner.test.ts+++ b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/fetch-runner.test.ts@@ -784,15 +784,50 @@ describe('FetchRunner', () => {expect(harness.republish).toHaveBeenCalledWith(expect.any(Object), expect.any(Object), 'not_ready', 600_000)})- it.each([- ['a fetch pass', OPTIONS],- ['the pod candidate pool', POOL_OPTIONS],- ])('returns pass-deadline work directly to the frontier from %s', async (_mode, options) => {+ it.each<[string, FetchRunnerOptions, 'none' | 'transient' | 'retry_after_1h', string, number]>([+ ['returns pass-deadline work directly to the frontier from a fetch pass', OPTIONS, 'none', 'pass_deadline', 0],+ [+ 'returns pass-deadline work directly to the frontier from the pod pool',+ POOL_OPTIONS,+ 'none',+ 'pass_deadline',+ 0,+ ],+ [+ 'delays pass-deadline work of a backed-off domain from a fetch pass',+ OPTIONS,+ 'transient',+ 'not_ready',+ 60_000,+ ],+ [+ 'delays pass-deadline work of a backed-off domain from the pod pool',+ POOL_OPTIONS,+ 'transient',+ 'not_ready',+ 60_000,+ ],+ [+ 'holds pass-deadline work of a long back-off in the ten-minute tier',+ OPTIONS,+ 'retry_after_1h',+ 'not_ready',+ 600_000,+ ],+ ])('%s', async (_name, options, backOff, reason, waitMs) => {const harness = build({}, {}, 'queued', { ...options, batchBudgetMs: -1 })+ if (backOff !== 'none') {+ harness.budget.recordTransientFailure(+ 'example.com',+ NOW_MS,+ backOff === 'retry_after_1h' ? 3_600_000 : undefined+ )+ }await harness.runner.run([candidate()], new Map())- expect(harness.republish).toHaveBeenCalledWith(expect.any(Object), expect.any(Object), 'pass_deadline', 0)+ expect(harness.fetch).not.toHaveBeenCalled()+ expect(harness.republish).toHaveBeenCalledWith(expect.any(Object), expect.any(Object), reason, waitMs)})it('finishes a retry when its last hop was the failed request', async () => {diff --git a/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/fetch-runner.ts b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/fetch-runner.tsindex bcfd4908c5f8..85a1994b262f 100644--- a/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/fetch-runner.ts+++ b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/fetch-runner.ts@@ -72,6 +72,7 @@ export interface FetchRunnerOptions {const TRANSIENT_OUTCOMES = new Set<TransientFetchOutcome>(['timeout', 'error', 'rate_limited', 'server_error'])const ONE_MINUTE_MS = 60_000+const TEN_MINUTES_MS = 10 * ONE_MINUTE_MSconst CONFIGURATION_RETRY_MS = 60 * 60 * 1000const STORAGE_FAILURE_LOG_INTERVAL_MS = 10_000@@ -440,7 +441,7 @@ export class FetchRunner implements FetchPass {return this.terminal(candidate, HOPS_EXHAUSTED, undefined, [])}if (Date.now() > deadlineMs) {- return await this.republish(republishBatch, candidate, 'deadline', 'pass_deadline', 0, [], 'pass_deadline')+ return await this.deferPastDeadline(republishBatch, candidate)}return await ImageFetchProcessingMetrics.runLimited(this.candidateWork,@@ -454,15 +455,7 @@ export class FetchRunner implements FetchPass {}try {if (Date.now() > deadlineMs) {- return await this.republish(- republishBatch,- candidate,- 'deadline',- 'pass_deadline',- 0,- [],- 'pass_deadline'- )+ return await this.deferPastDeadline(republishBatch, candidate)}return await this.fetchOne(candidate,@@ -815,6 +808,28 @@ export class FetchRunner implements FetchPass {}}+ /**+ * A URL that the pass did not reach goes back to the frontier, so the next pass can try it at once. A URL whose+ * registrable domain is backing off would only wait in the pool until that pass ends too, and loop through the+ * frontier until the back-off ends. It waits out the back-off in a delay tier instead. The wait stops at the+ * 10-minute tier because the one-hour tier runs one relay pod, and a back-off can outlast every tier.+ */+ private async deferPastDeadline(republishBatch: RepublishBatch, candidate: FetchCandidate): Promise<FetchAttempt> {+ const blockedMs = this.budget.blockedForMs(candidate.registrableDomain, Date.now())+ if (blockedMs > 0) {+ return await this.republish(+ republishBatch,+ candidate,+ 'backoff',+ 'not_ready',+ Math.min(Math.max(ONE_MINUTE_MS, blockedMs), TEN_MINUTES_MS),+ [],+ 'unknown_backoff'+ )+ }+ return await this.republish(republishBatch, candidate, 'deadline', 'pass_deadline', 0, [], 'pass_deadline')+ }+private async republish(republishBatch: RepublishBatch,candidate: FetchCandidate,