:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
fix(ai-research-v4): defer image jobs whose key read timed out (#124161)
Nicholas Waltzeadcbca15a84
diff --git a/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/job-admission.ts b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/job-admission.tsindex 9ca1793b2379..aac5f6d56750 100644--- a/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/job-admission.ts+++ b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/job-admission.ts@@ -136,10 +136,16 @@ export class V4JobAdmission {budget)} catch (error) {- if (budget.aborted) {+ // A key read that timed out is as transient as a spent budget. Thrown, it fails the whole batch and restarts the pod.+ if (budget.aborted || isTimeout(error)) {return DEFERRED}throw error}}}++/** The AWS SDK rejects a call whose own timeout fired with an `AbortError`, and `AbortSignal.timeout` with a `TimeoutError`. */+function isTimeout(error: unknown): boolean {+ return error instanceof Error && (error.name === 'AbortError' || error.name === 'TimeoutError')+}diff --git a/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/v4-lane.test.ts b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/v4-lane.test.tsindex bac29f17da60..278e5f1e0580 100644--- a/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/v4-lane.test.ts+++ b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/v4-lane.test.ts@@ -112,8 +112,12 @@ class MemoryCrawlHistory implements CrawlHistoryStore {}}-/** `keyReadsHang` makes every key row read wait until its caller gives up on it. */-function lane(rows: Record<string, Record<string, any> | undefined>, keyReadsHang = false, keyBudgetMs?: number) {+/** `hang` makes every key row read wait until its caller gives up on it. `time_out` fails it as a timed-out SDK call. */+function lane(+ rows: Record<string, Record<string, any> | undefined>,+ keyReads: 'answer' | 'hang' | 'time_out' = 'answer',+ keyBudgetMs?: number+) {const config = buildAiResearchV4ServerConfig({AI_RESEARCH_V4_IMAGE_FETCH_WEB_BOT_AUTH_PRIVATE_KEYS: generateKeyPairSync('ed25519').privateKey.export({ type: 'pkcs8', format: 'pem' })@@ -123,11 +127,14 @@ function lane(rows: Record<string, Record<string, any> | undefined>, keyReadsHanconst publisher = buildFrontierPublisher(producer, 10)const dynamo = {send: jest.fn((command: GetItemCommand, options?: any) => {- if (keyReadsHang) {+ if (keyReads === 'hang') {return new Promise<never>((_resolve, reject) =>options?.abortSignal?.addEventListener('abort', () => reject(options.abortSignal.reason)))}+ if (keyReads === 'time_out') {+ return Promise.reject(Object.assign(new Error('Request aborted'), { name: 'AbortError' }))+ }const id = `${command.input.Key!.pk.S}/${command.input.Key!.sk.S}`return Promise.resolve({ Item: rows[id] })}),@@ -268,7 +275,7 @@ describe('v4 image fetch lane', () => {})it('sends a job whose key was not read within the batch key budget to the first delay tier, with no request', async () => {- const harness = lane(row(7), true, 50)+ const harness = lane(row(7), 'hang', 50)runners.push(harness.runner)await harness.consumer.handleBatch([recorderRecord([{ ref: vector.url.ref, url: vector.url.url }])], NOW_MS)@@ -282,6 +289,17 @@ describe('v4 image fetch lane', () => {expect(readOptions.abortSignal.aborted).toBe(true)})+ it('sends a job whose key read timed out to the first delay tier, without failing the batch', async () => {+ const harness = lane(row(7), 'time_out')+ runners.push(harness.runner)++ await harness.consumer.handleBatch([recorderRecord([{ ref: vector.url.ref, url: vector.url.url }])], NOW_MS)++ expect(fetchStreamedMock).not.toHaveBeenCalled()+ const produced = (harness.producer.produce as jest.Mock).mock.calls.map(([record]) => record.topic)+ expect(produced).toEqual([KAFKA_AI_RESEARCH_V4_IMAGE_FETCH_RETRY_1M])+ })+describe('with a reference seen again later in its month', () => {const imageRequests = (): number =>fetchStreamedMock.mock.calls.filter(([url]) => url === vector.url.url).length