Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(ai-research-v4): defer image jobs whose key read timed out (#124161)

Nicholas Waltzeadcbca15a84
diff --git a/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/job-admission.ts b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/job-admission.ts
index 9ca1793b2379..aac5f6d56750 100644
--- a/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/job-admission.ts
+++ b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/job-admission.ts
@@ -136,10 +136,16 @@ export class V4JobAdmission {
budget
)
} catch (error) {
- if (budget.aborted) {
+ // A key read that timed out is as transient as a spent budget. Thrown, it fails the whole batch and restarts the pod.
+ if (budget.aborted || isTimeout(error)) {
return DEFERRED
}
throw error
}
}
}
+
+/** The AWS SDK rejects a call whose own timeout fired with an `AbortError`, and `AbortSignal.timeout` with a `TimeoutError`. */
+function isTimeout(error: unknown): boolean {
+ return error instanceof Error && (error.name === 'AbortError' || error.name === 'TimeoutError')
+}
diff --git a/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/v4-lane.test.ts b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/v4-lane.test.ts
index bac29f17da60..278e5f1e0580 100644
--- a/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/v4-lane.test.ts
+++ b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/v4-lane.test.ts
@@ -112,8 +112,12 @@ class MemoryCrawlHistory implements CrawlHistoryStore {
}
}
-/** `keyReadsHang` makes every key row read wait until its caller gives up on it. */
-function lane(rows: Record<string, Record<string, any> | undefined>, keyReadsHang = false, keyBudgetMs?: number) {
+/** `hang` makes every key row read wait until its caller gives up on it. `time_out` fails it as a timed-out SDK call. */
+function lane(
+ rows: Record<string, Record<string, any> | undefined>,
+ keyReads: 'answer' | 'hang' | 'time_out' = 'answer',
+ keyBudgetMs?: number
+) {
const config = buildAiResearchV4ServerConfig({
AI_RESEARCH_V4_IMAGE_FETCH_WEB_BOT_AUTH_PRIVATE_KEYS: generateKeyPairSync('ed25519')
.privateKey.export({ type: 'pkcs8', format: 'pem' })
@@ -123,11 +127,14 @@ function lane(rows: Record<string, Record<string, any> | undefined>, keyReadsHan
const publisher = buildFrontierPublisher(producer, 10)
const dynamo = {
send: jest.fn((command: GetItemCommand, options?: any) => {
- if (keyReadsHang) {
+ if (keyReads === 'hang') {
return new Promise<never>((_resolve, reject) =>
options?.abortSignal?.addEventListener('abort', () => reject(options.abortSignal.reason))
)
}
+ if (keyReads === 'time_out') {
+ return Promise.reject(Object.assign(new Error('Request aborted'), { name: 'AbortError' }))
+ }
const id = `${command.input.Key!.pk.S}/${command.input.Key!.sk.S}`
return Promise.resolve({ Item: rows[id] })
}),
@@ -268,7 +275,7 @@ describe('v4 image fetch lane', () => {
})
it('sends a job whose key was not read within the batch key budget to the first delay tier, with no request', async () => {
- const harness = lane(row(7), true, 50)
+ const harness = lane(row(7), 'hang', 50)
runners.push(harness.runner)
await harness.consumer.handleBatch([recorderRecord([{ ref: vector.url.ref, url: vector.url.url }])], NOW_MS)
@@ -282,6 +289,17 @@ describe('v4 image fetch lane', () => {
expect(readOptions.abortSignal.aborted).toBe(true)
})
+ it('sends a job whose key read timed out to the first delay tier, without failing the batch', async () => {
+ const harness = lane(row(7), 'time_out')
+ runners.push(harness.runner)
+
+ await harness.consumer.handleBatch([recorderRecord([{ ref: vector.url.ref, url: vector.url.url }])], NOW_MS)
+
+ expect(fetchStreamedMock).not.toHaveBeenCalled()
+ const produced = (harness.producer.produce as jest.Mock).mock.calls.map(([record]) => record.topic)
+ expect(produced).toEqual([KAFKA_AI_RESEARCH_V4_IMAGE_FETCH_RETRY_1M])
+ })
+
describe('with a reference seen again later in its month', () => {
const imageRequests = (): number =>
fetchStreamedMock.mock.calls.filter(([url]) => url === vector.url.url).length