🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
fix(worker): extend secondary ingestion retry window to outlast S3 throttling (#18371)
Niklas Semmler53924a83bc38
diff --git a/packages/shared/src/server/redis/ingestionQueue.ts b/packages/shared/src/server/redis/ingestionQueue.tsindex 5621d19b19..740d11a2fc 100644--- a/packages/shared/src/server/redis/ingestionQueue.ts+++ b/packages/shared/src/server/redis/ingestionQueue.ts@@ -146,10 +146,13 @@ export class SecondaryIngestionQueue {defaultJobOptions: {removeOnComplete: true,removeOnFail: 100_000,- attempts: 5,+ attempts: 8,backoff: {type: "exponential",delay: 5000,+ // Spreads retries of jobs that failed together (e.g. one S3+ // throttling episode) so they don't hit the prefix in lockstep.+ jitter: 0.5,},},})diff --git a/pnpm-lock.yaml b/pnpm-lock.yamlindex 6e845181a4..34e90aba8a 100644--- a/pnpm-lock.yaml+++ b/pnpm-lock.yaml@@ -1026,7 +1026,7 @@ importers:version: 10.8.0([email protected])([email protected])eslint-plugin-boundaries:specifier: 7.2.0- version: 7.2.0([email protected])([email protected]([email protected])([email protected]))([email protected])+ version: 7.2.0(@typescript-eslint/[email protected]([email protected]([email protected])([email protected]))([email protected])([email protected]))([email protected])([email protected]([email protected])([email protected]))([email protected])eslint-plugin-check-file:specifier: 3.3.2@@ -13648,10 +13648,10 @@ snapshots:'@blazediff/[email protected]': {}- '@boundaries/[email protected]([email protected])([email protected]([email protected])([email protected]))([email protected])':+ '@boundaries/[email protected](@typescript-eslint/[email protected]([email protected]([email protected])([email protected]))([email protected])([email protected]))([email protected])([email protected]([email protected])([email protected]))([email protected])':dependencies:eslint-import-resolver-node: 0.3.9([email protected])- eslint-module-utils: 2.12.1([email protected]([email protected]))([email protected])([email protected]([email protected])([email protected]))([email protected])+ eslint-module-utils: 2.12.1(@typescript-eslint/[email protected]([email protected]([email protected])([email protected]))([email protected])([email protected]))([email protected]([email protected]))([email protected])([email protected]([email protected])([email protected]))([email protected])handlebars: 4.7.9is-core-module: 2.16.1micromatch: 4.0.8@@ -14162,7 +14162,7 @@ snapshots:dependencies:'@floating-ui/dom': 1.8.0- '@floating-ui/utils': 0.2.10+ '@floating-ui/utils': 0.2.12vue-demi: 0.14.10([email protected]([email protected]))transitivePeerDependencies:- '@vue/composition-api'@@ -18449,6 +18449,19 @@ snapshots:transitivePeerDependencies:- supports-color+ '@typescript-eslint/[email protected]([email protected]([email protected])([email protected]))([email protected])([email protected])':+ dependencies:+ '@typescript-eslint/scope-manager': 8.67.0+ '@typescript-eslint/types': 8.67.0+ '@typescript-eslint/typescript-estree': 8.67.0([email protected])([email protected])+ '@typescript-eslint/visitor-keys': 8.67.0+ debug: 4.4.3([email protected])+ eslint: 10.8.0([email protected])([email protected])+ typescript: 7.0.2+ transitivePeerDependencies:+ - supports-color+ optional: true+dependencies:'@typescript-eslint/tsconfig-utils': 8.67.0(@typescript/[email protected])@@ -19146,11 +19159,11 @@ snapshots:dependencies:- acorn: 8.17.0+ acorn: 8.18.0dependencies:- acorn: 8.17.0+ acorn: 8.18.0@@ -20516,23 +20529,35 @@ snapshots:transitivePeerDependencies:- supports-color- [email protected]([email protected]([email protected]))([email protected])([email protected]([email protected])([email protected]))([email protected]):+ [email protected](@typescript-eslint/[email protected]([email protected]([email protected])([email protected]))([email protected])([email protected]))([email protected]([email protected]))([email protected])([email protected]([email protected])([email protected]))([email protected]):+ dependencies:+ debug: 3.2.7([email protected])+ optionalDependencies:+ '@typescript-eslint/parser': 8.67.0([email protected]([email protected])([email protected]))([email protected])([email protected])+ eslint: 10.8.0([email protected])([email protected])+ eslint-import-resolver-node: 0.3.10([email protected])+ eslint-import-resolver-typescript: 3.10.1([email protected])([email protected]([email protected])([email protected]))([email protected])+ transitivePeerDependencies:+ - supports-color++ [email protected](@typescript-eslint/[email protected]([email protected]([email protected])([email protected]))([email protected])([email protected]))([email protected]([email protected]))([email protected])([email protected]([email protected])([email protected]))([email protected]):dependencies:debug: 3.2.7([email protected])optionalDependencies:+ '@typescript-eslint/parser': 8.67.0([email protected]([email protected])([email protected]))([email protected])([email protected])eslint: 10.8.0([email protected])([email protected])eslint-import-resolver-node: 0.3.9([email protected])eslint-import-resolver-typescript: 3.10.1([email protected])([email protected]([email protected])([email protected]))([email protected])transitivePeerDependencies:- supports-color- [email protected]([email protected])([email protected]([email protected])([email protected]))([email protected]):+ [email protected](@typescript-eslint/[email protected]([email protected]([email protected])([email protected]))([email protected])([email protected]))([email protected])([email protected]([email protected])([email protected]))([email protected]):dependencies:- '@boundaries/elements': 3.1.1([email protected])([email protected]([email protected])([email protected]))([email protected])+ '@boundaries/elements': 3.1.1(@typescript-eslint/[email protected]([email protected]([email protected])([email protected]))([email protected])([email protected]))([email protected])([email protected]([email protected])([email protected]))([email protected])chalk: 4.1.2eslint: 10.8.0([email protected])([email protected])eslint-import-resolver-node: 0.3.9([email protected])- eslint-module-utils: 2.12.1([email protected]([email protected]))([email protected])([email protected]([email protected])([email protected]))([email protected])+ eslint-module-utils: 2.12.1(@typescript-eslint/[email protected]([email protected]([email protected])([email protected]))([email protected])([email protected]))([email protected]([email protected]))([email protected])([email protected]([email protected])([email protected]))([email protected])handlebars: 4.7.9micromatch: 4.0.8transitivePeerDependencies:@@ -20576,6 +20601,35 @@ snapshots:- eslint-import-resolver-webpack- supports-color+ [email protected](@typescript-eslint/[email protected]([email protected]([email protected])([email protected]))([email protected])([email protected]))([email protected])([email protected]([email protected])([email protected]))([email protected]):+ dependencies:+ '@rtsao/scc': 1.1.0+ array-includes: 3.1.9+ array.prototype.findlastindex: 1.2.6+ array.prototype.flat: 1.3.3+ array.prototype.flatmap: 1.3.3+ debug: 3.2.7([email protected])+ doctrine: 2.1.0+ eslint: 10.8.0([email protected])([email protected])+ eslint-import-resolver-node: 0.3.10([email protected])+ eslint-module-utils: 2.12.1(@typescript-eslint/[email protected]([email protected]([email protected])([email protected]))([email protected])([email protected]))([email protected]([email protected]))([email protected])([email protected]([email protected])([email protected]))([email protected])+ hasown: 2.0.4+ is-core-module: 2.16.2+ is-glob: 4.0.3+ minimatch: 3.1.5+ object.fromentries: 2.0.8+ object.groupby: 1.0.3+ object.values: 1.2.1+ semver: 6.3.1+ string.prototype.trimend: 1.0.9+ tsconfig-paths: 3.15.0+ optionalDependencies:+ '@typescript-eslint/parser': 8.67.0([email protected]([email protected])([email protected]))([email protected])([email protected])+ transitivePeerDependencies:+ - eslint-import-resolver-typescript+ - eslint-import-resolver-webpack+ - supports-color+dependencies:aria-query: 5.3.2diff --git a/worker/src/queues/ingestionQueue.ts b/worker/src/queues/ingestionQueue.tsindex e90f093edc..ca07e38058 100644--- a/worker/src/queues/ingestionQueue.ts+++ b/worker/src/queues/ingestionQueue.ts@@ -1,4 +1,4 @@-import { Job, Processor } from "bullmq";+import { Job, Processor, UnrecoverableError } from "bullmq";import {clickhouseClient,getClickhouseEntityType,@@ -29,7 +29,16 @@ import { IngestionService } from "../services/IngestionService";import { ClickhouseWriter, TableName } from "../services/ClickhouseWriter";import { chunk } from "lodash";import { randomUUID } from "crypto";-import { logRetryableJobFailure } from "./jobFailureLog";+import {+ exceedsNonSlowDownAttemptBudget,+ logRetryableJobFailure,+} from "./jobFailureLog";++/**+ * Attempts for secondary-queue failures other than S3 SlowDown. The queue's+ * larger default budget only exists to outlast throttling.+ */+const SECONDARY_QUEUE_NON_SLOWDOWN_ATTEMPTS = 5;export const ingestionQueueProcessorBuilder = (enableRedirectToSecondaryQueue: boolean,@@ -356,6 +365,18 @@ export const ingestionQueueProcessorBuilder = (error: e,job,});+ if (+ !enableRedirectToSecondaryQueue &&+ exceedsNonSlowDownAttemptBudget(+ job,+ e,+ SECONDARY_QUEUE_NON_SLOWDOWN_ATTEMPTS,+ )+ ) {+ throw new UnrecoverableError(+ e instanceof Error ? e.message : String(e),+ );+ }throw e;}};diff --git a/worker/src/queues/jobFailureLog.test.ts b/worker/src/queues/jobFailureLog.test.tsindex 1e0c121c68..46e5e69472 100644--- a/worker/src/queues/jobFailureLog.test.ts+++ b/worker/src/queues/jobFailureLog.test.ts@@ -19,7 +19,10 @@ vi.mock("@langfuse/shared/src/server", async () => {};});-import { logRetryableJobFailure } from "./jobFailureLog";+import {+ exceedsNonSlowDownAttemptBudget,+ logRetryableJobFailure,+} from "./jobFailureLog";function slowDown(): Error {const cause = Object.assign(new Error("Please reduce your request rate."), {@@ -98,3 +101,23 @@ describe("logRetryableJobFailure", () => {expect(mocks.warn).not.toHaveBeenCalled();});});++describe("exceedsNonSlowDownAttemptBudget", () => {+ const job = (attemptsMade: number) => ({+ attemptsMade,+ opts: { attempts: 8 },+ });++ it("allows non-throttle failures to retry until the budget's last attempt", () => {+ expect(+ exceedsNonSlowDownAttemptBudget(job(3), new Error("NoSuchKey"), 5),+ ).toBe(false);+ expect(+ exceedsNonSlowDownAttemptBudget(job(4), new Error("NoSuchKey"), 5),+ ).toBe(true);+ });++ it("never caps SlowDown, which keeps the full queue budget", () => {+ expect(exceedsNonSlowDownAttemptBudget(job(6), slowDown(), 5)).toBe(false);+ });+});diff --git a/worker/src/queues/jobFailureLog.ts b/worker/src/queues/jobFailureLog.tsindex 153977ac7a..b3111f40d2 100644--- a/worker/src/queues/jobFailureLog.ts+++ b/worker/src/queues/jobFailureLog.ts@@ -36,6 +36,19 @@ export function logRetryableJobFailure(params: {traceException(error);}+/**+ * Whether a non-SlowDown failure has used up `budget` attempts. Lets a queue+ * keep a long retry budget for S3 throttling without extending it to failures+ * that retrying will not fix.+ */+export function exceedsNonSlowDownAttemptBudget(+ job: AttemptState,+ error: unknown,+ budget: number,+): boolean {+ return !isS3SlowDownError(error) && (job.attemptsMade ?? 0) >= budget - 1;+}+function isTerminalAttempt(job: AttemptState,error: unknown,