Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

fix(worker): extend secondary ingestion retry window to outlast S3 throttling (#18371)

Niklas Semmler53924a83bc38
diff --git a/packages/shared/src/server/redis/ingestionQueue.ts b/packages/shared/src/server/redis/ingestionQueue.ts
index 5621d19b19..740d11a2fc 100644
--- a/packages/shared/src/server/redis/ingestionQueue.ts
+++ b/packages/shared/src/server/redis/ingestionQueue.ts
@@ -146,10 +146,13 @@ export class SecondaryIngestionQueue {
defaultJobOptions: {
removeOnComplete: true,
removeOnFail: 100_000,
- attempts: 5,
+ attempts: 8,
backoff: {
type: "exponential",
delay: 5000,
+ // Spreads retries of jobs that failed together (e.g. one S3
+ // throttling episode) so they don't hit the prefix in lockstep.
+ jitter: 0.5,
},
},
})
diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml
index 6e845181a4..34e90aba8a 100644
--- a/pnpm-lock.yaml
+++ b/pnpm-lock.yaml
@@ -1026,7 +1026,7 @@ importers:
eslint-plugin-boundaries:
specifier: 7.2.0
eslint-plugin-check-file:
specifier: 3.3.2
@@ -13648,10 +13648,10 @@ snapshots:
'@blazediff/[email protected]': {}
dependencies:
eslint-import-resolver-node: 0.3.9([email protected])
handlebars: 4.7.9
is-core-module: 2.16.1
micromatch: 4.0.8
@@ -14162,7 +14162,7 @@ snapshots:
dependencies:
'@floating-ui/dom': 1.8.0
- '@floating-ui/utils': 0.2.10
+ '@floating-ui/utils': 0.2.12
transitivePeerDependencies:
- '@vue/composition-api'
@@ -18449,6 +18449,19 @@ snapshots:
transitivePeerDependencies:
- supports-color
+ dependencies:
+ '@typescript-eslint/scope-manager': 8.67.0
+ '@typescript-eslint/types': 8.67.0
+ '@typescript-eslint/typescript-estree': 8.67.0([email protected])([email protected])
+ '@typescript-eslint/visitor-keys': 8.67.0
+ debug: 4.4.3([email protected])
+ typescript: 7.0.2
+ transitivePeerDependencies:
+ - supports-color
+ optional: true
+
'@typescript-eslint/[email protected](@typescript/[email protected])([email protected])':
dependencies:
'@typescript-eslint/tsconfig-utils': 8.67.0(@typescript/[email protected])
@@ -19146,11 +19159,11 @@ snapshots:
dependencies:
- acorn: 8.17.0
+ acorn: 8.18.0
dependencies:
- acorn: 8.17.0
+ acorn: 8.18.0
@@ -20516,23 +20529,35 @@ snapshots:
transitivePeerDependencies:
- supports-color
+ dependencies:
+ debug: 3.2.7([email protected])
+ optionalDependencies:
+ eslint-import-resolver-node: 0.3.10([email protected])
+ transitivePeerDependencies:
+ - supports-color
+
dependencies:
debug: 3.2.7([email protected])
optionalDependencies:
eslint-import-resolver-node: 0.3.9([email protected])
transitivePeerDependencies:
- supports-color
dependencies:
chalk: 4.1.2
eslint-import-resolver-node: 0.3.9([email protected])
handlebars: 4.7.9
micromatch: 4.0.8
transitivePeerDependencies:
@@ -20576,6 +20601,35 @@ snapshots:
- eslint-import-resolver-webpack
- supports-color
+ dependencies:
+ '@rtsao/scc': 1.1.0
+ array-includes: 3.1.9
+ array.prototype.findlastindex: 1.2.6
+ array.prototype.flat: 1.3.3
+ array.prototype.flatmap: 1.3.3
+ debug: 3.2.7([email protected])
+ doctrine: 2.1.0
+ eslint-import-resolver-node: 0.3.10([email protected])
+ hasown: 2.0.4
+ is-core-module: 2.16.2
+ is-glob: 4.0.3
+ minimatch: 3.1.5
+ object.fromentries: 2.0.8
+ object.groupby: 1.0.3
+ object.values: 1.2.1
+ semver: 6.3.1
+ string.prototype.trimend: 1.0.9
+ tsconfig-paths: 3.15.0
+ optionalDependencies:
+ transitivePeerDependencies:
+ - eslint-import-resolver-typescript
+ - eslint-import-resolver-webpack
+ - supports-color
+
dependencies:
aria-query: 5.3.2
diff --git a/worker/src/queues/ingestionQueue.ts b/worker/src/queues/ingestionQueue.ts
index e90f093edc..ca07e38058 100644
--- a/worker/src/queues/ingestionQueue.ts
+++ b/worker/src/queues/ingestionQueue.ts
@@ -1,4 +1,4 @@
-import { Job, Processor } from "bullmq";
+import { Job, Processor, UnrecoverableError } from "bullmq";
import {
clickhouseClient,
getClickhouseEntityType,
@@ -29,7 +29,16 @@ import { IngestionService } from "../services/IngestionService";
import { ClickhouseWriter, TableName } from "../services/ClickhouseWriter";
import { chunk } from "lodash";
import { randomUUID } from "crypto";
-import { logRetryableJobFailure } from "./jobFailureLog";
+import {
+ exceedsNonSlowDownAttemptBudget,
+ logRetryableJobFailure,
+} from "./jobFailureLog";
+
+/**
+ * Attempts for secondary-queue failures other than S3 SlowDown. The queue's
+ * larger default budget only exists to outlast throttling.
+ */
+const SECONDARY_QUEUE_NON_SLOWDOWN_ATTEMPTS = 5;
export const ingestionQueueProcessorBuilder = (
enableRedirectToSecondaryQueue: boolean,
@@ -356,6 +365,18 @@ export const ingestionQueueProcessorBuilder = (
error: e,
job,
});
+ if (
+ !enableRedirectToSecondaryQueue &&
+ exceedsNonSlowDownAttemptBudget(
+ job,
+ e,
+ SECONDARY_QUEUE_NON_SLOWDOWN_ATTEMPTS,
+ )
+ ) {
+ throw new UnrecoverableError(
+ e instanceof Error ? e.message : String(e),
+ );
+ }
throw e;
}
};
diff --git a/worker/src/queues/jobFailureLog.test.ts b/worker/src/queues/jobFailureLog.test.ts
index 1e0c121c68..46e5e69472 100644
--- a/worker/src/queues/jobFailureLog.test.ts
+++ b/worker/src/queues/jobFailureLog.test.ts
@@ -19,7 +19,10 @@ vi.mock("@langfuse/shared/src/server", async () => {
};
});
-import { logRetryableJobFailure } from "./jobFailureLog";
+import {
+ exceedsNonSlowDownAttemptBudget,
+ logRetryableJobFailure,
+} from "./jobFailureLog";
function slowDown(): Error {
const cause = Object.assign(new Error("Please reduce your request rate."), {
@@ -98,3 +101,23 @@ describe("logRetryableJobFailure", () => {
expect(mocks.warn).not.toHaveBeenCalled();
});
});
+
+describe("exceedsNonSlowDownAttemptBudget", () => {
+ const job = (attemptsMade: number) => ({
+ attemptsMade,
+ opts: { attempts: 8 },
+ });
+
+ it("allows non-throttle failures to retry until the budget's last attempt", () => {
+ expect(
+ exceedsNonSlowDownAttemptBudget(job(3), new Error("NoSuchKey"), 5),
+ ).toBe(false);
+ expect(
+ exceedsNonSlowDownAttemptBudget(job(4), new Error("NoSuchKey"), 5),
+ ).toBe(true);
+ });
+
+ it("never caps SlowDown, which keeps the full queue budget", () => {
+ expect(exceedsNonSlowDownAttemptBudget(job(6), slowDown(), 5)).toBe(false);
+ });
+});
diff --git a/worker/src/queues/jobFailureLog.ts b/worker/src/queues/jobFailureLog.ts
index 153977ac7a..b3111f40d2 100644
--- a/worker/src/queues/jobFailureLog.ts
+++ b/worker/src/queues/jobFailureLog.ts
@@ -36,6 +36,19 @@ export function logRetryableJobFailure(params: {
traceException(error);
}
+/**
+ * Whether a non-SlowDown failure has used up `budget` attempts. Lets a queue
+ * keep a long retry budget for S3 throttling without extending it to failures
+ * that retrying will not fix.
+ */
+export function exceedsNonSlowDownAttemptBudget(
+ job: AttemptState,
+ error: unknown,
+ budget: number,
+): boolean {
+ return !isS3SlowDownError(error) && (job.attemptsMade ?? 0) >= budget - 1;
+}
+
function isTerminalAttempt(
job: AttemptState,
error: unknown,