Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

fix(worker): fail batch exports on ClickHouse timeout and memory errors without retry (#18724)

Niklas Semmler4ca201226d3c
diff --git a/worker/src/features/batchExport/findInCauseChain.ts b/worker/src/features/batchExport/findInCauseChain.ts
new file mode 100644
index 0000000000..1dc1a7acdc
--- /dev/null
+++ b/worker/src/features/batchExport/findInCauseChain.ts
@@ -0,0 +1,14 @@
+/** Returns the first link of `error`'s cause chain matching `predicate`; cycle-safe. */
+export const findInCauseChain = (
+ error: unknown,
+ predicate: (link: object) => boolean,
+): object | undefined => {
+ const seen = new Set<unknown>();
+ let current: unknown = error;
+ while (current && typeof current === "object" && !seen.has(current)) {
+ if (predicate(current)) return current;
+ seen.add(current);
+ current = (current as { cause?: unknown }).cause;
+ }
+ return undefined;
+};
diff --git a/worker/src/features/batchExport/handleBatchExportJob.ts b/worker/src/features/batchExport/handleBatchExportJob.ts
index dcd9370df9..746c86d35d 100644
--- a/worker/src/features/batchExport/handleBatchExportJob.ts
+++ b/worker/src/features/batchExport/handleBatchExportJob.ts
@@ -25,6 +25,7 @@ import { getDatabaseReadStreamPaginated } from "../database-read-stream/getDatab
import { getObservationStream } from "../database-read-stream/observation-stream";
import { getTraceStream } from "../database-read-stream/trace-stream";
import { getEventsStream } from "../database-read-stream/event-stream";
+import { findInCauseChain } from "./findInCauseChain";
// Map table names to comment object types for preprocessing
const tableToCommentType: Record<string, CommentObjectType | undefined> = {
@@ -36,20 +37,13 @@ const tableToCommentType: Record<string, CommentObjectType | undefined> = {
const BATCH_EXPORT_CLICKHOUSE_SERVICE: PreferredClickhouseService = "ReadOnly";
-// Walks an error's `cause` chain to check whether `target` produced it. Used to
-// tell a read/transform failure (whose error the uploader rethrows as its cause)
-// apart from a genuine storage failure that merely aborts the stream as a side
-// effect. Guards against cause cycles.
-const isCausedBy = (error: unknown, target: unknown): boolean => {
- const seen = new Set<unknown>();
- let current: unknown = error;
- while (current && typeof current === "object" && !seen.has(current)) {
- if (current === target) return true;
- seen.add(current);
- current = (current as { cause?: unknown }).cause;
- }
- return false;
-};
+/**
+ * Checks whether `target` produced `error`, telling a read/transform failure
+ * (rethrown by the uploader as its cause) apart from a storage failure that
+ * merely aborts the stream as a side effect.
+ */
+const isCausedBy = (error: unknown, target: unknown): boolean =>
+ findInCauseChain(error, (link) => link === target) !== undefined;
export const handleBatchExportJob = async (
batchExportJob: BatchExportJobType,
diff --git a/worker/src/queues/__tests__/batchExportQueue.test.ts b/worker/src/queues/__tests__/batchExportQueue.test.ts
new file mode 100644
index 0000000000..a61e7cf2e3
--- /dev/null
+++ b/worker/src/queues/__tests__/batchExportQueue.test.ts
@@ -0,0 +1,91 @@
+import { Job } from "bullmq";
+import { beforeEach, describe, expect, it, vi } from "vitest";
+import { BatchExportStatus } from "@langfuse/shared";
+import {
+ ClickHouseResourceError,
+ QueueName,
+ TQueueJobTypes,
+} from "@langfuse/shared/src/server";
+
+const mocks = vi.hoisted(() => ({
+ handleBatchExportJob: vi.fn(),
+ updateBatchExport: vi.fn().mockResolvedValue(undefined),
+}));
+
+vi.mock("@langfuse/shared/src/db", () => ({
+ prisma: { batchExport: { update: mocks.updateBatchExport } },
+}));
+
+vi.mock("../../features/batchExport/handleBatchExportJob", () => ({
+ handleBatchExportJob: mocks.handleBatchExportJob,
+}));
+
+import { batchExportQueueProcessor } from "../batchExportQueue";
+
+const job = {
+ data: { payload: { projectId: "project-1", batchExportId: "export-1" } },
+} as Job<TQueueJobTypes[QueueName.BatchExport]>;
+
+// The handler rethrows read-stream failures as the cause of its own error.
+const rejectWithReadError = (message: string) => {
+ const resourceError = ClickHouseResourceError.wrapIfResourceError(
+ new Error(`${message} [query_id: q-1]`),
+ );
+ const handlerError = new Error("[BATCH EXPORT] Reading export data failed", {
+ cause: resourceError,
+ });
+ mocks.handleBatchExportJob.mockRejectedValue(handlerError);
+ return handlerError;
+};
+
+describe("batchExportQueueProcessor", () => {
+ beforeEach(() => {
+ vi.clearAllMocks();
+ });
+
+ it.each([
+ "Timeout exceeded: elapsed 185002.1 ms, maximum: 185000 ms.",
+ "(for query) memory limit exceeded: would use 10.00 GiB, maximum: 9.31 GiB.",
+ "Memory limit (for query) exceeded: would use 10.00 GiB, maximum: 9.31 GiB.",
+ ])("fails without retry on a query-bound limit: %s", async (message) => {
+ rejectWithReadError(message);
+
+ const thrown = await batchExportQueueProcessor(job).catch((e) => e);
+
+ expect(thrown.name).toBe("UnrecoverableError");
+ expect(mocks.updateBatchExport).toHaveBeenCalledWith(
+ expect.objectContaining({
+ data: expect.objectContaining({
+ status: BatchExportStatus.FAILED,
+ log: ClickHouseResourceError.ERROR_ADVICE_MESSAGE,
+ }),
+ }),
+ );
+ });
+
+ it.each([
+ "Timeout error.",
+ "Timeout exceeded while reading from socket (10.0.0.1:8443, 30000 ms)",
+ "(total) memory limit exceeded: would use 212.42 GiB, maximum: 212.40 GiB. OvercommitTracker decision: Query was selected to stop by OvercommitTracker.",
+ ])("keeps retrying a load-dependent failure: %s", async (message) => {
+ const handlerError = rejectWithReadError(message);
+
+ const thrown = await batchExportQueueProcessor(job).catch((e) => e);
+
+ expect(thrown).toBe(handlerError);
+ expect(mocks.updateBatchExport).toHaveBeenCalledWith(
+ expect.objectContaining({
+ data: expect.objectContaining({ log: "An internal error occurred" }),
+ }),
+ );
+ });
+
+ it("rethrows other errors so BullMQ retries them", async () => {
+ const uploadError = new Error("Failed to upload file to S3 (buffered)");
+ mocks.handleBatchExportJob.mockRejectedValue(uploadError);
+
+ const thrown = await batchExportQueueProcessor(job).catch((e) => e);
+
+ expect(thrown).toBe(uploadError);
+ });
+});
diff --git a/worker/src/queues/batchExportQueue.ts b/worker/src/queues/batchExportQueue.ts
index 4debd8e785..bf190bd049 100644
--- a/worker/src/queues/batchExportQueue.ts
+++ b/worker/src/queues/batchExportQueue.ts
@@ -1,4 +1,4 @@
-import { Job } from "bullmq";
+import { Job, UnrecoverableError } from "bullmq";
import {
BaseError,
@@ -7,9 +7,28 @@ import {
} from "@langfuse/shared";
import { prisma } from "@langfuse/shared/src/db";
-import { traceException, logger } from "@langfuse/shared/src/server";
+import {
+ ClickHouseResourceError,
+ traceException,
+ logger,
+} from "@langfuse/shared/src/server";
import { QueueName, TQueueJobTypes } from "@langfuse/shared/src/server";
import { handleBatchExportJob } from "../features/batchExport/handleBatchExportJob";
+import { findInCauseChain } from "../features/batchExport/findInCauseChain";
+
+/**
+ * Server-side kills of this query alone: TIMEOUT_EXCEEDED and the per-query
+ * MEMORY_LIMIT_EXCEEDED, in both the older and newer ClickHouse wording.
+ * Client/socket timeouts and total, per-user or overcommit memory kills depend
+ * on concurrent load and stay retryable.
+ */
+const QUERY_BOUND_RESOURCE_ERROR_PATTERNS: Partial<
+ Record<ClickHouseResourceError["errorType"], RegExp>
+> = {
+ TIMEOUT: /\btimeout exceeded: elapsed\b/i,
+ MEMORY_LIMIT:
+ /memory limit \(for query\) exceeded|\(for query\) memory limit exceeded/i,
+};
export const batchExportQueueProcessor = async (
job: Job<TQueueJobTypes[QueueName.BatchExport]>,
@@ -28,8 +47,8 @@ export const batchExportQueueProcessor = async (
);
return true;
}
- const displayError =
- e instanceof BaseError ? e.message : "An internal error occurred";
+ // A retry reruns the identical query, so it would hit the same limit again.
+ const isQueryBoundResourceError = hasQueryBoundResourceErrorCause(e);
await prisma.batchExport.update({
where: {
@@ -39,7 +58,7 @@ export const batchExportQueueProcessor = async (
data: {
status: BatchExportStatus.FAILED,
finishedAt: new Date(),
- log: displayError,
+ log: getDisplayError(e, isQueryBoundResourceError),
},
});
@@ -48,6 +67,30 @@ export const batchExportQueueProcessor = async (
e,
);
traceException(e);
+ if (isQueryBoundResourceError) {
+ throw new UnrecoverableError(e instanceof Error ? e.message : String(e));
+ }
throw e;
}
};
+
+/** Searches the whole cause chain, since the handler wraps read-stream failures. */
+const hasQueryBoundResourceErrorCause = (error: unknown): boolean =>
+ findInCauseChain(
+ error,
+ (link) =>
+ ClickHouseResourceError.is(link) &&
+ (QUERY_BOUND_RESOURCE_ERROR_PATTERNS[link.errorType]?.test(
+ link.message,
+ ) ??
+ false),
+ ) !== undefined;
+
+const getDisplayError = (
+ e: unknown,
+ isQueryBoundResourceError: boolean,
+): string => {
+ if (isQueryBoundResourceError)
+ return ClickHouseResourceError.ERROR_ADVICE_MESSAGE;
+ return e instanceof BaseError ? e.message : "An internal error occurred";
+};