🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
fix(worker): fail batch exports on ClickHouse timeout and memory errors without retry (#18724)
Niklas Semmler4ca201226d3c
diff --git a/worker/src/features/batchExport/findInCauseChain.ts b/worker/src/features/batchExport/findInCauseChain.tsnew file mode 100644index 0000000000..1dc1a7acdc--- /dev/null+++ b/worker/src/features/batchExport/findInCauseChain.ts@@ -0,0 +1,14 @@+/** Returns the first link of `error`'s cause chain matching `predicate`; cycle-safe. */+export const findInCauseChain = (+ error: unknown,+ predicate: (link: object) => boolean,+): object | undefined => {+ const seen = new Set<unknown>();+ let current: unknown = error;+ while (current && typeof current === "object" && !seen.has(current)) {+ if (predicate(current)) return current;+ seen.add(current);+ current = (current as { cause?: unknown }).cause;+ }+ return undefined;+};diff --git a/worker/src/features/batchExport/handleBatchExportJob.ts b/worker/src/features/batchExport/handleBatchExportJob.tsindex dcd9370df9..746c86d35d 100644--- a/worker/src/features/batchExport/handleBatchExportJob.ts+++ b/worker/src/features/batchExport/handleBatchExportJob.ts@@ -25,6 +25,7 @@ import { getDatabaseReadStreamPaginated } from "../database-read-stream/getDatabimport { getObservationStream } from "../database-read-stream/observation-stream";import { getTraceStream } from "../database-read-stream/trace-stream";import { getEventsStream } from "../database-read-stream/event-stream";+import { findInCauseChain } from "./findInCauseChain";// Map table names to comment object types for preprocessingconst tableToCommentType: Record<string, CommentObjectType | undefined> = {@@ -36,20 +37,13 @@ const tableToCommentType: Record<string, CommentObjectType | undefined> = {const BATCH_EXPORT_CLICKHOUSE_SERVICE: PreferredClickhouseService = "ReadOnly";-// Walks an error's `cause` chain to check whether `target` produced it. Used to-// tell a read/transform failure (whose error the uploader rethrows as its cause)-// apart from a genuine storage failure that merely aborts the stream as a side-// effect. Guards against cause cycles.-const isCausedBy = (error: unknown, target: unknown): boolean => {- const seen = new Set<unknown>();- let current: unknown = error;- while (current && typeof current === "object" && !seen.has(current)) {- if (current === target) return true;- seen.add(current);- current = (current as { cause?: unknown }).cause;- }- return false;-};+/**+ * Checks whether `target` produced `error`, telling a read/transform failure+ * (rethrown by the uploader as its cause) apart from a storage failure that+ * merely aborts the stream as a side effect.+ */+const isCausedBy = (error: unknown, target: unknown): boolean =>+ findInCauseChain(error, (link) => link === target) !== undefined;export const handleBatchExportJob = async (batchExportJob: BatchExportJobType,diff --git a/worker/src/queues/__tests__/batchExportQueue.test.ts b/worker/src/queues/__tests__/batchExportQueue.test.tsnew file mode 100644index 0000000000..a61e7cf2e3--- /dev/null+++ b/worker/src/queues/__tests__/batchExportQueue.test.ts@@ -0,0 +1,91 @@+import { Job } from "bullmq";+import { beforeEach, describe, expect, it, vi } from "vitest";+import { BatchExportStatus } from "@langfuse/shared";+import {+ ClickHouseResourceError,+ QueueName,+ TQueueJobTypes,+} from "@langfuse/shared/src/server";++const mocks = vi.hoisted(() => ({+ handleBatchExportJob: vi.fn(),+ updateBatchExport: vi.fn().mockResolvedValue(undefined),+}));++vi.mock("@langfuse/shared/src/db", () => ({+ prisma: { batchExport: { update: mocks.updateBatchExport } },+}));++vi.mock("../../features/batchExport/handleBatchExportJob", () => ({+ handleBatchExportJob: mocks.handleBatchExportJob,+}));++import { batchExportQueueProcessor } from "../batchExportQueue";++const job = {+ data: { payload: { projectId: "project-1", batchExportId: "export-1" } },+} as Job<TQueueJobTypes[QueueName.BatchExport]>;++// The handler rethrows read-stream failures as the cause of its own error.+const rejectWithReadError = (message: string) => {+ const resourceError = ClickHouseResourceError.wrapIfResourceError(+ new Error(`${message} [query_id: q-1]`),+ );+ const handlerError = new Error("[BATCH EXPORT] Reading export data failed", {+ cause: resourceError,+ });+ mocks.handleBatchExportJob.mockRejectedValue(handlerError);+ return handlerError;+};++describe("batchExportQueueProcessor", () => {+ beforeEach(() => {+ vi.clearAllMocks();+ });++ it.each([+ "Timeout exceeded: elapsed 185002.1 ms, maximum: 185000 ms.",+ "(for query) memory limit exceeded: would use 10.00 GiB, maximum: 9.31 GiB.",+ "Memory limit (for query) exceeded: would use 10.00 GiB, maximum: 9.31 GiB.",+ ])("fails without retry on a query-bound limit: %s", async (message) => {+ rejectWithReadError(message);++ const thrown = await batchExportQueueProcessor(job).catch((e) => e);++ expect(thrown.name).toBe("UnrecoverableError");+ expect(mocks.updateBatchExport).toHaveBeenCalledWith(+ expect.objectContaining({+ data: expect.objectContaining({+ status: BatchExportStatus.FAILED,+ log: ClickHouseResourceError.ERROR_ADVICE_MESSAGE,+ }),+ }),+ );+ });++ it.each([+ "Timeout error.",+ "Timeout exceeded while reading from socket (10.0.0.1:8443, 30000 ms)",+ "(total) memory limit exceeded: would use 212.42 GiB, maximum: 212.40 GiB. OvercommitTracker decision: Query was selected to stop by OvercommitTracker.",+ ])("keeps retrying a load-dependent failure: %s", async (message) => {+ const handlerError = rejectWithReadError(message);++ const thrown = await batchExportQueueProcessor(job).catch((e) => e);++ expect(thrown).toBe(handlerError);+ expect(mocks.updateBatchExport).toHaveBeenCalledWith(+ expect.objectContaining({+ data: expect.objectContaining({ log: "An internal error occurred" }),+ }),+ );+ });++ it("rethrows other errors so BullMQ retries them", async () => {+ const uploadError = new Error("Failed to upload file to S3 (buffered)");+ mocks.handleBatchExportJob.mockRejectedValue(uploadError);++ const thrown = await batchExportQueueProcessor(job).catch((e) => e);++ expect(thrown).toBe(uploadError);+ });+});diff --git a/worker/src/queues/batchExportQueue.ts b/worker/src/queues/batchExportQueue.tsindex 4debd8e785..bf190bd049 100644--- a/worker/src/queues/batchExportQueue.ts+++ b/worker/src/queues/batchExportQueue.ts@@ -1,4 +1,4 @@-import { Job } from "bullmq";+import { Job, UnrecoverableError } from "bullmq";import {BaseError,@@ -7,9 +7,28 @@ import {} from "@langfuse/shared";import { prisma } from "@langfuse/shared/src/db";-import { traceException, logger } from "@langfuse/shared/src/server";+import {+ ClickHouseResourceError,+ traceException,+ logger,+} from "@langfuse/shared/src/server";import { QueueName, TQueueJobTypes } from "@langfuse/shared/src/server";import { handleBatchExportJob } from "../features/batchExport/handleBatchExportJob";+import { findInCauseChain } from "../features/batchExport/findInCauseChain";++/**+ * Server-side kills of this query alone: TIMEOUT_EXCEEDED and the per-query+ * MEMORY_LIMIT_EXCEEDED, in both the older and newer ClickHouse wording.+ * Client/socket timeouts and total, per-user or overcommit memory kills depend+ * on concurrent load and stay retryable.+ */+const QUERY_BOUND_RESOURCE_ERROR_PATTERNS: Partial<+ Record<ClickHouseResourceError["errorType"], RegExp>+> = {+ TIMEOUT: /\btimeout exceeded: elapsed\b/i,+ MEMORY_LIMIT:+ /memory limit \(for query\) exceeded|\(for query\) memory limit exceeded/i,+};export const batchExportQueueProcessor = async (job: Job<TQueueJobTypes[QueueName.BatchExport]>,@@ -28,8 +47,8 @@ export const batchExportQueueProcessor = async ();return true;}- const displayError =- e instanceof BaseError ? e.message : "An internal error occurred";+ // A retry reruns the identical query, so it would hit the same limit again.+ const isQueryBoundResourceError = hasQueryBoundResourceErrorCause(e);await prisma.batchExport.update({where: {@@ -39,7 +58,7 @@ export const batchExportQueueProcessor = async (data: {status: BatchExportStatus.FAILED,finishedAt: new Date(),- log: displayError,+ log: getDisplayError(e, isQueryBoundResourceError),},});@@ -48,6 +67,30 @@ export const batchExportQueueProcessor = async (e,);traceException(e);+ if (isQueryBoundResourceError) {+ throw new UnrecoverableError(e instanceof Error ? e.message : String(e));+ }throw e;}};++/** Searches the whole cause chain, since the handler wraps read-stream failures. */+const hasQueryBoundResourceErrorCause = (error: unknown): boolean =>+ findInCauseChain(+ error,+ (link) =>+ ClickHouseResourceError.is(link) &&+ (QUERY_BOUND_RESOURCE_ERROR_PATTERNS[link.errorType]?.test(+ link.message,+ ) ??+ false),+ ) !== undefined;++const getDisplayError = (+ e: unknown,+ isQueryBoundResourceError: boolean,+): string => {+ if (isQueryBoundResourceError)+ return ClickHouseResourceError.ERROR_ADVICE_MESSAGE;+ return e instanceof BaseError ? e.message : "An internal error occurred";+};