🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
fix(batch-export): surface the real read failure, not a generic S3 error (#17891)
Niklas Semmler20a2613739fb
diff --git a/worker/src/__tests__/handleBatchExportJob-read-error.unit.test.ts b/worker/src/__tests__/handleBatchExportJob-read-error.unit.test.tsnew file mode 100644index 0000000000..f50ef50b94--- /dev/null+++ b/worker/src/__tests__/handleBatchExportJob-read-error.unit.test.ts@@ -0,0 +1,156 @@+import { PassThrough, Readable } from "stream";+import { expect, it, vi } from "vitest";+import {+ BatchExportFileFormat,+ BatchExportStatus,+ BatchExportTableName,+} from "@langfuse/shared";++const mocks = vi.hoisted(() => ({+ findBatchExport: vi.fn(),+ updateBatchExport: vi.fn().mockResolvedValue(undefined),+ applyCommentFilters: vi+ .fn()+ .mockResolvedValue({ filterState: [], hasNoMatches: false }),+ getEventsStream: vi.fn(),+ uploadFileBuffered: vi.fn(),+ getSignedUrl: vi.fn(),+}));++vi.mock("@langfuse/shared/src/db", () => ({+ prisma: {+ batchExport: {+ findFirst: mocks.findBatchExport,+ update: mocks.updateBatchExport,+ },+ },+}));++vi.mock("@langfuse/shared/src/server", () => ({+ applyCommentFilters: mocks.applyCommentFilters,+ getCurrentSpan: vi.fn(() => undefined),+ logger: { info: vi.fn(), warn: vi.fn(), error: vi.fn() },+ sendBatchExportSuccessEmail: vi.fn(),+ StorageServiceFactory: {+ getInstance: () => ({+ uploadFileBuffered: mocks.uploadFileBuffered,+ getSignedUrl: mocks.getSignedUrl,+ }),+ },+ // Object-mode passthrough so the piped stream error propagates to the uploader.+ streamTransformations: {+ [BatchExportFileFormat.JSONL]: () => new PassThrough({ objectMode: true }),+ },+}));++vi.mock("../env", () => ({+ env: {+ LANGFUSE_S3_BATCH_EXPORT_ENABLED: "true",+ LANGFUSE_S3_BATCH_EXPORT_BUCKET: "export-bucket",+ LANGFUSE_S3_BATCH_EXPORT_PREFIX: "",+ BATCH_EXPORT_DOWNLOAD_LINK_EXPIRATION_HOURS: 24,+ BATCH_EXPORT_S3_PART_SIZE_MIB: 20,+ },+}));++vi.mock("../features/database-read-stream/event-stream", () => ({+ getEventsStream: mocks.getEventsStream,+}));++import { handleBatchExportJob } from "../features/batchExport/handleBatchExportJob";++const seedQueuedExport = () =>+ mocks.findBatchExport.mockResolvedValue({+ createdAt: new Date(),+ status: BatchExportStatus.QUEUED,+ format: BatchExportFileFormat.JSONL,+ query: {+ tableName: BatchExportTableName.Events,+ filter: null,+ orderBy: null,+ },+ });++const runAndCatch = () =>+ handleBatchExportJob({+ projectId: "project-1",+ batchExportId: "export-1",+ }).then(+ () => {+ throw new Error("expected handleBatchExportJob to reject");+ },+ (err) => err,+ );++it("surfaces the read error, not the storage error, when the read stream fails", async () => {+ const readError = new Error(+ "Read timeout: query exceeded max_execution_time",+ );++ const readStream = new Readable({ objectMode: true, read() {} });+ setImmediate(() => readStream.destroy(readError));+ mocks.getEventsStream.mockResolvedValue(readStream);++ // Uploader rewraps a mid-stream failure as a storage error whose cause is the+ // piped read error, mirroring StorageService.handleStorageError.+ mocks.uploadFileBuffered.mockImplementation(+ ({ data }: { data: Readable }) =>+ new Promise((_resolve, reject) => {+ data.on("data", () => {});+ data.on("end", () => reject(new Error("stream ended without error")));+ data.on("error", (err) =>+ reject(+ new Error("Failed to upload file to S3 (buffered)", { cause: err }),+ ),+ );+ }),+ );++ seedQueuedExport();++ const thrown = await runAndCatch();++ expect(thrown).toBeInstanceOf(Error);+ expect(thrown.message).toContain(readError.message);+ expect(thrown.message).not.toContain("S3");+ expect(thrown.cause).toBe(readError);+ expect(mocks.getSignedUrl).not.toHaveBeenCalled();+});++it("keeps reporting a genuine storage failure as a storage error", async () => {+ // Read stream stays healthy and keeps producing rows.+ const readStream = new Readable({+ objectMode: true,+ read() {+ this.push({ id: "row" });+ },+ });+ mocks.getEventsStream.mockResolvedValue(readStream);++ const s3Error = new Error("S3 AccessDenied");+ const storageError = new Error("Failed to upload file to S3 (buffered)", {+ cause: s3Error,+ });++ // A real part-upload failure aborts the piped stream (as BufferedStreamUploader+ // does on its early break), which trips the pipeline's premature-close error,+ // then rejects with the storage error — whose cause is the S3 failure, not the+ // premature-close.+ mocks.uploadFileBuffered.mockImplementation(+ ({ data }: { data: Readable }) =>+ new Promise((_resolve, reject) => {+ data.once("data", () => {+ data.destroy();+ setImmediate(() => reject(storageError));+ });+ }),+ );++ seedQueuedExport();++ const thrown = await runAndCatch();++ expect(thrown).toBe(storageError);+ expect(thrown.message).toContain("S3");+ expect(mocks.getSignedUrl).not.toHaveBeenCalled();+});diff --git a/worker/src/features/batchExport/handleBatchExportJob.ts b/worker/src/features/batchExport/handleBatchExportJob.tsindex e52971552e..dcd9370df9 100644--- a/worker/src/features/batchExport/handleBatchExportJob.ts+++ b/worker/src/features/batchExport/handleBatchExportJob.ts@@ -36,6 +36,21 @@ const tableToCommentType: Record<string, CommentObjectType | undefined> = {const BATCH_EXPORT_CLICKHOUSE_SERVICE: PreferredClickhouseService = "ReadOnly";+// Walks an error's `cause` chain to check whether `target` produced it. Used to+// tell a read/transform failure (whose error the uploader rethrows as its cause)+// apart from a genuine storage failure that merely aborts the stream as a side+// effect. Guards against cause cycles.+const isCausedBy = (error: unknown, target: unknown): boolean => {+ const seen = new Set<unknown>();+ let current: unknown = error;+ while (current && typeof current === "object" && !seen.has(current)) {+ if (current === target) return true;+ seen.add(current);+ current = (current as { cause?: unknown }).cause;+ }+ return false;+};+export const handleBatchExportJob = async (batchExportJob: BatchExportJobType,) => {@@ -224,12 +239,19 @@ export const handleBatchExportJob = async (},});+ // The read/transform pipeline is piped into the buffered uploader below. A+ // mid-stream failure here (e.g. a ClickHouse query timeout) surfaces inside+ // the uploader's try/catch and gets rewrapped as a storage error, so we+ // capture the real cause here to re-attribute it after the upload rejects.+ let readStreamError: unknown = null;+const fileStream = pipeline(dbReadStream,loggingTransform,streamTransformations[jobDetails.format as BatchExportFileFormat](),(err) => {if (err) {+ readStreamError = err;logger.error("[BATCH EXPORT] Getting data from DB and transform failed: ",err,@@ -269,13 +291,34 @@ export const handleBatchExportJob = async (const storageService = StorageServiceFactory.getInstance(storageParams);- await storageService.uploadFileBuffered({- fileName,- fileType:- exportOptions[jobDetails.format as BatchExportFileFormat].fileType,- data: fileStream,- partSizeBytes: env.BATCH_EXPORT_S3_PART_SIZE_MIB * 1024 * 1024,- });+ try {+ await storageService.uploadFileBuffered({+ fileName,+ fileType:+ exportOptions[jobDetails.format as BatchExportFileFormat].fileType,+ data: fileStream,+ partSizeBytes: env.BATCH_EXPORT_S3_PART_SIZE_MIB * 1024 * 1024,+ });+ } catch (uploadError) {+ // A read/transform stream failure (ClickHouse/Postgres read or a transform)+ // is piped into the uploader, which rethrows it as the cause of its own+ // error. Surface the real origin only when the upload error's cause chain+ // traces back to that stream error — otherwise a genuine storage failure,+ // which aborts the stream as a side effect and trips readStreamError with an+ // unrelated premature-close, would be mislabeled. Thrown as a plain Error so+ // the customer-facing `log` stays generic and does not leak internals.+ if (readStreamError && isCausedBy(uploadError, readStreamError)) {+ const causeMessage =+ readStreamError instanceof Error+ ? readStreamError.message+ : String(readStreamError);+ throw new Error(+ `[BATCH EXPORT] Reading export data failed: ${causeMessage}`,+ { cause: readStreamError },+ );+ }+ throw uploadError;+ }// asAttachment must be explicit: S3 defaults it to true, but GCS and Azure// don't — and the web tier's downloadUrl fallback returns this stored URL