Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

fix(batch-export): surface the real read failure, not a generic S3 error (#17891)

Niklas Semmler20a2613739fb
diff --git a/worker/src/__tests__/handleBatchExportJob-read-error.unit.test.ts b/worker/src/__tests__/handleBatchExportJob-read-error.unit.test.ts
new file mode 100644
index 0000000000..f50ef50b94
--- /dev/null
+++ b/worker/src/__tests__/handleBatchExportJob-read-error.unit.test.ts
@@ -0,0 +1,156 @@
+import { PassThrough, Readable } from "stream";
+import { expect, it, vi } from "vitest";
+import {
+ BatchExportFileFormat,
+ BatchExportStatus,
+ BatchExportTableName,
+} from "@langfuse/shared";
+
+const mocks = vi.hoisted(() => ({
+ findBatchExport: vi.fn(),
+ updateBatchExport: vi.fn().mockResolvedValue(undefined),
+ applyCommentFilters: vi
+ .fn()
+ .mockResolvedValue({ filterState: [], hasNoMatches: false }),
+ getEventsStream: vi.fn(),
+ uploadFileBuffered: vi.fn(),
+ getSignedUrl: vi.fn(),
+}));
+
+vi.mock("@langfuse/shared/src/db", () => ({
+ prisma: {
+ batchExport: {
+ findFirst: mocks.findBatchExport,
+ update: mocks.updateBatchExport,
+ },
+ },
+}));
+
+vi.mock("@langfuse/shared/src/server", () => ({
+ applyCommentFilters: mocks.applyCommentFilters,
+ getCurrentSpan: vi.fn(() => undefined),
+ logger: { info: vi.fn(), warn: vi.fn(), error: vi.fn() },
+ sendBatchExportSuccessEmail: vi.fn(),
+ StorageServiceFactory: {
+ getInstance: () => ({
+ uploadFileBuffered: mocks.uploadFileBuffered,
+ getSignedUrl: mocks.getSignedUrl,
+ }),
+ },
+ // Object-mode passthrough so the piped stream error propagates to the uploader.
+ streamTransformations: {
+ [BatchExportFileFormat.JSONL]: () => new PassThrough({ objectMode: true }),
+ },
+}));
+
+vi.mock("../env", () => ({
+ env: {
+ LANGFUSE_S3_BATCH_EXPORT_ENABLED: "true",
+ LANGFUSE_S3_BATCH_EXPORT_BUCKET: "export-bucket",
+ LANGFUSE_S3_BATCH_EXPORT_PREFIX: "",
+ BATCH_EXPORT_DOWNLOAD_LINK_EXPIRATION_HOURS: 24,
+ BATCH_EXPORT_S3_PART_SIZE_MIB: 20,
+ },
+}));
+
+vi.mock("../features/database-read-stream/event-stream", () => ({
+ getEventsStream: mocks.getEventsStream,
+}));
+
+import { handleBatchExportJob } from "../features/batchExport/handleBatchExportJob";
+
+const seedQueuedExport = () =>
+ mocks.findBatchExport.mockResolvedValue({
+ createdAt: new Date(),
+ status: BatchExportStatus.QUEUED,
+ format: BatchExportFileFormat.JSONL,
+ query: {
+ tableName: BatchExportTableName.Events,
+ filter: null,
+ orderBy: null,
+ },
+ });
+
+const runAndCatch = () =>
+ handleBatchExportJob({
+ projectId: "project-1",
+ batchExportId: "export-1",
+ }).then(
+ () => {
+ throw new Error("expected handleBatchExportJob to reject");
+ },
+ (err) => err,
+ );
+
+it("surfaces the read error, not the storage error, when the read stream fails", async () => {
+ const readError = new Error(
+ "Read timeout: query exceeded max_execution_time",
+ );
+
+ const readStream = new Readable({ objectMode: true, read() {} });
+ setImmediate(() => readStream.destroy(readError));
+ mocks.getEventsStream.mockResolvedValue(readStream);
+
+ // Uploader rewraps a mid-stream failure as a storage error whose cause is the
+ // piped read error, mirroring StorageService.handleStorageError.
+ mocks.uploadFileBuffered.mockImplementation(
+ ({ data }: { data: Readable }) =>
+ new Promise((_resolve, reject) => {
+ data.on("data", () => {});
+ data.on("end", () => reject(new Error("stream ended without error")));
+ data.on("error", (err) =>
+ reject(
+ new Error("Failed to upload file to S3 (buffered)", { cause: err }),
+ ),
+ );
+ }),
+ );
+
+ seedQueuedExport();
+
+ const thrown = await runAndCatch();
+
+ expect(thrown).toBeInstanceOf(Error);
+ expect(thrown.message).toContain(readError.message);
+ expect(thrown.message).not.toContain("S3");
+ expect(thrown.cause).toBe(readError);
+ expect(mocks.getSignedUrl).not.toHaveBeenCalled();
+});
+
+it("keeps reporting a genuine storage failure as a storage error", async () => {
+ // Read stream stays healthy and keeps producing rows.
+ const readStream = new Readable({
+ objectMode: true,
+ read() {
+ this.push({ id: "row" });
+ },
+ });
+ mocks.getEventsStream.mockResolvedValue(readStream);
+
+ const s3Error = new Error("S3 AccessDenied");
+ const storageError = new Error("Failed to upload file to S3 (buffered)", {
+ cause: s3Error,
+ });
+
+ // A real part-upload failure aborts the piped stream (as BufferedStreamUploader
+ // does on its early break), which trips the pipeline's premature-close error,
+ // then rejects with the storage error — whose cause is the S3 failure, not the
+ // premature-close.
+ mocks.uploadFileBuffered.mockImplementation(
+ ({ data }: { data: Readable }) =>
+ new Promise((_resolve, reject) => {
+ data.once("data", () => {
+ data.destroy();
+ setImmediate(() => reject(storageError));
+ });
+ }),
+ );
+
+ seedQueuedExport();
+
+ const thrown = await runAndCatch();
+
+ expect(thrown).toBe(storageError);
+ expect(thrown.message).toContain("S3");
+ expect(mocks.getSignedUrl).not.toHaveBeenCalled();
+});
diff --git a/worker/src/features/batchExport/handleBatchExportJob.ts b/worker/src/features/batchExport/handleBatchExportJob.ts
index e52971552e..dcd9370df9 100644
--- a/worker/src/features/batchExport/handleBatchExportJob.ts
+++ b/worker/src/features/batchExport/handleBatchExportJob.ts
@@ -36,6 +36,21 @@ const tableToCommentType: Record<string, CommentObjectType | undefined> = {
const BATCH_EXPORT_CLICKHOUSE_SERVICE: PreferredClickhouseService = "ReadOnly";
+// Walks an error's `cause` chain to check whether `target` produced it. Used to
+// tell a read/transform failure (whose error the uploader rethrows as its cause)
+// apart from a genuine storage failure that merely aborts the stream as a side
+// effect. Guards against cause cycles.
+const isCausedBy = (error: unknown, target: unknown): boolean => {
+ const seen = new Set<unknown>();
+ let current: unknown = error;
+ while (current && typeof current === "object" && !seen.has(current)) {
+ if (current === target) return true;
+ seen.add(current);
+ current = (current as { cause?: unknown }).cause;
+ }
+ return false;
+};
+
export const handleBatchExportJob = async (
batchExportJob: BatchExportJobType,
) => {
@@ -224,12 +239,19 @@ export const handleBatchExportJob = async (
},
});
+ // The read/transform pipeline is piped into the buffered uploader below. A
+ // mid-stream failure here (e.g. a ClickHouse query timeout) surfaces inside
+ // the uploader's try/catch and gets rewrapped as a storage error, so we
+ // capture the real cause here to re-attribute it after the upload rejects.
+ let readStreamError: unknown = null;
+
const fileStream = pipeline(
dbReadStream,
loggingTransform,
streamTransformations[jobDetails.format as BatchExportFileFormat](),
(err) => {
if (err) {
+ readStreamError = err;
logger.error(
"[BATCH EXPORT] Getting data from DB and transform failed: ",
err,
@@ -269,13 +291,34 @@ export const handleBatchExportJob = async (
const storageService = StorageServiceFactory.getInstance(storageParams);
- await storageService.uploadFileBuffered({
- fileName,
- fileType:
- exportOptions[jobDetails.format as BatchExportFileFormat].fileType,
- data: fileStream,
- partSizeBytes: env.BATCH_EXPORT_S3_PART_SIZE_MIB * 1024 * 1024,
- });
+ try {
+ await storageService.uploadFileBuffered({
+ fileName,
+ fileType:
+ exportOptions[jobDetails.format as BatchExportFileFormat].fileType,
+ data: fileStream,
+ partSizeBytes: env.BATCH_EXPORT_S3_PART_SIZE_MIB * 1024 * 1024,
+ });
+ } catch (uploadError) {
+ // A read/transform stream failure (ClickHouse/Postgres read or a transform)
+ // is piped into the uploader, which rethrows it as the cause of its own
+ // error. Surface the real origin only when the upload error's cause chain
+ // traces back to that stream error — otherwise a genuine storage failure,
+ // which aborts the stream as a side effect and trips readStreamError with an
+ // unrelated premature-close, would be mislabeled. Thrown as a plain Error so
+ // the customer-facing `log` stays generic and does not leak internals.
+ if (readStreamError && isCausedBy(uploadError, readStreamError)) {
+ const causeMessage =
+ readStreamError instanceof Error
+ ? readStreamError.message
+ : String(readStreamError);
+ throw new Error(
+ `[BATCH EXPORT] Reading export data failed: ${causeMessage}`,
+ { cause: readStreamError },
+ );
+ }
+ throw uploadError;
+ }
// asAttachment must be explicit: S3 defaults it to true, but GCS and Azure
// don't — and the web tier's downloadUrl fallback returns this stored URL