Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

fix(blob-storage): raise S3 multipart part size to lift 48.8 GiB object cap (#17303)

Niklas Semmlerf27d64754dcf
diff --git a/packages/shared/src/server/services/StorageService.test.ts b/packages/shared/src/server/services/StorageService.test.ts
index c7f838544e..fe17be82a0 100644
--- a/packages/shared/src/server/services/StorageService.test.ts
+++ b/packages/shared/src/server/services/StorageService.test.ts
@@ -10,6 +10,23 @@ import { env } from "../../env";
import { resolveMediaStorageEndpoints } from "../s3";
import { StorageServiceFactory } from "./StorageService";
+// Capture the options handed to lib-storage's Upload so the non-buffered
+// fallback's part size can be asserted without real network I/O.
+const s3UploadCtorOptions = vi.hoisted(
+ () => [] as Array<{ partSize?: number; queueSize?: number }>,
+);
+
+vi.mock("@aws-sdk/lib-storage", () => ({
+ Upload: class {
+ constructor(options: { partSize?: number; queueSize?: number }) {
+ s3UploadCtorOptions.push(options);
+ }
+ done() {
+ return Promise.resolve(undefined);
+ }
+ },
+}));
+
describe("S3StorageService region normalization", () => {
it("trims a persisted region before configuring the AWS client", async () => {
const service = StorageServiceFactory.getInstance({
@@ -529,3 +546,62 @@ describe("S3StorageService DeleteObjects checksum", () => {
expect(findHeader(request, "x-amz-checksum-crc32")).toBeUndefined();
});
});
+
+describe("S3StorageService non-buffered upload part size", () => {
+ const bufferedKey = "LANGFUSE_S3_UPLOAD_ENABLE_BUFFERED" as const;
+ const originalBuffered = env[bufferedKey];
+
+ afterEach(() => {
+ (env as Record<string, unknown>)[bufferedKey] = originalBuffered;
+ s3UploadCtorOptions.length = 0;
+ });
+
+ const makeService = () =>
+ StorageServiceFactory.getInstance({
+ accessKeyId: "test-access-key",
+ secretAccessKey: "test-secret-key",
+ bucketName: "test-bucket",
+ endpoint: "http://127.0.0.1:9000",
+ region: "us-east-1",
+ forcePathStyle: true,
+ useAzureBlob: false,
+ useGoogleCloudStorage: false,
+ useOCIObjectStorage: false,
+ awsSse: undefined,
+ awsSseKmsKeyId: undefined,
+ });
+
+ it("forwards the caller's part size so the 10k-part cap does not truncate large exports", async () => {
+ (env as Record<string, unknown>)[bufferedKey] = "false";
+
+ await makeService().uploadFileBuffered({
+ fileName: "export.csv",
+ fileType: "text/csv",
+ data: Readable.from(["a,b,c\n"]),
+ partSizeBytes: 100 * 1024 * 1024,
+ });
+
+ expect(s3UploadCtorOptions).toHaveLength(1);
+ const { partSize } = s3UploadCtorOptions[0];
+ expect(partSize).toBe(100 * 1024 * 1024);
+ // Above lib-storage's 5 MiB default, which caps a single object at ~48.83 GiB.
+ expect(partSize).toBeGreaterThan(5 * 1024 * 1024);
+ });
+
+ it("caps concurrency so parallel callers do not multiply peak buffer memory", async () => {
+ (env as Record<string, unknown>)[bufferedKey] = "false";
+
+ await makeService().uploadFileBuffered({
+ fileName: "export.csv",
+ fileType: "text/csv",
+ data: Readable.from(["a,b,c\n"]),
+ partSizeBytes: 100 * 1024 * 1024,
+ maxConcurrentParts: 2,
+ });
+
+ expect(s3UploadCtorOptions).toHaveLength(1);
+ // Caller's concurrency is honored instead of lib-storage's default of 4,
+ // which would otherwise buffer partSize x 4 per concurrent upload.
+ expect(s3UploadCtorOptions[0].queueSize).toBe(2);
+ });
+});
diff --git a/packages/shared/src/server/services/StorageService.ts b/packages/shared/src/server/services/StorageService.ts
index 7f73629977..b2deafb21c 100644
--- a/packages/shared/src/server/services/StorageService.ts
+++ b/packages/shared/src/server/services/StorageService.ts
@@ -836,9 +836,9 @@ class S3StorageService implements StorageService {
Body: data,
ContentType: fileType,
}),
- // Use provided partSize and queueSize, or fall back to defaults
- // Default: 5 MB part size supports files up to ~50 GB (5 MB × 10,000 parts)
- // For large files, use partSize: 100 * 1024 * 1024 (100 MB) to support up to ~1 TB
+ // When partSize is undefined lib-storage falls back to 5 MiB, capping a
+ // single object at ~48.83 GiB (5 MiB × 10,000 parts). Callers uploading
+ // large objects must pass an explicit partSize to raise that ceiling.
partSize: partSize,
queueSize: queueSize,
}).done();
@@ -860,10 +860,20 @@ class S3StorageService implements StorageService {
stats,
}: UploadFileBuffered): Promise<UploadPartStats | undefined> {
if (env.LANGFUSE_S3_UPLOAD_ENABLE_BUFFERED !== "true") {
- // Tuning applies only on the buffered path. Forward no overrides so the
- // fallback keeps lib-storage's defaults — forwarding the resolved 100 MiB
- // partSize would ~20x per-upload memory (buffered is off by default).
- await this.uploadFile({ fileName, fileType, data });
+ // Forward the caller's own part size and concurrency instead of a global
+ // default. lib-storage otherwise falls back to 5 MiB parts, capping a
+ // single object at ~48.83 GiB (5 MiB × 10,000 parts) and silently
+ // truncating larger exports; and leaving queueSize undefined lets it
+ // buffer partSize × 4 per upload, unbounded across concurrent callers.
+ // Peak memory is now partSize × queueSize, both caller-controlled.
+ await this.uploadFile({
+ fileName,
+ fileType,
+ data,
+ partSize: partSizeBytes,
+ queueSize:
+ maxConcurrentParts ?? env.LANGFUSE_S3_UPLOAD_MAX_CONCURRENT_PARTS,
+ });
return undefined;
}