🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
fix(blob-storage): raise S3 multipart part size to lift 48.8 GiB object cap (#17303)
Niklas Semmlerf27d64754dcf
diff --git a/packages/shared/src/server/services/StorageService.test.ts b/packages/shared/src/server/services/StorageService.test.tsindex c7f838544e..fe17be82a0 100644--- a/packages/shared/src/server/services/StorageService.test.ts+++ b/packages/shared/src/server/services/StorageService.test.ts@@ -10,6 +10,23 @@ import { env } from "../../env";import { resolveMediaStorageEndpoints } from "../s3";import { StorageServiceFactory } from "./StorageService";+// Capture the options handed to lib-storage's Upload so the non-buffered+// fallback's part size can be asserted without real network I/O.+const s3UploadCtorOptions = vi.hoisted(+ () => [] as Array<{ partSize?: number; queueSize?: number }>,+);++vi.mock("@aws-sdk/lib-storage", () => ({+ Upload: class {+ constructor(options: { partSize?: number; queueSize?: number }) {+ s3UploadCtorOptions.push(options);+ }+ done() {+ return Promise.resolve(undefined);+ }+ },+}));+describe("S3StorageService region normalization", () => {it("trims a persisted region before configuring the AWS client", async () => {const service = StorageServiceFactory.getInstance({@@ -529,3 +546,62 @@ describe("S3StorageService DeleteObjects checksum", () => {expect(findHeader(request, "x-amz-checksum-crc32")).toBeUndefined();});});++describe("S3StorageService non-buffered upload part size", () => {+ const bufferedKey = "LANGFUSE_S3_UPLOAD_ENABLE_BUFFERED" as const;+ const originalBuffered = env[bufferedKey];++ afterEach(() => {+ (env as Record<string, unknown>)[bufferedKey] = originalBuffered;+ s3UploadCtorOptions.length = 0;+ });++ const makeService = () =>+ StorageServiceFactory.getInstance({+ accessKeyId: "test-access-key",+ secretAccessKey: "test-secret-key",+ bucketName: "test-bucket",+ endpoint: "http://127.0.0.1:9000",+ region: "us-east-1",+ forcePathStyle: true,+ useAzureBlob: false,+ useGoogleCloudStorage: false,+ useOCIObjectStorage: false,+ awsSse: undefined,+ awsSseKmsKeyId: undefined,+ });++ it("forwards the caller's part size so the 10k-part cap does not truncate large exports", async () => {+ (env as Record<string, unknown>)[bufferedKey] = "false";++ await makeService().uploadFileBuffered({+ fileName: "export.csv",+ fileType: "text/csv",+ data: Readable.from(["a,b,c\n"]),+ partSizeBytes: 100 * 1024 * 1024,+ });++ expect(s3UploadCtorOptions).toHaveLength(1);+ const { partSize } = s3UploadCtorOptions[0];+ expect(partSize).toBe(100 * 1024 * 1024);+ // Above lib-storage's 5 MiB default, which caps a single object at ~48.83 GiB.+ expect(partSize).toBeGreaterThan(5 * 1024 * 1024);+ });++ it("caps concurrency so parallel callers do not multiply peak buffer memory", async () => {+ (env as Record<string, unknown>)[bufferedKey] = "false";++ await makeService().uploadFileBuffered({+ fileName: "export.csv",+ fileType: "text/csv",+ data: Readable.from(["a,b,c\n"]),+ partSizeBytes: 100 * 1024 * 1024,+ maxConcurrentParts: 2,+ });++ expect(s3UploadCtorOptions).toHaveLength(1);+ // Caller's concurrency is honored instead of lib-storage's default of 4,+ // which would otherwise buffer partSize x 4 per concurrent upload.+ expect(s3UploadCtorOptions[0].queueSize).toBe(2);+ });+});diff --git a/packages/shared/src/server/services/StorageService.ts b/packages/shared/src/server/services/StorageService.tsindex 7f73629977..b2deafb21c 100644--- a/packages/shared/src/server/services/StorageService.ts+++ b/packages/shared/src/server/services/StorageService.ts@@ -836,9 +836,9 @@ class S3StorageService implements StorageService {Body: data,ContentType: fileType,}),- // Use provided partSize and queueSize, or fall back to defaults- // Default: 5 MB part size supports files up to ~50 GB (5 MB × 10,000 parts)- // For large files, use partSize: 100 * 1024 * 1024 (100 MB) to support up to ~1 TB+ // When partSize is undefined lib-storage falls back to 5 MiB, capping a+ // single object at ~48.83 GiB (5 MiB × 10,000 parts). Callers uploading+ // large objects must pass an explicit partSize to raise that ceiling.partSize: partSize,queueSize: queueSize,}).done();@@ -860,10 +860,20 @@ class S3StorageService implements StorageService {stats,}: UploadFileBuffered): Promise<UploadPartStats | undefined> {if (env.LANGFUSE_S3_UPLOAD_ENABLE_BUFFERED !== "true") {- // Tuning applies only on the buffered path. Forward no overrides so the- // fallback keeps lib-storage's defaults — forwarding the resolved 100 MiB- // partSize would ~20x per-upload memory (buffered is off by default).- await this.uploadFile({ fileName, fileType, data });+ // Forward the caller's own part size and concurrency instead of a global+ // default. lib-storage otherwise falls back to 5 MiB parts, capping a+ // single object at ~48.83 GiB (5 MiB × 10,000 parts) and silently+ // truncating larger exports; and leaving queueSize undefined lets it+ // buffer partSize × 4 per upload, unbounded across concurrent callers.+ // Peak memory is now partSize × queueSize, both caller-controlled.+ await this.uploadFile({+ fileName,+ fileType,+ data,+ partSize: partSizeBytes,+ queueSize:+ maxConcurrentParts ?? env.LANGFUSE_S3_UPLOAD_MAX_CONCURRENT_PARTS,+ });return undefined;}