🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
fix(shared): emit equal-sized S3 multipart parts from buffered uploader (#16754)
Niklas Semmler80998f4e0c9d
diff --git a/packages/shared/src/server/services/BufferedStreamUploader.ts b/packages/shared/src/server/services/BufferedStreamUploader.tsindex 394ad6521c..a455e9733d 100644--- a/packages/shared/src/server/services/BufferedStreamUploader.ts+++ b/packages/shared/src/server/services/BufferedStreamUploader.ts@@ -94,7 +94,12 @@ export class BufferedStreamUploader {private readonly stats: UploadPartStats;constructor(params: BufferedStreamUploaderParams) {- this.params = params;+ // Buffer.concat rejects a non-integer length. Callers can pass+ // MiB * 1024 * 1024 from a fractional env value (e.g. 5.1).+ this.params = {+ ...params,+ partSizeBytes: Math.floor(params.partSizeBytes),+ };this.stats = params.stats ?? emptyUploadPartStats();}@@ -118,15 +123,20 @@ export class BufferedStreamUploader {this.currentBuffer.push(buf);this.currentBufferSize += buf.byteLength;- if (this.currentBufferSize >= this.params.partSizeBytes) {- await this.flushBuffer();- if (this.errors.hasError()) break;+ // Slice exact partSizeBytes parts and carry the remainder. Some+ // S3-compatible endpoints reject complete-multipart when non-trailing+ // parts differ in length.+ while (+ this.currentBufferSize >= this.params.partSizeBytes &&+ !this.errors.hasError()+ ) {+ await this.flushExactPart();}}- // Flush remaining data+ // Final part is the only one allowed to be smaller than partSizeBytes.if (this.currentBufferSize > 0 && !this.errors.hasError()) {- await this.flushBuffer();+ await this.flushRemainder();}// Wait for all in-flight uploads to complete@@ -168,10 +178,34 @@ export class BufferedStreamUploader {return this.stats;}- private async flushBuffer(): Promise<void> {- const partData = Buffer.concat(this.currentBuffer);- this.currentBuffer = [];- this.currentBufferSize = 0;+ private async flushExactPart(): Promise<void> {+ await this.enqueuePart(this.takeBytes(this.params.partSizeBytes));+ }++ private async flushRemainder(): Promise<void> {+ await this.enqueuePart(this.takeBytes(this.currentBufferSize));+ }++ // Copies the first `byteLength` bytes out of currentBuffer and leaves any+ // leftover as the new buffer. Caller must pass byteLength <= currentBufferSize.+ private takeBytes(byteLength: number): Buffer {+ const partData = Buffer.concat(this.currentBuffer, byteLength);+ let remaining = byteLength;+ while (remaining > 0) {+ const chunk = this.currentBuffer[0];+ if (chunk.byteLength <= remaining) {+ remaining -= chunk.byteLength;+ this.currentBuffer.shift();+ } else {+ this.currentBuffer[0] = chunk.subarray(remaining);+ remaining = 0;+ }+ }+ this.currentBufferSize -= byteLength;+ return partData;+ }++ private async enqueuePart(partData: Buffer): Promise<void> {this.partNumber++;// Wait for a slot if all concurrent slots are fulldiff --git a/worker/src/__tests__/bufferedStreamUploader.test.ts b/worker/src/__tests__/bufferedStreamUploader.test.tsindex 32e52417a8..ffe828551f 100644--- a/worker/src/__tests__/bufferedStreamUploader.test.ts+++ b/worker/src/__tests__/bufferedStreamUploader.test.ts@@ -125,6 +125,62 @@ describe("BufferedStreamUploader", () => {expect(mock.uploadedParts[0].data.byteLength).toBe(10);expect(mock.uploadedParts[1].data.byteLength).toBe(5);});++ it("should emit exact partSizeBytes for every non-final part when chunks do not divide evenly", async () => {+ const mock = createMockStrategy();+ const uploader = new BufferedStreamUploader({+ ...defaultParams(mock.strategy),+ partSizeBytes: 10,+ });++ // 7+6=13 and 3+9=12: flushing the whole buffer would emit unequal+ // non-final parts. Slice to exact partSizeBytes instead.+ const chunks = ["1234567", "abcdef", "xyz", "123456789"];+ await uploader.upload(streamFrom(chunks));++ expect(mock.uploadedParts).toHaveLength(3);+ expect(mock.uploadedParts[0].data.byteLength).toBe(10);+ expect(mock.uploadedParts[1].data.byteLength).toBe(10);+ expect(mock.uploadedParts[2].data.byteLength).toBe(5);+ expect(+ Buffer.concat(mock.uploadedParts.map((p) => p.data)).toString(),+ ).toBe(chunks.join(""));+ });++ it("should coerce a fractional partSizeBytes to an integer byte length", async () => {+ const mock = createMockStrategy();+ const uploader = new BufferedStreamUploader({+ ...defaultParams(mock.strategy),+ partSizeBytes: 10.5,+ });++ // 15 bytes: floor(10.5)=10 full part + 5-byte remainder. A float+ // part size would throw in Buffer.concat before this change.+ await uploader.upload(streamFrom(["aaaaaaaaaa", "bbbbb"]));++ expect(mock.uploadedParts).toHaveLength(2);+ expect(mock.uploadedParts[0].data.byteLength).toBe(10);+ expect(mock.uploadedParts[1].data.byteLength).toBe(5);+ });++ it("should slice a single oversized chunk into exact partSizeBytes parts", async () => {+ const mock = createMockStrategy();+ const uploader = new BufferedStreamUploader({+ ...defaultParams(mock.strategy),+ partSizeBytes: 10,+ });++ const largeChunk = "a".repeat(25);+ await uploader.upload(streamFrom([largeChunk]));++ expect(mock.uploadedParts).toHaveLength(3);+ expect(mock.uploadedParts[0].data.byteLength).toBe(10);+ expect(mock.uploadedParts[1].data.byteLength).toBe(10);+ expect(mock.uploadedParts[2].data.byteLength).toBe(5);+ expect(+ Buffer.concat(mock.uploadedParts.map((p) => p.data)).toString(),+ ).toBe(largeChunk);+ });});describe("empty stream", () => {