🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
fix(shared): Revert emit equal-sized S3 multipart parts from buffered uploader (#16967)
Niklas Semmler2e79f0b5ee9e
diff --git a/packages/shared/src/server/services/BufferedStreamUploader.ts b/packages/shared/src/server/services/BufferedStreamUploader.tsindex a455e9733d..394ad6521c 100644--- a/packages/shared/src/server/services/BufferedStreamUploader.ts+++ b/packages/shared/src/server/services/BufferedStreamUploader.ts@@ -94,12 +94,7 @@ export class BufferedStreamUploader {private readonly stats: UploadPartStats;constructor(params: BufferedStreamUploaderParams) {- // Buffer.concat rejects a non-integer length. Callers can pass- // MiB * 1024 * 1024 from a fractional env value (e.g. 5.1).- this.params = {- ...params,- partSizeBytes: Math.floor(params.partSizeBytes),- };+ this.params = params;this.stats = params.stats ?? emptyUploadPartStats();}@@ -123,20 +118,15 @@ export class BufferedStreamUploader {this.currentBuffer.push(buf);this.currentBufferSize += buf.byteLength;- // Slice exact partSizeBytes parts and carry the remainder. Some- // S3-compatible endpoints reject complete-multipart when non-trailing- // parts differ in length.- while (- this.currentBufferSize >= this.params.partSizeBytes &&- !this.errors.hasError()- ) {- await this.flushExactPart();+ if (this.currentBufferSize >= this.params.partSizeBytes) {+ await this.flushBuffer();+ if (this.errors.hasError()) break;}}- // Final part is the only one allowed to be smaller than partSizeBytes.+ // Flush remaining dataif (this.currentBufferSize > 0 && !this.errors.hasError()) {- await this.flushRemainder();+ await this.flushBuffer();}// Wait for all in-flight uploads to complete@@ -178,34 +168,10 @@ export class BufferedStreamUploader {return this.stats;}- private async flushExactPart(): Promise<void> {- await this.enqueuePart(this.takeBytes(this.params.partSizeBytes));- }-- private async flushRemainder(): Promise<void> {- await this.enqueuePart(this.takeBytes(this.currentBufferSize));- }-- // Copies the first `byteLength` bytes out of currentBuffer and leaves any- // leftover as the new buffer. Caller must pass byteLength <= currentBufferSize.- private takeBytes(byteLength: number): Buffer {- const partData = Buffer.concat(this.currentBuffer, byteLength);- let remaining = byteLength;- while (remaining > 0) {- const chunk = this.currentBuffer[0];- if (chunk.byteLength <= remaining) {- remaining -= chunk.byteLength;- this.currentBuffer.shift();- } else {- this.currentBuffer[0] = chunk.subarray(remaining);- remaining = 0;- }- }- this.currentBufferSize -= byteLength;- return partData;- }-- private async enqueuePart(partData: Buffer): Promise<void> {+ private async flushBuffer(): Promise<void> {+ const partData = Buffer.concat(this.currentBuffer);+ this.currentBuffer = [];+ this.currentBufferSize = 0;this.partNumber++;// Wait for a slot if all concurrent slots are fulldiff --git a/worker/src/__tests__/bufferedStreamUploader.test.ts b/worker/src/__tests__/bufferedStreamUploader.test.tsindex ffe828551f..32e52417a8 100644--- a/worker/src/__tests__/bufferedStreamUploader.test.ts+++ b/worker/src/__tests__/bufferedStreamUploader.test.ts@@ -125,62 +125,6 @@ describe("BufferedStreamUploader", () => {expect(mock.uploadedParts[0].data.byteLength).toBe(10);expect(mock.uploadedParts[1].data.byteLength).toBe(5);});-- it("should emit exact partSizeBytes for every non-final part when chunks do not divide evenly", async () => {- const mock = createMockStrategy();- const uploader = new BufferedStreamUploader({- ...defaultParams(mock.strategy),- partSizeBytes: 10,- });-- // 7+6=13 and 3+9=12: flushing the whole buffer would emit unequal- // non-final parts. Slice to exact partSizeBytes instead.- const chunks = ["1234567", "abcdef", "xyz", "123456789"];- await uploader.upload(streamFrom(chunks));-- expect(mock.uploadedParts).toHaveLength(3);- expect(mock.uploadedParts[0].data.byteLength).toBe(10);- expect(mock.uploadedParts[1].data.byteLength).toBe(10);- expect(mock.uploadedParts[2].data.byteLength).toBe(5);- expect(- Buffer.concat(mock.uploadedParts.map((p) => p.data)).toString(),- ).toBe(chunks.join(""));- });-- it("should coerce a fractional partSizeBytes to an integer byte length", async () => {- const mock = createMockStrategy();- const uploader = new BufferedStreamUploader({- ...defaultParams(mock.strategy),- partSizeBytes: 10.5,- });-- // 15 bytes: floor(10.5)=10 full part + 5-byte remainder. A float- // part size would throw in Buffer.concat before this change.- await uploader.upload(streamFrom(["aaaaaaaaaa", "bbbbb"]));-- expect(mock.uploadedParts).toHaveLength(2);- expect(mock.uploadedParts[0].data.byteLength).toBe(10);- expect(mock.uploadedParts[1].data.byteLength).toBe(5);- });-- it("should slice a single oversized chunk into exact partSizeBytes parts", async () => {- const mock = createMockStrategy();- const uploader = new BufferedStreamUploader({- ...defaultParams(mock.strategy),- partSizeBytes: 10,- });-- const largeChunk = "a".repeat(25);- await uploader.upload(streamFrom([largeChunk]));-- expect(mock.uploadedParts).toHaveLength(3);- expect(mock.uploadedParts[0].data.byteLength).toBe(10);- expect(mock.uploadedParts[1].data.byteLength).toBe(10);- expect(mock.uploadedParts[2].data.byteLength).toBe(5);- expect(- Buffer.concat(mock.uploadedParts.map((p) => p.data)).toString(),- ).toBe(largeChunk);- });});describe("empty stream", () => {