Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

fix(shared): Revert emit equal-sized S3 multipart parts from buffered uploader (#16967)

Niklas Semmler2e79f0b5ee9e
diff --git a/packages/shared/src/server/services/BufferedStreamUploader.ts b/packages/shared/src/server/services/BufferedStreamUploader.ts
index a455e9733d..394ad6521c 100644
--- a/packages/shared/src/server/services/BufferedStreamUploader.ts
+++ b/packages/shared/src/server/services/BufferedStreamUploader.ts
@@ -94,12 +94,7 @@ export class BufferedStreamUploader {
private readonly stats: UploadPartStats;
constructor(params: BufferedStreamUploaderParams) {
- // Buffer.concat rejects a non-integer length. Callers can pass
- // MiB * 1024 * 1024 from a fractional env value (e.g. 5.1).
- this.params = {
- ...params,
- partSizeBytes: Math.floor(params.partSizeBytes),
- };
+ this.params = params;
this.stats = params.stats ?? emptyUploadPartStats();
}
@@ -123,20 +118,15 @@ export class BufferedStreamUploader {
this.currentBuffer.push(buf);
this.currentBufferSize += buf.byteLength;
- // Slice exact partSizeBytes parts and carry the remainder. Some
- // S3-compatible endpoints reject complete-multipart when non-trailing
- // parts differ in length.
- while (
- this.currentBufferSize >= this.params.partSizeBytes &&
- !this.errors.hasError()
- ) {
- await this.flushExactPart();
+ if (this.currentBufferSize >= this.params.partSizeBytes) {
+ await this.flushBuffer();
+ if (this.errors.hasError()) break;
}
}
- // Final part is the only one allowed to be smaller than partSizeBytes.
+ // Flush remaining data
if (this.currentBufferSize > 0 && !this.errors.hasError()) {
- await this.flushRemainder();
+ await this.flushBuffer();
}
// Wait for all in-flight uploads to complete
@@ -178,34 +168,10 @@ export class BufferedStreamUploader {
return this.stats;
}
- private async flushExactPart(): Promise<void> {
- await this.enqueuePart(this.takeBytes(this.params.partSizeBytes));
- }
-
- private async flushRemainder(): Promise<void> {
- await this.enqueuePart(this.takeBytes(this.currentBufferSize));
- }
-
- // Copies the first `byteLength` bytes out of currentBuffer and leaves any
- // leftover as the new buffer. Caller must pass byteLength <= currentBufferSize.
- private takeBytes(byteLength: number): Buffer {
- const partData = Buffer.concat(this.currentBuffer, byteLength);
- let remaining = byteLength;
- while (remaining > 0) {
- const chunk = this.currentBuffer[0];
- if (chunk.byteLength <= remaining) {
- remaining -= chunk.byteLength;
- this.currentBuffer.shift();
- } else {
- this.currentBuffer[0] = chunk.subarray(remaining);
- remaining = 0;
- }
- }
- this.currentBufferSize -= byteLength;
- return partData;
- }
-
- private async enqueuePart(partData: Buffer): Promise<void> {
+ private async flushBuffer(): Promise<void> {
+ const partData = Buffer.concat(this.currentBuffer);
+ this.currentBuffer = [];
+ this.currentBufferSize = 0;
this.partNumber++;
// Wait for a slot if all concurrent slots are full
diff --git a/worker/src/__tests__/bufferedStreamUploader.test.ts b/worker/src/__tests__/bufferedStreamUploader.test.ts
index ffe828551f..32e52417a8 100644
--- a/worker/src/__tests__/bufferedStreamUploader.test.ts
+++ b/worker/src/__tests__/bufferedStreamUploader.test.ts
@@ -125,62 +125,6 @@ describe("BufferedStreamUploader", () => {
expect(mock.uploadedParts[0].data.byteLength).toBe(10);
expect(mock.uploadedParts[1].data.byteLength).toBe(5);
});
-
- it("should emit exact partSizeBytes for every non-final part when chunks do not divide evenly", async () => {
- const mock = createMockStrategy();
- const uploader = new BufferedStreamUploader({
- ...defaultParams(mock.strategy),
- partSizeBytes: 10,
- });
-
- // 7+6=13 and 3+9=12: flushing the whole buffer would emit unequal
- // non-final parts. Slice to exact partSizeBytes instead.
- const chunks = ["1234567", "abcdef", "xyz", "123456789"];
- await uploader.upload(streamFrom(chunks));
-
- expect(mock.uploadedParts).toHaveLength(3);
- expect(mock.uploadedParts[0].data.byteLength).toBe(10);
- expect(mock.uploadedParts[1].data.byteLength).toBe(10);
- expect(mock.uploadedParts[2].data.byteLength).toBe(5);
- expect(
- Buffer.concat(mock.uploadedParts.map((p) => p.data)).toString(),
- ).toBe(chunks.join(""));
- });
-
- it("should coerce a fractional partSizeBytes to an integer byte length", async () => {
- const mock = createMockStrategy();
- const uploader = new BufferedStreamUploader({
- ...defaultParams(mock.strategy),
- partSizeBytes: 10.5,
- });
-
- // 15 bytes: floor(10.5)=10 full part + 5-byte remainder. A float
- // part size would throw in Buffer.concat before this change.
- await uploader.upload(streamFrom(["aaaaaaaaaa", "bbbbb"]));
-
- expect(mock.uploadedParts).toHaveLength(2);
- expect(mock.uploadedParts[0].data.byteLength).toBe(10);
- expect(mock.uploadedParts[1].data.byteLength).toBe(5);
- });
-
- it("should slice a single oversized chunk into exact partSizeBytes parts", async () => {
- const mock = createMockStrategy();
- const uploader = new BufferedStreamUploader({
- ...defaultParams(mock.strategy),
- partSizeBytes: 10,
- });
-
- const largeChunk = "a".repeat(25);
- await uploader.upload(streamFrom([largeChunk]));
-
- expect(mock.uploadedParts).toHaveLength(3);
- expect(mock.uploadedParts[0].data.byteLength).toBe(10);
- expect(mock.uploadedParts[1].data.byteLength).toBe(10);
- expect(mock.uploadedParts[2].data.byteLength).toBe(5);
- expect(
- Buffer.concat(mock.uploadedParts.map((p) => p.data)).toString(),
- ).toBe(largeChunk);
- });
});
describe("empty stream", () => {