Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

fix(shared): emit equal-sized S3 multipart parts from buffered uploader (#16754)

Niklas Semmler80998f4e0c9d
diff --git a/packages/shared/src/server/services/BufferedStreamUploader.ts b/packages/shared/src/server/services/BufferedStreamUploader.ts
index 394ad6521c..a455e9733d 100644
--- a/packages/shared/src/server/services/BufferedStreamUploader.ts
+++ b/packages/shared/src/server/services/BufferedStreamUploader.ts
@@ -94,7 +94,12 @@ export class BufferedStreamUploader {
private readonly stats: UploadPartStats;
constructor(params: BufferedStreamUploaderParams) {
- this.params = params;
+ // Buffer.concat rejects a non-integer length. Callers can pass
+ // MiB * 1024 * 1024 from a fractional env value (e.g. 5.1).
+ this.params = {
+ ...params,
+ partSizeBytes: Math.floor(params.partSizeBytes),
+ };
this.stats = params.stats ?? emptyUploadPartStats();
}
@@ -118,15 +123,20 @@ export class BufferedStreamUploader {
this.currentBuffer.push(buf);
this.currentBufferSize += buf.byteLength;
- if (this.currentBufferSize >= this.params.partSizeBytes) {
- await this.flushBuffer();
- if (this.errors.hasError()) break;
+ // Slice exact partSizeBytes parts and carry the remainder. Some
+ // S3-compatible endpoints reject complete-multipart when non-trailing
+ // parts differ in length.
+ while (
+ this.currentBufferSize >= this.params.partSizeBytes &&
+ !this.errors.hasError()
+ ) {
+ await this.flushExactPart();
}
}
- // Flush remaining data
+ // Final part is the only one allowed to be smaller than partSizeBytes.
if (this.currentBufferSize > 0 && !this.errors.hasError()) {
- await this.flushBuffer();
+ await this.flushRemainder();
}
// Wait for all in-flight uploads to complete
@@ -168,10 +178,34 @@ export class BufferedStreamUploader {
return this.stats;
}
- private async flushBuffer(): Promise<void> {
- const partData = Buffer.concat(this.currentBuffer);
- this.currentBuffer = [];
- this.currentBufferSize = 0;
+ private async flushExactPart(): Promise<void> {
+ await this.enqueuePart(this.takeBytes(this.params.partSizeBytes));
+ }
+
+ private async flushRemainder(): Promise<void> {
+ await this.enqueuePart(this.takeBytes(this.currentBufferSize));
+ }
+
+ // Copies the first `byteLength` bytes out of currentBuffer and leaves any
+ // leftover as the new buffer. Caller must pass byteLength <= currentBufferSize.
+ private takeBytes(byteLength: number): Buffer {
+ const partData = Buffer.concat(this.currentBuffer, byteLength);
+ let remaining = byteLength;
+ while (remaining > 0) {
+ const chunk = this.currentBuffer[0];
+ if (chunk.byteLength <= remaining) {
+ remaining -= chunk.byteLength;
+ this.currentBuffer.shift();
+ } else {
+ this.currentBuffer[0] = chunk.subarray(remaining);
+ remaining = 0;
+ }
+ }
+ this.currentBufferSize -= byteLength;
+ return partData;
+ }
+
+ private async enqueuePart(partData: Buffer): Promise<void> {
this.partNumber++;
// Wait for a slot if all concurrent slots are full
diff --git a/worker/src/__tests__/bufferedStreamUploader.test.ts b/worker/src/__tests__/bufferedStreamUploader.test.ts
index 32e52417a8..ffe828551f 100644
--- a/worker/src/__tests__/bufferedStreamUploader.test.ts
+++ b/worker/src/__tests__/bufferedStreamUploader.test.ts
@@ -125,6 +125,62 @@ describe("BufferedStreamUploader", () => {
expect(mock.uploadedParts[0].data.byteLength).toBe(10);
expect(mock.uploadedParts[1].data.byteLength).toBe(5);
});
+
+ it("should emit exact partSizeBytes for every non-final part when chunks do not divide evenly", async () => {
+ const mock = createMockStrategy();
+ const uploader = new BufferedStreamUploader({
+ ...defaultParams(mock.strategy),
+ partSizeBytes: 10,
+ });
+
+ // 7+6=13 and 3+9=12: flushing the whole buffer would emit unequal
+ // non-final parts. Slice to exact partSizeBytes instead.
+ const chunks = ["1234567", "abcdef", "xyz", "123456789"];
+ await uploader.upload(streamFrom(chunks));
+
+ expect(mock.uploadedParts).toHaveLength(3);
+ expect(mock.uploadedParts[0].data.byteLength).toBe(10);
+ expect(mock.uploadedParts[1].data.byteLength).toBe(10);
+ expect(mock.uploadedParts[2].data.byteLength).toBe(5);
+ expect(
+ Buffer.concat(mock.uploadedParts.map((p) => p.data)).toString(),
+ ).toBe(chunks.join(""));
+ });
+
+ it("should coerce a fractional partSizeBytes to an integer byte length", async () => {
+ const mock = createMockStrategy();
+ const uploader = new BufferedStreamUploader({
+ ...defaultParams(mock.strategy),
+ partSizeBytes: 10.5,
+ });
+
+ // 15 bytes: floor(10.5)=10 full part + 5-byte remainder. A float
+ // part size would throw in Buffer.concat before this change.
+ await uploader.upload(streamFrom(["aaaaaaaaaa", "bbbbb"]));
+
+ expect(mock.uploadedParts).toHaveLength(2);
+ expect(mock.uploadedParts[0].data.byteLength).toBe(10);
+ expect(mock.uploadedParts[1].data.byteLength).toBe(5);
+ });
+
+ it("should slice a single oversized chunk into exact partSizeBytes parts", async () => {
+ const mock = createMockStrategy();
+ const uploader = new BufferedStreamUploader({
+ ...defaultParams(mock.strategy),
+ partSizeBytes: 10,
+ });
+
+ const largeChunk = "a".repeat(25);
+ await uploader.upload(streamFrom([largeChunk]));
+
+ expect(mock.uploadedParts).toHaveLength(3);
+ expect(mock.uploadedParts[0].data.byteLength).toBe(10);
+ expect(mock.uploadedParts[1].data.byteLength).toBe(10);
+ expect(mock.uploadedParts[2].data.byteLength).toBe(5);
+ expect(
+ Buffer.concat(mock.uploadedParts.map((p) => p.data)).toString(),
+ ).toBe(largeChunk);
+ });
});
describe("empty stream", () => {