Skip to content
TypeScript

fix(server): coalesce tool output before it fills the live buffer (#16317)

Alyx Buckmann693ac5f5df23
diff --git a/apps/server/src/orchestration-v2/LiveStreamBudget.test.ts b/apps/server/src/orchestration-v2/LiveStreamBudget.test.ts
index 39f31bfedc7..44e66412fcd 100644
--- a/apps/server/src/orchestration-v2/LiveStreamBudget.test.ts
+++ b/apps/server/src/orchestration-v2/LiveStreamBudget.test.ts
@@ -87,6 +87,63 @@ describe("bufferLatestLiveStream", () => {
});
describe("replayAndBufferProjectedLiveEvents", () => {
+ it.effect.each(["items", "bytes"] as const)(
+ "replays the undelivered tail after a %s overflow without losing or duplicating events",
+ (limit) =>
+ Effect.scoped(
+ Effect.gen(function* () {
+ const pubsub = yield* PubSub.unbounded<Event>();
+ const subscribed = yield* Deferred.make<PubSub.Subscription<Event>>();
+ const events = [
+ { sequence: 1, text: "one" },
+ { sequence: 2, text: "two" },
+ { sequence: 3, text: "three" },
+ ];
+ const limits = limit === "items" ? { maxItems: 1 } : { maxSerializedBytes: 40 };
+ const pull = yield* Stream.toPull(
+ replayAndBufferProjectedLiveEvents(
+ {
+ project: (event: Event) => event,
+ subscribe: PubSub.subscribe(pubsub).pipe(
+ Effect.tap((subscription) => Deferred.succeed(subscribed, subscription)),
+ ),
+ latestSequence: Effect.succeed(1),
+ replay: () => Stream.succeed(events[0]!),
+ },
+ limits,
+ ),
+ );
+ const received = [...(yield* pull)];
+ // Hold the ACK while more durable events arrive.
+ yield* PubSub.publishAll(pubsub, events.slice(1));
+ yield* (yield* Deferred.await(subscribed)).shutdownHook.await;
+ const overflow = yield* pull.pipe(Effect.result);
+ expect(overflow._tag).toBe("Failure");
+ if (overflow._tag === "Failure")
+ expect(overflow.failure._tag).toBe("LiveStreamBufferError");
+
+ const afterSequence = received.at(-1)!.sequence;
+ const resumed = yield* replayAndBufferProjectedLiveEvents(
+ {
+ project: (event: Event) => event,
+ subscribe: PubSub.subscribe(pubsub),
+ afterSequence,
+ latestSequence: Effect.succeed(3),
+ replay: (throughSequence) =>
+ Stream.fromIterable(
+ events.filter(
+ (event) => event.sequence > afterSequence && event.sequence <= throughSequence,
+ ),
+ ),
+ },
+ limits,
+ ).pipe(Stream.take(2), Stream.runCollect);
+ expect([...received, ...resumed]).toEqual(events);
+ expect(yield* PubSub.size(pubsub)).toBe(0);
+ }),
+ ),
+ );
+
it.effect.each(["high-water", "replay"] as const)(
"unsubscribes and cancels a blocked %s read on live overflow",
(phase) =>
diff --git a/apps/server/src/orchestration-v2/ThreadLiveEventCoalescer.test.ts b/apps/server/src/orchestration-v2/ThreadLiveEventCoalescer.test.ts
index 294ed0622d6..7b91e479b98 100644
--- a/apps/server/src/orchestration-v2/ThreadLiveEventCoalescer.test.ts
+++ b/apps/server/src/orchestration-v2/ThreadLiveEventCoalescer.test.ts
@@ -36,6 +36,7 @@ function makeToolActivity(
readonly kind?: "tool.updated" | "tool.completed";
readonly toolCallId?: string;
readonly turnId?: RunId;
+ readonly output?: string;
} = {},
): ThreadEvent {
const itemRunId = options.turnId ?? runId;
@@ -66,7 +67,7 @@ function makeToolActivity(
completedAt: status === "completed" ? now : null,
updatedAt: now,
input: "echo app.ts",
- output: `output-${sequence}`,
+ output: options.output ?? `output-${sequence}`,
},
},
};
@@ -155,6 +156,62 @@ function makeTextItem(
}
describe("ThreadLiveEventCoalescer", () => {
+ it.effect.each(["items", "bytes"] as const)(
+ "replaces superseded tool output before charging the %s budget",
+ (limit) =>
+ Effect.scoped(
+ Effect.gen(function* () {
+ const updates = Array.from({ length: 1_100 }, (_, index) => makeToolActivity(index + 1));
+ const latest = updates.at(-1)!;
+ const coalescer = yield* makeThreadLiveEventCoalescer(
+ limit === "items"
+ ? { maxItems: 1 }
+ : { maxSerializedBytes: Buffer.byteLength(encodeEvent(latest)) },
+ );
+ yield* coalescer.offerAll(updates);
+ expect(yield* coalescer.usage).toEqual({
+ retainedItems: 1,
+ retainedSerializedBytes: Buffer.byteLength(encodeEvent(latest)),
+ });
+ yield* coalescer.end;
+ expect(yield* coalescer.stream.pipe(Stream.runCollect)).toEqual([latest]);
+ expect(yield* coalescer.usage).toEqual({ retainedItems: 0, retainedSerializedBytes: 0 });
+ }),
+ ),
+ );
+
+ it.effect("handles a tool output burst larger than the default 8 MiB buffer", () =>
+ Effect.scoped(
+ Effect.gen(function* () {
+ const coalescer = yield* makeThreadLiveEventCoalescer();
+ const updates = Array.from({ length: 150 }, (_, index) =>
+ makeToolActivity(index + 1, { output: "x".repeat(64 * 1024) }),
+ );
+ yield* coalescer.offerAll(updates);
+ yield* coalescer.end;
+ expect(yield* coalescer.stream.pipe(Stream.runCollect)).toEqual([updates.at(-1)]);
+ }),
+ ),
+ );
+
+ it.effect("still fails when a single replacement exceeds the byte budget", () =>
+ Effect.scoped(
+ Effect.gen(function* () {
+ const first = makeToolActivity(1);
+ const coalescer = yield* makeThreadLiveEventCoalescer({
+ maxSerializedBytes: Buffer.byteLength(encodeEvent(first)),
+ });
+ yield* coalescer.offer(first);
+ const overflow = yield* coalescer
+ .offer(makeToolActivity(2, { output: "x".repeat(2_000) }))
+ .pipe(Effect.result);
+ expect(overflow._tag).toBe("Failure");
+ yield* coalescer.closed;
+ expect(yield* coalescer.usage).toEqual({ retainedItems: 0, retainedSerializedBytes: 0 });
+ }),
+ ),
+ );
+
it("preserves tool-call identity switches as insertion-order boundaries", () => {
const events = [
makeToolActivity(1, { toolCallId: "call-a" }),
@@ -399,8 +456,8 @@ describe("ThreadLiveEventCoalescer", () => {
yield* TestClock.adjust("40 millis");
yield* coalescer.offerAll(latest);
expect(yield* coalescer.usage).toEqual({
- retainedItems: 4,
- retainedSerializedBytes: serializedBytes([...first, ...latest]),
+ retainedItems: 2,
+ retainedSerializedBytes: serializedBytes(latest),
});
yield* TestClock.adjust("10 millis");
const expectedUsage = {
diff --git a/apps/server/src/orchestration-v2/ThreadLiveEventCoalescer.ts b/apps/server/src/orchestration-v2/ThreadLiveEventCoalescer.ts
index 00adbb0cb54..33c82dfe4ef 100644
--- a/apps/server/src/orchestration-v2/ThreadLiveEventCoalescer.ts
+++ b/apps/server/src/orchestration-v2/ThreadLiveEventCoalescer.ts
@@ -90,7 +90,9 @@ export const makeThreadLiveEventCoalescer = Effect.fn(
E | LiveStreamBufferError | Cause.Done
>();
const mutex = yield* Semaphore.make(1);
- let pendingUpdates: Array<RetainedLiveItem<ThreadLiveEvent>> = [];
+ const pendingUpdates = new Set<RetainedLiveItem<ThreadLiveEvent>>();
+ const latestUpdates = new Map<string, RetainedLiveItem<ThreadLiveEvent>>();
+ const pendingKeys = new Map<ThreadLiveEvent["event"]["type"], string>();
let windowGeneration = 0;
let windowFiber: Fiber.Fiber<void, never> | null = null;
let closed = false;
@@ -106,14 +108,13 @@ export const makeThreadLiveEventCoalescer = Effect.fn(
});
const flushPending = Effect.fnUntraced(function* () {
- if (pendingUpdates.length === 0) {
+ if (pendingUpdates.size === 0) {
return;
}
- const items = yield* budget.replace(
- pendingUpdates,
- coalesceLiveUpdatedEvents(pendingUpdates.map((item) => item.value)),
- );
- pendingUpdates = [];
+ const items = Array.from(pendingUpdates);
+ pendingUpdates.clear();
+ latestUpdates.clear();
+ pendingKeys.clear();
yield* Queue.offerAll(output, items);
}, Effect.uninterruptible);
@@ -131,7 +132,6 @@ export const makeThreadLiveEventCoalescer = Effect.fn(
}
}),
),
- Effect.catchTags({ LiveStreamBufferError: () => Effect.void }),
);
// Keep each source batch together so a synchronization marker cannot pass
@@ -143,18 +143,28 @@ export const makeThreadLiveEventCoalescer = Effect.fn(
(input) =>
Effect.gen(function* () {
yield* budget.check;
- if (input.kind === "event") {
- yield* budget.retain(input).pipe(
- Effect.tap((item) => Effect.sync(() => pendingUpdates.push(item))),
+ const startsWindow = pendingUpdates.size === 0;
+ const key = input.kind === "event" ? coalescibleUpdateKey(input) : null;
+ if (input.kind === "event" && key !== null) {
+ const pendingKey = pendingKeys.get(input.event.type);
+ if (pendingKey !== undefined && pendingKey !== key) latestUpdates.clear();
+ pendingKeys.set(input.event.type, key);
+ const previous = latestUpdates.get(key);
+ yield* budget.replace(previous ? [previous] : [], [input]).pipe(
+ Effect.tap(([item]) =>
+ Effect.sync(() => {
+ if (previous) pendingUpdates.delete(previous);
+ pendingUpdates.add(item!);
+ latestUpdates.set(key, item!);
+ }),
+ ),
Effect.uninterruptible,
);
- }
- if (input.kind === "event" && coalescibleUpdateKey(input) !== null) {
- if (pendingUpdates.length === 1) {
+ if (startsWindow) {
const generation = ++windowGeneration;
windowFiber = yield* Effect.forkIn(flushWindow(generation), coalescerScope);
}
- if (pendingUpdates.length >= MAX_PENDING_UPDATES) {
+ if (pendingUpdates.size >= MAX_PENDING_UPDATES) {
yield* cancelWindow();
windowGeneration += 1;
yield* flushPending();
@@ -162,6 +172,12 @@ export const makeThreadLiveEventCoalescer = Effect.fn(
return;
}
+ if (input.kind === "event") {
+ yield* budget.retain(input).pipe(
+ Effect.tap((item) => Effect.sync(() => pendingUpdates.add(item))),
+ Effect.uninterruptible,
+ );
+ }
yield* cancelWindow();
windowGeneration += 1;
yield* flushPending();
@@ -187,7 +203,9 @@ export const makeThreadLiveEventCoalescer = Effect.fn(
windowGeneration += 1;
yield* cancelWindow();
budget.release(pendingUpdates);
- pendingUpdates = [];
+ pendingUpdates.clear();
+ latestUpdates.clear();
+ pendingKeys.clear();
budget.release(yield* Queue.clear(output).pipe(Effect.orElseSucceed(() => [])));
if (cause) {
yield* Queue.failCause(output, cause);