TypeScript
fix(server): coalesce tool output before it fills the live buffer (#16317)
Alyx Buckmann693ac5f5df23
diff --git a/apps/server/src/orchestration-v2/LiveStreamBudget.test.ts b/apps/server/src/orchestration-v2/LiveStreamBudget.test.tsindex 39f31bfedc7..44e66412fcd 100644--- a/apps/server/src/orchestration-v2/LiveStreamBudget.test.ts+++ b/apps/server/src/orchestration-v2/LiveStreamBudget.test.ts@@ -87,6 +87,63 @@ describe("bufferLatestLiveStream", () => {});describe("replayAndBufferProjectedLiveEvents", () => {+ it.effect.each(["items", "bytes"] as const)(+ "replays the undelivered tail after a %s overflow without losing or duplicating events",+ (limit) =>+ Effect.scoped(+ Effect.gen(function* () {+ const pubsub = yield* PubSub.unbounded<Event>();+ const subscribed = yield* Deferred.make<PubSub.Subscription<Event>>();+ const events = [+ { sequence: 1, text: "one" },+ { sequence: 2, text: "two" },+ { sequence: 3, text: "three" },+ ];+ const limits = limit === "items" ? { maxItems: 1 } : { maxSerializedBytes: 40 };+ const pull = yield* Stream.toPull(+ replayAndBufferProjectedLiveEvents(+ {+ project: (event: Event) => event,+ subscribe: PubSub.subscribe(pubsub).pipe(+ Effect.tap((subscription) => Deferred.succeed(subscribed, subscription)),+ ),+ latestSequence: Effect.succeed(1),+ replay: () => Stream.succeed(events[0]!),+ },+ limits,+ ),+ );+ const received = [...(yield* pull)];+ // Hold the ACK while more durable events arrive.+ yield* PubSub.publishAll(pubsub, events.slice(1));+ yield* (yield* Deferred.await(subscribed)).shutdownHook.await;+ const overflow = yield* pull.pipe(Effect.result);+ expect(overflow._tag).toBe("Failure");+ if (overflow._tag === "Failure")+ expect(overflow.failure._tag).toBe("LiveStreamBufferError");++ const afterSequence = received.at(-1)!.sequence;+ const resumed = yield* replayAndBufferProjectedLiveEvents(+ {+ project: (event: Event) => event,+ subscribe: PubSub.subscribe(pubsub),+ afterSequence,+ latestSequence: Effect.succeed(3),+ replay: (throughSequence) =>+ Stream.fromIterable(+ events.filter(+ (event) => event.sequence > afterSequence && event.sequence <= throughSequence,+ ),+ ),+ },+ limits,+ ).pipe(Stream.take(2), Stream.runCollect);+ expect([...received, ...resumed]).toEqual(events);+ expect(yield* PubSub.size(pubsub)).toBe(0);+ }),+ ),+ );+it.effect.each(["high-water", "replay"] as const)("unsubscribes and cancels a blocked %s read on live overflow",(phase) =>diff --git a/apps/server/src/orchestration-v2/ThreadLiveEventCoalescer.test.ts b/apps/server/src/orchestration-v2/ThreadLiveEventCoalescer.test.tsindex 294ed0622d6..7b91e479b98 100644--- a/apps/server/src/orchestration-v2/ThreadLiveEventCoalescer.test.ts+++ b/apps/server/src/orchestration-v2/ThreadLiveEventCoalescer.test.ts@@ -36,6 +36,7 @@ function makeToolActivity(readonly kind?: "tool.updated" | "tool.completed";readonly toolCallId?: string;readonly turnId?: RunId;+ readonly output?: string;} = {},): ThreadEvent {const itemRunId = options.turnId ?? runId;@@ -66,7 +67,7 @@ function makeToolActivity(completedAt: status === "completed" ? now : null,updatedAt: now,input: "echo app.ts",- output: `output-${sequence}`,+ output: options.output ?? `output-${sequence}`,},},};@@ -155,6 +156,62 @@ function makeTextItem(}describe("ThreadLiveEventCoalescer", () => {+ it.effect.each(["items", "bytes"] as const)(+ "replaces superseded tool output before charging the %s budget",+ (limit) =>+ Effect.scoped(+ Effect.gen(function* () {+ const updates = Array.from({ length: 1_100 }, (_, index) => makeToolActivity(index + 1));+ const latest = updates.at(-1)!;+ const coalescer = yield* makeThreadLiveEventCoalescer(+ limit === "items"+ ? { maxItems: 1 }+ : { maxSerializedBytes: Buffer.byteLength(encodeEvent(latest)) },+ );+ yield* coalescer.offerAll(updates);+ expect(yield* coalescer.usage).toEqual({+ retainedItems: 1,+ retainedSerializedBytes: Buffer.byteLength(encodeEvent(latest)),+ });+ yield* coalescer.end;+ expect(yield* coalescer.stream.pipe(Stream.runCollect)).toEqual([latest]);+ expect(yield* coalescer.usage).toEqual({ retainedItems: 0, retainedSerializedBytes: 0 });+ }),+ ),+ );++ it.effect("handles a tool output burst larger than the default 8 MiB buffer", () =>+ Effect.scoped(+ Effect.gen(function* () {+ const coalescer = yield* makeThreadLiveEventCoalescer();+ const updates = Array.from({ length: 150 }, (_, index) =>+ makeToolActivity(index + 1, { output: "x".repeat(64 * 1024) }),+ );+ yield* coalescer.offerAll(updates);+ yield* coalescer.end;+ expect(yield* coalescer.stream.pipe(Stream.runCollect)).toEqual([updates.at(-1)]);+ }),+ ),+ );++ it.effect("still fails when a single replacement exceeds the byte budget", () =>+ Effect.scoped(+ Effect.gen(function* () {+ const first = makeToolActivity(1);+ const coalescer = yield* makeThreadLiveEventCoalescer({+ maxSerializedBytes: Buffer.byteLength(encodeEvent(first)),+ });+ yield* coalescer.offer(first);+ const overflow = yield* coalescer+ .offer(makeToolActivity(2, { output: "x".repeat(2_000) }))+ .pipe(Effect.result);+ expect(overflow._tag).toBe("Failure");+ yield* coalescer.closed;+ expect(yield* coalescer.usage).toEqual({ retainedItems: 0, retainedSerializedBytes: 0 });+ }),+ ),+ );+it("preserves tool-call identity switches as insertion-order boundaries", () => {const events = [makeToolActivity(1, { toolCallId: "call-a" }),@@ -399,8 +456,8 @@ describe("ThreadLiveEventCoalescer", () => {yield* TestClock.adjust("40 millis");yield* coalescer.offerAll(latest);expect(yield* coalescer.usage).toEqual({- retainedItems: 4,- retainedSerializedBytes: serializedBytes([...first, ...latest]),+ retainedItems: 2,+ retainedSerializedBytes: serializedBytes(latest),});yield* TestClock.adjust("10 millis");const expectedUsage = {diff --git a/apps/server/src/orchestration-v2/ThreadLiveEventCoalescer.ts b/apps/server/src/orchestration-v2/ThreadLiveEventCoalescer.tsindex 00adbb0cb54..33c82dfe4ef 100644--- a/apps/server/src/orchestration-v2/ThreadLiveEventCoalescer.ts+++ b/apps/server/src/orchestration-v2/ThreadLiveEventCoalescer.ts@@ -90,7 +90,9 @@ export const makeThreadLiveEventCoalescer = Effect.fn(E | LiveStreamBufferError | Cause.Done>();const mutex = yield* Semaphore.make(1);- let pendingUpdates: Array<RetainedLiveItem<ThreadLiveEvent>> = [];+ const pendingUpdates = new Set<RetainedLiveItem<ThreadLiveEvent>>();+ const latestUpdates = new Map<string, RetainedLiveItem<ThreadLiveEvent>>();+ const pendingKeys = new Map<ThreadLiveEvent["event"]["type"], string>();let windowGeneration = 0;let windowFiber: Fiber.Fiber<void, never> | null = null;let closed = false;@@ -106,14 +108,13 @@ export const makeThreadLiveEventCoalescer = Effect.fn(});const flushPending = Effect.fnUntraced(function* () {- if (pendingUpdates.length === 0) {+ if (pendingUpdates.size === 0) {return;}- const items = yield* budget.replace(- pendingUpdates,- coalesceLiveUpdatedEvents(pendingUpdates.map((item) => item.value)),- );- pendingUpdates = [];+ const items = Array.from(pendingUpdates);+ pendingUpdates.clear();+ latestUpdates.clear();+ pendingKeys.clear();yield* Queue.offerAll(output, items);}, Effect.uninterruptible);@@ -131,7 +132,6 @@ export const makeThreadLiveEventCoalescer = Effect.fn(}}),),- Effect.catchTags({ LiveStreamBufferError: () => Effect.void }),);// Keep each source batch together so a synchronization marker cannot pass@@ -143,18 +143,28 @@ export const makeThreadLiveEventCoalescer = Effect.fn((input) =>Effect.gen(function* () {yield* budget.check;- if (input.kind === "event") {- yield* budget.retain(input).pipe(- Effect.tap((item) => Effect.sync(() => pendingUpdates.push(item))),+ const startsWindow = pendingUpdates.size === 0;+ const key = input.kind === "event" ? coalescibleUpdateKey(input) : null;+ if (input.kind === "event" && key !== null) {+ const pendingKey = pendingKeys.get(input.event.type);+ if (pendingKey !== undefined && pendingKey !== key) latestUpdates.clear();+ pendingKeys.set(input.event.type, key);+ const previous = latestUpdates.get(key);+ yield* budget.replace(previous ? [previous] : [], [input]).pipe(+ Effect.tap(([item]) =>+ Effect.sync(() => {+ if (previous) pendingUpdates.delete(previous);+ pendingUpdates.add(item!);+ latestUpdates.set(key, item!);+ }),+ ),Effect.uninterruptible,);- }- if (input.kind === "event" && coalescibleUpdateKey(input) !== null) {- if (pendingUpdates.length === 1) {+ if (startsWindow) {const generation = ++windowGeneration;windowFiber = yield* Effect.forkIn(flushWindow(generation), coalescerScope);}- if (pendingUpdates.length >= MAX_PENDING_UPDATES) {+ if (pendingUpdates.size >= MAX_PENDING_UPDATES) {yield* cancelWindow();windowGeneration += 1;yield* flushPending();@@ -162,6 +172,12 @@ export const makeThreadLiveEventCoalescer = Effect.fn(return;}+ if (input.kind === "event") {+ yield* budget.retain(input).pipe(+ Effect.tap((item) => Effect.sync(() => pendingUpdates.add(item))),+ Effect.uninterruptible,+ );+ }yield* cancelWindow();windowGeneration += 1;yield* flushPending();@@ -187,7 +203,9 @@ export const makeThreadLiveEventCoalescer = Effect.fn(windowGeneration += 1;yield* cancelWindow();budget.release(pendingUpdates);- pendingUpdates = [];+ pendingUpdates.clear();+ latestUpdates.clear();+ pendingKeys.clear();budget.release(yield* Queue.clear(output).pipe(Effect.orElseSucceed(() => [])));if (cause) {yield* Queue.failCause(output, cause);