Multiplayer agent harness for work.
Keep long live turn streams updating with bounded tails (#2260)
Regan Bellcb9aed0a9dfa
diff --git a/plugins/web-ui/package-lock.json b/plugins/web-ui/package-lock.jsonindex 8bc68762b..357d39b73 100644--- a/plugins/web-ui/package-lock.json+++ b/plugins/web-ui/package-lock.json@@ -21,7 +21,6 @@"@mariozechner/mini-lit": "^0.2.0","@sentry/browser": "10.75.0","@sentry/node": "10.75.0",- "@tanstack/ai": "0.54.0","@tanstack/ai-client": "0.31.1","date-fns": "^4.4.0","dockview-core": "^7.0.4",diff --git a/plugins/web-ui/package.json b/plugins/web-ui/package.jsonindex 2273440f6..19a9f1dfa 100644--- a/plugins/web-ui/package.json+++ b/plugins/web-ui/package.json@@ -29,7 +29,6 @@"@mariozechner/mini-lit": "^0.2.0","@sentry/browser": "10.75.0","@sentry/node": "10.75.0",- "@tanstack/ai": "0.54.0","@tanstack/ai-client": "0.31.1","date-fns": "^4.4.0","dockview-core": "^7.0.4",diff --git a/plugins/web-ui/src/chat.ts b/plugins/web-ui/src/chat.tsindex dac83456b..4c09d1de3 100644--- a/plugins/web-ui/src/chat.ts+++ b/plugins/web-ui/src/chat.ts@@ -121,7 +121,7 @@ import {buildTimeline,messageWorkTimeline,workTimelineSegments,- streamingTextTail,+ streamedAnswer,currentTextPhase,postSpeechText,toolCategory,@@ -1887,10 +1887,11 @@ export function createChatSurface(let workView: TemplateResult | typeof nothing =work && steeringOnly.length ? html`${steeringOnly.map((item) => renderTimelineItem(item, work))}` : nothing;if (showWork) {+ const workText = isStreaming ? messageText(msg) : text;workView = workBlock(work,isStreaming,- msg.stopReason === "aborted" || msg.stopReason === "error" ? "" : text,+ msg.stopReason === "aborted" || msg.stopReason === "error" ? "" : workText,(msg as AssistantWork).streamingBaseline ?? "",);}@@ -2116,7 +2117,7 @@ export function createChatSurface(const streamingFinal = isStreaming && phase?.phase === "final_answer";const workActive =hasWork && isStreaming && !streamingFinal && (work?.status === "working" || work?.status === "thinking");- const text = streamingFinal ? chunk.text.slice(phase.streamOffset) : chunk.text;+ const text = isStreaming && work ? streamedAnswer(chunk.text, work) : chunk.text;for (const [partIndex, part] of setupContent(assistantDisplayText(workActive ? "" : text, message.stopReason),).entries()) {@@ -2143,7 +2144,7 @@ export function createChatSurface(if (body.trim())parts.push(html`<div class="streaming-text ${animating ? "live-stream" : ""}" dir="auto">- ${markdown(body, animating, streamingFinal ? ((message as AssistantWork).streamingBaseline ?? "").slice(phase.streamOffset) : ((message as AssistantWork).streamingBaseline ?? ""))}+ ${markdown(body, animating, isStreaming && work ? streamedAnswer((message as AssistantWork).streamingBaseline ?? "", work) : ((message as AssistantWork).streamingBaseline ?? ""))}</div>`,);for (const link of links) parts.push(connectorWidget(link));@@ -2816,7 +2817,7 @@ export function createChatSurface(replies.push(speech);return false;});- const tail = active && last ? streamingTextTail(text, work.activity) : "";+ const tail = active && last ? streamedAnswer(text, work) : "";const stopping = isStreaming && runSlot.stopGeneration === runSlot.generation;const animating = active && last && !stopping;let label = last ? workLabel(work) : "Worked";@@ -2877,7 +2878,7 @@ export function createChatSurface(},),)}- ${tail.trim() ? html`<div class="work-said streaming-text ${animating ? "live-stream" : ""}">${markdown(tail, animating, streamingTextTail(baseline, work.activity))}</div>` : nothing}+ ${tail.trim() ? html`<div class="work-said streaming-text ${animating ? "live-stream" : ""}">${markdown(tail, animating, streamedAnswer(baseline, work))}</div>` : nothing}</div></details>`: nothing;diff --git a/plugins/web-ui/src/core-bridge.ts b/plugins/web-ui/src/core-bridge.tsindex 5e0270e50..b9120d3c1 100644--- a/plugins/web-ui/src/core-bridge.ts+++ b/plugins/web-ui/src/core-bridge.ts@@ -1,8 +1,7 @@import { reportHandledError, reportRequestTiming, stopBrowserErrors } from "./browser-errors.ts";import { captureMessage, stopAnalytics } from "./product-analytics.ts";import { streamedAnswer } from "./timeline.ts";-import { EventType } from "@tanstack/ai/client";-import { fetchServerSentEvents, StreamProcessor } from "@tanstack/ai-client";+import { fetchServerSentEvents } from "@tanstack/ai-client";import { postCallText, postResultOk } from "./surface-post.ts";import type { ModelMetadata } from "./pi-models.ts";import { createAssistantMessageEventStream } from "@earendil-works/pi-ai";@@ -431,6 +430,7 @@ export interface ToolActivity {}type WorkStatus = "thinking" | "working" | "complete" | "failed";export interface WorkBlock {+ streamStartOffset?: number;status: WorkStatus;startedAt?: number;finishedAt?: number;@@ -498,6 +498,7 @@ export interface RunPoll {attachments?: Array<{ name: string; mimetype?: string; sizeBytes?: number; artifactId?: string }>;} | null;partial?: string;+ partialStartOffset?: number;alive?: boolean;stale?: boolean;replyComplete?: boolean;@@ -1078,7 +1079,7 @@ async function driveApproval(work.status = "complete";work.finishedAt = Date.now();notify();- finish(stream, partial, { acc: "", lastProgressAt: now() }, submit.reply ?? "");+ finish(stream, partial, { acc: "", startOffset: 0, lastProgressAt: now() }, submit.reply ?? "");} catch (e) {work.status = "failed";work.finishedAt = Date.now();@@ -1247,7 +1248,7 @@ async function drive(work.status = "complete";work.finishedAt = Date.now();notify();- finish(stream, partial, { acc: "", lastProgressAt: now() }, submit.reply ?? "");+ finish(stream, partial, { acc: "", startOffset: 0, lastProgressAt: now() }, submit.reply ?? "");} catch (e) {work.status = "failed";work.finishedAt = Date.now();@@ -1287,7 +1288,7 @@ async function resumeDrive(notify();stream.push({ type: "start", partial });stream.push({ type: "text_start", contentIndex: 0, partial });- const st: Acc = { acc: "", lastProgressAt: now() };+ const st: Acc = { acc: "", startOffset: 0, lastProgressAt: now() };// Keep any assistant text the transcript already showed for this in-flight// run visible: seed the accumulator with it so attaching the live stream// never blanks text the person has already read. The server's own partial@@ -1311,7 +1312,7 @@ async function followRun(runId: string,signal?: AbortSignal,notify?: () => void,- st: Acc = { acc: "", lastProgressAt: now() },+ st: Acc = { acc: "", startOffset: 0, lastProgressAt: now() },slot?: RunSlot,gen = 0,): Promise<void> {@@ -1341,6 +1342,7 @@ function runPath(runId: string, suffix: string): string {}export interface Acc {+ startOffset: number;acc: string;lastProgressAt: number;staleSince?: number;@@ -1411,8 +1413,12 @@ function applyRun(}setWorkStale(work, run.stale === true, notify);const p = typeof run.partial === "string" ? run.partial : "";- if (p.length > st.acc.length) {- st.lastProgressAt = now();+ const startOffset = run.partialStartOffset ?? 0;+ if (p && startOffset + p.length >= st.startOffset + st.acc.length) {+ if (startOffset + p.length > st.startOffset + st.acc.length) st.lastProgressAt = now();+ if (startOffset !== st.startOffset) (partial as AssistantWork).streamingBaseline = "";+ st.startOffset = startOffset;+ if (work) work.streamStartOffset = startOffset;pushDelta(stream, partial, st, p);}if (!runIsTerminal(run)) return "open";@@ -1573,16 +1579,6 @@ async function streamRunViaSse(clearTimeout(timer);timer = setTimeout(abort, SSE_SILENCE_MS);};- const processor = new StreamProcessor({- initialMessages: [{ id: runId, role: "assistant", parts: [{ type: "text", content: st.acc }] }],- events: { onTextUpdate: (_id, text) => pushDelta(stream, partial, st, text) },- });- processor.processChunk({ type: EventType.TEXT_MESSAGE_START, messageId: runId, role: "assistant" });- const append = (delta: string): void => {- if (!delta) return;- st.lastProgressAt = now();- processor.processChunk({ type: EventType.TEXT_MESSAGE_CONTENT, messageId: runId, delta });- };const transport = fetchServerSentEvents(withBase(runPath(runId, "/events")), {credentials: "same-origin",});@@ -1591,7 +1587,7 @@ async function streamRunViaSse(received();if (event.type !== "CUSTOM") continue;if (event.name === "delta") {- const value = event.value as { offset?: unknown; delta?: unknown };+ const value = event.value as { offset?: unknown; delta?: unknown; startOffset?: number };if (typeof value.offset !== "number" ||!Number.isSafeInteger(value.offset) ||@@ -1599,13 +1595,10 @@ async function streamRunViaSse(typeof value.delta !== "string")return "fallback";- if (value.offset > st.acc.length) return "fallback";- const delta = value.delta.slice(st.acc.length - value.offset);- append(delta);+ if (value.offset > Math.max(st.startOffset + st.acc.length, value.startOffset ?? 0)) return "fallback";+ applyStreamText(stream, partial, st, value.delta, value.offset, value.startOffset ?? 0);} else if (event.name === "run") {const run = event.value as RunPoll;- if (typeof run.partial === "string" && run.partial.length > st.acc.length)- append(run.partial.slice(st.acc.length));// Completion must carry the final result, including files and approvals.const terminal = run.status === "done" || run.status === "failed" || run.result != null;if (applyRun(stream, partial, st, { ...run, replyComplete: terminal }, notify) === "terminal") return "done";@@ -1661,6 +1654,28 @@ function abortStream(stream: AssistantMessageEventStream, partial: AssistantMessstream.end(error);}+function applyStreamText(+ stream: AssistantMessageEventStream,+ partial: AssistantMessage,+ st: Acc,+ text: string,+ offset: number,+ startOffset: number,+): void {+ const end = st.startOffset + st.acc.length;+ if (offset + text.length < end || (offset + text.length === end && startOffset <= st.startOffset)) return;+ const start = Math.max(st.startOffset ?? 0, startOffset);+ const next = offset > end ? text : st.acc + text.slice(Math.max(0, end - offset));+ const nextStart = offset > end ? offset : st.startOffset;+ const work = (partial as AssistantWork).work;+ if (work) work.streamStartOffset = start;+ if (start !== st.startOffset) (partial as AssistantWork).streamingBaseline = "";+ st.acc = offset > end ? "" : st.acc.slice(start - st.startOffset);+ st.lastProgressAt = now();+ pushDelta(stream, partial, st, next.slice(start - nextStart));+ st.startOffset = start;+}+function pushDelta(stream: AssistantMessageEventStream, partial: AssistantMessage, st: Acc, next: string): void {const delta = next.slice(st.acc.length);const block = partial.content[0];diff --git a/plugins/web-ui/src/timeline.ts b/plugins/web-ui/src/timeline.tsindex 8e47aac6b..4adbda104 100644--- a/plugins/web-ui/src/timeline.ts+++ b/plugins/web-ui/src/timeline.ts@@ -293,7 +293,7 @@ export function sessionToolView(}export function currentTextPhase(work: WorkBlock): {- phase: "commentary" | "final_answer";+ phase: "commentary" | "final_answer" | undefined;streamOffset: number;startedAt: number;} | null {@@ -301,8 +301,8 @@ export function currentTextPhase(work: WorkBlock): {if (!entry) return null;const payload = entry.payload as { phase?: unknown; streamOffset?: unknown } | null;if (- (payload?.phase !== "commentary" && payload?.phase !== "final_answer") ||- typeof payload.streamOffset !== "number" ||+ (payload?.phase !== undefined && payload.phase !== "commentary" && payload.phase !== "final_answer") ||+ typeof payload?.streamOffset !== "number" ||!Number.isSafeInteger(payload.streamOffset) ||payload.streamOffset < 0)@@ -312,7 +312,21 @@ export function currentTextPhase(work: WorkBlock): {export function streamedAnswer(text: string, work: WorkBlock): string {const phase = currentTextPhase(work);- return phase?.phase === "final_answer" ? text.slice(phase.streamOffset) : streamingTextTail(text, work.activity);+ if (phase) {+ const lastText = work.activity.findLast((entry) => entry.type === "text_start" || entry.type === "text");+ if (phase.phase !== "final_answer" && lastText?.type === "text") return "";+ return streamTextFrom(text, phase.streamOffset, work.streamStartOffset ?? 0);+ }+ return work.streamStartOffset+ ? streamTextFrom(text, 0, work.streamStartOffset)+ : streamingTextTail(text, work.activity);+}++function streamTextFrom(text: string, offset: number, startOffset: number): string {+ if (offset >= startOffset) return text.slice(offset - startOffset);+ let boundary = 0;+ while (boundary < text.length && text[boundary]!.trim()) boundary++;+ return "…earlier output omitted\n\n" + text.slice(boundary).trimStart();}export function streamingTextTail(text: string, activity: ToolActivity[]): string {diff --git a/plugins/web-ui/test/completed-run-recovery.test.ts b/plugins/web-ui/test/completed-run-recovery.test.tsindex 82ae0b535..1dfe3acc2 100644--- a/plugins/web-ui/test/completed-run-recovery.test.ts+++ b/plugins/web-ui/test/completed-run-recovery.test.ts@@ -32,7 +32,7 @@ test("a poll timeout retains the exact run identity needed to recover its savedconst partial = entriesToMessages([answer])[0] as AssistantWork;partial.content = [{ type: "text", text: "" }];const stream = createAssistantMessageEventStream();- await pollRun(stream, partial, "run-current", { acc: "", lastProgressAt: 0 });+ await pollRun(stream, partial, "run-current", { acc: "", startOffset: 0, lastProgressAt: 0 });const error = (await stream.result()) as AssistantWork;assert.equal(error.stopReason, "error");assert.equal(error.interruptedRunId, "run-current");@@ -43,7 +43,11 @@ test("an idle running snapshot timeout also retains the run identity", async ()setClock(() => RUN_IDLE_MS + 1);globalThis.fetch = async () => new Response(JSON.stringify({ status: "running", result: null }));const stream = createAssistantMessageEventStream();- await pollRun(stream, entriesToMessages([answer])[0] as AssistantWork, "run-idle", { acc: "", lastProgressAt: 0 });+ await pollRun(stream, entriesToMessages([answer])[0] as AssistantWork, "run-idle", {+ acc: "",+ startOffset: 0,+ lastProgressAt: 0,+ });assert.equal(((await stream.result()) as AssistantWork).interruptedRunId, "run-idle");});@@ -51,7 +55,11 @@ test("definitive backend failures are not marked as interrupted polling", asyncglobalThis.fetch = async () =>new Response(JSON.stringify({ status: "failed", result: { status: "failed", reason: "backend error" } }));const stream = createAssistantMessageEventStream();- await pollRun(stream, entriesToMessages([answer])[0] as AssistantWork, "run-failed", { acc: "", lastProgressAt: 0 });+ await pollRun(stream, entriesToMessages([answer])[0] as AssistantWork, "run-failed", {+ acc: "",+ startOffset: 0,+ lastProgressAt: 0,+ });const error = (await stream.result()) as AssistantWork;assert.equal(error.stopReason, "error");assert.equal(error.interruptedRunId, undefined);diff --git a/plugins/web-ui/test/core-bridge-idle.test.ts b/plugins/web-ui/test/core-bridge-idle.test.tsindex 5a61791bc..dc1682cf4 100644--- a/plugins/web-ui/test/core-bridge-idle.test.ts+++ b/plugins/web-ui/test/core-bridge-idle.test.ts@@ -35,7 +35,7 @@ function blankAssistant(): AssistantMessage {}function freshAcc(t0: number): Acc {- return { acc: "", lastProgressAt: t0 };+ return { acc: "", startOffset: 0, lastProgressAt: t0 };}function drain(stream: ReturnType<typeof createAssistantMessageEventStream>): Promise<AssistantMessage> {@@ -447,3 +447,18 @@ test("live run polling includes only consumed visible steering in activity order["tool", "steer", "tool"],);});++test("polling advances a trimmed window even when every snapshot has the same length", async () => {+ instantSleep();+ stubRuns([+ { status: "running", result: null, partial: "first tail", partialStartOffset: 200_000 },+ { status: "running", result: null, partial: "later tail", partialStartOffset: 200_010 },+ { status: "done", result: { status: "ok" }, partial: "final tail", partialStartOffset: 200_020 },+ ]);+ const stream = createAssistantMessageEventStream();+ const partial = blankAssistant();+ const st = freshAcc(Date.now());+ await pollRun(stream, partial, "r", st);+ assert.deepEqual((await drain(stream)).content, [{ type: "text", text: "final tail" }]);+ assert.equal(st.startOffset, 200_020);+});diff --git a/plugins/web-ui/test/core-bridge-retry.test.ts b/plugins/web-ui/test/core-bridge-retry.test.tsindex 67c90d1ad..89a55cee1 100644--- a/plugins/web-ui/test/core-bridge-retry.test.ts+++ b/plugins/web-ui/test/core-bridge-retry.test.ts@@ -27,7 +27,7 @@ function blankAssistant(): AssistantMessage {}function freshAcc(t0: number): Acc {- return { acc: "", lastProgressAt: t0 };+ return { acc: "", startOffset: 0, lastProgressAt: t0 };}function drain(stream: ReturnType<typeof createAssistantMessageEventStream>): Promise<AssistantMessage> {diff --git a/plugins/web-ui/test/post-reply-recovery.test.ts b/plugins/web-ui/test/post-reply-recovery.test.tsindex 1841ae413..05fa29f50 100644--- a/plugins/web-ui/test/post-reply-recovery.test.ts+++ b/plugins/web-ui/test/post-reply-recovery.test.ts@@ -501,6 +501,30 @@ test("post replies remain visible in new and continuing conversations", async (thost.querySelector<HTMLElement & { content: string }>(".work-said qm-markdown")?.content,"Progress 2",);+ const priorActivity = work.activity;+ work.streamStartOffset = 200_001;+ work.activity = [+ ...priorActivity,+ { seq: 3, parentSeq: null, type: "text_start", createdAt: Date.now(), payload: { streamOffset: 200_003 } },+ ];+ streaming.content = [{ type: "text", text: " \nCommentary marker" }];+ conv!.drawActiveChat(agent);+ assert.equal(+ host.querySelector<HTMLElement & { content: string }>(".work-said qm-markdown")?.content,+ "Commentary marker",+ );+ work.activity = [+ { seq: 3, parentSeq: null, type: "text_start", createdAt: Date.now(), payload: { streamOffset: 0 } },+ ];+ streaming.content = [{ type: "text", text: "ipped tail complete" }];+ conv!.drawActiveChat(agent);+ assert.equal(+ host.querySelector<HTMLElement & { content: string }>(".assistant-body > .streaming-text qm-markdown")+ ?.content,+ "…earlier output omitted\n\ntail complete",+ );+ delete work.streamStartOffset;+ work.activity = priorActivity;work.activity = [...work.activity,{diff --git a/plugins/web-ui/test/run-connection.test.ts b/plugins/web-ui/test/run-connection.test.tsindex 2b84b956a..26698e053 100644--- a/plugins/web-ui/test/run-connection.test.ts+++ b/plugins/web-ui/test/run-connection.test.ts@@ -304,3 +304,40 @@ test("approval activity reaches the visible transcript before any response text"await completion;assert.ok(beforeCompletion.includes("approval-decision"), JSON.stringify(beforeCompletion));});++test("trimmed snapshots and live deltas use absolute offsets without freezing or duplicating text", async (t) => {+ const state = streamingFetch(t);+ const stream = await makeRunResumeStreamFn("r", {+ status: "running",+ result: null,+ partial: "ipped word",+ partialStartOffset: 200_001,+ })(model, {} as Context, {});+ await flush();+ state.send({ type: "CUSTOM", name: "delta", value: { offset: 200_011, delta: " more", startOffset: 200_003 } });+ state.send({+ type: "CUSTOM",+ name: "run",+ value: { status: "running", result: null, partial: "word more", partialStartOffset: 200_007 },+ });+ state.send({ type: "CUSTOM", name: "delta", value: { offset: 200_011, delta: " more!", startOffset: 200_007 } });+ state.send({ type: "CUSTOM", name: "run", value: { status: "done", result: { status: "ok" } } });+ assert.deepEqual((await stream.result()).content, [{ type: "text", text: "word more!" }]);+ assert.equal(state.polls, 0);+});++test("a resume snapshot replaces whitespace-normalized transcript seeds", async () => {+ const fn = makeRunResumeStreamFn(+ "r",+ {+ status: "done",+ result: { status: "ok" },+ partial: "\nChecking.\n\nNext part",+ },+ undefined,+ undefined,+ "Checking.",+ );+ const stream = await fn(model, {} as Context, {});+ assert.deepEqual((await stream.result()).content, [{ type: "text", text: "\nChecking.\n\nNext part" }]);+});diff --git a/plugins/web-ui/test/timeline.test.ts b/plugins/web-ui/test/timeline.test.tsindex cf190a215..0783a743f 100644--- a/plugins/web-ui/test/timeline.test.ts+++ b/plugins/web-ui/test/timeline.test.ts@@ -499,3 +499,50 @@ test("detached execution uses process presentation rather than foreground outputassert.equal(toolCategory({ tool: "execute", background: true }), "background");assert.equal(toolCategory({ tool: "execute", background: false }), "execute");});++test("trimmed phases show whole-word tails and newer phases retain absolute boundaries", () => {+ const work: WorkBlock = {+ status: "working",+ streamStartOffset: 101,+ activity: [+ {+ seq: 1,+ parentSeq: null,+ type: "text_start",+ createdAt: 1,+ payload: { phase: "final_answer", streamOffset: 80 },+ },+ ],+ };+ assert.equal(streamedAnswer("ipped word here", work), "…earlier output omitted\n\nword here");+ assert.equal(streamedAnswer("unbroken", work), "…earlier output omitted\n\n");+ work.activity = [+ {+ seq: 2,+ parentSeq: null,+ type: "text_start",+ createdAt: 2,+ payload: { phase: "final_answer", streamOffset: 107 },+ },+ ];+ assert.equal(streamedAnswer("older New answer", work), "New answer");+});++test("recorded commentary is not repeated after its live window is trimmed", () => {+ const work: WorkBlock = {+ status: "working",+ streamStartOffset: 101,+ activity: [+ { seq: 1, parentSeq: null, type: "text_start", createdAt: 1, payload: { phase: "commentary", streamOffset: 80 } },+ ],+ };+ assert.equal(streamedAnswer("ipped word here", work), "…earlier output omitted\n\nword here");+ work.activity.push({+ seq: 2,+ parentSeq: null,+ type: "text",+ createdAt: 2,+ payload: { text: "The whole commentary is now recorded." },+ });+ assert.equal(streamedAnswer("ipped word here", work), "");+});diff --git a/src/api/app-turn.ts b/src/api/app-turn.tsindex ac60943fe..5a0ec7589 100644--- a/src/api/app-turn.ts+++ b/src/api/app-turn.ts@@ -753,7 +753,7 @@ export function createTurnMethods(: {}),startedAt: run.startedAt,finishedAt: run.finishedAt,- ...(partial ? { partial } : {}),+ ...(partial ? { partial, partialStartOffset: deps.turnStream!.startOffset(runId) } : {}),...(firstBlock? { firstBlock: firstBlock.text, ...(firstBlock.closed ? { firstBlockClosed: true } : {}) }: {}),diff --git a/src/api/app-types.ts b/src/api/app-types.tsindex bef4df075..f5f03c3aa 100644--- a/src/api/app-types.ts+++ b/src/api/app-types.ts@@ -299,6 +299,7 @@ export interface App {attachments?: Array<{ name: string; mimetype: string; sizeBytes: number }>;};partial?: string;+ partialStartOffset?: number;alive?: boolean;stale?: boolean;replying?: boolean;diff --git a/src/api/routes/run-events.ts b/src/api/routes/run-events.tsindex 7b9242753..e9d0bcd62 100644--- a/src/api/routes/run-events.ts+++ b/src/api/routes/run-events.ts@@ -71,7 +71,6 @@ async function streamRun(ctx: ApiCtx): Promise<void> {if (syncing || closed) return;syncing = true;queueMicrotask(() => {- syncing = false;if (!closed) app.syncRunStream(runId, offset);});};@@ -82,7 +81,7 @@ async function streamRun(ctx: ApiCtx): Promise<void> {const snapshot = async (run: NonNullable<typeof initial>): Promise<void> => {const toolEntries = await app.getRunToolEntries(runId, actor?.p, toolCursor);if (closed) return;- offset = Math.max(offset, run.partial?.length ?? 0);+ offset = Math.max(offset, (run.partialStartOffset ?? 0) + (run.partial?.length ?? 0));for (const entry of toolEntries) {const event = toolEvent(entry);if (event && !sentToolEvents.has(event.id)) {@@ -124,6 +123,7 @@ async function streamRun(ctx: ApiCtx): Promise<void> {res.destroy();} finally {refreshing = false;+ syncing = false;sync();}};@@ -136,21 +136,26 @@ async function streamRun(ctx: ApiCtx): Promise<void> {return;}if (event.kind !== "delta" || refreshing) return;+ const startOffset = event.startOffset ?? 0;+ offset = Math.max(offset, startOffset);if (event.offset > offset) {sync();return;}const delta = event.text.slice(offset - event.offset);if (!delta) return;- send({ type: "CUSTOM", name: "delta", value: { offset, delta } }, `text:${offset + delta.length}`);+ syncing = false;+ send({ type: "CUSTOM", name: "delta", value: { offset, delta, startOffset } }, `text:${offset + delta.length}`);offset += delta.length;},() => {+ syncing = false;sync();void refresh();},);const heartbeat = setInterval(() => {+ syncing = false;sync();void refresh();}, HEARTBEAT_MS);diff --git a/src/core/orchestrator.ts b/src/core/orchestrator.tsindex 1a602569e..fec88d98f 100644--- a/src/core/orchestrator.ts+++ b/src/core/orchestrator.ts@@ -3918,14 +3918,17 @@ export function createOrchestrator(deps: OrchestratorDeps): Orchestrator {onTextBlockStart: async (phase) => {if (input.runId && deps.turnStream && !input.surfaceTools) deps.turnStream.publishBlockStart(input.runId);if (input.surfaceTools && spineFirstBlock) spineFirstBlockOpen = false;- if (phase && !input.surfaceTools) {+ if (!input.surfaceTools) {const streamOffset =- input.runId && deps.turnStream ? (deps.turnStream.snapshot(input.runId) ?? "").length : undefined;- await emit({+ input.runId && deps.turnStream+ ? deps.turnStream.startOffset(input.runId) + (deps.turnStream.snapshot(input.runId) ?? "").length+ : undefined;+ const started = emit({type: "text_start",- payload: { phase, ...(streamOffset !== undefined ? { streamOffset } : {}) },+ payload: { ...(phase ? { phase } : {}), ...(streamOffset !== undefined ? { streamOffset } : {}) },scopeLabel: scopeId,});+ await (phase ? started : started.catch(swallowAs("orchestrator: text-start activity", undefined)));}},onGapWork: (cb) => {diff --git a/src/persistence/postgres-notify-bus.ts b/src/persistence/postgres-notify-bus.tsindex 97a4a0936..415bf0b17 100644--- a/src/persistence/postgres-notify-bus.ts+++ b/src/persistence/postgres-notify-bus.ts@@ -23,6 +23,25 @@ export function createPostgresNotifyBus<T>(let stopListening: (() => Promise<void>) | null = null;let closed = false;let cleanup = Promise.resolve();+ let publishing: Promise<void> | null = null;+ let pending: string[] = [];++ const flush = async (): Promise<void> => {+ try {+ while (pending.length) {+ const batch = pending;+ pending = [];+ await pg+ .query(+ `SELECT pg_notify('${channel}', payload) FROM unnest($1::text[]) WITH ORDINALITY AS events(payload, ordinal) ORDER BY ordinal`,+ [batch],+ )+ .catch(swallowAs(`${label}: notify`, undefined));+ }+ } finally {+ publishing = null;+ }+ };function dropListenClient() {const stop = stopListening;@@ -42,9 +61,11 @@ export function createPostgresNotifyBus<T>(return {emit(event) {+ if (closed) return;const payload = encode(event);if (payload === null) return;- void pg.query(`SELECT pg_notify('${channel}', $1)`, [payload]).catch(swallowAs(`${label}: notify`, undefined));+ pending.push(payload);+ publishing ??= Promise.resolve().then(flush);},subscribe(cb, opts) {@@ -59,7 +80,7 @@ export function createPostgresNotifyBus<T>(async close() {closed = true;dropListenClient();- await cleanup;+ await Promise.all([cleanup, publishing]);await pg.close();},};diff --git a/src/runs/run-stream-events.ts b/src/runs/run-stream-events.tsindex 458d18a9e..1ceeefa9c 100644--- a/src/runs/run-stream-events.ts+++ b/src/runs/run-stream-events.ts@@ -1,13 +1,19 @@import type { EventBus } from "../util/event-bus.ts";export type RunStreamEvent =- | { runId: string; kind: "delta"; offset: number; text: string }+ | { runId: string; kind: "delta"; offset: number; text: string; startOffset: number }| { runId: string; kind: "sync"; offset: number }| { runId: string; kind: "refresh" };-export function emitRunText(bus: EventBus<RunStreamEvent>, runId: string, text: string, offset: number): void {+export function emitRunText(+ bus: EventBus<RunStreamEvent>,+ runId: string,+ text: string,+ offset: number,+ startOffset: number,+): void {// JSON escaping can cost six bytes per UTF-16 unit; stay below NOTIFY's byte limit.for (let at = 0; at < text.length; at += 1_000) {- bus.emit({ runId, kind: "delta", offset: offset + at, text: text.slice(at, at + 1_000) });+ bus.emit({ runId, kind: "delta", offset: offset + at, text: text.slice(at, at + 1_000), startOffset });}}diff --git a/src/runs/turn-stream.ts b/src/runs/turn-stream.tsindex 671a99cc8..9288f3705 100644--- a/src/runs/turn-stream.ts+++ b/src/runs/turn-stream.ts@@ -19,6 +19,7 @@ export interface TurnStream {markSurfacePosted(runId: string): void;surfacePosted(runId: string): boolean;snapshot(runId: string): string | null;+ startOffset(runId: string): number;markReplyDone(runId: string): void;isReplyDone(runId: string): boolean;end(runId: string): void;@@ -32,6 +33,7 @@ interface TurnStreamListener {interface Entry {text: string;+ totalChars: number;goal: GoalView | null;firstBlock: string;firstBlockOpen: boolean;@@ -45,7 +47,7 @@ interface Entry {export interface TurnStreamOptions {maxChars?: number;graceMs?: number;- onDelta?(runId: string, text: string, offset: number): void;+ onDelta?(runId: string, text: string, offset: number, startOffset: number): void;onChange?(runId: string): void;}@@ -57,6 +59,7 @@ const BLOCK_JOIN = "\n\n";function makeEntry(partial: Partial<Entry> = {}): Entry {return {text: "",+ totalChars: 0,goal: null,firstBlock: "",firstBlockOpen: true,@@ -70,7 +73,7 @@ function makeEntry(partial: Partial<Entry> = {}): Entry {}export function createTurnStream(opts: TurnStreamOptions = {}): TurnStream {- const maxChars = opts.maxChars ?? DEFAULT_MAX_CHARS;+ const maxChars = Math.max(2, opts.maxChars ?? DEFAULT_MAX_CHARS);const graceMs = opts.graceMs ?? DEFAULT_GRACE_MS;const runs = new Map<string, Entry>();const listeners = new Map<string, Set<TurnStreamListener>>();@@ -84,6 +87,15 @@ export function createTurnStream(opts: TurnStreamOptions = {}): TurnStream {return entry;};+ const append = (runId: string, entry: Entry, delta: string): void => {+ const offset = entry.totalChars;+ entry.totalChars += delta.length;+ entry.text += delta;+ if (entry.text.length > maxChars)+ entry.text = entry.text.slice(-Math.max(BLOCK_JOIN.length, Math.floor(maxChars / 2)));+ opts.onDelta?.(runId, delta, offset, entry.totalChars - entry.text.length);+ };+return {begin(runId) {const entry = runs.get(runId);@@ -106,12 +118,7 @@ export function createTurnStream(opts: TurnStreamOptions = {}): TurnStream {}if (entry.firstBlockOpen && entry.firstBlock.length < FIRST_BLOCK_MAX_CHARS)entry.firstBlock = (entry.firstBlock + delta).slice(0, FIRST_BLOCK_MAX_CHARS);- const offset = entry.text.length;- if (offset < maxChars) {- const added = delta.slice(0, maxChars - offset);- entry.text += added;- opts.onDelta?.(runId, added, offset);- }+ append(runId, entry, delta);},publishBlockStart(runId) {@@ -119,10 +126,7 @@ export function createTurnStream(opts: TurnStreamOptions = {}): TurnStream {if (!entry) return;if (entry.firstBlock) entry.firstBlockOpen = false;if (!entry.text || entry.text.endsWith(BLOCK_JOIN)) return;- const offset = entry.text.length;- const added = BLOCK_JOIN.slice(0, Math.max(0, maxChars - offset));- entry.text += added;- if (added) opts.onDelta?.(runId, added, offset);+ append(runId, entry, BLOCK_JOIN);},noteToolCall(runId) {@@ -165,6 +169,11 @@ export function createTurnStream(opts: TurnStreamOptions = {}): TurnStream {return text ? text : null;},+ startOffset(runId) {+ const entry = runs.get(runId);+ return entry ? entry.totalChars - entry.text.length : 0;+ },+markReplyDone(runId) {const entry = runs.get(runId);if (entry) entry.replyDone = true;diff --git a/src/wiring.ts b/src/wiring.tsindex e11370bad..49ab3316c 100644--- a/src/wiring.ts+++ b/src/wiring.ts@@ -1645,13 +1645,15 @@ export function buildApp(: createMemoryEventBus<RunStreamEvent>("run-stream");const refreshRunStream = (runId: string): void => runStreamEvents.emit({ runId, kind: "refresh" });const turnStream = createTurnStream({- onDelta: (runId, text, offset) => emitRunText(runStreamEvents, runId, text, offset),+ onDelta: (runId, text, offset, startOffset) => emitRunText(runStreamEvents, runId, text, offset, startOffset),onChange: refreshRunStream,});const stopStreamSync = runStreamEvents.subscribe((event) => {if (event.kind !== "sync") return;const text = turnStream.snapshot(event.runId);- if (text) emitRunText(runStreamEvents, event.runId, text.slice(event.offset), event.offset);+ const startOffset = turnStream.startOffset(event.runId);+ const offset = Math.max(event.offset, startOffset);+ if (text) emitRunText(runStreamEvents, event.runId, text.slice(offset - startOffset), offset, startOffset);});runs.onTerminal((run) => refreshRunStream(run.id));const sessionStateBus: SessionStateBus = config.databaseUrldiff --git a/test/orchestrator.test.ts b/test/orchestrator.test.tsindex 9496021ba..370cfef41 100644--- a/test/orchestrator.test.ts+++ b/test/orchestrator.test.ts@@ -4365,9 +4365,10 @@ test("narration reaches the live activity feed before its tool call", async () =const view = await app.getRun(run.id);assert.deepEqual(view?.activity?.map((entry) => entry.type),- ["text", "tool_call", "tool_result"],+ ["text", "tool_call", "tool_result", "text_start"],);assert.deepEqual(view?.activity?.[0]?.payload, { text: "I'll check the first item." });+ assert.deepEqual(view?.activity?.at(-1)?.payload, { streamOffset: "I'll check the first item.\n\n".length });});test("public text phases persist with exact stream offsets in session history and run activity", async () => {diff --git a/test/postgres-notify-order.test.ts b/test/postgres-notify-order.test.tsnew file mode 100644index 000000000..40acf95b6--- /dev/null+++ b/test/postgres-notify-order.test.ts@@ -0,0 +1,43 @@+import assert from "node:assert/strict";+import { mock, test } from "node:test";++const started: string[][] = [];+const pending: Array<() => void> = [];+let closed = false;+mock.module("../src/persistence/pg-pool.ts", {+ namedExports: {+ createPgPool: () => ({+ query: (_sql: string, [payload]: string[][]) => {+ started.push(payload!);+ return new Promise<void>((resolve) => pending.push(resolve));+ },+ close: async () => {+ closed = true;+ },+ }),+ },+});+const { createPostgresNotifyBus } = await import("../src/persistence/postgres-notify-bus.ts");+const tick = () => new Promise<void>((resolve) => setImmediate(resolve));++test("notification publication preserves order without concurrent pool queries and drains on close", async () => {+ const bus = createPostgresNotifyBus<number>("db", "events", "test");+ bus.emit(1);+ bus.emit(2);+ await tick();+ assert.deepEqual(started, [["1", "2"]]);+ bus.emit(3);+ bus.emit(4);+ const closing = bus.close!();+ assert.equal(closed, false);+ pending.shift()!();+ await tick();+ assert.deepEqual(started, [+ ["1", "2"],+ ["3", "4"],+ ]);+ assert.equal(closed, false);+ pending.shift()!();+ await closing;+ assert.equal(closed, true);+});diff --git a/test/postgres-run-stream.test.ts b/test/postgres-run-stream.test.tsindex 9cdfd04e7..1c730aa96 100644--- a/test/postgres-run-stream.test.ts+++ b/test/postgres-run-stream.test.ts@@ -14,7 +14,9 @@ test(const channel = `stream_${randomUUID().replaceAll("-", "")}`;const worker = createPostgresNotifyBus<RunStreamEvent>(database!, channel, "test-worker");const api = createPostgresNotifyBus<RunStreamEvent>(database!, channel, "test-api");- const stream = createTurnStream({ onDelta: (runId, text, offset) => emitRunText(worker, runId, text, offset) });+ const stream = createTurnStream({+ onDelta: (runId, text, offset, startOffset) => emitRunText(worker, runId, text, offset, startOffset),+ });const runId = randomUUID();stream.begin(runId);stream.publish(runId, "preexisting 🌍 ");@@ -27,8 +29,11 @@ test(const offWorker = await new Promise<() => void>((resolve) => {const off = worker.subscribe((event) => {- if (event.kind === "sync")- emitRunText(worker, runId, (stream.snapshot(runId) ?? "").slice(event.offset), event.offset);+ if (event.kind === "sync") {+ const startOffset = stream.startOffset(runId);+ const offset = Math.max(startOffset, event.offset);+ emitRunText(worker, runId, (stream.snapshot(runId) ?? "").slice(offset - startOffset), offset, startOffset);+ }},{ onResync: () => resolve(off) },);diff --git a/test/run-events.test.ts b/test/run-events.test.tsindex 8e9629678..a96b1974a 100644--- a/test/run-events.test.ts+++ b/test/run-events.test.ts@@ -15,6 +15,8 @@ test("run events push deltas without snapshot polling and recover a separate worconst bus = createMemoryEventBus<RunStreamEvent>("test");let reads = 0;let workerText = "prefix 🌍";+ let holdSync = false;+ let syncCalls = 0;let status: "running" | "done" = "running";const app = {async getRun() {@@ -30,7 +32,8 @@ test("run events push deltas without snapshot polling and recover a separate worreturn bus.subscribe(listener, { onResync });},syncRunStream(runId, offset) {- emitRunText(bus, runId, workerText.slice(offset), offset);+ syncCalls++;+ if (!holdSync) emitRunText(bus, runId, workerText.slice(offset), offset, 0);},async getRunToolEntries() {return [];@@ -57,12 +60,20 @@ test("run events push deltas without snapshot polling and recover a separate worawait sleep(120);assert.equal(reads, 2, "only initial authorization and race-free hydration read snapshots");workerText += " more";- bus.emit({ runId: "run", kind: "delta", offset: 9, text: " more" });+ bus.emit({ runId: "run", kind: "delta", startOffset: 0, offset: 9, text: " more" });await sleep(120);assert.equal(reads, 2, "text publications must not poll the run");assert.match(wire, /more/);workerText += " lost then found";- bus.emit({ runId: "run", kind: "delta", offset: 20, text: "found" });+ const beforeSync = syncCalls;+ holdSync = true;+ for (let i = 0; i < 3; i++) {+ bus.emit({ runId: "run", kind: "delta", startOffset: 0, offset: 20, text: "found" });+ await sleep(5);+ }+ assert.equal(syncCalls - beforeSync, 1, "later gaps must not start overlapping catch-up replays");+ holdSync = false;+ app.syncRunStream("run", 14);await sleep(50);assert.match(wire, /lost then found/);status = "done";@@ -105,7 +116,7 @@ test("run events deliver a pending activity snapshot before subsequent text andreturn bus.subscribe(listener, { onResync });},syncRunStream(runId, offset) {- emitRunText(bus, runId, workerText.slice(offset), offset);+ emitRunText(bus, runId, workerText.slice(offset), offset, 0);},async getRunToolEntries() {return [];@@ -134,7 +145,7 @@ test("run events deliver a pending activity snapshot before subsequent text andbus.emit({ runId: "run", kind: "refresh" });await entered.promise;workerText = "after-boundary";- bus.emit({ runId: "run", kind: "delta", offset: 0, text: workerText });+ bus.emit({ runId: "run", kind: "delta", startOffset: 0, offset: 0, text: workerText });await sleep(30);assert.doesNotMatch(wire, /after-boundary/);release.resolve();diff --git a/test/run-stream-events.test.ts b/test/run-stream-events.test.tsindex 0eacb2848..c4a1de76d 100644--- a/test/run-stream-events.test.ts+++ b/test/run-stream-events.test.ts@@ -25,7 +25,8 @@ test("text publications push exact offsets immediately, including block boundari[["hello", 0],["\n\n", 5],- ["wo", 7],+ ["world", 7],+ ["ignored", 12],],);assert.deepEqual(changes, ["run", "run"]);@@ -38,7 +39,7 @@ test("cross-process text frames stay under Postgres' payload limit and preserveif (event.kind === "delta") parts.push(event);});const text = '\u0000"\\🌍'.repeat(3_000);- emitRunText(bus, "r".repeat(36), text, 37);+ emitRunText(bus, "r".repeat(36), text, 37, 0);let offset = 37;for (const part of parts) {assert.equal(part.offset, offset);@@ -47,3 +48,22 @@ test("cross-process text frames stay under Postgres' payload limit and preserve}assert.equal(parts.map((part) => part.text).join(""), text);});++test("a boundary near the cap remains complete and later deltas keep absolute offsets", () => {+ const frames: Array<[string, number, number]> = [];+ const stream = createTurnStream({+ maxChars: 5,+ onDelta: (_id, text, offset, start) => frames.push([text, offset, start]),+ });+ stream.publish("r", "abcd");+ stream.publishBlockStart("r");+ stream.publishBlockStart("r");+ stream.publish("r", "next");+ assert.deepEqual(frames, [+ ["abcd", 0, 0],+ ["\n\n", 4, 4],+ ["next", 6, 8],+ ]);+ assert.equal(stream.snapshot("r"), "xt");+ assert.equal(stream.startOffset("r"), 8);+});diff --git a/test/turn-stream.test.ts b/test/turn-stream.test.tsindex 397952d67..7a5aec0e9 100644--- a/test/turn-stream.test.ts+++ b/test/turn-stream.test.ts@@ -74,7 +74,8 @@ test("caps a run's buffer at maxChars", () => {const s = createTurnStream({ maxChars: 5 });s.publish("r1", "abc");s.publish("r1", "defghij");- assert.equal(s.snapshot("r1"), "abcde", "buffer stops growing past the cap");+ assert.equal(s.snapshot("r1"), "ij", "buffer retains the newest text past the cap");+ assert.equal(s.startOffset("r1"), 8);});test("end() evicts a finished run's buffer after the grace period", async () => {diff --git a/test/web-ui-sse.test.ts b/test/web-ui-sse.test.tsindex 220bd8401..66eaf0636 100644--- a/test/web-ui-sse.test.ts+++ b/test/web-ui-sse.test.ts@@ -238,3 +238,33 @@ test("the events route requires a signed-in principal (401 without a cookie)", aconst res = await fetch(`${webBase}/api/runs/whatever/events`);assert.equal(res.status, 401);});++test("a completed long stream replays its bounded tail with absolute offsets and its full reply", async () => {+ const text = "long output ".repeat(20_000) + "last sentence";+ const submit = (await (+ await fetch(+ `${webBase}/api/turn`,+ asUser("alice", {+ method: "POST",+ body: JSON.stringify({ text, threadRef: `web:alice:${randomUUID()}` }),+ }),+ )+ ).json()) as { runId: string };+ await built.runs.waitFor(submit.runId, 10_000);+ const res = await fetch(+ `${webBase}/api/runs/${submit.runId}/events`,+ asUser("alice", {+ signal: AbortSignal.timeout(20_000),+ }),+ );+ const done = parseSse(await res.text()).find((event) => event.event === "done")!.data as {+ partial: string;+ partialStartOffset: number;+ result: { reply: string };+ };+ assert.ok(done.partial.length <= 200_000);+ assert.ok(done.partial.length >= 100_000);+ assert.ok(done.partialStartOffset > 0);+ assert.equal(done.result.reply.slice(done.partialStartOffset), done.partial);+ assert.ok(done.partial.includes("last sentence"));+});