Skip to content

Durable coding-agent harnesses (Claude Code, Codex, OpenCode, Pi, fx) on a Workflow SDK World

fix(pi): replace interrupted streamed replies on recovery (#27)

Pranay Prakashff32c13368f8
diff --git a/.changeset/pi-restart.md b/.changeset/pi-restart.md
new file mode 100644
index 0000000..5e3da5e
--- /dev/null
+++ b/.changeset/pi-restart.md
@@ -0,0 +1,5 @@
+---
+"durable-harness": patch
+---
+
+Pi signals when a recovered reply replaces interrupted streamed text, retaining completed text from the turn.
diff --git a/packages/harness-pi/src/index.ts b/packages/harness-pi/src/index.ts
index de2880d..4e2df0c 100644
--- a/packages/harness-pi/src/index.ts
+++ b/packages/harness-pi/src/index.ts
@@ -98,6 +98,8 @@ interface ModelInput {
/** Total messages in the request context (a replay divergence check). */
contextLength: number;
model: string;
+ /** First model call after `agent.prompt`, after the preceding `idle`. */
+ turnStart?: boolean;
}
/**
@@ -124,6 +126,7 @@ class PiDrive {
/** Messages already recorded by committed model steps. */
private recorded = 0;
private modelOrdinal = 0;
+ private turnStart?: boolean;
/** Why this drive must stop, set from inside Pi callbacks that cannot throw. */
private stop?: Error;
/** The drive-end hooks of its host tools (`afterDrive`), each once. */
@@ -173,6 +176,7 @@ class PiDrive {
this.recorded = ended ? restore.length : restore.length - lastAppended;
restore.push(...answered);
this.modelOrdinal = ended ? models.length : Math.max(0, models.length - 1);
+ this.turnStart = !ended && last ? (last.input as ModelInput).turnStart : undefined;
this.agent = new Agent({
initialState: {
@@ -212,13 +216,18 @@ class PiDrive {
this.ctx.signal.addEventListener('abort', onAbort);
if (this.ctx.signal.aborted) onAbort();
try {
- if (models.length > 0 && !ended) await this.agent.continue();
+ if (models.length > 0 && !ended) {
+ const restart = restartOf(state);
+ if (restart) this.ctx.live(restart);
+ await this.agent.continue();
+ }
this.throwIfStopped();
for (;;) {
await this.ctx.refresh();
const next = this.ctx.pending().filter((s) => !this.handed.has(s.id));
if (next.length === 0) break;
for (const s of next) this.handed.set(s.id, s);
+ this.turnStart = true;
await this.agent.prompt(next.map((s) => userMessage(s)));
this.throwIfStopped();
}
@@ -267,6 +276,8 @@ class PiDrive {
m.role === 'user' ? this.claimSubmission(m) : []
);
const replaying = this.ctx.isMemoized(key);
+ const turnStart = this.turnStart;
+ this.turnStart = false;
const run = async (): Promise<AssistantMessage> => {
if (this.stop) throw this.stop;
@@ -280,6 +291,7 @@ class PiDrive {
appended,
contextLength: messages.length,
model: `${model.provider}/${model.id}`,
+ ...(turnStart !== undefined ? { turnStart } : {}),
} satisfies ModelInput,
},
async (io): Promise<ModelOutput> => {
@@ -456,6 +468,31 @@ class PiDrive {
}
}
+/** Replace only the interrupted turn's text, keeping its committed model steps. */
+function restartOf(state: SessionState): { type: 'restart'; text: string; parts: string[] } | undefined {
+ const models = state.effectOrder.slice(lastUnavailable(state) + 1).filter((record) => record.name === 'model');
+ // A committed model failure makes Pi emit agent_end/idle too: it is not
+ // interrupted work for the next prompt to recover.
+ const finished = (record: typeof models[number]) =>
+ record.status === 'failed' || (record.status === 'completed' && record.output !== undefined &&
+ !messageOf(record.output).content.some((part) => part.type === 'toolCall'));
+ const last = models.at(-1);
+ if (!last || finished(last)) return;
+ let parts: string[] = [];
+ for (const [i, record] of models.entries()) {
+ const { turnStart } = record.input as ModelInput;
+ // Older logs predate the explicit boundary: a tool-less answer ended
+ // their previous turn. New logs also retain text before a steer or
+ // follow-up that continued the same Agent invocation without an idle.
+ if (turnStart === true || (turnStart === undefined && i > 0 && finished(models[i - 1]!))) parts = [];
+ if (record.status !== 'completed' || record.output === undefined) continue;
+ const text = messageOf(record.output).content.flatMap((part) => part.type === 'text' ? [part.text] : []).join('');
+ if (text) parts.push(text);
+ }
+ // Pi's live text deltas concatenate across model calls without separators.
+ return { type: 'restart', text: parts.join(''), parts };
+}
+
/** Stream events that count as the model's first output (TTFT). */
const FIRST_OUTPUT = new Set(['text_delta', 'thinking_delta', 'toolcall_start', 'toolcall_delta']);
diff --git a/packages/harness-pi/src/recovery.test.ts b/packages/harness-pi/src/recovery.test.ts
new file mode 100644
index 0000000..6043169
--- /dev/null
+++ b/packages/harness-pi/src/recovery.test.ts
@@ -0,0 +1,318 @@
+import { mkdtemp } from 'node:fs/promises';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+import { createAssistantMessageEventStream, createModels } from '@earendil-works/pi-ai';
+import { fauxAssistantMessage, fauxProvider, fauxText, fauxToolCall } from '@earendil-works/pi-ai/providers/faux';
+import { createWorld, type LocalWorld } from '@workflow/world-local';
+import { chatRoute, durableSessions, type Harness, type HostTool, liveResponse, QUEUE_PREFIX, Unavailable, vercelAnchors } from 'harness-world';
+import { leasedWorld } from 'harness-world/testing';
+import { afterEach, describe, expect, it } from 'vitest';
+import { piHarness, type PiHarnessOptions } from './index.ts';
+import { Type } from 'typebox';
+
+type LiveEvent = { type: string; text?: string; delta?: string; parts?: string[] };
+const worlds: LocalWorld[] = [];
+afterEach(async () => {
+ for (const world of worlds.splice(0)) await world.close?.();
+});
+
+async function until<T>(read: () => Promise<T | undefined>): Promise<T> {
+ const deadline = Date.now() + 10_000;
+ for (;;) {
+ const result = await read();
+ if (result !== undefined) return result;
+ if (Date.now() > deadline) throw new Error('timed out');
+ await new Promise((resolve) => setTimeout(resolve, 10));
+ }
+}
+
+function model(responses: Parameters<ReturnType<typeof fauxProvider>['setResponses']>[0]) {
+ const faux = fauxProvider();
+ faux.setResponses(responses);
+ const models = createModels();
+ models.setProvider(faux.provider);
+ return { model: faux.getModel(), streamFn: models.streamSimple.bind(models), faux };
+}
+
+/** Emit a complete text delta, then remain in flight until the process dies. */
+function interrupted(): ReturnType<typeof createAssistantMessageEventStream> {
+ const stream = createAssistantMessageEventStream();
+ const partial = fauxAssistantMessage(fauxText('cut off'));
+ stream.push({ type: 'start', partial });
+ stream.push({ type: 'text_start', contentIndex: 0, partial });
+ stream.push({ type: 'text_delta', contentIndex: 0, delta: 'cut off', partial });
+ return stream;
+}
+
+function events(sse: string): LiveEvent[] {
+ return sse.split('\n').flatMap((line) => line.startsWith('data: ') ? [JSON.parse(line.slice(6)) as LiveEvent] : []);
+}
+
+function display(initial: string, live: LiveEvent[]): string {
+ return live.reduce((text, event) => event.type === 'restart' ? event.text ?? '' : event.type === 'text' ? text + event.delta : text, initial);
+}
+
+/** Exercise existing sessions whose recorded inputs predate turn boundaries. */
+function legacyLog(harness: Harness): Harness {
+ return {
+ ...harness,
+ drive: (ctx) => harness.drive({ ...ctx, effect(key, options, run) {
+ if (options.name !== 'model') return ctx.effect(key, options, run);
+ const { turnStart: _, ...input } = options.input as Record<string, unknown>;
+ return ctx.effect(key, { ...options, input }, run);
+ } }),
+ };
+}
+
+/** Read through the crash point, retaining exactly the SSE cursor a client saw. */
+async function beforeCrash(response: Response) {
+ const reader = response.body!.getReader();
+ const decoder = new TextDecoder();
+ let sse = '';
+ for (;;) {
+ const next = await reader.read();
+ if (next.done) throw new Error('stream ended before the crash point');
+ sse += decoder.decode(next.value, { stream: true });
+ if (!events(sse).some((event) => event.delta === 'cut off')) continue;
+ const cursor = [...sse.matchAll(/^id: (.+)$/gm)].at(-1)![1]!;
+ await reader.cancel();
+ return { text: display('', events(sse)), cursor };
+ }
+}
+
+async function setup() {
+ const storage = createWorld({ dataDir: await mkdtemp(join(tmpdir(), 'hw-pi-restart-')), recoverActiveRuns: false });
+ worlds.push(storage);
+ const world = leasedWorld(storage, { visibilityMs: 60, secondMs: 10 });
+ const sessions = (options: PiHarnessOptions, wrap: (harness: Harness) => Harness = (harness) => harness) =>
+ durableSessions({ world, anchors: vercelAnchors(), marks: false, harness: wrap(piHarness(options)) });
+ return { world, sessions };
+}
+
+describe('Pi cursor reconnect after a crash', () => {
+ it.each([0, 1, 2].flatMap((kept) => [false, true].map((legacy) => ({ kept, legacy }))))('replaces an interrupted model reply, keeping $kept committed steps (legacy log: $legacy)', async ({ kept, legacy }) => {
+ const { world, sessions } = await setup();
+ let sideEffects = 0;
+ const tool = {
+ name: 'act', label: 'Act', description: 'Perform an action.', parameters: Type.Object({}), replay: 'never' as const,
+ async execute() {
+ sideEffects++;
+ return { content: [{ type: 'text' as const, text: 'done' }], details: undefined };
+ },
+ };
+ const parts = Array.from({ length: kept }, (_, i) => `Step ${i + 1}. `);
+ const first = model([
+ fauxAssistantMessage(fauxText('An earlier turn.')),
+ ...parts.map((text, i) => fauxAssistantMessage([fauxText(text), fauxToolCall('act', {}, { id: `call_${i}` })], { stopReason: 'toolUse' })),
+ ]);
+ let calls = 0;
+ const a = sessions({ ...first, tools: [tool], streamFn: (...args) => ++calls === kept + 2 ? interrupted() : first.streamFn(...args) }, legacy ? legacyLog : undefined);
+ world.registerHandler!(QUEUE_PREFIX, a.handler);
+ const { sessionId } = await a.start({ prompt: 'earlier' });
+ await world.broker.deliver()!.done;
+ const startIndex = await a.liveEnd(sessionId);
+ await a.send(sessionId, 'perform the actions');
+ const delivery = world.broker.deliver()!;
+ const before = await beforeCrash(await liveResponse(a, sessionId, { startIndex }));
+ expect(before.text).toBe(parts.join('') + 'cut off');
+ delivery.kill();
+
+ const second = model([fauxAssistantMessage(fauxText('Replacement answer.'))]);
+ const b = sessions({ ...second, tools: [tool] });
+ world.registerHandler!(QUEUE_PREFIX, b.handler);
+ const resumed = await chatRoute(b, { authorize: 'public' }).GET(new Request(`http://local/api/chat?sessionId=${sessionId}&cursor=${before.cursor}`));
+ await until(async () => world.broker.deliver(delivery.messageId));
+ const sse = await resumed.text();
+ const live = events(sse);
+ expect(display(before.text, live)).toBe(parts.join('') + 'Replacement answer.');
+ expect(live.filter((event) => event.type === 'restart')).toEqual([{ type: 'restart', text: parts.join(''), parts }]);
+ expect(live.findIndex((event) => event.type === 'restart')).toBeLessThan(live.findIndex((event) => event.type === 'text'));
+ expect(sse).toContain('event: end');
+ expect(second.faux.state.callCount).toBe(1);
+ expect(sideEffects).toBe(kept);
+ const state = await b.inspect(sessionId);
+ expect(state.effects.get(`model:${kept + 1}`)).toMatchObject({ attempt: 2, status: 'completed', output: { content: [{ type: 'text', text: 'Replacement answer.' }] } });
+ expect(state.effectOrder.filter((effect) => effect.name === 'tool.act')).toHaveLength(kept);
+
+ // A completed turn followed by a fresh prompt must not clear that new reply.
+ second.faux.setResponses([fauxAssistantMessage(fauxText('The next turn.'))]);
+ const route = chatRoute(b, { authorize: 'public' });
+ const next = await route.POST(new Request('http://local/api/chat', { method: 'POST', body: JSON.stringify({ sessionId, text: 'again' }) }));
+ await until(async () => world.broker.deliver());
+ const nextSse = await next.text();
+ expect(display('', events(nextSse))).toBe('The next turn.');
+ expect(events(nextSse).some((event) => event.type === 'restart')).toBe(false);
+ expect(nextSse).toContain('event: end');
+ // Reconnecting at the terminal cursor must stop, without replaying the answer.
+ const end = [...sse.matchAll(/^id: (.+;end)$/gm)].at(-1)![1]!;
+ expect((await route.GET(new Request(`http://local/api/chat?sessionId=${sessionId}&cursor=${end}`))).status).toBe(204);
+ });
+
+ it('leaves an uninterrupted answer alone and reconnects from its text cursor without replay', async () => {
+ const { world, sessions } = await setup();
+ const m = model([fauxAssistantMessage(fauxText('cut off'))]);
+ const a = sessions(m);
+ world.registerHandler!(QUEUE_PREFIX, a.handler);
+ const { sessionId } = await a.start();
+ const route = chatRoute(a, { authorize: 'public' });
+ const response = await route.POST(new Request('http://local/api/chat', { method: 'POST', body: JSON.stringify({ sessionId, text: 'hello' }) }));
+ await world.broker.deliver()!.done;
+ const sse = await response.text();
+ expect(display('', events(sse))).toBe('cut off');
+ expect(events(sse).some((event) => event.type === 'restart')).toBe(false);
+ const textRecord = sse.split('\n\n').find((record) => record.includes('"type":"text"'))!;
+ const cursor = /^id: (.+)$/m.exec(textRecord)![1]!;
+ const continuation = await route.GET(new Request(`http://local/api/chat?sessionId=${sessionId}&cursor=${cursor}`));
+ expect(display('cut off', events(await continuation.text()))).toBe('cut off');
+ expect(m.faux.state.callCount).toBe(1);
+ });
+
+ it('retains the text before an interrupted tool and does not repeat its side effect', async () => {
+ const { world, sessions } = await setup();
+ let sideEffects = 0;
+ const tool = (hang: boolean) => ({
+ name: 'act', label: 'Act', description: 'Perform an action.', parameters: Type.Object({}), replay: 'never' as const,
+ async execute() {
+ sideEffects++;
+ if (hang) await new Promise(() => {});
+ return { content: [{ type: 'text' as const, text: 'done' }], details: undefined };
+ },
+ });
+ const a = sessions({ ...model([fauxAssistantMessage([fauxText('cut off'), fauxToolCall('act', {}, { id: 'call_1' })], { stopReason: 'toolUse' })]), tools: [tool(true)] });
+ world.registerHandler!(QUEUE_PREFIX, a.handler);
+ const { sessionId } = await a.start({ prompt: 'act' });
+ const delivery = world.broker.deliver()!;
+ const before = await beforeCrash(await liveResponse(a, sessionId, { startIndex: 0 }));
+ await until(async () => (await a.inspect(sessionId)).effects.get('tool:call_1')?.status === 'running' ? true : undefined);
+ delivery.kill();
+
+ const replacement = model([fauxAssistantMessage(fauxText(' Recovered.'))]);
+ const b = sessions({ ...replacement, tools: [tool(false)] });
+ world.registerHandler!(QUEUE_PREFIX, b.handler);
+ const response = await chatRoute(b, { authorize: 'public' }).GET(new Request(`http://local/api/chat?sessionId=${sessionId}&cursor=${before.cursor}`));
+ await until(async () => world.broker.deliver(delivery.messageId));
+ const sse = await response.text();
+ expect(display(before.text, events(sse))).toBe('cut off Recovered.');
+ expect(events(sse).find((event) => event.type === 'restart')).toEqual({ type: 'restart', text: 'cut off', parts: ['cut off'] });
+ expect(sideEffects).toBe(1);
+ expect((await b.inspect(sessionId)).effects.get('tool:call_1')).toMatchObject({ status: 'failed', error: { interrupted: true } });
+ expect(replacement.faux.state.callCount).toBe(1);
+ expect(sse).toContain('event: end');
+ });
+
+ it('retains completed text before steering continued the same turn without an idle', async () => {
+ const { world, sessions } = await setup();
+ const first = model([]);
+ const stream = createAssistantMessageEventStream();
+ const message = fauxAssistantMessage(fauxText('Before steering. '));
+ stream.push({ type: 'start', partial: message });
+ stream.push({ type: 'text_start', contentIndex: 0, partial: message });
+ stream.push({ type: 'text_delta', contentIndex: 0, delta: 'Before steering. ', partial: message });
+ let calls = 0;
+ let readSteer!: () => void;
+ const steered = new Promise<void>((resolve) => (readSteer = resolve));
+ const a = sessions({ ...first, inboxPollMs: 1, streamFn: () => ++calls === 1 ? stream : interrupted() }, (harness) => ({
+ ...harness,
+ drive: (ctx) => harness.drive({ ...ctx, pending() {
+ const pending = ctx.pending();
+ if (pending.some((submission) => submission.kind === 'steer')) readSteer();
+ return pending;
+ } }),
+ }));
+ world.registerHandler!(QUEUE_PREFIX, a.handler);
+ const { sessionId } = await a.start({ prompt: 'start' });
+ const delivery = world.broker.deliver()!;
+ await until(async () => (await a.inspect(sessionId)).effects.get('model:0')?.status === 'running' ? true : undefined);
+ await a.send(sessionId, 'continue with this guidance', 'steer');
+ await steered;
+ stream.push({ type: 'done', reason: 'stop', message });
+ const before = await beforeCrash(await liveResponse(a, sessionId, { startIndex: 0 }));
+ expect(before.text).toBe('Before steering. cut off');
+ delivery.kill();
+
+ const b = sessions(model([fauxAssistantMessage(fauxText('After steering.'))]));
+ world.registerHandler!(QUEUE_PREFIX, b.handler);
+ const response = await chatRoute(b, { authorize: 'public' }).GET(new Request(`http://local/api/chat?sessionId=${sessionId}&cursor=${before.cursor}`));
+ await until(async () => world.broker.deliver(delivery.messageId));
+ const sse = await response.text();
+ expect(display(before.text, events(sse))).toBe('Before steering. After steering.');
+ expect(events(sse).find((event) => event.type === 'restart')).toEqual({ type: 'restart', text: 'Before steering. ', parts: ['Before steering. '] });
+ expect(sse).toContain('event: end');
+ });
+
+ it('excludes an earlier unavailable turn, including in a legacy log', async () => {
+ const { world, sessions } = await setup();
+ const tool: HostTool = {
+ name: 'unavailable', description: 'Fail this turn.', inputSchema: Type.Object({}), replay: 'safe',
+ async execute(_input, io) {
+ io.ctx.halt?.(new Unavailable('offline fixture is unavailable', { code: 'module_missing' }));
+ throw io.signal.reason;
+ },
+ };
+ const first = model([fauxAssistantMessage([fauxText('Earlier failed turn. '), fauxToolCall('unavailable', {})], { stopReason: 'toolUse' })]);
+ let calls = 0;
+ const a = sessions({ ...first, tools: [tool], streamFn: (...args) => ++calls === 1 ? first.streamFn(...args) : interrupted() }, legacyLog);
+ world.registerHandler!(QUEUE_PREFIX, a.handler);
+ const { sessionId } = await a.start({ prompt: 'fail this turn' });
+ await world.broker.deliver()!.done;
+ expect((await a.inspect(sessionId)).effectOrder.some((effect) => effect.name === 'unavailable')).toBe(true);
+ const startIndex = await a.liveEnd(sessionId);
+ await a.send(sessionId, 'a new turn');
+ const delivery = world.broker.deliver()!;
+ const before = await beforeCrash(await liveResponse(a, sessionId, { startIndex }));
+ delivery.kill();
+
+ const b = sessions(model([fauxAssistantMessage(fauxText('Replacement answer.'))]));
+ world.registerHandler!(QUEUE_PREFIX, b.handler);
+ const response = await chatRoute(b, { authorize: 'public' }).GET(new Request(`http://local/api/chat?sessionId=${sessionId}&cursor=${before.cursor}`));
+ await until(async () => world.broker.deliver(delivery.messageId));
+ const sse = await response.text();
+ expect(display(before.text, events(sse))).toBe('Replacement answer.');
+ expect(events(sse).find((event) => event.type === 'restart')).toEqual({ type: 'restart', text: '', parts: [] });
+ expect(sse).toContain('event: end');
+ });
+
+ it.each([false, true].flatMap((legacy) => [false, true].map((crash) => ({ legacy, crash }))))('excludes a terminal model failure from the next turn (legacy: $legacy, crash: $crash)', async ({ legacy, crash }) => {
+ const { world, sessions } = await setup();
+ let sideEffects = 0;
+ const tool = {
+ name: 'act', label: 'Act', description: 'Perform an action.', parameters: Type.Object({}), replay: 'never' as const,
+ async execute() {
+ sideEffects++;
+ return { content: [{ type: 'text' as const, text: 'done' }], details: undefined };
+ },
+ };
+ const first = model([
+ fauxAssistantMessage([fauxText('Earlier tool text. '), fauxToolCall('act', {}, { id: 'failed_turn_call' })], { stopReason: 'toolUse' }),
+ fauxAssistantMessage(fauxText(''), { stopReason: 'error', errorMessage: 'scripted terminal failure' }),
+ fauxAssistantMessage(fauxText('Fresh answer.')),
+ ]);
+ let calls = 0;
+ const a = sessions({ ...first, tools: [tool], modelAttempts: 1, streamFn: (...args) => ++calls > 2 && crash ? interrupted() : first.streamFn(...args) }, legacy ? legacyLog : undefined);
+ world.registerHandler!(QUEUE_PREFIX, a.handler);
+ const { sessionId } = await a.start({ prompt: 'act, then fail' });
+ await world.broker.deliver()!.done;
+ expect((await a.inspect(sessionId)).effects.get('model:1')).toMatchObject({ status: 'failed', error: { message: 'scripted terminal failure' } });
+ const response = await chatRoute(a, { authorize: 'public' }).POST(new Request('http://local/api/chat', { method: 'POST', body: JSON.stringify({ sessionId, text: 'a fresh prompt' }) }));
+ const delivery = world.broker.deliver()!;
+ if (!crash) {
+ await delivery.done;
+ const sse = await response.text();
+ expect(display('', events(sse))).toBe('Fresh answer.');
+ expect(events(sse).some((event) => event.type === 'restart')).toBe(false);
+ expect(sse).toContain('event: end');
+ } else {
+ const before = await beforeCrash(response);
+ delivery.kill();
+ const b = sessions({ ...model([fauxAssistantMessage(fauxText('Fresh answer.'))]), tools: [tool] });
+ world.registerHandler!(QUEUE_PREFIX, b.handler);
+ const reconnect = await chatRoute(b, { authorize: 'public' }).GET(new Request(`http://local/api/chat?sessionId=${sessionId}&cursor=${before.cursor}`));
+ await until(async () => world.broker.deliver(delivery.messageId));
+ const sse = await reconnect.text();
+ expect(display(before.text, events(sse))).toBe('Fresh answer.');
+ expect(events(sse).find((event) => event.type === 'restart')).toEqual({ type: 'restart', text: '', parts: [] });
+ expect(sse).toContain('event: end');
+ }
+ expect(sideEffects).toBe(1);
+ });
+});