Skip to content

armgit / qm

Public Imported from GitHub
0

Multiplayer agent harness for work.

Re-run retry-safe tool calls transparently when a deploy interrupts a turn (#2055)

Regan Bella096a496a99a
diff --git a/docs/session-tape-spec.md b/docs/session-tape-spec.md
index ef16ed88c..cceb7a296 100644
--- a/docs/session-tape-spec.md
+++ b/docs/session-tape-spec.md
@@ -102,8 +102,12 @@ tape changes the write contract:
1. take the per-session writer lease (existing code, unchanged)
2. rows = SELECT tape; ctx = fold(rows, audience)
3. if this is a RETRY of a reaped run whose user message is already the trailing
- turn on the tape → do not re-append it; append `interrupt` if a tool call
- dangles; prepend the resume note ("outcome unknown, don't redo side effects")
+ turn on the tape → do not re-append it. If a tool call dangles and the model
+ marked it `retrySafe` (every tool schema carries that optional boolean; the
+ `tool_call` entry records it plus the stripped input under `rerun`), the
+ harness re-runs it before the model is called and appends the real
+ `toolResult` row; otherwise append `interrupt` and prepend the resume note
+ ("outcome unknown, don't redo side effects")
4. else append the user message (bare_text + env footer as tagged block)
5. loop: call model; append response rows; run tools; append result rows —
each step's appends committed before the next dispatch
diff --git a/plugins/web-ui/src/chat.ts b/plugins/web-ui/src/chat.ts
index a01111699..576f524ef 100644
--- a/plugins/web-ui/src/chat.ts
+++ b/plugins/web-ui/src/chat.ts
@@ -3079,7 +3079,7 @@ export function createChatSurface(
}
function toolPayloadText(payload: ToolPayload): string {
- const hidden = new Set(["tool", "callId", "workStartedAt", "workFinishedAt", "isError"]);
+ const hidden = new Set(["tool", "callId", "workStartedAt", "workFinishedAt", "isError", "retrySafe", "rerun"]);
const entries = Object.entries(payload as Record<string, unknown>).filter(
([key, value]) => !hidden.has(key) && value !== undefined,
);
diff --git a/src/core/orchestrator.ts b/src/core/orchestrator.ts
index 6fa534371..fb97f928a 100644
--- a/src/core/orchestrator.ts
+++ b/src/core/orchestrator.ts
@@ -126,6 +126,7 @@ import {
filterTapeForAudience,
foldTape,
healFoldInterrupt,
+ openTapeToolCalls,
lastImportLacksScopes,
lintFold,
rehydrateFoldImages,
@@ -154,7 +155,7 @@ import {
withoutAlreadyIngested,
} from "./attachments.ts";
import { parseRef } from "../acl/resource-ref.ts";
-import { findTrailingPartialTurn, resumeNote, turnAtSeq } from "./turn-resume.ts";
+import { findTrailingPartialTurn, resumeNote, resumeStrategy, turnAtSeq } from "./turn-resume.ts";
import type { RecordedTurn } from "./turn-resume.ts";
import {
appendCoverageImport,
@@ -3069,6 +3070,22 @@ export function createOrchestrator(deps: OrchestratorDeps): Orchestrator {
MAX_HISTORY_IMAGE_BYTES,
);
};
+ const partial = isRetry ? (recordedTurn ?? findTrailingPartialTurn(visibleHistory, input.text)) : null;
+ let resumePlan = partial ? resumeStrategy(visibleHistory, partial) : null;
+ if (resumePlan?.kind === "retry") {
+ const retryCallId = resumePlan.call.callId;
+ const tapeAgrees = await deps.sessions
+ .getTape(session.id)
+ .then((rows) => {
+ if (rows.some((row) => row.kind === "message" && row.harness !== undefined && row.harness !== "pi"))
+ return true;
+ const tapeCalls = openTapeToolCalls(rows);
+ return tapeCalls.messages === 0 || (tapeCalls.open.length === 1 && tapeCalls.open[0] === retryCallId);
+ })
+ .catch(swallowAs("orchestrator: resume tape agreement", false));
+ if (!tapeAgrees) resumePlan = { kind: "note" };
+ }
+ const resume = resumePlan && resumePlan.kind !== "restart" ? partial : null;
const tapeRows = await (async () => {
if (memoryHistoryReset || historyHasSecurityTaint || contextWindow.totalEntries > TAPE_IMPORT_MAX_ENTRIES)
return undefined;
@@ -3085,8 +3102,10 @@ export function createOrchestrator(deps: OrchestratorDeps): Orchestrator {
const sameHarness = rows.every(
(row) => row.kind !== "message" || row.harness === undefined || row.harness === "pi",
);
+ const retrying = resumePlan?.kind === "retry";
if (
(!covered || lastImportLacksScopes(rows)) &&
+ !retrying &&
deps.sessionTapeMode === "serve" &&
sameHarness &&
participantHistorySeqs === undefined
@@ -3113,7 +3132,7 @@ export function createOrchestrator(deps: OrchestratorDeps): Orchestrator {
) &&
participantHistorySeqs === undefined;
let fold = eligible ? await rehydrateTape(foldTape(rows)) : undefined;
- if (eligible && rows.length && fold && tapeNeedsInterruptHeal(rows, fold)) {
+ if (eligible && rows.length && fold && !retrying && tapeNeedsInterruptHeal(rows, fold)) {
const interrupt = await deps.sessions.appendTape(lease, {
kind: "context_event",
payload: { event: "interrupt" },
@@ -3241,8 +3260,6 @@ export function createOrchestrator(deps: OrchestratorDeps): Orchestrator {
const approvalReplay =
!!pausedTurnUserEntry &&
String((pausedTurnUserEntry.payload as { text?: string } | null)?.text ?? "").trim() === input.text.trim();
- const partial = isRetry ? (recordedTurn ?? findTrailingPartialTurn(visibleHistory, input.text)) : null;
- const resume = partial && partial.workEntries > 0 ? partial : null;
if (partial)
postKeys.seed(
completedSurfaceEnqueues(
@@ -3267,10 +3284,12 @@ export function createOrchestrator(deps: OrchestratorDeps): Orchestrator {
: `attempt ${input.attempt}; re-running turn at seq ${partial.userSeq} (no recorded work to resume)`,
});
console.error(
- `[orchestrator] turn.resume attempt=${input.attempt} thread=${conversation.threadRef} userSeq=${partial.userSeq} workEntries=${partial.workEntries}`,
+ `[orchestrator] turn.resume attempt=${input.attempt} thread=${conversation.threadRef} userSeq=${partial.userSeq} workEntries=${partial.workEntries} strategy=${resumePlan!.kind}`,
);
}
- let turnInput = partial ? resumeNote({ backgroundJobs: !!backgroundBroker, workRecorded: !!resume }) : baseText;
+ let turnInput = resumePlan
+ ? resumeNote({ strategy: resumePlan, backgroundJobs: !!backgroundBroker })
+ : baseText;
if (partial && !history.some((entry) => entry.seq === partial.userSeq))
turnInput += `\nCurrent request (continue from recorded work; do not restart):\n${baseText}`;
if (releasedToolOutput) {
@@ -3898,6 +3917,7 @@ export function createOrchestrator(deps: OrchestratorDeps): Orchestrator {
...(selectedTape.fold ? { tapeFold: selectedTape.fold } : {}),
}
: {}),
+ ...(resumePlan?.kind === "retry" && !continuation ? { resumeToolCall: resumePlan.call } : {}),
tape: (rec) => {
turnProgress++;
if (rec.kind !== "message" || rec.meta?.bareText === undefined) {
@@ -4021,7 +4041,7 @@ export function createOrchestrator(deps: OrchestratorDeps): Orchestrator {
}
: undefined;
segment = await runHarnessSegment(
- resumeNote() +
+ resumeNote({ cause: "runtime-change" }) +
(recovery ? "\nContext reduced without a new summary." : "\nRuntime handoff completed.") +
" Continue the user's unfinished request using the saved conversation and tool results. Do not repeat completed actions or ask the user to repeat the request." +
(recovery &&
diff --git a/src/core/turn-resume.ts b/src/core/turn-resume.ts
index d31430f77..4b6b16225 100644
--- a/src/core/turn-resume.ts
+++ b/src/core/turn-resume.ts
@@ -70,18 +70,75 @@ export function turnAtSeq(entries: readonly SessionEntry[], userSeq: number): Re
return { userSeq, workEntries, ...(answer ? { answer } : {}) };
}
-export function resumeNote(opts?: { backgroundJobs?: boolean; workRecorded?: boolean }): string {
- if (opts?.workRecorded === false) {
- return `${NOTE_HEAD} your previous attempt at the request above was interrupted before it recorded any work, so there is nothing to pick up. Start the request now.)`;
+export interface ResumableToolCall {
+ callId: string;
+ tool: string;
+ input: Record<string, unknown>;
+}
+
+export type ResumeStrategy = { kind: "restart" } | { kind: "note" } | { kind: "retry"; call: ResumableToolCall };
+
+export function resumeStrategy(entries: readonly SessionEntry[], partial: PartialTurn): ResumeStrategy {
+ if (partial.workEntries === 0) return { kind: "restart" };
+ const turn = entries.filter((e) => e.seq > partial.userSeq);
+ const answered = new Set<unknown>();
+ for (const e of turn) {
+ const payload = e.payload as { callId?: unknown; interrupted?: unknown } | null;
+ if (e.type === "tool_result" && payload?.interrupted !== true) answered.add(payload?.callId);
+ }
+ const dangling = turn.filter(
+ (e) => e.type === "tool_call" && !answered.has((e.payload as { callId?: unknown } | null)?.callId),
+ );
+ if (dangling.length !== 1) return { kind: "note" };
+ const payload = dangling[0]!.payload as {
+ callId?: unknown;
+ retrySafe?: unknown;
+ rerun?: { tool?: unknown; input?: unknown };
+ } | null;
+ const rerun = payload?.rerun;
+ if (
+ payload?.retrySafe !== true ||
+ typeof payload.callId !== "string" ||
+ typeof rerun?.tool !== "string" ||
+ rerun.input === null ||
+ typeof rerun.input !== "object" ||
+ Array.isArray(rerun.input)
+ )
+ return { kind: "note" };
+ return {
+ kind: "retry",
+ call: { callId: payload.callId, tool: rerun.tool, input: rerun.input as Record<string, unknown> },
+ };
+}
+
+const ROUTINE = "This was a routine platform deploy; it is common and almost never worth mentioning to the user.";
+const CONTINUE = "Continue from where you left off; don't start over or repeat completed steps.)";
+
+export function resumeNote(opts?: {
+ strategy?: ResumeStrategy;
+ backgroundJobs?: boolean;
+ cause?: "deploy" | "runtime-change";
+}): string {
+ const strategy = opts?.strategy ?? { kind: "note" };
+ if (strategy.kind === "restart") {
+ return `${NOTE_HEAD} your previous attempt at the request above was interrupted before it recorded any work (a routine platform deploy), so there is nothing to pick up. Start the request now.)`;
}
- const parts = [
- `${NOTE_HEAD} your previous attempt at the request above was interrupted mid-turn.`,
- "Your work up to the interruption is recorded above; a tool result marked interrupted has an",
- "unknown outcome, so check what actually happened before redoing anything with side effects.",
- ];
+ const parts =
+ strategy.kind === "retry"
+ ? [
+ `${NOTE_HEAD} your previous attempt at the request above was paused mid-turn and has resumed.`,
+ ROUTINE,
+ "The result of the tool call that was in flight is recorded above.",
+ ]
+ : [
+ `${NOTE_HEAD} your previous attempt at the request above was interrupted mid-turn.`,
+ opts?.cause === "runtime-change" ? "" : ROUTINE,
+ "Your work up to the interruption is recorded above; a tool result marked interrupted has an",
+ "unknown outcome, so check what actually happened before redoing anything with side effects.",
+ ];
if (opts?.backgroundJobs) {
parts.push("Background jobs on your computer kept running — `background` list/poll to check on them.");
}
- parts.push("Continue from where you left off; don't start over or repeat completed steps.)");
- return parts.join(" ");
+ parts.push(CONTINUE);
+ return parts.filter(Boolean).join(" ");
}
diff --git a/src/harness/agent-tools.ts b/src/harness/agent-tools.ts
index d14760bc6..fb9468b1c 100644
--- a/src/harness/agent-tools.ts
+++ b/src/harness/agent-tools.ts
@@ -432,8 +432,15 @@ export function createAgentTools(ref: ToolContextRef, opts?: AgentToolsOptions):
await ref.emit({ type, payload, scopeLabel });
};
- const recordCall = (callId: string, payload: Record<string, unknown>): Promise<void> =>
- log("tool_call", { ...sandboxLog(payload), callId });
+ const retryMarks = new Map<string, RetryMark>();
+ const recordCall = (callId: string, payload: Record<string, unknown>): Promise<void> => {
+ const mark = retryMarks.get(callId);
+ return log("tool_call", {
+ ...sandboxLog(payload),
+ callId,
+ ...(mark ? { retrySafe: mark.safe, ...(mark.rerun ? { rerun: mark.rerun } : {}) } : {}),
+ });
+ };
const resultQueue = createKeyedQueue();
const quarantinedMessages = new Set<string>();
@@ -4347,10 +4354,65 @@ export function createAgentTools(ref: ToolContextRef, opts?: AgentToolsOptions):
const mcpNames = new Set(mcpTools.map((t) => t.name));
const active = opts?.readOnly ? tools.filter((t) => READ_ONLY_TOOL_NAMES.has(t.name) || mcpNames.has(t.name)) : tools;
return active.map((t) =>
- withRuntimeBarrier(withToolBodyTiming(withToolApprovalGate(t, ref, { recordCall, recordResult }), ref), ref),
+ withRetrySafety(
+ withRuntimeBarrier(withToolBodyTiming(withToolApprovalGate(t, ref, { recordCall, recordResult }), ref), ref),
+ retryMarks,
+ ),
);
}
+const RETRY_SAFE_FIELD = "retrySafe";
+
+const RETRY_SAFE_SCHEMA = Type.Optional(
+ Type.Boolean({
+ description:
+ "true if re-running this exact call is harmless (reads, searches, idempotent writes) so the platform may re-run it after an interruption; false if it would duplicate a side effect (sending a message, charging, creating a record).",
+ }),
+);
+const RERUN_INPUT_MAX_CHARS = 4_000;
+
+interface RetryMark {
+ safe: boolean;
+ rerun?: { tool: string; input: Record<string, unknown> };
+}
+
+function withRetrySafeField(parameters: unknown): unknown | null {
+ if (!isObj(parameters) || parameters.type !== "object" || !isObj(parameters.properties)) return null;
+ if (RETRY_SAFE_FIELD in parameters.properties) return null;
+ return { ...parameters, properties: { ...parameters.properties, [RETRY_SAFE_FIELD]: RETRY_SAFE_SCHEMA } };
+}
+
+function stripRetrySafeField(params: unknown): { params: unknown; retrySafe?: boolean } {
+ if (!isObj(params) || !(RETRY_SAFE_FIELD in params)) return { params };
+ const { [RETRY_SAFE_FIELD]: retrySafe, ...rest } = params;
+ return { params: rest, ...(typeof retrySafe === "boolean" ? { retrySafe } : {}) };
+}
+
+function withRetrySafety(tool: ToolDefinition, marks: Map<string, RetryMark>): ToolDefinition {
+ const parameters = withRetrySafeField(tool.parameters);
+ if (!parameters) return tool;
+ const inner = tool.execute.bind(tool);
+ return {
+ ...tool,
+ parameters: parameters as ToolDefinition["parameters"],
+ async execute(callId: string, params: unknown, ...rest: unknown[]) {
+ const stripped = stripRetrySafeField(params);
+ if (stripped.retrySafe !== undefined && isObj(stripped.params)) {
+ const replayable = stripped.retrySafe && JSON.stringify(stripped.params).length <= RERUN_INPUT_MAX_CHARS;
+ marks.set(callId, {
+ safe: stripped.retrySafe,
+ ...(replayable ? { rerun: { tool: tool.name, input: stripped.params } } : {}),
+ });
+ }
+ try {
+ return await (inner as (...args: unknown[]) => unknown)(callId, stripped.params, ...rest);
+ } finally {
+ marks.delete(callId);
+ }
+ },
+ } as ToolDefinition;
+}
+
const TOOL_APPROVAL_EXEMPT = new Set(["finish_silently"]);
const STRICT_TOOL_APPROVAL_REASON = "strict posture: this tool call requires human approval";
diff --git a/src/harness/claude-harness.ts b/src/harness/claude-harness.ts
index e2f184bf6..2cfc1692f 100644
--- a/src/harness/claude-harness.ts
+++ b/src/harness/claude-harness.ts
@@ -50,6 +50,7 @@ import {
type SteerIntake,
transitionTask,
type HarnessToolPlumbing,
+ withResumedToolCall,
} from "./harness-shared.ts";
import {
recordedMessageTimestamps,
@@ -336,6 +337,7 @@ export function createClaudeHarness(opts: ClaudeHarnessOptions = {}): Harness {
const controller = new AbortController();
ref.abortSignal = controller.signal;
const bridged = toolsEnabled ? bridgedTools(ref, harnessToolOptions(opts, turn)) : [];
+ turn = await withResumedToolCall(turn, ref, bridged);
const bridgedNames = bridged.map((definition) => `mcp__qm__${definition.name}`);
const childToolNames = bridged
.filter((definition) => nativeChildToolAllowed(definition.name))
diff --git a/src/harness/codex-harness.ts b/src/harness/codex-harness.ts
index 5a1504328..cb39c47cb 100644
--- a/src/harness/codex-harness.ts
+++ b/src/harness/codex-harness.ts
@@ -37,6 +37,7 @@ import {
transitionTask,
type BridgedTool,
type HarnessToolPlumbing,
+ withResumedToolCall,
} from "./harness-shared.ts";
import {
recordedMessageTimestamps,
@@ -937,6 +938,7 @@ export function createCodexHarness(opts: CodexHarnessOptions = {}): Harness {
toolAbort = new AbortController();
ref.abortSignal = toolAbort.signal;
tools = toolsEnabled ? bridgedTools(ref, harnessToolOptions(opts, turn)) : [];
+ turn = await withResumedToolCall(turn, ref, tools);
dynamicTools = tools.map((tool) => ({
type: "function",
name: tool.name,
diff --git a/src/harness/context-compaction.ts b/src/harness/context-compaction.ts
index ef8822409..e470db3ad 100644
--- a/src/harness/context-compaction.ts
+++ b/src/harness/context-compaction.ts
@@ -17,7 +17,7 @@ function headTailSlice(s: string, maxChars: number, tailChars: number): string {
}
export const INTERRUPTED_TOOL_RESULT =
- "[interrupted — the platform restarted while this tool call was running and its outcome was not recorded. Check what actually happened before redoing anything with side effects.]";
+ "[interrupted — a routine platform deploy restarted the agent while this tool call was running and its outcome was not recorded. Check what actually happened before redoing anything with side effects.]";
export const CONTEXT_SUMMARY_HEADER =
"[Earlier conversation summary — an index of turns compacted out of your context. Conversation turns and tool calls can be reopened with the history tool (seq parameter; a very long entry returns as head and tail), or searched (query). Tool results are excluded.]";
@@ -101,8 +101,9 @@ export function compactTranscript(history: SessionEntry[]): string {
const resultByCallId = new Map<string, true>();
for (const entry of history) {
if (entry.type !== "tool_result") continue;
- const cid = (entry.payload as { callId?: unknown } | null)?.callId;
- if (typeof cid === "string" && cid) resultByCallId.set(cid, true);
+ const payload = entry.payload as { callId?: unknown; interrupted?: unknown } | null;
+ if (typeof payload?.callId === "string" && payload.callId && payload.interrupted !== true)
+ resultByCallId.set(payload.callId, true);
}
const lines: string[] = [];
const push = (line: string) => lines.push(headTailSlice(line, MAX_COMPACT_ENTRY_CHARS, COMPACT_ENTRY_TAIL_CHARS));
diff --git a/src/harness/harness-shared.ts b/src/harness/harness-shared.ts
index 6cd554d39..4f0a5cd13 100644
--- a/src/harness/harness-shared.ts
+++ b/src/harness/harness-shared.ts
@@ -12,7 +12,8 @@ import { rehydrateOpenGoal } from "./goal.ts";
import type { HarnessLlmRequestRecord, HarnessModelUtilities, HarnessTurnInput, HarnessTurnResult } from "./harness.ts";
import { sanitizeTitle, TITLE_GENERATION_PROMPT, titleUserPrompt } from "./pi-harness.ts";
import { tapeCheckpointPayload, tapeEntryMirrorRecord, type NewTapeRecord } from "../sessions/session-store.ts";
-import { swallow, swallowAs } from "../util/errors.ts";
+import { errMessage, swallow, swallowAs } from "../util/errors.ts";
+import { INTERRUPTED_TOOL_RESULT } from "./context-compaction.ts";
export interface HarnessToolPlumbing {
scratchExec?: boolean;
@@ -162,6 +163,93 @@ export function bridgedTools(ref: ToolContextRef, options: AgentToolsOptions): B
return createAgentTools(ref, options) as unknown as BridgedTool[];
}
+export interface ResumedToolResult {
+ history: SessionEntry[];
+ message: {
+ role: "toolResult";
+ toolCallId: string;
+ toolName: string;
+ content: NonNullable<Awaited<ReturnType<BridgedTool["execute"]>>["content"]>;
+ isError: boolean;
+ timestamp: number;
+ };
+}
+
+export async function withResumedToolCall(
+ turn: HarnessTurnInput,
+ ref: ToolContextRef,
+ tools: readonly BridgedTool[],
+): Promise<HarnessTurnInput> {
+ const resumed = await resumeInterruptedToolCall(turn, ref, tools);
+ return resumed ? { ...turn, history: resumed.history } : turn;
+}
+
+export async function resumeInterruptedToolCall(
+ turn: HarnessTurnInput,
+ ref: ToolContextRef,
+ tools: readonly BridgedTool[],
+): Promise<ResumedToolResult | null> {
+ const call = turn.resumeToolCall;
+ if (!call) return null;
+ const tool = tools.find((candidate) => candidate.name === call.tool);
+ const emit = ref.emit;
+ const abortSignal = ref.abortSignal;
+ let recorded: SessionEntry | undefined;
+ ref.emit = async (entry) => {
+ if (entry.type === "tool_call") return;
+ const appended = await emit?.(entry);
+ if (entry.type === "tool_result" && !recorded) recorded = appended as SessionEntry;
+ return appended;
+ };
+ ref.abortSignal = turn.cancel;
+ let content: ResumedToolResult["message"]["content"] = [{ type: "text", text: INTERRUPTED_TOOL_RESULT }];
+ let isError = true;
+ try {
+ if (tool && !turn.cancel?.aborted) {
+ content = (await tool.execute(call.callId, call.input)).content ?? [];
+ isError = false;
+ } else {
+ console.error(
+ `[harness] resume: ${tool ? "turn cancelled" : `tool ${call.tool} unavailable on this turn`}; recording call ${call.callId} as interrupted`,
+ );
+ }
+ if (!recorded) {
+ await ref.emit({
+ type: "tool_result",
+ payload: { tool: call.tool, callId: call.callId, isError, result: content[0]?.text ?? "", interrupted: true },
+ scopeLabel: turn.scopeLabel,
+ });
+ }
+ } catch (error) {
+ const payload = recorded?.payload as { result?: unknown; isError?: unknown } | undefined;
+ isError = payload ? payload.isError === true : true;
+ content = [{ type: "text", text: payload ? String(payload.result ?? "") : `[error] ${errMessage(error)}` }];
+ if (!recorded)
+ await ref.emit({
+ type: "tool_result",
+ payload: { tool: call.tool, callId: call.callId, isError, result: content[0]!.text },
+ scopeLabel: turn.scopeLabel,
+ });
+ } finally {
+ ref.emit = emit;
+ ref.abortSignal = abortSignal;
+ }
+ if (!recorded) return null;
+ if (!isError)
+ console.log(`[harness] resume: re-ran retry-safe ${call.tool} call ${call.callId} as seq ${recorded.seq}`);
+ return {
+ history: [...turn.history, recorded],
+ message: {
+ role: "toolResult",
+ toolCallId: call.callId,
+ toolName: call.tool,
+ content,
+ isError,
+ timestamp: recorded.createdAt,
+ },
+ };
+}
+
export function bridgedToolText(result: Awaited<ReturnType<BridgedTool["execute"]>>): string {
return (result.content ?? [])
.filter((item): item is { type?: string; text: string } => typeof item.text === "string")
diff --git a/src/harness/harness.ts b/src/harness/harness.ts
index ad9529d6d..33d9183a1 100644
--- a/src/harness/harness.ts
+++ b/src/harness/harness.ts
@@ -1,3 +1,4 @@
+import type { ResumableToolCall } from "../core/turn-resume.ts";
import type { DocumentInput } from "../core/document-inputs.ts";
import type { RuntimeControl, HarnessHandoff } from "./runtime-types.ts";
import type {
@@ -137,6 +138,7 @@ export interface HarnessTurnInput {
tapeRows?: TapeRecord[];
tapeMode?: "shadow" | "serve";
tapeFold?: unknown[];
+ resumeToolCall?: ResumableToolCall;
scopeLabel: ScopeId;
orgScopeId: ScopeId;
providerKeys?: ProviderKeys;
diff --git a/src/harness/opencode-harness.ts b/src/harness/opencode-harness.ts
index 59fa5ddfb..fa6970cec 100644
--- a/src/harness/opencode-harness.ts
+++ b/src/harness/opencode-harness.ts
@@ -48,6 +48,7 @@ import {
type SteerIntake,
type BridgedTool,
type HarnessToolPlumbing,
+ withResumedToolCall,
} from "./harness-shared.ts";
import { recordedMessageTimestamps, reconstructMessagesFromHistory } from "./replay.ts";
import { countTokens } from "../util/tokens.ts";
@@ -899,6 +900,7 @@ export function createOpenCodeHarness(opts: OpenCodeHarnessOptions = {}): Harnes
const controller = new AbortController();
ref.abortSignal = controller.signal;
const tools = bridgedTools(ref, harnessToolOptions(opts, turn));
+ turn = await withResumedToolCall(turn, ref, tools);
const userEntry = await turn.emit({
type: "user",
payload: {
diff --git a/src/harness/pi-harness.ts b/src/harness/pi-harness.ts
index a5d8d13dd..70cb3267b 100644
--- a/src/harness/pi-harness.ts
+++ b/src/harness/pi-harness.ts
@@ -1,4 +1,4 @@
-import { recordSteerIntake, type SteerIntake } from "./harness-shared.ts";
+import { recordSteerIntake, resumeInterruptedToolCall, type BridgedTool, type SteerIntake } from "./harness-shared.ts";
import { withDocumentInputs, type DocumentModel } from "./document-inputs.ts";
import { gatewayModelsJson, gatewayModelsVersion } from "../model/gateway-models.ts";
import { Type } from "typebox";
@@ -11,6 +11,7 @@ import {
ModelRuntime,
SessionManager,
SettingsManager,
+ type ToolDefinition,
type AgentSession,
} from "@earendil-works/pi-coding-agent";
import {
@@ -44,7 +45,7 @@ const TURN_EFFORT_LEVELS = new Set<string>([
"default",
"adaptive",
]);
-import type { ClientToolDeclaration, ConversationTurn, ScopeId, SessionEntry } from "../types.ts";
+import type { ConversationTurn, ScopeId, SessionEntry } from "../types.ts";
import type {
GapPhase,
GapPhases,
@@ -1605,25 +1606,41 @@ export function createPiHarness(opts?: PiHarnessOptions): Harness {
const controlTools = opts?.controlTools ?? false;
const defaultTurnWallClockMs = opts?.turnWallClockMs ?? CONFIG_DEFAULTS.turnWallClockSec * 1000;
const signals = opts?.signals;
+ function createTurnTools(ref: ToolContextRef, turn: HarnessTurnInput): ToolDefinition[] {
+ return createAgentTools(ref, {
+ sessionTools: Boolean(turn.tools.sessionSyscalls),
+ delegateWork: turn.delegateWork === true,
+ scratchExec,
+ ownerAuthExec,
+ reachExec,
+ ...(mcpTools ? { mcpTools } : {}),
+ controlTools,
+ ...(turn.commandCredentialHandles?.length ? { commandCredentialHandles: turn.commandCredentialHandles } : {}),
+ ...(turn.surfaceTools ? { surfaceTools: true } : {}),
+ ...(turn.surfaceName ? { surfaceName: turn.surfaceName } : {}),
+ ...(turn.clientTools?.length ? { clientTools: turn.clientTools } : {}),
+ ...(turn.readOnly ? { readOnly: true } : {}),
+ ...(opts?.execTimeoutMs !== undefined ? { execTimeoutMs: opts.execTimeoutMs } : {}),
+ ...(opts?.execTimeoutCeilingMs !== undefined ? { execTimeoutCeilingMs: opts.execTimeoutCeilingMs } : {}),
+ ...(opts?.backgroundJobTtlMs !== undefined ? { backgroundJobTtlMs: opts.backgroundJobTtlMs } : {}),
+ ...(opts?.backgroundJobTtlMaxMs !== undefined ? { backgroundJobTtlMaxMs: opts.backgroundJobTtlMaxMs } : {}),
+ sandboxResources: opts?.sandboxResources,
+ });
+ }
async function createTurnSession(
model: Model<Api>,
sessionId: string,
systemPrompt: string,
history: SessionEntry[],
- priorTurns?: ConversationTurn[],
- readOnly?: boolean,
- surfaceTools?: boolean,
- surfaceName?: string,
- turnScope?: ScopeId,
- commandCredentialHandles?: readonly string[],
- tapeRows?: TapeRecord[],
- tapeMode?: "shadow" | "serve",
- tapeFold?: unknown[],
- tape?: HarnessTurnInput["tape"],
- turnProviderKeys?: ProviderKeys,
- sessionTools = false,
- delegateWork = false,
- clientTools?: readonly ClientToolDeclaration[],
+ priorTurns: ConversationTurn[] | undefined,
+ turnScope: ScopeId,
+ tapeRows: TapeRecord[] | undefined,
+ tapeMode: "shadow" | "serve" | undefined,
+ tapeFold: unknown[] | undefined,
+ tape: HarnessTurnInput["tape"],
+ turnProviderKeys: ProviderKeys | undefined,
+ ref: TurnSession["ref"],
+ tools: ToolDefinition[],
): Promise<{ entry: TurnSession; compileMs: number }> {
const compileStart = Date.now();
let reconstructed: PiReplayMessage[] | null;
@@ -1661,7 +1678,6 @@ export function createPiHarness(opts?: PiHarnessOptions): Harness {
turnProviderKeys ? undefined : modelGateway,
systemCacheSplit ? "long" : undefined,
);
- const ref: TurnSession["ref"] = { current: null };
const { resourceLoader, settingsManager, cwd, agentDir, ephemeralCwd } = await createIsolatedResources(
tempDirPrefix,
composedPrompt,
@@ -1675,25 +1691,7 @@ export function createPiHarness(opts?: PiHarnessOptions): Harness {
modelRuntime,
resourceLoader,
settingsManager,
- customTools: createAgentTools(ref, {
- sessionTools,
- delegateWork,
- scratchExec,
- ownerAuthExec,
- reachExec,
- ...(mcpTools ? { mcpTools } : {}),
- controlTools,
- ...(commandCredentialHandles?.length ? { commandCredentialHandles } : {}),
- ...(surfaceTools ? { surfaceTools: true } : {}),
- ...(surfaceName ? { surfaceName } : {}),
- ...(clientTools?.length ? { clientTools } : {}),
- ...(readOnly ? { readOnly: true } : {}),
- ...(opts?.execTimeoutMs !== undefined ? { execTimeoutMs: opts.execTimeoutMs } : {}),
- ...(opts?.execTimeoutCeilingMs !== undefined ? { execTimeoutCeilingMs: opts.execTimeoutCeilingMs } : {}),
- ...(opts?.backgroundJobTtlMs !== undefined ? { backgroundJobTtlMs: opts.backgroundJobTtlMs } : {}),
- ...(opts?.backgroundJobTtlMaxMs !== undefined ? { backgroundJobTtlMaxMs: opts.backgroundJobTtlMaxMs } : {}),
- sandboxResources: opts?.sandboxResources,
- }),
+ customTools: tools,
noTools: "builtin",
sessionManager: SessionManager.inMemory(undefined, { id: sessionId }),
cwd,
@@ -1724,7 +1722,7 @@ export function createPiHarness(opts?: PiHarnessOptions): Harness {
await tape({
kind: "context_event",
payload: { event: "legacy_import", messages: seeded },
- scopeLabel: turnScope!,
+ scopeLabel: turnScope,
});
} catch (err) {
removeIsolatedDirs({ agentDir, ephemeralCwd });
@@ -1838,50 +1836,61 @@ export function createPiHarness(opts?: PiHarnessOptions): Harness {
]),
},
{
- async runTurn(turn: HarnessTurnInput): Promise<HarnessTurnResult> {
- const desiredModelId = turn.runtime?.modelId ?? resolveModelId(turn.scopeLabel);
- const baseModel = getRequiredModel(desiredModelId, !turn.providerKeys);
- const turnModelGateway = turn.providerKeys ? undefined : modelGateway;
- const wantFast = wantsFastMode(turn.runtime?.fastMode, desiredModelId);
+ async runTurn(dispatched: HarnessTurnInput): Promise<HarnessTurnResult> {
+ const desiredModelId = dispatched.runtime?.modelId ?? resolveModelId(dispatched.scopeLabel);
+ const baseModel = getRequiredModel(desiredModelId, !dispatched.providerKeys);
+ const turnModelGateway = dispatched.providerKeys ? undefined : modelGateway;
+ const wantFast = wantsFastMode(dispatched.runtime?.fastMode, desiredModelId);
+ const ref: TurnSession["ref"] = {
+ current: dispatched.tools,
+ documents: dispatched.documents,
+ runtimeHandoff: undefined,
+ runtimeMutationPending: false,
+ runtimeInFlight: new Set(),
+ runtimeRunId: dispatched.runId,
+ runtimeActorId: dispatched.runtimeActorId,
+ pendingApprovals: [],
+ pausedOnApproval: undefined,
+ silentRequested: false,
+ pollFire: !!dispatched.pollFire,
+ screenToolResult: dispatched.screenToolResult,
+ verifyGoal: dispatched.verifyGoal,
+ emit: dispatched.emit,
+ scopeLabel: dispatched.scopeLabel,
+ orgScopeId: dispatched.orgScopeId,
+ toolApprovalGate: dispatched.toolApprovalGate,
+ };
+ const tools = createTurnTools(ref, dispatched);
+ const resumed = await resumeInterruptedToolCall(dispatched, ref, tools as unknown as BridgedTool[]);
+ const turn = resumed ? { ...dispatched, history: resumed.history } : dispatched;
+ if (resumed) {
+ const callId = resumed.message.toolCallId;
+ const resultScope = ref.tapeResultScopes?.get(callId);
+ ref.tapeResultScopes?.delete(callId);
+ await turn.tape?.({
+ kind: "message",
+ harness: "pi",
+ payload: stripImageBytes(resumed.message),
+ scopeLabel: resultScope ?? turn.scopeLabel,
+ });
+ }
const { entry, compileMs } = await createTurnSession(
withRequestHeaders(baseModel, !turnModelGateway?.models[desiredModelId], wantFast),
turn.session.id,
turn.systemPrompt,
turn.history,
turn.priorTurns,
- turn.readOnly,
- turn.surfaceTools,
- turn.surfaceName,
turn.scopeLabel,
- turn.commandCredentialHandles,
turn.tapeRows,
turn.tapeMode,
turn.tapeFold,
turn.tape,
turn.providerKeys,
- Boolean(turn.tools.sessionSyscalls),
- turn.delegateWork,
- turn.clientTools,
+ ref,
+ tools,
);
try {
const turnWallClockMs = turn.turnWallClockMs ?? defaultTurnWallClockMs;
- entry.ref.current = turn.tools;
- entry.ref.documents = turn.documents;
- entry.ref.runtimeHandoff = undefined;
- entry.ref.runtimeMutationPending = false;
- entry.ref.runtimeInFlight = new Set();
- entry.ref.runtimeRunId = turn.runId;
- entry.ref.runtimeActorId = turn.runtimeActorId;
- entry.ref.pendingApprovals = [];
- entry.ref.pausedOnApproval = undefined;
- entry.ref.silentRequested = false;
- entry.ref.pollFire = !!turn.pollFire;
- entry.ref.screenToolResult = turn.screenToolResult;
- entry.ref.verifyGoal = turn.verifyGoal;
- entry.ref.emit = turn.emit;
- entry.ref.scopeLabel = turn.scopeLabel;
- entry.ref.orgScopeId = turn.orgScopeId;
- entry.ref.toolApprovalGate = turn.toolApprovalGate;
const activeModel = entry.agentSession.model as { id?: string; headers?: Record<string, string> } | undefined;
entry.ref.fast = wantFast;
diff --git a/src/harness/replay.ts b/src/harness/replay.ts
index 2aa6c3e32..cc1f2381a 100644
--- a/src/harness/replay.ts
+++ b/src/harness/replay.ts
@@ -4,6 +4,7 @@ import { messageRevision, renderMessageRevision } from "../core/message-revision
import { CONTEXT_SUMMARY_HEADER, forModelContext, INTERRUPTED_TOOL_RESULT } from "./context-compaction.ts";
import { contextSummaryPayload } from "../sessions/session-store.ts";
import { isoFromTs, messageTag } from "../util/message-tag.ts";
+import { isObj } from "../util/objects.ts";
import { TAPE_IMPORT_MAX_ENTRIES } from "../sessions/session-store.ts";
import type { Lease, SessionStore, TapeRecord } from "../sessions/session-store.ts";
@@ -187,8 +188,10 @@ export function reconstructMessagesFromHistory(history: readonly SessionEntry[])
if (!cid) continue;
const re = resultByCallId.get(cid);
const name = typeof p.tool === "string" ? p.tool : "tool";
- const { tool: _t, callId: _c, ...recorded } = p;
+ const { tool: _t, callId: _c, rerun, ...recorded } = p;
const args = (() => {
+ const input = (rerun as { input?: unknown } | undefined)?.input;
+ if (isObj(input)) return { ...input, retrySafe: true };
if (typeof recorded.bytes !== "number" || typeof recorded.text === "string") return recorded;
if (recorded.action !== "post" && recorded.action !== "reach" && recorded.action !== "edit") return recorded;
const { bytes: _b, files: recordedFiles, ...rest } = recorded;
@@ -211,8 +214,10 @@ export function reconstructMessagesFromHistory(history: readonly SessionEntry[])
role: "toolResult",
toolCallId: cid,
toolName: name,
- content: [{ type: "text", text: String(rp.result ?? "") }],
- isError: rp.isError === true,
+ content: [
+ { type: "text", text: rp.interrupted === true ? INTERRUPTED_TOOL_RESULT : String(rp.result ?? "") },
+ ],
+ isError: rp.isError === true || rp.interrupted === true,
timestamp: re.createdAt,
});
} else {
diff --git a/src/harness/tape-fold.ts b/src/harness/tape-fold.ts
index d1ffe210b..674b6bcd6 100644
--- a/src/harness/tape-fold.ts
+++ b/src/harness/tape-fold.ts
@@ -261,11 +261,17 @@ export function foldTape(rows: readonly TapeRecord[]): unknown[] {
}
continue;
}
- if (row.kind === "message" && row.payload != null) f.out.push(row.payload);
+ if (row.kind === "message" && row.payload != null) f.out.push(interruptedResultPlaceholder(row.payload));
}
return f.out;
}
+function interruptedResultPlaceholder(message: unknown): unknown {
+ const msg = message as { role?: string; interrupted?: unknown; content?: unknown };
+ if (msg?.role !== "toolResult" || msg.interrupted !== true) return message;
+ return { ...msg, content: [{ type: "text", text: INTERRUPTED_TOOL_RESULT }], isError: true };
+}
+
const LEGACY_CONTINUATION_LINE = "(continuing after the tool result above)";
function healLegacyAssistantVoice(messages: readonly unknown[]): unknown[] {
@@ -319,6 +325,22 @@ export function planTapeSeed(
return { seed: mode === "serve" && lint.ok && fold.length ? fold : null, lint, fold };
}
+export function openTapeToolCalls(rows: readonly TapeRecord[]): { messages: number; open: string[] } {
+ const fold = foldTape(rows);
+ const answered = new Set<string>();
+ const calls: string[] = [];
+ for (const m of fold) {
+ const msg = m as { role?: string; toolCallId?: string; content?: unknown };
+ if (msg?.role === "toolResult" && typeof msg.toolCallId === "string") answered.add(msg.toolCallId);
+ if (msg?.role !== "assistant" || !Array.isArray(msg.content) || assistantDroppedAtReplay(m)) continue;
+ for (const block of msg.content) {
+ const b = block as { type?: string; id?: string };
+ if (b?.type === "toolCall" && typeof b.id === "string") calls.push(b.id);
+ }
+ }
+ return { messages: fold.length, open: calls.filter((id) => !answered.has(id)) };
+}
+
export function tapeNeedsInterruptHeal(rows: readonly TapeRecord[], folded?: readonly unknown[]): boolean {
const problems = lintFold(folded ?? foldTape(rows)).problems;
return problems.length > 0 && problems.every((p) => p.startsWith("end:"));
diff --git a/test/agent-tools.test.ts b/test/agent-tools.test.ts
index 6a53561b5..1f726fd63 100644
--- a/test/agent-tools.test.ts
+++ b/test/agent-tools.test.ts
@@ -4075,3 +4075,53 @@ test("background process guidance reflects the configured sandbox token lifetime
assert.match(unlimited, /does not expire those turn tokens/);
assert.doesNotMatch(unlimited, /turn tokens expire \d+ hours/);
});
+
+test("every tool schema carries the optional retrySafe flag, which is stripped before the tool runs and recorded on the call", async () => {
+ const emitted: Emitted[] = [];
+ const ref: ToolContextRef = {
+ current: fakeToolContext(),
+ emit: (e) => {
+ emitted.push(e as Emitted);
+ },
+ scopeLabel: "personal:U1",
+ };
+ const tools = createAgentTools(ref, {
+ controlTools: true,
+ clientTools: [
+ { name: "page_state", description: "Read page state", inputSchema: { type: "object", properties: {} } },
+ ],
+ });
+ for (const tool of tools) {
+ const schema = tool.parameters as { properties?: Record<string, { type?: string }>; required?: string[] };
+ assert.equal(schema.properties?.retrySafe?.type, "boolean", `${tool.name} exposes retrySafe`);
+ assert.ok(!schema.required?.includes("retrySafe"), `${tool.name} keeps retrySafe optional`);
+ }
+ const history = tools.find((t) => t.name === "history")!;
+ assert.ok(Check(history.parameters, { query: "budget", retrySafe: true }));
+
+ await callWith(history, "safe", { query: "budget", retrySafe: true });
+ await callWith(history, "unsafe", { seq: 3, retrySafe: false });
+ await callWith(history, "unmarked", { seq: 3 });
+
+ const calls = emitted.filter((e) => e.type === "tool_call");
+ assert.deepEqual(
+ calls.map((e) => e.payload),
+ [
+ {
+ tool: "history",
+ query: "budget",
+ callId: "safe",
+ retrySafe: true,
+ rerun: { tool: "history", input: { query: "budget" } },
+ },
+ { tool: "history", seq: 3, callId: "unsafe", retrySafe: false },
+ { tool: "history", seq: 3, callId: "unmarked" },
+ ],
+ );
+ assert.equal(emitted.filter((e) => e.type === "tool_result").length, 3);
+ assert.doesNotMatch(
+ JSON.stringify(emitted.filter((e) => e.type === "tool_result")),
+ /retrySafe/,
+ "the flag never reaches the tool body or its result",
+ );
+});
diff --git a/test/compact-transcript.test.ts b/test/compact-transcript.test.ts
index dd871377d..1c6932007 100644
--- a/test/compact-transcript.test.ts
+++ b/test/compact-transcript.test.ts
@@ -102,3 +102,27 @@ test("compactTranscript omits the stamp when an entry has no real timestamp", ()
assert.match(out, /^user#1: hello$/m);
assert.match(out, /^user#2: again$/m);
});
+
+test("a tool_result marked interrupted still leaves its call reported as interrupted in the compaction transcript", () => {
+ const out = compactTranscript([
+ {
+ sessionId: "s",
+ seq: 1,
+ parentSeq: null,
+ type: "tool_call",
+ payload: { tool: "execute", callId: "c1", command: "make" },
+ scopeLabel: "org:o",
+ createdAt: 1,
+ },
+ {
+ sessionId: "s",
+ seq: 2,
+ parentSeq: null,
+ type: "tool_result",
+ payload: { callId: "c1", result: "[exit 143]", interrupted: true },
+ scopeLabel: "org:o",
+ createdAt: 2,
+ },
+ ]);
+ assert.ok(out.includes(`tool_result for tool_call#1 (none recorded): ${INTERRUPTED_TOOL_RESULT}`));
+});
diff --git a/test/execute-scratch.test.ts b/test/execute-scratch.test.ts
index 388d6b88a..ade03dbfd 100644
--- a/test/execute-scratch.test.ts
+++ b/test/execute-scratch.test.ts
@@ -87,7 +87,14 @@ const schemaRequired = (tool: ReturnType<typeof createAgentTools>[number]): stri
test("flag OFF: the execute surface is exactly the legacy one (no scope/durable, scoped box)", async () => {
const { tc, seen } = sinkToolContext();
const [execute] = createAgentTools({ current: tc });
- assert.deepEqual(schemaProps(execute!), ["command", "sandbox_id", "purpose", "timeout_seconds", "credentials"]);
+ assert.deepEqual(schemaProps(execute!), [
+ "command",
+ "sandbox_id",
+ "purpose",
+ "timeout_seconds",
+ "credentials",
+ "retrySafe",
+ ]);
assert.deepEqual(schemaRequired(execute!), ["command", "purpose"]);
await call(execute, { command: "echo hi" });
assert.deepEqual(seen, [{ command: "echo hi", opts: undefined }]);
@@ -105,6 +112,7 @@ test("flag ON: scope defaults to the durable scoped box; scratch is an explicit
"credentials",
"scope",
"durable",
+ "retrySafe",
]);
await call(execute, { command: "echo hi" });
@@ -293,7 +301,14 @@ test("execute rejects unavailable, conflicting, and reached credential requests"
test("execute schema lists exact command credential handles", async () => {
const { tc, seen } = sinkToolContext();
const [execute] = createAgentTools({ current: tc }, { commandCredentialHandles: ["kc_github12345"] });
- assert.deepEqual(schemaProps(execute!), ["command", "sandbox_id", "purpose", "timeout_seconds", "credentials"]);
+ assert.deepEqual(schemaProps(execute!), [
+ "command",
+ "sandbox_id",
+ "purpose",
+ "timeout_seconds",
+ "credentials",
+ "retrySafe",
+ ]);
await call(execute, { command: "gh api user", credentials: ["kc_github12345"] });
assert.deepEqual(seen.at(-1)?.opts, { credentials: ["kc_github12345"] });
diff --git a/test/memory-tool.test.ts b/test/memory-tool.test.ts
index c56780fdd..ff773204d 100644
--- a/test/memory-tool.test.ts
+++ b/test/memory-tool.test.ts
@@ -214,7 +214,7 @@ test("the `memory` tool params expose NO scope field — the model cannot redire
const memoryTool = createAgentTools({ current: null }).find((t) => t.name === "memory");
assert.ok(memoryTool);
const props = (memoryTool.parameters as { properties?: Record<string, unknown> }).properties ?? {};
- assert.deepEqual(Object.keys(props).sort(), ["action", "content", "facts", "limit", "query"]);
+ assert.deepEqual(Object.keys(props).sort(), ["action", "content", "facts", "limit", "query", "retrySafe"]);
assert.equal("scope" in props, false);
});
diff --git a/test/pi-harness-resume.test.ts b/test/pi-harness-resume.test.ts
new file mode 100644
index 000000000..92bf06b79
--- /dev/null
+++ b/test/pi-harness-resume.test.ts
@@ -0,0 +1,134 @@
+import { test } from "node:test";
+import assert from "node:assert/strict";
+import { createPiHarness } from "../src/harness/pi-harness.ts";
+import type { HarnessTurnInput } from "../src/harness/harness.ts";
+import type { NewEntry, NewTapeRecord } from "../src/sessions/session-store.ts";
+import type { ToolContext } from "../src/tools/primitives.ts";
+import type { ScopeId, SessionEntry } from "../src/types.ts";
+
+function sse(events: Array<Record<string, unknown>>): Response {
+ const body = events.map((event) => `event: ${event.type as string}\ndata: ${JSON.stringify(event)}\n\n`).join("");
+ return new Response(body, { status: 200, headers: { "content-type": "text/event-stream" } });
+}
+
+function textReplyEvents(text: string): Array<Record<string, unknown>> {
+ return [
+ {
+ type: "message_start",
+ message: {
+ id: "msg_1",
+ type: "message",
+ role: "assistant",
+ content: [],
+ model: "claude-test",
+ stop_reason: null,
+ usage: { input_tokens: 5, output_tokens: 0 },
+ },
+ },
+ { type: "content_block_start", index: 0, content_block: { type: "text", text: "" } },
+ { type: "content_block_delta", index: 0, delta: { type: "text_delta", text } },
+ { type: "content_block_stop", index: 0 },
+ { type: "message_delta", delta: { stop_reason: "end_turn" }, usage: { output_tokens: 3 } },
+ { type: "message_stop" },
+ ];
+}
+
+const scope = "personal:tester" as ScopeId;
+const NOTE = "(system note: your previous attempt at the request above was paused mid-turn and has resumed.)";
+
+test("the pi harness re-runs a retry-safe interrupted call before the model sees the resumed turn", async () => {
+ const harness = createPiHarness({ apiKey: "sk-test" });
+ const entries: Array<{ seq: number; type: string; payload: unknown }> = [];
+ const tape: NewTapeRecord[] = [];
+ const queries: string[] = [];
+ const requests: Array<Array<{ role: string; content: Array<Record<string, unknown>> | string }>> = [];
+ const realFetch = globalThis.fetch;
+ globalThis.fetch = (async (_url: string | URL | Request, init?: RequestInit) => {
+ const body = JSON.parse(String(init?.body ?? "{}")) as { messages?: (typeof requests)[number] };
+ requests.push(body.messages ?? []);
+ return sse(textReplyEvents("we kept the Q2 budget flat"));
+ }) as typeof globalThis.fetch;
+ const history: SessionEntry[] = [
+ {
+ sessionId: "s1",
+ seq: 1,
+ parentSeq: null,
+ type: "user",
+ payload: { text: "what about the budget?" },
+ scopeLabel: scope,
+ createdAt: 1,
+ },
+ {
+ sessionId: "s1",
+ seq: 2,
+ parentSeq: null,
+ type: "tool_call",
+ payload: {
+ tool: "history",
+ query: "budget",
+ callId: "c-budget",
+ retrySafe: true,
+ rerun: { tool: "history", input: { query: "budget" } },
+ },
+ scopeLabel: scope,
+ createdAt: 2,
+ },
+ ];
+ let seq = 3;
+ const turn: HarnessTurnInput = {
+ session: { id: "s1" } as HarnessTurnInput["session"],
+ input: NOTE,
+ systemPrompt: "BASE",
+ history,
+ tools: {
+ async history(q: string) {
+ queries.push(q);
+ return ["user#1: the budget doc is in shared/q2.md"];
+ },
+ } as unknown as ToolContext,
+ scopeLabel: scope,
+ orgScopeId: "org:test" as ScopeId,
+ resumeToolCall: { callId: "c-budget", tool: "history", input: { query: "budget" } },
+ emit: async (entry: NewEntry) => {
+ const appended = { ...entry, sessionId: "s1", parentSeq: null, seq: seq++, createdAt: Date.now() };
+ entries.push({ seq: appended.seq, type: entry.type, payload: entry.payload });
+ return appended as SessionEntry;
+ },
+ tape: async (rec: NewTapeRecord) => {
+ tape.push(rec);
+ },
+ recordModelCall: () => {},
+ };
+ try {
+ const result = await harness.turns.runTurn(turn);
+ assert.equal(result.reply, "we kept the Q2 budget flat");
+ } finally {
+ globalThis.fetch = realFetch;
+ }
+
+ assert.deepEqual(queries, ["budget"], "the interrupted call ran exactly once");
+ assert.deepEqual(
+ entries.map((e) => e.type),
+ ["tool_result", "user", "assistant"],
+ "the result closes the dangling call before this attempt's note is recorded; no second tool_call",
+ );
+ assert.equal((entries[0]!.payload as { callId: string }).callId, "c-budget");
+
+ const roles = tape.filter((rec) => rec.kind === "message").map((rec) => (rec.payload as { role: string }).role);
+ assert.deepEqual(roles, ["toolResult", "user", "assistant"], "the tape closes the call before the trigger row");
+ const taped = tape.find((rec) => (rec.payload as { role?: string }).role === "toolResult")!.payload as {
+ toolCallId: string;
+ content: Array<{ text: string }>;
+ isError: boolean;
+ };
+ assert.equal(taped.toolCallId, "c-budget");
+ assert.equal(taped.isError, false);
+ assert.match(taped.content[0]!.text, /shared\/q2\.md/);
+
+ assert.equal(requests.length, 1);
+ const flattened = JSON.stringify(requests[0]);
+ assert.ok(!flattened.includes("[interrupted"), "the model never sees an interrupted placeholder");
+ const resultAt = flattened.indexOf("shared/q2.md");
+ const noteAt = flattened.indexOf("paused mid-turn and has resumed");
+ assert.ok(resultAt > 0 && noteAt > resultAt, "the real tool result precedes the resume note in the model's context");
+});
diff --git a/test/replay.test.ts b/test/replay.test.ts
index 55a80f1d2..5708830ff 100644
--- a/test/replay.test.ts
+++ b/test/replay.test.ts
@@ -533,3 +533,40 @@ test("overheard reconciliation deduplicates IDs within a batch but preserves ide
);
assert.deepEqual(selectOverheardToImport(incoming, new Set(picked.map((p) => p.ts))), []);
});
+
+test("a tool_result marked interrupted replays as the interrupted placeholder, not as a real result", () => {
+ const entries: SessionEntry[] = [
+ {
+ sessionId: "s",
+ seq: 1,
+ parentSeq: null,
+ type: "user",
+ payload: { text: "run it" },
+ scopeLabel: "org:o",
+ createdAt: 1,
+ },
+ {
+ sessionId: "s",
+ seq: 2,
+ parentSeq: null,
+ type: "tool_call",
+ payload: { tool: "execute", callId: "c1", command: "make" },
+ scopeLabel: "org:o",
+ createdAt: 2,
+ },
+ {
+ sessionId: "s",
+ seq: 3,
+ parentSeq: null,
+ type: "tool_result",
+ payload: { callId: "c1", result: "[exit 143]", interrupted: true },
+ scopeLabel: "org:o",
+ createdAt: 3,
+ },
+ ];
+ const result = reconstructMessagesFromHistory(entries).find((m) => m.role === "toolResult")!;
+ assert.equal(result.role, "toolResult");
+ if (result.role !== "toolResult") return;
+ assert.equal(result.content[0]!.text, INTERRUPTED_TOOL_RESULT);
+ assert.equal(result.isError, true);
+});
diff --git a/test/scope-reach.test.ts b/test/scope-reach.test.ts
index 8c3d34e9c..54bd2a83b 100644
--- a/test/scope-reach.test.ts
+++ b/test/scope-reach.test.ts
@@ -249,6 +249,7 @@ test("reachExec ON (no scratch): scope is a free string; a room routes to reachT
"timeout_seconds",
"credentials",
"scope",
+ "retrySafe",
]);
await call(execute, { command: "cat x", scope: "#project-alpha" });
assert.deepEqual(seen.at(-1)!.opts, { reachTarget: "#project-alpha" });
@@ -271,6 +272,7 @@ test("reachExec ON + scratchExec ON: scope accepts scoped, scratch, AND a room",
"credentials",
"scope",
"durable",
+ "retrySafe",
]);
await call(execute, { command: "x", scope: "scratch" });
assert.deepEqual(seen.at(-1)!.opts, { scratch: true });
diff --git a/test/tape-fold.test.ts b/test/tape-fold.test.ts
index 09d9a17d5..a89eef271 100644
--- a/test/tape-fold.test.ts
+++ b/test/tape-fold.test.ts
@@ -753,3 +753,26 @@ test("dangling calls heal only after every image is rehydrated", async () => {
);
assert.ok(tapeNeedsInterruptHeal(rows, hydrated));
});
+
+test("a toolResult row marked interrupted folds as the interrupted placeholder, so served turns match entry replay", () => {
+ const rows = [
+ user("run it"),
+ assistant([{ type: "toolCall", id: "c1", name: "execute", arguments: { command: "make" } }]),
+ row({
+ kind: "message",
+ payload: {
+ role: "toolResult",
+ toolCallId: "c1",
+ toolName: "execute",
+ content: [{ type: "text", text: "[exit 143]" }],
+ isError: false,
+ interrupted: true,
+ },
+ }),
+ ];
+ const fold = foldTape(rows) as Array<{ role: string; content: Array<{ text: string }>; isError?: boolean }>;
+ const result = fold.find((m) => m.role === "toolResult")!;
+ assert.equal(result.content[0]!.text, INTERRUPTED_TOOL_RESULT);
+ assert.equal(result.isError, true);
+ assert.ok(lintFold(fold).ok);
+});
diff --git a/test/tool-call-resume.test.ts b/test/tool-call-resume.test.ts
new file mode 100644
index 000000000..44d8213eb
--- /dev/null
+++ b/test/tool-call-resume.test.ts
@@ -0,0 +1,495 @@
+import { test } from "node:test";
+import assert from "node:assert/strict";
+import { mkdtempSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { createOrchestrator, type OrchestratorInput } from "../src/core/orchestrator.ts";
+import { isResumeNote, type ResumableToolCall } from "../src/core/turn-resume.ts";
+import { INTERRUPTED_TOOL_RESULT } from "../src/harness/context-compaction.ts";
+import { createIdentityService } from "../src/identity/identity-service.ts";
+import { createMemoryConfigStore } from "../src/resolution/config-store.ts";
+import { createAclStore } from "../src/acl/acl-store.ts";
+import { createResolutionService } from "../src/resolution/resolution-service.ts";
+import { createMemorySessionStore } from "../src/sessions/memory-session-store.ts";
+import { createMemoryRunStore } from "../src/runs/memory-run-store.ts";
+import { createLocalWorkspaceStore } from "../src/workspace/workspace-store.ts";
+import { createMemoryFileArtifactStore } from "../src/files/file-artifact-store.ts";
+import { createMemoryDurableByteStore } from "../src/files/durable-byte-store.ts";
+import { createMemoryService } from "../src/memory/memory-service.ts";
+import { createModelGateway } from "../src/model/model-gateway.ts";
+import { createAuditLog } from "../src/audit/audit-log.ts";
+import { createRateLimiter } from "../src/ratelimit/rate-limiter.ts";
+import { defineHarness, type HarnessTurnInput } from "../src/harness/harness.ts";
+import { bridgedTools, resumeInterruptedToolCall } from "../src/harness/harness-shared.ts";
+import type { ToolContextRef } from "../src/harness/agent-tools.ts";
+import { createDeployStore } from "../src/deploy/deploy-store.ts";
+import { createDockerDeployProvider } from "../src/deploy/docker-deploy-provider.ts";
+import { createDeployService } from "../src/deploy/deploy-service.ts";
+import { createDeliveryStore } from "../src/delivery/delivery-store.ts";
+import type { ToolContext } from "../src/tools/primitives.ts";
+import { scopeId, type Conversation, type Principal, type SessionEntry } from "../src/types.ts";
+import type { Sandbox } from "../src/sandbox/sandbox.ts";
+
+const ORG = "default-org";
+const actor: Principal = { id: "U1", type: "internal" };
+const conversation: Conversation = { kind: "dm", threadRef: "web:U1:resume", audience: [actor] };
+const personal = scopeId("personal", "U1");
+const ASK = "what did we decide about the budget?";
+
+function historyToolContext(calls: string[]): ToolContext {
+ return {
+ async history(q: string) {
+ calls.push(q);
+ return [`user#3: the budget doc is in shared/q2.md (${q})`];
+ },
+ } as unknown as ToolContext;
+}
+
+function turnFor(
+ history: SessionEntry[],
+ call: ResumableToolCall | undefined,
+ sink: Array<{ type: string; payload: unknown }>,
+ tools: ToolContext,
+): HarnessTurnInput {
+ let seq = history.length + 10;
+ return {
+ session: { id: "s1" } as HarnessTurnInput["session"],
+ input: "(system note: resumed)",
+ systemPrompt: "BASE",
+ history,
+ tools,
+ scopeLabel: personal,
+ orgScopeId: scopeId("org", ORG),
+ emit: async (entry) => {
+ sink.push({ type: entry.type, payload: entry.payload });
+ return { ...entry, sessionId: "s1", seq: seq++, parentSeq: null, createdAt: 1000 + seq } as SessionEntry;
+ },
+ recordModelCall: () => {},
+ ...(call ? { resumeToolCall: call } : {}),
+ };
+}
+
+const danglingCall: SessionEntry = {
+ sessionId: "s1",
+ seq: 2,
+ parentSeq: null,
+ type: "tool_call",
+ payload: {
+ tool: "history",
+ query: "budget",
+ callId: "c-budget",
+ retrySafe: true,
+ rerun: { tool: "history", input: { query: "budget" } },
+ },
+ scopeLabel: personal,
+ createdAt: 2,
+};
+
+test("resumeInterruptedToolCall re-runs the recorded input exactly once and records only the result", async () => {
+ const queries: string[] = [];
+ const sink: Array<{ type: string; payload: unknown }> = [];
+ const tools = historyToolContext(queries);
+ const turn = turnFor(
+ [danglingCall],
+ { callId: "c-budget", tool: "history", input: { query: "budget" } },
+ sink,
+ tools,
+ );
+ const ref: ToolContextRef = { current: tools, emit: turn.emit, scopeLabel: personal, orgScopeId: turn.orgScopeId };
+
+ const resumed = await resumeInterruptedToolCall(turn, ref, bridgedTools(ref, {}));
+
+ assert.deepEqual(queries, ["budget"], "the tool body ran exactly once with the stripped input");
+ assert.deepEqual(
+ sink.map((e) => e.type),
+ ["tool_result"],
+ "the dangling tool_call entry is not re-recorded; only its result lands",
+ );
+ const result = sink[0]!.payload as { callId: string; result: string; isError: boolean };
+ assert.equal(result.callId, "c-budget");
+ assert.equal(result.isError, false);
+ assert.match(result.result, /shared\/q2\.md/);
+ assert.ok(resumed);
+ assert.equal(resumed.history.length, 2, "the harness continues from history that now carries the result");
+ assert.equal(resumed.history[1]!.type, "tool_result");
+ assert.deepEqual(
+ { ...resumed.message, timestamp: 0 },
+ {
+ role: "toolResult",
+ toolCallId: "c-budget",
+ toolName: "history",
+ content: [{ type: "text", text: "- user#3: the budget doc is in shared/q2.md (budget)" }],
+ isError: false,
+ timestamp: 0,
+ },
+ );
+ assert.equal(ref.emit, turn.emit, "the emit hook is restored after the re-run");
+});
+
+test("resumeInterruptedToolCall does nothing without a call to resume", async () => {
+ const queries: string[] = [];
+ const sink: Array<{ type: string; payload: unknown }> = [];
+ const tools = historyToolContext(queries);
+ const idle = turnFor([danglingCall], undefined, sink, tools);
+ const ref: ToolContextRef = { current: tools, emit: idle.emit, scopeLabel: personal };
+ assert.equal(await resumeInterruptedToolCall(idle, ref, bridgedTools(ref, {})), null);
+ assert.deepEqual(queries, []);
+ assert.deepEqual(sink, []);
+});
+
+test("when the tool is unavailable on the retried turn the call is closed as interrupted instead of left dangling", async () => {
+ const queries: string[] = [];
+ const sink: Array<{ type: string; payload: unknown }> = [];
+ const tools = historyToolContext(queries);
+ const missing = turnFor([danglingCall], { callId: "c-x", tool: "no_such_tool", input: {} }, sink, tools);
+ const ref: ToolContextRef = { current: tools, emit: missing.emit, scopeLabel: personal };
+ const resumed = await resumeInterruptedToolCall(missing, ref, bridgedTools(ref, {}));
+ assert.ok(resumed);
+ assert.equal(resumed.message.isError, true);
+ assert.equal(resumed.message.content[0]!.text, INTERRUPTED_TOOL_RESULT);
+ assert.deepEqual(
+ sink.map((e) => [e.type, (e.payload as { callId: string; result: string }).result]),
+ [["tool_result", INTERRUPTED_TOOL_RESULT]],
+ );
+ assert.equal((sink[0]!.payload as { interrupted?: boolean }).interrupted, true, "the ledger carries the meaning");
+ assert.deepEqual(queries, []);
+});
+
+test("a cancelled turn does not re-run anything and closes the call as interrupted", async () => {
+ const queries: string[] = [];
+ const sink: Array<{ type: string; payload: unknown }> = [];
+ const tools = historyToolContext(queries);
+ const turn = turnFor(
+ [danglingCall],
+ { callId: "c-budget", tool: "history", input: { query: "budget" } },
+ sink,
+ tools,
+ );
+ turn.cancel = AbortSignal.abort();
+ const ref: ToolContextRef = { current: tools, emit: turn.emit, scopeLabel: personal };
+ const resumed = await resumeInterruptedToolCall(turn, ref, bridgedTools(ref, {}));
+ assert.equal(resumed?.message.isError, true);
+ assert.deepEqual(queries, []);
+ assert.equal(sink.length, 1);
+ assert.equal(ref.abortSignal, undefined, "the cancel signal is only borrowed for the re-run");
+});
+
+test("a tool that throws during the re-run still records an error result so the model sees a closed call", async () => {
+ const sink: Array<{ type: string; payload: unknown }> = [];
+ const tools = {
+ async history() {
+ throw new Error("transcript index offline");
+ },
+ } as unknown as ToolContext;
+ const turn = turnFor(
+ [danglingCall],
+ { callId: "c-budget", tool: "history", input: { query: "budget" } },
+ sink,
+ tools,
+ );
+ const ref: ToolContextRef = { current: tools, emit: turn.emit, scopeLabel: personal };
+ const resumed = await resumeInterruptedToolCall(turn, ref, bridgedTools(ref, {}));
+ assert.ok(resumed);
+ assert.equal(resumed.message.isError, true);
+ assert.match(resumed.message.content[0]!.text ?? "", /transcript index offline/);
+ assert.deepEqual(
+ sink.map((e) => [e.type, (e.payload as { callId: string; isError: boolean }).isError]),
+ [["tool_result", true]],
+ );
+});
+
+function fakeSandbox(): Sandbox {
+ const unreached = () => {
+ throw new Error("the resume tests must not provision a sandbox");
+ };
+ return {
+ profile: { backend: "fake", writablePersistence: "snapshot_to_workspace", processSessions: false },
+ provision: unreached as never,
+ run: unreached as never,
+ readFile: unreached as never,
+ writeFile: unreached as never,
+ writeFileBytes: unreached as never,
+ readFileBytes: unreached as never,
+ listDir: unreached as never,
+ removeDir: unreached as never,
+ teardown: unreached as never,
+ };
+}
+
+function buildScenario() {
+ const received: Array<{ input: string; resumeToolCall?: ResumableToolCall; historyTypes: string[] }> = [];
+ const harness = defineHarness(
+ {
+ id: "pi",
+ controlTransport: "in-process",
+ toolTransport: "in-process",
+ transcriptFormat: "pi",
+ capabilities: new Set(["native-tape"]),
+ },
+ {
+ async runTurn(turn) {
+ received.push({
+ input: turn.input,
+ ...(turn.resumeToolCall ? { resumeToolCall: turn.resumeToolCall } : {}),
+ historyTypes: turn.history.map((e) => e.type),
+ });
+ if (turn.resumeToolCall) {
+ await turn.emit({
+ type: "tool_result",
+ payload: { tool: "history", callId: turn.resumeToolCall.callId, isError: false, result: "- found it" },
+ scopeLabel: turn.scopeLabel,
+ });
+ await turn.tape?.({
+ kind: "message",
+ harness: "pi",
+ payload: {
+ role: "toolResult",
+ toolCallId: turn.resumeToolCall.callId,
+ toolName: "history",
+ content: [{ type: "text", text: "- found it" }],
+ isError: false,
+ timestamp: Date.now(),
+ },
+ scopeLabel: turn.scopeLabel,
+ });
+ }
+ const userEntry = await turn.emit({ type: "user", payload: { text: turn.input }, scopeLabel: turn.scopeLabel });
+ await turn.tape?.({
+ kind: "message",
+ harness: "pi",
+ payload: { role: "user", content: [{ type: "text", text: turn.input }], timestamp: Date.now() },
+ scopeLabel: turn.scopeLabel,
+ entrySeq: userEntry.seq,
+ meta: { bareText: turn.input },
+ });
+ const reply = "we kept the Q2 budget flat";
+ await turn.tape?.({
+ kind: "message",
+ harness: "pi",
+ payload: { role: "assistant", content: [{ type: "text", text: reply }], timestamp: Date.now() },
+ scopeLabel: turn.scopeLabel,
+ });
+ const finalEntry = await turn.emit({
+ type: "assistant",
+ payload: { text: reply },
+ scopeLabel: turn.scopeLabel,
+ });
+ await turn.tape?.({
+ kind: "annotation",
+ payload: { turnEnd: true },
+ scopeLabel: turn.scopeLabel,
+ entrySeq: finalEntry.seq,
+ });
+ return { reply, modelCalls: 1 };
+ },
+ async screenSecurity() {
+ return { decision: "auto" as const };
+ },
+ },
+ );
+ const sessions = createMemorySessionStore();
+ const { runs } = createMemoryRunStore();
+ const acl = createAclStore();
+ const auditLog = createAuditLog();
+ const workspace = createLocalWorkspaceStore(mkdtempSync(join(tmpdir(), "tool-call-resume-")));
+ const orchestrator = createOrchestrator({
+ identity: createIdentityService(),
+ resolution: createResolutionService(ORG, createMemoryConfigStore(ORG), acl),
+ sessionTapeMode: "serve",
+ sessions,
+ runs,
+ workspace,
+ files: createMemoryFileArtifactStore(createMemoryDurableByteStore()),
+ sandbox: fakeSandbox(),
+ modelGateway: createModelGateway(),
+ auditLog,
+ rateLimiter: createRateLimiter({ maxPerWindow: 100, windowMs: 60_000 }),
+ harness,
+ memory: createMemoryService(workspace),
+ deploy: createDeployService({
+ deployStore: createDeployStore(),
+ provider: createDockerDeployProvider(),
+ deployDir: join(tmpdir(), "tool-call-resume-deploy"),
+ auditLog,
+ acl,
+ }),
+ acl,
+ deliveries: createDeliveryStore(),
+ });
+ const input = (text: string, extra: Partial<OrchestratorInput> = {}): OrchestratorInput => ({
+ surface: "web",
+ actor,
+ conversation,
+ origin: { kind: "human" },
+ text,
+ ...extra,
+ });
+ return { orchestrator, sessions, runs, received, input };
+}
+
+async function seedDeadAttempt(
+ sessions: ReturnType<typeof buildScenario>["sessions"],
+ callPayload: Record<string, unknown>,
+ siblingWithResult?: Record<string, unknown>,
+ harness = "pi",
+): Promise<{ sessionId: string; userSeq: number }> {
+ const session = await sessions.getOrCreateByThread(conversation.threadRef, "dm", personal);
+ const { lease } = await sessions.acquireLease(session.id);
+ try {
+ const user = await sessions.append(lease!, { type: "user", payload: { text: ASK }, scopeLabel: personal });
+ await sessions.appendTape(lease!, {
+ kind: "message",
+ harness,
+ payload: { role: "user", content: [{ type: "text", text: ASK }], timestamp: user.createdAt },
+ scopeLabel: personal,
+ entrySeq: user.seq,
+ meta: { bareText: ASK },
+ });
+ if (siblingWithResult)
+ await sessions.append(lease!, { type: "tool_call", payload: siblingWithResult, scopeLabel: personal });
+ await sessions.append(lease!, { type: "tool_call", payload: callPayload, scopeLabel: personal });
+ await sessions.appendTape(lease!, {
+ kind: "message",
+ harness,
+ payload: {
+ role: "assistant",
+ content: [
+ ...(siblingWithResult
+ ? [{ type: "toolCall", id: "c-sibling", name: "history", arguments: { query: "q2" } }]
+ : []),
+ { type: "toolCall", id: "c-budget", name: "history", arguments: { query: "budget", retrySafe: true } },
+ ],
+ timestamp: Date.now(),
+ stopReason: "stop",
+ },
+ scopeLabel: personal,
+ });
+ if (siblingWithResult)
+ await sessions.append(lease!, {
+ type: "tool_result",
+ payload: { tool: "history", callId: "c-sibling", isError: false, result: "- q2 notes" },
+ scopeLabel: personal,
+ });
+ return { sessionId: session.id, userSeq: user.seq };
+ } finally {
+ await sessions.releaseLease(lease!);
+ }
+}
+
+const tapeEvents = async (sessions: ReturnType<typeof buildScenario>["sessions"], sessionId: string) =>
+ (await sessions.getTape(sessionId))
+ .filter((row) => row.kind === "context_event")
+ .map((row) => (row.payload as { event: string }).event);
+
+test("a retry whose dangling call was marked retry-safe re-runs it silently instead of warning about an unknown outcome", async () => {
+ const { orchestrator, sessions, runs, received, input } = buildScenario();
+ const run = (await runs.enqueue({ sessionId: conversation.threadRef, request: input(ASK) })).run;
+ const { sessionId, userSeq } = await seedDeadAttempt(sessions, {
+ tool: "history",
+ query: "budget",
+ callId: "c-budget",
+ retrySafe: true,
+ rerun: { tool: "history", input: { query: "budget" } },
+ });
+ await runs.noteTurnUserSeq(run.id, userSeq);
+
+ const retry = await orchestrator.handleTurn(input(ASK, { runId: run.id, attempt: 2 }));
+ assert.equal(retry.status, "ok");
+ assert.equal(received.length, 1);
+ const turn = received[0]!;
+ assert.deepEqual(turn.resumeToolCall, { callId: "c-budget", tool: "history", input: { query: "budget" } });
+ assert.ok(isResumeNote(turn.input));
+ assert.match(turn.input, /result of the tool call that was in flight is recorded above/);
+ assert.doesNotMatch(turn.input, /unknown outcome/);
+ assert.deepEqual(await tapeEvents(sessions, sessionId), [], "no interrupt or coverage import is baked into the tape");
+
+ const entries = await sessions.getEntries(sessionId);
+ const types = entries.map((e) => e.type);
+ assert.deepEqual(types, ["user", "tool_call", "tool_result", "user", "assistant"]);
+ const roles = (await sessions.getTape(sessionId))
+ .filter((row) => row.kind === "message")
+ .map((row) => (row.payload as { role: string }).role);
+ assert.deepEqual(
+ roles,
+ ["user", "assistant", "toolResult", "user", "assistant"],
+ "the tape closes the call before the note",
+ );
+});
+
+for (const [label, payload] of [
+ ["unsafe", { tool: "history", query: "budget", callId: "c-budget", retrySafe: false }],
+ ["unmarked", { tool: "history", query: "budget", callId: "c-budget" }],
+] as const) {
+ test(`a retry whose dangling call is ${label} gets the routine-deploy interrupted note and no re-run`, async () => {
+ const { orchestrator, sessions, runs, received, input } = buildScenario();
+ const run = (await runs.enqueue({ sessionId: conversation.threadRef, request: input(ASK) })).run;
+ const { sessionId, userSeq } = await seedDeadAttempt(sessions, payload);
+ await runs.noteTurnUserSeq(run.id, userSeq);
+
+ const retry = await orchestrator.handleTurn(input(ASK, { runId: run.id, attempt: 2 }));
+ assert.equal(retry.status, "ok");
+ assert.equal(received.length, 1);
+ const turn = received[0]!;
+ assert.equal(turn.resumeToolCall, undefined);
+ assert.ok(isResumeNote(turn.input));
+ assert.match(turn.input, /routine platform deploy/);
+ assert.match(turn.input, /almost never warrants mentioning|almost never worth mentioning/);
+ assert.match(turn.input, /unknown outcome/);
+ assert.match(turn.input, /check what actually happened before redoing anything with side effects/);
+ const types = (await sessions.getEntries(sessionId)).map((e) => e.type);
+ assert.deepEqual(types, ["user", "tool_call", "user", "assistant"], "the call stays open; nothing is re-run");
+ });
+}
+
+test("a parallel batch whose sibling result never reached the tape falls back to the note so the tape heals as one", async () => {
+ const { orchestrator, sessions, runs, received, input } = buildScenario();
+ const run = (await runs.enqueue({ sessionId: conversation.threadRef, request: input(ASK) })).run;
+ const { sessionId, userSeq } = await seedDeadAttempt(
+ sessions,
+ {
+ tool: "history",
+ query: "budget",
+ callId: "c-budget",
+ retrySafe: true,
+ rerun: { tool: "history", input: { query: "budget" } },
+ },
+ { tool: "history", query: "q2", callId: "c-sibling" },
+ );
+ await runs.noteTurnUserSeq(run.id, userSeq);
+
+ const retry = await orchestrator.handleTurn(input(ASK, { runId: run.id, attempt: 2 }));
+ assert.equal(retry.status, "ok");
+ assert.equal(
+ received[0]!.resumeToolCall,
+ undefined,
+ "the entries alone look retry-safe, but the tape still has two open calls",
+ );
+ assert.match(received[0]!.input, /unknown outcome/);
+ const roles = (await sessions.getTape(sessionId))
+ .filter((row) => row.kind === "message")
+ .map((row) => (row.payload as { role: string }).role);
+ assert.ok(!roles.includes("toolResult"), "no lone toolResult row is appended next to a still-open sibling");
+});
+
+test("a session taped by another harness still re-runs from entries, since its tape is never served", async () => {
+ const { orchestrator, sessions, runs, received, input } = buildScenario();
+ const run = (await runs.enqueue({ sessionId: conversation.threadRef, request: input(ASK) })).run;
+ const { userSeq } = await seedDeadAttempt(
+ sessions,
+ {
+ tool: "history",
+ query: "budget",
+ callId: "c-budget",
+ retrySafe: true,
+ rerun: { tool: "history", input: { query: "budget" } },
+ },
+ undefined,
+ "claude",
+ );
+ await runs.noteTurnUserSeq(run.id, userSeq);
+
+ const retry = await orchestrator.handleTurn(input(ASK, { runId: run.id, attempt: 2 }));
+ assert.equal(retry.status, "ok");
+ assert.deepEqual(received[0]!.resumeToolCall, { callId: "c-budget", tool: "history", input: { query: "budget" } });
+ assert.doesNotMatch(received[0]!.input, /unknown outcome/);
+});
diff --git a/test/turn-resume.test.ts b/test/turn-resume.test.ts
index e2481962e..b9a27591d 100644
--- a/test/turn-resume.test.ts
+++ b/test/turn-resume.test.ts
@@ -1,6 +1,12 @@
import { test } from "node:test";
import assert from "node:assert/strict";
-import { findTrailingPartialTurn, isResumeNote, resumeNote, turnAtSeq } from "../src/core/turn-resume.ts";
+import {
+ findTrailingPartialTurn,
+ isResumeNote,
+ resumeNote,
+ resumeStrategy,
+ turnAtSeq,
+} from "../src/core/turn-resume.ts";
import type { SessionEntry } from "../src/types.ts";
function ent(type: SessionEntry["type"], payload: unknown, seq: number): SessionEntry {
@@ -74,7 +80,7 @@ test("platform rails and render-only rows are NOT work — only tool rows replay
});
test("a prior no-work retry's restart note is skipped — later retries anchor on the human's entry", () => {
- const entries = [user("build and deploy the release", 1), user(resumeNote({ workRecorded: false }), 2)];
+ const entries = [user("build and deploy the release", 1), user(resumeNote({ strategy: { kind: "restart" } }), 2)];
const found = findTrailingPartialTurn(entries, "build and deploy the release");
assert.deepEqual(found, { userSeq: 1, workEntries: 0 }, "provenance stays on the original, not attempt 2's prompt");
});
@@ -113,7 +119,7 @@ test("resumeNote is recognized by isResumeNote and mentions background jobs only
});
test("the no-work wording promises nothing recorded, and neither wording re-sends the input", () => {
- const restart = resumeNote({ workRecorded: false });
+ const restart = resumeNote({ strategy: { kind: "restart" } });
assert.ok(isResumeNote(restart));
assert.doesNotMatch(restart, /recorded above|Continue from where you left off/);
assert.match(restart, /nothing to pick up\. Start the request now/);
@@ -199,3 +205,67 @@ test("turnAtSeq reports the turn's last answer, matching what the turn itself wo
answer: { seq: 13, text: "after the nudge" },
});
});
+
+const safeCall = (seq: number, retrySafe: boolean) =>
+ ent(
+ "tool_call",
+ {
+ tool: "history",
+ query: "deploy",
+ callId: `c${seq}`,
+ retrySafe,
+ ...(retrySafe ? { rerun: { tool: "history", input: { query: "deploy", limit: 5 } } } : {}),
+ },
+ seq,
+ );
+
+test("resumeStrategy re-runs a lone dangling call the model marked retry-safe, with its stripped input", () => {
+ const entries = [user("build and deploy the release", 1), toolCall(2), toolResult(3), safeCall(4, true)];
+ assert.deepEqual(resumeStrategy(entries, { userSeq: 1, workEntries: 3 }), {
+ kind: "retry",
+ call: { callId: "c4", tool: "history", input: { query: "deploy", limit: 5 } },
+ });
+});
+
+test("resumeStrategy falls back to the interrupted note when the dangling call is unsafe, unmarked, or ambiguous", () => {
+ const unsafe = [user("build and deploy the release", 1), safeCall(2, false)];
+ assert.deepEqual(resumeStrategy(unsafe, { userSeq: 1, workEntries: 1 }), { kind: "note" });
+ const unmarked = [user("build and deploy the release", 1), toolCall(2)];
+ assert.deepEqual(resumeStrategy(unmarked, { userSeq: 1, workEntries: 1 }), { kind: "note" });
+ const twoDangling = [user("build and deploy the release", 1), safeCall(2, true), safeCall(3, true)];
+ assert.deepEqual(resumeStrategy(twoDangling, { userSeq: 1, workEntries: 2 }), { kind: "note" });
+ const answered = [user("build and deploy the release", 1), safeCall(2, true), toolResult(3)];
+ assert.deepEqual(resumeStrategy(answered, { userSeq: 1, workEntries: 2 }), { kind: "note" });
+ const priorTurnCall = [safeCall(0, true), user("build and deploy the release", 1), toolCall(2), toolResult(3)];
+ assert.deepEqual(resumeStrategy(priorTurnCall, { userSeq: 1, workEntries: 2 }), { kind: "note" });
+});
+
+test("resumeStrategy restarts when the dead attempt recorded no work", () => {
+ assert.deepEqual(resumeStrategy([user("x", 1)], { userSeq: 1, workEntries: 0 }), { kind: "restart" });
+});
+
+test("the resume notes name the routine deploy, and only the retry note promises a complete record", () => {
+ const note = resumeNote();
+ assert.ok(isResumeNote(note));
+ assert.match(note, /routine platform deploy/);
+ assert.match(note, /almost never worth mentioning to the user/);
+ assert.match(note, /unknown outcome/);
+ const retried = resumeNote({ strategy: { kind: "retry", call: { callId: "c1", tool: "history", input: {} } } });
+ assert.ok(isResumeNote(retried));
+ assert.match(retried, /result of the tool call that was in flight is recorded above/);
+ assert.doesNotMatch(retried, /unknown outcome/);
+ assert.match(retried, /Continue from where you left off/);
+ assert.doesNotMatch(retried, /history|c1/);
+ const handoff = resumeNote({ cause: "runtime-change" });
+ assert.ok(isResumeNote(handoff));
+ assert.doesNotMatch(handoff, /deploy/);
+ assert.match(handoff, /unknown outcome/);
+});
+
+test("a tool_result marked interrupted does not answer its call, so the call is still resumable", () => {
+ const killed = ent("tool_result", { callId: "c4", result: "[exit 143]", interrupted: true }, 5);
+ const entries = [user("build and deploy the release", 1), safeCall(4, true), killed];
+ assert.equal(resumeStrategy(entries, { userSeq: 1, workEntries: 2 }).kind, "retry");
+ const unsafe = [user("build and deploy the release", 1), safeCall(4, false), killed];
+ assert.deepEqual(resumeStrategy(unsafe, { userSeq: 1, workEntries: 2 }), { kind: "note" });
+});