Multiplayer agent harness for work.
Tell resumed turns the recorded reason the last attempt stopped (#2361)
Regan Bell2fac75dc2f9b
diff --git a/src/core/orchestrator.ts b/src/core/orchestrator.tsindex 9b16df743..a19dcf8d1 100644--- a/src/core/orchestrator.ts+++ b/src/core/orchestrator.ts@@ -139,7 +139,14 @@ import {withoutAlreadyIngested,} from "./attachments.ts";import { parseRef } from "../acl/resource-ref.ts";-import { findTrailingPartialTurn, resumeNote, resumeStrategy, turnAtSeq } from "./turn-resume.ts";+import {+ findTrailingPartialTurn,+ resumeNote,+ resumeStrategy,+ runtimeSwitchNote,+ turnAtSeq,+ unrecordedToolResult,+} from "./turn-resume.ts";import type { RecordedTurn } from "./turn-resume.ts";import {recordedMessageTimestamps,@@ -2822,12 +2829,33 @@ export function createOrchestrator(deps: OrchestratorDeps): Orchestrator {const rawEntries = (await deps.sessions.getContextWindow(session.id)).entries;const priorTurns = memoryHistoryReset ? undefined : input.priorTurns;- const visibleHistory = filterHistory(forModelContext(rawEntries));+ let visibleHistory = filterHistory(forModelContext(rawEntries));const recallStart = Date.now();const recalled = await context.recallMemory(memoryRecallQuery(input.text, visibleHistory));const recallMs = Date.now() - recallStart;const partial = isRetry ? (recordedTurn ?? findTrailingPartialTurn(visibleHistory, input.text)) : null;- const resumePlan = partial ? resumeStrategy(visibleHistory, partial) : null;+ let resumePlan = partial ? resumeStrategy(visibleHistory, partial) : null;+ if (partial && resumePlan?.kind === "settle") {+ // A tool call's lost outcome is recorded as its result, so the model reads what happened to it there.+ const unrecorded = resumePlan.calls;+ const settled = await withManagedRosterVersion(() =>+ deps.sessions.appendMany(+ lease,+ unrecorded.map((call) => ({+ type: "tool_result" as const,+ payload: {+ tool: call.tool,+ callId: call.callId,+ isError: true,+ result: unrecordedToolResult(input.resumeCause),+ },+ scopeLabel: scopeId,+ })),+ ),+ );+ visibleHistory = filterHistory(forModelContext([...rawEntries, ...settled]));+ resumePlan = resumeStrategy(visibleHistory, partial);+ }const resume = resumePlan && resumePlan.kind !== "restart" ? partial : null;const compactStart = Date.now();const history = await withManagedRosterVersion(() =>@@ -2985,13 +3013,16 @@ export function createOrchestrator(deps: OrchestratorDeps): Orchestrator {);}let turnInput = resumePlan- ? resumeNote({ strategy: resumePlan, backgroundJobs: !!backgroundBroker })+ ? resumeNote({+ strategy: resumePlan,+ backgroundJobs: !!backgroundBroker,+ ...(input.resumeCause ? { cause: input.resumeCause } : {}),+ }): baseText;const requestInHistory = !!partial && history.some((entry) => entry.seq === partial.userSeq);const continueTurn =(resumePlan?.kind === "continue" || resumePlan?.kind === "retry") && requestInHistory && !releasedToolOutput;- if (partial && !requestInHistory)- turnInput += `\nCurrent request (continue from recorded work; do not restart):\n${baseText}`;+ if (partial && !requestInHistory) turnInput += `\nCurrent request:\n${baseText}`;if (partial) spine.turnUserEntrySeq = partial.userSeq;if (releasedToolOutput) {turnInput = `The human released quarantined tool output recorded in the conversation. Continue the original task using that output. The tool action already ran; do not repeat it. Original task: ${baseText}`;@@ -3660,9 +3691,7 @@ export function createOrchestrator(deps: OrchestratorDeps): Orchestrator {contextRecovered ||= recovery;turnAbort.signal.throwIfAborted();segment = await runHarnessSegment(- resumeNote({ cause: "runtime-change" }) +- (recovery ? "\nContext reduced without a new summary." : "\nRuntime handoff completed.") +- " Continue the user's unfinished request using the saved conversation and tool results. Do not repeat completed actions or ask the user to repeat the request." ++ runtimeSwitchNote("choice" in segment.runtimeHandoff ? segment.runtimeHandoff.choice : undefined) +(recovery &&!resumedHistory.some((entry) =>@@ -3671,7 +3700,7 @@ export function createOrchestrator(deps: OrchestratorDeps): Orchestrator {typeof entry.payload.text === "string" &&entry.payload.text.startsWith(baseText),)- ? `\nCurrent request (continue from recorded work; do not restart):\n${baseText}`+ ? `\nCurrent request:\n${baseText}`: ""),inbound.images.length ? { images: inbound.images } : {},{ history: resumedHistory },diff --git a/src/core/orchestrator/types.ts b/src/core/orchestrator/types.tsindex c394ab724..40da4842e 100644--- a/src/core/orchestrator/types.ts+++ b/src/core/orchestrator/types.ts@@ -1,3 +1,4 @@+import type { RunStop } from "../../runs/run-store.ts";import type { ExternalSlackPolicies } from "../../resolution/external-slack.ts";import type { RuntimeService } from "../../harness/runtime-types.ts";import type { SandboxResources } from "../../sandbox/sandbox-resources.ts";@@ -96,6 +97,8 @@ export interface OrchestratorInput extends Omit<origin: TurnOrigin;runId?: string;attempt?: number;+ /** Why the previous attempt of this run stopped (recorded by the run store); absent on a first attempt. */+ resumeCause?: RunStop;runLeaseToken?: string;runStartedAt?: number;diff --git a/src/core/turn-resume.ts b/src/core/turn-resume.tsindex 2e4e89440..ab75713cc 100644--- a/src/core/turn-resume.ts+++ b/src/core/turn-resume.ts@@ -1,7 +1,16 @@+import type { RuntimeChoice } from "../harness/harness.ts";+import { unrecordedToolResult } from "../harness/context-compaction.ts";++export { unrecordedToolResult };+import type { RunStop } from "../runs/run-store.ts";import type { SessionEntry } from "../types.ts";import { entryDeliveryKey, isOverheardEntry } from "../sessions/session-store.ts";-const NOTE_HEAD = "(system note:";+const RESUME_HEAD = "[system] Your previous attempt";+const RUNTIME_HEAD = "[system] Runtime switched";+const TRIMMED_HEAD = "[system] Context was trimmed";+/** Resume notes written before the [system] wording; still present in stored history. */+const LEGACY_NOTE_HEAD = "(system note:";export interface PartialTurn {userSeq: number;@@ -13,7 +22,8 @@ function entryText(e: SessionEntry): string {}export function isResumeNote(text: string): boolean {- return text.trimStart().startsWith(NOTE_HEAD);+ const t = text.trimStart();+ return [RESUME_HEAD, RUNTIME_HEAD, TRIMMED_HEAD, LEGACY_NOTE_HEAD].some((head) => t.startsWith(head));}const MODEL_TURN_ENTRY_TYPES = new Set<SessionEntry["type"]>(["user", "assistant", "tool_call", "tool_result"]);@@ -79,15 +89,23 @@ export interface ResumableToolCall {}export type ResumeStrategy =- { kind: "restart" } | { kind: "note" } | { kind: "continue" } | { kind: "retry"; call: ResumableToolCall };+ | { kind: "restart" }+ | { kind: "note" }+ | { kind: "continue" }+ | { kind: "retry"; call: ResumableToolCall }+ /** Tool calls whose outcome was never recorded: record what happened as their results, then continue. */+ | { kind: "settle"; calls: Array<{ callId: string; tool: string }> };export function resumeStrategy(entries: readonly SessionEntry[], partial: PartialTurn): ResumeStrategy {if (partial.workEntries === 0) return { kind: "restart" };const turn = entries.filter((e) => e.seq > partial.userSeq);const answered = new Set<unknown>();+ const withResult = new Set<unknown>();for (const e of turn) {const payload = e.payload as { callId?: unknown; interrupted?: unknown } | null;- if (e.type === "tool_result" && payload?.interrupted !== true) answered.add(payload?.callId);+ if (e.type !== "tool_result") continue;+ withResult.add(payload?.callId);+ if (payload?.interrupted !== true) answered.add(payload?.callId);}const dangling = turn.filter((e) => e.type === "tool_call" && !answered.has((e.payload as { callId?: unknown } | null)?.callId),@@ -96,7 +114,16 @@ export function resumeStrategy(entries: readonly SessionEntry[], partial: Partiaconst last = turn.findLast((e) => MODEL_TURN_ENTRY_TYPES.has(e.type));return last?.type === "tool_result" ? { kind: "continue" } : { kind: "note" };}- if (dangling.length !== 1) return { kind: "note" };+ // Calls with no result at all get one recording what happened; a call whose re-run was itself cut off+ // already has a result, so it only gets the note.+ const unrecorded = dangling.flatMap((e) => {+ const p = e.payload as { callId?: unknown; tool?: unknown } | null;+ return typeof p?.callId === "string" && !withResult.has(p.callId)+ ? [{ callId: p.callId, tool: typeof p.tool === "string" ? p.tool : "tool" }]+ : [];+ });+ const settle: ResumeStrategy = unrecorded.length ? { kind: "settle", calls: unrecorded } : { kind: "note" };+ if (dangling.length !== 1) return settle;const payload = dangling[0]!.payload as {callId?: unknown;retrySafe?: unknown;@@ -111,41 +138,59 @@ export function resumeStrategy(entries: readonly SessionEntry[], partial: Partiatypeof rerun.input !== "object" ||Array.isArray(rerun.input))- return { kind: "note" };+ return settle;return {kind: "retry",call: { callId: payload.callId, tool: rerun.tool, input: rerun.input as Record<string, unknown> },};}-const ROUTINE = "This was a routine platform deploy; it is common and almost never worth mentioning to the user.";-const CONTINUE = "Continue from where you left off; don't start over or repeat completed steps.)";+export type ResumeCause = RunStop;++/** Why the previous attempt stopped, from the cause the run store recorded. Never guessed. */+function causePhrase(cause: ResumeCause | undefined): string {+ switch (cause?.kind) {+ case "shutdown":+ return "a routine platform deploy";+ case "lease_expired":+ return "the worker crashed";+ case "error":+ return cause.code ? `a model-provider call failed (${providerDetail(cause)})` : "an internal error";+ default:+ return "cause not recorded";+ }+}++function providerDetail(cause: { code?: string; status?: number }): string {+ return `${cause.code}${cause.status !== undefined ? `, HTTP ${cause.status}` : ""}`;+}++const QUIET = "Don't mention this.";export function resumeNote(opts?: {strategy?: ResumeStrategy;backgroundJobs?: boolean;- cause?: "deploy" | "runtime-change";+ cause?: ResumeCause;}): string {- const strategy = opts?.strategy ?? { kind: "note" };- if (strategy.kind === "restart") {- return `${NOTE_HEAD} your previous attempt at the request above was interrupted before it recorded any work (a routine platform deploy), so there is nothing to pick up. Start the request now.)`;- }- const parts =- strategy.kind === "retry"- ? [- `${NOTE_HEAD} your previous attempt at the request above was paused mid-turn and has resumed.`,- ROUTINE,- "The result of the tool call that was in flight is recorded above.",- ]- : [- `${NOTE_HEAD} your previous attempt at the request above was interrupted mid-turn.`,- opts?.cause === "runtime-change" ? "" : ROUTINE,- "Your work up to the interruption is recorded above; a tool result marked interrupted has an",- "unknown outcome, so check what actually happened before redoing anything with side effects.",- ];- if (opts?.backgroundJobs) {- parts.push("Background jobs on your computer kept running — `background` list/poll to check on them.");- }- parts.push(CONTINUE);- return parts.filter(Boolean).join(" ");+ const why = causePhrase(opts?.cause);+ if (opts?.strategy?.kind === "restart")+ return `${RESUME_HEAD} stopped before recording any work — ${why}. Start the request now. ${QUIET}`;+ return [+ `${RESUME_HEAD} was interrupted mid-turn — ${why}. Continue from the recorded work; don't repeat completed steps. ${QUIET}`,+ opts?.backgroundJobs+ ? "Background jobs on your computer kept running — check them with `background` list/poll."+ : "",+ ]+ .filter(Boolean)+ .join(" ");+}++/** Told to the model when a task continues on a new runtime, or after its context was cut down. */+export function runtimeSwitchNote(choice: RuntimeChoice | undefined): string {+ const head = choice+ ? `${RUNTIME_HEAD} to ${choice.modelId}${choice.fastMode ? " fast" : ""}${+ choice.effortLevel && choice.effortLevel !== "auto" ? `; effort ${choice.effortLevel}` : ""+ }.`+ : `${TRIMMED_HEAD} without a new summary.`;+ return `${head} Continue from the recorded work; don't repeat completed actions or ask the user to restate.`;}diff --git a/src/harness/agent-tools.ts b/src/harness/agent-tools.tsindex c38569fd8..1682a4cec 100644--- a/src/harness/agent-tools.ts+++ b/src/harness/agent-tools.ts@@ -387,7 +387,8 @@ export function createAgentTools(ref: ToolContextRef, opts?: AgentToolsOptions):orgScopeId: ref.orgScopeId ?? ref.scopeLabel,sourceScopeId,});- if (type === "tool_result" && ref.shutdown?.aborted && isObj(payload)) payload = { ...payload, interrupted: true };+ if (type === "tool_result" && ref.shutdown?.aborted && isObj(payload))+ payload = { ...payload, interrupted: true, stop: { kind: "shutdown" } };await ref.emit({ type, payload, scopeLabel });};diff --git a/src/harness/context-compaction.ts b/src/harness/context-compaction.tsindex 04454657a..deb996bc1 100644--- a/src/harness/context-compaction.ts+++ b/src/harness/context-compaction.ts@@ -18,7 +18,34 @@ function headTailSlice(s: string, maxChars: number, tailChars: number): string {}export const INTERRUPTED_TOOL_RESULT =- "[interrupted — a routine platform deploy restarted the agent while this tool call was running and its outcome was not recorded. Check what actually happened before redoing anything with side effects.]";+ "[system] This call was interrupted; its outcome wasn't recorded. Check what happened before retrying.";++/** What stopped an attempt, as recorded by the run store (`RunStop`) or on an interrupted tool result. */+export type ToolInterruption = { kind: string; code?: string; status?: number } | undefined;++/** What interrupted a tool call, as the subject of "... interrupted this call". */+function interruptedBy(cause: ToolInterruption): string | null {+ switch (cause?.kind) {+ case "shutdown":+ return "A routine platform deploy";+ case "lease_expired":+ return "A worker crash";+ case "error":+ return cause.code+ ? `A model-provider failure (${cause.code}${cause.status !== undefined ? `, HTTP ${cause.status}` : ""})`+ : "An internal error";+ default:+ return null;+ }+}++/** The result a tool call gets when the attempt running it stopped before its outcome was recorded. */+export function unrecordedToolResult(cause: ToolInterruption): string {+ const by = interruptedBy(cause);+ return by+ ? `[system] ${by} interrupted this call; its outcome wasn't recorded. Check what happened before retrying. Don't mention this.`+ : INTERRUPTED_TOOL_RESULT;+}export const CONTEXT_SUMMARY_HEADER ="[Earlier conversation summary — an index of turns compacted out of your context. Conversation turns and tool calls can be reopened with the history tool (seq parameter; a very long entry returns as head and tail), or searched (query). Tool results are excluded.]";diff --git a/src/harness/mock-harness.ts b/src/harness/mock-harness.tsindex b7ac9e8ff..967730698 100644--- a/src/harness/mock-harness.ts+++ b/src/harness/mock-harness.ts@@ -1,3 +1,4 @@+import { isResumeNote } from "../core/turn-resume.ts";import {defineHarness,type Harness,@@ -181,7 +182,7 @@ export function createMockHarness(): Harness {} else if (command0 === "!boom-always" || boomAlwaysSessions.has(turn.session.id)) {boomAlwaysSessions.add(turn.session.id);throw new Error("boom: simulated turn fault");- } else if (command0.startsWith("(system note:") && resumePostSessions.has(turn.session.id)) {+ } else if (isResumeNote(command0) && resumePostSessions.has(turn.session.id)) {const msg = resumePostSessions.get(turn.session.id)!;resumePostSessions.delete(turn.session.id);await turn.emit({@@ -197,7 +198,7 @@ export function createMockHarness(): Harness {});usedTool = true;reply = r.ok ? "(posted after resume)" : `[not sent] ${r.message ?? "failed"}`;- } else if (command0.startsWith("(system note:") && shutdownResumeSessions.has(turn.session.id)) {+ } else if (isResumeNote(command0) && shutdownResumeSessions.has(turn.session.id)) {shutdownResumeSessions.delete(turn.session.id);reply = "finished after resume";} else if (command0 === "!work-until-shutdown") {diff --git a/src/harness/replay.ts b/src/harness/replay.tsindex 0e745e8e2..2f8b16fc4 100644--- a/src/harness/replay.ts+++ b/src/harness/replay.ts@@ -12,7 +12,7 @@ import type { ConversationTurn, OverheardMessage, SessionEntry } from "../types.import { deliveryNote } from "../core/attachments.ts";import { goalCancelledNote } from "./goal.ts";import { messageRevision, renderMessageRevision } from "../core/message-revisions.ts";-import { CONTEXT_SUMMARY_HEADER, INTERRUPTED_TOOL_RESULT } from "./context-compaction.ts";+import { CONTEXT_SUMMARY_HEADER, INTERRUPTED_TOOL_RESULT, unrecordedToolResult } from "./context-compaction.ts";import { contextSummaryPayload } from "../sessions/session-store.ts";import { isoFromTs, messageTag } from "../util/message-tag.ts";import { isObj } from "../util/objects.ts";@@ -228,9 +228,22 @@ function toolResultMessage(call: ToolCall, re: SessionEntry | undefined, at: numconst base = { role: "toolResult" as const, toolCallId: call.id, toolName: call.name };const text = (value: string) => [{ type: "text" as const, text: value }];if (!re) return { ...base, content: text(INTERRUPTED_TOOL_RESULT), isError: true, timestamp: at };- const p = (re.payload ?? {}) as { interrupted?: unknown; isError?: unknown; result?: unknown; wire?: ToolResultWire };+ const p = (re.payload ?? {}) as {+ interrupted?: unknown;+ stop?: { kind?: unknown };+ isError?: unknown;+ result?: unknown;+ wire?: ToolResultWire;+ };if (p.interrupted === true)- return { ...base, content: text(INTERRUPTED_TOOL_RESULT), isError: true, timestamp: re.createdAt };+ return {+ ...base,+ content: text(+ typeof p.stop?.kind === "string" ? unrecordedToolResult({ kind: p.stop.kind }) : INTERRUPTED_TOOL_RESULT,+ ),+ isError: true,+ timestamp: re.createdAt,+ };return {...base,content: p.wire?.content ? resultContent(p.wire.content) : text(String(p.result ?? "")),diff --git a/src/runs/memory-run-store.ts b/src/runs/memory-run-store.tsindex ea264b0ca..2e6af8408 100644--- a/src/runs/memory-run-store.ts+++ b/src/runs/memory-run-store.ts@@ -1,6 +1,6 @@import { randomUUID } from "node:crypto";import { EventEmitter } from "node:events";-import type { EnqueueInput, EnqueueResult, ReapEvent, Run, RunDeliveryState, RunStore } from "./run-store.ts";+import type { EnqueueInput, EnqueueResult, ReapEvent, Run, RunDeliveryState, RunStop, RunStore } from "./run-store.ts";import { isTerminal, leaseLapsed, releasesDedupKey } from "./run-store.ts";import type { LedgerBegin, ToolLedger } from "./tool-ledger.ts";@@ -106,13 +106,14 @@ export function createMemoryRunStore(opts?: { maxClaims?: number }): MemoryRuntireturn true;},- async releaseLease(runId, leaseToken) {+ async releaseLease(runId, leaseToken, stop) {const run = runs.get(runId);if (!run || run.status !== "running" || run.leaseToken !== leaseToken) return false;run.status = "pending";run.leaseToken = null;run.leaseExpiresAt = null;run.workerId = null;+ if (stop) run.lastStop = stop;return true;},@@ -136,8 +137,11 @@ export function createMemoryRunStore(opts?: { maxClaims?: number }): MemoryRunticonst run = runs.get(runId);if (!run || run.leaseToken !== leaseToken) return { requeued: false };return {- requeued: retire(run, error, opts?.retry !== false, { countsAsError: true, retryAfterMs: opts?.retryAfterMs })- .requeued,+ requeued: retire(run, error, opts?.retry !== false, {+ countsAsError: true,+ retryAfterMs: opts?.retryAfterMs,+ ...(opts?.stop ? { stop: opts.stop } : {}),+ }).requeued,};},@@ -363,7 +367,7 @@ export function createMemoryRunStore(opts?: { maxClaims?: number }): MemoryRuntirun: Run,error: string,retry: boolean,- opts?: { ifExpiredAt?: number; countsAsError?: boolean; retryAfterMs?: number },+ opts?: { ifExpiredAt?: number; countsAsError?: boolean; retryAfterMs?: number; stop?: RunStop },): { requeued: boolean; applied: boolean } {if (run.status !== "running") return { requeued: false, applied: false };if (opts?.ifExpiredAt !== undefined && (run.leaseExpiresAt === null || run.leaseExpiresAt > opts.ifExpiredAt)) {@@ -376,6 +380,7 @@ export function createMemoryRunStore(opts?: { maxClaims?: number }): MemoryRunticonst overClaimed = run.attempts >= maxClaims;if (retry && run.errorAttempts < run.maxAttempts && !overClaimed) {run.status = "pending";+ run.lastStop = opts?.stop ?? (opts?.countsAsError ? { kind: "error" } : { kind: "lease_expired" });retryAfter.set(run.id, Date.now() + Math.max(0, opts?.retryAfterMs ?? 0));return { requeued: true, applied: true };}diff --git a/src/runs/postgres-run-store.ts b/src/runs/postgres-run-store.tsindex 01d675e66..afac277e6 100644--- a/src/runs/postgres-run-store.ts+++ b/src/runs/postgres-run-store.ts@@ -8,7 +8,7 @@ import { isObj } from "../util/objects.ts";import type { TurnResult } from "../types.ts";import type { OrchestratorInput } from "../core/orchestrator.ts";import { resolveTurnOrigin } from "../core/turn-origin.ts";-import type { EnqueueInput, EnqueueResult, ReapEvent, Run, RunDeliveryState, RunStore } from "./run-store.ts";+import type { EnqueueInput, EnqueueResult, ReapEvent, Run, RunDeliveryState, RunStop, RunStore } from "./run-store.ts";import { isTerminal, releasesDedupKey } from "./run-store.ts";import { errMessage, swallow } from "../util/errors.ts";import type { LedgerBegin, ToolLedger } from "./tool-ledger.ts";@@ -43,6 +43,7 @@ function rowToRun(r: Record<string, unknown>): Run {createdAt: Number(r.created_at),startedAt: r.started_at === null ? null : Number(r.started_at),finishedAt: r.finished_at === null ? null : Number(r.finished_at),+ lastStop: r.last_stop != null ? (JSON.parse(r.last_stop as string) as RunStop) : null,};}@@ -132,6 +133,10 @@ export function createPostgresRunStore(connectionString: string, opts?: { maxCla`CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_runs_terminal_finished ON runs(finished_at, id) WHERE status IN ('done','failed')`,],},+ {+ id: "runs/store/0007-last-stop",+ statements: [`SET LOCAL lock_timeout = '3s'`, `ALTER TABLE runs ADD COLUMN IF NOT EXISTS last_stop TEXT`],+ },],[{@@ -219,7 +224,7 @@ export function createPostgresRunStore(connectionString: string, opts?: { maxClarun: Run,error: string,retry: boolean,- opts?: { ifExpiredAt?: number; countsAsError?: boolean; retryAfterMs?: number },+ opts?: { ifExpiredAt?: number; countsAsError?: boolean; retryAfterMs?: number; stop?: RunStop },): Promise<{ requeued: boolean; applied: boolean }> {const ifExpiredAt = opts?.ifExpiredAt ?? null;const countsAsError = opts?.countsAsError ?? false;@@ -228,9 +233,16 @@ export function createPostgresRunStore(connectionString: string, opts?: { maxClaif (retry && errorAttemptsAfter < run.maxAttempts && !overClaimed) {const { rowCount } = await q(`UPDATE runs SET status='pending', lease_token=NULL, lease_expires_at=NULL, worker_id=NULL,- error_attempts=error_attempts+$4, retry_after=$5+ error_attempts=error_attempts+$4, retry_after=$5, last_stop=$6WHERE id=$1 AND lease_token=$2 AND status='running' AND ($3::bigint IS NULL OR lease_expires_at <= $3) RETURNING pg_notify('qm_run_available', 'null')`,- [run.id, run.leaseToken, ifExpiredAt, countsAsError ? 1 : 0, Date.now() + Math.max(0, opts?.retryAfterMs ?? 0)],+ [+ run.id,+ run.leaseToken,+ ifExpiredAt,+ countsAsError ? 1 : 0,+ Date.now() + Math.max(0, opts?.retryAfterMs ?? 0),+ JSON.stringify(opts?.stop ?? (countsAsError ? { kind: "error" } : { kind: "lease_expired" })),+ ],);return { requeued: rowCount > 0, applied: rowCount > 0 };}@@ -329,10 +341,10 @@ export function createPostgresRunStore(connectionString: string, opts?: { maxClareturn rowCount > 0;},- async releaseLease(runId, leaseToken): Promise<boolean> {+ async releaseLease(runId, leaseToken, stop): Promise<boolean> {const { rowCount } = await q(- "UPDATE runs SET status='pending', lease_token=NULL, lease_expires_at=NULL, worker_id=NULL WHERE id=$1 AND lease_token=$2 AND status='running' RETURNING pg_notify('qm_run_available', 'null')",- [runId, leaseToken],+ "UPDATE runs SET status='pending', lease_token=NULL, lease_expires_at=NULL, worker_id=NULL, last_stop=COALESCE($3, last_stop) WHERE id=$1 AND lease_token=$2 AND status='running' RETURNING pg_notify('qm_run_available', 'null')",+ [runId, leaseToken, stop ? JSON.stringify(stop) : null],);return rowCount > 0;},@@ -356,7 +368,11 @@ export function createPostgresRunStore(connectionString: string, opts?: { maxClaif (!run || run.leaseToken !== leaseToken) return { requeued: false };return {requeued: (- await retire(run, error, opts?.retry !== false, { countsAsError: true, retryAfterMs: opts?.retryAfterMs })+ await retire(run, error, opts?.retry !== false, {+ countsAsError: true,+ retryAfterMs: opts?.retryAfterMs,+ ...(opts?.stop ? { stop: opts.stop } : {}),+ })).requeued,};},diff --git a/src/runs/run-store.ts b/src/runs/run-store.tsindex 4286db883..a9d5a2015 100644--- a/src/runs/run-store.ts+++ b/src/runs/run-store.ts@@ -19,6 +19,10 @@ export interface RunDeliveryState {editRef?: string;}+/** Why a run's previous attempt stopped, recorded when the run is handed back to the queue. */+export type RunStop =+ { kind: "shutdown" } | { kind: "lease_expired" } | { kind: "error"; code?: string; status?: number };+export interface Run {id: string;sessionId: string;@@ -37,6 +41,8 @@ export interface Run {createdAt: number;startedAt: number | null;finishedAt: number | null;+ /** Why the previous attempt stopped; null on a run's first attempt. */+ lastStop?: RunStop | null;}interface TerminalCursor {@@ -72,7 +78,8 @@ export interface RunStore {heartbeat(runId: string, leaseToken: string, ttlMs: number): Promise<boolean>;- releaseLease(runId: string, leaseToken: string): Promise<boolean>;+ /** Hands a running run back to the queue. `stop` records why the attempt ended; omit it when no attempt started. */+ releaseLease(runId: string, leaseToken: string, stop?: RunStop): Promise<boolean>;complete(runId: string, leaseToken: string, result: TurnResult): Promise<boolean>;@@ -80,7 +87,7 @@ export interface RunStore {runId: string,leaseToken: string,error: string,- opts?: { retry?: boolean; retryAfterMs?: number },+ opts?: { retry?: boolean; retryAfterMs?: number; stop?: RunStop },): Promise<{ requeued: boolean }>;setDeliveryState(runId: string, leaseToken: string | null, state: RunDeliveryState): Promise<boolean>;diff --git a/src/runs/worker.ts b/src/runs/worker.tsindex fb3315654..931f00c54 100644--- a/src/runs/worker.ts+++ b/src/runs/worker.ts@@ -4,9 +4,9 @@ import type { ErrorLog } from "../admin/error-log.ts";import { conversationScope } from "../resolution/resolution-service.ts";import type { TurnResult } from "../types.ts";import type { Orchestrator } from "../core/orchestrator.ts";-import { isNonRetryable, NonRetryableTurnError, turnFailureMessage } from "../core/turn-error.ts";+import { isNonRetryable, NonRetryableTurnError, ProviderTurnError, turnFailureMessage } from "../core/turn-error.ts";import { resolveTurnOrigin } from "../core/turn-origin.ts";-import { errorParks, type Run, type RunStore } from "./run-store.ts";+import { errorParks, type Run, type RunStop, type RunStore } from "./run-store.ts";import { errMessage, errorAlreadyReported, reportFailure, swallow } from "../util/errors.ts";import { sleep } from "../util/async.ts";import { retryDelay } from "./retry-delay.ts";@@ -37,6 +37,7 @@ export async function processRun(if (opts?.shutdown?.aborted) onShutdown();else opts?.shutdown?.addEventListener("abort", onShutdown, { once: true });let workDeadline: ReturnType<typeof setTimeout> | undefined;+ let started = false;let consecutiveLost = 0;let leaseLost = false;const beat = setInterval(() => {@@ -76,11 +77,13 @@ export async function processRun(}if (run.request.swarm && run.attempts > 3) throw new NonRetryableTurnError("swarm claim budget exhausted");const queueMs = run.startedAt !== null ? Math.max(0, run.startedAt - run.createdAt) : undefined;+ started = true;const result = await deps.orchestrator.handleTurn({...run.request,origin: resolveTurnOrigin(run.request),runId: run.id,attempt: run.attempts,+ ...(run.lastStop ? { resumeCause: run.lastStop } : {}),runLeaseToken: token,finalAttempt: errorParks(run, deps.runs.maxClaims),background: opts?.background ?? false,@@ -112,6 +115,7 @@ export async function processRun(await deps.runs.fail(run.id, token, turnFailureMessage(err), {retry: !isNonRetryable(err),retryAfterMs: retryDelay(run.errorAttempts),+ stop: runStopFor(err),});throw err;} finally {@@ -120,7 +124,7 @@ export async function processRun(opts?.shutdown?.removeEventListener("abort", onShutdown);if (opts?.shutdown?.aborted)await deps.runs- .releaseLease(run.id, token)+ .releaseLease(run.id, token, started ? { kind: "shutdown" } : undefined).catch((e) => swallow(`worker: shutdown handback failed run=${run.id}; lease will expire after exit`, e));}}@@ -273,3 +277,10 @@ export function createWorker(deps: WorkerDeps): Worker {},};}++/** What a resumed attempt is told about why this one failed: the typed provider code and status, never raw error text. */+function runStopFor(err: unknown): RunStop {+ if (err instanceof ProviderTurnError)+ return { kind: "error", code: err.code, ...(err.status !== undefined ? { status: err.status } : {}) };+ return { kind: "error" };+}diff --git a/test/claude-harness-turn.test.ts b/test/claude-harness-turn.test.tsindex 07f6df9c2..7f45df535 100644--- a/test/claude-harness-turn.test.ts+++ b/test/claude-harness-turn.test.ts@@ -758,7 +758,7 @@ test("Claude ignores continueTurn and still prompts the resume note as a user meassert.equal(result.reply, "picking up");assert.equal(entries[0]?.type, "user");assert.equal((entries[0]!.payload as { text: string }).text, note);- assert.ok(prompted[0]!.includes("(system note:"), "the note reaches the model as a user prompt");+ assert.ok(prompted[0]!.includes("[system] Your previous attempt"), "the note reaches the model as a user prompt");});for (const shutdown of [false, true]) {diff --git a/test/compact-transcript.test.ts b/test/compact-transcript.test.tsindex 1c6932007..6920f09b1 100644--- a/test/compact-transcript.test.ts+++ b/test/compact-transcript.test.ts@@ -15,7 +15,7 @@ test("compactTranscript marks an interrupted tool call (no recorded result) so tent("tool_call", { tool: "execute", command: "bash refresh.sh", callId: "c1" }, 2),ent("user", { text: "(system note: the platform restarted mid-turn...)" }, 3),]);- assert.match(out, /tool_result for tool_call#2 \(none recorded\): \[interrupted/);+ assert.match(out, /tool_result for tool_call#2 \(none recorded\): \[system\] This call was interrupted/);assert.ok(out.includes(INTERRUPTED_TOOL_RESULT));});diff --git a/test/context-compaction.test.ts b/test/context-compaction.test.tsindex 3dac28fa3..eea926d0c 100644--- a/test/context-compaction.test.ts+++ b/test/context-compaction.test.ts@@ -1448,7 +1448,7 @@ test("retry recovery supplies an excluded request without rerunning its completerunTurn: async (input) => {mainCalls++;assert.match(input.input, /Current request.*\nfinish the stored request/);- assert.match(input.input, /don't start over or repeat completed steps/);+ assert.match(input.input, /don't repeat completed steps/);assert.ok(!input.history.some((e) => e.type === "user"));await input.emit({ type: "assistant", payload: { text: "resumed" }, scopeLabel: input.scopeLabel });return { reply: "resumed" };diff --git a/test/orchestrator-steer-visibility.test.ts b/test/orchestrator-steer-visibility.test.tsindex 68b297e4e..fb90f297b 100644--- a/test/orchestrator-steer-visibility.test.ts+++ b/test/orchestrator-steer-visibility.test.ts@@ -48,7 +48,7 @@ for (const mode of ["retry", "handoff"] as const) {},};}- assert.match(turn.input, /previous attempt|Runtime handoff completed/);+ assert.match(turn.input, /previous attempt|Runtime switched/);await turn.emit({ type: "user", payload: { text: turn.input }, scopeLabel: turn.scopeLabel });await recordSteerIntake(turn, { text: "Change of plan", ts: "steer-visible" });await turn.emit({diff --git a/test/orchestrator.test.ts b/test/orchestrator.test.tsindex 465a63af9..5800c1fe0 100644--- a/test/orchestrator.test.ts+++ b/test/orchestrator.test.ts@@ -399,7 +399,7 @@ test("a retried run RESUMES the interrupted turn from the durable ledger insteadconst res = await app.turn(req);assert.equal(res.status, "ok");- assert.match(res.reply ?? "", /system note: your previous attempt at the request above was interrupted/);+ assert.match(res.reply ?? "", /\[system\] Your previous attempt was interrupted mid-turn/);assert.equal(res.sourceUserSeq, 0, "provenance points at the original interrupted user entry, not the resume note");assert.equal(res.sourceAssistantEntrySeq, 4);@@ -418,7 +418,7 @@ test("a retried run RESUMES the interrupted turn from the durable ledger instead);assert.match(userTexts[1]!,- /^\(system note: your previous attempt at the request above was interrupted/,+ /^\[system\] Your previous attempt was interrupted mid-turn/,"the retry prompts a continuation instead",);});@@ -436,7 +436,7 @@ test("the resume note is recorded hidden so no surface renders it as a typed useconst [original, note] = userEntries as [(typeof userEntries)[0], (typeof userEntries)[0]];assert.match(String((note.payload as { text?: string }).text ?? ""),- /^\(system note: your previous attempt at the request above was interrupted/,+ /^\[system\] Your previous attempt was interrupted mid-turn/,);assert.notEqual((original.payload as { hidden?: boolean }).hidden,@@ -458,7 +458,7 @@ test("a retry of an attempt that recorded NO work restarts it — never claims wconst res = await app.turn(req);assert.equal(res.status, "ok");- assert.match(res.reply ?? "", /interrupted before it recorded any work.*Start the request now/s);+ assert.match(res.reply ?? "", /stopped before recording any work.*Start the request now/s);assert.doesNotMatch(res.reply ?? "",/recorded above|don't start over/,diff --git a/test/pi-harness-continue.test.ts b/test/pi-harness-continue.test.tsindex 121265572..7aa0b5783 100644--- a/test/pi-harness-continue.test.ts+++ b/test/pi-harness-continue.test.ts@@ -105,7 +105,7 @@ async function runResume(history: SessionEntry[], overrides: Partial<HarnessTurn}function promptedNote(requests: RequestBody[]): boolean {- return JSON.stringify(requests[0]!.messages).includes("(system note:");+ return JSON.stringify(requests[0]!.messages).includes("[system] Your previous attempt");}test("a genuine resume whose context ends at a recorded tool result continues the assistant turn silently", async () => {diff --git a/test/process-run.test.ts b/test/process-run.test.tsindex 0dfcf1eb6..5c2091002 100644--- a/test/process-run.test.ts+++ b/test/process-run.test.ts@@ -194,6 +194,7 @@ test("a retryable turn failure requeues the run, rethrows, and stops the heartbeconst requeued = await runs.get(run!.id);assert.equal(requeued?.status, "pending", "an ordinary failure goes back on the queue");assert.equal(requeued?.attempts, 1);+ assert.deepEqual(requeued?.lastStop, { kind: "error" }, "an untyped error records no message, only that it failed");assert.equal(await runs.claim("w2", 9_000), null);t.mock.timers.tick(30_000);@@ -220,6 +221,7 @@ test("a retryable turn failure requeues the run, rethrows, and stops the heartbe2,"the retry carries its attempt number so the orchestrator can resume the interrupted turn",);+ assert.deepEqual(seen[0]?.resumeCause, { kind: "error" }, "the retry is told why the last attempt stopped");});test("finalAttempt marks the attempt whose error would park the run, from the claim-time budget", async (t) => {@@ -450,6 +452,7 @@ for (const rejects of [false, true]) {assert.equal(handedBack.errorAttempts, 0);assert.equal(handedBack.result, null);assert.equal(handedBack.leaseToken, null);+ assert.deepEqual(handedBack.lastStop, { kind: "shutdown" });assert.ok(await runs.claim("replacement", 5_000));});}@@ -505,5 +508,6 @@ test("a claim that lands after shutdown began is handed back without starting aassert.equal(handedBack.status, "pending");assert.equal(handedBack.errorAttempts, 0);assert.equal(handedBack.leaseToken, null);+ assert.equal(handedBack.lastStop ?? null, null, "a turn that never started records no stop cause");assert.ok(await runs.claim("replacement", 5_000));});diff --git a/test/replay.test.ts b/test/replay.test.tsindex 809a0bf99..1c2d6f787 100644--- a/test/replay.test.ts+++ b/test/replay.test.ts@@ -596,3 +596,13 @@ test("reconstructMessagesFromHistory keeps parallel tool calls in one assistant["user", "a,b", "toolResult", "toolResult", "c", "toolResult", "text"],);});++test("a tool result cut off by a deploy says so in its replayed result", () => {+ const history: SessionEntry[] = [+ ent("user", { text: "deploy it" }, 1),+ ent("tool_call", { tool: "execute", callId: "c1", command: "make deploy" }, 2),+ ent("tool_result", { callId: "c1", result: "[exit 143]", interrupted: true, stop: { kind: "shutdown" } }, 3),+ ];+ const text = JSON.stringify(reconstructMessagesFromHistory(history));+ assert.match(text, /A routine platform deploy interrupted this call/);+});diff --git a/test/run-store.test.ts b/test/run-store.test.tsindex 53abb9ac0..170dfef1a 100644--- a/test/run-store.test.ts+++ b/test/run-store.test.ts@@ -264,6 +264,33 @@ for (const backend of backends) {assert.equal(reclaimed?.errorAttempts, 0);});+ test(`[${backend.name}] a requeued run records why its last attempt stopped`, async () => {+ const { runs } = backend.make();+ const r = (await runs.enqueue({ sessionId: "s1", request: turn("long turn") })).run;+ assert.equal((await runs.get(r.id))?.lastStop ?? null, null, "a fresh run has no stop cause");++ const first = await runs.claim("w1", 60_000);+ await runs.releaseLease(r.id, first!.leaseToken!, { kind: "shutdown" });+ assert.deepEqual((await runs.get(r.id))?.lastStop, { kind: "shutdown" });++ const second = await runs.claim("w1", 60_000);+ const stop = { kind: "error" as const, code: "transient", status: 503 };+ await runs.fail(r.id, second!.leaseToken!, "boom", { retry: true, stop });+ assert.deepEqual((await runs.get(r.id))?.lastStop, stop);++ await sleep(5);+ const unstarted = await runs.claim("w1", 60_000);+ await runs.releaseLease(r.id, unstarted!.leaseToken!);+ assert.deepEqual((await runs.get(r.id))?.lastStop, stop, "a handback before the attempt started keeps the cause");++ await sleep(5);+ const third = await runs.claim("w1", 1);+ assert.ok(third, "claimable once the retry delay passes");+ await sleep(5);+ await runs.reapExpired();+ assert.deepEqual((await runs.get(r.id))?.lastStop, { kind: "lease_expired" });+ });+test(`[${backend.name}] releaseLease with a wrong/stale token is a no-op`, async () => {const { runs } = backend.make();const r = (await runs.enqueue({ sessionId: "s1", request: turn("hi") })).run;diff --git a/test/tool-call-resume.test.ts b/test/tool-call-resume.test.tsindex 247f9c4b9..04ab0ae5a 100644--- a/test/tool-call-resume.test.ts+++ b/test/tool-call-resume.test.ts@@ -322,8 +322,7 @@ test("a retry whose dangling call was marked retry-safe re-runs it silently instconst turn = received[0]!;assert.deepEqual(turn.resumeToolCall, { callId: "c-budget", tool: "history", input: { query: "budget" } });assert.ok(isResumeNote(turn.input));- assert.match(turn.input, /result of the tool call that was in flight is recorded above/);- assert.doesNotMatch(turn.input, /unknown outcome/);+ assert.match(turn.input, /interrupted mid-turn/);const entries = await sessions.getEntries(sessionId);const types = entries.map((e) => e.type);@@ -334,7 +333,7 @@ for (const [label, payload] of [["unsafe", { tool: "history", query: "budget", callId: "c-budget", retrySafe: false }],["unmarked", { tool: "history", query: "budget", callId: "c-budget" }],] as const) {- test(`a retry whose dangling call is ${label} gets the routine-deploy interrupted note and no re-run`, async () => {+ test(`a retry whose dangling call is ${label} records what happened as its result and does not re-run it`, async () => {const { orchestrator, sessions, runs, received, input } = buildScenario();const run = (await runs.enqueue({ sessionId: conversation.threadRef, request: input(ASK) })).run;const { sessionId, userSeq } = await seedDeadAttempt(sessions, payload);@@ -346,11 +345,16 @@ for (const [label, payload] of [const turn = received[0]!;assert.equal(turn.resumeToolCall, undefined);assert.ok(isResumeNote(turn.input));- assert.match(turn.input, /routine platform deploy/);- assert.match(turn.input, /almost never warrants mentioning|almost never worth mentioning/);- assert.match(turn.input, /unknown outcome/);- assert.match(turn.input, /check what actually happened before redoing anything with side effects/);- const types = (await sessions.getEntries(sessionId)).map((e) => e.type);- assert.deepEqual(types, ["user", "tool_call", "user", "assistant"], "the call stays open; nothing is re-run");+ const entries = await sessions.getEntries(sessionId);+ assert.deepEqual(+ entries.map((e) => e.type),+ ["user", "tool_call", "tool_result", "user", "assistant"],+ "nothing is re-run; the call gets a recorded result",+ );+ const result = entries[2]!.payload as { callId?: string; isError?: boolean; result?: string };+ assert.equal(result.callId, "c-budget");+ assert.equal(result.isError, true);+ assert.match(result.result ?? "", /^\[system\] This call was interrupted; its outcome wasn't recorded\./);+ assert.ok(turn.historyTypes.includes("tool_result"), "the model sees the recorded result");});}diff --git a/test/turn-resume.test.ts b/test/turn-resume.test.tsindex b4a66b8c2..b240e921d 100644--- a/test/turn-resume.test.ts+++ b/test/turn-resume.test.ts@@ -5,7 +5,9 @@ import {isResumeNote,resumeNote,resumeStrategy,+ runtimeSwitchNote,turnAtSeq,+ unrecordedToolResult,} from "../src/core/turn-resume.ts";import type { SessionEntry } from "../src/types.ts";@@ -115,14 +117,16 @@ test("resumeNote is recognized by isResumeNote and mentions background jobs onlyassert.ok(isResumeNote(resumeNote({ backgroundJobs: true })));assert.doesNotMatch(resumeNote(), /background/i);assert.match(resumeNote({ backgroundJobs: true }), /`background` list\/poll/);+ assert.ok(isResumeNote(runtimeSwitchNote(undefined)));+ assert.ok(isResumeNote(runtimeSwitchNote({ harnessId: "pi", modelId: "claude-opus-5-5" })));assert.ok(!isResumeNote("build and deploy the release"));});test("the no-work wording promises nothing recorded, and neither wording re-sends the input", () => {const restart = resumeNote({ strategy: { kind: "restart" } });assert.ok(isResumeNote(restart));- assert.doesNotMatch(restart, /recorded above|Continue from where you left off/);- assert.match(restart, /nothing to pick up\. Start the request now/);+ assert.doesNotMatch(restart, /recorded work|Continue/);+ assert.match(restart, /Start the request now/);for (const note of [resumeNote(), restart]) assert.doesNotMatch(note, /build and deploy the release/);});@@ -227,13 +231,25 @@ test("resumeStrategy re-runs a lone dangling call the model marked retry-safe, w});});-test("resumeStrategy falls back to the interrupted note when the dangling call is unsafe, unmarked, or ambiguous", () => {+test("resumeStrategy records what happened to calls it cannot safely re-run: unsafe, unmarked, or ambiguous", () => {const unsafe = [user("build and deploy the release", 1), safeCall(2, false)];- assert.deepEqual(resumeStrategy(unsafe, { userSeq: 1, workEntries: 1 }), { kind: "note" });+ assert.deepEqual(resumeStrategy(unsafe, { userSeq: 1, workEntries: 1 }), {+ kind: "settle",+ calls: [{ callId: "c2", tool: "history" }],+ });const unmarked = [user("build and deploy the release", 1), toolCall(2)];- assert.deepEqual(resumeStrategy(unmarked, { userSeq: 1, workEntries: 1 }), { kind: "note" });+ assert.deepEqual(resumeStrategy(unmarked, { userSeq: 1, workEntries: 1 }), {+ kind: "settle",+ calls: [{ callId: "c2", tool: "execute" }],+ });const twoDangling = [user("build and deploy the release", 1), safeCall(2, true), safeCall(3, true)];- assert.deepEqual(resumeStrategy(twoDangling, { userSeq: 1, workEntries: 2 }), { kind: "note" });+ assert.deepEqual(resumeStrategy(twoDangling, { userSeq: 1, workEntries: 2 }), {+ kind: "settle",+ calls: [+ { callId: "c2", tool: "history" },+ { callId: "c3", tool: "history" },+ ],+ });const priorTurnCall = [safeCall(0, true), user("build and deploy the release", 1), toolCall(2), toolResult(3)];assert.deepEqual(resumeStrategy(priorTurnCall, { userSeq: 1, workEntries: 2 }), { kind: "continue" });});@@ -242,22 +258,37 @@ test("resumeStrategy restarts when the dead attempt recorded no work", () => {assert.deepEqual(resumeStrategy([user("x", 1)], { userSeq: 1, workEntries: 0 }), { kind: "restart" });});-test("the resume notes name the routine deploy, and only the retry note promises a complete record", () => {+test("the resume notes state the recorded cause and are recognized as resume notes", () => {const note = resumeNote();assert.ok(isResumeNote(note));- assert.match(note, /routine platform deploy/);- assert.match(note, /almost never worth mentioning to the user/);- assert.match(note, /unknown outcome/);- const retried = resumeNote({ strategy: { kind: "retry", call: { callId: "c1", tool: "history", input: {} } } });- assert.ok(isResumeNote(retried));- assert.match(retried, /result of the tool call that was in flight is recorded above/);- assert.doesNotMatch(retried, /unknown outcome/);- assert.match(retried, /Continue from where you left off/);- assert.doesNotMatch(retried, /history|c1/);- const handoff = resumeNote({ cause: "runtime-change" });- assert.ok(isResumeNote(handoff));- assert.doesNotMatch(handoff, /deploy/);- assert.match(handoff, /unknown outcome/);+ assert.match(note, /cause not recorded/);+ assert.match(note, /Don't mention this/);+ assert.ok(+ isResumeNote(resumeNote({ strategy: { kind: "retry", call: { callId: "c1", tool: "history", input: {} } } })),+ );+});++test("a tool call whose outcome was lost records the cause as its own result", () => {+ assert.match(+ unrecordedToolResult({ kind: "shutdown" }),+ /^\[system\] A routine platform deploy interrupted this call/,+ );+ assert.match(unrecordedToolResult({ kind: "shutdown" }), /Don't mention this/);+ assert.match(+ unrecordedToolResult({ kind: "error", code: "transient", status: 503 }),+ /^\[system\] A model-provider failure \(transient, HTTP 503\) interrupted this call/,+ );+ assert.match(unrecordedToolResult({ kind: "lease_expired" }), /^\[system\] A worker crash interrupted this call/);+ assert.match(unrecordedToolResult(undefined), /^\[system\] This call was interrupted; its outcome wasn't recorded/);+});++test("the runtime switch note names the new runtime", () => {+ assert.match(+ runtimeSwitchNote({ harnessId: "pi", modelId: "claude-opus-5-5", fastMode: true, effortLevel: "high" }),+ /^\[system\] Runtime switched to claude-opus-5-5 fast; effort high\./,+ );+ assert.doesNotMatch(runtimeSwitchNote({ harnessId: "pi", modelId: "claude-opus-5-5" }), /fast|effort/);+ assert.match(runtimeSwitchNote(undefined), /^\[system\] Context was trimmed without a new summary\./);});test("a tool_result marked interrupted does not answer its call, so the call is still resumable", () => {@@ -265,7 +296,7 @@ test("a tool_result marked interrupted does not answer its call, so the call isconst entries = [user("build and deploy the release", 1), safeCall(4, true), killed];assert.equal(resumeStrategy(entries, { userSeq: 1, workEntries: 2 }).kind, "retry");const unsafe = [user("build and deploy the release", 1), safeCall(4, false), killed];- assert.deepEqual(resumeStrategy(unsafe, { userSeq: 1, workEntries: 2 }), { kind: "note" });+ assert.deepEqual(resumeStrategy(unsafe, { userSeq: 1, workEntries: 2 }), { kind: "note" }, "it already has a result");});test("resumeStrategy continues silently only when every call is answered and the ledger ends at a real tool result", () => {@@ -286,7 +317,8 @@ test("resumeStrategy continues silently only when every call is answered and the});assert.deepEqual(resumeStrategy([user("go", 1), toolCall(2), killed(3)], partial), { kind: "note" });assert.deepEqual(resumeStrategy([safeCall(0, true), user("go", 1), toolCall(2)], { userSeq: 1, workEntries: 1 }), {- kind: "note",+ kind: "settle",+ calls: [{ callId: "c2", tool: "execute" }],});assert.deepEqual(resumeStrategy([user("go", 1), toolCall(2), toolResult(3), safeCall(4, true), killed(5)], {@@ -296,3 +328,16 @@ test("resumeStrategy continues silently only when every call is answered and the"retry",);});++test("resume notes name exactly the cause the run store recorded", () => {+ assert.match(resumeNote({ cause: { kind: "shutdown" } }), /mid-turn — a routine platform deploy\./);+ assert.match(+ resumeNote({ cause: { kind: "error", code: "transient", status: 503 } }),+ /a model-provider call failed \(transient, HTTP 503\)/,+ );+ assert.match(+ resumeNote({ strategy: { kind: "restart" }, cause: { kind: "lease_expired" } }),+ /stopped before recording any work — the worker crashed\. Start the request now/,+ );+ assert.match(resumeNote({ cause: { kind: "error" } }), /— an internal error\./);+});diff --git a/test/turn-retry-replay.test.ts b/test/turn-retry-replay.test.tsindex 08d5ad561..1191f16e9 100644--- a/test/turn-retry-replay.test.ts+++ b/test/turn-retry-replay.test.ts@@ -1,3 +1,4 @@+import { isResumeNote } from "../src/core/turn-resume.ts";import { test } from "node:test";import assert from "node:assert/strict";import { mkdtempSync } from "node:fs";@@ -297,7 +298,7 @@ test("a resume whose recorded work ends at a tool result asks the harness to conassert.equal(retry.status, "ok");assert.deepEqual(continued, [undefined, true], "unchanged inventories preserve assistant continuation");assert.doesNotMatch(environments[1]!, /## Where you are|Environment changes/);- assert.match(turns[1]!, /^\(system note:/, "the note still travels as the fallback input");+ assert.ok(isResumeNote(turns[1]!), "the note still travels as the fallback input");assert.equal((await asks()).length, 1);});