Skip to content

armgit / qm

Public Imported from GitHub
0

Multiplayer agent harness for work.

Tell resumed turns the recorded reason the last attempt stopped (#2361)

Regan Bell2fac75dc2f9b
diff --git a/src/core/orchestrator.ts b/src/core/orchestrator.ts
index 9b16df743..a19dcf8d1 100644
--- a/src/core/orchestrator.ts
+++ b/src/core/orchestrator.ts
@@ -139,7 +139,14 @@ import {
withoutAlreadyIngested,
} from "./attachments.ts";
import { parseRef } from "../acl/resource-ref.ts";
-import { findTrailingPartialTurn, resumeNote, resumeStrategy, turnAtSeq } from "./turn-resume.ts";
+import {
+ findTrailingPartialTurn,
+ resumeNote,
+ resumeStrategy,
+ runtimeSwitchNote,
+ turnAtSeq,
+ unrecordedToolResult,
+} from "./turn-resume.ts";
import type { RecordedTurn } from "./turn-resume.ts";
import {
recordedMessageTimestamps,
@@ -2822,12 +2829,33 @@ export function createOrchestrator(deps: OrchestratorDeps): Orchestrator {
const rawEntries = (await deps.sessions.getContextWindow(session.id)).entries;
const priorTurns = memoryHistoryReset ? undefined : input.priorTurns;
- const visibleHistory = filterHistory(forModelContext(rawEntries));
+ let visibleHistory = filterHistory(forModelContext(rawEntries));
const recallStart = Date.now();
const recalled = await context.recallMemory(memoryRecallQuery(input.text, visibleHistory));
const recallMs = Date.now() - recallStart;
const partial = isRetry ? (recordedTurn ?? findTrailingPartialTurn(visibleHistory, input.text)) : null;
- const resumePlan = partial ? resumeStrategy(visibleHistory, partial) : null;
+ let resumePlan = partial ? resumeStrategy(visibleHistory, partial) : null;
+ if (partial && resumePlan?.kind === "settle") {
+ // A tool call's lost outcome is recorded as its result, so the model reads what happened to it there.
+ const unrecorded = resumePlan.calls;
+ const settled = await withManagedRosterVersion(() =>
+ deps.sessions.appendMany(
+ lease,
+ unrecorded.map((call) => ({
+ type: "tool_result" as const,
+ payload: {
+ tool: call.tool,
+ callId: call.callId,
+ isError: true,
+ result: unrecordedToolResult(input.resumeCause),
+ },
+ scopeLabel: scopeId,
+ })),
+ ),
+ );
+ visibleHistory = filterHistory(forModelContext([...rawEntries, ...settled]));
+ resumePlan = resumeStrategy(visibleHistory, partial);
+ }
const resume = resumePlan && resumePlan.kind !== "restart" ? partial : null;
const compactStart = Date.now();
const history = await withManagedRosterVersion(() =>
@@ -2985,13 +3013,16 @@ export function createOrchestrator(deps: OrchestratorDeps): Orchestrator {
);
}
let turnInput = resumePlan
- ? resumeNote({ strategy: resumePlan, backgroundJobs: !!backgroundBroker })
+ ? resumeNote({
+ strategy: resumePlan,
+ backgroundJobs: !!backgroundBroker,
+ ...(input.resumeCause ? { cause: input.resumeCause } : {}),
+ })
: baseText;
const requestInHistory = !!partial && history.some((entry) => entry.seq === partial.userSeq);
const continueTurn =
(resumePlan?.kind === "continue" || resumePlan?.kind === "retry") && requestInHistory && !releasedToolOutput;
- if (partial && !requestInHistory)
- turnInput += `\nCurrent request (continue from recorded work; do not restart):\n${baseText}`;
+ if (partial && !requestInHistory) turnInput += `\nCurrent request:\n${baseText}`;
if (partial) spine.turnUserEntrySeq = partial.userSeq;
if (releasedToolOutput) {
turnInput = `The human released quarantined tool output recorded in the conversation. Continue the original task using that output. The tool action already ran; do not repeat it. Original task: ${baseText}`;
@@ -3660,9 +3691,7 @@ export function createOrchestrator(deps: OrchestratorDeps): Orchestrator {
contextRecovered ||= recovery;
turnAbort.signal.throwIfAborted();
segment = await runHarnessSegment(
- resumeNote({ cause: "runtime-change" }) +
- (recovery ? "\nContext reduced without a new summary." : "\nRuntime handoff completed.") +
- " Continue the user's unfinished request using the saved conversation and tool results. Do not repeat completed actions or ask the user to repeat the request." +
+ runtimeSwitchNote("choice" in segment.runtimeHandoff ? segment.runtimeHandoff.choice : undefined) +
(recovery &&
!resumedHistory.some(
(entry) =>
@@ -3671,7 +3700,7 @@ export function createOrchestrator(deps: OrchestratorDeps): Orchestrator {
typeof entry.payload.text === "string" &&
entry.payload.text.startsWith(baseText),
)
- ? `\nCurrent request (continue from recorded work; do not restart):\n${baseText}`
+ ? `\nCurrent request:\n${baseText}`
: ""),
inbound.images.length ? { images: inbound.images } : {},
{ history: resumedHistory },
diff --git a/src/core/orchestrator/types.ts b/src/core/orchestrator/types.ts
index c394ab724..40da4842e 100644
--- a/src/core/orchestrator/types.ts
+++ b/src/core/orchestrator/types.ts
@@ -1,3 +1,4 @@
+import type { RunStop } from "../../runs/run-store.ts";
import type { ExternalSlackPolicies } from "../../resolution/external-slack.ts";
import type { RuntimeService } from "../../harness/runtime-types.ts";
import type { SandboxResources } from "../../sandbox/sandbox-resources.ts";
@@ -96,6 +97,8 @@ export interface OrchestratorInput extends Omit<
origin: TurnOrigin;
runId?: string;
attempt?: number;
+ /** Why the previous attempt of this run stopped (recorded by the run store); absent on a first attempt. */
+ resumeCause?: RunStop;
runLeaseToken?: string;
runStartedAt?: number;
diff --git a/src/core/turn-resume.ts b/src/core/turn-resume.ts
index 2e4e89440..ab75713cc 100644
--- a/src/core/turn-resume.ts
+++ b/src/core/turn-resume.ts
@@ -1,7 +1,16 @@
+import type { RuntimeChoice } from "../harness/harness.ts";
+import { unrecordedToolResult } from "../harness/context-compaction.ts";
+
+export { unrecordedToolResult };
+import type { RunStop } from "../runs/run-store.ts";
import type { SessionEntry } from "../types.ts";
import { entryDeliveryKey, isOverheardEntry } from "../sessions/session-store.ts";
-const NOTE_HEAD = "(system note:";
+const RESUME_HEAD = "[system] Your previous attempt";
+const RUNTIME_HEAD = "[system] Runtime switched";
+const TRIMMED_HEAD = "[system] Context was trimmed";
+/** Resume notes written before the [system] wording; still present in stored history. */
+const LEGACY_NOTE_HEAD = "(system note:";
export interface PartialTurn {
userSeq: number;
@@ -13,7 +22,8 @@ function entryText(e: SessionEntry): string {
}
export function isResumeNote(text: string): boolean {
- return text.trimStart().startsWith(NOTE_HEAD);
+ const t = text.trimStart();
+ return [RESUME_HEAD, RUNTIME_HEAD, TRIMMED_HEAD, LEGACY_NOTE_HEAD].some((head) => t.startsWith(head));
}
const MODEL_TURN_ENTRY_TYPES = new Set<SessionEntry["type"]>(["user", "assistant", "tool_call", "tool_result"]);
@@ -79,15 +89,23 @@ export interface ResumableToolCall {
}
export type ResumeStrategy =
- { kind: "restart" } | { kind: "note" } | { kind: "continue" } | { kind: "retry"; call: ResumableToolCall };
+ | { kind: "restart" }
+ | { kind: "note" }
+ | { kind: "continue" }
+ | { kind: "retry"; call: ResumableToolCall }
+ /** Tool calls whose outcome was never recorded: record what happened as their results, then continue. */
+ | { kind: "settle"; calls: Array<{ callId: string; tool: string }> };
export function resumeStrategy(entries: readonly SessionEntry[], partial: PartialTurn): ResumeStrategy {
if (partial.workEntries === 0) return { kind: "restart" };
const turn = entries.filter((e) => e.seq > partial.userSeq);
const answered = new Set<unknown>();
+ const withResult = new Set<unknown>();
for (const e of turn) {
const payload = e.payload as { callId?: unknown; interrupted?: unknown } | null;
- if (e.type === "tool_result" && payload?.interrupted !== true) answered.add(payload?.callId);
+ if (e.type !== "tool_result") continue;
+ withResult.add(payload?.callId);
+ if (payload?.interrupted !== true) answered.add(payload?.callId);
}
const dangling = turn.filter(
(e) => e.type === "tool_call" && !answered.has((e.payload as { callId?: unknown } | null)?.callId),
@@ -96,7 +114,16 @@ export function resumeStrategy(entries: readonly SessionEntry[], partial: Partia
const last = turn.findLast((e) => MODEL_TURN_ENTRY_TYPES.has(e.type));
return last?.type === "tool_result" ? { kind: "continue" } : { kind: "note" };
}
- if (dangling.length !== 1) return { kind: "note" };
+ // Calls with no result at all get one recording what happened; a call whose re-run was itself cut off
+ // already has a result, so it only gets the note.
+ const unrecorded = dangling.flatMap((e) => {
+ const p = e.payload as { callId?: unknown; tool?: unknown } | null;
+ return typeof p?.callId === "string" && !withResult.has(p.callId)
+ ? [{ callId: p.callId, tool: typeof p.tool === "string" ? p.tool : "tool" }]
+ : [];
+ });
+ const settle: ResumeStrategy = unrecorded.length ? { kind: "settle", calls: unrecorded } : { kind: "note" };
+ if (dangling.length !== 1) return settle;
const payload = dangling[0]!.payload as {
callId?: unknown;
retrySafe?: unknown;
@@ -111,41 +138,59 @@ export function resumeStrategy(entries: readonly SessionEntry[], partial: Partia
typeof rerun.input !== "object" ||
Array.isArray(rerun.input)
)
- return { kind: "note" };
+ return settle;
return {
kind: "retry",
call: { callId: payload.callId, tool: rerun.tool, input: rerun.input as Record<string, unknown> },
};
}
-const ROUTINE = "This was a routine platform deploy; it is common and almost never worth mentioning to the user.";
-const CONTINUE = "Continue from where you left off; don't start over or repeat completed steps.)";
+export type ResumeCause = RunStop;
+
+/** Why the previous attempt stopped, from the cause the run store recorded. Never guessed. */
+function causePhrase(cause: ResumeCause | undefined): string {
+ switch (cause?.kind) {
+ case "shutdown":
+ return "a routine platform deploy";
+ case "lease_expired":
+ return "the worker crashed";
+ case "error":
+ return cause.code ? `a model-provider call failed (${providerDetail(cause)})` : "an internal error";
+ default:
+ return "cause not recorded";
+ }
+}
+
+function providerDetail(cause: { code?: string; status?: number }): string {
+ return `${cause.code}${cause.status !== undefined ? `, HTTP ${cause.status}` : ""}`;
+}
+
+const QUIET = "Don't mention this.";
export function resumeNote(opts?: {
strategy?: ResumeStrategy;
backgroundJobs?: boolean;
- cause?: "deploy" | "runtime-change";
+ cause?: ResumeCause;
}): string {
- const strategy = opts?.strategy ?? { kind: "note" };
- if (strategy.kind === "restart") {
- return `${NOTE_HEAD} your previous attempt at the request above was interrupted before it recorded any work (a routine platform deploy), so there is nothing to pick up. Start the request now.)`;
- }
- const parts =
- strategy.kind === "retry"
- ? [
- `${NOTE_HEAD} your previous attempt at the request above was paused mid-turn and has resumed.`,
- ROUTINE,
- "The result of the tool call that was in flight is recorded above.",
- ]
- : [
- `${NOTE_HEAD} your previous attempt at the request above was interrupted mid-turn.`,
- opts?.cause === "runtime-change" ? "" : ROUTINE,
- "Your work up to the interruption is recorded above; a tool result marked interrupted has an",
- "unknown outcome, so check what actually happened before redoing anything with side effects.",
- ];
- if (opts?.backgroundJobs) {
- parts.push("Background jobs on your computer kept running — `background` list/poll to check on them.");
- }
- parts.push(CONTINUE);
- return parts.filter(Boolean).join(" ");
+ const why = causePhrase(opts?.cause);
+ if (opts?.strategy?.kind === "restart")
+ return `${RESUME_HEAD} stopped before recording any work — ${why}. Start the request now. ${QUIET}`;
+ return [
+ `${RESUME_HEAD} was interrupted mid-turn — ${why}. Continue from the recorded work; don't repeat completed steps. ${QUIET}`,
+ opts?.backgroundJobs
+ ? "Background jobs on your computer kept running — check them with `background` list/poll."
+ : "",
+ ]
+ .filter(Boolean)
+ .join(" ");
+}
+
+/** Told to the model when a task continues on a new runtime, or after its context was cut down. */
+export function runtimeSwitchNote(choice: RuntimeChoice | undefined): string {
+ const head = choice
+ ? `${RUNTIME_HEAD} to ${choice.modelId}${choice.fastMode ? " fast" : ""}${
+ choice.effortLevel && choice.effortLevel !== "auto" ? `; effort ${choice.effortLevel}` : ""
+ }.`
+ : `${TRIMMED_HEAD} without a new summary.`;
+ return `${head} Continue from the recorded work; don't repeat completed actions or ask the user to restate.`;
}
diff --git a/src/harness/agent-tools.ts b/src/harness/agent-tools.ts
index c38569fd8..1682a4cec 100644
--- a/src/harness/agent-tools.ts
+++ b/src/harness/agent-tools.ts
@@ -387,7 +387,8 @@ export function createAgentTools(ref: ToolContextRef, opts?: AgentToolsOptions):
orgScopeId: ref.orgScopeId ?? ref.scopeLabel,
sourceScopeId,
});
- if (type === "tool_result" && ref.shutdown?.aborted && isObj(payload)) payload = { ...payload, interrupted: true };
+ if (type === "tool_result" && ref.shutdown?.aborted && isObj(payload))
+ payload = { ...payload, interrupted: true, stop: { kind: "shutdown" } };
await ref.emit({ type, payload, scopeLabel });
};
diff --git a/src/harness/context-compaction.ts b/src/harness/context-compaction.ts
index 04454657a..deb996bc1 100644
--- a/src/harness/context-compaction.ts
+++ b/src/harness/context-compaction.ts
@@ -18,7 +18,34 @@ function headTailSlice(s: string, maxChars: number, tailChars: number): string {
}
export const INTERRUPTED_TOOL_RESULT =
- "[interrupted — a routine platform deploy restarted the agent while this tool call was running and its outcome was not recorded. Check what actually happened before redoing anything with side effects.]";
+ "[system] This call was interrupted; its outcome wasn't recorded. Check what happened before retrying.";
+
+/** What stopped an attempt, as recorded by the run store (`RunStop`) or on an interrupted tool result. */
+export type ToolInterruption = { kind: string; code?: string; status?: number } | undefined;
+
+/** What interrupted a tool call, as the subject of "... interrupted this call". */
+function interruptedBy(cause: ToolInterruption): string | null {
+ switch (cause?.kind) {
+ case "shutdown":
+ return "A routine platform deploy";
+ case "lease_expired":
+ return "A worker crash";
+ case "error":
+ return cause.code
+ ? `A model-provider failure (${cause.code}${cause.status !== undefined ? `, HTTP ${cause.status}` : ""})`
+ : "An internal error";
+ default:
+ return null;
+ }
+}
+
+/** The result a tool call gets when the attempt running it stopped before its outcome was recorded. */
+export function unrecordedToolResult(cause: ToolInterruption): string {
+ const by = interruptedBy(cause);
+ return by
+ ? `[system] ${by} interrupted this call; its outcome wasn't recorded. Check what happened before retrying. Don't mention this.`
+ : INTERRUPTED_TOOL_RESULT;
+}
export const CONTEXT_SUMMARY_HEADER =
"[Earlier conversation summary — an index of turns compacted out of your context. Conversation turns and tool calls can be reopened with the history tool (seq parameter; a very long entry returns as head and tail), or searched (query). Tool results are excluded.]";
diff --git a/src/harness/mock-harness.ts b/src/harness/mock-harness.ts
index b7ac9e8ff..967730698 100644
--- a/src/harness/mock-harness.ts
+++ b/src/harness/mock-harness.ts
@@ -1,3 +1,4 @@
+import { isResumeNote } from "../core/turn-resume.ts";
import {
defineHarness,
type Harness,
@@ -181,7 +182,7 @@ export function createMockHarness(): Harness {
} else if (command0 === "!boom-always" || boomAlwaysSessions.has(turn.session.id)) {
boomAlwaysSessions.add(turn.session.id);
throw new Error("boom: simulated turn fault");
- } else if (command0.startsWith("(system note:") && resumePostSessions.has(turn.session.id)) {
+ } else if (isResumeNote(command0) && resumePostSessions.has(turn.session.id)) {
const msg = resumePostSessions.get(turn.session.id)!;
resumePostSessions.delete(turn.session.id);
await turn.emit({
@@ -197,7 +198,7 @@ export function createMockHarness(): Harness {
});
usedTool = true;
reply = r.ok ? "(posted after resume)" : `[not sent] ${r.message ?? "failed"}`;
- } else if (command0.startsWith("(system note:") && shutdownResumeSessions.has(turn.session.id)) {
+ } else if (isResumeNote(command0) && shutdownResumeSessions.has(turn.session.id)) {
shutdownResumeSessions.delete(turn.session.id);
reply = "finished after resume";
} else if (command0 === "!work-until-shutdown") {
diff --git a/src/harness/replay.ts b/src/harness/replay.ts
index 0e745e8e2..2f8b16fc4 100644
--- a/src/harness/replay.ts
+++ b/src/harness/replay.ts
@@ -12,7 +12,7 @@ import type { ConversationTurn, OverheardMessage, SessionEntry } from "../types.
import { deliveryNote } from "../core/attachments.ts";
import { goalCancelledNote } from "./goal.ts";
import { messageRevision, renderMessageRevision } from "../core/message-revisions.ts";
-import { CONTEXT_SUMMARY_HEADER, INTERRUPTED_TOOL_RESULT } from "./context-compaction.ts";
+import { CONTEXT_SUMMARY_HEADER, INTERRUPTED_TOOL_RESULT, unrecordedToolResult } from "./context-compaction.ts";
import { contextSummaryPayload } from "../sessions/session-store.ts";
import { isoFromTs, messageTag } from "../util/message-tag.ts";
import { isObj } from "../util/objects.ts";
@@ -228,9 +228,22 @@ function toolResultMessage(call: ToolCall, re: SessionEntry | undefined, at: num
const base = { role: "toolResult" as const, toolCallId: call.id, toolName: call.name };
const text = (value: string) => [{ type: "text" as const, text: value }];
if (!re) return { ...base, content: text(INTERRUPTED_TOOL_RESULT), isError: true, timestamp: at };
- const p = (re.payload ?? {}) as { interrupted?: unknown; isError?: unknown; result?: unknown; wire?: ToolResultWire };
+ const p = (re.payload ?? {}) as {
+ interrupted?: unknown;
+ stop?: { kind?: unknown };
+ isError?: unknown;
+ result?: unknown;
+ wire?: ToolResultWire;
+ };
if (p.interrupted === true)
- return { ...base, content: text(INTERRUPTED_TOOL_RESULT), isError: true, timestamp: re.createdAt };
+ return {
+ ...base,
+ content: text(
+ typeof p.stop?.kind === "string" ? unrecordedToolResult({ kind: p.stop.kind }) : INTERRUPTED_TOOL_RESULT,
+ ),
+ isError: true,
+ timestamp: re.createdAt,
+ };
return {
...base,
content: p.wire?.content ? resultContent(p.wire.content) : text(String(p.result ?? "")),
diff --git a/src/runs/memory-run-store.ts b/src/runs/memory-run-store.ts
index ea264b0ca..2e6af8408 100644
--- a/src/runs/memory-run-store.ts
+++ b/src/runs/memory-run-store.ts
@@ -1,6 +1,6 @@
import { randomUUID } from "node:crypto";
import { EventEmitter } from "node:events";
-import type { EnqueueInput, EnqueueResult, ReapEvent, Run, RunDeliveryState, RunStore } from "./run-store.ts";
+import type { EnqueueInput, EnqueueResult, ReapEvent, Run, RunDeliveryState, RunStop, RunStore } from "./run-store.ts";
import { isTerminal, leaseLapsed, releasesDedupKey } from "./run-store.ts";
import type { LedgerBegin, ToolLedger } from "./tool-ledger.ts";
@@ -106,13 +106,14 @@ export function createMemoryRunStore(opts?: { maxClaims?: number }): MemoryRunti
return true;
},
- async releaseLease(runId, leaseToken) {
+ async releaseLease(runId, leaseToken, stop) {
const run = runs.get(runId);
if (!run || run.status !== "running" || run.leaseToken !== leaseToken) return false;
run.status = "pending";
run.leaseToken = null;
run.leaseExpiresAt = null;
run.workerId = null;
+ if (stop) run.lastStop = stop;
return true;
},
@@ -136,8 +137,11 @@ export function createMemoryRunStore(opts?: { maxClaims?: number }): MemoryRunti
const run = runs.get(runId);
if (!run || run.leaseToken !== leaseToken) return { requeued: false };
return {
- requeued: retire(run, error, opts?.retry !== false, { countsAsError: true, retryAfterMs: opts?.retryAfterMs })
- .requeued,
+ requeued: retire(run, error, opts?.retry !== false, {
+ countsAsError: true,
+ retryAfterMs: opts?.retryAfterMs,
+ ...(opts?.stop ? { stop: opts.stop } : {}),
+ }).requeued,
};
},
@@ -363,7 +367,7 @@ export function createMemoryRunStore(opts?: { maxClaims?: number }): MemoryRunti
run: Run,
error: string,
retry: boolean,
- opts?: { ifExpiredAt?: number; countsAsError?: boolean; retryAfterMs?: number },
+ opts?: { ifExpiredAt?: number; countsAsError?: boolean; retryAfterMs?: number; stop?: RunStop },
): { requeued: boolean; applied: boolean } {
if (run.status !== "running") return { requeued: false, applied: false };
if (opts?.ifExpiredAt !== undefined && (run.leaseExpiresAt === null || run.leaseExpiresAt > opts.ifExpiredAt)) {
@@ -376,6 +380,7 @@ export function createMemoryRunStore(opts?: { maxClaims?: number }): MemoryRunti
const overClaimed = run.attempts >= maxClaims;
if (retry && run.errorAttempts < run.maxAttempts && !overClaimed) {
run.status = "pending";
+ run.lastStop = opts?.stop ?? (opts?.countsAsError ? { kind: "error" } : { kind: "lease_expired" });
retryAfter.set(run.id, Date.now() + Math.max(0, opts?.retryAfterMs ?? 0));
return { requeued: true, applied: true };
}
diff --git a/src/runs/postgres-run-store.ts b/src/runs/postgres-run-store.ts
index 01d675e66..afac277e6 100644
--- a/src/runs/postgres-run-store.ts
+++ b/src/runs/postgres-run-store.ts
@@ -8,7 +8,7 @@ import { isObj } from "../util/objects.ts";
import type { TurnResult } from "../types.ts";
import type { OrchestratorInput } from "../core/orchestrator.ts";
import { resolveTurnOrigin } from "../core/turn-origin.ts";
-import type { EnqueueInput, EnqueueResult, ReapEvent, Run, RunDeliveryState, RunStore } from "./run-store.ts";
+import type { EnqueueInput, EnqueueResult, ReapEvent, Run, RunDeliveryState, RunStop, RunStore } from "./run-store.ts";
import { isTerminal, releasesDedupKey } from "./run-store.ts";
import { errMessage, swallow } from "../util/errors.ts";
import type { LedgerBegin, ToolLedger } from "./tool-ledger.ts";
@@ -43,6 +43,7 @@ function rowToRun(r: Record<string, unknown>): Run {
createdAt: Number(r.created_at),
startedAt: r.started_at === null ? null : Number(r.started_at),
finishedAt: r.finished_at === null ? null : Number(r.finished_at),
+ lastStop: r.last_stop != null ? (JSON.parse(r.last_stop as string) as RunStop) : null,
};
}
@@ -132,6 +133,10 @@ export function createPostgresRunStore(connectionString: string, opts?: { maxCla
`CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_runs_terminal_finished ON runs(finished_at, id) WHERE status IN ('done','failed')`,
],
},
+ {
+ id: "runs/store/0007-last-stop",
+ statements: [`SET LOCAL lock_timeout = '3s'`, `ALTER TABLE runs ADD COLUMN IF NOT EXISTS last_stop TEXT`],
+ },
],
[
{
@@ -219,7 +224,7 @@ export function createPostgresRunStore(connectionString: string, opts?: { maxCla
run: Run,
error: string,
retry: boolean,
- opts?: { ifExpiredAt?: number; countsAsError?: boolean; retryAfterMs?: number },
+ opts?: { ifExpiredAt?: number; countsAsError?: boolean; retryAfterMs?: number; stop?: RunStop },
): Promise<{ requeued: boolean; applied: boolean }> {
const ifExpiredAt = opts?.ifExpiredAt ?? null;
const countsAsError = opts?.countsAsError ?? false;
@@ -228,9 +233,16 @@ export function createPostgresRunStore(connectionString: string, opts?: { maxCla
if (retry && errorAttemptsAfter < run.maxAttempts && !overClaimed) {
const { rowCount } = await q(
`UPDATE runs SET status='pending', lease_token=NULL, lease_expires_at=NULL, worker_id=NULL,
- error_attempts=error_attempts+$4, retry_after=$5
+ error_attempts=error_attempts+$4, retry_after=$5, last_stop=$6
WHERE id=$1 AND lease_token=$2 AND status='running' AND ($3::bigint IS NULL OR lease_expires_at <= $3) RETURNING pg_notify('qm_run_available', 'null')`,
- [run.id, run.leaseToken, ifExpiredAt, countsAsError ? 1 : 0, Date.now() + Math.max(0, opts?.retryAfterMs ?? 0)],
+ [
+ run.id,
+ run.leaseToken,
+ ifExpiredAt,
+ countsAsError ? 1 : 0,
+ Date.now() + Math.max(0, opts?.retryAfterMs ?? 0),
+ JSON.stringify(opts?.stop ?? (countsAsError ? { kind: "error" } : { kind: "lease_expired" })),
+ ],
);
return { requeued: rowCount > 0, applied: rowCount > 0 };
}
@@ -329,10 +341,10 @@ export function createPostgresRunStore(connectionString: string, opts?: { maxCla
return rowCount > 0;
},
- async releaseLease(runId, leaseToken): Promise<boolean> {
+ async releaseLease(runId, leaseToken, stop): Promise<boolean> {
const { rowCount } = await q(
- "UPDATE runs SET status='pending', lease_token=NULL, lease_expires_at=NULL, worker_id=NULL WHERE id=$1 AND lease_token=$2 AND status='running' RETURNING pg_notify('qm_run_available', 'null')",
- [runId, leaseToken],
+ "UPDATE runs SET status='pending', lease_token=NULL, lease_expires_at=NULL, worker_id=NULL, last_stop=COALESCE($3, last_stop) WHERE id=$1 AND lease_token=$2 AND status='running' RETURNING pg_notify('qm_run_available', 'null')",
+ [runId, leaseToken, stop ? JSON.stringify(stop) : null],
);
return rowCount > 0;
},
@@ -356,7 +368,11 @@ export function createPostgresRunStore(connectionString: string, opts?: { maxCla
if (!run || run.leaseToken !== leaseToken) return { requeued: false };
return {
requeued: (
- await retire(run, error, opts?.retry !== false, { countsAsError: true, retryAfterMs: opts?.retryAfterMs })
+ await retire(run, error, opts?.retry !== false, {
+ countsAsError: true,
+ retryAfterMs: opts?.retryAfterMs,
+ ...(opts?.stop ? { stop: opts.stop } : {}),
+ })
).requeued,
};
},
diff --git a/src/runs/run-store.ts b/src/runs/run-store.ts
index 4286db883..a9d5a2015 100644
--- a/src/runs/run-store.ts
+++ b/src/runs/run-store.ts
@@ -19,6 +19,10 @@ export interface RunDeliveryState {
editRef?: string;
}
+/** Why a run's previous attempt stopped, recorded when the run is handed back to the queue. */
+export type RunStop =
+ { kind: "shutdown" } | { kind: "lease_expired" } | { kind: "error"; code?: string; status?: number };
+
export interface Run {
id: string;
sessionId: string;
@@ -37,6 +41,8 @@ export interface Run {
createdAt: number;
startedAt: number | null;
finishedAt: number | null;
+ /** Why the previous attempt stopped; null on a run's first attempt. */
+ lastStop?: RunStop | null;
}
interface TerminalCursor {
@@ -72,7 +78,8 @@ export interface RunStore {
heartbeat(runId: string, leaseToken: string, ttlMs: number): Promise<boolean>;
- releaseLease(runId: string, leaseToken: string): Promise<boolean>;
+ /** Hands a running run back to the queue. `stop` records why the attempt ended; omit it when no attempt started. */
+ releaseLease(runId: string, leaseToken: string, stop?: RunStop): Promise<boolean>;
complete(runId: string, leaseToken: string, result: TurnResult): Promise<boolean>;
@@ -80,7 +87,7 @@ export interface RunStore {
runId: string,
leaseToken: string,
error: string,
- opts?: { retry?: boolean; retryAfterMs?: number },
+ opts?: { retry?: boolean; retryAfterMs?: number; stop?: RunStop },
): Promise<{ requeued: boolean }>;
setDeliveryState(runId: string, leaseToken: string | null, state: RunDeliveryState): Promise<boolean>;
diff --git a/src/runs/worker.ts b/src/runs/worker.ts
index fb3315654..931f00c54 100644
--- a/src/runs/worker.ts
+++ b/src/runs/worker.ts
@@ -4,9 +4,9 @@ import type { ErrorLog } from "../admin/error-log.ts";
import { conversationScope } from "../resolution/resolution-service.ts";
import type { TurnResult } from "../types.ts";
import type { Orchestrator } from "../core/orchestrator.ts";
-import { isNonRetryable, NonRetryableTurnError, turnFailureMessage } from "../core/turn-error.ts";
+import { isNonRetryable, NonRetryableTurnError, ProviderTurnError, turnFailureMessage } from "../core/turn-error.ts";
import { resolveTurnOrigin } from "../core/turn-origin.ts";
-import { errorParks, type Run, type RunStore } from "./run-store.ts";
+import { errorParks, type Run, type RunStop, type RunStore } from "./run-store.ts";
import { errMessage, errorAlreadyReported, reportFailure, swallow } from "../util/errors.ts";
import { sleep } from "../util/async.ts";
import { retryDelay } from "./retry-delay.ts";
@@ -37,6 +37,7 @@ export async function processRun(
if (opts?.shutdown?.aborted) onShutdown();
else opts?.shutdown?.addEventListener("abort", onShutdown, { once: true });
let workDeadline: ReturnType<typeof setTimeout> | undefined;
+ let started = false;
let consecutiveLost = 0;
let leaseLost = false;
const beat = setInterval(() => {
@@ -76,11 +77,13 @@ export async function processRun(
}
if (run.request.swarm && run.attempts > 3) throw new NonRetryableTurnError("swarm claim budget exhausted");
const queueMs = run.startedAt !== null ? Math.max(0, run.startedAt - run.createdAt) : undefined;
+ started = true;
const result = await deps.orchestrator.handleTurn({
...run.request,
origin: resolveTurnOrigin(run.request),
runId: run.id,
attempt: run.attempts,
+ ...(run.lastStop ? { resumeCause: run.lastStop } : {}),
runLeaseToken: token,
finalAttempt: errorParks(run, deps.runs.maxClaims),
background: opts?.background ?? false,
@@ -112,6 +115,7 @@ export async function processRun(
await deps.runs.fail(run.id, token, turnFailureMessage(err), {
retry: !isNonRetryable(err),
retryAfterMs: retryDelay(run.errorAttempts),
+ stop: runStopFor(err),
});
throw err;
} finally {
@@ -120,7 +124,7 @@ export async function processRun(
opts?.shutdown?.removeEventListener("abort", onShutdown);
if (opts?.shutdown?.aborted)
await deps.runs
- .releaseLease(run.id, token)
+ .releaseLease(run.id, token, started ? { kind: "shutdown" } : undefined)
.catch((e) => swallow(`worker: shutdown handback failed run=${run.id}; lease will expire after exit`, e));
}
}
@@ -273,3 +277,10 @@ export function createWorker(deps: WorkerDeps): Worker {
},
};
}
+
+/** What a resumed attempt is told about why this one failed: the typed provider code and status, never raw error text. */
+function runStopFor(err: unknown): RunStop {
+ if (err instanceof ProviderTurnError)
+ return { kind: "error", code: err.code, ...(err.status !== undefined ? { status: err.status } : {}) };
+ return { kind: "error" };
+}
diff --git a/test/claude-harness-turn.test.ts b/test/claude-harness-turn.test.ts
index 07f6df9c2..7f45df535 100644
--- a/test/claude-harness-turn.test.ts
+++ b/test/claude-harness-turn.test.ts
@@ -758,7 +758,7 @@ test("Claude ignores continueTurn and still prompts the resume note as a user me
assert.equal(result.reply, "picking up");
assert.equal(entries[0]?.type, "user");
assert.equal((entries[0]!.payload as { text: string }).text, note);
- assert.ok(prompted[0]!.includes("(system note:"), "the note reaches the model as a user prompt");
+ assert.ok(prompted[0]!.includes("[system] Your previous attempt"), "the note reaches the model as a user prompt");
});
for (const shutdown of [false, true]) {
diff --git a/test/compact-transcript.test.ts b/test/compact-transcript.test.ts
index 1c6932007..6920f09b1 100644
--- a/test/compact-transcript.test.ts
+++ b/test/compact-transcript.test.ts
@@ -15,7 +15,7 @@ test("compactTranscript marks an interrupted tool call (no recorded result) so t
ent("tool_call", { tool: "execute", command: "bash refresh.sh", callId: "c1" }, 2),
ent("user", { text: "(system note: the platform restarted mid-turn...)" }, 3),
]);
- assert.match(out, /tool_result for tool_call#2 \(none recorded\): \[interrupted/);
+ assert.match(out, /tool_result for tool_call#2 \(none recorded\): \[system\] This call was interrupted/);
assert.ok(out.includes(INTERRUPTED_TOOL_RESULT));
});
diff --git a/test/context-compaction.test.ts b/test/context-compaction.test.ts
index 3dac28fa3..eea926d0c 100644
--- a/test/context-compaction.test.ts
+++ b/test/context-compaction.test.ts
@@ -1448,7 +1448,7 @@ test("retry recovery supplies an excluded request without rerunning its complete
runTurn: async (input) => {
mainCalls++;
assert.match(input.input, /Current request.*\nfinish the stored request/);
- assert.match(input.input, /don't start over or repeat completed steps/);
+ assert.match(input.input, /don't repeat completed steps/);
assert.ok(!input.history.some((e) => e.type === "user"));
await input.emit({ type: "assistant", payload: { text: "resumed" }, scopeLabel: input.scopeLabel });
return { reply: "resumed" };
diff --git a/test/orchestrator-steer-visibility.test.ts b/test/orchestrator-steer-visibility.test.ts
index 68b297e4e..fb90f297b 100644
--- a/test/orchestrator-steer-visibility.test.ts
+++ b/test/orchestrator-steer-visibility.test.ts
@@ -48,7 +48,7 @@ for (const mode of ["retry", "handoff"] as const) {
},
};
}
- assert.match(turn.input, /previous attempt|Runtime handoff completed/);
+ assert.match(turn.input, /previous attempt|Runtime switched/);
await turn.emit({ type: "user", payload: { text: turn.input }, scopeLabel: turn.scopeLabel });
await recordSteerIntake(turn, { text: "Change of plan", ts: "steer-visible" });
await turn.emit({
diff --git a/test/orchestrator.test.ts b/test/orchestrator.test.ts
index 465a63af9..5800c1fe0 100644
--- a/test/orchestrator.test.ts
+++ b/test/orchestrator.test.ts
@@ -399,7 +399,7 @@ test("a retried run RESUMES the interrupted turn from the durable ledger instead
const res = await app.turn(req);
assert.equal(res.status, "ok");
- assert.match(res.reply ?? "", /system note: your previous attempt at the request above was interrupted/);
+ assert.match(res.reply ?? "", /\[system\] Your previous attempt was interrupted mid-turn/);
assert.equal(res.sourceUserSeq, 0, "provenance points at the original interrupted user entry, not the resume note");
assert.equal(res.sourceAssistantEntrySeq, 4);
@@ -418,7 +418,7 @@ test("a retried run RESUMES the interrupted turn from the durable ledger instead
);
assert.match(
userTexts[1]!,
- /^\(system note: your previous attempt at the request above was interrupted/,
+ /^\[system\] Your previous attempt was interrupted mid-turn/,
"the retry prompts a continuation instead",
);
});
@@ -436,7 +436,7 @@ test("the resume note is recorded hidden so no surface renders it as a typed use
const [original, note] = userEntries as [(typeof userEntries)[0], (typeof userEntries)[0]];
assert.match(
String((note.payload as { text?: string }).text ?? ""),
- /^\(system note: your previous attempt at the request above was interrupted/,
+ /^\[system\] Your previous attempt was interrupted mid-turn/,
);
assert.notEqual(
(original.payload as { hidden?: boolean }).hidden,
@@ -458,7 +458,7 @@ test("a retry of an attempt that recorded NO work restarts it — never claims w
const res = await app.turn(req);
assert.equal(res.status, "ok");
- assert.match(res.reply ?? "", /interrupted before it recorded any work.*Start the request now/s);
+ assert.match(res.reply ?? "", /stopped before recording any work.*Start the request now/s);
assert.doesNotMatch(
res.reply ?? "",
/recorded above|don't start over/,
diff --git a/test/pi-harness-continue.test.ts b/test/pi-harness-continue.test.ts
index 121265572..7aa0b5783 100644
--- a/test/pi-harness-continue.test.ts
+++ b/test/pi-harness-continue.test.ts
@@ -105,7 +105,7 @@ async function runResume(history: SessionEntry[], overrides: Partial<HarnessTurn
}
function promptedNote(requests: RequestBody[]): boolean {
- return JSON.stringify(requests[0]!.messages).includes("(system note:");
+ return JSON.stringify(requests[0]!.messages).includes("[system] Your previous attempt");
}
test("a genuine resume whose context ends at a recorded tool result continues the assistant turn silently", async () => {
diff --git a/test/process-run.test.ts b/test/process-run.test.ts
index 0dfcf1eb6..5c2091002 100644
--- a/test/process-run.test.ts
+++ b/test/process-run.test.ts
@@ -194,6 +194,7 @@ test("a retryable turn failure requeues the run, rethrows, and stops the heartbe
const requeued = await runs.get(run!.id);
assert.equal(requeued?.status, "pending", "an ordinary failure goes back on the queue");
assert.equal(requeued?.attempts, 1);
+ assert.deepEqual(requeued?.lastStop, { kind: "error" }, "an untyped error records no message, only that it failed");
assert.equal(await runs.claim("w2", 9_000), null);
t.mock.timers.tick(30_000);
@@ -220,6 +221,7 @@ test("a retryable turn failure requeues the run, rethrows, and stops the heartbe
2,
"the retry carries its attempt number so the orchestrator can resume the interrupted turn",
);
+ assert.deepEqual(seen[0]?.resumeCause, { kind: "error" }, "the retry is told why the last attempt stopped");
});
test("finalAttempt marks the attempt whose error would park the run, from the claim-time budget", async (t) => {
@@ -450,6 +452,7 @@ for (const rejects of [false, true]) {
assert.equal(handedBack.errorAttempts, 0);
assert.equal(handedBack.result, null);
assert.equal(handedBack.leaseToken, null);
+ assert.deepEqual(handedBack.lastStop, { kind: "shutdown" });
assert.ok(await runs.claim("replacement", 5_000));
});
}
@@ -505,5 +508,6 @@ test("a claim that lands after shutdown began is handed back without starting a
assert.equal(handedBack.status, "pending");
assert.equal(handedBack.errorAttempts, 0);
assert.equal(handedBack.leaseToken, null);
+ assert.equal(handedBack.lastStop ?? null, null, "a turn that never started records no stop cause");
assert.ok(await runs.claim("replacement", 5_000));
});
diff --git a/test/replay.test.ts b/test/replay.test.ts
index 809a0bf99..1c2d6f787 100644
--- a/test/replay.test.ts
+++ b/test/replay.test.ts
@@ -596,3 +596,13 @@ test("reconstructMessagesFromHistory keeps parallel tool calls in one assistant
["user", "a,b", "toolResult", "toolResult", "c", "toolResult", "text"],
);
});
+
+test("a tool result cut off by a deploy says so in its replayed result", () => {
+ const history: SessionEntry[] = [
+ ent("user", { text: "deploy it" }, 1),
+ ent("tool_call", { tool: "execute", callId: "c1", command: "make deploy" }, 2),
+ ent("tool_result", { callId: "c1", result: "[exit 143]", interrupted: true, stop: { kind: "shutdown" } }, 3),
+ ];
+ const text = JSON.stringify(reconstructMessagesFromHistory(history));
+ assert.match(text, /A routine platform deploy interrupted this call/);
+});
diff --git a/test/run-store.test.ts b/test/run-store.test.ts
index 53abb9ac0..170dfef1a 100644
--- a/test/run-store.test.ts
+++ b/test/run-store.test.ts
@@ -264,6 +264,33 @@ for (const backend of backends) {
assert.equal(reclaimed?.errorAttempts, 0);
});
+ test(`[${backend.name}] a requeued run records why its last attempt stopped`, async () => {
+ const { runs } = backend.make();
+ const r = (await runs.enqueue({ sessionId: "s1", request: turn("long turn") })).run;
+ assert.equal((await runs.get(r.id))?.lastStop ?? null, null, "a fresh run has no stop cause");
+
+ const first = await runs.claim("w1", 60_000);
+ await runs.releaseLease(r.id, first!.leaseToken!, { kind: "shutdown" });
+ assert.deepEqual((await runs.get(r.id))?.lastStop, { kind: "shutdown" });
+
+ const second = await runs.claim("w1", 60_000);
+ const stop = { kind: "error" as const, code: "transient", status: 503 };
+ await runs.fail(r.id, second!.leaseToken!, "boom", { retry: true, stop });
+ assert.deepEqual((await runs.get(r.id))?.lastStop, stop);
+
+ await sleep(5);
+ const unstarted = await runs.claim("w1", 60_000);
+ await runs.releaseLease(r.id, unstarted!.leaseToken!);
+ assert.deepEqual((await runs.get(r.id))?.lastStop, stop, "a handback before the attempt started keeps the cause");
+
+ await sleep(5);
+ const third = await runs.claim("w1", 1);
+ assert.ok(third, "claimable once the retry delay passes");
+ await sleep(5);
+ await runs.reapExpired();
+ assert.deepEqual((await runs.get(r.id))?.lastStop, { kind: "lease_expired" });
+ });
+
test(`[${backend.name}] releaseLease with a wrong/stale token is a no-op`, async () => {
const { runs } = backend.make();
const r = (await runs.enqueue({ sessionId: "s1", request: turn("hi") })).run;
diff --git a/test/tool-call-resume.test.ts b/test/tool-call-resume.test.ts
index 247f9c4b9..04ab0ae5a 100644
--- a/test/tool-call-resume.test.ts
+++ b/test/tool-call-resume.test.ts
@@ -322,8 +322,7 @@ test("a retry whose dangling call was marked retry-safe re-runs it silently inst
const turn = received[0]!;
assert.deepEqual(turn.resumeToolCall, { callId: "c-budget", tool: "history", input: { query: "budget" } });
assert.ok(isResumeNote(turn.input));
- assert.match(turn.input, /result of the tool call that was in flight is recorded above/);
- assert.doesNotMatch(turn.input, /unknown outcome/);
+ assert.match(turn.input, /interrupted mid-turn/);
const entries = await sessions.getEntries(sessionId);
const types = entries.map((e) => e.type);
@@ -334,7 +333,7 @@ for (const [label, payload] of [
["unsafe", { tool: "history", query: "budget", callId: "c-budget", retrySafe: false }],
["unmarked", { tool: "history", query: "budget", callId: "c-budget" }],
] as const) {
- test(`a retry whose dangling call is ${label} gets the routine-deploy interrupted note and no re-run`, async () => {
+ test(`a retry whose dangling call is ${label} records what happened as its result and does not re-run it`, async () => {
const { orchestrator, sessions, runs, received, input } = buildScenario();
const run = (await runs.enqueue({ sessionId: conversation.threadRef, request: input(ASK) })).run;
const { sessionId, userSeq } = await seedDeadAttempt(sessions, payload);
@@ -346,11 +345,16 @@ for (const [label, payload] of [
const turn = received[0]!;
assert.equal(turn.resumeToolCall, undefined);
assert.ok(isResumeNote(turn.input));
- assert.match(turn.input, /routine platform deploy/);
- assert.match(turn.input, /almost never warrants mentioning|almost never worth mentioning/);
- assert.match(turn.input, /unknown outcome/);
- assert.match(turn.input, /check what actually happened before redoing anything with side effects/);
- const types = (await sessions.getEntries(sessionId)).map((e) => e.type);
- assert.deepEqual(types, ["user", "tool_call", "user", "assistant"], "the call stays open; nothing is re-run");
+ const entries = await sessions.getEntries(sessionId);
+ assert.deepEqual(
+ entries.map((e) => e.type),
+ ["user", "tool_call", "tool_result", "user", "assistant"],
+ "nothing is re-run; the call gets a recorded result",
+ );
+ const result = entries[2]!.payload as { callId?: string; isError?: boolean; result?: string };
+ assert.equal(result.callId, "c-budget");
+ assert.equal(result.isError, true);
+ assert.match(result.result ?? "", /^\[system\] This call was interrupted; its outcome wasn't recorded\./);
+ assert.ok(turn.historyTypes.includes("tool_result"), "the model sees the recorded result");
});
}
diff --git a/test/turn-resume.test.ts b/test/turn-resume.test.ts
index b4a66b8c2..b240e921d 100644
--- a/test/turn-resume.test.ts
+++ b/test/turn-resume.test.ts
@@ -5,7 +5,9 @@ import {
isResumeNote,
resumeNote,
resumeStrategy,
+ runtimeSwitchNote,
turnAtSeq,
+ unrecordedToolResult,
} from "../src/core/turn-resume.ts";
import type { SessionEntry } from "../src/types.ts";
@@ -115,14 +117,16 @@ test("resumeNote is recognized by isResumeNote and mentions background jobs only
assert.ok(isResumeNote(resumeNote({ backgroundJobs: true })));
assert.doesNotMatch(resumeNote(), /background/i);
assert.match(resumeNote({ backgroundJobs: true }), /`background` list\/poll/);
+ assert.ok(isResumeNote(runtimeSwitchNote(undefined)));
+ assert.ok(isResumeNote(runtimeSwitchNote({ harnessId: "pi", modelId: "claude-opus-5-5" })));
assert.ok(!isResumeNote("build and deploy the release"));
});
test("the no-work wording promises nothing recorded, and neither wording re-sends the input", () => {
const restart = resumeNote({ strategy: { kind: "restart" } });
assert.ok(isResumeNote(restart));
- assert.doesNotMatch(restart, /recorded above|Continue from where you left off/);
- assert.match(restart, /nothing to pick up\. Start the request now/);
+ assert.doesNotMatch(restart, /recorded work|Continue/);
+ assert.match(restart, /Start the request now/);
for (const note of [resumeNote(), restart]) assert.doesNotMatch(note, /build and deploy the release/);
});
@@ -227,13 +231,25 @@ test("resumeStrategy re-runs a lone dangling call the model marked retry-safe, w
});
});
-test("resumeStrategy falls back to the interrupted note when the dangling call is unsafe, unmarked, or ambiguous", () => {
+test("resumeStrategy records what happened to calls it cannot safely re-run: unsafe, unmarked, or ambiguous", () => {
const unsafe = [user("build and deploy the release", 1), safeCall(2, false)];
- assert.deepEqual(resumeStrategy(unsafe, { userSeq: 1, workEntries: 1 }), { kind: "note" });
+ assert.deepEqual(resumeStrategy(unsafe, { userSeq: 1, workEntries: 1 }), {
+ kind: "settle",
+ calls: [{ callId: "c2", tool: "history" }],
+ });
const unmarked = [user("build and deploy the release", 1), toolCall(2)];
- assert.deepEqual(resumeStrategy(unmarked, { userSeq: 1, workEntries: 1 }), { kind: "note" });
+ assert.deepEqual(resumeStrategy(unmarked, { userSeq: 1, workEntries: 1 }), {
+ kind: "settle",
+ calls: [{ callId: "c2", tool: "execute" }],
+ });
const twoDangling = [user("build and deploy the release", 1), safeCall(2, true), safeCall(3, true)];
- assert.deepEqual(resumeStrategy(twoDangling, { userSeq: 1, workEntries: 2 }), { kind: "note" });
+ assert.deepEqual(resumeStrategy(twoDangling, { userSeq: 1, workEntries: 2 }), {
+ kind: "settle",
+ calls: [
+ { callId: "c2", tool: "history" },
+ { callId: "c3", tool: "history" },
+ ],
+ });
const priorTurnCall = [safeCall(0, true), user("build and deploy the release", 1), toolCall(2), toolResult(3)];
assert.deepEqual(resumeStrategy(priorTurnCall, { userSeq: 1, workEntries: 2 }), { kind: "continue" });
});
@@ -242,22 +258,37 @@ test("resumeStrategy restarts when the dead attempt recorded no work", () => {
assert.deepEqual(resumeStrategy([user("x", 1)], { userSeq: 1, workEntries: 0 }), { kind: "restart" });
});
-test("the resume notes name the routine deploy, and only the retry note promises a complete record", () => {
+test("the resume notes state the recorded cause and are recognized as resume notes", () => {
const note = resumeNote();
assert.ok(isResumeNote(note));
- assert.match(note, /routine platform deploy/);
- assert.match(note, /almost never worth mentioning to the user/);
- assert.match(note, /unknown outcome/);
- const retried = resumeNote({ strategy: { kind: "retry", call: { callId: "c1", tool: "history", input: {} } } });
- assert.ok(isResumeNote(retried));
- assert.match(retried, /result of the tool call that was in flight is recorded above/);
- assert.doesNotMatch(retried, /unknown outcome/);
- assert.match(retried, /Continue from where you left off/);
- assert.doesNotMatch(retried, /history|c1/);
- const handoff = resumeNote({ cause: "runtime-change" });
- assert.ok(isResumeNote(handoff));
- assert.doesNotMatch(handoff, /deploy/);
- assert.match(handoff, /unknown outcome/);
+ assert.match(note, /cause not recorded/);
+ assert.match(note, /Don't mention this/);
+ assert.ok(
+ isResumeNote(resumeNote({ strategy: { kind: "retry", call: { callId: "c1", tool: "history", input: {} } } })),
+ );
+});
+
+test("a tool call whose outcome was lost records the cause as its own result", () => {
+ assert.match(
+ unrecordedToolResult({ kind: "shutdown" }),
+ /^\[system\] A routine platform deploy interrupted this call/,
+ );
+ assert.match(unrecordedToolResult({ kind: "shutdown" }), /Don't mention this/);
+ assert.match(
+ unrecordedToolResult({ kind: "error", code: "transient", status: 503 }),
+ /^\[system\] A model-provider failure \(transient, HTTP 503\) interrupted this call/,
+ );
+ assert.match(unrecordedToolResult({ kind: "lease_expired" }), /^\[system\] A worker crash interrupted this call/);
+ assert.match(unrecordedToolResult(undefined), /^\[system\] This call was interrupted; its outcome wasn't recorded/);
+});
+
+test("the runtime switch note names the new runtime", () => {
+ assert.match(
+ runtimeSwitchNote({ harnessId: "pi", modelId: "claude-opus-5-5", fastMode: true, effortLevel: "high" }),
+ /^\[system\] Runtime switched to claude-opus-5-5 fast; effort high\./,
+ );
+ assert.doesNotMatch(runtimeSwitchNote({ harnessId: "pi", modelId: "claude-opus-5-5" }), /fast|effort/);
+ assert.match(runtimeSwitchNote(undefined), /^\[system\] Context was trimmed without a new summary\./);
});
test("a tool_result marked interrupted does not answer its call, so the call is still resumable", () => {
@@ -265,7 +296,7 @@ test("a tool_result marked interrupted does not answer its call, so the call is
const entries = [user("build and deploy the release", 1), safeCall(4, true), killed];
assert.equal(resumeStrategy(entries, { userSeq: 1, workEntries: 2 }).kind, "retry");
const unsafe = [user("build and deploy the release", 1), safeCall(4, false), killed];
- assert.deepEqual(resumeStrategy(unsafe, { userSeq: 1, workEntries: 2 }), { kind: "note" });
+ assert.deepEqual(resumeStrategy(unsafe, { userSeq: 1, workEntries: 2 }), { kind: "note" }, "it already has a result");
});
test("resumeStrategy continues silently only when every call is answered and the ledger ends at a real tool result", () => {
@@ -286,7 +317,8 @@ test("resumeStrategy continues silently only when every call is answered and the
});
assert.deepEqual(resumeStrategy([user("go", 1), toolCall(2), killed(3)], partial), { kind: "note" });
assert.deepEqual(resumeStrategy([safeCall(0, true), user("go", 1), toolCall(2)], { userSeq: 1, workEntries: 1 }), {
- kind: "note",
+ kind: "settle",
+ calls: [{ callId: "c2", tool: "execute" }],
});
assert.deepEqual(
resumeStrategy([user("go", 1), toolCall(2), toolResult(3), safeCall(4, true), killed(5)], {
@@ -296,3 +328,16 @@ test("resumeStrategy continues silently only when every call is answered and the
"retry",
);
});
+
+test("resume notes name exactly the cause the run store recorded", () => {
+ assert.match(resumeNote({ cause: { kind: "shutdown" } }), /mid-turn — a routine platform deploy\./);
+ assert.match(
+ resumeNote({ cause: { kind: "error", code: "transient", status: 503 } }),
+ /a model-provider call failed \(transient, HTTP 503\)/,
+ );
+ assert.match(
+ resumeNote({ strategy: { kind: "restart" }, cause: { kind: "lease_expired" } }),
+ /stopped before recording any work — the worker crashed\. Start the request now/,
+ );
+ assert.match(resumeNote({ cause: { kind: "error" } }), /— an internal error\./);
+});
diff --git a/test/turn-retry-replay.test.ts b/test/turn-retry-replay.test.ts
index 08d5ad561..1191f16e9 100644
--- a/test/turn-retry-replay.test.ts
+++ b/test/turn-retry-replay.test.ts
@@ -1,3 +1,4 @@
+import { isResumeNote } from "../src/core/turn-resume.ts";
import { test } from "node:test";
import assert from "node:assert/strict";
import { mkdtempSync } from "node:fs";
@@ -297,7 +298,7 @@ test("a resume whose recorded work ends at a tool result asks the harness to con
assert.equal(retry.status, "ok");
assert.deepEqual(continued, [undefined, true], "unchanged inventories preserve assistant continuation");
assert.doesNotMatch(environments[1]!, /## Where you are|Environment changes/);
- assert.match(turns[1]!, /^\(system note:/, "the note still travels as the fallback input");
+ assert.ok(isResumeNote(turns[1]!), "the note still travels as the fallback input");
assert.equal((await asks()).length, 1);
});