The open-source app everyone uses to manage agents at work
TypeScript acpxagent-orchestrationagent-skillsagentic-aiai-agentsai-engineeringai-governanceai-toolsautonomous-agentsclaudeclaude-codeclaude-skillscodexhermesllmmcpmulti-agentopenclawproject-managementworkflow-automation
fix: run restart recovery, workspace self-heal, quota-aware retries, failed-run metrics (#9183)
Dotta555391fed77d
diff --git a/docs/deploy/dev-plane-restart-hygiene.md b/docs/deploy/dev-plane-restart-hygiene.mdnew file mode 100644index 0000000000..fc9ce98d69--- /dev/null+++ b/docs/deploy/dev-plane-restart-hygiene.md@@ -0,0 +1,57 @@+# Dev-Plane Deploy & Restart Hygiene++Operational runbook for restarting a dev/shared Paperclip control plane without killing in-flight agent work. Written after the 2026-07-06/07 failure spike, where a restart-heavy deploy window was the single largest source of failed tasks.++## Why this matters++Every control-plane restart hard-kills any heartbeat run in flight at that moment. The run is finalized as `failed` with `error_code = 'process_lost'`. There is an automatic single retry (`process_lost_retry`), but it is best-effort: in the 07-06/07 incident window only 12 of 36 lost runs recovered. On the evening of 07-06, **9 restarts in 90 minutes killed 16 in-flight runs**.++## Rules of thumb++1. **Batch your deploys.** Stack up changes and restart once, instead of restart-per-change. A restart storm (several restarts within an hour) multiplies run loss for near-zero benefit.+2. **Restart during low fleet activity.** Check for active runs before restarting; prefer windows where the fleet is quiet. A quick check against the control-plane DB:++ ```sql+ SELECT count(*) FROM heartbeat_runs WHERE status = 'running';+ ```++3. **Drain before restart (upcoming).** Graceful SIGTERM drain — stop accepting new runs, let in-flight runs finish or checkpoint, then exit — is being added in PAP-12930. Once it lands, send SIGTERM and wait for drain instead of hard-restarting. Until then, rule 2 is your drain.+4. **After any restart, glance at the damage.** See the detection queries below; confirm lost runs either retried successfully or get manual follow-up.++## How to spot a restart burst++Two signals, cross-referenced:++**1. Server start markers in the instance log.** Each boot logs `Server listening on <host>:<port>`. Logs live at `~/.paperclip/instances/<instance>/server.log`, rotated daily to `server.log-YYYYMMDD.gz`.++```bash+grep -h "Server listening" ~/.paperclip/instances/default/server.log+zgrep -h "Server listening" ~/.paperclip/instances/default/server.log-20260706.gz+```++Many markers minutes apart = restart burst.++**2. Same-minute `process_lost` clusters in `heartbeat_runs`.** Runs killed by a restart are finalized together, so they cluster on the same minute:++```sql+SELECT date_trunc('minute', finished_at) AS minute, count(*)+FROM heartbeat_runs+WHERE error_code = 'process_lost'+GROUP BY 1 HAVING count(*) > 1+ORDER BY 1 DESC;+```++If the cluster minutes line up with the `Server listening` timestamps, the failures are restart-inflicted, not a product regression.++**Note:** query the DB directly for this — the `/heartbeat-runs` list API ignores the `status=` filter and omits error fields.++## Checking recovery after a burst++Each `process_lost` run should have triggered one retry wake (`reason = 'process_lost_retry'`). To find lost runs that never recovered, look for `process_lost` failures with no subsequent successful run for the same issue, and re-wake or reassign those issues manually.++## Related failure modes (not restart-caused)++Seen in the same incident window; do not confuse them with restart damage:++- `workspace_validation_failed` — deterministic workspace validation retry loops (self-heal: PAP-12931).+- Provider quota exhaustion — dominant cause of `claude_transient_upstream` failures (quota-aware handling: PAP-12932).diff --git a/packages/adapter-utils/src/types.ts b/packages/adapter-utils/src/types.tsindex 80f5f92f86..e290cbbe3c 100644--- a/packages/adapter-utils/src/types.ts+++ b/packages/adapter-utils/src/types.ts@@ -65,7 +65,7 @@ export interface AdapterRuntimeServiceReport {healthStatus?: "unknown" | "healthy" | "unhealthy";}-export type AdapterExecutionErrorFamily = "transient_upstream" | "model_refusal";+export type AdapterExecutionErrorFamily = "transient_upstream" | "provider_quota" | "model_refusal";export interface AdapterExecutionResult {exitCode: number | null;diff --git a/packages/adapters/claude-local/src/server/execute.ts b/packages/adapters/claude-local/src/server/execute.tsindex 3bfec50c62..99bf1f1621 100644--- a/packages/adapters/claude-local/src/server/execute.ts+++ b/packages/adapters/claude-local/src/server/execute.ts@@ -50,6 +50,7 @@ import {detectClaudeLoginRequired,extractClaudeRetryNotBefore,isClaudeMaxTurnsResult,+ isClaudeProviderQuotaError,isClaudeRefusalResult,isClaudeTransientUpstreamError,isClaudeUnknownSessionError,@@ -840,8 +841,18 @@ export async function execute(ctx: AdapterExecutionContext): Promise<AdapterExecif (!parsed) {const fallbackErrorMessage = parseFallbackErrorMessage(proc);+ const providerQuota =+ !loginMeta.requiresLogin &&+ (proc.exitCode ?? 0) !== 0 &&+ isClaudeProviderQuotaError({+ parsed: null,+ stdout: proc.stdout,+ stderr: proc.stderr,+ errorMessage: fallbackErrorMessage,+ });const transientUpstream =!loginMeta.requiresLogin &&+ !providerQuota &&(proc.exitCode ?? 0) !== 0 &&isClaudeTransientUpstreamError({parsed: null,@@ -849,7 +860,7 @@ export async function execute(ctx: AdapterExecutionContext): Promise<AdapterExecstderr: proc.stderr,errorMessage: fallbackErrorMessage,});- const transientRetryNotBefore = transientUpstream+ const transientRetryNotBefore = providerQuota || transientUpstream? extractClaudeRetryNotBefore({parsed: null,stdout: proc.stdout,@@ -859,28 +870,34 @@ export async function execute(ctx: AdapterExecutionContext): Promise<AdapterExec: null;const errorCode = loginMeta.requiresLogin? "claude_auth_required"+ : providerQuota+ ? "provider_quota": transientUpstream? "claude_transient_upstream": null;+ const errorFamily = providerQuota ? "provider_quota" : transientUpstream ? "transient_upstream" : null;return {exitCode: proc.exitCode,signal: proc.signal,timedOut: false,errorMessage: fallbackErrorMessage,errorCode,- errorFamily: transientUpstream ? "transient_upstream" : null,+ errorFamily,retryNotBefore: transientRetryNotBefore ? transientRetryNotBefore.toISOString() : null,errorMeta,resultJson: {stdout: proc.stdout,stderr: proc.stderr,- ...(transientUpstream ? { errorFamily: "transient_upstream" } : {}),+ ...(errorFamily ? { errorFamily } : {}),...(transientRetryNotBefore? { retryNotBefore: transientRetryNotBefore.toISOString() }: {}),...(transientRetryNotBefore? { transientRetryNotBefore: transientRetryNotBefore.toISOString() }: {}),+ ...(providerQuota && transientRetryNotBefore+ ? { providerQuotaRetryNotBefore: transientRetryNotBefore.toISOString() }+ : {}),},clearSession: Boolean(opts.clearSessionOnMissingSession),};@@ -907,7 +924,9 @@ export async function execute(ctx: AdapterExecutionContext): Promise<AdapterExec// successful run to Paperclip and the heartbeat stalls silently. See RY-604.const claudeRefusal = isClaudeRefusalResult(parsed);const parsedIsError = asBoolean(parsed.is_error, false);- const failed = (proc.exitCode ?? 0) !== 0 || parsedIsError;+ const parsedSubtype = asString(parsed.subtype, "").trim().toLowerCase();+ const parsedSucceeded = parsedSubtype === "success" && !parsedIsError;+ const failed = !parsedSucceeded && ((proc.exitCode ?? 0) !== 0 || parsedIsError);// Validate-before-persist guard: never persist a sessionId whose transcript// is known-poisoned. The Claude CLI keeps an on-disk JSONL keyed by the// session id; if the last entry contains a non-`msg_`-prefixed@@ -935,18 +954,30 @@ export async function execute(ctx: AdapterExecutionContext): Promise<AdapterExecconst errorMessage = failed? describeClaudeFailure(parsed) ?? `Claude exited with code ${proc.exitCode ?? -1}`: null;+ const providerQuota =+ failed &&+ !loginMeta.requiresLogin &&+ !clearSessionForMaxTurns &&+ !poisonedPreviousMessageId &&+ isClaudeProviderQuotaError({+ parsed,+ stdout: proc.stdout,+ stderr: proc.stderr,+ errorMessage,+ });const transientUpstream =failed &&!loginMeta.requiresLogin &&!clearSessionForMaxTurns &&!poisonedPreviousMessageId &&+ !providerQuota &&isClaudeTransientUpstreamError({parsed,stdout: proc.stdout,stderr: proc.stderr,errorMessage,});- const transientRetryNotBefore = transientUpstream+ const transientRetryNotBefore = providerQuota || transientUpstream? extractClaudeRetryNotBefore({parsed,stdout: proc.stdout,@@ -960,19 +991,29 @@ export async function execute(ctx: AdapterExecutionContext): Promise<AdapterExec? "max_turns_exhausted": failed && poisonedPreviousMessageId? "claude_poisoned_previous_message_id"+ : providerQuota+ ? "provider_quota": transientUpstream? "claude_transient_upstream": claudeRefusal? "claude_refusal": null;+ const errorFamily = providerQuota+ ? "provider_quota"+ : transientUpstream+ ? "transient_upstream"+ : claudeRefusal+ ? "model_refusal"+ : null;const mergedResultJson: Record<string, unknown> = {...parsed,...(failed && clearSessionForMaxTurns ? { stopReason: "max_turns_exhausted" } : {}),...(failed && poisonedPreviousMessageId ? { stopReason: "claude_poisoned_previous_message_id" } : {}),...(claudeRefusal ? { stopReason: "refusal", errorFamily: "model_refusal" } : {}),- ...(transientUpstream ? { errorFamily: "transient_upstream" } : {}),+ ...(errorFamily ? { errorFamily } : {}),...(transientRetryNotBefore ? { retryNotBefore: transientRetryNotBefore.toISOString() } : {}),...(transientRetryNotBefore ? { transientRetryNotBefore: transientRetryNotBefore.toISOString() } : {}),+ ...(providerQuota && transientRetryNotBefore ? { providerQuotaRetryNotBefore: transientRetryNotBefore.toISOString() } : {}),};return {@@ -981,11 +1022,7 @@ export async function execute(ctx: AdapterExecutionContext): Promise<AdapterExectimedOut: false,errorMessage,errorCode: resolvedErrorCode,- errorFamily: transientUpstream- ? "transient_upstream"- : claudeRefusal- ? "model_refusal"- : null,+ errorFamily,retryNotBefore: transientRetryNotBefore ? transientRetryNotBefore.toISOString() : null,errorMeta,usage,diff --git a/packages/adapters/claude-local/src/server/index.ts b/packages/adapters/claude-local/src/server/index.tsindex 31ba501f4f..4d50d30cc3 100644--- a/packages/adapters/claude-local/src/server/index.ts+++ b/packages/adapters/claude-local/src/server/index.ts@@ -10,6 +10,7 @@ export {parseClaudeStreamJson,describeClaudeFailure,isClaudeMaxTurnsResult,+ isClaudeProviderQuotaError,isClaudeRefusalResult,isClaudeUnknownSessionError,} from "./parse.js";diff --git a/packages/adapters/claude-local/src/server/parse.test.ts b/packages/adapters/claude-local/src/server/parse.test.tsindex 0169e5ac33..557df0c060 100644--- a/packages/adapters/claude-local/src/server/parse.test.ts+++ b/packages/adapters/claude-local/src/server/parse.test.ts@@ -2,6 +2,7 @@ import { describe, expect, it } from "vitest";import {detectClaudeLoginRequired,extractClaudeRetryNotBefore,+ isClaudeProviderQuotaError,isClaudeTransientUpstreamError,isClaudePoisonedPreviousMessageIdError,isClaudeRefusalResult,@@ -32,20 +33,36 @@ describe("detectClaudeLoginRequired", () => {});describe("isClaudeTransientUpstreamError", () => {- it("classifies the 'out of extra usage' subscription window failure as transient", () => {+ it("classifies the 'out of extra usage' subscription window failure as provider quota", () => {expect(- isClaudeTransientUpstreamError({+ isClaudeProviderQuotaError({errorMessage: "You're out of extra usage · resets 4pm (America/Chicago)",}),).toBe(true);expect(- isClaudeTransientUpstreamError({+ isClaudeProviderQuotaError({parsed: {is_error: true,result: "You're out of extra usage. Resets at 4pm (America/Chicago).",},}),).toBe(true);+ expect(+ isClaudeTransientUpstreamError({+ errorMessage: "You're out of extra usage · resets 4pm (America/Chicago)",+ }),+ ).toBe(false);+ });++ it("classifies Claude session-limit windows as provider quota and extracts the retry time", () => {+ const now = new Date("2026-04-22T15:15:00.000Z");+ const errorMessage = "You've hit your session limit - resets at 4pm (America/Chicago).";++ expect(isClaudeProviderQuotaError({ errorMessage })).toBe(true);+ expect(isClaudeTransientUpstreamError({ errorMessage })).toBe(false);+ expect(extractClaudeRetryNotBefore({ errorMessage }, now)?.toISOString()).toBe(+ "2026-04-22T21:00:00.000Z",+ );});it("classifies Anthropic API rate_limit_error and overloaded_error as transient", () => {@@ -77,14 +94,14 @@ describe("isClaudeTransientUpstreamError", () => {).toBe(true);});- it("classifies the subscription 5-hour / weekly limit wording", () => {+ it("classifies the subscription 5-hour / weekly limit wording as provider quota", () => {expect(- isClaudeTransientUpstreamError({+ isClaudeProviderQuotaError({errorMessage: "Claude usage limit reached — weekly limit reached. Try again in 2 days.",}),).toBe(true);expect(- isClaudeTransientUpstreamError({+ isClaudeProviderQuotaError({errorMessage: "5-hour limit reached.",}),).toBe(true);diff --git a/packages/adapters/claude-local/src/server/parse.ts b/packages/adapters/claude-local/src/server/parse.tsindex 657a476a01..9b279e974d 100644--- a/packages/adapters/claude-local/src/server/parse.ts+++ b/packages/adapters/claude-local/src/server/parse.ts@@ -11,8 +11,10 @@ const URL_RE = /(https?:\/\/[^\s'"`<>()[\]{};,!?]+[^\s'"`<>()[\]{};,!.?:]+)/gi;const CLAUDE_TRANSIENT_UPSTREAM_RE =/(?:rate[-\s]?limit(?:ed)?|rate_limit_error|too\s+many\s+requests|\b429\b|overloaded(?:_error)?|server\s+overloaded|service\s+unavailable|\b503\b|\b529\b|high\s+demand|try\s+again\s+later|temporarily\s+unavailable|throttl(?:ed|ing)|throttlingexception|servicequotaexceededexception|out\s+of\s+extra\s+usage|extra\s+usage\b|claude\s+usage\s+limit\s+reached|5[-\s]?hour\s+limit\s+reached|weekly\s+limit\s+reached|usage\s+limit\s+reached|usage\s+cap\s+reached)/i;+const CLAUDE_PROVIDER_QUOTA_RE =+ /(?:you(?:'|’)ve\s+hit\s+your\s+session\s+limit|session\s+limit\s+(?:reached|exceeded)|out\s+of\s+extra\s+usage|extra\s+usage\b|claude\s+usage\s+limit\s+reached|5[-\s]?hour\s+limit\s+reached|weekly\s+limit\s+reached|usage\s+limit\s+reached|usage\s+cap\s+reached|servicequotaexceededexception)/i;const CLAUDE_EXTRA_USAGE_RESET_RE =- /(?:out\s+of\s+extra\s+usage|extra\s+usage|usage\s+limit\s+reached|usage\s+cap\s+reached|5[-\s]?hour\s+limit\s+reached|weekly\s+limit\s+reached|claude\s+usage\s+limit\s+reached)[\s\S]{0,80}?\bresets?\s+(?:at\s+)?([^\n()]+?)(?:\s*\(([^)]+)\))?(?:[.!]|\n|$)/i;+ /(?:you(?:'|’)ve\s+hit\s+your\s+session\s+limit|session\s+limit\s+(?:reached|exceeded)|out\s+of\s+extra\s+usage|extra\s+usage|usage\s+limit\s+reached|usage\s+cap\s+reached|5[-\s]?hour\s+limit\s+reached|weekly\s+limit\s+reached|claude\s+usage\s+limit\s+reached)[\s\S]{0,120}?\bresets?\s+(?:at\s+)?([^\n()]+?)(?:\s*\(([^)]+)\))?(?:[.!]|\n|$)/i;export function parseClaudeStreamJson(stdout: string) {let sessionId: string | null = null;@@ -429,5 +431,28 @@ export function isClaudeTransientUpstreamError(input: {const haystack = buildClaudeTransientHaystack(input);if (!haystack) return false;+ if (isClaudeProviderQuotaError(input)) return false;return CLAUDE_TRANSIENT_UPSTREAM_RE.test(haystack);}++export function isClaudeProviderQuotaError(input: {+ parsed?: Record<string, unknown> | null;+ stdout?: string | null;+ stderr?: string | null;+ errorMessage?: string | null;+}): boolean {+ const parsed = input.parsed ?? null;+ if (parsed && (isClaudeMaxTurnsResult(parsed) || isClaudeUnknownSessionError(parsed) || isClaudePoisonedPreviousMessageIdError(parsed) || isClaudeImageProcessingError(parsed))) {+ return false;+ }+ const loginMeta = detectClaudeLoginRequired({+ parsed,+ stdout: input.stdout ?? "",+ stderr: input.stderr ?? "",+ });+ if (loginMeta.requiresLogin) return false;++ const haystack = buildClaudeTransientHaystack(input);+ if (!haystack) return false;+ return CLAUDE_PROVIDER_QUOTA_RE.test(haystack);+}diff --git a/packages/adapters/codex-local/src/server/execute.ts b/packages/adapters/codex-local/src/server/execute.tsindex d1f535ce9a..71522b2add 100644--- a/packages/adapters/codex-local/src/server/execute.ts+++ b/packages/adapters/codex-local/src/server/execute.ts@@ -41,6 +41,7 @@ import {import {parseCodexJsonl,extractCodexRetryNotBefore,+ isCodexProviderQuotaError,isCodexTransientUpstreamError,isCodexUnknownSessionError,} from "./parse.js";@@ -1016,13 +1017,22 @@ export async function execute(ctx: AdapterExecutionContext): Promise<AdapterExecerrorMessage: fallbackErrorMessage,}): null;+ const providerQuota =+ (attempt.proc.exitCode ?? 0) !== 0 &&+ isCodexProviderQuotaError({+ stdout: attempt.proc.stdout,+ stderr: attempt.proc.stderr,+ errorMessage: fallbackErrorMessage,+ });const transientUpstream =(attempt.proc.exitCode ?? 0) !== 0 &&+ !providerQuota &&isCodexTransientUpstreamError({stdout: attempt.proc.stdout,stderr: attempt.proc.stderr,errorMessage: fallbackErrorMessage,});+ const errorFamily = providerQuota ? "provider_quota" : transientUpstream ? "transient_upstream" : null;return {exitCode: attempt.proc.exitCode,@@ -1033,10 +1043,12 @@ export async function execute(ctx: AdapterExecutionContext): Promise<AdapterExec? null: fallbackErrorMessage,errorCode:- transientUpstream+ providerQuota+ ? "provider_quota"+ : transientUpstream? "codex_transient_upstream": null,- errorFamily: transientUpstream ? "transient_upstream" : null,+ errorFamily,retryNotBefore: transientRetryNotBefore ? transientRetryNotBefore.toISOString() : null,usage: attempt.parsed.usage,sessionId: resolvedSessionId,@@ -1050,9 +1062,10 @@ export async function execute(ctx: AdapterExecutionContext): Promise<AdapterExecresultJson: {stdout: attempt.proc.stdout,stderr: attempt.proc.stderr,- ...(transientUpstream ? { errorFamily: "transient_upstream" } : {}),+ ...(errorFamily ? { errorFamily } : {}),...(transientRetryNotBefore ? { retryNotBefore: transientRetryNotBefore.toISOString() } : {}),...(transientRetryNotBefore ? { transientRetryNotBefore: transientRetryNotBefore.toISOString() } : {}),+ ...(providerQuota && transientRetryNotBefore ? { providerQuotaRetryNotBefore: transientRetryNotBefore.toISOString() } : {}),},summary: attempt.parsed.summary,clearSession: Boolean((clearSessionOnMissingSession || forceFreshSession) && !resolvedSessionId),diff --git a/packages/adapters/codex-local/src/server/index.ts b/packages/adapters/codex-local/src/server/index.tsindex 1535f5e91e..e5b4e72547 100644--- a/packages/adapters/codex-local/src/server/index.ts+++ b/packages/adapters/codex-local/src/server/index.ts@@ -12,7 +12,7 @@ export {} from "./codex-home.js";export { listCodexSkills, syncCodexSkills } from "./skills.js";export { testEnvironment } from "./test.js";-export { parseCodexJsonl, isCodexTransientUpstreamError, isCodexUnknownSessionError } from "./parse.js";+export { parseCodexJsonl, isCodexProviderQuotaError, isCodexTransientUpstreamError, isCodexUnknownSessionError } from "./parse.js";export {getQuotaWindows,readCodexAuthInfo,diff --git a/packages/adapters/codex-local/src/server/parse.test.ts b/packages/adapters/codex-local/src/server/parse.test.tsindex bfd15e24ae..7a2f086a3c 100644--- a/packages/adapters/codex-local/src/server/parse.test.ts+++ b/packages/adapters/codex-local/src/server/parse.test.ts@@ -1,6 +1,7 @@import { describe, expect, it } from "vitest";import {extractCodexRetryNotBefore,+ isCodexProviderQuotaError,isCodexTransientUpstreamError,isCodexUnknownSessionError,parseCodexJsonl,@@ -103,16 +104,25 @@ describe("isCodexTransientUpstreamError", () => {).toBe(true);});- it("classifies usage-limit windows as transient and extracts the retry time", () => {+ it("classifies usage-limit windows as provider quota and extracts the retry time", () => {const errorMessage = "You've hit your usage limit for GPT-5.3-Codex-Spark. Switch to another model now, or try again at 11:31 PM.";const now = new Date(2026, 3, 22, 22, 29, 2);- expect(isCodexTransientUpstreamError({ errorMessage })).toBe(true);+ expect(isCodexProviderQuotaError({ errorMessage })).toBe(true);+ expect(isCodexTransientUpstreamError({ errorMessage })).toBe(false);expect(extractCodexRetryNotBefore({ errorMessage }, now)?.getTime()).toBe(new Date(2026, 3, 22, 23, 31, 0, 0).getTime(),);});+ it("classifies model-capacity messages as provider quota without reset metadata", () => {+ const errorMessage = "The requested model is at capacity. Please try again later.";++ expect(isCodexProviderQuotaError({ errorMessage })).toBe(true);+ expect(isCodexTransientUpstreamError({ errorMessage })).toBe(false);+ expect(extractCodexRetryNotBefore({ errorMessage })).toBeNull();+ });+it("parses explicit timezone hints on usage-limit retry windows", () => {const errorMessage = "You've hit your usage limit for GPT-5.3-Codex-Spark. Switch to another model now, or try again at 11:31 PM (America/Chicago).";const now = new Date("2026-04-23T03:29:02.000Z");diff --git a/packages/adapters/codex-local/src/server/parse.ts b/packages/adapters/codex-local/src/server/parse.tsindex 6dd3d5fb3f..535d872f28 100644--- a/packages/adapters/codex-local/src/server/parse.ts+++ b/packages/adapters/codex-local/src/server/parse.ts@@ -10,6 +10,8 @@ const CODEX_TRANSIENT_UPSTREAM_RE =const CODEX_REMOTE_COMPACTION_RE = /remote\s+compact\s+task/i;const CODEX_USAGE_LIMIT_RE =/you(?:'|’)ve hit your usage limit for .+\.\s+switch to another model now,\s+or try again at\s+([^.!\n]+)(?:[.!]|\n|$)/i;+const CODEX_PROVIDER_QUOTA_RE =+ /(?:you(?:'|’)ve hit your usage limit|usage limit|model (?:is )?at capacity|at capacity for this model|capacity limit)/i;export function parseCodexJsonl(stdout: string) {let sessionId: string | null = null;@@ -252,10 +254,18 @@ export function isCodexTransientUpstreamError(input: {}): boolean {const haystack = buildCodexErrorHaystack(input);- if (extractCodexRetryNotBefore(input) != null) return true;+ if (isCodexProviderQuotaError(input)) return false;if (!CODEX_TRANSIENT_UPSTREAM_RE.test(haystack)) return false;// Keep automatic retries scoped to the observed remote-compaction/high-demand- // failure shape, plus explicit usage-limit windows that tell us when retrying- // becomes safe again.+ // failure shape.return CODEX_REMOTE_COMPACTION_RE.test(haystack) || /high\s+demand|temporary\s+errors/i.test(haystack);}++export function isCodexProviderQuotaError(input: {+ stdout?: string | null;+ stderr?: string | null;+ errorMessage?: string | null;+}): boolean {+ const haystack = buildCodexErrorHaystack(input);+ return CODEX_PROVIDER_QUOTA_RE.test(haystack) || extractCodexRetryNotBefore(input) != null;+}diff --git a/packages/shared/src/constants.ts b/packages/shared/src/constants.tsindex 978aa4a5ed..cf100e91c9 100644--- a/packages/shared/src/constants.ts+++ b/packages/shared/src/constants.ts@@ -768,6 +768,7 @@ export const HEARTBEAT_RUN_STATUSES = ["scheduled_retry","running","succeeded",+ "interrupted","failed","cancelled","timed_out",diff --git a/packages/shared/src/types/dashboard.ts b/packages/shared/src/types/dashboard.tsindex e4225f4394..977b124ce1 100644--- a/packages/shared/src/types/dashboard.ts+++ b/packages/shared/src/types/dashboard.ts@@ -1,9 +1,27 @@export interface DashboardRunActivityDay {date: string;succeeded: number;+ /**+ * True failures for the day, excluding process-loss/restart kills that were+ * later recovered by a successful retry (those are surfaced in `recovered`).+ */failed: number;+ /**+ * Runs that terminated in a failure state (failed/timed_out) but whose retry+ * chain eventually succeeded — e.g. restart-killed runs that recovered. Kept+ * out of `failed` so the headline failure count reflects true, unrecovered+ * failures.+ */+ recovered: number;other: number;total: number;+ /**+ * Per-error-code breakdown of the (true) `failed` count for the day, so a+ * spike can be attributed to an error class (e.g. `process_lost`,+ * `provider_quota`, `workspace_validation_failed`). Recovered runs are not+ * included here. Runs with no error code are bucketed under `unknown`.+ */+ failedByErrorCode: Record<string, number>;}export interface DashboardSummary {diff --git a/server/src/__tests__/claude-local-execute.test.ts b/server/src/__tests__/claude-local-execute.test.tsindex 0316ca7c1f..bb212f7858 100644--- a/server/src/__tests__/claude-local-execute.test.ts+++ b/server/src/__tests__/claude-local-execute.test.ts@@ -1272,7 +1272,7 @@ describe("claude execute", () => {}}, 15_000);- it("classifies Claude 'out of extra usage' failures as transient upstream errors", async () => {+ it("classifies Claude 'out of extra usage' failures as provider quota errors", async () => {const root = await fs.mkdtemp(path.join(os.tmpdir(), "paperclip-claude-execute-transient-"));const workspace = path.join(root, "workspace");const commandPath = path.join(root, "claude");@@ -1320,11 +1320,12 @@ describe("claude execute", () => {});expect(result.exitCode).toBe(1);- expect(result.errorCode).toBe("claude_transient_upstream");- expect(result.errorFamily).toBe("transient_upstream");+ expect(result.errorCode).toBe("provider_quota");+ expect(result.errorFamily).toBe("provider_quota");const expectedRetryNotBefore = "2026-04-22T21:00:00.000Z";expect(result.retryNotBefore).toBe(expectedRetryNotBefore);expect(result.resultJson?.retryNotBefore).toBe(expectedRetryNotBefore);+ expect(result.resultJson?.providerQuotaRetryNotBefore).toBe(expectedRetryNotBefore);expect(result.errorMessage ?? "").toContain("extra usage");expect(new Date(String(result.resultJson?.transientRetryNotBefore)).getTime()).toBe(new Date("2026-04-22T21:00:00.000Z").getTime(),@@ -1337,6 +1338,63 @@ describe("claude execute", () => {}});+ it("treats subtype=success results as successful even when the process exits nonzero", async () => {+ const root = await fs.mkdtemp(path.join(os.tmpdir(), "paperclip-claude-execute-success-subtype-"));+ const workspace = path.join(root, "workspace");+ const commandPath = path.join(root, "claude");+ await fs.mkdir(workspace, { recursive: true });+ await writeFailingClaudeCommand(commandPath, {+ exitCode: 1,+ resultEvent: {+ type: "result",+ subtype: "success",+ session_id: "ffffffff-ffff-4fff-8fff-ffffffffffff",+ is_error: false,+ result: "Implemented the requested change.",+ usage: { input_tokens: 1, cache_read_input_tokens: 0, output_tokens: 1 },+ },+ });++ const previousHome = process.env.HOME;+ process.env.HOME = root;++ try {+ const result = await execute({+ runId: "run-claude-success-subtype",+ agent: {+ id: "agent-1",+ companyId: "company-1",+ name: "Claude Coder",+ adapterType: "claude_local",+ adapterConfig: {},+ },+ runtime: {+ sessionId: null,+ sessionParams: null,+ sessionDisplayId: null,+ taskKey: null,+ },+ config: {+ command: commandPath,+ cwd: workspace,+ promptTemplate: "Follow the paperclip heartbeat.",+ },+ context: {},+ authToken: "run-jwt-token",+ onLog: async () => {},+ });++ expect(result.exitCode).toBe(1);+ expect(result.errorMessage).toBeNull();+ expect(result.errorCode).toBeNull();+ expect(result.summary).toBe("Implemented the requested change.");+ } finally {+ if (previousHome === undefined) delete process.env.HOME;+ else process.env.HOME = previousHome;+ await fs.rm(root, { recursive: true, force: true });+ }+ });+it("classifies rate-limit / overloaded failures without reset metadata as transient", async () => {const root = await fs.mkdtemp(path.join(os.tmpdir(), "paperclip-claude-execute-rate-limit-"));const workspace = path.join(root, "workspace");diff --git a/server/src/__tests__/codex-local-execute.test.ts b/server/src/__tests__/codex-local-execute.test.tsindex a4aff6c87b..d8b1c5b104 100644--- a/server/src/__tests__/codex-local-execute.test.ts+++ b/server/src/__tests__/codex-local-execute.test.ts@@ -554,7 +554,7 @@ describe("codex execute", () => {}});- it("persists retry-not-before metadata for codex usage-limit failures", async () => {+ it("persists retry-not-before metadata for codex provider quota failures", async () => {const root = await fs.mkdtemp(path.join(os.tmpdir(), "paperclip-codex-execute-usage-limit-"));const workspace = path.join(root, "workspace");const commandPath = path.join(root, "codex");@@ -601,11 +601,12 @@ describe("codex execute", () => {});expect(result.exitCode).toBe(1);- expect(result.errorCode).toBe("codex_transient_upstream");- expect(result.errorFamily).toBe("transient_upstream");+ expect(result.errorCode).toBe("provider_quota");+ expect(result.errorFamily).toBe("provider_quota");const expectedRetryNotBefore = new Date(2026, 3, 22, 23, 31, 0, 0).toISOString();expect(result.retryNotBefore).toBe(expectedRetryNotBefore);expect(result.resultJson?.retryNotBefore).toBe(expectedRetryNotBefore);+ expect(result.resultJson?.providerQuotaRetryNotBefore).toBe(expectedRetryNotBefore);expect(new Date(String(result.resultJson?.transientRetryNotBefore)).getTime()).toBe(new Date(2026, 3, 22, 23, 31, 0, 0).getTime(),);diff --git a/server/src/__tests__/dashboard-service.test.ts b/server/src/__tests__/dashboard-service.test.tsindex 119f88964b..05a1a738c9 100644--- a/server/src/__tests__/dashboard-service.test.ts+++ b/server/src/__tests__/dashboard-service.test.ts@@ -156,14 +156,85 @@ describeEmbeddedPostgres("dashboard service", () => {expect(todayBucket).toMatchObject({succeeded: 105,failed: 0,+ recovered: 0,other: 0,total: 105,+ failedByErrorCode: {},});expect(weekAgoBucket).toMatchObject({succeeded: 0,failed: 2,+ recovered: 0,other: 1,total: 3,+ // failed + timed_out with no error code both bucket under "unknown"+ failedByErrorCode: { unknown: 2 },});});++ it("separates recovered restart kills from true failures and breaks failures down by error code", async () => {+ const companyId = randomUUID();+ const agentId = randomUUID();+ const day = utcDay(-2);++ await db.insert(companies).values({+ id: companyId,+ name: "Paperclip",+ issuePrefix: `T${companyId.replace(/-/g, "").slice(0, 6).toUpperCase()}`,+ requireBoardApprovalForNewAgents: false,+ });++ await db.insert(agents).values({+ id: agentId,+ companyId,+ name: "CodexCoder",+ role: "engineer",+ status: "running",+ adapterType: "codex_local",+ adapterConfig: {},+ runtimeConfig: {},+ permissions: {},+ });++ const base = {+ companyId,+ agentId,+ invocationSource: "assignment",+ createdAt: day,+ };++ // Direct recovery: a process-loss kill whose retry succeeded.+ const original = randomUUID();+ const retry = randomUUID();+ // Chained recovery: kill -> failed retry -> succeeded retry (both kills recovered).+ const chainedOriginal = randomUUID();+ const chainedRetry = randomUUID();+ const chainedRetrySuccess = randomUUID();+ // A genuine, unrecovered failure that should remain in the failed count.+ const trueFailure = randomUUID();++ await db.insert(heartbeatRuns).values([+ { ...base, id: original, status: "failed", errorCode: "process_lost" },+ { ...base, id: retry, status: "succeeded", retryOfRunId: original },+ { ...base, id: chainedOriginal, status: "failed", errorCode: "process_lost" },+ { ...base, id: chainedRetry, status: "failed", errorCode: "process_lost", retryOfRunId: chainedOriginal },+ { ...base, id: chainedRetrySuccess, status: "succeeded", retryOfRunId: chainedRetry },+ { ...base, id: trueFailure, status: "failed", errorCode: "provider_quota" },+ ]);++ const summary = await dashboardService(db).summary(companyId);+ const bucket = summary.runActivity.find((b) => b.date === utcDateKey(day));++ expect(bucket).toMatchObject({+ succeeded: 2,+ // original + chainedOriginal + chainedRetry all recovered via a later success+ recovered: 3,+ failed: 1,+ other: 0,+ total: 6,+ failedByErrorCode: { provider_quota: 1 },+ });+ // process_lost kills that recovered must not leak into the failed breakdown.+ expect(bucket?.failedByErrorCode.process_lost).toBeUndefined();+ });});diff --git a/server/src/__tests__/heartbeat-process-recovery.test.ts b/server/src/__tests__/heartbeat-process-recovery.test.tsindex 22154cb9b3..d0af367378 100644--- a/server/src/__tests__/heartbeat-process-recovery.test.ts+++ b/server/src/__tests__/heartbeat-process-recovery.test.ts@@ -1293,6 +1293,213 @@ describeEmbeddedPostgres("heartbeat orphaned process recovery", () => {expect(checkoutReleasedIssue?.checkoutRunId).toBeNull();});+ it("interrupts running runs on graceful shutdown and queues restart recovery without recording a failure", async () => {+ const { agentId, runId, issueId, wakeupRequestId } = await seedRunFixture({+ agentStatus: "running",+ contextSnapshot: {+ modelProfile: "cheap",+ allowDeliverableWork: false,+ allowDocumentUpdates: false,+ resumeRequiresNormalModel: true,+ },+ });+ const heartbeat = heartbeatService(db);++ const result = await heartbeat.drainRunningRunsForShutdown(+ "SIGTERM",+ new Date("2026-03-19T00:06:00.000Z"),+ );+ expect(result.interrupted).toBe(1);+ expect(result.interruptedRunIds).toEqual([runId]);++ const runs = await db+ .select()+ .from(heartbeatRuns)+ .where(eq(heartbeatRuns.agentId, agentId));+ expect(runs).toHaveLength(2);++ const interruptedRun = runs.find((row) => row.id === runId);+ const retryRun = runs.find((row) => row.retryOfRunId === runId);+ expect(interruptedRun).toMatchObject({+ status: "interrupted",+ errorCode: "server_shutdown_interrupted",+ signal: "SIGTERM",+ livenessState: "needs_followup",+ });+ expect(interruptedRun?.resultJson).toMatchObject({+ stopReason: "interrupted",+ timeoutConfigured: false,+ timeoutFired: false,+ });+ expect(retryRun).toMatchObject({+ status: "queued",+ retryOfRunId: runId,+ processLossRetryCount: 1,+ });+ expect(retryRun?.contextSnapshot as Record<string, unknown>).toMatchObject({+ retryReason: "process_lost",+ retryOfRunId: runId,+ });+ expect(retryRun?.contextSnapshot as Record<string, unknown>).not.toHaveProperty("modelProfile");++ const wakeup = await db+ .select()+ .from(agentWakeupRequests)+ .where(eq(agentWakeupRequests.id, wakeupRequestId))+ .then((rows) => rows[0] ?? null);+ expect(wakeup?.status).toBe("cancelled");++ const issue = await db+ .select()+ .from(issues)+ .where(eq(issues.id, issueId))+ .then((rows) => rows[0] ?? null);+ expect(issue?.checkoutRunId).toBeNull();+ expect(issue?.executionRunId).toBe(retryRun?.id);+ });++ it("does not overwrite a run that is no longer running during graceful shutdown drain", async () => {+ const { runId, wakeupRequestId } = await seedRunFixture({+ agentStatus: "running",+ });+ const heartbeat = heartbeatService(db);++ await db+ .update(heartbeatRuns)+ .set({+ status: "succeeded",+ finishedAt: new Date("2026-03-19T00:05:30.000Z"),+ updatedAt: new Date("2026-03-19T00:05:30.000Z"),+ })+ .where(eq(heartbeatRuns.id, runId));++ const result = await heartbeat.drainRunningRunsForShutdown(+ "SIGTERM",+ new Date("2026-03-19T00:06:00.000Z"),+ );++ expect(result).toMatchObject({+ interrupted: 0,+ interruptedRunIds: [],+ retryRunIds: [],+ });+ const run = await db+ .select()+ .from(heartbeatRuns)+ .where(eq(heartbeatRuns.id, runId))+ .then((rows) => rows[0] ?? null);+ expect(run).toMatchObject({+ status: "succeeded",+ errorCode: null,+ signal: null,+ });+ const wakeup = await db+ .select()+ .from(agentWakeupRequests)+ .where(eq(agentWakeupRequests.id, wakeupRequestId))+ .then((rows) => rows[0] ?? null);+ expect(wakeup?.status).toBe("claimed");+ });++ it("does not enqueue duplicate restart recovery for the same interrupted run", async () => {+ const { agentId, runId, issueId, wakeupRequestId } = await seedRunFixture({+ agentStatus: "running",+ });+ const heartbeat = heartbeatService(db);++ await heartbeat.drainRunningRunsForShutdown("SIGTERM", new Date("2026-03-19T00:06:00.000Z"));+ const firstRetry = await db+ .select()+ .from(heartbeatRuns)+ .where(and(eq(heartbeatRuns.agentId, agentId), eq(heartbeatRuns.retryOfRunId, runId)))+ .then((rows) => rows[0] ?? null);+ expect(firstRetry?.id).toBeTruthy();++ await db+ .update(heartbeatRuns)+ .set({ status: "running", finishedAt: null, updatedAt: new Date("2026-03-19T00:07:00.000Z") })+ .where(eq(heartbeatRuns.id, runId));+ await db+ .update(agentWakeupRequests)+ .set({ status: "claimed", finishedAt: null, updatedAt: new Date("2026-03-19T00:07:00.000Z") })+ .where(eq(agentWakeupRequests.id, wakeupRequestId));+ await db+ .update(issues)+ .set({ checkoutRunId: runId, executionRunId: runId, updatedAt: new Date("2026-03-19T00:07:00.000Z") })+ .where(eq(issues.id, issueId));++ const secondDrain = await heartbeat.drainRunningRunsForShutdown(+ "SIGTERM",+ new Date("2026-03-19T00:08:00.000Z"),+ );+ expect(secondDrain.retryRunIds).toEqual([firstRetry?.id]);++ const retryRuns = await db+ .select()+ .from(heartbeatRuns)+ .where(and(eq(heartbeatRuns.agentId, agentId), eq(heartbeatRuns.retryOfRunId, runId)));+ expect(retryRuns).toHaveLength(1);+ expect(retryRuns[0]?.id).toBe(firstRetry?.id);+ });++ it("chains a single retry when restart recovery is interrupted by a second graceful shutdown", async () => {+ const { agentId, runId, issueId } = await seedRunFixture({+ agentStatus: "running",+ });+ const heartbeat = heartbeatService(db);++ await heartbeat.drainRunningRunsForShutdown("SIGTERM", new Date("2026-03-19T00:06:00.000Z"));+ const firstRetry = await db+ .select()+ .from(heartbeatRuns)+ .where(and(eq(heartbeatRuns.agentId, agentId), eq(heartbeatRuns.retryOfRunId, runId)))+ .then((rows) => rows[0] ?? null);+ expect(firstRetry?.id).toBeTruthy();++ await db+ .update(heartbeatRuns)+ .set({ status: "running", startedAt: new Date("2026-03-19T00:07:00.000Z"), updatedAt: new Date("2026-03-19T00:07:00.000Z") })+ .where(eq(heartbeatRuns.id, firstRetry!.id));+ await db+ .update(agentWakeupRequests)+ .set({ status: "claimed", claimedAt: new Date("2026-03-19T00:07:00.000Z"), updatedAt: new Date("2026-03-19T00:07:00.000Z") })+ .where(eq(agentWakeupRequests.id, firstRetry!.wakeupRequestId));+ await db+ .update(issues)+ .set({ checkoutRunId: firstRetry!.id, executionRunId: firstRetry!.id, updatedAt: new Date("2026-03-19T00:07:00.000Z") })+ .where(eq(issues.id, issueId));++ const secondDrain = await heartbeat.drainRunningRunsForShutdown(+ "SIGTERM",+ new Date("2026-03-19T00:08:00.000Z"),+ );+ expect(secondDrain.interruptedRunIds).toEqual([firstRetry!.id]);++ const runs = await db+ .select()+ .from(heartbeatRuns)+ .where(eq(heartbeatRuns.agentId, agentId));+ expect(runs).toHaveLength(3);+ expect(runs.find((row) => row.id === runId)?.status).toBe("interrupted");+ expect(runs.find((row) => row.id === firstRetry!.id)?.status).toBe("interrupted");++ const originalRetries = runs.filter((row) => row.retryOfRunId === runId);+ expect(originalRetries).toHaveLength(1);+ const secondRetry = runs.find((row) => row.retryOfRunId === firstRetry!.id);+ expect(secondRetry).toMatchObject({+ status: "queued",+ processLossRetryCount: 2,+ });++ const issue = await db+ .select()+ .from(issues)+ .where(eq(issues.id, issueId))+ .then((rows) => rows[0] ?? null);+ expect(issue?.checkoutRunId).toBeNull();+ expect(issue?.executionRunId).toBe(secondRetry?.id);+ });+it("releases active environment leases when an orphaned run is reaped", async () => {const { runId, issueId, companyId } = await seedRunFixture({processPid: 999_999_999,diff --git a/server/src/__tests__/heartbeat-retry-scheduling.test.ts b/server/src/__tests__/heartbeat-retry-scheduling.test.tsindex 4e59091057..80b05e5ef4 100644--- a/server/src/__tests__/heartbeat-retry-scheduling.test.ts+++ b/server/src/__tests__/heartbeat-retry-scheduling.test.ts@@ -5,8 +5,10 @@ import {agents,agentRuntimeState,agentWakeupRequests,+ activityLog,budgetPolicies,companies,+ companySkills,createDb,environmentLeases,heartbeatRunEvents,@@ -18,6 +20,7 @@ import {getEmbeddedPostgresTestSupport,startEmbeddedPostgresTestDatabase,} from "./helpers/embedded-postgres.js";+import { registerServerAdapter, unregisterServerAdapter } from "../adapters/index.ts";import {BOUNDED_TRANSIENT_HEARTBEAT_RETRY_DELAYS_MS,MAX_TURN_CONTINUATION_RETRY_REASON,@@ -27,6 +30,7 @@ import {const embeddedPostgresSupport = await getEmbeddedPostgresTestSupport();const describeEmbeddedPostgres = embeddedPostgresSupport.supported ? describe : describe.skip;+const PROVIDER_QUOTA_TEST_ADAPTER = "provider_quota_test";if (!embeddedPostgresSupport.supported) {console.warn(@@ -34,6 +38,20 @@ if (!embeddedPostgresSupport.supported) {);}+async function waitForRunToFinish(+ heartbeat: ReturnType<typeof heartbeatService>,+ runId: string,+ timeoutMs = 5_000,+) {+ const deadline = Date.now() + timeoutMs;+ while (Date.now() < deadline) {+ const run = await heartbeat.getRun(runId);+ if (run && !["queued", "running"].includes(run.status)) return run;+ await new Promise((resolve) => setTimeout(resolve, 50));+ }+ return await heartbeat.getRun(runId);+}+describeEmbeddedPostgres("heartbeat bounded retry scheduling", () => {let db!: ReturnType<typeof createDb>;let heartbeat!: ReturnType<typeof heartbeatService>;@@ -43,22 +61,49 @@ describeEmbeddedPostgres("heartbeat bounded retry scheduling", () => {tempDb = await startEmbeddedPostgresTestDatabase("paperclip-heartbeat-retry-scheduling-");db = createDb(tempDb.connectionString);heartbeat = heartbeatService(db);+ registerServerAdapter({+ type: PROVIDER_QUOTA_TEST_ADAPTER,+ execute: async () => ({+ exitCode: 1,+ signal: null,+ timedOut: false,+ errorMessage: "You've hit your session limit - resets at 4pm (America/Chicago).",+ errorCode: "provider_quota",+ errorFamily: "provider_quota",+ retryNotBefore: "2030-04-22T21:00:00.000Z",+ resultJson: {+ errorFamily: "provider_quota",+ retryNotBefore: "2030-04-22T21:00:00.000Z",+ providerQuotaRetryNotBefore: "2030-04-22T21:00:00.000Z",+ },+ }),+ testEnvironment: async () => ({+ adapterType: PROVIDER_QUOTA_TEST_ADAPTER,+ status: "pass",+ checks: [],+ testedAt: new Date().toISOString(),+ }),+ });}, 20_000);afterEach(async () => {+ await db.delete(activityLog);await db.delete(heartbeatRunEvents);await db.delete(environmentLeases);await db.delete(issueRelations);await db.delete(issues);+ await db.delete(activityLog);await db.delete(heartbeatRuns);await db.delete(agentWakeupRequests);await db.delete(agentRuntimeState);await db.delete(budgetPolicies);await db.delete(agents);+ await db.delete(companySkills);await db.delete(companies);});afterAll(async () => {+ unregisterServerAdapter(PROVIDER_QUOTA_TEST_ADAPTER);await tempDb?.cleanup();});@@ -68,11 +113,11 @@ describeEmbeddedPostgres("heartbeat bounded retry scheduling", () => {agentId: string;now: Date;errorCode: string;- errorFamily?: "transient_upstream" | null;+ errorFamily?: "transient_upstream" | "provider_quota" | null;retryNotBefore?: string | null;scheduledRetryAttempt?: number;resultJson?: Record<string, unknown> | null;- adapterType?: "codex_local" | "claude_local";+ adapterType?: string;agentName?: string;}) {const adapterType = input.adapterType ?? "codex_local";@@ -131,6 +176,88 @@ describeEmbeddedPostgres("heartbeat bounded retry scheduling", () => {});}+ it("records provider quota failures, schedules the reset-time retry, and leaves the agent idle", async () => {+ const companyId = randomUUID();+ const agentId = randomUUID();++ await db.insert(companies).values({+ id: companyId,+ name: "Paperclip",+ issuePrefix: `T${companyId.replace(/-/g, "").slice(0, 6).toUpperCase()}`,+ requireBoardApprovalForNewAgents: false,+ defaultResponsibleUserId: "responsible-user",+ });++ await db.insert(agents).values({+ id: agentId,+ companyId,+ name: "Quota Test",+ role: "engineer",+ status: "idle",+ adapterType: PROVIDER_QUOTA_TEST_ADAPTER,+ adapterConfig: {},+ runtimeConfig: {+ heartbeat: {+ wakeOnDemand: true,+ maxConcurrentRuns: 1,+ },+ },+ permissions: {},+ });++ const run = await heartbeat.invoke(agentId, "on_demand", {}, "manual");+ expect(run).not.toBeNull();++ const failedRun = await waitForRunToFinish(heartbeat, run!.id);+ expect(failedRun?.status).toBe("failed");+ expect(failedRun?.errorCode).toBe("provider_quota");+ expect((failedRun?.resultJson as Record<string, unknown> | null)?.errorFamily).toBe("provider_quota");++ await expect+ .poll(+ () =>+ db+ .select({ id: heartbeatRuns.id })+ .from(heartbeatRuns)+ .where(eq(heartbeatRuns.retryOfRunId, run!.id))+ .then((rows) => rows.length),+ { timeout: 5_000, interval: 50 },+ )+ .toBe(1);++ const retryRun = await db+ .select({+ id: heartbeatRuns.id,+ status: heartbeatRuns.status,+ scheduledRetryAt: heartbeatRuns.scheduledRetryAt,+ scheduledRetryReason: heartbeatRuns.scheduledRetryReason,+ contextSnapshot: heartbeatRuns.contextSnapshot,+ })+ .from(heartbeatRuns)+ .where(eq(heartbeatRuns.retryOfRunId, run!.id))+ .then((rows) => rows[0] ?? null);+ expect(retryRun?.status).toBe("scheduled_retry");+ expect(retryRun?.scheduledRetryReason).toBe("transient_failure");+ expect(retryRun?.scheduledRetryAt?.toISOString()).toBe("2030-04-22T21:00:00.000Z");+ expect((retryRun?.contextSnapshot as Record<string, unknown> | null)?.errorFamily).toBe("provider_quota");+ expect((retryRun?.contextSnapshot as Record<string, unknown> | null)?.providerQuotaRetryNotBefore).toBe(+ "2030-04-22T21:00:00.000Z",+ );+ expect((retryRun?.contextSnapshot as Record<string, unknown> | null)?.codexTransientFallbackMode ?? null).toBeNull();++ await expect+ .poll(+ () =>+ db+ .select({ status: agents.status, errorReason: agents.errorReason })+ .from(agents)+ .where(eq(agents.id, agentId))+ .then((rows) => rows[0] ?? null),+ { timeout: 5_000, interval: 50 },+ )+ .toEqual({ status: "idle", errorReason: null });+ });+async function seedMaxTurnFixture(input?: {companyId?: string;agentId?: string;@@ -684,6 +811,7 @@ describeEmbeddedPostgres("heartbeat bounded retry scheduling", () => {await db.delete(agentWakeupRequests);await db.delete(agentRuntimeState);await db.delete(agents);+ await db.delete(companySkills);await db.delete(companies);const dependencyBlocked = await seedMaxTurnFixture({ now: new Date("2026-04-20T17:00:00.000Z") });@@ -1350,6 +1478,7 @@ describeEmbeddedPostgres("heartbeat bounded retry scheduling", () => {await db.delete(heartbeatRuns);await db.delete(agentWakeupRequests);await db.delete(agents);+ await db.delete(companySkills);await db.delete(companies);}});diff --git a/server/src/__tests__/heartbeat-workspace-branch-containment.test.ts b/server/src/__tests__/heartbeat-workspace-branch-containment.test.tsindex c49152d254..1e23f826da 100644--- a/server/src/__tests__/heartbeat-workspace-branch-containment.test.ts+++ b/server/src/__tests__/heartbeat-workspace-branch-containment.test.ts@@ -139,10 +139,17 @@ async function createForwardBranchMismatch(input: {worktreePath: string;expectedBranch: string;actualBranch: string;+ divergeRecordedBranch?: boolean;}) {await mkdir(path.dirname(input.worktreePath), { recursive: true });await runGit(input.repoRoot, ["branch", input.expectedBranch]);await runGit(input.repoRoot, ["worktree", "add", "-b", input.actualBranch, input.worktreePath, input.expectedBranch]);+ if (input.divergeRecordedBranch) {+ await runGit(input.repoRoot, ["checkout", input.expectedBranch]);+ await writeFile(path.join(input.repoRoot, "recorded-branch.txt"), "recorded branch work\n", "utf8");+ await runGit(input.repoRoot, ["add", "recorded-branch.txt"]);+ await runGit(input.repoRoot, ["commit", "-m", "Add recorded branch work"]);+ }await writeFile(path.join(input.worktreePath, "actual-branch.txt"), "actual branch work\n", "utf8");await runGit(input.worktreePath, ["add", "actual-branch.txt"]);await runGit(input.worktreePath, ["commit", "-m", "Add actual branch work"]);@@ -362,6 +369,7 @@ async function seedBranchContainmentRun(worktreePath,expectedBranch,actualBranch,+ divergeRecordedBranch: opts.enableWorkspaceBranchReconcileForward !== true,});}@@ -594,12 +602,12 @@ async function expectContainedWorkspaceBranchFailure(input: {expectedBranchExists: true,actualBranchExists: true,sameHead: false,- ancestryVerdict: "ancestor",+ ancestryVerdict: "diverged",});expect(provenance.expectedHeadSha).toEqual(expect.stringMatching(/^[a-f0-9]{40}$/));expect(provenance.actualHeadSha).toEqual(expect.stringMatching(/^[a-f0-9]{40}$/));expect(provenance.expectedHeadSha).not.toBe(provenance.actualHeadSha);- expect(provenance.plainLanguageReason).toEqual(expect.stringContaining("forward of the recorded branch"));+ expect(provenance.plainLanguageReason).toEqual(expect.stringContaining("cannot prove a forward-only reconciliation"));const { issueRows, actionRows, comments } = await waitForContainmentSideEffects({db: input.db,@@ -647,7 +655,7 @@ async function expectContainedWorkspaceBranchFailure(input: {provenance: expect.objectContaining({expectedHeadSha: provenance.expectedHeadSha,actualHeadSha: provenance.actualHeadSha,- ancestryVerdict: "ancestor",+ ancestryVerdict: "diverged",plainLanguageReason: provenance.plainLanguageReason,}),}),@@ -710,9 +718,10 @@ async function expectForwardBranchReconciled(input: {}).from(executionWorkspaces).where(eq(executionWorkspaces.id, activeWorkspaceId));+ const expectedDurableBranch = input.expectsExistingRecordUpdate ? input.actualBranch : input.expectedBranch;expect(activeWorkspace).toMatchObject({- name: input.actualBranch,- branchName: input.actualBranch,+ name: expectedDurableBranch,+ branchName: expectedDurableBranch,providerRef: input.worktreePath,});@@ -738,19 +747,21 @@ async function expectForwardBranchReconciled(input: {.select().from(workspaceOperations).where(eq(workspaceOperations.heartbeatRunId, input.runId));- expect(operations).toEqual(- expect.arrayContaining([- expect.objectContaining({- status: "succeeded",- metadata: expect.objectContaining({- branchIncoherenceReconcileForward: true,- expectedBranchName: input.expectedBranch,- actualBranchName: input.actualBranch,- fingerprint: expect.stringMatching(/^workspace_incoherence:v1:sha256:/),+ if (input.expectsExistingRecordUpdate) {+ expect(operations).toEqual(+ expect.arrayContaining([+ expect.objectContaining({+ status: "succeeded",+ metadata: expect.objectContaining({+ branchIncoherenceReconcileForward: true,+ expectedBranchName: input.expectedBranch,+ actualBranchName: input.actualBranch,+ fingerprint: expect.stringMatching(/^workspace_incoherence:v1:sha256:/),+ }),}),- }),- ]),- );+ ]),+ );+ }if (input.expectsExistingRecordUpdate) {const [updatedWorkspace] = await input.db@@ -881,7 +892,6 @@ describeEmbeddedPostgres("heartbeat workspace branch containment", () => {it.each([["workspace-runtime fresh worktree reuse", "fresh_realize" as const, null],["workspace-runtime persisted restore", "persisted_restore" as const, "source-workspace"],- ["heartbeat finalization", "finalize" as const, "runtime-workspace"],])("contains mid-change branch divergence at %s", async (_name, callSite, expectedWorkspaceId) => {const repoRoot = await createGitRepo();tempRoots.push(repoRoot);@@ -939,9 +949,9 @@ describeEmbeddedPostgres("heartbeat workspace branch containment", () => {}, 30_000);it.each([- ["workspace-runtime fresh worktree reuse", "fresh_realize" as const, true],- ["workspace-runtime persisted restore", "persisted_restore" as const, true],- ["heartbeat finalization", "finalize" as const, true],+ ["workspace-runtime fresh worktree reuse", "fresh_realize" as const, false],+ ["workspace-runtime persisted restore", "persisted_restore" as const, false],+ ["heartbeat finalization", "finalize" as const, false],])("auto-reconciles forward branch divergence at %s when the flag is enabled", async (_name, callSite, expectsExistingRecordUpdate) => {const repoRoot = await createGitRepo();tempRoots.push(repoRoot);@@ -954,7 +964,7 @@ describeEmbeddedPostgres("heartbeat workspace branch containment", () => {status: callSite === "finalize" ? "" : await readGit(seeded.worktreePath, ["status", "--porcelain", "--untracked-files=all"]),};let expectedResolvedRecoveryActionFingerprint: string | null = null;- if (callSite === "fresh_realize") {+ if (callSite === "fresh_realize" && expectsExistingRecordUpdate) {const expectedHeadSha = await readGit(seeded.worktreePath, ["rev-parse", seeded.expectedBranch]);const actualHeadSha = await readGit(seeded.worktreePath, ["rev-parse", seeded.actualBranch]);expectedResolvedRecoveryActionFingerprint = fingerprintWorkspaceBranchIncoherenceForTest({diff --git a/server/src/__tests__/heartbeat-workspace-finalize-branch.test.ts b/server/src/__tests__/heartbeat-workspace-finalize-branch.test.tsindex fb30b06cfd..b4428786a5 100644--- a/server/src/__tests__/heartbeat-workspace-finalize-branch.test.ts+++ b/server/src/__tests__/heartbeat-workspace-finalize-branch.test.ts@@ -387,7 +387,7 @@ describeEmbeddedPostgres("heartbeat workspace finalization branch guard", () =>});}, 20_000);- it("fails unrecorded branch drift when the checked-out branch has different commits", async () => {+ it("adopts unrecorded forward branch drift for finalization without persisting it", async () => {const repoRoot = await createGitRepo();tempRoots.push(repoRoot);const { agentId, issueId } = await seedRunTarget(db, repoRoot);@@ -420,51 +420,40 @@ describeEmbeddedPostgres("heartbeat workspace finalization branch guard", () =>const finishedRun = await waitForRunToFinish(heartbeat, run!.id);expect(finishedRun).toMatchObject({- status: "failed",- errorCode: "workspace_validation_failed",- });- const workspaceValidation = (finishedRun?.resultJson as Record<string, unknown> | null)?.workspaceValidation;- expect(workspaceValidation).toMatchObject({- reason: "git_worktree_branch_incoherence",- sourceIssueId: issueId,- executionWorkspaceId,- expectedBranch: recordedBranch,- actualBranch: publishBranch,- cleanliness: "clean",- provenance: expect.objectContaining({- expectedBranchExists: true,- actualBranchExists: true,- sameHead: false,- }),- safeRepair: expect.objectContaining({- eligible: false,- attempted: false,- succeeded: false,- reason: "expected branch and current HEAD differ",- }),+ status: "succeeded",+ errorCode: null,+ error: null,});await waitForRuntimeStateLastRun(db, agentId, run!.id);expect(adapterExecute).toHaveBeenCalledTimes(1);+ const finalizedWorkspace = await db+ .select({ branchName: executionWorkspaces.branchName })+ .from(executionWorkspaces)+ .where(eq(executionWorkspaces.id, executionWorkspaceId!))+ .then((rows) => rows[0] ?? null);+ expect(finalizedWorkspace?.branchName).toBe(recordedBranch);+const finalizeOps = await listFinalizeOperations(db, run!.id);expect(finalizeOps).toHaveLength(1);expect(finalizeOps[0]).toMatchObject({- status: "failed",+ status: "succeeded",executionWorkspaceId,- stderrExcerpt: expect.stringContaining("Managed git worktree branch check failed"),});expect(finalizeOps[0]?.metadata).toMatchObject({managedGitWorktreeBranch: expect.objectContaining({executionWorkspaceId,- valid: false,- reasonCode: "branch_mismatch",- expectedBranchName: recordedBranch,+ valid: true,+ reasonCode: null,+ expectedBranchName: publishBranch,actualBranchName: publishBranch,}),- workspaceValidation: expect.objectContaining({- reason: "git_worktree_branch_incoherence",+ managedGitWorktreeBranchRepair: expect.objectContaining({+ attempted: true,+ succeeded: true,}),});+ expect(recordedBranch).not.toBe(publishBranch);}, 20_000);it("allows a successful adapter run when the branch transition is recorded before finalization", async () => {diff --git a/server/src/__tests__/heartbeat-workspace-session.test.ts b/server/src/__tests__/heartbeat-workspace-session.test.tsindex 3b505cb12d..cde79bf787 100644--- a/server/src/__tests__/heartbeat-workspace-session.test.ts+++ b/server/src/__tests__/heartbeat-workspace-session.test.ts@@ -1334,7 +1334,7 @@ describe("effective run execution workspace config freshness", () => {});});- it("fails explicit reuse restore errors without realizing a fallback workspace", async () => {+ it("fails loudly when explicit reuse restore errors", async () => {const base = buildWorkspaceConfigMetadata();const next = buildWorkspaceConfigMetadata({repoRef: "origin/release",@@ -1350,7 +1350,7 @@ describe("effective run execution workspace config freshness", () => {existingWorkspaceMetadata: persistedWorkspaceConfigFingerprint(base),nextMetadata: next,});- const realizeWorkspace = vi.fn(async () => ({ id: "fallback-workspace" }));+ const realizeWorkspace = vi.fn(async () => ({ id: "fallback-workspace", warnings: [] }));await expect(provisionExecutionWorkspaceForFreshnessDecision({requestedShouldReuseExisting: true,@@ -1362,28 +1362,14 @@ describe("effective run execution workspace config freshness", () => {throw new Error("restore command failed");},realizeWorkspace,- })).rejects.toMatchObject({- code: "workspace_validation_failed",- resultJson: {- workspaceValidation: expect.objectContaining({- reason: "inherited_workspace_reuse_failed",- issueId: "issue-1",- issueIdentifier: "PAP-42",- executionWorkspaceId: "workspace-old",- workspaceConfigFreshnessAction: "replace",- requestedReuseExisting: true,- replacementWorkspaceRealized: false,- remediation: expect.stringContaining("restore/provision logs"),- }),- },- });+ })).rejects.toThrow(/restore command failed/);expect(realizeWorkspace).not.toHaveBeenCalled();});it.each([{ name: "missing", status: null },{ name: "archived", status: "archived" },- ])("fails explicit reuse when the inherited workspace row is $name", async ({ status }) => {+ ])("fails loudly when the inherited workspace row is $name", async ({ status }) => {const reuseRequest = resolveExecutionWorkspaceReuseRequestForIssue({issueExecutionWorkspaceId: "workspace-old",issueExecutionWorkspacePreference: "reuse_existing",@@ -1403,7 +1389,7 @@ describe("effective run execution workspace config freshness", () => {existingWorkspaceMetadata: null,nextMetadata: metadata,});- const realizeWorkspace = vi.fn(async () => ({ id: "fallback-workspace" }));+ const realizeWorkspace = vi.fn(async () => ({ id: "fallback-workspace", warnings: [] }));await expect(provisionExecutionWorkspaceForFreshnessDecision({requestedShouldReuseExisting: reuseRequest.requestedShouldReuseExisting,@@ -1412,35 +1398,21 @@ describe("effective run execution workspace config freshness", () => {runId: "run-1",workspaceConfigFreshness: decision,restoreExistingWorkspace: reuseRequest.existingExecutionWorkspaceAvailable- ? async () => ({ id: "workspace-old" })+ ? async () => ({ id: "workspace-old", warnings: [] }): null,realizeWorkspace,- })).rejects.toMatchObject({- code: "workspace_validation_failed",- resultJson: {- workspaceValidation: expect.objectContaining({- reason: "inherited_workspace_reuse_unavailable",- issueId: "issue-1",- issueIdentifier: "PAP-42",- executionWorkspaceId: "workspace-old",- workspaceConfigFreshnessAction: "create",- requestedReuseExisting: true,- replacementWorkspaceRealized: false,- remediation: expect.stringContaining("clear the issue's reuse_existing workspace binding"),- }),- },- });+ })).rejects.toThrow(/could not be restored/);expect(realizeWorkspace).not.toHaveBeenCalled();});- it("fails explicit reuse restore misses without realizing a fallback workspace", async () => {+ it("fails loudly when explicit reuse restore returns no workspace", async () => {const metadata = buildWorkspaceConfigMetadata();const decision = resolveExecutionWorkspaceConfigFreshness({hasExistingWorkspace: true,existingWorkspaceMetadata: persistedWorkspaceConfigFingerprint(metadata),nextMetadata: metadata,});- const realizeWorkspace = vi.fn(async () => ({ id: "fallback-workspace" }));+ const realizeWorkspace = vi.fn(async () => ({ id: "fallback-workspace", warnings: [] }));await expect(provisionExecutionWorkspaceForFreshnessDecision({requestedShouldReuseExisting: true,@@ -1450,18 +1422,7 @@ describe("effective run execution workspace config freshness", () => {workspaceConfigFreshness: decision,restoreExistingWorkspace: async () => null,realizeWorkspace,- })).rejects.toMatchObject({- code: "workspace_validation_failed",- resultJson: {- workspaceValidation: expect.objectContaining({- reason: "inherited_workspace_reuse_unavailable",- workspaceConfigFreshnessAction: "reuse",- requestedReuseExisting: true,- replacementWorkspaceRealized: false,- remediation: expect.stringContaining("clear the issue's reuse_existing workspace binding"),- }),- },- });+ })).rejects.toThrow(/could not be restored/);expect(realizeWorkspace).not.toHaveBeenCalled();});diff --git a/server/src/__tests__/workspace-runtime.test.ts b/server/src/__tests__/workspace-runtime.test.tsindex bccd6c1e36..61c612bfdc 100644--- a/server/src/__tests__/workspace-runtime.test.ts+++ b/server/src/__tests__/workspace-runtime.test.ts@@ -2337,6 +2337,60 @@ describe("realizeExecutionWorkspace", () => {);}, 15_000);+ it("reattaches a clean forward detached HEAD to the recorded persisted git worktree branch", async () => {+ const repoRoot = await createTempRepo();+ const branchName = "PAP-454-reattach-detached-head";+ const worktreePath = path.join(repoRoot, ".paperclip", "worktrees", branchName);+ await fs.mkdir(path.dirname(worktreePath), { recursive: true });+ await runGit(repoRoot, ["branch", branchName]);+ await runGit(repoRoot, ["worktree", "add", worktreePath, branchName]);+ await runGit(worktreePath, ["checkout", "--detach"]);+ await fs.writeFile(path.join(worktreePath, "detached.txt"), "detached work\n", "utf8");+ await runGit(worktreePath, ["add", "detached.txt"]);+ await runGit(worktreePath, ["commit", "-m", "Add detached work"]);+ const detachedHead = await readGit(worktreePath, ["rev-parse", "HEAD"]);++ const restored = await ensurePersistedExecutionWorkspaceAvailable({+ base: {+ baseCwd: repoRoot,+ source: "project_primary",+ projectId: "project-1",+ workspaceId: "workspace-1",+ repoUrl: null,+ repoRef: "HEAD",+ },+ workspace: {+ id: "execution-workspace-detached",+ mode: "isolated_workspace",+ strategyType: "git_worktree",+ cwd: worktreePath,+ providerRef: worktreePath,+ projectId: "project-1",+ projectWorkspaceId: "workspace-1",+ repoUrl: null,+ baseRef: "HEAD",+ branchName,+ },+ issue: {+ id: "issue-detached",+ identifier: "PAP-454",+ title: "Repair detached branch mismatch",+ },+ agent: {+ id: "agent-1",+ name: "Codex Coder",+ companyId: "company-1",+ },+ });++ expect(restored?.branchName).toBe(branchName);+ expect(restored?.warnings).toEqual(expect.arrayContaining([+ expect.stringContaining("moved the recorded branch to that HEAD"),+ ]));+ await expect(readGit(worktreePath, ["branch", "--show-current"])).resolves.toBe(branchName);+ await expect(readGit(worktreePath, ["rev-parse", "HEAD"])).resolves.toBe(detachedHead);+ }, 15_000);+it("rejects dirty persisted git worktree branch incoherence with bounded recovery evidence", async () => {const repoRoot = await createTempRepo();const expectedBranch = "PAP-455-reject-dirty-branch-mismatch";@@ -2460,7 +2514,7 @@ describe("realizeExecutionWorkspace", () => {});}, 15_000);- it("rejects an existing persisted git worktree when the checked-out branch changed to a different commit", async () => {+ it("adopts an existing persisted git worktree when the checked-out branch is forward of the recorded branch", async () => {const repoRoot = await createTempRepo();const initial = await realizeExecutionWorkspace({@@ -2497,85 +2551,44 @@ describe("realizeExecutionWorkspace", () => {await runGit(initial.cwd, ["commit", "-m", "Add publish branch work"]);if (!initial.branchName) throw new Error("expected realized worktree branch name");- const expectedHeadSha = await readGit(repoRoot, ["rev-parse", `refs/heads/${initial.branchName}^{commit}`]);- const actualHeadSha = await readGit(initial.cwd, ["rev-parse", "HEAD"]);- const expectedFingerprint = workspaceBranchIncoherenceFingerprintForTest({- sourceIssueId: "issue-3",- executionWorkspaceId: "execution-workspace-3",- worktreePath: initial.cwd,- expectedBranch: initial.branchName,- actualBranch,- cleanliness: "clean",- expectedHeadSha,- actualHeadSha,- });-- let error: unknown = null;- try {- await ensurePersistedExecutionWorkspaceAvailable({- base: {- baseCwd: repoRoot,- source: "project_primary",- projectId: "project-1",- workspaceId: "workspace-1",- repoUrl: null,- repoRef: "HEAD",- },- workspace: {- id: "execution-workspace-3",- mode: "isolated_workspace",- strategyType: "git_worktree",- cwd: initial.cwd,- providerRef: initial.worktreePath,- projectId: "project-1",- projectWorkspaceId: "workspace-1",- repoUrl: null,- baseRef: "HEAD",- branchName: initial.branchName,- },- issue: {- id: "issue-3",- identifier: "PAP-456",- title: "Keep persisted branch coherent",- },- agent: {- id: "agent-1",- name: "Codex Coder",- companyId: "company-1",- },- });- } catch (err) {- error = err;- }-- expect(error).toMatchObject({- code: "workspace_validation_failed",- resultJson: {- workspaceValidation: expect.objectContaining({- reason: "git_worktree_branch_incoherence",- fingerprint: expectedFingerprint,- sourceIssueId: "issue-3",- sourceIdentifier: "PAP-456",- executionWorkspaceId: "execution-workspace-3",- expectedBranch: initial.branchName,- actualBranch,- cleanliness: "clean",- provenance: expect.objectContaining({- expectedBranchExists: true,- actualBranchExists: true,- sameHead: false,- ancestryVerdict: "ancestor",- plainLanguageReason: expect.stringContaining("forward of the recorded branch"),- }),- safeRepair: expect.objectContaining({- eligible: false,- attempted: false,- succeeded: false,- reason: "expected branch and current HEAD differ",- }),- }),+ const restored = await ensurePersistedExecutionWorkspaceAvailable({+ base: {+ baseCwd: repoRoot,+ source: "project_primary",+ projectId: "project-1",+ workspaceId: "workspace-1",+ repoUrl: null,+ repoRef: "HEAD",+ },+ workspace: {+ id: "execution-workspace-3",+ mode: "isolated_workspace",+ strategyType: "git_worktree",+ cwd: initial.cwd,+ providerRef: initial.worktreePath,+ projectId: "project-1",+ projectWorkspaceId: "workspace-1",+ repoUrl: null,+ baseRef: "HEAD",+ branchName: initial.branchName,+ },+ issue: {+ id: "issue-3",+ identifier: "PAP-456",+ title: "Keep persisted branch coherent",+ },+ agent: {+ id: "agent-1",+ name: "Codex Coder",+ companyId: "company-1",},});++ expect(restored?.branchName).toBe(actualBranch);+ expect(restored?.warnings).toEqual(expect.arrayContaining([+ expect.stringContaining("adopted it for subsequent runs"),+ ]));+ await expect(readGit(initial.cwd, ["branch", "--show-current"])).resolves.toBe(actualBranch);}, 15_000);it("classifies persisted git worktree branch incoherence as diverged when the checked-out branch is not forward", async () => {diff --git a/server/src/index.ts b/server/src/index.tsindex f4dab7a60e..b1fe7e6f48 100644--- a/server/src/index.ts+++ b/server/src/index.ts@@ -801,8 +801,28 @@ export async function startServer(): Promise<StartedServer> {throw err;}+ let drainHeartbeatRunsForShutdown: ((signal: "SIGINT" | "SIGTERM") => Promise<unknown>) | null = null;+ let heartbeatSchedulerStopped = false;+ let heartbeatSchedulerInterval: ReturnType<typeof setInterval> | null = null;+ const heartbeatSchedulerInFlight = new Set<Promise<void>>();+ const trackHeartbeatSchedulerWork = (work: Promise<unknown>) => {+ let tracked: Promise<void>;+ tracked = Promise.resolve(work)+ .then(() => undefined, () => undefined)+ .finally(() => {+ heartbeatSchedulerInFlight.delete(tracked);+ });+ heartbeatSchedulerInFlight.add(tracked);+ };+ const waitForHeartbeatSchedulerIdle = async () => {+ while (heartbeatSchedulerInFlight.size > 0) {+ await Promise.allSettled([...heartbeatSchedulerInFlight]);+ }+ };+if (config.heartbeatSchedulerEnabled) {const heartbeat = heartbeatService(db as any, { pluginWorkerManager });+ drainHeartbeatRunsForShutdown = heartbeat.drainRunningRunsForShutdown;const environmentCustomImages = environmentCustomImageService(db as any, { pluginWorkerManager });const routines = routineService(db as any, { pluginWorkerManager });const heartbeatSchedulingSuppression = resolveHeartbeatSchedulingSuppression();@@ -815,7 +835,7 @@ export async function startServer(): Promise<StartedServer> {"heartbeat scheduling suppressed for this runtime instance",);} else {- await (async () => {+ const startupHeartbeatRecovery = (async () => {for (let attempt = 1; attempt <= 2; attempt++) {try {const result = await heartbeat.reapOrphanedRuns();@@ -886,6 +906,8 @@ export async function startServer(): Promise<StartedServer> {})().catch((err) => {logger.error({ err }, "startup heartbeat recovery failed");});+ trackHeartbeatSchedulerWork(startupHeartbeatRecovery);+ await startupHeartbeatRecovery;}const setupCleanup = await environmentCustomImages.cleanupExpiredSetupSessions();@@ -893,7 +915,8 @@ export async function startServer(): Promise<StartedServer> {logger.warn({ ...setupCleanup }, "startup environment customImage setup cleanup changed sessions");}- setInterval(() => {+ heartbeatSchedulerInterval = setInterval(() => {+ if (heartbeatSchedulerStopped) return;const sweptRuntimeStatuses = heartbeat.sweepExpiredRuntimeStatuses();if (sweptRuntimeStatuses > 0) {logger.info(@@ -903,7 +926,7 @@ export async function startServer(): Promise<StartedServer> {}if (!resolveHeartbeatSchedulingSuppression().suppressed) {- void heartbeat+ trackHeartbeatSchedulerWork(heartbeat.tickTimers(new Date()).then((result) => {if (result.enqueued > 0) {@@ -912,10 +935,11 @@ export async function startServer(): Promise<StartedServer> {}).catch((err) => {logger.error({ err }, "heartbeat timer tick failed");- });+ }));}- void routines+ if (heartbeatSchedulerStopped) return;+ trackHeartbeatSchedulerWork(routines.tickScheduledTriggers(new Date()).then((result) => {if (result.triggered > 0) {@@ -924,9 +948,9 @@ export async function startServer(): Promise<StartedServer> {}).catch((err) => {logger.error({ err }, "routine scheduler tick failed");- });+ }));- void environmentCustomImages+ trackHeartbeatSchedulerWork(environmentCustomImages.cleanupExpiredSetupSessions().then((result) => {if (result.timedOut > 0 || result.failed > 0) {@@ -935,12 +959,13 @@ export async function startServer(): Promise<StartedServer> {}).catch((err) => {logger.error({ err }, "environment customImage setup cleanup failed");- });-+ }));++ if (heartbeatSchedulerStopped) return;if (!resolveHeartbeatSchedulingSuppression().suppressed) {// Periodically reap orphaned runs (5-min staleness threshold) and make sure// persisted queued work is still being driven forward.- void heartbeat+ trackHeartbeatSchedulerWork(heartbeat.reapOrphanedRuns({ staleThresholdMs: 5 * 60 * 1000 }).then(() => heartbeat.promoteDueScheduledRetries()).then(async (promotion) => {@@ -992,7 +1017,7 @@ export async function startServer(): Promise<StartedServer> {}).catch((err) => {logger.error({ err }, "periodic heartbeat recovery failed");- });+ }));}}, config.heartbeatSchedulerIntervalMs);}@@ -1096,12 +1121,28 @@ export async function startServer(): Promise<StartedServer> {{const shutdown = async (signal: "SIGINT" | "SIGTERM") => {+ heartbeatSchedulerStopped = true;+ if (heartbeatSchedulerInterval) {+ clearInterval(heartbeatSchedulerInterval);+ heartbeatSchedulerInterval = null;+ }+ await waitForHeartbeatSchedulerIdle();+const telemetryClient = getTelemetryClient();if (telemetryClient) {telemetryClient.stop();await telemetryClient.flush();}+ if (drainHeartbeatRunsForShutdown) {+ try {+ const drain = await drainHeartbeatRunsForShutdown(signal);+ logger.info({ signal, drain }, "graceful heartbeat run drain complete");+ } catch (err) {+ logger.error({ err, signal }, "graceful heartbeat run drain failed");+ }+ }+const appShutdown = (app as { locals?: { paperclipShutdown?: () => void } }).locals?.paperclipShutdown;appShutdown?.();diff --git a/server/src/services/dashboard.ts b/server/src/services/dashboard.tsindex 5d60ced564..1493a3f59e 100644--- a/server/src/services/dashboard.ts+++ b/server/src/services/dashboard.ts@@ -96,35 +96,79 @@ export function dashboardService(db: Db) {);const monthSpendCents = Number(monthSpend);- const runActivityDayExpr = sql<string>`to_char(${heartbeatRuns.createdAt} at time zone 'UTC', 'YYYY-MM-DD')`;- const runActivityRows = await db- .select({- date: runActivityDayExpr,- status: heartbeatRuns.status,- count: sql<number>`count(*)::double precision`,- })- .from(heartbeatRuns)- .where(- and(- eq(heartbeatRuns.companyId, companyId),- gte(heartbeatRuns.createdAt, runActivityStart),- ),+ // Per-day run breakdown. A run is "recovered" when its retry chain later+ // succeeded (recovered_runs = all ancestors of a succeeded retry), so a+ // restart-killed run whose retry succeeded is pulled out of the headline+ // failed count. error_code is carried through so a failure spike can be+ // attributed to an error class (e.g. process_lost, provider_quota).+ const runActivityRows = (await db.execute(sql`+ WITH RECURSIVE recovered_runs(id) AS (+ SELECT parent.id+ FROM ${heartbeatRuns} AS child+ JOIN ${heartbeatRuns} AS parent ON parent.id = child.retry_of_run_id+ WHERE child.company_id = ${companyId}+ AND child.status = 'succeeded'+ UNION+ SELECT parent.id+ FROM recovered_runs rr+ JOIN ${heartbeatRuns} AS child ON child.id = rr.id+ JOIN ${heartbeatRuns} AS parent ON parent.id = child.retry_of_run_id)- .groupBy(runActivityDayExpr, heartbeatRuns.status);+ SELECT+ to_char(run.created_at AT TIME ZONE 'UTC', 'YYYY-MM-DD') AS date,+ run.status AS status,+ run.error_code AS error_code,+ (run.id IN (SELECT id FROM recovered_runs)) AS recovered,+ count(*)::double precision AS count+ FROM ${heartbeatRuns} AS run+ WHERE run.company_id = ${companyId}+ AND run.created_at >= ${runActivityStart.toISOString()}::timestamptz+ GROUP BY date, run.status, run.error_code, recovered+ `)) as unknown as Iterable<{+ date: string;+ status: string;+ error_code: string | null;+ recovered: boolean | string;+ count: number | string;+ }>;const runActivity = new Map(runActivityDays.map((date) => [date,- { date, succeeded: 0, failed: 0, other: 0, total: 0 },+ {+ date,+ succeeded: 0,+ failed: 0,+ recovered: 0,+ other: 0,+ total: 0,+ failedByErrorCode: {} as Record<string, number>,+ },]),);for (const row of runActivityRows) {- const bucket = runActivity.get(row.date);+ const bucket = runActivity.get(String(row.date));if (!bucket) continue;const count = Number(row.count);- if (row.status === "succeeded") bucket.succeeded += count;- else if (row.status === "failed" || row.status === "timed_out") bucket.failed += count;- else bucket.other += count;+ const status = String(row.status);+ // Postgres booleans can arrive as JS boolean or "t"/"true" depending on driver.+ const recovered = row.recovered === true || row.recovered === "t" || row.recovered === "true";+ if (status === "succeeded") {+ bucket.succeeded += count;+ } else if (status === "failed" || status === "timed_out") {+ if (recovered) {+ bucket.recovered += count;+ } else {+ bucket.failed += count;+ const code =+ typeof row.error_code === "string" && row.error_code.length > 0+ ? row.error_code+ : "unknown";+ bucket.failedByErrorCode[code] = (bucket.failedByErrorCode[code] ?? 0) + count;+ }+ } else {+ bucket.other += count;+ }bucket.total += count;}diff --git a/server/src/services/heartbeat-stop-metadata.test.ts b/server/src/services/heartbeat-stop-metadata.test.tsindex fc6d54c82e..92624ea202 100644--- a/server/src/services/heartbeat-stop-metadata.test.ts+++ b/server/src/services/heartbeat-stop-metadata.test.ts@@ -64,6 +64,18 @@ describe("heartbeat stop metadata", () => {).toBe("cancelled");});+ it("records graceful interruption separately from failure", () => {+ expect(+ buildHeartbeatRunStopMetadata({+ adapterType: "codex_local",+ adapterConfig: {},+ outcome: "interrupted",+ errorCode: "server_shutdown_interrupted",+ errorMessage: "Interrupted by graceful server shutdown",+ }).stopReason,+ ).toBe("interrupted");+ });+it("normalizes max-turn exhaustion stop reasons", () => {expect(buildHeartbeatRunStopMetadata({diff --git a/server/src/services/heartbeat-stop-metadata.ts b/server/src/services/heartbeat-stop-metadata.tsindex de80a32bba..268a3722fc 100644--- a/server/src/services/heartbeat-stop-metadata.ts+++ b/server/src/services/heartbeat-stop-metadata.ts@@ -1,7 +1,8 @@-export type HeartbeatRunOutcome = "succeeded" | "failed" | "cancelled" | "timed_out";+export type HeartbeatRunOutcome = "succeeded" | "interrupted" | "failed" | "cancelled" | "timed_out";export type HeartbeatRunStopReason =| "completed"+ | "interrupted"| "timeout"| "cancelled"| "budget_paused"@@ -83,6 +84,7 @@ export function inferHeartbeatRunStopReason(input: {errorMessage?: string | null;}): HeartbeatRunStopReason {if (input.outcome === "succeeded") return "completed";+ if (input.outcome === "interrupted") return "interrupted";const maxTurnStopReason = normalizeMaxTurnStopReason(input.errorCode);if (maxTurnStopReason) return maxTurnStopReason;if (input.outcome === "timed_out") return "timeout";diff --git a/server/src/services/heartbeat.ts b/server/src/services/heartbeat.tsindex b23e562cbb..759f3a7b64 100644--- a/server/src/services/heartbeat.ts+++ b/server/src/services/heartbeat.ts@@ -105,7 +105,6 @@ import {inspectManagedGitWorktreeBranch,persistAdapterManagedRuntimeServices,realizeExecutionWorkspace,- reconcilePendingForwardBranchAfterPersistence,releaseRuntimeServicesForRun,type ExecutionWorkspaceInput,type RealizedExecutionWorkspace,@@ -269,7 +268,7 @@ const MAX_INLINE_WAKE_COMMENT_BODY_TOTAL_CHARS = 12_000;const execFile = promisify(execFileCallback);const EXECUTION_PATH_HEARTBEAT_RUN_STATUSES = ["queued", "running", "scheduled_retry"] as const;const CANCELLABLE_HEARTBEAT_RUN_STATUSES = ["queued", "running", "scheduled_retry"] as const;-const HEARTBEAT_RUN_TERMINAL_STATUSES = ["succeeded", "failed", "cancelled", "timed_out"] as const;+const HEARTBEAT_RUN_TERMINAL_STATUSES = ["succeeded", "interrupted", "failed", "cancelled", "timed_out"] as const;const UNSUCCESSFUL_HEARTBEAT_RUN_TERMINAL_STATUSES = ["failed", "cancelled", "timed_out"] as const;const TIMER_ACTIONABLE_ISSUE_STATUSES = ["todo", "in_progress"] as const;export {@@ -369,6 +368,9 @@ function readHeartbeatRunErrorFamily(const persistedFamily = readNonEmptyString(resultJson.errorFamily);if (persistedFamily) return persistedFamily;+ if (run.errorCode === "provider_quota") {+ return "provider_quota";+ }if (run.errorCode === "codex_transient_upstream" || run.errorCode === "claude_transient_upstream") {return "transient_upstream";}@@ -398,9 +400,10 @@ function readTransientRetryNotBeforeFromRun(run: Pick<typeof heartbeatRuns.$infefunction readTransientRecoveryContractFromRun(run: Pick<typeof heartbeatRuns.$inferSelect, "errorCode" | "resultJson">,) {- return readHeartbeatRunErrorFamily(run) === "transient_upstream"+ const errorFamily = readHeartbeatRunErrorFamily(run);+ return errorFamily === "transient_upstream" || errorFamily === "provider_quota"? {- errorFamily: "transient_upstream" as const,+ errorFamily,retryNotBefore: readTransientRetryNotBeforeFromRun(run),}: null;@@ -422,6 +425,7 @@ function mergeAdapterRecoveryMetadata(input: {? {retryNotBefore,transientRetryNotBefore: retryNotBefore,+ ...(errorFamily === "provider_quota" ? { providerQuotaRetryNotBefore: retryNotBefore } : {}),}: {}),};@@ -1631,8 +1635,8 @@ export async function assertGitSensitiveAdapterWorkspaceValid(input: {}const expectedManagedBranchName =- readNonEmptyString(input.persistedExecutionWorkspace?.branchName) ??- readNonEmptyString(input.executionWorkspace.branchName);+ readNonEmptyString(input.executionWorkspace.branchName) ??+ readNonEmptyString(input.persistedExecutionWorkspace?.branchName);if (input.persistedExecutionWorkspace?.strategyType === "git_worktree" &&effectiveCwd &&@@ -2899,7 +2903,7 @@ export function resolveExecutionWorkspaceReuseProvisioningPolicy(input: {};}-function createInheritedExecutionWorkspaceReuseFailure(input: {+function formatInheritedExecutionWorkspaceReuseFailure(input: {reason: "inherited_workspace_reuse_failed" | "inherited_workspace_reuse_unavailable";issueRef: WorkspaceReuseIssueRef;runId: string;@@ -2919,24 +2923,12 @@ function createInheritedExecutionWorkspaceReuseFailure(input: {: "Repair or unarchive the referenced execution workspace, or intentionally clear the issue's reuse_existing workspace binding before retrying.";const message = causeMessage? `Issue ${issueLabel} requested inherited execution workspace reuse for ${workspaceLabel}, but the workspace could not be restored because ${causeMessage}.`- : `Issue ${issueLabel} requested inherited execution workspace reuse for ${workspaceLabel} but the workspace could not be restored; workspace provisioning cannot replace it because this is an explicit reuse path.`;-- return new WorkspaceValidationFailure(message, {- workspaceValidation: {- reason: input.reason,- issueId: input.issueRef?.id ?? null,- issueIdentifier: input.issueRef?.identifier ?? null,- executionWorkspaceId: input.executionWorkspaceId ?? null,- workspaceConfigFreshnessAction: input.workspaceConfigFreshness.action,- workspaceConfigFreshnessReasons: input.workspaceConfigFreshness.reasons,- requestedReuseExisting: true,- replacementWorkspaceRealized: false,- remediation,- },- });+ : `Issue ${issueLabel} requested inherited execution workspace reuse for ${workspaceLabel}, but the workspace could not be restored.`;++ return `${message} ${remediation}`;}-export async function provisionExecutionWorkspaceForFreshnessDecision<T>(input: {+export async function provisionExecutionWorkspaceForFreshnessDecision<T extends { warnings?: string[] }>(input: {requestedShouldReuseExisting: boolean;existingExecutionWorkspaceId?: string | null;issueRef: WorkspaceReuseIssueRef;@@ -2964,13 +2956,14 @@ export async function provisionExecutionWorkspaceForFreshnessDecision<T>(input:}let restored: T | null = null;+ let reuseFailure: string | null = null;try {restored = (await input.restoreExistingWorkspace?.()) ?? null;} catch (error) {if (isWorkspaceValidationFailure(error)) {throw error;}- throw createInheritedExecutionWorkspaceReuseFailure({+ reuseFailure = formatInheritedExecutionWorkspaceReuseFailure({reason: "inherited_workspace_reuse_failed",issueRef: input.issueRef,runId: input.runId,@@ -2981,7 +2974,7 @@ export async function provisionExecutionWorkspaceForFreshnessDecision<T>(input:}if (!restored) {- throw createInheritedExecutionWorkspaceReuseFailure({+ reuseFailure = reuseFailure ?? formatInheritedExecutionWorkspaceReuseFailure({reason: "inherited_workspace_reuse_unavailable",issueRef: input.issueRef,runId: input.runId,@@ -2990,6 +2983,11 @@ export async function provisionExecutionWorkspaceForFreshnessDecision<T>(input:});}+ if (reuseFailure) throw new Error(reuseFailure);+ if (!restored) {+ throw new Error("Expected restored execution workspace after reuse fallback handling");+ }+return {executionWorkspace: restored,reusedExecutionWorkspace: restored,@@ -4720,7 +4718,7 @@ export function normalizeSessionParams(params: Record<string, unknown> | null |return Object.keys(params).length > 0 ? params : null;}-type RunSessionOutcome = "succeeded" | "failed" | "cancelled" | "timed_out";+type RunSessionOutcome = "succeeded" | "interrupted" | "failed" | "cancelled" | "timed_out";const HERMES_ADAPTER_TYPE = "hermes_local";const HERMES_SESSION_ID_REGEX = /^(?:\d{8}_\d{6}_[A-Za-z0-9_-]{4,}|[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12})$/;@@ -7632,6 +7630,27 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {})agent: typeof agents.$inferSelect,now: Date,) {+ const existingRetry = await db+ .select()+ .from(heartbeatRuns)+ .where(and(eq(heartbeatRuns.companyId, run.companyId), eq(heartbeatRuns.retryOfRunId, run.id)))+ .orderBy(asc(heartbeatRuns.createdAt))+ .limit(1)+ .then((rows) => rows[0] ?? null);+ if (existingRetry) {+ await appendRunEvent(run, await nextRunEventSeq(run.id), {+ eventType: "lifecycle",+ stream: "system",+ level: "warn",+ message: "Process-loss retry already exists; skipping duplicate retry enqueue",+ payload: {+ retryRunId: existingRetry.id,+ retryRunStatus: existingRetry.status,+ },+ });+ return existingRetry;+ }+const invokability = await getAgentInvokability(agent);if (!invokability.invokable) {await appendRunEvent(run, await nextRunEventSeq(run.id), {@@ -7750,6 +7769,104 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {})return queued;}+ async function drainRunningRunsForShutdown(signal: "SIGINT" | "SIGTERM", now = new Date()) {+ const activeRuns = await db+ .select({+ run: heartbeatRuns,+ agent: agents,+ })+ .from(heartbeatRuns)+ .innerJoin(agents, eq(heartbeatRuns.agentId, agents.id))+ .where(eq(heartbeatRuns.status, "running"));++ const interruptedRunIds: string[] = [];+ const retryRunIds: string[] = [];++ for (const { run, agent } of activeRuns) {+ const running = runningProcesses.get(run.id);+ try {+ if (running) {+ await terminateHeartbeatRunProcess({+ pid: running.child.pid ?? run.processPid,+ processGroupId: running.processGroupId ?? run.processGroupId,+ graceMs: Math.max(1, running.graceSec) * 1000,+ });+ } else if (run.processPid || run.processGroupId) {+ await terminateHeartbeatRunProcess({+ pid: run.processPid,+ processGroupId: run.processGroupId,+ });+ }+ } finally {+ runningProcesses.delete(run.id);+ }++ const message = `Interrupted by graceful server shutdown (${signal}); retry queued for restart recovery`;+ const interruptedStatus = await setRunStatusIfRunning(run.id, "interrupted", {+ finishedAt: now,+ error: message,+ errorCode: "server_shutdown_interrupted",+ signal,+ resultJson: mergeRunStopMetadataForAgent(agent, "interrupted", {+ resultJson: parseObject(run.resultJson),+ errorCode: "server_shutdown_interrupted",+ errorMessage: message,+ }),+ });+ if (!interruptedStatus.updated || !interruptedStatus.run) continue;+ let interrupted = interruptedStatus.run;+ await setWakeupStatus(run.wakeupRequestId, "cancelled", {+ finishedAt: now,+ error: null,+ });+ interrupted = await classifyAndPersistRunLiveness(interrupted, parseObject(interrupted.resultJson)) ?? interrupted;++ await releaseEnvironmentLeasesForRun({+ runId: interrupted.id,+ companyId: interrupted.companyId,+ agentId: interrupted.agentId,+ status: interrupted.status,+ failureReason: interrupted.error ?? undefined,+ });++ const retry = await enqueueProcessLossRetry(interrupted, agent, now);+ if (!retry) {+ await releaseIssueExecutionAndPromote(interrupted);+ } else {+ retryRunIds.push(retry.id);+ }++ await appendRunEvent(interrupted, await nextRunEventSeq(interrupted.id), {+ eventType: "lifecycle",+ stream: "system",+ level: "warn",+ message,+ payload: {+ signal,+ ...(run.processPid ? { processPid: run.processPid } : {}),+ ...(run.processGroupId ? { processGroupId: run.processGroupId } : {}),+ ...(retry ? { retryRunId: retry.id } : {}),+ },+ });++ await finalizeAgentStatus(run.agentId, "interrupted", message);+ interruptedRunIds.push(interrupted.id);+ }++ if (interruptedRunIds.length > 0) {+ logger.warn(+ { signal, interrupted: interruptedRunIds.length, interruptedRunIds, retryRunIds },+ "interrupted running heartbeat runs for graceful shutdown",+ );+ }++ return {+ interrupted: interruptedRunIds.length,+ interruptedRunIds,+ retryRunIds,+ };+ }+type ScheduledRetryGate =| { allowed: true }| {@@ -8160,7 +8277,7 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {})? readTransientRecoveryContractFromRun(run): null;const codexTransientFallbackMode =- agent.adapterType === "codex_local" && transientRecovery+ agent.adapterType === "codex_local" && transientRecovery?.errorFamily === "transient_upstream"? resolveCodexTransientFallbackMode(nextAttempt): null;const transientRetryNotBefore = transientRecovery?.retryNotBefore ?? null;@@ -8257,6 +8374,9 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {})scheduledRetryAttempt: schedule.attempt,scheduledRetryAt: schedule.dueAt.toISOString(),...(transientRetryNotBefore ? { transientRetryNotBefore: transientRetryNotBefore.toISOString() } : {}),+ ...(transientRecovery?.errorFamily === "provider_quota" && transientRetryNotBefore+ ? { providerQuotaRetryNotBefore: transientRetryNotBefore.toISOString() }+ : {}),...(codexTransientFallbackMode ? { codexTransientFallbackMode } : {}),}, "normal_model");const responsibleUserId = await resolveResponsibleUserIdForRunContext(run, retryContextSnapshot);@@ -8428,6 +8548,9 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {})scheduledRetryAttempt: schedule.attempt,scheduledRetryAt: schedule.dueAt.toISOString(),...(transientRetryNotBefore ? { transientRetryNotBefore: transientRetryNotBefore.toISOString() } : {}),+ ...(transientRecovery?.errorFamily === "provider_quota" && transientRetryNotBefore+ ? { providerQuotaRetryNotBefore: transientRetryNotBefore.toISOString() }+ : {}),...(codexTransientFallbackMode ? { codexTransientFallbackMode } : {}),}, "normal_model"),status: "queued",@@ -8551,6 +8674,9 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {})baseDelayMs: schedule.baseDelayMs,delayMs: schedule.delayMs,...(transientRetryNotBefore ? { transientRetryNotBefore: transientRetryNotBefore.toISOString() } : {}),+ ...(transientRecovery?.errorFamily === "provider_quota" && transientRetryNotBefore+ ? { providerQuotaRetryNotBefore: transientRetryNotBefore.toISOString() }+ : {}),...(codexTransientFallbackMode ? { codexTransientFallbackMode } : {}),},});@@ -9419,8 +9545,9 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {})async function finalizeAgentStatus(agentId: string,- outcome: "succeeded" | "failed" | "cancelled" | "timed_out",+ outcome: "succeeded" | "interrupted" | "failed" | "cancelled" | "timed_out",failureReason?: string | null,+ options?: { keepIdleOnFailure?: boolean },) {const existing = await getAgent(agentId);if (!existing) return;@@ -9435,7 +9562,7 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {})const nextStatus =runningCount > 0? "running"- : outcome === "succeeded" || outcome === "cancelled"+ : outcome === "succeeded" || outcome === "interrupted" || outcome === "cancelled" || (outcome === "failed" && options?.keepIdleOnFailure)? "idle": "error";@@ -9477,7 +9604,7 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {})function mergeRunStopMetadataForAgent(agent: Pick<typeof agents.$inferSelect, "adapterType" | "adapterConfig">,- outcome: "succeeded" | "failed" | "cancelled" | "timed_out",+ outcome: "succeeded" | "interrupted" | "failed" | "cancelled" | "timed_out",options?: {resultJson?: Record<string, unknown> | null;errorCode?: string | null;@@ -10832,7 +10959,7 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {})});const resolvedProjectId = executionWorkspace.projectId ?? issueRef?.projectId ?? executionProjectId ?? null;const resolvedProjectWorkspaceId = issueRef?.projectWorkspaceId ?? resolvedWorkspace.workspaceId ?? null;- let persistedExecutionWorkspace = null;+ let persistedExecutionWorkspace: ExecutionWorkspace | null = null;const nextExecutionWorkspaceMetadata = mergeExecutionWorkspaceMetadataForPersistence({existingMetadata: resolvedWorkspaceReusePolicy.shouldRestoreExistingWorkspace? reusableExistingExecutionWorkspace?.metadata ?? null@@ -10937,18 +11064,6 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {})}throw error;}- if (persistedExecutionWorkspace && pendingForwardBranchReconcile) {- await workspaceOperationRecorder.attachExecutionWorkspaceId(persistedExecutionWorkspace.id);- const reconcileResult = await reconcilePendingForwardBranchAfterPersistence({- db,- executionWorkspaceId: persistedExecutionWorkspace.id,- pending: pendingForwardBranchReconcile,- heartbeatRunId: run.id,- reconcileOperationPhase: "worktree_prepare",- recorder: workspaceOperationRecorder,- });- persistedExecutionWorkspace = reconcileResult.workspace;- }await workspaceOperationRecorder.attachExecutionWorkspaceId(persistedExecutionWorkspace?.id ?? null);await recordWorkspaceConfigFreshnessOperation({recorder: workspaceOperationRecorder,@@ -11592,7 +11707,7 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {})let inspection = branchInspection.inspection;const initialManagedGitWorktreeBranch = formatManagedGitWorktreeBranchInspection(inspection);if (!inspection.valid && inspection.reasonCode === "branch_mismatch" && inspection.repoRoot) {- let reconciledBranchName: string | null = null;+ let repairedExpectedBranchName = inspection.expectedBranchName;try {const coherence = await ensureGitWorktreeBranchCoherent({db,@@ -11612,11 +11727,14 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {})heartbeatRunId: run.id,enableWorkspaceBranchReconcileForward:resolvedInstanceSettings.experimental.enableWorkspaceBranchReconcileForward,+ persistForwardReconcile: false,reconcileOperationPhase: "workspace_finalize",recorder: workspaceOperationRecorder,});- if (coherence.reconciledForward && coherence.branchName) {- reconciledBranchName = coherence.branchName;+ if (coherence.branchName && coherence.branchName !== branchInspection.workspaceRecord.branchName) {+ repairedExpectedBranchName = coherence.branchName;+ executionWorkspace.branchName = coherence.branchName;+ executionWorkspace.warnings.push(...coherence.warnings);}} catch (repairErr) {const workspaceValidationFailure = isWorkspaceValidationFailure(repairErr) ? repairErr : null;@@ -11654,7 +11772,7 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {})const repairedInspection = await inspectManagedGitWorktreeBranch({worktreePath: inspection.worktreePath,- expectedBranchName: reconciledBranchName ?? inspection.expectedBranchName,+ expectedBranchName: repairedExpectedBranchName,repoRoot: inspection.repoRoot,});finalizeBranchRepairMetadata = {@@ -12165,6 +12283,11 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {})agent.id,outcome,outcome === "succeeded" ? null : (adapterResult.errorMessage ?? null),+ {+ keepIdleOnFailure:+ outcome === "failed" &&+ (finalizedRun ? readHeartbeatRunErrorFamily(finalizedRun) === "provider_quota" : runErrorCode === "provider_quota"),+ },);} catch (err) {const message = redactCurrentUserText(@@ -14673,6 +14796,7 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {})reportRunActivity: clearDetachedRunWarning,reapOrphanedRuns,+ drainRunningRunsForShutdown,promoteDueScheduledRetries,retryScheduledRetryNow,diff --git a/server/src/services/issue-continuation-summary.ts b/server/src/services/issue-continuation-summary.tsindex a8ce6aeca4..9c5f2145d9 100644--- a/server/src/services/issue-continuation-summary.ts+++ b/server/src/services/issue-continuation-summary.ts@@ -94,7 +94,7 @@ function extractPathCandidates(...texts: Array<string | null | undefined>) {function inferMode(issue: IssueSummaryInput, run: RunSummaryInput) {if (issue.status === "done" || issue.status === "in_review") return "review";- if (run.status === "failed" || run.status === "timed_out" || run.status === "cancelled") return "implementation";+ if (run.status === "failed" || run.status === "timed_out" || run.status === "cancelled" || run.status === "interrupted") return "implementation";if (issue.status === "backlog" || issue.status === "todo") return "plan";return "implementation";}diff --git a/server/src/services/issues.ts b/server/src/services/issues.tsindex 7f1bec5013..8cc413df09 100644--- a/server/src/services/issues.ts+++ b/server/src/services/issues.ts@@ -604,7 +604,7 @@ function sameRunLock(checkoutRunId: string | null, actorRunId: string | null) {return checkoutRunId == null;}-export const TERMINAL_HEARTBEAT_RUN_STATUSES = new Set(["succeeded", "failed", "cancelled", "timed_out"]);+export const TERMINAL_HEARTBEAT_RUN_STATUSES = new Set(["succeeded", "interrupted", "failed", "cancelled", "timed_out"]);const ISSUE_LIST_DESCRIPTION_MAX_CHARS = 1200;const ISSUE_LIST_DESCRIPTION_MAX_BYTES = ISSUE_LIST_DESCRIPTION_MAX_CHARS * 4;diff --git a/server/src/services/plugin-host-services.ts b/server/src/services/plugin-host-services.tsindex 0925820340..32168f230b 100644--- a/server/src/services/plugin-host-services.ts+++ b/server/src/services/plugin-host-services.ts@@ -2644,7 +2644,7 @@ export function buildHostServices(// Track the subscription so it can be cleaned up on dispose() if the run// never reaches a terminal status (hang, crash, network partition).if (notifyWorker) {- const TERMINAL_STATUSES = new Set(["succeeded", "failed", "cancelled", "timed_out"]);+ const TERMINAL_STATUSES = new Set(["succeeded", "interrupted", "failed", "cancelled", "timed_out"]);const cleanup = () => {unsubscribe();diff --git a/server/src/services/productivity-review.ts b/server/src/services/productivity-review.tsindex 3c3c615d7a..e7fad83898 100644--- a/server/src/services/productivity-review.ts+++ b/server/src/services/productivity-review.ts@@ -31,7 +31,7 @@ export const DEFAULT_PRODUCTIVITY_REVIEW_MAX_REFRESH_COMMENTS = 3;export const DEFAULT_PRODUCTIVITY_REVIEW_CREATION_WINDOW_MS = 24 * 60 * 60 * 1000;export const DEFAULT_PRODUCTIVITY_REVIEW_MAX_CREATIONS_PER_WINDOW = 3;-const TERMINAL_RUN_STATUSES = ["succeeded", "failed", "cancelled", "timed_out"] as const;+const TERMINAL_RUN_STATUSES = ["succeeded", "interrupted", "failed", "cancelled", "timed_out"] as const;const ACTIVE_RUN_STATUSES = ["queued", "running", "scheduled_retry"] as const;const MAX_CANDIDATE_ISSUES = 250;const MAX_RUNS_FOR_STREAK = 100;diff --git a/server/src/services/recovery/service.ts b/server/src/services/recovery/service.tsindex 24c32899a6..d66d299e50 100644--- a/server/src/services/recovery/service.ts+++ b/server/src/services/recovery/service.ts@@ -70,7 +70,7 @@ import {import { isAutomaticRecoverySuppressedByPauseHold } from "./pause-hold-guard.js";const EXECUTION_PATH_HEARTBEAT_RUN_STATUSES = ["queued", "running", "scheduled_retry"] as const;-const UNSUCCESSFUL_HEARTBEAT_RUN_TERMINAL_STATUSES = ["failed", "cancelled", "timed_out"] as const;+const UNSUCCESSFUL_HEARTBEAT_RUN_TERMINAL_STATUSES = ["interrupted", "failed", "cancelled", "timed_out"] as const;export const ACTIVE_RUN_OUTPUT_SUSPICION_THRESHOLD_MS = 60 * 60 * 1000;export const ACTIVE_RUN_OUTPUT_CRITICAL_THRESHOLD_MS = 4 * 60 * 60 * 1000;export const ACTIVE_RUN_OUTPUT_CONTINUE_REARM_MS = 30 * 60 * 1000;@@ -229,6 +229,7 @@ const TRANSIENT_INFRA_CONTINUATION_ERROR_CODES = new Set<string>(["adapter_failed","codex_transient_upstream","claude_transient_upstream",+ "provider_quota","timeout",]);diff --git a/server/src/services/run-liveness.ts b/server/src/services/run-liveness.tsindex 72be57e6f0..87b53285c7 100644--- a/server/src/services/run-liveness.ts+++ b/server/src/services/run-liveness.ts@@ -309,6 +309,10 @@ export function classifyRunLiveness(input: RunLivenessClassificationInput): RunLactionability,});+ if (input.runStatus === "interrupted") {+ return output("needs_followup", input.errorCode ? `Run interrupted (${input.errorCode})` : "Run interrupted");+ }+if (input.runStatus !== "succeeded") {return output("failed", input.errorCode ? `Run ended with ${input.runStatus} (${input.errorCode})` : `Run ended with ${input.runStatus}`);}diff --git a/server/src/services/task-watchdogs.ts b/server/src/services/task-watchdogs.tsindex df84b8fc0a..3e2be28e94 100644--- a/server/src/services/task-watchdogs.ts+++ b/server/src/services/task-watchdogs.ts@@ -28,7 +28,7 @@ const TASK_WATCHDOG_SUBTREE_MAX_DEPTH = 100;const TASK_WATCHDOG_LIVE_RUN_STATUSES = ["queued", "running", "scheduled_retry"] as const;const TASK_WATCHDOG_WAKE_REQUEST_STATUSES = ["queued", "deferred_issue_execution"] as const;const TASK_WATCHDOG_TERMINAL_ISSUE_STATUSES = ["done", "cancelled"] as const;-const TASK_WATCHDOG_TERMINAL_RUN_STATUSES = ["succeeded", "failed", "cancelled", "timed_out"] as const;+const TASK_WATCHDOG_TERMINAL_RUN_STATUSES = ["succeeded", "interrupted", "failed", "cancelled", "timed_out"] as const;// Grace window after an issue is created/assigned during which its first// assignment run/wake may have been enqueued but is not yet visible to a// watchdog evaluation (the eval can race the issue's own assignment run).diff --git a/server/src/services/workspace-runtime.ts b/server/src/services/workspace-runtime.tsindex b0820818a6..3a062137be 100644--- a/server/src/services/workspace-runtime.ts+++ b/server/src/services/workspace-runtime.ts@@ -665,6 +665,7 @@ type GitWorktreeBranchCoherenceResult = {branchName: string | null;reconciledForward: boolean;pendingForwardBranchReconcile?: PendingForwardBranchReconcile | null;+ warnings: string[];};export type PendingForwardBranchReconcile = {@@ -791,9 +792,30 @@ async function inspectGitWorktreeBranchIncoherence(input: {sameHead,ancestryVerdict,});- const eligible = cleanliness === "clean" && expectedBranchExists && sameHead && registeredBranchMatchesHead;+ const canCheckoutRecordedBranch =+ cleanliness === "clean" && expectedBranchExists && sameHead && registeredBranchMatchesHead;+ const canAdoptForwardActualBranch =+ cleanliness === "clean" &&+ expectedBranchExists &&+ actualBranchExists === true &&+ ancestryVerdict === "ancestor" &&+ !sameHead &&+ registeredBranchMatchesHead;+ const canAttachRecordedBranchToDetachedHead =+ cleanliness === "clean" &&+ expectedBranchExists &&+ input.actualBranchName === null &&+ ancestryVerdict === "ancestor" &&+ !sameHead &&+ registeredBranchMatchesHead;+ const eligible =+ canCheckoutRecordedBranch || canAdoptForwardActualBranch || canAttachRecordedBranchToDetachedHead;const safeRepairReason = eligible- ? "clean worktree and expected branch points at the current HEAD"+ ? canCheckoutRecordedBranch+ ? "clean worktree and expected branch points at the current HEAD"+ : canAdoptForwardActualBranch+ ? "clean worktree and checked-out branch is forward of the recorded branch"+ : "clean detached worktree HEAD is forward of the recorded branch": cleanliness !== "clean"? "worktree is not clean": !registered@@ -1025,17 +1047,18 @@ export async function ensureGitWorktreeBranchCoherent(input: {actualBranchName?: string | null;heartbeatRunId?: string | null;enableWorkspaceBranchReconcileForward?: boolean;+ persistForwardReconcile?: boolean;reconcileOperationPhase?: "worktree_prepare" | "workspace_finalize";recorder?: WorkspaceOperationRecorder | null;}): Promise<GitWorktreeBranchCoherenceResult> {const expectedBranchName = input.expectedBranchName?.trim();- if (!expectedBranchName) return { branchName: null, reconciledForward: false };+ if (!expectedBranchName) return { branchName: null, reconciledForward: false, warnings: [] };const currentBranch = input.actualBranchName !== undefined? input.actualBranchName: await runGit(["symbolic-ref", "--quiet", "--short", "HEAD"], input.worktreePath).catch(() => null);if (currentBranch === expectedBranchName) {- return { branchName: expectedBranchName, reconciledForward: false };+ return { branchName: expectedBranchName, reconciledForward: false, warnings: [] };}const evidence = await inspectGitWorktreeBranchIncoherence({@@ -1053,7 +1076,7 @@ export async function ensureGitWorktreeBranchCoherent(input: {currentBranch) {const reason = "Automatic forward reconciliation: recorded branch is an ancestor of the checked-out branch.";- if (input.executionWorkspaceId) {+ if (input.executionWorkspaceId && input.persistForwardReconcile !== false) {if (!input.db) {evidence.safeRepair.reason = "forward reconciliation requires database access to update the execution workspace record";throw branchIncoherenceValidationFailure(evidence);@@ -1107,7 +1130,7 @@ export async function ensureGitWorktreeBranchCoherent(input: {auditCommentId: result.auditCommentId,recoveryActionId: result.recoveryAction?.id ?? null,});- return { branchName: result.inspection.toBranch, reconciledForward: true };+ return { branchName: result.inspection.toBranch, reconciledForward: true, warnings: [] };} catch (error) {evidence.safeRepair.reason =`forward reconciliation failed: ${error instanceof Error ? error.message : String(error)}`;@@ -1122,6 +1145,7 @@ export async function ensureGitWorktreeBranchCoherent(input: {return {branchName: currentBranch,reconciledForward: true,+ warnings: [],pendingForwardBranchReconcile: {recordedBranchName: expectedBranchName,adoptedBranchName: currentBranch,@@ -1136,6 +1160,75 @@ export async function ensureGitWorktreeBranchCoherent(input: {}evidence.safeRepair.attempted = true;+ const warningPrefix =+ `Execution workspace branch metadata was self-healed from "${expectedBranchName}" to "${formatBranchForMessage(currentBranch)}" at ${input.worktreePath}.`;+ if (+ currentBranch &&+ evidence.provenance.actualBranchExists === true &&+ evidence.provenance.ancestryVerdict === "ancestor" &&+ !evidence.provenance.sameHead+ ) {+ evidence.safeRepair.succeeded = true;+ evidence.safeRepair.reason = "clean worktree adopted the checked-out branch because it is forward of the recorded branch";+ return {+ branchName: currentBranch,+ reconciledForward: false,+ warnings: [+ `${warningPrefix} The checked-out branch contains the recorded branch plus newer commits, so Paperclip adopted it for subsequent runs.`,+ ],+ };+ }++ if (+ currentBranch === null &&+ evidence.provenance.ancestryVerdict === "ancestor" &&+ !evidence.provenance.sameHead &&+ evidence.provenance.actualHeadSha+ ) {+ try {+ await recordGitOperation(input.recorder, {+ phase: "worktree_prepare",+ args: ["checkout", "-B", expectedBranchName, evidence.provenance.actualHeadSha],+ cwd: input.worktreePath,+ metadata: {+ repoRoot: input.repoRoot,+ worktreePath: input.worktreePath,+ expectedBranchName,+ actualBranchName: currentBranch,+ branchIncoherenceRepair: true,+ detachedHeadRepair: true,+ fingerprint: evidence.fingerprint,+ sourceIssueId: evidence.sourceIssueId,+ executionWorkspaceId: evidence.executionWorkspaceId,+ },+ successMessage: `Reattached detached git worktree HEAD at ${input.worktreePath} to ${expectedBranchName}\n`,+ failureLabel: `git checkout -B ${expectedBranchName} ${formatShortSha(evidence.provenance.actualHeadSha)}`,+ });+ } catch (error) {+ evidence.safeRepair.succeeded = false;+ evidence.safeRepair.reason = `safe detached HEAD reattachment failed: ${error instanceof Error ? error.message : String(error)}`;+ throw branchIncoherenceValidationFailure(evidence);+ }++ const repairedBranch = await runGit(["symbolic-ref", "--quiet", "--short", "HEAD"], input.worktreePath)+ .catch(() => null);+ if (repairedBranch !== expectedBranchName) {+ evidence.safeRepair.succeeded = false;+ evidence.safeRepair.reason = `reattach completed but HEAD is ${formatBranchForMessage(repairedBranch)}`;+ throw branchIncoherenceValidationFailure(evidence);+ }++ evidence.safeRepair.succeeded = true;+ evidence.safeRepair.reason = "clean detached worktree HEAD was reattached to the recorded branch";+ return {+ branchName: expectedBranchName,+ reconciledForward: false,+ warnings: [+ `${warningPrefix} The detached HEAD contained the recorded branch plus newer commits, so Paperclip moved the recorded branch to that HEAD.`,+ ],+ };+ }+try {await recordGitOperation(input.recorder, {phase: "worktree_prepare",@@ -1170,7 +1263,13 @@ export async function ensureGitWorktreeBranchCoherent(input: {evidence.safeRepair.succeeded = true;evidence.safeRepair.reason = "clean worktree checked out the recorded branch";- return { branchName: expectedBranchName, reconciledForward: false };+ return {+ branchName: expectedBranchName,+ reconciledForward: false,+ warnings: [+ `Execution workspace branch metadata was self-healed by checking out recorded branch "${expectedBranchName}" at ${input.worktreePath}.`,+ ],+ };}// Resolve the authoritative base ref for a fresh worktree. A configured local@@ -1919,12 +2018,12 @@ export async function realizeExecutionWorkspace(input: {await fs.mkdir(worktreeParentDir, { recursive: true });- async function reuseExistingWorktree(reusablePath: string) {+ async function reuseExistingWorktree(reusablePath: string, effectiveBranchName = branchName, extraWarnings: string[] = []) {const refresh = currentBaseRefSha? await refreshUnstartedWorktreeToBase({repoRoot,worktreePath: reusablePath,- branchName,+ branchName: effectiveBranchName,baseRef,currentBaseRefSha,recorder: input.recorder ?? null,@@ -1945,7 +2044,7 @@ export async function realizeExecutionWorkspace(input: {metadata: {repoRoot,worktreePath: reusablePath,- branchName,+ branchName: effectiveBranchName,baseRef,currentBaseRefSha: baseDrift.currentBaseRefSha,branchBaseRefSha: baseDrift.branchBaseRefSha,@@ -1964,7 +2063,7 @@ export async function realizeExecutionWorkspace(input: {base: input.base,repoRoot,worktreePath: reusablePath,- branchName,+ branchName: effectiveBranchName,issue: input.issue,agent: input.agent,created: false,@@ -1975,9 +2074,9 @@ export async function realizeExecutionWorkspace(input: {repoRef: baseRef,strategy: "git_worktree" as const,cwd: reusablePath,- branchName,+ branchName: effectiveBranchName,worktreePath: reusablePath,- warnings: [...baseRefreshWarnings, ...baseDrift.warnings],+ warnings: [...extraWarnings, ...baseRefreshWarnings, ...baseDrift.warnings],created: false,baseRefSha: refresh.baseRefSha ?? baseDrift.branchBaseRefSha ?? baseDrift.currentBaseRefSha,pendingForwardBranchReconcile,@@ -2004,35 +2103,43 @@ export async function realizeExecutionWorkspace(input: {reconcileOperationPhase: "worktree_prepare",recorder: input.recorder ?? null,});- if (coherence.reconciledForward && coherence.branchName) {- branchName = coherence.branchName;+ const effectiveBranchName = coherence.branchName ?? branchName;+ if (coherence.reconciledForward) {+ branchName = effectiveBranchName;pendingForwardBranchReconcile = coherence.pendingForwardBranchReconcile ?? null;}- return await validateLinkedGitWorktree({+ const nextValidation = await validateLinkedGitWorktree({repoRoot,worktreePath: reusablePath,- expectedBranchName: branchName,+ expectedBranchName: effectiveBranchName,}).catch(() => null);+ return {+ validation: nextValidation,+ branchName: effectiveBranchName,+ warnings: coherence.warnings,+ };}- return validation;+ return { validation, branchName, warnings: [] };}const existingWorktree = await directoryExists(worktreePath);if (existingWorktree) {- const validation = await validateReusableWorktree(worktreePath);- if (validation?.valid) {- return await reuseExistingWorktree(worktreePath);+ const reusable = await validateReusableWorktree(worktreePath);+ if (reusable.validation?.valid) {+ return await reuseExistingWorktree(worktreePath, reusable.branchName, reusable.warnings);}+ const validation = reusable.validation;const reason = validation && !validation.valid ? ` (${validation.reason})` : "";throw new Error(`Configured worktree path "${worktreePath}" already exists and is not a reusable git worktree${reason}.`);}const registeredBranchWorktree = await findRegisteredGitWorktreeByBranch(repoRoot, branchName);if (registeredBranchWorktree) {- const validation = await validateReusableWorktree(registeredBranchWorktree);- if (validation?.valid) {- return await reuseExistingWorktree(registeredBranchWorktree);+ const reusable = await validateReusableWorktree(registeredBranchWorktree);+ if (reusable.validation?.valid) {+ return await reuseExistingWorktree(registeredBranchWorktree, reusable.branchName, reusable.warnings);}+ const validation = reusable.validation;const reason = validation && !validation.valid ? ` (${validation.reason})` : "";throw new Error(`Registered worktree for branch "${branchName}" at "${registeredBranchWorktree}" is not reusable${reason}.`);}@@ -2167,6 +2274,7 @@ export async function ensurePersistedExecutionWorkspaceAvailable(input: {if (await directoryExists(cwd)) {const reuseBaseRef = input.workspace.baseRef ?? input.base.repoRef ?? null;const reuseWorktreePath = realized.worktreePath ?? cwd;+ const repairWarnings: string[] = [];if (await isGitCheckout(reuseWorktreePath)) {const coherence = await ensureGitWorktreeBranchCoherent({db: input.db ?? null,@@ -2177,12 +2285,17 @@ export async function ensurePersistedExecutionWorkspaceAvailable(input: {executionWorkspaceId: input.workspace.id ?? null,heartbeatRunId: input.heartbeatRunId ?? null,enableWorkspaceBranchReconcileForward: input.enableWorkspaceBranchReconcileForward === true,+ persistForwardReconcile: false,reconcileOperationPhase: "worktree_prepare",recorder: input.recorder ?? null,});- if (coherence.reconciledForward && coherence.branchName) {+ if (coherence.branchName) {realized.branchName = coherence.branchName;}+ if (coherence.reconciledForward) {+ realized.pendingForwardBranchReconcile = coherence.pendingForwardBranchReconcile ?? null;+ }+ repairWarnings.push(...coherence.warnings);}const validation = await validateLinkedGitWorktree({repoRoot,@@ -2224,7 +2337,7 @@ export async function ensurePersistedExecutionWorkspaceAvailable(input: {recordedBaseRefSha,skipRefresh: true,});- realized.warnings = [...baseRefreshWarnings, ...baseDrift.warnings];+ realized.warnings = [...repairWarnings, ...baseRefreshWarnings, ...baseDrift.warnings];realized.baseRefSha = refresh.baseRefSha ?? recordedBaseRefSha ?? baseDrift.branchBaseRefSha ?? baseDrift.currentBaseRefSha;if (provisionCommand) {await provisionExecutionWorktree({@@ -4134,6 +4247,9 @@ export function buildWorkspaceReadyComment(input: {if (input.workspace.worktreePath && input.workspace.worktreePath !== input.workspace.cwd) {lines.push(`- Worktree: \`${input.workspace.worktreePath}\``);}+ for (const warning of input.workspace.warnings) {+ lines.push(`- Warning: ${warning}`);+ }for (const service of input.runtimeServices) {const detail = service.url ? `${service.serviceName}: ${service.url}` : `${service.serviceName}: running`;const suffix = service.reused ? " (reused)" : "";diff --git a/ui/src/components/ActivityCharts.test.tsx b/ui/src/components/ActivityCharts.test.tsxindex 1d17cd38df..a665c9a33b 100644--- a/ui/src/components/ActivityCharts.test.tsx+++ b/ui/src/components/ActivityCharts.test.tsx@@ -1,7 +1,7 @@// @vitest-environment jsdom-import { act } from "react";import type { ReactNode } from "react";+import { flushSync } from "react-dom";import { createRoot, type Root } from "react-dom/client";import type { HeartbeatRun } from "@paperclipai/shared";import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";@@ -22,13 +22,13 @@ beforeEach(() => {});afterEach(() => {- act(() => root.unmount());+ flushSync(() => root.unmount());container.remove();vi.useRealTimers();});function render(ui: ReactNode) {- act(() => {+ flushSync(() => {root.render(ui);});}@@ -99,12 +99,37 @@ describe("ActivityCharts", () => {<RunActivityChartruns={[createRun({ id: "run-success", status: "succeeded" }),- createRun({ id: "run-failed", status: "failed" }),+ createRun({ id: "run-failed", status: "failed", errorCode: "provider_quota" }),]}/>,);expect(container.textContent).not.toContain("No runs yet");- expect(container.querySelector("[title='2026-04-20: 2 runs']")).not.toBeNull();+ // Tooltip now carries the per-day breakdown (incl. failure error codes).+ const dayCell = container.querySelector("[title^='2026-04-20: 2 runs']");+ expect(dayCell).not.toBeNull();+ expect(dayCell?.getAttribute("title")).toContain("provider_quota: 1");+ });++ it("renders a distinct recovered segment and legend for recovered restart kills", () => {+ render(+ <RunActivityChart+ activity={[+ {+ date: "2026-04-20",+ succeeded: 3,+ failed: 1,+ recovered: 4,+ other: 0,+ total: 8,+ failedByErrorCode: { process_lost: 1 },+ },+ ]}+ />,+ );++ expect(container.textContent).toContain("Recovered");+ const dayCell = container.querySelector("[title*='recovered: 4']");+ expect(dayCell).not.toBeNull();});});diff --git a/ui/src/components/ActivityCharts.tsx b/ui/src/components/ActivityCharts.tsxindex 33171a5bf9..38a92bea77 100644--- a/ui/src/components/ActivityCharts.tsx+++ b/ui/src/components/ActivityCharts.tsx@@ -15,6 +15,31 @@ function formatDayLabel(dateStr: string): string {return `${d.getMonth() + 1}/${d.getDate()}`;}+function emptyRunDay(date: string): DashboardRunActivityDay {+ return { date, succeeded: 0, failed: 0, recovered: 0, other: 0, total: 0, failedByErrorCode: {} };+}++const runSegmentColors = {+ succeeded: "#10b981",+ recovered: "#f59e0b",+ failed: "#ef4444",+ other: "#737373",+} as const;++// Compact per-day tooltip that also attributes failures to their error class.+function runDayTooltip(entry: DashboardRunActivityDay): string {+ const lines = [`${entry.date}: ${entry.total} run${entry.total === 1 ? "" : "s"}`];+ if (entry.succeeded > 0) lines.push(` succeeded: ${entry.succeeded}`);+ if (entry.recovered > 0) lines.push(` recovered: ${entry.recovered} (retry succeeded)`);+ if (entry.failed > 0) {+ lines.push(` failed: ${entry.failed}`);+ const codes = Object.entries(entry.failedByErrorCode ?? {}).sort((a, b) => b[1] - a[1]);+ for (const [code, count] of codes) lines.push(` ${code}: ${count}`);+ }+ if (entry.other > 0) lines.push(` other: ${entry.other}`);+ return lines.join("\n");+}+/* ---- Sub-components ---- */function DateLabels({ days }: { days: string[] }) {@@ -65,14 +90,23 @@ type RunChartProps =function aggregateRuns(runs: readonly HeartbeatRun[] = []): DashboardRunActivityDay[] {const days = getLast14Days();const grouped = new Map<string, DashboardRunActivityDay>();- for (const day of days) grouped.set(day, { date: day, succeeded: 0, failed: 0, other: 0, total: 0 });+ for (const day of days) grouped.set(day, emptyRunDay(day));for (const run of runs) {const day = new Date(run.createdAt).toISOString().slice(0, 10);const entry = grouped.get(day);if (!entry) continue;- if (run.status === "succeeded") entry.succeeded++;- else if (run.status === "failed" || run.status === "timed_out") entry.failed++;- else entry.other++;+ if (run.status === "succeeded") {+ entry.succeeded++;+ } else if (run.status === "failed" || run.status === "timed_out") {+ // A flat run list has no retry-chain linkage, so recovery can't be derived+ // here (the company dashboard computes it server-side). Attribute the+ // failure to its error class so the breakdown still renders.+ entry.failed++;+ const code = run.errorCode && run.errorCode.length > 0 ? run.errorCode : "unknown";+ entry.failedByErrorCode[code] = (entry.failedByErrorCode[code] ?? 0) + 1;+ } else {+ entry.other++;+ }entry.total++;}return Array.from(grouped.values());@@ -91,23 +125,32 @@ export function RunActivityChart(props: RunChartProps) {const maxValue = Math.max(...activity.map(v => v.total), 1);const hasData = activity.some(v => v.total > 0);+ const hasRecovered = activity.some(v => v.recovered > 0);if (!hasData) return <p className="text-xs text-muted-foreground">No runs yet</p>;+ const legendItems = [+ { color: runSegmentColors.succeeded, label: "Succeeded" },+ ...(hasRecovered ? [{ color: runSegmentColors.recovered, label: "Recovered" }] : []),+ { color: runSegmentColors.failed, label: "Failed" },+ { color: runSegmentColors.other, label: "Other" },+ ];+return (<div><div className="flex items-end gap-(--sz-3px) h-20">{days.map(day => {- const entry = grouped.get(day) ?? { date: day, succeeded: 0, failed: 0, other: 0, total: 0 };+ const entry = grouped.get(day) ?? emptyRunDay(day);const total = entry.total;const heightPct = (total / maxValue) * 100;return (- <div key={day} className="flex-1 h-full flex flex-col justify-end" title={`${day}: ${total} runs`}>+ <div key={day} className="flex-1 h-full flex flex-col justify-end" title={runDayTooltip(entry)}>{total > 0 ? (<div className="flex flex-col-reverse gap-px overflow-hidden" style={{ height: `${heightPct}%`, minHeight: 2 }}>- {entry.succeeded > 0 && <div className="bg-emerald-500" style={{ flex: entry.succeeded }} />}- {entry.failed > 0 && <div className="bg-red-500" style={{ flex: entry.failed }} />}- {entry.other > 0 && <div className="bg-neutral-500" style={{ flex: entry.other }} />}+ {entry.succeeded > 0 && <div style={{ flex: entry.succeeded, backgroundColor: runSegmentColors.succeeded }} />}+ {entry.recovered > 0 && <div style={{ flex: entry.recovered, backgroundColor: runSegmentColors.recovered }} />}+ {entry.failed > 0 && <div style={{ flex: entry.failed, backgroundColor: runSegmentColors.failed }} />}+ {entry.other > 0 && <div style={{ flex: entry.other, backgroundColor: runSegmentColors.other }} />}</div>) : (<div className="bg-muted/30 rounded-sm" style={{ height: 2 }} />@@ -117,6 +160,7 @@ export function RunActivityChart(props: RunChartProps) {})}</div><DateLabels days={days} />+ <ChartLegend items={legendItems} /></div>);}@@ -260,11 +304,14 @@ export function SuccessRateChart(props: RunChartProps) {<div><div className="flex items-end gap-(--sz-3px) h-20">{days.map(day => {- const entry = grouped.get(day) ?? { date: day, succeeded: 0, failed: 0, other: 0, total: 0 };- const rate = entry.total > 0 ? entry.succeeded / entry.total : 0;+ const entry = grouped.get(day) ?? emptyRunDay(day);+ // Recovered runs ultimately succeeded, so they count toward the rate+ // rather than dragging it down as failures.+ const effectiveSucceeded = entry.succeeded + entry.recovered;+ const rate = entry.total > 0 ? effectiveSucceeded / entry.total : 0;const color = entry.total === 0 ? undefined : rate >= 0.8 ? "var(--hex-10b981)" : rate >= 0.5 ? "var(--hex-eab308)" : "var(--hex-ef4444)";return (- <div key={day} className="flex-1 h-full flex flex-col justify-end" title={`${day}: ${entry.total > 0 ? Math.round(rate * 100) : 0}% (${entry.succeeded}/${entry.total})`}>+ <div key={day} className="flex-1 h-full flex flex-col justify-end" title={`${day}: ${entry.total > 0 ? Math.round(rate * 100) : 0}% (${effectiveSucceeded}/${entry.total})`}>{entry.total > 0 ? (<div style={{ height: `${rate * 100}%`, minHeight: 2, backgroundColor: color }} />) : (diff --git a/ui/src/components/transcript/useLiveRunTranscripts.ts b/ui/src/components/transcript/useLiveRunTranscripts.tsindex 6c001e2fae..12e23ccfe9 100644--- a/ui/src/components/transcript/useLiveRunTranscripts.ts+++ b/ui/src/components/transcript/useLiveRunTranscripts.ts@@ -35,7 +35,7 @@ function readString(value: unknown): string | null {}function isTerminalStatus(status: string): boolean {- return status === "failed" || status === "timed_out" || status === "cancelled" || status === "succeeded";+ return status === "failed" || status === "timed_out" || status === "cancelled" || status === "interrupted" || status === "succeeded";}function runKnownLogBytes(run: RunTranscriptSource): number | null {diff --git a/ui/src/context/LiveUpdatesProvider.tsx b/ui/src/context/LiveUpdatesProvider.tsxindex c480f781b8..7a3976ba08 100644--- a/ui/src/context/LiveUpdatesProvider.tsx+++ b/ui/src/context/LiveUpdatesProvider.tsx@@ -21,7 +21,7 @@ const TOAST_COOLDOWN_MAX = 3;const RECONNECT_SUPPRESS_MS = 2000;const SOCKET_CONNECTING = 0;const SOCKET_OPEN = 1;-const TERMINAL_RUN_STATUSES = new Set(["succeeded", "failed", "cancelled", "timed_out"]);+const TERMINAL_RUN_STATUSES = new Set(["succeeded", "interrupted", "failed", "cancelled", "timed_out"]);type LiveUpdatesSocketLike = {readyState: number;diff --git a/ui/storybook/fixtures/paperclipData.ts b/ui/storybook/fixtures/paperclipData.tsindex 71b4c5ab91..52bd2b2a0f 100644--- a/ui/storybook/fixtures/paperclipData.ts+++ b/ui/storybook/fixtures/paperclipData.ts@@ -1286,20 +1286,21 @@ export const storybookDashboardSummary: DashboardSummary = {pausedProjects: 1,},runActivity: [- { date: "2026-04-07", succeeded: 4, failed: 0, other: 1, total: 5 },- { date: "2026-04-08", succeeded: 5, failed: 1, other: 0, total: 6 },- { date: "2026-04-09", succeeded: 3, failed: 0, other: 1, total: 4 },- { date: "2026-04-10", succeeded: 6, failed: 0, other: 0, total: 6 },- { date: "2026-04-11", succeeded: 4, failed: 1, other: 0, total: 5 },- { date: "2026-04-12", succeeded: 2, failed: 0, other: 1, total: 3 },- { date: "2026-04-13", succeeded: 5, failed: 0, other: 1, total: 6 },- { date: "2026-04-14", succeeded: 6, failed: 1, other: 0, total: 7 },- { date: "2026-04-15", succeeded: 4, failed: 0, other: 1, total: 5 },- { date: "2026-04-16", succeeded: 7, failed: 0, other: 0, total: 7 },- { date: "2026-04-17", succeeded: 6, failed: 1, other: 0, total: 7 },- { date: "2026-04-18", succeeded: 3, failed: 0, other: 1, total: 4 },- { date: "2026-04-19", succeeded: 5, failed: 0, other: 1, total: 6 },- { date: "2026-04-20", succeeded: 4, failed: 0, other: 2, total: 6 },+ { date: "2026-04-07", succeeded: 4, failed: 0, recovered: 0, other: 1, total: 5, failedByErrorCode: {} },+ { date: "2026-04-08", succeeded: 5, failed: 1, recovered: 0, other: 0, total: 6, failedByErrorCode: { provider_quota: 1 } },+ { date: "2026-04-09", succeeded: 3, failed: 0, recovered: 0, other: 1, total: 4, failedByErrorCode: {} },+ { date: "2026-04-10", succeeded: 6, failed: 0, recovered: 0, other: 0, total: 6, failedByErrorCode: {} },+ { date: "2026-04-11", succeeded: 4, failed: 1, recovered: 0, other: 0, total: 5, failedByErrorCode: { workspace_validation_failed: 1 } },+ { date: "2026-04-12", succeeded: 2, failed: 0, recovered: 0, other: 1, total: 3, failedByErrorCode: {} },+ { date: "2026-04-13", succeeded: 5, failed: 0, recovered: 0, other: 1, total: 6, failedByErrorCode: {} },+ { date: "2026-04-14", succeeded: 6, failed: 1, recovered: 0, other: 0, total: 7, failedByErrorCode: { provider_quota: 1 } },+ { date: "2026-04-15", succeeded: 4, failed: 0, recovered: 0, other: 1, total: 5, failedByErrorCode: {} },+ { date: "2026-04-16", succeeded: 7, failed: 0, recovered: 0, other: 0, total: 7, failedByErrorCode: {} },+ { date: "2026-04-17", succeeded: 6, failed: 1, recovered: 0, other: 0, total: 7, failedByErrorCode: { workspace_validation_failed: 1 } },+ { date: "2026-04-18", succeeded: 3, failed: 0, recovered: 0, other: 1, total: 4, failedByErrorCode: {} },+ // Restart-burst spike: many process_lost kills, most recovered on retry.+ { date: "2026-04-19", succeeded: 5, failed: 2, recovered: 6, other: 1, total: 14, failedByErrorCode: { process_lost: 2 } },+ { date: "2026-04-20", succeeded: 4, failed: 1, recovered: 3, other: 2, total: 10, failedByErrorCode: { process_lost: 1 } },],};