Skip to content

The open-source app everyone uses to manage agents at work

test(server): retire duplicate process-loss recovery execution (#16088)

Dotta997ea5fd17d5
diff --git a/doc/plans/2026-10-12-test-audit-r16-heartbeat-recovery-controls.json b/doc/plans/2026-10-12-test-audit-r16-heartbeat-recovery-controls.json
new file mode 100644
index 0000000000..6179db6be7
--- /dev/null
+++ b/doc/plans/2026-10-12-test-audit-r16-heartbeat-recovery-controls.json
@@ -0,0 +1,97 @@
+{
+ "parentPR": 16086,
+ "parentBranch": "codex/test-audit-r15-environment-runtime",
+ "parentHead": "c0414ac02e625a1e635857895b74581c37201500",
+ "result": "PROVEN",
+ "registrationsBefore": 399,
+ "registrationsAfter": 398,
+ "declarationsBefore": 249,
+ "declarationsAfter": 248,
+ "retiredCase": "heartbeat orphaned process recovery re-enqueues continuation for stranded in-progress work with no active run",
+ "keeper": "heartbeat orphaned process recovery re-enqueues stranded in-progress work after a failed/process_lost run before escalating",
+ "deletedTestLOC": 31,
+ "movedAssertionLOC": 5,
+ "netTestLOCDelta": -26,
+ "candidateTestSHA256": "645fd53b52e513e97c83f1e074dc1bded0749be18fb086138d154c19ea18d1d6",
+ "parentSourceHashes": {
+ "server/src/__tests__/heartbeat-process-recovery.test.ts": "bf7c888eaba44a56d12823bb948026d5db44f2249a75a6cf083343edbd7c0673",
+ "server/src/services/heartbeat/retries.ts": "9f29feb6d986f281cbfd849e0152f49cf56e18974c6df1bc801010dcce4c877a",
+ "server/src/services/heartbeat.ts": "9ac217e97e5319c850924f083df32be43df11f33f8970ff8efb4a5f776a0c1d9",
+ "server/src/services/recovery/service.ts": "c414df876e7df64e701a752513db8e5cf7365589294d50da0e1396788a9207db"
+ },
+ "sourceRestoredBeforeBranchCreation": true,
+ "controls": [
+ {
+ "phase": "baseline",
+ "exitCode": 0,
+ "tests": 399,
+ "passed": 399,
+ "failed": 0,
+ "pending": 0,
+ "failures": [],
+ "ownerSHA256": "9f29feb6d986f281cbfd849e0152f49cf56e18974c6df1bc801010dcce4c877a",
+ "testSHA256": "bf7c888eaba44a56d12823bb948026d5db44f2249a75a6cf083343edbd7c0673"
+ },
+ {
+ "phase": "candidate",
+ "exitCode": 0,
+ "tests": 398,
+ "passed": 398,
+ "failed": 0,
+ "pending": 0,
+ "failures": [],
+ "ownerSHA256": "9f29feb6d986f281cbfd849e0152f49cf56e18974c6df1bc801010dcce4c877a",
+ "testSHA256": "645fd53b52e513e97c83f1e074dc1bded0749be18fb086138d154c19ea18d1d6"
+ },
+ {
+ "phase": "revert",
+ "exitCode": 0,
+ "tests": 399,
+ "passed": 399,
+ "failed": 0,
+ "pending": 0,
+ "failures": [],
+ "ownerSHA256": "9f29feb6d986f281cbfd849e0152f49cf56e18974c6df1bc801010dcce4c877a",
+ "testSHA256": "bf7c888eaba44a56d12823bb948026d5db44f2249a75a6cf083343edbd7c0673"
+ },
+ {
+ "phase": "mutation-keeper",
+ "exitCode": 1,
+ "tests": 398,
+ "passed": 0,
+ "failed": 1,
+ "pending": 397,
+ "failures": [
+ {
+ "name": "heartbeat orphaned process recovery re-enqueues stranded in-progress work after a failed/process_lost run before escalating",
+ "messages": [
+ "AssertionError: expected { \u2026(78) } to match object { status: 'scheduled_retry', \u2026(1) }\n(84 matching properties omitted from actual)\n at Proxy.<anonymous> (file://<dependency-checkout>/node_modules/.pnpm/[email protected]_@[email protected]_@[email protected][email protected]_@[email protected]__v_wr5mxfuu6kgge6st623q4v5egm/node_modules/vitest/dist/chunks/index.C4tKZ0yW.js:2050:10)\n at Proxy.<anonymous> (file://<dependency-checkout>/node_modules/.pnpm/[email protected]_@[email protected]_@[email protected][email protected]_@[email protected]__v_wr5mxfuu6kgge6st623q4v5egm/node_modules/vitest/dist/chunks/index.C4tKZ0yW.js:1944:14)\n at Proxy.methodWrapper (file://<dependency-checkout>/node_modules/.pnpm/[email protected]/node_modules/chai/index.js:1737:25)\n at server/src/__tests__/heartbeat-process-recovery.test.ts:10386:24\n at processTicksAndRejections (node:internal/process/task_queues:104:5)\n at file://<dependency-checkout>/node_modules/.pnpm/[email protected]_@[email protected]_@[email protected][email protected]_@[email protected]__v_wr5mxfuu6kgge6st623q4v5egm/node_modules/vitest/dist/chunks/run.BTlFXlnv.js:2783:20"
+ ]
+ }
+ ],
+ "ownerSHA256": "cff836de7a98e47e845ad7f7393e4c98fcfea3fecb0ba5bdb9fcc92fc9673c84",
+ "testSHA256": "645fd53b52e513e97c83f1e074dc1bded0749be18fb086138d154c19ea18d1d6"
+ },
+ {
+ "phase": "restoration",
+ "exitCode": 0,
+ "tests": 398,
+ "passed": 398,
+ "failed": 0,
+ "pending": 0,
+ "failures": [],
+ "ownerSHA256": "9f29feb6d986f281cbfd849e0152f49cf56e18974c6df1bc801010dcce4c877a",
+ "testSHA256": "645fd53b52e513e97c83f1e074dc1bded0749be18fb086138d154c19ea18d1d6"
+ }
+ ],
+ "temporaryMutation": {
+ "owner": "server/src/services/heartbeat/retries.ts",
+ "line": 1049,
+ "before": "scheduledRetryAttempt: schedule.attempt",
+ "after": "scheduledRetryAttempt: schedule.attempt + 1",
+ "keeperAssertionLine": 10386,
+ "selection": "Only exact failed/process_lost keeper selected; all other cases are unselected.",
+ "expectedAttempt": 1,
+ "observedAttempt": 2
+ }
+}
diff --git a/doc/plans/2026-10-12-test-audit-r16-heartbeat-recovery.md b/doc/plans/2026-10-12-test-audit-r16-heartbeat-recovery.md
new file mode 100644
index 0000000000..5174228c42
--- /dev/null
+++ b/doc/plans/2026-10-12-test-audit-r16-heartbeat-recovery.md
@@ -0,0 +1,84 @@
+# Heartbeat recovery test audit — 2026-10-12
+
+Parent: #16086 (`codex/test-audit-r15-environment-runtime`), exact head
+`c0414ac02e625a1e635857895b74581c37201500`.
+
+## Change and keeper
+
+The standalone `re-enqueues continuation for stranded in-progress work with no active run`
+case at `heartbeat-process-recovery.test.ts:11192–11221` repeats the existing
+`re-enqueues stranded in-progress work after a failed/process_lost run before escalating`
+table row at 10360.
+
+Both call `seedStrandedIssueFixture` with `status: in_progress` and
+`runStatus: failed`. The standalone test omits `runErrorCode`; the helper at 1132–1136
+persists `process_lost` as its default. The table supplies that same value.
+Every other supplied argument, default, and owner action matches. Generated
+record IDs vary between independent fixture runs.
+
+Move five unique assertion lines into the keeper: a truthy retry ID and
+`{ status: scheduled_retry, scheduledRetryAttempt: 1 }`. Existing keeper checks
+retain zero dispatch, one continuation, no escalation, exact issue IDs, two
+agent runs, `transient_failure`, and absent `modelProfile`. The keeper also
+checks source task/run binding, no recovery children, and eventual settlement.
+The failed/adapter_failed and timed_out/adapter_timed_out rows remain and gain
+the durable retry assertions.
+
+One test file: +5/−31 lines, net 26 fewer test lines. One registered execution
+is retired (399→398). This is execution retirement, with no production change.
+Crash/restart, ownership, cancellation, persistence, retry-budget, and failure
+isolation contracts remain. No CI time saving is claimed from local samples.
+
+## Exact-parent sequential proof
+
+| Phase | Passed | Failed | Filtered/pending |
+|---|---|---|---|
+| Full baseline | 399 | 0 | 0 |
+| Full candidate | 398 | 0 | 0 |
+| Full revert | 399 | 0 | 0 |
+| Temporary production mutation, exact keeper | 0 | 1 | 397 |
+| Owner restored, full candidate | 398 | 0 | 0 |
+
+The mutation changes the real scheduler write at `heartbeat/retries.ts:1049`
+from `schedule.attempt` to `schedule.attempt + 1`. The exact keeper fails at
+the moved retry-state assertion, candidate line 10386, expecting attempt 1 and receiving 2. Only that keeper is
+selected; other cases are filtered. The persisted attempt feeds durable
+failure-retry accounting in `execution-recovery-attempt.ts:40–49`.
+The recovery/scheduler owner is not mocked. The mutation is absent from this PR.
+After restoring the owner, all 398 candidate cases pass. Both source files were
+then restored byte-for-byte and the checkout was clean before branch creation.
+The final patch is the exact candidate bytes that passed restoration.
+
+Runtime-title comparison removes only the standalone registration and adds
+no cases. The original full-file ledger contains 249 declarations / 399 runtime
+cases; the candidate has 248 / 398. All declarations were reviewed. Near-overlaps
+with different queued wakes, timer claims, continuation ancestry, or crash points
+were retained.
+
+```sh
+PAPERCLIP_HOME=/private/tmp/r16-hbr/h \
+PAPERCLIP_CONFIG=/private/tmp/r16-hbr/h/config.json \
+PAPERCLIP_INSTANCE_ID=heartbeat-recovery-audit TMPDIR=/private/tmp/r16-hbr/t NODE_ENV=test \
+node node_modules/vitest/vitest.mjs run --project @paperclipai/server \
+server/src/__tests__/heartbeat-process-recovery.test.ts --exclude '**/dist/**'
+```
+
+Full repository typecheck/build/test preflight is delegated to isolated CI,
+as in the parent audit. The local worktree uses shared external dependency
+symlinks, and preflight rewrites workspace links. Every CI gate must pass
+before a green handoff. The full owner uses the checked-in server configuration
+and its real embedded Postgres.
+
+## Hashes and receipts
+
+- Parent test SHA-256: `bf7c888eaba44a56d12823bb948026d5db44f2249a75a6cf083343edbd7c0673`.
+- Candidate test SHA-256: `645fd53b52e513e97c83f1e074dc1bded0749be18fb086138d154c19ea18d1d6`.
+- Restored production retry owner SHA-256: `9f29feb6d986f281cbfd849e0152f49cf56e18974c6df1bc801010dcce4c877a`.
+
+Machine receipts are in `2026-10-12-test-audit-r16-heartbeat-recovery-controls.json`.
+Full JSON/log output, patch, exact keeper map, and whole-file ledger are in
+`/private/tmp/r16-heartbeat-recovery/`. The earlier read-only discovery is in
+`/private/tmp/heartbeat-recovery-full-audit/`. This parent replay uses fresh
+sequential controls; it does not reuse the earlier parent proof.
+
+Model: OpenAI `gpt-6.1-sol`; the host does not expose its context-window size.
diff --git a/server/src/__tests__/heartbeat-process-recovery.test.ts b/server/src/__tests__/heartbeat-process-recovery.test.ts
index 66f16134ef..d719c36421 100644
--- a/server/src/__tests__/heartbeat-process-recovery.test.ts
+++ b/server/src/__tests__/heartbeat-process-recovery.test.ts
@@ -10648,6 +10648,11 @@ describeEmbeddedPostgres("heartbeat orphaned process recovery", () => {
expect(runs).toHaveLength(2);
const retryRun = runs.find((row) => row.id !== runId);
+ expect(retryRun?.id).toBeTruthy();
+ expect(retryRun).toMatchObject({
+ status: "scheduled_retry",
+ scheduledRetryAttempt: 1,
+ });
expect(
retryRun?.contextSnapshot as Record<string, unknown> | undefined,
).toMatchObject({
@@ -11455,37 +11460,6 @@ describeEmbeddedPostgres("heartbeat orphaned process recovery", () => {
}
});
- it("re-enqueues continuation for stranded in-progress work with no active run", async () => {
- const { companyId, agentId, issueId, runId } =
- await seedStrandedIssueFixture({
- status: "in_progress",
- runStatus: "failed",
- });
- const heartbeat = heartbeatService(db);
-
- const result = await heartbeat.reconcileStrandedAssignedIssues();
- expect(result.dispatchRequeued).toBe(0);
- expect(result.continuationRequeued).toBe(1);
- expect(result.escalated).toBe(0);
- expect(result.issueIds).toEqual([issueId]);
-
- const runs = await selectRecoveryRunsByAgent(agentId);
- expect(runs).toHaveLength(2);
-
- const retryRun = runs.find((row) => row.id !== runId);
- expect(retryRun?.id).toBeTruthy();
- expect(
- (retryRun?.contextSnapshot as Record<string, unknown>)?.retryReason,
- ).toBe("transient_failure");
- expect(
- retryRun?.contextSnapshot as Record<string, unknown>,
- ).not.toHaveProperty("modelProfile");
- expect(retryRun).toMatchObject({
- status: "scheduled_retry",
- scheduledRetryAttempt: 1,
- });
- });
-
it("does not run generic continuation recovery for a paused unfinished session goal", async () => {
const { companyId, agentId, issueId, runId } =
await seedStrandedIssueFixture({