Skip to content

Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS

fix: recover retirement review for unpublished local tips (#13335)

picodot5713f700c15e
diff --git a/docs/features/branch-reconciliation.md b/docs/features/branch-reconciliation.md
index 9122f52716..97515fdba2 100644
--- a/docs/features/branch-reconciliation.md
+++ b/docs/features/branch-reconciliation.md
@@ -23,6 +23,30 @@ checkout on their own. Long-lived protected branches and the primary checkout
remain governed by the existing reconciliation policy; an empty reported work
list is not permission to remove them.
+## A parked sweep is not necessarily a clean repository
+
+`no coordinator task eligible; unresolved holds retained` means the inventory
+still contains work, but none can currently be dispatched. Inspect the per-app
+outcome rather than treating the absence of a generated task as successful cleanup.
+
+- `retirement-forge-unavailable` means PR evidence could not be read. Restore
+ the managed app's forge access and rerun. A missing local commit on the forge
+ is handled separately: reconciliation tries bounded merged-branch history,
+ requires exactly one matching-repository candidate, and still performs the
+ fresh checkout/policy/content review before any retirement. A branch-name
+ match alone never establishes that local work was integrated.
+- `retirement-pr-lookup-unavailable` and `retirement-review-unavailable`
+ distinguish an unexpected lookup failure from a failed checkout preview.
+- `retirement-receipt-unverified` means preservation is unproven. An exception
+ after retirement starts retains this obligation, including the exact checkout
+ identity; it is not downgraded to a failed preview or silently retried as one.
+- `worktree-unmanaged-location` requires a scoped release of the inventoried
+ checkout using the action below. Do not add all external checkouts to a trusted
+ root merely because their commits are merged: another tool may still own them.
+- `delivery-head-unverified` requires comparing the current local branch and
+ published PR. Preserve local-only changes and establish the current owner
+ before continuing delivery; do not reset the checkout to make the heads match.
+
## Retiring a held checkout
A row with a checkout path and HEAD offers **Archive and retire checkout**. The
diff --git a/server/services/branchReconcileRetirement.js b/server/services/branchReconcileRetirement.js
index 6083e4d413..e5770bca8f 100644
--- a/server/services/branchReconcileRetirement.js
+++ b/server/services/branchReconcileRetirement.js
@@ -38,8 +38,10 @@ export async function prepareReconcileRetirements(app, result, deps = {}) {
else held.push({ ...release, reason: outcome.reason || outcome.branchRetainedReason || 'retirement-receipt-unverified' });
continue;
}
+ let failureReason = 'retirement-pr-lookup-unavailable';
try {
const prUrl = await readPr(app, branch.branch, release.head);
+ failureReason = 'retirement-review-unavailable';
const prepared = await prepare({ appId: app.id, release, prUrl });
if (!prepared.ok) { held.push({ ...release, reason: prepared.reason }); continue; }
// The preview's policy gate requires completed successful legacy review
@@ -55,6 +57,9 @@ export async function prepareReconcileRetirements(app, result, deps = {}) {
&& prepared.review.comparison?.disposition === 'integrated'
&& prepared.review.comparison.difference === '') {
const retire = deps.retire || (await import('./heldCheckoutRecovery.js')).retireInventoriedCheckout;
+ // The service may have archived/removed the checkout before rejecting.
+ // Preserve an exact receipt obligation, not a retryable preview failure.
+ failureReason = 'retirement-receipt-unverified';
const result = await retire({ appId: app.id, release,
review: { prUrl, digest: prepared.review.digest, acceptSuperseded: false } }, { beforeMutation: assertAutomaticRetirementAllowed });
if (result.ok && result.archiveDir && result.branchDeleted === true) retired.push(release);
@@ -64,8 +69,8 @@ export async function prepareReconcileRetirements(app, result, deps = {}) {
candidates.push({ ...branch, state: 'REVIEW_RETIREMENT', retirement: { release, prUrl, digest: prepared.review.digest,
cognition: retirementCognitiveIdentity(prepared.review, release, prUrl) } });
} catch (error) {
- const reason = ['retirement-pr-ambiguous', 'retirement-forge-unsupported', 'daemon-stopped', 'automatic-retirement-disabled'].includes(error.message)
- ? error.message : 'retirement-review-unavailable';
+ const reason = ['retirement-pr-ambiguous', 'retirement-forge-unsupported', 'retirement-forge-unavailable', 'daemon-stopped', 'automatic-retirement-disabled'].includes(error.message)
+ ? error.message : failureReason;
held.push({ ...release, reason });
}
}
diff --git a/server/services/branchReconcileRetirement.test.js b/server/services/branchReconcileRetirement.test.js
index 2346b61953..c9434e27e7 100644
--- a/server/services/branchReconcileRetirement.test.js
+++ b/server/services/branchReconcileRetirement.test.js
@@ -1,4 +1,4 @@
-import { describe, expect, it, vi } from 'vitest';
+import { beforeEach, describe, expect, it, vi } from 'vitest';
import { prepareReconcileRetirements, retirementReviewInstructions, retirementCognitiveIdentity } from './branchReconcileRetirement.js';
const forge = vi.hoisted(() => ({ read: vi.fn() }));
@@ -18,6 +18,7 @@ const setup = () => ({
});
describe('historical completed-checkout review obligations', () => {
+ beforeEach(() => forge.read.mockReset());
it('archives and retires a checkout whose commits are all on the default branch without a PR read or dispatch', async () => {
const deps = setup();
deps.retire = vi.fn(async () => ({ ok: true, branchDeleted: true, archiveDir: '/private/verified-archive' }));
@@ -149,11 +150,84 @@ describe('historical completed-checkout review obligations', () => {
it('does not invent approval after forge failure or vanished historical policy', async () => {
const deps = setup(); deps.readPr.mockRejectedValueOnce(new Error('unreadable'));
- expect((await prepareReconcileRetirements(app, scan(), deps)).held[0].reason).toBe('retirement-review-unavailable');
+ expect((await prepareReconcileRetirements(app, scan(), deps)).held[0].reason).toBe('retirement-pr-lookup-unavailable');
deps.prepare.mockResolvedValue({ ok: true, review: { policy: { policies: [] } } });
expect((await prepareReconcileRetirements(app, scan(), deps)).held[0].reason).toBe('retirement-policy-changed');
});
+ it.each([404, 422])('recovers historical review when the local tip is not published on the forge (HTTP %s)', async status => {
+ const deps = setup();
+ forge.read.mockRejectedValueOnce(Object.assign(new Error('synthetic missing commit'), {
+ ghStderr: `gh: No commit found for SHA: ${head} (HTTP ${status})`, ghExitCode: 1,
+ })).mockResolvedValueOnce(JSON.stringify([{ url: 'https://github.com/example/repo/pull/42' }]));
+ const result = await prepareReconcileRetirements(app, scan(), { prepare: deps.prepare });
+ expect(result.candidates).toHaveLength(1);
+ expect(deps.prepare).toHaveBeenCalledWith(expect.objectContaining({ prUrl: 'https://github.com/example/repo/pull/42', release: expect.objectContaining({ head }) }));
+ expect(forge.read).toHaveBeenLastCalledWith(['pr', 'list', '--repo', 'github.com/example/repo', '--head', branch.branch,
+ '--state', 'merged', '--limit', '2', '--json', 'url'], undefined, expect.any(Object));
+ });
+
+ it.each([
+ { history: [] },
+ { history: [{ url: 'https://github.com/example/repo/pull/41' }, { url: 'https://github.com/example/repo/pull/42' }] },
+ { history: [{ url: 'https://github.com/other/repo/pull/42' }] },
+ ])('does not turn missing exact-head evidence into historical approval: %j', async ({ history }) => {
+ const deps = setup();
+ forge.read.mockRejectedValueOnce(Object.assign(new Error('missing'), { ghStderr: 'gh: Not Found (HTTP 404)' }))
+ .mockResolvedValueOnce(JSON.stringify(history));
+ expect(await prepareReconcileRetirements(app, scan(), { prepare: deps.prepare })).toMatchObject({
+ candidates: [], retired: [], held: [{ reason: 'retirement-pr-ambiguous' }],
+ });
+ expect(deps.prepare).not.toHaveBeenCalled();
+ });
+
+ it('retains a historical candidate when fresh preview refuses it', async () => {
+ const deps = setup();
+ forge.read.mockRejectedValueOnce(Object.assign(new Error('missing'), { ghStderr: 'gh: Not Found (HTTP 404)' }))
+ .mockResolvedValueOnce(JSON.stringify([{ url: 'https://github.com/example/repo/pull/42' }]));
+ deps.prepare.mockResolvedValue({ ok: false, reason: 'claim-owner-active' });
+ expect(await prepareReconcileRetirements(app, scan(), { prepare: deps.prepare })).toMatchObject({
+ candidates: [], retired: [], held: [{ reason: 'claim-owner-active' }],
+ });
+ });
+
+ it('distinguishes a failed preview from uncertain retirement without leaking raw errors', async () => {
+ const deps = setup();
+ deps.prepare.mockRejectedValueOnce(new Error('private diagnostic'));
+ expect((await prepareReconcileRetirements(app, scan(), deps)).held[0].reason).toBe('retirement-review-unavailable');
+ const prepared = await deps.prepare();
+ prepared.review.comparison = { disposition: 'integrated', localPatch: '', difference: '' };
+ deps.prepare.mockResolvedValue(prepared);
+ deps.retire = vi.fn(async () => { throw new Error('private diagnostic after possible removal'); });
+ expect(await prepareReconcileRetirements(app, scan(), deps)).toMatchObject({
+ candidates: [], retired: [], held: [{ reason: 'retirement-receipt-unverified' }],
+ });
+ expect(deps.retire).toHaveBeenCalledOnce();
+ });
+
+ it('keeps a failed historical fallback held rather than inferring a merged PR', async () => {
+ const deps = setup();
+ forge.read.mockRejectedValueOnce(Object.assign(new Error('gh: Not Found (HTTP 404)'), { ghStderr: 'gh: Not Found (HTTP 404)' }))
+ .mockRejectedValueOnce(new Error('synthetic unavailable'));
+ expect(await prepareReconcileRetirements(app, scan(), { prepare: deps.prepare })).toMatchObject({
+ candidates: [], retired: [], held: [{ reason: 'retirement-forge-unavailable' }],
+ });
+ expect(deps.prepare).not.toHaveBeenCalled();
+ });
+
+ it.each(['gh: rate limit exceeded (HTTP 403)', 'gh: authentication required (HTTP 401)', 'operation timed out',
+ 'gh: Validation Failed (HTTP 422)', `gh: No commit found for SHA: ${'b'.repeat(40)} (HTTP 422)`])(
+ 'does not replace an unreadable exact-head lookup with branch history: %s', async message => {
+ const deps = setup();
+ forge.read.mockRejectedValueOnce(Object.assign(new Error(message), { ghStderr: message }));
+ const calls = forge.read.mock.calls.length;
+ expect(await prepareReconcileRetirements(app, scan(), { prepare: deps.prepare })).toMatchObject({
+ candidates: [], retired: [], held: [{ reason: 'retirement-forge-unavailable' }],
+ });
+ expect(forge.read.mock.calls.length).toBe(calls + 1);
+ expect(deps.prepare).not.toHaveBeenCalled();
+ });
+
it('selects the unique exact head on a reused branch, then requires its fresh preview', async () => {
const deps = setup();
const row = { html_url: 'https://github.com/example/repo/pull/42', head: { ref: branch.branch, sha: head, repo: { full_name: 'example/repo' } },
diff --git a/server/services/cosTaskPreStepBlocks.compose.test.js b/server/services/cosTaskPreStepBlocks.compose.test.js
index e4f59ef12f..ae1d1313f5 100644
--- a/server/services/cosTaskPreStepBlocks.compose.test.js
+++ b/server/services/cosTaskPreStepBlocks.compose.test.js
@@ -220,6 +220,8 @@ describe('prepareManagedAppImprovementTask composes the extracted pre-step layer
for (let i = 0; i < 70; i++) expect(await generate()).toBeNull();
expect(recordPerpetualDispatch).not.toHaveBeenCalled();
const parkedMessage = emitLog.mock.calls.findLast(([, message]) => message.includes('branch-reconcile parked'))?.[1];
+ expect(parkedMessage).toContain('no coordinator task eligible; unresolved holds retained');
+ expect(parkedMessage).not.toContain('nothing actionable');
expect(parkedMessage).toContain('delivery branch(es) awaiting completion (delivery-ci-or-review-pending)');
expect(parkedMessage).not.toContain('skipped by disabled action toggles');
expect(recordBranchReconcileOutcome).toHaveBeenLastCalledWith(APP.id, expect.objectContaining({ remaining: expect.arrayContaining([
diff --git a/server/services/cosTaskPreStepBlocks.js b/server/services/cosTaskPreStepBlocks.js
index 4403a8c04b..5e618d3154 100644
--- a/server/services/cosTaskPreStepBlocks.js
+++ b/server/services/cosTaskPreStepBlocks.js
@@ -618,7 +618,8 @@ export async function resolveBranchReconcileBlock(app, taskType, metadata, taskS
// the abandoned-worktree case).
const gatedSuffix = countSuffix(cognitiveInFlight.filter(branch => !allActionable.includes(branch)), 'in-flight branch(es) skipped by disabled action toggles', (b) => b.state);
const deliverySuffix = countSuffix(routineDeliveryHolds, 'delivery branch(es) awaiting completion', branch => branch.reason);
- emitLog('info', `🔀 branch-reconcile parked for ${app.name}: nothing actionable (cleaned ${result.cleaned.length}${heldSuffix}${gatedSuffix}${deliverySuffix}${heldLiveSuffix}${supersededSuffix}${detachedSuffix})`, { appId: app.id });
+ const disposition = dispatchedOutcome.remaining?.length ? 'no coordinator task eligible; unresolved holds retained' : 'no remaining branch work';
+ emitLog('info', `🔀 branch-reconcile parked for ${app.name}: ${disposition} (cleaned ${result.cleaned.length}${heldSuffix}${gatedSuffix}${deliverySuffix}${heldLiveSuffix}${supersededSuffix}${detachedSuffix})`, { appId: app.id });
return { skip: true };
}
// Retain attempted revisions across caps/manual retries. A blocked prefix must
diff --git a/server/services/heldCheckoutReview.js b/server/services/heldCheckoutReview.js
index 3e74d4857f..3492d4e6dc 100644
--- a/server/services/heldCheckoutReview.js
+++ b/server/services/heldCheckoutReview.js
@@ -25,8 +25,18 @@ export async function findMergedPullRequest(repoPath, branch, tip, { forgeAccoun
if (!oid(tip)) throw new Error('retirement-pr-ambiguous');
// Query the exact commit, not an unbounded reused-branch history. A capped
// association result cannot establish uniqueness and stays held.
- const rows = JSON.parse(await execGh(['api', '--hostname', origin.host,
- `repos/${origin.fullName}/commits/${tip}/pulls?per_page=100`], undefined, options));
+ const association = await execGh(['api', '--hostname', origin.host,
+ `repos/${origin.fullName}/commits/${tip}/pulls?per_page=100`], undefined, options)
+ .catch(error => {
+ // An unpublished/rebased local tip may not exist on the forge. That
+ // endpoint's missing-commit response does not establish empty history: permit only the
+ // bounded historical lookup below, which must independently succeed.
+ const detail = error.ghStderr || '';
+ if (/\(HTTP 404\)/.test(detail)
+ || (detail.includes(`No commit found for SHA: ${tip}`) && /\(HTTP 422\)/.test(detail))) return null;
+ throw new Error('retirement-forge-unavailable');
+ });
+ const rows = association === null ? [] : JSON.parse(association);
if (!Array.isArray(rows) || rows.length >= 100
|| new Set(rows.map(row => row?.html_url)).size !== rows.length
|| rows.some(row => {
@@ -41,9 +51,11 @@ export async function findMergedPullRequest(repoPath, branch, tip, { forgeAccoun
if (matches.length === 1) return { url: matches[0].html_url, mergedAt: matches[0].merged_at };
// Preserve historical review of an older local tip only when branch history
// itself has exactly one candidate. Never choose among mismatching heads.
- const historical = JSON.parse(await execGh(['pr', 'list', '--repo', spec, '--head', branch, '--state', 'merged',
- '--limit', '2', '--json', 'url'], undefined, options));
- if (!Array.isArray(historical) || historical.length !== 1 || typeof historical[0]?.url !== 'string') throw new Error('retirement-pr-ambiguous');
+ const historical = await execGh(['pr', 'list', '--repo', spec, '--head', branch, '--state', 'merged',
+ '--limit', '2', '--json', 'url'], undefined, options).then(JSON.parse)
+ .catch(() => { throw new Error('retirement-forge-unavailable'); });
+ const selected = Array.isArray(historical) && historical.length === 1 && parsePullRequestUrl(historical[0]?.url);
+ if (!selected || selected.host !== origin.host || `${selected.owner}/${selected.repo}` !== origin.fullName) throw new Error('retirement-pr-ambiguous');
return { url: historical[0].url, mergedAt: null };
}
diff --git a/server/services/pm2.import.test.js b/server/services/pm2.import.test.js
new file mode 100644
index 0000000000..2bbb1311dd
--- /dev/null
+++ b/server/services/pm2.import.test.js
@@ -0,0 +1,10 @@
+import { expect, it, vi } from 'vitest';
+
+vi.mock('pm2', () => { throw new Error('Native PM2 client loaded before an RPC operation'); });
+
+it('loads the service without constructing the socket-probing native PM2 singleton', async () => {
+ const service = await import('./pm2.js');
+ expect(service.getDaemonIdentity).toBeTypeOf('function');
+ // The RPC boundary still loads the native client; it has not been disabled.
+ await expect(service.stopApp('synthetic-process')).rejects.toThrow();
+});
diff --git a/server/services/pm2.js b/server/services/pm2.js
index cdca02763c..95168a8d88 100644
--- a/server/services/pm2.js
+++ b/server/services/pm2.js
@@ -1,4 +1,3 @@
-import pm2 from 'pm2';
import { spawn } from '../lib/childProcess.js';
import { existsSync } from 'fs';
import { writeFile, unlink } from 'fs/promises';
@@ -281,7 +280,10 @@ async function spawnPm2Cli(action, name, pm2Home) {
}
/** Each operation owns its native client, including deadline teardown. */
-function connectAndRun(action, timeoutMs = COMMAND_TIMEOUT_MS) {
+async function connectAndRun(action, timeoutMs = COMMAND_TIMEOUT_MS) {
+ // Importing PM2 constructs its singleton and probes the interactor socket.
+ // Merely loading a reconciliation/peer-sync dependency must not contact it.
+ const { default: pm2 } = await import('pm2');
return new Promise((resolve, reject) => {
const client = new pm2.custom();
let settled = false;