Skip to content

Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS

fix: retain preservation obligations for vanished checkouts (#13271)

Adam Eivy47407168a51c
diff --git a/docs/TROUBLESHOOTING.md b/docs/TROUBLESHOOTING.md
index 9ba6a105dc..54a7ef3aa4 100644
--- a/docs/TROUBLESHOOTING.md
+++ b/docs/TROUBLESHOOTING.md
@@ -644,6 +644,26 @@ max_memory_restart: '500M'
## Development Issues
+### A merged checkout disappeared without an archive receipt
+
+`retirement-provenance-unverified` means a previously inventoried checkout or
+local ref is missing and its preservation has not been established. It is an
+unresolved obligation, not successful cleanup. The retirement API refuses this
+case, and the Schedule view retains the exact observed branch, path and head
+through later empty scans. A surviving branch, merged PR or integrated commit
+does not prove that ignored files, untracked work or private Git state survived.
+
+Trace the deletion actor and command separately from the PR's merge actor.
+Inspect the original `data/cos/held-checkout-archives/` and
+`data/cos/branch-retirement-archives/` manifests and retained execution evidence.
+If an original archive exists, verify its complete contents and exact identity
+through the supported recovery service before recording a retirement receipt.
+Preserve recoverable objects and original archives. Do not reconstruct a partial
+checkout as proof, delete another checkout, or clear the obligation merely
+because the source is gone. If provenance cannot be recovered, retain the
+evidence and missing source identity in the owning issue; #13271 records such
+an unresolved case without attributing its local deletion to the merge actor.
+
### "Another git process seems to be running" — but none is
**Symptom**: every git command against one repo (or, most often, every submodule
diff --git a/server/lib/reconcileRetirementEvidence.js b/server/lib/reconcileRetirementEvidence.js
index ec979a1b5b..d720fda0b4 100644
--- a/server/lib/reconcileRetirementEvidence.js
+++ b/server/lib/reconcileRetirementEvidence.js
@@ -1,6 +1,9 @@
/** Exact checkout obligations survive scans until fresh service preservation evidence. */
export const sameRetirementCheckout = (left, right) => left.branch === right.branch && left.path === right.path && left.head === right.head;
+export const isRetirementObligation = row => ['retirement-review-pending', 'retirement-receipt-unverified',
+ 'retirement-provenance-unverified'].includes(row?.reason);
+
export const hasRetirementReceipt = (outcome, release) => Number.isFinite(Date.parse(release.createdAt))
&& (outcome?.retired?.some(row => sameRetirementCheckout(row, release) && row.branchDeleted === true
&& typeof row.archiveDir === 'string' && row.archiveDir.length > 0
@@ -16,9 +19,13 @@ export function unresolvedRetirements(outcome, additional = []) {
const rows = [...(outcome?.retirementObligations || []), ...(outcome?.remaining || []), ...additional];
const obligations = [];
for (const row of rows) {
- if (!['retirement-review-pending', 'retirement-receipt-unverified'].includes(row.reason)
- || hasRetirementReceipt(outcome, row) || obligations.some(previous => sameRetirementCheckout(previous, row))) continue;
- obligations.push({ ...row, reason: 'retirement-receipt-unverified' });
+ if (!isRetirementObligation(row) || hasRetirementReceipt(outcome, row)) continue;
+ const previous = obligations.find(entry => sameRetirementCheckout(entry, row));
+ if (previous) {
+ if (row.reason === 'retirement-provenance-unverified') previous.reason = row.reason;
+ continue;
+ }
+ obligations.push({ ...row, reason: row.reason === 'retirement-provenance-unverified' ? row.reason : 'retirement-receipt-unverified' });
}
return obligations;
}
diff --git a/server/services/branchReconcileInventoryPrune.js b/server/services/branchReconcileInventoryPrune.js
index aed244c5a3..1a1173f288 100644
--- a/server/services/branchReconcileInventoryPrune.js
+++ b/server/services/branchReconcileInventoryPrune.js
@@ -1,15 +1,14 @@
/**
* Branch-reconcile persists its last inventory (`remaining`) so the Schedule view
- * can offer per-row actions. Anything else that retires a checkout or branch
- * (a claim run's own cleanup, `git worktree prune`, a later sweep) leaves that
- * stored row behind, and the UI then offers an action on something that no
- * longer exists. This decides which stored rows are such ghosts.
+ * can offer per-row actions. Disappearance establishes missing source, never
+ * preservation. Callers must retain it as an obligation without an exact receipt.
*/
import { execGit } from '../lib/execGit.js';
-import { pathExists } from '../lib/fileCore.js';
+import { readCheckoutPathPresence, readLocalBranchPresence } from './repoCleanupObservation.js';
// Rows that bind a retirement-receipt obligation are evidence, not display data.
-export const isObligationRow = row => ['retirement-review-pending', 'retirement-receipt-unverified'].includes(row?.reason);
+export const isObligationRow = row => ['retirement-review-pending', 'retirement-receipt-unverified',
+ 'retirement-provenance-unverified'].includes(row?.reason);
async function registeredPaths(repoPath) {
const { stdout } = await execGit(['worktree', 'list', '--porcelain'], repoPath);
@@ -19,8 +18,8 @@ async function registeredPaths(repoPath) {
/**
* Stored inventory rows whose checkout (or, for a branch-only row, local branch)
* is gone from the repository. A row whose branch survives but whose checkout is
- * gone is still a ghost checkout; the next sweep lists the branch afresh.
- * Fails open: any git read error yields no ghosts, so a flaky read hides nothing.
+ * gone still has unproven preservation; the next sweep lists the branch afresh.
+ * Unreadable paths are not absent. Git read errors reject for the caller to retain.
*
* @param {string} repoPath
* @param {object[]} rows
@@ -35,12 +34,11 @@ export async function findGoneInventoryRows(repoPath, rows, { includeObligations
// A remote-only hold has no local ref by definition. Only a new forge scan
// can establish that its remote preservation obligation is gone.
if (row?.reason === 'remote-branch-held') continue;
- if (!includeObligations && isObligationRow(row)) continue;
+ if (!includeObligations && (isObligationRow(row) || row?.reason === 'claim-reservation-owner-unverified')) continue;
if (typeof row?.path === 'string' && row.path) {
- if (!registered.has(row.path) && !await pathExists(row.path)) gone.push(row);
+ if (!registered.has(row.path) && await readCheckoutPathPresence(row.path) === false) gone.push(row);
} else if (typeof row?.branch === 'string' && row.branch) {
- const ref = await execGit(['rev-parse', '--verify', '--quiet', `refs/heads/${row.branch}`], repoPath, { ignoreExitCode: true });
- if (ref.exitCode === 1) gone.push(row);
+ if (await readLocalBranchPresence(repoPath, row.branch) === false) gone.push(row);
}
}
return gone;
diff --git a/server/services/heldCheckoutRecovery.js b/server/services/heldCheckoutRecovery.js
index 554246235d..4dfd964915 100644
--- a/server/services/heldCheckoutRecovery.js
+++ b/server/services/heldCheckoutRecovery.js
@@ -865,13 +865,13 @@ export async function retireInventoriedCheckout({ appId, release, review }, opts
if (!outcome?.remaining?.some?.(entry => sameRelease(entry, release))) return { ok: false, reason: 'release-not-in-inventory' };
const inventoryCurrent = async () => (await taskSchedule.getBranchReconcileOutcome(appId))?.remaining?.some(entry => sameRelease(entry, release)) === true;
const result = await retireHeldCheckout({ repoPath: app.repoPath, release, review }, { ...opts, inventoryCurrent, forgeAccount: app.forgeAccount ?? null });
- // The row was stale: something else already removed the checkout. That is the
- // outcome the operator asked for, so settle the row instead of refusing it.
+ // A missing original cannot be archived retroactively. Retain its exact
+ // identity for provenance recovery rather than equating absence with success.
if (!result.ok && result.reason === 'checkout-not-found') {
const gone = await findGoneInventoryRows(app.repoPath, [release]).catch(() => []);
if (gone.length) {
- await taskSchedule.dropInventoryRows(appId, gone);
- return { ok: true, alreadyRetired: true };
+ await taskSchedule.recordMissingInventoryRows(appId, gone);
+ return { ok: false, reason: 'retirement-provenance-unverified' };
}
}
if (result.ok) {
diff --git a/server/services/heldCheckoutRecovery.test.js b/server/services/heldCheckoutRecovery.test.js
index 608f485bd5..d3bd20b724 100644
--- a/server/services/heldCheckoutRecovery.test.js
+++ b/server/services/heldCheckoutRecovery.test.js
@@ -51,7 +51,7 @@ vi.mock('./taskScheduleStore.js', async (importOriginal) => ({
vi.mock('./apps.js', () => ({ getAppById: async (id) => (injected.app?.id === id ? injected.app : null) }));
const { retireInventoriedCheckout, prepareInventoriedCheckoutRetirement, retirePendingMergeCheckout, heldCheckoutArchiveRoot, preserveMergedCheckout, recoverMissingCheckoutPreservations, findMissingCheckoutPreservation } = await import('./heldCheckoutRecovery.js');
-const { recordBranchReconcileOutcome, getBranchReconcileOutcome } = await import('./taskSchedule.js');
+const { recordBranchReconcileOutcome, getBranchReconcileOutcome, recordHeldCheckoutRetirement, parkPerpetual } = await import('./taskSchedule.js');
const { withClaimOwnershipLock, checkReconcileOwnership } = await import('./cosClaimOwnership.js');
describe.skipIf(SKIP_HEAVY_INTEGRATION)('held checkout recovery (real git)', () => {
@@ -438,17 +438,54 @@ describe.skipIf(SKIP_HEAVY_INTEGRATION)('held checkout recovery (real git)', ()
expect(outcome.retired[0]).toMatchObject({ path: release.path, branch: 'feature/held', branchDeleted: true, archiveDir: result.archiveDir });
});
- it('settles a stale inventory row whose checkout was already removed instead of refusing it', async () => {
- const { path, release } = await heldCheckout('already-gone');
+ it('retains vanished active claim provenance instead of reporting an unarchived checkout retired', async () => {
+ const { path, release } = await heldCheckout('already-gone', { branch: 'claim/issue-42' });
+ const gitDir = (await execGit(['rev-parse', '--absolute-git-dir'], path)).stdout.trim();
+ await writeFile(join(gitDir, 'portos-review-status'), 'private synthetic evidence');
await recordBranchReconcileOutcome('app-1', {
- status: 'incomplete', remaining: [{ ...release, reason: 'worktree-locked' }, { branch: 'other', reason: 'actionable' }],
+ status: 'incomplete', remaining: [{ ...release, reason: 'retirement-review-pending' }, { branch: 'other', reason: 'actionable' }],
});
await execGit(['worktree', 'unlock', path], repo);
await execGit(['worktree', 'remove', '--force', path], repo);
- expect(await retireInventoriedCheckout({ appId: 'app-1', release }, { cosDir, readAgents: noOwners }))
- .toEqual({ ok: true, alreadyRetired: true });
- expect((await getBranchReconcileOutcome('app-1')).remaining).toEqual([{ branch: 'other', reason: 'actionable' }]);
+ const activeOwner = async () => [{ id: 'agent-live', status: 'running', metadata: {
+ sourceWorkspace: repo, claimBranches: [release.branch], claimReleasedBranches: [],
+ } }];
+ expect(await retireInventoriedCheckout({ appId: 'app-1', release }, { cosDir, readAgents: activeOwner }))
+ .toEqual({ ok: false, reason: 'retirement-provenance-unverified' });
+ const obligation = { ...release, reason: 'retirement-provenance-unverified', createdAt: expect.any(String) };
+ expect(await getBranchReconcileOutcome('app-1')).toMatchObject({ status: 'incomplete',
+ retirementObligations: [obligation], remaining: [obligation, { branch: 'other', reason: 'actionable' }],
+ });
+ // Neither ignored bytes nor private Git state can be reconstructed from the merged head.
+ expect(await archives()).toEqual([]);
+ expect((await execGit(['rev-parse', `refs/heads/${release.branch}`], repo)).stdout.trim()).toBe(release.head);
+ await recordBranchReconcileOutcome('app-1', { status: 'running', remaining: [] });
+ await parkPerpetual('branch-reconcile', 'app-1', { reason: 'no-in-flight-branches' });
+ expect(await getBranchReconcileOutcome('app-1')).toMatchObject({ status: 'incomplete', remaining: [obligation] });
+ });
+
+ it('retains a checkout that vanishes between sweeps until its exact preservation receipt arrives', async () => {
+ const { path, release } = await heldCheckout('gone-between-scans', { branch: 'feature/missing' });
+ const marker = { branch: 'claim/issue-99', issue: 99, reason: 'claim-reservation-owner-unverified' };
+ await recordBranchReconcileOutcome('app-1', { status: 'running', remaining: [{ ...release, reason: 'worktree-locked' }, marker] });
+ await execGit(['worktree', 'unlock', path], repo);
+ // Model an external actor; the retirement service never receives this deletion.
+ await execGit(['worktree', 'remove', '--force', path], repo);
+ await recordBranchReconcileOutcome('app-1', { status: 'drained', remaining: [] });
+ const pending = await getBranchReconcileOutcome('app-1');
+ expect(pending).toMatchObject({ status: 'incomplete', reason: 'retirement-provenance-unverified',
+ retirementObligations: [{ ...release, reason: 'retirement-provenance-unverified' }],
+ });
+ expect(pending.remaining).toHaveLength(1);
+ const createdAt = pending.retirementObligations[0].createdAt;
+ // A receipt for another head cannot explain the original disappearance.
+ await recordHeldCheckoutRetirement('app-1', { ...release, head: 'f'.repeat(40), branchDeleted: true, archiveDir: '/example/other-archive' });
+ await recordBranchReconcileOutcome('app-1', { status: 'running', remaining: [] });
+ expect((await getBranchReconcileOutcome('app-1')).retirementObligations[0].createdAt).toBe(createdAt);
+ // Only the trusted service writer can supply recovered preservation evidence.
+ await recordHeldCheckoutRetirement('app-1', { ...release, branchDeleted: true, archiveDir: '/example/recovered-original' });
+ expect(await getBranchReconcileOutcome('app-1')).toMatchObject({ retirementObligations: [], remaining: [] });
});
it('refuses a release that does not name a row of the last inventory', async () => {
diff --git a/server/services/taskSchedule.js b/server/services/taskSchedule.js
index 7c65be8420..95ee4116b6 100644
--- a/server/services/taskSchedule.js
+++ b/server/services/taskSchedule.js
@@ -774,7 +774,12 @@ export async function recordBranchReconcileOutcome(appId, { status, reason, rema
const record = ensureExecutionRecord(schedule, 'branch-reconcile', appId);
const now = new Date().toISOString();
const previous = record.branchReconcileOutcome;
- const retirementObligations = retainRetirementObligations(previous, [...(remaining || []), ...additionalObligations], now);
+ // A fresh empty scan must not erase the last observation of vanished source.
+ const omitted = remaining === undefined ? [] : (previous?.remaining || [])
+ .filter(row => !remaining.some(current => sameRetirementCheckout(current, row)));
+ const missing = await missingInventoryRows(appId, omitted);
+ const retirementObligations = retainRetirementObligations(previous,
+ [...(remaining || []), ...additionalObligations, ...missing.map(missingPreservationObligation)], now);
const inventory = remaining === undefined ? previous?.remaining ?? null : remaining;
const current = inventory?.map(row => {
const obligation = retirementObligations.find(entry => sameRetirementCheckout(entry, row));
@@ -785,6 +790,7 @@ export async function recordBranchReconcileOutcome(appId, { status, reason, rema
...previous, status, reason, nextAction, updatedAt: now, retirementObligations,
remaining: retained.length ? [...(current || []), ...retained] : current,
observedAt: remaining === undefined ? record.branchReconcileOutcome?.observedAt ?? null : now,
+ ...(missing.length && missingPreservationStatus),
};
return { result: record.branchReconcileOutcome, changed: true };
});
@@ -866,33 +872,43 @@ export async function getBranchReconcileOutcome(appId) {
return resolveExecutionRecord(schedule, 'branch-reconcile', appId)?.branchReconcileOutcome ?? null;
}
-/** Remove stored inventory rows (display data only; obligations are untouched). */
-export async function dropInventoryRows(appId, rows) {
+const missingPreservationObligation = row => ({ ...row, reason: 'retirement-provenance-unverified' });
+const missingPreservationStatus = {
+ status: 'incomplete', reason: 'retirement-provenance-unverified',
+ nextAction: 'Trace the missing original checkout/ref and recover its exact archive receipt; disappearance and merged commits do not prove preservation.',
+};
+
+async function missingInventoryRows(appId, rows, options) {
+ if (!appId || !rows.length) return [];
+ const app = await getAppById(appId);
+ if (!app?.repoPath) return [];
+ return findGoneInventoryRows(app.repoPath, rows, options);
+}
+
+/** Missing originals remain evidence until an exact service receipt settles them. */
+export async function recordMissingInventoryRows(appId, rows) {
return updateSchedule(async (schedule) => {
const outcome = ensureExecutionRecord(schedule, 'branch-reconcile', appId).branchReconcileOutcome;
if (!Array.isArray(outcome?.remaining)) return { result: outcome ?? null, changed: false };
- const kept = outcome.remaining.filter(entry => !rows.some(row => sameRetirementCheckout(row, entry)));
- if (kept.length === outcome.remaining.length) return { result: outcome, changed: false };
- outcome.remaining = kept;
+ const missing = outcome.remaining.filter(entry => rows.some(row => sameRetirementCheckout(row, entry)));
+ if (!missing.length) return { result: outcome, changed: false };
+ const now = new Date().toISOString();
+ const obligations = retainRetirementObligations(outcome, missing.map(missingPreservationObligation), now);
+ Object.assign(outcome, missingPreservationStatus, { updatedAt: now, retirementObligations: obligations,
+ remaining: outcome.remaining.map(row => obligations.find(entry => sameRetirementCheckout(entry, row)) || row),
+ });
return { result: outcome, changed: true };
});
}
-/**
- * Drop stored inventory rows whose checkout or branch no longer exists, so the
- * Schedule view never offers an action on something already cleaned up.
- * Returns the outcome as it now stands.
- */
-async function pruneGoneInventory(appId, outcome) {
+/** Reconcile display inventory with missing source without claiming archival success. */
+async function reconcileMissingInventory(appId, outcome) {
if (!appId || !outcome?.remaining?.length) return outcome;
- const app = await getAppById(appId).catch(() => null);
- if (!app?.repoPath) return outcome;
- const gone = await findGoneInventoryRows(app.repoPath, outcome.remaining, { includeObligations: true }).catch(() => []);
+ const gone = await missingInventoryRows(appId, outcome.remaining, { includeObligations: true }).catch(() => []);
if (!gone.length) return outcome;
- const obsolete = gone.filter(row => !isObligationRow(row));
- const current = obsolete.length ? await dropInventoryRows(appId, obsolete).catch(() => null) ?? outcome : outcome;
- // Missing originals do not prove preservation. Keep obligations, but do not
- // present their historical tuple as a checkout that can still be retired.
+ const unrecorded = gone.filter(row => !isObligationRow(row));
+ const current = unrecorded.length ? await recordMissingInventoryRows(appId, unrecorded).catch(() => null) ?? outcome : outcome;
+ // The original checkout is gone; flag it so the UI never offers a retire action on it.
return { ...current, remaining: current.remaining.map(row => ({
...row,
...(row.path && gone.some(missing => sameRetirementCheckout(missing, row)) && { checkoutMissing: true }),
@@ -1805,7 +1821,7 @@ export async function getScheduleStatus() {
.map(([appId, record]) => [appId, record.branchReconcileOutcome]),
];
taskStatus.reconcileOutcomes = Object.fromEntries(
- await mapWithConcurrency(outcomes, 8, async ([appId, outcome]) => [appId, await pruneGoneInventory(appId, outcome)])
+ await mapWithConcurrency(outcomes, 8, async ([appId, outcome]) => [appId, await reconcileMissingInventory(appId, outcome)])
);
}
status.tasks[taskType] = taskStatus;
diff --git a/server/services/taskSchedule.test.js b/server/services/taskSchedule.test.js
index 293228a7a1..ec06133195 100644
--- a/server/services/taskSchedule.test.js
+++ b/server/services/taskSchedule.test.js
@@ -78,6 +78,8 @@ vi.mock('./apps.js', () => ({
vi.mock('./taskInputGate.js', () => ({ inputsUnchangedSince: vi.fn().mockResolvedValue(false), readHead: vi.fn() }))
+vi.mock('./branchReconcileInventoryPrune.js', () => ({ findGoneInventoryRows: vi.fn().mockResolvedValue([]) }))
+
vi.mock('./instanceFeatures.js', () => ({
isInstanceFeatureEnabled: vi.fn().mockResolvedValue(true),
}))
@@ -205,7 +207,8 @@ import { loadState } from './cosState.js'
import { readJSONFile } from '../lib/fileUtils.js'
import { writeFile } from 'fs/promises'
-import { isTaskTypeEnabledForApp, getAppTaskTypeInterval, getAppTaskTypeOverrides, getActiveApps, clearAllPrWatcherState, clearAllIssueWatcherState } from './apps.js'
+import { isTaskTypeEnabledForApp, getAppById, getAppTaskTypeInterval, getAppTaskTypeOverrides, getActiveApps, clearAllPrWatcherState, clearAllIssueWatcherState } from './apps.js'
+import { findGoneInventoryRows } from './branchReconcileInventoryPrune.js'
import { getLocalParts } from '../lib/timezone.js'
import { getAdaptiveCooldownMultiplier } from './taskLearning.js'
import { parseCronToNextRun } from './eventScheduler.js'
@@ -3318,6 +3321,29 @@ describe('taskSchedule', () => {
.toMatchObject({ status: 'incomplete', reason: 'agent-failed', remaining, observedAt });
});
+ it('persists missing checkout and branch evidence when the Schedule view refreshes', async () => {
+ let saved = { version: 2, tasks: { 'branch-reconcile': { enabled: true, type: 'on-demand', perpetual: false } }, executions: {}, templates: [] };
+ readJSONFile.mockImplementation(async () => structuredClone(saved));
+ writeFile.mockImplementation(async (_path, contents) => { saved = JSON.parse(contents); });
+ getAppById.mockResolvedValueOnce({ repoPath: '/example/repo' });
+ const rows = [
+ { branch: 'fix/missing-checkout', path: '/example/checkout', head: 'a'.repeat(40), reason: 'worktree-locked' },
+ { branch: 'fix/missing-ref', head: 'b'.repeat(40), reason: 'branch-held' },
+ ];
+ await recordBranchReconcileOutcome('app-1', { status: 'running', remaining: rows });
+ findGoneInventoryRows.mockResolvedValueOnce(rows);
+ const outcome = (await getScheduleStatus()).tasks['branch-reconcile'].reconcileOutcomes['app-1'];
+ expect(outcome).toMatchObject({ status: 'incomplete', reason: 'retirement-provenance-unverified',
+ retirementObligations: rows.map(row => ({ ...row, reason: 'retirement-provenance-unverified', createdAt: expect.any(String) })),
+ });
+ expect(outcome.retired).toBeUndefined();
+ // Loading the persisted state again and an empty sweep cannot clear the evidence.
+ await recordBranchReconcileOutcome('app-1', { status: 'running', remaining: [] });
+ const reloaded = (await getScheduleStatus()).tasks['branch-reconcile'].reconcileOutcomes['app-1'];
+ expect(reloaded.retirementObligations).toEqual(outcome.retirementObligations);
+ expect(reloaded.remaining).toEqual(outcome.retirementObligations);
+ });
+
it('never loses exact retirement obligations across empty scans, restart reads or unrelated receipts', async () => {
let saved = { version: 2, tasks: { 'branch-reconcile': { enabled: true, type: 'on-demand', perpetual: true } }, executions: {}, templates: [] };
readJSONFile.mockImplementation(async () => structuredClone(saved));