Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS
JavaScript ai-agentsdashboarddeveloper-toolsdigital-twinquantified-selfsecond-brainself-hostedtailscale
fix: retain preservation obligations for vanished checkouts (#13271)
Adam Eivy47407168a51c
diff --git a/docs/TROUBLESHOOTING.md b/docs/TROUBLESHOOTING.mdindex 9ba6a105dc..54a7ef3aa4 100644--- a/docs/TROUBLESHOOTING.md+++ b/docs/TROUBLESHOOTING.md@@ -644,6 +644,26 @@ max_memory_restart: '500M'## Development Issues+### A merged checkout disappeared without an archive receipt++`retirement-provenance-unverified` means a previously inventoried checkout or+local ref is missing and its preservation has not been established. It is an+unresolved obligation, not successful cleanup. The retirement API refuses this+case, and the Schedule view retains the exact observed branch, path and head+through later empty scans. A surviving branch, merged PR or integrated commit+does not prove that ignored files, untracked work or private Git state survived.++Trace the deletion actor and command separately from the PR's merge actor.+Inspect the original `data/cos/held-checkout-archives/` and+`data/cos/branch-retirement-archives/` manifests and retained execution evidence.+If an original archive exists, verify its complete contents and exact identity+through the supported recovery service before recording a retirement receipt.+Preserve recoverable objects and original archives. Do not reconstruct a partial+checkout as proof, delete another checkout, or clear the obligation merely+because the source is gone. If provenance cannot be recovered, retain the+evidence and missing source identity in the owning issue; #13271 records such+an unresolved case without attributing its local deletion to the merge actor.+### "Another git process seems to be running" — but none is**Symptom**: every git command against one repo (or, most often, every submodulediff --git a/server/lib/reconcileRetirementEvidence.js b/server/lib/reconcileRetirementEvidence.jsindex ec979a1b5b..d720fda0b4 100644--- a/server/lib/reconcileRetirementEvidence.js+++ b/server/lib/reconcileRetirementEvidence.js@@ -1,6 +1,9 @@/** Exact checkout obligations survive scans until fresh service preservation evidence. */export const sameRetirementCheckout = (left, right) => left.branch === right.branch && left.path === right.path && left.head === right.head;+export const isRetirementObligation = row => ['retirement-review-pending', 'retirement-receipt-unverified',+ 'retirement-provenance-unverified'].includes(row?.reason);+export const hasRetirementReceipt = (outcome, release) => Number.isFinite(Date.parse(release.createdAt))&& (outcome?.retired?.some(row => sameRetirementCheckout(row, release) && row.branchDeleted === true&& typeof row.archiveDir === 'string' && row.archiveDir.length > 0@@ -16,9 +19,13 @@ export function unresolvedRetirements(outcome, additional = []) {const rows = [...(outcome?.retirementObligations || []), ...(outcome?.remaining || []), ...additional];const obligations = [];for (const row of rows) {- if (!['retirement-review-pending', 'retirement-receipt-unverified'].includes(row.reason)- || hasRetirementReceipt(outcome, row) || obligations.some(previous => sameRetirementCheckout(previous, row))) continue;- obligations.push({ ...row, reason: 'retirement-receipt-unverified' });+ if (!isRetirementObligation(row) || hasRetirementReceipt(outcome, row)) continue;+ const previous = obligations.find(entry => sameRetirementCheckout(entry, row));+ if (previous) {+ if (row.reason === 'retirement-provenance-unverified') previous.reason = row.reason;+ continue;+ }+ obligations.push({ ...row, reason: row.reason === 'retirement-provenance-unverified' ? row.reason : 'retirement-receipt-unverified' });}return obligations;}diff --git a/server/services/branchReconcileInventoryPrune.js b/server/services/branchReconcileInventoryPrune.jsindex aed244c5a3..1a1173f288 100644--- a/server/services/branchReconcileInventoryPrune.js+++ b/server/services/branchReconcileInventoryPrune.js@@ -1,15 +1,14 @@/*** Branch-reconcile persists its last inventory (`remaining`) so the Schedule view- * can offer per-row actions. Anything else that retires a checkout or branch- * (a claim run's own cleanup, `git worktree prune`, a later sweep) leaves that- * stored row behind, and the UI then offers an action on something that no- * longer exists. This decides which stored rows are such ghosts.+ * can offer per-row actions. Disappearance establishes missing source, never+ * preservation. Callers must retain it as an obligation without an exact receipt.*/import { execGit } from '../lib/execGit.js';-import { pathExists } from '../lib/fileCore.js';+import { readCheckoutPathPresence, readLocalBranchPresence } from './repoCleanupObservation.js';// Rows that bind a retirement-receipt obligation are evidence, not display data.-export const isObligationRow = row => ['retirement-review-pending', 'retirement-receipt-unverified'].includes(row?.reason);+export const isObligationRow = row => ['retirement-review-pending', 'retirement-receipt-unverified',+ 'retirement-provenance-unverified'].includes(row?.reason);async function registeredPaths(repoPath) {const { stdout } = await execGit(['worktree', 'list', '--porcelain'], repoPath);@@ -19,8 +18,8 @@ async function registeredPaths(repoPath) {/*** Stored inventory rows whose checkout (or, for a branch-only row, local branch)* is gone from the repository. A row whose branch survives but whose checkout is- * gone is still a ghost checkout; the next sweep lists the branch afresh.- * Fails open: any git read error yields no ghosts, so a flaky read hides nothing.+ * gone still has unproven preservation; the next sweep lists the branch afresh.+ * Unreadable paths are not absent. Git read errors reject for the caller to retain.** @param {string} repoPath* @param {object[]} rows@@ -35,12 +34,11 @@ export async function findGoneInventoryRows(repoPath, rows, { includeObligations// A remote-only hold has no local ref by definition. Only a new forge scan// can establish that its remote preservation obligation is gone.if (row?.reason === 'remote-branch-held') continue;- if (!includeObligations && isObligationRow(row)) continue;+ if (!includeObligations && (isObligationRow(row) || row?.reason === 'claim-reservation-owner-unverified')) continue;if (typeof row?.path === 'string' && row.path) {- if (!registered.has(row.path) && !await pathExists(row.path)) gone.push(row);+ if (!registered.has(row.path) && await readCheckoutPathPresence(row.path) === false) gone.push(row);} else if (typeof row?.branch === 'string' && row.branch) {- const ref = await execGit(['rev-parse', '--verify', '--quiet', `refs/heads/${row.branch}`], repoPath, { ignoreExitCode: true });- if (ref.exitCode === 1) gone.push(row);+ if (await readLocalBranchPresence(repoPath, row.branch) === false) gone.push(row);}}return gone;diff --git a/server/services/heldCheckoutRecovery.js b/server/services/heldCheckoutRecovery.jsindex 554246235d..4dfd964915 100644--- a/server/services/heldCheckoutRecovery.js+++ b/server/services/heldCheckoutRecovery.js@@ -865,13 +865,13 @@ export async function retireInventoriedCheckout({ appId, release, review }, optsif (!outcome?.remaining?.some?.(entry => sameRelease(entry, release))) return { ok: false, reason: 'release-not-in-inventory' };const inventoryCurrent = async () => (await taskSchedule.getBranchReconcileOutcome(appId))?.remaining?.some(entry => sameRelease(entry, release)) === true;const result = await retireHeldCheckout({ repoPath: app.repoPath, release, review }, { ...opts, inventoryCurrent, forgeAccount: app.forgeAccount ?? null });- // The row was stale: something else already removed the checkout. That is the- // outcome the operator asked for, so settle the row instead of refusing it.+ // A missing original cannot be archived retroactively. Retain its exact+ // identity for provenance recovery rather than equating absence with success.if (!result.ok && result.reason === 'checkout-not-found') {const gone = await findGoneInventoryRows(app.repoPath, [release]).catch(() => []);if (gone.length) {- await taskSchedule.dropInventoryRows(appId, gone);- return { ok: true, alreadyRetired: true };+ await taskSchedule.recordMissingInventoryRows(appId, gone);+ return { ok: false, reason: 'retirement-provenance-unverified' };}}if (result.ok) {diff --git a/server/services/heldCheckoutRecovery.test.js b/server/services/heldCheckoutRecovery.test.jsindex 608f485bd5..d3bd20b724 100644--- a/server/services/heldCheckoutRecovery.test.js+++ b/server/services/heldCheckoutRecovery.test.js@@ -51,7 +51,7 @@ vi.mock('./taskScheduleStore.js', async (importOriginal) => ({vi.mock('./apps.js', () => ({ getAppById: async (id) => (injected.app?.id === id ? injected.app : null) }));const { retireInventoriedCheckout, prepareInventoriedCheckoutRetirement, retirePendingMergeCheckout, heldCheckoutArchiveRoot, preserveMergedCheckout, recoverMissingCheckoutPreservations, findMissingCheckoutPreservation } = await import('./heldCheckoutRecovery.js');-const { recordBranchReconcileOutcome, getBranchReconcileOutcome } = await import('./taskSchedule.js');+const { recordBranchReconcileOutcome, getBranchReconcileOutcome, recordHeldCheckoutRetirement, parkPerpetual } = await import('./taskSchedule.js');const { withClaimOwnershipLock, checkReconcileOwnership } = await import('./cosClaimOwnership.js');describe.skipIf(SKIP_HEAVY_INTEGRATION)('held checkout recovery (real git)', () => {@@ -438,17 +438,54 @@ describe.skipIf(SKIP_HEAVY_INTEGRATION)('held checkout recovery (real git)', ()expect(outcome.retired[0]).toMatchObject({ path: release.path, branch: 'feature/held', branchDeleted: true, archiveDir: result.archiveDir });});- it('settles a stale inventory row whose checkout was already removed instead of refusing it', async () => {- const { path, release } = await heldCheckout('already-gone');+ it('retains vanished active claim provenance instead of reporting an unarchived checkout retired', async () => {+ const { path, release } = await heldCheckout('already-gone', { branch: 'claim/issue-42' });+ const gitDir = (await execGit(['rev-parse', '--absolute-git-dir'], path)).stdout.trim();+ await writeFile(join(gitDir, 'portos-review-status'), 'private synthetic evidence');await recordBranchReconcileOutcome('app-1', {- status: 'incomplete', remaining: [{ ...release, reason: 'worktree-locked' }, { branch: 'other', reason: 'actionable' }],+ status: 'incomplete', remaining: [{ ...release, reason: 'retirement-review-pending' }, { branch: 'other', reason: 'actionable' }],});await execGit(['worktree', 'unlock', path], repo);await execGit(['worktree', 'remove', '--force', path], repo);- expect(await retireInventoriedCheckout({ appId: 'app-1', release }, { cosDir, readAgents: noOwners }))- .toEqual({ ok: true, alreadyRetired: true });- expect((await getBranchReconcileOutcome('app-1')).remaining).toEqual([{ branch: 'other', reason: 'actionable' }]);+ const activeOwner = async () => [{ id: 'agent-live', status: 'running', metadata: {+ sourceWorkspace: repo, claimBranches: [release.branch], claimReleasedBranches: [],+ } }];+ expect(await retireInventoriedCheckout({ appId: 'app-1', release }, { cosDir, readAgents: activeOwner }))+ .toEqual({ ok: false, reason: 'retirement-provenance-unverified' });+ const obligation = { ...release, reason: 'retirement-provenance-unverified', createdAt: expect.any(String) };+ expect(await getBranchReconcileOutcome('app-1')).toMatchObject({ status: 'incomplete',+ retirementObligations: [obligation], remaining: [obligation, { branch: 'other', reason: 'actionable' }],+ });+ // Neither ignored bytes nor private Git state can be reconstructed from the merged head.+ expect(await archives()).toEqual([]);+ expect((await execGit(['rev-parse', `refs/heads/${release.branch}`], repo)).stdout.trim()).toBe(release.head);+ await recordBranchReconcileOutcome('app-1', { status: 'running', remaining: [] });+ await parkPerpetual('branch-reconcile', 'app-1', { reason: 'no-in-flight-branches' });+ expect(await getBranchReconcileOutcome('app-1')).toMatchObject({ status: 'incomplete', remaining: [obligation] });+ });++ it('retains a checkout that vanishes between sweeps until its exact preservation receipt arrives', async () => {+ const { path, release } = await heldCheckout('gone-between-scans', { branch: 'feature/missing' });+ const marker = { branch: 'claim/issue-99', issue: 99, reason: 'claim-reservation-owner-unverified' };+ await recordBranchReconcileOutcome('app-1', { status: 'running', remaining: [{ ...release, reason: 'worktree-locked' }, marker] });+ await execGit(['worktree', 'unlock', path], repo);+ // Model an external actor; the retirement service never receives this deletion.+ await execGit(['worktree', 'remove', '--force', path], repo);+ await recordBranchReconcileOutcome('app-1', { status: 'drained', remaining: [] });+ const pending = await getBranchReconcileOutcome('app-1');+ expect(pending).toMatchObject({ status: 'incomplete', reason: 'retirement-provenance-unverified',+ retirementObligations: [{ ...release, reason: 'retirement-provenance-unverified' }],+ });+ expect(pending.remaining).toHaveLength(1);+ const createdAt = pending.retirementObligations[0].createdAt;+ // A receipt for another head cannot explain the original disappearance.+ await recordHeldCheckoutRetirement('app-1', { ...release, head: 'f'.repeat(40), branchDeleted: true, archiveDir: '/example/other-archive' });+ await recordBranchReconcileOutcome('app-1', { status: 'running', remaining: [] });+ expect((await getBranchReconcileOutcome('app-1')).retirementObligations[0].createdAt).toBe(createdAt);+ // Only the trusted service writer can supply recovered preservation evidence.+ await recordHeldCheckoutRetirement('app-1', { ...release, branchDeleted: true, archiveDir: '/example/recovered-original' });+ expect(await getBranchReconcileOutcome('app-1')).toMatchObject({ retirementObligations: [], remaining: [] });});it('refuses a release that does not name a row of the last inventory', async () => {diff --git a/server/services/taskSchedule.js b/server/services/taskSchedule.jsindex 7c65be8420..95ee4116b6 100644--- a/server/services/taskSchedule.js+++ b/server/services/taskSchedule.js@@ -774,7 +774,12 @@ export async function recordBranchReconcileOutcome(appId, { status, reason, remaconst record = ensureExecutionRecord(schedule, 'branch-reconcile', appId);const now = new Date().toISOString();const previous = record.branchReconcileOutcome;- const retirementObligations = retainRetirementObligations(previous, [...(remaining || []), ...additionalObligations], now);+ // A fresh empty scan must not erase the last observation of vanished source.+ const omitted = remaining === undefined ? [] : (previous?.remaining || [])+ .filter(row => !remaining.some(current => sameRetirementCheckout(current, row)));+ const missing = await missingInventoryRows(appId, omitted);+ const retirementObligations = retainRetirementObligations(previous,+ [...(remaining || []), ...additionalObligations, ...missing.map(missingPreservationObligation)], now);const inventory = remaining === undefined ? previous?.remaining ?? null : remaining;const current = inventory?.map(row => {const obligation = retirementObligations.find(entry => sameRetirementCheckout(entry, row));@@ -785,6 +790,7 @@ export async function recordBranchReconcileOutcome(appId, { status, reason, rema...previous, status, reason, nextAction, updatedAt: now, retirementObligations,remaining: retained.length ? [...(current || []), ...retained] : current,observedAt: remaining === undefined ? record.branchReconcileOutcome?.observedAt ?? null : now,+ ...(missing.length && missingPreservationStatus),};return { result: record.branchReconcileOutcome, changed: true };});@@ -866,33 +872,43 @@ export async function getBranchReconcileOutcome(appId) {return resolveExecutionRecord(schedule, 'branch-reconcile', appId)?.branchReconcileOutcome ?? null;}-/** Remove stored inventory rows (display data only; obligations are untouched). */-export async function dropInventoryRows(appId, rows) {+const missingPreservationObligation = row => ({ ...row, reason: 'retirement-provenance-unverified' });+const missingPreservationStatus = {+ status: 'incomplete', reason: 'retirement-provenance-unverified',+ nextAction: 'Trace the missing original checkout/ref and recover its exact archive receipt; disappearance and merged commits do not prove preservation.',+};++async function missingInventoryRows(appId, rows, options) {+ if (!appId || !rows.length) return [];+ const app = await getAppById(appId);+ if (!app?.repoPath) return [];+ return findGoneInventoryRows(app.repoPath, rows, options);+}++/** Missing originals remain evidence until an exact service receipt settles them. */+export async function recordMissingInventoryRows(appId, rows) {return updateSchedule(async (schedule) => {const outcome = ensureExecutionRecord(schedule, 'branch-reconcile', appId).branchReconcileOutcome;if (!Array.isArray(outcome?.remaining)) return { result: outcome ?? null, changed: false };- const kept = outcome.remaining.filter(entry => !rows.some(row => sameRetirementCheckout(row, entry)));- if (kept.length === outcome.remaining.length) return { result: outcome, changed: false };- outcome.remaining = kept;+ const missing = outcome.remaining.filter(entry => rows.some(row => sameRetirementCheckout(row, entry)));+ if (!missing.length) return { result: outcome, changed: false };+ const now = new Date().toISOString();+ const obligations = retainRetirementObligations(outcome, missing.map(missingPreservationObligation), now);+ Object.assign(outcome, missingPreservationStatus, { updatedAt: now, retirementObligations: obligations,+ remaining: outcome.remaining.map(row => obligations.find(entry => sameRetirementCheckout(entry, row)) || row),+ });return { result: outcome, changed: true };});}-/**- * Drop stored inventory rows whose checkout or branch no longer exists, so the- * Schedule view never offers an action on something already cleaned up.- * Returns the outcome as it now stands.- */-async function pruneGoneInventory(appId, outcome) {+/** Reconcile display inventory with missing source without claiming archival success. */+async function reconcileMissingInventory(appId, outcome) {if (!appId || !outcome?.remaining?.length) return outcome;- const app = await getAppById(appId).catch(() => null);- if (!app?.repoPath) return outcome;- const gone = await findGoneInventoryRows(app.repoPath, outcome.remaining, { includeObligations: true }).catch(() => []);+ const gone = await missingInventoryRows(appId, outcome.remaining, { includeObligations: true }).catch(() => []);if (!gone.length) return outcome;- const obsolete = gone.filter(row => !isObligationRow(row));- const current = obsolete.length ? await dropInventoryRows(appId, obsolete).catch(() => null) ?? outcome : outcome;- // Missing originals do not prove preservation. Keep obligations, but do not- // present their historical tuple as a checkout that can still be retired.+ const unrecorded = gone.filter(row => !isObligationRow(row));+ const current = unrecorded.length ? await recordMissingInventoryRows(appId, unrecorded).catch(() => null) ?? outcome : outcome;+ // The original checkout is gone; flag it so the UI never offers a retire action on it.return { ...current, remaining: current.remaining.map(row => ({...row,...(row.path && gone.some(missing => sameRetirementCheckout(missing, row)) && { checkoutMissing: true }),@@ -1805,7 +1821,7 @@ export async function getScheduleStatus() {.map(([appId, record]) => [appId, record.branchReconcileOutcome]),];taskStatus.reconcileOutcomes = Object.fromEntries(- await mapWithConcurrency(outcomes, 8, async ([appId, outcome]) => [appId, await pruneGoneInventory(appId, outcome)])+ await mapWithConcurrency(outcomes, 8, async ([appId, outcome]) => [appId, await reconcileMissingInventory(appId, outcome)]));}status.tasks[taskType] = taskStatus;diff --git a/server/services/taskSchedule.test.js b/server/services/taskSchedule.test.jsindex 293228a7a1..ec06133195 100644--- a/server/services/taskSchedule.test.js+++ b/server/services/taskSchedule.test.js@@ -78,6 +78,8 @@ vi.mock('./apps.js', () => ({vi.mock('./taskInputGate.js', () => ({ inputsUnchangedSince: vi.fn().mockResolvedValue(false), readHead: vi.fn() }))+vi.mock('./branchReconcileInventoryPrune.js', () => ({ findGoneInventoryRows: vi.fn().mockResolvedValue([]) }))+vi.mock('./instanceFeatures.js', () => ({isInstanceFeatureEnabled: vi.fn().mockResolvedValue(true),}))@@ -205,7 +207,8 @@ import { loadState } from './cosState.js'import { readJSONFile } from '../lib/fileUtils.js'import { writeFile } from 'fs/promises'-import { isTaskTypeEnabledForApp, getAppTaskTypeInterval, getAppTaskTypeOverrides, getActiveApps, clearAllPrWatcherState, clearAllIssueWatcherState } from './apps.js'+import { isTaskTypeEnabledForApp, getAppById, getAppTaskTypeInterval, getAppTaskTypeOverrides, getActiveApps, clearAllPrWatcherState, clearAllIssueWatcherState } from './apps.js'+import { findGoneInventoryRows } from './branchReconcileInventoryPrune.js'import { getLocalParts } from '../lib/timezone.js'import { getAdaptiveCooldownMultiplier } from './taskLearning.js'import { parseCronToNextRun } from './eventScheduler.js'@@ -3318,6 +3321,29 @@ describe('taskSchedule', () => {.toMatchObject({ status: 'incomplete', reason: 'agent-failed', remaining, observedAt });});+ it('persists missing checkout and branch evidence when the Schedule view refreshes', async () => {+ let saved = { version: 2, tasks: { 'branch-reconcile': { enabled: true, type: 'on-demand', perpetual: false } }, executions: {}, templates: [] };+ readJSONFile.mockImplementation(async () => structuredClone(saved));+ writeFile.mockImplementation(async (_path, contents) => { saved = JSON.parse(contents); });+ getAppById.mockResolvedValueOnce({ repoPath: '/example/repo' });+ const rows = [+ { branch: 'fix/missing-checkout', path: '/example/checkout', head: 'a'.repeat(40), reason: 'worktree-locked' },+ { branch: 'fix/missing-ref', head: 'b'.repeat(40), reason: 'branch-held' },+ ];+ await recordBranchReconcileOutcome('app-1', { status: 'running', remaining: rows });+ findGoneInventoryRows.mockResolvedValueOnce(rows);+ const outcome = (await getScheduleStatus()).tasks['branch-reconcile'].reconcileOutcomes['app-1'];+ expect(outcome).toMatchObject({ status: 'incomplete', reason: 'retirement-provenance-unverified',+ retirementObligations: rows.map(row => ({ ...row, reason: 'retirement-provenance-unverified', createdAt: expect.any(String) })),+ });+ expect(outcome.retired).toBeUndefined();+ // Loading the persisted state again and an empty sweep cannot clear the evidence.+ await recordBranchReconcileOutcome('app-1', { status: 'running', remaining: [] });+ const reloaded = (await getScheduleStatus()).tasks['branch-reconcile'].reconcileOutcomes['app-1'];+ expect(reloaded.retirementObligations).toEqual(outcome.retirementObligations);+ expect(reloaded.remaining).toEqual(outcome.retirementObligations);+ });+it('never loses exact retirement obligations across empty scans, restart reads or unrelated receipts', async () => {let saved = { version: 2, tasks: { 'branch-reconcile': { enabled: true, type: 'on-demand', perpetual: true } }, executions: {}, templates: [] };readJSONFile.mockImplementation(async () => structuredClone(saved));