Skip to content

Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS

fix: recover verified preservation for missing checkouts (#13267)

Adam Eivy5edc0b5efe94
diff --git a/server/lib/README.md b/server/lib/README.md
index 44c74487d6..c3acae518e 100644
--- a/server/lib/README.md
+++ b/server/lib/README.md
@@ -28,6 +28,8 @@ The barrel `server/lib/index.js` is a machine-checkable enumeration of every pub
| `checkoutRetirementAdmission.js` | Durable common-repository publication/cut admission for explicit initialized-checkout quarantine; pending reservations survive restart and block managed checkout mutations until verified recovery. |
| `branchRetirementArchive.js` | `preserveBranchRetirement` privately archives exact shared configuration, the branch reflog and a verified standalone recovery bundle. Retirement retains Git's receiver lease/holder guards, then removes only the unchanged branch configuration under native config/ref locks. Changed evidence, recreated refs, external configuration sources and uncertain deletion retain live settings and leave retirement unresolved. |
| `reconcileRetirementEvidence.js` | Matches exact checkout retirement receipts and retains unresolved cleanup obligations. Server-created generation timestamps prevent an old archive receipt from completing a recreated checkout at the same branch, path and head. |
+| `branchRetirementArchive.js` | `preserveBranchRetirement` preserves the shared branch reflog verbatim, records its filesystem metadata, and verifies a standalone bundle containing the current tip and every old/new reflog commit before local branch retirement. Its final verifier holds changed refs or reflogs; unsupported storage, missing objects and archive failures retain the branch. |
+| `reconcileRetirementEvidence.js` | Matches exact checkout retirement and recovered-preservation receipts and retains unresolved cleanup obligations. Server-created generation timestamps prevent an old archive receipt from completing a recreated checkout at the same branch, path and head; verified historical preservation does not assert branch deletion. |
| `backupSharedAdmission.js` | Durable cross-process backup cuts and publication leases, with fail-closed owner recovery. |
| `runtimeFilePublication.js` | Admitted runtime recording updates with rollback of the prior file pair on failure. |
| `maintenanceAdmission.js` | Durable machine-local workflow drain, cross-process admission permits, continuation ownership and stale-resume fencing. |
diff --git a/server/lib/reconcileRetirementEvidence.js b/server/lib/reconcileRetirementEvidence.js
index edfdf947d2..ec979a1b5b 100644
--- a/server/lib/reconcileRetirementEvidence.js
+++ b/server/lib/reconcileRetirementEvidence.js
@@ -2,9 +2,15 @@
export const sameRetirementCheckout = (left, right) => left.branch === right.branch && left.path === right.path && left.head === right.head;
export const hasRetirementReceipt = (outcome, release) => Number.isFinite(Date.parse(release.createdAt))
- && outcome?.retired?.some(row => sameRetirementCheckout(row, release) && row.branchDeleted === true
+ && (outcome?.retired?.some(row => sameRetirementCheckout(row, release) && row.branchDeleted === true
&& typeof row.archiveDir === 'string' && row.archiveDir.length > 0
- && Date.parse(row.retiredAt) >= Date.parse(release.createdAt)) === true;
+ && Date.parse(row.retiredAt) >= Date.parse(release.createdAt)) === true
+ || outcome?.recoveredPreservations?.some(row => sameRetirementCheckout(row, release)
+ && row.obligationCreatedAt === release.createdAt && row.verified === true
+ && typeof row.archiveDir === 'string' && row.archiveDir.length > 0
+ && /^[a-f0-9]{64}$/.test(row.evidenceDigest || '')
+ && Date.parse(row.archivedAt) >= Date.parse(release.createdAt)
+ && Date.parse(row.verifiedAt) >= Date.parse(row.archivedAt)) === true);
export function unresolvedRetirements(outcome, additional = []) {
const rows = [...(outcome?.retirementObligations || []), ...(outcome?.remaining || []), ...additional];
@@ -26,8 +32,9 @@ export function retainRetirementObligations(outcome, additional, now) {
return pending.map(row => {
const previous = existing.find(entry => sameRetirementCheckout(entry, row));
if (Number.isFinite(Date.parse(previous?.createdAt))) return { ...row, createdAt: previous.createdAt };
- const earlier = (outcome?.retired || []).filter(entry => sameRetirementCheckout(entry, row))
- .map(entry => Date.parse(entry.retiredAt)).filter(Number.isFinite);
+ const earlier = [...(outcome?.retired || []), ...(outcome?.recoveredPreservations || [])]
+ .filter(entry => sameRetirementCheckout(entry, row))
+ .map(entry => Date.parse(entry.verifiedAt || entry.retiredAt)).filter(Number.isFinite);
return { ...row, createdAt: new Date(Math.max(Date.parse(now), ...earlier.map(time => time + 1))).toISOString() };
});
}
diff --git a/server/services/heldCheckoutRecovery.js b/server/services/heldCheckoutRecovery.js
index 53962d7910..f32136c0d3 100644
--- a/server/services/heldCheckoutRecovery.js
+++ b/server/services/heldCheckoutRecovery.js
@@ -20,9 +20,10 @@
import { findGoneInventoryRows } from './branchReconcileInventoryPrune.js';
import { createReadStream, realpathSync } from 'node:fs';
-import { lstat, stat, readlink, readdir, readFile, chmod } from 'node:fs/promises';
+import { lstat, stat, readlink, readdir, readFile, chmod, open } from 'node:fs/promises';
import { createHash, randomUUID } from 'node:crypto';
-import { basename, dirname, isAbsolute, join, resolve } from 'node:path';
+import { basename, dirname, isAbsolute, join, resolve, relative, sep } from 'node:path';
+import { sameRetirementCheckout, unresolvedRetirements } from '../lib/reconcileRetirementEvidence.js';
import { atomicWrite, copyFileGuarded, ensureDir, PATHS, rmGuarded, sha256File, writeFileGuarded } from '../lib/fileUtils.js';
import { execGit } from '../lib/execGit.js';
import { PROTECTED_BRANCHES } from '../lib/gitArgs.js';
@@ -34,6 +35,167 @@ import { withClaimOwnershipLock, readClaimOwnershipAgents, retireWithUnlockedArc
export const heldCheckoutArchiveRoot = (cosDir = PATHS.cos) => join(cosDir, 'held-checkout-archives');
+const evidenceHash = value => createHash('sha256').update(value).digest('hex');
+const historicalNextAction = 'Locate a complete archive for this exact historical checkout and retry branch reconciliation; if none exists, recover the original data from backup. Checkout or ref absence is not preservation evidence.';
+
+/** Never follow an archive link, including an intermediate directory link. */
+async function archiveFile(root, path) {
+ const rel = relative(root, path);
+ if (!rel || isAbsolute(rel) || rel.split(sep).includes('..')) throw new Error('archive-path-invalid');
+ const parts = rel.split(sep);
+ let current = root;
+ if (!(await lstat(root)).isDirectory()) throw new Error('archive-directory-invalid');
+ for (const [index, part] of parts.entries()) {
+ current = join(current, part);
+ const info = await lstat(current);
+ if (info.isSymbolicLink() || (index < parts.length - 1 ? !info.isDirectory() : !info.isFile())) throw new Error('archive-file-invalid');
+ }
+ return path;
+}
+
+async function archiveManifest(dir) {
+ const path = await archiveFile(dir, join(dir, 'manifest.json'));
+ if ((await lstat(path)).size > 16 * 1024 * 1024) throw new Error('archive-manifest-budget-exceeded');
+ const bytes = await readFile(path);
+ return { manifest: JSON.parse(bytes), digest: evidenceHash(bytes) };
+}
+
+/** Reuses the archive's recorded byte hashes and administration fingerprint.
+ * Older dirty archives lack patch hashes and cannot prove completeness later.
+ */
+async function verifyHeldArchive(repoPath, dir, release) {
+ const { manifest: saved, digest } = await archiveManifest(dir);
+ if (saved.repoPath !== repoPath || !sameRetirementCheckout(saved, release)
+ || (saved.release ? !sameRelease(saved.release, release) : release.locked !== false || release.lockReason !== null)) throw new Error('archive-identity-mismatch');
+ if (!(Date.parse(saved.archivedAt) >= Date.parse(release.createdAt)) || Date.parse(saved.archivedAt) > Date.now()) throw new Error('archive-generation-mismatch');
+ const captured = saved.captured;
+ if (captured?.commitBundle !== true || captured.missingCheckout || !Array.isArray(captured.files)
+ || !Array.isArray(captured.symlinks) || !Array.isArray(captured.gitlinks)
+ || !Array.isArray(saved.administration?.files) || !Array.isArray(saved.administration?.symlinks)
+ || !saved.administration.files.some(file => file.path === 'gitdir/HEAD')
+ || !saved.administration.files.some(file => file.path === 'git-indirection')) throw new Error('archive-preservation-incomplete');
+ if (captured.files.length + saved.administration.files.length > 1000) throw new Error('archive-file-budget-exceeded');
+ const stamps = [];
+ const stamp = async path => {
+ const info = await lstat(path, { bigint: true });
+ if (!info.isFile()) throw new Error('archive-file-invalid');
+ return [info.dev, info.ino, info.size, info.mtimeNs, info.ctimeNs].map(String).join(':');
+ };
+ const remember = async path => { stamps.push([path, await stamp(path)]); };
+ await remember(join(dir, 'manifest.json'));
+ let total = 0;
+ for (const [prefix, files] of [['files', captured.files], ['administration', saved.administration.files]]) {
+ for (const file of files) {
+ if (typeof file.path !== 'string' || !/^[a-f0-9]{64}$/.test(file.sha256 || '')) throw new Error('archive-file-proof-invalid');
+ const path = await archiveFile(dir, join(dir, prefix, file.path));
+ await remember(path);
+ const info = await lstat(path);
+ if (info.size !== file.bytes || info.size > MAX_FILE_BYTES || (total += info.size) > MAX_TOTAL_BYTES
+ || await sha256File(path) !== file.sha256) throw new Error('archive-file-changed');
+ }
+ }
+ const administrationFingerprint = evidenceHash(JSON.stringify({
+ files: saved.administration.files.filter(file => file.path !== 'gitdir/locked'), symlinks: saved.administration.symlinks,
+ }));
+ if (administrationFingerprint !== saved.administration.fingerprint) throw new Error('archive-administration-changed');
+ for (const name of ['index', 'worktree']) {
+ if (typeof captured[`${name}Diff`] !== 'boolean') throw new Error('archive-patch-proof-invalid');
+ if (!captured[`${name}Diff`]) continue;
+ const expected = saved.patchHashes?.[name];
+ if (!/^[a-f0-9]{64}$/.test(expected || '')) throw new Error('archive-patch-proof-unavailable');
+ const path = await archiveFile(dir, join(dir, `${name}.diff`));
+ await remember(path);
+ if (await sha256File(path) !== expected) throw new Error('archive-patch-changed');
+ }
+ const bundle = await archiveFile(dir, join(dir, 'commits.bundle'));
+ await remember(bundle);
+ // A thin bundle verifies against the current repository but is not a complete
+ // restore point. Inspect only its bounded textual header, never fetch it.
+ const handle = await open(bundle, 'r');
+ let header;
+ try {
+ const buffer = Buffer.alloc(256 * 1024);
+ const { bytesRead } = await handle.read(buffer, 0, buffer.length, 0);
+ header = buffer.subarray(0, bytesRead).toString('latin1').split('\n\n')[0];
+ if (!buffer.subarray(0, bytesRead).includes(Buffer.from('\n\n')) || /^-/m.test(header)) throw new Error('archive-bundle-incomplete');
+ } finally { await handle.close(); }
+ await git(['bundle', 'verify', bundle], repoPath);
+ const heads = new Set((await git(['bundle', 'list-heads', bundle], repoPath)).trim().split('\n').map(line => line.split(' ')[0]));
+ if (!Array.isArray(captured.recoveryHeads) || !captured.recoveryHeads.includes(release.head)
+ || captured.recoveryHeads.some(head => !heads.has(head))) throw new Error('archive-bundle-head-mismatch');
+ const bundleHash = await sha256File(bundle);
+ if (saved.bundleSha256 && bundleHash !== saved.bundleSha256) throw new Error('archive-bundle-changed');
+ if ((await archiveManifest(dir)).digest !== digest) throw new Error('archive-manifest-changed');
+ const recheck = async () => {
+ for (const [path, original] of stamps) {
+ if (await stamp(await archiveFile(dir, path)) !== original) return false;
+ }
+ return true;
+ };
+ if (!await recheck()) throw new Error('archive-changed-during-verification');
+ return { repoPath, archiveDir: dir, archivedAt: saved.archivedAt, evidenceDigest: evidenceHash(digest + bundleHash), kind: 'held', recheck };
+}
+
+async function historicalCheckoutAbsent(repoPath, release) {
+ const present = await lstat(release.path).catch(error => { if (error.code === 'ENOENT') return null; throw error; });
+ const trees = await listWorktrees(repoPath);
+ return !present && trees.length > 0 && !trees.some(tree => tree.path === release.path || (release.branch && shortBranch(tree.branch) === release.branch));
+}
+
+/** Read existing archives only. Verification stays outside both writer locks;
+ * the final ownership/absence recheck and schedule compare-and-set are fenced.
+ */
+export async function recoverMissingCheckoutPreservations({ appId, repoPath, outcome }, opts = {}) {
+ const expected = structuredClone(outcome);
+ const obligations = unresolvedRetirements(expected);
+ if (!obligations.length) return outcome;
+ const roots = [heldCheckoutArchiveRoot(opts.cosDir), join(opts.cosDir || PATHS.cos, 'initialized-checkout-retirements')];
+ const candidates = [];
+ let discoveryReason = null;
+ for (const root of roots) {
+ try {
+ const info = await lstat(root).catch(error => { if (error.code === 'ENOENT') return null; throw error; });
+ if (!info) continue;
+ if (!info.isDirectory()) throw new Error('archive-directory-invalid');
+ const names = await readdir(root, { withFileTypes: true });
+ if (names.length + candidates.length > 1000) throw new Error('archive-discovery-budget-exceeded');
+ for (const name of names) if (name.isDirectory()) candidates.push({ dir: join(root, name.name), kind: root === roots[0] ? 'held' : 'initialized' });
+ } catch { discoveryReason = 'archive-discovery-unavailable'; }
+ }
+ const observations = [];
+ for (const release of obligations) {
+ if (!await historicalCheckoutAbsent(repoPath, release).catch(() => false)) continue;
+ const evidence = [];
+ let proof = null;
+ for (const candidate of candidates) {
+ try {
+ proof = candidate.kind === 'held' ? await verifyHeldArchive(repoPath, candidate.dir, release)
+ : await (await import('./initializedCheckoutRecovery.js')).verifyMissingInitializedCheckoutPreservation({ appId, repoPath, release, archiveDir: candidate.dir });
+ evidence.push({ archiveDir: candidate.dir, reason: 'preservation-verified' });
+ break;
+ } catch (error) {
+ evidence.push({ archiveDir: candidate.dir, reason: /^[a-z][a-z-]+$/.test(error.message) ? error.message : 'archive-unverifiable' });
+ }
+ }
+ observations.push({ release, proof, evidence, reason: discoveryReason || (candidates.length ? 'archive-evidence-unverified' : 'archive-evidence-missing'), nextAction: historicalNextAction });
+ }
+ if (!observations.length) return outcome;
+ const { recordRecoveredCheckoutPreservations } = await import('./taskSchedule.js');
+ return withClaimOwnershipLock(() => recordRecoveredCheckoutPreservations(appId, expected, observations, async () => {
+ const { getAppById } = await import('./apps.js');
+ if ((await getAppById(appId))?.repoPath !== repoPath) return false;
+ for (const row of observations) {
+ if (!await historicalCheckoutAbsent(repoPath, row.release)) return false;
+ if (row.proof?.recheck && !await row.proof.recheck().catch(() => false)) {
+ row.evidence.push({ archiveDir: row.proof.archiveDir, reason: 'archive-changed-during-verification' });
+ row.proof = null;
+ row.reason = 'archive-changed-during-verification';
+ }
+ }
+ return true;
+ }));
+}
+
// Refuse rather than archive a subset: a partial archive followed by removal
// would delete the rest. Larger than the superseded reap's caps because this is
// an explicit operator action, but still bounded so a build tree cannot fill
@@ -393,6 +555,8 @@ async function writeArchive(dir, { repoPath, tree, branch, defaultBranch, releas
await atomicWrite(join(dir, 'manifest.json'), {
repoPath, defaultBranch, path: tree.path, head: tree.head, branch,
release, review, administration: { ...administration, files: administration.files.map(({ source: _source, ...file }) => file) }, archivedAt: new Date().toISOString(),
+ bundleSha256: await sha256File(bundle),
+ patchHashes: { index: evidenceHash(snapshot.indexDiff), worktree: evidenceHash(snapshot.worktreeDiff) },
captured: { commitBundle: true, recoveryHeads: [...recoveryHeads], ...(snapshot.missingCheckout ? { missingCheckout: true } : {}), gitlinks: snapshot.gitlinks, indexDiff: Boolean(snapshot.indexDiff), worktreeDiff: Boolean(snapshot.worktreeDiff), files, symlinks },
restore: [
'git -C <repo> fetch commits.bundle HEAD:refs/heads/<restored-branch>',
diff --git a/server/services/heldCheckoutRecovery.test.js b/server/services/heldCheckoutRecovery.test.js
index 81e37689bd..810fdf5474 100644
--- a/server/services/heldCheckoutRecovery.test.js
+++ b/server/services/heldCheckoutRecovery.test.js
@@ -20,13 +20,14 @@ vi.mock('../lib/paths.js', async importOriginal => makePathsProxy(await importOr
afterAll(cleanupTempDataRoots);
const SYMLINKS = process.platform !== 'win32';
-const injected = vi.hoisted(() => ({ policies: [], schedule: null, app: null, fetchRemote: null, fetches: [], onBundle: null }));
+const injected = vi.hoisted(() => ({ policies: [], schedule: null, app: null, fetchRemote: null, fetches: [], onBundle: null, onEvidence: null }));
vi.mock('../lib/execGit.js', async importOriginal => {
const actual = await importOriginal();
return { ...actual, execGit: async (args, cwd, options) => {
// Pauses an archive mid-write, after its first snapshot and before its re-read.
if (injected.onBundle && args.includes('bundle') && args.includes('create')) await injected.onBundle();
+ if (injected.onEvidence && args[0] === 'bundle' && args[1] === 'verify') await injected.onEvidence();
if (args.includes('credential.helper=!gh auth git-credential') && args.includes('fetch')) {
injected.fetches.push({ args, options });
if (injected.fetchRemote) args = args.map(arg => arg === 'https://github.com/example/repo.git' ? injected.fetchRemote : arg);
@@ -49,7 +50,7 @@ vi.mock('./taskScheduleStore.js', async (importOriginal) => ({
}));
vi.mock('./apps.js', () => ({ getAppById: async (id) => (injected.app?.id === id ? injected.app : null) }));
-const { retireInventoriedCheckout, prepareInventoriedCheckoutRetirement, retirePendingMergeCheckout, heldCheckoutArchiveRoot, preserveMergedCheckout } = await import('./heldCheckoutRecovery.js');
+const { retireInventoriedCheckout, prepareInventoriedCheckoutRetirement, retirePendingMergeCheckout, heldCheckoutArchiveRoot, preserveMergedCheckout, recoverMissingCheckoutPreservations } = await import('./heldCheckoutRecovery.js');
const { recordBranchReconcileOutcome, getBranchReconcileOutcome } = await import('./taskSchedule.js');
const { withClaimOwnershipLock, checkReconcileOwnership } = await import('./cosClaimOwnership.js');
@@ -61,7 +62,7 @@ describe.skipIf(SKIP_HEAVY_INTEGRATION)('held checkout recovery (real git)', ()
const noOwners = async () => [];
beforeEach(async () => {
- Object.assign(injected, { policies: [], schedule: { executions: {} }, app: { id: 'app-1', repoPath: null }, fetchRemote: null, fetches: [], onBundle: null });
+ Object.assign(injected, { policies: [], schedule: { executions: {} }, app: { id: 'app-1', repoPath: null }, fetchRemote: null, fetches: [], onBundle: null, onEvidence: null });
scratch = realpathSync(await mkdtemp(join(tmpdir(), 'portos-held-checkout-')));
const safeRepo = join(scratch, 'repo');
await mkdir(safeRepo);
@@ -104,6 +105,84 @@ describe.skipIf(SKIP_HEAVY_INTEGRATION)('held checkout recovery (real git)', ()
}
const archives = async () => (existsSync(heldCheckoutArchiveRoot(cosDir)) ? readdir(heldCheckoutArchiveRoot(cosDir)) : []);
+ async function historicalArchive() {
+ const path = join(external, 'historical');
+ const branch = 'feature/historical';
+ await execGit(['worktree', 'add', '-b', branch, path, 'main'], repo);
+ await writeFile(join(path, 'initial.txt'), 'staged historical work\n');
+ await execGit(['add', 'initial.txt'], path);
+ await writeFile(join(path, 'private.txt'), 'synthetic private bytes');
+ const release = { path, branch, head: (await execGit(['rev-parse', 'HEAD'], path)).stdout.trim(), locked: false, lockReason: null };
+ await recordBranchReconcileOutcome('app-1', { status: 'incomplete', remaining: [{ ...release, reason: 'retirement-review-pending' }] });
+ const saved = await preserveMergedCheckout(repo, { ...release, branch: `refs/heads/${branch}` }, 'main', { cosDir: PATHS.cos });
+ await execGit(['worktree', 'remove', '--force', path], repo);
+ return { release, saved };
+ }
+
+ it('recovers complete preservation through the schedule boundary with only the primary checkout, without claiming branch deletion', async () => {
+ const { release, saved } = await historicalArchive();
+ const before = (await execGit(['show-ref'], repo)).stdout;
+ const outcome = await recordBranchReconcileOutcome('app-1', { status: 'incomplete', remaining: [] });
+ expect(outcome.retirementObligations).toEqual([]);
+ expect(outcome.remaining).toEqual([]);
+ expect(outcome.recoveredPreservations).toEqual([expect.objectContaining({ path: release.path, branch: release.branch, head: release.head,
+ archiveDir: saved.archiveDir, verified: true,
+ evidenceDigest: expect.stringMatching(/^[a-f0-9]{64}$/),
+ })]);
+ expect(outcome.retired).toBeUndefined();
+ expect(outcome.recoveredPreservations[0]).not.toHaveProperty('branchDeleted');
+ expect(existsSync(release.path)).toBe(false);
+ expect((await execGit(['show-ref'], repo)).stdout).toBe(before);
+ // A new generation with the same branch/path/head cannot reuse the archive.
+ const recreated = await recordBranchReconcileOutcome('app-1', { status: 'incomplete', retirementObligations: [{ ...release, reason: 'retirement-review-pending' }] });
+ expect(recreated.retirementObligations).toHaveLength(1);
+ expect(recreated.retirementObligations[0].preservationEvidence.archives).toContainEqual({ archiveDir: saved.archiveDir, reason: 'archive-generation-mismatch' });
+ });
+
+ it.each(['missing', 'wrong-repository', 'wrong-path', 'wrong-head', 'changed-file', 'changed-patch', 'legacy-dirty', 'partial-original'])(
+ 'retains an actionable historical obligation for %s preservation evidence', async failure => {
+ const { release, saved } = await historicalArchive();
+ const manifestPath = join(saved.archiveDir, 'manifest.json');
+ const manifest = JSON.parse(await readFile(manifestPath, 'utf8'));
+ if (failure === 'missing') await rm(saved.archiveDir, { recursive: true, force: true });
+ if (failure === 'wrong-repository') manifest.repoPath = join(scratch, 'other-repo');
+ if (failure === 'wrong-path') manifest.path = join(external, 'other-checkout');
+ if (failure === 'wrong-head') manifest.head = 'f'.repeat(40);
+ if (failure === 'changed-file') await writeFile(join(saved.archiveDir, 'files', 'private.txt'), 'changed archive bytes');
+ if (failure === 'changed-patch') await writeFile(join(saved.archiveDir, 'index.diff'), 'changed patch bytes');
+ if (failure === 'legacy-dirty') delete manifest.patchHashes;
+ if (failure === 'partial-original') manifest.captured.missingCheckout = true;
+ if (failure !== 'missing') await writeFile(manifestPath, JSON.stringify(manifest));
+ const outcome = await recordBranchReconcileOutcome('app-1', { status: 'incomplete', remaining: [] });
+ expect(outcome.retirementObligations).toEqual([expect.objectContaining({ ...release,
+ preservationEvidence: expect.objectContaining({ status: 'unresolved', nextAction: expect.stringContaining('backup') }),
+ })]);
+ expect(outcome).toMatchObject({ status: 'incomplete', reason: 'historical-preservation-unresolved', recoveredPreservations: [] });
+ expect(outcome.remaining).toEqual(outcome.retirementObligations);
+ expect(existsSync(release.path)).toBe(false);
+ });
+
+ it.each(['recreated-path', 'new-holder', 'inventory-change', 'repository-change'])(
+ 'does not settle historical evidence after a concurrent %s', async change => {
+ const { release } = await historicalArchive();
+ const outcome = await getBranchReconcileOutcome('app-1');
+ injected.onEvidence = async () => {
+ injected.onEvidence = null;
+ if (change === 'recreated-path') await mkdir(release.path);
+ if (change === 'new-holder') await execGit(['worktree', 'add', join(external, 'new-holder'), release.branch], repo);
+ if (change === 'repository-change') injected.app.repoPath = join(scratch, 'changed-repo');
+ if (change === 'inventory-change') {
+ const current = injected.schedule.executions['task:branch-reconcile'].perApp['app-1'];
+ current.branchReconcileOutcome = { ...current.branchReconcileOutcome, observedAt: 'changed-inventory' };
+ }
+ };
+ await recoverMissingCheckoutPreservations({ appId: 'app-1', repoPath: repo, outcome });
+ const current = await getBranchReconcileOutcome('app-1');
+ expect(current.retirementObligations).toHaveLength(1);
+ expect(current.recoveredPreservations).toBeUndefined();
+ if (change === 'inventory-change') expect(current.observedAt).toBe('changed-inventory');
+ });
+
it('preserves ignored evidence before unattended cleanup and refuses later byte or head changes', async () => {
const path = join(external, 'merged');
await execGit(['worktree', 'add', '-b', 'feature/done', path, 'main'], repo);
diff --git a/server/services/initializedCheckoutRecovery.js b/server/services/initializedCheckoutRecovery.js
index e55f8bc749..5b782ba6f4 100644
--- a/server/services/initializedCheckoutRecovery.js
+++ b/server/services/initializedCheckoutRecovery.js
@@ -507,6 +507,29 @@ export function _createInitializedCheckoutRecovery(deps = {}) {
verified: true, branchDeleted: state.branchDeleted ?? false, proof };
};
+ const verifyMissingPreservation = async ({ appId, repoPath, release, archiveDir }) => {
+ const directory = location(archiveDir.split(/[\\/]/).at(-1));
+ if (directory !== archiveDir) fail('archive-location-mismatch');
+ const state = await readJournal(directory, { bounded: true });
+ if (state.appId !== appId || state.repoPath !== repoPath || !sameRelease(state.release, release)) fail('archive-identity-mismatch');
+ if (!['branch-retired', 'retired'].includes(state.phase) || state.branchDeleted !== true) fail('archive-retirement-unconfirmed');
+ const archivedAt = state.reservation?.owner?.startedAt;
+ if (!(Date.parse(archivedAt) >= Date.parse(release.createdAt)) || Date.parse(archivedAt) > Date.now()) fail('archive-generation-mismatch');
+ // The journal locates the manifest; the existing native verifier checks its
+ // digest, every preserved source and the standalone object closure.
+ const manifestPath = join(directory, 'manifest.json');
+ const info = await lstat(manifestPath);
+ if (!info.isFile() || info.isSymbolicLink() || info.size > 16 * 1024 * 1024) fail('archive-manifest-invalid');
+ const manifest = JSON.parse(await readFile(manifestPath, 'utf8'));
+ if (manifest.facts?.repoPath !== repoPath || !sameRelease(manifest.facts?.release, release)) fail('archive-identity-mismatch');
+ const result = await verifyArchive({ appId, transactionId: state.id, journalDigest: hash(state) });
+ if (result.proof?.verified !== true || result.proof?.standalone !== true) fail('archive-preservation-unverified');
+ if (hash(await readJournal(directory, { bounded: true })) !== hash(state)) fail('retirement-journal-changed');
+ const recheck = async () => hash(await readJournal(directory, { bounded: true })) === hash(state)
+ && createHash('sha256').update(await readFile(manifestPath)).digest('hex') === state.proof.manifestSha256;
+ return { repoPath, archiveDir: directory, archivedAt, evidenceDigest: state.proof.manifestSha256, kind: 'initialized', recheck };
+ };
+
// Explicit recovery restores the exact original inodes. It does not erase
// archives or auto-finish an interrupted ref deletion on stale forge evidence.
const restore = (input, { cancelOnly = false } = {}) => fenced(async () => {
@@ -959,7 +982,7 @@ export function _createInitializedCheckoutRecovery(deps = {}) {
const automatic = async (input, options) => recoverAutomatically(input, options)
.catch(error => ({ ok: false, reason: error.message, transactionId: input.transactionId, recoveryRequired: true }));
return { discover, recoverAutomatically: automatic, review, retire, status, restore,
- cancelPreservation: input => restore(input, { cancelOnly: true }), verifyArchive, reviewBranch, retireBranch, reviewMetadataRestore, restoreMetadata,
+ cancelPreservation: input => restore(input, { cancelOnly: true }), verifyArchive, verifyMissingPreservation, reviewBranch, retireBranch, reviewMetadataRestore, restoreMetadata,
reviewRestoredFinalization, finalizeRestored, reviewQuarantineResumption, resumeQuarantine };
}
@@ -970,6 +993,7 @@ export const initializedCheckoutStatus = recovery.status;
export const restoreInitializedCheckout = recovery.restore;
export const verifyInitializedCheckoutArchive = recovery.verifyArchive;
+export const verifyMissingInitializedCheckoutPreservation = recovery.verifyMissingPreservation;
export const reviewInitializedCheckoutBranch = recovery.reviewBranch;
export const retireInitializedCheckoutBranch = recovery.retireBranch;
diff --git a/server/services/initializedCheckoutRecovery.test.js b/server/services/initializedCheckoutRecovery.test.js
index 7e7b42b2a1..f852b72162 100644
--- a/server/services/initializedCheckoutRecovery.test.js
+++ b/server/services/initializedCheckoutRecovery.test.js
@@ -166,6 +166,7 @@ beforeEach(async () => {
}
if (fixture.failArchive && input.command === 'verify') throw new Error('archive-changed');
return { ok: true, placement: fixture.placement, fingerprint: fixture.snapshot.fingerprint,
+ ...(input.command === 'verify' && input.mode === 'standalone' ? { verified: true, standalone: true } : {}),
...(input.metadataRecovery && !alreadyRestored ? {
postMoveObservation: { file: `${input.part}-post-move.json`, sha256: fixture.metadataToken, digest: fixture.metadataToken },
} : {}),
@@ -962,6 +963,24 @@ describe('initialized checkout transaction', () => {
expect(fixture.cut).toBeNull();
expect((await fixture.service.status(input)).phase).toBe('retired');
});
+ it('binds missing-original recovery to the historical journal generation and native standalone proof', async () => {
+ const createdAt = new Date(Date.now() - 1000).toISOString();
+ const input = await quarantinedReview();
+ await fixture.service.retireBranch(input);
+ const archiveDir = join(root, input.transactionId);
+ await writeFile(join(archiveDir, 'manifest.json'), JSON.stringify({ facts: { repoPath: '/fixture/repo', release } }));
+ const request = { appId: 'app', repoPath: '/fixture/repo', archiveDir, release: { ...release, createdAt } };
+ fixture.deps.native.mockClear(); fixture.deps.deleteRef.mockClear();
+ expect(await fixture.service.verifyMissingPreservation(request)).toMatchObject({ archiveDir, kind: 'initialized', evidenceDigest: 'e'.repeat(64) });
+ expect(fixture.deps.native.mock.calls.map(([value]) => value.command)).toEqual(['verify']);
+ expect(fixture.deps.deleteRef).not.toHaveBeenCalled();
+ await expect(fixture.service.verifyMissingPreservation({ ...request, release: { ...request.release, head: 'f'.repeat(40) } }))
+ .rejects.toThrow('archive-identity-mismatch');
+ await expect(fixture.service.verifyMissingPreservation({ ...request, release: { ...release, createdAt: new Date(Date.now() + 1000).toISOString() } }))
+ .rejects.toThrow('archive-generation-mismatch');
+ fixture.failArchive = true;
+ await expect(fixture.service.verifyMissingPreservation(request)).rejects.toThrow('archive-changed');
+ });
it('invalidates branch finalization when the default advances or a holder appears', async () => {
const input = await quarantinedReview();
fixture.facts.defaultHead = 'e'.repeat(40);
diff --git a/server/services/taskSchedule.js b/server/services/taskSchedule.js
index 6082e5e710..226c9fd8b4 100644
--- a/server/services/taskSchedule.js
+++ b/server/services/taskSchedule.js
@@ -770,7 +770,7 @@ export async function isPerpetualParkActive(taskType, appId = null) {
/** Persist the latest deterministic sweep evidence independently of agent success. */
export async function recordBranchReconcileOutcome(appId, { status, reason, remaining, nextAction, retirementObligations: additionalObligations = [] }) {
- return updateSchedule(async (schedule) => {
+ const outcome = await updateSchedule(async (schedule) => {
const record = ensureExecutionRecord(schedule, 'branch-reconcile', appId);
const now = new Date().toISOString();
const previous = record.branchReconcileOutcome;
@@ -788,6 +788,50 @@ export async function recordBranchReconcileOutcome(appId, { status, reason, rema
};
return { result: record.branchReconcileOutcome, changed: true };
});
+ if (!appId || !unresolvedRetirements(outcome).length) return outcome;
+ const app = await getAppById(appId).catch(() => null);
+ if (!app?.repoPath) return outcome;
+ // Keep the expensive archive verifier off ordinary live-checkout writes.
+ const missing = await findGoneInventoryRows(app.repoPath, unresolvedRetirements(outcome), { includeObligations: true }).catch(() => []);
+ if (!missing.length) return outcome;
+ const { recoverMissingCheckoutPreservations } = await import('./heldCheckoutRecovery.js');
+ return recoverMissingCheckoutPreservations({ appId, repoPath: app.repoPath, outcome });
+}
+
+/** Compare the whole observed inventory and its generation inside the schedule
+ * writer. A verifier result never authorizes a newer/recreated checkout.
+ */
+export async function recordRecoveredCheckoutPreservations(appId, expected, observations, stillAbsent) {
+ return updateSchedule(async schedule => {
+ const record = ensureExecutionRecord(schedule, 'branch-reconcile', appId);
+ const outcome = record.branchReconcileOutcome;
+ if (JSON.stringify(outcome) !== JSON.stringify(expected) || !await stillAbsent()) return { result: outcome, changed: false };
+ const now = new Date().toISOString();
+ const recoveredPreservations = [...(outcome.recoveredPreservations || [])];
+ const diagnosed = unresolvedRetirements(outcome).map(release => {
+ const observation = observations.find(row => sameRetirementCheckout(row.release, release) && row.release.createdAt === release.createdAt);
+ if (!observation) return release;
+ const accepted = observation.proof && recoveredPreservations.length < 1000;
+ const { recheck: _recheck, ...proof } = observation.proof || {};
+ if (accepted) recoveredPreservations.push({
+ branch: release.branch, path: release.path, head: release.head,
+ obligationCreatedAt: release.createdAt, ...proof, verified: true, verifiedAt: now,
+ });
+ return { ...release, preservationEvidence: { status: 'unresolved',
+ reason: observation.proof && !accepted ? 'preservation-receipt-budget-exceeded' : observation.reason,
+ checkedAt: now, archives: observation.evidence, nextAction: observation.nextAction } };
+ });
+ const retirementObligations = unresolvedRetirements({ retirementObligations: diagnosed, recoveredPreservations });
+ const remaining = (outcome.remaining || []).flatMap(row => {
+ const diagnosedRow = diagnosed.find(release => sameRetirementCheckout(row, release) && row.createdAt === release.createdAt);
+ return diagnosedRow ? retirementObligations.filter(release => sameRetirementCheckout(release, diagnosedRow)) : [row];
+ });
+ const historical = retirementObligations.find(row => row.preservationEvidence);
+ record.branchReconcileOutcome = { ...outcome, retirementObligations, recoveredPreservations, remaining, updatedAt: now,
+ ...(historical ? { status: 'incomplete', reason: 'historical-preservation-unresolved', nextAction: historical.preservationEvidence.nextAction } : {}),
+ };
+ return { result: record.branchReconcileOutcome, changed: true };
+ });
}
/** Observations are diagnostic only; journals and fresh service proofs own recovery. */