Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS
JavaScript ai-agentsdashboarddeveloper-toolsdigital-twinquantified-selfsecond-brainself-hostedtailscale
fix: recover verified preservation for missing checkouts (#13267)
Adam Eivy5edc0b5efe94
diff --git a/server/lib/README.md b/server/lib/README.mdindex 44c74487d6..c3acae518e 100644--- a/server/lib/README.md+++ b/server/lib/README.md@@ -28,6 +28,8 @@ The barrel `server/lib/index.js` is a machine-checkable enumeration of every pub| `checkoutRetirementAdmission.js` | Durable common-repository publication/cut admission for explicit initialized-checkout quarantine; pending reservations survive restart and block managed checkout mutations until verified recovery. || `branchRetirementArchive.js` | `preserveBranchRetirement` privately archives exact shared configuration, the branch reflog and a verified standalone recovery bundle. Retirement retains Git's receiver lease/holder guards, then removes only the unchanged branch configuration under native config/ref locks. Changed evidence, recreated refs, external configuration sources and uncertain deletion retain live settings and leave retirement unresolved. || `reconcileRetirementEvidence.js` | Matches exact checkout retirement receipts and retains unresolved cleanup obligations. Server-created generation timestamps prevent an old archive receipt from completing a recreated checkout at the same branch, path and head. |+| `branchRetirementArchive.js` | `preserveBranchRetirement` preserves the shared branch reflog verbatim, records its filesystem metadata, and verifies a standalone bundle containing the current tip and every old/new reflog commit before local branch retirement. Its final verifier holds changed refs or reflogs; unsupported storage, missing objects and archive failures retain the branch. |+| `reconcileRetirementEvidence.js` | Matches exact checkout retirement and recovered-preservation receipts and retains unresolved cleanup obligations. Server-created generation timestamps prevent an old archive receipt from completing a recreated checkout at the same branch, path and head; verified historical preservation does not assert branch deletion. || `backupSharedAdmission.js` | Durable cross-process backup cuts and publication leases, with fail-closed owner recovery. || `runtimeFilePublication.js` | Admitted runtime recording updates with rollback of the prior file pair on failure. || `maintenanceAdmission.js` | Durable machine-local workflow drain, cross-process admission permits, continuation ownership and stale-resume fencing. |diff --git a/server/lib/reconcileRetirementEvidence.js b/server/lib/reconcileRetirementEvidence.jsindex edfdf947d2..ec979a1b5b 100644--- a/server/lib/reconcileRetirementEvidence.js+++ b/server/lib/reconcileRetirementEvidence.js@@ -2,9 +2,15 @@export const sameRetirementCheckout = (left, right) => left.branch === right.branch && left.path === right.path && left.head === right.head;export const hasRetirementReceipt = (outcome, release) => Number.isFinite(Date.parse(release.createdAt))- && outcome?.retired?.some(row => sameRetirementCheckout(row, release) && row.branchDeleted === true+ && (outcome?.retired?.some(row => sameRetirementCheckout(row, release) && row.branchDeleted === true&& typeof row.archiveDir === 'string' && row.archiveDir.length > 0- && Date.parse(row.retiredAt) >= Date.parse(release.createdAt)) === true;+ && Date.parse(row.retiredAt) >= Date.parse(release.createdAt)) === true+ || outcome?.recoveredPreservations?.some(row => sameRetirementCheckout(row, release)+ && row.obligationCreatedAt === release.createdAt && row.verified === true+ && typeof row.archiveDir === 'string' && row.archiveDir.length > 0+ && /^[a-f0-9]{64}$/.test(row.evidenceDigest || '')+ && Date.parse(row.archivedAt) >= Date.parse(release.createdAt)+ && Date.parse(row.verifiedAt) >= Date.parse(row.archivedAt)) === true);export function unresolvedRetirements(outcome, additional = []) {const rows = [...(outcome?.retirementObligations || []), ...(outcome?.remaining || []), ...additional];@@ -26,8 +32,9 @@ export function retainRetirementObligations(outcome, additional, now) {return pending.map(row => {const previous = existing.find(entry => sameRetirementCheckout(entry, row));if (Number.isFinite(Date.parse(previous?.createdAt))) return { ...row, createdAt: previous.createdAt };- const earlier = (outcome?.retired || []).filter(entry => sameRetirementCheckout(entry, row))- .map(entry => Date.parse(entry.retiredAt)).filter(Number.isFinite);+ const earlier = [...(outcome?.retired || []), ...(outcome?.recoveredPreservations || [])]+ .filter(entry => sameRetirementCheckout(entry, row))+ .map(entry => Date.parse(entry.verifiedAt || entry.retiredAt)).filter(Number.isFinite);return { ...row, createdAt: new Date(Math.max(Date.parse(now), ...earlier.map(time => time + 1))).toISOString() };});}diff --git a/server/services/heldCheckoutRecovery.js b/server/services/heldCheckoutRecovery.jsindex 53962d7910..f32136c0d3 100644--- a/server/services/heldCheckoutRecovery.js+++ b/server/services/heldCheckoutRecovery.js@@ -20,9 +20,10 @@import { findGoneInventoryRows } from './branchReconcileInventoryPrune.js';import { createReadStream, realpathSync } from 'node:fs';-import { lstat, stat, readlink, readdir, readFile, chmod } from 'node:fs/promises';+import { lstat, stat, readlink, readdir, readFile, chmod, open } from 'node:fs/promises';import { createHash, randomUUID } from 'node:crypto';-import { basename, dirname, isAbsolute, join, resolve } from 'node:path';+import { basename, dirname, isAbsolute, join, resolve, relative, sep } from 'node:path';+import { sameRetirementCheckout, unresolvedRetirements } from '../lib/reconcileRetirementEvidence.js';import { atomicWrite, copyFileGuarded, ensureDir, PATHS, rmGuarded, sha256File, writeFileGuarded } from '../lib/fileUtils.js';import { execGit } from '../lib/execGit.js';import { PROTECTED_BRANCHES } from '../lib/gitArgs.js';@@ -34,6 +35,167 @@ import { withClaimOwnershipLock, readClaimOwnershipAgents, retireWithUnlockedArcexport const heldCheckoutArchiveRoot = (cosDir = PATHS.cos) => join(cosDir, 'held-checkout-archives');+const evidenceHash = value => createHash('sha256').update(value).digest('hex');+const historicalNextAction = 'Locate a complete archive for this exact historical checkout and retry branch reconciliation; if none exists, recover the original data from backup. Checkout or ref absence is not preservation evidence.';++/** Never follow an archive link, including an intermediate directory link. */+async function archiveFile(root, path) {+ const rel = relative(root, path);+ if (!rel || isAbsolute(rel) || rel.split(sep).includes('..')) throw new Error('archive-path-invalid');+ const parts = rel.split(sep);+ let current = root;+ if (!(await lstat(root)).isDirectory()) throw new Error('archive-directory-invalid');+ for (const [index, part] of parts.entries()) {+ current = join(current, part);+ const info = await lstat(current);+ if (info.isSymbolicLink() || (index < parts.length - 1 ? !info.isDirectory() : !info.isFile())) throw new Error('archive-file-invalid');+ }+ return path;+}++async function archiveManifest(dir) {+ const path = await archiveFile(dir, join(dir, 'manifest.json'));+ if ((await lstat(path)).size > 16 * 1024 * 1024) throw new Error('archive-manifest-budget-exceeded');+ const bytes = await readFile(path);+ return { manifest: JSON.parse(bytes), digest: evidenceHash(bytes) };+}++/** Reuses the archive's recorded byte hashes and administration fingerprint.+ * Older dirty archives lack patch hashes and cannot prove completeness later.+ */+async function verifyHeldArchive(repoPath, dir, release) {+ const { manifest: saved, digest } = await archiveManifest(dir);+ if (saved.repoPath !== repoPath || !sameRetirementCheckout(saved, release)+ || (saved.release ? !sameRelease(saved.release, release) : release.locked !== false || release.lockReason !== null)) throw new Error('archive-identity-mismatch');+ if (!(Date.parse(saved.archivedAt) >= Date.parse(release.createdAt)) || Date.parse(saved.archivedAt) > Date.now()) throw new Error('archive-generation-mismatch');+ const captured = saved.captured;+ if (captured?.commitBundle !== true || captured.missingCheckout || !Array.isArray(captured.files)+ || !Array.isArray(captured.symlinks) || !Array.isArray(captured.gitlinks)+ || !Array.isArray(saved.administration?.files) || !Array.isArray(saved.administration?.symlinks)+ || !saved.administration.files.some(file => file.path === 'gitdir/HEAD')+ || !saved.administration.files.some(file => file.path === 'git-indirection')) throw new Error('archive-preservation-incomplete');+ if (captured.files.length + saved.administration.files.length > 1000) throw new Error('archive-file-budget-exceeded');+ const stamps = [];+ const stamp = async path => {+ const info = await lstat(path, { bigint: true });+ if (!info.isFile()) throw new Error('archive-file-invalid');+ return [info.dev, info.ino, info.size, info.mtimeNs, info.ctimeNs].map(String).join(':');+ };+ const remember = async path => { stamps.push([path, await stamp(path)]); };+ await remember(join(dir, 'manifest.json'));+ let total = 0;+ for (const [prefix, files] of [['files', captured.files], ['administration', saved.administration.files]]) {+ for (const file of files) {+ if (typeof file.path !== 'string' || !/^[a-f0-9]{64}$/.test(file.sha256 || '')) throw new Error('archive-file-proof-invalid');+ const path = await archiveFile(dir, join(dir, prefix, file.path));+ await remember(path);+ const info = await lstat(path);+ if (info.size !== file.bytes || info.size > MAX_FILE_BYTES || (total += info.size) > MAX_TOTAL_BYTES+ || await sha256File(path) !== file.sha256) throw new Error('archive-file-changed');+ }+ }+ const administrationFingerprint = evidenceHash(JSON.stringify({+ files: saved.administration.files.filter(file => file.path !== 'gitdir/locked'), symlinks: saved.administration.symlinks,+ }));+ if (administrationFingerprint !== saved.administration.fingerprint) throw new Error('archive-administration-changed');+ for (const name of ['index', 'worktree']) {+ if (typeof captured[`${name}Diff`] !== 'boolean') throw new Error('archive-patch-proof-invalid');+ if (!captured[`${name}Diff`]) continue;+ const expected = saved.patchHashes?.[name];+ if (!/^[a-f0-9]{64}$/.test(expected || '')) throw new Error('archive-patch-proof-unavailable');+ const path = await archiveFile(dir, join(dir, `${name}.diff`));+ await remember(path);+ if (await sha256File(path) !== expected) throw new Error('archive-patch-changed');+ }+ const bundle = await archiveFile(dir, join(dir, 'commits.bundle'));+ await remember(bundle);+ // A thin bundle verifies against the current repository but is not a complete+ // restore point. Inspect only its bounded textual header, never fetch it.+ const handle = await open(bundle, 'r');+ let header;+ try {+ const buffer = Buffer.alloc(256 * 1024);+ const { bytesRead } = await handle.read(buffer, 0, buffer.length, 0);+ header = buffer.subarray(0, bytesRead).toString('latin1').split('\n\n')[0];+ if (!buffer.subarray(0, bytesRead).includes(Buffer.from('\n\n')) || /^-/m.test(header)) throw new Error('archive-bundle-incomplete');+ } finally { await handle.close(); }+ await git(['bundle', 'verify', bundle], repoPath);+ const heads = new Set((await git(['bundle', 'list-heads', bundle], repoPath)).trim().split('\n').map(line => line.split(' ')[0]));+ if (!Array.isArray(captured.recoveryHeads) || !captured.recoveryHeads.includes(release.head)+ || captured.recoveryHeads.some(head => !heads.has(head))) throw new Error('archive-bundle-head-mismatch');+ const bundleHash = await sha256File(bundle);+ if (saved.bundleSha256 && bundleHash !== saved.bundleSha256) throw new Error('archive-bundle-changed');+ if ((await archiveManifest(dir)).digest !== digest) throw new Error('archive-manifest-changed');+ const recheck = async () => {+ for (const [path, original] of stamps) {+ if (await stamp(await archiveFile(dir, path)) !== original) return false;+ }+ return true;+ };+ if (!await recheck()) throw new Error('archive-changed-during-verification');+ return { repoPath, archiveDir: dir, archivedAt: saved.archivedAt, evidenceDigest: evidenceHash(digest + bundleHash), kind: 'held', recheck };+}++async function historicalCheckoutAbsent(repoPath, release) {+ const present = await lstat(release.path).catch(error => { if (error.code === 'ENOENT') return null; throw error; });+ const trees = await listWorktrees(repoPath);+ return !present && trees.length > 0 && !trees.some(tree => tree.path === release.path || (release.branch && shortBranch(tree.branch) === release.branch));+}++/** Read existing archives only. Verification stays outside both writer locks;+ * the final ownership/absence recheck and schedule compare-and-set are fenced.+ */+export async function recoverMissingCheckoutPreservations({ appId, repoPath, outcome }, opts = {}) {+ const expected = structuredClone(outcome);+ const obligations = unresolvedRetirements(expected);+ if (!obligations.length) return outcome;+ const roots = [heldCheckoutArchiveRoot(opts.cosDir), join(opts.cosDir || PATHS.cos, 'initialized-checkout-retirements')];+ const candidates = [];+ let discoveryReason = null;+ for (const root of roots) {+ try {+ const info = await lstat(root).catch(error => { if (error.code === 'ENOENT') return null; throw error; });+ if (!info) continue;+ if (!info.isDirectory()) throw new Error('archive-directory-invalid');+ const names = await readdir(root, { withFileTypes: true });+ if (names.length + candidates.length > 1000) throw new Error('archive-discovery-budget-exceeded');+ for (const name of names) if (name.isDirectory()) candidates.push({ dir: join(root, name.name), kind: root === roots[0] ? 'held' : 'initialized' });+ } catch { discoveryReason = 'archive-discovery-unavailable'; }+ }+ const observations = [];+ for (const release of obligations) {+ if (!await historicalCheckoutAbsent(repoPath, release).catch(() => false)) continue;+ const evidence = [];+ let proof = null;+ for (const candidate of candidates) {+ try {+ proof = candidate.kind === 'held' ? await verifyHeldArchive(repoPath, candidate.dir, release)+ : await (await import('./initializedCheckoutRecovery.js')).verifyMissingInitializedCheckoutPreservation({ appId, repoPath, release, archiveDir: candidate.dir });+ evidence.push({ archiveDir: candidate.dir, reason: 'preservation-verified' });+ break;+ } catch (error) {+ evidence.push({ archiveDir: candidate.dir, reason: /^[a-z][a-z-]+$/.test(error.message) ? error.message : 'archive-unverifiable' });+ }+ }+ observations.push({ release, proof, evidence, reason: discoveryReason || (candidates.length ? 'archive-evidence-unverified' : 'archive-evidence-missing'), nextAction: historicalNextAction });+ }+ if (!observations.length) return outcome;+ const { recordRecoveredCheckoutPreservations } = await import('./taskSchedule.js');+ return withClaimOwnershipLock(() => recordRecoveredCheckoutPreservations(appId, expected, observations, async () => {+ const { getAppById } = await import('./apps.js');+ if ((await getAppById(appId))?.repoPath !== repoPath) return false;+ for (const row of observations) {+ if (!await historicalCheckoutAbsent(repoPath, row.release)) return false;+ if (row.proof?.recheck && !await row.proof.recheck().catch(() => false)) {+ row.evidence.push({ archiveDir: row.proof.archiveDir, reason: 'archive-changed-during-verification' });+ row.proof = null;+ row.reason = 'archive-changed-during-verification';+ }+ }+ return true;+ }));+}+// Refuse rather than archive a subset: a partial archive followed by removal// would delete the rest. Larger than the superseded reap's caps because this is// an explicit operator action, but still bounded so a build tree cannot fill@@ -393,6 +555,8 @@ async function writeArchive(dir, { repoPath, tree, branch, defaultBranch, releasawait atomicWrite(join(dir, 'manifest.json'), {repoPath, defaultBranch, path: tree.path, head: tree.head, branch,release, review, administration: { ...administration, files: administration.files.map(({ source: _source, ...file }) => file) }, archivedAt: new Date().toISOString(),+ bundleSha256: await sha256File(bundle),+ patchHashes: { index: evidenceHash(snapshot.indexDiff), worktree: evidenceHash(snapshot.worktreeDiff) },captured: { commitBundle: true, recoveryHeads: [...recoveryHeads], ...(snapshot.missingCheckout ? { missingCheckout: true } : {}), gitlinks: snapshot.gitlinks, indexDiff: Boolean(snapshot.indexDiff), worktreeDiff: Boolean(snapshot.worktreeDiff), files, symlinks },restore: ['git -C <repo> fetch commits.bundle HEAD:refs/heads/<restored-branch>',diff --git a/server/services/heldCheckoutRecovery.test.js b/server/services/heldCheckoutRecovery.test.jsindex 81e37689bd..810fdf5474 100644--- a/server/services/heldCheckoutRecovery.test.js+++ b/server/services/heldCheckoutRecovery.test.js@@ -20,13 +20,14 @@ vi.mock('../lib/paths.js', async importOriginal => makePathsProxy(await importOrafterAll(cleanupTempDataRoots);const SYMLINKS = process.platform !== 'win32';-const injected = vi.hoisted(() => ({ policies: [], schedule: null, app: null, fetchRemote: null, fetches: [], onBundle: null }));+const injected = vi.hoisted(() => ({ policies: [], schedule: null, app: null, fetchRemote: null, fetches: [], onBundle: null, onEvidence: null }));vi.mock('../lib/execGit.js', async importOriginal => {const actual = await importOriginal();return { ...actual, execGit: async (args, cwd, options) => {// Pauses an archive mid-write, after its first snapshot and before its re-read.if (injected.onBundle && args.includes('bundle') && args.includes('create')) await injected.onBundle();+ if (injected.onEvidence && args[0] === 'bundle' && args[1] === 'verify') await injected.onEvidence();if (args.includes('credential.helper=!gh auth git-credential') && args.includes('fetch')) {injected.fetches.push({ args, options });if (injected.fetchRemote) args = args.map(arg => arg === 'https://github.com/example/repo.git' ? injected.fetchRemote : arg);@@ -49,7 +50,7 @@ vi.mock('./taskScheduleStore.js', async (importOriginal) => ({}));vi.mock('./apps.js', () => ({ getAppById: async (id) => (injected.app?.id === id ? injected.app : null) }));-const { retireInventoriedCheckout, prepareInventoriedCheckoutRetirement, retirePendingMergeCheckout, heldCheckoutArchiveRoot, preserveMergedCheckout } = await import('./heldCheckoutRecovery.js');+const { retireInventoriedCheckout, prepareInventoriedCheckoutRetirement, retirePendingMergeCheckout, heldCheckoutArchiveRoot, preserveMergedCheckout, recoverMissingCheckoutPreservations } = await import('./heldCheckoutRecovery.js');const { recordBranchReconcileOutcome, getBranchReconcileOutcome } = await import('./taskSchedule.js');const { withClaimOwnershipLock, checkReconcileOwnership } = await import('./cosClaimOwnership.js');@@ -61,7 +62,7 @@ describe.skipIf(SKIP_HEAVY_INTEGRATION)('held checkout recovery (real git)', ()const noOwners = async () => [];beforeEach(async () => {- Object.assign(injected, { policies: [], schedule: { executions: {} }, app: { id: 'app-1', repoPath: null }, fetchRemote: null, fetches: [], onBundle: null });+ Object.assign(injected, { policies: [], schedule: { executions: {} }, app: { id: 'app-1', repoPath: null }, fetchRemote: null, fetches: [], onBundle: null, onEvidence: null });scratch = realpathSync(await mkdtemp(join(tmpdir(), 'portos-held-checkout-')));const safeRepo = join(scratch, 'repo');await mkdir(safeRepo);@@ -104,6 +105,84 @@ describe.skipIf(SKIP_HEAVY_INTEGRATION)('held checkout recovery (real git)', ()}const archives = async () => (existsSync(heldCheckoutArchiveRoot(cosDir)) ? readdir(heldCheckoutArchiveRoot(cosDir)) : []);+ async function historicalArchive() {+ const path = join(external, 'historical');+ const branch = 'feature/historical';+ await execGit(['worktree', 'add', '-b', branch, path, 'main'], repo);+ await writeFile(join(path, 'initial.txt'), 'staged historical work\n');+ await execGit(['add', 'initial.txt'], path);+ await writeFile(join(path, 'private.txt'), 'synthetic private bytes');+ const release = { path, branch, head: (await execGit(['rev-parse', 'HEAD'], path)).stdout.trim(), locked: false, lockReason: null };+ await recordBranchReconcileOutcome('app-1', { status: 'incomplete', remaining: [{ ...release, reason: 'retirement-review-pending' }] });+ const saved = await preserveMergedCheckout(repo, { ...release, branch: `refs/heads/${branch}` }, 'main', { cosDir: PATHS.cos });+ await execGit(['worktree', 'remove', '--force', path], repo);+ return { release, saved };+ }++ it('recovers complete preservation through the schedule boundary with only the primary checkout, without claiming branch deletion', async () => {+ const { release, saved } = await historicalArchive();+ const before = (await execGit(['show-ref'], repo)).stdout;+ const outcome = await recordBranchReconcileOutcome('app-1', { status: 'incomplete', remaining: [] });+ expect(outcome.retirementObligations).toEqual([]);+ expect(outcome.remaining).toEqual([]);+ expect(outcome.recoveredPreservations).toEqual([expect.objectContaining({ path: release.path, branch: release.branch, head: release.head,+ archiveDir: saved.archiveDir, verified: true,+ evidenceDigest: expect.stringMatching(/^[a-f0-9]{64}$/),+ })]);+ expect(outcome.retired).toBeUndefined();+ expect(outcome.recoveredPreservations[0]).not.toHaveProperty('branchDeleted');+ expect(existsSync(release.path)).toBe(false);+ expect((await execGit(['show-ref'], repo)).stdout).toBe(before);+ // A new generation with the same branch/path/head cannot reuse the archive.+ const recreated = await recordBranchReconcileOutcome('app-1', { status: 'incomplete', retirementObligations: [{ ...release, reason: 'retirement-review-pending' }] });+ expect(recreated.retirementObligations).toHaveLength(1);+ expect(recreated.retirementObligations[0].preservationEvidence.archives).toContainEqual({ archiveDir: saved.archiveDir, reason: 'archive-generation-mismatch' });+ });++ it.each(['missing', 'wrong-repository', 'wrong-path', 'wrong-head', 'changed-file', 'changed-patch', 'legacy-dirty', 'partial-original'])(+ 'retains an actionable historical obligation for %s preservation evidence', async failure => {+ const { release, saved } = await historicalArchive();+ const manifestPath = join(saved.archiveDir, 'manifest.json');+ const manifest = JSON.parse(await readFile(manifestPath, 'utf8'));+ if (failure === 'missing') await rm(saved.archiveDir, { recursive: true, force: true });+ if (failure === 'wrong-repository') manifest.repoPath = join(scratch, 'other-repo');+ if (failure === 'wrong-path') manifest.path = join(external, 'other-checkout');+ if (failure === 'wrong-head') manifest.head = 'f'.repeat(40);+ if (failure === 'changed-file') await writeFile(join(saved.archiveDir, 'files', 'private.txt'), 'changed archive bytes');+ if (failure === 'changed-patch') await writeFile(join(saved.archiveDir, 'index.diff'), 'changed patch bytes');+ if (failure === 'legacy-dirty') delete manifest.patchHashes;+ if (failure === 'partial-original') manifest.captured.missingCheckout = true;+ if (failure !== 'missing') await writeFile(manifestPath, JSON.stringify(manifest));+ const outcome = await recordBranchReconcileOutcome('app-1', { status: 'incomplete', remaining: [] });+ expect(outcome.retirementObligations).toEqual([expect.objectContaining({ ...release,+ preservationEvidence: expect.objectContaining({ status: 'unresolved', nextAction: expect.stringContaining('backup') }),+ })]);+ expect(outcome).toMatchObject({ status: 'incomplete', reason: 'historical-preservation-unresolved', recoveredPreservations: [] });+ expect(outcome.remaining).toEqual(outcome.retirementObligations);+ expect(existsSync(release.path)).toBe(false);+ });++ it.each(['recreated-path', 'new-holder', 'inventory-change', 'repository-change'])(+ 'does not settle historical evidence after a concurrent %s', async change => {+ const { release } = await historicalArchive();+ const outcome = await getBranchReconcileOutcome('app-1');+ injected.onEvidence = async () => {+ injected.onEvidence = null;+ if (change === 'recreated-path') await mkdir(release.path);+ if (change === 'new-holder') await execGit(['worktree', 'add', join(external, 'new-holder'), release.branch], repo);+ if (change === 'repository-change') injected.app.repoPath = join(scratch, 'changed-repo');+ if (change === 'inventory-change') {+ const current = injected.schedule.executions['task:branch-reconcile'].perApp['app-1'];+ current.branchReconcileOutcome = { ...current.branchReconcileOutcome, observedAt: 'changed-inventory' };+ }+ };+ await recoverMissingCheckoutPreservations({ appId: 'app-1', repoPath: repo, outcome });+ const current = await getBranchReconcileOutcome('app-1');+ expect(current.retirementObligations).toHaveLength(1);+ expect(current.recoveredPreservations).toBeUndefined();+ if (change === 'inventory-change') expect(current.observedAt).toBe('changed-inventory');+ });+it('preserves ignored evidence before unattended cleanup and refuses later byte or head changes', async () => {const path = join(external, 'merged');await execGit(['worktree', 'add', '-b', 'feature/done', path, 'main'], repo);diff --git a/server/services/initializedCheckoutRecovery.js b/server/services/initializedCheckoutRecovery.jsindex e55f8bc749..5b782ba6f4 100644--- a/server/services/initializedCheckoutRecovery.js+++ b/server/services/initializedCheckoutRecovery.js@@ -507,6 +507,29 @@ export function _createInitializedCheckoutRecovery(deps = {}) {verified: true, branchDeleted: state.branchDeleted ?? false, proof };};+ const verifyMissingPreservation = async ({ appId, repoPath, release, archiveDir }) => {+ const directory = location(archiveDir.split(/[\\/]/).at(-1));+ if (directory !== archiveDir) fail('archive-location-mismatch');+ const state = await readJournal(directory, { bounded: true });+ if (state.appId !== appId || state.repoPath !== repoPath || !sameRelease(state.release, release)) fail('archive-identity-mismatch');+ if (!['branch-retired', 'retired'].includes(state.phase) || state.branchDeleted !== true) fail('archive-retirement-unconfirmed');+ const archivedAt = state.reservation?.owner?.startedAt;+ if (!(Date.parse(archivedAt) >= Date.parse(release.createdAt)) || Date.parse(archivedAt) > Date.now()) fail('archive-generation-mismatch');+ // The journal locates the manifest; the existing native verifier checks its+ // digest, every preserved source and the standalone object closure.+ const manifestPath = join(directory, 'manifest.json');+ const info = await lstat(manifestPath);+ if (!info.isFile() || info.isSymbolicLink() || info.size > 16 * 1024 * 1024) fail('archive-manifest-invalid');+ const manifest = JSON.parse(await readFile(manifestPath, 'utf8'));+ if (manifest.facts?.repoPath !== repoPath || !sameRelease(manifest.facts?.release, release)) fail('archive-identity-mismatch');+ const result = await verifyArchive({ appId, transactionId: state.id, journalDigest: hash(state) });+ if (result.proof?.verified !== true || result.proof?.standalone !== true) fail('archive-preservation-unverified');+ if (hash(await readJournal(directory, { bounded: true })) !== hash(state)) fail('retirement-journal-changed');+ const recheck = async () => hash(await readJournal(directory, { bounded: true })) === hash(state)+ && createHash('sha256').update(await readFile(manifestPath)).digest('hex') === state.proof.manifestSha256;+ return { repoPath, archiveDir: directory, archivedAt, evidenceDigest: state.proof.manifestSha256, kind: 'initialized', recheck };+ };+// Explicit recovery restores the exact original inodes. It does not erase// archives or auto-finish an interrupted ref deletion on stale forge evidence.const restore = (input, { cancelOnly = false } = {}) => fenced(async () => {@@ -959,7 +982,7 @@ export function _createInitializedCheckoutRecovery(deps = {}) {const automatic = async (input, options) => recoverAutomatically(input, options).catch(error => ({ ok: false, reason: error.message, transactionId: input.transactionId, recoveryRequired: true }));return { discover, recoverAutomatically: automatic, review, retire, status, restore,- cancelPreservation: input => restore(input, { cancelOnly: true }), verifyArchive, reviewBranch, retireBranch, reviewMetadataRestore, restoreMetadata,+ cancelPreservation: input => restore(input, { cancelOnly: true }), verifyArchive, verifyMissingPreservation, reviewBranch, retireBranch, reviewMetadataRestore, restoreMetadata,reviewRestoredFinalization, finalizeRestored, reviewQuarantineResumption, resumeQuarantine };}@@ -970,6 +993,7 @@ export const initializedCheckoutStatus = recovery.status;export const restoreInitializedCheckout = recovery.restore;export const verifyInitializedCheckoutArchive = recovery.verifyArchive;+export const verifyMissingInitializedCheckoutPreservation = recovery.verifyMissingPreservation;export const reviewInitializedCheckoutBranch = recovery.reviewBranch;export const retireInitializedCheckoutBranch = recovery.retireBranch;diff --git a/server/services/initializedCheckoutRecovery.test.js b/server/services/initializedCheckoutRecovery.test.jsindex 7e7b42b2a1..f852b72162 100644--- a/server/services/initializedCheckoutRecovery.test.js+++ b/server/services/initializedCheckoutRecovery.test.js@@ -166,6 +166,7 @@ beforeEach(async () => {}if (fixture.failArchive && input.command === 'verify') throw new Error('archive-changed');return { ok: true, placement: fixture.placement, fingerprint: fixture.snapshot.fingerprint,+ ...(input.command === 'verify' && input.mode === 'standalone' ? { verified: true, standalone: true } : {}),...(input.metadataRecovery && !alreadyRestored ? {postMoveObservation: { file: `${input.part}-post-move.json`, sha256: fixture.metadataToken, digest: fixture.metadataToken },} : {}),@@ -962,6 +963,24 @@ describe('initialized checkout transaction', () => {expect(fixture.cut).toBeNull();expect((await fixture.service.status(input)).phase).toBe('retired');});+ it('binds missing-original recovery to the historical journal generation and native standalone proof', async () => {+ const createdAt = new Date(Date.now() - 1000).toISOString();+ const input = await quarantinedReview();+ await fixture.service.retireBranch(input);+ const archiveDir = join(root, input.transactionId);+ await writeFile(join(archiveDir, 'manifest.json'), JSON.stringify({ facts: { repoPath: '/fixture/repo', release } }));+ const request = { appId: 'app', repoPath: '/fixture/repo', archiveDir, release: { ...release, createdAt } };+ fixture.deps.native.mockClear(); fixture.deps.deleteRef.mockClear();+ expect(await fixture.service.verifyMissingPreservation(request)).toMatchObject({ archiveDir, kind: 'initialized', evidenceDigest: 'e'.repeat(64) });+ expect(fixture.deps.native.mock.calls.map(([value]) => value.command)).toEqual(['verify']);+ expect(fixture.deps.deleteRef).not.toHaveBeenCalled();+ await expect(fixture.service.verifyMissingPreservation({ ...request, release: { ...request.release, head: 'f'.repeat(40) } }))+ .rejects.toThrow('archive-identity-mismatch');+ await expect(fixture.service.verifyMissingPreservation({ ...request, release: { ...release, createdAt: new Date(Date.now() + 1000).toISOString() } }))+ .rejects.toThrow('archive-generation-mismatch');+ fixture.failArchive = true;+ await expect(fixture.service.verifyMissingPreservation(request)).rejects.toThrow('archive-changed');+ });it('invalidates branch finalization when the default advances or a holder appears', async () => {const input = await quarantinedReview();fixture.facts.defaultHead = 'e'.repeat(40);diff --git a/server/services/taskSchedule.js b/server/services/taskSchedule.jsindex 6082e5e710..226c9fd8b4 100644--- a/server/services/taskSchedule.js+++ b/server/services/taskSchedule.js@@ -770,7 +770,7 @@ export async function isPerpetualParkActive(taskType, appId = null) {/** Persist the latest deterministic sweep evidence independently of agent success. */export async function recordBranchReconcileOutcome(appId, { status, reason, remaining, nextAction, retirementObligations: additionalObligations = [] }) {- return updateSchedule(async (schedule) => {+ const outcome = await updateSchedule(async (schedule) => {const record = ensureExecutionRecord(schedule, 'branch-reconcile', appId);const now = new Date().toISOString();const previous = record.branchReconcileOutcome;@@ -788,6 +788,50 @@ export async function recordBranchReconcileOutcome(appId, { status, reason, rema};return { result: record.branchReconcileOutcome, changed: true };});+ if (!appId || !unresolvedRetirements(outcome).length) return outcome;+ const app = await getAppById(appId).catch(() => null);+ if (!app?.repoPath) return outcome;+ // Keep the expensive archive verifier off ordinary live-checkout writes.+ const missing = await findGoneInventoryRows(app.repoPath, unresolvedRetirements(outcome), { includeObligations: true }).catch(() => []);+ if (!missing.length) return outcome;+ const { recoverMissingCheckoutPreservations } = await import('./heldCheckoutRecovery.js');+ return recoverMissingCheckoutPreservations({ appId, repoPath: app.repoPath, outcome });+}++/** Compare the whole observed inventory and its generation inside the schedule+ * writer. A verifier result never authorizes a newer/recreated checkout.+ */+export async function recordRecoveredCheckoutPreservations(appId, expected, observations, stillAbsent) {+ return updateSchedule(async schedule => {+ const record = ensureExecutionRecord(schedule, 'branch-reconcile', appId);+ const outcome = record.branchReconcileOutcome;+ if (JSON.stringify(outcome) !== JSON.stringify(expected) || !await stillAbsent()) return { result: outcome, changed: false };+ const now = new Date().toISOString();+ const recoveredPreservations = [...(outcome.recoveredPreservations || [])];+ const diagnosed = unresolvedRetirements(outcome).map(release => {+ const observation = observations.find(row => sameRetirementCheckout(row.release, release) && row.release.createdAt === release.createdAt);+ if (!observation) return release;+ const accepted = observation.proof && recoveredPreservations.length < 1000;+ const { recheck: _recheck, ...proof } = observation.proof || {};+ if (accepted) recoveredPreservations.push({+ branch: release.branch, path: release.path, head: release.head,+ obligationCreatedAt: release.createdAt, ...proof, verified: true, verifiedAt: now,+ });+ return { ...release, preservationEvidence: { status: 'unresolved',+ reason: observation.proof && !accepted ? 'preservation-receipt-budget-exceeded' : observation.reason,+ checkedAt: now, archives: observation.evidence, nextAction: observation.nextAction } };+ });+ const retirementObligations = unresolvedRetirements({ retirementObligations: diagnosed, recoveredPreservations });+ const remaining = (outcome.remaining || []).flatMap(row => {+ const diagnosedRow = diagnosed.find(release => sameRetirementCheckout(row, release) && row.createdAt === release.createdAt);+ return diagnosedRow ? retirementObligations.filter(release => sameRetirementCheckout(release, diagnosedRow)) : [row];+ });+ const historical = retirementObligations.find(row => row.preservationEvidence);+ record.branchReconcileOutcome = { ...outcome, retirementObligations, recoveredPreservations, remaining, updatedAt: now,+ ...(historical ? { status: 'incomplete', reason: 'historical-preservation-unresolved', nextAction: historical.preservationEvidence.nextAction } : {}),+ };+ return { result: record.branchReconcileOutcome, changed: true };+ });}/** Observations are diagnostic only; journals and fresh service proofs own recovery. */