Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS
JavaScript ai-agentsdashboarddeveloper-toolsdigital-twinquantified-selfsecond-brainself-hostedtailscale
fix: prove archived object closure and fence native evidence (#13267)
Adam Eivy70beb6423b26
diff --git a/server/services/heldCheckoutRecovery.js b/server/services/heldCheckoutRecovery.jsindex f32136c0d3..6c1753a70a 100644--- a/server/services/heldCheckoutRecovery.js+++ b/server/services/heldCheckoutRecovery.js@@ -20,7 +20,8 @@import { findGoneInventoryRows } from './branchReconcileInventoryPrune.js';import { createReadStream, realpathSync } from 'node:fs';-import { lstat, stat, readlink, readdir, readFile, chmod, open } from 'node:fs/promises';+import { lstat, stat, readlink, readdir, readFile, chmod, open, mkdtemp } from 'node:fs/promises';+import { devNull, tmpdir } from 'node:os';import { createHash, randomUUID } from 'node:crypto';import { basename, dirname, isAbsolute, join, resolve, relative, sep } from 'node:path';import { sameRetirementCheckout, unresolvedRetirements } from '../lib/reconcileRetirementEvidence.js';@@ -125,6 +126,19 @@ async function verifyHeldArchive(repoPath, dir, release) {|| captured.recoveryHeads.some(head => !heads.has(head))) throw new Error('archive-bundle-head-mismatch');const bundleHash = await sha256File(bundle);if (saved.bundleSha256 && bundleHash !== saved.bundleSha256) throw new Error('archive-bundle-changed');+ // Bundle verify/list-heads validate headers, not packed object integrity. Prove+ // the complete closure without borrowing any object from the live repository.+ const scratch = await mkdtemp(join(tmpdir(), 'portos-retirement-proof-'));+ const env = { ...Object.fromEntries(Object.entries(process.env).filter(([key]) => !key.toUpperCase().startsWith('GIT_'))),+ GIT_CONFIG_NOSYSTEM: '1', GIT_CONFIG_GLOBAL: devNull, GIT_NO_REPLACE_OBJECTS: '1', GIT_NO_LAZY_FETCH: '1', GIT_TERMINAL_PROMPT: '0' };+ const prove = args => execGit(['--no-optional-locks', '-c', `core.hooksPath=${devNull}`, '-c', 'core.fsmonitor=false', ...args], scratch,+ { env, maxBuffer: GIT_OUTPUT_BYTES });+ try {+ await prove(['init', '--bare', `--object-format=${release.head.length === 64 ? 'sha256' : 'sha1'}`, '.']);+ await prove(['bundle', 'unbundle', bundle]);+ await prove(['fsck', '--full', '--strict', '--no-reflogs', '--no-dangling', ...captured.recoveryHeads]);+ } catch { throw new Error('archive-bundle-object-proof-failed'); }+ finally { await rmGuarded(scratch, { recursive: true, force: true }); }if ((await archiveManifest(dir)).digest !== digest) throw new Error('archive-manifest-changed');const recheck = async () => {for (const [path, original] of stamps) {@@ -136,9 +150,8 @@ async function verifyHeldArchive(repoPath, dir, release) {return { repoPath, archiveDir: dir, archivedAt: saved.archivedAt, evidenceDigest: evidenceHash(digest + bundleHash), kind: 'held', recheck };}-async function historicalCheckoutAbsent(repoPath, release) {+async function historicalCheckoutAbsent(release, trees) {const present = await lstat(release.path).catch(error => { if (error.code === 'ENOENT') return null; throw error; });- const trees = await listWorktrees(repoPath);return !present && trees.length > 0 && !trees.some(tree => tree.path === release.path || (release.branch && shortBranch(tree.branch) === release.branch));}@@ -149,6 +162,8 @@ export async function recoverMissingCheckoutPreservations({ appId, repoPath, outconst expected = structuredClone(outcome);const obligations = unresolvedRetirements(expected);if (!obligations.length) return outcome;+ const trees = await listWorktrees(repoPath).catch(() => null);+ if (!trees?.length) return outcome;const roots = [heldCheckoutArchiveRoot(opts.cosDir), join(opts.cosDir || PATHS.cos, 'initialized-checkout-retirements')];const candidates = [];let discoveryReason = null;@@ -163,37 +178,44 @@ export async function recoverMissingCheckoutPreservations({ appId, repoPath, out} catch { discoveryReason = 'archive-discovery-unavailable'; }}const observations = [];- for (const release of obligations) {- if (!await historicalCheckoutAbsent(repoPath, release).catch(() => false)) continue;- const evidence = [];- let proof = null;- for (const candidate of candidates) {- try {- proof = candidate.kind === 'held' ? await verifyHeldArchive(repoPath, candidate.dir, release)- : await (await import('./initializedCheckoutRecovery.js')).verifyMissingInitializedCheckoutPreservation({ appId, repoPath, release, archiveDir: candidate.dir });- evidence.push({ archiveDir: candidate.dir, reason: 'preservation-verified' });- break;- } catch (error) {- evidence.push({ archiveDir: candidate.dir, reason: /^[a-z][a-z-]+$/.test(error.message) ? error.message : 'archive-unverifiable' });+ const evidenceFences = [];+ try {+ for (const release of obligations) {+ if (!await historicalCheckoutAbsent(release, trees).catch(() => false)) continue;+ const evidence = [];+ let proof = null;+ for (const candidate of candidates) {+ try {+ proof = candidate.kind === 'held' ? await verifyHeldArchive(repoPath, candidate.dir, release)+ : await (await import('./initializedCheckoutRecovery.js')).verifyMissingInitializedCheckoutPreservation({ appId, repoPath, release, archiveDir: candidate.dir });+ if (proof.releaseFence) evidenceFences.push(proof.releaseFence);+ evidence.push({ archiveDir: candidate.dir, reason: 'preservation-verified' });+ break;+ } catch (error) {+ evidence.push({ archiveDir: candidate.dir, reason: /^[a-z][a-z-]+$/.test(error.message) ? error.message : 'archive-unverifiable' });+ }}+ observations.push({ release, proof, evidence, reason: discoveryReason || (candidates.length ? 'archive-evidence-unverified' : 'archive-evidence-missing'), nextAction: historicalNextAction });}- observations.push({ release, proof, evidence, reason: discoveryReason || (candidates.length ? 'archive-evidence-unverified' : 'archive-evidence-missing'), nextAction: historicalNextAction });- }- if (!observations.length) return outcome;- const { recordRecoveredCheckoutPreservations } = await import('./taskSchedule.js');- return withClaimOwnershipLock(() => recordRecoveredCheckoutPreservations(appId, expected, observations, async () => {- const { getAppById } = await import('./apps.js');- if ((await getAppById(appId))?.repoPath !== repoPath) return false;- for (const row of observations) {- if (!await historicalCheckoutAbsent(repoPath, row.release)) return false;- if (row.proof?.recheck && !await row.proof.recheck().catch(() => false)) {- row.evidence.push({ archiveDir: row.proof.archiveDir, reason: 'archive-changed-during-verification' });- row.proof = null;- row.reason = 'archive-changed-during-verification';+ if (!observations.length) return outcome;+ const { recordRecoveredCheckoutPreservations } = await import('./taskSchedule.js');+ return await withClaimOwnershipLock(() => recordRecoveredCheckoutPreservations(appId, expected, observations, async () => {+ const { getAppById } = await import('./apps.js');+ if ((await getAppById(appId))?.repoPath !== repoPath) return false;+ const currentTrees = await listWorktrees(repoPath);+ for (const row of observations) {+ if (!await historicalCheckoutAbsent(row.release, currentTrees)) return false;+ if (row.proof?.recheck && !await row.proof.recheck().catch(() => false)) {+ row.evidence.push({ archiveDir: row.proof.archiveDir, reason: 'archive-changed-during-verification' });+ row.proof = null;+ row.reason = 'archive-changed-during-verification';+ }}- }- return true;- }));+ return true;+ }));+ } finally {+ for (const releaseFence of evidenceFences) await releaseFence();+ }}// Refuse rather than archive a subset: a partial archive followed by removaldiff --git a/server/services/heldCheckoutRecovery.test.js b/server/services/heldCheckoutRecovery.test.jsindex 810fdf5474..eb31d6dff5 100644--- a/server/services/heldCheckoutRecovery.test.js+++ b/server/services/heldCheckoutRecovery.test.js@@ -5,7 +5,7 @@* its lock and its branch as found.*/import { describe, it, expect, beforeEach, afterEach, afterAll, vi } from 'vitest';-import { cp, mkdtemp, mkdir, rm, rename, writeFile, readFile, readdir, symlink, utimes } from 'fs/promises';+import { cp, mkdtemp, mkdir, rm, rename, writeFile, readFile, readdir, symlink, utimes, truncate } from 'fs/promises';import { existsSync, realpathSync } from 'fs';import { tmpdir } from 'os';import { dirname, join } from 'path';@@ -139,7 +139,7 @@ describe.skipIf(SKIP_HEAVY_INTEGRATION)('held checkout recovery (real git)', ()expect(recreated.retirementObligations[0].preservationEvidence.archives).toContainEqual({ archiveDir: saved.archiveDir, reason: 'archive-generation-mismatch' });});- it.each(['missing', 'wrong-repository', 'wrong-path', 'wrong-head', 'changed-file', 'changed-patch', 'legacy-dirty', 'partial-original'])(+ it.each(['missing', 'wrong-repository', 'wrong-path', 'wrong-head', 'changed-file', 'changed-patch', 'legacy-dirty', 'legacy-corrupt-bundle', 'partial-original'])('retains an actionable historical obligation for %s preservation evidence', async failure => {const { release, saved } = await historicalArchive();const manifestPath = join(saved.archiveDir, 'manifest.json');@@ -150,6 +150,13 @@ describe.skipIf(SKIP_HEAVY_INTEGRATION)('held checkout recovery (real git)', ()if (failure === 'wrong-head') manifest.head = 'f'.repeat(40);if (failure === 'changed-file') await writeFile(join(saved.archiveDir, 'files', 'private.txt'), 'changed archive bytes');if (failure === 'changed-patch') await writeFile(join(saved.archiveDir, 'index.diff'), 'changed patch bytes');+ if (failure === 'legacy-corrupt-bundle') {+ delete manifest.bundleSha256;+ const bundlePath = join(saved.archiveDir, 'commits.bundle');+ const bundle = await readFile(bundlePath);+ // Preserve the valid textual header and destroy only the packed objects.+ await truncate(bundlePath, bundle.indexOf('\n\n') + 12);+ }if (failure === 'legacy-dirty') delete manifest.patchHashes;if (failure === 'partial-original') manifest.captured.missingCheckout = true;if (failure !== 'missing') await writeFile(manifestPath, JSON.stringify(manifest));diff --git a/server/services/initializedCheckoutRecovery.js b/server/services/initializedCheckoutRecovery.jsindex 5b782ba6f4..4c5191e872 100644--- a/server/services/initializedCheckoutRecovery.js+++ b/server/services/initializedCheckoutRecovery.js@@ -515,19 +515,31 @@ export function _createInitializedCheckoutRecovery(deps = {}) {if (!['branch-retired', 'retired'].includes(state.phase) || state.branchDeleted !== true) fail('archive-retirement-unconfirmed');const archivedAt = state.reservation?.owner?.startedAt;if (!(Date.parse(archivedAt) >= Date.parse(release.createdAt)) || Date.parse(archivedAt) > Date.now()) fail('archive-generation-mismatch');- // The journal locates the manifest; the existing native verifier checks its- // digest, every preserved source and the standalone object closure.- const manifestPath = join(directory, 'manifest.json');- const info = await lstat(manifestPath);- if (!info.isFile() || info.isSymbolicLink() || info.size > 16 * 1024 * 1024) fail('archive-manifest-invalid');- const manifest = JSON.parse(await readFile(manifestPath, 'utf8'));- if (manifest.facts?.repoPath !== repoPath || !sameRelease(manifest.facts?.release, release)) fail('archive-identity-mismatch');- const result = await verifyArchive({ appId, transactionId: state.id, journalDigest: hash(state) });- if (result.proof?.verified !== true || result.proof?.standalone !== true) fail('archive-preservation-unverified');- if (hash(await readJournal(directory, { bounded: true })) !== hash(state)) fail('retirement-journal-changed');- const recheck = async () => hash(await readJournal(directory, { bounded: true })) === hash(state)- && createHash('sha256').update(await readFile(manifestPath)).digest('hex') === state.proof.manifestSha256;- return { repoPath, archiveDir: directory, archivedAt, evidenceDigest: state.proof.manifestSha256, kind: 'initialized', recheck };+ // Every archive mutation takes this execution fence. Keep it until the+ // schedule writer accepts or refuses the evidence, not just through verify.+ const executor = await native({ command: 'lock', transactionDir: directory }, { keepOpen: true });+ try {+ if (hash(await readJournal(directory, { bounded: true })) !== hash(state)) fail('retirement-journal-changed');+ // The journal locates the manifest; the existing native verifier checks its+ // digest, every preserved source and the standalone object closure.+ const manifestPath = join(directory, 'manifest.json');+ const info = await lstat(manifestPath);+ if (!info.isFile() || info.isSymbolicLink() || info.size > 16 * 1024 * 1024) fail('archive-manifest-invalid');+ const manifest = JSON.parse(await readFile(manifestPath, 'utf8'));+ if (manifest.facts?.repoPath !== repoPath || !sameRelease(manifest.facts?.release, release)) fail('archive-identity-mismatch');+ const result = await verifyArchive({ appId, transactionId: state.id, journalDigest: hash(state) });+ if (result.proof?.verified !== true || result.proof?.standalone !== true) fail('archive-preservation-unverified');+ if (hash(await readJournal(directory, { bounded: true })) !== hash(state)) fail('retirement-journal-changed');+ const recheck = async () => {+ executor.assertHeld();+ const unchanged = hash(await readJournal(directory, { bounded: true })) === hash(state)+ && createHash('sha256').update(await readFile(manifestPath)).digest('hex') === state.proof.manifestSha256;+ executor.assertHeld();+ return unchanged;+ };+ return { repoPath, archiveDir: directory, archivedAt, evidenceDigest: state.proof.manifestSha256, kind: 'initialized', recheck,+ releaseFence: () => executor.release() };+ } catch (error) { await executor.release(); throw error; }};// Explicit recovery restores the exact original inodes. It does not erasediff --git a/server/services/initializedCheckoutRecovery.test.js b/server/services/initializedCheckoutRecovery.test.jsindex f852b72162..0363ab0808 100644--- a/server/services/initializedCheckoutRecovery.test.js+++ b/server/services/initializedCheckoutRecovery.test.js@@ -971,8 +971,20 @@ describe('initialized checkout transaction', () => {await writeFile(join(archiveDir, 'manifest.json'), JSON.stringify({ facts: { repoPath: '/fixture/repo', release } }));const request = { appId: 'app', repoPath: '/fixture/repo', archiveDir, release: { ...release, createdAt } };fixture.deps.native.mockClear(); fixture.deps.deleteRef.mockClear();- expect(await fixture.service.verifyMissingPreservation(request)).toMatchObject({ archiveDir, kind: 'initialized', evidenceDigest: 'e'.repeat(64) });- expect(fixture.deps.native.mock.calls.map(([value]) => value.command)).toEqual(['verify']);+ const originalNative = fixture.deps.native.getMockImplementation();+ const releaseFence = vi.fn(async () => {});+ fixture.deps.native.mockImplementation((input, options) => input.command === 'lock'+ ? { assertHeld: () => { if (fixture.fenceLost) throw new Error('retirement-executor-lost'); }, release: releaseFence }+ : originalNative(input, options));+ const recovered = await fixture.service.verifyMissingPreservation(request);+ expect(recovered).toMatchObject({ archiveDir, kind: 'initialized', evidenceDigest: 'e'.repeat(64) });+ expect(fixture.deps.native.mock.calls.map(([value]) => value.command)).toEqual(['lock', 'verify']);+ expect(releaseFence).not.toHaveBeenCalled();+ fixture.fenceLost = true;+ await expect(recovered.recheck()).rejects.toThrow('retirement-executor-lost');+ await recovered.releaseFence();+ expect(releaseFence).toHaveBeenCalledOnce();+ fixture.fenceLost = false;expect(fixture.deps.deleteRef).not.toHaveBeenCalled();await expect(fixture.service.verifyMissingPreservation({ ...request, release: { ...request.release, head: 'f'.repeat(40) } })).rejects.toThrow('archive-identity-mismatch');@@ -980,6 +992,7 @@ describe('initialized checkout transaction', () => {.rejects.toThrow('archive-generation-mismatch');fixture.failArchive = true;await expect(fixture.service.verifyMissingPreservation(request)).rejects.toThrow('archive-changed');+ expect(releaseFence).toHaveBeenCalledTimes(2);});it('invalidates branch finalization when the default advances or a holder appears', async () => {const input = await quarantinedReview();diff --git a/server/services/taskSchedule.js b/server/services/taskSchedule.jsindex 226c9fd8b4..8659c8518b 100644--- a/server/services/taskSchedule.js+++ b/server/services/taskSchedule.js@@ -812,7 +812,7 @@ export async function recordRecoveredCheckoutPreservations(appId, expected, obseconst observation = observations.find(row => sameRetirementCheckout(row.release, release) && row.release.createdAt === release.createdAt);if (!observation) return release;const accepted = observation.proof && recoveredPreservations.length < 1000;- const { recheck: _recheck, ...proof } = observation.proof || {};+ const { recheck: _recheck, releaseFence: _releaseFence, ...proof } = observation.proof || {};if (accepted) recoveredPreservations.push({branch: release.branch, path: release.path, head: release.head,obligationCreatedAt: release.createdAt, ...proof, verified: true, verifiedAt: now,