Skip to content

Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS

fix: prove archived object closure and fence native evidence (#13267)

Adam Eivy70beb6423b26
diff --git a/server/services/heldCheckoutRecovery.js b/server/services/heldCheckoutRecovery.js
index f32136c0d3..6c1753a70a 100644
--- a/server/services/heldCheckoutRecovery.js
+++ b/server/services/heldCheckoutRecovery.js
@@ -20,7 +20,8 @@
import { findGoneInventoryRows } from './branchReconcileInventoryPrune.js';
import { createReadStream, realpathSync } from 'node:fs';
-import { lstat, stat, readlink, readdir, readFile, chmod, open } from 'node:fs/promises';
+import { lstat, stat, readlink, readdir, readFile, chmod, open, mkdtemp } from 'node:fs/promises';
+import { devNull, tmpdir } from 'node:os';
import { createHash, randomUUID } from 'node:crypto';
import { basename, dirname, isAbsolute, join, resolve, relative, sep } from 'node:path';
import { sameRetirementCheckout, unresolvedRetirements } from '../lib/reconcileRetirementEvidence.js';
@@ -125,6 +126,19 @@ async function verifyHeldArchive(repoPath, dir, release) {
|| captured.recoveryHeads.some(head => !heads.has(head))) throw new Error('archive-bundle-head-mismatch');
const bundleHash = await sha256File(bundle);
if (saved.bundleSha256 && bundleHash !== saved.bundleSha256) throw new Error('archive-bundle-changed');
+ // Bundle verify/list-heads validate headers, not packed object integrity. Prove
+ // the complete closure without borrowing any object from the live repository.
+ const scratch = await mkdtemp(join(tmpdir(), 'portos-retirement-proof-'));
+ const env = { ...Object.fromEntries(Object.entries(process.env).filter(([key]) => !key.toUpperCase().startsWith('GIT_'))),
+ GIT_CONFIG_NOSYSTEM: '1', GIT_CONFIG_GLOBAL: devNull, GIT_NO_REPLACE_OBJECTS: '1', GIT_NO_LAZY_FETCH: '1', GIT_TERMINAL_PROMPT: '0' };
+ const prove = args => execGit(['--no-optional-locks', '-c', `core.hooksPath=${devNull}`, '-c', 'core.fsmonitor=false', ...args], scratch,
+ { env, maxBuffer: GIT_OUTPUT_BYTES });
+ try {
+ await prove(['init', '--bare', `--object-format=${release.head.length === 64 ? 'sha256' : 'sha1'}`, '.']);
+ await prove(['bundle', 'unbundle', bundle]);
+ await prove(['fsck', '--full', '--strict', '--no-reflogs', '--no-dangling', ...captured.recoveryHeads]);
+ } catch { throw new Error('archive-bundle-object-proof-failed'); }
+ finally { await rmGuarded(scratch, { recursive: true, force: true }); }
if ((await archiveManifest(dir)).digest !== digest) throw new Error('archive-manifest-changed');
const recheck = async () => {
for (const [path, original] of stamps) {
@@ -136,9 +150,8 @@ async function verifyHeldArchive(repoPath, dir, release) {
return { repoPath, archiveDir: dir, archivedAt: saved.archivedAt, evidenceDigest: evidenceHash(digest + bundleHash), kind: 'held', recheck };
}
-async function historicalCheckoutAbsent(repoPath, release) {
+async function historicalCheckoutAbsent(release, trees) {
const present = await lstat(release.path).catch(error => { if (error.code === 'ENOENT') return null; throw error; });
- const trees = await listWorktrees(repoPath);
return !present && trees.length > 0 && !trees.some(tree => tree.path === release.path || (release.branch && shortBranch(tree.branch) === release.branch));
}
@@ -149,6 +162,8 @@ export async function recoverMissingCheckoutPreservations({ appId, repoPath, out
const expected = structuredClone(outcome);
const obligations = unresolvedRetirements(expected);
if (!obligations.length) return outcome;
+ const trees = await listWorktrees(repoPath).catch(() => null);
+ if (!trees?.length) return outcome;
const roots = [heldCheckoutArchiveRoot(opts.cosDir), join(opts.cosDir || PATHS.cos, 'initialized-checkout-retirements')];
const candidates = [];
let discoveryReason = null;
@@ -163,37 +178,44 @@ export async function recoverMissingCheckoutPreservations({ appId, repoPath, out
} catch { discoveryReason = 'archive-discovery-unavailable'; }
}
const observations = [];
- for (const release of obligations) {
- if (!await historicalCheckoutAbsent(repoPath, release).catch(() => false)) continue;
- const evidence = [];
- let proof = null;
- for (const candidate of candidates) {
- try {
- proof = candidate.kind === 'held' ? await verifyHeldArchive(repoPath, candidate.dir, release)
- : await (await import('./initializedCheckoutRecovery.js')).verifyMissingInitializedCheckoutPreservation({ appId, repoPath, release, archiveDir: candidate.dir });
- evidence.push({ archiveDir: candidate.dir, reason: 'preservation-verified' });
- break;
- } catch (error) {
- evidence.push({ archiveDir: candidate.dir, reason: /^[a-z][a-z-]+$/.test(error.message) ? error.message : 'archive-unverifiable' });
+ const evidenceFences = [];
+ try {
+ for (const release of obligations) {
+ if (!await historicalCheckoutAbsent(release, trees).catch(() => false)) continue;
+ const evidence = [];
+ let proof = null;
+ for (const candidate of candidates) {
+ try {
+ proof = candidate.kind === 'held' ? await verifyHeldArchive(repoPath, candidate.dir, release)
+ : await (await import('./initializedCheckoutRecovery.js')).verifyMissingInitializedCheckoutPreservation({ appId, repoPath, release, archiveDir: candidate.dir });
+ if (proof.releaseFence) evidenceFences.push(proof.releaseFence);
+ evidence.push({ archiveDir: candidate.dir, reason: 'preservation-verified' });
+ break;
+ } catch (error) {
+ evidence.push({ archiveDir: candidate.dir, reason: /^[a-z][a-z-]+$/.test(error.message) ? error.message : 'archive-unverifiable' });
+ }
}
+ observations.push({ release, proof, evidence, reason: discoveryReason || (candidates.length ? 'archive-evidence-unverified' : 'archive-evidence-missing'), nextAction: historicalNextAction });
}
- observations.push({ release, proof, evidence, reason: discoveryReason || (candidates.length ? 'archive-evidence-unverified' : 'archive-evidence-missing'), nextAction: historicalNextAction });
- }
- if (!observations.length) return outcome;
- const { recordRecoveredCheckoutPreservations } = await import('./taskSchedule.js');
- return withClaimOwnershipLock(() => recordRecoveredCheckoutPreservations(appId, expected, observations, async () => {
- const { getAppById } = await import('./apps.js');
- if ((await getAppById(appId))?.repoPath !== repoPath) return false;
- for (const row of observations) {
- if (!await historicalCheckoutAbsent(repoPath, row.release)) return false;
- if (row.proof?.recheck && !await row.proof.recheck().catch(() => false)) {
- row.evidence.push({ archiveDir: row.proof.archiveDir, reason: 'archive-changed-during-verification' });
- row.proof = null;
- row.reason = 'archive-changed-during-verification';
+ if (!observations.length) return outcome;
+ const { recordRecoveredCheckoutPreservations } = await import('./taskSchedule.js');
+ return await withClaimOwnershipLock(() => recordRecoveredCheckoutPreservations(appId, expected, observations, async () => {
+ const { getAppById } = await import('./apps.js');
+ if ((await getAppById(appId))?.repoPath !== repoPath) return false;
+ const currentTrees = await listWorktrees(repoPath);
+ for (const row of observations) {
+ if (!await historicalCheckoutAbsent(row.release, currentTrees)) return false;
+ if (row.proof?.recheck && !await row.proof.recheck().catch(() => false)) {
+ row.evidence.push({ archiveDir: row.proof.archiveDir, reason: 'archive-changed-during-verification' });
+ row.proof = null;
+ row.reason = 'archive-changed-during-verification';
+ }
}
- }
- return true;
- }));
+ return true;
+ }));
+ } finally {
+ for (const releaseFence of evidenceFences) await releaseFence();
+ }
}
// Refuse rather than archive a subset: a partial archive followed by removal
diff --git a/server/services/heldCheckoutRecovery.test.js b/server/services/heldCheckoutRecovery.test.js
index 810fdf5474..eb31d6dff5 100644
--- a/server/services/heldCheckoutRecovery.test.js
+++ b/server/services/heldCheckoutRecovery.test.js
@@ -5,7 +5,7 @@
* its lock and its branch as found.
*/
import { describe, it, expect, beforeEach, afterEach, afterAll, vi } from 'vitest';
-import { cp, mkdtemp, mkdir, rm, rename, writeFile, readFile, readdir, symlink, utimes } from 'fs/promises';
+import { cp, mkdtemp, mkdir, rm, rename, writeFile, readFile, readdir, symlink, utimes, truncate } from 'fs/promises';
import { existsSync, realpathSync } from 'fs';
import { tmpdir } from 'os';
import { dirname, join } from 'path';
@@ -139,7 +139,7 @@ describe.skipIf(SKIP_HEAVY_INTEGRATION)('held checkout recovery (real git)', ()
expect(recreated.retirementObligations[0].preservationEvidence.archives).toContainEqual({ archiveDir: saved.archiveDir, reason: 'archive-generation-mismatch' });
});
- it.each(['missing', 'wrong-repository', 'wrong-path', 'wrong-head', 'changed-file', 'changed-patch', 'legacy-dirty', 'partial-original'])(
+ it.each(['missing', 'wrong-repository', 'wrong-path', 'wrong-head', 'changed-file', 'changed-patch', 'legacy-dirty', 'legacy-corrupt-bundle', 'partial-original'])(
'retains an actionable historical obligation for %s preservation evidence', async failure => {
const { release, saved } = await historicalArchive();
const manifestPath = join(saved.archiveDir, 'manifest.json');
@@ -150,6 +150,13 @@ describe.skipIf(SKIP_HEAVY_INTEGRATION)('held checkout recovery (real git)', ()
if (failure === 'wrong-head') manifest.head = 'f'.repeat(40);
if (failure === 'changed-file') await writeFile(join(saved.archiveDir, 'files', 'private.txt'), 'changed archive bytes');
if (failure === 'changed-patch') await writeFile(join(saved.archiveDir, 'index.diff'), 'changed patch bytes');
+ if (failure === 'legacy-corrupt-bundle') {
+ delete manifest.bundleSha256;
+ const bundlePath = join(saved.archiveDir, 'commits.bundle');
+ const bundle = await readFile(bundlePath);
+ // Preserve the valid textual header and destroy only the packed objects.
+ await truncate(bundlePath, bundle.indexOf('\n\n') + 12);
+ }
if (failure === 'legacy-dirty') delete manifest.patchHashes;
if (failure === 'partial-original') manifest.captured.missingCheckout = true;
if (failure !== 'missing') await writeFile(manifestPath, JSON.stringify(manifest));
diff --git a/server/services/initializedCheckoutRecovery.js b/server/services/initializedCheckoutRecovery.js
index 5b782ba6f4..4c5191e872 100644
--- a/server/services/initializedCheckoutRecovery.js
+++ b/server/services/initializedCheckoutRecovery.js
@@ -515,19 +515,31 @@ export function _createInitializedCheckoutRecovery(deps = {}) {
if (!['branch-retired', 'retired'].includes(state.phase) || state.branchDeleted !== true) fail('archive-retirement-unconfirmed');
const archivedAt = state.reservation?.owner?.startedAt;
if (!(Date.parse(archivedAt) >= Date.parse(release.createdAt)) || Date.parse(archivedAt) > Date.now()) fail('archive-generation-mismatch');
- // The journal locates the manifest; the existing native verifier checks its
- // digest, every preserved source and the standalone object closure.
- const manifestPath = join(directory, 'manifest.json');
- const info = await lstat(manifestPath);
- if (!info.isFile() || info.isSymbolicLink() || info.size > 16 * 1024 * 1024) fail('archive-manifest-invalid');
- const manifest = JSON.parse(await readFile(manifestPath, 'utf8'));
- if (manifest.facts?.repoPath !== repoPath || !sameRelease(manifest.facts?.release, release)) fail('archive-identity-mismatch');
- const result = await verifyArchive({ appId, transactionId: state.id, journalDigest: hash(state) });
- if (result.proof?.verified !== true || result.proof?.standalone !== true) fail('archive-preservation-unverified');
- if (hash(await readJournal(directory, { bounded: true })) !== hash(state)) fail('retirement-journal-changed');
- const recheck = async () => hash(await readJournal(directory, { bounded: true })) === hash(state)
- && createHash('sha256').update(await readFile(manifestPath)).digest('hex') === state.proof.manifestSha256;
- return { repoPath, archiveDir: directory, archivedAt, evidenceDigest: state.proof.manifestSha256, kind: 'initialized', recheck };
+ // Every archive mutation takes this execution fence. Keep it until the
+ // schedule writer accepts or refuses the evidence, not just through verify.
+ const executor = await native({ command: 'lock', transactionDir: directory }, { keepOpen: true });
+ try {
+ if (hash(await readJournal(directory, { bounded: true })) !== hash(state)) fail('retirement-journal-changed');
+ // The journal locates the manifest; the existing native verifier checks its
+ // digest, every preserved source and the standalone object closure.
+ const manifestPath = join(directory, 'manifest.json');
+ const info = await lstat(manifestPath);
+ if (!info.isFile() || info.isSymbolicLink() || info.size > 16 * 1024 * 1024) fail('archive-manifest-invalid');
+ const manifest = JSON.parse(await readFile(manifestPath, 'utf8'));
+ if (manifest.facts?.repoPath !== repoPath || !sameRelease(manifest.facts?.release, release)) fail('archive-identity-mismatch');
+ const result = await verifyArchive({ appId, transactionId: state.id, journalDigest: hash(state) });
+ if (result.proof?.verified !== true || result.proof?.standalone !== true) fail('archive-preservation-unverified');
+ if (hash(await readJournal(directory, { bounded: true })) !== hash(state)) fail('retirement-journal-changed');
+ const recheck = async () => {
+ executor.assertHeld();
+ const unchanged = hash(await readJournal(directory, { bounded: true })) === hash(state)
+ && createHash('sha256').update(await readFile(manifestPath)).digest('hex') === state.proof.manifestSha256;
+ executor.assertHeld();
+ return unchanged;
+ };
+ return { repoPath, archiveDir: directory, archivedAt, evidenceDigest: state.proof.manifestSha256, kind: 'initialized', recheck,
+ releaseFence: () => executor.release() };
+ } catch (error) { await executor.release(); throw error; }
};
// Explicit recovery restores the exact original inodes. It does not erase
diff --git a/server/services/initializedCheckoutRecovery.test.js b/server/services/initializedCheckoutRecovery.test.js
index f852b72162..0363ab0808 100644
--- a/server/services/initializedCheckoutRecovery.test.js
+++ b/server/services/initializedCheckoutRecovery.test.js
@@ -971,8 +971,20 @@ describe('initialized checkout transaction', () => {
await writeFile(join(archiveDir, 'manifest.json'), JSON.stringify({ facts: { repoPath: '/fixture/repo', release } }));
const request = { appId: 'app', repoPath: '/fixture/repo', archiveDir, release: { ...release, createdAt } };
fixture.deps.native.mockClear(); fixture.deps.deleteRef.mockClear();
- expect(await fixture.service.verifyMissingPreservation(request)).toMatchObject({ archiveDir, kind: 'initialized', evidenceDigest: 'e'.repeat(64) });
- expect(fixture.deps.native.mock.calls.map(([value]) => value.command)).toEqual(['verify']);
+ const originalNative = fixture.deps.native.getMockImplementation();
+ const releaseFence = vi.fn(async () => {});
+ fixture.deps.native.mockImplementation((input, options) => input.command === 'lock'
+ ? { assertHeld: () => { if (fixture.fenceLost) throw new Error('retirement-executor-lost'); }, release: releaseFence }
+ : originalNative(input, options));
+ const recovered = await fixture.service.verifyMissingPreservation(request);
+ expect(recovered).toMatchObject({ archiveDir, kind: 'initialized', evidenceDigest: 'e'.repeat(64) });
+ expect(fixture.deps.native.mock.calls.map(([value]) => value.command)).toEqual(['lock', 'verify']);
+ expect(releaseFence).not.toHaveBeenCalled();
+ fixture.fenceLost = true;
+ await expect(recovered.recheck()).rejects.toThrow('retirement-executor-lost');
+ await recovered.releaseFence();
+ expect(releaseFence).toHaveBeenCalledOnce();
+ fixture.fenceLost = false;
expect(fixture.deps.deleteRef).not.toHaveBeenCalled();
await expect(fixture.service.verifyMissingPreservation({ ...request, release: { ...request.release, head: 'f'.repeat(40) } }))
.rejects.toThrow('archive-identity-mismatch');
@@ -980,6 +992,7 @@ describe('initialized checkout transaction', () => {
.rejects.toThrow('archive-generation-mismatch');
fixture.failArchive = true;
await expect(fixture.service.verifyMissingPreservation(request)).rejects.toThrow('archive-changed');
+ expect(releaseFence).toHaveBeenCalledTimes(2);
});
it('invalidates branch finalization when the default advances or a holder appears', async () => {
const input = await quarantinedReview();
diff --git a/server/services/taskSchedule.js b/server/services/taskSchedule.js
index 226c9fd8b4..8659c8518b 100644
--- a/server/services/taskSchedule.js
+++ b/server/services/taskSchedule.js
@@ -812,7 +812,7 @@ export async function recordRecoveredCheckoutPreservations(appId, expected, obse
const observation = observations.find(row => sameRetirementCheckout(row.release, release) && row.release.createdAt === release.createdAt);
if (!observation) return release;
const accepted = observation.proof && recoveredPreservations.length < 1000;
- const { recheck: _recheck, ...proof } = observation.proof || {};
+ const { recheck: _recheck, releaseFence: _releaseFence, ...proof } = observation.proof || {};
if (accepted) recoveredPreservations.push({
branch: release.branch, path: release.path, head: release.head,
obligationCreatedAt: release.createdAt, ...proof, verified: true, verifiedAt: now,