Skip to content

Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS

fix: bound repeated checkout archive diagnostics (#13482)

Adam Eivy58da0838c180
diff --git a/server/services/heldCheckoutRecovery.js b/server/services/heldCheckoutRecovery.js
index 424e177db9..5765c9b6b8 100644
--- a/server/services/heldCheckoutRecovery.js
+++ b/server/services/heldCheckoutRecovery.js
@@ -155,6 +155,7 @@ async function archiveBytes(path, work, limitBytes, reason = 'retirement-archive
const { stream } = await archiveSource(path);
for await (const chunk of stream) {
work.check();
+ work.read?.(chunk.length);
if ((size += chunk.length) > limitBytes) throw Object.assign(new Error(reason), {
archiveLimit: { scope: 'metadata-read', kind: 'file', limitBytes, observedBytes: size },
});
@@ -337,6 +338,66 @@ export async function findMissingCheckoutPreservation({ repoPath, release }, opt
return { ...proof, recheck: async () => await absent() && await proof.recheck() };
}
+const RECOVERY_DETAIL_LIMIT = 20;
+const DISCOVERY_BYTE_LIMIT = 64 * 1024 * 1024;
+const archiveIdentityKey = row => row && [row.repoPath, row.path, row.head].every(value => typeof value === 'string')
+ && (typeof row.branch === 'string' || row.branch === null)
+ ? JSON.stringify([row.repoPath, row.branch, row.path, row.head]) : null;
+
+/** Metadata only selects work; every selected archive is independently verified. */
+async function discoverRecoveryArchives(cosDir, signal) {
+ const roots = [heldCheckoutArchiveRoot(cosDir), join(cosDir || PATHS.cos, 'initialized-checkout-retirements')];
+ const index = new Map();
+ const byDirectory = new Map();
+ const coverage = { entriesRead: 0, candidates: 0, indexed: 0, metadataBytes: 0, missingRoots: 0,
+ errors: 0, details: [], detailsOmitted: 0, complete: true, truncated: false };
+ const diagnose = (archiveDir, error) => {
+ coverage.complete = false;
+ coverage.errors++;
+ const refusal = archiveLimitRefusal(error);
+ const detail = { archiveDir, ...(refusal || { reason: /^[a-z][a-z-]+$/.test(error.message) ? error.message : 'archive-discovery-unavailable' }) };
+ if (coverage.details.length < RECOVERY_DETAIL_LIMIT) coverage.details.push(detail);
+ else coverage.detailsOmitted++;
+ };
+ const work = { ...archiveWork(signal), read: bytes => {
+ // Account before parsing, including partial, malformed and oversized reads.
+ coverage.metadataBytes += bytes;
+ if (coverage.metadataBytes > DISCOVERY_BYTE_LIMIT) throw new Error('archive-discovery-budget-exceeded');
+ } };
+ for (const root of roots) {
+ try {
+ const info = await lstat(root).catch(error => { if (error.code === 'ENOENT') return null; throw error; });
+ if (!info) { coverage.missingRoots++; continue; }
+ if (!info.isDirectory()) throw new Error('archive-directory-invalid');
+ const directory = await opendir(root);
+ for await (const entry of directory) {
+ if (coverage.entriesRead >= 1000 || coverage.metadataBytes >= DISCOVERY_BYTE_LIMIT) {
+ coverage.truncated = true;
+ diagnose(root, new Error('archive-discovery-budget-exceeded'));
+ break;
+ }
+ coverage.entriesRead++;
+ if (!entry.isDirectory()) continue;
+ const candidate = { dir: join(root, entry.name), kind: root === roots[0] ? 'held' : 'initialized' };
+ coverage.candidates++;
+ byDirectory.set(candidate.dir, candidate);
+ try {
+ const { manifest } = await archiveManifest(candidate.dir, work);
+ const identity = candidate.kind === 'held' ? manifest : { repoPath: manifest.facts?.repoPath, ...manifest.facts?.release };
+ const key = archiveIdentityKey(identity);
+ if (!key) throw new Error('archive-identity-unreadable');
+ if (!index.has(key)) index.set(key, []);
+ index.get(key).push(candidate);
+ coverage.indexed++;
+ } catch (error) {
+ diagnose(candidate.dir, error);
+ }
+ }
+ } catch (error) { diagnose(root, error); }
+ }
+ return { index, byDirectory, coverage };
+}
+
/** Read existing archives only. Verification stays outside both writer locks;
* the final ownership/absence recheck and schedule compare-and-set are fenced.
*/
@@ -346,45 +407,46 @@ export async function recoverMissingCheckoutPreservations({ appId, repoPath, out
if (!obligations.length) return outcome;
const trees = await listWorktrees(repoPath).catch(() => null);
if (!trees?.length) return outcome;
- const roots = [heldCheckoutArchiveRoot(opts.cosDir), join(opts.cosDir || PATHS.cos, 'initialized-checkout-retirements')];
- const candidates = [];
- let discoveryReason = null;
- for (const root of roots) {
- try {
- const info = await lstat(root).catch(error => { if (error.code === 'ENOENT') return null; throw error; });
- if (!info) continue;
- if (!info.isDirectory()) throw new Error('archive-directory-invalid');
- const names = await readdir(root, { withFileTypes: true });
- if (names.length + candidates.length > 1000) throw new Error('archive-discovery-budget-exceeded');
- for (const name of names) if (name.isDirectory()) candidates.push({ dir: join(root, name.name), kind: root === roots[0] ? 'held' : 'initialized' });
- } catch { discoveryReason = 'archive-discovery-unavailable'; }
- }
+ const { index, byDirectory, coverage } = await discoverRecoveryArchives(opts.cosDir, opts.signal);
const observations = [];
const evidenceFences = [];
+ let details = 0;
+ const addEvidence = (row, detail) => {
+ if (details < RECOVERY_DETAIL_LIMIT) { row.evidence.push(detail); details++; }
+ else row.coverage.detailsOmitted++;
+ };
try {
for (const release of obligations) {
if (!await historicalCheckoutAbsent(release, trees).catch(() => false)) continue;
- const evidence = [];
let proof = null;
+ let verified = 0;
+ const matching = index.get(archiveIdentityKey({ ...release, repoPath })) || [];
const scopedCandidates = Object.hasOwn(release, 'preservationReceipt')
- ? candidates.filter(candidate => release.preservationReceipt?.status === 'preserved'
+ ? [byDirectory.get(release.preservationReceipt?.archiveDir)].filter(candidate => candidate && release.preservationReceipt?.status === 'preserved'
&& JSON.stringify(release.preservationReceipt.incarnation) === JSON.stringify(release.checkoutIncarnation)
- && candidate.dir === release.preservationReceipt.archiveDir) : candidates;
+ && candidate.dir === release.preservationReceipt.archiveDir) : matching;
+ const row = { release, proof: null, evidence: [], coverage: { matching: scopedCandidates.length,
+ unrelated: coverage.indexed - matching.length, attempted: 0, detailsOmitted: 0 },
+ reason: !coverage.complete ? 'archive-discovery-incomplete' : coverage.candidates ? 'archive-evidence-unverified' : 'archive-evidence-missing', nextAction: historicalNextAction };
for (const candidate of scopedCandidates) {
+ row.coverage.attempted++;
try {
- proof = candidate.kind === 'held' ? await verifyHeldArchive(repoPath, candidate.dir, release, archiveWork(opts.signal))
+ const verifiedProof = candidate.kind === 'held' ? await verifyHeldArchive(repoPath, candidate.dir, release, archiveWork(opts.signal))
: await (await import('./initializedCheckoutRecovery.js')).verifyMissingInitializedCheckoutPreservation({ appId, repoPath, release, archiveDir: candidate.dir });
- if (proof.releaseFence) evidenceFences.push(proof.releaseFence);
- evidence.push({ archiveDir: candidate.dir, reason: 'preservation-verified' });
- break;
+ if (verifiedProof.releaseFence) evidenceFences.push(verifiedProof.releaseFence);
+ verified++;
+ proof = verifiedProof;
+ addEvidence(row, { archiveDir: candidate.dir, reason: 'preservation-verified' });
} catch (error) {
const refusal = archiveLimitRefusal(error);
- evidence.push({ archiveDir: candidate.dir, ...(refusal ? { reason: refusal.reason,
+ addEvidence(row, { archiveDir: candidate.dir, ...(refusal ? { reason: refusal.reason,
...(refusal.archiveLimit && { archiveLimit: refusal.archiveLimit }), nextAction: refusal.nextAction,
} : { reason: /^[a-z][a-z-]+$/.test(error.message) ? error.message : 'archive-unverifiable' }) });
}
}
- observations.push({ release, proof, evidence, reason: discoveryReason || (candidates.length ? 'archive-evidence-unverified' : 'archive-evidence-missing'), nextAction: historicalNextAction });
+ row.proof = verified === 1 && (coverage.complete || Object.hasOwn(release, 'preservationReceipt')) ? proof : null;
+ if (verified > 1) row.reason = 'archive-preservation-ambiguous';
+ observations.push(row);
}
if (!observations.length) return outcome;
const { recordRecoveredCheckoutPreservations } = await import('./taskSchedule.js');
@@ -395,13 +457,13 @@ export async function recoverMissingCheckoutPreservations({ appId, repoPath, out
for (const row of observations) {
if (!await historicalCheckoutAbsent(row.release, currentTrees)) return false;
if (row.proof?.recheck && !await row.proof.recheck().catch(() => false)) {
- row.evidence.push({ archiveDir: row.proof.archiveDir, reason: 'archive-changed-during-verification' });
+ addEvidence(row, { archiveDir: row.proof.archiveDir, reason: 'archive-changed-during-verification' });
row.proof = null;
row.reason = 'archive-changed-during-verification';
}
}
return true;
- }));
+ }, coverage));
} finally {
for (const releaseFence of evidenceFences) await releaseFence();
}
diff --git a/server/services/heldCheckoutRecovery.test.js b/server/services/heldCheckoutRecovery.test.js
index 37f92005db..c609710668 100644
--- a/server/services/heldCheckoutRecovery.test.js
+++ b/server/services/heldCheckoutRecovery.test.js
@@ -391,6 +391,94 @@ describe.skipIf(SKIP_HEAVY_INTEGRATION)('held checkout recovery (real git)', ()
expect(recreated.retirementObligations[0].preservationEvidence.archives).toContainEqual({ archiveDir: saved.archiveDir, reason: 'archive-generation-mismatch' });
});
+ describe('bounded historical evidence', () => {
+ beforeEach(cleanupTempDataRoots);
+ afterEach(cleanupTempDataRoots);
+
+ it('bounds persisted recovery diagnostics across obligations and unrelated archives while verifying the exact matching archive', async () => {
+ const { release, saved } = await historicalArchive();
+ const original = await getBranchReconcileOutcome('app-1');
+ const root = heldCheckoutArchiveRoot(PATHS.cos);
+ const obligations = Array.from({ length: 6 }, (_, i) => ({ ...original.retirementObligations[0],
+ branch: `feature/missing-${i}`, path: join(external, `missing-${i}`),
+ preservationEvidence: { status: 'unresolved', archives: [{ reason: 'legacy-diagnostic' }] } }));
+ const metadata = async (name, identity) => {
+ const dir = join(root, name);
+ await mkdir(dir);
+ await writeFile(join(dir, 'manifest.json'), JSON.stringify({ repoPath: repo, ...identity, archivedAt: '2000-01-01T00:00:00Z' }));
+ };
+ for (let i = 0; i < 80; i++) await metadata(`unrelated-${i}`, { ...release, path: join(external, `unrelated-${i}`) });
+ for (const [i, obligation] of obligations.entries()) {
+ for (let j = 0; j < 8; j++) await metadata(`matching-${i}-${j}`, obligation);
+ }
+ const receipt = { branch: 'feature/settled', path: join(external, 'settled'), head: release.head,
+ archiveDir: 'synthetic-previous-archive', branchDeleted: true, retiredAt: new Date().toISOString() };
+ original.retired = [receipt];
+ const outcome = await recordBranchReconcileOutcome('app-1', { status: 'incomplete', remaining: [],
+ retirementObligations: [...original.retirementObligations, ...obligations] });
+ expect(outcome.recoveredPreservations).toEqual([expect.objectContaining({ path: release.path, archiveDir: saved.archiveDir, verified: true })]);
+ expect(outcome.retirementObligations).toHaveLength(6);
+ expect(outcome.retired).toEqual([receipt]);
+ expect(outcome.preservationDiscovery).toMatchObject({ candidates: 129, indexed: 129, complete: true, errors: 0 });
+ const evidence = outcome.retirementObligations.map(row => row.preservationEvidence);
+ expect(evidence.every(row => row.coverage.matching === 8 && row.coverage.attempted === 8 && row.coverage.unrelated === 121)).toBe(true);
+ const details = evidence.flatMap(row => row.archives);
+ expect(details.length).toBeLessThanOrEqual(20);
+ expect(details.every(row => row.reason === 'archive-generation-mismatch')).toBe(true);
+ expect(evidence.reduce((total, row) => total + row.coverage.detailsOmitted, details.length)).toBe(48);
+ expect(JSON.stringify(outcome).length).toBeLessThan(30000);
+ expect(await getBranchReconcileOutcome('app-1')).toEqual(outcome);
+ });
+
+ it('bounds bytes read from oversized metadata across discovery candidates', async () => {
+ await historicalArchive();
+ const root = heldCheckoutArchiveRoot(PATHS.cos);
+ for (let i = 0; i < 6; i++) {
+ const dir = join(root, `oversized-${i}`);
+ await mkdir(dir);
+ const path = join(dir, 'manifest.json');
+ await writeFile(path, '');
+ await truncate(path, 17 * 1024 * 1024);
+ }
+ const outcome = await recordBranchReconcileOutcome('app-1', { status: 'incomplete', remaining: [] });
+ expect(outcome.recoveredPreservations).toEqual([]);
+ expect(outcome.retirementObligations).toHaveLength(1);
+ expect(outcome.preservationDiscovery).toMatchObject({ complete: false, truncated: true });
+ // Streaming discovery can consume at most one additional 64 KiB chunk.
+ expect(outcome.preservationDiscovery.metadataBytes).toBeLessThanOrEqual(64 * 1024 * 1024 + 64 * 1024);
+ expect(outcome.preservationDiscovery.details).toContainEqual(expect.objectContaining({ reason: 'archive-discovery-budget-exceeded' }));
+ });
+
+ it.each(['ambiguous', 'unreadable-metadata', 'missing-discovery', 'truncated-discovery'])(
+ 'retains an explicit hold for %s despite a complete matching archive', async failure => {
+ const { saved } = await historicalArchive();
+ const root = heldCheckoutArchiveRoot(PATHS.cos);
+ if (failure === 'ambiguous') await cp(saved.archiveDir, join(root, 'second-matching-archive'), { recursive: true });
+ if (failure === 'unreadable-metadata') {
+ for (let i = 0; i < 25; i++) {
+ await mkdir(join(root, `broken-metadata-${i}`));
+ await writeFile(join(root, `broken-metadata-${i}`, 'manifest.json'), '{');
+ }
+ }
+ if (failure === 'missing-discovery') await writeFile(join(PATHS.cos, 'initialized-checkout-retirements'), 'unreadable archive root');
+ if (failure === 'truncated-discovery') {
+ for (let i = 0; i < 1001; i++) await writeFile(join(root, `entry-${i}`), '');
+ }
+ const outcome = await recordBranchReconcileOutcome('app-1', { status: 'incomplete', remaining: [] });
+ expect(outcome.recoveredPreservations).toEqual([]);
+ expect(outcome.retirementObligations).toHaveLength(1);
+ expect(outcome.retirementObligations[0].preservationEvidence.reason).toBe(
+ failure === 'ambiguous' ? 'archive-preservation-ambiguous' : 'archive-discovery-incomplete');
+ if (failure !== 'ambiguous') {
+ expect(outcome.preservationDiscovery).toMatchObject({ complete: false, errors: failure === 'unreadable-metadata' ? 25 : 1 });
+ expect(outcome.preservationDiscovery.details).toHaveLength(failure === 'unreadable-metadata' ? 20 : 1);
+ expect(outcome.preservationDiscovery.detailsOmitted).toBe(failure === 'unreadable-metadata' ? 5 : 0);
+ expect(outcome.preservationDiscovery.truncated).toBe(failure === 'truncated-discovery');
+ }
+ });
+
+ });
+
it.each(['cancel', 'time', 'git-output', 'git-time'])('retains the %s diagnosis when isolated Git proof is interrupted without settling preservation', async mode => {
const { saved } = await historicalArchive();
const outcome = await getBranchReconcileOutcome('app-1');
diff --git a/server/services/taskSchedule.js b/server/services/taskSchedule.js
index a9f1665f8b..d23bec7d97 100644
--- a/server/services/taskSchedule.js
+++ b/server/services/taskSchedule.js
@@ -828,7 +828,7 @@ export async function recordBranchReconcileOutcome(appId, { status, reason, rema
/** Compare the whole observed inventory and its generation inside the schedule
* writer. A verifier result never authorizes a newer/recreated checkout.
*/
-export async function recordRecoveredCheckoutPreservations(appId, expected, observations, stillAbsent) {
+export async function recordRecoveredCheckoutPreservations(appId, expected, observations, stillAbsent, discovery) {
return updateSchedule(async schedule => {
const record = ensureExecutionRecord(schedule, 'branch-reconcile', appId);
const outcome = record.branchReconcileOutcome;
@@ -846,7 +846,7 @@ export async function recordRecoveredCheckoutPreservations(appId, expected, obse
});
return { ...release, preservationEvidence: { status: 'unresolved',
reason: observation.proof && !accepted ? 'preservation-receipt-budget-exceeded' : observation.reason,
- checkedAt: now, archives: observation.evidence, nextAction: observation.nextAction } };
+ checkedAt: now, archives: observation.evidence, ...(observation.coverage && { coverage: observation.coverage }), nextAction: observation.nextAction } };
});
const retirementObligations = unresolvedRetirements({ retirementObligations: diagnosed, recoveredPreservations });
const remaining = (outcome.remaining || []).flatMap(row => {
@@ -864,6 +864,7 @@ export async function recordRecoveredCheckoutPreservations(appId, expected, obse
: 'Historical preservation verified; run branch reconciliation again to confirm repository completion.' }
: {};
record.branchReconcileOutcome = { ...outcome, retirementObligations, recoveredPreservations, remaining, updatedAt: now,
+ ...(discovery && { preservationDiscovery: discovery }),
...diagnosis,
};
return { result: record.branchReconcileOutcome, changed: true };