Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS
JavaScript ai-agentsdashboarddeveloper-toolsdigital-twinquantified-selfsecond-brainself-hostedtailscale
fix: bound repeated checkout archive diagnostics (#13482)
Adam Eivy58da0838c180
diff --git a/server/services/heldCheckoutRecovery.js b/server/services/heldCheckoutRecovery.jsindex 424e177db9..5765c9b6b8 100644--- a/server/services/heldCheckoutRecovery.js+++ b/server/services/heldCheckoutRecovery.js@@ -155,6 +155,7 @@ async function archiveBytes(path, work, limitBytes, reason = 'retirement-archiveconst { stream } = await archiveSource(path);for await (const chunk of stream) {work.check();+ work.read?.(chunk.length);if ((size += chunk.length) > limitBytes) throw Object.assign(new Error(reason), {archiveLimit: { scope: 'metadata-read', kind: 'file', limitBytes, observedBytes: size },});@@ -337,6 +338,66 @@ export async function findMissingCheckoutPreservation({ repoPath, release }, optreturn { ...proof, recheck: async () => await absent() && await proof.recheck() };}+const RECOVERY_DETAIL_LIMIT = 20;+const DISCOVERY_BYTE_LIMIT = 64 * 1024 * 1024;+const archiveIdentityKey = row => row && [row.repoPath, row.path, row.head].every(value => typeof value === 'string')+ && (typeof row.branch === 'string' || row.branch === null)+ ? JSON.stringify([row.repoPath, row.branch, row.path, row.head]) : null;++/** Metadata only selects work; every selected archive is independently verified. */+async function discoverRecoveryArchives(cosDir, signal) {+ const roots = [heldCheckoutArchiveRoot(cosDir), join(cosDir || PATHS.cos, 'initialized-checkout-retirements')];+ const index = new Map();+ const byDirectory = new Map();+ const coverage = { entriesRead: 0, candidates: 0, indexed: 0, metadataBytes: 0, missingRoots: 0,+ errors: 0, details: [], detailsOmitted: 0, complete: true, truncated: false };+ const diagnose = (archiveDir, error) => {+ coverage.complete = false;+ coverage.errors++;+ const refusal = archiveLimitRefusal(error);+ const detail = { archiveDir, ...(refusal || { reason: /^[a-z][a-z-]+$/.test(error.message) ? error.message : 'archive-discovery-unavailable' }) };+ if (coverage.details.length < RECOVERY_DETAIL_LIMIT) coverage.details.push(detail);+ else coverage.detailsOmitted++;+ };+ const work = { ...archiveWork(signal), read: bytes => {+ // Account before parsing, including partial, malformed and oversized reads.+ coverage.metadataBytes += bytes;+ if (coverage.metadataBytes > DISCOVERY_BYTE_LIMIT) throw new Error('archive-discovery-budget-exceeded');+ } };+ for (const root of roots) {+ try {+ const info = await lstat(root).catch(error => { if (error.code === 'ENOENT') return null; throw error; });+ if (!info) { coverage.missingRoots++; continue; }+ if (!info.isDirectory()) throw new Error('archive-directory-invalid');+ const directory = await opendir(root);+ for await (const entry of directory) {+ if (coverage.entriesRead >= 1000 || coverage.metadataBytes >= DISCOVERY_BYTE_LIMIT) {+ coverage.truncated = true;+ diagnose(root, new Error('archive-discovery-budget-exceeded'));+ break;+ }+ coverage.entriesRead++;+ if (!entry.isDirectory()) continue;+ const candidate = { dir: join(root, entry.name), kind: root === roots[0] ? 'held' : 'initialized' };+ coverage.candidates++;+ byDirectory.set(candidate.dir, candidate);+ try {+ const { manifest } = await archiveManifest(candidate.dir, work);+ const identity = candidate.kind === 'held' ? manifest : { repoPath: manifest.facts?.repoPath, ...manifest.facts?.release };+ const key = archiveIdentityKey(identity);+ if (!key) throw new Error('archive-identity-unreadable');+ if (!index.has(key)) index.set(key, []);+ index.get(key).push(candidate);+ coverage.indexed++;+ } catch (error) {+ diagnose(candidate.dir, error);+ }+ }+ } catch (error) { diagnose(root, error); }+ }+ return { index, byDirectory, coverage };+}+/** Read existing archives only. Verification stays outside both writer locks;* the final ownership/absence recheck and schedule compare-and-set are fenced.*/@@ -346,45 +407,46 @@ export async function recoverMissingCheckoutPreservations({ appId, repoPath, outif (!obligations.length) return outcome;const trees = await listWorktrees(repoPath).catch(() => null);if (!trees?.length) return outcome;- const roots = [heldCheckoutArchiveRoot(opts.cosDir), join(opts.cosDir || PATHS.cos, 'initialized-checkout-retirements')];- const candidates = [];- let discoveryReason = null;- for (const root of roots) {- try {- const info = await lstat(root).catch(error => { if (error.code === 'ENOENT') return null; throw error; });- if (!info) continue;- if (!info.isDirectory()) throw new Error('archive-directory-invalid');- const names = await readdir(root, { withFileTypes: true });- if (names.length + candidates.length > 1000) throw new Error('archive-discovery-budget-exceeded');- for (const name of names) if (name.isDirectory()) candidates.push({ dir: join(root, name.name), kind: root === roots[0] ? 'held' : 'initialized' });- } catch { discoveryReason = 'archive-discovery-unavailable'; }- }+ const { index, byDirectory, coverage } = await discoverRecoveryArchives(opts.cosDir, opts.signal);const observations = [];const evidenceFences = [];+ let details = 0;+ const addEvidence = (row, detail) => {+ if (details < RECOVERY_DETAIL_LIMIT) { row.evidence.push(detail); details++; }+ else row.coverage.detailsOmitted++;+ };try {for (const release of obligations) {if (!await historicalCheckoutAbsent(release, trees).catch(() => false)) continue;- const evidence = [];let proof = null;+ let verified = 0;+ const matching = index.get(archiveIdentityKey({ ...release, repoPath })) || [];const scopedCandidates = Object.hasOwn(release, 'preservationReceipt')- ? candidates.filter(candidate => release.preservationReceipt?.status === 'preserved'+ ? [byDirectory.get(release.preservationReceipt?.archiveDir)].filter(candidate => candidate && release.preservationReceipt?.status === 'preserved'&& JSON.stringify(release.preservationReceipt.incarnation) === JSON.stringify(release.checkoutIncarnation)- && candidate.dir === release.preservationReceipt.archiveDir) : candidates;+ && candidate.dir === release.preservationReceipt.archiveDir) : matching;+ const row = { release, proof: null, evidence: [], coverage: { matching: scopedCandidates.length,+ unrelated: coverage.indexed - matching.length, attempted: 0, detailsOmitted: 0 },+ reason: !coverage.complete ? 'archive-discovery-incomplete' : coverage.candidates ? 'archive-evidence-unverified' : 'archive-evidence-missing', nextAction: historicalNextAction };for (const candidate of scopedCandidates) {+ row.coverage.attempted++;try {- proof = candidate.kind === 'held' ? await verifyHeldArchive(repoPath, candidate.dir, release, archiveWork(opts.signal))+ const verifiedProof = candidate.kind === 'held' ? await verifyHeldArchive(repoPath, candidate.dir, release, archiveWork(opts.signal)): await (await import('./initializedCheckoutRecovery.js')).verifyMissingInitializedCheckoutPreservation({ appId, repoPath, release, archiveDir: candidate.dir });- if (proof.releaseFence) evidenceFences.push(proof.releaseFence);- evidence.push({ archiveDir: candidate.dir, reason: 'preservation-verified' });- break;+ if (verifiedProof.releaseFence) evidenceFences.push(verifiedProof.releaseFence);+ verified++;+ proof = verifiedProof;+ addEvidence(row, { archiveDir: candidate.dir, reason: 'preservation-verified' });} catch (error) {const refusal = archiveLimitRefusal(error);- evidence.push({ archiveDir: candidate.dir, ...(refusal ? { reason: refusal.reason,+ addEvidence(row, { archiveDir: candidate.dir, ...(refusal ? { reason: refusal.reason,...(refusal.archiveLimit && { archiveLimit: refusal.archiveLimit }), nextAction: refusal.nextAction,} : { reason: /^[a-z][a-z-]+$/.test(error.message) ? error.message : 'archive-unverifiable' }) });}}- observations.push({ release, proof, evidence, reason: discoveryReason || (candidates.length ? 'archive-evidence-unverified' : 'archive-evidence-missing'), nextAction: historicalNextAction });+ row.proof = verified === 1 && (coverage.complete || Object.hasOwn(release, 'preservationReceipt')) ? proof : null;+ if (verified > 1) row.reason = 'archive-preservation-ambiguous';+ observations.push(row);}if (!observations.length) return outcome;const { recordRecoveredCheckoutPreservations } = await import('./taskSchedule.js');@@ -395,13 +457,13 @@ export async function recoverMissingCheckoutPreservations({ appId, repoPath, outfor (const row of observations) {if (!await historicalCheckoutAbsent(row.release, currentTrees)) return false;if (row.proof?.recheck && !await row.proof.recheck().catch(() => false)) {- row.evidence.push({ archiveDir: row.proof.archiveDir, reason: 'archive-changed-during-verification' });+ addEvidence(row, { archiveDir: row.proof.archiveDir, reason: 'archive-changed-during-verification' });row.proof = null;row.reason = 'archive-changed-during-verification';}}return true;- }));+ }, coverage));} finally {for (const releaseFence of evidenceFences) await releaseFence();}diff --git a/server/services/heldCheckoutRecovery.test.js b/server/services/heldCheckoutRecovery.test.jsindex 37f92005db..c609710668 100644--- a/server/services/heldCheckoutRecovery.test.js+++ b/server/services/heldCheckoutRecovery.test.js@@ -391,6 +391,94 @@ describe.skipIf(SKIP_HEAVY_INTEGRATION)('held checkout recovery (real git)', ()expect(recreated.retirementObligations[0].preservationEvidence.archives).toContainEqual({ archiveDir: saved.archiveDir, reason: 'archive-generation-mismatch' });});+ describe('bounded historical evidence', () => {+ beforeEach(cleanupTempDataRoots);+ afterEach(cleanupTempDataRoots);++ it('bounds persisted recovery diagnostics across obligations and unrelated archives while verifying the exact matching archive', async () => {+ const { release, saved } = await historicalArchive();+ const original = await getBranchReconcileOutcome('app-1');+ const root = heldCheckoutArchiveRoot(PATHS.cos);+ const obligations = Array.from({ length: 6 }, (_, i) => ({ ...original.retirementObligations[0],+ branch: `feature/missing-${i}`, path: join(external, `missing-${i}`),+ preservationEvidence: { status: 'unresolved', archives: [{ reason: 'legacy-diagnostic' }] } }));+ const metadata = async (name, identity) => {+ const dir = join(root, name);+ await mkdir(dir);+ await writeFile(join(dir, 'manifest.json'), JSON.stringify({ repoPath: repo, ...identity, archivedAt: '2000-01-01T00:00:00Z' }));+ };+ for (let i = 0; i < 80; i++) await metadata(`unrelated-${i}`, { ...release, path: join(external, `unrelated-${i}`) });+ for (const [i, obligation] of obligations.entries()) {+ for (let j = 0; j < 8; j++) await metadata(`matching-${i}-${j}`, obligation);+ }+ const receipt = { branch: 'feature/settled', path: join(external, 'settled'), head: release.head,+ archiveDir: 'synthetic-previous-archive', branchDeleted: true, retiredAt: new Date().toISOString() };+ original.retired = [receipt];+ const outcome = await recordBranchReconcileOutcome('app-1', { status: 'incomplete', remaining: [],+ retirementObligations: [...original.retirementObligations, ...obligations] });+ expect(outcome.recoveredPreservations).toEqual([expect.objectContaining({ path: release.path, archiveDir: saved.archiveDir, verified: true })]);+ expect(outcome.retirementObligations).toHaveLength(6);+ expect(outcome.retired).toEqual([receipt]);+ expect(outcome.preservationDiscovery).toMatchObject({ candidates: 129, indexed: 129, complete: true, errors: 0 });+ const evidence = outcome.retirementObligations.map(row => row.preservationEvidence);+ expect(evidence.every(row => row.coverage.matching === 8 && row.coverage.attempted === 8 && row.coverage.unrelated === 121)).toBe(true);+ const details = evidence.flatMap(row => row.archives);+ expect(details.length).toBeLessThanOrEqual(20);+ expect(details.every(row => row.reason === 'archive-generation-mismatch')).toBe(true);+ expect(evidence.reduce((total, row) => total + row.coverage.detailsOmitted, details.length)).toBe(48);+ expect(JSON.stringify(outcome).length).toBeLessThan(30000);+ expect(await getBranchReconcileOutcome('app-1')).toEqual(outcome);+ });++ it('bounds bytes read from oversized metadata across discovery candidates', async () => {+ await historicalArchive();+ const root = heldCheckoutArchiveRoot(PATHS.cos);+ for (let i = 0; i < 6; i++) {+ const dir = join(root, `oversized-${i}`);+ await mkdir(dir);+ const path = join(dir, 'manifest.json');+ await writeFile(path, '');+ await truncate(path, 17 * 1024 * 1024);+ }+ const outcome = await recordBranchReconcileOutcome('app-1', { status: 'incomplete', remaining: [] });+ expect(outcome.recoveredPreservations).toEqual([]);+ expect(outcome.retirementObligations).toHaveLength(1);+ expect(outcome.preservationDiscovery).toMatchObject({ complete: false, truncated: true });+ // Streaming discovery can consume at most one additional 64 KiB chunk.+ expect(outcome.preservationDiscovery.metadataBytes).toBeLessThanOrEqual(64 * 1024 * 1024 + 64 * 1024);+ expect(outcome.preservationDiscovery.details).toContainEqual(expect.objectContaining({ reason: 'archive-discovery-budget-exceeded' }));+ });++ it.each(['ambiguous', 'unreadable-metadata', 'missing-discovery', 'truncated-discovery'])(+ 'retains an explicit hold for %s despite a complete matching archive', async failure => {+ const { saved } = await historicalArchive();+ const root = heldCheckoutArchiveRoot(PATHS.cos);+ if (failure === 'ambiguous') await cp(saved.archiveDir, join(root, 'second-matching-archive'), { recursive: true });+ if (failure === 'unreadable-metadata') {+ for (let i = 0; i < 25; i++) {+ await mkdir(join(root, `broken-metadata-${i}`));+ await writeFile(join(root, `broken-metadata-${i}`, 'manifest.json'), '{');+ }+ }+ if (failure === 'missing-discovery') await writeFile(join(PATHS.cos, 'initialized-checkout-retirements'), 'unreadable archive root');+ if (failure === 'truncated-discovery') {+ for (let i = 0; i < 1001; i++) await writeFile(join(root, `entry-${i}`), '');+ }+ const outcome = await recordBranchReconcileOutcome('app-1', { status: 'incomplete', remaining: [] });+ expect(outcome.recoveredPreservations).toEqual([]);+ expect(outcome.retirementObligations).toHaveLength(1);+ expect(outcome.retirementObligations[0].preservationEvidence.reason).toBe(+ failure === 'ambiguous' ? 'archive-preservation-ambiguous' : 'archive-discovery-incomplete');+ if (failure !== 'ambiguous') {+ expect(outcome.preservationDiscovery).toMatchObject({ complete: false, errors: failure === 'unreadable-metadata' ? 25 : 1 });+ expect(outcome.preservationDiscovery.details).toHaveLength(failure === 'unreadable-metadata' ? 20 : 1);+ expect(outcome.preservationDiscovery.detailsOmitted).toBe(failure === 'unreadable-metadata' ? 5 : 0);+ expect(outcome.preservationDiscovery.truncated).toBe(failure === 'truncated-discovery');+ }+ });++ });+it.each(['cancel', 'time', 'git-output', 'git-time'])('retains the %s diagnosis when isolated Git proof is interrupted without settling preservation', async mode => {const { saved } = await historicalArchive();const outcome = await getBranchReconcileOutcome('app-1');diff --git a/server/services/taskSchedule.js b/server/services/taskSchedule.jsindex a9f1665f8b..d23bec7d97 100644--- a/server/services/taskSchedule.js+++ b/server/services/taskSchedule.js@@ -828,7 +828,7 @@ export async function recordBranchReconcileOutcome(appId, { status, reason, rema/** Compare the whole observed inventory and its generation inside the schedule* writer. A verifier result never authorizes a newer/recreated checkout.*/-export async function recordRecoveredCheckoutPreservations(appId, expected, observations, stillAbsent) {+export async function recordRecoveredCheckoutPreservations(appId, expected, observations, stillAbsent, discovery) {return updateSchedule(async schedule => {const record = ensureExecutionRecord(schedule, 'branch-reconcile', appId);const outcome = record.branchReconcileOutcome;@@ -846,7 +846,7 @@ export async function recordRecoveredCheckoutPreservations(appId, expected, obse});return { ...release, preservationEvidence: { status: 'unresolved',reason: observation.proof && !accepted ? 'preservation-receipt-budget-exceeded' : observation.reason,- checkedAt: now, archives: observation.evidence, nextAction: observation.nextAction } };+ checkedAt: now, archives: observation.evidence, ...(observation.coverage && { coverage: observation.coverage }), nextAction: observation.nextAction } };});const retirementObligations = unresolvedRetirements({ retirementObligations: diagnosed, recoveredPreservations });const remaining = (outcome.remaining || []).flatMap(row => {@@ -864,6 +864,7 @@ export async function recordRecoveredCheckoutPreservations(appId, expected, obse: 'Historical preservation verified; run branch reconciliation again to confirm repository completion.' }: {};record.branchReconcileOutcome = { ...outcome, retirementObligations, recoveredPreservations, remaining, updatedAt: now,+ ...(discovery && { preservationDiscovery: discovery }),...diagnosis,};return { result: record.branchReconcileOutcome, changed: true };