Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS
JavaScript ai-agentsdashboarddeveloper-toolsdigital-twinquantified-selfsecond-brainself-hostedtailscale
Verify existing preservation archives in absent checkout audits (#13354)
[._.]/3fe7009d05a4
diff --git a/server/services/agentRepoStateVerification.js b/server/services/agentRepoStateVerification.jsindex 8f1968b734..3251bfcfea 100644--- a/server/services/agentRepoStateVerification.js+++ b/server/services/agentRepoStateVerification.js@@ -317,7 +317,7 @@ const skipped = (skipReason, { issues = [], observed = null } = {}) => ({ verififunction branchLedger(agentId, location) {const record = async (status, reason, extra = {}) => {const { recordCleanupOutcome } = await import('./cosAgentLifecycle.js');- await recordCleanupOutcome(agentId, { status, reason, ...location, ...extra });+ return recordCleanupOutcome(agentId, { status, reason, ...location, ...extra });};const blockWith = async (reason, { skipReason = reason, record: extra, issues, observed } = {}) => {await record('blocked', reason, extra);@@ -437,7 +437,7 @@ async function verifyAgentBranchState({ agentId, task, agentState, success, prExif (!observed) return blockWith('probe-incomplete', { skipReason: REPO_STATE_SKIPS.PROBE_INCOMPLETE });if (observed.worktreePresent === false && observed.localBranchPresent === false) {- return settleAbsentCheckout({ app, sourceWorkspace, branchName, originalPrUrl, observed, record });+ return settleAbsentCheckout({ agentId, metadata, app, sourceWorkspace, branchName, worktreePath, originalPrUrl, observed, record, staysOpen: expectation.staysOpen });}const issues = classifyRepoStateIssues(expectation, { ...observed, branchName });if (issues.length === 0) {@@ -497,13 +497,34 @@ async function resolveCheckoutIncarnation({ metadata, sourceWorkspace, branchNam* Missing originals are an accounting gap, not a reason to hire an agent* to rediscover absence. Never invent a preservation receipt.*/-async function settleAbsentCheckout({ app, sourceWorkspace, branchName, originalPrUrl, observed, record }) {+async function settleAbsentCheckout({ agentId, metadata, app, sourceWorkspace, branchName, worktreePath, originalPrUrl, observed, record, staysOpen }) {const { readRetirementForgeEvidence } = await import('./heldCheckoutReview.js');const proofUrl = originalPrUrl || observed.prUrl;const forge = proofUrl && observed.prState === 'MERGED' ? await readRetirementForgeEvidence(sourceWorkspace, branchName, proofUrl,{ forgeAccount: app?.forgeAccount ?? null }).catch(() => null) : null;const integrated = forge && (await execGit(['merge-base', '--is-ancestor', forge.prHead, forge.defaultHead], sourceWorkspace,{ ignoreExitCode: true, env: { ...process.env, GIT_OPTIONAL_LOCKS: '0', GIT_NO_LAZY_FETCH: '1' } }).catch(() => null))?.exitCode === 0;+ const completed = metadata.completedCheckouts?.find(row => row.version === 1 && row.agentId === agentId+ && row.repoPath === sourceWorkspace && row.branch === branchName && row.path === worktreePath+ && Number.isFinite(Date.parse(row.startedAt)) && Date.parse(row.capturedAt) >= Date.parse(row.startedAt));+ if (!staysOpen && integrated && completed && completed.head === forge.prHead && observed.remoteBranchPresent === false) {+ const { findMissingCheckoutPreservation } = await import('./heldCheckoutRecovery.js');+ const release = { branch: branchName, path: worktreePath, head: completed.head,+ locked: false, lockReason: null, createdAt: completed.capturedAt };+ const proof = await findMissingCheckoutPreservation({ repoPath: sourceWorkspace, release }).catch(() => null);+ if (proof) {+ const { withClaimOwnershipLock } = await import('./cosClaimOwnership.js');+ const verified = await withClaimOwnershipLock(async () => {+ if (await resolveCheckoutIncarnation({ metadata, sourceWorkspace, branchName, worktreePath }) || !await proof.recheck()) return false;+ const { recheck: _recheck, ...evidence } = proof;+ const saved = await record('verified', 'already-absent-preservation-verified', { head: completed.head,+ prUrl: proofUrl, committedIntegrationVerified: true,+ preservation: { ...evidence, capturedAt: completed.capturedAt, verifiedAt: new Date().toISOString() } });+ return saved === true;+ }).catch(() => false);+ if (verified) return { verified: true, skipReason: null, issues: [], observed, recoveryTaskId: null };+ }+ }const reason = integrated ? 'already-absent-preservation-unverified' : 'absent-checkout-integration-unverified';await record(integrated ? 'already-absent' : 'blocked', reason, { prUrl: originalPrUrl, committedIntegrationVerified: !!integrated });return skipped(reason);diff --git a/server/services/agentRepoStateVerification.test.js b/server/services/agentRepoStateVerification.test.jsindex 66d27945de..2fdc9e129c 100644--- a/server/services/agentRepoStateVerification.test.js+++ b/server/services/agentRepoStateVerification.test.js@@ -29,9 +29,10 @@ vi.mock('./heldCheckoutReview.js', () => ({ readRetirementForgeEvidence: vi.fn(avi.mock('./completedCheckoutEvidence.js', () => ({ readCompletedCheckoutIdentity: vi.fn() }));vi.mock('./cos.js', () => ({}));vi.mock('./cosAgentLifecycle.js', () => ({ reserveRepoCleanupRecovery: vi.fn(async () => ({ admitted: true, attempt: 1 })), recordRepoCleanupRecovery: vi.fn(async () => true), recordCleanupOutcome: vi.fn(async () => true), updateAgent: vi.fn().mockResolvedValue({}), recordRepoStateRecovery: vi.fn().mockResolvedValue(true), recordRepoStateVerification: vi.fn().mockResolvedValue(true) }));-vi.mock('./cosClaimOwnership.js', () => ({ readClaimOwnershipAgents: vi.fn().mockResolvedValue([]) }));+vi.mock('./cosClaimOwnership.js', () => ({ readClaimOwnershipAgents: vi.fn().mockResolvedValue([]), withClaimOwnershipLock: fn => fn() }));vi.mock('./taskSchedule.js', () => ({ recordBranchReconcileOutcome: vi.fn().mockResolvedValue({}) }));vi.mock('./heldCheckoutRecovery.js', () => ({+ findMissingCheckoutPreservation: vi.fn(async () => null),retireInventoriedCheckout: vi.fn(async () => ({ ok: true, archiveDir: '/private/archive', branchDeleted: true })),prepareInventoriedCheckoutRetirement: vi.fn().mockResolvedValue({ ok: true, review: { digest: 'a'.repeat(64) } }),}));@@ -80,7 +81,7 @@ import { verifyAgentRepoState, sweepRepoStateVerifications, REPO_STATE_REMEDIATIimport { readBranchCompletionPolicies } from './branchCompletionPolicy.js';import { getTaskById } from './cosTaskStore.js';import { recordBranchReconcileOutcome } from './taskSchedule.js';-import { prepareInventoriedCheckoutRetirement, retireInventoriedCheckout } from './heldCheckoutRecovery.js';+import { findMissingCheckoutPreservation, prepareInventoriedCheckoutRetirement, retireInventoriedCheckout } from './heldCheckoutRecovery.js';import { addTask, getAllTasks } from './cosTaskStore.js';import { ensureTaskThread } from './brainTaskThreads.js';import { listWorktrees } from './worktreeManager.js';@@ -141,6 +142,7 @@ const run = (overrides = {}) => verifyAgentRepoState({beforeEach(() => {vi.clearAllMocks();+ findMissingCheckoutPreservation.mockResolvedValue(null);inspectOrQueueCompletedReview.mockResolvedValue({ disposition: 'held', reason: 'delivery-ci-or-review-pending' });readBranchCompletionPolicies.mockResolvedValue([]);getTaskById.mockResolvedValue(null);@@ -168,6 +170,29 @@ beforeEach(() => {});describe('verifyAgentRepoState — clean runs', () => {+ it('settles an absent completed checkout from verified original-generation archive evidence without deleting or queueing work', async () => {+ const completed = { version: 1, agentId: 'agent-1', repoPath: '/repo', branch: BRANCH, path: WORKTREE, head: HEAD,+ startedAt: CREATED_AT, capturedAt: CREATED_AT };+ const state = agentState({ completedCheckouts: [completed], reviewLoopMergeTarget: { url: 'https://github.com/example/repo/pull/7' }, repoStateVerification: { status: 'pending' } });+ const recheck = vi.fn(async () => true);+ findMissingCheckoutPreservation.mockResolvedValue({ archiveDir: '/example/archive', evidenceDigest: 'a'.repeat(64), recheck });+ expect(await run({ agentState: state })).toMatchObject({ verified: true, skipReason: null });+ expect(findMissingCheckoutPreservation).toHaveBeenCalledWith({ repoPath: '/repo', release: { ...RELEASE, createdAt: CREATED_AT } });+ expect(recordCleanupOutcome).toHaveBeenLastCalledWith('agent-1', expect.objectContaining({+ status: 'verified', reason: 'already-absent-preservation-verified', head: HEAD,+ preservation: expect.objectContaining({ capturedAt: CREATED_AT, evidenceDigest: 'a'.repeat(64) }),+ }));+ expect(recordRepoStateVerification).toHaveBeenLastCalledWith('agent-1', expect.objectContaining({ status: 'completed' }));+ expect(addTask).not.toHaveBeenCalled();+ expect(retireInventoriedCheckout).not.toHaveBeenCalled();+ recordCleanupOutcome.mockResolvedValueOnce(false);+ expect(await run({ agentState: state })).toMatchObject({ verified: false, skipReason: 'already-absent-preservation-unverified' });+ recheck.mockResolvedValue(false);+ expect(await run({ agentState: state })).toMatchObject({ verified: false, skipReason: 'already-absent-preservation-unverified' });+ findMissingCheckoutPreservation.mockResolvedValue(null);+ expect(await run({ agentState: state })).toMatchObject({ verified: false, skipReason: 'already-absent-preservation-unverified' });+ });+it.each([['broken leaf symlink', true], ['unreadable EACCES leaf', null]])('never treats %s as absent or verified cleanup', async (_name, presence) => {readCheckoutPathPresence.mockResolvedValue(presence);const result = await run();diff --git a/server/services/heldCheckoutRecovery.js b/server/services/heldCheckoutRecovery.jsindex 6c1753a70a..554246235d 100644--- a/server/services/heldCheckoutRecovery.js+++ b/server/services/heldCheckoutRecovery.js@@ -155,6 +155,38 @@ async function historicalCheckoutAbsent(release, trees) {return !present && trees.length > 0 && !trees.some(tree => tree.path === release.path || (release.branch && shortBranch(tree.branch) === release.branch));}+/** Completion audits have a server-captured checkout generation, but may never+ * have entered the scheduled retirement queue. Reuse its archive verifier+ * without creating an obligation or inventing a branch-deletion receipt.+ */+export async function findMissingCheckoutPreservation({ repoPath, release }, opts = {}) {+ if (!Number.isFinite(Date.parse(release?.createdAt)) || Date.parse(release.createdAt) > Date.now()+ || !/^[a-f0-9]{40}(?:[a-f0-9]{24})?$/.test(release?.head || '')) return null;+ const absent = async () => {+ const trees = await listWorktrees(repoPath);+ const ref = await execGit(['rev-parse', '--verify', '--quiet', `refs/heads/${release.branch}`], repoPath, { ignoreExitCode: true });+ return ref.exitCode === 1 && await historicalCheckoutAbsent(release, trees);+ };+ if (!await absent()) return null;+ const root = heldCheckoutArchiveRoot(opts.cosDir);+ const info = await lstat(root).catch(error => { if (error.code === 'ENOENT') return null; throw error; });+ if (!info?.isDirectory()) return null;+ const entries = await readdir(root, { withFileTypes: true });+ if (entries.length > 1000) return null;+ let selected = null;+ for (const entry of entries) {+ if (!entry.isDirectory()) continue;+ const proof = await verifyHeldArchive(repoPath, join(root, entry.name), release).catch(() => null);+ if (!proof) continue;+ // Multiple valid snapshots cannot identify the original private contents.+ if (selected) return null;+ selected = proof;+ }+ if (!selected) return null;+ const proof = selected;+ return { ...proof, recheck: async () => await absent() && await proof.recheck() };+}+/** Read existing archives only. Verification stays outside both writer locks;* the final ownership/absence recheck and schedule compare-and-set are fenced.*/diff --git a/server/services/heldCheckoutRecovery.test.js b/server/services/heldCheckoutRecovery.test.jsindex 8b077a7344..608f485bd5 100644--- a/server/services/heldCheckoutRecovery.test.js+++ b/server/services/heldCheckoutRecovery.test.js@@ -50,7 +50,7 @@ vi.mock('./taskScheduleStore.js', async (importOriginal) => ({}));vi.mock('./apps.js', () => ({ getAppById: async (id) => (injected.app?.id === id ? injected.app : null) }));-const { retireInventoriedCheckout, prepareInventoriedCheckoutRetirement, retirePendingMergeCheckout, heldCheckoutArchiveRoot, preserveMergedCheckout, recoverMissingCheckoutPreservations } = await import('./heldCheckoutRecovery.js');+const { retireInventoriedCheckout, prepareInventoriedCheckoutRetirement, retirePendingMergeCheckout, heldCheckoutArchiveRoot, preserveMergedCheckout, recoverMissingCheckoutPreservations, findMissingCheckoutPreservation } = await import('./heldCheckoutRecovery.js');const { recordBranchReconcileOutcome, getBranchReconcileOutcome } = await import('./taskSchedule.js');const { withClaimOwnershipLock, checkReconcileOwnership } = await import('./cosClaimOwnership.js');@@ -139,6 +139,26 @@ describe.skipIf(SKIP_HEAVY_INTEGRATION)('held checkout recovery (real git)', ()expect(recreated.retirementObligations[0].preservationEvidence.archives).toContainEqual({ archiveDir: saved.archiveDir, reason: 'archive-generation-mismatch' });});+ it('recognizes an absent completion archive without a schedule obligation and rejects changed or older evidence', async () => {+ const { release, saved } = await historicalArchive();+ const outcome = await getBranchReconcileOutcome('app-1');+ const original = { ...release, createdAt: outcome.retirementObligations[0].createdAt };+ injected.schedule = { executions: {} };+ const withoutObligation = await getBranchReconcileOutcome('app-1');+ await execGit(['branch', '-d', release.branch], repo);+ const proof = await findMissingCheckoutPreservation({ repoPath: repo, release: original });+ expect(proof).toMatchObject({ archiveDir: saved.archiveDir, evidenceDigest: expect.stringMatching(/^[a-f0-9]{64}$/) });+ expect(await proof.recheck()).toBe(true);+ await mkdir(release.path);+ expect(await proof.recheck()).toBe(false);+ await rm(release.path, { recursive: true });+ const manifest = JSON.parse(await readFile(join(saved.archiveDir, 'manifest.json'), 'utf8'));+ expect(await findMissingCheckoutPreservation({ repoPath: repo, release: { ...original, createdAt: new Date(Date.parse(manifest.archivedAt) + 1).toISOString() } })).toBeNull();+ await writeFile(join(saved.archiveDir, 'files', 'private.txt'), 'corrupt archived evidence');+ expect(await findMissingCheckoutPreservation({ repoPath: repo, release: original })).toBeNull();+ expect(await getBranchReconcileOutcome('app-1')).toEqual(withoutObligation);+ });+it.each([false, true])('clears the historical diagnosis after recovery retries, with other holds=%s', async otherHolds => {const { saved } = await historicalArchive();const unavailable = join(PATHS.cos, 'temporarily-unavailable-archive');