Skip to content

Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS

Verify existing preservation archives in absent checkout audits (#13354)

[._.]/3fe7009d05a4
diff --git a/server/services/agentRepoStateVerification.js b/server/services/agentRepoStateVerification.js
index 8f1968b734..3251bfcfea 100644
--- a/server/services/agentRepoStateVerification.js
+++ b/server/services/agentRepoStateVerification.js
@@ -317,7 +317,7 @@ const skipped = (skipReason, { issues = [], observed = null } = {}) => ({ verifi
function branchLedger(agentId, location) {
const record = async (status, reason, extra = {}) => {
const { recordCleanupOutcome } = await import('./cosAgentLifecycle.js');
- await recordCleanupOutcome(agentId, { status, reason, ...location, ...extra });
+ return recordCleanupOutcome(agentId, { status, reason, ...location, ...extra });
};
const blockWith = async (reason, { skipReason = reason, record: extra, issues, observed } = {}) => {
await record('blocked', reason, extra);
@@ -437,7 +437,7 @@ async function verifyAgentBranchState({ agentId, task, agentState, success, prEx
if (!observed) return blockWith('probe-incomplete', { skipReason: REPO_STATE_SKIPS.PROBE_INCOMPLETE });
if (observed.worktreePresent === false && observed.localBranchPresent === false) {
- return settleAbsentCheckout({ app, sourceWorkspace, branchName, originalPrUrl, observed, record });
+ return settleAbsentCheckout({ agentId, metadata, app, sourceWorkspace, branchName, worktreePath, originalPrUrl, observed, record, staysOpen: expectation.staysOpen });
}
const issues = classifyRepoStateIssues(expectation, { ...observed, branchName });
if (issues.length === 0) {
@@ -497,13 +497,34 @@ async function resolveCheckoutIncarnation({ metadata, sourceWorkspace, branchNam
* Missing originals are an accounting gap, not a reason to hire an agent
* to rediscover absence. Never invent a preservation receipt.
*/
-async function settleAbsentCheckout({ app, sourceWorkspace, branchName, originalPrUrl, observed, record }) {
+async function settleAbsentCheckout({ agentId, metadata, app, sourceWorkspace, branchName, worktreePath, originalPrUrl, observed, record, staysOpen }) {
const { readRetirementForgeEvidence } = await import('./heldCheckoutReview.js');
const proofUrl = originalPrUrl || observed.prUrl;
const forge = proofUrl && observed.prState === 'MERGED' ? await readRetirementForgeEvidence(sourceWorkspace, branchName, proofUrl,
{ forgeAccount: app?.forgeAccount ?? null }).catch(() => null) : null;
const integrated = forge && (await execGit(['merge-base', '--is-ancestor', forge.prHead, forge.defaultHead], sourceWorkspace,
{ ignoreExitCode: true, env: { ...process.env, GIT_OPTIONAL_LOCKS: '0', GIT_NO_LAZY_FETCH: '1' } }).catch(() => null))?.exitCode === 0;
+ const completed = metadata.completedCheckouts?.find(row => row.version === 1 && row.agentId === agentId
+ && row.repoPath === sourceWorkspace && row.branch === branchName && row.path === worktreePath
+ && Number.isFinite(Date.parse(row.startedAt)) && Date.parse(row.capturedAt) >= Date.parse(row.startedAt));
+ if (!staysOpen && integrated && completed && completed.head === forge.prHead && observed.remoteBranchPresent === false) {
+ const { findMissingCheckoutPreservation } = await import('./heldCheckoutRecovery.js');
+ const release = { branch: branchName, path: worktreePath, head: completed.head,
+ locked: false, lockReason: null, createdAt: completed.capturedAt };
+ const proof = await findMissingCheckoutPreservation({ repoPath: sourceWorkspace, release }).catch(() => null);
+ if (proof) {
+ const { withClaimOwnershipLock } = await import('./cosClaimOwnership.js');
+ const verified = await withClaimOwnershipLock(async () => {
+ if (await resolveCheckoutIncarnation({ metadata, sourceWorkspace, branchName, worktreePath }) || !await proof.recheck()) return false;
+ const { recheck: _recheck, ...evidence } = proof;
+ const saved = await record('verified', 'already-absent-preservation-verified', { head: completed.head,
+ prUrl: proofUrl, committedIntegrationVerified: true,
+ preservation: { ...evidence, capturedAt: completed.capturedAt, verifiedAt: new Date().toISOString() } });
+ return saved === true;
+ }).catch(() => false);
+ if (verified) return { verified: true, skipReason: null, issues: [], observed, recoveryTaskId: null };
+ }
+ }
const reason = integrated ? 'already-absent-preservation-unverified' : 'absent-checkout-integration-unverified';
await record(integrated ? 'already-absent' : 'blocked', reason, { prUrl: originalPrUrl, committedIntegrationVerified: !!integrated });
return skipped(reason);
diff --git a/server/services/agentRepoStateVerification.test.js b/server/services/agentRepoStateVerification.test.js
index 66d27945de..2fdc9e129c 100644
--- a/server/services/agentRepoStateVerification.test.js
+++ b/server/services/agentRepoStateVerification.test.js
@@ -29,9 +29,10 @@ vi.mock('./heldCheckoutReview.js', () => ({ readRetirementForgeEvidence: vi.fn(a
vi.mock('./completedCheckoutEvidence.js', () => ({ readCompletedCheckoutIdentity: vi.fn() }));
vi.mock('./cos.js', () => ({}));
vi.mock('./cosAgentLifecycle.js', () => ({ reserveRepoCleanupRecovery: vi.fn(async () => ({ admitted: true, attempt: 1 })), recordRepoCleanupRecovery: vi.fn(async () => true), recordCleanupOutcome: vi.fn(async () => true), updateAgent: vi.fn().mockResolvedValue({}), recordRepoStateRecovery: vi.fn().mockResolvedValue(true), recordRepoStateVerification: vi.fn().mockResolvedValue(true) }));
-vi.mock('./cosClaimOwnership.js', () => ({ readClaimOwnershipAgents: vi.fn().mockResolvedValue([]) }));
+vi.mock('./cosClaimOwnership.js', () => ({ readClaimOwnershipAgents: vi.fn().mockResolvedValue([]), withClaimOwnershipLock: fn => fn() }));
vi.mock('./taskSchedule.js', () => ({ recordBranchReconcileOutcome: vi.fn().mockResolvedValue({}) }));
vi.mock('./heldCheckoutRecovery.js', () => ({
+ findMissingCheckoutPreservation: vi.fn(async () => null),
retireInventoriedCheckout: vi.fn(async () => ({ ok: true, archiveDir: '/private/archive', branchDeleted: true })),
prepareInventoriedCheckoutRetirement: vi.fn().mockResolvedValue({ ok: true, review: { digest: 'a'.repeat(64) } }),
}));
@@ -80,7 +81,7 @@ import { verifyAgentRepoState, sweepRepoStateVerifications, REPO_STATE_REMEDIATI
import { readBranchCompletionPolicies } from './branchCompletionPolicy.js';
import { getTaskById } from './cosTaskStore.js';
import { recordBranchReconcileOutcome } from './taskSchedule.js';
-import { prepareInventoriedCheckoutRetirement, retireInventoriedCheckout } from './heldCheckoutRecovery.js';
+import { findMissingCheckoutPreservation, prepareInventoriedCheckoutRetirement, retireInventoriedCheckout } from './heldCheckoutRecovery.js';
import { addTask, getAllTasks } from './cosTaskStore.js';
import { ensureTaskThread } from './brainTaskThreads.js';
import { listWorktrees } from './worktreeManager.js';
@@ -141,6 +142,7 @@ const run = (overrides = {}) => verifyAgentRepoState({
beforeEach(() => {
vi.clearAllMocks();
+ findMissingCheckoutPreservation.mockResolvedValue(null);
inspectOrQueueCompletedReview.mockResolvedValue({ disposition: 'held', reason: 'delivery-ci-or-review-pending' });
readBranchCompletionPolicies.mockResolvedValue([]);
getTaskById.mockResolvedValue(null);
@@ -168,6 +170,29 @@ beforeEach(() => {
});
describe('verifyAgentRepoState — clean runs', () => {
+ it('settles an absent completed checkout from verified original-generation archive evidence without deleting or queueing work', async () => {
+ const completed = { version: 1, agentId: 'agent-1', repoPath: '/repo', branch: BRANCH, path: WORKTREE, head: HEAD,
+ startedAt: CREATED_AT, capturedAt: CREATED_AT };
+ const state = agentState({ completedCheckouts: [completed], reviewLoopMergeTarget: { url: 'https://github.com/example/repo/pull/7' }, repoStateVerification: { status: 'pending' } });
+ const recheck = vi.fn(async () => true);
+ findMissingCheckoutPreservation.mockResolvedValue({ archiveDir: '/example/archive', evidenceDigest: 'a'.repeat(64), recheck });
+ expect(await run({ agentState: state })).toMatchObject({ verified: true, skipReason: null });
+ expect(findMissingCheckoutPreservation).toHaveBeenCalledWith({ repoPath: '/repo', release: { ...RELEASE, createdAt: CREATED_AT } });
+ expect(recordCleanupOutcome).toHaveBeenLastCalledWith('agent-1', expect.objectContaining({
+ status: 'verified', reason: 'already-absent-preservation-verified', head: HEAD,
+ preservation: expect.objectContaining({ capturedAt: CREATED_AT, evidenceDigest: 'a'.repeat(64) }),
+ }));
+ expect(recordRepoStateVerification).toHaveBeenLastCalledWith('agent-1', expect.objectContaining({ status: 'completed' }));
+ expect(addTask).not.toHaveBeenCalled();
+ expect(retireInventoriedCheckout).not.toHaveBeenCalled();
+ recordCleanupOutcome.mockResolvedValueOnce(false);
+ expect(await run({ agentState: state })).toMatchObject({ verified: false, skipReason: 'already-absent-preservation-unverified' });
+ recheck.mockResolvedValue(false);
+ expect(await run({ agentState: state })).toMatchObject({ verified: false, skipReason: 'already-absent-preservation-unverified' });
+ findMissingCheckoutPreservation.mockResolvedValue(null);
+ expect(await run({ agentState: state })).toMatchObject({ verified: false, skipReason: 'already-absent-preservation-unverified' });
+ });
+
it.each([['broken leaf symlink', true], ['unreadable EACCES leaf', null]])('never treats %s as absent or verified cleanup', async (_name, presence) => {
readCheckoutPathPresence.mockResolvedValue(presence);
const result = await run();
diff --git a/server/services/heldCheckoutRecovery.js b/server/services/heldCheckoutRecovery.js
index 6c1753a70a..554246235d 100644
--- a/server/services/heldCheckoutRecovery.js
+++ b/server/services/heldCheckoutRecovery.js
@@ -155,6 +155,38 @@ async function historicalCheckoutAbsent(release, trees) {
return !present && trees.length > 0 && !trees.some(tree => tree.path === release.path || (release.branch && shortBranch(tree.branch) === release.branch));
}
+/** Completion audits have a server-captured checkout generation, but may never
+ * have entered the scheduled retirement queue. Reuse its archive verifier
+ * without creating an obligation or inventing a branch-deletion receipt.
+ */
+export async function findMissingCheckoutPreservation({ repoPath, release }, opts = {}) {
+ if (!Number.isFinite(Date.parse(release?.createdAt)) || Date.parse(release.createdAt) > Date.now()
+ || !/^[a-f0-9]{40}(?:[a-f0-9]{24})?$/.test(release?.head || '')) return null;
+ const absent = async () => {
+ const trees = await listWorktrees(repoPath);
+ const ref = await execGit(['rev-parse', '--verify', '--quiet', `refs/heads/${release.branch}`], repoPath, { ignoreExitCode: true });
+ return ref.exitCode === 1 && await historicalCheckoutAbsent(release, trees);
+ };
+ if (!await absent()) return null;
+ const root = heldCheckoutArchiveRoot(opts.cosDir);
+ const info = await lstat(root).catch(error => { if (error.code === 'ENOENT') return null; throw error; });
+ if (!info?.isDirectory()) return null;
+ const entries = await readdir(root, { withFileTypes: true });
+ if (entries.length > 1000) return null;
+ let selected = null;
+ for (const entry of entries) {
+ if (!entry.isDirectory()) continue;
+ const proof = await verifyHeldArchive(repoPath, join(root, entry.name), release).catch(() => null);
+ if (!proof) continue;
+ // Multiple valid snapshots cannot identify the original private contents.
+ if (selected) return null;
+ selected = proof;
+ }
+ if (!selected) return null;
+ const proof = selected;
+ return { ...proof, recheck: async () => await absent() && await proof.recheck() };
+}
+
/** Read existing archives only. Verification stays outside both writer locks;
* the final ownership/absence recheck and schedule compare-and-set are fenced.
*/
diff --git a/server/services/heldCheckoutRecovery.test.js b/server/services/heldCheckoutRecovery.test.js
index 8b077a7344..608f485bd5 100644
--- a/server/services/heldCheckoutRecovery.test.js
+++ b/server/services/heldCheckoutRecovery.test.js
@@ -50,7 +50,7 @@ vi.mock('./taskScheduleStore.js', async (importOriginal) => ({
}));
vi.mock('./apps.js', () => ({ getAppById: async (id) => (injected.app?.id === id ? injected.app : null) }));
-const { retireInventoriedCheckout, prepareInventoriedCheckoutRetirement, retirePendingMergeCheckout, heldCheckoutArchiveRoot, preserveMergedCheckout, recoverMissingCheckoutPreservations } = await import('./heldCheckoutRecovery.js');
+const { retireInventoriedCheckout, prepareInventoriedCheckoutRetirement, retirePendingMergeCheckout, heldCheckoutArchiveRoot, preserveMergedCheckout, recoverMissingCheckoutPreservations, findMissingCheckoutPreservation } = await import('./heldCheckoutRecovery.js');
const { recordBranchReconcileOutcome, getBranchReconcileOutcome } = await import('./taskSchedule.js');
const { withClaimOwnershipLock, checkReconcileOwnership } = await import('./cosClaimOwnership.js');
@@ -139,6 +139,26 @@ describe.skipIf(SKIP_HEAVY_INTEGRATION)('held checkout recovery (real git)', ()
expect(recreated.retirementObligations[0].preservationEvidence.archives).toContainEqual({ archiveDir: saved.archiveDir, reason: 'archive-generation-mismatch' });
});
+ it('recognizes an absent completion archive without a schedule obligation and rejects changed or older evidence', async () => {
+ const { release, saved } = await historicalArchive();
+ const outcome = await getBranchReconcileOutcome('app-1');
+ const original = { ...release, createdAt: outcome.retirementObligations[0].createdAt };
+ injected.schedule = { executions: {} };
+ const withoutObligation = await getBranchReconcileOutcome('app-1');
+ await execGit(['branch', '-d', release.branch], repo);
+ const proof = await findMissingCheckoutPreservation({ repoPath: repo, release: original });
+ expect(proof).toMatchObject({ archiveDir: saved.archiveDir, evidenceDigest: expect.stringMatching(/^[a-f0-9]{64}$/) });
+ expect(await proof.recheck()).toBe(true);
+ await mkdir(release.path);
+ expect(await proof.recheck()).toBe(false);
+ await rm(release.path, { recursive: true });
+ const manifest = JSON.parse(await readFile(join(saved.archiveDir, 'manifest.json'), 'utf8'));
+ expect(await findMissingCheckoutPreservation({ repoPath: repo, release: { ...original, createdAt: new Date(Date.parse(manifest.archivedAt) + 1).toISOString() } })).toBeNull();
+ await writeFile(join(saved.archiveDir, 'files', 'private.txt'), 'corrupt archived evidence');
+ expect(await findMissingCheckoutPreservation({ repoPath: repo, release: original })).toBeNull();
+ expect(await getBranchReconcileOutcome('app-1')).toEqual(withoutObligation);
+ });
+
it.each([false, true])('clears the historical diagnosis after recovery retries, with other holds=%s', async otherHolds => {
const { saved } = await historicalArchive();
const unavailable = join(PATHS.cos, 'temporarily-unavailable-archive');