Skip to content

Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS

fix: recover and archive deferred snapshot validation checkouts (#13269)

Adam Eivyc0cea3ca1fd2
diff --git a/docs/GITHUB_ACTIONS.md b/docs/GITHUB_ACTIONS.md
index 3d62665c6d..2d13b42cf1 100644
--- a/docs/GITHUB_ACTIONS.md
+++ b/docs/GITHUB_ACTIONS.md
@@ -195,7 +195,12 @@ the GitHub merge to that head. New and reused snapshot PRs use detached checkout
Failed guards, moved/unreadable inputs, or refused merges retain the PR, checkout,
and `.cache/quality-snapshot-merge.json` evidence for a later publish cycle;
they do not enter the generic pending-merge queue. Guard-capable GitLab MRs stay
-open until a supported pinned validation path is available. Repositories without
+open until a supported pinned validation path is available. Retained checkouts
+are located through Git's durable worktree inventory and reused for the same head
+on retry. After a verified merge, every retained attempt for that PR is archived
+and verified through checkout preservation before retirement; changed ownership,
+locks, or archive failures retain the checkout and report the remaining paths.
+Repositories without
this capability retain their existing snapshot merge behavior; arbitrary scripts
are never executed as a substitute.
diff --git a/server/services/appQualitySnapshotFile.js b/server/services/appQualitySnapshotFile.js
index acbdc4cbea..9eafd2149a 100644
--- a/server/services/appQualitySnapshotFile.js
+++ b/server/services/appQualitySnapshotFile.js
@@ -19,9 +19,8 @@
* generic repositories retain their existing merge-on-green fallback.
* The immediate merge is the one step that advances the default branch, so it
* runs under this install's repository merge admission (`cosMergeAdmission.js`)
- * like every other final merge: while a claim holds the lease the PR is left
- * open and handed to the local pending-merge queue, which takes admission at
- * its own later attempt. Reads never write. A v1 file or historical
+ * like every other final merge. A busy lease leaves the PR recoverable for
+ * a later admitted attempt. Reads never write. A v1 file or historical
* `quality-snapshot.json` is explicit legacy input. Before publication, live
* review ownership and non-snapshot branch changes defer replacement; the push
* lease pins only the exact head this invocation inspected.
@@ -461,7 +460,7 @@ async function landSnapshotPr(app, { body, count, defaultBranch, git, deps, mess
// If validation is interrupted, preserve the published checkout by default.
retainWorktree = true;
const queued = await mergeSnapshotPr(app, pr, deps, git, defaultBranch, { worktreePath, expectedHeadSha: hash });
- retainWorktree = queued.deferred === true && queued.guard?.required === true;
+ retainWorktree = queued.guardManaged === true;
console.log(`📊 Published quality snapshot for app ${app.id}: ${count} measurements → ${hash} (${pr.url})`);
return {
published: true, hash, path: APP_QUALITY_SNAPSHOT_FILENAME,
@@ -559,6 +558,7 @@ const mergeOutcomeFields = outcome => ({
queuedMerge: outcome.queued === true,
...(outcome.deferred === true ? { deferredMerge: true } : {}),
...(outcome.guard ? { mergeGuard: outcome.guard } : {}),
+ ...(outcome.retainedCheckouts?.length ? { retainedMergeCheckouts: outcome.retainedCheckouts } : {}),
});
/**
@@ -568,10 +568,8 @@ const mergeOutcomeFields = outcome => ({
*
* The immediate attempt needs this install's repository merge admission. When
* another owner holds it (or ownership cannot be verified) the merge is
- * deferred: GitHub hands the PR to the local pending-merge queue, which takes
- * admission at its own later attempt; GitLab's only queue is the forge's
- * auto-merge, which cannot honor a local lease, so the MR stays open for the
- * next publish cycle instead.
+ * deferred. Only generic GitHub repositories use the local pending-merge
+ * queue. Guard-capable PRs and GitLab admission refusals stay open for retry.
*
* @returns {{merged: boolean, queued: boolean, deferred?: boolean}} — merged
* and queued are never both true.
@@ -595,24 +593,33 @@ async function mergeSnapshotPr(app, pr, deps, git, defaultBranch, checkout = {})
}
async function guardedSnapshotMerge(app, pr, deps, git, guards, evidence, existingPath) {
- const deferred = () => ({ merged: false, queued: false, deferred: true, guard: evidence });
+ const deferred = () => ({ merged: false, queued: false, deferred: true, guard: evidence, guardManaged: true });
const run = async path => {
evidence.worktreePath = path;
try {
+ const current = await git.execGit(['rev-parse', '--verify', 'HEAD^{commit}'], path);
+ if (current.exitCode !== 0 || !/^[a-f0-9]{40}$/i.test(current.stdout?.trim() || '')) {
+ throw new Error('snapshot-checkout-head-unreadable');
+ }
+ evidence.checkoutHead = current.stdout.trim();
+ await guards.recordSnapshotMergeEvidence(pr, evidence, deps, { status: 'preparing' });
if (evidence.reason) throw new Error(evidence.reason);
- if (!existingPath) await addSnapshotWorktree(app, deps, path, evidence.headSha);
await guards.validateSnapshotMerge(app, pr, evidence, git, deps);
const attempt = await withSnapshotMergeAdmission(app, deps, async () => {
await guards.verifySnapshotMergeInputs(app, pr, evidence, git, deps);
return tryImmediateMerge(app, pr, deps, git, evidence.headSha);
});
- if (attempt.result) return { merged: true, queued: false };
+ if (attempt.result) {
+ const retire = deps.retireSnapshotMergeCheckouts || guards.retireSnapshotMergeCheckouts;
+ const retainedCheckouts = await retire(app, pr, deps).catch(() => [path]);
+ return { merged: true, queued: false, guardManaged: true, retainedCheckouts };
+ }
evidence.reason = attempt.reason || 'snapshot-merge-deferred';
} catch (error) {
- evidence.reason = error.message;
+ evidence.reason = error.stdout !== undefined || error.stderr !== undefined ? 'snapshot-merge-guards-failed' : error.message;
console.log(`⏳ Quality snapshot merge retained for app ${app.id}: ${evidence.reason}`);
await guards.recordSnapshotMergeEvidence(pr, evidence, deps, {
- status: 'deferred', stdout: error.stdout, stderr: error.stderr,
+ status: 'deferred', error: error.message, stdout: error.stdout, stderr: error.stderr,
});
return deferred();
}
@@ -623,10 +630,14 @@ async function guardedSnapshotMerge(app, pr, deps, git, guards, evidence, existi
if (evidence.reason) return deferred();
const { withCheckoutPublication } = await import('../lib/checkoutRetirementAdmission.js');
return withCheckoutPublication(app.repoPath, async () => {
- const path = await (deps.mkdtemp || mkdtemp)(join(tmpdir(), 'portos-quality-'));
- const result = await run(path);
- if (result.merged) await cleanupSnapshotWorktree(app, git, path, deps.rm || rm);
- return result;
+ const retained = await guards.retainedSnapshotMergeCheckouts(app, pr, deps);
+ const matching = retained.find(tree => tree.head === evidence.headSha);
+ const path = matching?.path || await (deps.mkdtemp || mkdtemp)(join(tmpdir(), 'portos-quality-'));
+ if (!matching) await addSnapshotWorktree(app, deps, path, evidence.headSha);
+ return run(path);
+ }).catch(error => {
+ evidence.reason = error.message;
+ return deferred();
});
}
diff --git a/server/services/appQualitySnapshotFile.test.js b/server/services/appQualitySnapshotFile.test.js
index e4802bc432..a1ca11da83 100644
--- a/server/services/appQualitySnapshotFile.test.js
+++ b/server/services/appQualitySnapshotFile.test.js
@@ -896,6 +896,8 @@ const guardedPublication = (reused = false) => {
const base = 'b'.repeat(40);
const live = { head, base, state: 'OPEN', manifest: { scripts: { 'pregate:merged': 'node scripts/pregate.js --guards-only' } } };
const events = [];
+ const trees = [];
+ const evidence = new Map();
const deps = testDeps({
git: gitDouble({
execGit: vi.fn(async args => {
@@ -926,15 +928,22 @@ const guardedPublication = (reused = false) => {
state: live.state, isDraft: false, isCrossRepository: false, headRefOid: live.head,
headRefName: QUALITY_SNAPSHOT_BRANCH, baseRefName: 'main',
})),
+ addWorktree: vi.fn(async () => { trees.push({ path: worktreePath, head, detached: true }); }),
linkWorktreeDependencies: vi.fn(async () => {}),
- writeMergeEvidence: vi.fn(async () => {}),
+ writeMergeEvidence: vi.fn(async (path, record) => { evidence.set(path, structuredClone(record)); }),
+ readMergeEvidence: vi.fn(async path => {
+ if (!evidence.has(path)) throw Object.assign(new Error('absent'), { code: 'ENOENT' });
+ return JSON.stringify(evidence.get(path));
+ }),
+ listSnapshotMergeWorktrees: vi.fn(async () => [...trees]),
+ retireSnapshotMergeCheckouts: vi.fn(async () => { trees.length = 0; evidence.clear(); return []; }),
execSnapshotMergeGuards: vi.fn(async () => { events.push('guards'); return { stdout: 'guards passed', stderr: '' }; }),
withMergeAdmission: vi.fn(async (_origin, work) => {
events.push('admission');
return { admitted: true, result: await work() };
}),
});
- return { deps, live, events, head, base };
+ return { deps, live, events, head, base, evidence, trees };
};
// These publisher-boundary regressions uniquely cover exact input validation
@@ -951,7 +960,7 @@ it.each([false, true])('validates the snapshot before admission and pins the hea
expect(deps.execGh).toHaveBeenCalledWith(expect.any(Array), undefined,
{ cwd: app.repoPath, env: { EXAMPLE_FORGE_ACCOUNT: 'selected' } });
expect(deps.queuePendingMerge).not.toHaveBeenCalled();
- expect(deps.rm).toHaveBeenCalledWith(worktreePath, { recursive: true, force: true });
+ expect(deps.retireSnapshotMergeCheckouts).toHaveBeenCalledTimes(1);
expect(deps.addWorktree).toHaveBeenCalledTimes(1);
expect(deps.addWorktree.mock.calls[0][0]).toEqual(
['worktree', 'add', '--detach', worktreePath, reused ? head : 'origin/main']);
@@ -965,7 +974,7 @@ it.each([false, true])('retains failed guard evidence without admission or a que
deps.execSnapshotMergeGuards.mockRejectedValue(Object.assign(new Error('guards failed'), { stdout: 'hostControlRoutes.test.js failed' }));
expect(await publishAppQualitySnapshot(app, deps)).toMatchObject({
published: true, merged: false, queuedMerge: false, deferredMerge: true,
- mergeGuard: { headSha: head, baseSha: base, worktreePath, reason: 'guards failed' },
+ mergeGuard: { headSha: head, baseSha: base, worktreePath, reason: 'snapshot-merge-guards-failed' },
});
expect(deps.withMergeAdmission).not.toHaveBeenCalled();
expect(deps.git.mergePR).not.toHaveBeenCalled();
@@ -1011,7 +1020,23 @@ it.each(['admission', 'merge'])('keeps guarded %s refusals retryable without que
expect(deps.rm).not.toHaveBeenCalled();
deps.withMergeAdmission.mockImplementation(async (_origin, work) => ({ admitted: true, result: await work() }));
deps.git.mergePR.mockResolvedValue({ success: true });
+ __resetQualitySnapshotPublishState();
expect(await publishAppQualitySnapshot(app, deps)).toMatchObject({ merged: true });
+ expect(deps.addWorktree).toHaveBeenCalledTimes(1);
+});
+
+it('preserves a retained checkout that acquired a lock before retry', async () => {
+ const { deps, trees } = guardedPublication(true);
+ deps.execSnapshotMergeGuards.mockRejectedValue(new Error('guard refused'));
+ expect(await publishAppQualitySnapshot(app, deps)).toMatchObject({ deferredMerge: true });
+ trees[0].locked = true;
+ __resetQualitySnapshotPublishState();
+ expect(await publishAppQualitySnapshot(app, deps)).toMatchObject({
+ merged: false, deferredMerge: true, mergeGuard: { reason: 'retained-snapshot-checkout-changed' },
+ });
+ expect(deps.addWorktree).toHaveBeenCalledTimes(1);
+ expect(deps.retireSnapshotMergeCheckouts).not.toHaveBeenCalled();
+ expect(deps.git.mergePR).not.toHaveBeenCalled();
});
it('leaves a guard-capable GitLab MR open instead of falling through to an unpinned merge', async () => {
diff --git a/server/services/appQualitySnapshotMerge.js b/server/services/appQualitySnapshotMerge.js
index 9290c9de53..bcae270225 100644
--- a/server/services/appQualitySnapshotMerge.js
+++ b/server/services/appQualitySnapshotMerge.js
@@ -1,4 +1,5 @@
-import { join } from 'node:path';
+import { basename, join } from 'node:path';
+import { readFile } from 'node:fs/promises';
import { atomicWrite, execFileAsync } from '../lib/fileCore.js';
const GUARD_COMMAND = 'node scripts/pregate.js --guards-only';
@@ -90,6 +91,52 @@ export async function validateSnapshotMerge(app, pr, evidence, git, deps) {
{ status: 'passed', stdout: output.stdout, stderr: output.stderr });
}
+/** Git's durable checkout inventory plus our evidence survives publisher restarts. */
+export async function retainedSnapshotMergeCheckouts(app, pr, deps) {
+ const list = deps.listSnapshotMergeWorktrees || (await import('./worktreeManager.js')).listWorktrees;
+ const read = deps.readMergeEvidence || readFile;
+ const retained = [];
+ for (const tree of await list(app.repoPath)) {
+ if (!basename(tree.path).startsWith('portos-quality-')) continue;
+ const body = await read(join(tree.path, '.cache', 'quality-snapshot-merge.json'), 'utf8')
+ .catch(error => { if (error.code === 'ENOENT') return null; throw error; });
+ if (body === null) continue;
+ const record = JSON.parse(body);
+ if (record.prUrl !== pr.url) continue;
+ if (record.checkoutHead !== tree.head || !tree.detached || tree.branch || tree.locked) {
+ throw new Error('retained-snapshot-checkout-changed');
+ }
+ retained.push(tree);
+ }
+ return retained;
+}
+
+/** Archive all attempts, including superseded heads, before verified retirement. */
+export async function retireSnapshotMergeCheckouts(app, pr, deps) {
+ const { preserveMergedCheckout } = await import('./heldCheckoutRecovery.js');
+ const { forceRemoveWorktreeDir } = await import('./worktreeManager.js');
+ const { retireWithUnlockedArchive } = await import('./cosClaimOwnershipState.js');
+ const checkOwner = deps.checkSnapshotPublicationOwnership
+ || (await import('./cosClaimOwnership.js')).checkSnapshotPublicationOwnership;
+ const held = [];
+ for (const tree of await retainedSnapshotMergeCheckouts(app, pr, deps)) {
+ const ownership = async () => (await checkOwner({ appId: app.id, branch: 'portos/quality-snapshot', prUrl: pr.url }))?.admitted === true;
+ const result = await retireWithUnlockedArchive({
+ busy: () => ({ removed: false }),
+ admit: async () => await ownership()
+ ? { fence: { repoPath: app.repoPath, path: tree.path, branch: null } }
+ : { result: { removed: false } },
+ archive: async () => ({ archive: await preserveMergedCheckout(app.repoPath, tree, null,
+ deps.snapshotArchiveCosDir ? { cosDir: deps.snapshotArchiveCosDir } : {}) }),
+ finish: async (_admitted, archive) => forceRemoveWorktreeDir(app.repoPath, tree.path, {
+ verifyRemoval: async holder => await ownership() ? archive.verify(holder) : 'snapshot-owner-changed',
+ }),
+ }).catch(() => ({ removed: false }));
+ if (!result.removed) held.push(tree.path);
+ }
+ return held;
+}
+
/** Called only inside admission. A moving base needs new guards, not a rebase. */
export async function verifySnapshotMergeInputs(app, pr, evidence, git, deps) {
const state = await readPr(app, pr, git, deps);