Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS
JavaScript ai-agentsdashboarddeveloper-toolsdigital-twinquantified-selfsecond-brainself-hostedtailscale
fix: recover and archive deferred snapshot validation checkouts (#13269)
Adam Eivyc0cea3ca1fd2
diff --git a/docs/GITHUB_ACTIONS.md b/docs/GITHUB_ACTIONS.mdindex 3d62665c6d..2d13b42cf1 100644--- a/docs/GITHUB_ACTIONS.md+++ b/docs/GITHUB_ACTIONS.md@@ -195,7 +195,12 @@ the GitHub merge to that head. New and reused snapshot PRs use detached checkoutFailed guards, moved/unreadable inputs, or refused merges retain the PR, checkout,and `.cache/quality-snapshot-merge.json` evidence for a later publish cycle;they do not enter the generic pending-merge queue. Guard-capable GitLab MRs stay-open until a supported pinned validation path is available. Repositories without+open until a supported pinned validation path is available. Retained checkouts+are located through Git's durable worktree inventory and reused for the same head+on retry. After a verified merge, every retained attempt for that PR is archived+and verified through checkout preservation before retirement; changed ownership,+locks, or archive failures retain the checkout and report the remaining paths.+Repositories withoutthis capability retain their existing snapshot merge behavior; arbitrary scriptsare never executed as a substitute.diff --git a/server/services/appQualitySnapshotFile.js b/server/services/appQualitySnapshotFile.jsindex acbdc4cbea..9eafd2149a 100644--- a/server/services/appQualitySnapshotFile.js+++ b/server/services/appQualitySnapshotFile.js@@ -19,9 +19,8 @@* generic repositories retain their existing merge-on-green fallback.* The immediate merge is the one step that advances the default branch, so it* runs under this install's repository merge admission (`cosMergeAdmission.js`)- * like every other final merge: while a claim holds the lease the PR is left- * open and handed to the local pending-merge queue, which takes admission at- * its own later attempt. Reads never write. A v1 file or historical+ * like every other final merge. A busy lease leaves the PR recoverable for+ * a later admitted attempt. Reads never write. A v1 file or historical* `quality-snapshot.json` is explicit legacy input. Before publication, live* review ownership and non-snapshot branch changes defer replacement; the push* lease pins only the exact head this invocation inspected.@@ -461,7 +460,7 @@ async function landSnapshotPr(app, { body, count, defaultBranch, git, deps, mess// If validation is interrupted, preserve the published checkout by default.retainWorktree = true;const queued = await mergeSnapshotPr(app, pr, deps, git, defaultBranch, { worktreePath, expectedHeadSha: hash });- retainWorktree = queued.deferred === true && queued.guard?.required === true;+ retainWorktree = queued.guardManaged === true;console.log(`📊 Published quality snapshot for app ${app.id}: ${count} measurements → ${hash} (${pr.url})`);return {published: true, hash, path: APP_QUALITY_SNAPSHOT_FILENAME,@@ -559,6 +558,7 @@ const mergeOutcomeFields = outcome => ({queuedMerge: outcome.queued === true,...(outcome.deferred === true ? { deferredMerge: true } : {}),...(outcome.guard ? { mergeGuard: outcome.guard } : {}),+ ...(outcome.retainedCheckouts?.length ? { retainedMergeCheckouts: outcome.retainedCheckouts } : {}),});/**@@ -568,10 +568,8 @@ const mergeOutcomeFields = outcome => ({** The immediate attempt needs this install's repository merge admission. When* another owner holds it (or ownership cannot be verified) the merge is- * deferred: GitHub hands the PR to the local pending-merge queue, which takes- * admission at its own later attempt; GitLab's only queue is the forge's- * auto-merge, which cannot honor a local lease, so the MR stays open for the- * next publish cycle instead.+ * deferred. Only generic GitHub repositories use the local pending-merge+ * queue. Guard-capable PRs and GitLab admission refusals stay open for retry.** @returns {{merged: boolean, queued: boolean, deferred?: boolean}} — merged* and queued are never both true.@@ -595,24 +593,33 @@ async function mergeSnapshotPr(app, pr, deps, git, defaultBranch, checkout = {})}async function guardedSnapshotMerge(app, pr, deps, git, guards, evidence, existingPath) {- const deferred = () => ({ merged: false, queued: false, deferred: true, guard: evidence });+ const deferred = () => ({ merged: false, queued: false, deferred: true, guard: evidence, guardManaged: true });const run = async path => {evidence.worktreePath = path;try {+ const current = await git.execGit(['rev-parse', '--verify', 'HEAD^{commit}'], path);+ if (current.exitCode !== 0 || !/^[a-f0-9]{40}$/i.test(current.stdout?.trim() || '')) {+ throw new Error('snapshot-checkout-head-unreadable');+ }+ evidence.checkoutHead = current.stdout.trim();+ await guards.recordSnapshotMergeEvidence(pr, evidence, deps, { status: 'preparing' });if (evidence.reason) throw new Error(evidence.reason);- if (!existingPath) await addSnapshotWorktree(app, deps, path, evidence.headSha);await guards.validateSnapshotMerge(app, pr, evidence, git, deps);const attempt = await withSnapshotMergeAdmission(app, deps, async () => {await guards.verifySnapshotMergeInputs(app, pr, evidence, git, deps);return tryImmediateMerge(app, pr, deps, git, evidence.headSha);});- if (attempt.result) return { merged: true, queued: false };+ if (attempt.result) {+ const retire = deps.retireSnapshotMergeCheckouts || guards.retireSnapshotMergeCheckouts;+ const retainedCheckouts = await retire(app, pr, deps).catch(() => [path]);+ return { merged: true, queued: false, guardManaged: true, retainedCheckouts };+ }evidence.reason = attempt.reason || 'snapshot-merge-deferred';} catch (error) {- evidence.reason = error.message;+ evidence.reason = error.stdout !== undefined || error.stderr !== undefined ? 'snapshot-merge-guards-failed' : error.message;console.log(`⏳ Quality snapshot merge retained for app ${app.id}: ${evidence.reason}`);await guards.recordSnapshotMergeEvidence(pr, evidence, deps, {- status: 'deferred', stdout: error.stdout, stderr: error.stderr,+ status: 'deferred', error: error.message, stdout: error.stdout, stderr: error.stderr,});return deferred();}@@ -623,10 +630,14 @@ async function guardedSnapshotMerge(app, pr, deps, git, guards, evidence, existiif (evidence.reason) return deferred();const { withCheckoutPublication } = await import('../lib/checkoutRetirementAdmission.js');return withCheckoutPublication(app.repoPath, async () => {- const path = await (deps.mkdtemp || mkdtemp)(join(tmpdir(), 'portos-quality-'));- const result = await run(path);- if (result.merged) await cleanupSnapshotWorktree(app, git, path, deps.rm || rm);- return result;+ const retained = await guards.retainedSnapshotMergeCheckouts(app, pr, deps);+ const matching = retained.find(tree => tree.head === evidence.headSha);+ const path = matching?.path || await (deps.mkdtemp || mkdtemp)(join(tmpdir(), 'portos-quality-'));+ if (!matching) await addSnapshotWorktree(app, deps, path, evidence.headSha);+ return run(path);+ }).catch(error => {+ evidence.reason = error.message;+ return deferred();});}diff --git a/server/services/appQualitySnapshotFile.test.js b/server/services/appQualitySnapshotFile.test.jsindex e4802bc432..a1ca11da83 100644--- a/server/services/appQualitySnapshotFile.test.js+++ b/server/services/appQualitySnapshotFile.test.js@@ -896,6 +896,8 @@ const guardedPublication = (reused = false) => {const base = 'b'.repeat(40);const live = { head, base, state: 'OPEN', manifest: { scripts: { 'pregate:merged': 'node scripts/pregate.js --guards-only' } } };const events = [];+ const trees = [];+ const evidence = new Map();const deps = testDeps({git: gitDouble({execGit: vi.fn(async args => {@@ -926,15 +928,22 @@ const guardedPublication = (reused = false) => {state: live.state, isDraft: false, isCrossRepository: false, headRefOid: live.head,headRefName: QUALITY_SNAPSHOT_BRANCH, baseRefName: 'main',})),+ addWorktree: vi.fn(async () => { trees.push({ path: worktreePath, head, detached: true }); }),linkWorktreeDependencies: vi.fn(async () => {}),- writeMergeEvidence: vi.fn(async () => {}),+ writeMergeEvidence: vi.fn(async (path, record) => { evidence.set(path, structuredClone(record)); }),+ readMergeEvidence: vi.fn(async path => {+ if (!evidence.has(path)) throw Object.assign(new Error('absent'), { code: 'ENOENT' });+ return JSON.stringify(evidence.get(path));+ }),+ listSnapshotMergeWorktrees: vi.fn(async () => [...trees]),+ retireSnapshotMergeCheckouts: vi.fn(async () => { trees.length = 0; evidence.clear(); return []; }),execSnapshotMergeGuards: vi.fn(async () => { events.push('guards'); return { stdout: 'guards passed', stderr: '' }; }),withMergeAdmission: vi.fn(async (_origin, work) => {events.push('admission');return { admitted: true, result: await work() };}),});- return { deps, live, events, head, base };+ return { deps, live, events, head, base, evidence, trees };};// These publisher-boundary regressions uniquely cover exact input validation@@ -951,7 +960,7 @@ it.each([false, true])('validates the snapshot before admission and pins the heaexpect(deps.execGh).toHaveBeenCalledWith(expect.any(Array), undefined,{ cwd: app.repoPath, env: { EXAMPLE_FORGE_ACCOUNT: 'selected' } });expect(deps.queuePendingMerge).not.toHaveBeenCalled();- expect(deps.rm).toHaveBeenCalledWith(worktreePath, { recursive: true, force: true });+ expect(deps.retireSnapshotMergeCheckouts).toHaveBeenCalledTimes(1);expect(deps.addWorktree).toHaveBeenCalledTimes(1);expect(deps.addWorktree.mock.calls[0][0]).toEqual(['worktree', 'add', '--detach', worktreePath, reused ? head : 'origin/main']);@@ -965,7 +974,7 @@ it.each([false, true])('retains failed guard evidence without admission or a quedeps.execSnapshotMergeGuards.mockRejectedValue(Object.assign(new Error('guards failed'), { stdout: 'hostControlRoutes.test.js failed' }));expect(await publishAppQualitySnapshot(app, deps)).toMatchObject({published: true, merged: false, queuedMerge: false, deferredMerge: true,- mergeGuard: { headSha: head, baseSha: base, worktreePath, reason: 'guards failed' },+ mergeGuard: { headSha: head, baseSha: base, worktreePath, reason: 'snapshot-merge-guards-failed' },});expect(deps.withMergeAdmission).not.toHaveBeenCalled();expect(deps.git.mergePR).not.toHaveBeenCalled();@@ -1011,7 +1020,23 @@ it.each(['admission', 'merge'])('keeps guarded %s refusals retryable without queexpect(deps.rm).not.toHaveBeenCalled();deps.withMergeAdmission.mockImplementation(async (_origin, work) => ({ admitted: true, result: await work() }));deps.git.mergePR.mockResolvedValue({ success: true });+ __resetQualitySnapshotPublishState();expect(await publishAppQualitySnapshot(app, deps)).toMatchObject({ merged: true });+ expect(deps.addWorktree).toHaveBeenCalledTimes(1);+});++it('preserves a retained checkout that acquired a lock before retry', async () => {+ const { deps, trees } = guardedPublication(true);+ deps.execSnapshotMergeGuards.mockRejectedValue(new Error('guard refused'));+ expect(await publishAppQualitySnapshot(app, deps)).toMatchObject({ deferredMerge: true });+ trees[0].locked = true;+ __resetQualitySnapshotPublishState();+ expect(await publishAppQualitySnapshot(app, deps)).toMatchObject({+ merged: false, deferredMerge: true, mergeGuard: { reason: 'retained-snapshot-checkout-changed' },+ });+ expect(deps.addWorktree).toHaveBeenCalledTimes(1);+ expect(deps.retireSnapshotMergeCheckouts).not.toHaveBeenCalled();+ expect(deps.git.mergePR).not.toHaveBeenCalled();});it('leaves a guard-capable GitLab MR open instead of falling through to an unpinned merge', async () => {diff --git a/server/services/appQualitySnapshotMerge.js b/server/services/appQualitySnapshotMerge.jsindex 9290c9de53..bcae270225 100644--- a/server/services/appQualitySnapshotMerge.js+++ b/server/services/appQualitySnapshotMerge.js@@ -1,4 +1,5 @@-import { join } from 'node:path';+import { basename, join } from 'node:path';+import { readFile } from 'node:fs/promises';import { atomicWrite, execFileAsync } from '../lib/fileCore.js';const GUARD_COMMAND = 'node scripts/pregate.js --guards-only';@@ -90,6 +91,52 @@ export async function validateSnapshotMerge(app, pr, evidence, git, deps) {{ status: 'passed', stdout: output.stdout, stderr: output.stderr });}+/** Git's durable checkout inventory plus our evidence survives publisher restarts. */+export async function retainedSnapshotMergeCheckouts(app, pr, deps) {+ const list = deps.listSnapshotMergeWorktrees || (await import('./worktreeManager.js')).listWorktrees;+ const read = deps.readMergeEvidence || readFile;+ const retained = [];+ for (const tree of await list(app.repoPath)) {+ if (!basename(tree.path).startsWith('portos-quality-')) continue;+ const body = await read(join(tree.path, '.cache', 'quality-snapshot-merge.json'), 'utf8')+ .catch(error => { if (error.code === 'ENOENT') return null; throw error; });+ if (body === null) continue;+ const record = JSON.parse(body);+ if (record.prUrl !== pr.url) continue;+ if (record.checkoutHead !== tree.head || !tree.detached || tree.branch || tree.locked) {+ throw new Error('retained-snapshot-checkout-changed');+ }+ retained.push(tree);+ }+ return retained;+}++/** Archive all attempts, including superseded heads, before verified retirement. */+export async function retireSnapshotMergeCheckouts(app, pr, deps) {+ const { preserveMergedCheckout } = await import('./heldCheckoutRecovery.js');+ const { forceRemoveWorktreeDir } = await import('./worktreeManager.js');+ const { retireWithUnlockedArchive } = await import('./cosClaimOwnershipState.js');+ const checkOwner = deps.checkSnapshotPublicationOwnership+ || (await import('./cosClaimOwnership.js')).checkSnapshotPublicationOwnership;+ const held = [];+ for (const tree of await retainedSnapshotMergeCheckouts(app, pr, deps)) {+ const ownership = async () => (await checkOwner({ appId: app.id, branch: 'portos/quality-snapshot', prUrl: pr.url }))?.admitted === true;+ const result = await retireWithUnlockedArchive({+ busy: () => ({ removed: false }),+ admit: async () => await ownership()+ ? { fence: { repoPath: app.repoPath, path: tree.path, branch: null } }+ : { result: { removed: false } },+ archive: async () => ({ archive: await preserveMergedCheckout(app.repoPath, tree, null,+ deps.snapshotArchiveCosDir ? { cosDir: deps.snapshotArchiveCosDir } : {}) }),+ finish: async (_admitted, archive) => forceRemoveWorktreeDir(app.repoPath, tree.path, {+ verifyRemoval: async holder => await ownership() ? archive.verify(holder) : 'snapshot-owner-changed',+ }),+ }).catch(() => ({ removed: false }));+ if (!result.removed) held.push(tree.path);+ }+ return held;+}+/** Called only inside admission. A moving base needs new guards, not a rebase. */export async function verifySnapshotMergeInputs(app, pr, evidence, git, deps) {const state = await readPr(app, pr, git, deps);