Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS
JavaScript ai-agentsdashboarddeveloper-toolsdigital-twinquantified-selfsecond-brainself-hostedtailscale
fix: retire checkouts with hashed review metadata (#13253)
Adam Eivy636e1b366d82
diff --git a/server/services/heldCheckoutRecovery.js b/server/services/heldCheckoutRecovery.jsindex 6c1753a70a..c77ece3983 100644--- a/server/services/heldCheckoutRecovery.js+++ b/server/services/heldCheckoutRecovery.js@@ -506,13 +506,78 @@ export async function preserveMergedCheckout(repoPath, tree, defaultBranch, { co};}+// All administration bytes are preserved separately. Only Git's reference+// formats establish object dependencies; messages and application JSON do not.+function recoveryReferenceKind(path) {+ const relative = path.replace(/^gitdir\//, '');+ if (/^(rebase-merge|rebase-apply|sequencer|reftable)(\/|$)/.test(relative)+ || /\.lock$/.test(relative) || (/^BISECT_/.test(relative) && relative !== 'BISECT_HEAD')) return 'unsafe';+ if (/^logs(?:\/|$)/.test(relative)) return 'reflog';+ if (/^refs(?:\/|$)/.test(relative)) return 'ref';+ if (relative === 'packed-refs') return 'packed';+ if (relative === 'FETCH_HEAD') return 'fetch';+ if (relative === 'MERGE_HEAD' || relative === 'shallow') return 'objects';+ if (['COMMIT_EDITMSG', 'MERGE_MSG', 'MERGE_MODE', 'MERGE_RR', 'SQUASH_MSG', 'TAG_EDITMSG', 'NOTES_EDITMSG'].includes(relative)) return null;+ // Worktree-private root refs use uppercase names. Unknown operation formats+ // must hold rather than silently omit recovery objects introduced by Git.+ return /^[A-Z][A-Z0-9_]*$/.test(relative) ? 'ref' : null;+}++async function recoveryObjectHeads(administration, { repoPath, tree, missingRoot }) {+ const heads = new Set([tree.head]);+ const oid = new RegExp(`^[a-f0-9]{${tree.head.length}}$`, 'i');+ const invalid = () => { throw new Error('unrecognized private recovery reference'); };+ const add = (value, allowZero = false) => {+ if (!oid.test(value)) invalid();+ if (/^0+$/.test(value)) { if (allowZero) return; invalid(); }+ heads.add(value.toLowerCase());+ if (heads.size > 1024) throw new Error('too many private recovery objects');+ };+ for (const link of administration.symlinks) {+ if (recoveryReferenceKind(link.path)) throw new Error('unsupported linked recovery reference');+ }+ for (const file of administration.files) {+ const kind = recoveryReferenceKind(file.path);+ if (!kind) continue;+ if (kind === 'unsafe') throw new Error('retirement-operation-active');+ const contents = await readFile(file.source, 'utf8');+ const lines = contents.split(/\r?\n/);+ if (lines.at(-1) === '') lines.pop();+ if (kind === 'ref') {+ if (lines.length !== 1) invalid();+ const symbolic = /^ref: (\S+)$/.exec(lines[0]);+ if (symbolic) {+ const prefix = missingRoot ? ['--git-dir', missingRoot, '--work-tree', tree.path] : [];+ await git([...prefix, 'check-ref-format', '--allow-onelevel', symbolic[1]], missingRoot ? repoPath : tree.path);+ add((await git([...prefix, 'rev-parse', '--verify', '--end-of-options', symbolic[1]], missingRoot ? repoPath : tree.path)).trim());+ } else add(lines[0]);+ continue;+ }+ if (kind === 'objects' && !lines.length) invalid();+ for (const line of lines) {+ if (kind === 'reflog') {+ const fields = /^([a-f0-9]+) ([a-f0-9]+) [^\t]+(?:\t.*)?$/i.exec(line);+ if (!fields) invalid();+ add(fields[1], true);+ add(fields[2], true);+ } else if (kind === 'fetch') {+ const fields = /^([a-f0-9]+)\t(?:not-for-merge)?\t.*$/i.exec(line);+ if (!fields) invalid();+ add(fields[1]);+ } else if (kind === 'packed') {+ if (line.startsWith('#')) continue;+ const fields = /^(?:([a-f0-9]+) refs\/\S+|\^([a-f0-9]+))$/i.exec(line);+ if (!fields) invalid();+ add(fields[1] || fields[2]);+ } else add(line);+ }+ }+ return heads;+}+async function writeArchive(dir, { repoPath, tree, branch, defaultBranch, release, review, missingRoot = null }) {const [snapshot, administration] = await archiveSnapshots(tree, repoPath, missingRoot);- // Sequencer state can contain abbreviated or external object references.- // Keep that checkout for recovery instead of claiming a complete bundle.- if (administration.files.some(file => /^gitdir\/(rebase-merge|rebase-apply|sequencer)\//.test(file.path))) {- throw new Error('retirement-operation-active');- }+ const recoveryHeads = await recoveryObjectHeads(administration, { repoPath, tree, missingRoot });await ensureDir(dir);await chmod(dir, 0o700);@@ -523,15 +588,6 @@ async function writeArchive(dir, { repoPath, tree, branch, defaultBranch, releas// names a tree containing conflict-resolution evidence, not a commit.// Named bundle roots preserve their reachable objects too. Unknown/missing// recovery objects still hold the checkout.- const recoveryHeads = new Set([tree.head]);- for (const file of administration.files) {- if (file.path === 'git-indirection' || /\/(index|gitdir|commondir|locked)$/.test(file.path)) continue;- const contents = await readFile(file.source, 'utf8');- for (const match of contents.matchAll(/(?<![a-f0-9])[a-f0-9]{40}(?:[a-f0-9]{24})?(?![a-f0-9])/gi)) {- if (!/^0+$/.test(match[0])) recoveryHeads.add(match[0].toLowerCase());- }- }- if (recoveryHeads.size > 1024) throw new Error('too many private recovery objects');const recoveryRefs = [];const namespace = `refs/portos/archive/${randomUUID()}`;try {diff --git a/server/services/heldCheckoutRecovery.test.js b/server/services/heldCheckoutRecovery.test.jsindex 8b077a7344..1a974ea282 100644--- a/server/services/heldCheckoutRecovery.test.js+++ b/server/services/heldCheckoutRecovery.test.js@@ -259,10 +259,15 @@ describe.skipIf(SKIP_HEAVY_INTEGRATION)('held checkout recovery (real git)', ()await writeFile(join(admin, 'AUTO_MERGE'), `${autoMerge}\n`);await execGit(['update-ref', 'refs/worktree/private-blob', blob], path);await execGit(['update-ref', 'refs/worktree/private-tag', tag], path);+ await execGit(['symbolic-ref', 'refs/worktree/symbolic-tag', 'refs/worktree/private-tag'], path);+ await execGit(['symbolic-ref', 'refs/worktree/current', 'HEAD'], path);+ const mergeHead = (await execGit(['commit-tree', autoMerge, '-m', 'private merge recovery'], path)).stdout.trim();+ const otherMergeHead = (await execGit(['commit-tree', autoMerge, '-m', 'other private merge recovery'], path)).stdout.trim();+ await writeFile(join(admin, 'MERGE_HEAD'), `${mergeHead}\n${otherMergeHead}\n`);const tree = { path, branch: 'refs/heads/feature/private-objects', head: (await execGit(['rev-parse', 'HEAD'], path)).stdout.trim() };const saved = await preserveMergedCheckout(repo, tree, 'main', { cosDir });const manifest = JSON.parse(await readFile(join(saved.archiveDir, 'manifest.json'), 'utf8'));- expect(manifest.captured.recoveryHeads).toEqual(expect.arrayContaining([autoMerge, blob, tag]));+ expect(manifest.captured.recoveryHeads).toEqual(expect.arrayContaining([autoMerge, blob, tag, mergeHead, otherMergeHead]));expect(await readFile(join(saved.archiveDir, 'administration', 'gitdir', 'AUTO_MERGE'), 'utf8')).toBe(`${autoMerge}\n`);expect(await saved.verify(tree)).toBe(null);// The archive must supply the conflict tree's children, not just list roots@@ -275,23 +280,79 @@ describe.skipIf(SKIP_HEAVY_INTEGRATION)('held checkout recovery (real git)', ()expect((await execGit(['cat-file', 'blob', conflictBlob], restored)).stdout).toContain('>>>>>>> theirs');expect((await execGit(['cat-file', 'blob', blob], restored)).stdout).toBe('private root blob bytes');expect((await execGit(['rev-parse', `${tag}^{tree}`], restored)).stdout.trim()).toBe(autoMerge);+ expect((await execGit(['rev-parse', `${mergeHead}^{tree}`], restored)).stdout.trim()).toBe(autoMerge);+ expect((await execGit(['rev-parse', `${otherMergeHead}^{tree}`], restored)).stdout.trim()).toBe(autoMerge);await execGit(['fsck', '--full', '--strict'], restored);});- it('retains the checkout when an AUTO_MERGE recovery tree is missing', async () => {+ it.each(['AUTO_MERGE', 'refs/worktree/missing', 'logs/HEAD:old', 'logs/HEAD:new', 'FETCH_HEAD'])(+ 'retains the checkout when a genuine recovery object is missing from %s', async reference => {const path = join(external, 'missing-auto-merge');await execGit(['worktree', 'add', '-b', 'feature/missing-auto-merge', path, 'main'], repo);const admin = (await execGit(['rev-parse', '--absolute-git-dir'], path)).stdout.trim();const missing = 'f'.repeat(40);- await writeFile(join(admin, 'AUTO_MERGE'), `${missing}\n`);const tree = { path, branch: 'refs/heads/feature/missing-auto-merge', head: (await execGit(['rev-parse', 'HEAD'], path)).stdout.trim() };+ const relative = reference.split(':')[0];+ const bytes = reference === 'logs/HEAD:old' ? `${missing} ${tree.head} Test <[email protected]> 1 +0000\tmissing old object\n`+ : reference === 'logs/HEAD:new' ? `${tree.head} ${missing} Test <[email protected]> 1 +0000\tmissing new object\n`+ : reference === 'FETCH_HEAD' ? `${missing}\t\tbranch 'missing' of https://example.invalid/repo\n` : `${missing}\n`;+ await mkdir(dirname(join(admin, relative)), { recursive: true });+ await writeFile(join(admin, relative), bytes);await expect(preserveMergedCheckout(repo, tree, 'main', { cosDir })).rejects.toThrow();expect(existsSync(path)).toBe(true);- expect(await readFile(join(admin, 'AUTO_MERGE'), 'utf8')).toBe(`${missing}\n`);+ expect(await readFile(join(admin, relative), 'utf8')).toBe(bytes);expect(await archives()).toEqual([]);expect((await execGit(['for-each-ref', '--format=%(refname)', 'refs/portos/archive'], repo)).stdout).toBe('');});+ it.each(['refs/worktree/unknown', 'logs/HEAD', 'FETCH_HEAD', 'FUTURE_HEAD', 'sequencer/todo', 'index.lock'])(+ 'retains unknown or active recovery state in %s', async relative => {+ const path = join(external, 'unknown-recovery');+ await execGit(['worktree', 'add', '-b', 'feature/unknown-recovery', path, 'main'], repo);+ const admin = (await execGit(['rev-parse', '--absolute-git-dir'], path)).stdout.trim();+ const tree = { path, branch: 'refs/heads/feature/unknown-recovery', head: (await execGit(['rev-parse', 'HEAD'], path)).stdout.trim() };+ await mkdir(dirname(join(admin, relative)), { recursive: true });+ const bytes = relative.startsWith('refs/') ? 'ref: HEAD^{tree}\n' : `unrecognized recovery format ${tree.head}\n`;+ await writeFile(join(admin, relative), bytes);+ await expect(preserveMergedCheckout(repo, tree, 'main', { cosDir })).rejects.toThrow();+ expect(await registered(path)).toBe(true);+ expect(await readFile(join(admin, relative), 'utf8')).toBe(bytes);+ expect(await archives()).toEqual([]);+ });++ it('preserves SHA-256 private refs and packed roots without interpreting metadata as objects', async () => {+ repo = join(scratch, 'sha256-repo');+ await execGit(['init', '--object-format=sha256', '-b', 'main', repo], scratch);+ await execGit(['config', 'user.name', 'Test'], repo);+ await execGit(['config', 'user.email', '[email protected]'], repo);+ await execGit(['config', 'commit.gpgsign', 'false'], repo);+ await writeFile(join(repo, 'initial.txt'), 'sha256 initial');+ await execGit(['add', 'initial.txt'], repo);+ await execGit(['commit', '-m', 'sha256 initial'], repo);+ const path = join(external, 'sha256');+ await execGit(['worktree', 'add', '-b', 'feature/sha256', path, 'main'], repo);+ const admin = (await execGit(['rev-parse', '--absolute-git-dir'], path)).stdout.trim();+ const tree = { path, branch: 'refs/heads/feature/sha256', head: (await execGit(['rev-parse', 'HEAD'], path)).stdout.trim() };+ await writeFile(join(path, 'private.txt'), 'private sha256 blob');+ const blob = (await execGit(['hash-object', '-w', 'private.txt'], path)).stdout.trim();+ await rm(join(path, 'private.txt'));+ await execGit(['update-ref', 'refs/worktree/private', blob], path);+ await execGit(['tag', '-a', 'temporary', blob, '-m', 'packed sha256 tag'], path);+ const tag = (await execGit(['rev-parse', 'refs/tags/temporary'], path)).stdout.trim();+ await execGit(['tag', '-d', 'temporary'], path);+ await writeFile(join(admin, 'packed-refs'), `# pack-refs with: peeled\n${tag} refs/worktree/packed\n^${blob}\n`);+ await writeFile(join(admin, 'portos-pr-review-outcome.json'), JSON.stringify({ avatar: '1'.repeat(64) }));+ const saved = await preserveMergedCheckout(repo, tree, 'main', { cosDir });+ expect(await saved.verify(tree)).toBe(null);+ await rename(repo, `${repo}-unavailable`);+ const restored = join(scratch, 'sha256-restored');+ await execGit(['init', '--bare', '--object-format=sha256', restored], scratch);+ await execGit(['fetch', join(saved.archiveDir, 'commits.bundle'), '+refs/portos/archive/*:refs/recovered/*'], restored);+ expect((await execGit(['cat-file', 'blob', blob], restored)).stdout).toBe('private sha256 blob');+ expect((await execGit(['rev-parse', `${tag}^{blob}`], restored)).stdout.trim()).toBe(blob);+ await execGit(['fsck', '--full', '--strict'], restored);+ });+it.each([false, true])('archives a missing checkout private index and staged contents before pruning, parentAlias=%s', async parentAlias => {const path = join(external, 'missing');await execGit(['worktree', 'add', '-b', 'feature/missing', path, 'main'], repo);@@ -840,6 +901,39 @@ describe.skipIf(SKIP_HEAVY_INTEGRATION)('held checkout recovery (real git)', ()return current;}+ it('retires reviewed metadata with unrelated hashes while recovering only structural reflog and fetch objects', async () => {+ const c = await reviewedCheckout('metadata-hashes');+ const absent = '1'.repeat(40);+ expect((await execGit(['cat-file', '-e', absent], repo, { ignoreExitCode: true })).exitCode).not.toBe(0);+ const metadata = Buffer.from(`${JSON.stringify({ reviews: [{ user: { avatar_url: `https://example.invalid/avatar/${absent}` } }], note: '2'.repeat(64) }, null, 2)}\n`);+ await writeFile(join(c.admin, 'portos-pr-review-outcome.json'), metadata);+ await writeFile(join(c.admin, 'COMMIT_EDITMSG'), `Unrelated message hash ${absent}\n`);+ const tree = (await execGit(['rev-parse', 'HEAD^{tree}'], c.path)).stdout.trim();+ const old = (await execGit(['commit-tree', tree, '-m', 'only in the old reflog field'], c.path)).stdout.trim();+ const next = (await execGit(['commit-tree', tree, '-m', 'only in the new reflog field'], c.path)).stdout.trim();+ const fetched = (await execGit(['commit-tree', tree, '-m', 'only in FETCH_HEAD'], c.path)).stdout.trim();+ await writeFile(join(c.admin, 'logs/HEAD'), `${old} ${next} Test <[email protected]> 1 +0000\tmessage ${absent}\n`);+ await writeFile(join(c.admin, 'FETCH_HEAD'), `${fetched}\tnot-for-merge\tbranch '${absent}' of https://example.invalid/repo\n`);+ const prepared = await c.prepare();+ expect(prepared.ok).toBe(true);+ const result = await c.confirm(prepared.review);+ expect(result).toMatchObject({ ok: true, branchDeleted: true, archiveDir: expect.any(String) });+ expect(existsSync(c.path)).toBe(false);+ expect(await readFile(join(result.archiveDir, 'administration/gitdir/portos-pr-review-outcome.json'))).toEqual(metadata);+ expect((await getBranchReconcileOutcome('app-1')).retired).toContainEqual(expect.objectContaining({+ path: c.release.path, branch: c.release.branch, head: c.release.head, archiveDir: result.archiveDir, branchDeleted: true,+ }));+ const manifest = JSON.parse(await readFile(join(result.archiveDir, 'manifest.json'), 'utf8'));+ expect(manifest.captured.recoveryHeads).toEqual(expect.arrayContaining([old, next, fetched]));+ expect(manifest.captured.recoveryHeads).not.toContain(absent);+ await rename(repo, `${repo}-unavailable`);+ const restored = join(scratch, 'metadata-restored');+ await execGit(['init', '--bare', restored], scratch);+ await execGit(['fetch', join(result.archiveDir, 'commits.bundle'), '+refs/portos/archive/*:refs/recovered/*'], restored);+ for (const oid of [old, next, fetched]) expect((await execGit(['cat-file', '-t', oid], restored)).stdout.trim()).toBe('commit');+ await execGit(['fsck', '--full', '--strict'], restored);+ });+it('retires a freshly reviewed reused-branch incarnation with a standalone archive and exact receipt', async () => {const c = await reviewedCheckout('quality-reused', { gitlink: true });reusedBranchOwners(c);