Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS
JavaScript ai-agentsdashboarddeveloper-toolsdigital-twinquantified-selfsecond-brainself-hostedtailscale
fix: verify historical staged patches against archived indexes (#13429)
Adam Eivy4c0779142a93
diff --git a/docs/features/branch-reconciliation.md b/docs/features/branch-reconciliation.mdindex 97515fdba2..c9424f1039 100644--- a/docs/features/branch-reconciliation.md+++ b/docs/features/branch-reconciliation.md@@ -68,6 +68,21 @@ immediately before removal; if that recheck refuses, the original lock isrestored. A retired row leaves the inventory and appears under *Recentlyretired* with its archive path; only the next sweep can report convergence.+Historical held-checkout archives must also prove that their staged patch+reconstructs the preserved private index. Verification imports the bundle into+a fresh, isolated Git repository, starts from the recorded HEAD, applies+`index.diff` to a new index, and compares every path, mode, stage and object ID+with the archived index. Split-index support uses only validated archived index+files; archived Git configuration, pointers and object stores are never execution+targets. Matching patch hashes alone cannot establish this proof: an older writer+could have hashed bytes already damaged during decoding.++A missing, unsupported or mismatched index retains the historical preservation+obligation with `archive-index-reconstruction-failed`. Locate a complete archive+for that exact checkout generation or recover the original data from backup, then+retry reconciliation. Failure never fabricates a retirement receipt or authorizes+deletion of a checkout, branch or archive.+## Explicit initialized-checkout preservationAutomatic cleanup continues to retain initialized submodules. The separatediff --git a/server/services/heldCheckoutRecovery.js b/server/services/heldCheckoutRecovery.jsindex b1c96f2844..1d4dd71341 100644--- a/server/services/heldCheckoutRecovery.js+++ b/server/services/heldCheckoutRecovery.js@@ -71,8 +71,8 @@ async function archiveManifest(dir) {return { manifest: JSON.parse(bytes), digest: evidenceHash(bytes) };}-/** Reuses the archive's recorded byte hashes and administration fingerprint.- * Older dirty archives lack patch hashes and cannot prove completeness later.+/** Reuses recorded byte hashes and independently reconstructs the private index.+ * Older dirty archives without patch hashes cannot prove completeness later.*/async function verifyHeldArchive(repoPath, dir, release) {const { manifest: saved, digest } = await archiveManifest(dir);@@ -143,14 +143,33 @@ async function verifyHeldArchive(repoPath, dir, release) {// (\\.\nul) is not the Git configuration-file contract.const env = { ...Object.fromEntries(Object.entries(process.env).filter(([key]) => !key.toUpperCase().startsWith('GIT_'))),GIT_CONFIG_NOSYSTEM: '1', GIT_CONFIG_GLOBAL: '/dev/null', GIT_NO_REPLACE_OBJECTS: '1', GIT_NO_LAZY_FETCH: '1', GIT_TERMINAL_PROMPT: '0' };- const prove = args => execGit(['--no-optional-locks', '-c', 'core.hooksPath=/dev/null', '-c', 'core.fsmonitor=false', ...args], scratch,- { env, maxBuffer: GIT_OUTPUT_BYTES });+ const prove = (args, index = null) => execGit(['--no-optional-locks', '-c', 'core.hooksPath=/dev/null', '-c', 'core.fsmonitor=false', ...args], scratch,+ { env: { ...env, ...(index && { GIT_INDEX_FILE: join(scratch, index) }) }, maxBuffer: GIT_OUTPUT_BYTES });try {- await prove(['init', '--bare', `--object-format=${release.head.length === 64 ? 'sha256' : 'sha1'}`, '.']);- await prove(['bundle', 'unbundle', bundle]);- await prove(['fsck', '--full', '--strict', '--no-reflogs', '--no-dangling', ...captured.recoveryHeads]);- } catch { throw new Error('archive-bundle-object-proof-failed'); }- finally { await rmGuarded(scratch, { recursive: true, force: true }); }+ try {+ await prove(['init', '--bare', `--object-format=${release.head.length === 64 ? 'sha256' : 'sha1'}`, '.']);+ await prove(['bundle', 'unbundle', bundle]);+ await prove(['fsck', '--full', '--strict', '--no-reflogs', '--no-dangling', ...captured.recoveryHeads]);+ } catch { throw new Error('archive-bundle-object-proof-failed'); }+ try {+ // Copy only validated index bytes into our own Git directory. Never run+ // Git against archived config, commondir, gitdir pointers or object stores.+ if (!saved.administration.files.some(file => file.path === 'gitdir/index')) throw new Error('missing archived index');+ for (const file of saved.administration.files.filter(file => file.path === 'gitdir/index'+ || /^gitdir\/sharedindex\.[a-f0-9]+$/.test(file.path))) {+ await copyFileGuarded(await archiveFile(dir, join(dir, 'administration', file.path)),+ join(scratch, file.path === 'gitdir/index' ? 'archived-index' : basename(file.path)));+ }+ // Quoted paths make this comparison ASCII and lossless even for opaque+ // filename bytes. --stage includes modes, object IDs and conflict stages.+ const entries = ['-c', 'core.quotePath=true', 'ls-files', '--stage'];+ const expected = await prove(entries, 'archived-index');+ await prove(['read-tree', release.head]);+ if (captured.indexDiff) await prove(['apply', '--cached', '--binary', join(dir, 'index.diff')]);+ const restored = await prove(entries);+ if (restored.stdout !== expected.stdout) throw new Error('index entries differ');+ } catch { throw new Error('archive-index-reconstruction-failed'); }+ } finally { await rmGuarded(scratch, { recursive: true, force: true }); }if ((await archiveManifest(dir)).digest !== digest) throw new Error('archive-manifest-changed');const recheck = async () => {for (const [path, original] of stamps) {diff --git a/server/services/heldCheckoutRecovery.test.js b/server/services/heldCheckoutRecovery.test.jsindex 8cbecc83fc..0c5532ba58 100644--- a/server/services/heldCheckoutRecovery.test.js+++ b/server/services/heldCheckoutRecovery.test.js@@ -8,6 +8,8 @@ import { describe, it, expect, beforeEach, afterEach, afterAll, vi } from 'vitesimport { cp, mkdtemp, mkdir, rm, rename, writeFile, readFile, readdir, symlink, utimes, truncate } from 'fs/promises';import { existsSync, realpathSync } from 'fs';import { tmpdir } from 'os';+import { promisify } from 'node:util';+import { execFile } from '../lib/childProcess.js';import { dirname, join } from 'path';import { readRetirementCheckoutIdentity, isCurrentRetirementCheckout } from '../lib/retirementCheckoutObservation.js';import { execGit } from '../lib/execGit.js';@@ -130,13 +132,14 @@ describe.skipIf(SKIP_HEAVY_INTEGRATION)('held checkout recovery (real git)', ()}const archives = async () => (existsSync(heldCheckoutArchiveRoot(cosDir)) ? readdir(heldCheckoutArchiveRoot(cosDir)) : []);- async function historicalArchive() {+ async function historicalArchive(prepare = async () => {}) {const path = join(external, 'historical');const branch = 'feature/historical';await execGit(['worktree', 'add', '-b', branch, path, 'main'], repo);await writeFile(join(path, 'initial.txt'), 'staged historical work\n');await execGit(['add', 'initial.txt'], path);await writeFile(join(path, 'private.txt'), 'synthetic private bytes');+ await prepare(path);const release = { path, branch, head: (await execGit(['rev-parse', 'HEAD'], path)).stdout.trim(), locked: false, lockReason: null };await recordBranchReconcileOutcome('app-1', { status: 'incomplete', remaining: [{ ...release, reason: 'retirement-review-pending' }] });const saved = await preserveMergedCheckout(repo, { ...release, branch: `refs/heads/${branch}` }, 'main', { cosDir: PATHS.cos });@@ -403,6 +406,61 @@ describe.skipIf(SKIP_HEAVY_INTEGRATION)('held checkout recovery (real git)', ()expect(await getBranchReconcileOutcome('app-1')).toEqual(withoutObligation);});+ // A matching patch hash cannot detect bytes damaged before the old writer hashed them.+ it.each(['compatible', 'split-index', 'damaged', 'omitted-patch', 'missing-index'])('independently reconstructs a legacy staged-only index: %s', async variant => {+ let patch;+ let originalEntries;+ const { release, saved } = await historicalArchive(async path => {+ await writeFile(join(path, 'staged-only.txt'), Buffer.concat([Buffer.from('café staged content\n'), Buffer.from([0xff, 0x0a])]));+ await execGit(['add', 'staged-only.txt'], path);+ if (variant === 'split-index') await execGit(['update-index', '--split-index'], path);+ originalEntries = (await execGit(['-c', 'core.quotePath=true', 'ls-files', '--stage'], path)).stdout;+ // Capture the original bytes independently of the legacy execGit text decoder.+ patch = (await promisify(execFile)('git', ['diff', '--cached', '--binary', '--no-textconv', '--no-ext-diff'], { cwd: path, encoding: 'buffer' })).stdout;+ await rm(join(path, 'staged-only.txt'));+ });+ const manifestPath = join(saved.archiveDir, 'manifest.json');+ const manifest = JSON.parse(await readFile(manifestPath, 'utf8'));+ delete manifest.resourceContract;+ if (variant === 'damaged') {+ const split = patch.indexOf(Buffer.from('é')) + 1;+ expect(split).toBeGreaterThan(0);+ patch = Buffer.from(patch.subarray(0, split).toString('utf8') + patch.subarray(split).toString('utf8'));+ }+ await writeFile(join(saved.archiveDir, 'index.diff'), patch);+ manifest.patchHashes.index = await sha256File(join(saved.archiveDir, 'index.diff'));+ if (variant === 'omitted-patch') manifest.captured.indexDiff = false;+ if (variant === 'missing-index') {+ manifest.administration.files = manifest.administration.files.filter(file => file.path !== 'gitdir/index');+ const { createHash } = await import('node:crypto');+ manifest.administration.fingerprint = createHash('sha256').update(JSON.stringify({+ files: manifest.administration.files.filter(file => file.path !== 'gitdir/locked'), symlinks: manifest.administration.symlinks,+ })).digest('hex');+ }+ await writeFile(manifestPath, JSON.stringify(manifest));+ const before = (await execGit(['show-ref'], repo)).stdout;+ const outcome = await recordBranchReconcileOutcome('app-1', { status: 'incomplete', remaining: [] });+ if (['damaged', 'omitted-patch', 'missing-index'].includes(variant)) {+ expect(outcome.retirementObligations).toEqual([expect.objectContaining({ ...release,+ preservationEvidence: expect.objectContaining({ status: 'unresolved', nextAction: expect.stringContaining('backup'),+ archives: expect.arrayContaining([expect.objectContaining({ reason: 'archive-index-reconstruction-failed' })]) }),+ })]);+ expect(outcome.recoveredPreservations).toEqual([]);+ } else {+ expect(outcome.retirementObligations, JSON.stringify(injected.proofFailures)).toEqual([]);+ expect(outcome.recoveredPreservations).toContainEqual(expect.objectContaining({ archiveDir: saved.archiveDir, verified: true }));+ const restored = join(scratch, 'legacy-restore.git');+ await execGit(['init', '--bare', restored], scratch);+ await execGit(['bundle', 'unbundle', join(saved.archiveDir, 'commits.bundle')], restored);+ await execGit(['read-tree', release.head], restored);+ await execGit(['apply', '--cached', join(saved.archiveDir, 'index.diff')], restored);+ expect((await execGit(['-c', 'core.quotePath=true', 'ls-files', '--stage'], restored)).stdout).toBe(originalEntries);+ }+ expect(outcome.retired).toBeUndefined();+ expect((await execGit(['show-ref'], repo)).stdout).toBe(before);+ expect(existsSync(saved.archiveDir)).toBe(true);+ });+it.each([false, true])('clears the historical diagnosis after recovery retries, with other holds=%s', async otherHolds => {const { saved } = await historicalArchive();const unavailable = join(PATHS.cos, 'temporarily-unavailable-archive');