Skip to content

Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS

fix: verify historical staged patches against archived indexes (#13429)

Adam Eivy4c0779142a93
diff --git a/docs/features/branch-reconciliation.md b/docs/features/branch-reconciliation.md
index 97515fdba2..c9424f1039 100644
--- a/docs/features/branch-reconciliation.md
+++ b/docs/features/branch-reconciliation.md
@@ -68,6 +68,21 @@ immediately before removal; if that recheck refuses, the original lock is
restored. A retired row leaves the inventory and appears under *Recently
retired* with its archive path; only the next sweep can report convergence.
+Historical held-checkout archives must also prove that their staged patch
+reconstructs the preserved private index. Verification imports the bundle into
+a fresh, isolated Git repository, starts from the recorded HEAD, applies
+`index.diff` to a new index, and compares every path, mode, stage and object ID
+with the archived index. Split-index support uses only validated archived index
+files; archived Git configuration, pointers and object stores are never execution
+targets. Matching patch hashes alone cannot establish this proof: an older writer
+could have hashed bytes already damaged during decoding.
+
+A missing, unsupported or mismatched index retains the historical preservation
+obligation with `archive-index-reconstruction-failed`. Locate a complete archive
+for that exact checkout generation or recover the original data from backup, then
+retry reconciliation. Failure never fabricates a retirement receipt or authorizes
+deletion of a checkout, branch or archive.
+
## Explicit initialized-checkout preservation
Automatic cleanup continues to retain initialized submodules. The separate
diff --git a/server/services/heldCheckoutRecovery.js b/server/services/heldCheckoutRecovery.js
index b1c96f2844..1d4dd71341 100644
--- a/server/services/heldCheckoutRecovery.js
+++ b/server/services/heldCheckoutRecovery.js
@@ -71,8 +71,8 @@ async function archiveManifest(dir) {
return { manifest: JSON.parse(bytes), digest: evidenceHash(bytes) };
}
-/** Reuses the archive's recorded byte hashes and administration fingerprint.
- * Older dirty archives lack patch hashes and cannot prove completeness later.
+/** Reuses recorded byte hashes and independently reconstructs the private index.
+ * Older dirty archives without patch hashes cannot prove completeness later.
*/
async function verifyHeldArchive(repoPath, dir, release) {
const { manifest: saved, digest } = await archiveManifest(dir);
@@ -143,14 +143,33 @@ async function verifyHeldArchive(repoPath, dir, release) {
// (\\.\nul) is not the Git configuration-file contract.
const env = { ...Object.fromEntries(Object.entries(process.env).filter(([key]) => !key.toUpperCase().startsWith('GIT_'))),
GIT_CONFIG_NOSYSTEM: '1', GIT_CONFIG_GLOBAL: '/dev/null', GIT_NO_REPLACE_OBJECTS: '1', GIT_NO_LAZY_FETCH: '1', GIT_TERMINAL_PROMPT: '0' };
- const prove = args => execGit(['--no-optional-locks', '-c', 'core.hooksPath=/dev/null', '-c', 'core.fsmonitor=false', ...args], scratch,
- { env, maxBuffer: GIT_OUTPUT_BYTES });
+ const prove = (args, index = null) => execGit(['--no-optional-locks', '-c', 'core.hooksPath=/dev/null', '-c', 'core.fsmonitor=false', ...args], scratch,
+ { env: { ...env, ...(index && { GIT_INDEX_FILE: join(scratch, index) }) }, maxBuffer: GIT_OUTPUT_BYTES });
try {
- await prove(['init', '--bare', `--object-format=${release.head.length === 64 ? 'sha256' : 'sha1'}`, '.']);
- await prove(['bundle', 'unbundle', bundle]);
- await prove(['fsck', '--full', '--strict', '--no-reflogs', '--no-dangling', ...captured.recoveryHeads]);
- } catch { throw new Error('archive-bundle-object-proof-failed'); }
- finally { await rmGuarded(scratch, { recursive: true, force: true }); }
+ try {
+ await prove(['init', '--bare', `--object-format=${release.head.length === 64 ? 'sha256' : 'sha1'}`, '.']);
+ await prove(['bundle', 'unbundle', bundle]);
+ await prove(['fsck', '--full', '--strict', '--no-reflogs', '--no-dangling', ...captured.recoveryHeads]);
+ } catch { throw new Error('archive-bundle-object-proof-failed'); }
+ try {
+ // Copy only validated index bytes into our own Git directory. Never run
+ // Git against archived config, commondir, gitdir pointers or object stores.
+ if (!saved.administration.files.some(file => file.path === 'gitdir/index')) throw new Error('missing archived index');
+ for (const file of saved.administration.files.filter(file => file.path === 'gitdir/index'
+ || /^gitdir\/sharedindex\.[a-f0-9]+$/.test(file.path))) {
+ await copyFileGuarded(await archiveFile(dir, join(dir, 'administration', file.path)),
+ join(scratch, file.path === 'gitdir/index' ? 'archived-index' : basename(file.path)));
+ }
+ // Quoted paths make this comparison ASCII and lossless even for opaque
+ // filename bytes. --stage includes modes, object IDs and conflict stages.
+ const entries = ['-c', 'core.quotePath=true', 'ls-files', '--stage'];
+ const expected = await prove(entries, 'archived-index');
+ await prove(['read-tree', release.head]);
+ if (captured.indexDiff) await prove(['apply', '--cached', '--binary', join(dir, 'index.diff')]);
+ const restored = await prove(entries);
+ if (restored.stdout !== expected.stdout) throw new Error('index entries differ');
+ } catch { throw new Error('archive-index-reconstruction-failed'); }
+ } finally { await rmGuarded(scratch, { recursive: true, force: true }); }
if ((await archiveManifest(dir)).digest !== digest) throw new Error('archive-manifest-changed');
const recheck = async () => {
for (const [path, original] of stamps) {
diff --git a/server/services/heldCheckoutRecovery.test.js b/server/services/heldCheckoutRecovery.test.js
index 8cbecc83fc..0c5532ba58 100644
--- a/server/services/heldCheckoutRecovery.test.js
+++ b/server/services/heldCheckoutRecovery.test.js
@@ -8,6 +8,8 @@ import { describe, it, expect, beforeEach, afterEach, afterAll, vi } from 'vites
import { cp, mkdtemp, mkdir, rm, rename, writeFile, readFile, readdir, symlink, utimes, truncate } from 'fs/promises';
import { existsSync, realpathSync } from 'fs';
import { tmpdir } from 'os';
+import { promisify } from 'node:util';
+import { execFile } from '../lib/childProcess.js';
import { dirname, join } from 'path';
import { readRetirementCheckoutIdentity, isCurrentRetirementCheckout } from '../lib/retirementCheckoutObservation.js';
import { execGit } from '../lib/execGit.js';
@@ -130,13 +132,14 @@ describe.skipIf(SKIP_HEAVY_INTEGRATION)('held checkout recovery (real git)', ()
}
const archives = async () => (existsSync(heldCheckoutArchiveRoot(cosDir)) ? readdir(heldCheckoutArchiveRoot(cosDir)) : []);
- async function historicalArchive() {
+ async function historicalArchive(prepare = async () => {}) {
const path = join(external, 'historical');
const branch = 'feature/historical';
await execGit(['worktree', 'add', '-b', branch, path, 'main'], repo);
await writeFile(join(path, 'initial.txt'), 'staged historical work\n');
await execGit(['add', 'initial.txt'], path);
await writeFile(join(path, 'private.txt'), 'synthetic private bytes');
+ await prepare(path);
const release = { path, branch, head: (await execGit(['rev-parse', 'HEAD'], path)).stdout.trim(), locked: false, lockReason: null };
await recordBranchReconcileOutcome('app-1', { status: 'incomplete', remaining: [{ ...release, reason: 'retirement-review-pending' }] });
const saved = await preserveMergedCheckout(repo, { ...release, branch: `refs/heads/${branch}` }, 'main', { cosDir: PATHS.cos });
@@ -403,6 +406,61 @@ describe.skipIf(SKIP_HEAVY_INTEGRATION)('held checkout recovery (real git)', ()
expect(await getBranchReconcileOutcome('app-1')).toEqual(withoutObligation);
});
+ // A matching patch hash cannot detect bytes damaged before the old writer hashed them.
+ it.each(['compatible', 'split-index', 'damaged', 'omitted-patch', 'missing-index'])('independently reconstructs a legacy staged-only index: %s', async variant => {
+ let patch;
+ let originalEntries;
+ const { release, saved } = await historicalArchive(async path => {
+ await writeFile(join(path, 'staged-only.txt'), Buffer.concat([Buffer.from('café staged content\n'), Buffer.from([0xff, 0x0a])]));
+ await execGit(['add', 'staged-only.txt'], path);
+ if (variant === 'split-index') await execGit(['update-index', '--split-index'], path);
+ originalEntries = (await execGit(['-c', 'core.quotePath=true', 'ls-files', '--stage'], path)).stdout;
+ // Capture the original bytes independently of the legacy execGit text decoder.
+ patch = (await promisify(execFile)('git', ['diff', '--cached', '--binary', '--no-textconv', '--no-ext-diff'], { cwd: path, encoding: 'buffer' })).stdout;
+ await rm(join(path, 'staged-only.txt'));
+ });
+ const manifestPath = join(saved.archiveDir, 'manifest.json');
+ const manifest = JSON.parse(await readFile(manifestPath, 'utf8'));
+ delete manifest.resourceContract;
+ if (variant === 'damaged') {
+ const split = patch.indexOf(Buffer.from('é')) + 1;
+ expect(split).toBeGreaterThan(0);
+ patch = Buffer.from(patch.subarray(0, split).toString('utf8') + patch.subarray(split).toString('utf8'));
+ }
+ await writeFile(join(saved.archiveDir, 'index.diff'), patch);
+ manifest.patchHashes.index = await sha256File(join(saved.archiveDir, 'index.diff'));
+ if (variant === 'omitted-patch') manifest.captured.indexDiff = false;
+ if (variant === 'missing-index') {
+ manifest.administration.files = manifest.administration.files.filter(file => file.path !== 'gitdir/index');
+ const { createHash } = await import('node:crypto');
+ manifest.administration.fingerprint = createHash('sha256').update(JSON.stringify({
+ files: manifest.administration.files.filter(file => file.path !== 'gitdir/locked'), symlinks: manifest.administration.symlinks,
+ })).digest('hex');
+ }
+ await writeFile(manifestPath, JSON.stringify(manifest));
+ const before = (await execGit(['show-ref'], repo)).stdout;
+ const outcome = await recordBranchReconcileOutcome('app-1', { status: 'incomplete', remaining: [] });
+ if (['damaged', 'omitted-patch', 'missing-index'].includes(variant)) {
+ expect(outcome.retirementObligations).toEqual([expect.objectContaining({ ...release,
+ preservationEvidence: expect.objectContaining({ status: 'unresolved', nextAction: expect.stringContaining('backup'),
+ archives: expect.arrayContaining([expect.objectContaining({ reason: 'archive-index-reconstruction-failed' })]) }),
+ })]);
+ expect(outcome.recoveredPreservations).toEqual([]);
+ } else {
+ expect(outcome.retirementObligations, JSON.stringify(injected.proofFailures)).toEqual([]);
+ expect(outcome.recoveredPreservations).toContainEqual(expect.objectContaining({ archiveDir: saved.archiveDir, verified: true }));
+ const restored = join(scratch, 'legacy-restore.git');
+ await execGit(['init', '--bare', restored], scratch);
+ await execGit(['bundle', 'unbundle', join(saved.archiveDir, 'commits.bundle')], restored);
+ await execGit(['read-tree', release.head], restored);
+ await execGit(['apply', '--cached', join(saved.archiveDir, 'index.diff')], restored);
+ expect((await execGit(['-c', 'core.quotePath=true', 'ls-files', '--stage'], restored)).stdout).toBe(originalEntries);
+ }
+ expect(outcome.retired).toBeUndefined();
+ expect((await execGit(['show-ref'], repo)).stdout).toBe(before);
+ expect(existsSync(saved.archiveDir)).toBe(true);
+ });
+
it.each([false, true])('clears the historical diagnosis after recovery retries, with other holds=%s', async otherHolds => {
const { saved } = await historicalArchive();
const unavailable = join(PATHS.cos, 'temporarily-unavailable-archive');