Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS
JavaScript ai-agentsdashboarddeveloper-toolsdigital-twinquantified-selfsecond-brainself-hostedtailscale
fix(cos): archive raw tracked bytes that normalization hides from diffs (#12002)
[._.]/ Adam Eivy67f29ccfcb1e
diff --git a/docs/features/branch-reconciliation.md b/docs/features/branch-reconciliation.mdindex b4c08dd09b..c737bdd444 100644--- a/docs/features/branch-reconciliation.md+++ b/docs/features/branch-reconciliation.md@@ -34,8 +34,9 @@ default branch and no live owner, claim binding or manual completion handoff.Before any mutation, every unique byte is archived under`data/cos/held-checkout-archives/`: a verified Git bundle of commits not on the-default branch, the staged and unstaged diffs, verbatim copies of untracked and-ignored files (hash-verified), and symlink targets. A nested repository or+default branch, the staged and unstaged diffs, raw bytes of tracked files that a+clean filter or line-ending conversion hides from those diffs, verbatim copies+of untracked and ignored files (all hash-verified), and symlink targets. A nested repository orsubmodule, unmerged or hidden index entries, an oversized file or a write during archiving fails the archive, which leaves the checkout, its lock and itsbranch untouched and removes the partial archive. The lock is released onlyafter a second fresh match, and ownership, HEAD and content are recheckeddiff --git a/server/services/heldCheckoutRecovery.js b/server/services/heldCheckoutRecovery.jsindex 426c23e020..5fbee9f20e 100644--- a/server/services/heldCheckoutRecovery.js+++ b/server/services/heldCheckoutRecovery.js@@ -11,13 +11,15 @@** Before any mutation every unique byte is archived under* `data/cos/held-checkout-archives/`: a bundle of commits not on the default- * branch, the staged and unstaged diffs, and verbatim copies of untracked AND- * ignored files (symlinks as recorded targets). The archive is verified by hash against the+ * branch, the staged and unstaged diffs, raw bytes of tracked files that differ+ * from their index blob, and verbatim copies of untracked AND ignored files+ * (symlinks as recorded targets). The archive is verified by hash against the* checkout, and the checkout's content hashes must not move between archive and* removal. A live owner, manual completion handoff, unreadable state or archive* failure leaves the checkout, its lock and its branch exactly as found.*/+import { createReadStream } from 'node:fs';import { lstat, readlink } from 'node:fs/promises';import { createHash } from 'node:crypto';import { dirname, join } from 'node:path';@@ -44,44 +46,64 @@ const shortBranch = (ref) => (ref ? String(ref).replace(/^refs\/heads\//, '') :const zLines = (stdout) => (stdout || '').split('\0').filter(Boolean);const git = (args, cwd) => execGit(args, cwd, { maxBuffer: GIT_OUTPUT_BYTES }).then(r => r.stdout || '');+/** Git's object id for raw bytes, with no clean filter or line-ending conversion. */+async function rawBlobId(source, size, format) {+ const hash = createHash(format).update(`blob ${size}\0`);+ for await (const chunk of createReadStream(source)) hash.update(chunk);+ return hash.digest('hex');+}+/*** Everything the archive must preserve, read in one pass: the staged and* unstaged diffs (separately, so a staged blob whose working copy is gone- * survives) and every untracked or ignored file by content hash, symlinks by- * target. Throws on anything that cannot be archived faithfully, before any- * copy is made. The fingerprint is compared again immediately before removal.+ * survives), raw copies of tracked files whose bytes differ from their index+ * blob (a clean filter or line-ending conversion hides those from diff), and+ * every untracked or ignored file by content hash, symlinks by target. Throws+ * on anything that cannot be archived faithfully, before any copy is made.+ * The fingerprint is compared again immediately before removal.*/async function snapshotContent(path) {if ((await git(['ls-files', '-u', '-z'], path)).length) throw new Error('unmerged index entries cannot be archived');+ const stage = zLines(await git(['ls-files', '--stage', '-z'], path));// A submodule's files and commits live in its own repository; its gitlink is not its content.- if (zLines(await git(['ls-files', '--stage', '-z'], path)).some(line => line.startsWith('160000 '))) {- throw new Error('submodules cannot be archived');- }+ if (stage.some(line => line.startsWith('160000 '))) throw new Error('submodules cannot be archived');// Assume-unchanged (lowercase tag) and skip-worktree (S) entries hide edits from diff.const hidden = zLines(await git(['ls-files', '-v', '-z'], path)).find(line => /^[a-zS]/.test(line));if (hidden) throw new Error(`${hidden.slice(2)} hides working-tree changes from Git`);- // Raw bytes only: a textconv or external diff driver can drop content.+ // A textconv or external diff driver can drop bytes from a patch.const indexDiff = await git(['diff', '--cached', '--binary', '--no-textconv', '--no-ext-diff'], path);const worktreeDiff = await git(['diff', '--binary', '--no-textconv', '--no-ext-diff'], path);- const others = [];++ const files = [];let total = 0;+ const addFile = async (rel, info, extra = {}) => {+ total += info.size;+ if (info.size > MAX_FILE_BYTES || total > MAX_TOTAL_BYTES) throw new Error(`${rel} exceeds the archive size limit`);+ files.push({ path: rel, bytes: info.size, sha256: await sha256File(join(path, rel)), ...extra });+ };+ const format = (await git(['rev-parse', '--show-object-format'], path)).trim() || 'sha1';+ for (const line of stage) {+ const tab = line.indexOf('\t');+ const [mode, blob] = line.slice(0, tab).split(' ');+ const rel = line.slice(tab + 1);+ // Symlink blobs are not filtered; deletions and type changes are in the diff.+ if (mode !== '100644' && mode !== '100755') continue;+ const info = await lstat(join(path, rel)).catch(err => (err.code === 'ENOENT' ? null : Promise.reject(err)));+ if (!info?.isFile()) continue;+ if (await rawBlobId(join(path, rel), info.size, format) !== blob) await addFile(rel, info, { tracked: true });+ }+ const symlinks = [];for (const rel of zLines(await git(['ls-files', '--others', '-z'], path))) {// `ls-files --others` names a nested repository as `dir/` without its files.if (rel.endsWith('/')) throw new Error(`nested repository at ${rel} cannot be archived`);- const source = join(path, rel);- const info = await lstat(source);- if (info.isSymbolicLink()) {- others.push({ path: rel, symlink: await readlink(source) });- continue;- }- if (!info.isFile()) throw new Error(`${rel} is not a regular file`);- total += info.size;- if (info.size > MAX_FILE_BYTES || total > MAX_TOTAL_BYTES) throw new Error(`${rel} exceeds the archive size limit`);- others.push({ path: rel, bytes: info.size, sha256: await sha256File(source) });+ const info = await lstat(join(path, rel));+ if (info.isSymbolicLink()) symlinks.push({ path: rel, target: await readlink(join(path, rel)) });+ else if (info.isFile()) await addFile(rel, info);+ else throw new Error(`${rel} is not a regular file`);}const fingerprint = createHash('sha256').update(indexDiff).update('\0').update(worktreeDiff).update('\0')- .update(JSON.stringify(others)).digest('hex');- return { indexDiff, worktreeDiff, others, fingerprint };+ .update(JSON.stringify({ files, symlinks })).digest('hex');+ return { indexDiff, worktreeDiff, files, symlinks, fingerprint };}/**@@ -116,8 +138,7 @@ async function writeArchive(dir, { repoPath, tree, branch, defaultBranch, releasif (snapshot.indexDiff) await writeFileGuarded(join(dir, 'index.diff'), snapshot.indexDiff);if (snapshot.worktreeDiff) await writeFileGuarded(join(dir, 'worktree.diff'), snapshot.worktreeDiff);- const files = snapshot.others.filter(entry => entry.symlink === undefined);- const symlinks = snapshot.others.filter(entry => entry.symlink !== undefined).map(({ path, symlink }) => ({ path, target: symlink }));+ const { files, symlinks } = snapshot;for (const file of files) {const destination = join(dir, 'files', file.path);await ensureDir(dirname(destination));@@ -135,6 +156,7 @@ async function writeArchive(dir, { repoPath, tree, branch, defaultBranch, releas`git -C <repo> worktree add <checkout> ${tree.head}`,snapshot.indexDiff ? 'git -C <checkout> apply --index index.diff' : null,snapshot.worktreeDiff ? 'git -C <checkout> apply worktree.diff' : null,+ // Last, so raw tracked bytes replace the filtered form the diffs restore.files.length ? 'cp -R files/. <checkout>/' : null,symlinks.length ? 'recreate manifest.captured.symlinks with ln -s' : null,].filter(Boolean),diff --git a/server/services/heldCheckoutRecovery.test.js b/server/services/heldCheckoutRecovery.test.jsindex cc67111817..617e304d51 100644--- a/server/services/heldCheckoutRecovery.test.js+++ b/server/services/heldCheckoutRecovery.test.js@@ -82,7 +82,14 @@ describe.skipIf(SKIP_HEAVY_INTEGRATION)('held checkout recovery (real git)', ()const archives = async () => (existsSync(heldCheckoutArchiveRoot(cosDir)) ? readdir(heldCheckoutArchiveRoot(cosDir)) : []);it('archives staged, unstaged, untracked, ignored and symlinked content, then retires a locked detached checkout', async () => {- const { path, release } = await heldCheckout('detached-held');+ await writeFile(join(repo, 'eol.txt'), 'one\ntwo\n');+ await execGit(['add', 'eol.txt'], repo);+ await execGit(['commit', '-m', 'chore: synthetic text file'], repo);+ const { path, release } = await heldCheckout('detached-held', { beforeLock: async (tree) => {+ // Normalization hides these CRLF bytes from every diff.+ await writeFile(join(tree, '.gitattributes'), 'eol.txt text eol=lf\n');+ await writeFile(join(tree, 'eol.txt'), 'one\r\ntwo\r\n');+ } });// Only the index holds this file once its working copy is gone.await writeFile(join(path, 'staged-only.txt'), 'staged work\n');await execGit(['add', 'staged-only.txt'], path);@@ -95,7 +102,9 @@ describe.skipIf(SKIP_HEAVY_INTEGRATION)('held checkout recovery (real git)', ()expect(await registered(release.path)).toBe(false);const manifest = JSON.parse(await readFile(join(result.archiveDir, 'manifest.json'), 'utf8'));expect(manifest.release).toEqual(release);- expect(manifest.captured.files.map(file => file.path).sort()).toEqual(['.gitignore', 'ignored.txt', 'untracked.txt']);+ expect(manifest.captured.files.map(file => `${file.path}${file.tracked ? ' (tracked)' : ''}`).sort())+ .toEqual(['.gitattributes', '.gitignore', 'eol.txt (tracked)', 'ignored.txt', 'initial.txt (tracked)', 'untracked.txt']);+ expect(await readFile(join(result.archiveDir, 'files', 'eol.txt'), 'utf8')).toBe('one\r\ntwo\r\n');expect(manifest.captured.symlinks).toEqual(SYMLINKS ? [{ path: 'linked', target: '../shared-target' }] : []);expect(await readFile(join(result.archiveDir, 'files', 'ignored.txt'), 'utf8')).toBe('ignored work\n');expect(await readFile(join(result.archiveDir, 'worktree.diff'), 'utf8')).toContain('+tracked edit');