Beautiful, AI-native markdown IDE and LLM wiki
TypeScript 2nd-brainagent-skillsclaudecodexcompany-braindocsknowledge-knowledge-baseknowledge-managementllmllm-wikillm-wiki-karpathymarkdownmarkdown-editormdmdxnotespkmsecond-brainskills
perf(server): cut shadow-repo write-path cost on large KBs (#2737)
Andrew Mikofalvy789ed92967a7
diff --git a/.changeset/shadow-repo-write-perf.md b/.changeset/shadow-repo-write-perf.mdnew file mode 100644index 000000000..6f2d1c9ed--- /dev/null+++ b/.changeset/shadow-repo-write-perf.md@@ -0,0 +1,5 @@+---+"@inkeep/open-knowledge": patch+---++Version-history recording is dramatically cheaper on large knowledge bases and slow (mounted/network) volumes. The background history snapshot now reuses a persistent git index so unchanged files are no longer re-read and re-hashed on every save cycle (up to ~11x faster tree builds on a 10k-file KB); switching git branches with many open documents batches its bookkeeping into a few git calls instead of four per document (a 100-document switch drops from ~15s to well under a second); and agent writes coalesce into the normal debounced history commit instead of forcing a full commit per write, while history reads still always reflect every completed write.diff --git a/packages/app/tests/integration/agent-write-commit-coalescing.test.ts b/packages/app/tests/integration/agent-write-commit-coalescing.test.tsnew file mode 100644index 000000000..86a869a44--- /dev/null+++ b/packages/app/tests/integration/agent-write-commit-coalescing.test.ts@@ -0,0 +1,86 @@+/**+ * Agent writes ride the L2 commit debounce instead of forcing a shadow commit+ * per write, and `/api/history` drains any pending commit before reading so+ * the debounce never costs read-your-writes.+ *+ * Pins the two halves of the contract:+ * 1. Coalescing — successive agent writes inside one debounce window produce+ * NO shadow commit on their own (the per-write forced flush is gone).+ * 2. Read-your-writes — a `/api/history` read issued right after a write+ * lists that write, because the handler flushes the pending commit first.+ *+ * Uses a deliberately huge `commitDebounceMs` so the debounce timer cannot+ * fire during the test — any commit observed before the history read would be+ * a regression back to per-write flushing.+ */++import { execFileSync } from 'node:child_process';+import { resolveShadowDir } from '@inkeep/open-knowledge-core/shadow-repo-layout';+import { afterEach, describe, expect, test } from 'vitest';+import { agentWriteMd, createTestServer, type TestServer } from './test-harness';++interface TimelineResponse {+ entries: Array<{ sha: string; type: string; message: string }>;+}++let server: TestServer | undefined;++afterEach(async () => {+ await server?.cleanup();+ server = undefined;+});++function listWipRefs(contentDir: string): string[] {+ const shadowDir = resolveShadowDir(contentDir);+ const raw = execFileSync('git', ['for-each-ref', '--format=%(refname)', 'refs/wip/'], {+ env: { ...process.env, GIT_DIR: shadowDir },+ encoding: 'utf-8',+ });+ return raw.trim().split('\n').filter(Boolean);+}++describe('agent-write L2 commit coalescing', () => {+ test('writes coalesce on the debounce; /api/history flushes before reading', async () => {+ server = await createTestServer({ gitEnabled: true, commitDebounceMs: 600_000 });++ await agentWriteMd(server.port, '# Doc A\n', {+ docName: 'coalesce-a',+ position: 'replace',+ agentId: 'coalesce-writer',+ agentName: 'Coalesce Writer',+ });+ await agentWriteMd(server.port, '# Doc B\n', {+ docName: 'coalesce-b',+ position: 'replace',+ agentId: 'coalesce-writer',+ agentName: 'Coalesce Writer',+ });++ // Both writes are on disk (the handler awaits the L1 store), but neither+ // forced an L2 shadow commit: the wip namespace is still empty.+ expect(listWipRefs(server.contentDir)).toEqual([]);++ // A history read right after the writes must still list them — the+ // handler drains the pending commit before querying.+ const res = await fetch(+ `${server.baseUrl}/api/history?docName=${encodeURIComponent('coalesce-a')}`,+ );+ expect(res.ok).toBe(true);+ const body = (await res.json()) as TimelineResponse;+ expect(body.entries.length).toBeGreaterThanOrEqual(1);+ expect(body.entries.some((e) => e.type === 'wip')).toBe(true);++ // The read's flush drained BOTH pending writes into one coalesced drain:+ // the second doc is queryable without any further write or flush, and the+ // writer advanced a single wip ref.+ const resB = await fetch(+ `${server.baseUrl}/api/history?docName=${encodeURIComponent('coalesce-b')}`,+ );+ expect(resB.ok).toBe(true);+ const bodyB = (await resB.json()) as TimelineResponse;+ expect(bodyB.entries.some((e) => e.type === 'wip')).toBe(true);++ const refs = listWipRefs(server.contentDir);+ expect(refs.length).toBe(1);+ }, 30_000);+});diff --git a/packages/app/tests/integration/test-harness.ts b/packages/app/tests/integration/test-harness.tsindex b802082c5..b07a4bd2f 100644--- a/packages/app/tests/integration/test-harness.ts+++ b/packages/app/tests/integration/test-harness.ts@@ -1005,18 +1005,18 @@ export async function awaitBacklinkIndexed(* return their shas (newest-first, as `/api/history` orders them — so the oldest* is `shas[shas.length - 1]`).*- * Agent-write handlers fire the L1 store → L2 git-commit chain FIRE-AND-FORGET- * (`flushDocToGit` in api-extension.ts: the HTTP response returns before the- * commit is durable). A bare `pollUntil` on `/api/history` therefore RACES the- * commit against a wall-clock budget — and under merge-queue contention the- * serial git-subprocess chain (global one-commit-in-flight mutex) blows any fixed- * budget. This helper AWAITS the commit instead: each iteration POSTs- * `/api/test-flush-git` (drains the pending L2 debounce timer + any in-flight- * commit), then checks. Flush every iteration — the first call can land before- * the fire-and-forget chain has even scheduled L2. The budget is a generous- * ceiling, not the thing being raced. Mirrors `rename-history.test.ts`'s- * `awaitWipCommit`; both close the same flake class (the version-rollback case- * is #1596).+ * Agent-write handlers leave the L2 git commit to the persistence debounce+ * (the HTTP response returns before any commit is durable; rename/rollback/+ * undo additionally fire `flushDocToGit` fire-and-forget). A bare `pollUntil`+ * on `/api/history` therefore RACES the commit against a wall-clock budget —+ * and under merge-queue contention the serial git-subprocess chain (global+ * one-commit-in-flight mutex) blows any fixed budget. This helper AWAITS the+ * commit instead: each iteration POSTs `/api/test-flush-git` (drains the+ * pending L2 debounce timer + any in-flight commit), then checks. Flush every+ * iteration — the first call can land before the store path has even armed the+ * L2 timer. The budget is a generous ceiling, not the thing being raced.+ * Mirrors `rename-history.test.ts`'s `awaitWipCommit`; both close the same+ * flake class (the version-rollback case is #1596).*/export async function awaitWipCommits(server: TestServer,diff --git a/packages/server/src/api-extension.ts b/packages/server/src/api-extension.tsindex add978b97..14d251402 100644--- a/packages/server/src/api-extension.ts+++ b/packages/server/src/api-extension.ts@@ -3243,6 +3243,14 @@ export function createApiExtension(options: ApiExtensionOptions): Extension {* L2 (disk → git): chained after L1 resolves to guarantee disk content is* up-to-date before the shadow-repo commit.*+ * Reserved for rare, semantically anchored actions (rename, rollback, undo)+ * whose commit must land promptly — a rename's log entries stay unanchored+ * until its shadow commit exists. High-frequency agent write handlers use+ * {@link flushDocToDisk} instead so their L2 commits coalesce on the+ * persistence debounce rather than paying a whole-corpus tree build per+ * write; `handleHistory` drains any pending commit before reading, so+ * history reads still observe every completed write.+ ** The returned promise is intentionally not awaited by callers — the HTTP* response fires immediately after the CRDT transaction; persistence is* best-effort background work.@@ -3257,6 +3265,21 @@ export function createApiExtension(options: ApiExtensionOptions): Extension {});}+ /**+ * Fire-and-forget L1-only flush: force the per-document disk store without+ * forcing the L2 shadow commit. The store path always arms the L2 debounce+ * timer, so the commit still lands — coalesced with every other write in the+ * window instead of one full tree build per agent write. No-op when the+ * handler already awaited `flushDiskAndDetectOutcome` (nothing debounced).+ */+ function flushDocToDisk(docName: string, label: string): void {+ const debounceId = `onStoreDocument-${docName}`;+ if (!hocuspocus.debouncer.isDebounced(debounceId)) return;+ hocuspocus.debouncer.executeNow(debounceId).catch((err: unknown) => {+ log.warn({ err }, `[${label}] post-write disk flush failed`);+ });+ }+/*** Force the debounced L1 disk store for `docName` to run now and await it,* then report whether it failed. Hocuspocus's `storeDocumentHooks` swallows@@ -5123,7 +5146,7 @@ export function createApiExtension(options: ApiExtensionOptions): Extension {respondDiskDivergence(res, 'agent-write');return;}- flushDocToGit(docName, 'agent-write');+ flushDocToDisk(docName, 'agent-write');onAgentWrite?.();// Success body is flat — no `{ ok: true }` wrapper. Clients@@ -5332,7 +5355,7 @@ export function createApiExtension(options: ApiExtensionOptions): Extension {return;}- flushDocToGit(resolvedDocName, 'agent-write-md');+ flushDocToDisk(resolvedDocName, 'agent-write-md');// Focus (attribution) on __system__ awareness. Focus drives browser// push-navigation to the doc the agent just wrote (writeKind); presence@@ -5704,7 +5727,7 @@ export function createApiExtension(options: ApiExtensionOptions): Extension {return;}}- flushDocToGit(resolvedDocName, 'frontmatter-patch');+ flushDocToDisk(resolvedDocName, 'frontmatter-patch');}agentFocusBroadcaster?.setFocus(agentId, {@@ -7148,7 +7171,7 @@ export function createApiExtension(options: ApiExtensionOptions): Extension {return;}- flushDocToGit(docName, 'agent-patch');+ flushDocToDisk(docName, 'agent-patch');// Focus (attribution) on __system__ awareness. Presence is separately// maintained via setPresence/touchMode pairs above.@@ -7603,15 +7626,16 @@ export function createApiExtension(options: ApiExtensionOptions): Extension {/*** POST /api/test-flush-git — await the L2 git-commit pipeline to settle.*- * Agent-write handlers fire `flushDocToGit` FIRE-AND-FORGET, so a test- * that needs the WIP commit durable can only poll the timeline against a- * wall-clock budget — and under CI load the serial git-subprocess chain- * (global one-commit-in-flight mutex in persistence.ts) blows any fixed- * budget. This route lets tests AWAIT the- * actual commit completion instead of racing it: it drains the pending- * L2 debounce timer and any in-flight commit before responding. Callers- * should flush-then-check inside their poll loop — the fire-and-forget- * chain may not have scheduled L2 yet on the first iteration.+ * Agent-write handlers fire `flushDocToDisk` FIRE-AND-FORGET and leave the+ * L2 shadow commit to the persistence debounce, so a test that needs the+ * WIP commit durable can only poll the timeline against a wall-clock+ * budget — and under CI load the serial git-subprocess chain (global+ * one-commit-in-flight mutex in persistence.ts) blows any fixed budget.+ * This route lets tests AWAIT the actual commit completion instead of+ * racing it: it drains the pending L2 debounce timer and any in-flight+ * commit before responding. Callers should flush-then-check inside their+ * poll loop — the fire-and-forget chain may not have scheduled L2 yet on+ * the first iteration.*/const handleTestFlushGit = withValidation(EmptyRequestSchema,@@ -8017,6 +8041,22 @@ export function createApiExtension(options: ApiExtensionOptions): Extension {return;}+ // Read-your-writes: agent write handlers no longer force an L2 shadow+ // commit per write (they ride the persistence debounce), so drain any+ // pending commit before querying — a `history` call issued right after a+ // write must list that write. No-op when nothing is pending. The flush+ // blocks the response, so surface slow (cold-index) drains in the logs.+ try {+ const flushStart = performance.now();+ await flushGitCommit?.();+ const flushMs = performance.now() - flushStart;+ if (flushMs > 1000) {+ log.warn({ durationMs: Math.round(flushMs) }, '[history] pre-read commit flush slow');+ }+ } catch (err) {+ log.warn({ err }, '[history] pre-read commit flush failed');+ }+const url = new URL(req.url ?? '/', `http://${req.headers.host ?? 'localhost'}`);const docName = url.searchParams.get('docName') ?? '';const folderParam = url.searchParams.get('folder');@@ -8469,10 +8509,11 @@ export function createApiExtension(options: ApiExtensionOptions): Extension {// Force-flush L1 (onStoreDocument debounce) then L2 (git commit) so the// restored version + attribution appear in the timeline within ~100ms- // rather than waiting for the natural ~4s L1+L2 debounce stack. Uses- // the shared `flushDocToGit` helper (same pattern as the three- // agent-write handlers) rather than a raw `flushGitCommit()` which- // no-ops when no L2 timer is set yet.+ // rather than waiting for the natural ~4s L1+L2 debounce stack.+ // Rollback intentionally keeps the prompt `flushDocToGit` helper —+ // unlike the high-frequency agent-write handlers, which use+ // `flushDocToDisk` and let the commit ride the debounce — rather than+ // a raw `flushGitCommit()` which no-ops when no L2 timer is set yet.// Await the L1 disk store so a swallowed persistence failure surfaces// as an error instead of a false success. Mirrors agent-write-md.const flushOutcome = await flushDiskAndDetectOutcome(docName);@@ -18449,7 +18490,7 @@ export function createApiExtension(options: ApiExtensionOptions): Extension {respondDiskDivergence(res, 'lint-fix');return;}- flushDocToGit(resolvedDocName, 'lint-fix');+ flushDocToDisk(resolvedDocName, 'lint-fix');// Remaining problems: re-lint the ACTUAL post-write source (bridge// normalization can shift bytes vs `fixed`), reusing the resolved cfg.diff --git a/packages/server/src/shadow-repo.test.ts b/packages/server/src/shadow-repo.test.tsindex dbcd1fd8a..05ab745df 100644--- a/packages/server/src/shadow-repo.test.ts+++ b/packages/server/src/shadow-repo.test.ts@@ -1,4 +1,3 @@-import { afterEach, beforeEach, describe, expect, spyOn, test } from 'bun:test';import { existsSync, mkdirSync, rmSync, writeFileSync } from 'node:fs';import { mkdtemp, rm } from 'node:fs/promises';import { tmpdir } from 'node:os';@@ -10,6 +9,7 @@ import {parseOkActor,} from '@inkeep/open-knowledge-core/shadow-repo-layout';import simpleGit from 'simple-git';+import { afterEach, beforeEach, describe, expect, test, vi } from 'vitest';import {buildWipTree,commitUpstreamImport,@@ -145,7 +145,7 @@ describe('initShadowRepo', () => {writeFileSync(resolve(legacyDir, 'LEGACY_SENTINEL'), 'legacy');writeFileSync(resolve(newDir, 'NEW_SENTINEL'), 'new');- const warnSpy = spyOn(console, 'warn').mockImplementation(() => {});+ const warnSpy = vi.spyOn(console, 'warn').mockImplementation(() => {});try {await initShadowRepo(projectRoot);@@ -185,6 +185,104 @@ describe('buildWipTree contentRoot pathspec', () => {});});+describe('buildWipTree persistent fan-out index', () => {+ let projectRoot: string;+ let shadow: ShadowHandle;+ let contentDir: string;++ beforeEach(async () => {+ projectRoot = resolve(tmpDir, 'project');+ contentDir = resolve(projectRoot, 'content');+ mkdirSync(contentDir, { recursive: true });+ const git = simpleGit(projectRoot);+ await git.init();+ shadow = await initShadowRepo(projectRoot);+ });++ test('reused index tracks modify, add, and delete across successive builds', async () => {+ writeFileSync(resolve(contentDir, 'a.md'), '# A v1\n');+ writeFileSync(resolve(contentDir, 'b.md'), '# B v1\n');+ const tree1 = await buildWipTree(shadow, 'content');+ // Index survives the call — this is the reuse the stat cache depends on.+ expect(existsSync(resolve(shadow.gitDir, 'index-wip-fanout'))).toBe(true);++ writeFileSync(resolve(contentDir, 'a.md'), '# A v2\n');+ rmSync(resolve(contentDir, 'b.md'));+ writeFileSync(resolve(contentDir, 'c.md'), '# C v1\n');+ const tree2 = await buildWipTree(shadow, 'content');+ expect(tree2).not.toBe(tree1);++ const sg = shadowGit(shadow);+ const listing = await sg.raw('ls-tree', '-r', '--name-only', tree2);+ const paths = listing.trim().split('\n').sort();+ expect(paths).toEqual(['content/a.md', 'content/c.md']);+ expect((await sg.raw('cat-file', '-p', `${tree2}:content/a.md`)).trim()).toBe('# A v2');+ });++ test('reused index produces the same tree a fresh index would', async () => {+ writeFileSync(resolve(contentDir, 'a.md'), '# A v1\n');+ await buildWipTree(shadow, 'content');+ writeFileSync(resolve(contentDir, 'a.md'), '# A v2\n');+ const warmTree = await buildWipTree(shadow, 'content');++ // Fresh-index reference build of the same working tree state.+ rmSync(resolve(shadow.gitDir, 'index-wip-fanout'));+ const freshTree = await buildWipTree(shadow, 'content');+ expect(warmTree).toBe(freshTree);+ });++ test('corrupt persistent index falls back to a fresh rebuild', async () => {+ writeFileSync(resolve(contentDir, 'a.md'), '# A v1\n');+ const tree1 = await buildWipTree(shadow, 'content');++ writeFileSync(resolve(shadow.gitDir, 'index-wip-fanout'), 'not a git index');+ const warnSpy = vi.spyOn(console, 'warn').mockImplementation(() => {});+ try {+ const tree2 = await buildWipTree(shadow, 'content');+ expect(tree2).toBe(tree1);+ const warnings = warnSpy.mock.calls.map((call) => String(call[0] ?? ''));+ expect(warnings.some((w) => w.includes('persistent fan-out index failed'))).toBe(true);+ } finally {+ warnSpy.mockRestore();+ }+ // Corrupt cache was dropped so the next cycle starts clean.+ expect(existsSync(resolve(shadow.gitDir, 'index-wip-fanout'))).toBe(false);+ });++ test('stale index.lock from a killed add falls back and is cleared', async () => {+ writeFileSync(resolve(contentDir, 'a.md'), '# A v1\n');+ const tree1 = await buildWipTree(shadow, 'content');++ writeFileSync(resolve(shadow.gitDir, 'index-wip-fanout.lock'), '');+ const warnSpy = vi.spyOn(console, 'warn').mockImplementation(() => {});+ try {+ const tree2 = await buildWipTree(shadow, 'content');+ expect(tree2).toBe(tree1);+ const warnings = warnSpy.mock.calls.map((call) => String(call[0] ?? ''));+ expect(warnings.some((w) => w.includes('persistent fan-out index failed'))).toBe(true);+ } finally {+ warnSpy.mockRestore();+ }+ expect(existsSync(resolve(shadow.gitDir, 'index-wip-fanout.lock'))).toBe(false);+ });++ test('boot sweep removes throwaway indices, stale lock, and orphaned park blob dirs but keeps the cache', async () => {+ writeFileSync(resolve(shadow.gitDir, 'index-wip-fanout-deadbeef'), '');+ writeFileSync(resolve(shadow.gitDir, 'index-wip-fanout.lock'), '');+ writeFileSync(resolve(shadow.gitDir, 'index-wip-fanout'), '');+ const orphanParkDir = resolve(shadow.gitDir, 'tmp-park-blobs-deadbeef');+ mkdirSync(orphanParkDir, { recursive: true });+ writeFileSync(resolve(orphanParkDir, '0-state'), '# orphaned park blob\n');++ // Re-init on the same project runs the sweep again.+ const reinit = await initShadowRepo(projectRoot);+ expect(existsSync(resolve(reinit.gitDir, 'index-wip-fanout-deadbeef'))).toBe(false);+ expect(existsSync(resolve(reinit.gitDir, 'index-wip-fanout.lock'))).toBe(false);+ expect(existsSync(orphanParkDir)).toBe(false);+ expect(existsSync(resolve(reinit.gitDir, 'index-wip-fanout'))).toBe(true);+ });+});+describe('commitWip', () => {let projectRoot: string;let shadow: ShadowHandle;@@ -526,6 +624,28 @@ describe('parkBranch', () => {expect(guideContent).toBe('# Guide');});+ test('parks enough documents to span multiple hash/stage batches', async () => {+ // 120 docs → 240 blobs/index entries, crossing the 200-entry chunk+ // boundary so both the multi-chunk hash-object and update-index legs run.+ const docs: ParkableDoc[] = Array.from({ length: 120 }, (_, i) => ({+ docName: `notes/doc-${i}`,+ markdown: `# Doc ${i} memory\n`,+ diskSnapshot: `# Doc ${i} disk\n`,+ }));++ const sha = await parkBranch(shadow, 'main', SERVICE_WRITER.id, docs);+ expect(sha).toHaveLength(40);++ const sg = shadowGit(shadow);+ // Spot-check first, chunk-straddling, and last entries on both tree sides.+ for (const i of [0, 99, 100, 119]) {+ const content = (await sg.raw('show', `${sha}:notes/doc-${i}`)).trim();+ expect(content).toBe(`# Doc ${i} memory`);+ const base = (await sg.raw('show', `${sha}:.park-base/notes/doc-${i}`)).trim();+ expect(base).toBe(`# Doc ${i} disk`);+ }+ });+test('isPairedWriteOrigin(PARK_SNAPSHOT_ORIGIN) returns true (US-017)', () => {// Import from standalone — verify paired: true is recognizedconst origin = {diff --git a/packages/server/src/shadow-repo.ts b/packages/server/src/shadow-repo.tsindex 14e96ab69..3c5641904 100644--- a/packages/server/src/shadow-repo.ts+++ b/packages/server/src/shadow-repo.ts@@ -257,9 +257,9 @@ export async function initShadowRepo(projectRoot: string): Promise<ShadowHandle>// Idempotent — no-op once all legacy refs are gone.await sweepLegacyShadowRefs(handle);- // Sweep orphaned temp-index files left behind by crashed `buildWipTree`- // calls from a prior process. Idempotent — no-op on a clean shutdown.- sweepOrphanedTmpIndexFiles(handle);+ // Sweep orphaned scratch state (temp indices, park blob dirs) left behind+ // by a crashed prior process. Idempotent — no-op on a clean shutdown.+ sweepOrphanedScratchState(handle);// Acquire exclusive writer lockacquireLock(shadowDir, projectRoot);@@ -478,25 +478,37 @@ async function commitWipInner(// ─── Per-writer fan-out helpers ────────────────────────────────────────────/**- * Stage the content directory and return a git tree SHA.- * Used by the per-writer L2 fan-out so all writers share the same tree.- * Uses a fresh index (no seeding from any ref) so the tree reflects- * current filesystem state of contentRoot.+ * Stable filename of the reusable fan-out index inside the shadow gitDir.+ * Reusing one index across drain cycles lets git's stat cache elide re-reading+ * and re-hashing unchanged files, so the per-drain cost of `buildWipTree` is a+ * stat walk instead of a whole-corpus re-hash. The name deliberately does NOT+ * match the `index-wip-fanout-<uuid>` throwaway pattern the boot sweep deletes:+ * the cache is valid across restarts (git re-validates every entry by stat).*/+const FANOUT_INDEX_NAME = 'index-wip-fanout';+/**- * Sweep orphaned `index-wip-fanout-*` files left in the shadow gitDir by a- * crashed `buildWipTree` call from a prior process. Each entry is a transient- * index scratch file; the owning process is always gone by the time we run- * (initShadowRepo acquires an exclusive writer lock immediately after), so- * unconditional deletion is safe.+ * Sweep transient scratch state left in the shadow gitDir by a crashed+ * process: `index-wip-fanout-<uuid>` throwaway scratch files, a stale+ * `index-wip-fanout.lock` from a `git add` killed mid-write (git would refuse+ * to reuse the persistent fan-out index while the lock exists), and+ * `tmp-park-blobs-<uuid>` blob scratch directories from a park killed before+ * its `finally` cleanup ran. The owning process is always gone by the time we+ * run (initShadowRepo acquires an exclusive writer lock immediately after),+ * so unconditional deletion is safe. The persistent `index-wip-fanout` file+ * itself is kept — it is a pure stat cache whose entries git re-validates on+ * every use.*/-function sweepOrphanedTmpIndexFiles(shadow: ShadowHandle): number {+function sweepOrphanedScratchState(shadow: ShadowHandle): number {let deleted = 0;try {for (const name of readdirSync(shadow.gitDir)) {- if (!name.startsWith('index-wip-fanout-')) continue;+ const isThrowaway = name.startsWith(`${FANOUT_INDEX_NAME}-`);+ const isStaleLock = name === `${FANOUT_INDEX_NAME}.lock`;+ const isParkScratchDir = name.startsWith('tmp-park-blobs-');+ if (!isThrowaway && !isStaleLock && !isParkScratchDir) continue;try {- rmSync(resolve(shadow.gitDir, name));+ rmSync(resolve(shadow.gitDir, name), { recursive: true, force: true });deleted++;} catch {// best effort — next startup will retry@@ -518,27 +530,72 @@ export async function buildWipTree(shadow: ShadowHandle, contentRoot: string): Preturn shadowOpGateFor(shadow).withMutator(() => buildWipTreeInner(shadow, contentRoot));}-async function buildWipTreeInner(shadow: ShadowHandle, contentRoot: string): Promise<string> {- const tmpIndex = resolve(shadow.gitDir, `index-wip-fanout-${randomUUID()}`);+/**+ * Stage the content directory into `indexFile` and return a git tree SHA.+ * Never seeded from any ref: a directory-pathspec `git add` fully syncs the+ * index subtree to the working tree (additions, modifications, AND deletions),+ * so the resulting tree always reflects current filesystem state of+ * contentRoot regardless of what the index held before.+ */+async function buildWipTreeWithIndex(+ shadow: ShadowHandle,+ contentRoot: string,+ indexFile: string,+): Promise<string> {const sg = shadowGit(shadow);const gitPathspec = contentRoot || '.';+ await sg+ .env({+ GIT_DIR: shadow.gitDir,+ GIT_WORK_TREE: shadow.workTree,+ GIT_INDEX_FILE: indexFile,+ })+ .raw('add', gitPathspec);+ return (+ await sg.env({ GIT_DIR: shadow.gitDir, GIT_INDEX_FILE: indexFile }).raw('write-tree')+ ).trim();+}+/**+ * Stage the content directory and return a git tree SHA.+ * Used by the per-writer L2 fan-out so all writers share the same tree.+ *+ * Fast path: reuse the persistent fan-out index so unchanged files hit git's+ * stat cache instead of being re-read and re-hashed every cycle (the previous+ * throwaway-index design made every drain O(total corpus bytes)). Safe to+ * share a single index because the L2 commit path is serialized (one commit+ * in flight per server; one server per contentDir via the writer lock).+ *+ * Fallback: on ANY failure (stale `index.lock` from a timed-out add, corrupt+ * index file, …) drop the persistent index and rebuild from a fresh throwaway+ * index — the pre-optimization behavior. Recorded history is identical either+ * way; only the caching differs.+ */+async function buildWipTreeInner(shadow: ShadowHandle, contentRoot: string): Promise<string> {+ const persistentIndex = resolve(shadow.gitDir, FANOUT_INDEX_NAME);try {- await sg- .env({- GIT_DIR: shadow.gitDir,- GIT_WORK_TREE: shadow.workTree,- GIT_INDEX_FILE: tmpIndex,- })- .raw('add', gitPathspec);- return (- await sg.env({ GIT_DIR: shadow.gitDir, GIT_INDEX_FILE: tmpIndex }).raw('write-tree')- ).trim();- } finally {+ return await buildWipTreeWithIndex(shadow, contentRoot, persistentIndex);+ } catch (e) {+ console.warn(+ '[shadow-repo] persistent fan-out index failed — rebuilding from a fresh index:',+ e,+ );+ for (const stale of [persistentIndex, `${persistentIndex}.lock`]) {+ try {+ rmSync(stale);+ } catch {+ // ignore cleanup failure+ }+ }+ const tmpIndex = resolve(shadow.gitDir, `${FANOUT_INDEX_NAME}-${randomUUID()}`);try {- rmSync(tmpIndex);- } catch {- // ignore cleanup failure+ return await buildWipTreeWithIndex(shadow, contentRoot, tmpIndex);+ } finally {+ try {+ rmSync(tmpIndex);+ } catch {+ // ignore cleanup failure+ }}}}@@ -1330,6 +1387,13 @@ export async function parkBranch();}+/**+ * Max blob/index entries per git invocation in the park batch. Keeps argv+ * comfortably under platform limits (200 absolute paths or `--cacheinfo`+ * tuples is a few tens of KB vs. the ~1 MB macOS/Linux ARG_MAX floor).+ */+const PARK_BATCH_CHUNK = 200;+async function parkBranchInner(shadow: ShadowHandle,branch: string,@@ -1341,31 +1405,58 @@ async function parkBranchInner(const tmpIndex = resolve(shadow.gitDir, `index-park-${branch.replace(/\//g, '-')}`);const ref = `refs/wip/${branch}/${writerId}`;- const tmpBlobFile = resolve(shadow.gitDir, 'tmp-park-blob');+ const tmpBlobDir = resolve(shadow.gitDir, `tmp-park-blobs-${randomUUID()}`);try {- // Build a tree with both Y.Doc state and disk snapshots- for (const doc of documents) {- // Store Y.Doc state at the doc's path- tracedWriteFileSync(tmpBlobFile, doc.markdown, 'utf-8');- const blobSha = (- await sg- .env({ GIT_DIR: shadow.gitDir, GIT_INDEX_FILE: tmpIndex })- .raw('hash-object', '-w', tmpBlobFile)- ).trim();- await sg- .env({ GIT_DIR: shadow.gitDir, GIT_INDEX_FILE: tmpIndex })- .raw('update-index', '--add', '--cacheinfo', `100644,${blobSha},${doc.docName}`);-- // Store disk snapshot at .park-base/<docName>- tracedWriteFileSync(tmpBlobFile, doc.diskSnapshot, 'utf-8');- const baseSha = (- await sg- .env({ GIT_DIR: shadow.gitDir, GIT_INDEX_FILE: tmpIndex })- .raw('hash-object', '-w', tmpBlobFile)- ).trim();+ // Build a tree with both Y.Doc state and disk snapshots. Write every blob+ // to a scratch dir first, then hash and stage them in chunked batches:+ // park runs `await`-ed on the branch-switch critical path with all writes+ // gated, and the previous per-doc form (2× hash-object + 2× update-index,+ // sequentially awaited) cost 4 git spawns per open doc — ~15s for a+ // 100-doc park. Batching collapses the spawn count to ~2 per 100 docs.+ tracedMkdirSync(tmpBlobDir, { recursive: true });+ const blobFiles: string[] = [];+ const treePaths: string[] = [];+ for (let i = 0; i < documents.length; i++) {+ const doc = documents[i];+ if (!doc) continue;+ const stateFile = resolve(tmpBlobDir, `${i}-state`);+ tracedWriteFileSync(stateFile, doc.markdown, 'utf-8');+ blobFiles.push(stateFile);+ treePaths.push(doc.docName);+ const baseFile = resolve(tmpBlobDir, `${i}-base`);+ tracedWriteFileSync(baseFile, doc.diskSnapshot, 'utf-8');+ blobFiles.push(baseFile);+ treePaths.push(`.park-base/${doc.docName}`);+ }++ // `git hash-object -w` accepts multiple files and prints one object id per+ // line in argument order, so blobShas[i] pairs with treePaths[i].+ const blobShas: string[] = [];+ for (let i = 0; i < blobFiles.length; i += PARK_BATCH_CHUNK) {+ const chunk = blobFiles.slice(i, i + PARK_BATCH_CHUNK);+ const out = await sg.env({ GIT_DIR: shadow.gitDir }).raw('hash-object', '-w', '--', ...chunk);+ const shas = out+ .trim()+ .split('\n')+ .map((s) => s.trim())+ .filter(Boolean);+ if (shas.length !== chunk.length || shas.some((s) => !/^[0-9a-f]{40}$/.test(s))) {+ throw new Error(+ `[shadow] park hash-object returned ${shas.length} ids for ${chunk.length} blobs`,+ );+ }+ blobShas.push(...shas);+ }++ for (let i = 0; i < treePaths.length; i += PARK_BATCH_CHUNK) {+ const cacheinfoArgs: string[] = [];+ const end = Math.min(i + PARK_BATCH_CHUNK, treePaths.length);+ for (let j = i; j < end; j++) {+ cacheinfoArgs.push('--cacheinfo', `100644,${blobShas[j]},${treePaths[j]}`);+ }await sg.env({ GIT_DIR: shadow.gitDir, GIT_INDEX_FILE: tmpIndex })- .raw('update-index', '--add', '--cacheinfo', `100644,${baseSha},.park-base/${doc.docName}`);+ .raw('update-index', '--add', ...cacheinfoArgs);}const treeSha = (@@ -1418,7 +1509,7 @@ async function parkBranchInner(// ignore cleanup failure}try {- rmSync(tmpBlobFile);+ rmSync(tmpBlobDir, { recursive: true, force: true });} catch {// ignore cleanup failure}