Skip to content

Beautiful, AI-native markdown IDE and LLM wiki

perf(server): cut shadow-repo write-path cost on large KBs (#2737)

Andrew Mikofalvy789ed92967a7
diff --git a/.changeset/shadow-repo-write-perf.md b/.changeset/shadow-repo-write-perf.md
new file mode 100644
index 000000000..6f2d1c9ed
--- /dev/null
+++ b/.changeset/shadow-repo-write-perf.md
@@ -0,0 +1,5 @@
+---
+"@inkeep/open-knowledge": patch
+---
+
+Version-history recording is dramatically cheaper on large knowledge bases and slow (mounted/network) volumes. The background history snapshot now reuses a persistent git index so unchanged files are no longer re-read and re-hashed on every save cycle (up to ~11x faster tree builds on a 10k-file KB); switching git branches with many open documents batches its bookkeeping into a few git calls instead of four per document (a 100-document switch drops from ~15s to well under a second); and agent writes coalesce into the normal debounced history commit instead of forcing a full commit per write, while history reads still always reflect every completed write.
diff --git a/packages/app/tests/integration/agent-write-commit-coalescing.test.ts b/packages/app/tests/integration/agent-write-commit-coalescing.test.ts
new file mode 100644
index 000000000..86a869a44
--- /dev/null
+++ b/packages/app/tests/integration/agent-write-commit-coalescing.test.ts
@@ -0,0 +1,86 @@
+/**
+ * Agent writes ride the L2 commit debounce instead of forcing a shadow commit
+ * per write, and `/api/history` drains any pending commit before reading so
+ * the debounce never costs read-your-writes.
+ *
+ * Pins the two halves of the contract:
+ * 1. Coalescing — successive agent writes inside one debounce window produce
+ * NO shadow commit on their own (the per-write forced flush is gone).
+ * 2. Read-your-writes — a `/api/history` read issued right after a write
+ * lists that write, because the handler flushes the pending commit first.
+ *
+ * Uses a deliberately huge `commitDebounceMs` so the debounce timer cannot
+ * fire during the test — any commit observed before the history read would be
+ * a regression back to per-write flushing.
+ */
+
+import { execFileSync } from 'node:child_process';
+import { resolveShadowDir } from '@inkeep/open-knowledge-core/shadow-repo-layout';
+import { afterEach, describe, expect, test } from 'vitest';
+import { agentWriteMd, createTestServer, type TestServer } from './test-harness';
+
+interface TimelineResponse {
+ entries: Array<{ sha: string; type: string; message: string }>;
+}
+
+let server: TestServer | undefined;
+
+afterEach(async () => {
+ await server?.cleanup();
+ server = undefined;
+});
+
+function listWipRefs(contentDir: string): string[] {
+ const shadowDir = resolveShadowDir(contentDir);
+ const raw = execFileSync('git', ['for-each-ref', '--format=%(refname)', 'refs/wip/'], {
+ env: { ...process.env, GIT_DIR: shadowDir },
+ encoding: 'utf-8',
+ });
+ return raw.trim().split('\n').filter(Boolean);
+}
+
+describe('agent-write L2 commit coalescing', () => {
+ test('writes coalesce on the debounce; /api/history flushes before reading', async () => {
+ server = await createTestServer({ gitEnabled: true, commitDebounceMs: 600_000 });
+
+ await agentWriteMd(server.port, '# Doc A\n', {
+ docName: 'coalesce-a',
+ position: 'replace',
+ agentId: 'coalesce-writer',
+ agentName: 'Coalesce Writer',
+ });
+ await agentWriteMd(server.port, '# Doc B\n', {
+ docName: 'coalesce-b',
+ position: 'replace',
+ agentId: 'coalesce-writer',
+ agentName: 'Coalesce Writer',
+ });
+
+ // Both writes are on disk (the handler awaits the L1 store), but neither
+ // forced an L2 shadow commit: the wip namespace is still empty.
+ expect(listWipRefs(server.contentDir)).toEqual([]);
+
+ // A history read right after the writes must still list them — the
+ // handler drains the pending commit before querying.
+ const res = await fetch(
+ `${server.baseUrl}/api/history?docName=${encodeURIComponent('coalesce-a')}`,
+ );
+ expect(res.ok).toBe(true);
+ const body = (await res.json()) as TimelineResponse;
+ expect(body.entries.length).toBeGreaterThanOrEqual(1);
+ expect(body.entries.some((e) => e.type === 'wip')).toBe(true);
+
+ // The read's flush drained BOTH pending writes into one coalesced drain:
+ // the second doc is queryable without any further write or flush, and the
+ // writer advanced a single wip ref.
+ const resB = await fetch(
+ `${server.baseUrl}/api/history?docName=${encodeURIComponent('coalesce-b')}`,
+ );
+ expect(resB.ok).toBe(true);
+ const bodyB = (await resB.json()) as TimelineResponse;
+ expect(bodyB.entries.some((e) => e.type === 'wip')).toBe(true);
+
+ const refs = listWipRefs(server.contentDir);
+ expect(refs.length).toBe(1);
+ }, 30_000);
+});
diff --git a/packages/app/tests/integration/test-harness.ts b/packages/app/tests/integration/test-harness.ts
index b802082c5..b07a4bd2f 100644
--- a/packages/app/tests/integration/test-harness.ts
+++ b/packages/app/tests/integration/test-harness.ts
@@ -1005,18 +1005,18 @@ export async function awaitBacklinkIndexed(
* return their shas (newest-first, as `/api/history` orders them — so the oldest
* is `shas[shas.length - 1]`).
*
- * Agent-write handlers fire the L1 store → L2 git-commit chain FIRE-AND-FORGET
- * (`flushDocToGit` in api-extension.ts: the HTTP response returns before the
- * commit is durable). A bare `pollUntil` on `/api/history` therefore RACES the
- * commit against a wall-clock budget — and under merge-queue contention the
- * serial git-subprocess chain (global one-commit-in-flight mutex) blows any fixed
- * budget. This helper AWAITS the commit instead: each iteration POSTs
- * `/api/test-flush-git` (drains the pending L2 debounce timer + any in-flight
- * commit), then checks. Flush every iteration — the first call can land before
- * the fire-and-forget chain has even scheduled L2. The budget is a generous
- * ceiling, not the thing being raced. Mirrors `rename-history.test.ts`'s
- * `awaitWipCommit`; both close the same flake class (the version-rollback case
- * is #1596).
+ * Agent-write handlers leave the L2 git commit to the persistence debounce
+ * (the HTTP response returns before any commit is durable; rename/rollback/
+ * undo additionally fire `flushDocToGit` fire-and-forget). A bare `pollUntil`
+ * on `/api/history` therefore RACES the commit against a wall-clock budget —
+ * and under merge-queue contention the serial git-subprocess chain (global
+ * one-commit-in-flight mutex) blows any fixed budget. This helper AWAITS the
+ * commit instead: each iteration POSTs `/api/test-flush-git` (drains the
+ * pending L2 debounce timer + any in-flight commit), then checks. Flush every
+ * iteration — the first call can land before the store path has even armed the
+ * L2 timer. The budget is a generous ceiling, not the thing being raced.
+ * Mirrors `rename-history.test.ts`'s `awaitWipCommit`; both close the same
+ * flake class (the version-rollback case is #1596).
*/
export async function awaitWipCommits(
server: TestServer,
diff --git a/packages/server/src/api-extension.ts b/packages/server/src/api-extension.ts
index add978b97..14d251402 100644
--- a/packages/server/src/api-extension.ts
+++ b/packages/server/src/api-extension.ts
@@ -3243,6 +3243,14 @@ export function createApiExtension(options: ApiExtensionOptions): Extension {
* L2 (disk → git): chained after L1 resolves to guarantee disk content is
* up-to-date before the shadow-repo commit.
*
+ * Reserved for rare, semantically anchored actions (rename, rollback, undo)
+ * whose commit must land promptly — a rename's log entries stay unanchored
+ * until its shadow commit exists. High-frequency agent write handlers use
+ * {@link flushDocToDisk} instead so their L2 commits coalesce on the
+ * persistence debounce rather than paying a whole-corpus tree build per
+ * write; `handleHistory` drains any pending commit before reading, so
+ * history reads still observe every completed write.
+ *
* The returned promise is intentionally not awaited by callers — the HTTP
* response fires immediately after the CRDT transaction; persistence is
* best-effort background work.
@@ -3257,6 +3265,21 @@ export function createApiExtension(options: ApiExtensionOptions): Extension {
});
}
+ /**
+ * Fire-and-forget L1-only flush: force the per-document disk store without
+ * forcing the L2 shadow commit. The store path always arms the L2 debounce
+ * timer, so the commit still lands — coalesced with every other write in the
+ * window instead of one full tree build per agent write. No-op when the
+ * handler already awaited `flushDiskAndDetectOutcome` (nothing debounced).
+ */
+ function flushDocToDisk(docName: string, label: string): void {
+ const debounceId = `onStoreDocument-${docName}`;
+ if (!hocuspocus.debouncer.isDebounced(debounceId)) return;
+ hocuspocus.debouncer.executeNow(debounceId).catch((err: unknown) => {
+ log.warn({ err }, `[${label}] post-write disk flush failed`);
+ });
+ }
+
/**
* Force the debounced L1 disk store for `docName` to run now and await it,
* then report whether it failed. Hocuspocus's `storeDocumentHooks` swallows
@@ -5123,7 +5146,7 @@ export function createApiExtension(options: ApiExtensionOptions): Extension {
respondDiskDivergence(res, 'agent-write');
return;
}
- flushDocToGit(docName, 'agent-write');
+ flushDocToDisk(docName, 'agent-write');
onAgentWrite?.();
// Success body is flat — no `{ ok: true }` wrapper. Clients
@@ -5332,7 +5355,7 @@ export function createApiExtension(options: ApiExtensionOptions): Extension {
return;
}
- flushDocToGit(resolvedDocName, 'agent-write-md');
+ flushDocToDisk(resolvedDocName, 'agent-write-md');
// Focus (attribution) on __system__ awareness. Focus drives browser
// push-navigation to the doc the agent just wrote (writeKind); presence
@@ -5704,7 +5727,7 @@ export function createApiExtension(options: ApiExtensionOptions): Extension {
return;
}
}
- flushDocToGit(resolvedDocName, 'frontmatter-patch');
+ flushDocToDisk(resolvedDocName, 'frontmatter-patch');
}
agentFocusBroadcaster?.setFocus(agentId, {
@@ -7148,7 +7171,7 @@ export function createApiExtension(options: ApiExtensionOptions): Extension {
return;
}
- flushDocToGit(docName, 'agent-patch');
+ flushDocToDisk(docName, 'agent-patch');
// Focus (attribution) on __system__ awareness. Presence is separately
// maintained via setPresence/touchMode pairs above.
@@ -7603,15 +7626,16 @@ export function createApiExtension(options: ApiExtensionOptions): Extension {
/**
* POST /api/test-flush-git — await the L2 git-commit pipeline to settle.
*
- * Agent-write handlers fire `flushDocToGit` FIRE-AND-FORGET, so a test
- * that needs the WIP commit durable can only poll the timeline against a
- * wall-clock budget — and under CI load the serial git-subprocess chain
- * (global one-commit-in-flight mutex in persistence.ts) blows any fixed
- * budget. This route lets tests AWAIT the
- * actual commit completion instead of racing it: it drains the pending
- * L2 debounce timer and any in-flight commit before responding. Callers
- * should flush-then-check inside their poll loop — the fire-and-forget
- * chain may not have scheduled L2 yet on the first iteration.
+ * Agent-write handlers fire `flushDocToDisk` FIRE-AND-FORGET and leave the
+ * L2 shadow commit to the persistence debounce, so a test that needs the
+ * WIP commit durable can only poll the timeline against a wall-clock
+ * budget — and under CI load the serial git-subprocess chain (global
+ * one-commit-in-flight mutex in persistence.ts) blows any fixed budget.
+ * This route lets tests AWAIT the actual commit completion instead of
+ * racing it: it drains the pending L2 debounce timer and any in-flight
+ * commit before responding. Callers should flush-then-check inside their
+ * poll loop — the fire-and-forget chain may not have scheduled L2 yet on
+ * the first iteration.
*/
const handleTestFlushGit = withValidation(
EmptyRequestSchema,
@@ -8017,6 +8041,22 @@ export function createApiExtension(options: ApiExtensionOptions): Extension {
return;
}
+ // Read-your-writes: agent write handlers no longer force an L2 shadow
+ // commit per write (they ride the persistence debounce), so drain any
+ // pending commit before querying — a `history` call issued right after a
+ // write must list that write. No-op when nothing is pending. The flush
+ // blocks the response, so surface slow (cold-index) drains in the logs.
+ try {
+ const flushStart = performance.now();
+ await flushGitCommit?.();
+ const flushMs = performance.now() - flushStart;
+ if (flushMs > 1000) {
+ log.warn({ durationMs: Math.round(flushMs) }, '[history] pre-read commit flush slow');
+ }
+ } catch (err) {
+ log.warn({ err }, '[history] pre-read commit flush failed');
+ }
+
const url = new URL(req.url ?? '/', `http://${req.headers.host ?? 'localhost'}`);
const docName = url.searchParams.get('docName') ?? '';
const folderParam = url.searchParams.get('folder');
@@ -8469,10 +8509,11 @@ export function createApiExtension(options: ApiExtensionOptions): Extension {
// Force-flush L1 (onStoreDocument debounce) then L2 (git commit) so the
// restored version + attribution appear in the timeline within ~100ms
- // rather than waiting for the natural ~4s L1+L2 debounce stack. Uses
- // the shared `flushDocToGit` helper (same pattern as the three
- // agent-write handlers) rather than a raw `flushGitCommit()` which
- // no-ops when no L2 timer is set yet.
+ // rather than waiting for the natural ~4s L1+L2 debounce stack.
+ // Rollback intentionally keeps the prompt `flushDocToGit` helper —
+ // unlike the high-frequency agent-write handlers, which use
+ // `flushDocToDisk` and let the commit ride the debounce — rather than
+ // a raw `flushGitCommit()` which no-ops when no L2 timer is set yet.
// Await the L1 disk store so a swallowed persistence failure surfaces
// as an error instead of a false success. Mirrors agent-write-md.
const flushOutcome = await flushDiskAndDetectOutcome(docName);
@@ -18449,7 +18490,7 @@ export function createApiExtension(options: ApiExtensionOptions): Extension {
respondDiskDivergence(res, 'lint-fix');
return;
}
- flushDocToGit(resolvedDocName, 'lint-fix');
+ flushDocToDisk(resolvedDocName, 'lint-fix');
// Remaining problems: re-lint the ACTUAL post-write source (bridge
// normalization can shift bytes vs `fixed`), reusing the resolved cfg.
diff --git a/packages/server/src/shadow-repo.test.ts b/packages/server/src/shadow-repo.test.ts
index dbcd1fd8a..05ab745df 100644
--- a/packages/server/src/shadow-repo.test.ts
+++ b/packages/server/src/shadow-repo.test.ts
@@ -1,4 +1,3 @@
-import { afterEach, beforeEach, describe, expect, spyOn, test } from 'bun:test';
import { existsSync, mkdirSync, rmSync, writeFileSync } from 'node:fs';
import { mkdtemp, rm } from 'node:fs/promises';
import { tmpdir } from 'node:os';
@@ -10,6 +9,7 @@ import {
parseOkActor,
} from '@inkeep/open-knowledge-core/shadow-repo-layout';
import simpleGit from 'simple-git';
+import { afterEach, beforeEach, describe, expect, test, vi } from 'vitest';
import {
buildWipTree,
commitUpstreamImport,
@@ -145,7 +145,7 @@ describe('initShadowRepo', () => {
writeFileSync(resolve(legacyDir, 'LEGACY_SENTINEL'), 'legacy');
writeFileSync(resolve(newDir, 'NEW_SENTINEL'), 'new');
- const warnSpy = spyOn(console, 'warn').mockImplementation(() => {});
+ const warnSpy = vi.spyOn(console, 'warn').mockImplementation(() => {});
try {
await initShadowRepo(projectRoot);
@@ -185,6 +185,104 @@ describe('buildWipTree contentRoot pathspec', () => {
});
});
+describe('buildWipTree persistent fan-out index', () => {
+ let projectRoot: string;
+ let shadow: ShadowHandle;
+ let contentDir: string;
+
+ beforeEach(async () => {
+ projectRoot = resolve(tmpDir, 'project');
+ contentDir = resolve(projectRoot, 'content');
+ mkdirSync(contentDir, { recursive: true });
+ const git = simpleGit(projectRoot);
+ await git.init();
+ shadow = await initShadowRepo(projectRoot);
+ });
+
+ test('reused index tracks modify, add, and delete across successive builds', async () => {
+ writeFileSync(resolve(contentDir, 'a.md'), '# A v1\n');
+ writeFileSync(resolve(contentDir, 'b.md'), '# B v1\n');
+ const tree1 = await buildWipTree(shadow, 'content');
+ // Index survives the call — this is the reuse the stat cache depends on.
+ expect(existsSync(resolve(shadow.gitDir, 'index-wip-fanout'))).toBe(true);
+
+ writeFileSync(resolve(contentDir, 'a.md'), '# A v2\n');
+ rmSync(resolve(contentDir, 'b.md'));
+ writeFileSync(resolve(contentDir, 'c.md'), '# C v1\n');
+ const tree2 = await buildWipTree(shadow, 'content');
+ expect(tree2).not.toBe(tree1);
+
+ const sg = shadowGit(shadow);
+ const listing = await sg.raw('ls-tree', '-r', '--name-only', tree2);
+ const paths = listing.trim().split('\n').sort();
+ expect(paths).toEqual(['content/a.md', 'content/c.md']);
+ expect((await sg.raw('cat-file', '-p', `${tree2}:content/a.md`)).trim()).toBe('# A v2');
+ });
+
+ test('reused index produces the same tree a fresh index would', async () => {
+ writeFileSync(resolve(contentDir, 'a.md'), '# A v1\n');
+ await buildWipTree(shadow, 'content');
+ writeFileSync(resolve(contentDir, 'a.md'), '# A v2\n');
+ const warmTree = await buildWipTree(shadow, 'content');
+
+ // Fresh-index reference build of the same working tree state.
+ rmSync(resolve(shadow.gitDir, 'index-wip-fanout'));
+ const freshTree = await buildWipTree(shadow, 'content');
+ expect(warmTree).toBe(freshTree);
+ });
+
+ test('corrupt persistent index falls back to a fresh rebuild', async () => {
+ writeFileSync(resolve(contentDir, 'a.md'), '# A v1\n');
+ const tree1 = await buildWipTree(shadow, 'content');
+
+ writeFileSync(resolve(shadow.gitDir, 'index-wip-fanout'), 'not a git index');
+ const warnSpy = vi.spyOn(console, 'warn').mockImplementation(() => {});
+ try {
+ const tree2 = await buildWipTree(shadow, 'content');
+ expect(tree2).toBe(tree1);
+ const warnings = warnSpy.mock.calls.map((call) => String(call[0] ?? ''));
+ expect(warnings.some((w) => w.includes('persistent fan-out index failed'))).toBe(true);
+ } finally {
+ warnSpy.mockRestore();
+ }
+ // Corrupt cache was dropped so the next cycle starts clean.
+ expect(existsSync(resolve(shadow.gitDir, 'index-wip-fanout'))).toBe(false);
+ });
+
+ test('stale index.lock from a killed add falls back and is cleared', async () => {
+ writeFileSync(resolve(contentDir, 'a.md'), '# A v1\n');
+ const tree1 = await buildWipTree(shadow, 'content');
+
+ writeFileSync(resolve(shadow.gitDir, 'index-wip-fanout.lock'), '');
+ const warnSpy = vi.spyOn(console, 'warn').mockImplementation(() => {});
+ try {
+ const tree2 = await buildWipTree(shadow, 'content');
+ expect(tree2).toBe(tree1);
+ const warnings = warnSpy.mock.calls.map((call) => String(call[0] ?? ''));
+ expect(warnings.some((w) => w.includes('persistent fan-out index failed'))).toBe(true);
+ } finally {
+ warnSpy.mockRestore();
+ }
+ expect(existsSync(resolve(shadow.gitDir, 'index-wip-fanout.lock'))).toBe(false);
+ });
+
+ test('boot sweep removes throwaway indices, stale lock, and orphaned park blob dirs but keeps the cache', async () => {
+ writeFileSync(resolve(shadow.gitDir, 'index-wip-fanout-deadbeef'), '');
+ writeFileSync(resolve(shadow.gitDir, 'index-wip-fanout.lock'), '');
+ writeFileSync(resolve(shadow.gitDir, 'index-wip-fanout'), '');
+ const orphanParkDir = resolve(shadow.gitDir, 'tmp-park-blobs-deadbeef');
+ mkdirSync(orphanParkDir, { recursive: true });
+ writeFileSync(resolve(orphanParkDir, '0-state'), '# orphaned park blob\n');
+
+ // Re-init on the same project runs the sweep again.
+ const reinit = await initShadowRepo(projectRoot);
+ expect(existsSync(resolve(reinit.gitDir, 'index-wip-fanout-deadbeef'))).toBe(false);
+ expect(existsSync(resolve(reinit.gitDir, 'index-wip-fanout.lock'))).toBe(false);
+ expect(existsSync(orphanParkDir)).toBe(false);
+ expect(existsSync(resolve(reinit.gitDir, 'index-wip-fanout'))).toBe(true);
+ });
+});
+
describe('commitWip', () => {
let projectRoot: string;
let shadow: ShadowHandle;
@@ -526,6 +624,28 @@ describe('parkBranch', () => {
expect(guideContent).toBe('# Guide');
});
+ test('parks enough documents to span multiple hash/stage batches', async () => {
+ // 120 docs → 240 blobs/index entries, crossing the 200-entry chunk
+ // boundary so both the multi-chunk hash-object and update-index legs run.
+ const docs: ParkableDoc[] = Array.from({ length: 120 }, (_, i) => ({
+ docName: `notes/doc-${i}`,
+ markdown: `# Doc ${i} memory\n`,
+ diskSnapshot: `# Doc ${i} disk\n`,
+ }));
+
+ const sha = await parkBranch(shadow, 'main', SERVICE_WRITER.id, docs);
+ expect(sha).toHaveLength(40);
+
+ const sg = shadowGit(shadow);
+ // Spot-check first, chunk-straddling, and last entries on both tree sides.
+ for (const i of [0, 99, 100, 119]) {
+ const content = (await sg.raw('show', `${sha}:notes/doc-${i}`)).trim();
+ expect(content).toBe(`# Doc ${i} memory`);
+ const base = (await sg.raw('show', `${sha}:.park-base/notes/doc-${i}`)).trim();
+ expect(base).toBe(`# Doc ${i} disk`);
+ }
+ });
+
test('isPairedWriteOrigin(PARK_SNAPSHOT_ORIGIN) returns true (US-017)', () => {
// Import from standalone — verify paired: true is recognized
const origin = {
diff --git a/packages/server/src/shadow-repo.ts b/packages/server/src/shadow-repo.ts
index 14e96ab69..3c5641904 100644
--- a/packages/server/src/shadow-repo.ts
+++ b/packages/server/src/shadow-repo.ts
@@ -257,9 +257,9 @@ export async function initShadowRepo(projectRoot: string): Promise<ShadowHandle>
// Idempotent — no-op once all legacy refs are gone.
await sweepLegacyShadowRefs(handle);
- // Sweep orphaned temp-index files left behind by crashed `buildWipTree`
- // calls from a prior process. Idempotent — no-op on a clean shutdown.
- sweepOrphanedTmpIndexFiles(handle);
+ // Sweep orphaned scratch state (temp indices, park blob dirs) left behind
+ // by a crashed prior process. Idempotent — no-op on a clean shutdown.
+ sweepOrphanedScratchState(handle);
// Acquire exclusive writer lock
acquireLock(shadowDir, projectRoot);
@@ -478,25 +478,37 @@ async function commitWipInner(
// ─── Per-writer fan-out helpers ────────────────────────────────────────────
/**
- * Stage the content directory and return a git tree SHA.
- * Used by the per-writer L2 fan-out so all writers share the same tree.
- * Uses a fresh index (no seeding from any ref) so the tree reflects
- * current filesystem state of contentRoot.
+ * Stable filename of the reusable fan-out index inside the shadow gitDir.
+ * Reusing one index across drain cycles lets git's stat cache elide re-reading
+ * and re-hashing unchanged files, so the per-drain cost of `buildWipTree` is a
+ * stat walk instead of a whole-corpus re-hash. The name deliberately does NOT
+ * match the `index-wip-fanout-<uuid>` throwaway pattern the boot sweep deletes:
+ * the cache is valid across restarts (git re-validates every entry by stat).
*/
+const FANOUT_INDEX_NAME = 'index-wip-fanout';
+
/**
- * Sweep orphaned `index-wip-fanout-*` files left in the shadow gitDir by a
- * crashed `buildWipTree` call from a prior process. Each entry is a transient
- * index scratch file; the owning process is always gone by the time we run
- * (initShadowRepo acquires an exclusive writer lock immediately after), so
- * unconditional deletion is safe.
+ * Sweep transient scratch state left in the shadow gitDir by a crashed
+ * process: `index-wip-fanout-<uuid>` throwaway scratch files, a stale
+ * `index-wip-fanout.lock` from a `git add` killed mid-write (git would refuse
+ * to reuse the persistent fan-out index while the lock exists), and
+ * `tmp-park-blobs-<uuid>` blob scratch directories from a park killed before
+ * its `finally` cleanup ran. The owning process is always gone by the time we
+ * run (initShadowRepo acquires an exclusive writer lock immediately after),
+ * so unconditional deletion is safe. The persistent `index-wip-fanout` file
+ * itself is kept — it is a pure stat cache whose entries git re-validates on
+ * every use.
*/
-function sweepOrphanedTmpIndexFiles(shadow: ShadowHandle): number {
+function sweepOrphanedScratchState(shadow: ShadowHandle): number {
let deleted = 0;
try {
for (const name of readdirSync(shadow.gitDir)) {
- if (!name.startsWith('index-wip-fanout-')) continue;
+ const isThrowaway = name.startsWith(`${FANOUT_INDEX_NAME}-`);
+ const isStaleLock = name === `${FANOUT_INDEX_NAME}.lock`;
+ const isParkScratchDir = name.startsWith('tmp-park-blobs-');
+ if (!isThrowaway && !isStaleLock && !isParkScratchDir) continue;
try {
- rmSync(resolve(shadow.gitDir, name));
+ rmSync(resolve(shadow.gitDir, name), { recursive: true, force: true });
deleted++;
} catch {
// best effort — next startup will retry
@@ -518,27 +530,72 @@ export async function buildWipTree(shadow: ShadowHandle, contentRoot: string): P
return shadowOpGateFor(shadow).withMutator(() => buildWipTreeInner(shadow, contentRoot));
}
-async function buildWipTreeInner(shadow: ShadowHandle, contentRoot: string): Promise<string> {
- const tmpIndex = resolve(shadow.gitDir, `index-wip-fanout-${randomUUID()}`);
+/**
+ * Stage the content directory into `indexFile` and return a git tree SHA.
+ * Never seeded from any ref: a directory-pathspec `git add` fully syncs the
+ * index subtree to the working tree (additions, modifications, AND deletions),
+ * so the resulting tree always reflects current filesystem state of
+ * contentRoot regardless of what the index held before.
+ */
+async function buildWipTreeWithIndex(
+ shadow: ShadowHandle,
+ contentRoot: string,
+ indexFile: string,
+): Promise<string> {
const sg = shadowGit(shadow);
const gitPathspec = contentRoot || '.';
+ await sg
+ .env({
+ GIT_DIR: shadow.gitDir,
+ GIT_WORK_TREE: shadow.workTree,
+ GIT_INDEX_FILE: indexFile,
+ })
+ .raw('add', gitPathspec);
+ return (
+ await sg.env({ GIT_DIR: shadow.gitDir, GIT_INDEX_FILE: indexFile }).raw('write-tree')
+ ).trim();
+}
+/**
+ * Stage the content directory and return a git tree SHA.
+ * Used by the per-writer L2 fan-out so all writers share the same tree.
+ *
+ * Fast path: reuse the persistent fan-out index so unchanged files hit git's
+ * stat cache instead of being re-read and re-hashed every cycle (the previous
+ * throwaway-index design made every drain O(total corpus bytes)). Safe to
+ * share a single index because the L2 commit path is serialized (one commit
+ * in flight per server; one server per contentDir via the writer lock).
+ *
+ * Fallback: on ANY failure (stale `index.lock` from a timed-out add, corrupt
+ * index file, …) drop the persistent index and rebuild from a fresh throwaway
+ * index — the pre-optimization behavior. Recorded history is identical either
+ * way; only the caching differs.
+ */
+async function buildWipTreeInner(shadow: ShadowHandle, contentRoot: string): Promise<string> {
+ const persistentIndex = resolve(shadow.gitDir, FANOUT_INDEX_NAME);
try {
- await sg
- .env({
- GIT_DIR: shadow.gitDir,
- GIT_WORK_TREE: shadow.workTree,
- GIT_INDEX_FILE: tmpIndex,
- })
- .raw('add', gitPathspec);
- return (
- await sg.env({ GIT_DIR: shadow.gitDir, GIT_INDEX_FILE: tmpIndex }).raw('write-tree')
- ).trim();
- } finally {
+ return await buildWipTreeWithIndex(shadow, contentRoot, persistentIndex);
+ } catch (e) {
+ console.warn(
+ '[shadow-repo] persistent fan-out index failed — rebuilding from a fresh index:',
+ e,
+ );
+ for (const stale of [persistentIndex, `${persistentIndex}.lock`]) {
+ try {
+ rmSync(stale);
+ } catch {
+ // ignore cleanup failure
+ }
+ }
+ const tmpIndex = resolve(shadow.gitDir, `${FANOUT_INDEX_NAME}-${randomUUID()}`);
try {
- rmSync(tmpIndex);
- } catch {
- // ignore cleanup failure
+ return await buildWipTreeWithIndex(shadow, contentRoot, tmpIndex);
+ } finally {
+ try {
+ rmSync(tmpIndex);
+ } catch {
+ // ignore cleanup failure
+ }
}
}
}
@@ -1330,6 +1387,13 @@ export async function parkBranch(
);
}
+/**
+ * Max blob/index entries per git invocation in the park batch. Keeps argv
+ * comfortably under platform limits (200 absolute paths or `--cacheinfo`
+ * tuples is a few tens of KB vs. the ~1 MB macOS/Linux ARG_MAX floor).
+ */
+const PARK_BATCH_CHUNK = 200;
+
async function parkBranchInner(
shadow: ShadowHandle,
branch: string,
@@ -1341,31 +1405,58 @@ async function parkBranchInner(
const tmpIndex = resolve(shadow.gitDir, `index-park-${branch.replace(/\//g, '-')}`);
const ref = `refs/wip/${branch}/${writerId}`;
- const tmpBlobFile = resolve(shadow.gitDir, 'tmp-park-blob');
+ const tmpBlobDir = resolve(shadow.gitDir, `tmp-park-blobs-${randomUUID()}`);
try {
- // Build a tree with both Y.Doc state and disk snapshots
- for (const doc of documents) {
- // Store Y.Doc state at the doc's path
- tracedWriteFileSync(tmpBlobFile, doc.markdown, 'utf-8');
- const blobSha = (
- await sg
- .env({ GIT_DIR: shadow.gitDir, GIT_INDEX_FILE: tmpIndex })
- .raw('hash-object', '-w', tmpBlobFile)
- ).trim();
- await sg
- .env({ GIT_DIR: shadow.gitDir, GIT_INDEX_FILE: tmpIndex })
- .raw('update-index', '--add', '--cacheinfo', `100644,${blobSha},${doc.docName}`);
-
- // Store disk snapshot at .park-base/<docName>
- tracedWriteFileSync(tmpBlobFile, doc.diskSnapshot, 'utf-8');
- const baseSha = (
- await sg
- .env({ GIT_DIR: shadow.gitDir, GIT_INDEX_FILE: tmpIndex })
- .raw('hash-object', '-w', tmpBlobFile)
- ).trim();
+ // Build a tree with both Y.Doc state and disk snapshots. Write every blob
+ // to a scratch dir first, then hash and stage them in chunked batches:
+ // park runs `await`-ed on the branch-switch critical path with all writes
+ // gated, and the previous per-doc form (2× hash-object + 2× update-index,
+ // sequentially awaited) cost 4 git spawns per open doc — ~15s for a
+ // 100-doc park. Batching collapses the spawn count to ~2 per 100 docs.
+ tracedMkdirSync(tmpBlobDir, { recursive: true });
+ const blobFiles: string[] = [];
+ const treePaths: string[] = [];
+ for (let i = 0; i < documents.length; i++) {
+ const doc = documents[i];
+ if (!doc) continue;
+ const stateFile = resolve(tmpBlobDir, `${i}-state`);
+ tracedWriteFileSync(stateFile, doc.markdown, 'utf-8');
+ blobFiles.push(stateFile);
+ treePaths.push(doc.docName);
+ const baseFile = resolve(tmpBlobDir, `${i}-base`);
+ tracedWriteFileSync(baseFile, doc.diskSnapshot, 'utf-8');
+ blobFiles.push(baseFile);
+ treePaths.push(`.park-base/${doc.docName}`);
+ }
+
+ // `git hash-object -w` accepts multiple files and prints one object id per
+ // line in argument order, so blobShas[i] pairs with treePaths[i].
+ const blobShas: string[] = [];
+ for (let i = 0; i < blobFiles.length; i += PARK_BATCH_CHUNK) {
+ const chunk = blobFiles.slice(i, i + PARK_BATCH_CHUNK);
+ const out = await sg.env({ GIT_DIR: shadow.gitDir }).raw('hash-object', '-w', '--', ...chunk);
+ const shas = out
+ .trim()
+ .split('\n')
+ .map((s) => s.trim())
+ .filter(Boolean);
+ if (shas.length !== chunk.length || shas.some((s) => !/^[0-9a-f]{40}$/.test(s))) {
+ throw new Error(
+ `[shadow] park hash-object returned ${shas.length} ids for ${chunk.length} blobs`,
+ );
+ }
+ blobShas.push(...shas);
+ }
+
+ for (let i = 0; i < treePaths.length; i += PARK_BATCH_CHUNK) {
+ const cacheinfoArgs: string[] = [];
+ const end = Math.min(i + PARK_BATCH_CHUNK, treePaths.length);
+ for (let j = i; j < end; j++) {
+ cacheinfoArgs.push('--cacheinfo', `100644,${blobShas[j]},${treePaths[j]}`);
+ }
await sg
.env({ GIT_DIR: shadow.gitDir, GIT_INDEX_FILE: tmpIndex })
- .raw('update-index', '--add', '--cacheinfo', `100644,${baseSha},.park-base/${doc.docName}`);
+ .raw('update-index', '--add', ...cacheinfoArgs);
}
const treeSha = (
@@ -1418,7 +1509,7 @@ async function parkBranchInner(
// ignore cleanup failure
}
try {
- rmSync(tmpBlobFile);
+ rmSync(tmpBlobDir, { recursive: true, force: true });
} catch {
// ignore cleanup failure
}