Skip to content

Beautiful, AI-native markdown IDE and LLM wiki

53.6 KB
Raw Blame
import { randomUUID } from 'node:crypto';
import { existsSync, readdirSync, rmSync } from 'node:fs';
import { isAbsolute, relative, resolve, sep } from 'node:path';
import {
  OK_DIR,
  OK_MACHINE_LOCAL_ROOT_DIRS,
  OK_MACHINE_LOCAL_ROOT_FILES,
  pathspecArgs,
  SYSTEM_WRITER_DISPLAY_NAMES,
} from '@inkeep/open-knowledge-core';
import { ATOMIC_TEMP_GLOB } from '@inkeep/open-knowledge-core/server';
import {
  type AutoConsolidationTrigger,
  CHECKPOINT_KIND_REGISTRY,
  CHECKPOINT_KINDS,
  type CheckpointKind,
  formatCheckpointBodyLine,
  formatCheckpointSubject,
  formatImportSubject,
  formatOkActor,
  formatParkSubject,
  OK_GENERATOR_WRITER_ID,
  type OkActorEntry,
  type ParsedCheckpoint,
  parseCheckpoint,
  parseWriterId,
  resolveGitDir,
  resolveShadowDir,
  type WriterClassification,
} from '@inkeep/open-knowledge-core/shadow-repo-layout';
import simpleGit from 'simple-git';
import { resolveCheckpointChainAnchors } from './checkpoint-chain.ts';
import { tracedMkdirSync, tracedRenameSync, tracedWriteFileSync } from './fs-traced.ts';
import { escapeGitPatternSegment, listTreeLongEntries, realpathOrResolved } from './git-paths.ts';
import { assertNotSymlink, ensureGitignoreEntries, SymlinkRefusedError } from './init-project.ts';
import { getLogger } from './logger.ts';
import {
  incrementShadowExcludeIndexEntriesDropped,
  incrementShadowExcludeIndexSweepFailures,
  incrementShadowMigrationLegacyRefsDeleted,
} from './metrics.ts';
import { acquireLock, releaseLock } from './shadow-lock.ts';
import { releaseShadowOpGate, shadowOpGateFor } from './shadow-op-gate.ts';
import { withSpan } from './telemetry.ts';

const log = getLogger('shadow-repo');

export interface ShadowHandle {
  gitDir: string;
  workTree: string;
}

export interface ShadowRef {
  current: ShadowHandle | undefined;
}

export interface WriterIdentity {
  id: string;
  name: string;
  email: string;
}

const GIT_TIMEOUT_MS = (() => {
  const raw = process.env.OK_GIT_TIMEOUT_MS;
  if (!raw) return 30_000;
  const parsed = Number.parseInt(raw, 10);
  return Number.isFinite(parsed) && parsed > 0 ? parsed : 30_000;
})();

export const MAINTENANCE_GIT_TIMEOUT_MS = (() => {
  const raw = process.env.OK_SHADOW_MAINTENANCE_GC_TIMEOUT_MS;
  if (!raw) return 600_000;
  const parsed = Number.parseInt(raw, 10);
  return Number.isFinite(parsed) && parsed > 0 ? parsed : 600_000;
})();

const CORPUS_STAGE_GIT_TIMEOUT_MS = (() => {
  const raw = process.env.OK_SHADOW_STAGE_TIMEOUT_MS;
  if (!raw) return 300_000;
  const parsed = Number.parseInt(raw, 10);
  return Number.isFinite(parsed) && parsed > 0 ? parsed : 300_000;
})();

const SHADOW_GC_AUTO = 512;

const MACHINE_LOCAL_DIRS = OK_MACHINE_LOCAL_ROOT_DIRS.map((name) => `${OK_DIR}/${name}/`);

const MACHINE_LOCAL_FILES = OK_MACHINE_LOCAL_ROOT_FILES.map((name) => `${OK_DIR}/${name}`);

const SHADOW_EXCLUDE_PATTERNS: readonly string[] = [
  ...MACHINE_LOCAL_DIRS.map((dir) => `**/${dir}`),
  ...MACHINE_LOCAL_FILES.map((file) => `**/${file}`),
  `**/${OK_DIR}/watcher-settle-fence-*`,
  ATOMIC_TEMP_GLOB,
];

export function gitDirExcludePatterns(shadow: ShadowHandle): string[] {
  const workTreeReal = realpathOrResolved(shadow.workTree);
  const patterns = new Set<string>();
  for (const candidate of [resolveGitDir(shadow.workTree), shadow.gitDir]) {
    if (!candidate || !existsSync(candidate)) continue;
    const rel = relative(workTreeReal, realpathOrResolved(candidate));
    if (rel === '' || rel === '..' || rel.startsWith(`..${sep}`) || isAbsolute(rel)) continue;
    const segments = rel.split(sep);
    if (segments.includes('.git')) continue;
    if (segments.some((segment) => /[\r\n]/.test(segment))) continue;
    patterns.add(`/${segments.map(escapeGitPatternSegment).join('/')}/`);
  }
  return [...patterns];
}

function shadowExcludePatterns(shadow: ShadowHandle): string[] {
  return [...SHADOW_EXCLUDE_PATTERNS, ...gitDirExcludePatterns(shadow)];
}

function shadowExcludeContent(shadow: ShadowHandle): string {
  return `# OpenKnowledge machine-local state, in-flight atomic writes, and any git dir inside the work tree: never part of a version, never staged.
${shadowExcludePatterns(shadow).join('\n')}
`;
}

const UPDATE_INDEX_ARGV_BUDGET_BYTES = 8000;

const shadowExcludesWritten = new Map<string, boolean>();

export function isShadowExcludesDegraded(shadow: ShadowHandle): boolean {
  return shadowExcludesWritten.get(shadow.gitDir) === false;
}

function chunkByArgvBudget(paths: readonly string[], budgetBytes: number): string[][] {
  const chunks: string[][] = [];
  let current: string[] = [];
  let used = 0;
  for (const path of paths) {
    const cost = Buffer.byteLength(path, 'utf-8') + 1;
    if (current.length > 0 && used + cost > budgetBytes) {
      chunks.push(current);
      current = [];
      used = 0;
    }
    current.push(path);
    used += cost;
  }
  if (current.length > 0) chunks.push(current);
  return chunks;
}

interface IndexSweepResult {
  failed: boolean;
}

async function dropExcludedIndexEntries(
  sg: ReturnType<typeof shadowGit>,
  shadow: ShadowHandle,
  indexFile: string,
): Promise<IndexSweepResult> {
  const env = { GIT_DIR: shadow.gitDir, GIT_INDEX_FILE: indexFile };
  let dropped = 0;
  let listedCount: number | undefined;
  try {
    const excludeArgs = shadowExcludePatterns(shadow).flatMap((pattern) => ['-x', pattern]);
    const listed = await sg.env(env).raw('ls-files', '-z', '-i', '-c', ...excludeArgs);
    const stale = listed.split('\0').filter((p) => p.length > 0);
    listedCount = stale.length;
    for (const chunk of chunkByArgvBudget(stale, UPDATE_INDEX_ARGV_BUDGET_BYTES)) {
      await sg.env(env).raw('update-index', '--force-remove', '--', ...chunk);
      dropped += chunk.length;
    }
    if (dropped > 0) {
      incrementShadowExcludeIndexEntriesDropped(dropped);
      log.info(
        { count: dropped },
        '[shadow-repo] dropped excluded entries a previous build had staged',
      );
    }
    return { failed: false };
  } catch (e) {
    const leftStaged = listedCount === undefined ? undefined : listedCount - dropped;
    incrementShadowExcludeIndexSweepFailures();
    log.warn(
      { err: e, dropped, leftStaged: leftStaged ?? 'unknown' },
      leftStaged === undefined
        ? '[shadow-repo] could not read the index to clear excluded entries — none were dropped and an unknown number are still staged (non-fatal)'
        : `[shadow-repo] could not finish clearing excluded entries from the index — dropped ${dropped}, left ${leftStaged} still staged (non-fatal)`,
    );
    return { failed: true };
  }
}

async function ensureShadowExcludes(shadow: ShadowHandle): Promise<void> {
  const infoDir = resolve(shadow.gitDir, 'info');
  const excludeFile = resolve(infoDir, 'exclude');
  try {
    assertNotSymlink(infoDir, 'shadow info/');
    tracedMkdirSync(infoDir, { recursive: true });
    const outcome = ensureGitignoreEntries(
      excludeFile,
      shadowExcludeContent(shadow),
      'shadow info/exclude',
    );
    shadowExcludesWritten.set(shadow.gitDir, true);
    log.info({ outcome, excludeFile }, `[shadow-repo] shadow exclude file ${outcome}`);
  } catch (e) {
    shadowExcludesWritten.set(shadow.gitDir, false);
    if (e instanceof SymlinkRefusedError) {
      log.error(
        { err: e, excludeFile },
        '[shadow-repo] refused to write the shadow exclude file: a symlink is planted on a component of the shadow info/exclude path',
      );
    } else {
      log.warn(
        { err: e, excludeFile },
        '[shadow-repo] could not write the shadow exclude file — excluded paths stay out of versions only through the index sweep',
      );
    }
  }
  const fanoutIndex = resolve(shadow.gitDir, FANOUT_INDEX_NAME);
  if (!existsSync(fanoutIndex)) return;
  const sweep = await dropExcludedIndexEntries(
    shadowGit(shadow, { timeoutMs: CORPUS_STAGE_GIT_TIMEOUT_MS }),
    shadow,
    fanoutIndex,
  );
  if (sweep.failed) shadowExcludesWritten.set(shadow.gitDir, false);
}

export function shadowGit(shadow: ShadowHandle, opts?: { timeoutMs?: number }) {
  return simpleGit({
    baseDir: shadow.workTree,
    timeout: { block: opts?.timeoutMs ?? GIT_TIMEOUT_MS },
  }).env({
    GIT_DIR: shadow.gitDir,
    GIT_WORK_TREE: shadow.workTree,
  });
}

export async function configureShadowGc(shadow: ShadowHandle): Promise<void> {
  const sg = shadowGit(shadow);
  await sg.raw('config', 'gc.auto', String(SHADOW_GC_AUTO));
  await sg.raw('config', 'gc.autoDetach', 'false');
  await sg.raw('config', 'gc.writeCommitGraph', 'true');
  await sg.raw('config', 'commitGraph.changedPaths', 'true');
}

export interface WipChainInfo {
  writerId: string;
  tipSha: string;
  classification: WriterClassification;
  isPark: boolean;
  committedAtMs: number;
}

export async function enumerateWipChains(
  shadow: ShadowHandle,
  branch: string,
): Promise<WipChainInfo[]> {
  const sg = shadowGit(shadow);
  let lines: string[];
  try {
    lines = (
      await sg.raw(
        'for-each-ref',
        '--format=%(refname)%00%(objectname)%00%(committerdate:unix)%00%(contents:subject)',
        `refs/wip/${branch}/`,
      )
    )
      .trim()
      .split('\n')
      .filter(Boolean);
  } catch {
    return [];
  }
  const out: WipChainInfo[] = [];
  for (const line of lines) {
    const [refname = '', tipSha = '', committerUnix = '', subject = ''] = line.split('\x00');
    const writerId = refname.split('/').slice(3).join('/');
    if (!writerId) continue;
    const unix = Number.parseInt(committerUnix, 10);
    out.push({
      writerId,
      tipSha,
      classification: parseWriterId(writerId).classification,
      isPark: subject.startsWith('park:'),
      committedAtMs: Number.isFinite(unix) ? unix * 1000 : 0,
    });
  }
  return out;
}

export async function initShadowRepo(
  projectRoot: string,
  opts?: {
    deferGcConfig?: boolean;
  },
): Promise<ShadowHandle> {
  const shadowDir = resolveShadowDir(projectRoot);

  const legacyDir = resolve(projectRoot, '.git/openknowledge');
  const legacyExists = existsSync(legacyDir);
  const newExists = existsSync(shadowDir);
  if (legacyExists && !newExists) {
    tracedRenameSync(legacyDir, shadowDir);
  } else if (legacyExists && newExists) {
    log.warn(
      { legacyDir, shadowDir },
      '[shadow-repo] unexpected legacy + new shadow both present — no rename performed',
    );
  }

  const alreadyInit = existsSync(resolve(shadowDir, 'HEAD'));
  if (!alreadyInit) {
    tracedMkdirSync(shadowDir, { recursive: true });

    const git = simpleGit({ baseDir: projectRoot, timeout: { block: GIT_TIMEOUT_MS } });
    await git.raw('init', '--bare', shadowDir);

    const sg = simpleGit({ timeout: { block: GIT_TIMEOUT_MS } }).env({ GIT_DIR: shadowDir });
    await sg.raw('config', '--unset', 'core.bare');
    await sg.raw('config', 'core.worktree', projectRoot);
    await sg.raw('config', 'user.name', 'openknowledge');
    await sg.raw('config', 'user.email', '[email protected]');
  }

  const handle: ShadowHandle = { gitDir: shadowDir, workTree: projectRoot };

  await ensureShadowExcludes(handle);

  if (!opts?.deferGcConfig) {
    try {
      await configureShadowGc(handle);
    } catch (e) {
      log.warn({ err: e }, 'failed to write gc config (non-fatal)');
    }
  }

  await sweepLegacyShadowRefs(handle);

  sweepOrphanedScratchState(handle);

  acquireLock(shadowDir, projectRoot);

  return handle;
}

export function destroyShadowRepo(shadow: ShadowHandle): void {
  releaseLock(shadow.gitDir);
  releaseShadowOpGate(shadow.gitDir);
}

export async function sweepLegacyShadowRefs(shadow: ShadowHandle): Promise<number> {
  const sg = shadowGit(shadow);
  let allRefs: string[];
  try {
    const raw = await sg.raw('for-each-ref', '--format=%(refname)', 'refs/wip');
    allRefs = raw
      .trim()
      .split('\n')
      .filter((r) => r.length > 0);
  } catch {
    return 0;
  }

  const toDelete: string[] = [];
  const breakdown: Record<string, number> = { server: 0, 'human-': 0, upstream: 0 };

  for (const refname of allRefs) {
    const parts = refname.split('/');
    if (parts.length < 4) continue;
    const writerId = parts.slice(3).join('/');

    const classification = parseWriterId(writerId).classification;
    if (classification !== 'unknown') continue;

    if (writerId === 'server') {
      toDelete.push(refname);
      breakdown.server++;
    } else if (writerId.startsWith('human-')) {
      toDelete.push(refname);
      breakdown['human-']++;
    } else if (writerId === 'upstream') {
      toDelete.push(refname);
      breakdown.upstream++;
    }
  }

  if (toDelete.length === 0) return 0;

  let deleted = 0;
  await shadowOpGateFor(shadow).withMutator(async () => {
    for (const ref of toDelete) {
      try {
        await sg.raw('update-ref', '-d', ref);
        deleted++;
      } catch (e) {
        log.warn({ ref, err: e }, `[shadow-migration] failed to delete legacy ref ${ref}`);
      }
    }
  });
  incrementShadowMigrationLegacyRefsDeleted(deleted);
  log.warn(
    { deleted, server: breakdown.server, human: breakdown['human-'], upstream: breakdown.upstream },
    `[shadow-migration] deleted ${deleted} legacy refs: server=${breakdown.server} human-=${breakdown['human-']} upstream=${breakdown.upstream}`,
  );

  return deleted;
}

export interface CommitWipOptions {
  date?: string;
}

const wipCommitQueues = new Map<string, Promise<unknown>>();

export async function commitWip(
  shadow: ShadowHandle,
  writer: WriterIdentity,
  contentRoot: string,
  message: string,
  branch = 'main',
  opts?: CommitWipOptions,
): Promise<string> {
  return withSpan(
    'shadow.commitWip',
    {
      attributes: {
        'shadow.writer': writer.id,
        'shadow.branch': branch,
      },
    },
    async () => {
      const key = `${shadow.gitDir}\0${writer.id}`;
      const flush = () =>
        shadowOpGateFor(shadow).withMutator(() =>
          commitWipInner(shadow, writer, contentRoot, message, branch, opts?.date),
        );
      const run = (wipCommitQueues.get(key) ?? Promise.resolve()).then(flush, flush);
      wipCommitQueues.set(key, run);
      return run;
    },
  );
}

async function commitWipInner(
  shadow: ShadowHandle,
  writer: WriterIdentity,
  contentRoot: string,
  message: string,
  branch = 'main',
  date?: string,
): Promise<string> {
  const tmpIndex = resolve(shadow.gitDir, `index-wip-${writer.id}`);
  const ref = `refs/wip/${branch}/${writer.id}`;
  const sg = shadowGit(shadow, { timeoutMs: CORPUS_STAGE_GIT_TIMEOUT_MS });
  const gitPathspec = contentRoot || '.';

  try {
    try {
      const refTree = (await sg.raw('rev-parse', `${ref}^{tree}`)).trim();
      await sg.env({ GIT_DIR: shadow.gitDir, GIT_INDEX_FILE: tmpIndex }).raw('read-tree', refTree);
    } catch (e) {
      const msg = e instanceof Error ? e.message : String(e);
      if (msg.includes('unknown revision') || msg.includes('bad revision')) {
      } else {
        log.error({ ref, err: e }, `Unexpected error seeding index for ${ref}`);
        throw e;
      }
    }

    await sg
      .env({
        GIT_DIR: shadow.gitDir,
        GIT_WORK_TREE: shadow.workTree,
        GIT_INDEX_FILE: tmpIndex,
      })
      .raw('add', ...pathspecArgs([gitPathspec]));
    await dropExcludedIndexEntries(sg, shadow, tmpIndex);
    const treeSha = (
      await sg.env({ GIT_DIR: shadow.gitDir, GIT_INDEX_FILE: tmpIndex }).raw('write-tree')
    ).trim();

    let parentSha: string | null = null;
    try {
      parentSha = (await sg.raw('rev-parse', ref)).trim();
    } catch (e) {
      const msg = e instanceof Error ? e.message : String(e);
      if (!msg.includes('unknown revision') && !msg.includes('bad revision')) {
        log.error({ ref, err: e }, `Unexpected error resolving ${ref}`);
        throw e;
      }
    }

    const args = ['commit-tree', treeSha, '-m', message];
    if (parentSha) args.push('-p', parentSha);

    const commitEnv: Record<string, string> = {
      GIT_DIR: shadow.gitDir,
      GIT_AUTHOR_NAME: writer.name,
      GIT_AUTHOR_EMAIL: writer.email,
      GIT_COMMITTER_NAME: 'openknowledge',
      GIT_COMMITTER_EMAIL: '[email protected]',
    };
    if (date) {
      commitEnv.GIT_AUTHOR_DATE = date;
      commitEnv.GIT_COMMITTER_DATE = date;
    }
    const commitSha = (await sg.env(commitEnv).raw(...args)).trim();

    await sg.raw('update-ref', ref, commitSha);
    return commitSha;
  } finally {
    try {
      rmSync(tmpIndex);
    } catch {}
  }
}

export const FANOUT_INDEX_NAME = 'index-wip-fanout';

function sweepOrphanedScratchState(shadow: ShadowHandle): number {
  let deleted = 0;
  try {
    for (const name of readdirSync(shadow.gitDir)) {
      const isThrowaway = name.startsWith(`${FANOUT_INDEX_NAME}-`);
      const isStaleLock = name === `${FANOUT_INDEX_NAME}.lock`;
      const isParkScratchDir = name.startsWith('tmp-park-blobs-');
      if (!isThrowaway && !isStaleLock && !isParkScratchDir) continue;
      try {
        rmSync(resolve(shadow.gitDir, name), { recursive: true, force: true });
        deleted++;
      } catch {}
    }
  } catch {}
  return deleted;
}

export async function buildWipTree(shadow: ShadowHandle, contentRoot: string): Promise<string> {
  return shadowOpGateFor(shadow).withMutator(() => buildWipTreeInner(shadow, contentRoot));
}

async function buildWipTreeWithIndex(
  shadow: ShadowHandle,
  contentRoot: string,
  indexFile: string,
): Promise<string> {
  const sg = shadowGit(shadow, { timeoutMs: CORPUS_STAGE_GIT_TIMEOUT_MS });
  const gitPathspec = contentRoot || '.';
  await sg
    .env({
      GIT_DIR: shadow.gitDir,
      GIT_WORK_TREE: shadow.workTree,
      GIT_INDEX_FILE: indexFile,
    })
    .raw('add', ...pathspecArgs([gitPathspec]));
  await dropExcludedIndexEntries(sg, shadow, indexFile);
  return (
    await sg.env({ GIT_DIR: shadow.gitDir, GIT_INDEX_FILE: indexFile }).raw('write-tree')
  ).trim();
}

async function buildWipTreeInner(shadow: ShadowHandle, contentRoot: string): Promise<string> {
  const persistentIndex = resolve(shadow.gitDir, FANOUT_INDEX_NAME);
  try {
    return await buildWipTreeWithIndex(shadow, contentRoot, persistentIndex);
  } catch (e) {
    log.warn(
      { err: e },
      '[shadow-repo] persistent fan-out index failed — rebuilding from a fresh index',
    );
    const isTimeout = e instanceof Error && e.message.includes('block timeout');
    for (const stale of isTimeout
      ? [`${persistentIndex}.lock`]
      : [persistentIndex, `${persistentIndex}.lock`]) {
      try {
        rmSync(stale);
      } catch {}
    }
    const tmpIndex = resolve(shadow.gitDir, `${FANOUT_INDEX_NAME}-${randomUUID()}`);
    try {
      return await buildWipTreeWithIndex(shadow, contentRoot, tmpIndex);
    } finally {
      try {
        rmSync(tmpIndex);
      } catch {}
    }
  }
}

export async function commitWipFromTree(
  shadow: ShadowHandle,
  writer: WriterIdentity,
  treeSha: string,
  message: string,
  branch = 'main',
): Promise<string> {
  return withSpan(
    'shadow.commitWipFromTree',
    {
      attributes: {
        'shadow.writer': writer.id,
        'shadow.branch': branch,
        'shadow.tree': treeSha.slice(0, 8),
      },
    },
    async () =>
      shadowOpGateFor(shadow).withMutator(() =>
        commitWipFromTreeInner(shadow, writer, treeSha, message, branch),
      ),
  );
}

async function commitWipFromTreeInner(
  shadow: ShadowHandle,
  writer: WriterIdentity,
  treeSha: string,
  message: string,
  branch = 'main',
): Promise<string> {
  const ref = `refs/wip/${branch}/${writer.id}`;
  const sg = shadowGit(shadow);

  let parentSha: string | null = null;
  try {
    parentSha = (await sg.raw('rev-parse', ref)).trim();
  } catch (e) {
    const msg = e instanceof Error ? e.message : String(e);
    if (!msg.includes('unknown revision') && !msg.includes('bad revision')) {
      log.error({ ref, err: e }, `Unexpected error resolving ${ref}`);
      throw e;
    }
  }

  const args = ['commit-tree', treeSha, '-m', message];
  if (parentSha) args.push('-p', parentSha);

  const commitSha = (
    await sg
      .env({
        GIT_DIR: shadow.gitDir,
        GIT_AUTHOR_NAME: writer.name,
        GIT_AUTHOR_EMAIL: writer.email,
        GIT_COMMITTER_NAME: 'openknowledge',
        GIT_COMMITTER_EMAIL: '[email protected]',
      })
      .raw(...args)
  ).trim();

  await sg.raw('update-ref', ref, commitSha);
  return commitSha;
}

export const FILE_SYSTEM_WRITER: WriterIdentity = {
  id: 'file-system',
  name: SYSTEM_WRITER_DISPLAY_NAMES.fileSystem,
  email: '[email protected]',
};

export const GIT_UPSTREAM_WRITER: WriterIdentity = {
  id: 'git-upstream',
  name: SYSTEM_WRITER_DISPLAY_NAMES.gitUpstream,
  email: '[email protected]',
};

export const SERVICE_WRITER: WriterIdentity = {
  id: 'openknowledge-service',
  name: SYSTEM_WRITER_DISPLAY_NAMES.service,
  email: '[email protected]',
};

/**
 * Deliberately NOT `SERVICE_WRITER`: that one is the fallback for work with no contributor behind
 * it, and precedent #25 reserves it for exactly that.
 */
export const OK_GENERATOR_WRITER: WriterIdentity = {
  id: OK_GENERATOR_WRITER_ID,
  name: SYSTEM_WRITER_DISPLAY_NAMES.generator,
  email: `${OK_GENERATOR_WRITER_ID}@openknowledge.local`,
};

const UPSTREAM_WRITER: WriterIdentity = GIT_UPSTREAM_WRITER;

export async function commitUpstreamImport(
  shadow: ShadowHandle,
  contentRoot: string,
  oldHead: string | null,
  newHead: string,
  branch = 'main',
): Promise<string> {
  return withSpan(
    'shadow.commitUpstreamImport',
    { attributes: { 'shadow.branch': branch, 'shadow.new_head': newHead.slice(0, 8) } },
    async () => commitUpstreamImportInner(shadow, contentRoot, oldHead, newHead, branch),
  );
}

async function commitUpstreamImportInner(
  shadow: ShadowHandle,
  contentRoot: string,
  oldHead: string | null,
  newHead: string,
  branch = 'main',
): Promise<string> {
  const subject = formatImportSubject(oldHead, newHead);
  const actorEntry: OkActorEntry = {
    v: 1,
    writer_id: UPSTREAM_WRITER.id,
    principal: null,
    agent_session: null,
    agent_type: null,
    client_name: null,
    client_version: null,
    label: null,
    display_name: UPSTREAM_WRITER.name,
    color_seed: UPSTREAM_WRITER.id,
    docs: [],
  };
  const message = `${subject}\n\n${formatOkActor(actorEntry)}`;
  return commitWip(shadow, UPSTREAM_WRITER, contentRoot, message, branch);
}

export interface SafetyCheckpointParams {
  action: string;
  context: Record<string, unknown>;
}

const SAFETY_WRITER: WriterIdentity = SERVICE_WRITER;

export async function safetyCheckpoint(
  shadow: ShadowHandle,
  contentRoot: string,
  params: SafetyCheckpointParams,
  branch = 'main',
): Promise<string> {
  const subject = formatCheckpointSubject(`pre-${params.action}`);
  const actorEntry: OkActorEntry = {
    v: 1,
    writer_id: SAFETY_WRITER.id,
    principal: null,
    agent_session: null,
    agent_type: null,
    client_name: null,
    client_version: null,
    label: null,
    display_name: SAFETY_WRITER.name,
    color_seed: SAFETY_WRITER.id,
    docs: [],
  };
  const message = `${subject}\n\n${formatOkActor(actorEntry)}`;
  return commitWip(shadow, SAFETY_WRITER, contentRoot, message, branch);
}

export type InMemoryCheckpointParams = (
  | {
      kind: 'bridge-merge-loss';
      docName: string;
      contents: string;
      label: string;
      branch?: string;
      metadata: { lostSubstrings: string[]; which?: string };
    }
  | {
      kind: 'producer-guard-loss';
      docName: string;
      contents: string;
      label: string;
      branch?: string;
      metadata: { construct: string };
    }
  | {
      kind: 'observer-a-duplication';
      docName: string;
      contents: string;
      label: string;
      branch?: string;
      metadata: { duplicatedLineCount: number };
    }
  | {
      kind: 'external-change-rescue';
      docName: string;
      contents: string;
      label: string;
      branch?: string;
      metadata: { incomingDiskSha: string };
    }
  | {
      kind: 'external-change-rescue-disk-update';
      docName: string;
      contents: string;
      label: string;
      branch?: string;
      metadata: { incomingDiskSha: string };
    }
  | {
      kind: 'defer-exhaustion-loss';
      docName: string;
      contents: string;
      label: string;
      branch?: string;
      metadata: { deferCount: number };
    }
  | {
      kind: 'observer-a-apply-loss';
      docName: string;
      contents: string;
      label: string;
      branch?: string;
      metadata: { lostSubstrings: string[] };
    }
  | {
      kind: 'bridge-derive-loss';
      docName: string;
      contents: string;
      label: string;
      branch?: string;
      metadata: { lostSubstrings: string[] };
    }
  | {
      kind: 'persistence-reconcile-loss';
      docName: string;
      contents: string;
      label: string;
      branch?: string;
      metadata: { atRiskLines: number; witnessAvailable: boolean };
    }
  | {
      kind: 'bridge-backstop-trip';
      docName: string;
      contents: string;
      label: string;
      branch?: string;
      metadata: { rounds: number };
    }
  | {
      kind: 'persistence-duplication-reset';
      docName: string;
      contents: string;
      label: string;
      branch?: string;
      metadata: { copies: number; fragmentChildren: number };
    }
  | {
      kind: 'persistence-divergence-realign';
      docName: string;
      contents: string;
      label: string;
      branch?: string;
      metadata: { diskBytes: number; discardedBytes: number };
    }
  | {
      kind: 'managed-artifact-reconcile';
      docName: string;
      contents: string;
      label: string;
      branch?: string;
      metadata: { diskBytes: number; discardedBytes: number };
    }
  | {
      kind: 'external-change-both-kept';
      docName: string;
      contents: string;
      label: string;
      branch?: string;
      metadata: Extract<ParsedCheckpoint, { kind: 'external-change-both-kept' }>['metadata'];
    }
  | {
      kind: 'external-change-kept-live';
      docName: string;
      contents: string;
      label: string;
      branch?: string;
      metadata: Extract<ParsedCheckpoint, { kind: 'external-change-kept-live' }>['metadata'];
    }
  | {
      kind: 'external-change-no-engine';
      docName: string;
      contents: string;
      label: string;
      branch?: string;
      metadata: Extract<ParsedCheckpoint, { kind: 'external-change-no-engine' }>['metadata'];
    }
) & {
  date?: string;
};

export async function saveInMemoryCheckpoint(
  shadow: ShadowHandle,
  contentRoot: string,
  params: InMemoryCheckpointParams,
): Promise<string> {
  return shadowOpGateFor(shadow).withMutator(() =>
    saveInMemoryCheckpointInner(shadow, contentRoot, params),
  );
}

async function saveInMemoryCheckpointInner(
  shadow: ShadowHandle,
  contentRoot: string,
  params: InMemoryCheckpointParams,
): Promise<string> {
  const branch = params.branch ?? 'main';
  const sg = shadowGit(shadow);
  const token = randomUUID();
  const tmpIndex = resolve(shadow.gitDir, `index-checkpoint-${token}`);
  const tmpBlobFile = resolve(shadow.gitDir, `tmp-checkpoint-blob-${token}`);

  const normalizedRoot =
    contentRoot === '.' ? '' : contentRoot.replace(/^\.\//, '').replace(/\/$/, '');
  const treePath = normalizedRoot ? `${normalizedRoot}/${params.docName}` : params.docName;
  const size = Buffer.byteLength(params.contents, 'utf-8');
  let parsed: ParsedCheckpoint;
  switch (params.kind) {
    case 'bridge-merge-loss':
      parsed = {
        kind: 'bridge-merge-loss',
        docName: params.docName,
        size,
        metadata: params.metadata,
      };
      break;
    case 'producer-guard-loss':
      parsed = {
        kind: 'producer-guard-loss',
        docName: params.docName,
        size,
        metadata: params.metadata,
      };
      break;
    case 'observer-a-duplication':
      parsed = {
        kind: 'observer-a-duplication',
        docName: params.docName,
        size,
        metadata: params.metadata,
      };
      break;
    case 'external-change-rescue':
      parsed = {
        kind: 'external-change-rescue',
        docName: params.docName,
        size,
        metadata: params.metadata,
      };
      break;
    case 'external-change-rescue-disk-update':
      parsed = {
        kind: 'external-change-rescue-disk-update',
        docName: params.docName,
        size,
        metadata: params.metadata,
      };
      break;
    case 'defer-exhaustion-loss':
      parsed = {
        kind: 'defer-exhaustion-loss',
        docName: params.docName,
        size,
        metadata: params.metadata,
      };
      break;
    case 'observer-a-apply-loss':
      parsed = {
        kind: 'observer-a-apply-loss',
        docName: params.docName,
        size,
        metadata: params.metadata,
      };
      break;
    case 'bridge-derive-loss':
      parsed = {
        kind: 'bridge-derive-loss',
        docName: params.docName,
        size,
        metadata: params.metadata,
      };
      break;
    case 'bridge-backstop-trip':
      parsed = {
        kind: 'bridge-backstop-trip',
        docName: params.docName,
        size,
        metadata: params.metadata,
      };
      break;
    case 'persistence-reconcile-loss':
      parsed = {
        kind: 'persistence-reconcile-loss',
        docName: params.docName,
        size,
        metadata: params.metadata,
      };
      break;
    case 'persistence-duplication-reset':
      parsed = {
        kind: 'persistence-duplication-reset',
        docName: params.docName,
        size,
        metadata: params.metadata,
      };
      break;
    case 'persistence-divergence-realign':
      parsed = {
        kind: 'persistence-divergence-realign',
        docName: params.docName,
        size,
        metadata: params.metadata,
      };
      break;
    case 'managed-artifact-reconcile':
      parsed = {
        kind: 'managed-artifact-reconcile',
        docName: params.docName,
        size,
        metadata: params.metadata,
      };
      break;
    case 'external-change-both-kept':
      parsed = {
        kind: 'external-change-both-kept',
        docName: params.docName,
        size,
        metadata: params.metadata,
      };
      break;
    case 'external-change-kept-live':
      parsed = {
        kind: 'external-change-kept-live',
        docName: params.docName,
        size,
        metadata: params.metadata,
      };
      break;
    case 'external-change-no-engine':
      parsed = {
        kind: 'external-change-no-engine',
        docName: params.docName,
        size,
        metadata: params.metadata,
      };
      break;
  }
  const bodyLine = formatCheckpointBodyLine(parsed);
  const message = `checkpoint: ${params.label}\n\n${bodyLine}`;

  try {
    tracedWriteFileSync(tmpBlobFile, params.contents, 'utf-8');
    const blobSha = (
      await sg
        .env({ GIT_DIR: shadow.gitDir, GIT_INDEX_FILE: tmpIndex })
        .raw('hash-object', '-w', tmpBlobFile)
    ).trim();
    await sg
      .env({ GIT_DIR: shadow.gitDir, GIT_INDEX_FILE: tmpIndex })
      .raw('update-index', '--add', '--cacheinfo', `100644,${blobSha},${treePath}`);
    const treeSha = (
      await sg.env({ GIT_DIR: shadow.gitDir, GIT_INDEX_FILE: tmpIndex }).raw('write-tree')
    ).trim();

    const commitEnv: Record<string, string> = {
      GIT_DIR: shadow.gitDir,
      GIT_AUTHOR_NAME: 'openknowledge',
      GIT_AUTHOR_EMAIL: '[email protected]',
      GIT_COMMITTER_NAME: 'openknowledge',
      GIT_COMMITTER_EMAIL: '[email protected]',
    };
    if (params.date) {
      commitEnv.GIT_AUTHOR_DATE = params.date;
      commitEnv.GIT_COMMITTER_DATE = params.date;
    }
    const commitSha = (await sg.env(commitEnv).raw('commit-tree', treeSha, '-m', message)).trim();

    await sg.raw('update-ref', `refs/checkpoints/${branch}/${commitSha}`, commitSha);
    return commitSha;
  } finally {
    try {
      rmSync(tmpIndex);
    } catch {}
    try {
      rmSync(tmpBlobFile);
    } catch {}
  }
}

export interface TimelineRescueEntry {
  docName: string;
  timestamp: string;
  size: number;
  sha: string;
  label: string;
  incomingDiskSha: string;
}

export async function listRescueCheckpoints(
  shadow: ShadowHandle,
  branch = 'main',
): Promise<TimelineRescueEntry[]> {
  const sg = shadowGit(shadow);
  let refOutput: string;
  try {
    refOutput = await sg.raw(
      'for-each-ref',
      '--format=%(objectname)',
      `refs/checkpoints/${branch}/`,
    );
  } catch {
    return [];
  }
  const shas = refOutput
    .trim()
    .split('\n')
    .filter((s) => s.length === 40);
  if (shas.length === 0) return [];

  let logRaw: string;
  try {
    logRaw = await sg.raw(
      'log',
      '--no-walk',
      '--author-date-order',
      '--format=%H%x00%aI%x00%s%x00%B%x1e',
      ...shas,
    );
  } catch {
    return [];
  }

  const out: TimelineRescueEntry[] = [];
  for (const record of logRaw.split('\x1e')) {
    const trimmed = record.trimStart();
    if (!trimmed) continue;
    const [sha = '', timestamp = '', subject = '', body = ''] = trimmed.split('\x00');
    const parsed = parseCheckpoint(body);
    if (
      parsed?.kind !== 'external-change-rescue' &&
      parsed?.kind !== 'external-change-rescue-disk-update'
    )
      continue;

    let docName = parsed.docName ?? '';
    let size = parsed.size ?? 0;

    if (!docName) {
      try {
        const entry = (await listTreeLongEntries(sg, ['ls-tree', '-r', '--long', sha]))[0];
        if (entry) {
          if (size === 0) size = entry.size;
          docName =
            entry.path
              .replace(/\.mdx?$/, '')
              .split('/')
              .slice(-1)[0] ?? '';
        }
      } catch {}
    }
    if (!docName) continue;
    out.push({
      docName,
      timestamp,
      size,
      sha,
      label: subject.replace(/^checkpoint:\s*/, ''),
      incomingDiskSha: parsed.metadata.incomingDiskSha,
    });
  }
  return out;
}

export interface CheckpointRetentionPolicy {
  maxBridgeMergeLoss: number;
  maxProducerGuardLoss: number;
  maxObserverADuplication: number;
  maxExternalChangeRescue: number;
  maxExternalChangeRescueDiskUpdate: number;
  maxDeferExhaustionLoss: number;
  maxBridgeDeriveLoss: number;
  maxObserverAApplyLoss: number;
  maxBridgeBackstopTrip: number;
  maxPersistenceReconcileLoss: number;
  maxPersistenceDuplicationReset: number;
  maxPersistenceDivergenceRealign: number;
  maxManagedArtifactReconcile: number;
  maxExternalChangeBothKeptPerDocument: number;
  maxExternalChangeKeptLivePerDocument: number;
  maxExternalChangeNoEnginePerDocument: number;
  maxAutoConsolidation: number;
  ttlMs: number;
}

export const DEFAULT_CHECKPOINT_RETENTION: CheckpointRetentionPolicy = {
  maxBridgeMergeLoss: 50,
  maxProducerGuardLoss: 50,
  maxObserverADuplication: 50,
  maxExternalChangeRescue: 50,
  maxExternalChangeRescueDiskUpdate: 50,
  maxDeferExhaustionLoss: 50,
  maxBridgeDeriveLoss: 50,
  maxObserverAApplyLoss: 50,
  maxBridgeBackstopTrip: 50,
  maxPersistenceReconcileLoss: 50,
  maxPersistenceDuplicationReset: 50,
  maxPersistenceDivergenceRealign: 50,
  maxManagedArtifactReconcile: 50,
  maxExternalChangeBothKeptPerDocument: 50,
  maxExternalChangeKeptLivePerDocument: 50,
  maxExternalChangeNoEnginePerDocument: 50,
  maxAutoConsolidation: 2,
  ttlMs: 30 * 24 * 60 * 60 * 1000,
};

export interface CheckpointGcResult {
  scanned: number;
  deletedBridgeMergeLoss: number;
  deletedProducerGuardLoss: number;
  deletedObserverADuplication: number;
  deletedExternalChangeRescue: number;
  deletedExternalChangeRescueDiskUpdate: number;
  deletedDeferExhaustionLoss: number;
  deletedBridgeDeriveLoss: number;
  deletedObserverAApplyLoss: number;
  deletedBridgeBackstopTrip: number;
  deletedPersistenceReconcileLoss: number;
  deletedPersistenceDuplicationReset: number;
  deletedPersistenceDivergenceRealign: number;
  deletedManagedArtifactReconcile: number;
  deletedExternalChangeBothKept: number;
  deletedExternalChangeKeptLive: number;
  deletedExternalChangeNoEngine: number;
  deletedAutoConsolidation: number;
  retained: number;
}

type GcDeletionCounter = Exclude<keyof CheckpointGcResult, 'scanned' | 'retained'>;

interface GcBucketPolicy {
  limit: (policy: CheckpointRetentionPolicy) => number;
  counter: GcDeletionCounter;
  applyTtl: boolean;
  scope: 'branch' | 'document';
}

export const GC_BUCKET_POLICY = {
  'bridge-merge-loss': {
    limit: (p) => p.maxBridgeMergeLoss,
    counter: 'deletedBridgeMergeLoss',
    applyTtl: true,
    scope: 'branch',
  },
  'producer-guard-loss': {
    limit: (p) => p.maxProducerGuardLoss,
    counter: 'deletedProducerGuardLoss',
    applyTtl: true,
    scope: 'branch',
  },
  'observer-a-duplication': {
    limit: (p) => p.maxObserverADuplication,
    counter: 'deletedObserverADuplication',
    applyTtl: true,
    scope: 'branch',
  },
  'external-change-rescue': {
    limit: (p) => p.maxExternalChangeRescue,
    counter: 'deletedExternalChangeRescue',
    applyTtl: true,
    scope: 'branch',
  },
  'external-change-rescue-disk-update': {
    limit: (p) => p.maxExternalChangeRescueDiskUpdate,
    counter: 'deletedExternalChangeRescueDiskUpdate',
    applyTtl: true,
    scope: 'branch',
  },
  'defer-exhaustion-loss': {
    limit: (p) => p.maxDeferExhaustionLoss,
    counter: 'deletedDeferExhaustionLoss',
    applyTtl: true,
    scope: 'branch',
  },
  'observer-a-apply-loss': {
    limit: (p) => p.maxObserverAApplyLoss,
    counter: 'deletedObserverAApplyLoss',
    applyTtl: true,
    scope: 'branch',
  },
  'bridge-derive-loss': {
    limit: (p) => p.maxBridgeDeriveLoss,
    counter: 'deletedBridgeDeriveLoss',
    applyTtl: true,
    scope: 'branch',
  },
  'bridge-backstop-trip': {
    limit: (p) => p.maxBridgeBackstopTrip,
    counter: 'deletedBridgeBackstopTrip',
    applyTtl: true,
    scope: 'branch',
  },
  'persistence-reconcile-loss': {
    limit: (p) => p.maxPersistenceReconcileLoss,
    counter: 'deletedPersistenceReconcileLoss',
    applyTtl: true,
    scope: 'branch',
  },
  'persistence-duplication-reset': {
    limit: (p) => p.maxPersistenceDuplicationReset,
    counter: 'deletedPersistenceDuplicationReset',
    applyTtl: true,
    scope: 'branch',
  },
  'persistence-divergence-realign': {
    limit: (p) => p.maxPersistenceDivergenceRealign,
    counter: 'deletedPersistenceDivergenceRealign',
    applyTtl: true,
    scope: 'branch',
  },
  'managed-artifact-reconcile': {
    limit: (p) => p.maxManagedArtifactReconcile,
    counter: 'deletedManagedArtifactReconcile',
    applyTtl: true,
    scope: 'branch',
  },
  'external-change-both-kept': {
    limit: (p) => p.maxExternalChangeBothKeptPerDocument,
    counter: 'deletedExternalChangeBothKept',
    applyTtl: false,
    scope: 'document',
  },
  'external-change-kept-live': {
    limit: (p) => p.maxExternalChangeKeptLivePerDocument,
    counter: 'deletedExternalChangeKeptLive',
    applyTtl: false,
    scope: 'document',
  },
  'external-change-no-engine': {
    limit: (p) => p.maxExternalChangeNoEnginePerDocument,
    counter: 'deletedExternalChangeNoEngine',
    applyTtl: false,
    scope: 'document',
  },
  'auto-consolidation': {
    limit: (p) => p.maxAutoConsolidation,
    counter: 'deletedAutoConsolidation',
    applyTtl: false,
    scope: 'branch',
  },
} as const satisfies Record<CheckpointKind, GcBucketPolicy>;

export async function gcCheckpointRefs(
  shadow: ShadowHandle,
  branch = 'main',
  policy: CheckpointRetentionPolicy = DEFAULT_CHECKPOINT_RETENTION,
): Promise<CheckpointGcResult> {
  return shadowOpGateFor(shadow).withMutator(() => gcCheckpointRefsInner(shadow, branch, policy));
}

async function gcCheckpointRefsInner(
  shadow: ShadowHandle,
  branch = 'main',
  policy: CheckpointRetentionPolicy = DEFAULT_CHECKPOINT_RETENTION,
): Promise<CheckpointGcResult> {
  const result: CheckpointGcResult = {
    scanned: 0,
    deletedBridgeMergeLoss: 0,
    deletedProducerGuardLoss: 0,
    deletedObserverADuplication: 0,
    deletedExternalChangeRescue: 0,
    deletedExternalChangeRescueDiskUpdate: 0,
    deletedDeferExhaustionLoss: 0,
    deletedBridgeDeriveLoss: 0,
    deletedObserverAApplyLoss: 0,
    deletedBridgeBackstopTrip: 0,
    deletedPersistenceReconcileLoss: 0,
    deletedPersistenceDuplicationReset: 0,
    deletedPersistenceDivergenceRealign: 0,
    deletedManagedArtifactReconcile: 0,
    deletedExternalChangeBothKept: 0,
    deletedExternalChangeKeptLive: 0,
    deletedExternalChangeNoEngine: 0,
    deletedAutoConsolidation: 0,
    retained: 0,
  };
  const sg = shadowGit(shadow);
  let refOutput: string;
  try {
    refOutput = await sg.raw(
      'for-each-ref',
      '--format=%(objectname) %(refname)',
      `refs/checkpoints/${branch}/`,
    );
  } catch {
    return result;
  }
  const refLines = refOutput
    .trim()
    .split('\n')
    .map((l) => l.trim())
    .filter(Boolean);
  if (refLines.length === 0) return result;

  const shaToRef = new Map<string, string>();
  const shas: string[] = [];
  for (const line of refLines) {
    const spaceIdx = line.indexOf(' ');
    if (spaceIdx < 0) continue;
    const sha = line.slice(0, spaceIdx);
    const ref = line.slice(spaceIdx + 1);
    if (sha.length !== 40) continue;
    shaToRef.set(sha, ref);
    shas.push(sha);
  }
  result.scanned = shas.length;
  if (shas.length === 0) return result;

  let logRaw: string;
  try {
    logRaw = await sg.raw(
      'log',
      '--no-walk',
      '--author-date-order',
      '--format=%H%x00%aI%x00%B%x1e',
      ...shas,
    );
  } catch {
    return result;
  }

  interface Entry {
    sha: string;
    timestamp: number;
    kind: CheckpointKind | null;
    docName: string | null;
  }
  const entries: Entry[] = [];
  for (const record of logRaw.split('\x1e')) {
    const trimmed = record.trimStart();
    if (!trimmed) continue;
    const [sha = '', timestamp = '', body = ''] = trimmed.split('\x00');
    if (!sha) continue;
    const parsed = parseCheckpoint(body);
    const kind = parsed?.kind ?? null;
    const ts = Date.parse(timestamp);
    entries.push({
      sha,
      timestamp: Number.isFinite(ts) ? ts : 0,
      kind,
      docName: parsed?.docName ?? null,
    });
  }

  const byBucket = Object.fromEntries(
    CHECKPOINT_KINDS.map((kind) => [kind, [] as Entry[]]),
  ) as Record<CheckpointKind, Entry[]>;
  let retainedUntyped = 0;
  for (const e of entries) {
    if (e.kind === null) {
      retainedUntyped++;
      continue;
    }
    byBucket[CHECKPOINT_KIND_REGISTRY[e.kind].gcBucket].push(e);
  }

  const now = Date.now();
  const deleteRefs: string[] = [];
  const planDeletions = (
    list: Entry[],
    limit: number,
    counter: GcDeletionCounter,
    applyTtl = true,
  ): void => {
    list.sort((a, b) => b.timestamp - a.timestamp);
    let keep = limit;
    while (keep > 0 && keep < list.length) {
      const lastKept = list[keep - 1];
      const firstDropped = list[keep];
      if (!lastKept || !firstDropped || lastKept.timestamp !== firstDropped.timestamp) break;
      keep++;
    }
    for (let i = 0; i < list.length; i++) {
      const entry = list[i];
      if (!entry) continue;
      const overCount = i >= keep;
      const overTtl =
        applyTtl && policy.ttlMs > 0 && entry.timestamp > 0 && now - entry.timestamp > policy.ttlMs;
      if (overCount || overTtl) {
        const ref = shaToRef.get(entry.sha);
        if (ref) {
          deleteRefs.push(ref);
          result[counter]++;
        }
      }
    }
  };
  const groupsOf = (list: Entry[], scope: GcBucketPolicy['scope']): Entry[][] => {
    if (scope === 'branch') return [list];
    const byDocument = new Map<string, Entry[]>();
    for (const entry of list) {
      const key = entry.docName ?? '';
      const group = byDocument.get(key);
      if (group) group.push(entry);
      else byDocument.set(key, [entry]);
    }
    return [...byDocument.values()];
  };
  for (const bucket of CHECKPOINT_KINDS) {
    const bucketPolicy = GC_BUCKET_POLICY[bucket];
    for (const group of groupsOf(byBucket[bucket], bucketPolicy.scope)) {
      planDeletions(group, bucketPolicy.limit(policy), bucketPolicy.counter, bucketPolicy.applyTtl);
    }
  }

  for (const ref of deleteRefs) {
    try {
      await sg.raw('update-ref', '-d', ref);
    } catch (err) {
      log.warn({ ref, err }, `[checkpoint-gc] failed to delete ${ref}`);
    }
  }

  result.retained = retainedUntyped + (result.scanned - deleteRefs.length - retainedUntyped);
  return result;
}

export interface ParkableDoc {
  docName: string;
  markdown: string;
  diskSnapshot: string;
}

export async function parkBranch(
  shadow: ShadowHandle,
  branch: string,
  writerId: string,
  documents: ParkableDoc[],
  newBranch?: string,
): Promise<string | null> {
  if (documents.length === 0) return null;
  return withSpan(
    'shadow.parkBranch',
    {
      attributes: {
        'shadow.branch': branch,
        'shadow.new_branch': newBranch ?? '',
        'shadow.doc_count': documents.length,
      },
    },
    async () =>
      shadowOpGateFor(shadow).withMutator(() =>
        parkBranchInner(shadow, branch, writerId, documents, newBranch),
      ),
  );
}

const PARK_BATCH_CHUNK = 200;

async function parkBranchInner(
  shadow: ShadowHandle,
  branch: string,
  writerId: string,
  documents: ParkableDoc[],
  newBranch?: string,
): Promise<string | null> {
  const sg = shadowGit(shadow);
  const tmpIndex = resolve(shadow.gitDir, `index-park-${branch.replace(/\//g, '-')}`);
  const ref = `refs/wip/${branch}/${writerId}`;

  const tmpBlobDir = resolve(shadow.gitDir, `tmp-park-blobs-${randomUUID()}`);
  try {
    tracedMkdirSync(tmpBlobDir, { recursive: true });
    const blobFiles: string[] = [];
    const treePaths: string[] = [];
    for (let i = 0; i < documents.length; i++) {
      const doc = documents[i];
      if (!doc) continue;
      const stateFile = resolve(tmpBlobDir, `${i}-state`);
      tracedWriteFileSync(stateFile, doc.markdown, 'utf-8');
      blobFiles.push(stateFile);
      treePaths.push(doc.docName);
      const baseFile = resolve(tmpBlobDir, `${i}-base`);
      tracedWriteFileSync(baseFile, doc.diskSnapshot, 'utf-8');
      blobFiles.push(baseFile);
      treePaths.push(`.park-base/${doc.docName}`);
    }

    const blobShas: string[] = [];
    for (let i = 0; i < blobFiles.length; i += PARK_BATCH_CHUNK) {
      const chunk = blobFiles.slice(i, i + PARK_BATCH_CHUNK);
      const out = await sg.env({ GIT_DIR: shadow.gitDir }).raw('hash-object', '-w', '--', ...chunk);
      const shas = out
        .trim()
        .split('\n')
        .map((s) => s.trim())
        .filter(Boolean);
      if (shas.length !== chunk.length || shas.some((s) => !/^[0-9a-f]{40}$/.test(s))) {
        throw new Error(
          `[shadow] park hash-object returned ${shas.length} ids for ${chunk.length} blobs`,
        );
      }
      blobShas.push(...shas);
    }

    for (let i = 0; i < treePaths.length; i += PARK_BATCH_CHUNK) {
      const cacheinfoArgs: string[] = [];
      const end = Math.min(i + PARK_BATCH_CHUNK, treePaths.length);
      for (let j = i; j < end; j++) {
        cacheinfoArgs.push('--cacheinfo', `100644,${blobShas[j]},${treePaths[j]}`);
      }
      await sg
        .env({ GIT_DIR: shadow.gitDir, GIT_INDEX_FILE: tmpIndex })
        .raw('update-index', '--add', ...cacheinfoArgs);
    }

    const treeSha = (
      await sg.env({ GIT_DIR: shadow.gitDir, GIT_INDEX_FILE: tmpIndex }).raw('write-tree')
    ).trim();

    let parentSha: string | null = null;
    try {
      parentSha = (await sg.raw('rev-parse', ref)).trim();
    } catch {}

    const parkActorEntry: OkActorEntry = {
      v: 1,
      writer_id: SERVICE_WRITER.id,
      principal: null,
      agent_session: null,
      agent_type: null,
      client_name: null,
      client_version: null,
      label: null,
      display_name: SERVICE_WRITER.name,
      color_seed: SERVICE_WRITER.id,
      docs: documents.map((d) => d.docName),
    };
    const parkMessage = `${formatParkSubject(branch, newBranch ?? branch)}\n\n${formatOkActor(parkActorEntry)}`;
    const args = ['commit-tree', treeSha, '-m', parkMessage];
    if (parentSha) args.push('-p', parentSha);

    const commitSha = (
      await sg
        .env({
          GIT_DIR: shadow.gitDir,
          GIT_AUTHOR_NAME: 'openknowledge',
          GIT_AUTHOR_EMAIL: '[email protected]',
          GIT_COMMITTER_NAME: 'openknowledge',
          GIT_COMMITTER_EMAIL: '[email protected]',
        })
        .raw(...args)
    ).trim();

    await sg.raw('update-ref', ref, commitSha);
    return commitSha;
  } finally {
    try {
      rmSync(tmpIndex);
    } catch {}
    try {
      rmSync(tmpBlobDir, { recursive: true, force: true });
    } catch {}
  }
}

export async function readParkedState(
  shadow: ShadowHandle,
  branch: string,
  writerId: string,
  docName: string,
): Promise<{ markdown: string; diskSnapshot: string } | null> {
  const sg = shadowGit(shadow);
  const ref = `refs/wip/${branch}/${writerId}`;

  let refSha: string;
  try {
    refSha = (await sg.raw('rev-parse', ref)).trim();
  } catch {
    return null;
  }

  try {
    const msg = (await sg.raw('log', '-1', '--format=%s', refSha)).trim();
    if (!msg.startsWith('park:')) return null;

    const markdown = await sg.raw('show', `${refSha}:${docName}`);
    const diskSnapshot = await sg.raw('show', `${refSha}:.park-base/${docName}`);
    return { markdown, diskSnapshot };
  } catch (e) {
    log.error({ docName, ref, err: e }, `Failed to read parked state for ${docName} from ${ref}`);
    throw e;
  }
}

export interface SaveVersionResult {
  checkpointRef: string;
}

export interface SaveVersionOptions {
  checkpointKind?: { foldedRefs: number; trigger: AutoConsolidationTrigger };
  includeUpstream?: boolean;
  timeoutMs?: number;
  date?: string;
}

export async function saveVersion(
  shadow: ShadowHandle,
  contentRoot: string,
  writers: WriterIdentity[],
  branch = 'main',
  summary?: string,
  options?: SaveVersionOptions,
): Promise<SaveVersionResult> {
  return withSpan(
    'shadow.saveVersion',
    {
      attributes: {
        'shadow.branch': branch,
        'shadow.writer_count': writers.length,
        'shadow.checkpoint_kind': options?.checkpointKind ? 'auto-consolidation' : 'user',
      },
    },
    async () =>
      shadowOpGateFor(shadow).withMutator(() =>
        saveVersionInner(shadow, contentRoot, writers, branch, summary, options),
      ),
  );
}

async function saveVersionInner(
  shadow: ShadowHandle,
  contentRoot: string,
  writers: WriterIdentity[],
  branch = 'main',
  summary?: string,
  options?: SaveVersionOptions,
): Promise<SaveVersionResult> {
  const sg = shadowGit(shadow, options?.timeoutMs ? { timeoutMs: options.timeoutMs } : undefined);
  const gitPathspec = contentRoot || '.';

  const shadowTmpIndex = resolve(shadow.gitDir, `index-checkpoint-${randomUUID()}`);
  try {
    await sg
      .env({
        GIT_DIR: shadow.gitDir,
        GIT_WORK_TREE: shadow.workTree,
        GIT_INDEX_FILE: shadowTmpIndex,
      })
      .raw('add', ...pathspecArgs([gitPathspec]));
    await dropExcludedIndexEntries(
      shadowGit(shadow, {
        timeoutMs: Math.max(options?.timeoutMs ?? 0, CORPUS_STAGE_GIT_TIMEOUT_MS),
      }),
      shadow,
      shadowTmpIndex,
    );
    const shadowTreeSha = (
      await sg.env({ GIT_DIR: shadow.gitDir, GIT_INDEX_FILE: shadowTmpIndex }).raw('write-tree')
    ).trim();

    const foldWriters =
      options?.includeUpstream === false ? writers : [...writers, GIT_UPSTREAM_WRITER];
    const shadowParentShas: string[] = [];
    const wipSnapshotShas = new Map<string, string>();
    for (const w of foldWriters) {
      try {
        const sha = (await sg.raw('rev-parse', `refs/wip/${branch}/${w.id}`)).trim();
        shadowParentShas.push(sha);
        wipSnapshotShas.set(w.id, sha);
      } catch {}
    }
    const uniqueParents = [...new Set(shadowParentShas)];

    const chainAnchors = await resolveCheckpointChainAnchors(sg, branch);
    for (const anchor of chainAnchors) {
      if (!uniqueParents.includes(anchor)) uniqueParents.push(anchor);
    }
    if (chainAnchors.length > 1) {
      log.info(
        { branch, anchors: chainAnchors.length },
        '[shadow] checkpoint chain re-anchored across multiple dangling tips',
      );
    }

    const checkpointActorEntry: OkActorEntry = {
      v: 1,
      writer_id: SERVICE_WRITER.id,
      principal: null,
      agent_session: null,
      agent_type: null,
      client_name: null,
      client_version: null,
      label: null,
      display_name: SERVICE_WRITER.name,
      color_seed: SERVICE_WRITER.id,
      docs: [],
    };
    const checkpointSubject = summary?.trim() ? summary.trim() : 'Checkpoint version';
    let checkpointMessage = `${formatCheckpointSubject(checkpointSubject)}\n\n${formatOkActor(checkpointActorEntry)}`;
    if (options?.checkpointKind) {
      checkpointMessage += `\n${formatCheckpointBodyLine({
        kind: 'auto-consolidation',
        docName: null,
        size: null,
        metadata: {
          foldedRefs: options.checkpointKind.foldedRefs,
          trigger: options.checkpointKind.trigger,
        },
      })}`;
    }
    const checkpointArgs = ['commit-tree', shadowTreeSha, '-m', checkpointMessage];
    for (const p of uniqueParents) {
      checkpointArgs.push('-p', p);
    }

    const checkpointEnv: Record<string, string> = {
      GIT_DIR: shadow.gitDir,
      GIT_AUTHOR_NAME: 'openknowledge',
      GIT_AUTHOR_EMAIL: '[email protected]',
      GIT_COMMITTER_NAME: 'openknowledge',
      GIT_COMMITTER_EMAIL: '[email protected]',
    };
    if (options?.date) {
      checkpointEnv.GIT_AUTHOR_DATE = options.date;
      checkpointEnv.GIT_COMMITTER_DATE = options.date;
    }
    const checkpointSha = (await sg.env(checkpointEnv).raw(...checkpointArgs)).trim();

    const checkpointRef = `refs/checkpoints/${branch}/${checkpointSha}`;
    await sg.raw('update-ref', checkpointRef, checkpointSha);

    await resetFoldedWipRefs(sg, branch, foldWriters, wipSnapshotShas);

    return { checkpointRef };
  } finally {
    try {
      rmSync(shadowTmpIndex);
    } catch {}
  }
}

export async function resetFoldedWipRefs(
  sg: ReturnType<typeof shadowGit>,
  branch: string,
  writers: readonly { id: string }[],
  wipSnapshotShas: ReadonlyMap<string, string>,
): Promise<void> {
  for (const w of writers) {
    const ref = `refs/wip/${branch}/${w.id}`;
    const expected = wipSnapshotShas.get(w.id);
    if (expected === undefined) continue;
    try {
      await sg.raw('update-ref', '-d', ref, expected);
    } catch {}
  }
}