Skip to content

Turbocharge Claude Code, Cursor, Codex, Gemini & every coding agent: faster, cheaper, with contextual understanding specific to your codebase.

20.0 KB
Raw Blame
/**
 * `init` — build the `.context/` graph from a code repository.
 *
 * Pipeline (no database, no embeddings):
 *   1. Walk the repo for source files.
 *   2. Summarize each file to prose (one LLM call per file, cached by content hash).
 *   3. Synthesize a CURATED node set from the labeled summaries (the synthesizer
 *      decides granularity: subsystems, notable files, and concepts). This is
 *      one LLM call per batch of summaries; the batches are independent, so they
 *      run concurrently, and each is cached by content. Where the batches are cut,
 *      and why that decides what the next build costs, is `./batches.ts`.
 *   4. Resolve node names → slugs and links → edges; attribute each node to its
 *      source files so staleness stays exact.
 *   5. Write one markdown file per node (preserving human notes) + a manifest.
 */
import { readFileSync, writeFileSync, renameSync, mkdirSync, existsSync } from "node:fs";
import { join, resolve } from "node:path";
import { walkDir } from "../ingest/fs.js";
import { filterByOnlyDirs } from "../graph/source-files.js";
import { readFingerprint } from "../graph/fingerprint.js";
import { contentHash } from "../util/id.js";
import { relPosix } from "../util/paths.js";
import { readSourceFile } from "../util/source.js";
import { readFollowNestedRepos, readFollowSubmodules, readIncludeDirs } from "../util/state.js";
import type { Summarizer } from "../ai/summarize.js";
import { LlmFailureGate } from "../ai/failure.js";
import type { FileSummary, SynthNode, Synthesizer } from "../ai/synthesize.js";
import { planSynthesis, type SynthesisBatch } from "./batches.js";
import {
  CACHE_DIR,
  MANIFEST_VERSION,
  contextDirFor,
  deleteNode,
  digestSources,
  existingNodeSlugs,
  slugify,
  writeManifest,
  writeNode,
  type ContextNode,
  type Manifest,
  type NodeLink,
  type SourceRef,
} from "./node-file.js";

/** Extensions treated as source code. */
export const CODE_EXTENSIONS = [
  ".ts", ".tsx", ".js", ".jsx", ".mjs", ".cjs",
  ".py", ".go", ".rs", ".java", ".kt", ".scala",
  ".rb", ".php", ".c", ".h", ".cpp", ".hpp", ".cc",
  ".cs", ".swift", ".gleam", ".sql", ".sh", ".proto",
];

/**
 * Synthesis calls in flight at once. Small on purpose, and separate from phase 1's
 * `-j`: there each call is one small file summary, here each is a whole batch of
 * summaries — a long, expensive request, of which a big repo has a handful rather
 * than hundreds.
 */
const DEFAULT_SYNTH_CONCURRENCY = 4;

export interface BuildProgress {
  phase: "summarize" | "synthesize" | "write";
  index: number;
  total: number;
  file: string;
}

export interface BuildOptions {
  /** Override the output dir (default: `<root>/.context`). */
  contextDir?: string;
  /** Extensions to treat as code. Default: {@link CODE_EXTENSIONS}. */
  extensions?: string[];
  /** Repo-relative directory prefixes to limit the concept pass (`--only-dir`).
   * Same prefix semantics as the wiring walk. When omitted, falls back to the
   * whitelist recorded in the graph fingerprint (mirrors `checkGraph`). */
  onlyDirs?: string[];
  /** Human label for the model, recorded in the manifest (e.g. "openrouter:openai/gpt-4o-mini"). */
  model: string;
  summarizer: Summarizer;
  synthesizer: Synthesizer;
  /** Files summarized in parallel during phase 1. Default 8. Raised via `graft build -j`. */
  concurrency?: number;
  /** Synthesis batches synthesized in parallel during phase 2. Default
   * {@link DEFAULT_SYNTH_CONCURRENCY}. Raised via `graft build --synth-concurrency`. */
  synthConcurrency?: number;
  onProgress?: (info: BuildProgress) => void;
}

export interface BuildResult {
  contextDir: string;
  files: number;
  summarized: number;
  cached: number;
  batches: number;
  nodes: number;
  links: number;
  errors: string[];
  /** Files whose summary call failed, and files never attempted once the pass gave
   * up — reported as data so the CLI can exit non-zero without reading messages (#127).
   * A unit is a file in phase 1 and a whole batch of them in phase 2, and the
   * counts cover both: the failure gate is shared. */
  failedFiles: number;
  skippedFiles: number;
  /** Why the LLM passes stopped early, when they did. */
  fatal?: string;
}

/** CLI/API `--only-dir` wins; otherwise the whitelist recorded in the graph
 * fingerprint — the same source `checkGraph` / `probeDrift` use, so a later
 * `--deep` without the flag still skips files the wiring pass excluded. */
function resolveOnlyDirs(outDir: string, explicit?: readonly string[]): Set<string> | undefined {
  const list = explicit && explicit.length > 0 ? explicit : (readFingerprint(outDir)?.onlyDirs ?? []);
  return list.length > 0 ? new Set(list) : undefined;
}

/**
 * Files the concept pass summarizes (and `checkContext` re-hashes): the same
 * walk as before (`--include-dir` / submodule flags from state, minus the
 * output dir), then {@link filterByOnlyDirs}. Shared so build and check cannot
 * disagree about what "current" means under a whitelist.
 */
export function listContextFiles(
  root: string,
  outDir: string,
  exts: readonly string[],
  explicitOnlyDirs?: readonly string[],
): string[] {
  const walked = walkDir(root, readIncludeDirs(root), {
    followSubmodules: readFollowSubmodules(root),
    followNestedRepos: readFollowNestedRepos(root),
  })
    .filter((f) => exts.some((e) => f.toLowerCase().endsWith(e)))
    .filter((f) => !f.startsWith(outDir));
  return filterByOnlyDirs(walked, root, resolveOnlyDirs(outDir, explicitOnlyDirs));
}

/** The gitignored LLM-call cache: per-file summaries + per-batch synthesis. */
interface BuildCache {
  summaries: Record<string, { hash: string; summary: string }>;
  synth: Record<string, SynthNode[]>;
}

interface FileWork {
  rel: string;
  hash: string;
  summary?: string;
}

/** A node under construction, before its digest/links are finalized. */
interface NodeDraft {
  name: string;
  slug: string;
  type: string;
  summary: string;
  sources: Map<string, string>; // path → hash
  links: Map<string, NodeLink>; // "to|relation" → link
}

/** What one synthesis batch ended up contributing, for the per-batch log line. */
interface BatchOutcome {
  nodes: SynthNode[];
  /** Served from the cache, so no call was made for it. */
  cached: boolean;
  state: "ok" | "failed" | "skipped";
}

export async function buildContext(dir: string, opts: BuildOptions): Promise<BuildResult> {
  const root = resolve(dir);
  const outDir = contextDirFor(root, opts.contextDir);
  const exts = opts.extensions ?? CODE_EXTENSIONS;
  // Read the same persisted walk choices as the Tier-1 wiring graph, so the
  // Tier-2 concept pipeline sees exactly the same directories and submodules.
  // `--only-dir` is applied with the same prefix match as wiring (CLI/API, else
  // the fingerprint) so out-of-scope files are never summarized or synthesized.
  const files = listContextFiles(root, outDir, exts, opts.onlyDirs);

  const cache = loadCache(outDir);
  // Flush the summary cache to disk during phase 1 so a build interrupted
  // partway (session/rate limit, crash, Ctrl-C) resumes without re-summarizing
  // the files it already did. Throttled to keep disk churn negligible; on the
  // next run every already-summarized file is a content-hash cache hit ($0).
  let lastFlush = Date.now();
  const flushEveryMs = summaryCheckpointMs();
  const maybeFlush = (): void => {
    const now = Date.now();
    if (now - lastFlush < flushEveryMs) return;
    lastFlush = now;
    saveCache(outDir, cache);
  };
  const result: BuildResult = {
    contextDir: outDir,
    files: 0,
    summarized: 0,
    cached: 0,
    batches: 0,
    nodes: 0,
    links: 0,
    errors: [],
    failedFiles: 0,
    skippedFiles: 0,
  };

  // Phase 1: summarize each file, concurrent, content-hash cached.
  //
  // The gate is shared with the crux pass (`graph/enrich.ts`): once the provider has
  // clearly stopped serving — a spent quota, a rejected key — the remaining files are
  // not attempted. This pass is where #127's 1,617 doomed calls were spent, one per
  // file, before the build exited 0.
  const gate = new LlmFailureGate();
  const work = await mapWithConcurrency(files, Math.max(1, opts.concurrency ?? 8), async (file, i): Promise<FileWork | undefined> => {
    const rel = relPosix(root, file);
    opts.onProgress?.({ phase: "summarize", index: i, total: files.length, file: rel });
    let code: string;
    try {
      const decoded = readSourceFile(file);
      if (decoded === null) return undefined; // unsupported encoding (e.g. UTF-16BE) — skip, not an error
      code = decoded;
    } catch (err) {
      result.errors.push(`${rel}: ${errMsg(err)}`);
      return undefined;
    }
    const hash = contentHash(code);
    const hit = cache.summaries[rel];
    if (hit && hit.hash === hash) {
      result.cached++;
      return { rel, hash, summary: hit.summary };
    }
    // A cache hit is still served after the gate closes (it costs nothing) — only
    // the call is skipped.
    if (gate.stopped) {
      gate.skip();
      return { rel, hash };
    }
    try {
      const summary = await opts.summarizer.summarize(code, { path: rel });
      cache.summaries[rel] = { hash, summary };
      result.summarized++;
      maybeFlush();
      gate.succeeded();
      return { rel, hash, summary };
    } catch (err) {
      const message = errMsg(err);
      result.errors.push(`${rel}: ${message}`);
      gate.record(message);
      return { rel, hash }; // covered (counts against staleness) but not summarized
    }
  });
  result.failedFiles = gate.failed;
  result.skippedFiles = gate.skipped;
  result.fatal = gate.fatal;

  // Phase 1 done: persist every summary before the (also LLM-backed) synthesis
  // phase, so an interruption there never discards phase-1 work.
  saveCache(outDir, cache);

  const processed = work.filter((w): w is FileWork => w !== undefined);
  result.files = processed.length;
  const hashByPath = new Map(processed.map((w) => [w.rel, w.hash]));

  // Phase 2: synthesize curated nodes from the summaries, batched + cached.
  const summarized: FileSummary[] = processed
    .filter((w): w is FileWork & { summary: string } => Boolean(w.summary))
    .map((w) => ({ path: w.rel, summary: w.summary }))
    .sort((a, b) => a.path.localeCompare(b.path));
  const batches = planSynthesis(summarized, hashByPath);
  result.batches = batches.length;

  /**
   * One batch's synthesis, cached and failure-gated exactly as phase 1 is: a cache
   * hit is served even once the gate has closed (it costs nothing), only the call
   * is skipped.
   */
  async function synthesizeBatch(batch: SynthesisBatch, b: number): Promise<BatchOutcome> {
    let nodes = cache.synth[batch.key];
    // An empty array is a miss, not a hit: caching [] made a silent empty
    // synthesis permanent, the same trap #177 closed for the meaning pass (#129).
    if (Array.isArray(nodes) && nodes.length > 0) return { nodes, cached: true, state: "ok" };
    if (gate.stopped) {
      gate.skip();
      return { nodes: [], cached: false, state: "skipped" };
    }
    try {
      nodes = await opts.synthesizer.synthesize(batch.files);
    } catch (err) {
      // Recorded, not thrown. A provider that has stopped serving fails every
      // batch the same way, so the gate turns the first of those into a loud
      // fatal and the rest into skipped calls, and the CLI already reports a
      // concept pass with errors as a degraded meaning tier. Rethrowing instead
      // would write no graph at all and leave the synthesis cache unsaved, so the
      // next run would re-synthesize every batch that did work.
      const message = errMsg(err);
      result.errors.push(`synthesis batch ${b + 1}/${batches.length}: ${message}`);
      gate.record(message);
      return { nodes: [], cached: false, state: "failed" };
    }
    if (nodes.length > 0) cache.synth[batch.key] = nodes;
    else delete cache.synth[batch.key];
    gate.succeeded();
    return { nodes, cached: false, state: "ok" };
  }

  // A batch is an independent unit of work — its call sees only its own summaries,
  // and nothing is merged until every call is in (phase 3) — so the batches run
  // concurrently, the way phase 1's per-file calls already do. The slow part is
  // the round-trip, and a big repo otherwise serializes every one of them.
  //
  // Results are collected in BATCH order, whatever order the calls finish in.
  // That is what makes this safe: the merge below is order-sensitive (first name
  // registered wins, longest summary wins), so consuming batches as they completed
  // would make the graph a function of network timing, and two builds of the same
  // repo could disagree.
  let synthesized = 0;
  const outcomes = await mapWithConcurrency(batches, Math.max(1, opts.synthConcurrency ?? DEFAULT_SYNTH_CONCURRENCY), async (batch, b): Promise<BatchOutcome> => {
    const outcome = await synthesizeBatch(batch, b);
    // Reported on completion, like the crux pass (`graph/enrich.ts`): with several
    // calls in flight the next batch is not the one that is running, so `index`
    // counts finished batches and `file` names the one that just landed. The CLI
    // prints them as one overwritten line, which only reads as progress if the
    // counter never goes backwards.
    opts.onProgress?.({ phase: "synthesize", index: synthesized++, total: batches.length, file: `batch ${b + 1}` });
    return outcome;
  });

  // One log line per batch, in batch order: printed as the calls land it would
  // report which call was fastest, which is noise on a run whose output does not
  // depend on how the calls interleaved.
  const synthNodes: SynthNode[] = [];
  for (const [b, out] of outcomes.entries()) {
    const links = out.nodes.reduce((n, node) => n + node.links.length, 0);
    const note = out.cached ? " (cached)" : out.state === "failed" ? " (failed)" : out.state === "skipped" ? " (skipped)" : "";
    console.error(`  synthesis batch ${b + 1}/${batches.length}: ${out.nodes.length} nodes, ${links} links${note}`);
    synthNodes.push(...out.nodes);
  }
  // The gate is shared with phase 1, so these counts now cover both LLM passes.
  result.failedFiles = gate.failed;
  result.skippedFiles = gate.skipped;
  result.fatal = gate.fatal;

  // Drop cache entries for batches we no longer produce, so it can't grow forever.
  // Skip empty arrays so a failed batch is retried on the next --deep, not frozen.
  cache.synth = Object.fromEntries(
    batches.flatMap(({ key }) => {
      const v = cache.synth[key];
      return v && v.length > 0 ? [[key, v] as [string, SynthNode[]]] : [];
    }),
  );
  saveCache(outDir, cache);

  // Phase 3: merge synth nodes by slug, building a name→slug resolution table.
  const drafts = new Map<string, NodeDraft>();
  const nameToSlug = new Map<string, string>();
  for (const n of synthNodes) {
    const slug = slugify(n.name);
    let draft = drafts.get(slug);
    if (!draft) {
      draft = { name: n.name, slug, type: n.type || "concept", summary: n.summary ?? "", sources: new Map(), links: new Map() };
      drafts.set(slug, draft);
    }
    if ((n.summary?.length ?? 0) > draft.summary.length) draft.summary = n.summary;
    if (n.type && draft.type === "concept") draft.type = n.type;
    for (const src of n.sources) {
      const hash = hashByPath.get(src);
      if (hash) draft.sources.set(src, hash);
    }
    registerName(nameToSlug, n.name, slug);
  }

  // Phase 4: resolve links to slugs (both endpoints must be defined nodes).
  for (const n of synthNodes) {
    const from = drafts.get(slugify(n.name));
    if (!from) continue;
    for (const link of n.links) {
      const to = resolveSlug(nameToSlug, link.to);
      if (!to || to === from.slug) continue;
      const key = `${to}|${link.relation}`;
      if (!from.links.has(key)) from.links.set(key, { to, relation: link.relation, description: link.description });
    }
  }

  // Concept nodes the model didn't attribute to files inherit the provenance of
  // the nodes they link to, so they still go stale when their subject changes.
  for (const draft of drafts.values()) {
    if (draft.sources.size > 0) continue;
    for (const link of draft.links.values()) {
      const target = drafts.get(link.to);
      if (target) for (const [p, h] of target.sources) draft.sources.set(p, h);
    }
  }

  // Phase 5: finalize, reconcile with disk, write.
  const nodes: ContextNode[] = [...drafts.values()].map((d) => {
    const sources: SourceRef[] = [...d.sources.entries()]
      .map(([path, hash]) => ({ path, hash }))
      .sort((a, b) => a.path.localeCompare(b.path));
    return {
      name: d.name,
      slug: d.slug,
      type: d.type || "concept",
      summary: d.summary,
      sources,
      sourcesDigest: digestSources(sources),
      links: [...d.links.values()].sort((a, b) => a.to.localeCompare(b.to)),
      human: "",
    };
  });
  nodes.sort((a, b) => a.slug.localeCompare(b.slug));

  const liveSlugs = new Set(nodes.map((n) => n.slug));
  for (const slug of existingNodeSlugs(outDir)) {
    if (!liveSlugs.has(slug)) deleteNode(outDir, slug);
  }
  for (let i = 0; i < nodes.length; i++) {
    opts.onProgress?.({ phase: "write", index: i, total: nodes.length, file: nodes[i].slug });
    writeNode(outDir, nodes[i]);
  }
  result.nodes = nodes.length;
  result.links = nodes.reduce((n, node) => n + node.links.length, 0);
  // Failure mode 2 (#129): a model can emit real tool_calls whose quality
  // collapsed (many batches, zero links) with nothing per-batch in the log.
  if (result.links === 0 && result.batches > 1) {
    console.error(
      `⚠ synthesis produced 0 links across ${result.batches} batches — the model may be degrading; see per-batch counts above`,
    );
  }

  // Manifest: authoritative file→hash map (every processed file) + node roster.
  const fileRefs: SourceRef[] = processed
    .map((w) => ({ path: w.rel, hash: w.hash }))
    .sort((a, b) => a.path.localeCompare(b.path));
  const manifest: Manifest = {
    version: MANIFEST_VERSION,
    model: opts.model,
    repoDigest: digestSources(fileRefs),
    files: fileRefs,
    nodes: nodes.map((n) => ({
      slug: n.slug,
      name: n.name,
      type: n.type,
      sources: n.sources.map((s) => s.path),
      sourcesDigest: n.sourcesDigest,
    })),
  };
  writeManifest(outDir, manifest);

  return result;
}

function registerName(table: Map<string, string>, name: string, slug: string): void {
  const key = name.trim().toLowerCase();
  if (key && !table.has(key)) table.set(key, slug);
}

function resolveSlug(table: Map<string, string>, name: string): string | undefined {
  return table.get(name.trim().toLowerCase());
}

function errMsg(err: unknown): string {
  return err instanceof Error ? err.message : String(err);
}

function cachePath(outDir: string): string {
  return join(outDir, CACHE_DIR, "summaries.json");
}

function loadCache(outDir: string): BuildCache {
  const path = cachePath(outDir);
  if (existsSync(path)) {
    try {
      const parsed = JSON.parse(readFileSync(path, "utf8")) as Partial<BuildCache>;
      return { summaries: parsed.summaries ?? {}, synth: parsed.synth ?? {} };
    } catch {
      /* fall through to empty */
    }
  }
  return { summaries: {}, synth: {} };
}

function saveCache(outDir: string, cache: BuildCache): void {
  const path = cachePath(outDir);
  mkdirSync(join(outDir, CACHE_DIR), { recursive: true });
  // Atomic: a kill mid-write can't leave a truncated (unparseable) cache that
  // would throw away every prior summary on the next load.
  const tmp = `${path}.tmp`;
  writeFileSync(tmp, JSON.stringify(cache, null, 2));
  renameSync(tmp, path);
}

/** Min interval between phase-1 cache flushes. Env seam for tests. */
function summaryCheckpointMs(): number {
  const raw = Number(process.env.GRAFT_SUMMARY_CHECKPOINT_MS);
  return Number.isFinite(raw) && raw >= 0 ? raw : 15_000;
}

/** Run `fn` over `items` with at most `limit` in flight, preserving order. */
async function mapWithConcurrency<T, R>(
  items: T[],
  limit: number,
  fn: (item: T, index: number) => Promise<R>,
): Promise<R[]> {
  const results: R[] = new Array(items.length);
  let next = 0;
  const workers = Array.from({ length: Math.min(limit, items.length) }, async () => {
    while (next < items.length) {
      const i = next++;
      results[i] = await fn(items[i], i);
    }
  });
  await Promise.all(workers);
  return results;
}