Turbocharge Claude Code, Cursor, Codex, Gemini & every coding agent: faster, cheaper, with contextual understanding specific to your codebase.
TypeScript ai-agentsanthropicclaude-codeclicode-graphcodexcontext-engineeringcursordeveloper-toolsgeminiknowledge-graphllmmcpmcp-serveropen-sourceopenaitree-sitter
feat(ask): build-time token/IDF index sidecar — ask stops re-tokenizing the corpus per query
Shrish Dwivedia3c1bdff41cd
diff --git a/src/ask/ask.ts b/src/ask/ask.tsindex b9f2abd5..1f938eaa 100644--- a/src/ask/ask.ts+++ b/src/ask/ask.ts@@ -20,6 +20,7 @@ import { contextDirFor } from "../context/node-file.js";import { readGraph, wiringPath } from "../graph/write.js";import type { EdgeV1, GraphV1, NodeV1, Relation } from "../graph/types.js";import { personalizedPageRank } from "./graphrank.js";+import { counts, readAskIndex, tokenize, type AskIndex, type AskIndexDoc } from "./index-file.js";export interface AskHit {kind: "concept" | "symbol" | "caller" | "callee";@@ -53,28 +54,6 @@ export interface AskResult {saved?: { files: number; baselineChars: number };}-const STOP = new Set([- "the", "a", "an", "of", "to", "in", "is", "are", "how", "does", "do", "what",- "where", "which", "that", "this", "it", "for", "on", "and", "or", "with",- "i", "we", "get", "set", "use", "used", "using", "when", "why", "can",-]);--/** Split prose + identifiers into lowercased subword tokens (camelCase, snake, kebab). */-function tokenize(text: string): string[] {- return text- .replace(/([a-z0-9])([A-Z])/g, "$1 $2") // camelCase → camel Case- .toLowerCase()- .split(/[^a-z0-9]+/)- .filter((t) => t.length > 1 && !STOP.has(t));-}--/** Term-frequency count map. */-function counts(tokens: string[]): Map<string, number> {- const m = new Map<string, number>();- for (const t of tokens) m.set(t, (m.get(t) ?? 0) + 1);- return m;-}-/** First real prose line of a node body (skips headings, markers, blanks). */function firstProse(body: string): string {for (const raw of body.split("\n")) {@@ -88,6 +67,10 @@ function firstProse(body: string): string {interface Corpus {concepts: Array<{ slug: string; name: string; sources: string[]; related: string[]; snippet: string; text: string }>;graph: GraphV1 | null;+ /** Build-time token/df sidecar (`.graph/index.json`), or null when absent,+ * unparseable, an unknown version, or stale (checked against `graph.nodes.length`+ * by the caller — see `lexical()`). Null means: fall back to live tokenization. */+ askIndex: AskIndex | null;}function loadCorpus(outDir: string): Corpus {@@ -114,7 +97,7 @@ function loadCorpus(outDir: string): Corpus {});}}- return { concepts, graph: readGraph(wiringPath(outDir)) };+ return { concepts, graph: readGraph(wiringPath(outDir)), askIndex: readAskIndex(outDir) };}/** Score a document's token counts against the query counts (name field@@ -141,14 +124,31 @@ function score(* documents (concept nodes + symbol nodes) contain each token; N is the corpus* size. `log(1 + N/(1+df))` is the smoothed standard form: monotonically* decreasing in df, always positive, ~0 extra weight for a token in every doc. */+function idfFromDf(df: Map<string, number>, n: number): Map<string, number> {+ const idf = new Map<string, number>();+ for (const [t, d] of df) idf.set(t, Math.log(1 + n / (1 + d)));+ return idf;+}+function computeIdf(docBags: Array<Set<string>>): Map<string, number> {- const n = docBags.length;const df = new Map<string, number>();for (const bag of docBags)for (const t of bag) df.set(t, (df.get(t) ?? 0) + 1);- const idf = new Map<string, number>();- for (const [t, d] of df) idf.set(t, Math.log(1 + n / (1 + d)));- return idf;+ return idfFromDf(df, docBags.length);+}++/** Same idf as {@link computeIdf}, but the symbol half of `df` and the doc+ * count come from the build-time sidecar instead of tokenizing the corpus+ * live; only the concept bags (always tokenized live — there are only dozens)+ * are folded in here, the same way `computeIdf` would fold them into the full+ * docBags array. `df` is a commutative sum and `n` is the same total document+ * count either way, so this produces byte-identical idf values to `computeIdf`+ * run over concept-bags + live-tokenized-symbol-bags. */+function computeIdfFromIndex(index: AskIndex, conceptBags: Array<Set<string>>): Map<string, number> {+ const df = new Map(index.df);+ for (const bag of conceptBags)+ for (const t of bag) df.set(t, (df.get(t) ?? 0) + 1);+ return idfFromDf(df, index.docCount + conceptBags.length);}/** BM25 term score for a document field, used for the (potentially large) body@@ -273,25 +273,47 @@ function lexical(query: string, corpus: Corpus, limit: number, graphRank: booleaname: counts(tokenize(c.name)),body: counts(tokenize(c.text)),}));++ // A build-time sidecar (`.graph/index.json`) is usable only when it covers+ // every node in the current graph: `docs.length === graph.nodes.length` is a+ // cheap staleness guard (an id-level check below catches a same-count but+ // mismatched set, e.g. a hand-edited or corrupt sidecar). Anything short of+ // that — missing, unparseable, unknown version, wrong count, mismatched ids —+ // falls back to live tokenization so results never depend on the sidecar+ // existing; see `readAskIndex`'s own null-on-anything-off contract.+ let docById: Map<string, AskIndexDoc> | null = null;+ if (corpus.askIndex && graph && corpus.askIndex.docs.length === graph.nodes.length) {+ const map = new Map(corpus.askIndex.docs.map((d) => [d.id, d]));+ if (graph.nodes.every((n) => map.has(n.id))) docById = map;+ }+ const askIndex = docById ? corpus.askIndex : null;+// Symbols AND file nodes are scored: a symbol's body_text is its definition;// a file's body_text is the module-level residual (imports/constants not in// any symbol). Including files closes the recall gap where a gold file's only// query-relevant text lives outside every function/class.- const symbolDocs = (graph?.nodes ?? []).map((n) => ({- n,- name: counts(tokenize(n.name)),- path: counts(tokenize(n.path)),- // The body (indexed at build) joins the signature + summary as the low-weight- // body field, so a term that appears only in the code — not the name/signature- // — still makes the node findable. IDF (from these same bags) keeps a word- // common across many bodies from dominating.- body: counts(tokenize(`${n.signature ?? ""} ${n.summary ?? ""} ${n.body_text ?? ""}`)),- }));- const docBags: Array<Set<string>> = [];- for (const d of conceptDocs) docBags.push(new Set([...d.name.keys(), ...d.body.keys()]));- for (const d of symbolDocs)- docBags.push(new Set([...d.name.keys(), ...d.path.keys(), ...d.body.keys()]));- const idf = computeIdf(docBags);+ const symbolDocs = (graph?.nodes ?? []).map((n) => {+ const d = docById?.get(n.id);+ if (d) return { n, name: new Map(d.name), path: new Map(d.path), body: new Map(d.body) };+ return {+ n,+ name: counts(tokenize(n.name)),+ path: counts(tokenize(n.path)),+ // The body (indexed at build) joins the signature + summary as the low-weight+ // body field, so a term that appears only in the code — not the name/signature+ // — still makes the node findable. IDF (from these same bags) keeps a word+ // common across many bodies from dominating.+ body: counts(tokenize(`${n.signature ?? ""} ${n.summary ?? ""} ${n.body_text ?? ""}`)),+ };+ });++ const conceptBags = conceptDocs.map((d) => new Set([...d.name.keys(), ...d.body.keys()]));+ // IDF must see the SAME inputs either way: with a sidecar, its `df` (stored+ // WITHOUT concepts) plus these live concept bags reproduces exactly what+ // `computeIdf` below would compute from concept+symbol bags tokenized live.+ const idf = askIndex+ ? computeIdfFromIndex(askIndex, conceptBags)+ : computeIdf([...conceptBags, ...symbolDocs.map((d) => new Set([...d.name.keys(), ...d.path.keys(), ...d.body.keys()]))]);// ── Concepts (prose nodes; not in the wiring graph) ──const conceptHits: AskHit[] = [];@@ -318,9 +340,11 @@ function lexical(query: string, corpus: Corpus, limit: number, graphRank: booleafor (const v of m.values()) s += v;return s;};- const avgBodyLen = symbolDocs.length- ? symbolDocs.reduce((a, d) => a + bodyLen(d.body), 0) / symbolDocs.length- : 0;+ const avgBodyLen = askIndex+ ? askIndex.avgBodyLen+ : symbolDocs.length+ ? symbolDocs.reduce((a, d) => a + bodyLen(d.body), 0) / symbolDocs.length+ : 0;const lex = new Map<string, number>(); // node id → lexical score (>0 only)let maxLex = 0;for (const { n, name, path, body } of symbolDocs) {diff --git a/src/ask/index-file.ts b/src/ask/index-file.tsnew file mode 100644index 00000000..e092fec2--- /dev/null+++ b/src/ask/index-file.ts@@ -0,0 +1,151 @@+/**+ * Build-time sidecar for `graft ask` — `<outDir>/.graph/index.json`.+ *+ * `ask`'s lexical pass tokenizes every symbol node's name/path/body on every+ * query; at 32k nodes that re-tokenization is ~45% of query time (profiled).+ * `graft build` writes this sidecar once, alongside `wiring.json`, with the+ * token→count bags per node plus the corpus-wide document frequencies, so a+ * query just reads counts instead of re-splitting every node's text.+ *+ * `tokenize`/`counts` live here (not duplicated in `ask.ts`) so build-time and+ * query-time text-splitting are provably the same function — the sidecar can+ * only be a correct cache of `ask.ts`'s own math if both sides call the same+ * code. `ask.ts` imports both back from here.+ *+ * Concept (markdown) docs are NOT part of this sidecar — there are only dozens+ * of them, they're still tokenized live at query time, and their doc-frequency+ * contribution is folded into the stored `df` at query time (see `ask.ts`),+ * which is why `df` here counts symbol/file nodes only.+ */+import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs";+import { dirname, join } from "node:path";+import type { GraphV1 } from "../graph/types.js";++/** Words too common/short to carry query intent — dropped before scoring. */+const STOP = new Set([+ "the", "a", "an", "of", "to", "in", "is", "are", "how", "does", "do", "what",+ "where", "which", "that", "this", "it", "for", "on", "and", "or", "with",+ "i", "we", "get", "set", "use", "used", "using", "when", "why", "can",+]);++/** Split prose + identifiers into lowercased subword tokens (camelCase, snake, kebab).+ * The single source of truth for tokenization — shared by build-time indexing+ * (this file) and query-time fallback (`ask.ts`) so the sidecar is a provably+ * exact cache of the live path. */+export function tokenize(text: string): string[] {+ return text+ .replace(/([a-z0-9])([A-Z])/g, "$1 $2") // camelCase → camel Case+ .toLowerCase()+ .split(/[^a-z0-9]+/)+ .filter((t) => t.length > 1 && !STOP.has(t));+}++/** Term-frequency count map. */+export function counts(tokens: string[]): Map<string, number> {+ const m = new Map<string, number>();+ for (const t of tokens) m.set(t, (m.get(t) ?? 0) + 1);+ return m;+}++/** One node's token bags, JSON-friendly (`Map` → sorted `[token, count][]`). */+export interface AskIndexDoc {+ id: string;+ name: [string, number][];+ path: [string, number][];+ body: [string, number][];+}++/** The build-time sidecar. `df`/`docCount` cover symbol+file nodes only (no+ * concepts — see module docstring); `avgBodyLen` is the BM25 corpus average. */+export interface AskIndex {+ version: 1;+ avgBodyLen: number;+ df: [string, number][];+ docCount: number;+ docs: AskIndexDoc[];+}++const ASK_INDEX_DIR = ".graph";+const ASK_INDEX_FILE = "index.json";++/** Absolute path to the ask sidecar for a context dir: `<dir>/.graph/index.json`. */+export function askIndexPath(outDir: string): string {+ return join(outDir, ASK_INDEX_DIR, ASK_INDEX_FILE);+}++function pairs(m: Map<string, number>): [string, number][] {+ return [...m.entries()];+}++/** Sum of a token→count bag's counts (a document's field length). */+function bagLen(p: [string, number][]): number {+ let s = 0;+ for (const [, c] of p) s += c;+ return s;+}++/**+ * Tokenize every node in `graph` (exactly as `ask.ts`'s lexical pass does) and+ * write the resulting bags + document frequencies to `<outDir>/.graph/index.json`.+ * Returns the path written. Deterministic: nodes are indexed in id order, so an+ * unchanged graph produces a byte-identical sidecar.+ */+export function writeAskIndex(outDir: string, graph: GraphV1): string {+ const nodes = [...graph.nodes].sort((a, b) => a.id.localeCompare(b.id));+ const docs: AskIndexDoc[] = [];+ const df = new Map<string, number>();++ for (const n of nodes) {+ const name = counts(tokenize(n.name));+ const path = counts(tokenize(n.path));+ const body = counts(+ tokenize(`${n.signature ?? ""} ${n.summary ?? ""} ${n.body_text ?? ""}`),+ );+ docs.push({ id: n.id, name: pairs(name), path: pairs(path), body: pairs(body) });++ const bag = new Set<string>([...name.keys(), ...path.keys(), ...body.keys()]);+ for (const t of bag) df.set(t, (df.get(t) ?? 0) + 1);+ }++ const avgBodyLen = docs.length+ ? docs.reduce((a, d) => a + bagLen(d.body), 0) / docs.length+ : 0;++ const index: AskIndex = {+ version: 1,+ avgBodyLen,+ df: pairs(df),+ docCount: nodes.length,+ docs,+ };++ const outPath = askIndexPath(outDir);+ mkdirSync(dirname(outPath), { recursive: true });+ writeFileSync(outPath, JSON.stringify(index) + "\n");+ return outPath;+}++/** Read the ask sidecar. Returns null on a missing file, unparseable JSON, an+ * unrecognized shape, or an unknown `version` — any of which means the caller+ * should fall back to live tokenization, never crash. */+export function readAskIndex(outDir: string): AskIndex | null {+ const path = askIndexPath(outDir);+ if (!existsSync(path)) return null;+ try {+ const raw = JSON.parse(readFileSync(path, "utf8"));+ if (+ !raw ||+ typeof raw !== "object" ||+ raw.version !== 1 ||+ typeof raw.docCount !== "number" ||+ typeof raw.avgBodyLen !== "number" ||+ !Array.isArray(raw.df) ||+ !Array.isArray(raw.docs)+ ) {+ return null;+ }+ return raw as AskIndex;+ } catch {+ return null;+ }+}diff --git a/src/graph/build.ts b/src/graph/build.tsindex 09ca6575..b7606b65 100644--- a/src/graph/build.ts+++ b/src/graph/build.ts@@ -16,6 +16,7 @@ import { resolveEdges, type GoModule } from "./resolve.js";import { enrichGraph, type EnrichStats } from "./enrich.js";import { readGraph, writeGraph, wiringPath } from "./write.js";import { writeCards, writeIndex, writeCovers } from "./cards.js";+import { writeAskIndex } from "../ask/index-file.js";import type { GraphV1, Kind, NodeV1, Relation } from "./types.js";import type { CruxSummarizer } from "../ai/crux.js";@@ -136,6 +137,10 @@ export async function buildGraph(};const graphPath = writeGraph(graph, outDir);+ // `ask`'s token/IDF sidecar — moves per-query corpus tokenization to build+ // time (~45% of query time on a 32k-node graph, profiled). Lands next to+ // wiring.json; `ask` falls back to live tokenization when it's absent/stale.+ writeAskIndex(outDir, graph);// Tier-2 passive surface: project the nodes into per-file markdown cards, and// refresh the INDEX roster. Pure projection — no LLM, no network.diff --git a/test/ask-index.test.ts b/test/ask-index.test.tsnew file mode 100644index 00000000..5e0e00d7--- /dev/null+++ b/test/ask-index.test.ts@@ -0,0 +1,182 @@+/**+ * Tests for the `ask` build-time sidecar (`.graph/index.json`).+ *+ * `graft build` writes token/document-frequency bags once so `ask` doesn't+ * re-tokenize the whole corpus per query. These tests pin down the contract+ * that makes the sidecar safe to ship: it is a byte-for-byte reproduction of+ * live tokenization, consuming it never changes a single `ask` result (hits+ * AND scores), and any way the sidecar can be missing/wrong falls back to the+ * live path rather than crashing or silently drifting.+ */+import { test } from "node:test";+import assert from "node:assert/strict";+import { mkdtempSync, writeFileSync, rmSync, readFileSync, unlinkSync } from "node:fs";+import { join } from "node:path";+import { tmpdir } from "node:os";+import { buildGraph } from "../src/graph/build.js";+import { ask } from "../src/ask/ask.js";+import { contextDirFor } from "../src/context/node-file.js";+import { readGraph, wiringPath } from "../src/graph/write.js";+import { askIndexPath, readAskIndex, tokenize, counts, writeAskIndex } from "../src/ask/index-file.js";++/** A small multi-file fixture with enough overlapping vocabulary that IDF and+ * BM25 actually differentiate hits, so a parity test on scores is meaningful. */+function makeFixture(): string {+ const dir = mkdtempSync(join(tmpdir(), "graft-ask-index-"));+ writeFileSync(+ join(dir, "auth.ts"),+ `/** Validate an incoming API request's auth token. */\n` ++ `export function validateAuthToken(token: string): boolean {\n` ++ ` return token.length > 0;\n` ++ `}\n\n` ++ `export function authMiddleware(req: unknown): boolean {\n` ++ ` return validateAuthToken("x");\n` ++ `}\n`,+ );+ writeFileSync(+ join(dir, "requests.ts"),+ `/** Parse an incoming API request body. */\n` ++ `export function parseRequestBody(raw: string): unknown {\n` ++ ` return JSON.parse(raw);\n` ++ `}\n`,+ );+ writeFileSync(+ join(dir, "unrelated.ts"),+ `export function addTwo(a: number, b: number): number {\n return a + b;\n}\n`,+ );+ return dir;+}++const QUERY = "How does authentication middleware validate incoming API requests?";++test("writeAskIndex + readAskIndex round-trip matches live tokenization exactly", async () => {+ const dir = makeFixture();+ try {+ await buildGraph(dir);+ const outDir = contextDirFor(dir);+ const graph = readGraph(wiringPath(outDir));+ assert.ok(graph, "wiring graph should exist after build");++ const index = readAskIndex(outDir);+ assert.ok(index, "sidecar should exist after build");+ assert.equal(index!.version, 1);+ assert.equal(index!.docCount, graph!.nodes.length);+ assert.equal(index!.docs.length, graph!.nodes.length);++ const sortPairs = (p: [string, number][]) => [...p].sort((a, b) => a[0].localeCompare(b[0]));+ const byId = new Map(index!.docs.map((d) => [d.id, d]));+ for (const n of graph!.nodes) {+ const d = byId.get(n.id);+ assert.ok(d, `sidecar should carry a doc for ${n.id}`);+ const liveName = [...counts(tokenize(n.name)).entries()];+ const livePath = [...counts(tokenize(n.path)).entries()];+ const liveBody = [+ ...counts(tokenize(`${n.signature ?? ""} ${n.summary ?? ""} ${n.body_text ?? ""}`)).entries(),+ ];+ assert.deepEqual(sortPairs(d!.name), sortPairs(liveName), `name bag for ${n.id}`);+ assert.deepEqual(sortPairs(d!.path), sortPairs(livePath), `path bag for ${n.id}`);+ assert.deepEqual(sortPairs(d!.body), sortPairs(liveBody), `body bag for ${n.id}`);+ }++ // Round-trip through writeAskIndex again (idempotent, deterministic).+ const path2 = writeAskIndex(outDir, graph!);+ assert.equal(path2, askIndexPath(outDir));+ const reread = readAskIndex(outDir);+ assert.deepEqual(reread, index, "re-writing an unchanged graph reproduces the same sidecar");+ } finally {+ rmSync(dir, { recursive: true, force: true });+ }+});++test("ask results are IDENTICAL with and without the sidecar (same hits, same scores)", async () => {+ const dir = makeFixture();+ try {+ await buildGraph(dir);+ const outDir = contextDirFor(dir);+ const idxPath = askIndexPath(outDir);++ const withIndex = ask(dir, QUERY, { source: false });+ assert.ok(readAskIndex(outDir), "sanity: sidecar is present for the first run");++ const backup = readFileSync(idxPath);+ unlinkSync(idxPath);+ try {+ assert.equal(readAskIndex(outDir), null, "sanity: sidecar is really gone");+ const withoutIndex = ask(dir, QUERY, { source: false });+ assert.deepEqual(withoutIndex, withIndex, "sidecar vs live fallback must produce identical AskResult");+ assert.ok(withIndex.hits.length > 0, "the fixture query should actually match something");+ } finally {+ writeFileSync(idxPath, backup);+ }+ } finally {+ rmSync(dir, { recursive: true, force: true });+ }+});++test("unknown sidecar version falls back to live tokenization without crashing", async () => {+ const dir = makeFixture();+ try {+ await buildGraph(dir);+ const outDir = contextDirFor(dir);+ const idxPath = askIndexPath(outDir);++ const live = ask(dir, QUERY, { source: false });++ const raw = JSON.parse(readFileSync(idxPath, "utf8"));+ raw.version = 2;+ writeFileSync(idxPath, JSON.stringify(raw));++ assert.equal(readAskIndex(outDir), null, "an unknown version reads as null (fallback signal)");+ const withBadVersion = ask(dir, QUERY, { source: false });+ assert.deepEqual(withBadVersion, live, "unknown-version sidecar must not change results");+ } finally {+ rmSync(dir, { recursive: true, force: true });+ }+});++test("a stale sidecar (docCount mismatch) falls back to live tokenization", async () => {+ const dir = makeFixture();+ try {+ await buildGraph(dir);+ const outDir = contextDirFor(dir);+ const idxPath = askIndexPath(outDir);++ const live = ask(dir, QUERY, { source: false });++ const raw = JSON.parse(readFileSync(idxPath, "utf8"));+ raw.docs = raw.docs.slice(1); // drop one doc — docs.length !== graph.nodes.length now+ writeFileSync(idxPath, JSON.stringify(raw));++ const stale = ask(dir, QUERY, { source: false });+ assert.deepEqual(stale, live, "a docCount-mismatched sidecar must not change results");+ } finally {+ rmSync(dir, { recursive: true, force: true });+ }+});++test("an unparseable sidecar file falls back to live tokenization", async () => {+ const dir = makeFixture();+ try {+ await buildGraph(dir);+ const outDir = contextDirFor(dir);+ const idxPath = askIndexPath(outDir);++ const live = ask(dir, QUERY, { source: false });+ writeFileSync(idxPath, "{not json");++ assert.equal(readAskIndex(outDir), null);+ const withGarbage = ask(dir, QUERY, { source: false });+ assert.deepEqual(withGarbage, live);+ } finally {+ rmSync(dir, { recursive: true, force: true });+ }+});++test("readAskIndex returns null when the sidecar is simply missing", () => {+ const dir = mkdtempSync(join(tmpdir(), "graft-ask-index-missing-"));+ try {+ assert.equal(readAskIndex(contextDirFor(dir)), null);+ } finally {+ rmSync(dir, { recursive: true, force: true });+ }+});