Skip to content

Turbocharge Claude Code, Cursor, Codex, Gemini & every coding agent: faster, cheaper, with contextual understanding specific to your codebase.

feat(ask): build-time token/IDF index sidecar — ask stops re-tokenizing the corpus per query

Shrish Dwivedia3c1bdff41cd
diff --git a/src/ask/ask.ts b/src/ask/ask.ts
index b9f2abd5..1f938eaa 100644
--- a/src/ask/ask.ts
+++ b/src/ask/ask.ts
@@ -20,6 +20,7 @@ import { contextDirFor } from "../context/node-file.js";
import { readGraph, wiringPath } from "../graph/write.js";
import type { EdgeV1, GraphV1, NodeV1, Relation } from "../graph/types.js";
import { personalizedPageRank } from "./graphrank.js";
+import { counts, readAskIndex, tokenize, type AskIndex, type AskIndexDoc } from "./index-file.js";
export interface AskHit {
kind: "concept" | "symbol" | "caller" | "callee";
@@ -53,28 +54,6 @@ export interface AskResult {
saved?: { files: number; baselineChars: number };
}
-const STOP = new Set([
- "the", "a", "an", "of", "to", "in", "is", "are", "how", "does", "do", "what",
- "where", "which", "that", "this", "it", "for", "on", "and", "or", "with",
- "i", "we", "get", "set", "use", "used", "using", "when", "why", "can",
-]);
-
-/** Split prose + identifiers into lowercased subword tokens (camelCase, snake, kebab). */
-function tokenize(text: string): string[] {
- return text
- .replace(/([a-z0-9])([A-Z])/g, "$1 $2") // camelCase → camel Case
- .toLowerCase()
- .split(/[^a-z0-9]+/)
- .filter((t) => t.length > 1 && !STOP.has(t));
-}
-
-/** Term-frequency count map. */
-function counts(tokens: string[]): Map<string, number> {
- const m = new Map<string, number>();
- for (const t of tokens) m.set(t, (m.get(t) ?? 0) + 1);
- return m;
-}
-
/** First real prose line of a node body (skips headings, markers, blanks). */
function firstProse(body: string): string {
for (const raw of body.split("\n")) {
@@ -88,6 +67,10 @@ function firstProse(body: string): string {
interface Corpus {
concepts: Array<{ slug: string; name: string; sources: string[]; related: string[]; snippet: string; text: string }>;
graph: GraphV1 | null;
+ /** Build-time token/df sidecar (`.graph/index.json`), or null when absent,
+ * unparseable, an unknown version, or stale (checked against `graph.nodes.length`
+ * by the caller — see `lexical()`). Null means: fall back to live tokenization. */
+ askIndex: AskIndex | null;
}
function loadCorpus(outDir: string): Corpus {
@@ -114,7 +97,7 @@ function loadCorpus(outDir: string): Corpus {
});
}
}
- return { concepts, graph: readGraph(wiringPath(outDir)) };
+ return { concepts, graph: readGraph(wiringPath(outDir)), askIndex: readAskIndex(outDir) };
}
/** Score a document's token counts against the query counts (name field
@@ -141,14 +124,31 @@ function score(
* documents (concept nodes + symbol nodes) contain each token; N is the corpus
* size. `log(1 + N/(1+df))` is the smoothed standard form: monotonically
* decreasing in df, always positive, ~0 extra weight for a token in every doc. */
+function idfFromDf(df: Map<string, number>, n: number): Map<string, number> {
+ const idf = new Map<string, number>();
+ for (const [t, d] of df) idf.set(t, Math.log(1 + n / (1 + d)));
+ return idf;
+}
+
function computeIdf(docBags: Array<Set<string>>): Map<string, number> {
- const n = docBags.length;
const df = new Map<string, number>();
for (const bag of docBags)
for (const t of bag) df.set(t, (df.get(t) ?? 0) + 1);
- const idf = new Map<string, number>();
- for (const [t, d] of df) idf.set(t, Math.log(1 + n / (1 + d)));
- return idf;
+ return idfFromDf(df, docBags.length);
+}
+
+/** Same idf as {@link computeIdf}, but the symbol half of `df` and the doc
+ * count come from the build-time sidecar instead of tokenizing the corpus
+ * live; only the concept bags (always tokenized live — there are only dozens)
+ * are folded in here, the same way `computeIdf` would fold them into the full
+ * docBags array. `df` is a commutative sum and `n` is the same total document
+ * count either way, so this produces byte-identical idf values to `computeIdf`
+ * run over concept-bags + live-tokenized-symbol-bags. */
+function computeIdfFromIndex(index: AskIndex, conceptBags: Array<Set<string>>): Map<string, number> {
+ const df = new Map(index.df);
+ for (const bag of conceptBags)
+ for (const t of bag) df.set(t, (df.get(t) ?? 0) + 1);
+ return idfFromDf(df, index.docCount + conceptBags.length);
}
/** BM25 term score for a document field, used for the (potentially large) body
@@ -273,25 +273,47 @@ function lexical(query: string, corpus: Corpus, limit: number, graphRank: boolea
name: counts(tokenize(c.name)),
body: counts(tokenize(c.text)),
}));
+
+ // A build-time sidecar (`.graph/index.json`) is usable only when it covers
+ // every node in the current graph: `docs.length === graph.nodes.length` is a
+ // cheap staleness guard (an id-level check below catches a same-count but
+ // mismatched set, e.g. a hand-edited or corrupt sidecar). Anything short of
+ // that — missing, unparseable, unknown version, wrong count, mismatched ids —
+ // falls back to live tokenization so results never depend on the sidecar
+ // existing; see `readAskIndex`'s own null-on-anything-off contract.
+ let docById: Map<string, AskIndexDoc> | null = null;
+ if (corpus.askIndex && graph && corpus.askIndex.docs.length === graph.nodes.length) {
+ const map = new Map(corpus.askIndex.docs.map((d) => [d.id, d]));
+ if (graph.nodes.every((n) => map.has(n.id))) docById = map;
+ }
+ const askIndex = docById ? corpus.askIndex : null;
+
// Symbols AND file nodes are scored: a symbol's body_text is its definition;
// a file's body_text is the module-level residual (imports/constants not in
// any symbol). Including files closes the recall gap where a gold file's only
// query-relevant text lives outside every function/class.
- const symbolDocs = (graph?.nodes ?? []).map((n) => ({
- n,
- name: counts(tokenize(n.name)),
- path: counts(tokenize(n.path)),
- // The body (indexed at build) joins the signature + summary as the low-weight
- // body field, so a term that appears only in the code — not the name/signature
- // — still makes the node findable. IDF (from these same bags) keeps a word
- // common across many bodies from dominating.
- body: counts(tokenize(`${n.signature ?? ""} ${n.summary ?? ""} ${n.body_text ?? ""}`)),
- }));
- const docBags: Array<Set<string>> = [];
- for (const d of conceptDocs) docBags.push(new Set([...d.name.keys(), ...d.body.keys()]));
- for (const d of symbolDocs)
- docBags.push(new Set([...d.name.keys(), ...d.path.keys(), ...d.body.keys()]));
- const idf = computeIdf(docBags);
+ const symbolDocs = (graph?.nodes ?? []).map((n) => {
+ const d = docById?.get(n.id);
+ if (d) return { n, name: new Map(d.name), path: new Map(d.path), body: new Map(d.body) };
+ return {
+ n,
+ name: counts(tokenize(n.name)),
+ path: counts(tokenize(n.path)),
+ // The body (indexed at build) joins the signature + summary as the low-weight
+ // body field, so a term that appears only in the code — not the name/signature
+ // — still makes the node findable. IDF (from these same bags) keeps a word
+ // common across many bodies from dominating.
+ body: counts(tokenize(`${n.signature ?? ""} ${n.summary ?? ""} ${n.body_text ?? ""}`)),
+ };
+ });
+
+ const conceptBags = conceptDocs.map((d) => new Set([...d.name.keys(), ...d.body.keys()]));
+ // IDF must see the SAME inputs either way: with a sidecar, its `df` (stored
+ // WITHOUT concepts) plus these live concept bags reproduces exactly what
+ // `computeIdf` below would compute from concept+symbol bags tokenized live.
+ const idf = askIndex
+ ? computeIdfFromIndex(askIndex, conceptBags)
+ : computeIdf([...conceptBags, ...symbolDocs.map((d) => new Set([...d.name.keys(), ...d.path.keys(), ...d.body.keys()]))]);
// ── Concepts (prose nodes; not in the wiring graph) ──
const conceptHits: AskHit[] = [];
@@ -318,9 +340,11 @@ function lexical(query: string, corpus: Corpus, limit: number, graphRank: boolea
for (const v of m.values()) s += v;
return s;
};
- const avgBodyLen = symbolDocs.length
- ? symbolDocs.reduce((a, d) => a + bodyLen(d.body), 0) / symbolDocs.length
- : 0;
+ const avgBodyLen = askIndex
+ ? askIndex.avgBodyLen
+ : symbolDocs.length
+ ? symbolDocs.reduce((a, d) => a + bodyLen(d.body), 0) / symbolDocs.length
+ : 0;
const lex = new Map<string, number>(); // node id → lexical score (>0 only)
let maxLex = 0;
for (const { n, name, path, body } of symbolDocs) {
diff --git a/src/ask/index-file.ts b/src/ask/index-file.ts
new file mode 100644
index 00000000..e092fec2
--- /dev/null
+++ b/src/ask/index-file.ts
@@ -0,0 +1,151 @@
+/**
+ * Build-time sidecar for `graft ask` — `<outDir>/.graph/index.json`.
+ *
+ * `ask`'s lexical pass tokenizes every symbol node's name/path/body on every
+ * query; at 32k nodes that re-tokenization is ~45% of query time (profiled).
+ * `graft build` writes this sidecar once, alongside `wiring.json`, with the
+ * token→count bags per node plus the corpus-wide document frequencies, so a
+ * query just reads counts instead of re-splitting every node's text.
+ *
+ * `tokenize`/`counts` live here (not duplicated in `ask.ts`) so build-time and
+ * query-time text-splitting are provably the same function — the sidecar can
+ * only be a correct cache of `ask.ts`'s own math if both sides call the same
+ * code. `ask.ts` imports both back from here.
+ *
+ * Concept (markdown) docs are NOT part of this sidecar — there are only dozens
+ * of them, they're still tokenized live at query time, and their doc-frequency
+ * contribution is folded into the stored `df` at query time (see `ask.ts`),
+ * which is why `df` here counts symbol/file nodes only.
+ */
+import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs";
+import { dirname, join } from "node:path";
+import type { GraphV1 } from "../graph/types.js";
+
+/** Words too common/short to carry query intent — dropped before scoring. */
+const STOP = new Set([
+ "the", "a", "an", "of", "to", "in", "is", "are", "how", "does", "do", "what",
+ "where", "which", "that", "this", "it", "for", "on", "and", "or", "with",
+ "i", "we", "get", "set", "use", "used", "using", "when", "why", "can",
+]);
+
+/** Split prose + identifiers into lowercased subword tokens (camelCase, snake, kebab).
+ * The single source of truth for tokenization — shared by build-time indexing
+ * (this file) and query-time fallback (`ask.ts`) so the sidecar is a provably
+ * exact cache of the live path. */
+export function tokenize(text: string): string[] {
+ return text
+ .replace(/([a-z0-9])([A-Z])/g, "$1 $2") // camelCase → camel Case
+ .toLowerCase()
+ .split(/[^a-z0-9]+/)
+ .filter((t) => t.length > 1 && !STOP.has(t));
+}
+
+/** Term-frequency count map. */
+export function counts(tokens: string[]): Map<string, number> {
+ const m = new Map<string, number>();
+ for (const t of tokens) m.set(t, (m.get(t) ?? 0) + 1);
+ return m;
+}
+
+/** One node's token bags, JSON-friendly (`Map` → sorted `[token, count][]`). */
+export interface AskIndexDoc {
+ id: string;
+ name: [string, number][];
+ path: [string, number][];
+ body: [string, number][];
+}
+
+/** The build-time sidecar. `df`/`docCount` cover symbol+file nodes only (no
+ * concepts — see module docstring); `avgBodyLen` is the BM25 corpus average. */
+export interface AskIndex {
+ version: 1;
+ avgBodyLen: number;
+ df: [string, number][];
+ docCount: number;
+ docs: AskIndexDoc[];
+}
+
+const ASK_INDEX_DIR = ".graph";
+const ASK_INDEX_FILE = "index.json";
+
+/** Absolute path to the ask sidecar for a context dir: `<dir>/.graph/index.json`. */
+export function askIndexPath(outDir: string): string {
+ return join(outDir, ASK_INDEX_DIR, ASK_INDEX_FILE);
+}
+
+function pairs(m: Map<string, number>): [string, number][] {
+ return [...m.entries()];
+}
+
+/** Sum of a token→count bag's counts (a document's field length). */
+function bagLen(p: [string, number][]): number {
+ let s = 0;
+ for (const [, c] of p) s += c;
+ return s;
+}
+
+/**
+ * Tokenize every node in `graph` (exactly as `ask.ts`'s lexical pass does) and
+ * write the resulting bags + document frequencies to `<outDir>/.graph/index.json`.
+ * Returns the path written. Deterministic: nodes are indexed in id order, so an
+ * unchanged graph produces a byte-identical sidecar.
+ */
+export function writeAskIndex(outDir: string, graph: GraphV1): string {
+ const nodes = [...graph.nodes].sort((a, b) => a.id.localeCompare(b.id));
+ const docs: AskIndexDoc[] = [];
+ const df = new Map<string, number>();
+
+ for (const n of nodes) {
+ const name = counts(tokenize(n.name));
+ const path = counts(tokenize(n.path));
+ const body = counts(
+ tokenize(`${n.signature ?? ""} ${n.summary ?? ""} ${n.body_text ?? ""}`),
+ );
+ docs.push({ id: n.id, name: pairs(name), path: pairs(path), body: pairs(body) });
+
+ const bag = new Set<string>([...name.keys(), ...path.keys(), ...body.keys()]);
+ for (const t of bag) df.set(t, (df.get(t) ?? 0) + 1);
+ }
+
+ const avgBodyLen = docs.length
+ ? docs.reduce((a, d) => a + bagLen(d.body), 0) / docs.length
+ : 0;
+
+ const index: AskIndex = {
+ version: 1,
+ avgBodyLen,
+ df: pairs(df),
+ docCount: nodes.length,
+ docs,
+ };
+
+ const outPath = askIndexPath(outDir);
+ mkdirSync(dirname(outPath), { recursive: true });
+ writeFileSync(outPath, JSON.stringify(index) + "\n");
+ return outPath;
+}
+
+/** Read the ask sidecar. Returns null on a missing file, unparseable JSON, an
+ * unrecognized shape, or an unknown `version` — any of which means the caller
+ * should fall back to live tokenization, never crash. */
+export function readAskIndex(outDir: string): AskIndex | null {
+ const path = askIndexPath(outDir);
+ if (!existsSync(path)) return null;
+ try {
+ const raw = JSON.parse(readFileSync(path, "utf8"));
+ if (
+ !raw ||
+ typeof raw !== "object" ||
+ raw.version !== 1 ||
+ typeof raw.docCount !== "number" ||
+ typeof raw.avgBodyLen !== "number" ||
+ !Array.isArray(raw.df) ||
+ !Array.isArray(raw.docs)
+ ) {
+ return null;
+ }
+ return raw as AskIndex;
+ } catch {
+ return null;
+ }
+}
diff --git a/src/graph/build.ts b/src/graph/build.ts
index 09ca6575..b7606b65 100644
--- a/src/graph/build.ts
+++ b/src/graph/build.ts
@@ -16,6 +16,7 @@ import { resolveEdges, type GoModule } from "./resolve.js";
import { enrichGraph, type EnrichStats } from "./enrich.js";
import { readGraph, writeGraph, wiringPath } from "./write.js";
import { writeCards, writeIndex, writeCovers } from "./cards.js";
+import { writeAskIndex } from "../ask/index-file.js";
import type { GraphV1, Kind, NodeV1, Relation } from "./types.js";
import type { CruxSummarizer } from "../ai/crux.js";
@@ -136,6 +137,10 @@ export async function buildGraph(
};
const graphPath = writeGraph(graph, outDir);
+ // `ask`'s token/IDF sidecar — moves per-query corpus tokenization to build
+ // time (~45% of query time on a 32k-node graph, profiled). Lands next to
+ // wiring.json; `ask` falls back to live tokenization when it's absent/stale.
+ writeAskIndex(outDir, graph);
// Tier-2 passive surface: project the nodes into per-file markdown cards, and
// refresh the INDEX roster. Pure projection — no LLM, no network.
diff --git a/test/ask-index.test.ts b/test/ask-index.test.ts
new file mode 100644
index 00000000..5e0e00d7
--- /dev/null
+++ b/test/ask-index.test.ts
@@ -0,0 +1,182 @@
+/**
+ * Tests for the `ask` build-time sidecar (`.graph/index.json`).
+ *
+ * `graft build` writes token/document-frequency bags once so `ask` doesn't
+ * re-tokenize the whole corpus per query. These tests pin down the contract
+ * that makes the sidecar safe to ship: it is a byte-for-byte reproduction of
+ * live tokenization, consuming it never changes a single `ask` result (hits
+ * AND scores), and any way the sidecar can be missing/wrong falls back to the
+ * live path rather than crashing or silently drifting.
+ */
+import { test } from "node:test";
+import assert from "node:assert/strict";
+import { mkdtempSync, writeFileSync, rmSync, readFileSync, unlinkSync } from "node:fs";
+import { join } from "node:path";
+import { tmpdir } from "node:os";
+import { buildGraph } from "../src/graph/build.js";
+import { ask } from "../src/ask/ask.js";
+import { contextDirFor } from "../src/context/node-file.js";
+import { readGraph, wiringPath } from "../src/graph/write.js";
+import { askIndexPath, readAskIndex, tokenize, counts, writeAskIndex } from "../src/ask/index-file.js";
+
+/** A small multi-file fixture with enough overlapping vocabulary that IDF and
+ * BM25 actually differentiate hits, so a parity test on scores is meaningful. */
+function makeFixture(): string {
+ const dir = mkdtempSync(join(tmpdir(), "graft-ask-index-"));
+ writeFileSync(
+ join(dir, "auth.ts"),
+ `/** Validate an incoming API request's auth token. */\n` +
+ `export function validateAuthToken(token: string): boolean {\n` +
+ ` return token.length > 0;\n` +
+ `}\n\n` +
+ `export function authMiddleware(req: unknown): boolean {\n` +
+ ` return validateAuthToken("x");\n` +
+ `}\n`,
+ );
+ writeFileSync(
+ join(dir, "requests.ts"),
+ `/** Parse an incoming API request body. */\n` +
+ `export function parseRequestBody(raw: string): unknown {\n` +
+ ` return JSON.parse(raw);\n` +
+ `}\n`,
+ );
+ writeFileSync(
+ join(dir, "unrelated.ts"),
+ `export function addTwo(a: number, b: number): number {\n return a + b;\n}\n`,
+ );
+ return dir;
+}
+
+const QUERY = "How does authentication middleware validate incoming API requests?";
+
+test("writeAskIndex + readAskIndex round-trip matches live tokenization exactly", async () => {
+ const dir = makeFixture();
+ try {
+ await buildGraph(dir);
+ const outDir = contextDirFor(dir);
+ const graph = readGraph(wiringPath(outDir));
+ assert.ok(graph, "wiring graph should exist after build");
+
+ const index = readAskIndex(outDir);
+ assert.ok(index, "sidecar should exist after build");
+ assert.equal(index!.version, 1);
+ assert.equal(index!.docCount, graph!.nodes.length);
+ assert.equal(index!.docs.length, graph!.nodes.length);
+
+ const sortPairs = (p: [string, number][]) => [...p].sort((a, b) => a[0].localeCompare(b[0]));
+ const byId = new Map(index!.docs.map((d) => [d.id, d]));
+ for (const n of graph!.nodes) {
+ const d = byId.get(n.id);
+ assert.ok(d, `sidecar should carry a doc for ${n.id}`);
+ const liveName = [...counts(tokenize(n.name)).entries()];
+ const livePath = [...counts(tokenize(n.path)).entries()];
+ const liveBody = [
+ ...counts(tokenize(`${n.signature ?? ""} ${n.summary ?? ""} ${n.body_text ?? ""}`)).entries(),
+ ];
+ assert.deepEqual(sortPairs(d!.name), sortPairs(liveName), `name bag for ${n.id}`);
+ assert.deepEqual(sortPairs(d!.path), sortPairs(livePath), `path bag for ${n.id}`);
+ assert.deepEqual(sortPairs(d!.body), sortPairs(liveBody), `body bag for ${n.id}`);
+ }
+
+ // Round-trip through writeAskIndex again (idempotent, deterministic).
+ const path2 = writeAskIndex(outDir, graph!);
+ assert.equal(path2, askIndexPath(outDir));
+ const reread = readAskIndex(outDir);
+ assert.deepEqual(reread, index, "re-writing an unchanged graph reproduces the same sidecar");
+ } finally {
+ rmSync(dir, { recursive: true, force: true });
+ }
+});
+
+test("ask results are IDENTICAL with and without the sidecar (same hits, same scores)", async () => {
+ const dir = makeFixture();
+ try {
+ await buildGraph(dir);
+ const outDir = contextDirFor(dir);
+ const idxPath = askIndexPath(outDir);
+
+ const withIndex = ask(dir, QUERY, { source: false });
+ assert.ok(readAskIndex(outDir), "sanity: sidecar is present for the first run");
+
+ const backup = readFileSync(idxPath);
+ unlinkSync(idxPath);
+ try {
+ assert.equal(readAskIndex(outDir), null, "sanity: sidecar is really gone");
+ const withoutIndex = ask(dir, QUERY, { source: false });
+ assert.deepEqual(withoutIndex, withIndex, "sidecar vs live fallback must produce identical AskResult");
+ assert.ok(withIndex.hits.length > 0, "the fixture query should actually match something");
+ } finally {
+ writeFileSync(idxPath, backup);
+ }
+ } finally {
+ rmSync(dir, { recursive: true, force: true });
+ }
+});
+
+test("unknown sidecar version falls back to live tokenization without crashing", async () => {
+ const dir = makeFixture();
+ try {
+ await buildGraph(dir);
+ const outDir = contextDirFor(dir);
+ const idxPath = askIndexPath(outDir);
+
+ const live = ask(dir, QUERY, { source: false });
+
+ const raw = JSON.parse(readFileSync(idxPath, "utf8"));
+ raw.version = 2;
+ writeFileSync(idxPath, JSON.stringify(raw));
+
+ assert.equal(readAskIndex(outDir), null, "an unknown version reads as null (fallback signal)");
+ const withBadVersion = ask(dir, QUERY, { source: false });
+ assert.deepEqual(withBadVersion, live, "unknown-version sidecar must not change results");
+ } finally {
+ rmSync(dir, { recursive: true, force: true });
+ }
+});
+
+test("a stale sidecar (docCount mismatch) falls back to live tokenization", async () => {
+ const dir = makeFixture();
+ try {
+ await buildGraph(dir);
+ const outDir = contextDirFor(dir);
+ const idxPath = askIndexPath(outDir);
+
+ const live = ask(dir, QUERY, { source: false });
+
+ const raw = JSON.parse(readFileSync(idxPath, "utf8"));
+ raw.docs = raw.docs.slice(1); // drop one doc — docs.length !== graph.nodes.length now
+ writeFileSync(idxPath, JSON.stringify(raw));
+
+ const stale = ask(dir, QUERY, { source: false });
+ assert.deepEqual(stale, live, "a docCount-mismatched sidecar must not change results");
+ } finally {
+ rmSync(dir, { recursive: true, force: true });
+ }
+});
+
+test("an unparseable sidecar file falls back to live tokenization", async () => {
+ const dir = makeFixture();
+ try {
+ await buildGraph(dir);
+ const outDir = contextDirFor(dir);
+ const idxPath = askIndexPath(outDir);
+
+ const live = ask(dir, QUERY, { source: false });
+ writeFileSync(idxPath, "{not json");
+
+ assert.equal(readAskIndex(outDir), null);
+ const withGarbage = ask(dir, QUERY, { source: false });
+ assert.deepEqual(withGarbage, live);
+ } finally {
+ rmSync(dir, { recursive: true, force: true });
+ }
+});
+
+test("readAskIndex returns null when the sidecar is simply missing", () => {
+ const dir = mkdtempSync(join(tmpdir(), "graft-ask-index-missing-"));
+ try {
+ assert.equal(readAskIndex(contextDirFor(dir)), null);
+ } finally {
+ rmSync(dir, { recursive: true, force: true });
+ }
+});