Turbocharge Claude Code, Cursor, Codex, Gemini & every coding agent: faster, cheaper, with contextual understanding specific to your codebase.
TypeScript ai-agentsanthropicclaude-codeclicode-graphcodexcontext-engineeringcursordeveloper-toolsgeminiknowledge-graphllmmcpmcp-serveropen-sourceopenaitree-sitter
perf(graph): slim wiring.json — body_text lives only in the ask index sidecar
Shrish Dwivedia67ae16735e8
diff --git a/src/ask/ask.ts b/src/ask/ask.tsindex 81b4de5f..f5df2a48 100644--- a/src/ask/ask.ts+++ b/src/ask/ask.ts@@ -302,7 +302,11 @@ function lexical(query: string, corpus: Corpus, limit: number, graphRank: boolea// The body (indexed at build) joins the signature + summary as the low-weight// body field, so a term that appears only in the code — not the name/signature// — still makes the node findable. IDF (from these same bags) keeps a word- // common across many bodies from dominating.+ // common across many bodies from dominating. `body_text` is absent on every+ // node loaded from a wiring.json written after the slim-serialization change+ // (it's stripped there — see `write.ts`) as well as on file nodes and+ // pre-body_text graphs; `?? ""` degrades gracefully to signature+summary+ // only, it never crashes on the missing field.body: counts(tokenize(`${n.signature ?? ""} ${n.summary ?? ""} ${n.body_text ?? ""}`)),};});diff --git a/src/graph/build.ts b/src/graph/build.tsindex 08e7c8a5..ee43c4af 100644--- a/src/graph/build.ts+++ b/src/graph/build.ts@@ -143,6 +143,11 @@ export async function buildGraph(// Best-effort: wiring.json is already on disk at this point, so a sidecar// write failure (e.g. an unwritable cache dir) must not abort cards/index/// covers below — record it and keep going, same as other recoverable errors.+ //+ // MUST pass the in-memory `graph` here, never a re-read of wiringPath(outDir):+ // `writeGraph` strips `body_text` from what it serializes (dead weight once+ // this sidecar exists), so the nodes on disk no longer carry it — only this+ // in-memory object, still holding what `extractFile` populated, does.try {writeAskIndex(outDir, graph);} catch (err) {diff --git a/src/graph/write.ts b/src/graph/write.tsindex e969b669..8b993a88 100644--- a/src/graph/write.ts+++ b/src/graph/write.ts@@ -10,7 +10,7 @@*/import { mkdirSync, readFileSync, writeFileSync } from "node:fs";import { dirname, join } from "node:path";-import type { EdgeV1, GraphV1 } from "./types.js";+import type { EdgeV1, GraphV1, NodeV1 } from "./types.js";/** Hidden subdir under the context dir that holds machine-only graph artifacts. */export const GRAPH_DIR = ".graph";@@ -36,7 +36,7 @@ export function readGraph(path: string): GraphV1 | null {export function writeGraph(graph: GraphV1, outDir: string): string {const sorted: GraphV1 = {...graph,- nodes: [...graph.nodes].sort((a, b) => a.id.localeCompare(b.id)),+ nodes: [...graph.nodes].sort((a, b) => a.id.localeCompare(b.id)).map(stripBodyText),edges: [...graph.edges].sort(edgeOrder),};const path = wiringPath(outDir);@@ -45,6 +45,21 @@ export function writeGraph(graph: GraphV1, outDir: string): string {return path;}+/**+ * Drop `body_text` from the SERIALIZED copy of a node — it is ~65% of+ * wiring.json's bytes on a large graph, and every byte of it is already+ * duplicated in the `ask` sidecar (`.cache/ask-index.json`), tokenized, which+ * is the only place anything reads it from. Callers must pass this the+ * in-memory graph BEFORE this stripped copy is produced (see `build.ts`:+ * `writeAskIndex` runs on the original `graph` object, never on a re-read of+ * this slimmed file) — this function never mutates the input node.+ */+function stripBodyText(node: NodeV1): NodeV1 {+ if (node.body_text === undefined) return node;+ const { body_text: _body_text, ...rest } = node;+ return rest as NodeV1;+}+function edgeOrder(a: EdgeV1, b: EdgeV1): number {return (a.source.localeCompare(b.source) ||diff --git a/test/ask-index.test.ts b/test/ask-index.test.tsindex 6e80a4b1..234b6b63 100644--- a/test/ask-index.test.ts+++ b/test/ask-index.test.ts@@ -10,7 +10,7 @@*/import { test } from "node:test";import assert from "node:assert/strict";-import { mkdtempSync, writeFileSync, mkdirSync, rmSync, readFileSync, unlinkSync } from "node:fs";+import { mkdtempSync, writeFileSync, mkdirSync, rmSync, readFileSync, readdirSync, unlinkSync } from "node:fs";import { join, dirname } from "node:path";import { tmpdir } from "node:os";import { buildGraph } from "../src/graph/build.js";@@ -18,6 +18,8 @@ import { ask } from "../src/ask/ask.js";import { contextDirFor } from "../src/context/node-file.js";import { readGraph, wiringPath } from "../src/graph/write.js";import { askIndexPath, readAskIndex, tokenize, counts, writeAskIndex } from "../src/ask/index-file.js";+import { extractFile, languageOf } from "../src/graph/extract.js";+import type { GraphV1, NodeV1 } from "../src/graph/types.js";/** A small multi-file fixture with enough overlapping vocabulary that IDF and* BM25 actually differentiate hits, so a parity test on scores is meaningful. */@@ -49,11 +51,49 @@ function makeFixture(): string {const QUERY = "How does authentication middleware validate incoming API requests?";+/**+ * Reconstruct a "fat" wiring.json — body_text present on every node, as EVERY+ * build produced before the slim-serialization change (see `write.ts`) — by+ * re-extracting it from each fixture source file and merging it back onto the+ * (now-slim) wiring.json already on disk. `extractFile` is a pure function of+ * source text, so this reproduces byte-identical values to what the sidecar+ * was actually tokenized from at build time.+ *+ * Several tests below pin down sidecar-corruption fallback behavior (unknown+ * version / stale docCount / unparseable file) — a concern orthogonal to+ * whether wiring.json itself is slim or fat. Before the slim-serialization+ * change those tests got a fat graph for free; this helper keeps them testing+ * the same thing (does falling back to live tokenization reproduce the+ * sidecar exactly?) now that a fresh build no longer hands them one.+ */+function reinjectBodyText(dir: string, outDir: string): void {+ const wpath = wiringPath(outDir);+ const graph = JSON.parse(readFileSync(wpath, "utf8")) as GraphV1;+ const bodyById = new Map<string, string>();+ for (const entry of readdirSync(dir)) {+ const lang = languageOf(entry);+ if (!lang) continue;+ const source = readFileSync(join(dir, entry), "utf8");+ const { nodes } = extractFile(entry, source, lang);+ for (const n of nodes) if (n.body_text !== undefined) bodyById.set(n.id, n.body_text);+ }+ for (const n of graph.nodes as NodeV1[]) {+ const bt = bodyById.get(n.id);+ if (bt !== undefined) n.body_text = bt;+ }+ writeFileSync(wpath, JSON.stringify(graph));+}+test("writeAskIndex + readAskIndex round-trip matches live tokenization exactly", async () => {const dir = makeFixture();try {await buildGraph(dir);const outDir = contextDirFor(dir);+ // This test asserts the sidecar is a byte-for-byte cache of tokenizing the+ // graph's OWN fields — which requires body_text, no longer present on a+ // freshly-built wiring.json. Reconstruct it (deterministically) so the+ // cache-correctness check below is meaningful.+ reinjectBodyText(dir, outDir);const graph = readGraph(wiringPath(outDir));assert.ok(graph, "wiring graph should exist after build");@@ -93,6 +133,12 @@ test("ask results are IDENTICAL with and without the sidecar (same hits, same sctry {await buildGraph(dir);const outDir = contextDirFor(dir);+ // A fresh build's wiring.json is slim (no body_text) by design (Task 2) —+ // the live-tokenization fallback only reproduces the sidecar exactly when+ // the underlying graph is fat, so reconstruct that here to test what this+ // test is actually about: does the fallback path correctly reproduce the+ // sidecar's numbers?+ reinjectBodyText(dir, outDir);const idxPath = askIndexPath(outDir);const withIndex = ask(dir, QUERY, { source: false });@@ -118,6 +164,7 @@ test("unknown sidecar version falls back to live tokenization without crashing",try {await buildGraph(dir);const outDir = contextDirFor(dir);+ reinjectBodyText(dir, outDir); // see comment above: fallback == live tokenization only on a fat graphconst idxPath = askIndexPath(outDir);const live = ask(dir, QUERY, { source: false });@@ -139,6 +186,7 @@ test("a stale sidecar (docCount mismatch) falls back to live tokenization", asyntry {await buildGraph(dir);const outDir = contextDirFor(dir);+ reinjectBodyText(dir, outDir); // see comment above: fallback == live tokenization only on a fat graphconst idxPath = askIndexPath(outDir);const live = ask(dir, QUERY, { source: false });@@ -159,6 +207,7 @@ test("an unparseable sidecar file falls back to live tokenization", async () =>try {await buildGraph(dir);const outDir = contextDirFor(dir);+ reinjectBodyText(dir, outDir); // see comment above: fallback == live tokenization only on a fat graphconst idxPath = askIndexPath(outDir);const live = ask(dir, QUERY, { source: false });@@ -226,3 +275,103 @@ test("a failed sidecar write is recorded in build errors, not fatal", async () =rmSync(dir, { recursive: true, force: true });}});++// ── Task-2: body_text moves out of wiring.json entirely ────────────────────++test("ask WITH sidecar: identical hits/scores whether the underlying wiring.json is slim or fat", async () => {+ const dir = makeFixture();+ try {+ await buildGraph(dir);+ const outDir = contextDirFor(dir);+ const wpath = wiringPath(outDir);++ const slimResult = ask(dir, QUERY, { source: false });+ assert.ok(slimResult.hits.length > 0, "sanity: the fixture query matches something");++ // Simulate a "fat" wiring.json (as an old build, or a hand-edit, would+ // produce) by re-injecting an arbitrary body_text onto every node. When+ // the sidecar is present, scoring reads token bags from IT, not from+ // node.body_text, so this must have zero effect on the result.+ const raw = JSON.parse(readFileSync(wpath, "utf8")) as GraphV1;+ for (const n of raw.nodes) (n as { body_text?: string }).body_text = `junk filler text for ${n.id}`;+ writeFileSync(wpath, JSON.stringify(raw));++ const fatResult = ask(dir, QUERY, { source: false });+ assert.deepEqual(+ fatResult,+ slimResult,+ "sidecar-driven ask must be identical whether wiring.json is slim or carries body_text",+ );+ } finally {+ rmSync(dir, { recursive: true, force: true });+ }+});++test("ask WITHOUT sidecar on a SLIM graph: no crash, body contributions absent, name matching still works", async () => {+ const dir = mkdtempSync(join(tmpdir(), "graft-ask-index-slim-nosidecar-"));+ try {+ // "stripe" appears only inside checkout's body — never in its name/signature.+ writeFileSync(+ join(dir, "pay.ts"),+ `export function checkout(amount: number): string {\n` ++ ` const token = createStripeCharge(amount);\n` ++ ` return token;\n` ++ `}\n`,+ );+ await buildGraph(dir); // writes a slim wiring.json + the sidecar+ const outDir = contextDirFor(dir);+ const idxPath = askIndexPath(outDir);+ unlinkSync(idxPath); // no sidecar from here on+ assert.equal(readAskIndex(outDir), null, "sanity: sidecar is really gone");++ // Body-only term: with no sidecar AND a slim graph, body text isn't+ // available anywhere — must not crash, and must simply find nothing.+ const byBody = ask(dir, "stripe");+ assert.equal(+ byBody.hits.find((h) => h.title.startsWith("checkout")),+ undefined,+ "a body-only term cannot be found once body_text lives only in the (now-missing) sidecar",+ );++ // Name matching is unaffected — it never depended on body_text.+ const byName = ask(dir, "checkout");+ const hit = byName.hits.find((h) => h.title.startsWith("checkout"));+ assert.ok(hit, "name-field matching still works with no sidecar and a slim graph");+ } finally {+ rmSync(dir, { recursive: true, force: true });+ }+});++test("ask on an OLD fat graph (body_text present, no sidecar): unchanged behavior — body_text still used", () => {+ const dir = mkdtempSync(join(tmpdir(), "graft-ask-index-old-fat-"));+ try {+ // Build a wiring.json by hand via extractFile directly (bypassing+ // buildGraph/writeGraph entirely), so body_text survives on disk exactly+ // as a pre-slim-serialization build would have written it. No sidecar.+ const source =+ `export function checkout(amount: number): string {\n` ++ ` const token = createStripeCharge(amount);\n` ++ ` return token;\n` ++ `}\n`;+ const { nodes } = extractFile("pay.ts", source, "typescript");+ const graph: GraphV1 = {+ meta: { version: 1, nodeCount: nodes.length, edgeCount: 0, languages: ["typescript"] },+ nodes,+ edges: [],+ };+ const outDir = contextDirFor(dir);+ const wpath = wiringPath(outDir);+ mkdirSync(dirname(wpath), { recursive: true });+ writeFileSync(wpath, JSON.stringify(graph));+ assert.ok(+ nodes.some((n) => n.body_text?.toLowerCase().includes("stripe")),+ "sanity: extractFile actually populated body_text for this fixture",+ );++ const r = ask(dir, "stripe");+ const hit = r.hits.find((h) => h.title.startsWith("checkout"));+ assert.ok(hit, "an old fat graph with no sidecar must still find a body-only term via node.body_text");+ } finally {+ rmSync(dir, { recursive: true, force: true });+ }+});diff --git a/test/graph-write.test.ts b/test/graph-write.test.tsnew file mode 100644index 00000000..5a7eaf65--- /dev/null+++ b/test/graph-write.test.ts@@ -0,0 +1,127 @@+/**+ * Tests for `writeGraph`'s slim serialization: `body_text` is dead weight in+ * `wiring.json` once the `ask` sidecar (`.cache/ask-index.json`) exists — every+ * byte of it is already duplicated there, tokenized. These tests pin the+ * behavior contract: the field never reaches disk, the in-memory graph object+ * `build.ts` still needs for the sidecar is untouched, and every other field+ * round-trips exactly as before.+ */+import { test } from "node:test";+import assert from "node:assert/strict";+import { mkdtempSync, writeFileSync, rmSync, readFileSync } from "node:fs";+import { join } from "node:path";+import { tmpdir } from "node:os";+import { buildGraph } from "../src/graph/build.js";+import { writeGraph, readGraph, wiringPath } from "../src/graph/write.js";+import { contextDirFor } from "../src/context/node-file.js";+import type { GraphV1, NodeV1 } from "../src/graph/types.js";++function makeNode(overrides: Partial<NodeV1> = {}): NodeV1 {+ return {+ id: "a.ts#f",+ name: "f",+ kind: "function",+ path: "a.ts",+ span: "L1-L3",+ signature: "f(): void",+ exported: true,+ origin: "ast",+ body_hash: "hash",+ summary_state: "pending",+ summary: null,+ crux: null,+ ...overrides,+ };+}++test("writeGraph strips body_text from the serialized node but keeps every other field", () => {+ const dir = mkdtempSync(join(tmpdir(), "graft-write-"));+ try {+ const node = makeNode({ body_text: "some searchable definition body" });+ const graph: GraphV1 = {+ meta: { version: 1, nodeCount: 1, edgeCount: 0, languages: ["typescript"] },+ nodes: [node],+ edges: [],+ };+ const path = writeGraph(graph, dir);++ const raw = JSON.parse(readFileSync(path, "utf8"));+ assert.equal(raw.nodes.length, 1);+ assert.ok(!("body_text" in raw.nodes[0]), "serialized node must not carry body_text");+ // Every other field is untouched.+ assert.equal(raw.nodes[0].id, node.id);+ assert.equal(raw.nodes[0].name, node.name);+ assert.equal(raw.nodes[0].signature, node.signature);+ assert.equal(raw.nodes[0].body_hash, node.body_hash);++ const reread = readGraph(path);+ assert.equal(reread!.nodes[0].body_text, undefined);+ } finally {+ rmSync(dir, { recursive: true, force: true });+ }+});++test("writeGraph does not mutate the in-memory node — build.ts's sidecar pass needs body_text intact", () => {+ const dir = mkdtempSync(join(tmpdir(), "graft-write-mutate-"));+ try {+ const node = makeNode({ body_text: "untouched body text" });+ const graph: GraphV1 = {+ meta: { version: 1, nodeCount: 1, edgeCount: 0, languages: ["typescript"] },+ nodes: [node],+ edges: [],+ };+ writeGraph(graph, dir);+ assert.equal(node.body_text, "untouched body text", "original node object must be unchanged");+ assert.equal(graph.nodes[0].body_text, "untouched body text", "graph.nodes must still reference the live object");+ } finally {+ rmSync(dir, { recursive: true, force: true });+ }+});++test("a node with no body_text (e.g. a file node) round-trips unchanged", () => {+ const dir = mkdtempSync(join(tmpdir(), "graft-write-nobody-"));+ try {+ const node = makeNode({ kind: "file", signature: null });+ const graph: GraphV1 = {+ meta: { version: 1, nodeCount: 1, edgeCount: 0, languages: ["typescript"] },+ nodes: [node],+ edges: [],+ };+ const path = writeGraph(graph, dir);+ const raw = JSON.parse(readFileSync(path, "utf8"));+ assert.ok(!("body_text" in raw.nodes[0]));+ assert.equal(raw.nodes[0].kind, "file");+ } finally {+ rmSync(dir, { recursive: true, force: true });+ }+});++// ── Contract line 1: new builds' wiring.json has NO body_text key on any node ──++test("buildGraph: the serialized wiring.json has no body_text key on ANY node", async () => {+ const dir = mkdtempSync(join(tmpdir(), "graft-write-build-"));+ try {+ writeFileSync(+ join(dir, "auth.ts"),+ `/** Validate an incoming API request's auth token. */\n` ++ `export function validateAuthToken(token: string): boolean {\n` ++ ` return token.length > 0;\n` ++ `}\n\n` ++ `export const FEATURE_FLAG = true;\n`,+ );+ const result = await buildGraph(dir);+ assert.ok(result.nodes > 0, "sanity: the fixture actually produced nodes");++ const outDir = contextDirFor(dir);+ const raw = readFileSync(wiringPath(outDir), "utf8");+ assert.ok(!raw.includes("body_text"), "raw wiring.json text must not contain the body_text key at all");++ const parsed = JSON.parse(raw) as GraphV1;+ assert.ok(parsed.nodes.length > 0);+ for (const n of parsed.nodes) {+ assert.ok(!("body_text" in n), `node ${n.id} must not carry body_text in the serialized graph`);+ }+ } finally {+ rmSync(dir, { recursive: true, force: true });+ }+});