Skip to content

Turbocharge Claude Code, Cursor, Codex, Gemini & every coding agent: faster, cheaper, with contextual understanding specific to your codebase.

fix(ask): best-effort sidecar write, docCount cross-check, move index to .cache

Shrish Dwivedieef949377652
diff --git a/src/ask/ask.ts b/src/ask/ask.ts
index 1f938eaa..408c64a5 100644
--- a/src/ask/ask.ts
+++ b/src/ask/ask.ts
@@ -67,7 +67,7 @@ function firstProse(body: string): string {
interface Corpus {
concepts: Array<{ slug: string; name: string; sources: string[]; related: string[]; snippet: string; text: string }>;
graph: GraphV1 | null;
- /** Build-time token/df sidecar (`.graph/index.json`), or null when absent,
+ /** Build-time token/df sidecar (`.cache/ask-index.json`), or null when absent,
* unparseable, an unknown version, or stale (checked against `graph.nodes.length`
* by the caller — see `lexical()`). Null means: fall back to live tokenization. */
askIndex: AskIndex | null;
@@ -274,7 +274,7 @@ function lexical(query: string, corpus: Corpus, limit: number, graphRank: boolea
body: counts(tokenize(c.text)),
}));
- // A build-time sidecar (`.graph/index.json`) is usable only when it covers
+ // A build-time sidecar (`.cache/ask-index.json`) is usable only when it covers
// every node in the current graph: `docs.length === graph.nodes.length` is a
// cheap staleness guard (an id-level check below catches a same-count but
// mismatched set, e.g. a hand-edited or corrupt sidecar). Anything short of
diff --git a/src/ask/index-file.ts b/src/ask/index-file.ts
index e092fec2..6d414bbe 100644
--- a/src/ask/index-file.ts
+++ b/src/ask/index-file.ts
@@ -1,11 +1,13 @@
/**
- * Build-time sidecar for `graft ask` — `<outDir>/.graph/index.json`.
+ * Build-time sidecar for `graft ask` — `<outDir>/.cache/ask-index.json`.
*
* `ask`'s lexical pass tokenizes every symbol node's name/path/body on every
* query; at 32k nodes that re-tokenization is ~45% of query time (profiled).
- * `graft build` writes this sidecar once, alongside `wiring.json`, with the
- * token→count bags per node plus the corpus-wide document frequencies, so a
- * query just reads counts instead of re-splitting every node's text.
+ * `graft build` writes this sidecar once, with the token→count bags per node
+ * plus the corpus-wide document frequencies, so a query just reads counts
+ * instead of re-splitting every node's text. It's a derived cache, not
+ * checked-in graph data, so it lives under the gitignored `.cache/` dir
+ * (see `CACHE_DIR` in `context/node-file.ts`) rather than `.graph/`.
*
* `tokenize`/`counts` live here (not duplicated in `ask.ts`) so build-time and
* query-time text-splitting are provably the same function — the sidecar can
@@ -20,6 +22,7 @@
import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs";
import { dirname, join } from "node:path";
import type { GraphV1 } from "../graph/types.js";
+import { CACHE_DIR } from "../context/node-file.js";
/** Words too common/short to carry query intent — dropped before scoring. */
const STOP = new Set([
@@ -65,12 +68,13 @@ export interface AskIndex {
docs: AskIndexDoc[];
}
-const ASK_INDEX_DIR = ".graph";
-const ASK_INDEX_FILE = "index.json";
+const ASK_INDEX_FILE = "ask-index.json";
-/** Absolute path to the ask sidecar for a context dir: `<dir>/.graph/index.json`. */
+/** Absolute path to the ask sidecar for a context dir: `<dir>/.cache/ask-index.json`.
+ * `.cache/` is the established uncommitted-cache location — this sidecar is a
+ * derived, regenerate-anytime cache, not checked-in graph data. */
export function askIndexPath(outDir: string): string {
- return join(outDir, ASK_INDEX_DIR, ASK_INDEX_FILE);
+ return join(outDir, CACHE_DIR, ASK_INDEX_FILE);
}
function pairs(m: Map<string, number>): [string, number][] {
@@ -86,9 +90,10 @@ function bagLen(p: [string, number][]): number {
/**
* Tokenize every node in `graph` (exactly as `ask.ts`'s lexical pass does) and
- * write the resulting bags + document frequencies to `<outDir>/.graph/index.json`.
- * Returns the path written. Deterministic: nodes are indexed in id order, so an
- * unchanged graph produces a byte-identical sidecar.
+ * write the resulting bags + document frequencies to
+ * `<outDir>/.cache/ask-index.json`. Returns the path written. Deterministic:
+ * nodes are indexed in id order, so an unchanged graph produces a
+ * byte-identical sidecar.
*/
export function writeAskIndex(outDir: string, graph: GraphV1): string {
const nodes = [...graph.nodes].sort((a, b) => a.id.localeCompare(b.id));
@@ -126,8 +131,10 @@ export function writeAskIndex(outDir: string, graph: GraphV1): string {
}
/** Read the ask sidecar. Returns null on a missing file, unparseable JSON, an
- * unrecognized shape, or an unknown `version` — any of which means the caller
- * should fall back to live tokenization, never crash. */
+ * unrecognized shape, an unknown `version`, or a `docCount` that doesn't match
+ * the number of docs actually stored (a corrupted/truncated sidecar would
+ * otherwise silently skew IDF) — any of which means the caller should fall
+ * back to live tokenization, never crash or trust bad data. */
export function readAskIndex(outDir: string): AskIndex | null {
const path = askIndexPath(outDir);
if (!existsSync(path)) return null;
@@ -140,7 +147,8 @@ export function readAskIndex(outDir: string): AskIndex | null {
typeof raw.docCount !== "number" ||
typeof raw.avgBodyLen !== "number" ||
!Array.isArray(raw.df) ||
- !Array.isArray(raw.docs)
+ !Array.isArray(raw.docs) ||
+ raw.docCount !== raw.docs.length
) {
return null;
}
diff --git a/src/graph/build.ts b/src/graph/build.ts
index b7606b65..08e7c8a5 100644
--- a/src/graph/build.ts
+++ b/src/graph/build.ts
@@ -138,9 +138,16 @@ export async function buildGraph(
const graphPath = writeGraph(graph, outDir);
// `ask`'s token/IDF sidecar — moves per-query corpus tokenization to build
- // time (~45% of query time on a 32k-node graph, profiled). Lands next to
- // wiring.json; `ask` falls back to live tokenization when it's absent/stale.
- writeAskIndex(outDir, graph);
+ // time (~45% of query time on a 32k-node graph, profiled). Lives in the
+ // cache dir; `ask` falls back to live tokenization when it's absent/stale.
+ // Best-effort: wiring.json is already on disk at this point, so a sidecar
+ // write failure (e.g. an unwritable cache dir) must not abort cards/index/
+ // covers below — record it and keep going, same as other recoverable errors.
+ try {
+ writeAskIndex(outDir, graph);
+ } catch (err) {
+ errors.push(`ask-index: ${err instanceof Error ? err.message : String(err)}`);
+ }
// Tier-2 passive surface: project the nodes into per-file markdown cards, and
// refresh the INDEX roster. Pure projection — no LLM, no network.
diff --git a/test/ask-index.test.ts b/test/ask-index.test.ts
index 5e0e00d7..6e80a4b1 100644
--- a/test/ask-index.test.ts
+++ b/test/ask-index.test.ts
@@ -1,5 +1,5 @@
/**
- * Tests for the `ask` build-time sidecar (`.graph/index.json`).
+ * Tests for the `ask` build-time sidecar (`.cache/ask-index.json`).
*
* `graft build` writes token/document-frequency bags once so `ask` doesn't
* re-tokenize the whole corpus per query. These tests pin down the contract
@@ -10,8 +10,8 @@
*/
import { test } from "node:test";
import assert from "node:assert/strict";
-import { mkdtempSync, writeFileSync, rmSync, readFileSync, unlinkSync } from "node:fs";
-import { join } from "node:path";
+import { mkdtempSync, writeFileSync, mkdirSync, rmSync, readFileSync, unlinkSync } from "node:fs";
+import { join, dirname } from "node:path";
import { tmpdir } from "node:os";
import { buildGraph } from "../src/graph/build.js";
import { ask } from "../src/ask/ask.js";
@@ -180,3 +180,49 @@ test("readAskIndex returns null when the sidecar is simply missing", () => {
rmSync(dir, { recursive: true, force: true });
}
});
+
+test("readAskIndex returns null when docCount doesn't match docs.length", () => {
+ const dir = mkdtempSync(join(tmpdir(), "graft-ask-index-doccount-"));
+ try {
+ const outDir = contextDirFor(dir);
+ const idxPath = askIndexPath(outDir);
+ mkdirSync(dirname(idxPath), { recursive: true });
+ const corrupted = {
+ version: 1,
+ avgBodyLen: 3,
+ df: [["foo", 1]],
+ docCount: 5, // deliberately mismatched with docs below
+ docs: [{ id: "a", name: [], path: [], body: [["foo", 1]] }],
+ };
+ writeFileSync(idxPath, JSON.stringify(corrupted));
+ assert.equal(readAskIndex(outDir), null, "docCount !== docs.length must read as null");
+ } finally {
+ rmSync(dir, { recursive: true, force: true });
+ }
+});
+
+test("a failed sidecar write is recorded in build errors, not fatal", async () => {
+ const dir = makeFixture();
+ try {
+ const outDir = contextDirFor(dir);
+ const idxPath = askIndexPath(outDir);
+ // Pre-create the sidecar's own path AS A DIRECTORY so writeAskIndex's
+ // writeFileSync fails with EISDIR — simulates any write failure without
+ // needing real permission tricks.
+ mkdirSync(idxPath, { recursive: true });
+
+ const result = await buildGraph(dir);
+
+ assert.ok(
+ result.errors.some((e) => e.startsWith("ask-index:")),
+ `expected an "ask-index: ..." entry in result.errors, got: ${JSON.stringify(result.errors)}`,
+ );
+ // The rest of the build must still have completed: wiring.json, cards,
+ // and INDEX all still get written even though the sidecar write failed.
+ const graph = readGraph(wiringPath(outDir));
+ assert.ok(graph, "wiring graph should still be written despite the sidecar failure");
+ assert.ok(result.cards > 0, "cards should still be written despite the sidecar failure");
+ } finally {
+ rmSync(dir, { recursive: true, force: true });
+ }
+});