Turbocharge Claude Code, Cursor, Codex, Gemini & every coding agent: faster, cheaper, with contextual understanding specific to your codebase.
TypeScript ai-agentsanthropicclaude-codeclicode-graphcodexcontext-engineeringcursordeveloper-toolsgeminiknowledge-graphllmmcpmcp-serveropen-sourceopenaitree-sitter
identify the extractor by hashing it, not by one file's mtime
anirudhkumar-nanonets1e3fbdba653b
diff --git a/src/graph/extract-cache.ts b/src/graph/extract-cache.tsindex 11b5dfcc..a3209d4b 100644--- a/src/graph/extract-cache.ts+++ b/src/graph/extract-cache.ts@@ -19,13 +19,14 @@** Lives under `.cache/` (gitignored, regenerate-anytime) next to `ask-index.json`* and `summaries.json`. Two things invalidate the whole file: a bump of- * {@link CACHE_VERSION}, and a change to the extractor module itself (stamped by- * its mtime+size, so an npm upgrade or a local rebuild drops stale parses without- * anyone remembering to bump anything).+ * {@link CACHE_VERSION} for a change of on-disk shape, and a change to the+ * extraction code itself, caught by content-hashing it — see+ * {@link extractorStamp}. Neither asks anyone to remember to bump anything.*/-import { statSync } from "node:fs";+import { readFileSync, readdirSync } from "node:fs";+import { createHash } from "node:crypto";import { fileURLToPath } from "node:url";-import { join } from "node:path";+import { dirname, extname, join } from "node:path";import { CACHE_DIR } from "../context/node-file.js";import { readJson, writeJsonAtomic } from "../util/state.js";import type { RawEdge } from "./extract.js";@@ -61,20 +62,83 @@ export function extractCachePath(outDir: string): string {return join(outDir, CACHE_DIR, EXTRACT_CACHE_FILE);}-/** `mtime:size` of the extractor module, or `"unknown"` when it can't be stat'd.- * Resolved by rewriting this module's own URL, so it works both in `dist/` (.js)- * and under tsx (.ts) without knowing which layout we're in. */+/** Computed once per process — this can't change under a running process without+ * the module graph itself being swapped, and a long-lived MCP server holding the+ * identity of the code it actually loaded is the correct answer anyway. */+let memoizedStamp: string | null = null;++/**+ * Identity of the code that produces the cached parses. Both sidecars key on it,+ * so *anything* that can change extraction output has to change this string.+ *+ * Content-hashed over every sibling module in this directory, plus the package+ * version. Two earlier instincts are deliberately rejected:+ *+ * - **Not a timestamp.** This was `mtime:size` of `extract.js` alone, which is+ * wrong in both directions. Too loose: the parse of `db.count()` depends on+ * `bindings.ts` deciding that `db` is a `UserRepo` (that answer is stored *in*+ * the cached entry, as an edge's `recvType`), and fixing a binding bug leaves+ * `extract.js`'s emitted bytes untouched — so any build that skips rewriting+ * unchanged output keeps the old stamp and silently replays pre-fix edges. Too+ * tight: a rebuild that rewrites identical content moves the mtime and throws+ * away the whole memo for nothing.+ * - **Not a hand-kept list of modules.** A list is correct exactly until someone+ * moves parse logic into a module nobody added to it, and the failure is silent.+ *+ * Hashing the directory over-invalidates a little — editing any `graph/` module+ * costs one cold rebuild — which is no more often than a version bump already+ * costs, and always in the safe direction. The package version is folded in so a+ * tree-sitter grammar upgrade (which changes parse output without changing any of+ * graft's own files) invalidates too.+ *+ * Measured at ~0.5ms for 21 files / 556KB, paid once per process.+ */export function extractorStamp(): string {+ if (memoizedStamp === null) memoizedStamp = computeStamp();+ return memoizedStamp;+}++function computeStamp(): string {try {- const url = import.meta.url.replace(/extract-cache\.(c|m)?([jt]s)$/, "extract.$1$2");- if (url === import.meta.url) return "unknown";- const s = statSync(fileURLToPath(url));- return `${s.mtimeMs}:${s.size}`;+ const self = fileURLToPath(import.meta.url);+ // `.js` when running from `dist/`, `.ts` under tsx — take the extension from+ // our own filename rather than guessing which layout we're in.+ const dir = dirname(self);+ return stampDir(dir, extname(self), packageVersion(dir));} catch {return "unknown";}}+/**+ * Hash every `ext` file in `dir`, plus `version`. Separated from+ * {@link extractorStamp} so a test can prove the property that matters: a change+ * to *any* module in the directory moves the stamp, not just the one the old+ * implementation happened to watch.+ */+export function stampDir(dir: string, ext: string, version = ""): string {+ const files = readdirSync(dir).filter((f) => f.endsWith(ext)).sort();+ if (!files.length) return "unknown";+ const h = createHash("sha256");+ h.update(version);+ for (const f of files) {+ h.update(f); // a rename is a change, even at identical content+ h.update(readFileSync(join(dir, f)));+ }+ return h.digest("hex").slice(0, 16);+}++/** graft's own version, or `""` when it can't be read (then the content hash+ * alone carries the stamp — still correct for every local edit). */+function packageVersion(graphDir: string): string {+ try {+ const pkg = JSON.parse(readFileSync(join(graphDir, "..", "..", "package.json"), "utf8")) as { version?: string };+ return pkg.version ?? "";+ } catch {+ return "";+ }+}+export function emptyExtractCache(): ExtractCache {return { version: CACHE_VERSION, extractor: extractorStamp(), files: {} };}diff --git a/test/graph-incremental.test.ts b/test/graph-incremental.test.tsindex 453cb788..583bfa9a 100644--- a/test/graph-incremental.test.ts+++ b/test/graph-incremental.test.ts@@ -10,7 +10,7 @@ import { chmodSync, mkdtempSync, mkdirSync, readFileSync, rmSync, utimesSync, wrimport { tmpdir } from "node:os";import { join } from "node:path";import { buildGraph } from "../src/graph/build.js";-import { readExtractCache } from "../src/graph/extract-cache.js";+import { extractCachePath, extractorStamp, readExtractCache, stampDir } from "../src/graph/extract-cache.js";import { isClean, probeDrift, readFingerprint } from "../src/graph/fingerprint.js";import { readAskIndex } from "../src/ask/index-file.js";import { readGraph, wiringPath } from "../src/graph/write.js";@@ -189,3 +189,59 @@ test("an unreadable file is still recorded, so it can't look new on every probe"const g = readGraph(wiringPath(outOf(d))) as GraphV1;assert.ok(g.nodes.some((n) => n.id === "src/secret.ts#secretFn"), "the once-unreadable file is indexed now");});++/**+ * The stamp is the memo's second key: the file hash says "these bytes", the stamp+ * says "and this is the code that parsed them". Get it wrong and a fixed extractor+ * silently keeps serving output from the broken one.+ */+test("extractorStamp: a real identity for the loaded extraction code", () => {+ const s = extractorStamp();+ assert.notEqual(s, "unknown", "path resolution must work under both tsx and dist/, or invalidation silently stops");+ assert.match(s, /^[0-9a-f]{16}$/);+ assert.equal(s, extractorStamp(), "memoized — the cost is paid once per process");+});++test("the stamp moves for any module in the extractor directory, not just one", () => {+ const dir = mkdtempSync(join(tmpdir(), "graft-stamp-"));+ writeFileSync(join(dir, "extract.js"), "export const a = 1;\n");+ writeFileSync(join(dir, "bindings.js"), "export const b = 1;\n");+ writeFileSync(join(dir, "notes.txt"), "not code\n");+ const base = stampDir(dir, ".js", "0.8.0");++ assert.equal(stampDir(dir, ".js", "0.8.0"), base, "stable for identical input");++ // The bug this replaces: only `extract.js` was watched, so a receiver-type fix+ // in bindings.js left the stamp — and therefore every cached parse — untouched.+ writeFileSync(join(dir, "bindings.js"), "export const b = 2;\n");+ const afterSibling = stampDir(dir, ".js", "0.8.0");+ assert.notEqual(afterSibling, base, "a sibling module's content must count");++ // A grammar upgrade changes parse output without touching any of graft's files.+ assert.notEqual(stampDir(dir, ".js", "0.8.1"), afterSibling, "the package version must count");++ // Non-code in the same directory must not churn the memo.+ writeFileSync(join(dir, "notes.txt"), "still not code\n");+ assert.equal(stampDir(dir, ".js", "0.8.0"), afterSibling, "a non-module file in the directory must not churn the memo");++ const beforeRename = afterSibling;+ rmSync(join(dir, "bindings.js"));+ writeFileSync(join(dir, "renamed.js"), "export const b = 2;\n");+ assert.notEqual(stampDir(dir, ".js", "0.8.0"), beforeRename, "a rename at identical content is still a change");+});++test("a memo written by a different extractor is dropped, not replayed", async () => {+ const d = repo();+ await buildGraph(d);+ const path = extractCachePath(outOf(d));+ const cache = JSON.parse(readFileSync(path, "utf8"));+ assert.ok(Object.keys(cache.files).length > 0);++ writeFileSync(path, JSON.stringify({ ...cache, extractor: "deadbeefdeadbeef" }));+ assert.deepEqual(readExtractCache(outOf(d)).files, {}, "entries from an unknown extractor are worthless");++ // So the next build re-parses everything rather than trusting them.+ const cold = await buildGraph(d);+ assert.equal(cold.reused, 0);+ assert.equal(cold.parsed, cold.files);+});