Understand any codebase instantly. System intelligence for codebases, built for humans and AI.
TypeScript aiai-memoryclaude-codeclicode-analysiscode-mappingcode-memorycodebase-understandingcodexdeveloper-toolsllmopenclawpersistent-memoryprogram-analysissoftware-architecturesystem-intelligencesystem-mapping
perf(context): key the BM25 index on a tokenizer fingerprint
KageBinarya541cb0f2a81
diff --git a/ix-cli/src/cli/__tests__/bm25-cache.test.ts b/ix-cli/src/cli/__tests__/bm25-cache.test.tsindex a12125e6..1240dbb6 100644--- a/ix-cli/src/cli/__tests__/bm25-cache.test.ts+++ b/ix-cli/src/cli/__tests__/bm25-cache.test.ts@@ -6,7 +6,7 @@ import { tmpdir } from "node:os";import { dirname, join } from "node:path";import { afterEach, beforeEach, describe, expect, it } from "vitest";-import { bm25IndexPath, cachedBm25Ranker, readGitState } from "../explain/bm25-cache.js";+import { bm25IndexPath, cachedBm25Ranker, readGitState, TOKENIZER_FINGERPRINT } from "../explain/bm25-cache.js";import { bm25Rank, isSourcePath, MAX_BM25_BYTES } from "../explain/issue.js";import { gitRepoAccess, type RepoAccess } from "../explain/text-references.js";@@ -121,6 +121,22 @@ describe("cachedBm25Ranker", () => {expect(JSON.parse(readFileSync(bm25IndexPath(dir), "utf-8")).head).not.toBe(before);});+ it("rebuilds an index a different tokenizer built", async () => {+ const repo = gitRepoAccess(dir)!;+ const files = sourceFiles(repo);+ cachedBm25Ranker(dir)(repo, files, QUERIES[0]);+ const path = bm25IndexPath(dir);+ const stored = JSON.parse(readFileSync(path, "utf-8"));+ expect(stored.tok).toBe(TOKENIZER_FINGERPRINT);+ // As an older build would have left it: same HEAD, other tokens.+ writeFileSync(path, JSON.stringify({ ...stored, tok: "an-older-tokenizer", postings: {} }));++ const watched = counting(repo);+ expect(cachedBm25Ranker(dir)(watched, files, QUERIES[0])).toEqual(bm25Rank(repo, files, QUERIES[0]));+ expect(watched.reads.length).toBeGreaterThan(0);+ expect(JSON.parse(readFileSync(path, "utf-8")).tok).toBe(TOKENIZER_FINGERPRINT);+ });+it("keeps skipping a file too large to score without reading it again", async () => {const repo = gitRepoAccess(dir)!;const files = sourceFiles(repo);diff --git a/ix-cli/src/cli/explain/bm25-cache.ts b/ix-cli/src/cli/explain/bm25-cache.tsindex d713279b..1ed30486 100644--- a/ix-cli/src/cli/explain/bm25-cache.ts+++ b/ix-cli/src/cli/explain/bm25-cache.ts@@ -11,6 +11,7 @@ import {bm25QueryTerms,bm25Rank,bm25Score,+ bm25Tokens,MAX_BM25_BYTES,type Bm25Doc,type Bm25Hit,@@ -38,8 +39,22 @@ import type { RepoAccess } from "./text-references.js";*/const INDEX_VERSION = 1;+/**+ * What the tokenizer does to a probe that exercises its rules: case, camel+ * and snake splitting, digits, punctuation, stopwords and short words. Part of+ * the index key, so a change to `bm25Tokens` that shows here invalidates every+ * stored index even if nobody bumps INDEX_VERSION -- an upgrade must never+ * score new queries against an old tokenizer's postings.+ */+const TOKENIZER_PROBE =+ "The saveModel() of get_openai_callback fails: JsonWriter.toJsonTree in src/a-b/C.ts line 42, a to is HTTPServer2";+export const TOKENIZER_FINGERPRINT = createHash("sha256")+ .update(JSON.stringify(bm25Tokens(TOKENIZER_PROBE))).digest("hex").slice(0, 16);+interface Bm25Index {v: number;+ /** `TOKENIZER_FINGERPRINT` when built. */+ tok: string;root: string;head: string;maxBytes: number;@@ -154,7 +169,7 @@ function buildIndex(root: string,): { docs: Map<string, Bm25Doc>; built: Bm25Index } {const built: Bm25Index = {- v: INDEX_VERSION, root, head: state.head, maxBytes: MAX_BM25_BYTES,+ v: INDEX_VERSION, tok: TOKENIZER_FINGERPRINT, root, head: state.head, maxBytes: MAX_BM25_BYTES,paths: [], lengths: [], skipped: [],// No prototype: `constructor` and `tostring` are words too.postings: Object.create(null) as Record<string, number[]>,@@ -186,7 +201,7 @@ function buildIndex(function loadIndex(path: string, root: string, head: string): Bm25Index | undefined {try {const data = JSON.parse(readFileSync(path, "utf-8")) as Partial<Bm25Index>;- if (data.v !== INDEX_VERSION || data.root !== root || data.head !== head || data.maxBytes !== MAX_BM25_BYTES) return undefined;+ if (data.v !== INDEX_VERSION || data.tok !== TOKENIZER_FINGERPRINT || data.root !== root || data.head !== head || data.maxBytes !== MAX_BM25_BYTES) return undefined;if (!Array.isArray(data.paths) || !Array.isArray(data.lengths) || !Array.isArray(data.skipped)) return undefined;if (data.paths.length !== data.lengths.length || !data.postings || typeof data.postings !== "object") return undefined;return data as Bm25Index;