Turbocharge Claude Code, Cursor, Codex, Gemini & every coding agent: faster, cheaper, with contextual understanding specific to your codebase.
TypeScript ai-agentsanthropicclaude-codeclicode-graphcodexcontext-engineeringcursordeveloper-toolsgeminiknowledge-graphllmmcpmcp-serveropen-sourceopenaitree-sitter
Report the language, not the grammar that parsed it
anirudhkumar-nanonets65c99302e91e
diff --git a/src/graph/build.ts b/src/graph/build.tsindex 1f917e7a..79360227 100644--- a/src/graph/build.ts+++ b/src/graph/build.ts@@ -17,7 +17,7 @@ import { readFileSync } from "node:fs";import { basename, dirname, resolve } from "node:path";import { walkDir } from "../ingest/fs.js";import { contextDirFor, ensureGitignored, ensureSearchable } from "../context/node-file.js";-import { extractFile, languageOf, type Language, type RawEdge } from "./extract.js";+import { extractFile, languageLabelOf, languageOf, type RawEdge } from "./extract.js";import { contentHash } from "../util/id.js";import { relPosix } from "../util/paths.js";import {@@ -142,7 +142,9 @@ export async function buildGraph(const nodes: NodeV1[] = [];const rawEdges: RawEdge[] = [];const sources = new Map<string, string>();- const langs = new Set<Language>();+ /** Display labels, not grammars — `.mjs` is parsed as typescript but reported as+ * javascript, or the banner claims a repo's JavaScript went unindexed. */+ const langs = new Set<string>();const errors: string[] = [];// In a git worktree there is nothing to reuse *yet* — `graft/` is gitignored, so@@ -165,6 +167,7 @@ export async function buildGraph(const rel = f.rel;opts.onProgress?.({ phase: "parse", index: i, total: files.length, file: rel });const lang = languageOf(f.abs)!;+ const label = languageLabelOf(f.abs)!;const cached = priorExtract.files[rel];// Every file is read and hashed, every build — only the *parse* is memoized.@@ -199,7 +202,7 @@ export async function buildGraph(}nodes.push(...cached.nodes);rawEdges.push(...cached.rawEdges);- langs.add(lang);+ langs.add(label);return;}@@ -209,7 +212,7 @@ export async function buildGraph(nodes.push(...fileNodes);rawEdges.push(...fileEdges);sources.set(rel, source);- langs.add(lang);+ langs.add(label);entries[rel] = { size: f.size, mtimeMs: f.mtimeMs, hash, nodes: fileNodes, rawEdges: fileEdges };} catch (err) {const message = `${rel}: parse failed — ${err instanceof Error ? err.message : String(err)}`;diff --git a/src/graph/extract.ts b/src/graph/extract.tsindex 63c2d6b0..7de33b42 100644--- a/src/graph/extract.ts+++ b/src/graph/extract.ts@@ -17,14 +17,52 @@ import type { Kind, NodeV1, Relation } from "./types.js";export type Language = "typescript" | "tsx" | "python" | "go";+/**+ * Extension → the tree-sitter grammar that parses it, and the label a human expects+ * to see for it.+ *+ * The two are not the same, and conflating them under-reported coverage: `.mjs` is+ * parsed by the typescript grammar, so a JS repo's build banner read `[typescript]`+ * and a `.jsx` one read `[tsx]`. Both are true about the *parser* and misleading+ * about the repo — people went looking for why their JavaScript hadn't been indexed+ * when it had, and could not tell a language that was merely unlabelled from one+ * that really was skipped (see issue #36).+ *+ * One table, both readings derived from it, so adding an extension cannot fix+ * extraction and forget the label. Ordered longest-suffix-first: `.tsx` has to be+ * tested before `.ts` would match it.+ */+const EXTENSIONS: ReadonlyArray<{ ext: string; grammar: Language; label: string }> = [+ { ext: ".tsx", grammar: "tsx", label: "tsx" },+ { ext: ".jsx", grammar: "tsx", label: "jsx" },+ { ext: ".mts", grammar: "typescript", label: "typescript" },+ { ext: ".cts", grammar: "typescript", label: "typescript" },+ { ext: ".ts", grammar: "typescript", label: "typescript" },+ { ext: ".mjs", grammar: "typescript", label: "javascript" },+ { ext: ".cjs", grammar: "typescript", label: "javascript" },+ { ext: ".js", grammar: "typescript", label: "javascript" },+ { ext: ".pyi", grammar: "python", label: "python" },+ { ext: ".py", grammar: "python", label: "python" },+ { ext: ".go", grammar: "go", label: "go" },+];++function entryFor(path: string): (typeof EXTENSIONS)[number] | undefined {+ const p = path.toLowerCase();+ return EXTENSIONS.find((e) => p.endsWith(e.ext));+}+/** Map a file path to a supported language, or null if unsupported. */export function languageOf(path: string): Language | null {- const p = path.toLowerCase();- if (p.endsWith(".tsx") || p.endsWith(".jsx")) return "tsx";- if (/\.(ts|mts|cts|js|mjs|cjs)$/.test(p)) return "typescript";- if (p.endsWith(".py") || p.endsWith(".pyi")) return "python";- if (p.endsWith(".go")) return "go";- return null;+ return entryFor(path)?.grammar ?? null;+}++/**+ * What to *call* the language of this file, for a banner or a repo map — or null when+ * the file isn't indexed at all, which is the distinction {@link languageOf} shares+ * and the one that matters to a reader checking coverage.+ */+export function languageLabelOf(path: string): string | null {+ return entryFor(path)?.label ?? null;}/**diff --git a/src/graph/map.ts b/src/graph/map.tsindex 4af4eeb3..b7db88a0 100644--- a/src/graph/map.ts+++ b/src/graph/map.ts@@ -19,7 +19,7 @@* `buildRepoMap` + `formatRepoMap` directly.*/import type { GraphV1, NodeV1 } from "./types.js";-import { languageOf } from "./extract.js";+import { languageLabelOf } from "./extract.js";import { WALK_RELATIONS } from "./relations.js";import { scopeLabel, scopeOf, scopesOfGraph } from "./scopes.js";import { savingsFooter, savingsFor, type Savings } from "../context/savings.js";@@ -127,11 +127,13 @@ function topHubs(nodes: NodeV1[], inDegree: Map<string, number>, cap: number): H.slice(0, cap);}+/** Display labels, not tree-sitter grammars: a `scripts/` tree of `.mjs` files is+ * "javascript" here, not "typescript". See {@link languageLabelOf}. */function sortedLanguages(paths: string[]): string[] {const set = new Set<string>();for (const p of paths) {- const lang = languageOf(p);- if (lang) set.add(lang);+ const label = languageLabelOf(p);+ if (label) set.add(label);}return [...set].sort();}diff --git a/test/graph-languages.test.ts b/test/graph-languages.test.tsnew file mode 100644index 00000000..a63fa44c--- /dev/null+++ b/test/graph-languages.test.ts@@ -0,0 +1,92 @@+/**+ * What graft *calls* the languages it indexed, as distinct from the tree-sitter+ * grammar it used to parse them.+ *+ * The bug (#36): `.mjs` and `.js` are parsed by the typescript grammar and `.jsx` by+ * the tsx one, and those grammar names were reported verbatim as the languages+ * present. A repo of four `.ts` files and one `.mjs` announced `[typescript]` while+ * the `.mjs` file's symbols were perfectly queryable, so the banner — the natural+ * place to check coverage — sent people looking for a problem that wasn't there, and+ * gave them no way to tell a language that was merely unlabelled from one that+ * really had been skipped.+ */+import { test } from "node:test";+import assert from "node:assert/strict";+import { mkdirSync, mkdtempSync, writeFileSync } from "node:fs";+import { tmpdir } from "node:os";+import { join } from "node:path";+import { buildGraph } from "../src/graph/build.js";+import { buildRepoMap } from "../src/graph/map.js";+import { languageLabelOf, languageOf } from "../src/graph/extract.js";+import { readGraph, wiringPath } from "../src/graph/write.js";++/** Every extension graft claims to index, one per grammar/label pairing. */+const INDEXED = [+ "a.ts", "a.mts", "a.cts",+ "a.js", "a.mjs", "a.cjs",+ "a.tsx", "a.jsx",+ "a.py", "a.pyi",+ "a.go",+];++test("a file is labelled exactly when it is indexed", () => {+ // The guard that keeps #36 from coming back: both readings come off one table, so+ // adding an extension cannot teach the extractor about it and leave the banner+ // silent (or vice versa — claiming coverage of a file nothing can parse).+ for (const f of [...INDEXED, "a.txt", "a.rs", "README.md", "noextension", ""]) {+ assert.equal(+ languageOf(f) === null,+ languageLabelOf(f) === null,+ `${f}: languageOf and languageLabelOf disagree about whether it is indexed`,+ );+ }+});++test("labels name the language, not the grammar that parses it", () => {+ assert.equal(languageLabelOf("scripts/tool.mjs"), "javascript");+ assert.equal(languageLabelOf("scripts/tool.cjs"), "javascript");+ assert.equal(languageLabelOf("web/app.js"), "javascript");+ assert.equal(languageLabelOf("web/app.jsx"), "jsx");+ assert.equal(languageLabelOf("src/app.tsx"), "tsx");+ assert.equal(languageLabelOf("src/app.ts"), "typescript");+ assert.equal(languageLabelOf("src/app.mts"), "typescript");+ assert.equal(languageLabelOf("api/main.py"), "python");+ assert.equal(languageLabelOf("api/main.pyi"), "python");+ assert.equal(languageLabelOf("cmd/main.go"), "go");++ // The grammar is unchanged — extraction, the extract cache and every `Language`+ // switch still see exactly what they saw before.+ assert.equal(languageOf("scripts/tool.mjs"), "typescript");+ assert.equal(languageOf("web/app.jsx"), "tsx");+});++test("labels are case-insensitive, like the grammar lookup", () => {+ assert.equal(languageLabelOf("src/App.TSX"), "tsx");+ assert.equal(languageLabelOf("scripts/TOOL.MJS"), "javascript");+});++test("the build banner and repo map report every language they indexed", async () => {+ // #36's repro, generalized: one file per label, all of them genuinely parsed.+ const d = mkdtempSync(join(tmpdir(), "graft-langs-"));+ mkdirSync(join(d, "src"), { recursive: true });+ mkdirSync(join(d, "scripts"), { recursive: true });+ writeFileSync(join(d, "src", "a.ts"), "export function tsOnly(): number {\n return 1;\n}\n");+ writeFileSync(join(d, "src", "b.tsx"), "export function TsxOnly(): null {\n return null;\n}\n");+ writeFileSync(join(d, "scripts", "tool.mjs"), "export function mjsOnlySymbol() {\n return 1;\n}\n");+ writeFileSync(join(d, "scripts", "web.jsx"), "export function JsxOnly() {\n return null;\n}\n");+ writeFileSync(join(d, "src", "c.py"), "def py_only():\n return 1\n");++ const r = await buildGraph(d);+ assert.deepEqual(r.languages, ["javascript", "jsx", "python", "tsx", "typescript"]);++ // The reported symbol was queryable all along — that mismatch is what the issue was+ // about, so pin both halves together.+ const graph = readGraph(wiringPath(r.contextDir));+ assert.ok(+ graph?.nodes.some((n) => n.name === "mjsOnlySymbol"),+ "the .mjs file really is indexed, which is why omitting its label misled",+ );++ // `map` derives labels from paths independently, so it gets its own assertion.+ assert.deepEqual(buildRepoMap(graph!).totals.languages, r.languages);+});