Skip to content

Turbocharge Claude Code, Cursor, Codex, Gemini & every coding agent: faster, cheaper, with contextual understanding specific to your codebase.

Report the language, not the grammar that parsed it

anirudhkumar-nanonets65c99302e91e
diff --git a/src/graph/build.ts b/src/graph/build.ts
index 1f917e7a..79360227 100644
--- a/src/graph/build.ts
+++ b/src/graph/build.ts
@@ -17,7 +17,7 @@ import { readFileSync } from "node:fs";
import { basename, dirname, resolve } from "node:path";
import { walkDir } from "../ingest/fs.js";
import { contextDirFor, ensureGitignored, ensureSearchable } from "../context/node-file.js";
-import { extractFile, languageOf, type Language, type RawEdge } from "./extract.js";
+import { extractFile, languageLabelOf, languageOf, type RawEdge } from "./extract.js";
import { contentHash } from "../util/id.js";
import { relPosix } from "../util/paths.js";
import {
@@ -142,7 +142,9 @@ export async function buildGraph(
const nodes: NodeV1[] = [];
const rawEdges: RawEdge[] = [];
const sources = new Map<string, string>();
- const langs = new Set<Language>();
+ /** Display labels, not grammars — `.mjs` is parsed as typescript but reported as
+ * javascript, or the banner claims a repo's JavaScript went unindexed. */
+ const langs = new Set<string>();
const errors: string[] = [];
// In a git worktree there is nothing to reuse *yet* — `graft/` is gitignored, so
@@ -165,6 +167,7 @@ export async function buildGraph(
const rel = f.rel;
opts.onProgress?.({ phase: "parse", index: i, total: files.length, file: rel });
const lang = languageOf(f.abs)!;
+ const label = languageLabelOf(f.abs)!;
const cached = priorExtract.files[rel];
// Every file is read and hashed, every build — only the *parse* is memoized.
@@ -199,7 +202,7 @@ export async function buildGraph(
}
nodes.push(...cached.nodes);
rawEdges.push(...cached.rawEdges);
- langs.add(lang);
+ langs.add(label);
return;
}
@@ -209,7 +212,7 @@ export async function buildGraph(
nodes.push(...fileNodes);
rawEdges.push(...fileEdges);
sources.set(rel, source);
- langs.add(lang);
+ langs.add(label);
entries[rel] = { size: f.size, mtimeMs: f.mtimeMs, hash, nodes: fileNodes, rawEdges: fileEdges };
} catch (err) {
const message = `${rel}: parse failed — ${err instanceof Error ? err.message : String(err)}`;
diff --git a/src/graph/extract.ts b/src/graph/extract.ts
index 63c2d6b0..7de33b42 100644
--- a/src/graph/extract.ts
+++ b/src/graph/extract.ts
@@ -17,14 +17,52 @@ import type { Kind, NodeV1, Relation } from "./types.js";
export type Language = "typescript" | "tsx" | "python" | "go";
+/**
+ * Extension → the tree-sitter grammar that parses it, and the label a human expects
+ * to see for it.
+ *
+ * The two are not the same, and conflating them under-reported coverage: `.mjs` is
+ * parsed by the typescript grammar, so a JS repo's build banner read `[typescript]`
+ * and a `.jsx` one read `[tsx]`. Both are true about the *parser* and misleading
+ * about the repo — people went looking for why their JavaScript hadn't been indexed
+ * when it had, and could not tell a language that was merely unlabelled from one
+ * that really was skipped (see issue #36).
+ *
+ * One table, both readings derived from it, so adding an extension cannot fix
+ * extraction and forget the label. Ordered longest-suffix-first: `.tsx` has to be
+ * tested before `.ts` would match it.
+ */
+const EXTENSIONS: ReadonlyArray<{ ext: string; grammar: Language; label: string }> = [
+ { ext: ".tsx", grammar: "tsx", label: "tsx" },
+ { ext: ".jsx", grammar: "tsx", label: "jsx" },
+ { ext: ".mts", grammar: "typescript", label: "typescript" },
+ { ext: ".cts", grammar: "typescript", label: "typescript" },
+ { ext: ".ts", grammar: "typescript", label: "typescript" },
+ { ext: ".mjs", grammar: "typescript", label: "javascript" },
+ { ext: ".cjs", grammar: "typescript", label: "javascript" },
+ { ext: ".js", grammar: "typescript", label: "javascript" },
+ { ext: ".pyi", grammar: "python", label: "python" },
+ { ext: ".py", grammar: "python", label: "python" },
+ { ext: ".go", grammar: "go", label: "go" },
+];
+
+function entryFor(path: string): (typeof EXTENSIONS)[number] | undefined {
+ const p = path.toLowerCase();
+ return EXTENSIONS.find((e) => p.endsWith(e.ext));
+}
+
/** Map a file path to a supported language, or null if unsupported. */
export function languageOf(path: string): Language | null {
- const p = path.toLowerCase();
- if (p.endsWith(".tsx") || p.endsWith(".jsx")) return "tsx";
- if (/\.(ts|mts|cts|js|mjs|cjs)$/.test(p)) return "typescript";
- if (p.endsWith(".py") || p.endsWith(".pyi")) return "python";
- if (p.endsWith(".go")) return "go";
- return null;
+ return entryFor(path)?.grammar ?? null;
+}
+
+/**
+ * What to *call* the language of this file, for a banner or a repo map — or null when
+ * the file isn't indexed at all, which is the distinction {@link languageOf} shares
+ * and the one that matters to a reader checking coverage.
+ */
+export function languageLabelOf(path: string): string | null {
+ return entryFor(path)?.label ?? null;
}
/**
diff --git a/src/graph/map.ts b/src/graph/map.ts
index 4af4eeb3..b7db88a0 100644
--- a/src/graph/map.ts
+++ b/src/graph/map.ts
@@ -19,7 +19,7 @@
* `buildRepoMap` + `formatRepoMap` directly.
*/
import type { GraphV1, NodeV1 } from "./types.js";
-import { languageOf } from "./extract.js";
+import { languageLabelOf } from "./extract.js";
import { WALK_RELATIONS } from "./relations.js";
import { scopeLabel, scopeOf, scopesOfGraph } from "./scopes.js";
import { savingsFooter, savingsFor, type Savings } from "../context/savings.js";
@@ -127,11 +127,13 @@ function topHubs(nodes: NodeV1[], inDegree: Map<string, number>, cap: number): H
.slice(0, cap);
}
+/** Display labels, not tree-sitter grammars: a `scripts/` tree of `.mjs` files is
+ * "javascript" here, not "typescript". See {@link languageLabelOf}. */
function sortedLanguages(paths: string[]): string[] {
const set = new Set<string>();
for (const p of paths) {
- const lang = languageOf(p);
- if (lang) set.add(lang);
+ const label = languageLabelOf(p);
+ if (label) set.add(label);
}
return [...set].sort();
}
diff --git a/test/graph-languages.test.ts b/test/graph-languages.test.ts
new file mode 100644
index 00000000..a63fa44c
--- /dev/null
+++ b/test/graph-languages.test.ts
@@ -0,0 +1,92 @@
+/**
+ * What graft *calls* the languages it indexed, as distinct from the tree-sitter
+ * grammar it used to parse them.
+ *
+ * The bug (#36): `.mjs` and `.js` are parsed by the typescript grammar and `.jsx` by
+ * the tsx one, and those grammar names were reported verbatim as the languages
+ * present. A repo of four `.ts` files and one `.mjs` announced `[typescript]` while
+ * the `.mjs` file's symbols were perfectly queryable, so the banner — the natural
+ * place to check coverage — sent people looking for a problem that wasn't there, and
+ * gave them no way to tell a language that was merely unlabelled from one that
+ * really had been skipped.
+ */
+import { test } from "node:test";
+import assert from "node:assert/strict";
+import { mkdirSync, mkdtempSync, writeFileSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { buildGraph } from "../src/graph/build.js";
+import { buildRepoMap } from "../src/graph/map.js";
+import { languageLabelOf, languageOf } from "../src/graph/extract.js";
+import { readGraph, wiringPath } from "../src/graph/write.js";
+
+/** Every extension graft claims to index, one per grammar/label pairing. */
+const INDEXED = [
+ "a.ts", "a.mts", "a.cts",
+ "a.js", "a.mjs", "a.cjs",
+ "a.tsx", "a.jsx",
+ "a.py", "a.pyi",
+ "a.go",
+];
+
+test("a file is labelled exactly when it is indexed", () => {
+ // The guard that keeps #36 from coming back: both readings come off one table, so
+ // adding an extension cannot teach the extractor about it and leave the banner
+ // silent (or vice versa — claiming coverage of a file nothing can parse).
+ for (const f of [...INDEXED, "a.txt", "a.rs", "README.md", "noextension", ""]) {
+ assert.equal(
+ languageOf(f) === null,
+ languageLabelOf(f) === null,
+ `${f}: languageOf and languageLabelOf disagree about whether it is indexed`,
+ );
+ }
+});
+
+test("labels name the language, not the grammar that parses it", () => {
+ assert.equal(languageLabelOf("scripts/tool.mjs"), "javascript");
+ assert.equal(languageLabelOf("scripts/tool.cjs"), "javascript");
+ assert.equal(languageLabelOf("web/app.js"), "javascript");
+ assert.equal(languageLabelOf("web/app.jsx"), "jsx");
+ assert.equal(languageLabelOf("src/app.tsx"), "tsx");
+ assert.equal(languageLabelOf("src/app.ts"), "typescript");
+ assert.equal(languageLabelOf("src/app.mts"), "typescript");
+ assert.equal(languageLabelOf("api/main.py"), "python");
+ assert.equal(languageLabelOf("api/main.pyi"), "python");
+ assert.equal(languageLabelOf("cmd/main.go"), "go");
+
+ // The grammar is unchanged — extraction, the extract cache and every `Language`
+ // switch still see exactly what they saw before.
+ assert.equal(languageOf("scripts/tool.mjs"), "typescript");
+ assert.equal(languageOf("web/app.jsx"), "tsx");
+});
+
+test("labels are case-insensitive, like the grammar lookup", () => {
+ assert.equal(languageLabelOf("src/App.TSX"), "tsx");
+ assert.equal(languageLabelOf("scripts/TOOL.MJS"), "javascript");
+});
+
+test("the build banner and repo map report every language they indexed", async () => {
+ // #36's repro, generalized: one file per label, all of them genuinely parsed.
+ const d = mkdtempSync(join(tmpdir(), "graft-langs-"));
+ mkdirSync(join(d, "src"), { recursive: true });
+ mkdirSync(join(d, "scripts"), { recursive: true });
+ writeFileSync(join(d, "src", "a.ts"), "export function tsOnly(): number {\n return 1;\n}\n");
+ writeFileSync(join(d, "src", "b.tsx"), "export function TsxOnly(): null {\n return null;\n}\n");
+ writeFileSync(join(d, "scripts", "tool.mjs"), "export function mjsOnlySymbol() {\n return 1;\n}\n");
+ writeFileSync(join(d, "scripts", "web.jsx"), "export function JsxOnly() {\n return null;\n}\n");
+ writeFileSync(join(d, "src", "c.py"), "def py_only():\n return 1\n");
+
+ const r = await buildGraph(d);
+ assert.deepEqual(r.languages, ["javascript", "jsx", "python", "tsx", "typescript"]);
+
+ // The reported symbol was queryable all along — that mismatch is what the issue was
+ // about, so pin both halves together.
+ const graph = readGraph(wiringPath(r.contextDir));
+ assert.ok(
+ graph?.nodes.some((n) => n.name === "mjsOnlySymbol"),
+ "the .mjs file really is indexed, which is why omitting its label misled",
+ );
+
+ // `map` derives labels from paths independently, so it gets its own assertion.
+ assert.deepEqual(buildRepoMap(graph!).totals.languages, r.languages);
+});