Skip to content

Turbocharge Claude Code, Cursor, Codex, Gemini & every coding agent: faster, cheaper, with contextual understanding specific to your codebase.

fix(extract): load native grammars lazily and fall back to WASM when one is missing (#594)

Shrish Dwivedic4411a69d387
diff --git a/src/app/review-process.ts b/src/app/review-process.ts
index 1be03f03..d97dd97d 100644
--- a/src/app/review-process.ts
+++ b/src/app/review-process.ts
@@ -26,11 +26,12 @@
*
* A child process rather than a `worker_threads` worker, deliberately:
*
- * - **The native grammars keep loading the way they already do.** `graph/extract.ts`
- * imports nine tree-sitter addons at module scope and constructs a `Parser` there
- * too, and `graph/generic.ts` loads more through `createRequire`. In a worker all
- * of that is re-instantiated per thread, which puts the N-API-in-worker behaviour
- * of nine third-party native modules between the App and every review. A fork runs
+ * - **The native grammars keep loading the way they already do.** `graph/native.ts`
+ * requires up to nine tree-sitter addons and constructs a `Parser` the first time
+ * a build needs them, and `graph/generic.ts` loads more through `createRequire`.
+ * In a worker all of that is re-instantiated per thread, which puts the
+ * N-API-in-worker behaviour of nine third-party native modules between the App
+ * and every review. A fork runs
* byte-identical code to what production runs today.
* - **A review can be killed.** A wedged parse or a 10-minute clone is a process
* to SIGKILL, not a thread with no safe way to stop it.
diff --git a/src/cli.ts b/src/cli.ts
index b0990f9d..bfb50199 100644
--- a/src/cli.ts
+++ b/src/cli.ts
@@ -48,7 +48,8 @@ import { planRetract, runRetract, changed, type Retraction } from "./hosts/retra
import { compactWrites, formatNonInteractiveHelp, formatPlan, runPicker } from "./cli-picker.js";
import { buildGraphWithProgress } from "./claude/build-progress.js";
import { homedir } from "node:os";
-import { formatUpgradeReport, formatVersionReport, getNpmViewVersion, readCurrentVersion, runUpgrade } from "./cli-meta.js";
+import { formatUpgradeReport, formatVersionReport, getNpmViewVersion, readCurrentVersion, readPackageName, runUpgrade } from "./cli-meta.js";
+import { nativeFallbackNotice } from "./graph/native.js";
import { patchBuildConfig, type BuildConfig } from "./util/state.js";
import { normalizePathPrefix } from "./util/paths.js";
import { latestSession, formatSessionStats, sessionInputRate } from "./claude/session-metrics.js";
@@ -595,6 +596,7 @@ grouped(program.command("build"), GROUP.setup)
includeDirs: opts.includeDir,
followSubmodules: followSubmodulesWasExplicit ? opts.followSubmodules : undefined,
followNestedRepos: followNestedReposWasExplicit ? opts.followNestedRepos : undefined,
+ packageName: readPackageName(import.meta.url),
});
return;
}
@@ -663,6 +665,10 @@ grouped(program.command("build"), GROUP.setup)
{ repo: buildRoot },
);
for (const e of g.errors) console.error(`✗ ${e}`);
+ // A language whose native grammar didn't load was parsed by its WASM stand-in:
+ // say which, and the install that fixes it. Nothing when everything loaded.
+ const fallbackNotice = nativeFallbackNotice(g.nativeFallback, readPackageName(import.meta.url));
+ if (fallbackNotice) console.error(fallbackNotice);
const rel = relative(process.cwd(), g.contextDir) || "graft";
if (process.env.GRAFT_NO_GITIGNORE) {
diff --git a/src/graph/build.ts b/src/graph/build.ts
index b3094051..178f7f58 100644
--- a/src/graph/build.ts
+++ b/src/graph/build.ts
@@ -17,9 +17,10 @@ import { readFileSync } from "node:fs";
import { basename, dirname, resolve } from "node:path";
import { walkDir } from "../ingest/fs.js";
import { contextDirFor, ensureGitignored, ensureSearchable } from "../context/node-file.js";
-import { extractFile, languageLabelOf, languageOf, type RawEdge } from "./extract.js";
-import { extractGeneric, genericLangOf, warmGenericGrammars } from "./generic.js";
-import { containerLangOf, extractContainer, warmContainerGrammars } from "./container.js";
+import { extractFile, languageLabelOf, languageOf, type Language, type RawEdge } from "./extract.js";
+import { extractGeneric, fallbackLangOf, genericLangOf, warmGenericGrammars, type GenericLang } from "./generic.js";
+import { containerLangOf, extractContainer, warmContainerGrammars, type ContainerLang } from "./container.js";
+import { missingNativePackages, nativeReady, type NativeFallback } from "./native.js";
import { contentHash } from "../util/id.js";
import { relPosix } from "../util/paths.js";
import { readSourceFile } from "../util/source.js";
@@ -44,6 +45,55 @@ import type { CruxSummarizer } from "../ai/crux.js";
export { listSourceFiles } from "./source-files.js";
+/**
+ * Which tier extracts a file. {@link buildGraph} and `checkGraph` both dispatch on
+ * this, so a tier the build extracts can never be one the check can't see.
+ *
+ * Depth tier (hand-written, native grammar) when a language claims the file and its
+ * native grammar loads. When it doesn't load (npm 12 skips the install script that
+ * builds `tree-sitter-kotlin`), the file goes to that language's WASM stand-in in
+ * the breadth tier and `fellBack` names the language. A container (.vue) is checked
+ * before the breadth tier so a future grammar claiming its extension can't shadow
+ * it; anything else goes to the breadth tier (generic tags.scm over a WASM grammar).
+ */
+export interface FileTier {
+ lang: Language | null;
+ container: ContainerLang | null;
+ generic: GenericLang | null;
+ fellBack: Language | null;
+}
+
+export function tierOf(path: string): FileTier {
+ const depth = languageOf(path);
+ if (depth) {
+ return nativeReady(depth)
+ ? { lang: depth, container: null, generic: null, fellBack: null }
+ : { lang: null, container: null, generic: fallbackLangOf(depth), fellBack: depth };
+ }
+ const container = containerLangOf(path);
+ if (container) return { lang: null, container, generic: null, fellBack: null };
+ return { lang: null, container: null, generic: genericLangOf(path), fellBack: null };
+}
+
+/** Load every WASM grammar these files need, before a synchronous parse loop can
+ * call `extractGeneric`/`extractContainer`: the breadth tier's, the containers',
+ * and the stand-ins for depth languages (a container's embedded one included)
+ * whose native grammar didn't load. */
+export async function warmGrammarsFor(paths: Iterable<string>): Promise<void> {
+ const generic = new Set<string>();
+ const containers = new Set<string>();
+ for (const path of paths) {
+ const t = tierOf(path);
+ if (t.generic) generic.add(t.generic.name);
+ if (t.container) {
+ containers.add(t.container.name);
+ if (!nativeReady(t.container.inner)) generic.add(fallbackLangOf(t.container.inner).name);
+ }
+ }
+ await warmGenericGrammars(generic);
+ await warmContainerGrammars(containers);
+}
+
/** Minimum non-file node count for a discovered sub-scope to stand on its own
* (over-split guard 3). A scope with fewer nodes than this is folded into the
* root scope — node counts aren't known until after the graph is assembled,
@@ -119,6 +169,9 @@ export interface GraphBuildResult {
byKind: Record<Kind, number>;
byRelation: Record<Relation, number>;
languages: string[];
+ /** Languages parsed by a WASM stand-in because their native grammar didn't load,
+ * and the packages to allow at install to fix it. Absent when everything loaded. */
+ nativeFallback?: NativeFallback;
meaning: EnrichStats;
errors: string[];
}
@@ -190,30 +243,36 @@ export async function buildGraph(
let parsed = 0;
let reused = 0;
- // Breadth tier: WASM grammars load asynchronously, so warm the ones this repo
- // needs ONCE here (buildGraph is async) before the synchronous parse loop below
- // can call extractGeneric. Depth-tier (native) grammars need no warmup.
- await warmGenericGrammars(
- new Set(files.map((f) => genericLangOf(f.abs)?.name).filter((n): n is string => !!n)),
- );
- // Container tier (.vue and friends) loads its wrapper grammars the same way,
- // for the same reason: extractContainer runs inside the sync loop below.
- await warmContainerGrammars(
- new Set(files.map((f) => containerLangOf(f.abs)?.name).filter((n): n is string => !!n)),
- );
+ // Breadth and container tiers: WASM grammars load asynchronously, so warm the
+ // ones this repo needs ONCE here (buildGraph is async) before the synchronous
+ // parse loop below can call extractGeneric/extractContainer. This is also where
+ // the depth tier's native grammars load, each on first ask, and where one that
+ // can't load gets its WASM stand-in warmed instead.
+ await warmGrammarsFor(files.map((f) => f.abs));
+ /** Languages whose files went to a stand-in: grammars for the fix, labels for people. */
+ const fellBack = new Set<Language>();
+ const fellBackLabels = new Set<string>();
files.forEach((f, i) => {
const rel = f.rel;
opts.onProgress?.({ phase: "parse", index: i, total: files.length, file: rel });
- // Depth tier (hand-written, native grammar) if a language claims the file;
- // otherwise the breadth tier (generic tags.scm over a WASM grammar).
- const lang = languageOf(f.abs);
- // A container is neither tier: its wrapper grammar only locates the embedded
- // block, which then goes to the depth-tier extractor. Checked before the
- // breadth tier so a future grammar claiming .vue can't shadow it.
- const container = lang ? null : containerLangOf(f.abs);
- const generic = lang || container ? null : genericLangOf(f.abs);
+ const { lang, container, generic, fellBack: standIn } = tierOf(f.abs);
const label = languageLabelOf(f.abs) ?? container?.name ?? generic?.name ?? "unknown";
+ // A .vue file's embedded script falls back the same way its language does.
+ const inner = container && !nativeReady(container.inner) ? container.inner : null;
+ if (standIn) {
+ fellBack.add(standIn);
+ fellBackLabels.add(label);
+ }
+ if (inner) {
+ fellBack.add(inner);
+ fellBackLabels.add(inner);
+ }
+ // Which stand-in parsed this file, if one did. A cached parse is replayed only
+ // when the same parser would produce it now: otherwise a file parsed by the
+ // stand-in would keep its basic parse after a reinstall restored the native
+ // grammar, for as long as its bytes didn't change.
+ const via = standIn ? generic!.name : inner ? fallbackLangOf(inner).name : undefined;
const cached = priorExtract.files[rel];
// Every file is read and hashed, every build — only the *parse* is memoized.
@@ -244,7 +303,7 @@ export async function buildGraph(
}
const hash = contentHash(source);
- if (cached && hash === cached.hash) {
+ if (cached && hash === cached.hash && cached.via === via) {
entries[rel] = { ...cached, size: f.size, mtimeMs: f.mtimeMs };
sources.set(rel, source);
reused++;
@@ -269,11 +328,11 @@ export async function buildGraph(
rawEdges.push(...fileEdges);
sources.set(rel, source);
langs.add(label);
- entries[rel] = { size: f.size, mtimeMs: f.mtimeMs, hash, nodes: fileNodes, rawEdges: fileEdges };
+ entries[rel] = { size: f.size, mtimeMs: f.mtimeMs, hash, nodes: fileNodes, rawEdges: fileEdges, ...(via ? { via } : {}) };
} catch (err) {
const message = `${rel}: parse failed — ${err instanceof Error ? err.message : String(err)}`;
errors.push(message);
- entries[rel] = { size: f.size, mtimeMs: f.mtimeMs, hash, nodes: [], rawEdges: [], error: message };
+ entries[rel] = { size: f.size, mtimeMs: f.mtimeMs, hash, nodes: [], rawEdges: [], error: message, ...(via ? { via } : {}) };
}
});
@@ -404,6 +463,9 @@ export async function buildGraph(
byKind,
byRelation,
languages: [...langs].sort(),
+ ...(fellBack.size > 0
+ ? { nativeFallback: { languages: [...fellBackLabels].sort(), packages: missingNativePackages(fellBack) } }
+ : {}),
meaning,
errors,
};
diff --git a/src/graph/check.ts b/src/graph/check.ts
index 77acdafe..c503f00b 100644
--- a/src/graph/check.ts
+++ b/src/graph/check.ts
@@ -20,10 +20,10 @@
import { resolve } from "node:path";
import { relPosix } from "../util/paths.js";
import { contextDirFor } from "../context/node-file.js";
-import { extractFile, languageOf } from "./extract.js";
-import { extractGeneric, genericLangOf, warmGenericGrammars } from "./generic.js";
-import { containerLangOf, extractContainer, warmContainerGrammars } from "./container.js";
-import { listSourceFiles } from "./build.js";
+import { extractFile } from "./extract.js";
+import { extractGeneric } from "./generic.js";
+import { extractContainer } from "./container.js";
+import { listSourceFiles, tierOf, warmGrammarsFor } from "./build.js";
import { readGraph, wiringPath } from "./write.js";
import { readFingerprint } from "./fingerprint.js";
import { readSourceFile } from "../util/source.js";
@@ -88,25 +88,19 @@ export async function checkGraph(
const fpOnlyDirs = readFingerprint(outDir)?.onlyDirs;
const onlyDirs = fpOnlyDirs && fpOnlyDirs.length > 0 ? new Set(fpOnlyDirs) : undefined;
const sourceFiles = listSourceFiles(root, outDir, undefined, onlyDirs);
- await warmGenericGrammars(
- new Set(sourceFiles.map((f) => genericLangOf(f)?.name).filter((n): n is string => !!n)),
- );
// Container-tier grammars need the same warmup as the generic ones, for the same
- // reason: extraction below is synchronous. Missing this is what made `graft
+ // reason: extraction below is synchronous. Missing that is what made `graft
// check` report every `.vue` node as `removed` right after a clean build (#236)
// — the tier extracted fine, and then the check had no branch that could see it.
- await warmContainerGrammars(
- new Set(sourceFiles.map((f) => containerLangOf(f)?.name).filter((n): n is string => !!n)),
- );
+ // The same call warms the WASM stand-in for a depth language whose native
+ // grammar didn't load, exactly as the build does.
+ await warmGrammarsFor(sourceFiles);
const current = new Map<string, string>(); // id → body_hash
for (const file of sourceFiles) {
- // The same three-way branch `buildGraph` uses, in the same order. The two must
- // stay in step: a tier the build extracts and the check cannot see reports as
- // `removed` forever, and the `graft build` the check tells you to run can never
- // repair it.
- const lang = languageOf(file);
- const container = lang ? null : containerLangOf(file);
- const generic = lang || container ? null : genericLangOf(file);
+ // The same dispatch `buildGraph` uses (`tierOf`). The two must stay in step: a
+ // tier the build extracts and the check cannot see reports as `removed`
+ // forever, and the `graft build` the check tells you to run can never repair it.
+ const { lang, container, generic } = tierOf(file);
let source: string | null;
try {
source = readSourceFile(file);
diff --git a/src/graph/container.ts b/src/graph/container.ts
index 3015069e..f9bd72ff 100644
--- a/src/graph/container.ts
+++ b/src/graph/container.ts
@@ -22,7 +22,8 @@
* fixtures whose true line numbers are known.
*/
import { extractFile, mintId, type ExtractResult, type Language, type RawEdge } from "./extract.js";
-import { loadWasmLanguage, parseWasm, type TsNode } from "./generic.js";
+import { extractGeneric, fallbackLangOf, loadWasmLanguage, parseWasm, type TsNode } from "./generic.js";
+import { nativeReady } from "./native.js";
import { contentHash } from "../util/id.js";
import type { NodeV1 } from "./types.js";
@@ -168,7 +169,11 @@ export function extractContainer(rel: string, source: string, lang: ContainerLan
let inner: ExtractResult;
try {
- inner = extractFile(rel, script, lang.inner);
+ // No native grammar for the embedded language (see native.ts): its WASM
+ // stand-in gives the block's symbols, warmed by the build like any other.
+ inner = nativeReady(lang.inner)
+ ? extractFile(rel, script, lang.inner)
+ : extractGeneric(rel, script, fallbackLangOf(lang.inner).name);
} catch {
continue; // one bad block, not a bad build
}
diff --git a/src/graph/extract-cache.ts b/src/graph/extract-cache.ts
index 2acf8b25..db3ba22c 100644
--- a/src/graph/extract-cache.ts
+++ b/src/graph/extract-cache.ts
@@ -48,6 +48,10 @@ export interface ExtractEntry {
* re-reports the same error and contributes no nodes, exactly as a cold build
* would. */
error?: string;
+ /** The breadth-tier grammar that parsed this file standing in for its depth-tier
+ * one, whose native grammar didn't load. Absent for every other file. The entry is
+ * replayed only while the same stand-in would parse it again. */
+ via?: string;
}
export interface ExtractCache {
diff --git a/src/graph/extract.ts b/src/graph/extract.ts
index d0f9dff1..a8215fa4 100644
--- a/src/graph/extract.ts
+++ b/src/graph/extract.ts
@@ -6,18 +6,13 @@
* arrow-function consts) plus unresolved edge intents. Edge *targets* are
* resolved against the whole-repo node index later, in build.ts.
*/
-import Parser from "tree-sitter";
-import TypeScript from "tree-sitter-typescript";
-import Python from "tree-sitter-python";
-import Go from "tree-sitter-go";
-import R from "tree-sitter-r";
-import Java from "tree-sitter-java";
-import Kotlin from "tree-sitter-kotlin";
-import Swift from "tree-sitter-swift";
-import PHP from "tree-sitter-php";
+// Types only: the native grammars themselves load on first use, in native.ts, so
+// a missing binary costs its own language instead of every command.
+import type Parser from "tree-sitter";
import { basename } from "node:path";
import { contentHash } from "../util/id.js";
import { collectBindings, goReceiverVarOf, resolveRecvType, type FileBindings } from "./bindings.js";
+import { nativeParser } from "./native.js";
import type { Kind, NodeV1, Relation } from "./types.js";
export type Language = "typescript" | "tsx" | "python" | "go" | "java" | "kotlin" | "swift" | "php" | "r";
@@ -310,19 +305,6 @@ const FUNCTION_VALUE_TYPES = new Set([
const EMPTY_SET: ReadonlySet<string> = new Set();
-const parser = new Parser();
-const GRAMMARS: Record<Language, unknown> = {
- typescript: TypeScript.typescript,
- tsx: TypeScript.tsx,
- python: Python,
- go: Go,
- r: R,
- java: Java,
- kotlin: Kotlin,
- swift: Swift,
- php: PHP.php,
-};
-
export interface WalkCtx {
rel: string;
source: string;
@@ -380,13 +362,14 @@ interface DefDescriptor {
* no such limit as long as each returned chunk is under 32 KB, so we always feed
* the source in <32 KB slices. Code-unit indexing matches `String.slice`. */
const PARSE_CHUNK = 16384;
-function parseSource(source: string): Parser.SyntaxNode {
+function parseSource(parser: Parser, source: string): Parser.SyntaxNode {
return parser.parse((index: number) => source.slice(index, index + PARSE_CHUNK)).rootNode;
}
+/** Throws when `lang`'s native grammar is unavailable: callers check `nativeReady`
+ * (native.ts) first and hand such a file to its breadth-tier stand-in instead. */
export function extractFile(rel: string, source: string, lang: Language): ExtractResult {
- parser.setLanguage(GRAMMARS[lang] as never);
- const root = parseSource(source);
+ const root = parseSource(nativeParser(lang), source);
const bindings = collectBindings(root, lang);
const importedSymbols = collectImportedSymbols(root, lang);
const rGenerics = lang === "r" ? collectRGenerics(root) : EMPTY_SET;
diff --git a/src/graph/generic.ts b/src/graph/generic.ts
index b599deb7..def35f6e 100644
--- a/src/graph/generic.ts
+++ b/src/graph/generic.ts
@@ -23,7 +23,7 @@ import { dirname, join } from "node:path";
import { createRequire } from "node:module";
import { contentHash } from "../util/id.js";
import type { Kind, NodeV1 } from "./types.js";
-import type { ExtractResult, RawEdge } from "./extract.js";
+import type { ExtractResult, Language, RawEdge } from "./extract.js";
const HERE = dirname(fileURLToPath(import.meta.url));
const require = createRequire(import.meta.url);
@@ -36,6 +36,9 @@ export interface GenericLang {
name: string;
exts: string[];
wasm: string;
+ /** Bundle dirs whose `tags.scm` to use, concatenated, when there is no vendored
+ * queries/<name>.scm. Only the depth-tier stand-ins below set it. */
+ bundledTags?: string[];
}
/** The breadth registry. Add a row + a queries/<name>.scm to support a language.
@@ -60,6 +63,36 @@ export const GENERIC_LANGS: readonly GenericLang[] = [
{ name: "lua", exts: [".lua"], wasm: "lua" },
];
+/**
+ * WASM stand-ins for the depth tier, used only for a language whose native grammar
+ * did not load (native.ts): npm 12 skips the install script that builds
+ * `tree-sitter-kotlin`, for one. They claim no extension, so `genericLangOf` never
+ * returns them; the build picks one by asking {@link fallbackLangOf}. Output is the
+ * breadth tier's: symbols and bare-name calls, no receiver types or bindings.
+ * Kotlin, Swift and PHP use their vendored queries/<name>.scm; the rest use the
+ * bundle's own tags.scm (typescript's builds on javascript's, as upstream does).
+ */
+const FALLBACK_LANGS: Record<Exclude<Language, "java">, GenericLang> = {
+ typescript: { name: "typescript", exts: [], wasm: "typescript", bundledTags: ["javascript", "typescript"] },
+ tsx: { name: "tsx", exts: [], wasm: "tsx", bundledTags: ["javascript", "tsx"] },
+ python: { name: "python", exts: [], wasm: "python", bundledTags: ["python"] },
+ go: { name: "go", exts: [], wasm: "go", bundledTags: ["go"] },
+ r: { name: "r", exts: [], wasm: "r", bundledTags: ["r"] },
+ kotlin: { name: "kotlin", exts: [], wasm: "kotlin" },
+ swift: { name: "swift", exts: [], wasm: "swift" },
+ php: { name: "php", exts: [], wasm: "php" },
+};
+
+/** The breadth-tier grammar that stands in for a depth-tier language whose native
+ * grammar can't load. Java already has a breadth row, so it is its own stand-in. */
+export function fallbackLangOf(lang: Language): GenericLang {
+ return lang === "java" ? GENERIC_LANGS.find((l) => l.name === "java")! : FALLBACK_LANGS[lang];
+}
+
+function rowNamed(name: string): GenericLang | undefined {
+ return GENERIC_LANGS.find((l) => l.name === name) ?? Object.values(FALLBACK_LANGS).find((l) => l.name === name);
+}
+
const byExt = new Map<string, GenericLang>();
for (const l of GENERIC_LANGS) for (const e of l.exts) byExt.set(e, l);
@@ -101,20 +134,29 @@ function requireWasm(wasm: string): Buffer | null {
}
}
-function loadQuery(name: string): string | null {
+// Sanitize editor-specific query predicates the tree-sitter Query compiler rejects.
+function sanitizeQuery(raw: string): string {
+ return raw.replace(/\(#(?:strip!|set!|set-adjacent!|select-adjacent!|make-range!|offset!|gsub!)[^()]*\)/g, "");
+}
+
+function loadQuery(row: GenericLang): string | null {
for (const dir of QUERY_DIRS) {
try {
- const raw = readFileSync(join(dir, `${name}.scm`), "utf8");
- // Sanitize editor-specific query predicates the tree-sitter Query compiler rejects.
- return raw.replace(
- /\(#(?:strip!|set!|set-adjacent!|select-adjacent!|make-range!|offset!|gsub!)[^()]*\)/g,
- "",
- );
+ return sanitizeQuery(readFileSync(join(dir, `${row.name}.scm`), "utf8"));
} catch {
/* try next dir */
}
}
- return null;
+ if (!row.bundledTags) return null;
+ const parts: string[] = [];
+ for (const dir of row.bundledTags) {
+ try {
+ parts.push(readFileSync(require.resolve(`tree-sitter-wasm/${dir}/tags.scm`), "utf8"));
+ } catch {
+ /* no tags.scm for this one */
+ }
+ }
+ return parts.length > 0 ? sanitizeQuery(parts.join("\n")) : null;
}
/** Warm the WASM grammars for the given graft lang names. Idempotent; must be
@@ -122,7 +164,7 @@ function loadQuery(name: string): string | null {
* grammars are silently skipped (their files then extract as file-only). */
export async function warmGenericGrammars(langNames: Iterable<string>): Promise<void> {
const want = new Set(langNames);
- const need = [...want].filter((n) => !loaded.has(n) && GENERIC_LANGS.some((l) => l.name === n));
+ const need = [...want].filter((n) => !loaded.has(n) && rowNamed(n) !== undefined);
if (need.length === 0) return;
if (!tsMod) {
tsMod = await import("web-tree-sitter");
@@ -131,12 +173,12 @@ export async function warmGenericGrammars(langNames: Iterable<string>): Promise<
await initPromise;
const { Language, Query } = tsMod;
for (const name of need) {
- const row = GENERIC_LANGS.find((l) => l.name === name)!;
+ const row = rowNamed(name)!;
const bytes = requireWasm(row.wasm);
if (!bytes) continue;
try {
const language = await Language.load(bytes);
- const scm = loadQuery(name);
+ const scm = loadQuery(row);
let query: unknown | null = null;
if (scm) {
try { query = new Query(language, scm); } catch { query = null; }
diff --git a/src/graph/native.ts b/src/graph/native.ts
new file mode 100644
index 00000000..32ac41a1
--- /dev/null
+++ b/src/graph/native.ts
@@ -0,0 +1,187 @@
+/**
+ * The depth tier's native grammars, loaded on first use and never fatally.
+ *
+ * Every depth-tier grammar is a native addon, and so is `tree-sitter` itself. Most
+ * ship prebuilt binaries that load with no install step, but not all of them:
+ * `tree-sitter-kotlin` ships none at all, and `tree-sitter` and
+ * `tree-sitter-python` have no linux-arm64 build. Those work only if npm ran their
+ * install script, which compiles one. npm 12 stopped running dependency install
+ * scripts by default, so after a plain `npm install -g` the binary is not there.
+ *
+ * These used to be static imports at the top of extract.ts. One missing binary
+ * then threw while the module graph loaded and took every command down with it,
+ * the MCP server and the hooks included, even in a repo with no Kotlin in it.
+ * Now each is required the first time a file of its language is parsed, and a
+ * failure is remembered: that language's files go to its WASM stand-in in the
+ * breadth tier instead (`fallbackLangOf` in generic.ts), and the build says which
+ * languages and how to get the native parser back ({@link nativeFallbackNotice}).
+ */
+import { createRequire } from "node:module";
+import type Parser from "tree-sitter";
+import type { Language } from "./extract.js";
+
+const nodeRequire = createRequire(import.meta.url);
+
+const CORE = "tree-sitter";
+
+/** The module each depth-tier grammar comes from, and the export that is the
+ * grammar when the module ships more than one (typescript+tsx, php+php_only). */
+const GRAMMAR_MODULES: Record<Language, { module: string; pick?: string }> = {
+ typescript: { module: "tree-sitter-typescript", pick: "typescript" },
+ tsx: { module: "tree-sitter-typescript", pick: "tsx" },
+ python: { module: "tree-sitter-python" },
+ go: { module: "tree-sitter-go" },
+ r: { module: "tree-sitter-r" },
+ java: { module: "tree-sitter-java" },
+ kotlin: { module: "tree-sitter-kotlin" },
+ swift: { module: "tree-sitter-swift" },
+ php: { module: "tree-sitter-php", pick: "php" },
+};
+
+/** The name `npm --allow-scripts` needs, where it differs from the module name.
+ * `tree-sitter-r` is an npm alias, and npm matches an allow entry against the real
+ * package only: an alias as the key matches nothing. */
+const NPM_NAME: Record<string, string> = {
+ "tree-sitter-r": "@davisvaughan/tree-sitter-r",
+};
+
+type Loaded = { ok: true; value: unknown } | { ok: false; error: string };
+
+let requireModule: (id: string) => unknown = (id) => nodeRequire(id);
+const modules = new Map<string, Loaded>();
+const ready = new Map<Language, boolean>();
+/** undefined: not tried yet. null: tried, and the core binding is unusable. */
+let parser: Parser | null | undefined;
+
+function message(err: unknown): string {
+ return err instanceof Error ? err.message : String(err);
+}
+
+function load(id: string): Loaded {
+ let hit = modules.get(id);
+ if (!hit) {
+ try {
+ hit = { ok: true, value: requireModule(id) };
+ } catch (err) {
+ hit = { ok: false, error: message(err) };
+ }
+ modules.set(id, hit);
+ }
+ return hit;
+}
+
+function coreParser(): Parser | null {
+ if (parser === undefined) {
+ const core = load(CORE);
+ parser = null;
+ if (core.ok) {
+ try {
+ parser = new (core.value as typeof Parser)();
+ } catch (err) {
+ modules.set(CORE, { ok: false, error: message(err) });
+ }
+ }
+ }
+ return parser;
+}
+
+function grammarOf(lang: Language): unknown {
+ const { module, pick } = GRAMMAR_MODULES[lang];
+ const m = load(module);
+ if (!m.ok) return null;
+ return pick ? ((m.value as Record<string, unknown>)[pick] ?? null) : m.value;
+}
+
+/**
+ * True when `lang`'s native grammar loads and the parser accepts it. The first call
+ * per language does the loading; the answer is cached for the life of the process.
+ * Never throws.
+ */
+export function nativeReady(lang: Language): boolean {
+ let ok = ready.get(lang);
+ if (ok === undefined) {
+ ok = false;
+ // Load the grammar even when the core binding failed, so the notice can name
+ // every package that is missing its binary, not only the first.
+ const grammar = grammarOf(lang);
+ const p = coreParser();
+ if (p && grammar) {
+ try {
+ p.setLanguage(grammar as never);
+ ok = true;
+ } catch {
+ /* an addon built for another tree-sitter ABI: unusable, though it loaded */
+ }
+ }
+ ready.set(lang, ok);
+ }
+ return ok;
+}
+
+/** The shared native parser, set to `lang`. Throws when that grammar is not
+ * available; callers ask {@link nativeReady} first and send the file elsewhere. */
+export function nativeParser(lang: Language): Parser {
+ if (!nativeReady(lang)) {
+ const { module } = GRAMMAR_MODULES[lang];
+ const why = [CORE, module].map((id) => modules.get(id)).find((m) => m && !m.ok);
+ throw new Error(`the native ${lang} parser is not available${why && !why.ok ? `: ${why.error}` : ""}`);
+ }
+ const p = parser as Parser;
+ p.setLanguage(grammarOf(lang) as never);
+ return p;
+}
+
+/**
+ * The npm packages whose missing binary keeps these languages off the native
+ * parser: what `npm install --allow-scripts=…` has to name. Empty when every one
+ * loaded, or when the ones that failed did load and were rejected by the parser
+ * (an ABI mismatch, which no install flag fixes).
+ */
+export function missingNativePackages(langs: Iterable<Language>): string[] {
+ const out = new Set<string>();
+ for (const lang of langs) {
+ if (nativeReady(lang)) continue;
+ for (const id of [CORE, GRAMMAR_MODULES[lang].module]) {
+ const m = modules.get(id);
+ if (m && !m.ok) out.add(NPM_NAME[id] ?? id);
+ }
+ }
+ return [...out].sort();
+}
+
+/** The languages a build parsed with a WASM stand-in instead of their native
+ * grammar, and the packages to allow so the next install builds them. */
+export interface NativeFallback {
+ /** Display labels (`javascript`, not the `typescript` grammar that parses it). */
+ languages: string[];
+ packages: string[];
+}
+
+/**
+ * The one line a build prints when some languages fell back, or null when none did.
+ * One line on purpose: `graft init` passes on only the `⚠`/`✗` lines of the build
+ * it runs, so a second line holding the fix would be dropped there.
+ */
+export function nativeFallbackNotice(fallback: NativeFallback | undefined, packageName: string): string | null {
+ if (!fallback || fallback.languages.length === 0) return null;
+ const langs = fallback.languages.join(", ");
+ const what = fallback.languages.length === 1 ? "the native parser is" : "the native parsers are";
+ const head = `⚠ ${langs}: basic parsing only, ${what} missing.`;
+ if (fallback.packages.length === 0) return head;
+ return `${head} To fix: npm install -g --allow-scripts=${fallback.packages.join(",")} ${packageName}`;
+}
+
+/** Test seam: replace how native modules are required (null restores the real
+ * `require`), and forget everything loaded so far. A missing binary is simulated by
+ * a function that throws for that module id, the way node-gyp-build does. */
+export function setNativeRequireForTest(fn: ((id: string) => unknown) | null): void {
+ requireModule = fn ?? ((id) => nodeRequire(id));
+ modules.clear();
+ ready.clear();
+ parser = undefined;
+}
+
+/** The real `require` the seam above wraps, for a test's pass-through. */
+export function realNativeRequire(id: string): unknown {
+ return nodeRequire(id);
+}
diff --git a/src/graph/workspace-cli.ts b/src/graph/workspace-cli.ts
index f6dc57e8..7fb1296b 100644
--- a/src/graph/workspace-cli.ts
+++ b/src/graph/workspace-cli.ts
@@ -8,6 +8,8 @@
import { Graft } from "../engine.js";
import { contextDirFor, ensureGitignored } from "../context/node-file.js";
import { patchBuildConfig, type BuildConfig } from "../util/state.js";
+import { brand, PACKAGE_FOR } from "../brand.js";
+import { nativeFallbackNotice } from "./native.js";
import type { EngineConfig } from "../ai/providers.js";
import { formatAsk } from "../ask/ask.js";
import type { Direction } from "./traverse.js";
@@ -40,12 +42,17 @@ export interface WorkspaceBuildOptions {
followSubmodules?: boolean;
/** An explicit CLI nested-clone choice to persist into every child repo. */
followNestedRepos?: boolean;
+ /** The npm package this runs from, for the fix in the native-parser notice. */
+ packageName?: string;
}
/** Build every git child into its own committable `graft/`, then replace the
* parent's `graft/` with `workspace.json`. Prints the one-time split warning
* first when migrating away from a mega-graph. */
export async function runWorkspaceBuild(root: string, opts: WorkspaceBuildOptions): Promise<void> {
+ // Native grammars load once per process, so the children share one answer:
+ // collected here and printed once at the end, not once per repo.
+ const fellBack = { languages: new Set<string>(), packages: new Set<string>() };
const buildChild = async (childDir: string, childName: string): Promise<void> => {
// Persisted BEFORE the child build itself runs, same as the single-repo
// path in cli.ts, so this build and every later no-flag child build agree.
@@ -67,6 +74,8 @@ export async function runWorkspaceBuild(root: string, opts: WorkspaceBuildOption
const g = await engine.graph(childDir, { llm: opts.deep, concurrency: opts.concurrency });
console.log(`✓ ${childName}/: ${g.nodes} nodes, ${g.edges} edges, ${g.cards} cards [${g.languages.join(", ")}]`);
for (const e of g.errors) console.error(`✗ ${childName}/: ${e}`);
+ for (const l of g.nativeFallback?.languages ?? []) fellBack.languages.add(l);
+ for (const p of g.nativeFallback?.packages ?? []) fellBack.packages.add(p);
};
const { children } = await splitWorkspace(
@@ -83,6 +92,11 @@ export async function runWorkspaceBuild(root: string, opts: WorkspaceBuildOption
ensureGitignored(root, contextDirFor(root, opts.override));
console.log(`✓ workspace: ${children.length} repos federated → graft/workspace.json`);
console.log(` graft/ is git-ignored — each teammate runs \`graft build\` to regenerate it locally.`);
+ const notice = nativeFallbackNotice(
+ { languages: [...fellBack.languages].sort(), packages: [...fellBack.packages].sort() },
+ opts.packageName ?? PACKAGE_FOR[brand()],
+ );
+ if (notice) console.error(notice);
}
export function runWorkspaceAsk(
diff --git a/test/graph-native-fallback.test.ts b/test/graph-native-fallback.test.ts
new file mode 100644
index 00000000..b5b540dd
--- /dev/null
+++ b/test/graph-native-fallback.test.ts
@@ -0,0 +1,185 @@
+/**
+ * A native grammar that won't load costs its own language, never the command.
+ *
+ * The bug: npm 12 stops running dependency install scripts by default, and
+ * `tree-sitter-kotlin` ships no prebuilt binary, so after `npm install -g` its addon
+ * is simply absent. extract.ts imported every grammar statically, so the require
+ * threw while the module graph loaded and `graft build`, the MCP server and the
+ * hooks all crashed, in a repo with no Kotlin in it at all.
+ *
+ * Simulated here through native.ts's require seam, the way node-gyp-build fails:
+ * the require throws "No native build was found".
+ */
+import { afterEach, test } from "node:test";
+import assert from "node:assert/strict";
+import { mkdirSync, writeFileSync } from "node:fs";
+import { join } from "node:path";
+import { buildGraph } from "../src/graph/build.js";
+import { checkGraph } from "../src/graph/check.js";
+import {
+ missingNativePackages,
+ nativeFallbackNotice,
+ nativeReady,
+ realNativeRequire,
+ setNativeRequireForTest,
+} from "../src/graph/native.js";
+import { tmpRepo } from "./helpers.js";
+
+const KOTLIN = [
+ "package demo",
+ "",
+ "class Greeter(val name: String) {",
+ ' fun greet(): String = "hi " + name',
+ "}",
+ "",
+ "fun helper(x: Int): Int = x + 1",
+ "",
+ "fun main() {",
+ ' val g = Greeter("a")',
+ " println(g.greet())",
+ " helper(2)",
+ "}",
+ "",
+].join("\n");
+
+const PY = ["class Store:", " def load(self):", " return fetch()", "", "def fetch():", " return 1", ""].join("\n");
+
+const TS = ["export function add(a: number, b: number): number {", " return a + b;", "}", "export function twice(a: number): number {", " return add(a, a);", "}", ""].join("\n");
+
+function repo(): string {
+ const d = tmpRepo("native-fallback");
+ mkdirSync(join(d, "src"), { recursive: true });
+ writeFileSync(join(d, "src", "math.ts"), TS);
+ writeFileSync(join(d, "src", "store.py"), PY);
+ writeFileSync(join(d, "src", "Main.kt"), KOTLIN);
+ return d;
+}
+
+/** A require that fails for `missing`, exactly as node-gyp-build does, and records
+ * every module asked for. */
+function failing(missing: string[], asked: string[] = []): (id: string) => unknown {
+ return (id) => {
+ asked.push(id);
+ if (missing.includes(id)) {
+ throw new Error(`No native build was found for platform=${process.platform} arch=${process.arch} loaded from node_modules/${id}`);
+ }
+ return realNativeRequire(id);
+ };
+}
+
+afterEach(() => setNativeRequireForTest(null));
+
+test("all grammars load: native parse everywhere, and no notice", async () => {
+ setNativeRequireForTest(null);
+ const r = await buildGraph(repo());
+ assert.deepEqual(r.errors, []);
+ assert.equal(r.nativeFallback, undefined);
+ assert.equal(nativeFallbackNotice(r.nativeFallback, "@trailhq/trail"), null);
+ assert.deepEqual(r.languages, ["kotlin", "python", "typescript"]);
+ assert.ok(nativeReady("kotlin"));
+});
+
+test("a missing kotlin binary: kotlin goes to the WASM stand-in, the build completes, the rest stay native", async () => {
+ setNativeRequireForTest(failing(["tree-sitter-kotlin"]));
+ const dir = repo();
+ const r = await buildGraph(dir);
+ assert.deepEqual(r.errors, []);
+ assert.deepEqual(r.languages, ["kotlin", "python", "typescript"]);
+ assert.deepEqual(r.nativeFallback, { languages: ["kotlin"], packages: ["tree-sitter-kotlin"] });
+
+ const { readGraph, wiringPath } = await import("../src/graph/write.js");
+ const g = readGraph(wiringPath(r.contextDir))!;
+ const kt = g.nodes.filter((n) => n.path === "src/Main.kt" && n.kind !== "file");
+ const names = kt.map((n) => n.name).sort();
+ for (const want of ["Greeter", "greet", "helper", "main"]) assert.ok(names.includes(want), `kotlin ${want} missing: ${names}`);
+ assert.ok(kt.every((n) => n.origin === "generic"), "kotlin symbols come from the stand-in");
+ // The vendored kotlin tags query carries calls, so `graft callers helper` works.
+ assert.ok(
+ g.edges.some((e) => e.relation === "calls" && e.source === "src/Main.kt#main" && e.target === "src/Main.kt#helper"),
+ "main → helper call edge",
+ );
+ // Languages with a binary are untouched.
+ for (const path of ["src/math.ts", "src/store.py"]) {
+ const syms = g.nodes.filter((n) => n.path === path && n.kind !== "file");
+ assert.ok(syms.length > 0 && syms.every((n) => n.origin === "ast"), `${path} stays on the native parser`);
+ }
+
+ // `graft check` dispatches the same way, so the fallback graph reads as in sync.
+ const c = await checkGraph(dir);
+ assert.deepEqual([c.added, c.removed, c.changed], [[], [], []]);
+});
+
+test("the notice names only the grammar that failed, with the install that fixes it", async () => {
+ setNativeRequireForTest(failing(["tree-sitter-kotlin"]));
+ const r = await buildGraph(repo());
+ assert.equal(
+ nativeFallbackNotice(r.nativeFallback, "@trailhq/trail"),
+ "⚠ kotlin: basic parsing only, the native parser is missing. To fix: npm install -g --allow-scripts=tree-sitter-kotlin @trailhq/trail",
+ );
+ assert.match(nativeFallbackNotice(r.nativeFallback, "@nanonets/graft")!, / @nanonets\/graft$/);
+});
+
+test("a repo without kotlin never asks for its grammar", async () => {
+ const asked: string[] = [];
+ setNativeRequireForTest(failing(["tree-sitter-kotlin"], asked));
+ const d = tmpRepo("native-lazy");
+ writeFileSync(join(d, "math.ts"), TS);
+ const r = await buildGraph(d);
+ assert.deepEqual(r.errors, []);
+ assert.equal(r.nativeFallback, undefined);
+ assert.ok(!asked.includes("tree-sitter-kotlin"), `asked for: ${asked.join(", ")}`);
+});
+
+test("no tree-sitter core binding: every depth language falls back, nothing crashes", async () => {
+ setNativeRequireForTest(failing(["tree-sitter"]));
+ const dir = repo();
+ const r = await buildGraph(dir);
+ assert.deepEqual(r.errors, []);
+ assert.deepEqual(r.nativeFallback, { languages: ["kotlin", "python", "typescript"], packages: ["tree-sitter"] });
+ const { readGraph, wiringPath } = await import("../src/graph/write.js");
+ const g = readGraph(wiringPath(r.contextDir))!;
+ const names = new Set(g.nodes.filter((n) => n.kind !== "file").map((n) => n.name));
+ // The bundle's own tags queries stand in for typescript and python.
+ for (const want of ["add", "twice", "Store", "load", "fetch", "helper"]) assert.ok(names.has(want), `${want} missing`);
+ assert.equal(
+ nativeFallbackNotice(r.nativeFallback, "@trailhq/trail"),
+ "⚠ kotlin, python, typescript: basic parsing only, the native parsers are missing. To fix: npm install -g --allow-scripts=tree-sitter @trailhq/trail",
+ );
+});
+
+test("a stand-in parse is not replayed once the native grammar loads again", async () => {
+ const dir = repo();
+ setNativeRequireForTest(failing(["tree-sitter-kotlin"]));
+ await buildGraph(dir);
+ // The reinstall that the notice asks for: same bytes, the grammar now loads.
+ setNativeRequireForTest(null);
+ const r = await buildGraph(dir);
+ assert.equal(r.nativeFallback, undefined);
+ assert.equal(r.parsed, 1, "only the kotlin file is parsed again");
+ const { readGraph, wiringPath } = await import("../src/graph/write.js");
+ const g = readGraph(wiringPath(r.contextDir))!;
+ const kt = g.nodes.filter((n) => n.path === "src/Main.kt" && n.kind !== "file");
+ assert.ok(kt.length > 0 && kt.every((n) => n.origin === "ast"), "kotlin is back on the native parser");
+});
+
+test("the allow-scripts name is the real package, not the npm alias", () => {
+ setNativeRequireForTest(failing(["tree-sitter-r"]));
+ assert.equal(nativeReady("r"), false);
+ assert.deepEqual(missingNativePackages(["r", "python"]), ["@davisvaughan/tree-sitter-r"]);
+});
+
+test("a .vue file's script falls back with its language, spans still on the .vue line", async () => {
+ setNativeRequireForTest(failing(["tree-sitter-typescript"]));
+ const d = tmpRepo("native-vue");
+ writeFileSync(
+ join(d, "Card.vue"),
+ ["<template>", " <p>hi</p>", "</template>", "", '<script lang="ts">', "export function shout(what: string) {", " return what.toUpperCase();", "}", "</script>", ""].join("\n"),
+ );
+ const r = await buildGraph(d);
+ assert.deepEqual(r.errors, []);
+ assert.deepEqual(r.nativeFallback, { languages: ["typescript"], packages: ["tree-sitter-typescript"] });
+ const { readGraph, wiringPath } = await import("../src/graph/write.js");
+ const shout = readGraph(wiringPath(r.contextDir))!.nodes.find((n) => n.name === "shout");
+ assert.ok(shout, "the script's function is still indexed");
+ assert.equal(shout.span, "L6-L8");
+});