Turbocharge Claude Code, Cursor, Codex, Gemini & every coding agent: faster, cheaper, with contextual understanding specific to your codebase.
TypeScript ai-agentsanthropicclaude-codeclicode-graphcodexcontext-engineeringcursordeveloper-toolsgeminiknowledge-graphllmmcpmcp-serveropen-sourceopenaitree-sitter
fix(extract): load native grammars lazily and fall back to WASM when one is missing (#594)
Shrish Dwivedic4411a69d387
diff --git a/src/app/review-process.ts b/src/app/review-process.tsindex 1be03f03..d97dd97d 100644--- a/src/app/review-process.ts+++ b/src/app/review-process.ts@@ -26,11 +26,12 @@** A child process rather than a `worker_threads` worker, deliberately:*- * - **The native grammars keep loading the way they already do.** `graph/extract.ts`- * imports nine tree-sitter addons at module scope and constructs a `Parser` there- * too, and `graph/generic.ts` loads more through `createRequire`. In a worker all- * of that is re-instantiated per thread, which puts the N-API-in-worker behaviour- * of nine third-party native modules between the App and every review. A fork runs+ * - **The native grammars keep loading the way they already do.** `graph/native.ts`+ * requires up to nine tree-sitter addons and constructs a `Parser` the first time+ * a build needs them, and `graph/generic.ts` loads more through `createRequire`.+ * In a worker all of that is re-instantiated per thread, which puts the+ * N-API-in-worker behaviour of nine third-party native modules between the App+ * and every review. A fork runs* byte-identical code to what production runs today.* - **A review can be killed.** A wedged parse or a 10-minute clone is a process* to SIGKILL, not a thread with no safe way to stop it.diff --git a/src/cli.ts b/src/cli.tsindex b0990f9d..bfb50199 100644--- a/src/cli.ts+++ b/src/cli.ts@@ -48,7 +48,8 @@ import { planRetract, runRetract, changed, type Retraction } from "./hosts/retraimport { compactWrites, formatNonInteractiveHelp, formatPlan, runPicker } from "./cli-picker.js";import { buildGraphWithProgress } from "./claude/build-progress.js";import { homedir } from "node:os";-import { formatUpgradeReport, formatVersionReport, getNpmViewVersion, readCurrentVersion, runUpgrade } from "./cli-meta.js";+import { formatUpgradeReport, formatVersionReport, getNpmViewVersion, readCurrentVersion, readPackageName, runUpgrade } from "./cli-meta.js";+import { nativeFallbackNotice } from "./graph/native.js";import { patchBuildConfig, type BuildConfig } from "./util/state.js";import { normalizePathPrefix } from "./util/paths.js";import { latestSession, formatSessionStats, sessionInputRate } from "./claude/session-metrics.js";@@ -595,6 +596,7 @@ grouped(program.command("build"), GROUP.setup)includeDirs: opts.includeDir,followSubmodules: followSubmodulesWasExplicit ? opts.followSubmodules : undefined,followNestedRepos: followNestedReposWasExplicit ? opts.followNestedRepos : undefined,+ packageName: readPackageName(import.meta.url),});return;}@@ -663,6 +665,10 @@ grouped(program.command("build"), GROUP.setup){ repo: buildRoot },);for (const e of g.errors) console.error(`✗ ${e}`);+ // A language whose native grammar didn't load was parsed by its WASM stand-in:+ // say which, and the install that fixes it. Nothing when everything loaded.+ const fallbackNotice = nativeFallbackNotice(g.nativeFallback, readPackageName(import.meta.url));+ if (fallbackNotice) console.error(fallbackNotice);const rel = relative(process.cwd(), g.contextDir) || "graft";if (process.env.GRAFT_NO_GITIGNORE) {diff --git a/src/graph/build.ts b/src/graph/build.tsindex b3094051..178f7f58 100644--- a/src/graph/build.ts+++ b/src/graph/build.ts@@ -17,9 +17,10 @@ import { readFileSync } from "node:fs";import { basename, dirname, resolve } from "node:path";import { walkDir } from "../ingest/fs.js";import { contextDirFor, ensureGitignored, ensureSearchable } from "../context/node-file.js";-import { extractFile, languageLabelOf, languageOf, type RawEdge } from "./extract.js";-import { extractGeneric, genericLangOf, warmGenericGrammars } from "./generic.js";-import { containerLangOf, extractContainer, warmContainerGrammars } from "./container.js";+import { extractFile, languageLabelOf, languageOf, type Language, type RawEdge } from "./extract.js";+import { extractGeneric, fallbackLangOf, genericLangOf, warmGenericGrammars, type GenericLang } from "./generic.js";+import { containerLangOf, extractContainer, warmContainerGrammars, type ContainerLang } from "./container.js";+import { missingNativePackages, nativeReady, type NativeFallback } from "./native.js";import { contentHash } from "../util/id.js";import { relPosix } from "../util/paths.js";import { readSourceFile } from "../util/source.js";@@ -44,6 +45,55 @@ import type { CruxSummarizer } from "../ai/crux.js";export { listSourceFiles } from "./source-files.js";+/**+ * Which tier extracts a file. {@link buildGraph} and `checkGraph` both dispatch on+ * this, so a tier the build extracts can never be one the check can't see.+ *+ * Depth tier (hand-written, native grammar) when a language claims the file and its+ * native grammar loads. When it doesn't load (npm 12 skips the install script that+ * builds `tree-sitter-kotlin`), the file goes to that language's WASM stand-in in+ * the breadth tier and `fellBack` names the language. A container (.vue) is checked+ * before the breadth tier so a future grammar claiming its extension can't shadow+ * it; anything else goes to the breadth tier (generic tags.scm over a WASM grammar).+ */+export interface FileTier {+ lang: Language | null;+ container: ContainerLang | null;+ generic: GenericLang | null;+ fellBack: Language | null;+}++export function tierOf(path: string): FileTier {+ const depth = languageOf(path);+ if (depth) {+ return nativeReady(depth)+ ? { lang: depth, container: null, generic: null, fellBack: null }+ : { lang: null, container: null, generic: fallbackLangOf(depth), fellBack: depth };+ }+ const container = containerLangOf(path);+ if (container) return { lang: null, container, generic: null, fellBack: null };+ return { lang: null, container: null, generic: genericLangOf(path), fellBack: null };+}++/** Load every WASM grammar these files need, before a synchronous parse loop can+ * call `extractGeneric`/`extractContainer`: the breadth tier's, the containers',+ * and the stand-ins for depth languages (a container's embedded one included)+ * whose native grammar didn't load. */+export async function warmGrammarsFor(paths: Iterable<string>): Promise<void> {+ const generic = new Set<string>();+ const containers = new Set<string>();+ for (const path of paths) {+ const t = tierOf(path);+ if (t.generic) generic.add(t.generic.name);+ if (t.container) {+ containers.add(t.container.name);+ if (!nativeReady(t.container.inner)) generic.add(fallbackLangOf(t.container.inner).name);+ }+ }+ await warmGenericGrammars(generic);+ await warmContainerGrammars(containers);+}+/** Minimum non-file node count for a discovered sub-scope to stand on its own* (over-split guard 3). A scope with fewer nodes than this is folded into the* root scope — node counts aren't known until after the graph is assembled,@@ -119,6 +169,9 @@ export interface GraphBuildResult {byKind: Record<Kind, number>;byRelation: Record<Relation, number>;languages: string[];+ /** Languages parsed by a WASM stand-in because their native grammar didn't load,+ * and the packages to allow at install to fix it. Absent when everything loaded. */+ nativeFallback?: NativeFallback;meaning: EnrichStats;errors: string[];}@@ -190,30 +243,36 @@ export async function buildGraph(let parsed = 0;let reused = 0;- // Breadth tier: WASM grammars load asynchronously, so warm the ones this repo- // needs ONCE here (buildGraph is async) before the synchronous parse loop below- // can call extractGeneric. Depth-tier (native) grammars need no warmup.- await warmGenericGrammars(- new Set(files.map((f) => genericLangOf(f.abs)?.name).filter((n): n is string => !!n)),- );- // Container tier (.vue and friends) loads its wrapper grammars the same way,- // for the same reason: extractContainer runs inside the sync loop below.- await warmContainerGrammars(- new Set(files.map((f) => containerLangOf(f.abs)?.name).filter((n): n is string => !!n)),- );+ // Breadth and container tiers: WASM grammars load asynchronously, so warm the+ // ones this repo needs ONCE here (buildGraph is async) before the synchronous+ // parse loop below can call extractGeneric/extractContainer. This is also where+ // the depth tier's native grammars load, each on first ask, and where one that+ // can't load gets its WASM stand-in warmed instead.+ await warmGrammarsFor(files.map((f) => f.abs));+ /** Languages whose files went to a stand-in: grammars for the fix, labels for people. */+ const fellBack = new Set<Language>();+ const fellBackLabels = new Set<string>();files.forEach((f, i) => {const rel = f.rel;opts.onProgress?.({ phase: "parse", index: i, total: files.length, file: rel });- // Depth tier (hand-written, native grammar) if a language claims the file;- // otherwise the breadth tier (generic tags.scm over a WASM grammar).- const lang = languageOf(f.abs);- // A container is neither tier: its wrapper grammar only locates the embedded- // block, which then goes to the depth-tier extractor. Checked before the- // breadth tier so a future grammar claiming .vue can't shadow it.- const container = lang ? null : containerLangOf(f.abs);- const generic = lang || container ? null : genericLangOf(f.abs);+ const { lang, container, generic, fellBack: standIn } = tierOf(f.abs);const label = languageLabelOf(f.abs) ?? container?.name ?? generic?.name ?? "unknown";+ // A .vue file's embedded script falls back the same way its language does.+ const inner = container && !nativeReady(container.inner) ? container.inner : null;+ if (standIn) {+ fellBack.add(standIn);+ fellBackLabels.add(label);+ }+ if (inner) {+ fellBack.add(inner);+ fellBackLabels.add(inner);+ }+ // Which stand-in parsed this file, if one did. A cached parse is replayed only+ // when the same parser would produce it now: otherwise a file parsed by the+ // stand-in would keep its basic parse after a reinstall restored the native+ // grammar, for as long as its bytes didn't change.+ const via = standIn ? generic!.name : inner ? fallbackLangOf(inner).name : undefined;const cached = priorExtract.files[rel];// Every file is read and hashed, every build — only the *parse* is memoized.@@ -244,7 +303,7 @@ export async function buildGraph(}const hash = contentHash(source);- if (cached && hash === cached.hash) {+ if (cached && hash === cached.hash && cached.via === via) {entries[rel] = { ...cached, size: f.size, mtimeMs: f.mtimeMs };sources.set(rel, source);reused++;@@ -269,11 +328,11 @@ export async function buildGraph(rawEdges.push(...fileEdges);sources.set(rel, source);langs.add(label);- entries[rel] = { size: f.size, mtimeMs: f.mtimeMs, hash, nodes: fileNodes, rawEdges: fileEdges };+ entries[rel] = { size: f.size, mtimeMs: f.mtimeMs, hash, nodes: fileNodes, rawEdges: fileEdges, ...(via ? { via } : {}) };} catch (err) {const message = `${rel}: parse failed — ${err instanceof Error ? err.message : String(err)}`;errors.push(message);- entries[rel] = { size: f.size, mtimeMs: f.mtimeMs, hash, nodes: [], rawEdges: [], error: message };+ entries[rel] = { size: f.size, mtimeMs: f.mtimeMs, hash, nodes: [], rawEdges: [], error: message, ...(via ? { via } : {}) };}});@@ -404,6 +463,9 @@ export async function buildGraph(byKind,byRelation,languages: [...langs].sort(),+ ...(fellBack.size > 0+ ? { nativeFallback: { languages: [...fellBackLabels].sort(), packages: missingNativePackages(fellBack) } }+ : {}),meaning,errors,};diff --git a/src/graph/check.ts b/src/graph/check.tsindex 77acdafe..c503f00b 100644--- a/src/graph/check.ts+++ b/src/graph/check.ts@@ -20,10 +20,10 @@import { resolve } from "node:path";import { relPosix } from "../util/paths.js";import { contextDirFor } from "../context/node-file.js";-import { extractFile, languageOf } from "./extract.js";-import { extractGeneric, genericLangOf, warmGenericGrammars } from "./generic.js";-import { containerLangOf, extractContainer, warmContainerGrammars } from "./container.js";-import { listSourceFiles } from "./build.js";+import { extractFile } from "./extract.js";+import { extractGeneric } from "./generic.js";+import { extractContainer } from "./container.js";+import { listSourceFiles, tierOf, warmGrammarsFor } from "./build.js";import { readGraph, wiringPath } from "./write.js";import { readFingerprint } from "./fingerprint.js";import { readSourceFile } from "../util/source.js";@@ -88,25 +88,19 @@ export async function checkGraph(const fpOnlyDirs = readFingerprint(outDir)?.onlyDirs;const onlyDirs = fpOnlyDirs && fpOnlyDirs.length > 0 ? new Set(fpOnlyDirs) : undefined;const sourceFiles = listSourceFiles(root, outDir, undefined, onlyDirs);- await warmGenericGrammars(- new Set(sourceFiles.map((f) => genericLangOf(f)?.name).filter((n): n is string => !!n)),- );// Container-tier grammars need the same warmup as the generic ones, for the same- // reason: extraction below is synchronous. Missing this is what made `graft+ // reason: extraction below is synchronous. Missing that is what made `graft// check` report every `.vue` node as `removed` right after a clean build (#236)// — the tier extracted fine, and then the check had no branch that could see it.- await warmContainerGrammars(- new Set(sourceFiles.map((f) => containerLangOf(f)?.name).filter((n): n is string => !!n)),- );+ // The same call warms the WASM stand-in for a depth language whose native+ // grammar didn't load, exactly as the build does.+ await warmGrammarsFor(sourceFiles);const current = new Map<string, string>(); // id → body_hashfor (const file of sourceFiles) {- // The same three-way branch `buildGraph` uses, in the same order. The two must- // stay in step: a tier the build extracts and the check cannot see reports as- // `removed` forever, and the `graft build` the check tells you to run can never- // repair it.- const lang = languageOf(file);- const container = lang ? null : containerLangOf(file);- const generic = lang || container ? null : genericLangOf(file);+ // The same dispatch `buildGraph` uses (`tierOf`). The two must stay in step: a+ // tier the build extracts and the check cannot see reports as `removed`+ // forever, and the `graft build` the check tells you to run can never repair it.+ const { lang, container, generic } = tierOf(file);let source: string | null;try {source = readSourceFile(file);diff --git a/src/graph/container.ts b/src/graph/container.tsindex 3015069e..f9bd72ff 100644--- a/src/graph/container.ts+++ b/src/graph/container.ts@@ -22,7 +22,8 @@* fixtures whose true line numbers are known.*/import { extractFile, mintId, type ExtractResult, type Language, type RawEdge } from "./extract.js";-import { loadWasmLanguage, parseWasm, type TsNode } from "./generic.js";+import { extractGeneric, fallbackLangOf, loadWasmLanguage, parseWasm, type TsNode } from "./generic.js";+import { nativeReady } from "./native.js";import { contentHash } from "../util/id.js";import type { NodeV1 } from "./types.js";@@ -168,7 +169,11 @@ export function extractContainer(rel: string, source: string, lang: ContainerLanlet inner: ExtractResult;try {- inner = extractFile(rel, script, lang.inner);+ // No native grammar for the embedded language (see native.ts): its WASM+ // stand-in gives the block's symbols, warmed by the build like any other.+ inner = nativeReady(lang.inner)+ ? extractFile(rel, script, lang.inner)+ : extractGeneric(rel, script, fallbackLangOf(lang.inner).name);} catch {continue; // one bad block, not a bad build}diff --git a/src/graph/extract-cache.ts b/src/graph/extract-cache.tsindex 2acf8b25..db3ba22c 100644--- a/src/graph/extract-cache.ts+++ b/src/graph/extract-cache.ts@@ -48,6 +48,10 @@ export interface ExtractEntry {* re-reports the same error and contributes no nodes, exactly as a cold build* would. */error?: string;+ /** The breadth-tier grammar that parsed this file standing in for its depth-tier+ * one, whose native grammar didn't load. Absent for every other file. The entry is+ * replayed only while the same stand-in would parse it again. */+ via?: string;}export interface ExtractCache {diff --git a/src/graph/extract.ts b/src/graph/extract.tsindex d0f9dff1..a8215fa4 100644--- a/src/graph/extract.ts+++ b/src/graph/extract.ts@@ -6,18 +6,13 @@* arrow-function consts) plus unresolved edge intents. Edge *targets* are* resolved against the whole-repo node index later, in build.ts.*/-import Parser from "tree-sitter";-import TypeScript from "tree-sitter-typescript";-import Python from "tree-sitter-python";-import Go from "tree-sitter-go";-import R from "tree-sitter-r";-import Java from "tree-sitter-java";-import Kotlin from "tree-sitter-kotlin";-import Swift from "tree-sitter-swift";-import PHP from "tree-sitter-php";+// Types only: the native grammars themselves load on first use, in native.ts, so+// a missing binary costs its own language instead of every command.+import type Parser from "tree-sitter";import { basename } from "node:path";import { contentHash } from "../util/id.js";import { collectBindings, goReceiverVarOf, resolveRecvType, type FileBindings } from "./bindings.js";+import { nativeParser } from "./native.js";import type { Kind, NodeV1, Relation } from "./types.js";export type Language = "typescript" | "tsx" | "python" | "go" | "java" | "kotlin" | "swift" | "php" | "r";@@ -310,19 +305,6 @@ const FUNCTION_VALUE_TYPES = new Set([const EMPTY_SET: ReadonlySet<string> = new Set();-const parser = new Parser();-const GRAMMARS: Record<Language, unknown> = {- typescript: TypeScript.typescript,- tsx: TypeScript.tsx,- python: Python,- go: Go,- r: R,- java: Java,- kotlin: Kotlin,- swift: Swift,- php: PHP.php,-};-export interface WalkCtx {rel: string;source: string;@@ -380,13 +362,14 @@ interface DefDescriptor {* no such limit as long as each returned chunk is under 32 KB, so we always feed* the source in <32 KB slices. Code-unit indexing matches `String.slice`. */const PARSE_CHUNK = 16384;-function parseSource(source: string): Parser.SyntaxNode {+function parseSource(parser: Parser, source: string): Parser.SyntaxNode {return parser.parse((index: number) => source.slice(index, index + PARSE_CHUNK)).rootNode;}+/** Throws when `lang`'s native grammar is unavailable: callers check `nativeReady`+ * (native.ts) first and hand such a file to its breadth-tier stand-in instead. */export function extractFile(rel: string, source: string, lang: Language): ExtractResult {- parser.setLanguage(GRAMMARS[lang] as never);- const root = parseSource(source);+ const root = parseSource(nativeParser(lang), source);const bindings = collectBindings(root, lang);const importedSymbols = collectImportedSymbols(root, lang);const rGenerics = lang === "r" ? collectRGenerics(root) : EMPTY_SET;diff --git a/src/graph/generic.ts b/src/graph/generic.tsindex b599deb7..def35f6e 100644--- a/src/graph/generic.ts+++ b/src/graph/generic.ts@@ -23,7 +23,7 @@ import { dirname, join } from "node:path";import { createRequire } from "node:module";import { contentHash } from "../util/id.js";import type { Kind, NodeV1 } from "./types.js";-import type { ExtractResult, RawEdge } from "./extract.js";+import type { ExtractResult, Language, RawEdge } from "./extract.js";const HERE = dirname(fileURLToPath(import.meta.url));const require = createRequire(import.meta.url);@@ -36,6 +36,9 @@ export interface GenericLang {name: string;exts: string[];wasm: string;+ /** Bundle dirs whose `tags.scm` to use, concatenated, when there is no vendored+ * queries/<name>.scm. Only the depth-tier stand-ins below set it. */+ bundledTags?: string[];}/** The breadth registry. Add a row + a queries/<name>.scm to support a language.@@ -60,6 +63,36 @@ export const GENERIC_LANGS: readonly GenericLang[] = [{ name: "lua", exts: [".lua"], wasm: "lua" },];+/**+ * WASM stand-ins for the depth tier, used only for a language whose native grammar+ * did not load (native.ts): npm 12 skips the install script that builds+ * `tree-sitter-kotlin`, for one. They claim no extension, so `genericLangOf` never+ * returns them; the build picks one by asking {@link fallbackLangOf}. Output is the+ * breadth tier's: symbols and bare-name calls, no receiver types or bindings.+ * Kotlin, Swift and PHP use their vendored queries/<name>.scm; the rest use the+ * bundle's own tags.scm (typescript's builds on javascript's, as upstream does).+ */+const FALLBACK_LANGS: Record<Exclude<Language, "java">, GenericLang> = {+ typescript: { name: "typescript", exts: [], wasm: "typescript", bundledTags: ["javascript", "typescript"] },+ tsx: { name: "tsx", exts: [], wasm: "tsx", bundledTags: ["javascript", "tsx"] },+ python: { name: "python", exts: [], wasm: "python", bundledTags: ["python"] },+ go: { name: "go", exts: [], wasm: "go", bundledTags: ["go"] },+ r: { name: "r", exts: [], wasm: "r", bundledTags: ["r"] },+ kotlin: { name: "kotlin", exts: [], wasm: "kotlin" },+ swift: { name: "swift", exts: [], wasm: "swift" },+ php: { name: "php", exts: [], wasm: "php" },+};++/** The breadth-tier grammar that stands in for a depth-tier language whose native+ * grammar can't load. Java already has a breadth row, so it is its own stand-in. */+export function fallbackLangOf(lang: Language): GenericLang {+ return lang === "java" ? GENERIC_LANGS.find((l) => l.name === "java")! : FALLBACK_LANGS[lang];+}++function rowNamed(name: string): GenericLang | undefined {+ return GENERIC_LANGS.find((l) => l.name === name) ?? Object.values(FALLBACK_LANGS).find((l) => l.name === name);+}+const byExt = new Map<string, GenericLang>();for (const l of GENERIC_LANGS) for (const e of l.exts) byExt.set(e, l);@@ -101,20 +134,29 @@ function requireWasm(wasm: string): Buffer | null {}}-function loadQuery(name: string): string | null {+// Sanitize editor-specific query predicates the tree-sitter Query compiler rejects.+function sanitizeQuery(raw: string): string {+ return raw.replace(/\(#(?:strip!|set!|set-adjacent!|select-adjacent!|make-range!|offset!|gsub!)[^()]*\)/g, "");+}++function loadQuery(row: GenericLang): string | null {for (const dir of QUERY_DIRS) {try {- const raw = readFileSync(join(dir, `${name}.scm`), "utf8");- // Sanitize editor-specific query predicates the tree-sitter Query compiler rejects.- return raw.replace(- /\(#(?:strip!|set!|set-adjacent!|select-adjacent!|make-range!|offset!|gsub!)[^()]*\)/g,- "",- );+ return sanitizeQuery(readFileSync(join(dir, `${row.name}.scm`), "utf8"));} catch {/* try next dir */}}- return null;+ if (!row.bundledTags) return null;+ const parts: string[] = [];+ for (const dir of row.bundledTags) {+ try {+ parts.push(readFileSync(require.resolve(`tree-sitter-wasm/${dir}/tags.scm`), "utf8"));+ } catch {+ /* no tags.scm for this one */+ }+ }+ return parts.length > 0 ? sanitizeQuery(parts.join("\n")) : null;}/** Warm the WASM grammars for the given graft lang names. Idempotent; must be@@ -122,7 +164,7 @@ function loadQuery(name: string): string | null {* grammars are silently skipped (their files then extract as file-only). */export async function warmGenericGrammars(langNames: Iterable<string>): Promise<void> {const want = new Set(langNames);- const need = [...want].filter((n) => !loaded.has(n) && GENERIC_LANGS.some((l) => l.name === n));+ const need = [...want].filter((n) => !loaded.has(n) && rowNamed(n) !== undefined);if (need.length === 0) return;if (!tsMod) {tsMod = await import("web-tree-sitter");@@ -131,12 +173,12 @@ export async function warmGenericGrammars(langNames: Iterable<string>): Promise<await initPromise;const { Language, Query } = tsMod;for (const name of need) {- const row = GENERIC_LANGS.find((l) => l.name === name)!;+ const row = rowNamed(name)!;const bytes = requireWasm(row.wasm);if (!bytes) continue;try {const language = await Language.load(bytes);- const scm = loadQuery(name);+ const scm = loadQuery(row);let query: unknown | null = null;if (scm) {try { query = new Query(language, scm); } catch { query = null; }diff --git a/src/graph/native.ts b/src/graph/native.tsnew file mode 100644index 00000000..32ac41a1--- /dev/null+++ b/src/graph/native.ts@@ -0,0 +1,187 @@+/**+ * The depth tier's native grammars, loaded on first use and never fatally.+ *+ * Every depth-tier grammar is a native addon, and so is `tree-sitter` itself. Most+ * ship prebuilt binaries that load with no install step, but not all of them:+ * `tree-sitter-kotlin` ships none at all, and `tree-sitter` and+ * `tree-sitter-python` have no linux-arm64 build. Those work only if npm ran their+ * install script, which compiles one. npm 12 stopped running dependency install+ * scripts by default, so after a plain `npm install -g` the binary is not there.+ *+ * These used to be static imports at the top of extract.ts. One missing binary+ * then threw while the module graph loaded and took every command down with it,+ * the MCP server and the hooks included, even in a repo with no Kotlin in it.+ * Now each is required the first time a file of its language is parsed, and a+ * failure is remembered: that language's files go to its WASM stand-in in the+ * breadth tier instead (`fallbackLangOf` in generic.ts), and the build says which+ * languages and how to get the native parser back ({@link nativeFallbackNotice}).+ */+import { createRequire } from "node:module";+import type Parser from "tree-sitter";+import type { Language } from "./extract.js";++const nodeRequire = createRequire(import.meta.url);++const CORE = "tree-sitter";++/** The module each depth-tier grammar comes from, and the export that is the+ * grammar when the module ships more than one (typescript+tsx, php+php_only). */+const GRAMMAR_MODULES: Record<Language, { module: string; pick?: string }> = {+ typescript: { module: "tree-sitter-typescript", pick: "typescript" },+ tsx: { module: "tree-sitter-typescript", pick: "tsx" },+ python: { module: "tree-sitter-python" },+ go: { module: "tree-sitter-go" },+ r: { module: "tree-sitter-r" },+ java: { module: "tree-sitter-java" },+ kotlin: { module: "tree-sitter-kotlin" },+ swift: { module: "tree-sitter-swift" },+ php: { module: "tree-sitter-php", pick: "php" },+};++/** The name `npm --allow-scripts` needs, where it differs from the module name.+ * `tree-sitter-r` is an npm alias, and npm matches an allow entry against the real+ * package only: an alias as the key matches nothing. */+const NPM_NAME: Record<string, string> = {+ "tree-sitter-r": "@davisvaughan/tree-sitter-r",+};++type Loaded = { ok: true; value: unknown } | { ok: false; error: string };++let requireModule: (id: string) => unknown = (id) => nodeRequire(id);+const modules = new Map<string, Loaded>();+const ready = new Map<Language, boolean>();+/** undefined: not tried yet. null: tried, and the core binding is unusable. */+let parser: Parser | null | undefined;++function message(err: unknown): string {+ return err instanceof Error ? err.message : String(err);+}++function load(id: string): Loaded {+ let hit = modules.get(id);+ if (!hit) {+ try {+ hit = { ok: true, value: requireModule(id) };+ } catch (err) {+ hit = { ok: false, error: message(err) };+ }+ modules.set(id, hit);+ }+ return hit;+}++function coreParser(): Parser | null {+ if (parser === undefined) {+ const core = load(CORE);+ parser = null;+ if (core.ok) {+ try {+ parser = new (core.value as typeof Parser)();+ } catch (err) {+ modules.set(CORE, { ok: false, error: message(err) });+ }+ }+ }+ return parser;+}++function grammarOf(lang: Language): unknown {+ const { module, pick } = GRAMMAR_MODULES[lang];+ const m = load(module);+ if (!m.ok) return null;+ return pick ? ((m.value as Record<string, unknown>)[pick] ?? null) : m.value;+}++/**+ * True when `lang`'s native grammar loads and the parser accepts it. The first call+ * per language does the loading; the answer is cached for the life of the process.+ * Never throws.+ */+export function nativeReady(lang: Language): boolean {+ let ok = ready.get(lang);+ if (ok === undefined) {+ ok = false;+ // Load the grammar even when the core binding failed, so the notice can name+ // every package that is missing its binary, not only the first.+ const grammar = grammarOf(lang);+ const p = coreParser();+ if (p && grammar) {+ try {+ p.setLanguage(grammar as never);+ ok = true;+ } catch {+ /* an addon built for another tree-sitter ABI: unusable, though it loaded */+ }+ }+ ready.set(lang, ok);+ }+ return ok;+}++/** The shared native parser, set to `lang`. Throws when that grammar is not+ * available; callers ask {@link nativeReady} first and send the file elsewhere. */+export function nativeParser(lang: Language): Parser {+ if (!nativeReady(lang)) {+ const { module } = GRAMMAR_MODULES[lang];+ const why = [CORE, module].map((id) => modules.get(id)).find((m) => m && !m.ok);+ throw new Error(`the native ${lang} parser is not available${why && !why.ok ? `: ${why.error}` : ""}`);+ }+ const p = parser as Parser;+ p.setLanguage(grammarOf(lang) as never);+ return p;+}++/**+ * The npm packages whose missing binary keeps these languages off the native+ * parser: what `npm install --allow-scripts=…` has to name. Empty when every one+ * loaded, or when the ones that failed did load and were rejected by the parser+ * (an ABI mismatch, which no install flag fixes).+ */+export function missingNativePackages(langs: Iterable<Language>): string[] {+ const out = new Set<string>();+ for (const lang of langs) {+ if (nativeReady(lang)) continue;+ for (const id of [CORE, GRAMMAR_MODULES[lang].module]) {+ const m = modules.get(id);+ if (m && !m.ok) out.add(NPM_NAME[id] ?? id);+ }+ }+ return [...out].sort();+}++/** The languages a build parsed with a WASM stand-in instead of their native+ * grammar, and the packages to allow so the next install builds them. */+export interface NativeFallback {+ /** Display labels (`javascript`, not the `typescript` grammar that parses it). */+ languages: string[];+ packages: string[];+}++/**+ * The one line a build prints when some languages fell back, or null when none did.+ * One line on purpose: `graft init` passes on only the `⚠`/`✗` lines of the build+ * it runs, so a second line holding the fix would be dropped there.+ */+export function nativeFallbackNotice(fallback: NativeFallback | undefined, packageName: string): string | null {+ if (!fallback || fallback.languages.length === 0) return null;+ const langs = fallback.languages.join(", ");+ const what = fallback.languages.length === 1 ? "the native parser is" : "the native parsers are";+ const head = `⚠ ${langs}: basic parsing only, ${what} missing.`;+ if (fallback.packages.length === 0) return head;+ return `${head} To fix: npm install -g --allow-scripts=${fallback.packages.join(",")} ${packageName}`;+}++/** Test seam: replace how native modules are required (null restores the real+ * `require`), and forget everything loaded so far. A missing binary is simulated by+ * a function that throws for that module id, the way node-gyp-build does. */+export function setNativeRequireForTest(fn: ((id: string) => unknown) | null): void {+ requireModule = fn ?? ((id) => nodeRequire(id));+ modules.clear();+ ready.clear();+ parser = undefined;+}++/** The real `require` the seam above wraps, for a test's pass-through. */+export function realNativeRequire(id: string): unknown {+ return nodeRequire(id);+}diff --git a/src/graph/workspace-cli.ts b/src/graph/workspace-cli.tsindex f6dc57e8..7fb1296b 100644--- a/src/graph/workspace-cli.ts+++ b/src/graph/workspace-cli.ts@@ -8,6 +8,8 @@import { Graft } from "../engine.js";import { contextDirFor, ensureGitignored } from "../context/node-file.js";import { patchBuildConfig, type BuildConfig } from "../util/state.js";+import { brand, PACKAGE_FOR } from "../brand.js";+import { nativeFallbackNotice } from "./native.js";import type { EngineConfig } from "../ai/providers.js";import { formatAsk } from "../ask/ask.js";import type { Direction } from "./traverse.js";@@ -40,12 +42,17 @@ export interface WorkspaceBuildOptions {followSubmodules?: boolean;/** An explicit CLI nested-clone choice to persist into every child repo. */followNestedRepos?: boolean;+ /** The npm package this runs from, for the fix in the native-parser notice. */+ packageName?: string;}/** Build every git child into its own committable `graft/`, then replace the* parent's `graft/` with `workspace.json`. Prints the one-time split warning* first when migrating away from a mega-graph. */export async function runWorkspaceBuild(root: string, opts: WorkspaceBuildOptions): Promise<void> {+ // Native grammars load once per process, so the children share one answer:+ // collected here and printed once at the end, not once per repo.+ const fellBack = { languages: new Set<string>(), packages: new Set<string>() };const buildChild = async (childDir: string, childName: string): Promise<void> => {// Persisted BEFORE the child build itself runs, same as the single-repo// path in cli.ts, so this build and every later no-flag child build agree.@@ -67,6 +74,8 @@ export async function runWorkspaceBuild(root: string, opts: WorkspaceBuildOptionconst g = await engine.graph(childDir, { llm: opts.deep, concurrency: opts.concurrency });console.log(`✓ ${childName}/: ${g.nodes} nodes, ${g.edges} edges, ${g.cards} cards [${g.languages.join(", ")}]`);for (const e of g.errors) console.error(`✗ ${childName}/: ${e}`);+ for (const l of g.nativeFallback?.languages ?? []) fellBack.languages.add(l);+ for (const p of g.nativeFallback?.packages ?? []) fellBack.packages.add(p);};const { children } = await splitWorkspace(@@ -83,6 +92,11 @@ export async function runWorkspaceBuild(root: string, opts: WorkspaceBuildOptionensureGitignored(root, contextDirFor(root, opts.override));console.log(`✓ workspace: ${children.length} repos federated → graft/workspace.json`);console.log(` graft/ is git-ignored — each teammate runs \`graft build\` to regenerate it locally.`);+ const notice = nativeFallbackNotice(+ { languages: [...fellBack.languages].sort(), packages: [...fellBack.packages].sort() },+ opts.packageName ?? PACKAGE_FOR[brand()],+ );+ if (notice) console.error(notice);}export function runWorkspaceAsk(diff --git a/test/graph-native-fallback.test.ts b/test/graph-native-fallback.test.tsnew file mode 100644index 00000000..b5b540dd--- /dev/null+++ b/test/graph-native-fallback.test.ts@@ -0,0 +1,185 @@+/**+ * A native grammar that won't load costs its own language, never the command.+ *+ * The bug: npm 12 stops running dependency install scripts by default, and+ * `tree-sitter-kotlin` ships no prebuilt binary, so after `npm install -g` its addon+ * is simply absent. extract.ts imported every grammar statically, so the require+ * threw while the module graph loaded and `graft build`, the MCP server and the+ * hooks all crashed, in a repo with no Kotlin in it at all.+ *+ * Simulated here through native.ts's require seam, the way node-gyp-build fails:+ * the require throws "No native build was found".+ */+import { afterEach, test } from "node:test";+import assert from "node:assert/strict";+import { mkdirSync, writeFileSync } from "node:fs";+import { join } from "node:path";+import { buildGraph } from "../src/graph/build.js";+import { checkGraph } from "../src/graph/check.js";+import {+ missingNativePackages,+ nativeFallbackNotice,+ nativeReady,+ realNativeRequire,+ setNativeRequireForTest,+} from "../src/graph/native.js";+import { tmpRepo } from "./helpers.js";++const KOTLIN = [+ "package demo",+ "",+ "class Greeter(val name: String) {",+ ' fun greet(): String = "hi " + name',+ "}",+ "",+ "fun helper(x: Int): Int = x + 1",+ "",+ "fun main() {",+ ' val g = Greeter("a")',+ " println(g.greet())",+ " helper(2)",+ "}",+ "",+].join("\n");++const PY = ["class Store:", " def load(self):", " return fetch()", "", "def fetch():", " return 1", ""].join("\n");++const TS = ["export function add(a: number, b: number): number {", " return a + b;", "}", "export function twice(a: number): number {", " return add(a, a);", "}", ""].join("\n");++function repo(): string {+ const d = tmpRepo("native-fallback");+ mkdirSync(join(d, "src"), { recursive: true });+ writeFileSync(join(d, "src", "math.ts"), TS);+ writeFileSync(join(d, "src", "store.py"), PY);+ writeFileSync(join(d, "src", "Main.kt"), KOTLIN);+ return d;+}++/** A require that fails for `missing`, exactly as node-gyp-build does, and records+ * every module asked for. */+function failing(missing: string[], asked: string[] = []): (id: string) => unknown {+ return (id) => {+ asked.push(id);+ if (missing.includes(id)) {+ throw new Error(`No native build was found for platform=${process.platform} arch=${process.arch} loaded from node_modules/${id}`);+ }+ return realNativeRequire(id);+ };+}++afterEach(() => setNativeRequireForTest(null));++test("all grammars load: native parse everywhere, and no notice", async () => {+ setNativeRequireForTest(null);+ const r = await buildGraph(repo());+ assert.deepEqual(r.errors, []);+ assert.equal(r.nativeFallback, undefined);+ assert.equal(nativeFallbackNotice(r.nativeFallback, "@trailhq/trail"), null);+ assert.deepEqual(r.languages, ["kotlin", "python", "typescript"]);+ assert.ok(nativeReady("kotlin"));+});++test("a missing kotlin binary: kotlin goes to the WASM stand-in, the build completes, the rest stay native", async () => {+ setNativeRequireForTest(failing(["tree-sitter-kotlin"]));+ const dir = repo();+ const r = await buildGraph(dir);+ assert.deepEqual(r.errors, []);+ assert.deepEqual(r.languages, ["kotlin", "python", "typescript"]);+ assert.deepEqual(r.nativeFallback, { languages: ["kotlin"], packages: ["tree-sitter-kotlin"] });++ const { readGraph, wiringPath } = await import("../src/graph/write.js");+ const g = readGraph(wiringPath(r.contextDir))!;+ const kt = g.nodes.filter((n) => n.path === "src/Main.kt" && n.kind !== "file");+ const names = kt.map((n) => n.name).sort();+ for (const want of ["Greeter", "greet", "helper", "main"]) assert.ok(names.includes(want), `kotlin ${want} missing: ${names}`);+ assert.ok(kt.every((n) => n.origin === "generic"), "kotlin symbols come from the stand-in");+ // The vendored kotlin tags query carries calls, so `graft callers helper` works.+ assert.ok(+ g.edges.some((e) => e.relation === "calls" && e.source === "src/Main.kt#main" && e.target === "src/Main.kt#helper"),+ "main → helper call edge",+ );+ // Languages with a binary are untouched.+ for (const path of ["src/math.ts", "src/store.py"]) {+ const syms = g.nodes.filter((n) => n.path === path && n.kind !== "file");+ assert.ok(syms.length > 0 && syms.every((n) => n.origin === "ast"), `${path} stays on the native parser`);+ }++ // `graft check` dispatches the same way, so the fallback graph reads as in sync.+ const c = await checkGraph(dir);+ assert.deepEqual([c.added, c.removed, c.changed], [[], [], []]);+});++test("the notice names only the grammar that failed, with the install that fixes it", async () => {+ setNativeRequireForTest(failing(["tree-sitter-kotlin"]));+ const r = await buildGraph(repo());+ assert.equal(+ nativeFallbackNotice(r.nativeFallback, "@trailhq/trail"),+ "⚠ kotlin: basic parsing only, the native parser is missing. To fix: npm install -g --allow-scripts=tree-sitter-kotlin @trailhq/trail",+ );+ assert.match(nativeFallbackNotice(r.nativeFallback, "@nanonets/graft")!, / @nanonets\/graft$/);+});++test("a repo without kotlin never asks for its grammar", async () => {+ const asked: string[] = [];+ setNativeRequireForTest(failing(["tree-sitter-kotlin"], asked));+ const d = tmpRepo("native-lazy");+ writeFileSync(join(d, "math.ts"), TS);+ const r = await buildGraph(d);+ assert.deepEqual(r.errors, []);+ assert.equal(r.nativeFallback, undefined);+ assert.ok(!asked.includes("tree-sitter-kotlin"), `asked for: ${asked.join(", ")}`);+});++test("no tree-sitter core binding: every depth language falls back, nothing crashes", async () => {+ setNativeRequireForTest(failing(["tree-sitter"]));+ const dir = repo();+ const r = await buildGraph(dir);+ assert.deepEqual(r.errors, []);+ assert.deepEqual(r.nativeFallback, { languages: ["kotlin", "python", "typescript"], packages: ["tree-sitter"] });+ const { readGraph, wiringPath } = await import("../src/graph/write.js");+ const g = readGraph(wiringPath(r.contextDir))!;+ const names = new Set(g.nodes.filter((n) => n.kind !== "file").map((n) => n.name));+ // The bundle's own tags queries stand in for typescript and python.+ for (const want of ["add", "twice", "Store", "load", "fetch", "helper"]) assert.ok(names.has(want), `${want} missing`);+ assert.equal(+ nativeFallbackNotice(r.nativeFallback, "@trailhq/trail"),+ "⚠ kotlin, python, typescript: basic parsing only, the native parsers are missing. To fix: npm install -g --allow-scripts=tree-sitter @trailhq/trail",+ );+});++test("a stand-in parse is not replayed once the native grammar loads again", async () => {+ const dir = repo();+ setNativeRequireForTest(failing(["tree-sitter-kotlin"]));+ await buildGraph(dir);+ // The reinstall that the notice asks for: same bytes, the grammar now loads.+ setNativeRequireForTest(null);+ const r = await buildGraph(dir);+ assert.equal(r.nativeFallback, undefined);+ assert.equal(r.parsed, 1, "only the kotlin file is parsed again");+ const { readGraph, wiringPath } = await import("../src/graph/write.js");+ const g = readGraph(wiringPath(r.contextDir))!;+ const kt = g.nodes.filter((n) => n.path === "src/Main.kt" && n.kind !== "file");+ assert.ok(kt.length > 0 && kt.every((n) => n.origin === "ast"), "kotlin is back on the native parser");+});++test("the allow-scripts name is the real package, not the npm alias", () => {+ setNativeRequireForTest(failing(["tree-sitter-r"]));+ assert.equal(nativeReady("r"), false);+ assert.deepEqual(missingNativePackages(["r", "python"]), ["@davisvaughan/tree-sitter-r"]);+});++test("a .vue file's script falls back with its language, spans still on the .vue line", async () => {+ setNativeRequireForTest(failing(["tree-sitter-typescript"]));+ const d = tmpRepo("native-vue");+ writeFileSync(+ join(d, "Card.vue"),+ ["<template>", " <p>hi</p>", "</template>", "", '<script lang="ts">', "export function shout(what: string) {", " return what.toUpperCase();", "}", "</script>", ""].join("\n"),+ );+ const r = await buildGraph(d);+ assert.deepEqual(r.errors, []);+ assert.deepEqual(r.nativeFallback, { languages: ["typescript"], packages: ["tree-sitter-typescript"] });+ const { readGraph, wiringPath } = await import("../src/graph/write.js");+ const shout = readGraph(wiringPath(r.contextDir))!.nodes.find((n) => n.name === "shout");+ assert.ok(shout, "the script's function is still indexed");+ assert.equal(shout.span, "L6-L8");+});