Find code by asking what it does. A CLI for coding agents that uses Jev to discover relevant files and source context.
TypeScript ai-sdkclaude-codeclicode-searchcodexcoding-agentscontext-retrievaldeveloper-toolsjevsemantic-searchtypescriptvercel-ai-gateway
Add named Go and Rust declaration retrieval (#35)
David Zhang82ef1fd3f431
diff --git a/README.md b/README.mdindex 583e448..4046fa3 100644--- a/README.md+++ b/README.md@@ -88,7 +88,7 @@ confidently return an excerpt; it does not force every search into a fixed top-tlist.The summary and compact file list come first, followed by selected source with-line references, then detailed declaration and call locations. Python and TypeScript/JavaScript support declaration+line references, then detailed declaration and call locations. Python, TypeScript/JavaScript, Go and Rust support declarationparsing; other text uses a fallback. The output is evidence for the agent to use,not a generated answer or a guarantee that every relevant file was found.[See a recorded output example](specs/done/jevgrep/assets/stdout-example.txt).diff --git a/apps/cli/package.json b/apps/cli/package.jsonindex 0e8e8a3..c1bfe3d 100644--- a/apps/cli/package.json+++ b/apps/cli/package.json@@ -1,6 +1,6 @@{"name": "@dzhng/jevgrep",- "version": "0.6.0",+ "version": "0.7.0","description": "Content-aware source retrieval for coding agents","license": "MIT","repository": {diff --git a/apps/cli/test/package.test.ts b/apps/cli/test/package.test.tsindex d361770..58b6334 100644--- a/apps/cli/test/package.test.ts+++ b/apps/cli/test/package.test.ts@@ -37,7 +37,8 @@ testInDocker(const filenames = packed.files.map((file: { path: string }) => file.path);expect(filenames).toContain("dist/bin/index.js");expect(filenames).toContain("dist/bin/parser-worker.mjs");- expect(filenames).toContain("dist/assets/tree-sitter/tree-sitter-python.wasm");+ for (const name of ["python", "go", "rust"])+ expect(filenames).toContain(`dist/assets/tree-sitter/tree-sitter-${name}.wasm`);expect(filenames).toContain("dist/THIRD_PARTY_NOTICES.txt");expect(filenames).toContain("dist/skills/jevgrep/SKILL.md");expect(filenames.some((path: string) => /^(evals|src|node_modules|test)\//.test(path))).toBe(diff --git a/bun.lock b/bun.lockindex b200cb9..f50be71 100644--- a/bun.lock+++ b/bun.lock@@ -13,7 +13,7 @@},"apps/cli": {"name": "@dzhng/jevgrep",- "version": "0.6.0",+ "version": "0.7.0","bin": {"jg": "./dist/bin/index.js",},@@ -42,7 +42,9 @@"devDependencies": {"@repo/typescript-config": "workspace:*","@types/node": "22",+ "tree-sitter-go": "0.25.0","tree-sitter-python": "0.25.0",+ "tree-sitter-rust": "0.24.0",},},"packages/typescript-config": {@@ -199,8 +201,12 @@"tinypool": ["[email protected]", "", {}, "sha512-Pugqs6M0m7Lv1I7FtxN4aoyToKg1C4tu+/381vH35y8oENM/Ai7f7C4StcoK4/+BSw9ebcS8jRiVrORFKCALLw=="],+ "tree-sitter-go": ["[email protected]", "", { "dependencies": { "node-addon-api": "^8.3.1", "node-gyp-build": "^4.8.4" }, "peerDependencies": { "tree-sitter": "^0.25.0" }, "optionalPeers": ["tree-sitter"] }, "sha512-APBc/Dq3xz/e35Xpkhb1blu5UgW+2E3RyGWawZSCNcbGwa7jhSQPS8KsUupuzBla8PCo8+lz9W/JDJjmfRa2tw=="],+"tree-sitter-python": ["[email protected]", "", { "dependencies": { "node-addon-api": "^8.5.0", "node-gyp-build": "^4.8.4" }, "peerDependencies": { "tree-sitter": "^0.25.0" }, "optionalPeers": ["tree-sitter"] }, "sha512-eCmJx6zQa35GxaCtQD+wXHOhYqBxEL+bp71W/s3fcDMu06MrtzkVXR437dRrCrbrDbyLuUDJpAgycs7ncngLXw=="],+ "tree-sitter-rust": ["[email protected]", "", { "dependencies": { "node-addon-api": "^8.2.2", "node-gyp-build": "^4.8.4" }, "peerDependencies": { "tree-sitter": "^0.22.1" }, "optionalPeers": ["tree-sitter"] }, "sha512-NWemUDf629Tfc90Y0Z55zuwPCAHkLxWnMf2RznYu4iBkkrQl2o/CHGB7Cr52TyN5F1DAx8FmUnDtCy9iUkXZEQ=="],+"turbo": ["[email protected]", "", { "optionalDependencies": { "@turbo/darwin-64": "2.10.3", "@turbo/darwin-arm64": "2.10.3", "@turbo/linux-64": "2.10.3", "@turbo/linux-arm64": "2.10.3", "@turbo/windows-64": "2.10.3", "@turbo/windows-arm64": "2.10.3" }, "bin": { "turbo": "bin/turbo" } }, "sha512-uZIqzfgtWbyqu1Tqwdd0giRnPGgL0ejbcdF8eZLJhtTTVSrOgArZGzYeXTD6XNcc7ANgdhqex11Y9bHBeyiQLQ=="],"typescript": ["[email protected]", "", { "bin": { "tsc": "bin/tsc", "tsserver": "bin/tsserver" } }, "sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw=="],diff --git a/docs/architecture.md b/docs/architecture.mdindex c9f37bf..2194f94 100644--- a/docs/architecture.md+++ b/docs/architecture.md@@ -26,19 +26,24 @@ concurrency and token-aware admission.Source selection and presentation are separate. Declaration units, comments,structural class headers and bounded local-call context preserve meaning without-requiring complete files in the initial output. Parsing supports Python and+requiring complete files in the initial output. Parsing supports Python, Go, Rust andTypeScript/JavaScript; other or invalid text falls back to bounded source chunks.Source ranges always refer to the same immutable snapshot used for classification.## Parsing-Python uses a [packaged Tree-sitter WASM grammar](../packages/core/assets/README.md)-in a cancellable worker. This avoids a Python installation requirement and the+Python, Go and Rust use [packaged Tree-sitter WASM grammars](../packages/core/assets/README.md)+in a shared cancellable worker. This avoids a Python installation requirement and thestartup cost of embedding an interpreter. TypeScript/JavaScript use the TypeScriptcompiler parser; other eligible text remains searchable through bounded chunks.-The syntax tree supplies declarations and source coordinates. Python query-previews, structural neighbours and inherited-method reading leads are retrieval+The syntax tree supplies declarations and source coordinates. Go declaration+groups remain intact where earlier values affect later constants. Rust methods+retain module/impl headers and attributes, including inner attributes. Macro+expansion and type resolution are outside this boundary. See the+[Go/Rust retrieval contract](../specs/go-rust-parsing.md).++Python query previews, structural neighbours and inherited-method reading leads are retrievalpolicy on top of that tree, not a proof of runtime dispatch. Preserve originalsource bytes; never reconstruct returned code from the tree.diff --git a/evals/README.md b/evals/README.mdindex 4bb8086..386200a 100644--- a/evals/README.md+++ b/evals/README.md@@ -40,3 +40,6 @@ why lower retrieval charges can be offset by greater coding-agent expense.The [skill-impact pilot](results/skill-impact-2026-09-29.md) compares old and newskills on identical CLI builds, separately from automatic-trigger checks.++The [Python parser checkpoint](results/python-parser-checkpoint-2026-09-29.md)+records one official task against the merged Tree-sitter runtime.diff --git a/evals/results/python-parser-checkpoint-2026-09-29.md b/evals/results/python-parser-checkpoint-2026-09-29.mdnew file mode 100644index 0000000..1600fac--- /dev/null+++ b/evals/results/python-parser-checkpoint-2026-09-29.md@@ -0,0 +1,40 @@+# Python parser checkpoint — 2026-09-29++The merged Tree-sitter Python parser passed one official SWE-bench task,+`psf__requests-1142`. The installed candidate was built from `ae10a9e`; although+its manifest reads 0.6.0, it is not the published 0.6.0 package. Its SHA-256 is+`3309d67d7ae8d5a9ca824eaf7d1d5a0c2193df6422998d18cd8f34d61c2cff38`.++| Observation | Result |+|---|---:|+| Official resolved tasks | 1 / 1 |+| Coding-agent cost | $0.397791 |+| Jev cost (native input-usage estimate) | $0.005494944 |+| Total task cost | $0.403285944 |+| Jev requests | 16 |+| Agent wall time | 117.79 s |+| Charged work time | 105.14 s |+| Credited standalone retrieval wait | 12.64 s |++Required retrieval and timing protocol checks passed; all billing was reconciled.+Six declaration-selection requests covered `requests/models.py`,+`requests/adapters.py`, and `test_requests.py`, confirming structural Python+parsing was exercised rather than only directory navigation.+The retained no-Jev baseline also passed and cost $0.2685004. That historical+comparison does not isolate the parser change, and this checkpoint is not a cost+win or a full-cohort acceptance claim. No baseline was rerun and no favorable+attempt was selected from repeated treatments.++The [installed harness](../implementation/swebench/installed.md) froze Sol+`openai/gpt-5.6-sol` at medium effort, the public skill, source tree and official+grader. Jev used the native TypeSafe route. The original runtime image was absent;+the retained rebuilt Requests image passed pinned base-layer, source and tool+identity checks. Its image ID is+`sha256:1a4338ea36e3f3fc9737f0625a4e4c07f1a2424f844aada5ac0d6b6a5cdf543a`.++Raw plan, lifecycle receipt, agent patch, official grade, provider traces and+accounting remain in ignored run storage at+`evals/runs/swebench/tree-sitter-python-checkpoint-2026-09-29/`.+The plan SHA-256 is `e1747710a865f690ff90ed933cf8179553aad37f5b29bbc390d708d7b9a6977a`.+The result supports this task's correctness checkpoint; it does not prove all+Python programs or repositories are regression-free.diff --git a/packages/core/assets/README.md b/packages/core/assets/README.mdindex 26f8b08..84064f1 100644--- a/packages/core/assets/README.md+++ b/packages/core/assets/README.md@@ -1,10 +1,10 @@# Parser assets-The build copies the Python grammar WASM from its exact-version official-Tree-sitter package into `tree-sitter/`. The generated asset and its license-notice ship in the CLI archive; searches never download a grammar. The external+The build copies Python, Go and Rust grammar WASM from exact-version official+Tree-sitter packages into `tree-sitter/`. The generated assets and their license+notices ship in the CLI archive; searches never download a grammar. The external`web-tree-sitter` runtime supplies its own WASM and license.[Asset preparation](../../../scripts/parser-assets.mjs) owns the copy and license-provenance. It also prepares the canonical grammar used by release validation;+provenance. It also prepares the canonical grammars used by release validation;a clean validation checkout must install the locked development dependencies first.diff --git a/packages/core/package.json b/packages/core/package.jsonindex 77ee435..0f026e4 100644--- a/packages/core/package.json+++ b/packages/core/package.json@@ -22,6 +22,8 @@"devDependencies": {"@repo/typescript-config": "workspace:*","@types/node": "22",- "tree-sitter-python": "0.25.0"+ "tree-sitter-python": "0.25.0",+ "tree-sitter-go": "0.25.0",+ "tree-sitter-rust": "0.24.0"}}diff --git a/packages/core/src/evaluator.ts b/packages/core/src/evaluator.tsindex 27c19f6..360ddbc 100644--- a/packages/core/src/evaluator.ts+++ b/packages/core/src/evaluator.ts@@ -126,7 +126,7 @@ export function createEvaluator(options: {endpoint: endpoint.baseURL,protocol: "typesafe-ai-3.0.8",policyVersion: options.policyVersion ?? "1",- parserVersion: "tree-sitter-0.27.0-python-0.25.0-ts-5.9.3",+ parserVersion: "tree-sitter-0.27.0-python-0.25.0-go-0.25.0-rust-0.24.0-ts-5.9.3",promptVersion: "unit-locators-1",},};diff --git a/packages/core/src/parser-declarations.mjs b/packages/core/src/parser-declarations.mjsnew file mode 100644index 0000000..aa81645--- /dev/null+++ b/packages/core/src/parser-declarations.mjs@@ -0,0 +1,113 @@+const field = (node, name) => node.childForFieldName(name);+const comment = (node) => node.type === "comment" || node.type.endsWith("_comment");+const range = (node) => ({+ startLine: node.startPosition.row + 1,+ endLine: node.endPosition.row + (node.endPosition.column ? 1 : 0),+});++export function declarations(root, language) {+ const comments = [],+ stack = [root];+ while (stack.length) {+ const node = stack.pop();+ if (comment(node)) comments.push(range(node));+ else for (const child of node.namedChildren) stack.push(child);+ }+ const units = [];+ if (language === "go")+ for (const node of root.namedChildren) {+ if (comment(node)) continue;+ let name = field(node, "name")?.text ?? node.type;+ if (node.type === "method_declaration") {+ const receiver = field(node, "receiver")?.namedChildren.find(+ (child) => child.type === "parameter_declaration",+ );+ const owner = receiver && field(receiver, "type")?.text.replace(/^\*/, "");+ if (owner) name = `${owner}.${name}`;+ } else if (["type_declaration", "var_declaration", "const_declaration"].includes(node.type)) {+ // Keep groups intact: iota and omitted constant values depend on earlier specs.+ const specs = node.namedChildren.flatMap((child) =>+ child.type === "var_spec_list" ? child.namedChildren : [child],+ );+ name =+ specs+ .flatMap((spec) =>+ spec+ .childrenForFieldName("name")+ .filter((child) => child.type === "identifier" || child.type === "type_identifier")+ .map((child) => child.text),+ )+ .join(", ") || node.type;+ } else if (node.type === "package_clause")+ name = `package ${node.namedChildren.find((child) => child.type === "package_identifier").text}`;+ units.push({ name, range: range(node) });+ }+ else {+ function startWithAttributes(node) {+ let start = node;+ while (+ start.previousNamedSibling &&+ (start.previousNamedSibling.type === "attribute_item" ||+ comment(start.previousNamedSibling))+ ) {+ const previous = start.previousNamedSibling;+ const before = previous.previousNamedSibling;+ if (+ comment(previous) &&+ before &&+ !comment(before) &&+ before.type !== "attribute_item" &&+ before.endPosition.row === previous.startPosition.row+ )+ break;+ start = start.previousNamedSibling;+ }+ return range(start).startLine;+ }+ function visit(nodes, prefix = "", headers = []) {+ const ownedHeaders = [+ ...headers,+ ...nodes.filter((node) => node.type === "inner_attribute_item").map(range),+ ];+ for (const node of nodes) {+ if (comment(node) || ["attribute_item", "inner_attribute_item"].includes(node.type))+ continue;+ const body =+ ["impl_item", "trait_item", "mod_item", "foreign_mod_item"].includes(node.type) &&+ field(node, "body");+ const owner =+ field(node, "name")?.text ||+ field(node, "type")?.text ||+ field(node, "macro")?.text ||+ node.type;+ const startLine = startWithAttributes(node);+ if (+ body &&+ body.namedChildren.some(+ (child) =>+ !comment(child) && !["attribute_item", "inner_attribute_item"].includes(child.type),+ )+ ) {+ const header = { startLine, endLine: body.startPosition.row + 1 };+ units.push({+ name: prefix + owner + ".context",+ range: header,+ ownerHeaders: [...ownedHeaders, header],+ });+ visit(+ body.namedChildren,+ node.type === "foreign_mod_item" ? prefix : prefix + owner + ".",+ [...ownedHeaders, header],+ );+ } else+ units.push({+ name: prefix + owner,+ range: { ...range(node), startLine },+ ownerHeaders: ownedHeaders,+ });+ }+ }+ visit(root.namedChildren);+ }+ return { units, comments };+}diff --git a/packages/core/src/parser-helpers.mjs b/packages/core/src/parser-helpers.mjsindex 9c2415b..b06d262 100644--- a/packages/core/src/parser-helpers.mjs+++ b/packages/core/src/parser-helpers.mjs@@ -1,22 +1,29 @@import { Parser, Language, Query } from "web-tree-sitter";import { fileURLToPath } from "node:url";+import { declarations } from "./parser-declarations.mjs";import { preview } from "./parser-preview.mjs";let initialized;-let language;-async function parse(source, action) {+const languages = new Map();+async function parse(source, action, languageName = "python") {// Retrieval coordinates count LF lines; normalizing bare CR would mislabel original bytes.- if (/\r(?!\n)/.test(source)) return null;+ if (languageName === "python" && /\r(?!\n)/.test(source)) return null;await (initialized ??= Parser.init());- language ??= await Language.load(- fileURLToPath(new URL("../assets/tree-sitter/tree-sitter-python.wasm", import.meta.url)),- );+ if (!languages.has(languageName))+ languages.set(+ languageName,+ await Language.load(+ fileURLToPath(+ new URL(`../assets/tree-sitter/tree-sitter-${languageName}.wasm`, import.meta.url),+ ),+ ),+ );const parser = new Parser();let tree;try {- parser.setLanguage(language);+ parser.setLanguage(languages.get(languageName));tree = parser.parse(source);if (!tree || tree.rootNode.hasError) return null;- if (!validPython(tree.rootNode)) return null;+ if (languageName === "python" && !validPython(tree.rootNode)) return null;return action(tree.rootNode);} finally {tree?.delete();@@ -48,7 +55,7 @@ let syntaxChecks;function validPython(root) {// Let the native query engine skip nodes that cannot affect this compatibility policy.syntaxChecks ??= new Query(- language,+ languages.get("python"),`[(exec_statement) (print_statement) (except_clause) (raise_statement) (for_in_clause)(concatenated_string) (function_definition) (class_definition) (integer)@@ -414,6 +421,8 @@ function calls(root, source, ranges) {export async function execute(helper, input) {if (helper === "inspect") return parse(input, inspectPython);const data = JSON.parse(input);+ if (helper === "declarations")+ return parse(data.source, (root) => declarations(root, data.language), data.language);if (helper === "neighborhood")return parse(data.source, (root) => neighborhood(root, data.ranges));if (helper === "calls")diff --git a/packages/core/src/parser.ts b/packages/core/src/parser.tsindex 5f1165b..cdd5e21 100644--- a/packages/core/src/parser.ts+++ b/packages/core/src/parser.ts@@ -1,6 +1,6 @@import { fork, type ChildProcess } from "node:child_process";-type Helper = "inspect" | "preview" | "neighborhood" | "calls";+type Helper = "inspect" | "preview" | "neighborhood" | "calls" | "declarations";type Pending = {helper: Helper;input: string;diff --git a/packages/core/src/retrieve.ts b/packages/core/src/retrieve.tsindex ba65a57..18388bd 100644--- a/packages/core/src/retrieve.ts+++ b/packages/core/src/retrieve.ts@@ -277,7 +277,7 @@ export async function retrieve(input: SearchInput, evaluator: Evaluator): Promis}if (truncated &&- /\.(?:pyi?|[cm]?[jt]s|[jt]sx)$/.test(source.path) &&+ /\.(?:pyi?|[cm]?[jt]s|[jt]sx|go|rs)$/.test(source.path) &&bytes.length <= 1_000_000) {const syntax = await inspect(source, {diff --git a/packages/core/src/source.ts b/packages/core/src/source.tsindex 8e50dce..efda028 100644--- a/packages/core/src/source.ts+++ b/packages/core/src/source.ts@@ -15,7 +15,7 @@ export type SourceUnit = {export type Inspection = {units: SourceUnit[];comments: Range[];- mode: "python" | "typescript" | "text";+ mode: "python" | "typescript" | "go" | "rust" | "text";fallback?: "unsupported" | "syntax" | "size";};@@ -167,6 +167,16 @@ export async function inspect(};visit(file);mode = "typescript";+ } else if (/\.(go|rs)$/.test(path)) {+ const parsed = await runParser<{ units: typeof units; comments: Range[] }>(+ "declarations",+ JSON.stringify({ source, language: path.endsWith(".go") ? "go" : "rust" }),+ options.signal,+ );+ if (!parsed) return fallback("syntax");+ units = parsed.units;+ comments = parsed.comments;+ mode = path.endsWith(".go") ? "go" : "rust";} else return fallback("unsupported");comments = [...new Map(comments.map((r) => [`${r.startLine}:${r.endLine}`, r])).values()].sort((a, b) => a.startLine - b.startLine,diff --git a/scripts/RELEASING.md b/scripts/RELEASING.mdindex 46751fb..bc79bd7 100644--- a/scripts/RELEASING.md+++ b/scripts/RELEASING.md@@ -30,7 +30,7 @@ asset checks. [The build](build-cli.ts) copies the authored MIT license and[collects third-party notices](package-notices.mjs) from emitted bundle inputs.[Retained license sources](licenses/README.md) document upstream distribution gaps.The archive excludes test fixtures, evaluation evidence, node_modules directories, source maps,-and unbundled development source. The parser worker, extraction code, and packaged Python grammar+and unbundled development source. The parser worker, extraction code, and packaged grammarsare required runtime assets and are included. Direct runtime parser dependenciesinstall from exact npm pins; transitive dependencies resolve during npm installation.The registry-verification checkout installs locked development dependencies and@@ -40,7 +40,7 @@ bytes and license notices with the downloaded archive.On an Apple Silicon Mac, run `node scripts/test-native.mjs` after the normal`bun install --frozen-lockfile` setup. This builds and packs the candidate, installsit with npm outside the checkout, and runs the credential-free commands and-synthetic Python search against a loopback Gateway fixture. To verify an existing+synthetic Python/Go/Rust searches against a loopback Gateway fixture. To verify an existingarchive without Bun, use `node scripts/test-native.mjs --prebuilt /path/package.tgz`.The command prints the archive hash and actual Node/macOS runtime for retainedevidence. It requires Node 22 or newer; a result proves the printed version only.diff --git a/scripts/build-cli.ts b/scripts/build-cli.tsindex 9c19970..8abae2e 100644--- a/scripts/build-cli.ts+++ b/scripts/build-cli.ts@@ -25,7 +25,7 @@ await writeFile(join(out, "THIRD_PARTY_NOTICES.txt"),(await bundledNotices(build.metafile!, process.cwd())) + grammarNotices,);-for (const name of ["parser-worker", "parser-helpers", "parser-preview"])+for (const name of ["parser-worker", "parser-helpers", "parser-preview", "parser-declarations"])await cp(join(root, `packages/core/src/${name}.mjs`), join(out, `bin/${name}.mjs`));await cp(join(root, "LICENSE"), join(out, "LICENSE"));await cp(join(root, "packages/core/assets"), join(out, "assets"), { recursive: true });diff --git a/scripts/licenses/README.md b/scripts/licenses/README.mdindex c6388fe..8e6c284 100644--- a/scripts/licenses/README.md+++ b/scripts/licenses/README.md@@ -14,5 +14,5 @@ there is no build-time or runtime license download.## Parser grammarsThe build copies license notices directly from the pinned official Tree-sitter-Python grammar package. Its WASM file is shipped unmodified. The external+grammar packages. Their WASM files are shipped unmodified. The externalweb-tree-sitter and TypeScript packages retain their installed license files.diff --git a/scripts/parser-assets.mjs b/scripts/parser-assets.mjsindex 4a4a62a..528cb3f 100644--- a/scripts/parser-assets.mjs+++ b/scripts/parser-assets.mjs@@ -2,6 +2,7 @@ import { createRequire } from "node:module";import { readFile, mkdir, cp } from "node:fs/promises";import { dirname, join, resolve } from "node:path";import { fileURLToPath } from "node:url";+export const grammars = ["python", "go", "rust"];export async function grammarAssets(root = fileURLToPath(new URL("../", import.meta.url)),copy = false,@@ -10,13 +11,17 @@ export async function grammarAssets(const core = JSON.parse(await readFile(join(root, "packages/core/package.json"), "utf8"));const destination = join(root, "packages/core/assets/tree-sitter");if (copy) await mkdir(destination, { recursive: true });- const name = "tree-sitter-python",- directory = dirname(require.resolve(`${name}/package.json`));- const metadata = JSON.parse(await readFile(join(directory, "package.json"), "utf8"));- if (metadata.version !== core.devDependencies[name])- throw new Error(`Grammar version differs from manifest: ${name}`);- if (copy) await cp(join(directory, `${name}.wasm`), join(destination, `${name}.wasm`));- return `\nBundled Tree-sitter grammar notices\n\n=== ${name}@${metadata.version} (${metadata.license}) ===\nSource: https://github.com/tree-sitter/${name}\n${await readFile(join(directory, "LICENSE"), "utf8")}\n`;+ let notices = "\nBundled Tree-sitter grammar notices\n\n";+ for (const language of grammars) {+ const name = `tree-sitter-${language}`;+ const directory = dirname(require.resolve(`${name}/package.json`));+ const metadata = JSON.parse(await readFile(join(directory, "package.json"), "utf8"));+ if (metadata.version !== core.devDependencies[name])+ throw new Error(`Grammar version differs from manifest: ${name}`);+ if (copy) await cp(join(directory, `${name}.wasm`), join(destination, `${name}.wasm`));+ notices += `=== ${name}@${metadata.version} (${metadata.license}) ===\nSource: https://github.com/tree-sitter/${name}\n${await readFile(join(directory, "LICENSE"), "utf8")}\n`;+ }+ return notices;}if (process.argv[1] && resolve(process.argv[1]) === fileURLToPath(import.meta.url))diff --git a/scripts/test-native.mjs b/scripts/test-native.mjsindex 3dfe2b4..f1ee2cf 100644--- a/scripts/test-native.mjs+++ b/scripts/test-native.mjs@@ -131,7 +131,7 @@ try {"--test-reporter=tap","--test-concurrency=1","--test-name-pattern",- "^(installed local commands match the package without credentials|skill command delegates installation to npx without credentials|actual installed search parses Python and returns every relevant hierarchy branch|installed saved-provider journey: (vercel|typesafe|openrouter) auth doctor search cache replacement|installed legacy credentials use Vercel without rewriting saved bytes|installed saved credentials defeat conflicting environment and environment-only auth fails|installed invalid saved providers fail before HTTP without rewriting credentials|installed invalid auth preserves saved bytes and leaves no temporary credentials)$",+ "^(installed Go/Rust search indexes late methods and selects their source instead of distant noise|installed local commands match the package without credentials|skill command delegates installation to npx without credentials|actual installed search parses Python and returns every relevant hierarchy branch|installed saved-provider journey: (vercel|typesafe|openrouter) auth doctor search cache replacement|installed legacy credentials use Vercel without rewriting saved bytes|installed saved credentials defeat conflicting environment and environment-only auth fails|installed invalid saved providers fail before HTTP without rewriting credentials|installed invalid auth preserves saved bytes and leaves no temporary credentials)$",join(root, "test/installed.test.mjs"),],{ cwd: scratch, env: runtimeEnv, timeout: 120_000, maxBuffer: 8_000_000 },@@ -139,7 +139,7 @@ try {assert.equal(tested.stderr, "");assert.match(tested.stdout,- /^# pass 10$/m,+ /^# pass 11$/m,"All native smoke journeys must execute; renamed selectors cannot silently pass",);console.log(tested.stdout.trimEnd());diff --git a/scripts/validate-release.mjs b/scripts/validate-release.mjsindex c1bccd4..0971c65 100644--- a/scripts/validate-release.mjs+++ b/scripts/validate-release.mjs@@ -4,7 +4,7 @@ import { readFile, stat } from "node:fs/promises";import { resolve } from "node:path";import { fileURLToPath } from "node:url";import { promisify } from "node:util";-import { grammarAssets } from "./parser-assets.mjs";+import { grammarAssets, grammars } from "./parser-assets.mjs";const execute = promisify(execFile);export const repository = fileURLToPath(new URL("../", import.meta.url));@@ -48,9 +48,10 @@ export async function validateRelease(tarball, tag, root = repository) {if (new Set(files).size !== files.length) throw new Error("Duplicate archive entries");for (const path of files)if (- !/^package\/(?:package\.json|README(?:\.md)?|LICENSE|dist\/(?:bin\/(?:index\.js|parser-(?:worker|helpers|preview)\.mjs)|assets\/(?:tree-sitter\/tree-sitter-python\.wasm|README\.md)|skills\/jevgrep\/SKILL\.md|LICENSE|THIRD_PARTY_NOTICES\.txt))$/.test(+ !/^package\/(?:package\.json|README(?:\.md)?|LICENSE|dist\/(?:bin\/(?:index\.js|parser-(?:worker|helpers|preview|declarations)\.mjs)|assets\/(?:README\.md)|skills\/jevgrep\/SKILL\.md|LICENSE|THIRD_PARTY_NOTICES\.txt))$/.test(path,- )+ ) &&+ !grammars.some((name) => path === `package/dist/assets/tree-sitter/tree-sitter-${name}.wasm`))throw new Error(`Unexpected published file: ${path}`);const listing = await execute("tar", ["-tvzf", tarball], { maxBuffer: 8_000_000 });@@ -76,14 +77,14 @@ export async function validateRelease(tarball, tag, root = repository) {for (const [packed, original] of [["dist/LICENSE", "LICENSE"],["dist/skills/jevgrep/SKILL.md", "skills/jevgrep/SKILL.md"],- ...["parser-worker", "parser-helpers", "parser-preview"].map((name) => [+ ...["parser-worker", "parser-helpers", "parser-preview", "parser-declarations"].map((name) => [`dist/bin/${name}.mjs`,`packages/core/src/${name}.mjs`,]),- [- "dist/assets/tree-sitter/tree-sitter-python.wasm",- "packages/core/assets/tree-sitter/tree-sitter-python.wasm",- ],+ ...grammars.map((name) => [+ `dist/assets/tree-sitter/tree-sitter-${name}.wasm`,+ `packages/core/assets/tree-sitter/tree-sitter-${name}.wasm`,+ ]),])if (!(await extract(packed)).equals(await readFile(resolve(root, original))))throw new Error(`Packaged ${packed} differs from its canonical source`);diff --git a/specs/go-rust-parsing.md b/specs/go-rust-parsing.mdnew file mode 100644index 0000000..e58b9e5--- /dev/null+++ b/specs/go-rust-parsing.md@@ -0,0 +1,47 @@+# Go/Rust declaration retrieval++Go and Rust use named source selection through the existing bounded parser+worker. Python and TypeScript behavior stays fixed; the Python prerequisite is+the official `psf__requests-1142` checkpoint against merged `ae10a9e`.++The useful contract is observable through `inspect` and installed search:+declaration names appear in large-file previews, selection can isolate a function,+and returned source preserves original UTF-8/CRLF bytes and declaration context.+This establishes structural retrieval capability, not measured model recall.++The pinned official grammars (`tree-sitter-go` 0.25.0 and `tree-sitter-rust`+0.24.0) supply syntax, following their upstream tag conventions. Grammar assets and licenses share+the existing build and release-validation owner. No new runtime download, native+compiler, worker pool, or cache is introduced.++`parser-declarations.mjs` owns Go/Rust syntax extraction; the worker owns parser+lifetime. Go declaration groups remain intact because constants can depend on+group order (`iota`). Rust modules, traits, impls, attributes and inner attributes+retain their enclosing headers. Generic receivers keep their source spelling;+this is neither type resolution nor macro expansion.++Verification: focused tests fail against text fallback and pass with named+declarations; installed tests verify declaration indexes and returned source;+malformed/oversized source keeps lossless fallback; missing grammars fail visibly.+Cold/warm cost and source-unit size are measured against `ae10a9e`; overhead+is reported without calling it a speedup. Full release gates and independent+review cover the integrated change.++References: [Go tags](https://github.com/tree-sitter/tree-sitter-go/blob/v0.25.0/queries/tags.scm)+and [Rust tags](https://github.com/tree-sitter/tree-sitter-rust/blob/v0.24.0/queries/tags.scm).++[Measured precision and parsing cost](go-rust-parsing/RESULTS.md) records the+tradeoff. Python query previews and inherited-call analysis remain Python-specific.++## Verification record — 2026-09-29++`bun run verify` passed 210 source checks and 41 installed-package scenarios.+Native Apple Silicon smoke passed all 11 journeys, including Go/Rust search.+Typechecks passed; lint retains the existing filesystem control-regex warning.+A clean checkout installed the locked grammar packages and validated the exact+archive, its canonical assets, license notices, and integrity.++Independent review's multi-name Go constant-label finding was reproduced and+fixed by excluding punctuation from identifier fields. Focused tests failed+before each extraction fix and passed afterward. Python parsing retains its+existing differential and cancellation coverage.diff --git a/specs/go-rust-parsing/RESULTS.md b/specs/go-rust-parsing/RESULTS.mdnew file mode 100644index 0000000..2911450--- /dev/null+++ b/specs/go-rust-parsing/RESULTS.md@@ -0,0 +1,36 @@+# Go/Rust parsing measurements++Three fresh-process trials of the final candidate on darwin/arm64 with Node+24.14.0 are compared with the unchanged `ae10a9e` baseline trials from+[development](development-measurements.json). Baseline trials were not rerun. Cold time+includes module import and first inspection; warm time is the median of ten+inspections, summarized across the three processes. No provider calls were made.++| Language | Functions | Cold before → after (ms) | Warm before → after (ms) | Target unit bytes before → after |+|---|---:|---:|---:|---:|+| go | 100 | 143.38 → 281.41 | 0.06 → 2.65 | 5,195 → 52 |+| go | 1,500 | 151.98 → 454.15 | 0.63 → 30.63 | 23,976 → 54 |+| rust | 100 | 148.12 → 275.91 | 0.06 → 2.42 | 5,180 → 52 |+| rust | 1,500 | 152.00 → 257.82 | 0.62 → 22.19 | 23,976 → 54 |++This is a deliberate precision/cost tradeoff. Parsing adds work and worker memory;+text fallback had no parser worker. The measured unit contains a known target+marker, not a model-selected final excerpt. Smaller units do not establish lower+end-to-end cost, better recall, or faster coding-agent tasks. Worker RSS snapshots+and source identities are in the [raw measurements](measurements.json).++The installed search fixture supplies deterministic relevance answers. It verifies+that late methods reach large-file declaration indexes, named selection returns+their source and Rust attributes, and distant noise is not returned. This is a+capability check, not a live-provider quality benchmark.++Reproduce each workload in prepared baseline/candidate checkouts using the same+[measurement script](bench.mjs), with `go` or `rust` and 100 or 1500 functions:++```sh+node --experimental-strip-types specs/go-rust-parsing/bench.mjs /absolute/checkout/packages/core/src/source.ts go 1500+```++Run each arm in alternating fresh processes. The candidate must first prepare its+pinned grammar assets through the CLI build. Further extraction fixes may change+timings; the recorded source hashes identify this measurement cohort.diff --git a/specs/go-rust-parsing/bench.mjs b/specs/go-rust-parsing/bench.mjsnew file mode 100644index 0000000..f4197e6--- /dev/null+++ b/specs/go-rust-parsing/bench.mjs@@ -0,0 +1,47 @@+import { ChildProcess, execFileSync } from "node:child_process";+const [module, language, countText] = process.argv.slice(2),+ count = Number(countText);+if (!module || !["go", "rust"].includes(language) || !Number.isSafeInteger(count) || count < 1)+ throw new Error("Use a source module path, go or rust, and a positive function count");+let child;+const emit = ChildProcess.prototype.emit;+ChildProcess.prototype.emit = function (event, ...args) {+ if (event === "spawn") child = this;+ return emit.call(this, event, ...args);+};+const source =+ (language === "go" ? "package sample\n" : "") ++ Array.from({ length: count }, (_, i) =>+ language === "go"+ ? `func target_${i}() string { return "marker_${i}_🙂" }\n`+ : `fn target_${i}() -> &'static str { "marker_${i}_🙂" }\n`,+ ).join("");+const snapshot = { source, path: language === "go" ? "x.go" : "x.rs", contentHash: "fixture" };+const t = performance.now();+const { inspect, sourceForUnit } = await import(module);+const result = await inspect(snapshot);+const cold = performance.now() - t;+const warm = [];+for (let i = 0; i < 10; i++) {+ const start = performance.now();+ await inspect({ ...snapshot });+ warm.push(performance.now() - start);+}+const marker = `"marker_${Math.floor(count / 2)}_🙂"`;+const unit = result.units.find((u) => sourceForUnit(snapshot, u).includes(marker));+if (!unit) throw Error("Target source missing");+console.log(+ JSON.stringify({+ language,+ count,+ cold_ms: cold,+ warm_ms: warm,+ units: result.units.length,+ target_unit_bytes: Buffer.byteLength(sourceForUnit(snapshot, unit)),+ worker_rss_kib: child+ ? Number(+ execFileSync("ps", ["-o", "rss=", "-p", String(child.pid)], { encoding: "utf8" }).trim(),+ )+ : null,+ }),+);diff --git a/specs/go-rust-parsing/development-measurements.json b/specs/go-rust-parsing/development-measurements.jsonnew file mode 100644index 0000000..90d3107--- /dev/null+++ b/specs/go-rust-parsing/development-measurements.json@@ -0,0 +1,540 @@+{+ "baseline": "ae10a9ee8ee55d5a262e04efc5e0ac41c6733a33",+ "node": "v24.14.0",+ "platform": "darwin/arm64",+ "records": [+ {+ "arm": "before",+ "trial": 0,+ "language": "go",+ "count": 100,+ "cold_ms": 137.24620800000002,+ "warm_ms": [+ 0.07979199999999764,+ 0.048082999999991216,+ 0.058458000000001675,+ 0.053665999999992664,+ 0.06212500000000887,+ 0.043708000000009406,+ 0.04200000000000159,+ 0.051124999999984766,+ 0.12204099999999585,+ 0.06295800000000895+ ],+ "units": 1,+ "target_unit_bytes": 5195,+ "worker_rss_kib": null+ },+ {+ "arm": "after",+ "trial": 0,+ "language": "go",+ "count": 100,+ "cold_ms": 184.870916,+ "warm_ms": [+ 2.373000000000019,+ 2.6346670000000074,+ 1.7156249999999886,+ 1.6431670000000054,+ 1.6285000000000025,+ 1.6205420000000004,+ 1.404042000000004,+ 1.348792000000003,+ 1.6571670000000154,+ 1.6779999999999973+ ],+ "units": 101,+ "target_unit_bytes": 52,+ "worker_rss_kib": 87856+ },+ {+ "arm": "before",+ "trial": 1,+ "language": "go",+ "count": 100,+ "cold_ms": 143.433708,+ "warm_ms": [+ 0.08291700000000901,+ 0.04862500000001546,+ 0.0675420000000031,+ 0.19645799999997848,+ 0.060291000000006534,+ 0.0625840000000153,+ 0.049000000000006594,+ 0.05349999999998545,+ 0.05025000000000546,+ 0.04816700000000651+ ],+ "units": 1,+ "target_unit_bytes": 5195,+ "worker_rss_kib": null+ },+ {+ "arm": "after",+ "trial": 1,+ "language": "go",+ "count": 100,+ "cold_ms": 189.528541,+ "warm_ms": [+ 2.2134169999999926,+ 2.1358749999999986,+ 1.5052919999999972,+ 1.5457499999999982,+ 1.372917000000001,+ 1.3282079999999894,+ 1.3761250000000018,+ 1.548374999999993,+ 1.4377920000000017,+ 1.3357919999999979+ ],+ "units": 101,+ "target_unit_bytes": 52,+ "worker_rss_kib": 86480+ },+ {+ "arm": "before",+ "trial": 2,+ "language": "go",+ "count": 100,+ "cold_ms": 143.38341599999998,+ "warm_ms": [+ 0.08362499999998363,+ 0.051124999999984766,+ 0.19695899999999256,+ 0.07433400000002166,+ 0.08033299999999599,+ 0.0503750000000025,+ 0.04466700000000401,+ 0.05883300000002123,+ 0.0449169999999981,+ 0.05287500000000023+ ],+ "units": 1,+ "target_unit_bytes": 5195,+ "worker_rss_kib": null+ },+ {+ "arm": "after",+ "trial": 2,+ "language": "go",+ "count": 100,+ "cold_ms": 188.95291699999999,+ "warm_ms": [+ 2.240167000000014,+ 2.1582500000000095,+ 1.712208000000004,+ 1.825082999999978,+ 1.5590419999999767,+ 1.4155839999999955,+ 1.3925419999999917,+ 1.5942499999999882,+ 1.4927909999999827,+ 1.381208000000015+ ],+ "units": 101,+ "target_unit_bytes": 52,+ "worker_rss_kib": 86128+ },+ {+ "arm": "before",+ "trial": 0,+ "language": "go",+ "count": 1500,+ "cold_ms": 151.97924999999998,+ "warm_ms": [+ 1.0316660000000013,+ 1.353749999999991,+ 0.626041999999984,+ 0.6341249999999832,+ 0.645875000000018,+ 0.5972919999999817,+ 0.5961250000000007,+ 2.3009580000000085,+ 0.6234579999999994,+ 0.597124999999977+ ],+ "units": 4,+ "target_unit_bytes": 23976,+ "worker_rss_kib": null+ },+ {+ "arm": "after",+ "trial": 0,+ "language": "go",+ "count": 1500,+ "cold_ms": 233.470833,+ "warm_ms": [+ 17.710791999999998,+ 16.610041999999964,+ 16.287041999999985,+ 16.11908299999999,+ 16.978707999999983,+ 16.572374999999965,+ 16.696500000000015,+ 16.670375000000035,+ 14.712041,+ 14.781457999999986+ ],+ "units": 1501,+ "target_unit_bytes": 54,+ "worker_rss_kib": 124112+ },+ {+ "arm": "before",+ "trial": 1,+ "language": "go",+ "count": 1500,+ "cold_ms": 147.158917,+ "warm_ms": [+ 0.5936670000000106,+ 1.227374999999995,+ 0.5569580000000087,+ 0.5495409999999765,+ 0.551667000000009,+ 0.5903329999999869,+ 2.2135419999999897,+ 0.5703330000000051,+ 0.49962500000000887,+ 0.5096670000000074+ ],+ "units": 4,+ "target_unit_bytes": 23976,+ "worker_rss_kib": null+ },+ {+ "arm": "after",+ "trial": 1,+ "language": "go",+ "count": 1500,+ "cold_ms": 218.675959,+ "warm_ms": [+ 16.304583000000008,+ 15.606334000000032,+ 15.568875000000048,+ 15.659583999999995,+ 15.790166,+ 14.82854199999997,+ 15.850458000000003,+ 15.488165999999978,+ 14.437000000000012,+ 15.014250000000004+ ],+ "units": 1501,+ "target_unit_bytes": 54,+ "worker_rss_kib": 125840+ },+ {+ "arm": "before",+ "trial": 2,+ "language": "go",+ "count": 1500,+ "cold_ms": 152.123584,+ "warm_ms": [+ 0.6299579999999878,+ 1.213624999999979,+ 0.6246669999999881,+ 0.6185420000000192,+ 0.6034999999999968,+ 0.5918329999999798,+ 0.6312499999999943,+ 0.7666659999999865,+ 2.126249999999999,+ 0.5161250000000166+ ],+ "units": 4,+ "target_unit_bytes": 23976,+ "worker_rss_kib": null+ },+ {+ "arm": "after",+ "trial": 2,+ "language": "go",+ "count": 1500,+ "cold_ms": 218.740959,+ "warm_ms": [+ 18.075458000000026,+ 16.563999999999993,+ 16.398249999999962,+ 16.785124999999994,+ 17.857124999999996,+ 17.722458000000017,+ 17.748582999999996,+ 17.488207999999986,+ 16.60895800000003,+ 17.034457999999972+ ],+ "units": 1501,+ "target_unit_bytes": 54,+ "worker_rss_kib": 124480+ },+ {+ "arm": "before",+ "trial": 0,+ "language": "rust",+ "count": 100,+ "cold_ms": 146.79166700000002,+ "warm_ms": [+ 0.08687499999999204,+ 0.0514169999999865,+ 0.06166600000000244,+ 0.059208999999981415,+ 0.06337500000000773,+ 0.18091699999999378,+ 0.1322499999999991,+ 0.09183299999997985,+ 0.0681250000000091,+ 0.057332999999999856+ ],+ "units": 1,+ "target_unit_bytes": 5180,+ "worker_rss_kib": null+ },+ {+ "arm": "after",+ "trial": 0,+ "language": "rust",+ "count": 100,+ "cold_ms": 194.910792,+ "warm_ms": [+ 2.7383749999999907,+ 2.433667000000014,+ 1.9113329999999848,+ 2.068708000000015,+ 1.7388750000000073,+ 2.1834169999999915,+ 1.7560410000000104,+ 1.7517080000000078,+ 1.883291999999983,+ 1.3318749999999966+ ],+ "units": 100,+ "target_unit_bytes": 52,+ "worker_rss_kib": 98288+ },+ {+ "arm": "before",+ "trial": 1,+ "language": "rust",+ "count": 100,+ "cold_ms": 149.737667,+ "warm_ms": [+ 0.0831670000000031,+ 0.05624999999997726,+ 0.06049999999999045,+ 0.058041000000002896,+ 0.05962500000001114,+ 0.04845799999998235,+ 0.04087499999999977,+ 0.06095899999999688,+ 0.04724999999999113,+ 0.043042000000014013+ ],+ "units": 1,+ "target_unit_bytes": 5180,+ "worker_rss_kib": null+ },+ {+ "arm": "after",+ "trial": 1,+ "language": "rust",+ "count": 100,+ "cold_ms": 191.64425,+ "warm_ms": [+ 2.760165999999998,+ 2.118957999999992,+ 1.9867909999999824,+ 2.2094999999999914,+ 1.7807500000000118,+ 1.9852500000000077,+ 1.86016699999999,+ 1.8732090000000028,+ 1.9823749999999905,+ 1.5636670000000095+ ],+ "units": 100,+ "target_unit_bytes": 52,+ "worker_rss_kib": 100016+ },+ {+ "arm": "before",+ "trial": 2,+ "language": "rust",+ "count": 100,+ "cold_ms": 148.11629100000002,+ "warm_ms": [+ 0.08866700000001515,+ 0.05441600000000335,+ 0.061916999999994005,+ 0.049915999999996075,+ 0.06829199999998536,+ 0.051832999999987805,+ 0.056250000000005684,+ 0.06733399999998824,+ 0.05629199999998491,+ 0.06070800000000531+ ],+ "units": 1,+ "target_unit_bytes": 5180,+ "worker_rss_kib": null+ },+ {+ "arm": "after",+ "trial": 2,+ "language": "rust",+ "count": 100,+ "cold_ms": 203.695833,+ "warm_ms": [+ 2.434833999999995,+ 2.351165999999978,+ 1.9988749999999982,+ 2.392791999999986,+ 1.8858749999999986,+ 1.9526250000000118,+ 2.0176669999999888,+ 1.7651250000000118,+ 2.0050830000000133,+ 1.7243749999999807+ ],+ "units": 100,+ "target_unit_bytes": 52,+ "worker_rss_kib": 96592+ },+ {+ "arm": "before",+ "trial": 0,+ "language": "rust",+ "count": 1500,+ "cold_ms": 149.60775,+ "warm_ms": [+ 0.5886250000000075,+ 1.167082999999991,+ 0.6224170000000129,+ 0.5444999999999993,+ 0.6081250000000011,+ 0.7984160000000031,+ 0.6195000000000164,+ 2.0960830000000215,+ 0.5019999999999811,+ 0.49229199999999196+ ],+ "units": 4,+ "target_unit_bytes": 23976,+ "worker_rss_kib": null+ },+ {+ "arm": "after",+ "trial": 0,+ "language": "rust",+ "count": 1500,+ "cold_ms": 230.225125,+ "warm_ms": [+ 23.293958000000003,+ 20.277667000000008,+ 19.939791999999954,+ 19.792250000000024,+ 19.433791999999983,+ 17.85012499999999,+ 18.151792,+ 20.731832999999995,+ 22.549916999999994,+ 21.633459000000016+ ],+ "units": 1500,+ "target_unit_bytes": 54,+ "worker_rss_kib": 137440+ },+ {+ "arm": "before",+ "trial": 1,+ "language": "rust",+ "count": 1500,+ "cold_ms": 157.93491699999998,+ "warm_ms": [+ 0.6371670000000051,+ 1.436000000000007,+ 0.6196659999999952,+ 0.6478749999999991,+ 0.6295000000000073,+ 0.5977090000000089,+ 0.5875409999999874,+ 2.2151250000000005,+ 0.5549579999999992,+ 0.5406670000000133+ ],+ "units": 4,+ "target_unit_bytes": 23976,+ "worker_rss_kib": null+ },+ {+ "arm": "after",+ "trial": 1,+ "language": "rust",+ "count": 1500,+ "cold_ms": 224.057708,+ "warm_ms": [+ 23.153624999999977,+ 19.902083000000005,+ 19.706542000000013,+ 19.600291000000027,+ 19.47666700000002,+ 19.28383299999996,+ 19.863500000000045,+ 20.620416999999975,+ 21.63929200000001,+ 20.715625000000045+ ],+ "units": 1500,+ "target_unit_bytes": 54,+ "worker_rss_kib": 136928+ },+ {+ "arm": "before",+ "trial": 2,+ "language": "rust",+ "count": 1500,+ "cold_ms": 152.00225,+ "warm_ms": [+ 0.5918340000000057,+ 1.2329170000000147,+ 0.6595829999999978,+ 0.6225419999999815,+ 0.6970829999999921,+ 0.6174579999999992,+ 2.4705419999999947,+ 0.6350419999999986,+ 0.5098330000000146,+ 0.5191670000000101+ ],+ "units": 4,+ "target_unit_bytes": 23976,+ "worker_rss_kib": null+ },+ {+ "arm": "after",+ "trial": 2,+ "language": "rust",+ "count": 1500,+ "cold_ms": 233.576708,+ "warm_ms": [+ 22.596708000000007,+ 21.197791999999993,+ 19.434916000000044,+ 18.358124999999973,+ 19.243042000000003,+ 18.87183399999998,+ 19.183208999999977,+ 19.496290999999985,+ 18.825042000000053,+ 17.756665999999996+ ],+ "units": 1500,+ "target_unit_bytes": 54,+ "worker_rss_kib": 139296+ }+ ],+ "source_sha256": {+ "packages/core/src/source.ts": "3d7b820dac964a903c3d6a7e7205e4353b51b301d5b78dab4139276364be112f",+ "packages/core/src/parser-helpers.mjs": "4d65d7f3d2bf2763c3e407b83f60174f4725be085eecc4461a55b9329a669ece",+ "packages/core/src/parser-declarations.mjs": "e70973ad70b420fba39dd443a1d5f568e90f26a9687ced7d0b2e9c16f206c9d3"+ }+}diff --git a/specs/go-rust-parsing/measurements.json b/specs/go-rust-parsing/measurements.jsonnew file mode 100644index 0000000..f476776--- /dev/null+++ b/specs/go-rust-parsing/measurements.json@@ -0,0 +1,541 @@+{+ "baseline": "ae10a9ee8ee55d5a262e04efc5e0ac41c6733a33",+ "node": "v24.14.0",+ "platform": "darwin/arm64",+ "records": [+ {+ "arm": "before",+ "trial": 0,+ "language": "go",+ "count": 100,+ "cold_ms": 137.24620800000002,+ "warm_ms": [+ 0.07979199999999764,+ 0.048082999999991216,+ 0.058458000000001675,+ 0.053665999999992664,+ 0.06212500000000887,+ 0.043708000000009406,+ 0.04200000000000159,+ 0.051124999999984766,+ 0.12204099999999585,+ 0.06295800000000895+ ],+ "units": 1,+ "target_unit_bytes": 5195,+ "worker_rss_kib": null+ },+ {+ "arm": "before",+ "trial": 1,+ "language": "go",+ "count": 100,+ "cold_ms": 143.433708,+ "warm_ms": [+ 0.08291700000000901,+ 0.04862500000001546,+ 0.0675420000000031,+ 0.19645799999997848,+ 0.060291000000006534,+ 0.0625840000000153,+ 0.049000000000006594,+ 0.05349999999998545,+ 0.05025000000000546,+ 0.04816700000000651+ ],+ "units": 1,+ "target_unit_bytes": 5195,+ "worker_rss_kib": null+ },+ {+ "arm": "before",+ "trial": 2,+ "language": "go",+ "count": 100,+ "cold_ms": 143.38341599999998,+ "warm_ms": [+ 0.08362499999998363,+ 0.051124999999984766,+ 0.19695899999999256,+ 0.07433400000002166,+ 0.08033299999999599,+ 0.0503750000000025,+ 0.04466700000000401,+ 0.05883300000002123,+ 0.0449169999999981,+ 0.05287500000000023+ ],+ "units": 1,+ "target_unit_bytes": 5195,+ "worker_rss_kib": null+ },+ {+ "arm": "before",+ "trial": 0,+ "language": "go",+ "count": 1500,+ "cold_ms": 151.97924999999998,+ "warm_ms": [+ 1.0316660000000013,+ 1.353749999999991,+ 0.626041999999984,+ 0.6341249999999832,+ 0.645875000000018,+ 0.5972919999999817,+ 0.5961250000000007,+ 2.3009580000000085,+ 0.6234579999999994,+ 0.597124999999977+ ],+ "units": 4,+ "target_unit_bytes": 23976,+ "worker_rss_kib": null+ },+ {+ "arm": "before",+ "trial": 1,+ "language": "go",+ "count": 1500,+ "cold_ms": 147.158917,+ "warm_ms": [+ 0.5936670000000106,+ 1.227374999999995,+ 0.5569580000000087,+ 0.5495409999999765,+ 0.551667000000009,+ 0.5903329999999869,+ 2.2135419999999897,+ 0.5703330000000051,+ 0.49962500000000887,+ 0.5096670000000074+ ],+ "units": 4,+ "target_unit_bytes": 23976,+ "worker_rss_kib": null+ },+ {+ "arm": "before",+ "trial": 2,+ "language": "go",+ "count": 1500,+ "cold_ms": 152.123584,+ "warm_ms": [+ 0.6299579999999878,+ 1.213624999999979,+ 0.6246669999999881,+ 0.6185420000000192,+ 0.6034999999999968,+ 0.5918329999999798,+ 0.6312499999999943,+ 0.7666659999999865,+ 2.126249999999999,+ 0.5161250000000166+ ],+ "units": 4,+ "target_unit_bytes": 23976,+ "worker_rss_kib": null+ },+ {+ "arm": "before",+ "trial": 0,+ "language": "rust",+ "count": 100,+ "cold_ms": 146.79166700000002,+ "warm_ms": [+ 0.08687499999999204,+ 0.0514169999999865,+ 0.06166600000000244,+ 0.059208999999981415,+ 0.06337500000000773,+ 0.18091699999999378,+ 0.1322499999999991,+ 0.09183299999997985,+ 0.0681250000000091,+ 0.057332999999999856+ ],+ "units": 1,+ "target_unit_bytes": 5180,+ "worker_rss_kib": null+ },+ {+ "arm": "before",+ "trial": 1,+ "language": "rust",+ "count": 100,+ "cold_ms": 149.737667,+ "warm_ms": [+ 0.0831670000000031,+ 0.05624999999997726,+ 0.06049999999999045,+ 0.058041000000002896,+ 0.05962500000001114,+ 0.04845799999998235,+ 0.04087499999999977,+ 0.06095899999999688,+ 0.04724999999999113,+ 0.043042000000014013+ ],+ "units": 1,+ "target_unit_bytes": 5180,+ "worker_rss_kib": null+ },+ {+ "arm": "before",+ "trial": 2,+ "language": "rust",+ "count": 100,+ "cold_ms": 148.11629100000002,+ "warm_ms": [+ 0.08866700000001515,+ 0.05441600000000335,+ 0.061916999999994005,+ 0.049915999999996075,+ 0.06829199999998536,+ 0.051832999999987805,+ 0.056250000000005684,+ 0.06733399999998824,+ 0.05629199999998491,+ 0.06070800000000531+ ],+ "units": 1,+ "target_unit_bytes": 5180,+ "worker_rss_kib": null+ },+ {+ "arm": "before",+ "trial": 0,+ "language": "rust",+ "count": 1500,+ "cold_ms": 149.60775,+ "warm_ms": [+ 0.5886250000000075,+ 1.167082999999991,+ 0.6224170000000129,+ 0.5444999999999993,+ 0.6081250000000011,+ 0.7984160000000031,+ 0.6195000000000164,+ 2.0960830000000215,+ 0.5019999999999811,+ 0.49229199999999196+ ],+ "units": 4,+ "target_unit_bytes": 23976,+ "worker_rss_kib": null+ },+ {+ "arm": "before",+ "trial": 1,+ "language": "rust",+ "count": 1500,+ "cold_ms": 157.93491699999998,+ "warm_ms": [+ 0.6371670000000051,+ 1.436000000000007,+ 0.6196659999999952,+ 0.6478749999999991,+ 0.6295000000000073,+ 0.5977090000000089,+ 0.5875409999999874,+ 2.2151250000000005,+ 0.5549579999999992,+ 0.5406670000000133+ ],+ "units": 4,+ "target_unit_bytes": 23976,+ "worker_rss_kib": null+ },+ {+ "arm": "before",+ "trial": 2,+ "language": "rust",+ "count": 1500,+ "cold_ms": 152.00225,+ "warm_ms": [+ 0.5918340000000057,+ 1.2329170000000147,+ 0.6595829999999978,+ 0.6225419999999815,+ 0.6970829999999921,+ 0.6174579999999992,+ 2.4705419999999947,+ 0.6350419999999986,+ 0.5098330000000146,+ 0.5191670000000101+ ],+ "units": 4,+ "target_unit_bytes": 23976,+ "worker_rss_kib": null+ },+ {+ "arm": "after",+ "trial": 0,+ "language": "go",+ "count": 100,+ "cold_ms": 281.413959,+ "warm_ms": [+ 3.7676250000000095,+ 3.538333000000023,+ 2.3937080000000037,+ 2.3475839999999835,+ 2.265209000000027,+ 2.2827910000000315,+ 1.7748750000000086,+ 1.6493750000000205,+ 2.4490410000000224,+ 1.9844590000000153+ ],+ "units": 101,+ "target_unit_bytes": 52,+ "worker_rss_kib": 87552+ },+ {+ "arm": "after",+ "trial": 1,+ "language": "go",+ "count": 100,+ "cold_ms": 274.43008299999997,+ "warm_ms": [+ 3.634416999999985,+ 3.4551660000000197,+ 2.8712919999999826,+ 2.983542,+ 2.658042000000023,+ 2.306041999999991,+ 2.636665999999991,+ 2.04200000000003,+ 2.4957919999999945,+ 2.1512500000000045+ ],+ "units": 101,+ "target_unit_bytes": 52,+ "worker_rss_kib": 88176+ },+ {+ "arm": "after",+ "trial": 2,+ "language": "go",+ "count": 100,+ "cold_ms": 298.107375,+ "warm_ms": [+ 4.054875000000038,+ 3.8750839999999585,+ 2.976875000000007,+ 2.90137500000003,+ 7.425334000000021,+ 2.4610419999999635,+ 2.4359159999999633,+ 2.226165999999978,+ 2.8776659999999765,+ 1.939833999999962+ ],+ "units": 101,+ "target_unit_bytes": 52,+ "worker_rss_kib": 87904+ },+ {+ "arm": "after",+ "trial": 0,+ "language": "go",+ "count": 1500,+ "cold_ms": 356.514125,+ "warm_ms": [+ 28.278250000000014,+ 27.882791999999995,+ 26.994541999999967,+ 27.660124999999994,+ 29.289207999999974,+ 26.81933300000003,+ 27.31016699999998,+ 30.04458299999999,+ 26.755333000000064,+ 26.54733299999998+ ],+ "units": 1501,+ "target_unit_bytes": 54,+ "worker_rss_kib": 126960+ },+ {+ "arm": "after",+ "trial": 1,+ "language": "go",+ "count": 1500,+ "cold_ms": 576.862917,+ "warm_ms": [+ 58.58487500000001,+ 48.688290999999936,+ 53.28912500000001,+ 55.83275000000003,+ 50.115125000000035,+ 49.88945799999999,+ 40.7941669999999,+ 40.29383399999995,+ 37.68033400000013,+ 44.40812499999993+ ],+ "units": 1501,+ "target_unit_bytes": 54,+ "worker_rss_kib": 132528+ },+ {+ "arm": "after",+ "trial": 2,+ "language": "go",+ "count": 1500,+ "cold_ms": 454.14966699999997,+ "warm_ms": [+ 35.27245799999997,+ 31.073709000000008,+ 28.73512500000004,+ 27.060791999999992,+ 33.53875000000005,+ 30.777041000000054,+ 30.48791600000004,+ 31.089208999999983,+ 27.006000000000085,+ 26.838499999999954+ ],+ "units": 1501,+ "target_unit_bytes": 54,+ "worker_rss_kib": 124640+ },+ {+ "arm": "after",+ "trial": 0,+ "language": "rust",+ "count": 100,+ "cold_ms": 313.84383299999996,+ "warm_ms": [+ 4.176040999999998,+ 3.3393749999999613,+ 2.789416000000017,+ 3.03612499999997,+ 2.5884169999999926,+ 2.784249999999986,+ 2.527084000000002,+ 2.575791000000038,+ 2.6677500000000123,+ 2.425374999999974+ ],+ "units": 100,+ "target_unit_bytes": 52,+ "worker_rss_kib": 98176+ },+ {+ "arm": "after",+ "trial": 1,+ "language": "rust",+ "count": 100,+ "cold_ms": 275.91083299999997,+ "warm_ms": [+ 3.591542000000004,+ 2.9383339999999976,+ 2.3992920000000026,+ 2.5857500000000186,+ 5.5227500000000305,+ 2.214916999999957,+ 2.2708339999999794,+ 2.4445420000000126,+ 2.0651659999999765,+ 1.6691249999999513+ ],+ "units": 100,+ "target_unit_bytes": 52,+ "worker_rss_kib": 97824+ },+ {+ "arm": "after",+ "trial": 2,+ "language": "rust",+ "count": 100,+ "cold_ms": 248.10275,+ "warm_ms": [+ 3.0264999999999986,+ 4.40304100000003,+ 2.2428330000000187,+ 2.676125000000013,+ 2.415916999999979,+ 2.4104580000000055,+ 1.9123329999999896,+ 1.8734999999999786,+ 1.9861659999999688,+ 1.890874999999994+ ],+ "units": 100,+ "target_unit_bytes": 52,+ "worker_rss_kib": 97008+ },+ {+ "arm": "after",+ "trial": 0,+ "language": "rust",+ "count": 1500,+ "cold_ms": 286.304666,+ "warm_ms": [+ 27.033957999999984,+ 25.34395900000004,+ 23.853083999999967,+ 23.628750000000025,+ 23.65779200000003,+ 23.839292,+ 24.00958399999996,+ 25.172166000000004,+ 25.879500000000007,+ 22.61445900000001+ ],+ "units": 1500,+ "target_unit_bytes": 54,+ "worker_rss_kib": 136528+ },+ {+ "arm": "after",+ "trial": 1,+ "language": "rust",+ "count": 1500,+ "cold_ms": 257.817042,+ "warm_ms": [+ 26.579834000000005,+ 22.514583000000016,+ 21.54216699999995,+ 21.478083000000026,+ 22.332291999999995,+ 22.055042000000014,+ 21.191667000000052,+ 22.786292000000003,+ 23.906542,+ 21.541999999999973+ ],+ "units": 1500,+ "target_unit_bytes": 54,+ "worker_rss_kib": 135392+ },+ {+ "arm": "after",+ "trial": 2,+ "language": "rust",+ "count": 1500,+ "cold_ms": 234.315,+ "warm_ms": [+ 22.147875,+ 18.847125000000005,+ 18.749833000000024,+ 18.745709000000033,+ 18.414999999999964,+ 18.300959000000034,+ 18.435000000000002,+ 20.005833999999993,+ 20.14783299999999,+ 18.70795800000002+ ],+ "units": 1500,+ "target_unit_bytes": 54,+ "worker_rss_kib": 136384+ }+ ],+ "source_sha256": {+ "packages/core/src/source.ts": "3d7b820dac964a903c3d6a7e7205e4353b51b301d5b78dab4139276364be112f",+ "packages/core/src/parser-helpers.mjs": "4d65d7f3d2bf2763c3e407b83f60174f4725be085eecc4461a55b9329a669ece",+ "packages/core/src/parser-declarations.mjs": "aa7f67d9030b4c5cf5ab5d43e9a2457da02e61f2b225039a85d7ce68a2d7b830"+ },+ "comparison": "Final candidate confirmation; the three baseline trials per workload are reused unchanged from development-measurements.json."+}diff --git a/test/installed.test.mjs b/test/installed.test.mjsindex 0756abd..b2cb828 100644--- a/test/installed.test.mjs+++ b/test/installed.test.mjs@@ -467,6 +467,73 @@ test("skill command delegates installation to npx without credentials", async (tassert.match(unavailable.stdout, /requires npx/);});+test("installed Go/Rust search indexes late methods and selects their source instead of distant noise", async (t) => {+ const fixture = await context(t, async ({ body, response }) => {+ const answers = Object.fromEntries(+ Object.keys(body.questions).map((id) => {+ const match = /^(q|scope|ref)(\d+)$/.exec(id);+ const selected =+ body.state.declarations && match+ ? match[1] === "scope" ||+ (match[1] === "q" &&+ body.state.declarations[Number(match[2])].name.endsWith(".record_event"))+ : !id.startsWith("ref");+ return [id, { type: "noul", noul: selected ? 0.95 : 0.05 }];+ }),+ );+ response.writeHead(200, { "content-type": "application/json" });+ response.end(JSON.stringify({ answers }));+ return true;+ });+ const files = [+ [+ "sample.go",+ "package sample\n" ++ Array.from(+ { length: 160 },+ (_, i) => `func noise${i}() string { return "DISTANT_NOISE_${i}_${"x".repeat(100)}" }\n`,+ ).join("") ++ 'type Box struct {}\nfunc (b *Box) record_event() string { return "go-evidence" }\n',+ "go-evidence",+ ],+ [+ "sample.rs",+ Array.from(+ { length: 160 },+ (_, i) => `fn noise${i}() -> &'static str { "DISTANT_NOISE_${i}_${"x".repeat(100)}" }\n`,+ ).join("") ++ 'struct Box;\n#[allow(dead_code)]\nimpl Box {\n #[inline]\n pub fn record_event(&self) -> &str { "rust-evidence" }\n}\n',+ "rust-evidence",+ ],+ ];+ for (const [path, source] of files) await writeFile(join(fixture.tree, path), source);+ const result = await fixture.run([query, fixture.tree, "--no-cache"]);+ assert.equal(result.code, 0, result.stdout);+ for (const [path, , marker] of files) {+ assert.ok(+ fixture.requests.some(+ ({ body }) =>+ body.state.path === path &&+ body.state.preview?.truncated &&+ body.state.preview.declarations?.some((d) => d.name === "Box.record_event"),+ ),+ `Missing late-method preview for ${path}`,+ );+ assert.ok(+ fixture.requests.some(+ ({ body }) =>+ body.state.path === path &&+ body.state.declarations?.some((d) => d.name === "Box.record_event"),+ ),+ `Missing named selection for ${path}`,+ );+ assert.ok(result.stdout.includes(marker), result.stdout);+ }+ assert.ok(result.stdout.includes("#[allow(dead_code)]"), result.stdout);+ assert.ok(result.stdout.includes("#[inline]"), result.stdout);+ assert.ok(!result.stdout.includes("DISTANT_NOISE_0_"), result.stdout);+});+test("actual installed search parses Python and returns every relevant hierarchy branch", async (t) => {const fixture = await context(t);const result = await fixture.run([query, fixture.tree, "--no-cache"]);@@ -991,23 +1058,51 @@ test("source budget preserves every file and lead while explicitly omitting sourtest("missing or corrupt packaged parser assets fail closed without downloads", async (t) => {const scratch = await mkdtemp(join(tmpdir(), "jg-missing-python-"));t.after(() => rm(scratch, { recursive: true, force: true }));- for (const [asset, corrupt] of [+ for (const [asset, corrupt, extension] of [["dist/bin/parser-worker.mjs", false],["dist/assets/tree-sitter/tree-sitter-python.wasm", false],["node_modules/web-tree-sitter/web-tree-sitter.wasm", false],["dist/assets/tree-sitter/tree-sitter-python.wasm", true],+ ["dist/assets/tree-sitter/tree-sitter-go.wasm", false, "go"],+ ["dist/assets/tree-sitter/tree-sitter-go.wasm", true, "go"],+ ["dist/assets/tree-sitter/tree-sitter-rust.wasm", false, "rs"],+ ["dist/assets/tree-sitter/tree-sitter-rust.wasm", true, "rs"],]) {const copy = join(scratch, "package");await cp(packageDirectory, copy, { recursive: true, dereference: true });await access(join(copy, asset));if (corrupt) await writeFile(join(copy, asset), "corrupt runtime fixture");else await rm(join(copy, asset));- const fixture = await context(t, "healthy", join(copy, "dist/bin/index.js"));+ const fixture = await context(+ t,+ extension+ ? async ({ body, response }) => {+ response.writeHead(200, { "content-type": "application/json" });+ response.end(+ JSON.stringify({+ answers: Object.fromEntries(+ Object.keys(body.questions).map((id) => [id, { type: "noul", noul: 0.95 }]),+ ),+ }),+ );+ return true;+ }+ : "healthy",+ join(copy, "dist/bin/index.js"),+ );+ if (extension)+ await writeFile(+ join(fixture.tree, `broken.${extension}`),+ extension === "go" ? "package sample\nfunc record_event() {}\n" : "fn record_event() {}\n",+ );const result = await fixture.run([query, fixture.tree, "--no-cache"]);assert.equal(result.code, 1, `${asset} (corrupt=${corrupt}): ${result.stdout}`);assert.ok(result.stdout.trim(), "Asset failure must produce a diagnostic");assert.ok(- !fixture.requests.some(({ body }) => body.state.declarations),+ !fixture.requests.some(+ ({ body }) =>+ body.state.declarations && (!extension || body.state.path === `broken.${extension}`),+ ),"Unavailable parser assets must not fabricate declaration evidence",);await fixture.removeCredentials();diff --git a/test/parser/README.md b/test/parser/README.mdindex 0a2f835..d52e1dc 100644--- a/test/parser/README.md+++ b/test/parser/README.md@@ -23,3 +23,7 @@ misrepresented as malformed user source.Inherited-call tests preserve useful reading leads when a receiver or callee isparenthesized, or a selected excerpt contains only a multiline call’s openingline. They assert returned locations rather than the grammar’s node shape.++Go/Rust tests exercise named selection, declaration groups, generic receivers,+Rust owner/attribute context, and byte-preserving fallback. Installed coverage+checks late-method previews, exact selected source, and missing/corrupt grammars.diff --git a/test/parser/languages.test.ts b/test/parser/languages.test.tsnew file mode 100644index 0000000..96d7365--- /dev/null+++ b/test/parser/languages.test.ts@@ -0,0 +1,103 @@+import { test } from "node:test";+import assert from "node:assert/strict";+import { inspect, sourceForUnit } from "../../packages/core/src/source.ts";++test("Go selects named methods and preserves declaration groups and Unicode bytes", async () => {+ const source =+ "package sample\r\nconst (\r\n First = iota\r\n Second\r\n)\r\ntype Box[T any] struct { value T }\r\nfunc (b *Box[T]) Read() T {\r\n // 🙂\r\n return b.value\r\n}\r\n";+ const snapshot = { path: "sample.go", source, contentHash: "fixture" };+ const result = await inspect(snapshot);+ assert.equal(result.mode, "go");+ const method = result.units.find((unit) => unit.name === "Box[T].Read");+ assert.ok(method);+ assert.equal(sourceForUnit(snapshot, method), source.split("\r\n").slice(6).join("\r\n"));+ const group = result.units.find((unit) => unit.name === "First, Second");+ assert.ok(group);+ assert.equal(sourceForUnit(snapshot, group), "const (\r\n First = iota\r\n Second\r\n)\r\n");+});++test("Rust methods retain module, impl, attribute and original source context", async () => {+ const source =+ '#![allow(dead_code)]\r\n#[cfg(feature = "demo")]\r\nmod inner {\r\n #![allow(unused)]\r\n struct Box;\r\n #[cfg(feature = "demo")]\r\n impl Box {\r\n /// Reads the value.\r\n #[inline]\r\n pub fn read(&self) -> &str { "🙂" }\r\n }\r\n}\r\n';+ const snapshot = { path: "sample.rs", source, contentHash: "fixture" };+ const result = await inspect(snapshot);+ assert.equal(result.mode, "rust");+ const method = result.units.find((unit) => unit.name === "inner.Box.read");+ assert.ok(method);+ assert.deepEqual(method.ownerHeaders, [+ { startLine: 1, endLine: 1 },+ { startLine: 2, endLine: 3 },+ { startLine: 4, endLine: 4 },+ { startLine: 6, endLine: 7 },+ ]);+ assert.equal(+ sourceForUnit(snapshot, method),+ ' /// Reads the value.\r\n #[inline]\r\n pub fn read(&self) -> &str { "🙂" }\r\n',+ );+});++test("Rust comments between attributes and declarations retain the attributes", async () => {+ const source =+ '#[cfg(feature = "demo")]\n// builds only with demo\nmod inner {\n #[inline]\n // useful method\n pub fn read() {}\n}\n';+ const snapshot = { path: "sample.rs", source, contentHash: "fixture" };+ const result = await inspect(snapshot);+ const method = result.units.find((unit) => unit.name === "inner.read");+ assert.ok(method);+ assert.deepEqual(method.ownerHeaders, [{ startLine: 1, endLine: 3 }]);+ assert.equal(+ sourceForUnit(snapshot, method),+ " #[inline]\n // useful method\n pub fn read() {}\n",+ );+});++test("Rust attributes do not attach the previous declaration's trailing comment", async () => {+ const source =+ "fn earlier() {} // about earlier\n#[inline]\n// about target\npub fn target() {}\n";+ const snapshot = { path: "sample.rs", source, contentHash: "fixture" };+ const result = await inspect(snapshot);+ const target = result.units.find((unit) => unit.name === "target");+ assert.ok(target);+ assert.equal(sourceForUnit(snapshot, target), "#[inline]\n// about target\npub fn target() {}\n");+});++test("declaration labels cover grouped Go names and Rust traits, foreign items and macros", async () => {+ for (const [path, source, names] of [+ [+ "sample.go",+ "package x\nconst Alpha, Beta = 1, 2\ntype (\n Alias = string\n Value struct{}\n)\nvar First, Second int\nvar (\n Third int\n Fourth = 1\n)\nfunc K() {}\nfunc K() {}\n",+ ["Alpha, Beta", "Alias, Value", "First, Second", "Third, Fourth", "K", "K"],+ ],+ [+ "sample.rs",+ 'mod inner {\n pub trait View {\n fn show(&self);\n }\n}\nextern "C" {\n fn foreign();\n static VALUE: u8;\n}\nmacro_rules! make { () => {} }\nfn K() {}\nfn K() {}\n',+ ["inner.View.show", "foreign", "VALUE", "make", "K", "K"],+ ],+ ] as const) {+ const result = await inspect({ path, source, contentHash: "fixture" });+ for (const name of names)+ assert.ok(+ result.units.some((unit) => unit.name === name),+ name,+ );+ }+});++test("Go/Rust syntax and size fallbacks preserve every Unicode source byte", async () => {+ for (const [path, source, options, reason] of [+ ["bad.go", "package x\nfunc broken( { 🙂\n", { maxUnitBytes: 8 }, "syntax"],+ ["bad.rs", "fn broken( { 🙂\n", { maxUnitBytes: 8 }, "syntax"],+ [+ "big.go",+ 'package x\nfunc target() string { return "🙂" }\r\n',+ { maxParseBytes: 1, maxUnitBytes: 8 },+ "size",+ ],+ ["big.rs", 'fn target() -> &str { "🙂" }\r\n', { maxParseBytes: 1, maxUnitBytes: 8 }, "size"],+ ] as const) {+ const snapshot = { path, source, contentHash: "fixture" };+ const result = await inspect(snapshot, options);+ assert.equal(result.fallback, reason);+ assert.equal(result.units.map((unit) => sourceForUnit(snapshot, unit)).join(""), source);+ assert.ok(result.units.every((unit) => Buffer.byteLength(sourceForUnit(snapshot, unit)) <= 8));+ }+});diff --git a/test/release.test.mjs b/test/release.test.mjsindex 6892ae0..cc898d2 100644--- a/test/release.test.mjs+++ b/test/release.test.mjs@@ -89,21 +89,26 @@ test("archive validation rejects changed skill bytes and accidental source payloLICENSE: "MIT canonical copyright\n","skills/jevgrep/SKILL.md": "Use jg for unfamiliar code.\n",...Object.fromEntries(- ["parser-worker", "parser-helpers", "parser-preview"].map((name) => [+ ["parser-worker", "parser-helpers", "parser-preview", "parser-declarations"].map((name) => [`packages/core/src/${name}.mjs`,"// fixture",]),),"packages/core/package.json": JSON.stringify({- devDependencies: { "tree-sitter-python": "1.0.0" },+ devDependencies: Object.fromEntries(+ ["python", "go", "rust"].map((name) => [`tree-sitter-${name}`, "1.0.0"]),+ ),}),- "packages/core/node_modules/tree-sitter-python/package.json": JSON.stringify({- name: "tree-sitter-python",- version: "1.0.0",- license: "MIT",- }),- "packages/core/node_modules/tree-sitter-python/LICENSE": "MIT fixture grammar license",- "packages/core/assets/tree-sitter/tree-sitter-python.wasm": "Python grammar fixture",+ ...Object.fromEntries(+ ["python", "go", "rust"].flatMap((name) => [+ [+ `packages/core/node_modules/tree-sitter-${name}/package.json`,+ JSON.stringify({ name: `tree-sitter-${name}`, version: "1.0.0", license: "MIT" }),+ ],+ [`packages/core/node_modules/tree-sitter-${name}/LICENSE`, "MIT fixture grammar license"],+ [`packages/core/assets/tree-sitter/tree-sitter-${name}.wasm`, `${name} grammar fixture`],+ ]),+ ),};for (const [path, text] of Object.entries(files)) {await mkdir(join(root, path, ".."), { recursive: true });@@ -118,14 +123,14 @@ test("archive validation rejects changed skill bytes and accidental source paylo["apps/cli/package.json", "package.json"],["LICENSE", "dist/LICENSE"],["skills/jevgrep/SKILL.md", "dist/skills/jevgrep/SKILL.md"],- ...["parser-worker", "parser-helpers", "parser-preview"].map((name) => [+ ...["parser-worker", "parser-helpers", "parser-preview", "parser-declarations"].map((name) => [`packages/core/src/${name}.mjs`,`dist/bin/${name}.mjs`,]),- [- "packages/core/assets/tree-sitter/tree-sitter-python.wasm",- "dist/assets/tree-sitter/tree-sitter-python.wasm",- ],+ ...["python", "go", "rust"].map((name) => [+ `packages/core/assets/tree-sitter/tree-sitter-${name}.wasm`,+ `dist/assets/tree-sitter/tree-sitter-${name}.wasm`,+ ]),])await cp(join(root, from), join(pkg, to));await writeFile(join(pkg, "dist/bin/index.js"), '#!/usr/bin/env node\nconsole.log("jg");\n');@@ -169,9 +174,9 @@ test("archive validation rejects changed skill bytes and accidental source payloawait assert.rejects(validateRelease(tarball, "v1.2.3", root), /Unexpected published file/);});-test("grammar notices retain the official Python parser license", async () => {+test("grammar notices retain all official parser licenses", async () => {const { grammarAssets } = await import("../scripts/parser-assets.mjs");const notices = await grammarAssets();- assert.match(notices, /tree-sitter-python@/);+ for (const name of ["python", "go", "rust"]) assert.ok(notices.includes(`tree-sitter-${name}@`));assert.match(notices, /Permission is hereby granted/);});