Skip to content

Find code by asking what it does. A CLI for coding agents that uses Jev to discover relevant files and source context.

Add named Go and Rust declaration retrieval (#35)

David Zhang82ef1fd3f431
diff --git a/README.md b/README.md
index 583e448..4046fa3 100644
--- a/README.md
+++ b/README.md
@@ -88,7 +88,7 @@ confidently return an excerpt; it does not force every search into a fixed top-t
list.
The summary and compact file list come first, followed by selected source with
-line references, then detailed declaration and call locations. Python and TypeScript/JavaScript support declaration
+line references, then detailed declaration and call locations. Python, TypeScript/JavaScript, Go and Rust support declaration
parsing; other text uses a fallback. The output is evidence for the agent to use,
not a generated answer or a guarantee that every relevant file was found.
[See a recorded output example](specs/done/jevgrep/assets/stdout-example.txt).
diff --git a/apps/cli/package.json b/apps/cli/package.json
index 0e8e8a3..c1bfe3d 100644
--- a/apps/cli/package.json
+++ b/apps/cli/package.json
@@ -1,6 +1,6 @@
{
"name": "@dzhng/jevgrep",
- "version": "0.6.0",
+ "version": "0.7.0",
"description": "Content-aware source retrieval for coding agents",
"license": "MIT",
"repository": {
diff --git a/apps/cli/test/package.test.ts b/apps/cli/test/package.test.ts
index d361770..58b6334 100644
--- a/apps/cli/test/package.test.ts
+++ b/apps/cli/test/package.test.ts
@@ -37,7 +37,8 @@ testInDocker(
const filenames = packed.files.map((file: { path: string }) => file.path);
expect(filenames).toContain("dist/bin/index.js");
expect(filenames).toContain("dist/bin/parser-worker.mjs");
- expect(filenames).toContain("dist/assets/tree-sitter/tree-sitter-python.wasm");
+ for (const name of ["python", "go", "rust"])
+ expect(filenames).toContain(`dist/assets/tree-sitter/tree-sitter-${name}.wasm`);
expect(filenames).toContain("dist/THIRD_PARTY_NOTICES.txt");
expect(filenames).toContain("dist/skills/jevgrep/SKILL.md");
expect(filenames.some((path: string) => /^(evals|src|node_modules|test)\//.test(path))).toBe(
diff --git a/bun.lock b/bun.lock
index b200cb9..f50be71 100644
--- a/bun.lock
+++ b/bun.lock
@@ -13,7 +13,7 @@
},
"apps/cli": {
"name": "@dzhng/jevgrep",
- "version": "0.6.0",
+ "version": "0.7.0",
"bin": {
"jg": "./dist/bin/index.js",
},
@@ -42,7 +42,9 @@
"devDependencies": {
"@repo/typescript-config": "workspace:*",
"@types/node": "22",
+ "tree-sitter-go": "0.25.0",
"tree-sitter-python": "0.25.0",
+ "tree-sitter-rust": "0.24.0",
},
},
"packages/typescript-config": {
@@ -199,8 +201,12 @@
"tinypool": ["[email protected]", "", {}, "sha512-Pugqs6M0m7Lv1I7FtxN4aoyToKg1C4tu+/381vH35y8oENM/Ai7f7C4StcoK4/+BSw9ebcS8jRiVrORFKCALLw=="],
+ "tree-sitter-go": ["[email protected]", "", { "dependencies": { "node-addon-api": "^8.3.1", "node-gyp-build": "^4.8.4" }, "peerDependencies": { "tree-sitter": "^0.25.0" }, "optionalPeers": ["tree-sitter"] }, "sha512-APBc/Dq3xz/e35Xpkhb1blu5UgW+2E3RyGWawZSCNcbGwa7jhSQPS8KsUupuzBla8PCo8+lz9W/JDJjmfRa2tw=="],
+
"tree-sitter-python": ["[email protected]", "", { "dependencies": { "node-addon-api": "^8.5.0", "node-gyp-build": "^4.8.4" }, "peerDependencies": { "tree-sitter": "^0.25.0" }, "optionalPeers": ["tree-sitter"] }, "sha512-eCmJx6zQa35GxaCtQD+wXHOhYqBxEL+bp71W/s3fcDMu06MrtzkVXR437dRrCrbrDbyLuUDJpAgycs7ncngLXw=="],
+ "tree-sitter-rust": ["[email protected]", "", { "dependencies": { "node-addon-api": "^8.2.2", "node-gyp-build": "^4.8.4" }, "peerDependencies": { "tree-sitter": "^0.22.1" }, "optionalPeers": ["tree-sitter"] }, "sha512-NWemUDf629Tfc90Y0Z55zuwPCAHkLxWnMf2RznYu4iBkkrQl2o/CHGB7Cr52TyN5F1DAx8FmUnDtCy9iUkXZEQ=="],
+
"turbo": ["[email protected]", "", { "optionalDependencies": { "@turbo/darwin-64": "2.10.3", "@turbo/darwin-arm64": "2.10.3", "@turbo/linux-64": "2.10.3", "@turbo/linux-arm64": "2.10.3", "@turbo/windows-64": "2.10.3", "@turbo/windows-arm64": "2.10.3" }, "bin": { "turbo": "bin/turbo" } }, "sha512-uZIqzfgtWbyqu1Tqwdd0giRnPGgL0ejbcdF8eZLJhtTTVSrOgArZGzYeXTD6XNcc7ANgdhqex11Y9bHBeyiQLQ=="],
"typescript": ["[email protected]", "", { "bin": { "tsc": "bin/tsc", "tsserver": "bin/tsserver" } }, "sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw=="],
diff --git a/docs/architecture.md b/docs/architecture.md
index c9f37bf..2194f94 100644
--- a/docs/architecture.md
+++ b/docs/architecture.md
@@ -26,19 +26,24 @@ concurrency and token-aware admission.
Source selection and presentation are separate. Declaration units, comments,
structural class headers and bounded local-call context preserve meaning without
-requiring complete files in the initial output. Parsing supports Python and
+requiring complete files in the initial output. Parsing supports Python, Go, Rust and
TypeScript/JavaScript; other or invalid text falls back to bounded source chunks.
Source ranges always refer to the same immutable snapshot used for classification.
## Parsing
-Python uses a [packaged Tree-sitter WASM grammar](../packages/core/assets/README.md)
-in a cancellable worker. This avoids a Python installation requirement and the
+Python, Go and Rust use [packaged Tree-sitter WASM grammars](../packages/core/assets/README.md)
+in a shared cancellable worker. This avoids a Python installation requirement and the
startup cost of embedding an interpreter. TypeScript/JavaScript use the TypeScript
compiler parser; other eligible text remains searchable through bounded chunks.
-The syntax tree supplies declarations and source coordinates. Python query
-previews, structural neighbours and inherited-method reading leads are retrieval
+The syntax tree supplies declarations and source coordinates. Go declaration
+groups remain intact where earlier values affect later constants. Rust methods
+retain module/impl headers and attributes, including inner attributes. Macro
+expansion and type resolution are outside this boundary. See the
+[Go/Rust retrieval contract](../specs/go-rust-parsing.md).
+
+Python query previews, structural neighbours and inherited-method reading leads are retrieval
policy on top of that tree, not a proof of runtime dispatch. Preserve original
source bytes; never reconstruct returned code from the tree.
diff --git a/evals/README.md b/evals/README.md
index 4bb8086..386200a 100644
--- a/evals/README.md
+++ b/evals/README.md
@@ -40,3 +40,6 @@ why lower retrieval charges can be offset by greater coding-agent expense.
The [skill-impact pilot](results/skill-impact-2026-09-29.md) compares old and new
skills on identical CLI builds, separately from automatic-trigger checks.
+
+The [Python parser checkpoint](results/python-parser-checkpoint-2026-09-29.md)
+records one official task against the merged Tree-sitter runtime.
diff --git a/evals/results/python-parser-checkpoint-2026-09-29.md b/evals/results/python-parser-checkpoint-2026-09-29.md
new file mode 100644
index 0000000..1600fac
--- /dev/null
+++ b/evals/results/python-parser-checkpoint-2026-09-29.md
@@ -0,0 +1,40 @@
+# Python parser checkpoint — 2026-09-29
+
+The merged Tree-sitter Python parser passed one official SWE-bench task,
+`psf__requests-1142`. The installed candidate was built from `ae10a9e`; although
+its manifest reads 0.6.0, it is not the published 0.6.0 package. Its SHA-256 is
+`3309d67d7ae8d5a9ca824eaf7d1d5a0c2193df6422998d18cd8f34d61c2cff38`.
+
+| Observation | Result |
+|---|---:|
+| Official resolved tasks | 1 / 1 |
+| Coding-agent cost | $0.397791 |
+| Jev cost (native input-usage estimate) | $0.005494944 |
+| Total task cost | $0.403285944 |
+| Jev requests | 16 |
+| Agent wall time | 117.79 s |
+| Charged work time | 105.14 s |
+| Credited standalone retrieval wait | 12.64 s |
+
+Required retrieval and timing protocol checks passed; all billing was reconciled.
+Six declaration-selection requests covered `requests/models.py`,
+`requests/adapters.py`, and `test_requests.py`, confirming structural Python
+parsing was exercised rather than only directory navigation.
+The retained no-Jev baseline also passed and cost $0.2685004. That historical
+comparison does not isolate the parser change, and this checkpoint is not a cost
+win or a full-cohort acceptance claim. No baseline was rerun and no favorable
+attempt was selected from repeated treatments.
+
+The [installed harness](../implementation/swebench/installed.md) froze Sol
+`openai/gpt-5.6-sol` at medium effort, the public skill, source tree and official
+grader. Jev used the native TypeSafe route. The original runtime image was absent;
+the retained rebuilt Requests image passed pinned base-layer, source and tool
+identity checks. Its image ID is
+`sha256:1a4338ea36e3f3fc9737f0625a4e4c07f1a2424f844aada5ac0d6b6a5cdf543a`.
+
+Raw plan, lifecycle receipt, agent patch, official grade, provider traces and
+accounting remain in ignored run storage at
+`evals/runs/swebench/tree-sitter-python-checkpoint-2026-09-29/`.
+The plan SHA-256 is `e1747710a865f690ff90ed933cf8179553aad37f5b29bbc390d708d7b9a6977a`.
+The result supports this task's correctness checkpoint; it does not prove all
+Python programs or repositories are regression-free.
diff --git a/packages/core/assets/README.md b/packages/core/assets/README.md
index 26f8b08..84064f1 100644
--- a/packages/core/assets/README.md
+++ b/packages/core/assets/README.md
@@ -1,10 +1,10 @@
# Parser assets
-The build copies the Python grammar WASM from its exact-version official
-Tree-sitter package into `tree-sitter/`. The generated asset and its license
-notice ship in the CLI archive; searches never download a grammar. The external
+The build copies Python, Go and Rust grammar WASM from exact-version official
+Tree-sitter packages into `tree-sitter/`. The generated assets and their license
+notices ship in the CLI archive; searches never download a grammar. The external
`web-tree-sitter` runtime supplies its own WASM and license.
[Asset preparation](../../../scripts/parser-assets.mjs) owns the copy and license
-provenance. It also prepares the canonical grammar used by release validation;
+provenance. It also prepares the canonical grammars used by release validation;
a clean validation checkout must install the locked development dependencies first.
diff --git a/packages/core/package.json b/packages/core/package.json
index 77ee435..0f026e4 100644
--- a/packages/core/package.json
+++ b/packages/core/package.json
@@ -22,6 +22,8 @@
"devDependencies": {
"@repo/typescript-config": "workspace:*",
"@types/node": "22",
- "tree-sitter-python": "0.25.0"
+ "tree-sitter-python": "0.25.0",
+ "tree-sitter-go": "0.25.0",
+ "tree-sitter-rust": "0.24.0"
}
}
diff --git a/packages/core/src/evaluator.ts b/packages/core/src/evaluator.ts
index 27c19f6..360ddbc 100644
--- a/packages/core/src/evaluator.ts
+++ b/packages/core/src/evaluator.ts
@@ -126,7 +126,7 @@ export function createEvaluator(options: {
endpoint: endpoint.baseURL,
protocol: "typesafe-ai-3.0.8",
policyVersion: options.policyVersion ?? "1",
- parserVersion: "tree-sitter-0.27.0-python-0.25.0-ts-5.9.3",
+ parserVersion: "tree-sitter-0.27.0-python-0.25.0-go-0.25.0-rust-0.24.0-ts-5.9.3",
promptVersion: "unit-locators-1",
},
};
diff --git a/packages/core/src/parser-declarations.mjs b/packages/core/src/parser-declarations.mjs
new file mode 100644
index 0000000..aa81645
--- /dev/null
+++ b/packages/core/src/parser-declarations.mjs
@@ -0,0 +1,113 @@
+const field = (node, name) => node.childForFieldName(name);
+const comment = (node) => node.type === "comment" || node.type.endsWith("_comment");
+const range = (node) => ({
+ startLine: node.startPosition.row + 1,
+ endLine: node.endPosition.row + (node.endPosition.column ? 1 : 0),
+});
+
+export function declarations(root, language) {
+ const comments = [],
+ stack = [root];
+ while (stack.length) {
+ const node = stack.pop();
+ if (comment(node)) comments.push(range(node));
+ else for (const child of node.namedChildren) stack.push(child);
+ }
+ const units = [];
+ if (language === "go")
+ for (const node of root.namedChildren) {
+ if (comment(node)) continue;
+ let name = field(node, "name")?.text ?? node.type;
+ if (node.type === "method_declaration") {
+ const receiver = field(node, "receiver")?.namedChildren.find(
+ (child) => child.type === "parameter_declaration",
+ );
+ const owner = receiver && field(receiver, "type")?.text.replace(/^\*/, "");
+ if (owner) name = `${owner}.${name}`;
+ } else if (["type_declaration", "var_declaration", "const_declaration"].includes(node.type)) {
+ // Keep groups intact: iota and omitted constant values depend on earlier specs.
+ const specs = node.namedChildren.flatMap((child) =>
+ child.type === "var_spec_list" ? child.namedChildren : [child],
+ );
+ name =
+ specs
+ .flatMap((spec) =>
+ spec
+ .childrenForFieldName("name")
+ .filter((child) => child.type === "identifier" || child.type === "type_identifier")
+ .map((child) => child.text),
+ )
+ .join(", ") || node.type;
+ } else if (node.type === "package_clause")
+ name = `package ${node.namedChildren.find((child) => child.type === "package_identifier").text}`;
+ units.push({ name, range: range(node) });
+ }
+ else {
+ function startWithAttributes(node) {
+ let start = node;
+ while (
+ start.previousNamedSibling &&
+ (start.previousNamedSibling.type === "attribute_item" ||
+ comment(start.previousNamedSibling))
+ ) {
+ const previous = start.previousNamedSibling;
+ const before = previous.previousNamedSibling;
+ if (
+ comment(previous) &&
+ before &&
+ !comment(before) &&
+ before.type !== "attribute_item" &&
+ before.endPosition.row === previous.startPosition.row
+ )
+ break;
+ start = start.previousNamedSibling;
+ }
+ return range(start).startLine;
+ }
+ function visit(nodes, prefix = "", headers = []) {
+ const ownedHeaders = [
+ ...headers,
+ ...nodes.filter((node) => node.type === "inner_attribute_item").map(range),
+ ];
+ for (const node of nodes) {
+ if (comment(node) || ["attribute_item", "inner_attribute_item"].includes(node.type))
+ continue;
+ const body =
+ ["impl_item", "trait_item", "mod_item", "foreign_mod_item"].includes(node.type) &&
+ field(node, "body");
+ const owner =
+ field(node, "name")?.text ||
+ field(node, "type")?.text ||
+ field(node, "macro")?.text ||
+ node.type;
+ const startLine = startWithAttributes(node);
+ if (
+ body &&
+ body.namedChildren.some(
+ (child) =>
+ !comment(child) && !["attribute_item", "inner_attribute_item"].includes(child.type),
+ )
+ ) {
+ const header = { startLine, endLine: body.startPosition.row + 1 };
+ units.push({
+ name: prefix + owner + ".context",
+ range: header,
+ ownerHeaders: [...ownedHeaders, header],
+ });
+ visit(
+ body.namedChildren,
+ node.type === "foreign_mod_item" ? prefix : prefix + owner + ".",
+ [...ownedHeaders, header],
+ );
+ } else
+ units.push({
+ name: prefix + owner,
+ range: { ...range(node), startLine },
+ ownerHeaders: ownedHeaders,
+ });
+ }
+ }
+ visit(root.namedChildren);
+ }
+ return { units, comments };
+}
diff --git a/packages/core/src/parser-helpers.mjs b/packages/core/src/parser-helpers.mjs
index 9c2415b..b06d262 100644
--- a/packages/core/src/parser-helpers.mjs
+++ b/packages/core/src/parser-helpers.mjs
@@ -1,22 +1,29 @@
import { Parser, Language, Query } from "web-tree-sitter";
import { fileURLToPath } from "node:url";
+import { declarations } from "./parser-declarations.mjs";
import { preview } from "./parser-preview.mjs";
let initialized;
-let language;
-async function parse(source, action) {
+const languages = new Map();
+async function parse(source, action, languageName = "python") {
// Retrieval coordinates count LF lines; normalizing bare CR would mislabel original bytes.
- if (/\r(?!\n)/.test(source)) return null;
+ if (languageName === "python" && /\r(?!\n)/.test(source)) return null;
await (initialized ??= Parser.init());
- language ??= await Language.load(
- fileURLToPath(new URL("../assets/tree-sitter/tree-sitter-python.wasm", import.meta.url)),
- );
+ if (!languages.has(languageName))
+ languages.set(
+ languageName,
+ await Language.load(
+ fileURLToPath(
+ new URL(`../assets/tree-sitter/tree-sitter-${languageName}.wasm`, import.meta.url),
+ ),
+ ),
+ );
const parser = new Parser();
let tree;
try {
- parser.setLanguage(language);
+ parser.setLanguage(languages.get(languageName));
tree = parser.parse(source);
if (!tree || tree.rootNode.hasError) return null;
- if (!validPython(tree.rootNode)) return null;
+ if (languageName === "python" && !validPython(tree.rootNode)) return null;
return action(tree.rootNode);
} finally {
tree?.delete();
@@ -48,7 +55,7 @@ let syntaxChecks;
function validPython(root) {
// Let the native query engine skip nodes that cannot affect this compatibility policy.
syntaxChecks ??= new Query(
- language,
+ languages.get("python"),
`[
(exec_statement) (print_statement) (except_clause) (raise_statement) (for_in_clause)
(concatenated_string) (function_definition) (class_definition) (integer)
@@ -414,6 +421,8 @@ function calls(root, source, ranges) {
export async function execute(helper, input) {
if (helper === "inspect") return parse(input, inspectPython);
const data = JSON.parse(input);
+ if (helper === "declarations")
+ return parse(data.source, (root) => declarations(root, data.language), data.language);
if (helper === "neighborhood")
return parse(data.source, (root) => neighborhood(root, data.ranges));
if (helper === "calls")
diff --git a/packages/core/src/parser.ts b/packages/core/src/parser.ts
index 5f1165b..cdd5e21 100644
--- a/packages/core/src/parser.ts
+++ b/packages/core/src/parser.ts
@@ -1,6 +1,6 @@
import { fork, type ChildProcess } from "node:child_process";
-type Helper = "inspect" | "preview" | "neighborhood" | "calls";
+type Helper = "inspect" | "preview" | "neighborhood" | "calls" | "declarations";
type Pending = {
helper: Helper;
input: string;
diff --git a/packages/core/src/retrieve.ts b/packages/core/src/retrieve.ts
index ba65a57..18388bd 100644
--- a/packages/core/src/retrieve.ts
+++ b/packages/core/src/retrieve.ts
@@ -277,7 +277,7 @@ export async function retrieve(input: SearchInput, evaluator: Evaluator): Promis
}
if (
truncated &&
- /\.(?:pyi?|[cm]?[jt]s|[jt]sx)$/.test(source.path) &&
+ /\.(?:pyi?|[cm]?[jt]s|[jt]sx|go|rs)$/.test(source.path) &&
bytes.length <= 1_000_000
) {
const syntax = await inspect(source, {
diff --git a/packages/core/src/source.ts b/packages/core/src/source.ts
index 8e50dce..efda028 100644
--- a/packages/core/src/source.ts
+++ b/packages/core/src/source.ts
@@ -15,7 +15,7 @@ export type SourceUnit = {
export type Inspection = {
units: SourceUnit[];
comments: Range[];
- mode: "python" | "typescript" | "text";
+ mode: "python" | "typescript" | "go" | "rust" | "text";
fallback?: "unsupported" | "syntax" | "size";
};
@@ -167,6 +167,16 @@ export async function inspect(
};
visit(file);
mode = "typescript";
+ } else if (/\.(go|rs)$/.test(path)) {
+ const parsed = await runParser<{ units: typeof units; comments: Range[] }>(
+ "declarations",
+ JSON.stringify({ source, language: path.endsWith(".go") ? "go" : "rust" }),
+ options.signal,
+ );
+ if (!parsed) return fallback("syntax");
+ units = parsed.units;
+ comments = parsed.comments;
+ mode = path.endsWith(".go") ? "go" : "rust";
} else return fallback("unsupported");
comments = [...new Map(comments.map((r) => [`${r.startLine}:${r.endLine}`, r])).values()].sort(
(a, b) => a.startLine - b.startLine,
diff --git a/scripts/RELEASING.md b/scripts/RELEASING.md
index 46751fb..bc79bd7 100644
--- a/scripts/RELEASING.md
+++ b/scripts/RELEASING.md
@@ -30,7 +30,7 @@ asset checks. [The build](build-cli.ts) copies the authored MIT license and
[collects third-party notices](package-notices.mjs) from emitted bundle inputs.
[Retained license sources](licenses/README.md) document upstream distribution gaps.
The archive excludes test fixtures, evaluation evidence, node_modules directories, source maps,
-and unbundled development source. The parser worker, extraction code, and packaged Python grammar
+and unbundled development source. The parser worker, extraction code, and packaged grammars
are required runtime assets and are included. Direct runtime parser dependencies
install from exact npm pins; transitive dependencies resolve during npm installation.
The registry-verification checkout installs locked development dependencies and
@@ -40,7 +40,7 @@ bytes and license notices with the downloaded archive.
On an Apple Silicon Mac, run `node scripts/test-native.mjs` after the normal
`bun install --frozen-lockfile` setup. This builds and packs the candidate, installs
it with npm outside the checkout, and runs the credential-free commands and
-synthetic Python search against a loopback Gateway fixture. To verify an existing
+synthetic Python/Go/Rust searches against a loopback Gateway fixture. To verify an existing
archive without Bun, use `node scripts/test-native.mjs --prebuilt /path/package.tgz`.
The command prints the archive hash and actual Node/macOS runtime for retained
evidence. It requires Node 22 or newer; a result proves the printed version only.
diff --git a/scripts/build-cli.ts b/scripts/build-cli.ts
index 9c19970..8abae2e 100644
--- a/scripts/build-cli.ts
+++ b/scripts/build-cli.ts
@@ -25,7 +25,7 @@ await writeFile(
join(out, "THIRD_PARTY_NOTICES.txt"),
(await bundledNotices(build.metafile!, process.cwd())) + grammarNotices,
);
-for (const name of ["parser-worker", "parser-helpers", "parser-preview"])
+for (const name of ["parser-worker", "parser-helpers", "parser-preview", "parser-declarations"])
await cp(join(root, `packages/core/src/${name}.mjs`), join(out, `bin/${name}.mjs`));
await cp(join(root, "LICENSE"), join(out, "LICENSE"));
await cp(join(root, "packages/core/assets"), join(out, "assets"), { recursive: true });
diff --git a/scripts/licenses/README.md b/scripts/licenses/README.md
index c6388fe..8e6c284 100644
--- a/scripts/licenses/README.md
+++ b/scripts/licenses/README.md
@@ -14,5 +14,5 @@ there is no build-time or runtime license download.
## Parser grammars
The build copies license notices directly from the pinned official Tree-sitter
-Python grammar package. Its WASM file is shipped unmodified. The external
+grammar packages. Their WASM files are shipped unmodified. The external
web-tree-sitter and TypeScript packages retain their installed license files.
diff --git a/scripts/parser-assets.mjs b/scripts/parser-assets.mjs
index 4a4a62a..528cb3f 100644
--- a/scripts/parser-assets.mjs
+++ b/scripts/parser-assets.mjs
@@ -2,6 +2,7 @@ import { createRequire } from "node:module";
import { readFile, mkdir, cp } from "node:fs/promises";
import { dirname, join, resolve } from "node:path";
import { fileURLToPath } from "node:url";
+export const grammars = ["python", "go", "rust"];
export async function grammarAssets(
root = fileURLToPath(new URL("../", import.meta.url)),
copy = false,
@@ -10,13 +11,17 @@ export async function grammarAssets(
const core = JSON.parse(await readFile(join(root, "packages/core/package.json"), "utf8"));
const destination = join(root, "packages/core/assets/tree-sitter");
if (copy) await mkdir(destination, { recursive: true });
- const name = "tree-sitter-python",
- directory = dirname(require.resolve(`${name}/package.json`));
- const metadata = JSON.parse(await readFile(join(directory, "package.json"), "utf8"));
- if (metadata.version !== core.devDependencies[name])
- throw new Error(`Grammar version differs from manifest: ${name}`);
- if (copy) await cp(join(directory, `${name}.wasm`), join(destination, `${name}.wasm`));
- return `\nBundled Tree-sitter grammar notices\n\n=== ${name}@${metadata.version} (${metadata.license}) ===\nSource: https://github.com/tree-sitter/${name}\n${await readFile(join(directory, "LICENSE"), "utf8")}\n`;
+ let notices = "\nBundled Tree-sitter grammar notices\n\n";
+ for (const language of grammars) {
+ const name = `tree-sitter-${language}`;
+ const directory = dirname(require.resolve(`${name}/package.json`));
+ const metadata = JSON.parse(await readFile(join(directory, "package.json"), "utf8"));
+ if (metadata.version !== core.devDependencies[name])
+ throw new Error(`Grammar version differs from manifest: ${name}`);
+ if (copy) await cp(join(directory, `${name}.wasm`), join(destination, `${name}.wasm`));
+ notices += `=== ${name}@${metadata.version} (${metadata.license}) ===\nSource: https://github.com/tree-sitter/${name}\n${await readFile(join(directory, "LICENSE"), "utf8")}\n`;
+ }
+ return notices;
}
if (process.argv[1] && resolve(process.argv[1]) === fileURLToPath(import.meta.url))
diff --git a/scripts/test-native.mjs b/scripts/test-native.mjs
index 3dfe2b4..f1ee2cf 100644
--- a/scripts/test-native.mjs
+++ b/scripts/test-native.mjs
@@ -131,7 +131,7 @@ try {
"--test-reporter=tap",
"--test-concurrency=1",
"--test-name-pattern",
- "^(installed local commands match the package without credentials|skill command delegates installation to npx without credentials|actual installed search parses Python and returns every relevant hierarchy branch|installed saved-provider journey: (vercel|typesafe|openrouter) auth doctor search cache replacement|installed legacy credentials use Vercel without rewriting saved bytes|installed saved credentials defeat conflicting environment and environment-only auth fails|installed invalid saved providers fail before HTTP without rewriting credentials|installed invalid auth preserves saved bytes and leaves no temporary credentials)$",
+ "^(installed Go/Rust search indexes late methods and selects their source instead of distant noise|installed local commands match the package without credentials|skill command delegates installation to npx without credentials|actual installed search parses Python and returns every relevant hierarchy branch|installed saved-provider journey: (vercel|typesafe|openrouter) auth doctor search cache replacement|installed legacy credentials use Vercel without rewriting saved bytes|installed saved credentials defeat conflicting environment and environment-only auth fails|installed invalid saved providers fail before HTTP without rewriting credentials|installed invalid auth preserves saved bytes and leaves no temporary credentials)$",
join(root, "test/installed.test.mjs"),
],
{ cwd: scratch, env: runtimeEnv, timeout: 120_000, maxBuffer: 8_000_000 },
@@ -139,7 +139,7 @@ try {
assert.equal(tested.stderr, "");
assert.match(
tested.stdout,
- /^# pass 10$/m,
+ /^# pass 11$/m,
"All native smoke journeys must execute; renamed selectors cannot silently pass",
);
console.log(tested.stdout.trimEnd());
diff --git a/scripts/validate-release.mjs b/scripts/validate-release.mjs
index c1bccd4..0971c65 100644
--- a/scripts/validate-release.mjs
+++ b/scripts/validate-release.mjs
@@ -4,7 +4,7 @@ import { readFile, stat } from "node:fs/promises";
import { resolve } from "node:path";
import { fileURLToPath } from "node:url";
import { promisify } from "node:util";
-import { grammarAssets } from "./parser-assets.mjs";
+import { grammarAssets, grammars } from "./parser-assets.mjs";
const execute = promisify(execFile);
export const repository = fileURLToPath(new URL("../", import.meta.url));
@@ -48,9 +48,10 @@ export async function validateRelease(tarball, tag, root = repository) {
if (new Set(files).size !== files.length) throw new Error("Duplicate archive entries");
for (const path of files)
if (
- !/^package\/(?:package\.json|README(?:\.md)?|LICENSE|dist\/(?:bin\/(?:index\.js|parser-(?:worker|helpers|preview)\.mjs)|assets\/(?:tree-sitter\/tree-sitter-python\.wasm|README\.md)|skills\/jevgrep\/SKILL\.md|LICENSE|THIRD_PARTY_NOTICES\.txt))$/.test(
+ !/^package\/(?:package\.json|README(?:\.md)?|LICENSE|dist\/(?:bin\/(?:index\.js|parser-(?:worker|helpers|preview|declarations)\.mjs)|assets\/(?:README\.md)|skills\/jevgrep\/SKILL\.md|LICENSE|THIRD_PARTY_NOTICES\.txt))$/.test(
path,
- )
+ ) &&
+ !grammars.some((name) => path === `package/dist/assets/tree-sitter/tree-sitter-${name}.wasm`)
)
throw new Error(`Unexpected published file: ${path}`);
const listing = await execute("tar", ["-tvzf", tarball], { maxBuffer: 8_000_000 });
@@ -76,14 +77,14 @@ export async function validateRelease(tarball, tag, root = repository) {
for (const [packed, original] of [
["dist/LICENSE", "LICENSE"],
["dist/skills/jevgrep/SKILL.md", "skills/jevgrep/SKILL.md"],
- ...["parser-worker", "parser-helpers", "parser-preview"].map((name) => [
+ ...["parser-worker", "parser-helpers", "parser-preview", "parser-declarations"].map((name) => [
`dist/bin/${name}.mjs`,
`packages/core/src/${name}.mjs`,
]),
- [
- "dist/assets/tree-sitter/tree-sitter-python.wasm",
- "packages/core/assets/tree-sitter/tree-sitter-python.wasm",
- ],
+ ...grammars.map((name) => [
+ `dist/assets/tree-sitter/tree-sitter-${name}.wasm`,
+ `packages/core/assets/tree-sitter/tree-sitter-${name}.wasm`,
+ ]),
])
if (!(await extract(packed)).equals(await readFile(resolve(root, original))))
throw new Error(`Packaged ${packed} differs from its canonical source`);
diff --git a/specs/go-rust-parsing.md b/specs/go-rust-parsing.md
new file mode 100644
index 0000000..e58b9e5
--- /dev/null
+++ b/specs/go-rust-parsing.md
@@ -0,0 +1,47 @@
+# Go/Rust declaration retrieval
+
+Go and Rust use named source selection through the existing bounded parser
+worker. Python and TypeScript behavior stays fixed; the Python prerequisite is
+the official `psf__requests-1142` checkpoint against merged `ae10a9e`.
+
+The useful contract is observable through `inspect` and installed search:
+declaration names appear in large-file previews, selection can isolate a function,
+and returned source preserves original UTF-8/CRLF bytes and declaration context.
+This establishes structural retrieval capability, not measured model recall.
+
+The pinned official grammars (`tree-sitter-go` 0.25.0 and `tree-sitter-rust`
+0.24.0) supply syntax, following their upstream tag conventions. Grammar assets and licenses share
+the existing build and release-validation owner. No new runtime download, native
+compiler, worker pool, or cache is introduced.
+
+`parser-declarations.mjs` owns Go/Rust syntax extraction; the worker owns parser
+lifetime. Go declaration groups remain intact because constants can depend on
+group order (`iota`). Rust modules, traits, impls, attributes and inner attributes
+retain their enclosing headers. Generic receivers keep their source spelling;
+this is neither type resolution nor macro expansion.
+
+Verification: focused tests fail against text fallback and pass with named
+declarations; installed tests verify declaration indexes and returned source;
+malformed/oversized source keeps lossless fallback; missing grammars fail visibly.
+Cold/warm cost and source-unit size are measured against `ae10a9e`; overhead
+is reported without calling it a speedup. Full release gates and independent
+review cover the integrated change.
+
+References: [Go tags](https://github.com/tree-sitter/tree-sitter-go/blob/v0.25.0/queries/tags.scm)
+and [Rust tags](https://github.com/tree-sitter/tree-sitter-rust/blob/v0.24.0/queries/tags.scm).
+
+[Measured precision and parsing cost](go-rust-parsing/RESULTS.md) records the
+tradeoff. Python query previews and inherited-call analysis remain Python-specific.
+
+## Verification record — 2026-09-29
+
+`bun run verify` passed 210 source checks and 41 installed-package scenarios.
+Native Apple Silicon smoke passed all 11 journeys, including Go/Rust search.
+Typechecks passed; lint retains the existing filesystem control-regex warning.
+A clean checkout installed the locked grammar packages and validated the exact
+archive, its canonical assets, license notices, and integrity.
+
+Independent review's multi-name Go constant-label finding was reproduced and
+fixed by excluding punctuation from identifier fields. Focused tests failed
+before each extraction fix and passed afterward. Python parsing retains its
+existing differential and cancellation coverage.
diff --git a/specs/go-rust-parsing/RESULTS.md b/specs/go-rust-parsing/RESULTS.md
new file mode 100644
index 0000000..2911450
--- /dev/null
+++ b/specs/go-rust-parsing/RESULTS.md
@@ -0,0 +1,36 @@
+# Go/Rust parsing measurements
+
+Three fresh-process trials of the final candidate on darwin/arm64 with Node
+24.14.0 are compared with the unchanged `ae10a9e` baseline trials from
+[development](development-measurements.json). Baseline trials were not rerun. Cold time
+includes module import and first inspection; warm time is the median of ten
+inspections, summarized across the three processes. No provider calls were made.
+
+| Language | Functions | Cold before → after (ms) | Warm before → after (ms) | Target unit bytes before → after |
+|---|---:|---:|---:|---:|
+| go | 100 | 143.38 → 281.41 | 0.06 → 2.65 | 5,195 → 52 |
+| go | 1,500 | 151.98 → 454.15 | 0.63 → 30.63 | 23,976 → 54 |
+| rust | 100 | 148.12 → 275.91 | 0.06 → 2.42 | 5,180 → 52 |
+| rust | 1,500 | 152.00 → 257.82 | 0.62 → 22.19 | 23,976 → 54 |
+
+This is a deliberate precision/cost tradeoff. Parsing adds work and worker memory;
+text fallback had no parser worker. The measured unit contains a known target
+marker, not a model-selected final excerpt. Smaller units do not establish lower
+end-to-end cost, better recall, or faster coding-agent tasks. Worker RSS snapshots
+and source identities are in the [raw measurements](measurements.json).
+
+The installed search fixture supplies deterministic relevance answers. It verifies
+that late methods reach large-file declaration indexes, named selection returns
+their source and Rust attributes, and distant noise is not returned. This is a
+capability check, not a live-provider quality benchmark.
+
+Reproduce each workload in prepared baseline/candidate checkouts using the same
+[measurement script](bench.mjs), with `go` or `rust` and 100 or 1500 functions:
+
+```sh
+node --experimental-strip-types specs/go-rust-parsing/bench.mjs /absolute/checkout/packages/core/src/source.ts go 1500
+```
+
+Run each arm in alternating fresh processes. The candidate must first prepare its
+pinned grammar assets through the CLI build. Further extraction fixes may change
+timings; the recorded source hashes identify this measurement cohort.
diff --git a/specs/go-rust-parsing/bench.mjs b/specs/go-rust-parsing/bench.mjs
new file mode 100644
index 0000000..f4197e6
--- /dev/null
+++ b/specs/go-rust-parsing/bench.mjs
@@ -0,0 +1,47 @@
+import { ChildProcess, execFileSync } from "node:child_process";
+const [module, language, countText] = process.argv.slice(2),
+ count = Number(countText);
+if (!module || !["go", "rust"].includes(language) || !Number.isSafeInteger(count) || count < 1)
+ throw new Error("Use a source module path, go or rust, and a positive function count");
+let child;
+const emit = ChildProcess.prototype.emit;
+ChildProcess.prototype.emit = function (event, ...args) {
+ if (event === "spawn") child = this;
+ return emit.call(this, event, ...args);
+};
+const source =
+ (language === "go" ? "package sample\n" : "") +
+ Array.from({ length: count }, (_, i) =>
+ language === "go"
+ ? `func target_${i}() string { return "marker_${i}_🙂" }\n`
+ : `fn target_${i}() -> &'static str { "marker_${i}_🙂" }\n`,
+ ).join("");
+const snapshot = { source, path: language === "go" ? "x.go" : "x.rs", contentHash: "fixture" };
+const t = performance.now();
+const { inspect, sourceForUnit } = await import(module);
+const result = await inspect(snapshot);
+const cold = performance.now() - t;
+const warm = [];
+for (let i = 0; i < 10; i++) {
+ const start = performance.now();
+ await inspect({ ...snapshot });
+ warm.push(performance.now() - start);
+}
+const marker = `"marker_${Math.floor(count / 2)}_🙂"`;
+const unit = result.units.find((u) => sourceForUnit(snapshot, u).includes(marker));
+if (!unit) throw Error("Target source missing");
+console.log(
+ JSON.stringify({
+ language,
+ count,
+ cold_ms: cold,
+ warm_ms: warm,
+ units: result.units.length,
+ target_unit_bytes: Buffer.byteLength(sourceForUnit(snapshot, unit)),
+ worker_rss_kib: child
+ ? Number(
+ execFileSync("ps", ["-o", "rss=", "-p", String(child.pid)], { encoding: "utf8" }).trim(),
+ )
+ : null,
+ }),
+);
diff --git a/specs/go-rust-parsing/development-measurements.json b/specs/go-rust-parsing/development-measurements.json
new file mode 100644
index 0000000..90d3107
--- /dev/null
+++ b/specs/go-rust-parsing/development-measurements.json
@@ -0,0 +1,540 @@
+{
+ "baseline": "ae10a9ee8ee55d5a262e04efc5e0ac41c6733a33",
+ "node": "v24.14.0",
+ "platform": "darwin/arm64",
+ "records": [
+ {
+ "arm": "before",
+ "trial": 0,
+ "language": "go",
+ "count": 100,
+ "cold_ms": 137.24620800000002,
+ "warm_ms": [
+ 0.07979199999999764,
+ 0.048082999999991216,
+ 0.058458000000001675,
+ 0.053665999999992664,
+ 0.06212500000000887,
+ 0.043708000000009406,
+ 0.04200000000000159,
+ 0.051124999999984766,
+ 0.12204099999999585,
+ 0.06295800000000895
+ ],
+ "units": 1,
+ "target_unit_bytes": 5195,
+ "worker_rss_kib": null
+ },
+ {
+ "arm": "after",
+ "trial": 0,
+ "language": "go",
+ "count": 100,
+ "cold_ms": 184.870916,
+ "warm_ms": [
+ 2.373000000000019,
+ 2.6346670000000074,
+ 1.7156249999999886,
+ 1.6431670000000054,
+ 1.6285000000000025,
+ 1.6205420000000004,
+ 1.404042000000004,
+ 1.348792000000003,
+ 1.6571670000000154,
+ 1.6779999999999973
+ ],
+ "units": 101,
+ "target_unit_bytes": 52,
+ "worker_rss_kib": 87856
+ },
+ {
+ "arm": "before",
+ "trial": 1,
+ "language": "go",
+ "count": 100,
+ "cold_ms": 143.433708,
+ "warm_ms": [
+ 0.08291700000000901,
+ 0.04862500000001546,
+ 0.0675420000000031,
+ 0.19645799999997848,
+ 0.060291000000006534,
+ 0.0625840000000153,
+ 0.049000000000006594,
+ 0.05349999999998545,
+ 0.05025000000000546,
+ 0.04816700000000651
+ ],
+ "units": 1,
+ "target_unit_bytes": 5195,
+ "worker_rss_kib": null
+ },
+ {
+ "arm": "after",
+ "trial": 1,
+ "language": "go",
+ "count": 100,
+ "cold_ms": 189.528541,
+ "warm_ms": [
+ 2.2134169999999926,
+ 2.1358749999999986,
+ 1.5052919999999972,
+ 1.5457499999999982,
+ 1.372917000000001,
+ 1.3282079999999894,
+ 1.3761250000000018,
+ 1.548374999999993,
+ 1.4377920000000017,
+ 1.3357919999999979
+ ],
+ "units": 101,
+ "target_unit_bytes": 52,
+ "worker_rss_kib": 86480
+ },
+ {
+ "arm": "before",
+ "trial": 2,
+ "language": "go",
+ "count": 100,
+ "cold_ms": 143.38341599999998,
+ "warm_ms": [
+ 0.08362499999998363,
+ 0.051124999999984766,
+ 0.19695899999999256,
+ 0.07433400000002166,
+ 0.08033299999999599,
+ 0.0503750000000025,
+ 0.04466700000000401,
+ 0.05883300000002123,
+ 0.0449169999999981,
+ 0.05287500000000023
+ ],
+ "units": 1,
+ "target_unit_bytes": 5195,
+ "worker_rss_kib": null
+ },
+ {
+ "arm": "after",
+ "trial": 2,
+ "language": "go",
+ "count": 100,
+ "cold_ms": 188.95291699999999,
+ "warm_ms": [
+ 2.240167000000014,
+ 2.1582500000000095,
+ 1.712208000000004,
+ 1.825082999999978,
+ 1.5590419999999767,
+ 1.4155839999999955,
+ 1.3925419999999917,
+ 1.5942499999999882,
+ 1.4927909999999827,
+ 1.381208000000015
+ ],
+ "units": 101,
+ "target_unit_bytes": 52,
+ "worker_rss_kib": 86128
+ },
+ {
+ "arm": "before",
+ "trial": 0,
+ "language": "go",
+ "count": 1500,
+ "cold_ms": 151.97924999999998,
+ "warm_ms": [
+ 1.0316660000000013,
+ 1.353749999999991,
+ 0.626041999999984,
+ 0.6341249999999832,
+ 0.645875000000018,
+ 0.5972919999999817,
+ 0.5961250000000007,
+ 2.3009580000000085,
+ 0.6234579999999994,
+ 0.597124999999977
+ ],
+ "units": 4,
+ "target_unit_bytes": 23976,
+ "worker_rss_kib": null
+ },
+ {
+ "arm": "after",
+ "trial": 0,
+ "language": "go",
+ "count": 1500,
+ "cold_ms": 233.470833,
+ "warm_ms": [
+ 17.710791999999998,
+ 16.610041999999964,
+ 16.287041999999985,
+ 16.11908299999999,
+ 16.978707999999983,
+ 16.572374999999965,
+ 16.696500000000015,
+ 16.670375000000035,
+ 14.712041,
+ 14.781457999999986
+ ],
+ "units": 1501,
+ "target_unit_bytes": 54,
+ "worker_rss_kib": 124112
+ },
+ {
+ "arm": "before",
+ "trial": 1,
+ "language": "go",
+ "count": 1500,
+ "cold_ms": 147.158917,
+ "warm_ms": [
+ 0.5936670000000106,
+ 1.227374999999995,
+ 0.5569580000000087,
+ 0.5495409999999765,
+ 0.551667000000009,
+ 0.5903329999999869,
+ 2.2135419999999897,
+ 0.5703330000000051,
+ 0.49962500000000887,
+ 0.5096670000000074
+ ],
+ "units": 4,
+ "target_unit_bytes": 23976,
+ "worker_rss_kib": null
+ },
+ {
+ "arm": "after",
+ "trial": 1,
+ "language": "go",
+ "count": 1500,
+ "cold_ms": 218.675959,
+ "warm_ms": [
+ 16.304583000000008,
+ 15.606334000000032,
+ 15.568875000000048,
+ 15.659583999999995,
+ 15.790166,
+ 14.82854199999997,
+ 15.850458000000003,
+ 15.488165999999978,
+ 14.437000000000012,
+ 15.014250000000004
+ ],
+ "units": 1501,
+ "target_unit_bytes": 54,
+ "worker_rss_kib": 125840
+ },
+ {
+ "arm": "before",
+ "trial": 2,
+ "language": "go",
+ "count": 1500,
+ "cold_ms": 152.123584,
+ "warm_ms": [
+ 0.6299579999999878,
+ 1.213624999999979,
+ 0.6246669999999881,
+ 0.6185420000000192,
+ 0.6034999999999968,
+ 0.5918329999999798,
+ 0.6312499999999943,
+ 0.7666659999999865,
+ 2.126249999999999,
+ 0.5161250000000166
+ ],
+ "units": 4,
+ "target_unit_bytes": 23976,
+ "worker_rss_kib": null
+ },
+ {
+ "arm": "after",
+ "trial": 2,
+ "language": "go",
+ "count": 1500,
+ "cold_ms": 218.740959,
+ "warm_ms": [
+ 18.075458000000026,
+ 16.563999999999993,
+ 16.398249999999962,
+ 16.785124999999994,
+ 17.857124999999996,
+ 17.722458000000017,
+ 17.748582999999996,
+ 17.488207999999986,
+ 16.60895800000003,
+ 17.034457999999972
+ ],
+ "units": 1501,
+ "target_unit_bytes": 54,
+ "worker_rss_kib": 124480
+ },
+ {
+ "arm": "before",
+ "trial": 0,
+ "language": "rust",
+ "count": 100,
+ "cold_ms": 146.79166700000002,
+ "warm_ms": [
+ 0.08687499999999204,
+ 0.0514169999999865,
+ 0.06166600000000244,
+ 0.059208999999981415,
+ 0.06337500000000773,
+ 0.18091699999999378,
+ 0.1322499999999991,
+ 0.09183299999997985,
+ 0.0681250000000091,
+ 0.057332999999999856
+ ],
+ "units": 1,
+ "target_unit_bytes": 5180,
+ "worker_rss_kib": null
+ },
+ {
+ "arm": "after",
+ "trial": 0,
+ "language": "rust",
+ "count": 100,
+ "cold_ms": 194.910792,
+ "warm_ms": [
+ 2.7383749999999907,
+ 2.433667000000014,
+ 1.9113329999999848,
+ 2.068708000000015,
+ 1.7388750000000073,
+ 2.1834169999999915,
+ 1.7560410000000104,
+ 1.7517080000000078,
+ 1.883291999999983,
+ 1.3318749999999966
+ ],
+ "units": 100,
+ "target_unit_bytes": 52,
+ "worker_rss_kib": 98288
+ },
+ {
+ "arm": "before",
+ "trial": 1,
+ "language": "rust",
+ "count": 100,
+ "cold_ms": 149.737667,
+ "warm_ms": [
+ 0.0831670000000031,
+ 0.05624999999997726,
+ 0.06049999999999045,
+ 0.058041000000002896,
+ 0.05962500000001114,
+ 0.04845799999998235,
+ 0.04087499999999977,
+ 0.06095899999999688,
+ 0.04724999999999113,
+ 0.043042000000014013
+ ],
+ "units": 1,
+ "target_unit_bytes": 5180,
+ "worker_rss_kib": null
+ },
+ {
+ "arm": "after",
+ "trial": 1,
+ "language": "rust",
+ "count": 100,
+ "cold_ms": 191.64425,
+ "warm_ms": [
+ 2.760165999999998,
+ 2.118957999999992,
+ 1.9867909999999824,
+ 2.2094999999999914,
+ 1.7807500000000118,
+ 1.9852500000000077,
+ 1.86016699999999,
+ 1.8732090000000028,
+ 1.9823749999999905,
+ 1.5636670000000095
+ ],
+ "units": 100,
+ "target_unit_bytes": 52,
+ "worker_rss_kib": 100016
+ },
+ {
+ "arm": "before",
+ "trial": 2,
+ "language": "rust",
+ "count": 100,
+ "cold_ms": 148.11629100000002,
+ "warm_ms": [
+ 0.08866700000001515,
+ 0.05441600000000335,
+ 0.061916999999994005,
+ 0.049915999999996075,
+ 0.06829199999998536,
+ 0.051832999999987805,
+ 0.056250000000005684,
+ 0.06733399999998824,
+ 0.05629199999998491,
+ 0.06070800000000531
+ ],
+ "units": 1,
+ "target_unit_bytes": 5180,
+ "worker_rss_kib": null
+ },
+ {
+ "arm": "after",
+ "trial": 2,
+ "language": "rust",
+ "count": 100,
+ "cold_ms": 203.695833,
+ "warm_ms": [
+ 2.434833999999995,
+ 2.351165999999978,
+ 1.9988749999999982,
+ 2.392791999999986,
+ 1.8858749999999986,
+ 1.9526250000000118,
+ 2.0176669999999888,
+ 1.7651250000000118,
+ 2.0050830000000133,
+ 1.7243749999999807
+ ],
+ "units": 100,
+ "target_unit_bytes": 52,
+ "worker_rss_kib": 96592
+ },
+ {
+ "arm": "before",
+ "trial": 0,
+ "language": "rust",
+ "count": 1500,
+ "cold_ms": 149.60775,
+ "warm_ms": [
+ 0.5886250000000075,
+ 1.167082999999991,
+ 0.6224170000000129,
+ 0.5444999999999993,
+ 0.6081250000000011,
+ 0.7984160000000031,
+ 0.6195000000000164,
+ 2.0960830000000215,
+ 0.5019999999999811,
+ 0.49229199999999196
+ ],
+ "units": 4,
+ "target_unit_bytes": 23976,
+ "worker_rss_kib": null
+ },
+ {
+ "arm": "after",
+ "trial": 0,
+ "language": "rust",
+ "count": 1500,
+ "cold_ms": 230.225125,
+ "warm_ms": [
+ 23.293958000000003,
+ 20.277667000000008,
+ 19.939791999999954,
+ 19.792250000000024,
+ 19.433791999999983,
+ 17.85012499999999,
+ 18.151792,
+ 20.731832999999995,
+ 22.549916999999994,
+ 21.633459000000016
+ ],
+ "units": 1500,
+ "target_unit_bytes": 54,
+ "worker_rss_kib": 137440
+ },
+ {
+ "arm": "before",
+ "trial": 1,
+ "language": "rust",
+ "count": 1500,
+ "cold_ms": 157.93491699999998,
+ "warm_ms": [
+ 0.6371670000000051,
+ 1.436000000000007,
+ 0.6196659999999952,
+ 0.6478749999999991,
+ 0.6295000000000073,
+ 0.5977090000000089,
+ 0.5875409999999874,
+ 2.2151250000000005,
+ 0.5549579999999992,
+ 0.5406670000000133
+ ],
+ "units": 4,
+ "target_unit_bytes": 23976,
+ "worker_rss_kib": null
+ },
+ {
+ "arm": "after",
+ "trial": 1,
+ "language": "rust",
+ "count": 1500,
+ "cold_ms": 224.057708,
+ "warm_ms": [
+ 23.153624999999977,
+ 19.902083000000005,
+ 19.706542000000013,
+ 19.600291000000027,
+ 19.47666700000002,
+ 19.28383299999996,
+ 19.863500000000045,
+ 20.620416999999975,
+ 21.63929200000001,
+ 20.715625000000045
+ ],
+ "units": 1500,
+ "target_unit_bytes": 54,
+ "worker_rss_kib": 136928
+ },
+ {
+ "arm": "before",
+ "trial": 2,
+ "language": "rust",
+ "count": 1500,
+ "cold_ms": 152.00225,
+ "warm_ms": [
+ 0.5918340000000057,
+ 1.2329170000000147,
+ 0.6595829999999978,
+ 0.6225419999999815,
+ 0.6970829999999921,
+ 0.6174579999999992,
+ 2.4705419999999947,
+ 0.6350419999999986,
+ 0.5098330000000146,
+ 0.5191670000000101
+ ],
+ "units": 4,
+ "target_unit_bytes": 23976,
+ "worker_rss_kib": null
+ },
+ {
+ "arm": "after",
+ "trial": 2,
+ "language": "rust",
+ "count": 1500,
+ "cold_ms": 233.576708,
+ "warm_ms": [
+ 22.596708000000007,
+ 21.197791999999993,
+ 19.434916000000044,
+ 18.358124999999973,
+ 19.243042000000003,
+ 18.87183399999998,
+ 19.183208999999977,
+ 19.496290999999985,
+ 18.825042000000053,
+ 17.756665999999996
+ ],
+ "units": 1500,
+ "target_unit_bytes": 54,
+ "worker_rss_kib": 139296
+ }
+ ],
+ "source_sha256": {
+ "packages/core/src/source.ts": "3d7b820dac964a903c3d6a7e7205e4353b51b301d5b78dab4139276364be112f",
+ "packages/core/src/parser-helpers.mjs": "4d65d7f3d2bf2763c3e407b83f60174f4725be085eecc4461a55b9329a669ece",
+ "packages/core/src/parser-declarations.mjs": "e70973ad70b420fba39dd443a1d5f568e90f26a9687ced7d0b2e9c16f206c9d3"
+ }
+}
diff --git a/specs/go-rust-parsing/measurements.json b/specs/go-rust-parsing/measurements.json
new file mode 100644
index 0000000..f476776
--- /dev/null
+++ b/specs/go-rust-parsing/measurements.json
@@ -0,0 +1,541 @@
+{
+ "baseline": "ae10a9ee8ee55d5a262e04efc5e0ac41c6733a33",
+ "node": "v24.14.0",
+ "platform": "darwin/arm64",
+ "records": [
+ {
+ "arm": "before",
+ "trial": 0,
+ "language": "go",
+ "count": 100,
+ "cold_ms": 137.24620800000002,
+ "warm_ms": [
+ 0.07979199999999764,
+ 0.048082999999991216,
+ 0.058458000000001675,
+ 0.053665999999992664,
+ 0.06212500000000887,
+ 0.043708000000009406,
+ 0.04200000000000159,
+ 0.051124999999984766,
+ 0.12204099999999585,
+ 0.06295800000000895
+ ],
+ "units": 1,
+ "target_unit_bytes": 5195,
+ "worker_rss_kib": null
+ },
+ {
+ "arm": "before",
+ "trial": 1,
+ "language": "go",
+ "count": 100,
+ "cold_ms": 143.433708,
+ "warm_ms": [
+ 0.08291700000000901,
+ 0.04862500000001546,
+ 0.0675420000000031,
+ 0.19645799999997848,
+ 0.060291000000006534,
+ 0.0625840000000153,
+ 0.049000000000006594,
+ 0.05349999999998545,
+ 0.05025000000000546,
+ 0.04816700000000651
+ ],
+ "units": 1,
+ "target_unit_bytes": 5195,
+ "worker_rss_kib": null
+ },
+ {
+ "arm": "before",
+ "trial": 2,
+ "language": "go",
+ "count": 100,
+ "cold_ms": 143.38341599999998,
+ "warm_ms": [
+ 0.08362499999998363,
+ 0.051124999999984766,
+ 0.19695899999999256,
+ 0.07433400000002166,
+ 0.08033299999999599,
+ 0.0503750000000025,
+ 0.04466700000000401,
+ 0.05883300000002123,
+ 0.0449169999999981,
+ 0.05287500000000023
+ ],
+ "units": 1,
+ "target_unit_bytes": 5195,
+ "worker_rss_kib": null
+ },
+ {
+ "arm": "before",
+ "trial": 0,
+ "language": "go",
+ "count": 1500,
+ "cold_ms": 151.97924999999998,
+ "warm_ms": [
+ 1.0316660000000013,
+ 1.353749999999991,
+ 0.626041999999984,
+ 0.6341249999999832,
+ 0.645875000000018,
+ 0.5972919999999817,
+ 0.5961250000000007,
+ 2.3009580000000085,
+ 0.6234579999999994,
+ 0.597124999999977
+ ],
+ "units": 4,
+ "target_unit_bytes": 23976,
+ "worker_rss_kib": null
+ },
+ {
+ "arm": "before",
+ "trial": 1,
+ "language": "go",
+ "count": 1500,
+ "cold_ms": 147.158917,
+ "warm_ms": [
+ 0.5936670000000106,
+ 1.227374999999995,
+ 0.5569580000000087,
+ 0.5495409999999765,
+ 0.551667000000009,
+ 0.5903329999999869,
+ 2.2135419999999897,
+ 0.5703330000000051,
+ 0.49962500000000887,
+ 0.5096670000000074
+ ],
+ "units": 4,
+ "target_unit_bytes": 23976,
+ "worker_rss_kib": null
+ },
+ {
+ "arm": "before",
+ "trial": 2,
+ "language": "go",
+ "count": 1500,
+ "cold_ms": 152.123584,
+ "warm_ms": [
+ 0.6299579999999878,
+ 1.213624999999979,
+ 0.6246669999999881,
+ 0.6185420000000192,
+ 0.6034999999999968,
+ 0.5918329999999798,
+ 0.6312499999999943,
+ 0.7666659999999865,
+ 2.126249999999999,
+ 0.5161250000000166
+ ],
+ "units": 4,
+ "target_unit_bytes": 23976,
+ "worker_rss_kib": null
+ },
+ {
+ "arm": "before",
+ "trial": 0,
+ "language": "rust",
+ "count": 100,
+ "cold_ms": 146.79166700000002,
+ "warm_ms": [
+ 0.08687499999999204,
+ 0.0514169999999865,
+ 0.06166600000000244,
+ 0.059208999999981415,
+ 0.06337500000000773,
+ 0.18091699999999378,
+ 0.1322499999999991,
+ 0.09183299999997985,
+ 0.0681250000000091,
+ 0.057332999999999856
+ ],
+ "units": 1,
+ "target_unit_bytes": 5180,
+ "worker_rss_kib": null
+ },
+ {
+ "arm": "before",
+ "trial": 1,
+ "language": "rust",
+ "count": 100,
+ "cold_ms": 149.737667,
+ "warm_ms": [
+ 0.0831670000000031,
+ 0.05624999999997726,
+ 0.06049999999999045,
+ 0.058041000000002896,
+ 0.05962500000001114,
+ 0.04845799999998235,
+ 0.04087499999999977,
+ 0.06095899999999688,
+ 0.04724999999999113,
+ 0.043042000000014013
+ ],
+ "units": 1,
+ "target_unit_bytes": 5180,
+ "worker_rss_kib": null
+ },
+ {
+ "arm": "before",
+ "trial": 2,
+ "language": "rust",
+ "count": 100,
+ "cold_ms": 148.11629100000002,
+ "warm_ms": [
+ 0.08866700000001515,
+ 0.05441600000000335,
+ 0.061916999999994005,
+ 0.049915999999996075,
+ 0.06829199999998536,
+ 0.051832999999987805,
+ 0.056250000000005684,
+ 0.06733399999998824,
+ 0.05629199999998491,
+ 0.06070800000000531
+ ],
+ "units": 1,
+ "target_unit_bytes": 5180,
+ "worker_rss_kib": null
+ },
+ {
+ "arm": "before",
+ "trial": 0,
+ "language": "rust",
+ "count": 1500,
+ "cold_ms": 149.60775,
+ "warm_ms": [
+ 0.5886250000000075,
+ 1.167082999999991,
+ 0.6224170000000129,
+ 0.5444999999999993,
+ 0.6081250000000011,
+ 0.7984160000000031,
+ 0.6195000000000164,
+ 2.0960830000000215,
+ 0.5019999999999811,
+ 0.49229199999999196
+ ],
+ "units": 4,
+ "target_unit_bytes": 23976,
+ "worker_rss_kib": null
+ },
+ {
+ "arm": "before",
+ "trial": 1,
+ "language": "rust",
+ "count": 1500,
+ "cold_ms": 157.93491699999998,
+ "warm_ms": [
+ 0.6371670000000051,
+ 1.436000000000007,
+ 0.6196659999999952,
+ 0.6478749999999991,
+ 0.6295000000000073,
+ 0.5977090000000089,
+ 0.5875409999999874,
+ 2.2151250000000005,
+ 0.5549579999999992,
+ 0.5406670000000133
+ ],
+ "units": 4,
+ "target_unit_bytes": 23976,
+ "worker_rss_kib": null
+ },
+ {
+ "arm": "before",
+ "trial": 2,
+ "language": "rust",
+ "count": 1500,
+ "cold_ms": 152.00225,
+ "warm_ms": [
+ 0.5918340000000057,
+ 1.2329170000000147,
+ 0.6595829999999978,
+ 0.6225419999999815,
+ 0.6970829999999921,
+ 0.6174579999999992,
+ 2.4705419999999947,
+ 0.6350419999999986,
+ 0.5098330000000146,
+ 0.5191670000000101
+ ],
+ "units": 4,
+ "target_unit_bytes": 23976,
+ "worker_rss_kib": null
+ },
+ {
+ "arm": "after",
+ "trial": 0,
+ "language": "go",
+ "count": 100,
+ "cold_ms": 281.413959,
+ "warm_ms": [
+ 3.7676250000000095,
+ 3.538333000000023,
+ 2.3937080000000037,
+ 2.3475839999999835,
+ 2.265209000000027,
+ 2.2827910000000315,
+ 1.7748750000000086,
+ 1.6493750000000205,
+ 2.4490410000000224,
+ 1.9844590000000153
+ ],
+ "units": 101,
+ "target_unit_bytes": 52,
+ "worker_rss_kib": 87552
+ },
+ {
+ "arm": "after",
+ "trial": 1,
+ "language": "go",
+ "count": 100,
+ "cold_ms": 274.43008299999997,
+ "warm_ms": [
+ 3.634416999999985,
+ 3.4551660000000197,
+ 2.8712919999999826,
+ 2.983542,
+ 2.658042000000023,
+ 2.306041999999991,
+ 2.636665999999991,
+ 2.04200000000003,
+ 2.4957919999999945,
+ 2.1512500000000045
+ ],
+ "units": 101,
+ "target_unit_bytes": 52,
+ "worker_rss_kib": 88176
+ },
+ {
+ "arm": "after",
+ "trial": 2,
+ "language": "go",
+ "count": 100,
+ "cold_ms": 298.107375,
+ "warm_ms": [
+ 4.054875000000038,
+ 3.8750839999999585,
+ 2.976875000000007,
+ 2.90137500000003,
+ 7.425334000000021,
+ 2.4610419999999635,
+ 2.4359159999999633,
+ 2.226165999999978,
+ 2.8776659999999765,
+ 1.939833999999962
+ ],
+ "units": 101,
+ "target_unit_bytes": 52,
+ "worker_rss_kib": 87904
+ },
+ {
+ "arm": "after",
+ "trial": 0,
+ "language": "go",
+ "count": 1500,
+ "cold_ms": 356.514125,
+ "warm_ms": [
+ 28.278250000000014,
+ 27.882791999999995,
+ 26.994541999999967,
+ 27.660124999999994,
+ 29.289207999999974,
+ 26.81933300000003,
+ 27.31016699999998,
+ 30.04458299999999,
+ 26.755333000000064,
+ 26.54733299999998
+ ],
+ "units": 1501,
+ "target_unit_bytes": 54,
+ "worker_rss_kib": 126960
+ },
+ {
+ "arm": "after",
+ "trial": 1,
+ "language": "go",
+ "count": 1500,
+ "cold_ms": 576.862917,
+ "warm_ms": [
+ 58.58487500000001,
+ 48.688290999999936,
+ 53.28912500000001,
+ 55.83275000000003,
+ 50.115125000000035,
+ 49.88945799999999,
+ 40.7941669999999,
+ 40.29383399999995,
+ 37.68033400000013,
+ 44.40812499999993
+ ],
+ "units": 1501,
+ "target_unit_bytes": 54,
+ "worker_rss_kib": 132528
+ },
+ {
+ "arm": "after",
+ "trial": 2,
+ "language": "go",
+ "count": 1500,
+ "cold_ms": 454.14966699999997,
+ "warm_ms": [
+ 35.27245799999997,
+ 31.073709000000008,
+ 28.73512500000004,
+ 27.060791999999992,
+ 33.53875000000005,
+ 30.777041000000054,
+ 30.48791600000004,
+ 31.089208999999983,
+ 27.006000000000085,
+ 26.838499999999954
+ ],
+ "units": 1501,
+ "target_unit_bytes": 54,
+ "worker_rss_kib": 124640
+ },
+ {
+ "arm": "after",
+ "trial": 0,
+ "language": "rust",
+ "count": 100,
+ "cold_ms": 313.84383299999996,
+ "warm_ms": [
+ 4.176040999999998,
+ 3.3393749999999613,
+ 2.789416000000017,
+ 3.03612499999997,
+ 2.5884169999999926,
+ 2.784249999999986,
+ 2.527084000000002,
+ 2.575791000000038,
+ 2.6677500000000123,
+ 2.425374999999974
+ ],
+ "units": 100,
+ "target_unit_bytes": 52,
+ "worker_rss_kib": 98176
+ },
+ {
+ "arm": "after",
+ "trial": 1,
+ "language": "rust",
+ "count": 100,
+ "cold_ms": 275.91083299999997,
+ "warm_ms": [
+ 3.591542000000004,
+ 2.9383339999999976,
+ 2.3992920000000026,
+ 2.5857500000000186,
+ 5.5227500000000305,
+ 2.214916999999957,
+ 2.2708339999999794,
+ 2.4445420000000126,
+ 2.0651659999999765,
+ 1.6691249999999513
+ ],
+ "units": 100,
+ "target_unit_bytes": 52,
+ "worker_rss_kib": 97824
+ },
+ {
+ "arm": "after",
+ "trial": 2,
+ "language": "rust",
+ "count": 100,
+ "cold_ms": 248.10275,
+ "warm_ms": [
+ 3.0264999999999986,
+ 4.40304100000003,
+ 2.2428330000000187,
+ 2.676125000000013,
+ 2.415916999999979,
+ 2.4104580000000055,
+ 1.9123329999999896,
+ 1.8734999999999786,
+ 1.9861659999999688,
+ 1.890874999999994
+ ],
+ "units": 100,
+ "target_unit_bytes": 52,
+ "worker_rss_kib": 97008
+ },
+ {
+ "arm": "after",
+ "trial": 0,
+ "language": "rust",
+ "count": 1500,
+ "cold_ms": 286.304666,
+ "warm_ms": [
+ 27.033957999999984,
+ 25.34395900000004,
+ 23.853083999999967,
+ 23.628750000000025,
+ 23.65779200000003,
+ 23.839292,
+ 24.00958399999996,
+ 25.172166000000004,
+ 25.879500000000007,
+ 22.61445900000001
+ ],
+ "units": 1500,
+ "target_unit_bytes": 54,
+ "worker_rss_kib": 136528
+ },
+ {
+ "arm": "after",
+ "trial": 1,
+ "language": "rust",
+ "count": 1500,
+ "cold_ms": 257.817042,
+ "warm_ms": [
+ 26.579834000000005,
+ 22.514583000000016,
+ 21.54216699999995,
+ 21.478083000000026,
+ 22.332291999999995,
+ 22.055042000000014,
+ 21.191667000000052,
+ 22.786292000000003,
+ 23.906542,
+ 21.541999999999973
+ ],
+ "units": 1500,
+ "target_unit_bytes": 54,
+ "worker_rss_kib": 135392
+ },
+ {
+ "arm": "after",
+ "trial": 2,
+ "language": "rust",
+ "count": 1500,
+ "cold_ms": 234.315,
+ "warm_ms": [
+ 22.147875,
+ 18.847125000000005,
+ 18.749833000000024,
+ 18.745709000000033,
+ 18.414999999999964,
+ 18.300959000000034,
+ 18.435000000000002,
+ 20.005833999999993,
+ 20.14783299999999,
+ 18.70795800000002
+ ],
+ "units": 1500,
+ "target_unit_bytes": 54,
+ "worker_rss_kib": 136384
+ }
+ ],
+ "source_sha256": {
+ "packages/core/src/source.ts": "3d7b820dac964a903c3d6a7e7205e4353b51b301d5b78dab4139276364be112f",
+ "packages/core/src/parser-helpers.mjs": "4d65d7f3d2bf2763c3e407b83f60174f4725be085eecc4461a55b9329a669ece",
+ "packages/core/src/parser-declarations.mjs": "aa7f67d9030b4c5cf5ab5d43e9a2457da02e61f2b225039a85d7ce68a2d7b830"
+ },
+ "comparison": "Final candidate confirmation; the three baseline trials per workload are reused unchanged from development-measurements.json."
+}
diff --git a/test/installed.test.mjs b/test/installed.test.mjs
index 0756abd..b2cb828 100644
--- a/test/installed.test.mjs
+++ b/test/installed.test.mjs
@@ -467,6 +467,73 @@ test("skill command delegates installation to npx without credentials", async (t
assert.match(unavailable.stdout, /requires npx/);
});
+test("installed Go/Rust search indexes late methods and selects their source instead of distant noise", async (t) => {
+ const fixture = await context(t, async ({ body, response }) => {
+ const answers = Object.fromEntries(
+ Object.keys(body.questions).map((id) => {
+ const match = /^(q|scope|ref)(\d+)$/.exec(id);
+ const selected =
+ body.state.declarations && match
+ ? match[1] === "scope" ||
+ (match[1] === "q" &&
+ body.state.declarations[Number(match[2])].name.endsWith(".record_event"))
+ : !id.startsWith("ref");
+ return [id, { type: "noul", noul: selected ? 0.95 : 0.05 }];
+ }),
+ );
+ response.writeHead(200, { "content-type": "application/json" });
+ response.end(JSON.stringify({ answers }));
+ return true;
+ });
+ const files = [
+ [
+ "sample.go",
+ "package sample\n" +
+ Array.from(
+ { length: 160 },
+ (_, i) => `func noise${i}() string { return "DISTANT_NOISE_${i}_${"x".repeat(100)}" }\n`,
+ ).join("") +
+ 'type Box struct {}\nfunc (b *Box) record_event() string { return "go-evidence" }\n',
+ "go-evidence",
+ ],
+ [
+ "sample.rs",
+ Array.from(
+ { length: 160 },
+ (_, i) => `fn noise${i}() -> &'static str { "DISTANT_NOISE_${i}_${"x".repeat(100)}" }\n`,
+ ).join("") +
+ 'struct Box;\n#[allow(dead_code)]\nimpl Box {\n #[inline]\n pub fn record_event(&self) -> &str { "rust-evidence" }\n}\n',
+ "rust-evidence",
+ ],
+ ];
+ for (const [path, source] of files) await writeFile(join(fixture.tree, path), source);
+ const result = await fixture.run([query, fixture.tree, "--no-cache"]);
+ assert.equal(result.code, 0, result.stdout);
+ for (const [path, , marker] of files) {
+ assert.ok(
+ fixture.requests.some(
+ ({ body }) =>
+ body.state.path === path &&
+ body.state.preview?.truncated &&
+ body.state.preview.declarations?.some((d) => d.name === "Box.record_event"),
+ ),
+ `Missing late-method preview for ${path}`,
+ );
+ assert.ok(
+ fixture.requests.some(
+ ({ body }) =>
+ body.state.path === path &&
+ body.state.declarations?.some((d) => d.name === "Box.record_event"),
+ ),
+ `Missing named selection for ${path}`,
+ );
+ assert.ok(result.stdout.includes(marker), result.stdout);
+ }
+ assert.ok(result.stdout.includes("#[allow(dead_code)]"), result.stdout);
+ assert.ok(result.stdout.includes("#[inline]"), result.stdout);
+ assert.ok(!result.stdout.includes("DISTANT_NOISE_0_"), result.stdout);
+});
+
test("actual installed search parses Python and returns every relevant hierarchy branch", async (t) => {
const fixture = await context(t);
const result = await fixture.run([query, fixture.tree, "--no-cache"]);
@@ -991,23 +1058,51 @@ test("source budget preserves every file and lead while explicitly omitting sour
test("missing or corrupt packaged parser assets fail closed without downloads", async (t) => {
const scratch = await mkdtemp(join(tmpdir(), "jg-missing-python-"));
t.after(() => rm(scratch, { recursive: true, force: true }));
- for (const [asset, corrupt] of [
+ for (const [asset, corrupt, extension] of [
["dist/bin/parser-worker.mjs", false],
["dist/assets/tree-sitter/tree-sitter-python.wasm", false],
["node_modules/web-tree-sitter/web-tree-sitter.wasm", false],
["dist/assets/tree-sitter/tree-sitter-python.wasm", true],
+ ["dist/assets/tree-sitter/tree-sitter-go.wasm", false, "go"],
+ ["dist/assets/tree-sitter/tree-sitter-go.wasm", true, "go"],
+ ["dist/assets/tree-sitter/tree-sitter-rust.wasm", false, "rs"],
+ ["dist/assets/tree-sitter/tree-sitter-rust.wasm", true, "rs"],
]) {
const copy = join(scratch, "package");
await cp(packageDirectory, copy, { recursive: true, dereference: true });
await access(join(copy, asset));
if (corrupt) await writeFile(join(copy, asset), "corrupt runtime fixture");
else await rm(join(copy, asset));
- const fixture = await context(t, "healthy", join(copy, "dist/bin/index.js"));
+ const fixture = await context(
+ t,
+ extension
+ ? async ({ body, response }) => {
+ response.writeHead(200, { "content-type": "application/json" });
+ response.end(
+ JSON.stringify({
+ answers: Object.fromEntries(
+ Object.keys(body.questions).map((id) => [id, { type: "noul", noul: 0.95 }]),
+ ),
+ }),
+ );
+ return true;
+ }
+ : "healthy",
+ join(copy, "dist/bin/index.js"),
+ );
+ if (extension)
+ await writeFile(
+ join(fixture.tree, `broken.${extension}`),
+ extension === "go" ? "package sample\nfunc record_event() {}\n" : "fn record_event() {}\n",
+ );
const result = await fixture.run([query, fixture.tree, "--no-cache"]);
assert.equal(result.code, 1, `${asset} (corrupt=${corrupt}): ${result.stdout}`);
assert.ok(result.stdout.trim(), "Asset failure must produce a diagnostic");
assert.ok(
- !fixture.requests.some(({ body }) => body.state.declarations),
+ !fixture.requests.some(
+ ({ body }) =>
+ body.state.declarations && (!extension || body.state.path === `broken.${extension}`),
+ ),
"Unavailable parser assets must not fabricate declaration evidence",
);
await fixture.removeCredentials();
diff --git a/test/parser/README.md b/test/parser/README.md
index 0a2f835..d52e1dc 100644
--- a/test/parser/README.md
+++ b/test/parser/README.md
@@ -23,3 +23,7 @@ misrepresented as malformed user source.
Inherited-call tests preserve useful reading leads when a receiver or callee is
parenthesized, or a selected excerpt contains only a multiline call’s opening
line. They assert returned locations rather than the grammar’s node shape.
+
+Go/Rust tests exercise named selection, declaration groups, generic receivers,
+Rust owner/attribute context, and byte-preserving fallback. Installed coverage
+checks late-method previews, exact selected source, and missing/corrupt grammars.
diff --git a/test/parser/languages.test.ts b/test/parser/languages.test.ts
new file mode 100644
index 0000000..96d7365
--- /dev/null
+++ b/test/parser/languages.test.ts
@@ -0,0 +1,103 @@
+import { test } from "node:test";
+import assert from "node:assert/strict";
+import { inspect, sourceForUnit } from "../../packages/core/src/source.ts";
+
+test("Go selects named methods and preserves declaration groups and Unicode bytes", async () => {
+ const source =
+ "package sample\r\nconst (\r\n First = iota\r\n Second\r\n)\r\ntype Box[T any] struct { value T }\r\nfunc (b *Box[T]) Read() T {\r\n // 🙂\r\n return b.value\r\n}\r\n";
+ const snapshot = { path: "sample.go", source, contentHash: "fixture" };
+ const result = await inspect(snapshot);
+ assert.equal(result.mode, "go");
+ const method = result.units.find((unit) => unit.name === "Box[T].Read");
+ assert.ok(method);
+ assert.equal(sourceForUnit(snapshot, method), source.split("\r\n").slice(6).join("\r\n"));
+ const group = result.units.find((unit) => unit.name === "First, Second");
+ assert.ok(group);
+ assert.equal(sourceForUnit(snapshot, group), "const (\r\n First = iota\r\n Second\r\n)\r\n");
+});
+
+test("Rust methods retain module, impl, attribute and original source context", async () => {
+ const source =
+ '#![allow(dead_code)]\r\n#[cfg(feature = "demo")]\r\nmod inner {\r\n #![allow(unused)]\r\n struct Box;\r\n #[cfg(feature = "demo")]\r\n impl Box {\r\n /// Reads the value.\r\n #[inline]\r\n pub fn read(&self) -> &str { "🙂" }\r\n }\r\n}\r\n';
+ const snapshot = { path: "sample.rs", source, contentHash: "fixture" };
+ const result = await inspect(snapshot);
+ assert.equal(result.mode, "rust");
+ const method = result.units.find((unit) => unit.name === "inner.Box.read");
+ assert.ok(method);
+ assert.deepEqual(method.ownerHeaders, [
+ { startLine: 1, endLine: 1 },
+ { startLine: 2, endLine: 3 },
+ { startLine: 4, endLine: 4 },
+ { startLine: 6, endLine: 7 },
+ ]);
+ assert.equal(
+ sourceForUnit(snapshot, method),
+ ' /// Reads the value.\r\n #[inline]\r\n pub fn read(&self) -> &str { "🙂" }\r\n',
+ );
+});
+
+test("Rust comments between attributes and declarations retain the attributes", async () => {
+ const source =
+ '#[cfg(feature = "demo")]\n// builds only with demo\nmod inner {\n #[inline]\n // useful method\n pub fn read() {}\n}\n';
+ const snapshot = { path: "sample.rs", source, contentHash: "fixture" };
+ const result = await inspect(snapshot);
+ const method = result.units.find((unit) => unit.name === "inner.read");
+ assert.ok(method);
+ assert.deepEqual(method.ownerHeaders, [{ startLine: 1, endLine: 3 }]);
+ assert.equal(
+ sourceForUnit(snapshot, method),
+ " #[inline]\n // useful method\n pub fn read() {}\n",
+ );
+});
+
+test("Rust attributes do not attach the previous declaration's trailing comment", async () => {
+ const source =
+ "fn earlier() {} // about earlier\n#[inline]\n// about target\npub fn target() {}\n";
+ const snapshot = { path: "sample.rs", source, contentHash: "fixture" };
+ const result = await inspect(snapshot);
+ const target = result.units.find((unit) => unit.name === "target");
+ assert.ok(target);
+ assert.equal(sourceForUnit(snapshot, target), "#[inline]\n// about target\npub fn target() {}\n");
+});
+
+test("declaration labels cover grouped Go names and Rust traits, foreign items and macros", async () => {
+ for (const [path, source, names] of [
+ [
+ "sample.go",
+ "package x\nconst Alpha, Beta = 1, 2\ntype (\n Alias = string\n Value struct{}\n)\nvar First, Second int\nvar (\n Third int\n Fourth = 1\n)\nfunc K() {}\nfunc K() {}\n",
+ ["Alpha, Beta", "Alias, Value", "First, Second", "Third, Fourth", "K", "K"],
+ ],
+ [
+ "sample.rs",
+ 'mod inner {\n pub trait View {\n fn show(&self);\n }\n}\nextern "C" {\n fn foreign();\n static VALUE: u8;\n}\nmacro_rules! make { () => {} }\nfn K() {}\nfn K() {}\n',
+ ["inner.View.show", "foreign", "VALUE", "make", "K", "K"],
+ ],
+ ] as const) {
+ const result = await inspect({ path, source, contentHash: "fixture" });
+ for (const name of names)
+ assert.ok(
+ result.units.some((unit) => unit.name === name),
+ name,
+ );
+ }
+});
+
+test("Go/Rust syntax and size fallbacks preserve every Unicode source byte", async () => {
+ for (const [path, source, options, reason] of [
+ ["bad.go", "package x\nfunc broken( { 🙂\n", { maxUnitBytes: 8 }, "syntax"],
+ ["bad.rs", "fn broken( { 🙂\n", { maxUnitBytes: 8 }, "syntax"],
+ [
+ "big.go",
+ 'package x\nfunc target() string { return "🙂" }\r\n',
+ { maxParseBytes: 1, maxUnitBytes: 8 },
+ "size",
+ ],
+ ["big.rs", 'fn target() -> &str { "🙂" }\r\n', { maxParseBytes: 1, maxUnitBytes: 8 }, "size"],
+ ] as const) {
+ const snapshot = { path, source, contentHash: "fixture" };
+ const result = await inspect(snapshot, options);
+ assert.equal(result.fallback, reason);
+ assert.equal(result.units.map((unit) => sourceForUnit(snapshot, unit)).join(""), source);
+ assert.ok(result.units.every((unit) => Buffer.byteLength(sourceForUnit(snapshot, unit)) <= 8));
+ }
+});
diff --git a/test/release.test.mjs b/test/release.test.mjs
index 6892ae0..cc898d2 100644
--- a/test/release.test.mjs
+++ b/test/release.test.mjs
@@ -89,21 +89,26 @@ test("archive validation rejects changed skill bytes and accidental source paylo
LICENSE: "MIT canonical copyright\n",
"skills/jevgrep/SKILL.md": "Use jg for unfamiliar code.\n",
...Object.fromEntries(
- ["parser-worker", "parser-helpers", "parser-preview"].map((name) => [
+ ["parser-worker", "parser-helpers", "parser-preview", "parser-declarations"].map((name) => [
`packages/core/src/${name}.mjs`,
"// fixture",
]),
),
"packages/core/package.json": JSON.stringify({
- devDependencies: { "tree-sitter-python": "1.0.0" },
+ devDependencies: Object.fromEntries(
+ ["python", "go", "rust"].map((name) => [`tree-sitter-${name}`, "1.0.0"]),
+ ),
}),
- "packages/core/node_modules/tree-sitter-python/package.json": JSON.stringify({
- name: "tree-sitter-python",
- version: "1.0.0",
- license: "MIT",
- }),
- "packages/core/node_modules/tree-sitter-python/LICENSE": "MIT fixture grammar license",
- "packages/core/assets/tree-sitter/tree-sitter-python.wasm": "Python grammar fixture",
+ ...Object.fromEntries(
+ ["python", "go", "rust"].flatMap((name) => [
+ [
+ `packages/core/node_modules/tree-sitter-${name}/package.json`,
+ JSON.stringify({ name: `tree-sitter-${name}`, version: "1.0.0", license: "MIT" }),
+ ],
+ [`packages/core/node_modules/tree-sitter-${name}/LICENSE`, "MIT fixture grammar license"],
+ [`packages/core/assets/tree-sitter/tree-sitter-${name}.wasm`, `${name} grammar fixture`],
+ ]),
+ ),
};
for (const [path, text] of Object.entries(files)) {
await mkdir(join(root, path, ".."), { recursive: true });
@@ -118,14 +123,14 @@ test("archive validation rejects changed skill bytes and accidental source paylo
["apps/cli/package.json", "package.json"],
["LICENSE", "dist/LICENSE"],
["skills/jevgrep/SKILL.md", "dist/skills/jevgrep/SKILL.md"],
- ...["parser-worker", "parser-helpers", "parser-preview"].map((name) => [
+ ...["parser-worker", "parser-helpers", "parser-preview", "parser-declarations"].map((name) => [
`packages/core/src/${name}.mjs`,
`dist/bin/${name}.mjs`,
]),
- [
- "packages/core/assets/tree-sitter/tree-sitter-python.wasm",
- "dist/assets/tree-sitter/tree-sitter-python.wasm",
- ],
+ ...["python", "go", "rust"].map((name) => [
+ `packages/core/assets/tree-sitter/tree-sitter-${name}.wasm`,
+ `dist/assets/tree-sitter/tree-sitter-${name}.wasm`,
+ ]),
])
await cp(join(root, from), join(pkg, to));
await writeFile(join(pkg, "dist/bin/index.js"), '#!/usr/bin/env node\nconsole.log("jg");\n');
@@ -169,9 +174,9 @@ test("archive validation rejects changed skill bytes and accidental source paylo
await assert.rejects(validateRelease(tarball, "v1.2.3", root), /Unexpected published file/);
});
-test("grammar notices retain the official Python parser license", async () => {
+test("grammar notices retain all official parser licenses", async () => {
const { grammarAssets } = await import("../scripts/parser-assets.mjs");
const notices = await grammarAssets();
- assert.match(notices, /tree-sitter-python@/);
+ for (const name of ["python", "go", "rust"]) assert.ok(notices.includes(`tree-sitter-${name}@`));
assert.match(notices, /Permission is hereby granted/);
});