Find code by asking what it does. A CLI for coding agents that uses Jev to discover relevant files and source context.
TypeScript ai-sdkclaude-codeclicode-searchcodexcoding-agentscontext-retrievaldeveloper-toolsjevsemantic-searchtypescriptvercel-ai-gateway
Parse extensionless Python and Node scripts by their shebang
Fred Amartey051169ed7cd7
diff --git a/README.md b/README.mdindex 63f3422..71e0195 100644--- a/README.md+++ b/README.md@@ -89,7 +89,8 @@ list.The summary and compact file list come first, followed by selected source withline references, then detailed declaration and call locations. Python, TypeScript/JavaScript, Go and Rust support declaration-parsing; other text uses a fallback. The output is evidence for the agent to use,+parsing, with extensionless Python and Node scripts recognised by their shebang;+other text uses a fallback. The output is evidence for the agent to use,not a generated answer or a guarantee that every relevant file was found.[See a recorded output example](specs/done/jevgrep/assets/stdout-example.txt).diff --git a/packages/core/src/call-context.ts b/packages/core/src/call-context.tsindex 976612d..1a2f9ff 100644--- a/packages/core/src/call-context.ts+++ b/packages/core/src/call-context.ts@@ -1,4 +1,5 @@import { runParser } from "./parser";+import { resolveLanguage } from "./source";import type { Snapshot } from "./filesystem";import type { FileEvidence, Range } from "./types";type Call = Range & {@@ -14,7 +15,7 @@ export async function localCallContext() {const original = file.presentationExcerpts ?? file.excerpts;if (- !/\.pyi?$/.test(snapshot.path) ||+ resolveLanguage(snapshot.path, snapshot.source) !== "python" ||Buffer.byteLength(snapshot.source) > 1_000_000 ||file.sourceOmitted ||original.some((e) => e.partial || e.sourceByteStart !== undefined)diff --git a/packages/core/src/parser-helpers.mjs b/packages/core/src/parser-helpers.mjsindex b06d262..28bd880 100644--- a/packages/core/src/parser-helpers.mjs+++ b/packages/core/src/parser-helpers.mjs@@ -432,7 +432,7 @@ export async function execute(helper, input) {data.query.normalize("NFKC").match(/[_\p{ID_Start}][_\p{ID_Continue}]*/gu) ?? [],);let matches = [];- if (/\.pyi?$/.test(data.path) && Buffer.byteLength(data.text) > data.budget)+ if (data.language === "python" && Buffer.byteLength(data.text) > data.budget)matches = await parse(data.text, (root) => previewMatches(root, tokens));return preview(data, matches);}diff --git a/packages/core/src/retrieve.ts b/packages/core/src/retrieve.tsindex efab99e..8c8ce0d 100644--- a/packages/core/src/retrieve.ts+++ b/packages/core/src/retrieve.ts@@ -1,7 +1,7 @@import { basename, extname } from "node:path";import { createFilesystem, type Snapshot, type DirectoryEntry } from "./filesystem";import { type EvaluationRequest, EvaluationFailure } from "./evaluator";-import { inspect, pythonPreview, sourceForUnit, splitSource } from "./source";+import { inspect, pythonPreview, resolveLanguage, sourceForUnit, splitSource } from "./source";import {navigationRequest,fileAssessmentRequest,@@ -304,6 +304,7 @@ export async function retrieve(input: SearchInput, evaluator: Evaluator): Promisreturn buffered({ ...item, childPreview: preview }, sources);}async function previewFile(source: Snapshot): Promise<FilePreview> {+ const language = resolveLanguage(source.path, source.source);const bytes = Buffer.from(source.source);let text = new TextDecoder("utf8", { fatal: true }).decode(bytes.subarray(0, 16384), {stream: bytes.length > 16384,@@ -326,7 +327,7 @@ export async function retrieve(input: SearchInput, evaluator: Evaluator): Promisdeclarations: [],declarationIndexTruncated: false,};- if (truncated && /\.pyi?$/.test(source.path) && bytes.length <= 1_000_000) {+ if (truncated && language === "python" && bytes.length <= 1_000_000) {const sampled = await pythonPreview(source, input.query, 16384, input.signal);if (sampled?.truncated &&@@ -339,11 +340,7 @@ export async function retrieve(input: SearchInput, evaluator: Evaluator): Promispreview.range = "sampled source ranges";}}- if (- truncated &&- /\.(?:pyi?|[cm]?[jt]s|[jt]sx|go|rs)$/.test(source.path) &&- bytes.length <= 1_000_000- ) {+ if (truncated && language !== undefined && bytes.length <= 1_000_000) {const syntax = await inspect(source, {signal: input.signal,maxUnitBytes: Math.max(4, bytes.length),diff --git a/packages/core/src/source.ts b/packages/core/src/source.tsindex efda028..47a9e17 100644--- a/packages/core/src/source.ts+++ b/packages/core/src/source.ts@@ -1,4 +1,5 @@import ts from "typescript";+import { basename, extname } from "node:path";import { runParser } from "./parser.ts";export type Range = { startLine: number; endLine: number };@@ -19,6 +20,29 @@ export type Inspection = {fallback?: "unsupported" | "syntax" | "size";};+export function resolveLanguage(+ path: string,+ source: string,+): "python" | "go" | "rust" | "TS" | "TSX" | "JS" | "JSX" | undefined {+ if (/\.pyi?$/.test(path)) return "python";+ if (path.endsWith(".tsx")) return "TSX";+ if (path.endsWith(".jsx")) return "JSX";+ if (/\.[cm]?js$/.test(path)) return "JS";+ if (/\.[cm]?ts$/.test(path)) return "TS";+ if (path.endsWith(".go")) return "go";+ if (path.endsWith(".rs")) return "rust";+ if (extname(path) !== "" || !source.startsWith("#!")) return;+ const [program = "", ...args] = source.split("\n", 1)[0]!.slice(2).trim().split(/\s+/);+ // env options, the operand of -u/-C/-P/-a and NAME=VALUE assignments come before the command.+ const command = args.find(+ (arg, index) =>+ !arg.startsWith("-") && !arg.includes("=") && !/^-[uCPa]$/.test(args[index - 1] ?? ""),+ );+ const interpreter = basename(basename(program) === "env" ? (command ?? "") : program);+ if (/^python(\d+(\.\d+)?)?$/.test(interpreter)) return "python";+ if (interpreter === "node" || interpreter === "nodejs") return "JS";+}+type SourceText = { bytes: Buffer; offsets: number[]; lineCount: number };function sourceText(source: string): SourceText {const lines = source.split("\n"),@@ -82,15 +106,17 @@ export async function inspect("Source bounds must be integers; unit bytes at least 4 and parse bytes positive",);const { source, path } = snapshot;+ const language = resolveLanguage(path, source);const text = sourceText(source);// Context windows use conservative whole-line Python comments, including inside multiline strings.- const pythonComments = /\.pyi?$/.test(path)- ? source- .split("\n")- .flatMap((line, index) =>- /^\s*#/.test(line) ? [{ startLine: index + 1, endLine: index + 1 }] : [],- )- : [];+ const pythonComments =+ language === "python"+ ? source+ .split("\n")+ .flatMap((line, index) =>+ /^\s*#/.test(line) ? [{ startLine: index + 1, endLine: index + 1 }] : [],+ )+ : [];const fallback = (reason: Inspection["fallback"]): Inspection => ({mode: "text",fallback: reason,@@ -104,7 +130,7 @@ export async function inspect(comments: Range[] = [],mode: Inspection["mode"];let syntaxFallback = false;- if (/\.pyi?$/.test(path)) {+ if (language === "python") {// A missing/incompatible packaged parser is a setup failure, never syntax fallback.const parsed = await runParser<Array<Range & { name: string; ownerHeaders: Range[] }>>("inspect",@@ -119,15 +145,14 @@ export async function inspect(}));comments = pythonComments;mode = "python";- } else if (/\.(?:[cm]?[jt]s|[jt]sx)$/.test(path)) {- const kind = path.endsWith(".tsx")- ? ts.ScriptKind.TSX- : path.endsWith(".jsx")- ? ts.ScriptKind.JSX- : /\.[cm]?js$/.test(path)- ? ts.ScriptKind.JS- : ts.ScriptKind.TS;- const file = ts.createSourceFile(path, source, ts.ScriptTarget.Latest, true, kind);+ } else if (language === "TS" || language === "TSX" || language === "JS" || language === "JSX") {+ const file = ts.createSourceFile(+ path,+ source,+ ts.ScriptTarget.Latest,+ true,+ ts.ScriptKind[language],+ );syntaxFallback = !!(file as ts.SourceFile & { parseDiagnostics?: unknown[] }).parseDiagnostics?.length;const line = (position: number) => file.getLineAndCharacterOfPosition(position).line + 1;@@ -167,16 +192,16 @@ export async function inspect(};visit(file);mode = "typescript";- } else if (/\.(go|rs)$/.test(path)) {+ } else if (language === "go" || language === "rust") {const parsed = await runParser<{ units: typeof units; comments: Range[] }>("declarations",- JSON.stringify({ source, language: path.endsWith(".go") ? "go" : "rust" }),+ JSON.stringify({ source, language }),options.signal,);if (!parsed) return fallback("syntax");units = parsed.units;comments = parsed.comments;- mode = path.endsWith(".go") ? "go" : "rust";+ mode = language;} else return fallback("unsupported");comments = [...new Map(comments.map((r) => [`${r.startLine}:${r.endLine}`, r])).values()].sort((a, b) => a.startLine - b.startLine,@@ -233,7 +258,11 @@ export async function pythonNeighborhood(selected: Range[],signal?: AbortSignal,): Promise<Range[]> {- if (!/\.pyi?$/.test(snapshot.path) || Buffer.byteLength(snapshot.source) > 1_000_000) return [];+ if (+ resolveLanguage(snapshot.path, snapshot.source) !== "python" ||+ Buffer.byteLength(snapshot.source) > 1_000_000+ )+ return [];return ((await runParser<Range[]>("neighborhood",@@ -275,7 +304,12 @@ export async function pythonPreview(throw new Error("Preview allowance must be at least 256 bytes");const result = await runParser<SourcePreview>("preview",- JSON.stringify({ text: snapshot.source, path: snapshot.path, query, budget }),+ JSON.stringify({+ text: snapshot.source,+ language: resolveLanguage(snapshot.path, snapshot.source),+ query,+ budget,+ }),signal,);if (result?.truncated) {diff --git a/packages/core/src/test-body-selection.ts b/packages/core/src/test-body-selection.tsindex 241c55f..1796f06 100644--- a/packages/core/src/test-body-selection.ts+++ b/packages/core/src/test-body-selection.ts@@ -1,4 +1,4 @@-import { inspect } from "./source";+import { inspect, resolveLanguage } from "./source";import type { Snapshot } from "./filesystem";import type { Evaluator, FileEvidence, Range } from "./types";import { EvaluationFailure } from "./evaluator";@@ -26,7 +26,7 @@ export async function selectTestBodies(if (!file.roles.includes("test") ||file.sourceOmitted ||- !/\.pyi?$/.test(snapshot.path) ||+ resolveLanguage(snapshot.path, snapshot.source) !== "python" ||Buffer.byteLength(snapshot.source) > 1_000_000 ||shown.some((e) => e.partial || e.sourceByteStart !== undefined))diff --git a/test/parser/parity.test.ts b/test/parser/parity.test.tsindex cad3f83..a19d8a5 100644--- a/test/parser/parity.test.ts+++ b/test/parser/parity.test.ts@@ -34,7 +34,13 @@ test("Python declaration, preview, neighborhood and inherited-call outputs match? source: JSON.stringify(helper === "preview"- ? { path: "x.py", text: source, query: "target first K", budget: 600 }+ ? {+ path: "x.py",+ language: "python",+ text: source,+ query: "target first K",+ budget: 600,+ }: {source,ranges: [diff --git a/test/parser/source.test.ts b/test/parser/source.test.tsindex caa5674..01f7e85 100644--- a/test/parser/source.test.ts+++ b/test/parser/source.test.ts@@ -1,6 +1,44 @@import { test } from "node:test";import assert from "node:assert/strict";-import { inspect } from "../../packages/core/src/source.ts";+import { inspect, pythonNeighborhood } from "../../packages/core/src/source.ts";++test("extensionless Python shebangs preserve declarations, comments and neighboring source", async () => {+ for (const shebang of [+ "#!/usr/bin/env python3",+ "#!/usr/bin/env -S python3 -u",+ "#!/usr/bin/env -S PYTHONUNBUFFERED=1 python3",+ "#!/usr/bin/env -S -u PYTHONPATH python3",+ "#! /usr/bin/python3.12\r",+ ]) {+ const source =+ shebang ++ '\n# module\nclass Example:\n """context"""\n def first(self):\n return "é"\n @decorator\n def middle(self):\n return self.first()\n def last(self):\n pass\n';+ const snapshot = { path: "bin/tool", contentHash: "fixture", source };+ const withExtension = { ...snapshot, path: "bin/tool.py" };+ const expected = await inspect(withExtension);+ assert.equal(expected.mode, "python");+ assert.deepEqual(await inspect(snapshot), expected, shebang);+ const selected = [{ startLine: 8, endLine: 9 }];+ const neighbors = await pythonNeighborhood(withExtension, selected);+ assert.ok(neighbors.some((range) => range.startLine === 5 && range.endLine === 6));+ assert.deepEqual(await pythonNeighborhood(snapshot, selected), neighbors, shebang);+ }+});++test("extensionless Node shebangs preserve JavaScript declarations and selected source", async () => {+ for (const shebang of [+ "#!/usr/bin/env node",+ "#!/usr/bin/nodejs",+ "#!/usr/bin/env -S NODE_ENV=production node",+ ]) {+ const source = shebang + '\n// module\nfunction target() {\n return "🙂";\n}\n';+ const snapshot = { path: "bin/tool", contentHash: "fixture", source };+ const expected = await inspect({ ...snapshot, path: "bin/tool.js" });+ assert.equal(expected.mode, "typescript");+ assert.deepEqual(await inspect(snapshot), expected, shebang);+ assert.equal(expected.units[0]!.name, "target");+ }+});test("Python preserves decorators, class context and nested declarations", async () => {const source =@@ -58,6 +96,11 @@ test("syntax errors and unsupported source fall back without losing source lines["bad.py", "def broken(:\n return 2\n", "syntax"],["bad.ts", "const = ;", "syntax"],["readme.md", "éé\r\nnext\r\nlast", "unsupported"],+ ["tool", "def target():\n pass\n", "unsupported"],+ ["tool", "#!/bin/bash\ndef target():\n pass\n", "unsupported"],+ ["tool", "\n#!/usr/bin/env python3\ndef target():\n pass\n", "unsupported"],+ ["tool.sh", "#!/usr/bin/env python3\ndef target():\n pass\n", "unsupported"],+ ["tool.js", "#!/usr/bin/env python3\ndef target():\n pass\n", "syntax"],]) {const result = await inspect({ path: path!, source: source!, contentHash: "fixture" },diff --git a/test/retrieval.test.ts b/test/retrieval.test.tsindex 4c02a17..f0ff617 100644--- a/test/retrieval.test.ts+++ b/test/retrieval.test.ts@@ -1,6 +1,6 @@import { renderResult } from "../apps/cli/src/render";import { routeProviderFetch } from "./fixtures/provider-route.mjs";-import { expect } from "bun:test";+import { expect, test } from "bun:test";import { mkdtemp, mkdir, writeFile, rm } from "node:fs/promises";import { tmpdir } from "node:os";import { join } from "node:path";@@ -8,6 +8,82 @@ import { testIfDocker } from "./helpers/docker";import { retrieve } from "../packages/core/src/retrieve";import { createEvaluator, EvaluationFailure } from "../packages/core/src/evaluator";+test("extensionless scripts preserve extension-based provider contents and returned evidence", async () => {+ for (const [extension, shebang, body] of [+ [+ ".py",+ "#!/usr/bin/env -S python3 -u",+ 'class Base:\n def helper(self):\n return "🙂"\nclass Child(Base):\n def test_target(self):\n assert self.helper() == "🙂"\n',+ ],+ [".js", "#!/usr/bin/env node", 'function target() {\n return "🙂";\n}\n'],+ ]) {+ const root = await mkdtemp(join(tmpdir(), "jg-shebang-"));+ const padding = `${extension === ".py" ? "#" : "//"} ${"padding ".repeat(12)}\n`;+ const source = `${shebang}\n${padding.repeat(200)}${body}`;+ const runs = [];+ try {+ for (const path of [`tool${extension}`, "tool"]) {+ const requests: string[] = [];+ await writeFile(join(root, path), source);+ const signal = new AbortController().signal;+ const result = await retrieve(+ { root, query: "test_target target", signal },+ createEvaluator({+ provider: "vercel",+ apiKey: "fixture",+ signal,+ fetch: async (_input, init) => {+ const request = String(init?.body);+ requests.push(request);+ const body = JSON.parse(request) as {+ questions: Record<string, { instructions: string }>;+ };+ return Response.json({+ answers: Object.fromEntries(+ Object.entries(body.questions).map(([id, question]) => [+ id,+ {+ type: "noul",+ noul: question.instructions.includes("(Base.helper,") ? 0.1 : 0.95,+ },+ ]),+ ),+ });+ },+ }),+ );+ expect(result.status).toBe("complete");+ expect(requests.join("\n")).toContain(`"extension":"${path === "tool" ? "" : extension}"`);+ runs.push({+ requests: requests.map((request) =>+ request+ .replaceAll(`tool${extension}`, "tool")+ .replaceAll(`"extension":"${extension}"`, '"extension":""'),+ ),+ files: JSON.parse(JSON.stringify(result.files).replaceAll(`tool${extension}`, "tool")),+ });+ await rm(join(root, path));+ }+ expect(runs[0]!.requests.join("\n")).toContain('"name":');+ if (extension === ".py") {+ expect(runs[0]!.requests.join("\n")).toContain('"range":"sampled source ranges"');+ expect(runs[0]!.requests.join("\n")).toContain('"candidates":');+ expect(runs[0]!.files[0].callLeads).toEqual([+ {+ caller: "Child.test_target",+ name: "Base.helper",+ range: { startLine: 203, endLine: 204 },+ unknownEarlierBases: [],+ },+ ]);+ }+ expect(runs[1]).toEqual(runs[0]);+ } finally {+ await rm(root, { recursive: true, force: true });+ }+ }+});+testIfDocker("hierarchical retrieval returns source from multiple files without uploading excluded data",async () => {