Find code by asking what it does. A CLI for coding agents that uses Jev to discover relevant files and source context.
TypeScript ai-sdkclaude-codeclicode-searchcodexcoding-agentscontext-retrievaldeveloper-toolsjevsemantic-searchtypescriptvercel-ai-gateway
import { test } from "node:test";
import assert from "node:assert/strict";
import {
inspect,
pythonNeighborhood,
sourceForUnit,
splitSource,
} from "../../packages/core/src/source.ts";
test("extensionless Python shebangs preserve declarations, comments and neighboring source", async () => {
for (const shebang of [
"#!/usr/bin/env python3",
"#!/usr/bin/env -S python3 -u",
"#!/usr/bin/env -S PYTHONUNBUFFERED=1 python3",
"#!/usr/bin/env -S -u PYTHONPATH python3",
"#! /usr/bin/python3.12\r",
]) {
const source =
shebang +
'\n# module\nclass Example:\n """context"""\n def first(self):\n return "é"\n @decorator\n def middle(self):\n return self.first()\n def last(self):\n pass\n';
const snapshot = { path: "bin/tool", contentHash: "fixture", source };
const withExtension = { ...snapshot, path: "bin/tool.py" };
const expected = await inspect(withExtension);
assert.equal(expected.mode, "python");
assert.deepEqual(await inspect(snapshot), expected, shebang);
const selected = [{ startLine: 8, endLine: 9 }];
const neighbors = await pythonNeighborhood(withExtension, selected);
assert.ok(neighbors.some((range) => range.startLine === 5 && range.endLine === 6));
assert.deepEqual(await pythonNeighborhood(snapshot, selected), neighbors, shebang);
}
});
test("extensionless Node shebangs preserve JavaScript declarations and selected source", async () => {
for (const shebang of [
"#!/usr/bin/env node",
"#!/usr/bin/nodejs",
"#!/usr/bin/env -S NODE_ENV=production node",
]) {
const source = shebang + '\n// module\nfunction target() {\n return "🙂";\n}\n';
const snapshot = { path: "bin/tool", contentHash: "fixture", source };
const expected = await inspect({ ...snapshot, path: "bin/tool.js" });
assert.equal(expected.mode, "typescript");
assert.deepEqual(await inspect(snapshot), expected, shebang);
assert.equal(expected.units[0]!.name, "target");
}
});
test("Python preserves decorators, class context and nested declarations", async () => {
const source =
'# module\n@decorator\nclass Café:\n """docs"""\n value = 1\n @property\n def first(self):\n def nested():\n return "é"\n return nested()\n\n class Inner:\n def method(self):\n pass\n tail = 2\n';
const result = await inspect({ path: "sample.py", contentHash: "fixture", source });
assert.equal(result.mode, "python");
assert.deepEqual(
result.units.map(({ name, range }) => ({ name, ...range })),
[
{ name: "Café.context", startLine: 2, endLine: 5 },
{ name: "Café.first", startLine: 6, endLine: 10 },
{ name: "Café.context", startLine: 11, endLine: 11 },
{ name: "Café.Inner.context", startLine: 12, endLine: 12 },
{ name: "Café.Inner.method", startLine: 13, endLine: 14 },
{ name: "Café.context", startLine: 15, endLine: 15 },
],
);
assert.deepEqual(result.comments, [{ startLine: 1, endLine: 1 }]);
});
test("comments between adjacent byte strings preserve structural source selection", async () => {
const source = "def target():\n return (b'a' # comment\n b'b')\n";
const snapshot = { path: "sample.py", contentHash: "fixture", source };
const result = await inspect(snapshot);
assert.equal(result.mode, "python");
assert.deepEqual(
result.units.map(({ name, range }) => ({ name, ...range })),
[{ name: "target", startLine: 1, endLine: 3 }],
);
const { sourceForUnit } = await import("../../packages/core/src/source.ts");
assert.equal(sourceForUnit(snapshot, result.units[0]!), source);
});
test("parenthesized deletion targets preserve Python declaration ranges", async () => {
for (const target of [
"(self.cached)",
"(\n # target\n self.cached\n )",
"((items[0]))",
]) {
const source = `def target(self, items):\n del ${target}\n`;
const snapshot = { path: "sample.py", contentHash: "fixture", source };
const result = await inspect(snapshot);
assert.equal(result.mode, "python");
assert.deepEqual(
result.units.map(({ name, range }) => ({ name, ...range })),
[{ name: "target", startLine: 1, endLine: source.trimEnd().split("\n").length }],
);
const { sourceForUnit } = await import("../../packages/core/src/source.ts");
assert.equal(sourceForUnit(snapshot, result.units[0]!), source);
}
});
test("syntax errors and unsupported source fall back without losing source lines", async () => {
for (const [path, source, reason] of [
["bad.py", "def broken(:\n return 2\n", "syntax"],
["bad.ts", "const = ;", "syntax"],
["readme.md", "éé\r\nnext\r\nlast", "unsupported"],
["tool", "def target():\n pass\n", "unsupported"],
["tool", "#!/bin/bash\ndef target():\n pass\n", "unsupported"],
["tool", "\n#!/usr/bin/env python3\ndef target():\n pass\n", "unsupported"],
["tool.sh", "#!/usr/bin/env python3\ndef target():\n pass\n", "unsupported"],
["tool.js", "#!/usr/bin/env python3\ndef target():\n pass\n", "syntax"],
]) {
const result = await inspect(
{ path: path!, source: source!, contentHash: "fixture" },
{ maxUnitBytes: 8 },
);
assert.equal(result.mode, "text");
assert.equal(result.fallback, reason);
assert.equal(result.units[0]!.range.startLine, 1);
assert.equal(result.units.at(-1)!.range.endLine, source!.split("\n").length);
}
});
test("TypeScript syntax fallback retains block-comment ranges and exact source bytes", async () => {
const source =
"/*\n" + ("comment " + "x".repeat(70) + "\n").repeat(100) + "*/\nconst broken = ;\n";
const snapshot = { path: "broken.ts", source, contentHash: "fixture" };
const result = await inspect(snapshot, { maxUnitBytes: 3000 });
assert.equal(result.mode, "text");
assert.equal(result.fallback, "syntax");
assert.deepEqual(result.comments, [{ startLine: 1, endLine: 102 }]);
const { sourceForUnit } = await import("../../packages/core/src/source.ts");
assert.equal(result.units.map((unit) => sourceForUnit(snapshot, unit)).join(""), source);
});
test("oversized Unicode lines are losslessly split into bounded byte spans", async () => {
const source = "é漢🙂".repeat(10) + "\r\nend";
const result = await inspect(
{ path: "large.txt", source, contentHash: "fixture" },
{ maxUnitBytes: 16 },
);
const bytes = Buffer.from(source);
const pieces = result.units.map((unit) =>
bytes.subarray(unit.sourceByteStart, unit.sourceByteEnd),
);
assert.ok(pieces.every((piece) => piece.length <= 16));
assert.equal(
pieces.map((piece) => new TextDecoder("utf-8", { fatal: true }).decode(piece)).join(""),
source,
);
assert.ok(result.units.every((unit) => unit.partial));
});
test("selected methods add only their class header and small immediate neighbors", async () => {
const { pythonNeighborhood } = await import("../../packages/core/src/source.ts");
const source =
'class Example:\n """context"""\n def first(self):\n pass\n @decorator\n def middle(self):\n pass\n def last(self):\n pass\n';
assert.deepEqual(
await pythonNeighborhood({ path: "example.py", source, contentHash: "fixture" }, [
{ startLine: 6, endLine: 7 },
]),
[
{ startLine: 1, endLine: 2 },
{ startLine: 3, endLine: 4 },
{ startLine: 8, endLine: 9 },
],
);
});
test("TS/JS uses original source coordinates, comments, and decorator-bearing members", async () => {
const source =
'/** header */\r\nexport class Box {\r\n // getter\r\n @trace\r\n get value() { return "🙂"; }\r\n}\r\n';
const result = await inspect({ path: "box.ts", source, contentHash: "fixture" });
assert.equal(result.mode, "typescript");
assert.deepEqual(
result.units.map(({ name, range }) => ({ name, ...range })),
[
{ name: "Box.context", startLine: 2, endLine: 3 },
{ name: "Box.value", startLine: 4, endLine: 5 },
],
);
assert.deepEqual(result.comments, [
{ startLine: 1, endLine: 1 },
{ startLine: 3, endLine: 3 },
]);
});
test("unsupported Python 2 syntax falls back across inspection, preview and neighbors", async () => {
const { pythonNeighborhood, pythonPreview, sourceForUnit } =
await import("../../packages/core/src/source.ts");
for (const source of [
'def target():\n print "old"\n',
'def target():\n exec "x=1" in globals(), locals()\n',
"try:\n pass\nexcept Exception, e:\n pass\n",
'raise Error, "message", traceback\n',
"value = left <> right\n",
"value = 123L\n",
"value = 0xFFl\n",
"value = 0755\n",
"value = 0_1\n",
"value = `expression`\n",
'value = ur"text"\n',
'value = rU"text"\n',
"def target((first, second)):\n return first\n",
"def target((first, second)=(1, 2)):\n return first\n",
"value = lambda (first, second): first\n",
]) {
const snapshot = {
path: "old.py",
source: source + "# padding\n".repeat(60),
contentHash: "fixture",
};
const result = await inspect(snapshot);
assert.equal(result.mode, "text", source);
assert.equal(result.fallback, "syntax", source);
assert.equal(
result.units.map((unit) => sourceForUnit(snapshot, unit)).join(""),
snapshot.source,
);
assert.deepEqual(
await pythonNeighborhood(snapshot, [{ startLine: 1, endLine: 2 }]),
[],
source,
);
const preview = await pythonPreview(snapshot, "target", 512);
assert.equal(preview.parseUnavailable, true, source);
assert.equal(preview.matchedDeclarations, 0, source);
}
});
test("Python 3 equivalents and Python 2-looking strings remain parsed", async () => {
for (const source of [
"type Alias = int\n",
"def target[T](value: T):\n return value\n",
'value = f"{mapping["key"]}"\n',
"value = f\"{'\\n'}\"\n",
'value = f"""{value # comment\n}"""\n',
'value = f"{(value +\n other)}"\n',
"value = f'{1:{mapping['width']}}'\n",
"value = f'{1:{\"\\n\"}}'\n",
'print("old")\n',
"print >> stream, value\n",
"print +1\nprint [1,2]\n",
"value = f\"{mapping['key']}\"\n",
'value = f"""{mapping["key"]}"""\n',
'value = f"""{(value +\n other)}"""\n',
"value = f\"{'#value'}\"\n",
"value = f'{1:\\\n}'\n",
"type = int\ndef target(value: int):\n return value\n",
'exec("x=1", globals(), locals())\n',
"try:\n pass\nexcept Exception as e:\n pass\n",
"try:\n pass\nexcept (TypeError, ValueError):\n pass\n",
'raise Error("message") from cause\n',
"raise (Error, value)\n",
"value = 0o755 + 0xFF + 0b11 + 000 + 0_0\n",
'value = u"text" + r"raw" + rb"bytes".decode()\n',
"def target(value=[a for (a,b) in pairs]):\n return value\n",
"for (a,b) in pairs:\n pass\n",
'# print "old"; exec "x"; 123L\ntext = "ur\\\"text\\\" <> `value`"\n',
]) {
const result = await inspect({ path: "modern.py", source, contentHash: "fixture" });
assert.equal(result.mode, "python", source);
}
});
test("trailing Python comments stay separate from AST declaration ends", async () => {
const source = "class A:\n def x(self):\n pass\n # tail\n";
const result = await inspect({ path: "a.py", source, contentHash: "fixture" });
assert.deepEqual(
result.units.map(({ name, range }) => ({ name, ...range })),
[
{ name: "A.context", startLine: 1, endLine: 1 },
{ name: "A.x", startLine: 2, endLine: 3 },
],
);
assert.deepEqual(result.comments, [{ startLine: 4, endLine: 4 }]);
});
test("an f-string expression is implementation, not a Python docstring", async () => {
const { pythonPreview } = await import("../../packages/core/src/source.ts");
const source = 'def target():\n f"compute {value}"\n return 2\n' + "# filler\n".repeat(120);
const preview = await pythonPreview(
{ path: "a.py", source, contentHash: "fixture" },
"target",
600,
);
assert.ok(
preview.spans.some(
(span) => span.basis === "query-named implementation" && span.startLine === 2,
),
);
});
test("parse ceiling still returns complete bounded text without losing CRLF", async () => {
const { sourceForUnit } = await import("../../packages/core/src/source.ts");
const source = 'def example():\r\n return "🙂"\r\n';
const snapshot = { path: "big.py", source, contentHash: "fixture" };
const result = await inspect(snapshot, { maxParseBytes: 12, maxUnitBytes: 8 });
assert.equal(result.fallback, "size");
assert.equal(result.mode, "text");
assert.equal(result.units.map((unit) => sourceForUnit(snapshot, unit)).join(""), source);
assert.ok(result.units.every((unit) => Buffer.byteLength(sourceForUnit(snapshot, unit)) <= 8));
});
test("query previews skip parenthesized Python docstrings", async () => {
const { pythonPreview } = await import("../../packages/core/src/source.ts");
const source =
'def target():\n ("""' +
"docs ".repeat(250) +
'""")\n return "IMPORTANT"\n' +
"# filler\n".repeat(400);
const result = await pythonPreview(
{ path: "a.py", source, contentHash: "fixture" },
"target",
600,
);
assert.ok(
result.spans.some(
(span) =>
span.basis === "query-named implementation" &&
span.startLine === 3 &&
span.text.includes("IMPORTANT"),
),
);
});
test("Python identifiers match normalized query names without changing source", async () => {
const { pythonPreview } = await import("../../packages/core/src/source.ts");
const source =
"# opening\n".repeat(40) + 'def K():\n return "important"\n' + "# filler\n".repeat(100);
const snapshot = { path: "a.py", source, contentHash: "fixture" };
assert.equal((await inspect(snapshot)).units[0]!.name, "K");
for (const query of ["K", "K"]) {
const result = await pythonPreview(snapshot, query, 600);
assert.equal(result.matchedDeclarations, 1);
assert.ok(
result.spans.some(
(span) => span.basis === "query-named implementation" && span.text.includes("important"),
),
);
}
});
test("empty Python suites fall back instead of crashing query previews", async () => {
const { pythonPreview } = await import("../../packages/core/src/source.ts");
const source = "def f():\n" + "\n".repeat(700),
snapshot = { path: "a.py", source, contentHash: "fixture" };
const result = await pythonPreview(snapshot, "f", 600);
assert.equal(result.parseUnavailable, true);
assert.equal(result.matchedDeclarations, 0);
assert.equal((await inspect(snapshot)).mode, "text");
});
test("TypeScript source units preserve embedded comments verbatim", async () => {
const source =
'function x() {\n // end\n}\nconst x = /*important*/ 1;\nconst url="https://example.test";\nconst regexp=/https?:\\/\\//;\n';
const snapshot = { path: "comments.ts", source, contentHash: "fixture" };
const result = await inspect(snapshot);
const { sourceForUnit } = await import("../../packages/core/src/source.ts");
assert.equal(
sourceForUnit(snapshot, result.units[0]!),
source.split("\n").slice(0, 3).join("\n") + "\n",
);
});
test("same-named classes retain their own structural headers", async () => {
for (const [path, source] of [
[
"same.py",
"class Same:\n def first(self): pass\n\nclass Same:\n def second(self): pass\n",
],
["same.ts", "class Same {\n first() {}\n}\nclass Same {\n second() {}\n}\n"],
]) {
const result = await inspect({ path: path!, source: source!, contentHash: "fixture" });
assert.deepEqual(result.units.find((unit) => unit.name === "Same.first")?.ownerHeaders, [
{ startLine: 1, endLine: 1 },
]);
assert.deepEqual(result.units.find((unit) => unit.name === "Same.second")?.ownerHeaders, [
{ startLine: 4, endLine: 4 },
]);
}
});
test("generated TypeScript deeper than the call stack parses or falls back to text", async () => {
const { sourceForUnit } = await import("../../packages/core/src/source.ts");
// A long flat expression nests one level per operator in the syntax tree.
const chain = "// generated\nexport const table = " + "1 + ".repeat(50_000) + "1;\n";
const parsed = await inspect(
{ path: "chain.ts", source: chain, contentHash: "fixture" },
{ maxUnitBytes: Buffer.byteLength(chain) },
);
assert.equal(parsed.mode, "typescript");
assert.deepEqual(
parsed.units.map(({ name, range }) => ({ name, ...range })),
[{ name: "table", startLine: 2, endLine: 2 }],
);
assert.deepEqual(parsed.comments, [{ startLine: 1, endLine: 1 }]);
const nested = "export const x = " + "(".repeat(20_000) + "1" + ")".repeat(20_000) + ";\n";
const snapshot = { path: "nested.ts", source: nested, contentHash: "fixture" };
const fallback = await inspect(snapshot, { maxUnitBytes: 3000 });
assert.equal(fallback.mode, "text");
assert.equal(fallback.fallback, "syntax");
assert.equal(fallback.units.map((unit) => sourceForUnit(snapshot, unit)).join(""), nested);
});
test("Python nested beyond the query depth falls back quickly while wide files parse", async () => {
const { sourceForUnit } = await import("../../packages/core/src/source.ts");
// One level per `+`: past 65,535 levels Tree-sitter queries miss matches and slow down sharply.
const deep = "def f():\n return 1\nvalue = " + "1+".repeat(100_000) + "1\n";
const snapshot = { path: "generated.py", source: deep, contentHash: "fixture" };
const started = performance.now();
const result = await inspect(snapshot, { maxUnitBytes: 3000 });
assert.ok(performance.now() - started < 10_000, "deep source was not rejected quickly");
assert.equal(result.mode, "text");
assert.equal(result.fallback, "syntax");
assert.equal(result.units.map((unit) => sourceForUnit(snapshot, unit)).join(""), deep);
// Many statements or elements make a wide tree, not a deep one.
const wide =
"def f():\n return 1\n" +
Array.from({ length: 30_000 }, (_, i) => `x${i} = [${i}, ${i}]`).join("\n") +
"\n";
const parsed = await inspect({ path: "wide.py", source: wide, contentHash: "fixture" });
assert.equal(parsed.mode, "python");
assert.equal(parsed.units[0]?.name, "f");
});
test("class context is marked structurally, not inferred from member names", async () => {
const marked = async (path: string, source: string) =>
(await inspect({ path, source, contentHash: "fixture" })).units.map(
({ name, range, classContext }) =>
`${name}@${range.startLine}-${range.endLine}${classContext ? " [class context]" : ""}`,
);
assert.deepEqual(
await marked(
"template.py",
'class Template:\n """Render with a context."""\n engine = None\n\n def context(self, request):\n return {"request": request}\n\n def render(self, request):\n return self.context(request)\n',
),
[
"Template.context@1-4 [class context]",
"Template.context@5-6",
"Template.context@7-7 [class context]",
"Template.render@8-9",
],
);
assert.deepEqual(
await marked(
"widget.ts",
'export class Widget {\n static kind = "w";\n context() {\n return {};\n }\n}\nexport class Inline { context() { return 1; } }\n',
),
[
"Widget.context@1-1 [class context]",
"Widget.kind@2-2",
"Widget.context@3-5",
"Inline.context@7-7",
],
);
assert.deepEqual(
await marked(
"widget.rs",
"struct Widget;\nimpl Widget {\n fn context(&self) -> u32 {\n 1\n }\n}\n",
),
["Widget@1-1", "Widget.context@2-2 [class context]", "Widget.context@3-5"],
);
});
test("fragment extraction reuses one encoded source allocation", () => {
const source = "é漢🙂 source line\r\n".repeat(2_000);
const snapshot = { path: "wide.txt", contentHash: "fixture", source };
const originalFrom = Buffer.from;
let encodedBytes = 0;
// Keep this global allocation probe in a non-concurrent test.
Buffer.from = ((...args: Parameters<typeof Buffer.from>) => {
if (typeof args[0] === "string") encodedBytes += Buffer.byteLength(args[0]);
return Reflect.apply(originalFrom, Buffer, args);
}) as typeof Buffer.from;
try {
const pieces = splitSource(snapshot, 1_000).map((unit) => sourceForUnit(snapshot, unit));
assert.equal(pieces.join(""), source);
assert.ok(encodedBytes <= Buffer.byteLength(source) * 2, `encoded ${encodedBytes} bytes`);
} finally {
Buffer.from = originalFrom;
}
});
test("cached source fragments preserve UTF-8 bytes and line coordinates", async () => {
for (const source of ["", "é漢🙂", "é漢🙂\n", "é漢🙂\r\nlast", "é漢🙂".repeat(8_000)]) {
const snapshot = { path: "text.txt", contentHash: "fixture", source };
const fragments = splitSource(snapshot, 16);
assert.equal(fragments.map((unit) => sourceForUnit(snapshot, unit)).join(""), source);
assert.deepEqual(splitSource(snapshot, 16), fragments);
const parsed = await inspect(snapshot, { maxUnitBytes: 16 });
assert.equal(parsed.units.map((unit) => sourceForUnit(snapshot, unit)).join(""), source);
if (source) {
assert.equal(fragments[0]!.sourceByteStart, 0);
assert.equal(fragments.at(-1)!.sourceByteEnd, Buffer.byteLength(source));
assert.equal(fragments.at(-1)!.range.endLine, source.split("\n").length);
}
}
});
test("snapshot-keyed source cache refreshes after source mutation", async () => {
const snapshot = { path: "same.txt", contentHash: "fixture", source: "old\nsource" };
const oldFragments = splitSource(snapshot, 4);
assert.equal(oldFragments.map((unit) => sourceForUnit(snapshot, unit)).join(""), snapshot.source);
const other = { ...snapshot, source: "é漢🙂\r\nnew" };
assert.equal(
splitSource(other, 4)
.map((unit) => sourceForUnit(other, unit))
.join(""),
other.source,
);
snapshot.source = "changed\n🙂";
const changed = await inspect(snapshot, { maxUnitBytes: 4 });
assert.equal(
changed.units.map((unit) => sourceForUnit(snapshot, unit)).join(""),
snapshot.source,
);
assert.equal(
splitSource(snapshot, 4)
.map((unit) => sourceForUnit(snapshot, unit))
.join(""),
snapshot.source,
);
assert.equal(
splitSource(other, 4)
.map((unit) => sourceForUnit(other, unit))
.join(""),
other.source,
);
});
test("long Python comment runs parse quickly without changing declarations", async () => {
const run = (indent: string, count: number) =>
Array.from({ length: count }, (_, i) => `${indent}# n${i} don't "q" {x}`).join("\n");
// Parsing was quadratic in a comment run: 8,000 lines took 9.5 s.
const source =
`class Box:\n${run(" ", 12_000)}\n def first(self):\n return 1\n` +
`${run("", 12_000)}\ndef second():\n return 2\n`;
const started = performance.now();
const result = await inspect(
{ path: "notes.py", source, contentHash: "fixture" },
{ maxUnitBytes: Buffer.byteLength(source) },
);
assert.ok(performance.now() - started < 5_000, "comment runs were not parsed quickly");
assert.deepEqual(
result.units.map(({ name, range }) => `${name}@${range.startLine}-${range.endLine}`),
["Box.context@1-12001", "Box.first@12002-12003", "second@24004-24005"],
);
// Comment-looking lines inside a string are string content, even with quotes and braces.
const inString = `value = f"""\n${run("", 300)}\n"""\n` + 'def after():\n return """#"""\n';
const parsed = await inspect({ path: "doc.py", source: inString, contentHash: "fixture" });
assert.equal(parsed.mode, "python");
assert.deepEqual(
parsed.units.map(({ name, range }) => `${name}@${range.startLine}-${range.endLine}`),
["after@303-304"],
);
});