Skip to content

Find code by asking what it does. A CLI for coding agents that uses Jev to discover relevant files and source context.

perf(parser): reuse encoded source snapshots

David Zhang6c2e6332a55a
diff --git a/packages/core/src/source.ts b/packages/core/src/source.ts
index 98771fa..277760b 100644
--- a/packages/core/src/source.ts
+++ b/packages/core/src/source.ts
@@ -47,11 +47,18 @@ export function resolveLanguage(
}
type SourceText = { bytes: Buffer; offsets: number[]; lineCount: number };
-function sourceText(source: string): SourceText {
+const sourceTexts = new WeakMap<Snapshot, { source: string; text: SourceText }>();
+function sourceText(snapshot: Snapshot): SourceText {
+ const { source } = snapshot;
+ const cached = sourceTexts.get(snapshot);
+ // Readonly is a type-level promise; mutable callers must not receive stale bytes.
+ if (cached?.source === source) return cached.text;
const lines = source.split("\n"),
offsets = [0];
for (const line of lines) offsets.push(offsets.at(-1)! + Buffer.byteLength(line) + 1);
- return { bytes: Buffer.from(source), offsets, lineCount: lines.length };
+ const text = { bytes: Buffer.from(source), offsets, lineCount: lines.length };
+ sourceTexts.set(snapshot, { source, text });
+ return text;
}
function textUnits(
text: SourceText,
@@ -110,7 +117,7 @@ export async function inspect(
);
const { source, path } = snapshot;
const language = resolveLanguage(path, source);
- const text = sourceText(source);
+ const text = sourceText(snapshot);
// Context windows use conservative whole-line Python comments, including inside multiline strings.
const pythonComments =
language === "python"
@@ -379,8 +386,8 @@ export async function pythonPreview(
/** Byte spans include original line endings; never widen a partial unit to whole lines. */
export function sourceForUnit(snapshot: Snapshot, unit: SourceUnit): string {
- return Buffer.from(snapshot.source)
- .subarray(unit.sourceByteStart, unit.sourceByteEnd)
+ return sourceText(snapshot)
+ .bytes.subarray(unit.sourceByteStart, unit.sourceByteEnd)
.toString("utf8");
}
@@ -388,6 +395,6 @@ export function sourceForUnit(snapshot: Snapshot, unit: SourceUnit): string {
export function splitSource(snapshot: Snapshot, maxBytes = 12_000): SourceUnit[] {
if (!Number.isSafeInteger(maxBytes) || maxBytes < 4)
throw new Error("Invalid source byte allowance");
- const text = sourceText(snapshot.source);
+ const text = sourceText(snapshot);
return textUnits(text, { startLine: 1, endLine: text.lineCount }, "source", maxBytes, false);
}
diff --git a/test/parser/source.test.ts b/test/parser/source.test.ts
index 89ce1f6..cfbd835 100644
--- a/test/parser/source.test.ts
+++ b/test/parser/source.test.ts
@@ -1,6 +1,11 @@
import { test } from "node:test";
import assert from "node:assert/strict";
-import { inspect, pythonNeighborhood } from "../../packages/core/src/source.ts";
+import {
+ inspect,
+ pythonNeighborhood,
+ sourceForUnit,
+ splitSource,
+} from "../../packages/core/src/source.ts";
test("extensionless Python shebangs preserve declarations, comments and neighboring source", async () => {
for (const shebang of [
@@ -455,6 +460,74 @@ test("class context is marked structurally, not inferred from member names", asy
);
});
+test("fragment extraction reuses one encoded source allocation", () => {
+ const source = "é漢🙂 source line\r\n".repeat(2_000);
+ const snapshot = { path: "wide.txt", contentHash: "fixture", source };
+ const originalFrom = Buffer.from;
+ let encodedBytes = 0;
+ // Keep this global allocation probe in a non-concurrent test.
+ Buffer.from = ((...args: Parameters<typeof Buffer.from>) => {
+ if (typeof args[0] === "string") encodedBytes += Buffer.byteLength(args[0]);
+ return Reflect.apply(originalFrom, Buffer, args);
+ }) as typeof Buffer.from;
+ try {
+ const pieces = splitSource(snapshot, 1_000).map((unit) => sourceForUnit(snapshot, unit));
+ assert.equal(pieces.join(""), source);
+ assert.ok(encodedBytes <= Buffer.byteLength(source) * 2, `encoded ${encodedBytes} bytes`);
+ } finally {
+ Buffer.from = originalFrom;
+ }
+});
+
+test("cached source fragments preserve UTF-8 bytes and line coordinates", async () => {
+ for (const source of ["", "é漢🙂", "é漢🙂\n", "é漢🙂\r\nlast", "é漢🙂".repeat(8_000)]) {
+ const snapshot = { path: "text.txt", contentHash: "fixture", source };
+ const fragments = splitSource(snapshot, 16);
+ assert.equal(fragments.map((unit) => sourceForUnit(snapshot, unit)).join(""), source);
+ assert.deepEqual(splitSource(snapshot, 16), fragments);
+ const parsed = await inspect(snapshot, { maxUnitBytes: 16 });
+ assert.equal(parsed.units.map((unit) => sourceForUnit(snapshot, unit)).join(""), source);
+ if (source) {
+ assert.equal(fragments[0]!.sourceByteStart, 0);
+ assert.equal(fragments.at(-1)!.sourceByteEnd, Buffer.byteLength(source));
+ assert.equal(fragments.at(-1)!.range.endLine, source.split("\n").length);
+ }
+ }
+});
+
+test("snapshot-keyed source cache refreshes after source mutation", async () => {
+ const snapshot = { path: "same.txt", contentHash: "fixture", source: "old\nsource" };
+ const oldFragments = splitSource(snapshot, 4);
+ assert.equal(oldFragments.map((unit) => sourceForUnit(snapshot, unit)).join(""), snapshot.source);
+
+ const other = { ...snapshot, source: "é漢🙂\r\nnew" };
+ assert.equal(
+ splitSource(other, 4)
+ .map((unit) => sourceForUnit(other, unit))
+ .join(""),
+ other.source,
+ );
+
+ snapshot.source = "changed\n🙂";
+ const changed = await inspect(snapshot, { maxUnitBytes: 4 });
+ assert.equal(
+ changed.units.map((unit) => sourceForUnit(snapshot, unit)).join(""),
+ snapshot.source,
+ );
+ assert.equal(
+ splitSource(snapshot, 4)
+ .map((unit) => sourceForUnit(snapshot, unit))
+ .join(""),
+ snapshot.source,
+ );
+ assert.equal(
+ splitSource(other, 4)
+ .map((unit) => sourceForUnit(other, unit))
+ .join(""),
+ other.source,
+ );
+});
+
test("long Python comment runs parse quickly without changing declarations", async () => {
const run = (indent: string, count: number) =>
Array.from({ length: count }, (_, i) => `${indent}# n${i} don't "q" {x}`).join("\n");