Find code by asking what it does. A CLI for coding agents that uses Jev to discover relevant files and source context.
TypeScript ai-sdkclaude-codeclicode-searchcodexcoding-agentscontext-retrievaldeveloper-toolsjevsemantic-searchtypescriptvercel-ai-gateway
perf(parser): reuse encoded source snapshots
David Zhang6c2e6332a55a
diff --git a/packages/core/src/source.ts b/packages/core/src/source.tsindex 98771fa..277760b 100644--- a/packages/core/src/source.ts+++ b/packages/core/src/source.ts@@ -47,11 +47,18 @@ export function resolveLanguage(}type SourceText = { bytes: Buffer; offsets: number[]; lineCount: number };-function sourceText(source: string): SourceText {+const sourceTexts = new WeakMap<Snapshot, { source: string; text: SourceText }>();+function sourceText(snapshot: Snapshot): SourceText {+ const { source } = snapshot;+ const cached = sourceTexts.get(snapshot);+ // Readonly is a type-level promise; mutable callers must not receive stale bytes.+ if (cached?.source === source) return cached.text;const lines = source.split("\n"),offsets = [0];for (const line of lines) offsets.push(offsets.at(-1)! + Buffer.byteLength(line) + 1);- return { bytes: Buffer.from(source), offsets, lineCount: lines.length };+ const text = { bytes: Buffer.from(source), offsets, lineCount: lines.length };+ sourceTexts.set(snapshot, { source, text });+ return text;}function textUnits(text: SourceText,@@ -110,7 +117,7 @@ export async function inspect();const { source, path } = snapshot;const language = resolveLanguage(path, source);- const text = sourceText(source);+ const text = sourceText(snapshot);// Context windows use conservative whole-line Python comments, including inside multiline strings.const pythonComments =language === "python"@@ -379,8 +386,8 @@ export async function pythonPreview(/** Byte spans include original line endings; never widen a partial unit to whole lines. */export function sourceForUnit(snapshot: Snapshot, unit: SourceUnit): string {- return Buffer.from(snapshot.source)- .subarray(unit.sourceByteStart, unit.sourceByteEnd)+ return sourceText(snapshot)+ .bytes.subarray(unit.sourceByteStart, unit.sourceByteEnd).toString("utf8");}@@ -388,6 +395,6 @@ export function sourceForUnit(snapshot: Snapshot, unit: SourceUnit): string {export function splitSource(snapshot: Snapshot, maxBytes = 12_000): SourceUnit[] {if (!Number.isSafeInteger(maxBytes) || maxBytes < 4)throw new Error("Invalid source byte allowance");- const text = sourceText(snapshot.source);+ const text = sourceText(snapshot);return textUnits(text, { startLine: 1, endLine: text.lineCount }, "source", maxBytes, false);}diff --git a/test/parser/source.test.ts b/test/parser/source.test.tsindex 89ce1f6..cfbd835 100644--- a/test/parser/source.test.ts+++ b/test/parser/source.test.ts@@ -1,6 +1,11 @@import { test } from "node:test";import assert from "node:assert/strict";-import { inspect, pythonNeighborhood } from "../../packages/core/src/source.ts";+import {+ inspect,+ pythonNeighborhood,+ sourceForUnit,+ splitSource,+} from "../../packages/core/src/source.ts";test("extensionless Python shebangs preserve declarations, comments and neighboring source", async () => {for (const shebang of [@@ -455,6 +460,74 @@ test("class context is marked structurally, not inferred from member names", asy);});+test("fragment extraction reuses one encoded source allocation", () => {+ const source = "é漢🙂 source line\r\n".repeat(2_000);+ const snapshot = { path: "wide.txt", contentHash: "fixture", source };+ const originalFrom = Buffer.from;+ let encodedBytes = 0;+ // Keep this global allocation probe in a non-concurrent test.+ Buffer.from = ((...args: Parameters<typeof Buffer.from>) => {+ if (typeof args[0] === "string") encodedBytes += Buffer.byteLength(args[0]);+ return Reflect.apply(originalFrom, Buffer, args);+ }) as typeof Buffer.from;+ try {+ const pieces = splitSource(snapshot, 1_000).map((unit) => sourceForUnit(snapshot, unit));+ assert.equal(pieces.join(""), source);+ assert.ok(encodedBytes <= Buffer.byteLength(source) * 2, `encoded ${encodedBytes} bytes`);+ } finally {+ Buffer.from = originalFrom;+ }+});++test("cached source fragments preserve UTF-8 bytes and line coordinates", async () => {+ for (const source of ["", "é漢🙂", "é漢🙂\n", "é漢🙂\r\nlast", "é漢🙂".repeat(8_000)]) {+ const snapshot = { path: "text.txt", contentHash: "fixture", source };+ const fragments = splitSource(snapshot, 16);+ assert.equal(fragments.map((unit) => sourceForUnit(snapshot, unit)).join(""), source);+ assert.deepEqual(splitSource(snapshot, 16), fragments);+ const parsed = await inspect(snapshot, { maxUnitBytes: 16 });+ assert.equal(parsed.units.map((unit) => sourceForUnit(snapshot, unit)).join(""), source);+ if (source) {+ assert.equal(fragments[0]!.sourceByteStart, 0);+ assert.equal(fragments.at(-1)!.sourceByteEnd, Buffer.byteLength(source));+ assert.equal(fragments.at(-1)!.range.endLine, source.split("\n").length);+ }+ }+});++test("snapshot-keyed source cache refreshes after source mutation", async () => {+ const snapshot = { path: "same.txt", contentHash: "fixture", source: "old\nsource" };+ const oldFragments = splitSource(snapshot, 4);+ assert.equal(oldFragments.map((unit) => sourceForUnit(snapshot, unit)).join(""), snapshot.source);++ const other = { ...snapshot, source: "é漢🙂\r\nnew" };+ assert.equal(+ splitSource(other, 4)+ .map((unit) => sourceForUnit(other, unit))+ .join(""),+ other.source,+ );++ snapshot.source = "changed\n🙂";+ const changed = await inspect(snapshot, { maxUnitBytes: 4 });+ assert.equal(+ changed.units.map((unit) => sourceForUnit(snapshot, unit)).join(""),+ snapshot.source,+ );+ assert.equal(+ splitSource(snapshot, 4)+ .map((unit) => sourceForUnit(snapshot, unit))+ .join(""),+ snapshot.source,+ );+ assert.equal(+ splitSource(other, 4)+ .map((unit) => sourceForUnit(other, unit))+ .join(""),+ other.source,+ );+});+test("long Python comment runs parse quickly without changing declarations", async () => {const run = (indent: string, count: number) =>Array.from({ length: count }, (_, i) => `${indent}# n${i} don't "q" {x}`).join("\n");