Skip to content

Find code by asking what it does. A CLI for coding agents that uses Jev to discover relevant files and source context.

14.5 KB
Raw Blame
import ts from "typescript";
import { basename, extname } from "node:path";
import { runParser } from "./parser.ts";

export type Range = { startLine: number; endLine: number };
import type { Snapshot } from "./filesystem";
export type SourceUnit = {
  id: string;
  name: string;
  range: Range;
  sourceByteStart: number;
  sourceByteEnd: number;
  partial?: boolean;
  ownerHeaders?: Range[];
  /** Structural class, impl or module source between members. The name alone cannot identify
   * it: a real member named `context` shares the `<Owner>.context` name. */
  classContext?: true;
};
export type Inspection = {
  units: SourceUnit[];
  comments: Range[];
  mode: "python" | "typescript" | "go" | "rust" | "text";
  fallback?: "unsupported" | "syntax" | "size";
};

export function resolveLanguage(
  path: string,
  source: string,
): "python" | "go" | "rust" | "TS" | "TSX" | "JS" | "JSX" | undefined {
  if (/\.pyi?$/.test(path)) return "python";
  if (path.endsWith(".tsx")) return "TSX";
  if (path.endsWith(".jsx")) return "JSX";
  if (/\.[cm]?js$/.test(path)) return "JS";
  if (/\.[cm]?ts$/.test(path)) return "TS";
  if (path.endsWith(".go")) return "go";
  if (path.endsWith(".rs")) return "rust";
  if (extname(path) !== "" || !source.startsWith("#!")) return;
  const [program = "", ...args] = source.split("\n", 1)[0]!.slice(2).trim().split(/\s+/);
  // env options, the operand of -u/-C/-P/-a and NAME=VALUE assignments come before the command.
  const command = args.find(
    (arg, index) =>
      !arg.startsWith("-") && !arg.includes("=") && !/^-[uCPa]$/.test(args[index - 1] ?? ""),
  );
  const interpreter = basename(basename(program) === "env" ? (command ?? "") : program);
  if (/^python(\d+(\.\d+)?)?$/.test(interpreter)) return "python";
  if (interpreter === "node" || interpreter === "nodejs") return "JS";
}

type SourceText = { bytes: Buffer; offsets: number[]; lineCount: number };
const sourceTexts = new WeakMap<Snapshot, { source: string; text: SourceText }>();
function sourceText(snapshot: Snapshot): SourceText {
  const { source } = snapshot;
  const cached = sourceTexts.get(snapshot);
  // Readonly is a type-level promise; mutable callers must not receive stale bytes.
  if (cached?.source === source) return cached.text;
  const lines = source.split("\n"),
    offsets = [0];
  for (const line of lines) offsets.push(offsets.at(-1)! + Buffer.byteLength(line) + 1);
  const text = { bytes: Buffer.from(source), offsets, lineCount: lines.length };
  sourceTexts.set(snapshot, { source, text });
  return text;
}
function textUnits(
  text: SourceText,
  range: Range,
  name: string,
  maxBytes: number,
  partial: boolean,
): SourceUnit[] {
  const { bytes: raw, offsets, lineCount } = text;
  const units: SourceUnit[] = [];
  const first = Math.min(raw.length, offsets[range.startLine - 1] ?? raw.length);
  const end = Math.min(raw.length, offsets[range.endLine] ?? raw.length);
  let start = first,
    line = range.startLine;
  while (start < end) {
    let finish = Math.min(end, start + maxBytes);
    if (finish < end) {
      while (finish > start && (raw[finish]! & 0xc0) === 0x80) finish--;
      const newline = raw.subarray(start, finish).lastIndexOf(10);
      if (newline >= 0) finish = start + newline + 1;
    }
    const part = raw.subarray(start, finish).toString("utf8");
    const newlines = part.split("\n").length - 1;
    const endLine = line + newlines - (part.endsWith("\n") ? 1 : 0);
    units.push({
      id: `${name}:${start}:${finish}`,
      name,
      range: { startLine: line, endLine },
      sourceByteStart: start,
      sourceByteEnd: finish,
      ...(partial || first !== start || finish !== end ? { partial: true } : {}),
    });
    line += newlines;
    start = finish;
  }
  // The final empty line has no bytes but still belongs to the snapshot coordinates.
  if (raw.at(-1) === 10 && range.endLine === lineCount && units.length)
    units.at(-1)!.range.endLine = lineCount;
  return units;
}

export async function inspect(
  snapshot: Snapshot,
  options: { maxUnitBytes?: number; maxParseBytes?: number; signal?: AbortSignal } = {},
): Promise<Inspection> {
  const maxUnitBytes = options.maxUnitBytes ?? 24_000;
  const maxParseBytes = options.maxParseBytes ?? 1_000_000;
  if (
    !Number.isSafeInteger(maxUnitBytes) ||
    maxUnitBytes < 4 ||
    !Number.isSafeInteger(maxParseBytes) ||
    maxParseBytes < 1
  )
    throw new Error(
      "Source bounds must be integers; unit bytes at least 4 and parse bytes positive",
    );
  const { source, path } = snapshot;
  const language = resolveLanguage(path, source);
  const text = sourceText(snapshot);
  // Context windows use conservative whole-line Python comments, including inside multiline strings.
  const pythonComments =
    language === "python"
      ? source
          .split("\n")
          .flatMap((line, index) =>
            /^\s*#/.test(line) ? [{ startLine: index + 1, endLine: index + 1 }] : [],
          )
      : [];
  const fallback = (reason: Inspection["fallback"]): Inspection => ({
    mode: "text",
    fallback: reason,
    comments: pythonComments,
    units: source
      ? textUnits(text, { startLine: 1, endLine: text.lineCount }, "source", maxUnitBytes, true)
      : [],
  });
  if (Buffer.byteLength(source) > maxParseBytes) return fallback("size");
  let units: { name: string; range: Range; ownerHeaders?: Range[]; classContext?: true }[] = [],
    comments: Range[] = [],
    mode: Inspection["mode"];
  let syntaxFallback = false;
  if (language === "python") {
    // A missing/incompatible packaged parser is a setup failure, never syntax fallback.
    const parsed = await runParser<
      Array<Range & { name: string; ownerHeaders: Range[]; classContext?: boolean }>
    >("inspect", source, options.signal);
    if (parsed === null) return fallback("syntax");
    units = parsed.map(({ name, startLine, endLine, ownerHeaders, classContext }) => ({
      name,
      range: { startLine, endLine },
      ownerHeaders,
      ...(classContext === true ? { classContext: true as const } : {}),
    }));
    comments = pythonComments;
    mode = "python";
  } else if (language === "TS" || language === "TSX" || language === "JS" || language === "JSX") {
    let file: ts.SourceFile;
    try {
      file = ts.createSourceFile(path, source, ts.ScriptTarget.Latest, true, ts.ScriptKind[language]);
    } catch (error) {
      // Deeply nested source can exhaust the parser's call stack: unparseable here, like a syntax error.
      if (error instanceof RangeError) return fallback("syntax");
      throw error;
    }
    syntaxFallback = !!(file as ts.SourceFile & { parseDiagnostics?: unknown[] }).parseDiagnostics
      ?.length;
    const line = (position: number) => file.getLineAndCharacterOfPosition(position).line + 1;
    const add = (node: ts.Node, prefix = "", ownerHeaders: Range[] = []) => {
      const named = node as ts.Node & { name?: ts.Node };
      const name =
        prefix +
        (named.name?.getText(file) ||
          (ts.isVariableStatement(node)
            ? node.declarationList.declarations.map((d) => d.name.getText(file)).join(", ")
            : "source"));
      if (ts.isClassDeclaration(node) && node.members.length) {
        const start = line(node.getStart(file)),
          first = line(node.members[0]!.getStart(file));
        const header = first > start ? { startLine: start, endLine: first - 1 } : undefined;
        const headers = header ? [...ownerHeaders, header] : ownerHeaders;
        if (header)
          units.push({
            name: name + ".context",
            range: header,
            ownerHeaders: headers,
            classContext: true,
          });
        for (const member of node.members) add(member, name + ".", headers);
      } else
        units.push({
          name,
          ownerHeaders,
          range: {
            startLine: line(node.getStart(file)),
            endLine: line(Math.max(node.getStart(file), node.end - 1)),
          },
        });
    };
    if (!syntaxFallback) for (const statement of file.statements) add(statement);
    // Pre-order walk with an explicit stack: long generated expressions nest deeper than the call
    // stack allows. Children are pushed in reverse so they are visited in source order.
    const walk: ts.Node[] = [file];
    while (walk.length) {
      const node = walk.pop()!;
      for (const comment of [
        ...(ts.getLeadingCommentRanges(source, node.getFullStart()) ?? []),
        ...(ts.getTrailingCommentRanges(source, node.end) ?? []),
      ])
        comments.push({ startLine: line(comment.pos), endLine: line(comment.end - 1) });
      const children: ts.Node[] = [];
      // The callback must return undefined: a truthy result stops forEachChild early.
      ts.forEachChild(node, (child) => {
        children.push(child);
      });
      for (let index = children.length - 1; index >= 0; index--) walk.push(children[index]!);
    }
    mode = "typescript";
  } else if (language === "go" || language === "rust") {
    const parsed = await runParser<{ units: typeof units; comments: Range[] }>(
      "declarations",
      JSON.stringify({ source, language }),
      options.signal,
    );
    if (!parsed) return fallback("syntax");
    units = parsed.units.map(({ classContext, ...unit }) => ({
      ...unit,
      ...(classContext === true ? { classContext: true as const } : {}),
    }));
    comments = parsed.comments;
    mode = language;
  } else return fallback("unsupported");
  comments = [...new Map(comments.map((r) => [`${r.startLine}:${r.endLine}`, r])).values()].sort(
    (a, b) => a.startLine - b.startLine,
  );
  // Invalid declarations fall back to text, but comments still own the context-window boundaries.
  if (syntaxFallback) return { ...fallback("syntax"), comments };
  if (!units.length && source)
    return {
      units: textUnits(
        text,
        { startLine: 1, endLine: text.lineCount },
        "source",
        maxUnitBytes,
        true,
      ),
      comments,
      mode,
    };
  return {
    mode,
    comments,
    units: units.flatMap((unit) => {
      // Parser ranges and returned byte spans must remain tied to the original snapshot.
      const start = Math.min(
        text.bytes.length,
        text.offsets[unit.range.startLine - 1] ?? text.bytes.length,
      );
      const end = Math.min(
        text.bytes.length,
        text.offsets[unit.range.endLine] ?? text.bytes.length,
      );
      const marker = unit.classContext ? { classContext: true as const } : {};
      if (end - start <= maxUnitBytes)
        return [
          {
            id: `${unit.name}:${start}:${end}`,
            name: unit.name,
            range: unit.range,
            ownerHeaders: unit.ownerHeaders,
            sourceByteStart: start,
            sourceByteEnd: end,
            ...marker,
          },
        ];
      return textUnits(text, unit.range, unit.name, maxUnitBytes, true).map((part) => ({
        ...part,
        ownerHeaders: unit.ownerHeaders,
        ...marker,
      }));
    }),
  };
}

/** Additive policy only: callers retain selected ranges separately. */
export async function pythonNeighborhood(
  snapshot: Snapshot,
  selected: Range[],
  signal?: AbortSignal,
): Promise<Range[]> {
  if (
    resolveLanguage(snapshot.path, snapshot.source) !== "python" ||
    Buffer.byteLength(snapshot.source) > 1_000_000
  )
    return [];
  return (
    (await runParser<Range[]>(
      "neighborhood",
      JSON.stringify({ source: snapshot.source, ranges: selected }),
      signal,
    )) ?? []
  );
}

export type PreviewSpan = Range & {
  text: string;
  basis: string;
  sourceByteStart: number;
  sourceByteEnd: number;
  partialLine?: boolean;
  columnStartByte?: number;
  columnEndByte?: number;
};
export type SourcePreview = {
  text: string;
  spans: PreviewSpan[];
  truncated: boolean;
  sourceBytes: number;
  previewBytes: number;
  method: string;
  matchedDeclarations: number;
  unrepresentedMatches: number;
  parseUnavailable?: boolean;
  scope?: string;
};
/** The preview worker handles Python declarations and generic text windows. */
export async function pythonPreview(
  snapshot: Snapshot,
  query: string,
  budget = 16_384,
  signal?: AbortSignal,
): Promise<SourcePreview | null> {
  if (!Number.isSafeInteger(budget) || budget < 256)
    throw new Error("Preview allowance must be at least 256 bytes");
  const result = await runParser<SourcePreview>(
    "preview",
    JSON.stringify({
      text: snapshot.source,
      language: resolveLanguage(snapshot.path, snapshot.source),
      query,
      budget,
    }),
    signal,
  );
  if (result?.truncated) {
    const raw = Buffer.from(snapshot.source),
      lines = snapshot.source.split("\n");
    if (
      !result.spans.length ||
      result.spans.some((span) => {
        if (
          !Number.isInteger(span.startLine) ||
          !Number.isInteger(span.endLine) ||
          span.startLine < 1 ||
          span.endLine < span.startLine ||
          span.endLine > lines.length
        )
          return true;
        if (
          !Number.isInteger(span.sourceByteStart) ||
          !Number.isInteger(span.sourceByteEnd) ||
          span.sourceByteStart < 0 ||
          span.sourceByteEnd < span.sourceByteStart ||
          span.sourceByteEnd > raw.length ||
          raw.subarray(span.sourceByteStart, span.sourceByteEnd).toString() !== span.text
        )
          return true;
        const original = lines.slice(span.startLine - 1, span.endLine).join("\n");
        if (span.columnStartByte !== undefined)
          return (
            span.startLine !== span.endLine ||
            !Number.isInteger(span.columnStartByte) ||
            !Number.isInteger(span.columnEndByte) ||
            span.columnStartByte < 0 ||
            span.columnEndByte! < span.columnStartByte ||
            Buffer.from(original).subarray(span.columnStartByte, span.columnEndByte).toString() !==
              span.text
          );
        return span.partialLine ? !original.includes(span.text) : original !== span.text;
      })
    )
      return null;
  }
  return result;
}

/** Byte spans include original line endings; never widen a partial unit to whole lines. */
export function sourceForUnit(snapshot: Snapshot, unit: SourceUnit): string {
  return sourceText(snapshot)
    .bytes.subarray(unit.sourceByteStart, unit.sourceByteEnd)
    .toString("utf8");
}

/** Complete-file fragments keep admission independent of declaration-name sampling. */
export function splitSource(snapshot: Snapshot, maxBytes = 12_000): SourceUnit[] {
  if (!Number.isSafeInteger(maxBytes) || maxBytes < 4)
    throw new Error("Invalid source byte allowance");
  const text = sourceText(snapshot);
  return textUnits(text, { startLine: 1, endLine: text.lineCount }, "source", maxBytes, false);
}