Skip to content

Find code by asking what it does. A CLI for coding agents that uses Jev to discover relevant files and source context.

37.3 KB
Raw Blame
import { basename, extname } from "node:path";
import { createFilesystem, type Snapshot, type DirectoryEntry } from "./filesystem";
import { type EvaluationRequest, EvaluationFailure } from "./evaluator";
import { inspect, pythonPreview, resolveLanguage, sourceForUnit, splitSource } from "./source";
import {
  navigationRequest,
  fileAssessmentRequest,
  type DirectoryPreview,
  type FilePreview,
  type NavigationItem,
  type Evidence,
} from "./requests";
import {
  selectFile,
  wholeFileContextBytes,
  sourceBatchSize,
  sourceConfidenceBoundary,
  type SelectionResult,
} from "./selection";
import { localCallContext } from "./call-context";
import { selectTestBodies } from "./test-body-selection";
import { repositoryContext } from "./repository-context";
import type { Evaluator, FileEvidence, RetrievalResult, SearchInput } from "./types";

// Bound per-stage source work; the evaluator separately caps shared provider attempts.
const stageWorkers = 32;
const relevanceBoundary = 0.5;
const weakScoreCeiling = 0.6;
const minimumPositiveScores = 32;
export const navigationByteBudget = 8_000_000;
function navigationAdmissions<T extends { score: number }>(
  classified: T[],
  hasStrongEvidence: boolean,
) {
  const suppressWeak =
    !hasStrongEvidence &&
    classified.filter(({ score }) => score > relevanceBoundary && score < weakScoreCeiling)
      .length >= minimumPositiveScores;
  return {
    admitted: classified.filter(
      ({ score }) => score >= weakScoreCeiling || (!suppressWeak && score > relevanceBoundary),
    ),
    suppressWeak,
  };
}
/** Traversal owns admission; every stage reads through the same eligibility policy. */
export async function retrieve(input: SearchInput, evaluator: Evaluator): Promise<RetrievalResult> {
  const reader = await createFilesystem({
    root: input.root,
    policy: input.policy,
    signal: input.signal,
    protectedPaths: input.protectedPaths,
  });
  const issues = new Map<string, number>();
  let providerFailure: string | undefined;
  const inspected = new Set<string>();
  const candidates = new Map<string, { path: string; contentHash: string; score: number }>();
  const files = new Map<string, FileEvidence>();
  const declarations = new Map<string, SelectionResult["declarations"]>();
  const groupCounts = new Map<string, number>();
  const visited = new Set<string>();
  const pruned = new Map<string, NavigationItem>();
  const previews = new Map<string, FilePreview>();
  type Donor = { path: string; contentHash: string };
  const donors = new WeakMap<NavigationItem, Donor[]>();
  const anchors = new WeakMap<{ path: string; classes: string[] }, Donor>();
  function buffered(item: NavigationItem, sources: Donor[]) {
    donors.set(
      item,
      sources.map(({ path, contentHash }) => ({ path, contentHash })),
    );
    return item;
  }
  let validationQueue: Promise<void> = Promise.resolve();
  async function freshEvaluation(request: EvaluationRequest, sources: Donor[], navigation = false) {
    const validate = async () => {
      for (const source of new Map(sources.map((source) => [source.path, source])).values()) {
        if (!(await unchanged(source))) throw new EvaluationFailure("source-invalid");
      }
    };
    const beforeAttempt = () => {
      const pending = validationQueue.then(validate);
      validationQueue = pending.catch(() => {});
      return pending;
    };
    return evaluator.evaluate(request, { navigation, beforeAttempt });
  }
  let entriesSeen = 0;
  let navigationBytes = 0;
  let positiveScoresSeen = 0;
  let strongScoreSeen = false;
  let confidentSourceSeen = false;
  const sourceProbes = new Set<string>();
  const firstSelections = new Map<string, { contentHash: string; selection: SelectionResult }>();
  let progressBytes = 0;
  let stop = false;
  function issue(kind: string, count = 1, message?: string) {
    if (kind === "provider") providerFailure ??= message;
    issues.set(kind, (issues.get(kind) ?? 0) + count);
    if (["authentication", "request-limit", "cancelled", "interrupted"].includes(kind)) stop = true;
  }
  function reportSnapshot(snapshot: Snapshot) {
    if (!inspected.add(snapshot.path)) return;
    progressBytes += Buffer.byteLength(snapshot.source);
    try {
      input.progress?.({ files: inspected.size, bytes: progressBytes });
    } catch {
      // Progress is observational and must never change search behavior.
    }
  }
  async function snapshot(path: string) {
    const result = await reader.readSnapshot(path, { reuse: true });
    if (result.status === "issue") {
      issue(result.issue.kind);
      return;
    }
    if (result.status === "excluded") return;
    reportSnapshot(result.snapshot);
    return result.snapshot;
  }
  function isCancellation(error: unknown) {
    return (
      (error instanceof EvaluationFailure && error.kind === "cancelled") ||
      (input.signal.aborted &&
        (error === input.signal.reason || (error instanceof Error && error.name === "AbortError")))
    );
  }
  async function score(
    items: Iterable<NavigationItem> | AsyncIterable<NavigationItem>,
    anchor: { path: string; classes: string[] } | undefined,
    consume: (item: NavigationItem, probability: number) => void,
  ) {
    type Group = {
      items: NavigationItem[];
      state: "queued" | "active" | "done";
      results?: Array<{ item: NavigationItem; score: number }>;
    };
    const active = new Set<Promise<void>>();
    const queued: Group[] = [];
    const ordered: Group[] = [];
    const results: Array<{ item: NavigationItem; score: number }> = [];
    let commit = 0;
    let failure: unknown;
    let budgetPaused = false;
    function flush(force = false) {
      while (commit < ordered.length && ordered[commit]!.state === "done") {
        const group = ordered[commit++]!;
        for (const result of group.results ?? []) {
          consume(result.item, result.score);
        }
      }
      if (force) {
        for (; commit < ordered.length; commit++) {
          const group = ordered[commit]!;
          if (group.state !== "done") continue;
          for (const result of group.results ?? []) consume(result.item, result.score);
        }
      }
    }
    function split(group: Group) {
      const middle = Math.ceil(group.items.length / 2);
      const left: Group = { items: group.items.slice(0, middle), state: "queued" };
      const right: Group = { items: group.items.slice(middle), state: "queued" };
      ordered.splice(ordered.indexOf(group), 1, left, right);
      queued.push(left, right);
    }
    async function scoreGroup(group: Group) {
      if (stop || input.signal.aborted) return;
      const request = navigationRequest(input.query, group.items, anchor);
      const requestBytes = Buffer.byteLength(JSON.stringify(request));
      if (
        !confidentSourceSeen &&
        positiveScoresSeen >= minimumPositiveScores &&
        navigationBytes + requestBytes > navigationByteBudget
      ) {
        queued.unshift(group);
        budgetPaused = true;
        return;
      }
      navigationBytes += requestBytes;
      const sources = group.items.flatMap((item) => donors.get(item) ?? []);
      if (anchor) sources.push(anchors.get(anchor)!);
      let scores: Record<string, number>;
      try {
        scores = await freshEvaluation(request, sources, true);
      } catch (error) {
        if (
          error instanceof EvaluationFailure &&
          (error.kind === "source-invalid" || (error.kind === "provider" && error.splitEligible)) &&
          group.items.length > 1
        ) {
          split(group);
          return;
        }
        if (!(error instanceof EvaluationFailure && error.kind === "source-invalid"))
          issue(
            error instanceof EvaluationFailure ? error.kind : "provider",
            1,
            error instanceof EvaluationFailure ? error.message : undefined,
          );
        group.state = "done";
        flush();
        return;
      }
      const classified = group.items.map((item, index) => ({ item, score: scores[`q${index}`]! }));
      for (const { score } of classified) {
        if (score >= weakScoreCeiling) strongScoreSeen = true;
        if (score > relevanceBoundary) positiveScoresSeen++;
      }
      results.push(...classified);
      group.results = classified;
      group.state = "done";
      flush();
      await confirmSource(classified);
    }
    function pump() {
      while (
        !failure &&
        !budgetPaused &&
        active.size < stageWorkers &&
        queued.length &&
        !stop &&
        !input.signal.aborted
      ) {
        const group = queued.shift()!;
        group.state = "active";
        let pending!: Promise<void>;
        pending = scoreGroup(group)
          .catch((error: unknown) => {
            if (isCancellation(error)) issue("cancelled");
            else failure ??= error;
          })
          .finally(() => {
            active.delete(pending);
            pump();
          });
        active.add(pending);
      }
    }
    async function waitForProgress() {
      while (active.size) await Promise.race(active);
    }
    async function settleBudget() {
      if (!budgetPaused) return;
      await waitForProgress();
      if (failure || stop || input.signal.aborted) return;
      if (!confidentSourceSeen) await confirmSource(results, true);
      if (stop || input.signal.aborted) return;
      if (!confidentSourceSeen) {
        issue("low_confidence_budget");
        stop = true;
        return;
      }
      budgetPaused = false;
      pump();
    }
    async function drain() {
      for (;;) {
        if (failure || stop || input.signal.aborted) break;
        if (budgetPaused) {
          await settleBudget();
          continue;
        }
        pump();
        if (active.size) {
          await Promise.race(active);
          continue;
        }
        if (queued.length) continue;
        break;
      }
      await waitForProgress();
      flush(Boolean(failure || stop || input.signal.aborted));
    }
    async function submit(group: NavigationItem[]) {
      const entry: Group = { items: group, state: "queued" };
      ordered.push(entry);
      queued.push(entry);
      pump();
      while (
        !failure &&
        !stop &&
        !input.signal.aborted &&
        (active.size >= stageWorkers || queued.length)
      ) {
        if (budgetPaused) await settleBudget();
        else if (active.size) await Promise.race(active);
        else pump();
      }
    }
    let batch: NavigationItem[] = [];
    try {
      for await (const item of items) {
        if (failure || stop || input.signal.aborted) break;
        if (budgetPaused) await settleBudget();
        if (failure || stop || input.signal.aborted) break;
        if (
          Buffer.byteLength(JSON.stringify(navigationRequest(input.query, [item], anchor))) > 38_000
        ) {
          issue("request-size");
          continue;
        }
        if (
          batch.length &&
          (batch.length >= 128 ||
            Buffer.byteLength(
              JSON.stringify(navigationRequest(input.query, [...batch, item], anchor)),
            ) > 38_000)
        ) {
          await submit(batch);
          batch = [];
        }
        batch.push(item);
      }
      if (batch.length) await submit(batch);
      await drain();
    } catch (error) {
      if (isCancellation(error)) issue("cancelled");
      else failure ??= error;
    } finally {
      await drain();
    }
    if (failure) throw failure;
    flush(Boolean(stop || input.signal.aborted));
    return results;
  }
  async function previewDirectory(path: string): Promise<DirectoryPreview | undefined> {
    const preview: DirectoryPreview = {
      entries: [],
      truncated: false,
      sampledFiles: 0,
      sampledDirectories: 0,
      sampledExtensions: {},
    };
    let cursor: string | undefined;
    let bytes = 0;
    try {
      do {
        const page = await reader.listPage(path, cursor);
        cursor = page.nextCursor;
        for (const entry of page.issues) issue(entry.kind);
        if (page.issues.length) return;
        for (const entry of page.entries) {
          const child = { name: basename(entry.path), kind: entry.kind };
          const size = Buffer.byteLength(JSON.stringify(child));
          if (preview.entries.length >= 64 || bytes + size > 4096) {
            preview.truncated = true;
            break;
          }
          preview.entries.push(child);
          bytes += size;
          if (entry.kind === "directory") preview.sampledDirectories++;
          else {
            preview.sampledFiles++;
            const extension = extname(entry.path) || "[no extension]";
            preview.sampledExtensions[extension] = (preview.sampledExtensions[extension] ?? 0) + 1;
          }
        }
        if (preview.truncated) break;
      } while (cursor && !stop);
      if (cursor) preview.truncated = true;
    } finally {
      if (cursor) await reader.closeCursor(cursor);
    }
    preview.entries.sort((a, b) => a.name.localeCompare(b.name));
    return preview;
  }
  async function withDirectoryContent(item: NavigationItem): Promise<NavigationItem> {
    const preview = {
      ...item.childPreview!,
      contentSamples: [] as NonNullable<DirectoryPreview["contentSamples"]>,
    };
    const sources: Donor[] = [];
    const children = preview.entries.filter((child) => child.kind === "file");
    const perFile = Math.max(80, Math.floor(16000 / Math.max(1, children.length)));
    for (const child of children) {
      if (stop) break;
      const snapshotValue = await snapshot(`${item.path}/${child.name}`);
      if (!snapshotValue || Buffer.byteLength(snapshotValue.source) > 1_000_000) continue;
      sources.push({ path: snapshotValue.path, contentHash: snapshotValue.contentHash });
      const source = snapshotValue.source;
      const part = Math.floor(perFile / 3);
      const offsets = [
        0,
        Math.max(0, Math.floor(source.length / 2) - Math.floor(part / 2)),
        Math.max(0, source.length - part),
      ];
      preview.contentSamples.push({
        name: child.name,
        truncated: source.length > perFile,
        source:
          source.length <= perFile
            ? source
            : offsets
                .map((start) => `[character offset ${start}]\n${source.slice(start, start + part)}`)
                .join("\n...\n"),
      });
    }
    while (
      Buffer.byteLength(JSON.stringify(preview)) > 28000 &&
      preview.contentSamples.some((sample) => sample.source.length > 80)
    ) {
      for (const sample of preview.contentSamples) {
        sample.source = sample.source.slice(
          0,
          Math.max(80, Math.floor(sample.source.length * 0.8)),
        );
        sample.truncated = true;
      }
    }
    return buffered({ ...item, childPreview: preview }, sources);
  }
  async function previewFile(source: Snapshot): Promise<FilePreview> {
    const language = resolveLanguage(source.path, source.source);
    const bytes = Buffer.from(source.source);
    let text = new TextDecoder("utf8", { fatal: true }).decode(bytes.subarray(0, 16384), {
      stream: bytes.length > 16384,
    });
    let truncated = bytes.length > 16384;
    while (Buffer.byteLength(JSON.stringify(text)) > 24000) {
      let end = Math.floor(text.length * 0.75);
      const last = text.charCodeAt(end - 1);
      if (last >= 0xd800 && last <= 0xdbff) end--;
      text = text.slice(0, end);
      truncated = true;
    }
    const preview: FilePreview = {
      sizeBytes: bytes.length,
      extension: extname(source.path),
      text,
      previewBytes: Buffer.byteLength(text),
      truncated,
      range: "opening bytes",
      declarations: [],
      declarationIndexTruncated: false,
    };
    if (truncated && language === "python" && bytes.length <= 1_000_000) {
      const sampled = await pythonPreview(source, input.query, 16384, input.signal);
      if (
        sampled?.truncated &&
        sampled.text &&
        Buffer.byteLength(sampled.text) <= 16384 &&
        Buffer.byteLength(JSON.stringify(sampled.text)) <= 24000
      ) {
        preview.text = sampled.text;
        preview.previewBytes = sampled.previewBytes;
        preview.range = "sampled source ranges";
      }
    }
    if (truncated && language !== undefined && bytes.length <= 1_000_000) {
      const syntax = await inspect(source, {
        signal: input.signal,
        maxUnitBytes: Math.max(4, bytes.length),
      });
      preview.declarations = syntax.units
        .filter((unit) => !unit.partial)
        .map((unit) => ({ name: unit.name, ...unit.range }));
      // Keep the longest prefix of declarations that fits. Serialized size grows with each entry,
      // so a binary search finds the same prefix as removing one declaration at a time.
      const all = preview.declarations;
      const fits = (count: number) => {
        preview.declarations = all.slice(0, count);
        return Buffer.byteLength(JSON.stringify(preview)) <= 32000;
      };
      if (all.length && !fits(all.length)) {
        // Removal measured every shorter prefix with the truncation flag already set.
        preview.declarationIndexTruncated = true;
        let low = 0,
          high = all.length - 1;
        while (low < high) {
          const middle = Math.ceil((low + high) / 2);
          if (fits(middle)) low = middle;
          else high = middle - 1;
        }
        preview.declarations = all.slice(0, low);
      } else preview.declarations = all;
    }
    return preview;
  }
  async function discover(seeds: string[], anchor?: { path: string; classes: string[] }) {
    const directories = [...seeds];
    while (directories.length && !stop && entriesSeen < 100_000) {
      const level = directories.splice(0).map((path) => ({ path, depth: 0 }));
      const hashes = new Map<string, string>();
      async function* items(): AsyncGenerator<NavigationItem> {
        const expanded: NavigationItem[] = [];
        for (let index = 0; index < level.length && !stop; index++) {
          const current = level[index]!;
          if (entriesSeen >= 100_000) {
            issue("resource_limit");
            break;
          }
          if (visited.has(current.path)) continue;
          visited.add(current.path);
          const entries: DirectoryEntry[] = [];
          let cursor: string | undefined;
          try {
            do {
              const page = await reader.listPage(current.path, cursor);
              cursor = page.nextCursor;
              for (const entry of page.issues) issue(entry.kind);
              entries.push(...page.entries);
              if (
                entriesSeen + entries.length > 100_000 ||
                (cursor && entriesSeen + entries.length === 100_000)
              ) {
                issue("resource_limit");
                break;
              }
            } while (cursor && !stop);
          } finally {
            if (cursor) await reader.closeCursor(cursor);
          }
          for (const entry of entries.sort((a, b) => a.path.localeCompare(b.path))) {
            if (stop) break;
            if (entriesSeen++ >= 100_000) {
              issue("resource_limit");
              break;
            }
            if (entry.kind === "directory") {
              if (current.depth === 0) level.push({ path: entry.path, depth: 1 });
              else {
                const childPreview = await previewDirectory(entry.path);
                if (!childPreview) continue;
                const item: NavigationItem = { path: entry.path, kind: "directory", childPreview };
                yield anchor ? await withDirectoryContent(item) : item;
              }
            } else {
              const source = await snapshot(entry.path);
              if (!source) continue;
              hashes.set(entry.path, source.contentHash);
              const filePreview = await previewFile(source);
              previews.set(entry.path, filePreview);
              const item = buffered({ path: entry.path, kind: "file", filePreview }, [source]);
              if (Buffer.byteLength(source.source) > 1_000_000) {
                issue("resource_limit");
                yield item;
                continue;
              }
              if (
                Buffer.byteLength(JSON.stringify(navigationRequest(input.query, [item], anchor))) >
                38_000
              ) {
                for (const chunk of splitSource(source, 12_000)) {
                  const text = sourceForUnit(source, chunk);
                  expanded.push(
                    buffered(
                      {
                        path: entry.path,
                        kind: "file",
                        filePreview: {
                          sizeBytes: filePreview.sizeBytes,
                          extension: filePreview.extension,
                          text,
                          previewBytes: Buffer.byteLength(text),
                          truncated: true,
                          range: "sampled source ranges",
                        },
                      },
                      [source],
                    ),
                  );
                }
              } else yield item;
            }
          }
        }
        yield* expanded;
      }
      const classified: Array<{ item: NavigationItem; score: number }> = [];
      await score(items(), anchor, (item, probability) => {
        classified.push({ item, score: probability });
      });
      const { admitted, suppressWeak } = navigationAdmissions(classified, strongScoreSeen);
      for (const { item, score: probability } of admitted) {
        if (item.kind === "directory") {
          directories.push(item.path);
        } else {
          const prior = candidates.get(item.path);
          if (!prior || probability > prior.score)
            candidates.set(item.path, {
              path: item.path,
              contentHash: hashes.get(item.path)!,
              score: probability,
            });
        }
      }
      for (const { item, score: probability } of classified)
        if (
          item.kind === "directory" &&
          (probability <= relevanceBoundary || (suppressWeak && probability < weakScoreCeiling)) &&
          !anchor
        )
          pruned.set(item.path, item);
    }
    if (directories.length) issue("resource_limit");
  }
  function sortedCandidates() {
    return [...candidates.values()].sort(
      (a, b) => b.score - a.score || a.path.localeCompare(b.path),
    );
  }
  async function unchanged(candidate: { path: string; contentHash: string }) {
    // Unchanged file identity reuses the reader's earlier snapshot; any change rereads and rehashes.
    const result = await reader.readSnapshot(candidate.path, { reuse: true });
    if (result.status === "ok" && result.snapshot.contentHash === candidate.contentHash) {
      reportSnapshot(result.snapshot);
      return result.snapshot;
    }
    issue(result.status === "issue" ? result.issue.kind : "changed");
    if (result.status === "issue" && result.issue.kind === "interrupted") return;
    const prior = files.get(candidate.path);
    if (prior)
      files.set(candidate.path, {
        ...prior,
        roles: [],
        priority: undefined,
        presentationExcerpts: [],
        selectedPresentationExcerpts: [],
        callLeads: [],
        presentationSelected: [],
        sourceDecisions: [],
        leads: [],
        selected: [],
        rendered: [],
        excerpts: [],
        sourceOmitted: true,
      });
  }

  async function selectSource(
    candidate: { path: string; contentHash: string; score: number },
    source: Snapshot,
    previous?: FileEvidence,
    evidence?: () => Promise<Evidence[] | undefined>,
  ) {
    if (Buffer.byteLength(source.source) > 1_000_000) {
      issue("source_inspection_limit");
      return;
    }
    const selection = await selectFile(
      source,
      input.query,
      candidate.score,
      {
        get requests() {
          return evaluator.requests;
        },
        evaluate: (request) => {
          const context = request.state as { selectedEvidence?: Evidence[] };
          return freshEvaluation(request, [
            candidate,
            ...(context.selectedEvidence ?? []).map((entry) => candidates.get(entry.path)!),
          ]);
        },
      },
      async () => {
        if (input.signal.aborted) throw new EvaluationFailure("cancelled");
        const current = await unchanged(candidate);
        if (input.signal.aborted) throw new EvaluationFailure("cancelled");
        if (!current) return null;
        return { evidence: await evidence?.() };
      },
      previous,
      input.signal,
    );
    for (const entry of selection.issues)
      if (entry.kind !== "source-invalid")
        issue(entry.kind, entry.count, selection.providerFailure);
    return selection;
  }
  async function confirmSource(
    choices: Array<{ item: NavigationItem; score: number }>,
    finalProbe = false,
  ) {
    if (
      confidentSourceSeen ||
      (!finalProbe && sourceProbes.size >= 4) ||
      stop ||
      input.signal.aborted
    )
      return;
    const available = choices.flatMap(({ item, score }) =>
      item.kind === "file"
        ? [{ ...donors.get(item)!.find((source) => source.path === item.path)!, score }]
        : [],
    );
    if (finalProbe) available.push(...candidates.values());
    const ordered = finalProbe
      ? available.sort((a, b) => b.score - a.score || a.path.localeCompare(b.path))
      : available;
    const candidate = ordered.find(
      ({ path, score }) =>
        previews.get(path)!.sizeBytes <= (finalProbe ? 1_000_000 : wholeFileContextBytes) &&
        score >= weakScoreCeiling &&
        !(finalProbe ? firstSelections.has(path) : sourceProbes.has(path)),
    );
    if (!candidate) return;
    sourceProbes.add(candidate.path);
    const source = await unchanged(candidate);
    if (
      !source ||
      Buffer.byteLength(source.source) > (finalProbe ? 1_000_000 : wholeFileContextBytes)
    )
      return;
    if (!finalProbe) {
      const syntax = await inspect(source, { signal: input.signal });
      if (syntax.units.length > sourceBatchSize) return;
    }
    const selection = await selectSource(candidate, source);
    if (!selection) return;
    firstSelections.set(candidate.path, { contentHash: candidate.contentHash, selection });
    if (
      !selection.issues.length &&
      !selection.file.sourceOmitted &&
      selection.file.sourceDecisions?.some(({ score }) => score > sourceConfidenceBoundary) &&
      (await unchanged(candidate))
    )
      confidentSourceSeen = true;
  }

  async function parallel<T>(items: T[], work: (item: T) => Promise<void>) {
    let next = 0;
    const results = await Promise.allSettled(
      Array.from({ length: Math.min(stageWorkers, items.length) }, async () => {
        try {
          while (next < items.length && !stop && !input.signal.aborted) await work(items[next++]!);
        } catch (error) {
          stop = true;
          throw error;
        }
      }),
    );
    const failed = results.find((result) => result.status === "rejected");
    if (failed?.status === "rejected") throw failed.reason;
  }
  try {
    try {
      await discover(["."]);
      let anchor: { path: string; classes: string[] } | undefined;
      for (const candidate of sortedCandidates()) {
        if (candidate.score <= relevanceBoundary || stop) break;
        const source = await unchanged(candidate);
        if (!source) continue;
        const size = Buffer.byteLength(source.source);
        const units = (
          await inspect(source, {
            signal: input.signal,
            maxParseBytes: Math.max(1, size),
            maxUnitBytes: Math.max(4, size),
          })
        ).units;
        const classes = [
          ...new Set(
            units.filter((unit) => unit.classContext).map((unit) => unit.name.split(".")[0]!),
          ),
        ];
        if (classes.length && Buffer.byteLength(JSON.stringify(classes)) < 4000) {
          anchor = { path: candidate.path, classes };
          anchors.set(anchor, candidate);
          break;
        }
      }
      if (anchor && !stop) {
        // Only one relationship reconsideration, anchored before new candidates are admitted.
        async function* relationships() {
          for (const item of pruned.values()) {
            if (stop) break;
            yield await withDirectoryContent(item);
          }
        }
        const classified: Array<{ item: NavigationItem; score: number }> = [];
        await score(relationships(), anchor, (item, probability) => {
          classified.push({ item, score: probability });
        });
        const seeds = navigationAdmissions(classified, strongScoreSeen).admitted.map(
          (decision) => decision.item.path,
        );
        await discover(seeds, anchor);
      }
      const ordered = [...candidates.values()];
      // All admitted paths survive even if subsequent source inspection is unavailable.
      for (const candidate of ordered) {
        const prior = firstSelections.get(candidate.path);
        files.set(
          candidate.path,
          prior?.contentHash === candidate.contentHash
            ? {
                ...prior.selection.file,
                score: candidate.score,
              }
            : {
                ...candidate,
                roles: [],
                leads: [],
                selected: [],
                rendered: [],
                excerpts: [],
                sourceOmitted: false,
              },
        );
      }
      const select = async (evidence?: () => Promise<Evidence[] | undefined>) => {
        const selectCandidate = async (candidate: (typeof ordered)[number]) => {
          const source = await unchanged(candidate);
          if (!source) return;
          const prior = firstSelections.get(candidate.path);
          const selection =
            !evidence && prior?.contentHash === candidate.contentHash
              ? prior.selection
              : await selectSource(candidate, source, files.get(candidate.path), evidence);
          if (!selection) return;
          selection.file.score = candidate.score;
          files.set(candidate.path, selection.file);
          declarations.set(candidate.path, selection.declarations);
          groupCounts.set(candidate.path, selection.groups);
        };
        if (evidence) {
          for (const candidate of ordered) {
            if (stop || input.signal.aborted) break;
            await selectCandidate(candidate);
          }
        } else await parallel(ordered, selectCandidate);
      };
      const selectEvidence = async () => {
        await select();
        const evidence: Evidence[] = [];
        // Donor order is part of every follow-up request and its cache key. Files are selected
        // concurrently, so completion order depends on network timing; donors use a deterministic
        // policy instead: fewer declaration groups first, then path.
        const donors = [...declarations.keys()].sort(
          (a, b) => groupCounts.get(a)! - groupCounts.get(b)! || (a < b ? -1 : a > b ? 1 : 0),
        );
        for (const path of donors) {
          const candidate = candidates.get(path)!;
          if (stop || input.signal.aborted) break;
          if (!files.get(candidate.path)!.excerpts.length) continue;
          // Context donors obey the same current eligibility/hash check as target files.
          if (!(await unchanged(candidate))) continue;
          evidence.push(
            ...files.get(candidate.path)!.excerpts.map((excerpt) => ({
              path: candidate.path,
              ...excerpt.range,
              source: excerpt.source,
            })),
          );
        }

        if (evidence.length && Buffer.byteLength(JSON.stringify(evidence)) <= 64_000 && !stop)
          await select(async () => {
            const current = new Set<string>();
            for (const path of new Set(evidence.map((entry) => entry.path))) {
              const candidate = candidates.get(path)!;
              if (await unchanged(candidate)) current.add(path);
            }
            const fresh = evidence.filter((entry) => current.has(entry.path));
            return fresh.length ? fresh : undefined;
          });
      };
      // File assessment reads only the discovery preview, so it runs alongside evidence selection.
      const assessments = new Map<string, { labels: string[]; priority: number }>();
      const assessFiles = () =>
        parallel(ordered, async (candidate) => {
          const source = await unchanged(candidate);
          if (!source) return;
          const preview = previews.get(candidate.path)!;
          try {
            const scores = await freshEvaluation(
              fileAssessmentRequest(input.query, candidate.path, preview),
              [candidate],
            );
            assessments.set(candidate.path, {
              labels: Object.keys(scores).filter(
                (role) => role !== "priority" && scores[role]! > 0.5,
              ),
              priority: scores.priority!,
            });
          } catch (error) {
            if (!(error instanceof EvaluationFailure && error.kind === "source-invalid"))
              issue(
                error instanceof EvaluationFailure ? error.kind : "provider",
                1,
                error instanceof EvaluationFailure ? error.message : undefined,
              );
          }
        });
      await Promise.all([selectEvidence(), assessFiles()]);
      await parallel(ordered, async (candidate) => {
        const file = files.get(candidate.path);
        if (!file || file.sourceOmitted) return;
        const snapshot = await unchanged(candidate);
        if (!snapshot) return;
        if (assessments.get(candidate.path)?.labels.includes("test")) {
          file.presentationExcerpts =
            file.selectedPresentationExcerpts ?? file.presentationExcerpts;
          file.presentationSelected = file.selected;
        }
        try {
          const context = await localCallContext(snapshot, file, input.signal);
          if (context && (await unchanged(candidate)) && files.get(candidate.path) === file)
            Object.assign(file, context);
        } catch (error) {
          if (input.signal.aborted) throw error;
          // Optional structural context cannot discard already selected evidence.
          issue("local-call-context");
        }
      });
      // Selection replaces file records, so assessments attach afterward; invalidated files keep none.
      for (const [path, assessment] of assessments) {
        const file = files.get(path)!;
        if (!file.sourceOmitted) {
          file.roles = assessment.labels;
          file.priority = assessment.priority;
        }
      }
      const testInputs = [];
      for (const candidate of ordered) {
        const file = files.get(candidate.path)!;
        if (file.sourceOmitted || !file.roles.includes("test")) continue;
        const snapshot = await unchanged(candidate);
        if (snapshot) testInputs.push({ snapshot, file });
      }
      try {
        const changes = await selectTestBodies(
          input.query,
          testInputs,
          {
            get requests() {
              return evaluator.requests;
            },
            evaluate: (request) => {
              const state = request.state as { candidates: Record<string, { path: string }> };
              return freshEvaluation(
                request,
                [...new Set(Object.values(state.candidates).map((c) => c.path))].map(
                  (path) => candidates.get(path)!,
                ),
              );
            },
          },
          input.signal,
        );
        let current = true;
        for (const item of testInputs) {
          if (
            !(await unchanged(candidates.get(item.snapshot.path)!)) ||
            files.get(item.snapshot.path) !== item.file
          )
            current = false;
        }
        if (current)
          for (const { file, ...presentation } of changes) Object.assign(file, presentation);
      } catch (error) {
        if (input.signal.aborted) throw input.signal.reason;
        issue(
          error instanceof EvaluationFailure ? error.kind : "test-body-selection",
          1,
          error instanceof EvaluationFailure ? error.message : undefined,
        );
      }
      if (issues.has("authentication") && !files.size)
        throw new EvaluationFailure("authentication");
    } catch (error) {
      if (
        !input.signal.aborted ||
        (error !== input.signal.reason && !(error instanceof Error && error.name === "AbortError"))
      )
        throw error;
      issue("cancelled");
    }
    // Cancellation can occur before the selection phase creates admitted-file records.
    for (const candidate of candidates.values())
      if (!files.has(candidate.path))
        files.set(candidate.path, {
          ...candidate,
          roles: [],
          leads: [],
          selected: [],
          rendered: [],
          excerpts: [],
          sourceOmitted: false,
        });
    const context = await repositoryContext(
      reader,
      sortedCandidates().map((candidate) => files.get(candidate.path)!),
    );
    // File assessment may outlive the bytes it classified, for every language.
    for (const candidate of candidates.values()) await unchanged(candidate);
    return {
      root: reader.root,
      query: input.query,
      status: input.signal.aborted ? "interrupted" : issues.size ? "incomplete" : "complete",
      files: sortedCandidates().map((candidate) => files.get(candidate.path)!),
      repositoryContext: context,
      issues: [...issues].map(([kind, count]) => ({ kind, count })),
      warnings: evaluator.cacheIssues,
      providerFailure,
      counts: {
        requests: evaluator.requests,
        cacheHits: evaluator.cacheHits ?? 0,
        inspectedFiles: inspected.size,
      },
    };
  } finally {
    await reader.close();
  }
}