Turbocharge Claude Code, Cursor, Codex, Gemini & every coding agent: faster, cheaper, with contextual understanding specific to your codebase.
TypeScript ai-agentsanthropicclaude-codeclicode-graphcodexcontext-engineeringcursordeveloper-toolsgeminiknowledge-graphllmmcpmcp-serveropen-sourceopenaitree-sitter
fix(deep): don't abort a 96% meaning pass the same way as a quota error (#235) (#254)
Frankie-Xu415a2de37554
diff --git a/src/ai/crux.ts b/src/ai/crux.tsindex 924ddd2c..dd90d3c8 100644--- a/src/ai/crux.ts+++ b/src/ai/crux.ts@@ -14,7 +14,7 @@* sees each symbol's neighbours, which sharpens the summaries. Line numbers are* consumed once, at write time, to slice the crux text verbatim from source.*/-import type { ChatModel } from "./llm/types.js";+import type { ChatModel, ChatResponse } from "./llm/types.js";import { recoverToolArgsFromContent, warnToolChoiceIgnored } from "./llm/recover-tool.js";import type { Kind } from "../graph/types.js";@@ -42,6 +42,40 @@ export interface NodeCrux {export interface CruxSummarizer {describeFile(input: FileCruxInput): Promise<NodeCrux[]>;+ /** Set by {@link ChatCruxSummarizer} after each call; optional on fakes. */+ lastMiss?: CruxMiss | null;+}++/** Why a crux call produced no usable summaries (#235). */+export type CruxMissKind = "empty-toolCalls" | "unparseable" | "truncated" | "empty-parsed";++export interface CruxMiss {+ kind: CruxMissKind;+ finishReason: string | null;+}++function isTruncatedStop(reason: string | null): boolean {+ if (!reason) return false;+ const r = reason.toLowerCase();+ return r === "length" || r === "max_tokens";+}++/** Classify an empty/unusable crux reply. `null` means at least one usable summary. */+export function classifyCruxMiss(res: ChatResponse, parsed: NodeCrux[]): CruxMiss | null {+ const finishReason = res.stopReason;+ if (parsed.some((p) => p.summary.trim())) return null;+ if (isTruncatedStop(finishReason)) return { kind: "truncated", finishReason };+ if (parsed.length > 0) return { kind: "empty-parsed", finishReason };+ const emptyTools = res.toolCalls.length === 0;+ const emptyText = !res.text?.trim();+ if (emptyTools && emptyText) return { kind: "empty-toolCalls", finishReason };+ return { kind: "unparseable", finishReason };+}++/** Per-file error text: miss class + the provider's finish_reason (#235). */+export function formatCruxMiss(kind: CruxMissKind, finishReason: string | null): string {+ const fr = finishReason == null || finishReason === "" ? "null" : finishReason;+ return `model returned no usable symbol summaries [${kind}, finish_reason=${fr}]`;}const SYSTEM_PROMPT = `You explain code definitions for a code graph that helps engineers navigate a codebase.@@ -139,9 +173,12 @@ function argsFromResponse(res: { text: string; toolCalls: { name: string; args:/** Crux summarizer backed by any {@link ChatModel} via forced tool calling. */export class ChatCruxSummarizer implements CruxSummarizer {+ lastMiss: CruxMiss | null = null;+constructor(private model: ChatModel) {}async describeFile(input: FileCruxInput): Promise<NodeCrux[]> {+ this.lastMiss = null;if (input.nodes.length === 0) return [];const res = await this.model.create({temperature: 0,@@ -159,6 +196,8 @@ export class ChatCruxSummarizer implements CruxSummarizer {{ role: "user", content: userContent(input) },],});- return parseResults(argsFromResponse(res));+ const parsed = parseResults(argsFromResponse(res));+ this.lastMiss = classifyCruxMiss(res, parsed);+ return parsed;}}diff --git a/src/ai/failure.ts b/src/ai/failure.tsindex 7aa1d76c..45939a2c 100644--- a/src/ai/failure.ts+++ b/src/ai/failure.ts@@ -10,6 +10,11 @@** One gate, shared by both passes, so they agree on what "the provider stopped* working" means and a caller can report it once.+ *+ * Content-quality misses (empty/unusable crux, #235) still count as failed files+ * so they are not cached as success (#177), but they do not increment the+ * consecutive-failure cutoff — the provider is answering, just not usefully for+ * those files. Quota/auth stay immediately terminal (#127).*//** Consecutive failures that end a pass. One flaky file is normal; five in a row is@@ -55,14 +60,21 @@ export class LlmFailureGate {return this.fatal !== undefined;}- /** Record a failed unit; sets {@link fatal} when this failure is the last straw. */- record(message: string): void {+ /**+ * Record a failed unit; sets {@link fatal} when this failure is the last straw.+ * Pass `{ quality: true }` for a content-quality miss (#235): counted, not fatal,+ * unless the message is quota/auth (those still stop immediately).+ */+ record(message: string, opts?: { quality?: boolean }): void {this.failed++;- this.consecutive++;const terminal = terminalReason(message);if (terminal) {this.fatal = `${terminal} — stopped after ${this.failed} failed file(s). First error: ${message}`;- } else if (this.consecutive >= MAX_CONSECUTIVE_FAILURES) {+ return;+ }+ if (opts?.quality) return;+ this.consecutive++;+ if (this.consecutive >= MAX_CONSECUTIVE_FAILURES) {this.fatal = `${this.consecutive} files in a row failed, so the pass stopped rather than keep calling. Last error: ${message}`;}}diff --git a/src/graph/enrich.ts b/src/graph/enrich.tsindex bbbd180e..f18a3b2c 100644--- a/src/graph/enrich.ts+++ b/src/graph/enrich.ts@@ -19,7 +19,7 @@* once, to cut `crux.code` verbatim from the source. `crux.span` is a pointer* only and is never used to re-slice.*/-import type { CruxSummarizer, NodeCrux, NodeRef } from "../ai/crux.js";+import { formatCruxMiss, type CruxMissKind, type CruxSummarizer, type NodeCrux, type NodeRef } from "../ai/crux.js";import { LlmFailureGate } from "../ai/failure.js";import type { Crux, NodeV1 } from "./types.js";@@ -65,9 +65,9 @@ export interface EnrichStats {failedFiles: number;/** Files never attempted, because {@link EnrichStats.fatal} stopped the pass. */skippedFiles: number;- /** Set when the pass gave up early: quota/auth rejection, or a run of failures- * that says the provider is not going to start working. The reason is written- * for a human — it is what `graft build --deep` exits non-zero with. */+ /** Set when the pass gave up early: quota/auth rejection, or a run of+ * provider failures. Content-quality misses (#235) count in `failedFiles` but+ * do not set this. The reason is what `graft build --deep` exits non-zero with. */fatal?: string;}@@ -168,11 +168,11 @@ export async function enrichGraph(return { id: n.id, kind: n.kind, signature: n.signature, startLine, endLine };});- const { results, error } = await collectFileCrux(summarizer, path, source, refs);+ const { results, error, quality } = await collectFileCrux(summarizer, path, source, refs);let fileError = error;if (fileError) {stats.errors.push(`${path}: ${fileError}`);- gate.record(fileError);+ gate.record(fileError, { quality });}let applied = 0;@@ -197,9 +197,9 @@ export async function enrichGraph(if (!fileError && refs.length > 0 && applied === 0) {// collectFileCrux already labels a total miss; this catches "got entries// but every summary was blank" so the CLI's #127 degraded-exit path fires.- fileError = "model returned no usable symbol summaries";+ fileError = cruxMissMessage(summarizer, "empty-parsed");stats.errors.push(`${path}: ${fileError}`);- gate.record(fileError);+ gate.record(fileError, { quality: true });} else if (!fileError) {gate.succeeded();}@@ -240,6 +240,11 @@ async function mapWithConcurrency<T, R>(return results;}+function cruxMissMessage(summarizer: CruxSummarizer, fallback: CruxMissKind): string {+ const miss = summarizer.lastMiss;+ return formatCruxMiss(miss?.kind ?? fallback, miss?.finishReason ?? null);+}+/*** Describe every requested definition in a file, re-asking for any the model* omits (it sometimes drops entries from a batch). Returns whatever it collected@@ -250,7 +255,7 @@ async function collectFileCrux(path: string,source: string,refs: NodeRef[],-): Promise<{ results: Map<string, NodeCrux>; error?: string }> {+): Promise<{ results: Map<string, NodeCrux>; error?: string; quality?: boolean }> {const results = new Map<string, NodeCrux>();let missing = refs;let error: string | undefined;@@ -267,9 +272,9 @@ async function collectFileCrux(// A total miss used to return `{ results: ∅ }` with no error — enrich left// every node `pending`, the CLI exited 0, and `graft check` told the user to// re-run `--deep` forever (#172). Surface it as a failure like a thrown error.+ // Content-quality, not quota: count the file, keep going (#235).if (!error && refs.length > 0 && results.size === 0) {- error =- "model returned no symbol summaries (empty tool response — the provider may ignore forced tool_choice)";+ return { results, error: cruxMissMessage(summarizer, "empty-toolCalls"), quality: true };}return { results, error };}diff --git a/test/graph-enrich-quality.test.ts b/test/graph-enrich-quality.test.tsnew file mode 100644index 00000000..1e09a24f--- /dev/null+++ b/test/graph-enrich-quality.test.ts@@ -0,0 +1,186 @@+/**+ * #235: after #177, empty/unusable crux replies are real failures (not silent+ * pending). Combined with LlmFailureGate's consecutive-failure cutoff that+ * turned a 96% meaning pass into a hard abort — resume then died on the same+ * files. Content-quality misses must stay failed (and uncached), but must not+ * trip the gate the way quota/auth does.+ */+import { test } from "node:test";+import assert from "node:assert/strict";+import { enrichGraph } from "../src/graph/enrich.js";+import { ChatCruxSummarizer } from "../src/ai/crux.js";+import { LlmFailureGate } from "../src/ai/failure.js";+import type { CruxSummarizer, FileCruxInput, NodeCrux } from "../src/ai/crux.js";+import type { ChatModel, ChatRequest, ChatResponse, ToolCall } from "../src/ai/llm/types.js";+import type { NodeV1 } from "../src/graph/types.js";++class EmptyCrux implements CruxSummarizer {+ calls = 0;+ async describeFile(_input: FileCruxInput): Promise<NodeCrux[]> {+ this.calls++;+ return [];+ }+}++class CannedModel implements ChatModel {+ readonly label = "fake:quality";+ constructor(+ private readonly reply: {+ text?: string;+ toolCalls?: ToolCall[];+ stopReason?: string | null;+ },+ ) {}+ async create(_req: ChatRequest): Promise<ChatResponse> {+ const text = this.reply.text ?? "";+ return {+ text,+ toolCalls: this.reply.toolCalls ?? [],+ usage: { input: 0, output: 0, cacheRead: 0, cacheCreate: 0 },+ stopReason: this.reply.stopReason ?? "stop",+ assistant: { role: "assistant", content: text },+ };+ }+}++function node(path: string): NodeV1 {+ return {+ id: `${path}#run`,+ name: "run",+ kind: "function",+ path,+ span: "L1-L3",+ signature: "run()",+ exported: true,+ origin: "ast",+ body_hash: `h-${path}`,+ summary_state: "pending",+ summary: null,+ crux: null,+ };+}++function fixture(n: number): { nodes: NodeV1[]; sources: Map<string, string> } {+ const paths = Array.from({ length: n }, (_, i) => `f${i}.ts`);+ return {+ nodes: paths.map(node),+ sources: new Map(paths.map((p) => [p, "export function run() {\n return 1;\n}\n"])),+ };+}++function oneFile(summarizer: CruxSummarizer) {+ const { nodes, sources } = fixture(1);+ return enrichGraph(nodes, new Map(), sources, { summarizer, concurrency: 1 }).then((stats) => ({+ stats,+ nodes,+ }));+}++test("#235: consecutive empty crux replies do not abort the rest of the pass", async () => {+ const { nodes, sources } = fixture(12);+ const summarizer = new EmptyCrux();++ const stats = await enrichGraph(nodes, new Map(), sources, { summarizer, concurrency: 1 });++ assert.equal(stats.fatal, undefined, "content-quality misses must not trip the consecutive-failure gate");+ assert.ok(summarizer.calls >= 12, `every file must be attempted, saw ${summarizer.calls} calls`);+ assert.equal(stats.failedFiles, 12);+ assert.equal(stats.skippedFiles, 0);+ assert.equal(stats.computed, 0);+ for (const n of nodes) {+ assert.equal(n.summary_state, "pending", "empty replies must not be cached as ready (#177)");+ assert.equal(n.summary, null);+ }+});++test("#235: consecutive 401s still stop the pass immediately (#127)", async () => {+ const { nodes, sources } = fixture(10);+ class AuthFail implements CruxSummarizer {+ calls = 0;+ async describeFile(): Promise<NodeCrux[]> {+ this.calls++;+ throw new Error("401 Unauthorized: invalid api key");+ }+ }+ const summarizer = new AuthFail();++ const stats = await enrichGraph(nodes, new Map(), sources, { summarizer, concurrency: 1 });++ assert.match(stats.fatal ?? "", /rejected the API key/);+ assert.equal(summarizer.calls, 1, "quota/auth is still terminal on the first file");+ assert.equal(stats.failedFiles, 1);+ assert.equal(stats.skippedFiles, 9);+});++test("#235: LlmFailureGate.record quality misses never set fatal; quota still does", () => {+ const quality = new LlmFailureGate();+ for (let i = 0; i < 12; i++) {+ quality.record("model returned no usable symbol summaries [empty-toolCalls, finish_reason=stop]", {+ quality: true,+ });+ }+ assert.equal(quality.fatal, undefined);+ assert.equal(quality.failed, 12);+ assert.equal(quality.stopped, false);++ const auth = new LlmFailureGate();+ auth.record("401 Unauthorized: invalid api key", { quality: true });+ assert.match(auth.fatal ?? "", /rejected the API key/);+});++test("#235: empty toolCalls names empty-toolCalls and finish_reason", async () => {+ const summarizer = new ChatCruxSummarizer(new CannedModel({ toolCalls: [], text: "", stopReason: "stop" }));+ const { stats } = await oneFile(summarizer);+ assert.equal(stats.failedFiles, 1);+ assert.match(stats.errors[0] ?? "", /empty-toolCalls/);+ assert.match(stats.errors[0] ?? "", /finish_reason=stop/);+});++test("#235: truncated output names truncated and finish_reason=length", async () => {+ const summarizer = new ChatCruxSummarizer(+ new CannedModel({+ text: '{"symbols":[{"id":"f0.ts#run","summary":"partial',+ toolCalls: [],+ stopReason: "length",+ }),+ );+ const { stats } = await oneFile(summarizer);+ assert.equal(stats.failedFiles, 1);+ assert.match(stats.errors[0] ?? "", /truncated/);+ assert.match(stats.errors[0] ?? "", /finish_reason=length/);+});++test("#235: unparseable content names unparseable", async () => {+ const summarizer = new ChatCruxSummarizer(+ new CannedModel({+ text: "The architecture is a layered monolith.",+ toolCalls: [],+ stopReason: "stop",+ }),+ );+ const { stats } = await oneFile(summarizer);+ assert.equal(stats.failedFiles, 1);+ assert.match(stats.errors[0] ?? "", /unparseable/);+ assert.match(stats.errors[0] ?? "", /finish_reason=stop/);+});++test("#235: blank parsed summaries name empty-parsed and stay pending", async () => {+ const summarizer = new ChatCruxSummarizer(+ new CannedModel({+ toolCalls: [+ {+ id: "1",+ name: "record_symbols",+ args: { symbols: [{ id: "f0.ts#run", summary: " ", crux_start: 0, crux_end: 0 }] },+ },+ ],+ stopReason: "stop",+ }),+ );+ const { stats, nodes } = await oneFile(summarizer);+ assert.equal(stats.failedFiles, 1);+ assert.equal(stats.computed, 0);+ assert.match(stats.errors[0] ?? "", /empty-parsed/);+ assert.equal(nodes[0].summary_state, "pending");+ assert.equal(nodes[0].summary, null);+});