Skip to content

Turbocharge Claude Code, Cursor, Codex, Gemini & every coding agent: faster, cheaper, with contextual understanding specific to your codebase.

fix(deep): don't abort a 96% meaning pass the same way as a quota error (#235) (#254)

Frankie-Xu415a2de37554
diff --git a/src/ai/crux.ts b/src/ai/crux.ts
index 924ddd2c..dd90d3c8 100644
--- a/src/ai/crux.ts
+++ b/src/ai/crux.ts
@@ -14,7 +14,7 @@
* sees each symbol's neighbours, which sharpens the summaries. Line numbers are
* consumed once, at write time, to slice the crux text verbatim from source.
*/
-import type { ChatModel } from "./llm/types.js";
+import type { ChatModel, ChatResponse } from "./llm/types.js";
import { recoverToolArgsFromContent, warnToolChoiceIgnored } from "./llm/recover-tool.js";
import type { Kind } from "../graph/types.js";
@@ -42,6 +42,40 @@ export interface NodeCrux {
export interface CruxSummarizer {
describeFile(input: FileCruxInput): Promise<NodeCrux[]>;
+ /** Set by {@link ChatCruxSummarizer} after each call; optional on fakes. */
+ lastMiss?: CruxMiss | null;
+}
+
+/** Why a crux call produced no usable summaries (#235). */
+export type CruxMissKind = "empty-toolCalls" | "unparseable" | "truncated" | "empty-parsed";
+
+export interface CruxMiss {
+ kind: CruxMissKind;
+ finishReason: string | null;
+}
+
+function isTruncatedStop(reason: string | null): boolean {
+ if (!reason) return false;
+ const r = reason.toLowerCase();
+ return r === "length" || r === "max_tokens";
+}
+
+/** Classify an empty/unusable crux reply. `null` means at least one usable summary. */
+export function classifyCruxMiss(res: ChatResponse, parsed: NodeCrux[]): CruxMiss | null {
+ const finishReason = res.stopReason;
+ if (parsed.some((p) => p.summary.trim())) return null;
+ if (isTruncatedStop(finishReason)) return { kind: "truncated", finishReason };
+ if (parsed.length > 0) return { kind: "empty-parsed", finishReason };
+ const emptyTools = res.toolCalls.length === 0;
+ const emptyText = !res.text?.trim();
+ if (emptyTools && emptyText) return { kind: "empty-toolCalls", finishReason };
+ return { kind: "unparseable", finishReason };
+}
+
+/** Per-file error text: miss class + the provider's finish_reason (#235). */
+export function formatCruxMiss(kind: CruxMissKind, finishReason: string | null): string {
+ const fr = finishReason == null || finishReason === "" ? "null" : finishReason;
+ return `model returned no usable symbol summaries [${kind}, finish_reason=${fr}]`;
}
const SYSTEM_PROMPT = `You explain code definitions for a code graph that helps engineers navigate a codebase.
@@ -139,9 +173,12 @@ function argsFromResponse(res: { text: string; toolCalls: { name: string; args:
/** Crux summarizer backed by any {@link ChatModel} via forced tool calling. */
export class ChatCruxSummarizer implements CruxSummarizer {
+ lastMiss: CruxMiss | null = null;
+
constructor(private model: ChatModel) {}
async describeFile(input: FileCruxInput): Promise<NodeCrux[]> {
+ this.lastMiss = null;
if (input.nodes.length === 0) return [];
const res = await this.model.create({
temperature: 0,
@@ -159,6 +196,8 @@ export class ChatCruxSummarizer implements CruxSummarizer {
{ role: "user", content: userContent(input) },
],
});
- return parseResults(argsFromResponse(res));
+ const parsed = parseResults(argsFromResponse(res));
+ this.lastMiss = classifyCruxMiss(res, parsed);
+ return parsed;
}
}
diff --git a/src/ai/failure.ts b/src/ai/failure.ts
index 7aa1d76c..45939a2c 100644
--- a/src/ai/failure.ts
+++ b/src/ai/failure.ts
@@ -10,6 +10,11 @@
*
* One gate, shared by both passes, so they agree on what "the provider stopped
* working" means and a caller can report it once.
+ *
+ * Content-quality misses (empty/unusable crux, #235) still count as failed files
+ * so they are not cached as success (#177), but they do not increment the
+ * consecutive-failure cutoff — the provider is answering, just not usefully for
+ * those files. Quota/auth stay immediately terminal (#127).
*/
/** Consecutive failures that end a pass. One flaky file is normal; five in a row is
@@ -55,14 +60,21 @@ export class LlmFailureGate {
return this.fatal !== undefined;
}
- /** Record a failed unit; sets {@link fatal} when this failure is the last straw. */
- record(message: string): void {
+ /**
+ * Record a failed unit; sets {@link fatal} when this failure is the last straw.
+ * Pass `{ quality: true }` for a content-quality miss (#235): counted, not fatal,
+ * unless the message is quota/auth (those still stop immediately).
+ */
+ record(message: string, opts?: { quality?: boolean }): void {
this.failed++;
- this.consecutive++;
const terminal = terminalReason(message);
if (terminal) {
this.fatal = `${terminal} — stopped after ${this.failed} failed file(s). First error: ${message}`;
- } else if (this.consecutive >= MAX_CONSECUTIVE_FAILURES) {
+ return;
+ }
+ if (opts?.quality) return;
+ this.consecutive++;
+ if (this.consecutive >= MAX_CONSECUTIVE_FAILURES) {
this.fatal = `${this.consecutive} files in a row failed, so the pass stopped rather than keep calling. Last error: ${message}`;
}
}
diff --git a/src/graph/enrich.ts b/src/graph/enrich.ts
index bbbd180e..f18a3b2c 100644
--- a/src/graph/enrich.ts
+++ b/src/graph/enrich.ts
@@ -19,7 +19,7 @@
* once, to cut `crux.code` verbatim from the source. `crux.span` is a pointer
* only and is never used to re-slice.
*/
-import type { CruxSummarizer, NodeCrux, NodeRef } from "../ai/crux.js";
+import { formatCruxMiss, type CruxMissKind, type CruxSummarizer, type NodeCrux, type NodeRef } from "../ai/crux.js";
import { LlmFailureGate } from "../ai/failure.js";
import type { Crux, NodeV1 } from "./types.js";
@@ -65,9 +65,9 @@ export interface EnrichStats {
failedFiles: number;
/** Files never attempted, because {@link EnrichStats.fatal} stopped the pass. */
skippedFiles: number;
- /** Set when the pass gave up early: quota/auth rejection, or a run of failures
- * that says the provider is not going to start working. The reason is written
- * for a human — it is what `graft build --deep` exits non-zero with. */
+ /** Set when the pass gave up early: quota/auth rejection, or a run of
+ * provider failures. Content-quality misses (#235) count in `failedFiles` but
+ * do not set this. The reason is what `graft build --deep` exits non-zero with. */
fatal?: string;
}
@@ -168,11 +168,11 @@ export async function enrichGraph(
return { id: n.id, kind: n.kind, signature: n.signature, startLine, endLine };
});
- const { results, error } = await collectFileCrux(summarizer, path, source, refs);
+ const { results, error, quality } = await collectFileCrux(summarizer, path, source, refs);
let fileError = error;
if (fileError) {
stats.errors.push(`${path}: ${fileError}`);
- gate.record(fileError);
+ gate.record(fileError, { quality });
}
let applied = 0;
@@ -197,9 +197,9 @@ export async function enrichGraph(
if (!fileError && refs.length > 0 && applied === 0) {
// collectFileCrux already labels a total miss; this catches "got entries
// but every summary was blank" so the CLI's #127 degraded-exit path fires.
- fileError = "model returned no usable symbol summaries";
+ fileError = cruxMissMessage(summarizer, "empty-parsed");
stats.errors.push(`${path}: ${fileError}`);
- gate.record(fileError);
+ gate.record(fileError, { quality: true });
} else if (!fileError) {
gate.succeeded();
}
@@ -240,6 +240,11 @@ async function mapWithConcurrency<T, R>(
return results;
}
+function cruxMissMessage(summarizer: CruxSummarizer, fallback: CruxMissKind): string {
+ const miss = summarizer.lastMiss;
+ return formatCruxMiss(miss?.kind ?? fallback, miss?.finishReason ?? null);
+}
+
/**
* Describe every requested definition in a file, re-asking for any the model
* omits (it sometimes drops entries from a batch). Returns whatever it collected
@@ -250,7 +255,7 @@ async function collectFileCrux(
path: string,
source: string,
refs: NodeRef[],
-): Promise<{ results: Map<string, NodeCrux>; error?: string }> {
+): Promise<{ results: Map<string, NodeCrux>; error?: string; quality?: boolean }> {
const results = new Map<string, NodeCrux>();
let missing = refs;
let error: string | undefined;
@@ -267,9 +272,9 @@ async function collectFileCrux(
// A total miss used to return `{ results: ∅ }` with no error — enrich left
// every node `pending`, the CLI exited 0, and `graft check` told the user to
// re-run `--deep` forever (#172). Surface it as a failure like a thrown error.
+ // Content-quality, not quota: count the file, keep going (#235).
if (!error && refs.length > 0 && results.size === 0) {
- error =
- "model returned no symbol summaries (empty tool response — the provider may ignore forced tool_choice)";
+ return { results, error: cruxMissMessage(summarizer, "empty-toolCalls"), quality: true };
}
return { results, error };
}
diff --git a/test/graph-enrich-quality.test.ts b/test/graph-enrich-quality.test.ts
new file mode 100644
index 00000000..1e09a24f
--- /dev/null
+++ b/test/graph-enrich-quality.test.ts
@@ -0,0 +1,186 @@
+/**
+ * #235: after #177, empty/unusable crux replies are real failures (not silent
+ * pending). Combined with LlmFailureGate's consecutive-failure cutoff that
+ * turned a 96% meaning pass into a hard abort — resume then died on the same
+ * files. Content-quality misses must stay failed (and uncached), but must not
+ * trip the gate the way quota/auth does.
+ */
+import { test } from "node:test";
+import assert from "node:assert/strict";
+import { enrichGraph } from "../src/graph/enrich.js";
+import { ChatCruxSummarizer } from "../src/ai/crux.js";
+import { LlmFailureGate } from "../src/ai/failure.js";
+import type { CruxSummarizer, FileCruxInput, NodeCrux } from "../src/ai/crux.js";
+import type { ChatModel, ChatRequest, ChatResponse, ToolCall } from "../src/ai/llm/types.js";
+import type { NodeV1 } from "../src/graph/types.js";
+
+class EmptyCrux implements CruxSummarizer {
+ calls = 0;
+ async describeFile(_input: FileCruxInput): Promise<NodeCrux[]> {
+ this.calls++;
+ return [];
+ }
+}
+
+class CannedModel implements ChatModel {
+ readonly label = "fake:quality";
+ constructor(
+ private readonly reply: {
+ text?: string;
+ toolCalls?: ToolCall[];
+ stopReason?: string | null;
+ },
+ ) {}
+ async create(_req: ChatRequest): Promise<ChatResponse> {
+ const text = this.reply.text ?? "";
+ return {
+ text,
+ toolCalls: this.reply.toolCalls ?? [],
+ usage: { input: 0, output: 0, cacheRead: 0, cacheCreate: 0 },
+ stopReason: this.reply.stopReason ?? "stop",
+ assistant: { role: "assistant", content: text },
+ };
+ }
+}
+
+function node(path: string): NodeV1 {
+ return {
+ id: `${path}#run`,
+ name: "run",
+ kind: "function",
+ path,
+ span: "L1-L3",
+ signature: "run()",
+ exported: true,
+ origin: "ast",
+ body_hash: `h-${path}`,
+ summary_state: "pending",
+ summary: null,
+ crux: null,
+ };
+}
+
+function fixture(n: number): { nodes: NodeV1[]; sources: Map<string, string> } {
+ const paths = Array.from({ length: n }, (_, i) => `f${i}.ts`);
+ return {
+ nodes: paths.map(node),
+ sources: new Map(paths.map((p) => [p, "export function run() {\n return 1;\n}\n"])),
+ };
+}
+
+function oneFile(summarizer: CruxSummarizer) {
+ const { nodes, sources } = fixture(1);
+ return enrichGraph(nodes, new Map(), sources, { summarizer, concurrency: 1 }).then((stats) => ({
+ stats,
+ nodes,
+ }));
+}
+
+test("#235: consecutive empty crux replies do not abort the rest of the pass", async () => {
+ const { nodes, sources } = fixture(12);
+ const summarizer = new EmptyCrux();
+
+ const stats = await enrichGraph(nodes, new Map(), sources, { summarizer, concurrency: 1 });
+
+ assert.equal(stats.fatal, undefined, "content-quality misses must not trip the consecutive-failure gate");
+ assert.ok(summarizer.calls >= 12, `every file must be attempted, saw ${summarizer.calls} calls`);
+ assert.equal(stats.failedFiles, 12);
+ assert.equal(stats.skippedFiles, 0);
+ assert.equal(stats.computed, 0);
+ for (const n of nodes) {
+ assert.equal(n.summary_state, "pending", "empty replies must not be cached as ready (#177)");
+ assert.equal(n.summary, null);
+ }
+});
+
+test("#235: consecutive 401s still stop the pass immediately (#127)", async () => {
+ const { nodes, sources } = fixture(10);
+ class AuthFail implements CruxSummarizer {
+ calls = 0;
+ async describeFile(): Promise<NodeCrux[]> {
+ this.calls++;
+ throw new Error("401 Unauthorized: invalid api key");
+ }
+ }
+ const summarizer = new AuthFail();
+
+ const stats = await enrichGraph(nodes, new Map(), sources, { summarizer, concurrency: 1 });
+
+ assert.match(stats.fatal ?? "", /rejected the API key/);
+ assert.equal(summarizer.calls, 1, "quota/auth is still terminal on the first file");
+ assert.equal(stats.failedFiles, 1);
+ assert.equal(stats.skippedFiles, 9);
+});
+
+test("#235: LlmFailureGate.record quality misses never set fatal; quota still does", () => {
+ const quality = new LlmFailureGate();
+ for (let i = 0; i < 12; i++) {
+ quality.record("model returned no usable symbol summaries [empty-toolCalls, finish_reason=stop]", {
+ quality: true,
+ });
+ }
+ assert.equal(quality.fatal, undefined);
+ assert.equal(quality.failed, 12);
+ assert.equal(quality.stopped, false);
+
+ const auth = new LlmFailureGate();
+ auth.record("401 Unauthorized: invalid api key", { quality: true });
+ assert.match(auth.fatal ?? "", /rejected the API key/);
+});
+
+test("#235: empty toolCalls names empty-toolCalls and finish_reason", async () => {
+ const summarizer = new ChatCruxSummarizer(new CannedModel({ toolCalls: [], text: "", stopReason: "stop" }));
+ const { stats } = await oneFile(summarizer);
+ assert.equal(stats.failedFiles, 1);
+ assert.match(stats.errors[0] ?? "", /empty-toolCalls/);
+ assert.match(stats.errors[0] ?? "", /finish_reason=stop/);
+});
+
+test("#235: truncated output names truncated and finish_reason=length", async () => {
+ const summarizer = new ChatCruxSummarizer(
+ new CannedModel({
+ text: '{"symbols":[{"id":"f0.ts#run","summary":"partial',
+ toolCalls: [],
+ stopReason: "length",
+ }),
+ );
+ const { stats } = await oneFile(summarizer);
+ assert.equal(stats.failedFiles, 1);
+ assert.match(stats.errors[0] ?? "", /truncated/);
+ assert.match(stats.errors[0] ?? "", /finish_reason=length/);
+});
+
+test("#235: unparseable content names unparseable", async () => {
+ const summarizer = new ChatCruxSummarizer(
+ new CannedModel({
+ text: "The architecture is a layered monolith.",
+ toolCalls: [],
+ stopReason: "stop",
+ }),
+ );
+ const { stats } = await oneFile(summarizer);
+ assert.equal(stats.failedFiles, 1);
+ assert.match(stats.errors[0] ?? "", /unparseable/);
+ assert.match(stats.errors[0] ?? "", /finish_reason=stop/);
+});
+
+test("#235: blank parsed summaries name empty-parsed and stay pending", async () => {
+ const summarizer = new ChatCruxSummarizer(
+ new CannedModel({
+ toolCalls: [
+ {
+ id: "1",
+ name: "record_symbols",
+ args: { symbols: [{ id: "f0.ts#run", summary: " ", crux_start: 0, crux_end: 0 }] },
+ },
+ ],
+ stopReason: "stop",
+ }),
+ );
+ const { stats, nodes } = await oneFile(summarizer);
+ assert.equal(stats.failedFiles, 1);
+ assert.equal(stats.computed, 0);
+ assert.match(stats.errors[0] ?? "", /empty-parsed/);
+ assert.equal(nodes[0].summary_state, "pending");
+ assert.equal(nodes[0].summary, null);
+});