Skip to content

Codebase intelligence for AI and humans: code health scores, auto-generated docs, git analytics, dead code detection, and architectural decisions via MCP.

fix(chat): stop suggesting questions no tool can answer (#2315)

Raghav Chamadiyaa3fd8b21a6ea
diff --git a/packages/ui/__tests__/chat/source-citations.test.tsx b/packages/ui/__tests__/chat/source-citations.test.tsx
index 742155d9c..020e47dde 100644
--- a/packages/ui/__tests__/chat/source-citations.test.tsx
+++ b/packages/ui/__tests__/chat/source-citations.test.tsx
@@ -17,6 +17,86 @@ function searchCall(results: Array<Record<string, unknown>>): ChatUIToolCall {
}
describe("extractSources", () => {
+ // These five tools produced no citation at all before: a reader could not
+ // check the scored claim the answer was built on. Field names are the live
+ // wire shapes, which differ per tool (`file` on get_symbol, `file_path`
+ // elsewhere, a path-keyed object on get_risk).
+ it("cites the files the scored tools actually read", () => {
+ const calls: ChatUIToolCall[] = [
+ {
+ id: "r1",
+ name: "get_risk",
+ arguments: {},
+ result: { targets: { "packages/core/persist.py": { hotspot_score: 0.9 } } },
+ status: "done",
+ },
+ {
+ id: "h1",
+ name: "get_health",
+ arguments: {},
+ result: { findings: [{ file_path: "packages/cli/command.py" }] },
+ status: "done",
+ },
+ {
+ id: "d1",
+ name: "get_dead_code",
+ arguments: {},
+ result: { high_confidence: [{ file_path: "packages/web/unused.ts" }] },
+ status: "done",
+ },
+ {
+ id: "s1",
+ name: "get_symbol",
+ arguments: {},
+ result: { file: "packages/ui/widget.tsx", name: "Widget" },
+ status: "done",
+ },
+ ];
+
+ const paths = extractSources(calls, "repo1").map((s) => s.targetPath);
+
+ expect(paths).toContain("packages/core/persist.py");
+ expect(paths).toContain("packages/cli/command.py");
+ expect(paths).toContain("packages/web/unused.ts");
+ expect(paths).toContain("packages/ui/widget.tsx");
+ });
+
+ it("does not cite a tool call that failed", () => {
+ // A failed read is marked status "error"; citing it would advertise
+ // evidence the answer never had.
+ const sources = extractSources(
+ [
+ {
+ id: "r2",
+ name: "get_risk",
+ arguments: {},
+ result: { error: "no git metadata available" },
+ status: "error",
+ },
+ ],
+ "repo1",
+ );
+
+ expect(sources).toHaveLength(0);
+ });
+
+ it("survives a malformed payload rather than throwing", () => {
+ const sources = extractSources(
+ [
+ {
+ id: "h2",
+ name: "get_health",
+ arguments: {},
+ result: { findings: [{ file_path: 42 }], targets: [null, "ok.py"] },
+ status: "done",
+ },
+ ],
+ "repo1",
+ );
+
+ expect(sources.map((s) => s.targetPath)).toEqual(["ok.py"]);
+ });
+
it("reads the rank-normalized confidence, not the raw backend score", () => {
// BM25 fallback scores are unbounded — reading relevance_score here is what
// rendered "1808%" in the sources list.
diff --git a/packages/ui/src/chat/chat-context.ts b/packages/ui/src/chat/chat-context.ts
index e05637baf..492c836d0 100644
--- a/packages/ui/src/chat/chat-context.ts
+++ b/packages/ui/src/chat/chat-context.ts
@@ -49,7 +49,7 @@ const PRESENTATION_COPY: Record<ChatContextKind, PresentationCopy> = {
placeholder: "Ask about this repository, or paste a file path",
suggestions: [
"Give me an overview of this codebase",
- "What are the highest-risk files to modify?",
+ "Which files have the worst code health?",
"What architectural decisions have been made?",
"What dead code can be safely removed?",
],
@@ -95,11 +95,11 @@ const PRESENTATION_COPY: Record<ChatContextKind, PresentationCopy> = {
],
},
refactoring: {
- placeholder: "Ask about this refactoring opportunity",
+ placeholder: "Ask about this repository's refactoring targets",
suggestions: [
"Turn this into a safe refactoring plan",
- "Which tests should protect this change?",
- "What could break if I modify these files?",
+ "Which refactoring opportunity is safest to start with?",
+ "Which refactoring targets affect the most dependents?",
],
},
file: {
@@ -175,19 +175,19 @@ const PRESENTATION_COPY: Record<ChatContextKind, PresentationCopy> = {
],
},
security: {
- placeholder: "Ask about these security findings",
+ placeholder: "Ask how this repository tracks security evidence",
suggestions: [
- "Which security finding should be investigated first?",
+ "How does this repository detect and track security findings?",
"Explain the evidence without overstating certainty",
- "Which code paths are affected?",
],
},
usage: {
- placeholder: "Ask about usage and savings",
+ // Usage and billing figures live on the page, not behind any chat tool.
+ placeholder: "Ask about this repository (usage figures are on the page)",
suggestions: [
- "Explain the largest source of usage",
- "Which figures are measured versus estimated?",
- "Where could usage be reduced safely?",
+ "Give me an overview of this codebase",
+ "Which files have the worst code health?",
+ "What dead code can be safely removed?",
],
},
settings: {
@@ -202,7 +202,7 @@ const PRESENTATION_COPY: Record<ChatContextKind, PresentationCopy> = {
placeholder: "Ask a follow-up, or paste a file path",
suggestions: [
"Give me an overview of this codebase",
- "What are the highest-risk files to modify?",
+ "Which files have the worst code health?",
"Score the change risk of HEAD",
"What architectural decisions have been made?",
],
@@ -216,22 +216,19 @@ const COLLECTION_PRESENTATION_COPY: Partial<
placeholder: "Ask about this repository's documentation",
suggestions: [
"Which documentation should I read first?",
- "Where is documentation missing or stale?",
- "Connect the documentation to its source files",
+ "Where should a new contributor start?",
],
},
file: {
placeholder: "Ask about files in this repository",
suggestions: [
"Which files are the main entry points?",
- "Find the files responsible for a feature",
- "Which files are riskiest to modify?",
+ "Which files have the worst code health?",
],
},
symbol: {
placeholder: "Ask about symbols in this repository",
suggestions: [
- "Find the symbol responsible for a behavior",
"Which symbols have the widest impact?",
"Show the most important public interfaces",
],
@@ -247,8 +244,8 @@ const COLLECTION_PRESENTATION_COPY: Partial<
commit: {
placeholder: "Ask about repository history",
suggestions: [
- "Summarize the most important recent changes",
- "Which files change together most often?",
+ "Review the risk of the latest commit",
+ "Which files have the most bug-fix history?",
"Find the history behind an architectural choice",
],
},
diff --git a/packages/ui/src/chat/chat-interface.tsx b/packages/ui/src/chat/chat-interface.tsx
index dececba5e..447de42bb 100644
--- a/packages/ui/src/chat/chat-interface.tsx
+++ b/packages/ui/src/chat/chat-interface.tsx
@@ -53,7 +53,7 @@ import { useChatScroll } from "./use-chat-scroll";
const DEFAULT_SUGGESTIONS: readonly ChatSuggestion[] = [
"Give me an overview of this codebase",
- "What are the highest-risk files to modify?",
+ "Which files have the worst code health?",
"Score the change risk of HEAD",
"What dead code can be safely removed?",
"What architectural decisions have been made?",
diff --git a/packages/ui/src/chat/source-citations.tsx b/packages/ui/src/chat/source-citations.tsx
index 6cdaf05a5..b1d0a603b 100644
--- a/packages/ui/src/chat/source-citations.tsx
+++ b/packages/ui/src/chat/source-citations.tsx
@@ -28,6 +28,24 @@ export function extractSources(
const seen = new Set<string>();
const sources: SourceReference[] = [];
+ // The scored tools (risk, health, dead code, change risk, one symbol) cite
+ // the files they scored. Without this the claims a reader most needs to
+ // check, a risk score or a health finding, carry no link at all.
+ const pushFile = (toolCallId: string, toolName: string, filePath: unknown) => {
+ if (typeof filePath !== "string" || !filePath) return;
+ const pageId = filePageId(filePath);
+ if (seen.has(pageId)) return;
+ seen.add(pageId);
+ sources.push({
+ id: `${toolCallId}:${pageId}`,
+ pageId,
+ title: filePath.split("/").pop() ?? filePath,
+ pageType: "file_page",
+ targetPath: filePath,
+ toolName,
+ });
+ };
+
for (const tc of toolCalls) {
if (tc.status !== "done" || !tc.result) continue;
@@ -98,6 +116,45 @@ export function extractSources(
}
}
+ // `targets` is a path-keyed object on the MCP wire and an array on legacy
+ // fixtures; both shapes appear in stored conversations.
+ if (tc.name === "get_risk" || tc.name === "get_change_risk") {
+ const targets = result.targets as Record<string, unknown> | unknown[] | undefined;
+ if (Array.isArray(targets)) {
+ for (const row of targets.slice(0, 8)) {
+ const r = row as Record<string, unknown>;
+ pushFile(tc.id, tc.name, r.file_path ?? r.target);
+ }
+ } else if (targets && typeof targets === "object") {
+ for (const key of Object.keys(targets).slice(0, 8)) pushFile(tc.id, tc.name, key);
+ }
+ const fixHistory = result.fix_history as { files?: Array<{ path?: string }> } | undefined;
+ for (const f of (fixHistory?.files ?? []).slice(0, 5)) pushFile(tc.id, tc.name, f.path);
+ }
+
+ if (tc.name === "get_health") {
+ const findings = (result.findings as Array<Record<string, unknown>>) ?? [];
+ for (const f of findings.slice(0, 8)) pushFile(tc.id, tc.name, f.file_path);
+ const opportunities =
+ (result.refactoring_opportunities as Array<Record<string, unknown>>) ?? [];
+ for (const o of opportunities.slice(0, 5)) pushFile(tc.id, tc.name, o.file_path);
+ const targets = (result.targets as unknown[]) ?? [];
+ for (const t of targets.slice(0, 8)) pushFile(tc.id, tc.name, t);
+ }
+
+ if (tc.name === "get_dead_code") {
+ const rows = [
+ ...((result.high_confidence as Array<Record<string, unknown>>) ?? []),
+ ...((result.medium_confidence as Array<Record<string, unknown>>) ?? []),
+ ];
+ for (const r of rows.slice(0, 10)) pushFile(tc.id, tc.name, r.file_path);
+ }
+
+ // get_symbol names the file it verified as `file`, not `file_path`.
+ if (tc.name === "get_symbol") {
+ pushFile(tc.id, tc.name, result.file);
+ }
+
if (tc.name === "get_why") {
const decisions = (result.decisions as Array<Record<string, unknown>>)
?? (result.matching_decisions as Array<Record<string, unknown>>)
diff --git a/packages/ui/src/chat/tool-call-block.tsx b/packages/ui/src/chat/tool-call-block.tsx
index eb29b67a7..c4b06b09f 100644
--- a/packages/ui/src/chat/tool-call-block.tsx
+++ b/packages/ui/src/chat/tool-call-block.tsx
@@ -76,14 +76,20 @@ export function ToolCallBlock({
<span className="font-medium text-[var(--color-text-secondary)]">
{label}
</span>
- {toolCall.summary && !isRunning && (
- <span className="text-[var(--color-text-tertiary)] truncate ml-1">
- — {toolCall.summary}
+ {!isRunning && (toolCall.summary || isError) && (
+ // The server already composes a failed summary as "Error: ...", so
+ // a separate Failed badge beside it just says the same thing twice.
+ <span
+ className={cn(
+ "truncate ml-1",
+ isError
+ ? "text-[var(--color-error)]"
+ : "text-[var(--color-text-tertiary)]",
+ )}
+ >
+ — {toolCall.summary || "Failed"}
</span>
)}
- {isError && (
- <span className="text-[var(--color-error)]">— Failed</span>
- )}
</button>
<span className="ml-auto flex items-center gap-2 shrink-0">
{toolCall.artifact && onViewArtifact && !isRunning && (
diff --git a/packages/web/src/lib/chat/to-chat-ui-messages.test.ts b/packages/web/src/lib/chat/to-chat-ui-messages.test.ts
index 8c2240688..4e29e734d 100644
--- a/packages/web/src/lib/chat/to-chat-ui-messages.test.ts
+++ b/packages/web/src/lib/chat/to-chat-ui-messages.test.ts
@@ -3,6 +3,28 @@ import type { ChatMessageResponse } from "@/lib/api/types";
import { toChatUiMessages } from "./to-chat-ui-messages";
describe("toChatUiMessages", () => {
+ it.each([
+ ["an errored read", { error: "no git metadata available" }, "error"],
+ ["a good read", { score: 8.4 }, "done"],
+ ])("reloads %s with the right status", (_label, result, expected) => {
+ // The wire persists no status. Hardcoding "done" here laundered a failed
+ // read back into a successful one on every reload, and it was cited as
+ // evidence again.
+ const stored: ChatMessageResponse[] = [
+ {
+ id: "m1",
+ conversation_id: "c1",
+ role: "assistant",
+ content: {
+ tool_calls: [{ id: "t1", name: "get_risk", result }],
+ },
+ created_at: "2026-08-28T00:00:00Z",
+ },
+ ];
+
+ expect(toChatUiMessages(stored)[0]?.toolCalls[0]?.status).toBe(expected);
+ });
+
it("restores the persisted artifact envelope without rebuilding it", () => {
const stored: ChatMessageResponse[] = [
{
diff --git a/packages/web/src/lib/chat/to-chat-ui-messages.ts b/packages/web/src/lib/chat/to-chat-ui-messages.ts
index 9cfeb0775..74e6739eb 100644
--- a/packages/web/src/lib/chat/to-chat-ui-messages.ts
+++ b/packages/web/src/lib/chat/to-chat-ui-messages.ts
@@ -38,7 +38,15 @@ export function toChatUiMessages(
},
}
: {}),
- status: "done" as const,
+ // The wire never persists a status, so reload it from the same `error`
+ // key the live stream reads. Without this a failed read comes back from
+ // storage looking like a good one, and is cited as evidence again.
+ status:
+ typeof toolCall.result === "object" &&
+ toolCall.result !== null &&
+ "error" in toolCall.result
+ ? ("error" as const)
+ : ("done" as const),
...(toolCall.origin ? { origin: toolCall.origin } : {}),
})),
isStreaming: false,
diff --git a/packages/web/src/lib/hooks/use-chat.test.tsx b/packages/web/src/lib/hooks/use-chat.test.tsx
index ac07728d2..53a9cb9fb 100644
--- a/packages/web/src/lib/hooks/use-chat.test.tsx
+++ b/packages/web/src/lib/hooks/use-chat.test.tsx
@@ -124,6 +124,58 @@ describe("useChat lifecycle", () => {
vi.unstubAllGlobals();
});
+ it.each([
+ ["an errored read", { error: "no git metadata available" }, "error"],
+ ["a good read", { targets: { "a.py": {} } }, "done"],
+ ] as const)("marks %s on the tool call", async (_label, data, expected) => {
+ // The server reports an unservable read as an `error` key. Before this,
+ // tool_result set "done" unconditionally, so a failed read rendered
+ // identically to a good one and the answer above it looked evidence-backed.
+ let streamController!: ReadableStreamDefaultController<Uint8Array>;
+ const stream = new ReadableStream<Uint8Array>({
+ start(controller) {
+ streamController = controller;
+ },
+ });
+ mocks.postChatMessage.mockImplementationOnce(() =>
+ Promise.resolve(new Response(stream)),
+ );
+ const { result } = renderHook(() => useChat("r1"));
+
+ act(() => {
+ void result.current.sendMessage("What is risky here?");
+ });
+ await act(async () => {
+ streamController.enqueue(
+ new TextEncoder().encode(
+ [
+ `data: ${JSON.stringify({
+ type: "tool_start",
+ tool_id: "t1",
+ tool_name: "get_risk",
+ input: {},
+ })}
+`,
+ `data: ${JSON.stringify({
+ type: "tool_result",
+ tool_id: "t1",
+ tool_name: "get_risk",
+ summary: "Risk assessment",
+ artifact: { type: "risk_report", data },
+ })}
+`,
+ ].join(""),
+ ),
+ );
+ await Promise.resolve();
+ await Promise.resolve();
+ });
+
+ expect(result.current.messages.at(-1)?.toolCalls?.[0]?.status).toBe(expected);
+
+ act(() => result.current.cancel());
+ });
+
it.each(["resolve", "reject"] as const)(
"ignores a late conversation %s from the previous repository",
async (outcome) => {
diff --git a/packages/web/src/lib/hooks/use-chat.ts b/packages/web/src/lib/hooks/use-chat.ts
index ecf04ac9d..0fcea23db 100644
--- a/packages/web/src/lib/hooks/use-chat.ts
+++ b/packages/web/src/lib/hooks/use-chat.ts
@@ -257,21 +257,29 @@ export function useChat(repoId: string) {
],
};
- case "tool_result":
+ case "tool_result": {
+ // The server reports an unservable read as an `error` key plus
+ // an "Error: ..." summary. Without this the failed row renders
+ // exactly like a good one, so the answer above it reads as
+ // evidence-backed when nothing was read.
+ const data = ev.artifact.data as unknown as Record<string, unknown>;
+ const failed =
+ typeof data === "object" && data !== null && "error" in data;
return {
...m,
toolCalls: m.toolCalls.map((tc) =>
tc.id === ev.tool_id
? {
...tc,
- result: ev.artifact.data as unknown as Record<string, unknown>,
+ result: data,
summary: ev.summary,
artifact: ev.artifact,
- status: "done" as const,
+ status: failed ? ("error" as const) : ("done" as const),
}
: tc,
),
};
+ }
case "truncated":
return { ...m, truncated: true };