Skip to content

Codebase intelligence for AI and humans: code health scores, auto-generated docs, git analytics, dead code detection, and architectural decisions via MCP.

fix(chat): show normalized confidence in source citations, not raw score (#846)

Raghav Chamadiya43e0cf7bf8a0
diff --git a/packages/ui/__tests__/chat/source-citations.test.tsx b/packages/ui/__tests__/chat/source-citations.test.tsx
new file mode 100644
index 000000000..f9b990e26
--- /dev/null
+++ b/packages/ui/__tests__/chat/source-citations.test.tsx
@@ -0,0 +1,65 @@
+import { describe, it, expect } from "vitest";
+import { extractSources } from "../../src/chat/source-citations.js";
+import type { ChatUIToolCall } from "@repowise-dev/types/chat";
+
+function searchCall(results: Array<Record<string, unknown>>): ChatUIToolCall {
+ return {
+ id: "tc1",
+ name: "search_codebase",
+ arguments: { query: "where is auth handled" },
+ result: { results },
+ status: "done",
+ };
+}
+
+describe("extractSources", () => {
+ it("reads the rank-normalized confidence, not the raw backend score", () => {
+ // BM25 fallback scores are unbounded — reading relevance_score here is what
+ // rendered "1808%" in the sources list.
+ const sources = extractSources(
+ [
+ searchCall([
+ {
+ page_id: "file_page:auth.py",
+ title: "auth.py",
+ relevance_score: 18.08,
+ confidence_score: 1,
+ },
+ ]),
+ ],
+ "repo1",
+ );
+
+ expect(sources).toHaveLength(1);
+ expect(sources[0]?.confidence).toBe(1);
+ });
+
+ it("keeps every confidence renderable as a 0-100% badge", () => {
+ const sources = extractSources(
+ [
+ searchCall([
+ { page_id: "a", title: "a", relevance_score: 18.08, confidence_score: 1 },
+ { page_id: "b", title: "b", relevance_score: 14.75, confidence_score: 0.82 },
+ { page_id: "c", title: "c", relevance_score: 0.43, confidence_score: 0.02 },
+ ]),
+ ],
+ "repo1",
+ );
+
+ expect(sources).toHaveLength(3);
+ for (const s of sources) {
+ expect(s.confidence).toBeGreaterThanOrEqual(0);
+ expect(s.confidence).toBeLessThanOrEqual(1);
+ }
+ });
+
+ it("omits confidence when the server sends no confidence_score", () => {
+ const sources = extractSources(
+ [searchCall([{ page_id: "file_page:a.py", title: "a.py", relevance_score: 9.1 }])],
+ "repo1",
+ );
+
+ // Badge hides rather than falling back to the unbounded raw score.
+ expect(sources[0]?.confidence).toBeUndefined();
+ });
+});
diff --git a/packages/ui/src/chat/source-citations.tsx b/packages/ui/src/chat/source-citations.tsx
index 56a5a72c0..fb6bf50c9 100644
--- a/packages/ui/src/chat/source-citations.tsx
+++ b/packages/ui/src/chat/source-citations.tsx
@@ -11,7 +11,12 @@ export interface SourceReference {
title: string;
pageType: string;
targetPath: string;
- score?: number;
+ /**
+ * Rank-normalized 0–1 confidence, never the raw backend score: embedding
+ * hits are cosine (0–1) but the BM25 fallback is unbounded, so the raw
+ * score has no fixed scale to render as a percentage.
+ */
+ confidence?: number | undefined;
toolName: string;
}
@@ -39,7 +44,7 @@ export function extractSources(
title: (r.title as string) ?? pageId,
pageType: (r.page_type as string) ?? "file_page",
targetPath: (r.target_path as string) ?? "",
- score: r.relevance_score as number ?? r.score as number,
+ confidence: r.confidence_score as number | undefined,
toolName: tc.name,
});
}
@@ -176,9 +181,9 @@ export function SourceCitations({
</span>
<SourceIcon pageType={source.pageType} className="h-3 w-3 shrink-0 opacity-60" />
<span className="truncate max-w-[160px] font-medium">{source.title}</span>
- {source.score != null && (
+ {source.confidence != null && (
<span className="text-[10px] text-[var(--color-text-tertiary)] tabular-nums">
- {(source.score * 100).toFixed(0)}%
+ {(source.confidence * 100).toFixed(0)}%
</span>
)}
<ArrowUpRight className="h-2.5 w-2.5 opacity-0 group-hover:opacity-100 transition-opacity shrink-0" />