Skip to content

Local code intelligence for developers and AI agents. Connect code, docs and infrastructure in a persistent graph, with CBM Lens, MCP tools, agent integrations and change-aware test selection.

fix(ui): flag only real names in the unknown names note

Bernhard Jackiewicz4ba4d5a38fd5
diff --git a/graph-ui/src/browser-ai/BrowserChatDock.tsx b/graph-ui/src/browser-ai/BrowserChatDock.tsx
index 2ec19548d..179756c96 100644
--- a/graph-ui/src/browser-ai/BrowserChatDock.tsx
+++ b/graph-ui/src/browser-ai/BrowserChatDock.tsx
@@ -124,7 +124,7 @@ function TokenLimitNote({ limit, automatic, chat, model, onChangeOutput }: Limit
function AnswerNotes({ shortened, packet, model, historyOmitted, unsupported = [] }: { shortened?: LimitNote; packet?: PreparedExplanationContext; model: string; historyOmitted?: number; unsupported?: readonly string[] }): JSX.Element {
const capacity = packet?.capacity;
return <>
- {unsupported.length > 0 && <small className="cbm-chat-answer-note">{browserChatText.unsupportedNames(unsupported.slice(0, 6))}</small>}
+ {unsupported.length > 0 && <small className="cbm-chat-answer-note">{browserChatText.unsupportedNames(unsupported)}</small>}
{shortened && <TokenLimitNote {...shortened} />}
{capacity && <small className="cbm-chat-answer-note">{browserChatText.capacity(capacity.nodes, capacity.edges, model, capacity.shown)}</small>}
{!!historyOmitted && <small className="cbm-chat-answer-note">{browserChatText.historyTrimmed(historyOmitted)}</small>}
diff --git a/graph-ui/src/browser-ai/BrowserChatDock.wording.test.tsx b/graph-ui/src/browser-ai/BrowserChatDock.wording.test.tsx
index 9c2d2a44e..7b8ce6ae8 100644
--- a/graph-ui/src/browser-ai/BrowserChatDock.wording.test.tsx
+++ b/graph-ui/src/browser-ai/BrowserChatDock.wording.test.tsx
@@ -149,3 +149,15 @@ describe('the token limit note (W6)', () => {
expect(noteButtons('.cbm-chat-explanation')).toEqual(['Change the output limit']);
});
});
+
+describe('the note about names the answer was not given (W7)', () => {
+ it('counts the names it does not show and says plainly when the answer is likely made up', async () => {
+ const { props, runtime } = fixture();
+ runtime.chat.mockResolvedValueOnce(`It uses ${Array.from({ length: 9 }, (_, index) => `\`helper_${index}\``).join(', ')}, \`%s\`, \`False\` and runs on PostgreSQL.`);
+ await render({ ...props, proactiveSelection: jsonbAggEvidence() }); await load();
+ await ask('Explain this class in detail, line by line.');
+ const note = [...last().querySelectorAll('.cbm-chat-answer-note')].map(item => item.textContent ?? '').find(text => text.includes('graph facts')) ?? '';
+ expect(note).toBe('9 names in this answer are not in the source or graph facts it was given: helper_0, helper_1, helper_2, helper_3, helper_4, helper_5, +3 more. '
+ + 'The answer is likely made up; do not rely on it.');
+ });
+});
diff --git a/graph-ui/src/browser-ai/chat-wording.test.ts b/graph-ui/src/browser-ai/chat-wording.test.ts
index e84482ec4..a835cb251 100644
--- a/graph-ui/src/browser-ai/chat-wording.test.ts
+++ b/graph-ui/src/browser-ai/chat-wording.test.ts
@@ -1,6 +1,6 @@
import { describe, expect, it } from 'vitest';
import { prepareExplanationContext, selectionSummary } from './explanation-context';
-import { explanationSentence, formatExplanationEvidence } from './explanation-response';
+import { explanationSentence, formatExplanationEvidence, namesNotIn } from './explanation-response';
import { jsonbAggEvidence, jsonbAggRenderLimited } from './galaxy-evidence.fixture';
import { relationshipAnswer } from './relationship-answer';
import { browserChatText, groundedText } from './strings';
@@ -159,3 +159,28 @@ describe('why the model sentence was left out (W5)', () => {
expect(groundedText.de.sentenceDropped()).not.toMatch(/hier:|Fakten nicht zeigen/);
});
});
+
+describe('the note about names the answer was not given (W7)', () => {
+ const given = 'Selected: JSONBAgg (Class) in django/contrib/postgres/aggregates/general.py:50-54.\nclass JSONBAgg(OrderableAggMixin, Aggregate):\n function = "JSONB_AGG"';
+
+ it('flags no placeholders, literals or tokens without letters', () => {
+ const answer = 'It wraps `SUM`, `AVG`, `%()`, `%s`, `%(distinct)s`, `{}`, `{0}`, `False`, `True`, `None`, `null`, `undefined` and `42` in JavaScript.';
+ expect(namesNotIn(answer, given)).toEqual(['SUM', 'AVG', 'JavaScript']);
+ });
+
+ it('accepts a product name that only extends a long folder name of the selection, and nothing looser', () => {
+ expect(namesNotIn('It runs only on PostgreSQL.', given)).toEqual([]);
+ expect(namesNotIn('It calls PostgresHelperFactory and postgres_json_helper.', given)).toEqual(['PostgresHelperFactory', 'postgres_json_helper']);
+ // A short segment ("aggregates" is long, "py" is not) never widens the check.
+ expect(namesNotIn('It uses PyTorch.', given)).toEqual(['PyTorch']);
+ });
+
+ it('shows six names with the rest counted, and says plainly when the answer is likely made up', () => {
+ const names = (count: number) => Array.from({ length: count }, (_, index) => `made_up_${index}`);
+ expect(browserChatText.unsupportedNames(names(7))).toBe('Not in the source or graph facts this answer was given: made_up_0, made_up_1, made_up_2, '
+ + 'made_up_3, made_up_4, made_up_5, +1 more. Check these names before relying on them.');
+ expect(browserChatText.unsupportedNames(names(21))).toBe('21 names in this answer are not in the source or graph facts it was given: made_up_0, made_up_1, '
+ + 'made_up_2, made_up_3, made_up_4, made_up_5, +15 more. The answer is likely made up; do not rely on it.');
+ expect(browserChatText.unsupportedNames(names(2))).toBe('Not in the source or graph facts this answer was given: made_up_0, made_up_1. Check these names before relying on them.');
+ });
+});
diff --git a/graph-ui/src/browser-ai/explanation-response.ts b/graph-ui/src/browser-ai/explanation-response.ts
index 1815006a7..adf875495 100644
--- a/graph-ui/src/browser-ai/explanation-response.ts
+++ b/graph-ui/src/browser-ai/explanation-response.ts
@@ -78,15 +78,26 @@ const IDENTIFIER = /\b(?:[A-Za-z]+_\w+|[a-z]+[A-Z]\w*|[A-Z][a-z0-9]+[A-Z]\w*|[A-
function nameCheck(given: string): (name: string) => boolean {
const words = new Set((given.match(/[A-Za-z_][A-Za-z0-9_]*/g) ?? []).map(word => word.toLowerCase()));
const text = given.toLowerCase();
- return name => /^[A-Za-z_][A-Za-z0-9_]*$/.test(name) ? words.has(name.toLowerCase()) : text.includes(name.toLowerCase());
+ // Folders and files of at least six letters in the paths given ("postgres" of django/contrib/postgres).
+ const segments = [...new Set((given.match(/[\w.-]+(?:\/[\w.-]+)+/g) ?? []).flatMap(path => path.split('/'))
+ .map(segment => segment.replace(/\.\w+$/, '').toLowerCase()).filter(segment => /^[a-z]{6,}$/.test(segment)))];
+ // "PostgreSQL" beside django/contrib/postgres names the product of that folder, not a made-up
+ // symbol: a word of letters only, at most three letters longer than the segment (W7).
+ const extendsFolder = (name: string) => /^[A-Za-z]+$/.test(name) && segments.some(segment => name.toLowerCase().startsWith(segment) && name.length - segment.length <= 3);
+ return name => (/^[A-Za-z_][A-Za-z0-9_]*$/.test(name) ? words.has(name.toLowerCase()) : text.includes(name.toLowerCase())) || extendsFolder(name);
}
+/** Literals and format placeholders are no names: `False`, `None`, `%s`, `%(distinct)s`, `{0}` (W7). */
+const LITERAL = /^(?:true|false|none|null|undefined|nil|nan)$/i;
+const PLACEHOLDER = /^(?:%(?:\([^)]*\))?[-#0 +]*\d*(?:\.\d+)?[a-z%]?|\{[^{}]*\})$/i;
+const nameLike = (name: string) => /\p{L}/u.test(name) && !LITERAL.test(name) && !PLACEHOLDER.test(name);
+
/** Names an answer uses that the request it was given does not contain: written in
* backticks or shaped like an identifier (flake8, json_agg_helper). Shown under the answer (K12). */
export function namesNotIn(answer: string, given: string): string[] {
const named = [...answer.matchAll(/`([^`\n]+)`/g)].map(match => match[1].trim()).concat(answer.replace(/`[^`]*`/g, ' ').match(IDENTIFIER) ?? []);
const known = nameCheck(given);
- return [...new Set(named.filter(name => name && !known(name)))];
+ return [...new Set(named.filter(name => name && nameLike(name) && !known(name)))];
}
/** Why the model's sentence was left out, as the sentence wrote it: a claim word the source does
@@ -120,7 +131,7 @@ export function explanationSentence(output: string, packet: PreparedExplanationC
const claims = [...sentence.matchAll(new RegExp(UNSUPPORTED_CLAIM.source, 'gi'))]
.map(match => ({ word: match[0].toLowerCase().replace(/(?:s|ing)$/, '').split(' ')[0], shown: match[0] }))
.concat(GERMAN_CLAIMS.flatMap(([pattern, word]) => { const match = pattern.exec(sentence); return match ? [{ word, shown: shownClaim(match[0]) }] : []; }));
- const unknown = named.find(name => !known(name));
+ const unknown = named.find(name => nameLike(name) && !known(name));
if (unknown) return { dropped: 'unsupported', reason: { kind: 'name', text: unknown } };
const claim = mode === 'code' ? undefined : claims.find(item => mode === 'graph' || !code.includes(item.word));
if (claim) return { dropped: 'unsupported', reason: { kind: 'claim', text: claim.shown } };
diff --git a/graph-ui/src/browser-ai/strings.ts b/graph-ui/src/browser-ai/strings.ts
index 3e5c0e313..955007a3f 100644
--- a/graph-ui/src/browser-ai/strings.ts
+++ b/graph-ui/src/browser-ai/strings.ts
@@ -212,7 +212,12 @@ export const browserChatText = {
autoLoadNote: 'Only from this browser\'s cache; nothing is downloaded on start. A project switch stays in this page and keeps a loaded model without loading it again, so this option only matters when the page is opened or reloaded.',
resumeFailed: 'The cached model could not be loaded without a download. Load it in the agent configuration.',
/** Under an answer that names what its source and graph facts do not contain (K12). */
- unsupportedNames: (names: readonly string[]) => `Not in the source or graph facts this answer was given: ${names.join(', ')}. Check these names before relying on them.`,
+ /** Six names and the rest counted; with many unknown names the answer is most likely invented (W7). */
+ unsupportedNames: (names: readonly string[]) => {
+ const listed = `${names.slice(0, 6).join(', ')}${names.length > 6 ? `, +${(names.length - 6).toLocaleString('en-US')} more` : ''}`;
+ return names.length >= 8 ? `${names.length.toLocaleString('en-US')} names in this answer are not in the source or graph facts it was given: ${listed}. The answer is likely made up; do not rely on it.`
+ : `Not in the source or graph facts this answer was given: ${listed}. Check these names before relying on them.`;
+ },
/** Who wrote which part of a grounded automatic explanation (K7). */
factsAndSentence: 'Facts listed from the indexed graph; the last sentence is generated by the model.',
factsOnly: 'Listed from the indexed graph; not generated by the model.',