Skip to content

Local code intelligence for developers and AI agents. Connect code, docs and infrastructure in a persistent graph, with CBM Lens, MCP tools, agent integrations and change-aware test selection.

test(ui): expect file questions answered from the file in the chat check (C7)

Bernhard Jackiewicz63e2f40469c4
diff --git a/graph-ui/tools/handtest-fixes-chat.mjs b/graph-ui/tools/handtest-fixes-chat.mjs
index 83e51e3ff..5a9cb46ac 100644
--- a/graph-ui/tools/handtest-fixes-chat.mjs
+++ b/graph-ui/tools/handtest-fixes-chat.mjs
@@ -367,7 +367,8 @@ async function k17(page) {
await loadModel(page);
await openChat(page);
await explanationDone(page);
- const exploreAnswer = await ask(page, 'was kannst du mir über dieses aktuelle File sagen');
+ // A question the model answers: "was kannst du mir über ..." is answered from the file since C7, and the point here is a model answer that must not come back.
+ const exploreAnswer = await ask(page, 'Welche Aktion nutzt dieses aktuelle File?');
await shot(page, 'K17', 'explore-answer', 'A question about the YAML workflow in Explore and its answer.');
await tab(page, 'galaxy');
await galaxyReady(page);
@@ -397,11 +398,11 @@ async function k17(page) {
if (dividerBox) await shot(page, 'K17', 'explore-return-crop', 'Crop of the divider above the returned question.', { clip: { x: dividerBox.x - 8, y: Math.max(0, dividerBox.y - 8), width: dividerBox.width + 16, height: 220 } });
const backAssistant = (back?.messages ?? []).filter((message) => message.role === 'assistant').length;
const backUsers = (back?.messages ?? []).filter((message) => message.role === 'user').map((message) => message.content.split('\n').at(-1));
- check('K17', 'Earlier answers about another file are not resent, also not on the way back; topic breaks and New conversation visible', Boolean(request) && !/was kannst du mir über dieses aktuelle File/.test(text)
+ check('K17', 'Earlier answers about another file are not resent, also not on the way back; topic breaks and New conversation visible', Boolean(request) && !/Welche Aktion nutzt dieses aktuelle File/.test(text)
&& !text.includes(exploreAnswer.slice(-60).trim()) && assistantTurns === 0 && /New conversation/.test(header) && divider.some((item) => /New topic: JSONBAgg/.test(item))
- && Boolean(back) && backAssistant === 0 && !/was kannst du mir über dieses aktuelle File/.test(backText) && !backText.includes(exploreAnswer.slice(-60).trim())
+ && Boolean(back) && backAssistant === 0 && !/Welche Aktion nutzt dieses aktuelle File/.test(backText) && !backText.includes(exploreAnswer.slice(-60).trim())
&& JSON.stringify(backUsers) === JSON.stringify(['Und was noch?']) && divider.some((item) => /New topic: \.github\/workflows\/new_contributor_pr\.yml\. Earlier messages are not sent/.test(item)),
- { assistantMessagesInPrompt: assistantTurns, flake8InPrompt: /flake8/i.test(text), exploreQuestionInPrompt: /aktuelle File/.test(text), header, divider, galaxyAnswer: galaxyAnswer.slice(0, 200),
+ { assistantMessagesInPrompt: assistantTurns, flake8InPrompt: /flake8/i.test(text), exploreQuestionInPrompt: /aktuelle File/.test(text), header, divider, exploreAnswer: exploreAnswer.slice(0, 200), galaxyAnswer: galaxyAnswer.slice(0, 200),
returnPrompt: { assistantMessages: backAssistant, userMessages: backUsers, earlierExploreAnswerInPrompt: backText.includes(exploreAnswer.slice(-60).trim()) }, returnAnswer: returnAnswer.slice(0, 200) });
await save('K17', 'prompt-galaxy.txt', text);
await save('K17', 'prompt-explore-return.txt', backText);
@@ -679,6 +680,7 @@ async function k12(page) {
check('K12', 'YAML facts count a list of mappings as a list with item keys (.pre-commit-config.yaml: repos, list of 5) and a mapping by its keys (.readthedocs.yml)',
yaml.length === 2 && yaml.every((item) => item.opened && item.wanted && !item.repeatedKeys && item.factsOnly && item.modelCalls === 0), { yaml });
// Explicit questions about the workflow go to the model, as before (back by its tree row: no reload, the model stays loaded).
+ // Since C7 "what does this file do" questions are answered from the file instead: no request, the counted facts, the read-from-file note.
await openTreeFile(page, '.github/workflows/new_contributor_pr.yml');
await explanationDone(page);
const questions = ['was kannst du mir über dieses aktuelle File sagen', 'Wie viele Jobs gibt es in dieser Datei?', 'Welche Jobs gibt es in dieser Datei?', 'What does this file do?', 'How many jobs does this workflow have?'];
@@ -694,10 +696,15 @@ async function k12(page) {
}
await save('K12', 'outputs.json', { outputs, others });
const questionsAsked = outputs.filter((item) => item.kind === 'question');
+ const FROM_FILE = ['was kannst du mir über dieses aktuelle File sagen', 'What does this file do?'];
+ const modelQuestions = questionsAsked.filter((item) => !FROM_FILE.includes(item.question));
+ const fromFile = questionsAsked.filter((item) => FROM_FILE.includes(item.question));
+ const answeredFromFile = fromFile.length > 0 && fromFile.every((item) => !item.prompt && /1 (?:job|Job): build/.test(item.answer)
+ && /Read from the file; not generated by the model\.|Aus der Datei gelesen, nicht vom Modell erzeugt\./.test(item.answer));
// The open file is in the system section of the request, named as a workflow, with its counted facts, and its JSON record is gone.
- const fileInPrompt = questionsAsked.every((item) => /--- BEGIN EXACT SOURCE TEXT ---\nname: New contributor message/.test(item.prompt)
+ const fileInPrompt = modelQuestions.every((item) => /--- BEGIN EXACT SOURCE TEXT ---\nname: New contributor message/.test(item.prompt)
&& /The current file is a GitHub Actions workflow \(YAML configuration, not program code\)\./.test(item.prompt) && !/"status":"ready"/.test(item.prompt));
- const factsInPrompt = questionsAsked.every((item) => /Facts read from the file \(counted, not guessed\):/.test(item.prompt) && /1 job: `build`/.test(item.prompt))
+ const factsInPrompt = modelQuestions.every((item) => /Facts read from the file \(counted, not guessed\):/.test(item.prompt) && /1 job: `build`/.test(item.prompt))
&& /Facts read from the file \(counted, not guessed\):/.test(promptText(asked));
const factsInCard = /1 job: build \("Hello new contributor", runs on ubuntu-latest, 1 step\)/.test(card) && /Trigger: pull_request_target \(types: opened\)/.test(card);
const factsOnly = !automatic && askButton === 1 && /Read from the file; not generated by the model\./.test(card) && card.split('\n').filter(Boolean).every((line) => /^(?:Agent|ⓘ Source|Workflow name|Trigger|1 job|Actions used|Read from the file|Ask the model)/.test(line.trim()));
@@ -709,8 +716,8 @@ async function k12(page) {
const wrongCount = counts.filter((answer) => /\b(?:[2-9]|zwei|drei|vier|two|three|four)\s+(?:jobs?|Jobs?)\b/i.test(answer));
const unsupportedNotes = await page.locator('.cbm-chat-answer-note').count();
check('K12', 'The automatic card of a configuration file shows only facts read from it; the model writes only on "Ask the model" or a question, marked as generated; answers invent no flake8/Python and no wrong job count',
- factsOnly && factsInCard && askedLabelled && othersFactsOnly && fileInPrompt && factsInPrompt && invented.length === 0 && wrongCount.length === 0,
- { factsOnly, factsInCard, askedLabelled, othersFactsOnly, fileInPrompt, factsInPrompt, invented: invented.length, wrongCount: wrongCount.length, counts, unsupportedNotes,
+ factsOnly && factsInCard && askedLabelled && othersFactsOnly && fileInPrompt && factsInPrompt && answeredFromFile && invented.length === 0 && wrongCount.length === 0,
+ { factsOnly, factsInCard, askedLabelled, othersFactsOnly, fileInPrompt, factsInPrompt, answeredFromFile, invented: invented.length, wrongCount: wrongCount.length, counts, unsupportedNotes,
automatic: card.replace(/\s+/g, ' ').slice(0, 420), askTheModel: askedCard.replace(/\s+/g, ' ').slice(0, 520), others,
answers: questionsAsked.map((item) => `${item.question} => ${item.answer.replace(/\s+/g, ' ').slice(0, 260)}`) });
}