Skip to content

Local code intelligence for developers and AI agents. Connect code, docs and infrastructure in a persistent graph, with CBM Lens, MCP tools, agent integrations and change-aware test selection.

test(ui): browser checks for the in-page project switch and the chat gaps (K24, K16, K12, K7, K1)

Bernhard Jackiewicz1a60ba83521c
diff --git a/graph-ui/tools/handtest-fixes-chat.mjs b/graph-ui/tools/handtest-fixes-chat.mjs
index 8ac2abc7a..387f094be 100644
--- a/graph-ui/tools/handtest-fixes-chat.mjs
+++ b/graph-ui/tools/handtest-fixes-chat.mjs
@@ -5,7 +5,11 @@
* K11, K12, K14, K15, K16, K17, K24).
*
* node tools/handtest-fixes-chat.mjs --origin http://127.0.0.1:4392 --out /tmp/chat \
- * --profile /tmp/profile-with-cached-model [--only K14,K7] [--runs 3]
+ * --profile /tmp/profile-with-cached-model [--only K14,K7] [--runs 3] [--headed]
+ *
+ * The browser runs headless unless --headed is given; ANGLE on Metal gives the headless
+ * browser the GPU with shader-f16 the local model needs (without it WebGPU falls back to
+ * SwiftShader, which has no f16).
*
* One check per item. Each drives the flow the plan describes, measures it in the page
* (DOM text and boxes, the messages sent to the model worker, the agent lamp) and prints
@@ -33,6 +37,7 @@ const PROFILE = resolve(String(arg('profile', join(OUT, '..', 'profile'))));
const PROJECT = String(arg('project', 'django-demo'));
const CONTROL = String(arg('control', 'cbm'));
const RUNS = Number(arg('runs', 3));
+const HEADED = arg('headed', false) === true;
const ALL = ['K15', 'K16', 'K11', 'K17', 'K14', 'K7', 'K4', 'K1', 'K12', 'K10', 'K24'];
const ONLY = String(arg('only', ALL.join(','))).split(',');
const VIEWPORT = { width: 1600, height: 1000 };
@@ -46,10 +51,24 @@ function pageProbe() {
// Everything sent to the model worker, including workers created later.
window.__probeWorker = [];
window.__probeAnswers = [];
+ // Model workers created in this page: a project switch must not create another (K24).
+ window.__probeModelWorkers = 0;
+ // Every request of the page, with the address the page showed when it was sent (K24).
+ window.__probeFetches = [];
+ const nativeFetch = window.fetch.bind(window);
+ window.fetch = (input, init) => {
+ try {
+ const url = typeof input === 'string' ? input : input instanceof URL ? input.href : input.url;
+ window.__probeFetches.push({ t: Math.round(performance.now()), url, body: typeof init?.body === 'string' ? init.body.slice(0, 600) : '', page: location.search });
+ } catch { /* observation only */ }
+ return nativeFetch(input, init);
+ };
+ window.__probePage = Math.random().toString(36).slice(2);
const NativeWorker = window.Worker;
window.Worker = class ProbeWorker extends NativeWorker {
constructor(url, options) {
super(url, options);
+ if (/browser-ai/.test(String(url))) window.__probeModelWorkers += 1;
// The model's raw answers, before the chat checks or shortens them.
this.addEventListener('message', (event) => {
if (event.data?.kind === 'answer') window.__probeAnswers.push({ t: Math.round(performance.now()), output: event.data.output, stopReason: event.data.stopReason });
@@ -279,6 +298,28 @@ async function k16(page) {
&& JSON.stringify(englishButtons) === JSON.stringify(['Show the list', 'Ask the model']) && modelCalls.length === 0 && luftCalls.length === 1 && !/Meintest du|Did you mean/.test(luft),
{ typoAnswerStart: typo.slice(0, 120), shortTypo: shortTypo.slice(0, 120), suggestion: suggestion.slice(0, 200), germanButtons, afterShowList: listed.slice(0, 120), listedButtons,
direction: direction.slice(0, 200), englishButtons, modelCallsBeforeLuft: modelCalls.length, luftModelCalls: luftCalls.length, luft: luft.slice(0, 200) });
+ // A typo of up to two letters in the selected name, any case, and "wo" anywhere in the question (completeness check of 2026-10-04).
+ const typoMark = await markWorker(page);
+ const orders = [];
+ for (const question of ['jsonbagg wird wo aufgerufen', 'wo wird jsonbagg aufgerufen', 'von wo wird jsonbagg aufgerufen']) orders.push({ question, answer: await ask(page, question) });
+ await shot(page, 'K16', 'word-order-listed', '"jsonbagg wird wo aufgerufen", "wo wird ..." and "von wo wird ...": each lists the callers of JSONBAgg from the graph.');
+ const typos = [];
+ for (const [question, language] of [['wer ruft JSONBAg auf', 'de'], ['wer ruft jsonbgg auf?', 'de'], ['who calls jsonbag', 'en'], ['callers of jsonbag', 'en']]) typos.push({ question, language, answer: await ask(page, question) });
+ const typoBox = await last().boundingBox();
+ await shot(page, 'K16', 'name-typo-suggested', '"callers of jsonbag" (and "wer ruft JSONBAg auf", "wer ruft jsonbgg auf?", "who calls jsonbag"): "Did you mean: callers of JSONBAgg?" with Show the list and Ask the model.');
+ if (typoBox) await shot(page, 'K16', 'name-typo-crop', 'Crop of the suggestion for "callers of jsonbag".', { clip: { x: typoBox.x - 8, y: Math.max(0, typoBox.y - 8), width: typoBox.width + 16, height: Math.min(typoBox.height + 16, 400) } });
+ await last().getByRole('button', { name: 'Show the list' }).click();
+ await wait(600);
+ const typoListed = await last().innerText();
+ await shot(page, 'K16', 'name-typo-listed', 'After "Show the list": the complete callers of JSONBAgg.');
+ const typoModelCalls = (await sentSince(page, typoMark)).length;
+ check('K16', 'A typo in the selected name (up to two letters, any case) is offered for the selection, a free word order with "wo" is listed; none of them reach the model',
+ orders.every((item) => /Aufrufer von JSONBAgg im geladenen Graphen/.test(item.answer) && callers.every((name) => item.answer.includes(name)))
+ && typos.every((item) => item.language === 'de' ? /Meintest du: Aufrufer von JSONBAgg\?/.test(item.answer) && /ist nicht genau der Name der Auswahl/.test(item.answer)
+ : /Did you mean: callers of JSONBAgg\?/.test(item.answer) && /is not exactly the name of the selection/.test(item.answer))
+ && /Callers of JSONBAgg in the loaded graph/.test(typoListed) && typoModelCalls === 0,
+ { orders: orders.map((item) => `${item.question} => ${item.answer.replace(/\s+/g, ' ').slice(0, 140)}`), typos: typos.map((item) => `${item.question} => ${item.answer.replace(/\s+/g, ' ').slice(0, 160)}`),
+ afterShowList: typoListed.replace(/\s+/g, ' ').slice(0, 140), modelCalls: typoModelCalls });
}
async function k11(page) {
@@ -455,6 +496,18 @@ async function k7(page) {
// Without source the listed facts are the explanation; the model is not asked to guess.
const areaOk = !request && /Selected source area: django/.test(card) && /Connections /.test(card) && !/Selected\.|members\[\d+\]|startLine|Finding a|Visible operations/.test(card)
&& /not generated by the model/.test(card);
+ // The same area opened, with nothing selected inside it (hand test 17:55): facts about the area, not "Opened: django".
+ mark = await markWorker(page); from = await now(page);
+ await page.getByRole('button', { name: 'Open area →' }).first().click();
+ await wait(2500);
+ await explanationDone(page);
+ [request] = (await sentSince(page, mark)).filter((entry) => entry.profile === 'automatic-explanation');
+ const openedCard = await explanationText(page);
+ outputs.push({ selection: 'Architecture overview, django opened, nothing selected', card: openedCard, model: (await answersSince(page, from))[0]?.output, prompt: promptText(request) });
+ await shot(page, 'K7', 'architecture-area-opened', 'Overview inside django with nothing selected: the card names the opened area, its lines and languages, the parts shown, its hotspot findings and the connections between its parts.');
+ const openedOk = !request && /Opened source area: django\./.test(openedCard) && /indexed lines in [\d,]+ measured files of [\d,]+; files by language: /.test(openedCard)
+ && /Parts shown \([\d,]+, largest first\): /.test(openedCard) && /hotspot findings?: /.test(openedCard) && /Connections of its parts: /.test(openedCard) && /not generated by the model/.test(openedCard)
+ && !/^Opened: /m.test(openedCard);
// Architecture, Behavior: main of manage.py-tpl.
mark = await markWorker(page); from = await now(page);
await page.getByRole('button', { name: 'Behavior' }).click();
@@ -467,8 +520,8 @@ async function k7(page) {
const behaviorOk = /Starting operation: main/.test(card) && (!request || /def main\(\)/.test(promptText(request))) && !/Finding a|Visible operations/.test(card)
&& /line 9 calls os\.environ\.setdefault/.test(card) && /line 18 calls execute_from_command_line/.test(card);
await save('K7', 'outputs.json', outputs);
- check('K7', 'Automatic explanations rest on listed facts (Galaxy incoming CALLS, Architecture area, Behavior)', galaxyOk && areaOk && behaviorOk,
- { galaxyOk, areaOk, behaviorOk, cards: outputs.map((item) => `${item.selection}: ${item.card.replace(/\s+/g, ' ').slice(0, 400)}`) });
+ check('K7', 'Automatic explanations rest on listed facts (Galaxy incoming CALLS, Architecture area selected and opened, Behavior)', galaxyOk && areaOk && openedOk && behaviorOk,
+ { galaxyOk, areaOk, openedOk, behaviorOk, cards: outputs.map((item) => `${item.selection}: ${item.card.replace(/\s+/g, ' ').slice(0, 520)}`) });
}
async function k4(page) {
@@ -522,7 +575,44 @@ async function k1(page) {
check('K1', 'Cut answer explains the token limit, opens the configuration at Output and lists larger models', summary === 'Token limit reached: the answer was cut short'
&& /all 48 output tokens/.test(expanded) && /2,048 tokens/.test(expanded) && /Qwen3 0\.6B · 579 MB download/.test(expanded) && /LFM2\.5 1\.2B · 764 MB download/.test(expanded)
&& /Qwen3\.5 2B · 1\.40 GB download/.test(expanded) && focus.dialog && focus.id === 'cbm-chat-output-tokens', { summary, expanded, focus });
+ // The automatic variant names both limits as well (completeness check of 2026-10-04). A code file in Explore
+ // gets up to two model sentences; with 32 output tokens the explanation runs into the limit.
+ await setOutputLimit(page, 32);
+ await tab(page, 'explore');
+ await openTreeFile(page, 'django/contrib/postgres/aggregates/general.py');
+ await explanationDone(page);
+ let automatic = '';
+ for (let run = 0; run < RUNS + 3 && !automatic; run++) {
+ const cut = page.locator('.cbm-chat-explanation details.cbm-chat-limit-note');
+ if (await cut.count()) {
+ await cut.locator('summary').click();
+ await wait(400);
+ automatic = await cut.innerText().catch(() => '');
+ } else await clickExplainAgain(page);
+ }
+ await shot(page, 'K1', 'automatic-note-expanded', 'An automatic explanation cut at 32 output tokens: the note names the output limit and the input limit of automatic explanations and of a chat question.');
+ const box = await page.locator('.cbm-chat-explanation details.cbm-chat-limit-note').boundingBox().catch(() => null);
+ if (box) await shot(page, 'K1', 'automatic-note-crop', 'Crop of the expanded note under the automatic explanation.', { clip: { x: box.x - 8, y: Math.max(0, box.y - 8), width: box.width + 16, height: Math.min(box.height + 16, 420) } });
+ await setOutputLimit(page, 512);
+ check('K1', 'The note under a cut automatic explanation names its input limit and its output limit, and those of a question', /Automatic explanations stop after 32 output tokens/.test(automatic)
+ && /read at most 1,536 input tokens/.test(automatic) && /may read up to 2,048 input tokens/.test(automatic), { automatic });
+}
+
+/** Opens a file of the project root (or below) in Explore by clicking its tree row. */
+async function openTreeFile(page, path) {
+ const parts = path.split('/');
+ for (let depth = 1; depth < parts.length; depth++) {
+ const row = page.locator(`.atlas-tree-row[data-path="${parts.slice(0, depth).join('/')}"]`).first();
+ await row.waitFor({ timeout: 30000 });
+ for (let attempt = 0; attempt < 4 && (await row.getAttribute('data-expanded')) !== 'true'; attempt++) { await row.click(); await wait(1200); }
+ }
+ const file = page.locator(`.atlas-tree-row[data-path="${path}"]`).first();
+ if (!(await file.count())) return false;
+ await file.click();
+ await wait(3500);
+ return true;
}
+const cardButton = (page, name) => page.locator('.cbm-chat-explanation').getByRole('button', { name, exact: true });
async function k12(page) {
const outputs = [];
@@ -530,11 +620,37 @@ async function k12(page) {
await loadModel(page);
await openChat(page);
let mark = await markWorker(page);
- const from = await now(page);
await explanationDone(page);
+ // The automatic card of a configuration file: the facts read from it, and no model text (K12).
const [automatic] = (await sentSince(page, mark)).filter((entry) => entry.profile === 'automatic-explanation');
- outputs.push({ kind: 'automatic', card: await explanationText(page), model: (await answersSince(page, from))[0]?.output, prompt: promptText(automatic) });
- await shot(page, 'K12', 'automatic', 'Explore with the workflow open: the automatic explanation lists name, trigger, the one job and the action, read from the file, above the model text.');
+ const card = await explanationText(page);
+ const askButton = await cardButton(page, 'Ask the model').count();
+ outputs.push({ kind: 'automatic', card, modelCalled: Boolean(automatic) });
+ await shot(page, 'K12', 'automatic', 'Explore with the workflow open: the automatic card lists name, trigger, the one job and the action read from the file, says it is not generated by the model, and offers "Ask the model".');
+ // Only on request: "Ask the model" lets the model write about the file, marked as generated.
+ mark = await markWorker(page);
+ let from = await now(page);
+ await cardButton(page, 'Ask the model').click();
+ await wait(800);
+ await explanationDone(page);
+ const [asked] = (await sentSince(page, mark)).filter((entry) => entry.profile === 'automatic-explanation');
+ const askedCard = await explanationText(page);
+ outputs.push({ kind: 'ask-the-model', card: askedCard, model: (await answersSince(page, from))[0]?.output, prompt: promptText(asked) });
+ await shot(page, 'K12', 'ask-the-model', 'After "Ask the model": the facts stay above, the model text follows and is marked as generated (or left out when it names what the file does not show).');
+ // Other configuration and text files of the project: facts only, no model.
+ const others = [];
+ for (const path of ['package.json', 'tox.ini', 'pyproject.toml', 'README.rst', '.editorconfig']) {
+ mark = await markWorker(page);
+ if (!(await openTreeFile(page, path))) continue;
+ await explanationDone(page);
+ const calls = (await sentSince(page, mark)).filter((entry) => entry.profile === 'automatic-explanation').length;
+ const text = await explanationText(page);
+ others.push({ path, card: text.replace(/\s+/g, ' ').slice(0, 300), factsOnly: /Read from the file; not generated by the model\./.test(text), modelCalls: calls });
+ if (others.length <= 3) await shot(page, 'K12', `facts-${path.replace(/[^\w]+/g, '-')}`, `${path}: the automatic card shows only what is read from the file; no model call.`);
+ }
+ // Explicit questions about the workflow go to the model, as before (back by its tree row: no reload, the model stays loaded).
+ await openTreeFile(page, '.github/workflows/new_contributor_pr.yml');
+ await explanationDone(page);
const questions = ['was kannst du mir über dieses aktuelle File sagen', 'Wie viele Jobs gibt es in dieser Datei?', 'Welche Jobs gibt es in dieser Datei?', 'What does this file do?', 'How many jobs does this workflow have?'];
const runs = Math.max(RUNS, questions.length);
for (let run = 0; run < runs; run++) {
@@ -546,23 +662,27 @@ async function k12(page) {
if (run === 1) await shot(page, 'K12', 'jobs-count', 'The answer to "Wie viele Jobs gibt es in dieser Datei?": the prompt carries the counted facts (1 job: build).');
await newConversation(page);
}
- await save('K12', 'outputs.json', outputs);
- const asked = outputs.filter((item) => item.kind === 'question');
+ await save('K12', 'outputs.json', { outputs, others });
+ const questionsAsked = outputs.filter((item) => item.kind === 'question');
// The open file is in the system section of the request, named as a workflow, with its counted facts, and its JSON record is gone.
- const fileInPrompt = asked.every((item) => /--- BEGIN EXACT SOURCE TEXT ---\nname: New contributor message/.test(item.prompt)
+ const fileInPrompt = questionsAsked.every((item) => /--- BEGIN EXACT SOURCE TEXT ---\nname: New contributor message/.test(item.prompt)
&& /The current file is a GitHub Actions workflow \(YAML configuration, not program code\)\./.test(item.prompt) && !/"status":"ready"/.test(item.prompt));
- const factsInPrompt = asked.every((item) => /Facts read from the file \(counted, not guessed\):/.test(item.prompt) && /1 job: `build`/.test(item.prompt))
- && /Facts read from the file \(counted, not guessed\):/.test(outputs[0].prompt);
- const card = outputs[0].card;
- const factsInCard = /1 job: build \("Hello new contributor", runs on ubuntu-latest, 1 step\)/.test(card) && /Trigger: pull_request_target \(types: opened\)/.test(card)
- && /Facts read from the file/.test(card);
- const invented = asked.filter((item) => /flake8|python|\bpip\b|\.py\b/i.test(item.answer));
- const counts = asked.filter((item) => /Wie viele|How many/.test(item.question)).map((item) => item.answer.replace(/^You\s+[^\n]*\n+Agent\s+(?:ⓘ Source\s+)?/, '').replace(/\s+/g, ' ').slice(0, 200));
+ const factsInPrompt = questionsAsked.every((item) => /Facts read from the file \(counted, not guessed\):/.test(item.prompt) && /1 job: `build`/.test(item.prompt))
+ && /Facts read from the file \(counted, not guessed\):/.test(promptText(asked));
+ const factsInCard = /1 job: build \("Hello new contributor", runs on ubuntu-latest, 1 step\)/.test(card) && /Trigger: pull_request_target \(types: opened\)/.test(card);
+ const factsOnly = !automatic && askButton === 1 && /Read from the file; not generated by the model\./.test(card) && card.split('\n').filter(Boolean).every((line) => /^(?:Agent|ⓘ Source|Workflow name|Trigger|1 job|Actions used|Read from the file|Ask the model)/.test(line.trim()));
+ const askedLabelled = Boolean(asked) && /1 job: build/.test(askedCard)
+ && (/Facts read from the file; the text after them is generated by the model\./.test(askedCard) || /The model's text named something the file does not show and was left out\./.test(askedCard));
+ const othersFactsOnly = others.length >= 2 && others.every((item) => item.modelCalls === 0 && item.factsOnly);
+ const invented = questionsAsked.filter((item) => /flake8|python|\bpip\b|\.py\b/i.test(item.answer));
+ const counts = questionsAsked.filter((item) => /Wie viele|How many/.test(item.question)).map((item) => item.answer.replace(/^You\s+[^\n]*\n+Agent\s+(?:ⓘ Source\s+)?/, '').replace(/\s+/g, ' ').slice(0, 200));
const wrongCount = counts.filter((answer) => /\b(?:[2-9]|zwei|drei|vier|two|three|four)\s+(?:jobs?|Jobs?)\b/i.test(answer));
- check('K12', 'The open YAML file and its counted facts are in the prompt and the card; answers invent no flake8/Python and no wrong job count', fileInPrompt && factsInPrompt && factsInCard
- && invented.length === 0 && wrongCount.length === 0,
- { fileInPrompt, factsInPrompt, factsInCard, invented: invented.length, wrongCount: wrongCount.length, counts, automatic: card.replace(/\s+/g, ' ').slice(0, 420),
- answers: asked.map((item) => `${item.question} => ${item.answer.replace(/\s+/g, ' ').slice(0, 260)}`) });
+ const unsupportedNotes = await page.locator('.cbm-chat-answer-note').count();
+ check('K12', 'The automatic card of a configuration file shows only facts read from it; the model writes only on "Ask the model" or a question, marked as generated; answers invent no flake8/Python and no wrong job count',
+ factsOnly && factsInCard && askedLabelled && othersFactsOnly && fileInPrompt && factsInPrompt && invented.length === 0 && wrongCount.length === 0,
+ { factsOnly, factsInCard, askedLabelled, othersFactsOnly, fileInPrompt, factsInPrompt, invented: invented.length, wrongCount: wrongCount.length, counts, unsupportedNotes,
+ automatic: card.replace(/\s+/g, ' ').slice(0, 420), askTheModel: askedCard.replace(/\s+/g, ' ').slice(0, 520), others,
+ answers: questionsAsked.map((item) => `${item.question} => ${item.answer.replace(/\s+/g, ' ').slice(0, 260)}`) });
}
async function k10(page) {
@@ -601,32 +721,158 @@ async function k10(page) {
{ lampAfterReload, buttons, option, cacheEntries: cache, autoDefault, autoLoadActiveAfterSeconds: active ? seconds : null, modelNetworkRequests: network });
}
-async function k24(page) {
- await open(page, PROJECT, 'galaxy');
- await galaxyReady(page);
- const loaded = await loadModel(page);
- await shot(page, 'K24', 'before-switch', `${PROJECT} with the agent active (green lamp).`);
- const requestsBefore = modelRequests.length;
- await page.locator('.atlas-project-switcher summary').first().click();
- await page.locator('.atlas-project-picker button', { has: page.locator('.atlas-project-result-name', { hasText: new RegExp(`^${CONTROL}`) }) }).first().click();
+/* K24: what each workspace shows of a project, after a switch in the page. */
+const K24_VIEWS = [
+ { id: 'explore', workspace: 'explore' },
+ { id: 'galaxy', workspace: 'galaxy' },
+ { id: 'architecture-overview', workspace: 'architecture', view: 'Overview', sub: 'Structure' },
+ { id: 'architecture-entry-points', workspace: 'architecture', view: 'Overview', sub: 'Entry points' },
+ { id: 'architecture-service-map', workspace: 'architecture', view: 'Routes', sub: 'Service map' },
+ { id: 'architecture-endpoints', workspace: 'architecture', view: 'Routes', sub: 'Endpoints' },
+ { id: 'architecture-hotspots', workspace: 'architecture', view: 'Hotspots' },
+ { id: 'architecture-system-structure', workspace: 'architecture', view: 'System structure' },
+ { id: 'architecture-behavior', workspace: 'architecture', view: 'Behavior' },
+ { id: 'adr', workspace: 'adr' },
+ { id: 'coverage', workspace: 'coverage' },
+ { id: 'system-logs', workspace: 'system', sub: 'Logs' },
+];
+/** Words only the other project's views show: its source paths and symbol prefixes. */
+const PROJECT_MARKERS = {
+ 'django-demo': /\bdjango\/|manage\.py-tpl|\bjs_tests\b|django-demo\/|JSONBAgg/,
+ cbm: /src\/main\.c|\bgraph-ui\/|\bcbm_[a-z]|\bcbm\/|test-infrastructure/,
+};
+const BUSY = /Loading|Reading|Preparing|Checking|Resolving|Updating view/;
+/** The workspace text once it stopped changing (or after `timeout`). */
+async function settledStage(page, timeout = 30000) {
+ let last = '', same = 0;
const t0 = Date.now();
- await page.waitForURL(new RegExp(`project=${CONTROL}`), { timeout: 30000 });
- await page.waitForSelector('.atlas-shell');
+ while (Date.now() - t0 < timeout) {
+ await wait(900);
+ const text = await page.locator('.atlas-workspace-stage').innerText().catch(() => '');
+ same = text === last && !BUSY.test(text) ? same + 1 : 0;
+ if (same >= 2) return text;
+ last = text;
+ }
+ return last;
+}
+async function showView(page, item) {
+ await page.locator(`[data-workspace-tab="${item.workspace}"]`).first().click();
+ await wait(700);
+ if (item.view) { await page.locator('.atlas-workspace-stage').getByRole('button', { name: item.view, exact: true }).first().click().catch(() => {}); await wait(700); }
+ if (item.sub) { await page.locator('.atlas-workspace-stage').getByRole('button', { name: item.sub, exact: true }).first().click().catch(() => {}); await wait(700); }
+}
+/** Paths and dotted names a view shows; a stale view shows ones the fresh view of its project does not. */
+const pathTokens = (text) => new Set(text.match(/[\w.@-]+\/[\w./@-]+/g) ?? []);
+async function viewTexts(page, label, shots) {
+ const texts = {};
+ for (const item of K24_VIEWS) {
+ await showView(page, item);
+ texts[item.id] = await settledStage(page, item.workspace === 'galaxy' ? 45000 : 30000);
+ if (item.workspace === 'system') texts[item.id] += `\n${await page.locator('select option[value="project"]').first().innerText().catch(() => '')}`;
+ if (shots) await shot(page, 'K24', `${label}-${item.id}`, `${label}: ${item.id} after the switch.`);
+ }
+ return texts;
+}
+/** Requests sent once the address named `shown` that name `project` (URL parameter or JSON argument);
+ * the log transport carries entries logged before the switch by design. */
+async function requestsNaming(page, from, shown, project) {
+ const fetches = (await page.evaluate(() => window.__probeFetches.slice()))
+ .filter((entry) => entry.t >= from && new URLSearchParams(entry.page).get('project') === shown && !/\/api\/ui-log/.test(entry.url));
+ const names = (entry) => {
+ const named = new Set();
+ try { const value = new URL(entry.url, location.href).searchParams.get('project'); if (value) named.add(value); } catch { /* not a URL */ }
+ for (const match of entry.body.matchAll(/"(?:project|project_name)"\s*:\s*"([^"]+)"/g)) named.add(match[1]);
+ return named;
+ };
+ return { total: fetches.length, naming: fetches.filter((entry) => names(entry).has(project)).map((entry) => ({ t: entry.t, url: entry.url.slice(0, 120), body: entry.body.slice(0, 160), page: entry.page })) };
+}
+const lampFrames = async (page, label, t0) => {
const frames = [];
for (const at of [0, 250, 1000, 3000]) {
const delay = at - (Date.now() - t0);
if (delay > 0) await wait(delay);
- frames.push({ at, label: await agentLabel(page), file: await shot(page, 'K24', `after-switch-${at}ms`, `${at} ms after the switch to ${CONTROL}: the agent lamp state.`) });
+ frames.push({ at, label: await agentLabel(page), file: await shot(page, 'K24', `${label}-${at}ms`, `${label}, ${at} ms after the click: the agent lamp.`) });
}
- const active = await agentActive(page, 120000);
- const seconds = (Date.now() - t0) / 1000;
- await shot(page, 'K24', 'after-switch-active', `${CONTROL} with the agent active again, without a click.`);
- const project = await page.locator('.atlas-project-switcher summary').first().getAttribute('aria-label');
- // Back, so later runs start in the main project.
+ return frames;
+};
+const activeWorkspace = (page) => page.locator('[data-workspace-tab][aria-selected="true"]').first().getAttribute('data-workspace-tab').catch(() => '');
+/** The same text as the fresh view of the project, apart from numbers (times, memory) and spacing. */
+const sameText = (left, right) => left.replace(/\d[\d,.:]*/g, '#').replace(/\s+/g, ' ') === right.replace(/\d[\d,.:]*/g, '#').replace(/\s+/g, ' ');
+async function switchTo(page, project) {
+ await page.locator('.atlas-project-switcher summary').first().click();
+ await page.locator('.atlas-project-picker button', { has: page.locator('.atlas-project-result-name', { hasText: new RegExp(`^${project}`) }) }).first().click();
+}
+
+async function k24(page) {
+ // Fresh page loads of both projects are the reference for what each view shows (no switch involved).
+ const fresh = await page.context().newPage();
+ const reference = {};
+ for (const project of [PROJECT, CONTROL]) {
+ await open(fresh, project, 'explore');
+ await wait(4000);
+ reference[project] = await viewTexts(fresh, `fresh-${project}`, false);
+ }
+ await fresh.close();
+ await save('K24', 'reference.json', reference);
+ // The flow of the hand test (18:04): django-demo with the agent active and a selection, then cbm, then back.
await open(page, PROJECT, 'galaxy');
+ await galaxyReady(page);
+ const loaded = await loadModel(page);
+ await openChat(page);
+ await select(page, 'JSONBAgg');
+ await scopeSettled(page);
+ await explanationDone(page);
+ await ask(page, 'Who calls JSONBAgg?');
+ await shot(page, 'K24', 'before-switch', `${PROJECT} with the agent active, JSONBAgg selected and a listed answer in the chat.`);
+ const page0 = await page.evaluate(() => ({ id: window.__probePage, origin: performance.timeOrigin, workers: window.__probeModelWorkers }));
+ const requestsBefore = modelRequests.length;
+ const legs = [];
+ for (const [from, to, how] of [[PROJECT, CONTROL, 'switcher'], [CONTROL, PROJECT, 'switcher'], [PROJECT, CONTROL, 'back'], [CONTROL, PROJECT, 'forward']]) {
+ const mark = await markWorker(page);
+ const t = await now(page);
+ const workspaceBefore = await activeWorkspace(page);
+ const t0 = Date.now();
+ if (how === 'switcher') await switchTo(page, to);
+ else if (how === 'back') await page.goBack({ waitUntil: 'commit' });
+ else await page.goForward({ waitUntil: 'commit' });
+ const frames = await lampFrames(page, `${how}-to-${to}`, t0);
+ await page.waitForFunction((name) => new URLSearchParams(location.search).get('project') === name, to, { timeout: 30000 }).catch(() => {});
+ const shown = await page.locator('.atlas-project-switcher summary').first().getAttribute('aria-label').catch(() => '');
+ const workspaceAfter = await activeWorkspace(page);
+ const chat = await page.locator('.cbm-chat-transcript').innerText().catch(() => '');
+ const texts = how === 'switcher' ? await viewTexts(page, `${how}-to-${to}`, true) : {};
+ const views = Object.entries(texts).map(([id, text]) => {
+ const stale = PROJECT_MARKERS[from].exec(text)?.[0];
+ const expected = pathTokens(reference[to][id] ?? '');
+ const unexpected = [...pathTokens(text)].filter((token) => !expected.has(token) && PROJECT_MARKERS[from].test(token));
+ return { id, stale: stale ?? null, unexpected: unexpected.slice(0, 5), chars: text.length, referenceChars: (reference[to][id] ?? '').length, sameAsFresh: sameText(text, reference[to][id] ?? '') };
+ });
+ await wait(1500);
+ const leaks = await requestsNaming(page, t, to, from);
+ const state = await page.evaluate(() => ({ id: window.__probePage, origin: performance.timeOrigin, workers: window.__probeModelWorkers }));
+ const prepares = (await workerMessages(page)).slice(mark).filter((entry) => entry.kind === 'prepare').length;
+ legs.push({ from, to, how, project: shown, url: await page.evaluate(() => location.search), workspace: { before: workspaceBefore, after: workspaceAfter }, samePage: state.id === page0.id && state.origin === page0.origin,
+ modelWorkersCreated: state.workers - page0.workers, prepares, lamp: frames.map(({ at, label }) => ({ at, label })),
+ chatHasOldTurn: to === CONTROL ? /Who calls JSONBAgg/.test(chat) : null, chatKeptOwnTurn: to === PROJECT ? /Who calls JSONBAgg/.test(chat) : null,
+ views, requestsAfterSwitch: leaks.total, requestsNamingOldProject: leaks.naming });
+ }
const downloads = modelRequests.length - requestsBefore;
- check('K24', 'The agent stays loaded across a project switch (reloads from cache by itself)', loaded && active && /cbm/.test(project ?? '') && downloads === 0,
- { project, frames: frames.map(({ at, label }) => ({ at, label })), activeAfterSeconds: active ? seconds : null, modelRequests: downloads });
+ await save('K24', 'legs.json', legs);
+ const legOk = (leg) => leg.samePage && leg.workspace.before === leg.workspace.after && leg.modelWorkersCreated === 0 && leg.prepares === 0 && leg.lamp.every((frame) => /Agent active/.test(frame.label ?? ''))
+ && new RegExp(leg.to).test(leg.project ?? '') && leg.requestsNamingOldProject.length === 0 && leg.views.every((view) => !view.stale && !view.unexpected.length)
+ && (leg.to !== CONTROL || leg.chatHasOldTurn === false) && (leg.to !== PROJECT || leg.how !== 'switcher' || leg.chatKeptOwnTurn === true);
+ check('K24', 'A project switch stays in the page: the model stays loaded (no reload, no new worker, lamp active), every workspace shows only the new project, no request names the old one; Back/Forward switch the same way',
+ loaded && downloads === 0 && legs.every(legOk),
+ { modelDownloads: downloads, legs: legs.map((leg) => ({ ...leg, views: leg.views.filter((view) => view.stale || view.unexpected.length), viewsChecked: leg.views.length,
+ viewsSameAsFresh: leg.views.filter((view) => view.sameAsFresh).map((view) => view.id), viewsDifferingFromFresh: leg.views.filter((view) => !view.sameAsFresh).map((view) => view.id),
+ requestsNamingOldProject: leg.requestsNamingOldProject.slice(0, 5) })) });
+ // The configuration says what a switch does with the model.
+ await openConfig(page);
+ const tooltip = await page.locator('#cbm-chat-auto-load').locator('xpath=..').getAttribute('title').catch(() => '');
+ await shot(page, 'K24', 'config-tooltip', 'The agent configuration; the tooltip of "Load the chosen model on start" says a project switch keeps the model in the page.');
+ await closeConfig(page);
+ check('K24', 'The setting tooltip says a switch keeps the model in the page and the option is for opening or reloading', /A project switch stays in this page and keeps a loaded model without loading it again/.test(tooltip ?? '')
+ && /opened or reloaded/.test(tooltip ?? ''), { tooltip });
}
/* ------------------------------------------------------------------ */
@@ -634,7 +880,7 @@ async function k24(page) {
const CHECKS = { K15: k15, K16: k16, K11: k11, K17: k17, K14: k14, K7: k7, K4: k4, K1: k1, K12: k12, K10: k10, K24: k24 };
await mkdir(OUT, { recursive: true });
const context = await chromium.launchPersistentContext(PROFILE, {
- headless: false, viewport: VIEWPORT, deviceScaleFactor: 2, args: ['--enable-unsafe-webgpu', '--ignore-gpu-blocklist'],
+ headless: !HEADED, viewport: VIEWPORT, deviceScaleFactor: 2, args: ['--enable-unsafe-webgpu', '--ignore-gpu-blocklist', '--use-angle=metal', '--enable-gpu'],
});
await context.addInitScript(pageProbe);
// Model downloads, from the page or its workers: a cached load makes none.