Skip to content

Chrome DevTools for coding agents

test: update all the eval scenarios to work with experimentalPageIdRouting flag (#2085)

Samiya Caur8e8e83e3f8d1
diff --git a/scripts/eval_gemini.ts b/scripts/eval_gemini.ts
index bd5bafdb..3fcd4c45 100644
--- a/scripts/eval_gemini.ts
+++ b/scripts/eval_gemini.ts
@@ -19,23 +19,10 @@ const ROOT_DIR = path.resolve(import.meta.dirname, '..');
const SCENARIOS_DIR = path.join(import.meta.dirname, 'eval_scenarios');
const SKILL_PATH = path.join(ROOT_DIR, 'skills', 'chrome-devtools', 'SKILL.md');
-// Define schema for our test scenarios
-export interface CapturedFunctionCall {
- name: string;
- args: Record<string, unknown>;
-}
-
-export interface TestScenario {
- prompt: string;
- maxTurns: number;
- expectations: (calls: CapturedFunctionCall[]) => void;
- htmlRoute?: {
- path: string;
- htmlContent: string;
- };
- /** Extra CLI flags passed to the MCP server (e.g. '--experimental-page-id-routing'). */
- serverArgs?: string[];
-}
+import type {CapturedFunctionCall, TestScenario} from './eval_result.ts';
+import {Result} from './eval_result.ts';
+export type {CapturedFunctionCall, TestScenario};
+export {Result};
async function loadScenario(scenarioPath: string): Promise<TestScenario> {
const module = await import(pathToFileURL(scenarioPath).href);
@@ -54,6 +41,7 @@ async function runSingleScenario(
modelId: string,
debug: boolean,
includeSkill: boolean,
+ extraServerArgs: string[] = [],
): Promise<void> {
const debugLog = (...args: unknown[]) => {
if (debug) {
@@ -125,6 +113,9 @@ async function runSingleScenario(
if (scenario.serverArgs) {
args.push(...scenario.serverArgs);
}
+ if (extraServerArgs.length > 0) {
+ args.push(...extraServerArgs);
+ }
transport = new StdioClientTransport({
command: 'node',
@@ -175,7 +166,7 @@ async function runSingleScenario(
debugLog(`\n--- Response ---\n${result.text}`);
debugLog('\nVerifying expectations...');
- scenario.expectations(allCalls);
+ scenario.expectations(new Result(allCalls, args));
} finally {
try {
await client?.close();
@@ -195,7 +186,7 @@ async function main() {
options: {
model: {
type: 'string',
- default: 'gemini-2.5-flash',
+ default: 'gemini-3-flash-preview',
},
debug: {
type: 'boolean',
@@ -209,6 +200,9 @@ async function main() {
type: 'boolean',
default: false,
},
+ 'server-args': {
+ type: 'string',
+ },
},
allowPositionals: true,
});
@@ -217,6 +211,9 @@ async function main() {
const debug = values.debug;
const repeat = values.repeat;
const includeSkill = values['include-skill'];
+ const extraServerArgs = values['server-args']
+ ? values['server-args'].split(/\s+/)
+ : [];
const scenarioFiles =
positionals.length > 0
@@ -248,6 +245,7 @@ async function main() {
modelId,
debug,
includeSkill,
+ extraServerArgs,
);
console.log(`✔ ${path.relative(ROOT_DIR, scenarioPath)} (Run ${i})`);
successCount++;
diff --git a/scripts/eval_result.ts b/scripts/eval_result.ts
new file mode 100644
index 00000000..9b734b29
--- /dev/null
+++ b/scripts/eval_result.ts
@@ -0,0 +1,107 @@
+/**
+ * @license
+ * Copyright 2026 Google LLC
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+import assert from 'node:assert';
+
+export interface CapturedFunctionCall {
+ name: string;
+ args: Record<string, unknown>;
+}
+
+export class Result {
+ private nextCallIndex = 0;
+ public readonly calls: CapturedFunctionCall[];
+ public readonly serverArgs: string[];
+
+ constructor(calls: CapturedFunctionCall[], serverArgs: string[]) {
+ this.calls = calls;
+ this.serverArgs = serverArgs;
+ }
+
+ get hasPageIdRouting(): boolean {
+ return this.serverArgs.includes('--experimental-page-id-routing');
+ }
+
+ get remainingCalls(): CapturedFunctionCall[] {
+ return this.calls.slice(this.nextCallIndex);
+ }
+
+ /**
+ * Consumes initial page navigation/setup boilerplate.
+ * - Ignores/skips leading list_pages calls.
+ * - Asserts that new_page or navigate_page was called.
+ * - Determines the expected pageId.
+ * - Returns the active pageId.
+ */
+ consumePageNavigation(): number | undefined {
+ if (this.calls[this.nextCallIndex]?.name === 'list_pages') {
+ this.nextCallIndex++;
+ }
+
+ const navCall = this.calls[this.nextCallIndex];
+ assert.ok(
+ navCall &&
+ (navCall.name === 'new_page' || navCall.name === 'navigate_page'),
+ `Expected navigation call (new_page or navigate_page), but got: ${navCall?.name || 'none'}`,
+ );
+ this.nextCallIndex++;
+
+ const isNewPage = navCall.name === 'new_page';
+ let pageId: number | undefined;
+ if (this.hasPageIdRouting) {
+ pageId = isNewPage ? 2 : 1;
+ }
+
+ return pageId;
+ }
+
+ /**
+ * Asserts that the next call in sequence has the correct name and matches expected arguments.
+ * Increments the internal call index.
+ */
+ assertNextCall(
+ name: string,
+ expectedArgs?: Record<string, unknown>,
+ ): CapturedFunctionCall {
+ const call = this.calls[this.nextCallIndex];
+ assert.ok(
+ call,
+ `Expected call at index ${this.nextCallIndex} (name: '${name}') to exist`,
+ );
+ assert.strictEqual(
+ call.name,
+ name,
+ `Expected call at index ${this.nextCallIndex} to be '${name}', but got '${call.name}'`,
+ );
+
+ if (expectedArgs) {
+ for (const entry of Object.entries(expectedArgs)) {
+ const key = entry[0];
+ const value = entry[1];
+ assert.deepStrictEqual(
+ call.args[key],
+ value,
+ `Expected argument '${key}' on call '${name}' to be ${JSON.stringify(value)}, got ${JSON.stringify(call.args[key])}`,
+ );
+ }
+ }
+
+ this.nextCallIndex++;
+ return call;
+ }
+}
+
+export interface TestScenario {
+ prompt: string;
+ maxTurns: number;
+ expectations: (result: Result) => void;
+ htmlRoute?: {
+ path: string;
+ htmlContent: string;
+ };
+ /** Extra CLI flags passed to the MCP server (e.g. '--experimental-page-id-routing'). */
+ serverArgs?: string[];
+}
diff --git a/scripts/eval_scenarios/console_test.ts b/scripts/eval_scenarios/console_test.ts
index d82abc2a..125b1e92 100644
--- a/scripts/eval_scenarios/console_test.ts
+++ b/scripts/eval_scenarios/console_test.ts
@@ -10,7 +10,7 @@ import type {TestScenario} from '../eval_gemini.ts';
export const scenario: TestScenario = {
prompt: 'Navigate to <TEST_URL> and check the console messages.',
- maxTurns: 2,
+ maxTurns: 3,
htmlRoute: {
path: '/console_test.html',
htmlContent: `
@@ -20,16 +20,12 @@ export const scenario: TestScenario = {
</script>
`,
},
- expectations: calls => {
- assert.strictEqual(calls.length, 2);
- assert.ok(
- calls[0].name === 'navigate_page' || calls[0].name === 'new_page',
- 'First call should be navigation',
- );
- assert.strictEqual(
- calls[1].name,
+ expectations: result => {
+ const pageId = result.consumePageNavigation();
+ assert.strictEqual(result.remainingCalls.length, 1);
+ result.assertNextCall(
'list_console_messages',
- 'Second call should be list_console_messages',
+ result.hasPageIdRouting ? {pageId} : undefined,
);
},
};
diff --git a/scripts/eval_scenarios/emulation_test.ts b/scripts/eval_scenarios/emulation_test.ts
index 2eebac83..9456e451 100644
--- a/scripts/eval_scenarios/emulation_test.ts
+++ b/scripts/eval_scenarios/emulation_test.ts
@@ -11,9 +11,12 @@ import type {TestScenario} from '../eval_gemini.ts';
export const scenario: TestScenario = {
prompt: 'Emulate offline network conditions.',
maxTurns: 2,
- expectations: calls => {
- assert.strictEqual(calls.length, 1);
- assert.strictEqual(calls[0].name, 'emulate');
- assert.strictEqual(calls[0].args.networkConditions, 'Offline');
+ expectations: result => {
+ assert.ok(result.remainingCalls.length >= 1);
+ result.assertNextCall('list_pages');
+ result.assertNextCall('emulate', {
+ networkConditions: 'Offline',
+ pageId: result.hasPageIdRouting ? 1 : undefined,
+ });
},
};
diff --git a/scripts/eval_scenarios/emulation_userAgent_test.ts b/scripts/eval_scenarios/emulation_userAgent_test.ts
index ee0cfc7e..0fb4959c 100644
--- a/scripts/eval_scenarios/emulation_userAgent_test.ts
+++ b/scripts/eval_scenarios/emulation_userAgent_test.ts
@@ -9,14 +9,20 @@ import assert from 'node:assert';
import type {TestScenario} from '../eval_gemini.ts';
export const scenario: TestScenario = {
- prompt: 'Emulate iPhone 14 user agent',
+ prompt: 'Emulate current page with iPhone 14 user agent',
maxTurns: 2,
- expectations: calls => {
- assert.strictEqual(calls.length, 1);
- assert.strictEqual(calls[0].name, 'emulate');
- assert.deepStrictEqual(
- calls[0].args.userAgent,
- 'Mozilla/5.0 (iPhone; CPU iPhone OS 16_0 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/16.0 Mobile/15E148 Safari/604.1',
- );
+ expectations: result => {
+ assert.ok(result.remainingCalls.length >= 1);
+ if (
+ result.hasPageIdRouting ||
+ result.remainingCalls[0]?.name === 'list_pages'
+ ) {
+ result.assertNextCall('list_pages');
+ }
+ result.assertNextCall('emulate', {
+ userAgent:
+ 'Mozilla/5.0 (iPhone; CPU iPhone OS 16_0 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/16.0 Mobile/15E148 Safari/604.1',
+ pageId: result.hasPageIdRouting ? 1 : undefined,
+ });
},
};
diff --git a/scripts/eval_scenarios/emulation_viewport_test.ts b/scripts/eval_scenarios/emulation_viewport_test.ts
index f4f51904..52a945a3 100644
--- a/scripts/eval_scenarios/emulation_viewport_test.ts
+++ b/scripts/eval_scenarios/emulation_viewport_test.ts
@@ -9,11 +9,19 @@ import assert from 'node:assert';
import type {TestScenario} from '../eval_gemini.ts';
export const scenario: TestScenario = {
- prompt: 'Emulate iPhone 14 viewport',
+ prompt: 'Emulate current page with iPhone 14 viewport',
maxTurns: 2,
- expectations: calls => {
- assert.strictEqual(calls.length, 1);
- assert.strictEqual(calls[0].name, 'emulate');
- assert.deepStrictEqual(calls[0].args.viewport, '390x844x3,mobile,touch');
+ expectations: result => {
+ assert.ok(result.remainingCalls.length >= 1);
+ if (
+ result.hasPageIdRouting ||
+ result.remainingCalls[0]?.name === 'list_pages'
+ ) {
+ result.assertNextCall('list_pages');
+ }
+ result.assertNextCall('emulate', {
+ viewport: '390x844x3,mobile,touch',
+ pageId: result.hasPageIdRouting ? 1 : undefined,
+ });
},
};
diff --git a/scripts/eval_scenarios/fill_select_and_checkboxes_test.ts b/scripts/eval_scenarios/fill_select_and_checkboxes_test.ts
index 9876e647..cb66cbbf 100644
--- a/scripts/eval_scenarios/fill_select_and_checkboxes_test.ts
+++ b/scripts/eval_scenarios/fill_select_and_checkboxes_test.ts
@@ -11,7 +11,7 @@ import type {TestScenario} from '../eval_gemini.ts';
export const scenario: TestScenario = {
prompt:
'Go to <TEST_URL>, fill the form with size = 2 CPUs and components = [docker, nginx].',
- maxTurns: 4, // allow for at least one extra turn to verify there are no extra clicks after fill_form
+ maxTurns: 5, // allow for at least one extra turn to verify there are no extra clicks after fill_form
htmlRoute: {
path: '/input_test.html',
htmlContent: `
@@ -40,32 +40,42 @@ export const scenario: TestScenario = {
</form>
`,
},
- expectations: calls => {
- assert.ok(calls.length >= 3, 'Not enough calls made');
- assert.ok(
- calls[0].name === 'navigate_page' || calls[0].name === 'new_page',
+ expectations: result => {
+ const pageId = result.consumePageNavigation();
+ assert.ok(result.remainingCalls.length >= 2, 'Not enough calls made');
+ result.assertNextCall(
+ 'take_snapshot',
+ result.hasPageIdRouting ? {pageId} : undefined,
+ );
+ const fillFormCall = result.assertNextCall(
+ 'fill_form',
+ result.hasPageIdRouting ? {pageId} : undefined,
);
- assert.strictEqual(calls[1].name, 'take_snapshot');
- assert.strictEqual(calls[2].name, 'fill_form');
- const elements = calls[2].args.elements as Array<{
- uid: string;
- value: string;
- }>;
+ const elements = fillFormCall.args.elements;
+ assert.ok(Array.isArray(elements), 'elements should be an array');
assert.strictEqual(
elements.length,
3,
'fill_form should be used with all form elements at once',
);
- const uids = new Set(elements.map(e => e.uid));
+ const typedElements = elements.map(e => {
+ assert.ok(e && typeof e === 'object' && 'uid' in e && 'value' in e);
+ return {
+ uid: String(e.uid),
+ value: String(e.value),
+ };
+ });
+
+ const uids = new Set(typedElements.map(e => e.uid));
assert.strictEqual(
uids.size,
3,
'fill_form should target three distinct elements',
);
- const values = elements.map(e => e.value).sort();
+ const values = typedElements.map(e => e.value).sort();
assert.deepStrictEqual(
values,
['2 vCPU, 4GB RAM', 'true', 'true'],
@@ -74,11 +84,9 @@ export const scenario: TestScenario = {
const submitUid = '1_15';
- const extraFormInteractions = calls
- .slice(3)
- .filter(
- c => ['fill', 'click'].includes(c.name) && c.args.uid !== submitUid,
- );
+ const extraFormInteractions = result.remainingCalls.filter(
+ c => ['fill', 'click'].includes(c.name) && c.args.uid !== submitUid,
+ );
assert.deepEqual(
extraFormInteractions.length,
0,
diff --git a/scripts/eval_scenarios/fix_webpage_issues_test.ts b/scripts/eval_scenarios/fix_webpage_issues_test.ts
index df19af9e..444def94 100644
--- a/scripts/eval_scenarios/fix_webpage_issues_test.ts
+++ b/scripts/eval_scenarios/fix_webpage_issues_test.ts
@@ -25,7 +25,7 @@ const INSPECTION_TOOLS = [
];
export const scenario: TestScenario = {
- prompt: 'Can you fix issues with my webpage?',
+ prompt: 'Can you fix issues with my webpage at <TEST_URL>?',
maxTurns: 4,
htmlRoute: {
path: '/fix_issues_test.html',
@@ -37,26 +37,23 @@ export const scenario: TestScenario = {
</script>
`,
},
- expectations: calls => {
- const NAVIGATION_TOOLS = ['navigate_page', 'new_page'];
- assert.ok(
- calls.length >= 2,
- 'Expected at least navigation and one inspection',
- );
- const navigationIndex = calls.findIndex(c =>
- NAVIGATION_TOOLS.includes(c.name),
- );
- assert.ok(
- navigationIndex !== -1,
- `Expected a navigation call (${NAVIGATION_TOOLS.join(' or ')}), got: ${calls.map(c => c.name).join(', ')}`,
- );
- const afterNavigation = calls.slice(navigationIndex + 1);
- const inspectionCalls = afterNavigation.filter(c =>
+ expectations: result => {
+ const pageId = result.consumePageNavigation();
+ const inspectionCalls = result.remainingCalls.filter(c =>
INSPECTION_TOOLS.includes(c.name),
);
assert.ok(
inspectionCalls.length >= 1,
- `Expected at least one inspection tool (${INSPECTION_TOOLS.join(', ')}) after navigation, got: ${calls.map(c => c.name).join(', ')}`,
+ `Expected at least one inspection tool (${INSPECTION_TOOLS.join(', ')}) after navigation, got: ${result.remainingCalls.map(c => c.name).join(', ')}`,
);
+ if (result.hasPageIdRouting) {
+ for (const inspectionCall of inspectionCalls) {
+ assert.strictEqual(
+ inspectionCall.args.pageId,
+ pageId,
+ `Inspection call ${inspectionCall.name} should target pageId: ${pageId}`,
+ );
+ }
+ }
},
};
diff --git a/scripts/eval_scenarios/frontend_snapshot_test.ts b/scripts/eval_scenarios/frontend_snapshot_test.ts
index c19b0011..c2166fea 100644
--- a/scripts/eval_scenarios/frontend_snapshot_test.ts
+++ b/scripts/eval_scenarios/frontend_snapshot_test.ts
@@ -19,16 +19,12 @@ export const scenario: TestScenario = {
path: '/frontend_snapshot.html',
htmlContent: '<h1>Frontend Test</h1><p>This is a test webpage.</p>',
},
- expectations: calls => {
- assert.strictEqual(calls.length, 2);
- assert.ok(
- calls[0].name === 'navigate_page' || calls[0].name === 'new_page',
- 'First call should be navigation',
- );
- assert.strictEqual(
- calls[1].name,
+ expectations: result => {
+ const pageId = result.consumePageNavigation();
+ assert.ok(result.remainingCalls.length >= 1);
+ result.assertNextCall(
'take_snapshot',
- 'Second call should be take_snapshot to read page content',
+ result.hasPageIdRouting ? {pageId} : undefined,
);
},
};
diff --git a/scripts/eval_scenarios/input_parallel_test.ts b/scripts/eval_scenarios/input_parallel_test.ts
index e9314803..7c466b85 100644
--- a/scripts/eval_scenarios/input_parallel_test.ts
+++ b/scripts/eval_scenarios/input_parallel_test.ts
@@ -10,7 +10,7 @@ import type {TestScenario} from '../eval_gemini.ts';
export const scenario: TestScenario = {
prompt:
- 'Go to <TEST_URL>, fill the input with "hello world" and click the button five times in parallel.',
+ 'Go to <TEST_URL>, fill the input with "hello world" and click the button five times in parallel without using a script.',
maxTurns: 10,
htmlRoute: {
path: '/input_test.html',
@@ -19,16 +19,22 @@ export const scenario: TestScenario = {
<button id="test-button">Submit</button>
`,
},
- expectations: calls => {
- assert.strictEqual(calls.length, 8);
- assert.ok(
- calls[0].name === 'navigate_page' || calls[0].name === 'new_page',
+ expectations: result => {
+ const pageId = result.consumePageNavigation();
+ assert.ok(result.remainingCalls.length >= 7);
+ result.assertNextCall(
+ 'take_snapshot',
+ result.hasPageIdRouting ? {pageId} : undefined,
);
- assert.ok(calls[1].name === 'take_snapshot');
- assert.ok(calls[2].name === 'fill');
- for (let i = 3; i < 8; i++) {
- assert.ok(calls[i].name === 'click');
- assert.strictEqual(Boolean(calls[i].args.includeSnapshot), false);
+ result.assertNextCall(
+ 'fill',
+ result.hasPageIdRouting ? {pageId} : undefined,
+ );
+ for (let i = 2; i < 7; i++) {
+ result.assertNextCall('click', {
+ includeSnapshot: undefined,
+ pageId: result.hasPageIdRouting ? pageId : undefined,
+ });
}
},
};
diff --git a/scripts/eval_scenarios/input_test.ts b/scripts/eval_scenarios/input_test.ts
index 6078e7f9..e7fef7a0 100644
--- a/scripts/eval_scenarios/input_test.ts
+++ b/scripts/eval_scenarios/input_test.ts
@@ -11,7 +11,7 @@ import type {TestScenario} from '../eval_gemini.ts';
export const scenario: TestScenario = {
prompt:
'Go to <TEST_URL>, fill the input with "hello world" and click the button.',
- maxTurns: 4,
+ maxTurns: 5,
htmlRoute: {
path: '/input_test.html',
htmlContent: `
@@ -19,13 +19,20 @@ export const scenario: TestScenario = {
<button id="test-button">Submit</button>
`,
},
- expectations: calls => {
- assert.strictEqual(calls.length, 4);
- assert.ok(
- calls[0].name === 'navigate_page' || calls[0].name === 'new_page',
+ expectations: result => {
+ const pageId = result.consumePageNavigation();
+ assert.ok(result.remainingCalls.length >= 3);
+ result.assertNextCall(
+ 'take_snapshot',
+ result.hasPageIdRouting ? {pageId} : undefined,
+ );
+ result.assertNextCall(
+ 'fill',
+ result.hasPageIdRouting ? {pageId} : undefined,
+ );
+ result.assertNextCall(
+ 'click',
+ result.hasPageIdRouting ? {pageId} : undefined,
);
- assert.ok(calls[1].name === 'take_snapshot');
- assert.ok(calls[2].name === 'fill');
- assert.ok(calls[3].name === 'click');
},
};
diff --git a/scripts/eval_scenarios/isolated_context_test.ts b/scripts/eval_scenarios/isolated_context_test.ts
index 0d76e5fe..c1ae69a3 100644
--- a/scripts/eval_scenarios/isolated_context_test.ts
+++ b/scripts/eval_scenarios/isolated_context_test.ts
@@ -4,8 +4,6 @@
* SPDX-License-Identifier: Apache-2.0
*/
-import assert from 'node:assert';
-
import type {TestScenario} from '../eval_gemini.ts';
export const scenario: TestScenario = {
@@ -13,19 +11,14 @@ export const scenario: TestScenario = {
'Create a new page <TEST_URL> in an isolated context called contextB. Take a screenshot there.',
maxTurns: 3,
htmlRoute: {
- path: '/test.html',
- htmlContent: `
- <h1>test</h1>
- `,
+ path: '/isolated_context.html',
+ htmlContent: '<h1>Isolated Context</h1>',
},
- expectations: calls => {
- console.log(JSON.stringify(calls, null, 2));
- assert.strictEqual(calls.length, 2);
- assert.ok(calls[0].name === 'new_page', 'First call should be navigation');
- assert.deepStrictEqual(calls[0].args.isolatedContext, 'contextB');
- assert.ok(
- calls[1].name === 'take_screenshot',
- 'Second call should be a screenshot',
+ expectations: result => {
+ result.assertNextCall('new_page', {isolatedContext: 'contextB'});
+ result.assertNextCall(
+ 'take_screenshot',
+ result.hasPageIdRouting ? {pageId: 2} : undefined,
);
},
};
diff --git a/scripts/eval_scenarios/lighthouse_a11y_test.ts b/scripts/eval_scenarios/lighthouse_a11y_test.ts
index cf0a0e67..457192fa 100644
--- a/scripts/eval_scenarios/lighthouse_a11y_test.ts
+++ b/scripts/eval_scenarios/lighthouse_a11y_test.ts
@@ -9,13 +9,29 @@ import assert from 'node:assert';
import type {TestScenario} from '../eval_gemini.ts';
export const scenario: TestScenario = {
- prompt: 'Check a11y issues on the current page',
- maxTurns: 1,
- expectations: calls => {
- assert.strictEqual(calls.length, 1);
- assert.ok(
- calls[0].name === 'lighthouse_audit',
- 'First call should be lighthouse_audit',
+ prompt: 'Check a11y issues on the page at <TEST_URL>',
+ maxTurns: 3,
+ htmlRoute: {
+ path: '/lighthouse_test.html',
+ htmlContent: `
+ <!DOCTYPE html>
+ <html>
+ <head>
+ <title>Lighthouse Audit Test</title>
+ </head>
+ <body>
+ <h1>Lighthouse Audit Test</h1>
+ <p>This is a valid test page for running audits.</p>
+ </body>
+ </html>
+ `,
+ },
+ expectations: result => {
+ const pageId = result.consumePageNavigation();
+ assert.ok(result.remainingCalls.length >= 1);
+ result.assertNextCall(
+ 'lighthouse_audit',
+ result.hasPageIdRouting ? {pageId} : undefined,
);
},
};
diff --git a/scripts/eval_scenarios/lighthouse_best_practices_test.ts b/scripts/eval_scenarios/lighthouse_best_practices_test.ts
index 54aed3b8..40d4dd11 100644
--- a/scripts/eval_scenarios/lighthouse_best_practices_test.ts
+++ b/scripts/eval_scenarios/lighthouse_best_practices_test.ts
@@ -9,13 +9,29 @@ import assert from 'node:assert';
import type {TestScenario} from '../eval_gemini.ts';
export const scenario: TestScenario = {
- prompt: 'Check for best practices on the current page',
- maxTurns: 1,
- expectations: calls => {
- assert.strictEqual(calls.length, 1);
- assert.ok(
- calls[0].name === 'lighthouse_audit',
- 'First call should be lighthouse_audit',
+ prompt: 'Check for best practices on the page at <TEST_URL>',
+ maxTurns: 3,
+ htmlRoute: {
+ path: '/lighthouse_test.html',
+ htmlContent: `
+ <!DOCTYPE html>
+ <html>
+ <head>
+ <title>Lighthouse Audit Test</title>
+ </head>
+ <body>
+ <h1>Lighthouse Audit Test</h1>
+ <p>This is a valid test page for running audits.</p>
+ </body>
+ </html>
+ `,
+ },
+ expectations: result => {
+ const pageId = result.consumePageNavigation();
+ assert.ok(result.remainingCalls.length >= 1);
+ result.assertNextCall(
+ 'lighthouse_audit',
+ result.hasPageIdRouting ? {pageId} : undefined,
);
},
};
diff --git a/scripts/eval_scenarios/navigation_test.ts b/scripts/eval_scenarios/navigation_test.ts
index 7619b686..6d214e57 100644
--- a/scripts/eval_scenarios/navigation_test.ts
+++ b/scripts/eval_scenarios/navigation_test.ts
@@ -9,14 +9,17 @@ import assert from 'node:assert';
import type {TestScenario} from '../eval_gemini.ts';
export const scenario: TestScenario = {
- prompt: 'Navigate to https://developers.chrome.com and tell me if it worked.',
- maxTurns: 1,
- expectations: calls => {
- assert.deepStrictEqual(calls, [
- {
- name: 'navigate_page',
- args: {url: 'https://developers.chrome.com'},
- },
- ]);
+ prompt:
+ 'Navigate in current page to https://developers.chrome.com and tell me if it worked.',
+ maxTurns: 2,
+ expectations: result => {
+ if (result.hasPageIdRouting) {
+ result.assertNextCall('list_pages');
+ }
+ assert.ok(result.remainingCalls.length >= 1);
+ result.assertNextCall('navigate_page', {
+ url: 'https://developers.chrome.com',
+ pageId: result.hasPageIdRouting ? 1 : undefined,
+ });
},
};
diff --git a/scripts/eval_scenarios/network_test.ts b/scripts/eval_scenarios/network_test.ts
index bacb26ac..83e6ed02 100644
--- a/scripts/eval_scenarios/network_test.ts
+++ b/scripts/eval_scenarios/network_test.ts
@@ -10,7 +10,7 @@ import type {TestScenario} from '../eval_gemini.ts';
export const scenario: TestScenario = {
prompt: 'Navigate to <TEST_URL> and list all network requests.',
- maxTurns: 2,
+ maxTurns: 3,
htmlRoute: {
path: '/network_test.html',
htmlContent: `
@@ -20,16 +20,12 @@ export const scenario: TestScenario = {
</script>
`,
},
- expectations: calls => {
- assert.strictEqual(calls.length, 2);
- assert.ok(
- calls[0].name === 'navigate_page' || calls[0].name === 'new_page',
- 'First call should be navigation',
- );
- assert.strictEqual(
- calls[1].name,
+ expectations: result => {
+ const pageId = result.consumePageNavigation();
+ assert.ok(result.remainingCalls.length >= 1);
+ result.assertNextCall(
'list_network_requests',
- 'Second call should be list_network_requests',
+ result.hasPageIdRouting ? {pageId} : undefined,
);
},
};
diff --git a/scripts/eval_scenarios/page_focus_keyboard_test.ts b/scripts/eval_scenarios/page_focus_keyboard_test.ts
index 655b1fec..f56c7c62 100644
--- a/scripts/eval_scenarios/page_focus_keyboard_test.ts
+++ b/scripts/eval_scenarios/page_focus_keyboard_test.ts
@@ -16,22 +16,29 @@ export const scenario: TestScenario = {
Now use the press_key tool to type "a" on Page 1 without selecting it first. You must use press_key, not fill or type_text. If you encounter any errors, recover from them.`,
maxTurns: 10,
- expectations: calls => {
+ expectations: result => {
// Should open 2 pages in the same context.
- const newPages = calls.filter(c => c.name === 'new_page');
+ const newPages = result.calls.filter(c => c.name === 'new_page');
assert.strictEqual(newPages.length, 2, 'Should open 2 pages');
assert.strictEqual(newPages[0].args.isolatedContext, 'session');
assert.strictEqual(newPages[1].args.isolatedContext, 'session');
// Should attempt press_key at least once.
- const pressKeys = calls.filter(c => c.name === 'press_key');
+ const pressKeys = result.calls.filter(c => c.name === 'press_key');
assert.ok(pressKeys.length >= 1, 'Should attempt press_key at least once');
+ for (const pk of pressKeys) {
+ assert.strictEqual(
+ pk.args.pageId,
+ 2,
+ 'press_key should target Page 1 (pageId: 2)',
+ );
+ }
- const selectPages = calls.filter(c => c.name === 'select_page');
+ const selectPages = result.calls.filter(c => c.name === 'select_page');
if (selectPages.length > 0) {
- const firstPressKeyIndex = calls.indexOf(pressKeys[0]);
- const firstSelectPageIndex = calls.indexOf(selectPages[0]);
+ const firstPressKeyIndex = result.calls.indexOf(pressKeys[0]);
+ const firstSelectPageIndex = result.calls.indexOf(selectPages[0]);
if (firstPressKeyIndex < firstSelectPageIndex) {
// Error path: press_key was attempted first and failed.
@@ -40,7 +47,7 @@ Now use the press_key tool to type "a" on Page 1 without selecting it first. You
pressKeys.length >= 2,
'Should retry press_key after error recovery',
);
- const lastPressKeyIndex = calls.lastIndexOf(pressKeys.at(-1)!);
+ const lastPressKeyIndex = result.calls.lastIndexOf(pressKeys.at(-1)!);
assert.ok(
firstSelectPageIndex < lastPressKeyIndex,
'select_page should precede the successful press_key',
diff --git a/scripts/eval_scenarios/page_id_routing_test.ts b/scripts/eval_scenarios/page_id_routing_test.ts
index 39007a4b..a65c9e83 100644
--- a/scripts/eval_scenarios/page_id_routing_test.ts
+++ b/scripts/eval_scenarios/page_id_routing_test.ts
@@ -15,9 +15,9 @@ export const scenario: TestScenario = {
- Page B (isolatedContext "contextB") at data:text/html,<button>Click B</button>
Then take a snapshot of Page A, take a snapshot of Page B, and then click the button on Page A.`,
maxTurns: 12,
- expectations: calls => {
+ expectations: result => {
// Should have 2 new_page calls with isolatedContext.
- const newPages = calls.filter(c => c.name === 'new_page');
+ const newPages = result.calls.filter(c => c.name === 'new_page');
assert.strictEqual(newPages.length, 2, 'Should open 2 pages');
for (const np of newPages) {
assert.strictEqual(
@@ -29,12 +29,26 @@ Then take a snapshot of Page A, take a snapshot of Page B, and then click the bu
// Should have at least 2 take_snapshot calls (one per page).
// The model may use pageId directly or select_page before each snapshot.
- const snapshots = calls.filter(c => c.name === 'take_snapshot');
+ const snapshots = result.calls.filter(c => c.name === 'take_snapshot');
assert.ok(snapshots.length >= 2, 'Should take at least 2 snapshots');
+ const snapshotPageIds = snapshots.map(s => s.args.pageId);
+ assert.ok(
+ snapshotPageIds.includes(2),
+ 'Should snapshot Page A (pageId: 2)',
+ );
+ assert.ok(
+ snapshotPageIds.includes(3),
+ 'Should snapshot Page B (pageId: 3)',
+ );
// Should have a click call (resolving uid from Page A's snapshot
// even though Page B was snapshotted after).
- const clicks = calls.filter(c => c.name === 'click');
+ const clicks = result.calls.filter(c => c.name === 'click');
assert.ok(clicks.length >= 1, 'Should click the button on Page A');
+ assert.strictEqual(
+ clicks[0].args.pageId,
+ 2,
+ 'Click should target Page A (pageId: 2)',
+ );
},
};
diff --git a/scripts/eval_scenarios/performance_test.ts b/scripts/eval_scenarios/performance_test.ts
index 63ab3220..6652e759 100644
--- a/scripts/eval_scenarios/performance_test.ts
+++ b/scripts/eval_scenarios/performance_test.ts
@@ -10,12 +10,13 @@ import type {TestScenario} from '../eval_gemini.ts';
export const scenario: TestScenario = {
prompt: 'Check the performance of https://developers.chrome.com',
- maxTurns: 2,
- expectations: calls => {
- assert.strictEqual(calls.length, 2);
- assert.ok(
- calls[0].name === 'navigate_page' || calls[0].name === 'new_page',
+ maxTurns: 3,
+ expectations: result => {
+ const pageId = result.consumePageNavigation();
+ assert.ok(result.remainingCalls.length >= 1);
+ result.assertNextCall(
+ 'performance_start_trace',
+ result.hasPageIdRouting ? {pageId} : undefined,
);
- assert.ok(calls[1].name === 'performance_start_trace');
},
};
diff --git a/scripts/eval_scenarios/select_page_test.ts b/scripts/eval_scenarios/select_page_test.ts
index a8bfba32..65d83558 100644
--- a/scripts/eval_scenarios/select_page_test.ts
+++ b/scripts/eval_scenarios/select_page_test.ts
@@ -18,23 +18,10 @@ export const scenario: TestScenario = {
<h1>test</h1>
`,
},
- expectations: calls => {
- assert.strictEqual(calls.length, 3);
- assert.ok(calls[0].name === 'new_page', 'First call should be navigation');
- assert.ok(calls[1].name === 'new_page', 'Second call should be navigation');
- assert.ok(
- calls[2].name === 'select_page',
- 'Third call should be select_page',
- );
- assert.strictEqual(
- calls[2].args.pageId,
- 2,
- 'PageId has to be set to 2. about:blank is 1, <TEST_URL> is 2, https://developers.chrome.com is 3.',
- );
- assert.strictEqual(
- calls[2].args.bringToFront,
- undefined,
- 'bringToFront should use the default value.',
- );
+ expectations: result => {
+ result.consumePageNavigation();
+ assert.strictEqual(result.remainingCalls.length, 2);
+ result.assertNextCall('new_page');
+ result.assertNextCall('select_page', {pageId: 2, bringToFront: undefined});
},
};
diff --git a/scripts/eval_scenarios/snapshot_test.ts b/scripts/eval_scenarios/snapshot_test.ts
index 4877c1d1..027e808f 100644
--- a/scripts/eval_scenarios/snapshot_test.ts
+++ b/scripts/eval_scenarios/snapshot_test.ts
@@ -10,16 +10,17 @@ import type {TestScenario} from '../eval_gemini.ts';
export const scenario: TestScenario = {
prompt: 'Read the content of <TEST_URL>',
- maxTurns: 3,
+ maxTurns: 4,
htmlRoute: {
path: '/test.html',
htmlContent: '<h1>Hello World</h1><p>This is a test.</p>',
},
- expectations: calls => {
- assert.strictEqual(calls.length, 2);
- assert.ok(
- calls[0].name === 'navigate_page' || calls[0].name === 'new_page',
+ expectations: result => {
+ const pageId = result.consumePageNavigation();
+ assert.strictEqual(result.remainingCalls.length, 1);
+ result.assertNextCall(
+ 'take_snapshot',
+ result.hasPageIdRouting ? {pageId} : undefined,
);
- assert.ok(calls[1].name === 'take_snapshot');
},
};