Chrome DevTools for coding agents
test: update all the eval scenarios to work with experimentalPageIdRouting flag (#2085)
Samiya Caur8e8e83e3f8d1
diff --git a/scripts/eval_gemini.ts b/scripts/eval_gemini.tsindex bd5bafdb..3fcd4c45 100644--- a/scripts/eval_gemini.ts+++ b/scripts/eval_gemini.ts@@ -19,23 +19,10 @@ const ROOT_DIR = path.resolve(import.meta.dirname, '..');const SCENARIOS_DIR = path.join(import.meta.dirname, 'eval_scenarios');const SKILL_PATH = path.join(ROOT_DIR, 'skills', 'chrome-devtools', 'SKILL.md');-// Define schema for our test scenarios-export interface CapturedFunctionCall {- name: string;- args: Record<string, unknown>;-}--export interface TestScenario {- prompt: string;- maxTurns: number;- expectations: (calls: CapturedFunctionCall[]) => void;- htmlRoute?: {- path: string;- htmlContent: string;- };- /** Extra CLI flags passed to the MCP server (e.g. '--experimental-page-id-routing'). */- serverArgs?: string[];-}+import type {CapturedFunctionCall, TestScenario} from './eval_result.ts';+import {Result} from './eval_result.ts';+export type {CapturedFunctionCall, TestScenario};+export {Result};async function loadScenario(scenarioPath: string): Promise<TestScenario> {const module = await import(pathToFileURL(scenarioPath).href);@@ -54,6 +41,7 @@ async function runSingleScenario(modelId: string,debug: boolean,includeSkill: boolean,+ extraServerArgs: string[] = [],): Promise<void> {const debugLog = (...args: unknown[]) => {if (debug) {@@ -125,6 +113,9 @@ async function runSingleScenario(if (scenario.serverArgs) {args.push(...scenario.serverArgs);}+ if (extraServerArgs.length > 0) {+ args.push(...extraServerArgs);+ }transport = new StdioClientTransport({command: 'node',@@ -175,7 +166,7 @@ async function runSingleScenario(debugLog(`\n--- Response ---\n${result.text}`);debugLog('\nVerifying expectations...');- scenario.expectations(allCalls);+ scenario.expectations(new Result(allCalls, args));} finally {try {await client?.close();@@ -195,7 +186,7 @@ async function main() {options: {model: {type: 'string',- default: 'gemini-2.5-flash',+ default: 'gemini-3-flash-preview',},debug: {type: 'boolean',@@ -209,6 +200,9 @@ async function main() {type: 'boolean',default: false,},+ 'server-args': {+ type: 'string',+ },},allowPositionals: true,});@@ -217,6 +211,9 @@ async function main() {const debug = values.debug;const repeat = values.repeat;const includeSkill = values['include-skill'];+ const extraServerArgs = values['server-args']+ ? values['server-args'].split(/\s+/)+ : [];const scenarioFiles =positionals.length > 0@@ -248,6 +245,7 @@ async function main() {modelId,debug,includeSkill,+ extraServerArgs,);console.log(`✔ ${path.relative(ROOT_DIR, scenarioPath)} (Run ${i})`);successCount++;diff --git a/scripts/eval_result.ts b/scripts/eval_result.tsnew file mode 100644index 00000000..9b734b29--- /dev/null+++ b/scripts/eval_result.ts@@ -0,0 +1,107 @@+/**+ * @license+ * Copyright 2026 Google LLC+ * SPDX-License-Identifier: Apache-2.0+ */++import assert from 'node:assert';++export interface CapturedFunctionCall {+ name: string;+ args: Record<string, unknown>;+}++export class Result {+ private nextCallIndex = 0;+ public readonly calls: CapturedFunctionCall[];+ public readonly serverArgs: string[];++ constructor(calls: CapturedFunctionCall[], serverArgs: string[]) {+ this.calls = calls;+ this.serverArgs = serverArgs;+ }++ get hasPageIdRouting(): boolean {+ return this.serverArgs.includes('--experimental-page-id-routing');+ }++ get remainingCalls(): CapturedFunctionCall[] {+ return this.calls.slice(this.nextCallIndex);+ }++ /**+ * Consumes initial page navigation/setup boilerplate.+ * - Ignores/skips leading list_pages calls.+ * - Asserts that new_page or navigate_page was called.+ * - Determines the expected pageId.+ * - Returns the active pageId.+ */+ consumePageNavigation(): number | undefined {+ if (this.calls[this.nextCallIndex]?.name === 'list_pages') {+ this.nextCallIndex++;+ }++ const navCall = this.calls[this.nextCallIndex];+ assert.ok(+ navCall &&+ (navCall.name === 'new_page' || navCall.name === 'navigate_page'),+ `Expected navigation call (new_page or navigate_page), but got: ${navCall?.name || 'none'}`,+ );+ this.nextCallIndex++;++ const isNewPage = navCall.name === 'new_page';+ let pageId: number | undefined;+ if (this.hasPageIdRouting) {+ pageId = isNewPage ? 2 : 1;+ }++ return pageId;+ }++ /**+ * Asserts that the next call in sequence has the correct name and matches expected arguments.+ * Increments the internal call index.+ */+ assertNextCall(+ name: string,+ expectedArgs?: Record<string, unknown>,+ ): CapturedFunctionCall {+ const call = this.calls[this.nextCallIndex];+ assert.ok(+ call,+ `Expected call at index ${this.nextCallIndex} (name: '${name}') to exist`,+ );+ assert.strictEqual(+ call.name,+ name,+ `Expected call at index ${this.nextCallIndex} to be '${name}', but got '${call.name}'`,+ );++ if (expectedArgs) {+ for (const entry of Object.entries(expectedArgs)) {+ const key = entry[0];+ const value = entry[1];+ assert.deepStrictEqual(+ call.args[key],+ value,+ `Expected argument '${key}' on call '${name}' to be ${JSON.stringify(value)}, got ${JSON.stringify(call.args[key])}`,+ );+ }+ }++ this.nextCallIndex++;+ return call;+ }+}++export interface TestScenario {+ prompt: string;+ maxTurns: number;+ expectations: (result: Result) => void;+ htmlRoute?: {+ path: string;+ htmlContent: string;+ };+ /** Extra CLI flags passed to the MCP server (e.g. '--experimental-page-id-routing'). */+ serverArgs?: string[];+}diff --git a/scripts/eval_scenarios/console_test.ts b/scripts/eval_scenarios/console_test.tsindex d82abc2a..125b1e92 100644--- a/scripts/eval_scenarios/console_test.ts+++ b/scripts/eval_scenarios/console_test.ts@@ -10,7 +10,7 @@ import type {TestScenario} from '../eval_gemini.ts';export const scenario: TestScenario = {prompt: 'Navigate to <TEST_URL> and check the console messages.',- maxTurns: 2,+ maxTurns: 3,htmlRoute: {path: '/console_test.html',htmlContent: `@@ -20,16 +20,12 @@ export const scenario: TestScenario = {</script>`,},- expectations: calls => {- assert.strictEqual(calls.length, 2);- assert.ok(- calls[0].name === 'navigate_page' || calls[0].name === 'new_page',- 'First call should be navigation',- );- assert.strictEqual(- calls[1].name,+ expectations: result => {+ const pageId = result.consumePageNavigation();+ assert.strictEqual(result.remainingCalls.length, 1);+ result.assertNextCall('list_console_messages',- 'Second call should be list_console_messages',+ result.hasPageIdRouting ? {pageId} : undefined,);},};diff --git a/scripts/eval_scenarios/emulation_test.ts b/scripts/eval_scenarios/emulation_test.tsindex 2eebac83..9456e451 100644--- a/scripts/eval_scenarios/emulation_test.ts+++ b/scripts/eval_scenarios/emulation_test.ts@@ -11,9 +11,12 @@ import type {TestScenario} from '../eval_gemini.ts';export const scenario: TestScenario = {prompt: 'Emulate offline network conditions.',maxTurns: 2,- expectations: calls => {- assert.strictEqual(calls.length, 1);- assert.strictEqual(calls[0].name, 'emulate');- assert.strictEqual(calls[0].args.networkConditions, 'Offline');+ expectations: result => {+ assert.ok(result.remainingCalls.length >= 1);+ result.assertNextCall('list_pages');+ result.assertNextCall('emulate', {+ networkConditions: 'Offline',+ pageId: result.hasPageIdRouting ? 1 : undefined,+ });},};diff --git a/scripts/eval_scenarios/emulation_userAgent_test.ts b/scripts/eval_scenarios/emulation_userAgent_test.tsindex ee0cfc7e..0fb4959c 100644--- a/scripts/eval_scenarios/emulation_userAgent_test.ts+++ b/scripts/eval_scenarios/emulation_userAgent_test.ts@@ -9,14 +9,20 @@ import assert from 'node:assert';import type {TestScenario} from '../eval_gemini.ts';export const scenario: TestScenario = {- prompt: 'Emulate iPhone 14 user agent',+ prompt: 'Emulate current page with iPhone 14 user agent',maxTurns: 2,- expectations: calls => {- assert.strictEqual(calls.length, 1);- assert.strictEqual(calls[0].name, 'emulate');- assert.deepStrictEqual(- calls[0].args.userAgent,- 'Mozilla/5.0 (iPhone; CPU iPhone OS 16_0 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/16.0 Mobile/15E148 Safari/604.1',- );+ expectations: result => {+ assert.ok(result.remainingCalls.length >= 1);+ if (+ result.hasPageIdRouting ||+ result.remainingCalls[0]?.name === 'list_pages'+ ) {+ result.assertNextCall('list_pages');+ }+ result.assertNextCall('emulate', {+ userAgent:+ 'Mozilla/5.0 (iPhone; CPU iPhone OS 16_0 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/16.0 Mobile/15E148 Safari/604.1',+ pageId: result.hasPageIdRouting ? 1 : undefined,+ });},};diff --git a/scripts/eval_scenarios/emulation_viewport_test.ts b/scripts/eval_scenarios/emulation_viewport_test.tsindex f4f51904..52a945a3 100644--- a/scripts/eval_scenarios/emulation_viewport_test.ts+++ b/scripts/eval_scenarios/emulation_viewport_test.ts@@ -9,11 +9,19 @@ import assert from 'node:assert';import type {TestScenario} from '../eval_gemini.ts';export const scenario: TestScenario = {- prompt: 'Emulate iPhone 14 viewport',+ prompt: 'Emulate current page with iPhone 14 viewport',maxTurns: 2,- expectations: calls => {- assert.strictEqual(calls.length, 1);- assert.strictEqual(calls[0].name, 'emulate');- assert.deepStrictEqual(calls[0].args.viewport, '390x844x3,mobile,touch');+ expectations: result => {+ assert.ok(result.remainingCalls.length >= 1);+ if (+ result.hasPageIdRouting ||+ result.remainingCalls[0]?.name === 'list_pages'+ ) {+ result.assertNextCall('list_pages');+ }+ result.assertNextCall('emulate', {+ viewport: '390x844x3,mobile,touch',+ pageId: result.hasPageIdRouting ? 1 : undefined,+ });},};diff --git a/scripts/eval_scenarios/fill_select_and_checkboxes_test.ts b/scripts/eval_scenarios/fill_select_and_checkboxes_test.tsindex 9876e647..cb66cbbf 100644--- a/scripts/eval_scenarios/fill_select_and_checkboxes_test.ts+++ b/scripts/eval_scenarios/fill_select_and_checkboxes_test.ts@@ -11,7 +11,7 @@ import type {TestScenario} from '../eval_gemini.ts';export const scenario: TestScenario = {prompt:'Go to <TEST_URL>, fill the form with size = 2 CPUs and components = [docker, nginx].',- maxTurns: 4, // allow for at least one extra turn to verify there are no extra clicks after fill_form+ maxTurns: 5, // allow for at least one extra turn to verify there are no extra clicks after fill_formhtmlRoute: {path: '/input_test.html',htmlContent: `@@ -40,32 +40,42 @@ export const scenario: TestScenario = {</form>`,},- expectations: calls => {- assert.ok(calls.length >= 3, 'Not enough calls made');- assert.ok(- calls[0].name === 'navigate_page' || calls[0].name === 'new_page',+ expectations: result => {+ const pageId = result.consumePageNavigation();+ assert.ok(result.remainingCalls.length >= 2, 'Not enough calls made');+ result.assertNextCall(+ 'take_snapshot',+ result.hasPageIdRouting ? {pageId} : undefined,+ );+ const fillFormCall = result.assertNextCall(+ 'fill_form',+ result.hasPageIdRouting ? {pageId} : undefined,);- assert.strictEqual(calls[1].name, 'take_snapshot');- assert.strictEqual(calls[2].name, 'fill_form');- const elements = calls[2].args.elements as Array<{- uid: string;- value: string;- }>;+ const elements = fillFormCall.args.elements;+ assert.ok(Array.isArray(elements), 'elements should be an array');assert.strictEqual(elements.length,3,'fill_form should be used with all form elements at once',);- const uids = new Set(elements.map(e => e.uid));+ const typedElements = elements.map(e => {+ assert.ok(e && typeof e === 'object' && 'uid' in e && 'value' in e);+ return {+ uid: String(e.uid),+ value: String(e.value),+ };+ });++ const uids = new Set(typedElements.map(e => e.uid));assert.strictEqual(uids.size,3,'fill_form should target three distinct elements',);- const values = elements.map(e => e.value).sort();+ const values = typedElements.map(e => e.value).sort();assert.deepStrictEqual(values,['2 vCPU, 4GB RAM', 'true', 'true'],@@ -74,11 +84,9 @@ export const scenario: TestScenario = {const submitUid = '1_15';- const extraFormInteractions = calls- .slice(3)- .filter(- c => ['fill', 'click'].includes(c.name) && c.args.uid !== submitUid,- );+ const extraFormInteractions = result.remainingCalls.filter(+ c => ['fill', 'click'].includes(c.name) && c.args.uid !== submitUid,+ );assert.deepEqual(extraFormInteractions.length,0,diff --git a/scripts/eval_scenarios/fix_webpage_issues_test.ts b/scripts/eval_scenarios/fix_webpage_issues_test.tsindex df19af9e..444def94 100644--- a/scripts/eval_scenarios/fix_webpage_issues_test.ts+++ b/scripts/eval_scenarios/fix_webpage_issues_test.ts@@ -25,7 +25,7 @@ const INSPECTION_TOOLS = [];export const scenario: TestScenario = {- prompt: 'Can you fix issues with my webpage?',+ prompt: 'Can you fix issues with my webpage at <TEST_URL>?',maxTurns: 4,htmlRoute: {path: '/fix_issues_test.html',@@ -37,26 +37,23 @@ export const scenario: TestScenario = {</script>`,},- expectations: calls => {- const NAVIGATION_TOOLS = ['navigate_page', 'new_page'];- assert.ok(- calls.length >= 2,- 'Expected at least navigation and one inspection',- );- const navigationIndex = calls.findIndex(c =>- NAVIGATION_TOOLS.includes(c.name),- );- assert.ok(- navigationIndex !== -1,- `Expected a navigation call (${NAVIGATION_TOOLS.join(' or ')}), got: ${calls.map(c => c.name).join(', ')}`,- );- const afterNavigation = calls.slice(navigationIndex + 1);- const inspectionCalls = afterNavigation.filter(c =>+ expectations: result => {+ const pageId = result.consumePageNavigation();+ const inspectionCalls = result.remainingCalls.filter(c =>INSPECTION_TOOLS.includes(c.name),);assert.ok(inspectionCalls.length >= 1,- `Expected at least one inspection tool (${INSPECTION_TOOLS.join(', ')}) after navigation, got: ${calls.map(c => c.name).join(', ')}`,+ `Expected at least one inspection tool (${INSPECTION_TOOLS.join(', ')}) after navigation, got: ${result.remainingCalls.map(c => c.name).join(', ')}`,);+ if (result.hasPageIdRouting) {+ for (const inspectionCall of inspectionCalls) {+ assert.strictEqual(+ inspectionCall.args.pageId,+ pageId,+ `Inspection call ${inspectionCall.name} should target pageId: ${pageId}`,+ );+ }+ }},};diff --git a/scripts/eval_scenarios/frontend_snapshot_test.ts b/scripts/eval_scenarios/frontend_snapshot_test.tsindex c19b0011..c2166fea 100644--- a/scripts/eval_scenarios/frontend_snapshot_test.ts+++ b/scripts/eval_scenarios/frontend_snapshot_test.ts@@ -19,16 +19,12 @@ export const scenario: TestScenario = {path: '/frontend_snapshot.html',htmlContent: '<h1>Frontend Test</h1><p>This is a test webpage.</p>',},- expectations: calls => {- assert.strictEqual(calls.length, 2);- assert.ok(- calls[0].name === 'navigate_page' || calls[0].name === 'new_page',- 'First call should be navigation',- );- assert.strictEqual(- calls[1].name,+ expectations: result => {+ const pageId = result.consumePageNavigation();+ assert.ok(result.remainingCalls.length >= 1);+ result.assertNextCall('take_snapshot',- 'Second call should be take_snapshot to read page content',+ result.hasPageIdRouting ? {pageId} : undefined,);},};diff --git a/scripts/eval_scenarios/input_parallel_test.ts b/scripts/eval_scenarios/input_parallel_test.tsindex e9314803..7c466b85 100644--- a/scripts/eval_scenarios/input_parallel_test.ts+++ b/scripts/eval_scenarios/input_parallel_test.ts@@ -10,7 +10,7 @@ import type {TestScenario} from '../eval_gemini.ts';export const scenario: TestScenario = {prompt:- 'Go to <TEST_URL>, fill the input with "hello world" and click the button five times in parallel.',+ 'Go to <TEST_URL>, fill the input with "hello world" and click the button five times in parallel without using a script.',maxTurns: 10,htmlRoute: {path: '/input_test.html',@@ -19,16 +19,22 @@ export const scenario: TestScenario = {<button id="test-button">Submit</button>`,},- expectations: calls => {- assert.strictEqual(calls.length, 8);- assert.ok(- calls[0].name === 'navigate_page' || calls[0].name === 'new_page',+ expectations: result => {+ const pageId = result.consumePageNavigation();+ assert.ok(result.remainingCalls.length >= 7);+ result.assertNextCall(+ 'take_snapshot',+ result.hasPageIdRouting ? {pageId} : undefined,);- assert.ok(calls[1].name === 'take_snapshot');- assert.ok(calls[2].name === 'fill');- for (let i = 3; i < 8; i++) {- assert.ok(calls[i].name === 'click');- assert.strictEqual(Boolean(calls[i].args.includeSnapshot), false);+ result.assertNextCall(+ 'fill',+ result.hasPageIdRouting ? {pageId} : undefined,+ );+ for (let i = 2; i < 7; i++) {+ result.assertNextCall('click', {+ includeSnapshot: undefined,+ pageId: result.hasPageIdRouting ? pageId : undefined,+ });}},};diff --git a/scripts/eval_scenarios/input_test.ts b/scripts/eval_scenarios/input_test.tsindex 6078e7f9..e7fef7a0 100644--- a/scripts/eval_scenarios/input_test.ts+++ b/scripts/eval_scenarios/input_test.ts@@ -11,7 +11,7 @@ import type {TestScenario} from '../eval_gemini.ts';export const scenario: TestScenario = {prompt:'Go to <TEST_URL>, fill the input with "hello world" and click the button.',- maxTurns: 4,+ maxTurns: 5,htmlRoute: {path: '/input_test.html',htmlContent: `@@ -19,13 +19,20 @@ export const scenario: TestScenario = {<button id="test-button">Submit</button>`,},- expectations: calls => {- assert.strictEqual(calls.length, 4);- assert.ok(- calls[0].name === 'navigate_page' || calls[0].name === 'new_page',+ expectations: result => {+ const pageId = result.consumePageNavigation();+ assert.ok(result.remainingCalls.length >= 3);+ result.assertNextCall(+ 'take_snapshot',+ result.hasPageIdRouting ? {pageId} : undefined,+ );+ result.assertNextCall(+ 'fill',+ result.hasPageIdRouting ? {pageId} : undefined,+ );+ result.assertNextCall(+ 'click',+ result.hasPageIdRouting ? {pageId} : undefined,);- assert.ok(calls[1].name === 'take_snapshot');- assert.ok(calls[2].name === 'fill');- assert.ok(calls[3].name === 'click');},};diff --git a/scripts/eval_scenarios/isolated_context_test.ts b/scripts/eval_scenarios/isolated_context_test.tsindex 0d76e5fe..c1ae69a3 100644--- a/scripts/eval_scenarios/isolated_context_test.ts+++ b/scripts/eval_scenarios/isolated_context_test.ts@@ -4,8 +4,6 @@* SPDX-License-Identifier: Apache-2.0*/-import assert from 'node:assert';-import type {TestScenario} from '../eval_gemini.ts';export const scenario: TestScenario = {@@ -13,19 +11,14 @@ export const scenario: TestScenario = {'Create a new page <TEST_URL> in an isolated context called contextB. Take a screenshot there.',maxTurns: 3,htmlRoute: {- path: '/test.html',- htmlContent: `- <h1>test</h1>- `,+ path: '/isolated_context.html',+ htmlContent: '<h1>Isolated Context</h1>',},- expectations: calls => {- console.log(JSON.stringify(calls, null, 2));- assert.strictEqual(calls.length, 2);- assert.ok(calls[0].name === 'new_page', 'First call should be navigation');- assert.deepStrictEqual(calls[0].args.isolatedContext, 'contextB');- assert.ok(- calls[1].name === 'take_screenshot',- 'Second call should be a screenshot',+ expectations: result => {+ result.assertNextCall('new_page', {isolatedContext: 'contextB'});+ result.assertNextCall(+ 'take_screenshot',+ result.hasPageIdRouting ? {pageId: 2} : undefined,);},};diff --git a/scripts/eval_scenarios/lighthouse_a11y_test.ts b/scripts/eval_scenarios/lighthouse_a11y_test.tsindex cf0a0e67..457192fa 100644--- a/scripts/eval_scenarios/lighthouse_a11y_test.ts+++ b/scripts/eval_scenarios/lighthouse_a11y_test.ts@@ -9,13 +9,29 @@ import assert from 'node:assert';import type {TestScenario} from '../eval_gemini.ts';export const scenario: TestScenario = {- prompt: 'Check a11y issues on the current page',- maxTurns: 1,- expectations: calls => {- assert.strictEqual(calls.length, 1);- assert.ok(- calls[0].name === 'lighthouse_audit',- 'First call should be lighthouse_audit',+ prompt: 'Check a11y issues on the page at <TEST_URL>',+ maxTurns: 3,+ htmlRoute: {+ path: '/lighthouse_test.html',+ htmlContent: `+ <!DOCTYPE html>+ <html>+ <head>+ <title>Lighthouse Audit Test</title>+ </head>+ <body>+ <h1>Lighthouse Audit Test</h1>+ <p>This is a valid test page for running audits.</p>+ </body>+ </html>+ `,+ },+ expectations: result => {+ const pageId = result.consumePageNavigation();+ assert.ok(result.remainingCalls.length >= 1);+ result.assertNextCall(+ 'lighthouse_audit',+ result.hasPageIdRouting ? {pageId} : undefined,);},};diff --git a/scripts/eval_scenarios/lighthouse_best_practices_test.ts b/scripts/eval_scenarios/lighthouse_best_practices_test.tsindex 54aed3b8..40d4dd11 100644--- a/scripts/eval_scenarios/lighthouse_best_practices_test.ts+++ b/scripts/eval_scenarios/lighthouse_best_practices_test.ts@@ -9,13 +9,29 @@ import assert from 'node:assert';import type {TestScenario} from '../eval_gemini.ts';export const scenario: TestScenario = {- prompt: 'Check for best practices on the current page',- maxTurns: 1,- expectations: calls => {- assert.strictEqual(calls.length, 1);- assert.ok(- calls[0].name === 'lighthouse_audit',- 'First call should be lighthouse_audit',+ prompt: 'Check for best practices on the page at <TEST_URL>',+ maxTurns: 3,+ htmlRoute: {+ path: '/lighthouse_test.html',+ htmlContent: `+ <!DOCTYPE html>+ <html>+ <head>+ <title>Lighthouse Audit Test</title>+ </head>+ <body>+ <h1>Lighthouse Audit Test</h1>+ <p>This is a valid test page for running audits.</p>+ </body>+ </html>+ `,+ },+ expectations: result => {+ const pageId = result.consumePageNavigation();+ assert.ok(result.remainingCalls.length >= 1);+ result.assertNextCall(+ 'lighthouse_audit',+ result.hasPageIdRouting ? {pageId} : undefined,);},};diff --git a/scripts/eval_scenarios/navigation_test.ts b/scripts/eval_scenarios/navigation_test.tsindex 7619b686..6d214e57 100644--- a/scripts/eval_scenarios/navigation_test.ts+++ b/scripts/eval_scenarios/navigation_test.ts@@ -9,14 +9,17 @@ import assert from 'node:assert';import type {TestScenario} from '../eval_gemini.ts';export const scenario: TestScenario = {- prompt: 'Navigate to https://developers.chrome.com and tell me if it worked.',- maxTurns: 1,- expectations: calls => {- assert.deepStrictEqual(calls, [- {- name: 'navigate_page',- args: {url: 'https://developers.chrome.com'},- },- ]);+ prompt:+ 'Navigate in current page to https://developers.chrome.com and tell me if it worked.',+ maxTurns: 2,+ expectations: result => {+ if (result.hasPageIdRouting) {+ result.assertNextCall('list_pages');+ }+ assert.ok(result.remainingCalls.length >= 1);+ result.assertNextCall('navigate_page', {+ url: 'https://developers.chrome.com',+ pageId: result.hasPageIdRouting ? 1 : undefined,+ });},};diff --git a/scripts/eval_scenarios/network_test.ts b/scripts/eval_scenarios/network_test.tsindex bacb26ac..83e6ed02 100644--- a/scripts/eval_scenarios/network_test.ts+++ b/scripts/eval_scenarios/network_test.ts@@ -10,7 +10,7 @@ import type {TestScenario} from '../eval_gemini.ts';export const scenario: TestScenario = {prompt: 'Navigate to <TEST_URL> and list all network requests.',- maxTurns: 2,+ maxTurns: 3,htmlRoute: {path: '/network_test.html',htmlContent: `@@ -20,16 +20,12 @@ export const scenario: TestScenario = {</script>`,},- expectations: calls => {- assert.strictEqual(calls.length, 2);- assert.ok(- calls[0].name === 'navigate_page' || calls[0].name === 'new_page',- 'First call should be navigation',- );- assert.strictEqual(- calls[1].name,+ expectations: result => {+ const pageId = result.consumePageNavigation();+ assert.ok(result.remainingCalls.length >= 1);+ result.assertNextCall('list_network_requests',- 'Second call should be list_network_requests',+ result.hasPageIdRouting ? {pageId} : undefined,);},};diff --git a/scripts/eval_scenarios/page_focus_keyboard_test.ts b/scripts/eval_scenarios/page_focus_keyboard_test.tsindex 655b1fec..f56c7c62 100644--- a/scripts/eval_scenarios/page_focus_keyboard_test.ts+++ b/scripts/eval_scenarios/page_focus_keyboard_test.ts@@ -16,22 +16,29 @@ export const scenario: TestScenario = {Now use the press_key tool to type "a" on Page 1 without selecting it first. You must use press_key, not fill or type_text. If you encounter any errors, recover from them.`,maxTurns: 10,- expectations: calls => {+ expectations: result => {// Should open 2 pages in the same context.- const newPages = calls.filter(c => c.name === 'new_page');+ const newPages = result.calls.filter(c => c.name === 'new_page');assert.strictEqual(newPages.length, 2, 'Should open 2 pages');assert.strictEqual(newPages[0].args.isolatedContext, 'session');assert.strictEqual(newPages[1].args.isolatedContext, 'session');// Should attempt press_key at least once.- const pressKeys = calls.filter(c => c.name === 'press_key');+ const pressKeys = result.calls.filter(c => c.name === 'press_key');assert.ok(pressKeys.length >= 1, 'Should attempt press_key at least once');+ for (const pk of pressKeys) {+ assert.strictEqual(+ pk.args.pageId,+ 2,+ 'press_key should target Page 1 (pageId: 2)',+ );+ }- const selectPages = calls.filter(c => c.name === 'select_page');+ const selectPages = result.calls.filter(c => c.name === 'select_page');if (selectPages.length > 0) {- const firstPressKeyIndex = calls.indexOf(pressKeys[0]);- const firstSelectPageIndex = calls.indexOf(selectPages[0]);+ const firstPressKeyIndex = result.calls.indexOf(pressKeys[0]);+ const firstSelectPageIndex = result.calls.indexOf(selectPages[0]);if (firstPressKeyIndex < firstSelectPageIndex) {// Error path: press_key was attempted first and failed.@@ -40,7 +47,7 @@ Now use the press_key tool to type "a" on Page 1 without selecting it first. YoupressKeys.length >= 2,'Should retry press_key after error recovery',);- const lastPressKeyIndex = calls.lastIndexOf(pressKeys.at(-1)!);+ const lastPressKeyIndex = result.calls.lastIndexOf(pressKeys.at(-1)!);assert.ok(firstSelectPageIndex < lastPressKeyIndex,'select_page should precede the successful press_key',diff --git a/scripts/eval_scenarios/page_id_routing_test.ts b/scripts/eval_scenarios/page_id_routing_test.tsindex 39007a4b..a65c9e83 100644--- a/scripts/eval_scenarios/page_id_routing_test.ts+++ b/scripts/eval_scenarios/page_id_routing_test.ts@@ -15,9 +15,9 @@ export const scenario: TestScenario = {- Page B (isolatedContext "contextB") at data:text/html,<button>Click B</button>Then take a snapshot of Page A, take a snapshot of Page B, and then click the button on Page A.`,maxTurns: 12,- expectations: calls => {+ expectations: result => {// Should have 2 new_page calls with isolatedContext.- const newPages = calls.filter(c => c.name === 'new_page');+ const newPages = result.calls.filter(c => c.name === 'new_page');assert.strictEqual(newPages.length, 2, 'Should open 2 pages');for (const np of newPages) {assert.strictEqual(@@ -29,12 +29,26 @@ Then take a snapshot of Page A, take a snapshot of Page B, and then click the bu// Should have at least 2 take_snapshot calls (one per page).// The model may use pageId directly or select_page before each snapshot.- const snapshots = calls.filter(c => c.name === 'take_snapshot');+ const snapshots = result.calls.filter(c => c.name === 'take_snapshot');assert.ok(snapshots.length >= 2, 'Should take at least 2 snapshots');+ const snapshotPageIds = snapshots.map(s => s.args.pageId);+ assert.ok(+ snapshotPageIds.includes(2),+ 'Should snapshot Page A (pageId: 2)',+ );+ assert.ok(+ snapshotPageIds.includes(3),+ 'Should snapshot Page B (pageId: 3)',+ );// Should have a click call (resolving uid from Page A's snapshot// even though Page B was snapshotted after).- const clicks = calls.filter(c => c.name === 'click');+ const clicks = result.calls.filter(c => c.name === 'click');assert.ok(clicks.length >= 1, 'Should click the button on Page A');+ assert.strictEqual(+ clicks[0].args.pageId,+ 2,+ 'Click should target Page A (pageId: 2)',+ );},};diff --git a/scripts/eval_scenarios/performance_test.ts b/scripts/eval_scenarios/performance_test.tsindex 63ab3220..6652e759 100644--- a/scripts/eval_scenarios/performance_test.ts+++ b/scripts/eval_scenarios/performance_test.ts@@ -10,12 +10,13 @@ import type {TestScenario} from '../eval_gemini.ts';export const scenario: TestScenario = {prompt: 'Check the performance of https://developers.chrome.com',- maxTurns: 2,- expectations: calls => {- assert.strictEqual(calls.length, 2);- assert.ok(- calls[0].name === 'navigate_page' || calls[0].name === 'new_page',+ maxTurns: 3,+ expectations: result => {+ const pageId = result.consumePageNavigation();+ assert.ok(result.remainingCalls.length >= 1);+ result.assertNextCall(+ 'performance_start_trace',+ result.hasPageIdRouting ? {pageId} : undefined,);- assert.ok(calls[1].name === 'performance_start_trace');},};diff --git a/scripts/eval_scenarios/select_page_test.ts b/scripts/eval_scenarios/select_page_test.tsindex a8bfba32..65d83558 100644--- a/scripts/eval_scenarios/select_page_test.ts+++ b/scripts/eval_scenarios/select_page_test.ts@@ -18,23 +18,10 @@ export const scenario: TestScenario = {<h1>test</h1>`,},- expectations: calls => {- assert.strictEqual(calls.length, 3);- assert.ok(calls[0].name === 'new_page', 'First call should be navigation');- assert.ok(calls[1].name === 'new_page', 'Second call should be navigation');- assert.ok(- calls[2].name === 'select_page',- 'Third call should be select_page',- );- assert.strictEqual(- calls[2].args.pageId,- 2,- 'PageId has to be set to 2. about:blank is 1, <TEST_URL> is 2, https://developers.chrome.com is 3.',- );- assert.strictEqual(- calls[2].args.bringToFront,- undefined,- 'bringToFront should use the default value.',- );+ expectations: result => {+ result.consumePageNavigation();+ assert.strictEqual(result.remainingCalls.length, 2);+ result.assertNextCall('new_page');+ result.assertNextCall('select_page', {pageId: 2, bringToFront: undefined});},};diff --git a/scripts/eval_scenarios/snapshot_test.ts b/scripts/eval_scenarios/snapshot_test.tsindex 4877c1d1..027e808f 100644--- a/scripts/eval_scenarios/snapshot_test.ts+++ b/scripts/eval_scenarios/snapshot_test.ts@@ -10,16 +10,17 @@ import type {TestScenario} from '../eval_gemini.ts';export const scenario: TestScenario = {prompt: 'Read the content of <TEST_URL>',- maxTurns: 3,+ maxTurns: 4,htmlRoute: {path: '/test.html',htmlContent: '<h1>Hello World</h1><p>This is a test.</p>',},- expectations: calls => {- assert.strictEqual(calls.length, 2);- assert.ok(- calls[0].name === 'navigate_page' || calls[0].name === 'new_page',+ expectations: result => {+ const pageId = result.consumePageNavigation();+ assert.strictEqual(result.remainingCalls.length, 1);+ result.assertNextCall(+ 'take_snapshot',+ result.hasPageIdRouting ? {pageId} : undefined,);- assert.ok(calls[1].name === 'take_snapshot');},};