Chrome DevTools for coding agents
chore: support an html route in evals (#770)
Alex Rudenko0284d18b7b4b
diff --git a/scripts/eval_gemini.ts b/scripts/eval_gemini.tsindex 8163dc64..c099399c 100644--- a/scripts/eval_gemini.ts+++ b/scripts/eval_gemini.ts@@ -6,7 +6,7 @@import fs from 'node:fs';import path from 'node:path';-import {describe, test} from 'node:test';+import {describe, test, before, after, afterEach} from 'node:test';import {GoogleGenerativeAI,@@ -18,6 +18,7 @@ import {StdioClientTransport} from '@modelcontextprotocol/sdk/client/stdio.js';const ROOT_DIR = path.resolve(import.meta.dirname, '..');const SCENARIOS_DIR = path.join(import.meta.dirname, 'eval_scenarios');+import {TestServer} from '../build/tests/server.js';// Define schema for our test scenariosexport interface CapturedFunctionCall {@@ -29,6 +30,10 @@ export interface TestScenario {prompt: string;maxTurns: number;expectations: (calls: CapturedFunctionCall[]) => void;+ htmlRoute?: {+ path: string;+ htmlContent: string;+ };}async function loadScenario(scenarioPath: string): Promise<TestScenario> {@@ -78,6 +83,7 @@ const cleanSchemaRecursive = (schema: unknown): unknown => {async function runSingleScenario(scenarioPath: string,apiKey: string,+ server: TestServer,): Promise<void> {const absolutePath = path.resolve(scenarioPath);console.log(`\n### Running Scenario: ${absolutePath} ###`);@@ -88,6 +94,17 @@ async function runSingleScenario(try {const scenario = await loadScenario(absolutePath);+ if (scenario.htmlRoute) {+ server.addHtmlRoute(+ scenario.htmlRoute.path,+ scenario.htmlRoute.htmlContent,+ );+ scenario.prompt = scenario.prompt.replace(+ '<TEST_URL>',+ server.getRoute(scenario.htmlRoute.path),+ );+ }+// Path to the compiled MCP serverconst serverPath = path.join(ROOT_DIR, 'build/src/index.js');if (!fs.existsSync(serverPath)) {@@ -148,7 +165,7 @@ async function runSingleScenario(const genAI = new GoogleGenerativeAI(apiKey);const model = genAI.getGenerativeModel({- model: 'gemini-3-pro-preview',+ model: 'gemini-2.5-flash',tools: [{functionDeclarations}],});@@ -167,7 +184,9 @@ async function runSingleScenario(console.log(`\n--- Turn 1 (User) ---`);console.log(scenario.prompt);- let result = await chat.sendMessage(scenario.prompt);+ let result = await chat.sendMessage(scenario.prompt, {+ timeout: 5000,+ });let response = result.response;while (turnCount < scenario.maxTurns) {@@ -256,13 +275,27 @@ if (!apiKey) {}void describe('Gemini Eval Scenarios', () => {+ const server = new TestServer(TestServer.randomPort());++ before(async () => {+ await server.start();+ });++ after(async () => {+ await server.stop();+ });++ afterEach(() => {+ server.restore();+ });+const files = fs.readdirSync(SCENARIOS_DIR).filter(file => {return file.endsWith('.ts') || file.endsWith('.js');});for (const file of files) {- void test(file, async () => {- await runSingleScenario(path.join(SCENARIOS_DIR, file), apiKey);+ void test(file, {timeout: 60_000}, async () => {+ await runSingleScenario(path.join(SCENARIOS_DIR, file), apiKey, server);});}});diff --git a/scripts/eval_scenarios/snapshot_test.ts b/scripts/eval_scenarios/snapshot_test.tsnew file mode 100644index 00000000..c5f43ff5--- /dev/null+++ b/scripts/eval_scenarios/snapshot_test.ts@@ -0,0 +1,24 @@+/**+ * @license+ * Copyright 2026 Google LLC+ * SPDX-License-Identifier: Apache-2.0+ */+import assert from 'node:assert';++import type {TestScenario} from '../eval_gemini.ts';++export const scenario: TestScenario = {+ prompt: 'Read the content of <TEST_URL>',+ maxTurns: 3,+ htmlRoute: {+ path: '/test.html',+ htmlContent: '<h1>Hello World</h1><p>This is a test.</p>',+ },+ expectations: calls => {+ assert.strictEqual(calls.length, 2);+ assert.ok(+ calls[0].name === 'navigate_page' || calls[0].name === 'new_page',+ );+ assert.ok(calls[1].name === 'take_snapshot');+ },+};diff --git a/tests/server.ts b/tests/server.tsindex fb6fcd5b..7278861a 100644--- a/tests/server.ts+++ b/tests/server.ts@@ -13,7 +13,7 @@ import {before, after, afterEach} from 'node:test';import {html} from './utils.js';-class TestServer {+export class TestServer {#port: number;#server: Server;