Skip to content

Chrome DevTools for coding agents

chore: support an html route in evals (#770)

Alex Rudenko0284d18b7b4b
diff --git a/scripts/eval_gemini.ts b/scripts/eval_gemini.ts
index 8163dc64..c099399c 100644
--- a/scripts/eval_gemini.ts
+++ b/scripts/eval_gemini.ts
@@ -6,7 +6,7 @@
import fs from 'node:fs';
import path from 'node:path';
-import {describe, test} from 'node:test';
+import {describe, test, before, after, afterEach} from 'node:test';
import {
GoogleGenerativeAI,
@@ -18,6 +18,7 @@ import {StdioClientTransport} from '@modelcontextprotocol/sdk/client/stdio.js';
const ROOT_DIR = path.resolve(import.meta.dirname, '..');
const SCENARIOS_DIR = path.join(import.meta.dirname, 'eval_scenarios');
+import {TestServer} from '../build/tests/server.js';
// Define schema for our test scenarios
export interface CapturedFunctionCall {
@@ -29,6 +30,10 @@ export interface TestScenario {
prompt: string;
maxTurns: number;
expectations: (calls: CapturedFunctionCall[]) => void;
+ htmlRoute?: {
+ path: string;
+ htmlContent: string;
+ };
}
async function loadScenario(scenarioPath: string): Promise<TestScenario> {
@@ -78,6 +83,7 @@ const cleanSchemaRecursive = (schema: unknown): unknown => {
async function runSingleScenario(
scenarioPath: string,
apiKey: string,
+ server: TestServer,
): Promise<void> {
const absolutePath = path.resolve(scenarioPath);
console.log(`\n### Running Scenario: ${absolutePath} ###`);
@@ -88,6 +94,17 @@ async function runSingleScenario(
try {
const scenario = await loadScenario(absolutePath);
+ if (scenario.htmlRoute) {
+ server.addHtmlRoute(
+ scenario.htmlRoute.path,
+ scenario.htmlRoute.htmlContent,
+ );
+ scenario.prompt = scenario.prompt.replace(
+ '<TEST_URL>',
+ server.getRoute(scenario.htmlRoute.path),
+ );
+ }
+
// Path to the compiled MCP server
const serverPath = path.join(ROOT_DIR, 'build/src/index.js');
if (!fs.existsSync(serverPath)) {
@@ -148,7 +165,7 @@ async function runSingleScenario(
const genAI = new GoogleGenerativeAI(apiKey);
const model = genAI.getGenerativeModel({
- model: 'gemini-3-pro-preview',
+ model: 'gemini-2.5-flash',
tools: [{functionDeclarations}],
});
@@ -167,7 +184,9 @@ async function runSingleScenario(
console.log(`\n--- Turn 1 (User) ---`);
console.log(scenario.prompt);
- let result = await chat.sendMessage(scenario.prompt);
+ let result = await chat.sendMessage(scenario.prompt, {
+ timeout: 5000,
+ });
let response = result.response;
while (turnCount < scenario.maxTurns) {
@@ -256,13 +275,27 @@ if (!apiKey) {
}
void describe('Gemini Eval Scenarios', () => {
+ const server = new TestServer(TestServer.randomPort());
+
+ before(async () => {
+ await server.start();
+ });
+
+ after(async () => {
+ await server.stop();
+ });
+
+ afterEach(() => {
+ server.restore();
+ });
+
const files = fs.readdirSync(SCENARIOS_DIR).filter(file => {
return file.endsWith('.ts') || file.endsWith('.js');
});
for (const file of files) {
- void test(file, async () => {
- await runSingleScenario(path.join(SCENARIOS_DIR, file), apiKey);
+ void test(file, {timeout: 60_000}, async () => {
+ await runSingleScenario(path.join(SCENARIOS_DIR, file), apiKey, server);
});
}
});
diff --git a/scripts/eval_scenarios/snapshot_test.ts b/scripts/eval_scenarios/snapshot_test.ts
new file mode 100644
index 00000000..c5f43ff5
--- /dev/null
+++ b/scripts/eval_scenarios/snapshot_test.ts
@@ -0,0 +1,24 @@
+/**
+ * @license
+ * Copyright 2026 Google LLC
+ * SPDX-License-Identifier: Apache-2.0
+ */
+import assert from 'node:assert';
+
+import type {TestScenario} from '../eval_gemini.ts';
+
+export const scenario: TestScenario = {
+ prompt: 'Read the content of <TEST_URL>',
+ maxTurns: 3,
+ htmlRoute: {
+ path: '/test.html',
+ htmlContent: '<h1>Hello World</h1><p>This is a test.</p>',
+ },
+ expectations: calls => {
+ assert.strictEqual(calls.length, 2);
+ assert.ok(
+ calls[0].name === 'navigate_page' || calls[0].name === 'new_page',
+ );
+ assert.ok(calls[1].name === 'take_snapshot');
+ },
+};
diff --git a/tests/server.ts b/tests/server.ts
index fb6fcd5b..7278861a 100644
--- a/tests/server.ts
+++ b/tests/server.ts
@@ -13,7 +13,7 @@ import {before, after, afterEach} from 'node:test';
import {html} from './utils.js';
-class TestServer {
+export class TestServer {
#port: number;
#server: Server;