Skip to content

Playwright is a framework for Web Testing and Automation. It allows testing Chromium, Firefox and WebKit with a single API.

test(mcp): introduce eval tests (#38318)

Dmitry Gozman0a1d764126bb
diff --git a/package.json b/package.json
index e5289cf10b..0eb8f8930f 100644
--- a/package.json
+++ b/package.json
@@ -30,7 +30,7 @@
"test": "playwright test --config=tests/library/playwright.config.ts",
"test-mcp": "playwright test --config=tests/mcp/playwright.config.ts",
"ctest-mcp": "playwright test --config=tests/mcp/playwright.config.ts --project=chrome",
- "etest-mcp": "playwright test --config=tests/mcp/playwright.config.extension.ts",
+ "eval-mcp": "playwright test --config=tests/mcp/playwright.config.ts --project=eval",
"eslint": "eslint --cache",
"tsc": "tsc -p . && tsc -p packages/html-reporter/",
"doc": "node utils/doclint/cli.js",
diff --git a/packages/playwright/src/agents/agent.ts b/packages/playwright/src/agents/agent.ts
index 923a674ae3..6510ffb134 100644
--- a/packages/playwright/src/agents/agent.ts
+++ b/packages/playwright/src/agents/agent.ts
@@ -52,8 +52,9 @@ export class Agent<T extends z.ZodSchema<any>> {
try {
return await this.loop.run<z.output<T>>(`${prompt}\n\nTask:\n${task}\n\nParams:\n${JSON.stringify(params, null, 2)}`, {
...options,
- tools,
- callTool,
+ // TODO: fix types in tiny-loop
+ tools: tools as any,
+ callTool: callTool as any,
resultSchema: this.resultSchema
});
} finally {
diff --git a/packages/playwright/src/agents/performTask.ts b/packages/playwright/src/agents/performTask.ts
index df11ef6378..d425b5508e 100644
--- a/packages/playwright/src/agents/performTask.ts
+++ b/packages/playwright/src/agents/performTask.ts
@@ -36,8 +36,9 @@ export async function performTask(context: playwright.BrowserContext, task: stri
try {
return await loop.run(task, {
- tools: await backend.listTools(),
- callTool,
+ // TODO: fix types in tiny-loop
+ tools: await backend.listTools() as any,
+ callTool: callTool as any,
logger,
});
} finally {
diff --git a/tests/assets/evals/fill-form-submit.html b/tests/assets/evals/fill-form-submit.html
new file mode 100644
index 0000000000..a115d6e941
--- /dev/null
+++ b/tests/assets/evals/fill-form-submit.html
@@ -0,0 +1,369 @@
+<!DOCTYPE html>
+<html lang="en">
+<head>
+ <meta charset="UTF-8">
+ <meta name="viewport" content="width=device-width, initial-scale=1.0">
+ <title>Order Confirmation</title>
+ <style>
+ * {
+ margin: 0;
+ padding: 0;
+ box-sizing: border-box;
+ }
+
+ body {
+ font-family: 'Segoe UI', Tahoma, Geneva, Verdana, sans-serif;
+ background: linear-gradient(135deg, #667eea 0%, #764ba2 100%);
+ padding: 20px;
+ min-height: 100vh;
+ display: flex;
+ justify-content: center;
+ align-items: center;
+ }
+
+ .container {
+ background: white;
+ border-radius: 10px;
+ box-shadow: 0 10px 40px rgba(0, 0, 0, 0.2);
+ padding: 40px;
+ max-width: 700px;
+ width: 100%;
+ }
+
+ .success-icon {
+ text-align: center;
+ margin-bottom: 20px;
+ }
+
+ .success-icon svg {
+ width: 80px;
+ height: 80px;
+ stroke: #4CAF50;
+ stroke-width: 2;
+ stroke-linecap: round;
+ stroke-linejoin: round;
+ fill: none;
+ }
+
+ h1 {
+ color: #333;
+ margin-bottom: 10px;
+ text-align: center;
+ font-size: 28px;
+ }
+
+ .subtitle {
+ text-align: center;
+ color: #666;
+ margin-bottom: 30px;
+ font-size: 16px;
+ }
+
+ .order-details {
+ background: #f8f9fa;
+ border-radius: 8px;
+ padding: 25px;
+ margin-bottom: 25px;
+ }
+
+ .detail-section {
+ margin-bottom: 20px;
+ }
+
+ .detail-section:last-child {
+ margin-bottom: 0;
+ }
+
+ .section-title {
+ font-size: 14px;
+ font-weight: 600;
+ color: #888;
+ text-transform: uppercase;
+ margin-bottom: 10px;
+ letter-spacing: 0.5px;
+ }
+
+ .detail-row {
+ display: flex;
+ justify-content: space-between;
+ padding: 8px 0;
+ border-bottom: 1px solid #e0e0e0;
+ }
+
+ .detail-row:last-child {
+ border-bottom: none;
+ }
+
+ .detail-label {
+ color: #555;
+ font-weight: 500;
+ }
+
+ .detail-value {
+ color: #333;
+ font-weight: 400;
+ text-align: right;
+ }
+
+ .price-section {
+ background: linear-gradient(135deg, #667eea 0%, #764ba2 100%);
+ color: white;
+ border-radius: 8px;
+ padding: 20px;
+ margin-bottom: 25px;
+ }
+
+ .price-row {
+ display: flex;
+ justify-content: space-between;
+ padding: 8px 0;
+ }
+
+ .total-price {
+ font-size: 32px;
+ font-weight: 700;
+ margin-top: 10px;
+ }
+
+ .delivery-section {
+ background: #e3f2fd;
+ border-left: 4px solid #2196F3;
+ padding: 20px;
+ border-radius: 5px;
+ margin-bottom: 25px;
+ }
+
+ .delivery-section h3 {
+ color: #1976D2;
+ margin-bottom: 10px;
+ font-size: 18px;
+ }
+
+ .delivery-info {
+ color: #555;
+ line-height: 1.6;
+ }
+
+ .delivery-days {
+ font-size: 24px;
+ font-weight: 700;
+ color: #1976D2;
+ }
+
+ .btn-back {
+ width: 100%;
+ padding: 15px;
+ background: #e0e0e0;
+ color: #333;
+ border: none;
+ border-radius: 5px;
+ font-size: 16px;
+ font-weight: 600;
+ cursor: pointer;
+ transition: all 0.2s;
+ }
+
+ .btn-back:hover {
+ background: #d0d0d0;
+ }
+
+ .error-message {
+ background: #ffebee;
+ color: #c62828;
+ padding: 20px;
+ border-radius: 8px;
+ border-left: 4px solid #c62828;
+ text-align: center;
+ }
+ </style>
+</head>
+<body>
+ <div class="container" id="content">
+ <!-- Content will be generated by JavaScript -->
+ </div>
+
+ <script>
+ // Product prices in USD
+ const productPrices = {
+ 'table': 299.99,
+ 'chair': 89.99,
+ 'sofa': 699.99,
+ 'desk': 249.99,
+ 'cabinet': 399.99
+ };
+
+ // Calculate delivery days based on ZIP code
+ function calculateDeliveryDays(zipCode) {
+ if (!zipCode) return 7;
+
+ // Simple logic: use first digit of ZIP to determine delivery zone
+ const firstDigit = parseInt(zipCode.toString().charAt(0));
+
+ // ZIP codes 0-2: East Coast
+ // ZIP codes 3-5: Central
+ // ZIP codes 6-9: West Coast
+
+ if (firstDigit >= 0 && firstDigit <= 2) {
+ return 3;
+ } else if (firstDigit >= 3 && firstDigit <= 5) {
+ return 5;
+ } else if (firstDigit >= 6 && firstDigit <= 9) {
+ return 7;
+ }
+
+ return 7; // Default
+ }
+
+ // Get URL parameters
+ function getUrlParams() {
+ // Handle both URL search params and hash params
+ let queryString = window.location.search;
+ if (!queryString && window.location.hash) {
+ // Extract query string from hash (format: #?param=value)
+ queryString = window.location.hash.substring(2); // Remove '#?'
+ }
+ const params = new URLSearchParams(queryString);
+ return {
+ productType: params.get('productType'),
+ color: params.get('color'),
+ name: params.get('name'),
+ email: params.get('email'),
+ address: params.get('address'),
+ city: params.get('city'),
+ state: params.get('state'),
+ zipCode: params.get('zipCode')
+ };
+ }
+
+ // Format currency
+ function formatCurrency(amount) {
+ return new Intl.NumberFormat('en-US', {
+ style: 'currency',
+ currency: 'USD'
+ }).format(amount);
+ }
+
+ // Capitalize first letter
+ function capitalize(str) {
+ if (!str) return '';
+ return str.charAt(0).toUpperCase() + str.slice(1);
+ }
+
+ // Generate confirmation page
+ function generateConfirmation() {
+ const params = getUrlParams();
+ const container = document.getElementById('content');
+
+ // Check if we have required parameters
+ if (!params.productType || !params.name) {
+ container.innerHTML = `
+ <div class="error-message">
+ <h2>Order Information Missing</h2>
+ <p>Unable to process order confirmation. Please return to the order form.</p>
+ <button class="btn-back" onclick="window.history.back()">Go Back</button>
+ </div>
+ `;
+ return;
+ }
+
+ // Calculate price and delivery
+ const basePrice = productPrices[params.productType] || 0;
+ const tax = basePrice * 0.08; // 8% tax
+ const shipping = 29.99;
+ const totalPrice = basePrice + tax + shipping;
+ const deliveryDays = calculateDeliveryDays(params.zipCode);
+
+ // Generate HTML
+ container.innerHTML = `
+ <div class="success-icon">
+ <svg viewBox="0 0 52 52">
+ <circle cx="26" cy="26" r="25"/>
+ <path d="M14.1 27.2l7.1 7.2 16.7-16.8"/>
+ </svg>
+ </div>
+
+ <h1>Order Confirmed!</h1>
+ <p class="subtitle">Thank you for your purchase, ${params.name}</p>
+
+ <div class="order-details">
+ <div class="detail-section">
+ <div class="section-title">Product Details</div>
+ <div class="detail-row">
+ <span class="detail-label">Product:</span>
+ <span class="detail-value">${capitalize(params.productType)}</span>
+ </div>
+ <div class="detail-row">
+ <span class="detail-label">Color:</span>
+ <span class="detail-value">${capitalize(params.color)}</span>
+ </div>
+ </div>
+
+ <div class="detail-section">
+ <div class="section-title">Customer Information</div>
+ <div class="detail-row">
+ <span class="detail-label">Name:</span>
+ <span class="detail-value">${params.name}</span>
+ </div>
+ <div class="detail-row">
+ <span class="detail-label">Email:</span>
+ <span class="detail-value">${params.email}</span>
+ </div>
+ </div>
+
+ <div class="detail-section">
+ <div class="section-title">Shipping Address</div>
+ <div class="detail-row">
+ <span class="detail-label">Address:</span>
+ <span class="detail-value">${params.address}</span>
+ </div>
+ <div class="detail-row">
+ <span class="detail-label">City:</span>
+ <span class="detail-value">${params.city}</span>
+ </div>
+ <div class="detail-row">
+ <span class="detail-label">State:</span>
+ <span class="detail-value">${params.state}</span>
+ </div>
+ <div class="detail-row">
+ <span class="detail-label">ZIP Code:</span>
+ <span class="detail-value">${params.zipCode}</span>
+ </div>
+ </div>
+ </div>
+
+ <div class="price-section">
+ <div class="price-row">
+ <span>Product Price:</span>
+ <span>${formatCurrency(basePrice)}</span>
+ </div>
+ <div class="price-row">
+ <span>Tax (8%):</span>
+ <span>${formatCurrency(tax)}</span>
+ </div>
+ <div class="price-row">
+ <span>Shipping:</span>
+ <span>${formatCurrency(shipping)}</span>
+ </div>
+ <div class="price-row total-price">
+ <span>Total:</span>
+ <span>${formatCurrency(totalPrice)}</span>
+ </div>
+ </div>
+
+ <div class="delivery-section">
+ <h3>📦 Estimated Delivery</h3>
+ <div class="delivery-info">
+ <p>Your order will arrive in approximately <span class="delivery-days">${deliveryDays} days</span></p>
+ <p style="margin-top: 10px; font-size: 14px;">We'll send tracking information to ${params.email} once your order ships.</p>
+ </div>
+ </div>
+
+ <button class="btn-back" onclick="window.location.href='./fill-form.html'">Place Another Order</button>
+ `;
+ }
+
+ // Initialize page
+ document.addEventListener('DOMContentLoaded', generateConfirmation);
+ </script>
+</body>
+</html>
diff --git a/tests/assets/evals/fill-form.html b/tests/assets/evals/fill-form.html
new file mode 100644
index 0000000000..02de222e1e
--- /dev/null
+++ b/tests/assets/evals/fill-form.html
@@ -0,0 +1,186 @@
+<head>
+ <meta charset="UTF-8">
+ <meta name="viewport" content="width=device-width, initial-scale=1.0">
+ <title>Order Form</title>
+ <style>
+ * {
+ margin: 0;
+ padding: 0;
+ box-sizing: border-box;
+ }
+
+ body {
+ font-family: 'Segoe UI', Tahoma, Geneva, Verdana, sans-serif;
+ background: linear-gradient(135deg, #667eea 0%, #764ba2 100%);
+ padding: 20px;
+ min-height: 100vh;
+ display: flex;
+ justify-content: center;
+ align-items: center;
+ }
+
+ .container {
+ background: white;
+ border-radius: 10px;
+ box-shadow: 0 10px 40px rgba(0, 0, 0, 0.2);
+ padding: 40px;
+ max-width: 600px;
+ width: 100%;
+ }
+
+ h1 {
+ color: #333;
+ margin-bottom: 30px;
+ text-align: center;
+ font-size: 28px;
+ }
+
+ .form-group {
+ margin-bottom: 20px;
+ }
+
+ label {
+ display: block;
+ margin-bottom: 8px;
+ color: #555;
+ font-weight: 600;
+ font-size: 14px;
+ }
+
+ input[type="text"],
+ input[type="email"],
+ select,
+ textarea {
+ width: 100%;
+ padding: 12px;
+ border: 2px solid #e0e0e0;
+ border-radius: 5px;
+ font-size: 14px;
+ transition: border-color 0.3s;
+ font-family: inherit;
+ }
+
+ input[type="text"]:focus,
+ input[type="email"]:focus,
+ select:focus,
+ textarea:focus {
+ outline: none;
+ border-color: #667eea;
+ }
+
+ textarea {
+ resize: vertical;
+ min-height: 80px;
+ }
+
+ .btn-order {
+ width: 100%;
+ padding: 15px;
+ background: linear-gradient(135deg, #667eea 0%, #764ba2 100%);
+ color: white;
+ border: none;
+ border-radius: 5px;
+ font-size: 16px;
+ font-weight: 600;
+ cursor: pointer;
+ transition: transform 0.2s, box-shadow 0.2s;
+ margin-top: 10px;
+ }
+
+ .btn-order:hover {
+ transform: translateY(-2px);
+ box-shadow: 0 5px 20px rgba(102, 126, 234, 0.4);
+ }
+
+ .btn-order:active {
+ transform: translateY(0);
+ }
+
+ </style>
+</head>
+<body>
+ <div class="container">
+ <h1>Product Order Form</h1>
+ <form id="orderForm">
+
+ <div class="form-group">
+ <label for="productType">Product Type *</label>
+ <select id="productType" name="productType" required>
+ <option value="">Select a product</option>
+ <option value="chair">Chair</option>
+ <option value="sofa">Sofa</option>
+ <option value="desk">Desk</option>
+ <option value="table">Table</option>
+ <option value="cabinet">Cabinet</option>
+ </select>
+ </div>
+
+ <div class="form-group">
+ <label for="color">Color *</label>
+ <select id="color" name="color" required>
+ <option value="">Select a color</option>
+ <option value="red">Red</option>
+ <option value="green">Green</option>
+ <option value="blue">Blue</option>
+ <option value="black">Black</option>
+ <option value="white">White</option>
+ <option value="brown">Brown</option>
+ </select>
+ </div>
+
+ <div class="form-group">
+ <label for="name">Full Name *</label>
+ <input type="text" id="name" name="name" placeholder="Enter your full name" required>
+ </div>
+
+ <div class="form-group">
+ <label for="email">Email Address *</label>
+ <input type="email" id="email" name="email" placeholder="[email protected]" required>
+ </div>
+
+ <div class="form-group">
+ <label for="address">Street Address *</label>
+ <input type="text" id="address" name="address" placeholder="123 Main Street" required>
+ </div>
+
+ <div class="form-group">
+ <label for="city">City *</label>
+ <input type="text" id="city" name="city" placeholder="City" required>
+ </div>
+
+ <div class="form-group">
+ <label for="state">State/Province *</label>
+ <input type="text" id="state" name="state" placeholder="State" required>
+ </div>
+
+ <div class="form-group">
+ <label for="zipCode">ZIP/Postal Code *</label>
+ <input type="text" id="zipCode" name="zipCode" placeholder="12345" required>
+ </div>
+
+ <button type="button" class="btn-order">Order</button>
+ </form>
+ </div>
+
+ <script>
+ const form = document.getElementById('orderForm');
+
+ const submit = () => {
+ location.href = './fill-form-submit.html#?' + new URLSearchParams(new FormData(form)).toString();
+ };
+
+ document.querySelector('.btn-order').addEventListener('click', () => {
+ if (form.checkValidity()) {
+ if (window.confirm(`Please confirm your order for a ${form.color.value} ${form.productType.value}.`))
+ submit();
+ } else {
+ form.reportValidity();
+ }
+ });
+
+ form.addEventListener('submit', e => {
+ e.preventDefault();
+ submit();
+ });
+ </script>
+</body>
diff --git a/tests/mcp/__cache__/copilot/mcp-browser-eval-fill-the-form-repeat0.json b/tests/mcp/__cache__/copilot/mcp-browser-eval-fill-the-form-repeat0.json
new file mode 100644
index 0000000000..8b4e8b1238
--- /dev/null
+++ b/tests/mcp/__cache__/copilot/mcp-browser-eval-fill-the-form-repeat0.json
@@ -0,0 +1,330 @@
+{
+ "facb950333d3cbebad4cb469c846c6a03a72dce8": {
+ "result": {
+ "role": "assistant",
+ "content": "I'll help you navigate to the form, fill it out with the specified details, and report back the total price and delivery information.",
+ "toolCalls": []
+ },
+ "usage": {
+ "input": 4484,
+ "output": 132
+ }
+ },
+ "a8e0596eb3a8ab52837b1544d1c328f0886e26a1": {
+ "result": {
+ "role": "assistant",
+ "content": "Navigate to the form page",
+ "toolCalls": [
+ {
+ "name": "browser_navigate",
+ "arguments": {
+ "url": "http://localhost:<PORT>/evals/fill-form.html"
+ },
+ "id": "tooluse_GPuE3K9kQwSXLSPV2UAyEw"
+ }
+ ]
+ },
+ "usage": {
+ "input": 4535,
+ "output": 88
+ }
+ },
+ "bbdb8dff944cf47be23aa17fa55a678a29f5cd4a": {
+ "result": {
+ "role": "assistant",
+ "content": "Capture snapshot to see the form",
+ "toolCalls": []
+ },
+ "usage": {
+ "input": 5232,
+ "output": 69
+ }
+ },
+ "3b8c1b6c21376d9a91c21a9e33a35439a3990c54": {
+ "result": {
+ "role": "assistant",
+ "content": "I'll fill out the form for the first item (blue table)",
+ "toolCalls": []
+ },
+ "usage": {
+ "input": 5263,
+ "output": 130
+ }
+ },
+ "40b838f47364c51fe3512a4fc385fa16763d53a4": {
+ "result": {
+ "role": "assistant",
+ "content": "Fill form for first item - blue table",
+ "toolCalls": [
+ {
+ "name": "browser_fill_form",
+ "arguments": {
+ "fields": [
+ {
+ "name": "Product Type",
+ "ref": "e7",
+ "type": "combobox",
+ "value": "Table"
+ },
+ {
+ "name": "Color",
+ "ref": "e10",
+ "type": "combobox",
+ "value": "Blue"
+ },
+ {
+ "name": "Full Name",
+ "ref": "e13",
+ "type": "textbox",
+ "value": "John Doe"
+ },
+ {
+ "name": "Email Address",
+ "ref": "e16",
+ "type": "textbox",
+ "value": "[email protected]"
+ },
+ {
+ "name": "Street Address",
+ "ref": "e19",
+ "type": "textbox",
+ "value": "123 Main St"
+ },
+ {
+ "name": "City",
+ "ref": "e22",
+ "type": "textbox",
+ "value": "Anytown"
+ },
+ {
+ "name": "State/Province",
+ "ref": "e25",
+ "type": "textbox",
+ "value": "XYZ"
+ },
+ {
+ "name": "ZIP/Postal Code",
+ "ref": "e28",
+ "type": "textbox",
+ "value": "12345"
+ }
+ ]
+ },
+ "id": "tooluse_9iiHR97qRZWzf8YVVbkHmA"
+ }
+ ]
+ },
+ "usage": {
+ "input": 5301,
+ "output": 331
+ }
+ },
+ "30b7c82f8a226b17b040f846903656852e7a6e36": {
+ "result": {
+ "role": "assistant",
+ "content": "Click the Order button for the first item",
+ "toolCalls": []
+ },
+ "usage": {
+ "input": 5830,
+ "output": 81
+ }
+ },
+ "d4c841c1aa8400cc2aeaf0a4e4db2c4df625a320": {
+ "result": {
+ "role": "assistant",
+ "content": "Click the Order button",
+ "toolCalls": []
+ },
+ "usage": {
+ "input": 5862,
+ "output": 77
+ }
+ },
+ "51ee9f20ae47d1660b4ed9dbdaf6d44b4b951339": {
+ "result": {
+ "role": "assistant",
+ "content": "",
+ "toolCalls": [
+ {
+ "name": "browser_click",
+ "arguments": {
+ "element": "Order button",
+ "ref": "e29"
+ },
+ "id": "tooluse_WtpFqw9JScur-lAHZEsKRQ"
+ }
+ ]
+ },
+ "usage": {
+ "input": 5890,
+ "output": 72
+ }
+ },
+ "4493b8b9ef4b1cfaff25033b937f9a3bc81de116": {
+ "result": {
+ "role": "assistant",
+ "content": "",
+ "toolCalls": [
+ {
+ "name": "browser_handle_dialog",
+ "arguments": {
+ "accept": true
+ },
+ "id": "tooluse_oUddkbggQ3aEEYKh8BU8wg"
+ }
+ ]
+ },
+ "usage": {
+ "input": 6042,
+ "output": 55
+ }
+ },
+ "890e35cf92b660a3fa3ade73f150a929caf92a1d": {
+ "result": {
+ "role": "assistant",
+ "content": "",
+ "toolCalls": [
+ {
+ "name": "browser_click",
+ "arguments": {
+ "element": "Place Another Order button",
+ "ref": "e58"
+ },
+ "id": "tooluse_AlBQSRtuQT2Yot3LPY5F-Q"
+ }
+ ]
+ },
+ "usage": {
+ "input": 6903,
+ "output": 74
+ }
+ },
+ "9bee0bcd86f45f9622412b4c01bc0fa871054aa3": {
+ "result": {
+ "role": "assistant",
+ "content": "",
+ "toolCalls": [
+ {
+ "name": "browser_fill_form",
+ "arguments": {
+ "fields": [
+ {
+ "name": "Product Type",
+ "ref": "e7",
+ "type": "combobox",
+ "value": "Desk"
+ },
+ {
+ "name": "Color",
+ "ref": "e10",
+ "type": "combobox",
+ "value": "Green"
+ },
+ {
+ "name": "Full Name",
+ "ref": "e13",
+ "type": "textbox",
+ "value": "John Doe"
+ },
+ {
+ "name": "Email Address",
+ "ref": "e16",
+ "type": "textbox",
+ "value": "[email protected]"
+ },
+ {
+ "name": "Street Address",
+ "ref": "e19",
+ "type": "textbox",
+ "value": "123 Main St"
+ },
+ {
+ "name": "City",
+ "ref": "e22",
+ "type": "textbox",
+ "value": "Anytown"
+ },
+ {
+ "name": "State/Province",
+ "ref": "e25",
+ "type": "textbox",
+ "value": "XYZ"
+ },
+ {
+ "name": "ZIP/Postal Code",
+ "ref": "e28",
+ "type": "textbox",
+ "value": "12345"
+ }
+ ]
+ },
+ "id": "tooluse_YGgfR5fAR6OJcd7Q9M4t7w"
+ }
+ ]
+ },
+ "usage": {
+ "input": 7601,
+ "output": 298
+ }
+ },
+ "faae385281a92076a48da71d391fa0e477894e6d": {
+ "result": {
+ "role": "assistant",
+ "content": "",
+ "toolCalls": [
+ {
+ "name": "browser_click",
+ "arguments": {
+ "element": "Order button",
+ "ref": "e29"
+ },
+ "id": "tooluse_qo1cC8s9SB-sPPPgTk36sA"
+ }
+ ]
+ },
+ "usage": {
+ "input": 8123,
+ "output": 72
+ }
+ },
+ "eaab8ebfa4b63c74ba919e849ba983fcb874528f": {
+ "result": {
+ "role": "assistant",
+ "content": "",
+ "toolCalls": [
+ {
+ "name": "browser_handle_dialog",
+ "arguments": {
+ "accept": true
+ },
+ "id": "tooluse_EWFnsSPoTY6i4Qf7bpEwjg"
+ }
+ ]
+ },
+ "usage": {
+ "input": 8274,
+ "output": 55
+ }
+ },
+ "2c31a0c54acc5a4c4f9b8e4c7acdbf94a81eebbb": {
+ "result": {
+ "role": "assistant",
+ "content": "",
+ "toolCalls": [
+ {
+ "name": "report_result",
+ "arguments": {
+ "deliveryDays": 3,
+ "price": 653.96
+ },
+ "id": "tooluse_I1b-G92TS7qIuUsLtl020w"
+ }
+ ]
+ },
+ "usage": {
+ "input": 9136,
+ "output": 75
+ }
+ }
+}
\ No newline at end of file
diff --git a/tests/mcp/__cache__/copilot/mcp-browser-eval-fill-the-form-repeat1.json b/tests/mcp/__cache__/copilot/mcp-browser-eval-fill-the-form-repeat1.json
new file mode 100644
index 0000000000..d17add266f
--- /dev/null
+++ b/tests/mcp/__cache__/copilot/mcp-browser-eval-fill-the-form-repeat1.json
@@ -0,0 +1,492 @@
+{
+ "facb950333d3cbebad4cb469c846c6a03a72dce8": {
+ "result": {
+ "role": "assistant",
+ "content": "I'll help you navigate to the form, fill it out with the specified details, and complete the order. Let me start by navigating to the page.",
+ "toolCalls": []
+ },
+ "usage": {
+ "input": 4484,
+ "output": 126
+ }
+ },
+ "f65dd22f0f79ffd6398d42230797a4a53c72c9ab": {
+ "result": {
+ "role": "assistant",
+ "content": "Navigate to the form page to start the ordering process",
+ "toolCalls": [
+ {
+ "name": "browser_navigate",
+ "arguments": {
+ "url": "http://localhost:<PORT>/evals/fill-form.html"
+ },
+ "id": "tooluse_BCc-AvDnQWmc8O81Stj2YQ"
+ }
+ ]
+ },
+ "usage": {
+ "input": 4540,
+ "output": 93
+ }
+ },
+ "eb63c3301417583c538a6501c36797d42e161a1a": {
+ "result": {
+ "role": "assistant",
+ "content": "Take a snapshot to see the current page state",
+ "toolCalls": [
+ {
+ "name": "browser_snapshot",
+ "arguments": {},
+ "id": "tooluse_EQPPTsKBT_G_YsVMHNZnAw"
+ }
+ ]
+ },
+ "usage": {
+ "input": 5242,
+ "output": 61
+ }
+ },
+ "190598029fe374115a87b39e2d631689f712f8c5": {
+ "result": {
+ "role": "assistant",
+ "content": "Fill out the form for the first item - blue table",
+ "toolCalls": []
+ },
+ "usage": {
+ "input": 5878,
+ "output": 350
+ }
+ },
+ "73845baea403b9e42331b19667593fbf028c3429": {
+ "result": {
+ "role": "assistant",
+ "content": "Select Table as the product type",
+ "toolCalls": [
+ {
+ "name": "browser_select_option",
+ "arguments": {
+ "element": "Product Type dropdown",
+ "ref": "e7",
+ "values": [
+ "Table"
+ ]
+ },
+ "id": "tooluse_WVHv7CI3R-WqyOj6O7LTKg"
+ }
+ ]
+ },
+ "usage": {
+ "input": 5913,
+ "output": 116
+ }
+ },
+ "5cac5e0e4742f5674f0c4c7fd933d7e4c84f0513": {
+ "result": {
+ "role": "assistant",
+ "content": "Select Blue as the color",
+ "toolCalls": []
+ },
+ "usage": {
+ "input": 6165,
+ "output": 99
+ }
+ },
+ "67a472d6a9e6e9c965eaa82a70a4bdbb7485eca6": {
+ "result": {
+ "role": "assistant",
+ "content": "",
+ "toolCalls": [
+ {
+ "name": "browser_select_option",
+ "arguments": {
+ "element": "Color dropdown",
+ "ref": "e10",
+ "values": [
+ "Blue"
+ ]
+ },
+ "id": "tooluse_f86G0fVES260pD2OGjYdCA"
+ }
+ ]
+ },
+ "usage": {
+ "input": 6194,
+ "output": 93
+ }
+ },
+ "65f822bec8f70205e0d58e9a2ee09a3707ed6401": {
+ "result": {
+ "role": "assistant",
+ "content": "Fill in customer information for the order",
+ "toolCalls": [
+ {
+ "name": "browser_fill_form",
+ "arguments": {
+ "fields": [
+ {
+ "name": "Full Name",
+ "ref": "e13",
+ "type": "textbox",
+ "value": "John Doe"
+ },
+ {
+ "name": "Email Address",
+ "ref": "e16",
+ "type": "textbox",
+ "value": "[email protected]"
+ },
+ {
+ "name": "Street Address",
+ "ref": "e19",
+ "type": "textbox",
+ "value": "123 Main St"
+ },
+ {
+ "name": "City",
+ "ref": "e22",
+ "type": "textbox",
+ "value": "Anytown"
+ },
+ {
+ "name": "State/Province",
+ "ref": "e25",
+ "type": "textbox",
+ "value": "XYZ"
+ },
+ {
+ "name": "ZIP/Postal Code",
+ "ref": "e28",
+ "type": "textbox",
+ "value": "12345"
+ }
+ ]
+ },
+ "id": "tooluse_ztCsxsUlTueuOPgHsO83dA"
+ }
+ ]
+ },
+ "usage": {
+ "input": 6441,
+ "output": 343
+ }
+ },
+ "6b1f789361769bb90eb776cff6df695394e90ff5": {
+ "result": {
+ "role": "assistant",
+ "content": "Click the Order button to submit the first order",
+ "toolCalls": []
+ },
+ "usage": {
+ "input": 6879,
+ "output": 82
+ }
+ },
+ "f036e1c5e71f07be6729fc6fc0ef5a7514564633": {
+ "result": {
+ "role": "assistant",
+ "content": "",
+ "toolCalls": [
+ {
+ "name": "browser_click",
+ "arguments": {
+ "element": "Order button",
+ "ref": "e29"
+ },
+ "id": "tooluse_7Bw_du6XShCs0VM46yds3A"
+ }
+ ]
+ },
+ "usage": {
+ "input": 6912,
+ "output": 72
+ }
+ },
+ "44e579cf6dfa27d84426d8afbffda3bb20e978b5": {
+ "result": {
+ "role": "assistant",
+ "content": "Accept the order confirmation dialog",
+ "toolCalls": []
+ },
+ "usage": {
+ "input": 7064,
+ "output": 61
+ }
+ },
+ "b1043d04c5761a4273d3c6d358e18727ae53bedd": {
+ "result": {
+ "role": "assistant",
+ "content": "",
+ "toolCalls": [
+ {
+ "name": "browser_handle_dialog",
+ "arguments": {
+ "accept": true
+ },
+ "id": "tooluse_CBKRGvaMT8-0bxI3LmpgLQ"
+ }
+ ]
+ },
+ "usage": {
+ "input": 7093,
+ "output": 55
+ }
+ },
+ "4a20e94c802689f0e5795cc01d0754593c95b8d9": {
+ "result": {
+ "role": "assistant",
+ "content": "Click Place Another Order button to order the second item",
+ "toolCalls": []
+ },
+ "usage": {
+ "input": 7955,
+ "output": 85
+ }
+ },
+ "02d0f1acdf58e14bfacf4fb3d1a04973726b3f6a": {
+ "result": {
+ "role": "assistant",
+ "content": "",
+ "toolCalls": [
+ {
+ "name": "browser_click",
+ "arguments": {
+ "element": "Place Another Order button",
+ "ref": "e58"
+ },
+ "id": "tooluse_mEVX8xrBQ-u7BPjnZ7PPKQ"
+ }
+ ]
+ },
+ "usage": {
+ "input": 7989,
+ "output": 74
+ }
+ },
+ "19504162e60fbdd96072af88bd669b3e88860524": {
+ "result": {
+ "role": "assistant",
+ "content": "Select Desk as the product type for the second order",
+ "toolCalls": []
+ },
+ "usage": {
+ "input": 8688,
+ "output": 107
+ }
+ },
+ "302dcb0c44750d90fdf69073db4a6533d657af9c": {
+ "result": {
+ "role": "assistant",
+ "content": "",
+ "toolCalls": [
+ {
+ "name": "browser_select_option",
+ "arguments": {
+ "element": "Product Type dropdown",
+ "ref": "e7",
+ "values": [
+ "Desk"
+ ]
+ },
+ "id": "tooluse_BaMRqzC-QbeyPBC-9RP0aw"
+ }
+ ]
+ },
+ "usage": {
+ "input": 8723,
+ "output": 95
+ }
+ },
+ "b40f9a90caa41102d3b795a22a005ad9042b659e": {
+ "result": {
+ "role": "assistant",
+ "content": "Select Green as the color for the desk",
+ "toolCalls": []
+ },
+ "usage": {
+ "input": 8971,
+ "output": 11
+ }
+ },
+ "9c502c32a6f3797f941b683c19473c89cec2a1b4": {
+ "result": {
+ "role": "assistant",
+ "content": "",
+ "toolCalls": [
+ {
+ "name": "browser_select_option",
+ "arguments": {
+ "element": "Color dropdown",
+ "ref": "e10",
+ "values": [
+ "Green"
+ ]
+ },
+ "id": "tooluse_bvxZZXldSu28CR0zYOoIng"
+ }
+ ]
+ },
+ "usage": {
+ "input": 9003,
+ "output": 93
+ }
+ },
+ "ce3d6682dda9339c3efd384e5bd23e9aef4852fc": {
+ "result": {
+ "role": "assistant",
+ "content": "Fill in the customer information for the second order",
+ "toolCalls": []
+ },
+ "usage": {
+ "input": 9250,
+ "output": 250
+ }
+ },
+ "92858cba54b40fc65bc94e7f1ffd16f427f3f1b0": {
+ "result": {
+ "role": "assistant",
+ "content": "",
+ "toolCalls": [
+ {
+ "name": "browser_fill_form",
+ "arguments": {
+ "fields": [
+ {
+ "name": "Full Name",
+ "ref": "e13",
+ "type": "textbox",
+ "value": "John Doe"
+ },
+ {
+ "name": "Email Address",
+ "ref": "e16",
+ "type": "textbox",
+ "value": "[email protected]"
+ },
+ {
+ "name": "Street Address",
+ "ref": "e19",
+ "type": "textbox",
+ "value": "123 Main St"
+ },
+ {
+ "name": "City",
+ "ref": "e22",
+ "type": "textbox",
+ "value": "Anytown"
+ },
+ {
+ "name": "State/Province",
+ "ref": "e25",
+ "type": "textbox",
+ "value": "XYZ"
+ },
+ {
+ "name": "ZIP/Postal Code",
+ "ref": "e28",
+ "type": "textbox",
+ "value": "12345"
+ }
+ ]
+ },
+ "id": "tooluse_MiTMyTOaRoCmK0EapkxU7g"
+ }
+ ]
+ },
+ "usage": {
+ "input": 9283,
+ "output": 240
+ }
+ },
+ "3c11d68c47519b7364a7dd5c1be26d6b7cd16a66": {
+ "result": {
+ "role": "assistant",
+ "content": "Click the Order button to submit the second order",
+ "toolCalls": []
+ },
+ "usage": {
+ "input": 9714,
+ "output": 12
+ }
+ },
+ "334effd07d305facef120ed49d47eca9546b9ed2": {
+ "result": {
+ "role": "assistant",
+ "content": "",
+ "toolCalls": [
+ {
+ "name": "browser_click",
+ "arguments": {
+ "element": "Order button",
+ "ref": "e29"
+ },
+ "id": "tooluse_jana5W7FSLiGM40Y58vxuA"
+ }
+ ]
+ },
+ "usage": {
+ "input": 9747,
+ "output": 72
+ }
+ },
+ "3e70d82f485d3bb257053e605a888e391d8458f3": {
+ "result": {
+ "role": "assistant",
+ "content": "Accept the order confirmation dialog for the second order",
+ "toolCalls": []
+ },
+ "usage": {
+ "input": 9899,
+ "output": 12
+ }
+ },
+ "f6df99806ce1301aedf58fd2ebfc77b3645f849b": {
+ "result": {
+ "role": "assistant",
+ "content": "",
+ "toolCalls": [
+ {
+ "name": "browser_handle_dialog",
+ "arguments": {
+ "accept": true
+ },
+ "id": "tooluse_j3MT0SvtSBuucN6YXXxfZg"
+ }
+ ]
+ },
+ "usage": {
+ "input": 9932,
+ "output": 55
+ }
+ },
+ "c6d1b1ddc05aa572b48a0abfebaf84117d38aefd": {
+ "result": {
+ "role": "assistant",
+ "content": "Report the total price and maximum estimated delivery time for both orders",
+ "toolCalls": []
+ },
+ "usage": {
+ "input": 10795,
+ "output": 88
+ }
+ },
+ "b2b70eb2a30bcd06f1cd7819b9a7d4e275a528b6": {
+ "result": {
+ "role": "assistant",
+ "content": "",
+ "toolCalls": [
+ {
+ "name": "report_result",
+ "arguments": {
+ "deliveryDays": 3,
+ "price": 653.96
+ },
+ "id": "tooluse_CR5rFd09QRGnH5-Lt3-pGw"
+ }
+ ]
+ },
+ "usage": {
+ "input": 10831,
+ "output": 75
+ }
+ }
+}
\ No newline at end of file
diff --git a/tests/mcp/__cache__/copilot/mcp-browser-eval-fill-the-form-repeat2.json b/tests/mcp/__cache__/copilot/mcp-browser-eval-fill-the-form-repeat2.json
new file mode 100644
index 0000000000..d936c0aac0
--- /dev/null
+++ b/tests/mcp/__cache__/copilot/mcp-browser-eval-fill-the-form-repeat2.json
@@ -0,0 +1,404 @@
+{
+ "facb950333d3cbebad4cb469c846c6a03a72dce8": {
+ "result": {
+ "role": "assistant",
+ "content": "I'll help you navigate to the form, fill it out with the specified details, and report back the results. Let me start by navigating to the page.",
+ "toolCalls": []
+ },
+ "usage": {
+ "input": 4484,
+ "output": 136
+ }
+ },
+ "e9daf936478b73139df8f3ea61bc82a1a13c3cdb": {
+ "result": {
+ "role": "assistant",
+ "content": "Navigate to the form page",
+ "toolCalls": [
+ {
+ "name": "browser_navigate",
+ "arguments": {
+ "url": "http://localhost:<PORT>/evals/fill-form.html"
+ },
+ "id": "tooluse_2QTlFMmcQlqV0a1rVultZw"
+ }
+ ]
+ },
+ "usage": {
+ "input": 4541,
+ "output": 88
+ }
+ },
+ "7dc1192eb36584ccdef322c804036a775f23f7a2": {
+ "result": {
+ "role": "assistant",
+ "content": "Check the current state of the form before filling",
+ "toolCalls": [
+ {
+ "name": "browser_snapshot",
+ "arguments": {},
+ "id": "tooluse_Xq-IYmyLTua6t1KRVizOew"
+ }
+ ]
+ },
+ "usage": {
+ "input": 5238,
+ "output": 61
+ }
+ },
+ "e960fd42b63a75629d10e2b064a712cf4bb222c8": {
+ "result": {
+ "role": "assistant",
+ "content": "Fill the first order form for a blue table",
+ "toolCalls": []
+ },
+ "usage": {
+ "input": 5874,
+ "output": 126
+ }
+ },
+ "ab8368051e5eb0335afc2627359409ff6e527b22": {
+ "result": {
+ "role": "assistant",
+ "content": "Select Table as the product type for the first order",
+ "toolCalls": [
+ {
+ "name": "browser_select_option",
+ "arguments": {
+ "element": "Product Type dropdown",
+ "ref": "e7",
+ "values": [
+ "Table"
+ ]
+ },
+ "id": "tooluse_qDJgPasgTZazcKmtzotHZg"
+ }
+ ]
+ },
+ "usage": {
+ "input": 5907,
+ "output": 120
+ }
+ },
+ "eefa023ed6d14b88656c764ec83bf98bee091d0a": {
+ "result": {
+ "role": "assistant",
+ "content": "Select Blue as the color",
+ "toolCalls": []
+ },
+ "usage": {
+ "input": 6163,
+ "output": 99
+ }
+ },
+ "c7ddf8898251a2d2a7fcadf36e226a14f2d32955": {
+ "result": {
+ "role": "assistant",
+ "content": "",
+ "toolCalls": [
+ {
+ "name": "browser_select_option",
+ "arguments": {
+ "element": "Color dropdown",
+ "ref": "e10",
+ "values": [
+ "Blue"
+ ]
+ },
+ "id": "tooluse_K_TcaZuDTaaPDZT-qhxH0g"
+ }
+ ]
+ },
+ "usage": {
+ "input": 6192,
+ "output": 93
+ }
+ },
+ "5ccb74547a127df0533ca011b1ac02fb9df840c3": {
+ "result": {
+ "role": "assistant",
+ "content": "Fill in all the customer details in the form",
+ "toolCalls": [
+ {
+ "name": "browser_fill_form",
+ "arguments": {
+ "fields": [
+ {
+ "name": "Full Name",
+ "ref": "e13",
+ "type": "textbox",
+ "value": "John Doe"
+ },
+ {
+ "name": "Email Address",
+ "ref": "e16",
+ "type": "textbox",
+ "value": "[email protected]"
+ },
+ {
+ "name": "Street Address",
+ "ref": "e19",
+ "type": "textbox",
+ "value": "123 Main St"
+ },
+ {
+ "name": "City",
+ "ref": "e22",
+ "type": "textbox",
+ "value": "Anytown"
+ },
+ {
+ "name": "State/Province",
+ "ref": "e25",
+ "type": "textbox",
+ "value": "XYZ"
+ },
+ {
+ "name": "ZIP/Postal Code",
+ "ref": "e28",
+ "type": "textbox",
+ "value": "12345"
+ }
+ ]
+ },
+ "id": "tooluse_fevjVnF0S2OhEUg_pdQNog"
+ }
+ ]
+ },
+ "usage": {
+ "input": 6439,
+ "output": 345
+ }
+ },
+ "1ef5b1de2508c7962e6e4b4dcefe59c11179680d": {
+ "result": {
+ "role": "assistant",
+ "content": "Submit the order form",
+ "toolCalls": []
+ },
+ "usage": {
+ "input": 6879,
+ "output": 77
+ }
+ },
+ "f096f02c459d6fa500047ad6ba5e65c0e5222e20": {
+ "result": {
+ "role": "assistant",
+ "content": "",
+ "toolCalls": [
+ {
+ "name": "browser_click",
+ "arguments": {
+ "element": "Order button",
+ "ref": "e29"
+ },
+ "id": "tooluse_cDuf7VYITsOk3SPizGkdbw"
+ }
+ ]
+ },
+ "usage": {
+ "input": 6907,
+ "output": 72
+ }
+ },
+ "a28b9ead7a286733ddbf6c52fd0c3ed1d5954725": {
+ "result": {
+ "role": "assistant",
+ "content": "",
+ "toolCalls": [
+ {
+ "name": "browser_handle_dialog",
+ "arguments": {
+ "accept": true
+ },
+ "id": "tooluse_A0g51QouRsWL9CsOTx5Ycg"
+ }
+ ]
+ },
+ "usage": {
+ "input": 7059,
+ "output": 55
+ }
+ },
+ "fb6f4514207d7e27fa9a5c6da79320dd6d5ab572": {
+ "result": {
+ "role": "assistant",
+ "content": "",
+ "toolCalls": [
+ {
+ "name": "browser_click",
+ "arguments": {
+ "element": "Place Another Order button",
+ "ref": "e58"
+ },
+ "id": "tooluse_4U7nwg7ZTMiom22WRl-FJw"
+ }
+ ]
+ },
+ "usage": {
+ "input": 7920,
+ "output": 74
+ }
+ },
+ "4a73f42e2d9bdb78c9a2e5390bc7a964fbb343c0": {
+ "result": {
+ "role": "assistant",
+ "content": "",
+ "toolCalls": [
+ {
+ "name": "browser_select_option",
+ "arguments": {
+ "element": "Product Type dropdown",
+ "ref": "e7",
+ "values": [
+ "Desk"
+ ]
+ },
+ "id": "tooluse_67fzJrlgS06obb4sPHxk-Q"
+ }
+ ]
+ },
+ "usage": {
+ "input": 8618,
+ "output": 95
+ }
+ },
+ "cf8bc3f7c495171c5541706d297582791b7c316f": {
+ "result": {
+ "role": "assistant",
+ "content": "",
+ "toolCalls": [
+ {
+ "name": "browser_select_option",
+ "arguments": {
+ "element": "Color dropdown",
+ "ref": "e10",
+ "values": [
+ "Green"
+ ]
+ },
+ "id": "tooluse_CroDBxgYS1akDERsrWsRAg"
+ }
+ ]
+ },
+ "usage": {
+ "input": 8865,
+ "output": 93
+ }
+ },
+ "995ae7de03de31d0deaa9f9c415b682386ea0290": {
+ "result": {
+ "role": "assistant",
+ "content": "",
+ "toolCalls": [
+ {
+ "name": "browser_fill_form",
+ "arguments": {
+ "fields": [
+ {
+ "name": "Full Name",
+ "ref": "e13",
+ "type": "textbox",
+ "value": "John Doe"
+ },
+ {
+ "name": "Email Address",
+ "ref": "e16",
+ "type": "textbox",
+ "value": "[email protected]"
+ },
+ {
+ "name": "Street Address",
+ "ref": "e19",
+ "type": "textbox",
+ "value": "123 Main St"
+ },
+ {
+ "name": "City",
+ "ref": "e22",
+ "type": "textbox",
+ "value": "Anytown"
+ },
+ {
+ "name": "State/Province",
+ "ref": "e25",
+ "type": "textbox",
+ "value": "XYZ"
+ },
+ {
+ "name": "ZIP/Postal Code",
+ "ref": "e28",
+ "type": "textbox",
+ "value": "12345"
+ }
+ ]
+ },
+ "id": "tooluse_DLFD9lwrQb2Oc_4NpnYOYg"
+ }
+ ]
+ },
+ "usage": {
+ "input": 9111,
+ "output": 240
+ }
+ },
+ "6ec90af600847bd4dd2a0fa6b8ea5c08fd0e59d0": {
+ "result": {
+ "role": "assistant",
+ "content": "",
+ "toolCalls": [
+ {
+ "name": "browser_click",
+ "arguments": {
+ "element": "Order button",
+ "ref": "e29"
+ },
+ "id": "tooluse_oNWDYmu7RTClOEuCTrn7Sg"
+ }
+ ]
+ },
+ "usage": {
+ "input": 9541,
+ "output": 72
+ }
+ },
+ "c46467b8f90fdfbfdf5081dda99e2055b6f6152f": {
+ "result": {
+ "role": "assistant",
+ "content": "",
+ "toolCalls": [
+ {
+ "name": "browser_handle_dialog",
+ "arguments": {
+ "accept": true
+ },
+ "id": "tooluse_qBV8FnNVTMiBZ8oLuUVYOQ"
+ }
+ ]
+ },
+ "usage": {
+ "input": 9692,
+ "output": 55
+ }
+ },
+ "c9ebdba51d456673a83fd1ec4e484fea864ef5b7": {
+ "result": {
+ "role": "assistant",
+ "content": "",
+ "toolCalls": [
+ {
+ "name": "report_result",
+ "arguments": {
+ "deliveryDays": 3,
+ "price": 653.96
+ },
+ "id": "tooluse_Tsf2nC_CR6CMtZL91D0ovA"
+ }
+ ]
+ },
+ "usage": {
+ "input": 10554,
+ "output": 75
+ }
+ }
+}
\ No newline at end of file
diff --git a/tests/mcp/browser.eval.ts b/tests/mcp/browser.eval.ts
new file mode 100644
index 0000000000..c44bf25e92
--- /dev/null
+++ b/tests/mcp/browser.eval.ts
@@ -0,0 +1,45 @@
+/**
+ * Copyright (c) Microsoft Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import z from 'zod';
+import { zodToJsonSchema } from 'zod-to-json-schema';
+
+import { test, expect } from './fixtures';
+
+import type * as tinyLoop from 'tiny-loop';
+
+test.use({ mcpHeadless: async ({ headless }, use) => use(headless) });
+
+test('fill the form', async ({ loop, client, server }) => {
+ expect(await loop.run<{ price: number, deliveryDays: number }>(`
+ Navigate to ${server.PREFIX + '/evals/fill-form.html'} via Playwright MCP.
+ Order a blue table and a green desk.
+ Use the following details for the order form:
+ - Name: John Doe
+ - Address: 123 Main St, Anytown, XYZ state
+ - Zip Code: 12345
+ Report back total price for both items, and maximum estimated delivery.
+ Do not take screenshots.
+ `, {
+ tools: (await client.listTools()).tools as tinyLoop.Tool[],
+ callTool: (params => client.callTool(params)) as tinyLoop.ToolCallback,
+ resultSchema: zodToJsonSchema(z.object({
+ price: z.number().describe('Total price in USD.'),
+ deliveryDays: z.number().describe('Maximum estimated delivery time in days.'),
+ })) as tinyLoop.Schema,
+ })).toEqual({ price: 653.96, deliveryDays: 3 });
+});
diff --git a/tests/mcp/fixtures.ts b/tests/mcp/fixtures.ts
index 9f55b0c643..ba92eaa84d 100644
--- a/tests/mcp/fixtures.ts
+++ b/tests/mcp/fixtures.ts
@@ -17,6 +17,8 @@
import fs from 'fs';
import path from 'path';
import { chromium } from 'playwright';
+import { Loop } from 'tiny-loop';
+import debug from 'debug';
import { test as baseTest, expect as baseExpect } from '@playwright/test';
import { StdioClientTransport } from '@modelcontextprotocol/sdk/client/stdio.js';
@@ -65,6 +67,7 @@ type TestFixtures = {
server: TestServer;
httpsServer: TestServer;
mcpHeadless: boolean;
+ loop: Loop;
};
type WorkerFixtures = {
@@ -184,6 +187,28 @@ export const test = serverTest.extend<TestFixtures & TestOptions, WorkerFixtures
mcpBrowser: ['chrome', { option: true }],
mcpServerType: ['mcp', { option: true }],
+
+ loop: async ({ server }, use) => {
+ const provider = 'copilot';
+ const cacheFile = path.join(__dirname, '__cache__', provider, sanitizeFileName(test.info().titlePath.join(' ') + '-repeat' + test.info().repeatEachIndex) + '.json');
+ const dataBefore = await fs.promises.readFile(cacheFile, 'utf-8').catch(() => '{}');
+ let cache = {};
+ try {
+ cache = JSON.parse(dataBefore);
+ } catch {
+ }
+ const caches = { before: cache, after: {} };
+ await use(new Loop(provider, {
+ caches,
+ secrets: { PORT: String(server.PORT) },
+ logger: (category, ...args) => debug(category)(...args),
+ }));
+ const dataAfter = JSON.stringify(caches.after, null, 2);
+ if (test.info().status === 'passed' && dataBefore !== dataAfter) {
+ await fs.promises.mkdir(path.dirname(cacheFile), { recursive: true });
+ await fs.promises.writeFile(cacheFile, dataAfter);
+ }
+ },
});
async function createTransport(mcpServerType: TestOptions['mcpServerType'], options: { args: string[], env: NodeJS.ProcessEnv, cwd: string }): Promise<{
@@ -312,3 +337,7 @@ export async function prepareDebugTest(startClient: StartClient, testFile?: stri
const [, id] = listResult.content[0].text.match(/\[id=([^\]]+)\]/);
return { client, id };
}
+
+function sanitizeFileName(name: string): string {
+ return name.replace('.ts', '').replace(/[^a-zA-Z0-9_]+/g, '-');
+}
diff --git a/tests/mcp/playwright.config.extension.ts b/tests/mcp/playwright.config.extension.ts
deleted file mode 100644
index 526fda2001..0000000000
--- a/tests/mcp/playwright.config.extension.ts
+++ /dev/null
@@ -1,32 +0,0 @@
-/**
- * Copyright (c) Microsoft Corporation.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-import { defineConfig } from '@playwright/test';
-
-import type { TestOptions } from './fixtures';
-
-export default defineConfig<TestOptions>({
- testDir: './',
- grep: /extension/,
- fullyParallel: true,
- forbidOnly: !!process.env.CI,
- retries: process.env.CI ? 2 : 0,
- workers: process.env.CI ? 1 : undefined,
- reporter: 'list',
- projects: [
- { name: 'chromium', use: { mcpBrowser: 'chromium' } },
- ],
-});
diff --git a/tests/mcp/playwright.config.ts b/tests/mcp/playwright.config.ts
index 11ab165bf8..965b9bb67f 100644
--- a/tests/mcp/playwright.config.ts
+++ b/tests/mcp/playwright.config.ts
@@ -14,12 +14,15 @@
* limitations under the License.
*/
import * as path from 'path';
+import dotenv from 'dotenv';
import { defineConfig } from '@playwright/test';
import type { TestOptions } from './fixtures';
import type { ReporterDescription } from '@playwright/test';
+dotenv.config({ path: path.resolve(__dirname, '../../.env'), quiet: true });
+
const rootTestDir = path.join(__dirname, '..');
const testDir = path.join(rootTestDir, 'mcp');
const outputDir = path.join(__dirname, '..', '..', 'test-results');
@@ -44,7 +47,6 @@ const metadata = {
export default defineConfig<TestOptions>({
testDir: rootTestDir,
- grepInvert: /extension/,
fullyParallel: true,
forbidOnly: !!process.env.CI,
workers: process.env.CI ? 2 : undefined,
@@ -55,5 +57,14 @@ export default defineConfig<TestOptions>({
{ name: 'firefox', use: { mcpBrowser: 'firefox' }, metadata: { ...metadata, browserName: 'firefox' }, testDir },
{ name: 'webkit', use: { mcpBrowser: 'webkit' }, metadata: { ...metadata, browserName: 'webkit' }, testDir },
... process.platform === 'win32' ? [{ name: 'msedge', use: { mcpBrowser: 'msedge' }, metadata: { ...metadata, browserName: 'chromium', channel: 'msedge' }, testDir }] : [],
+ {
+ name: 'eval',
+ testDir,
+ testMatch: /.*\.eval\.ts/,
+ repeatEach: 3, // Generate three different trajectories per test.
+ timeout: process.env.CI ? 30_000 : 180_000, // CI should be cache-only, but local use of LLM is slow.
+ use: { mcpBrowser: 'chromium' },
+ metadata: { ...metadata, browserName: 'chromium' },
+ }
],
});