Playwright is a framework for Web Testing and Automation. It allows testing Chromium, Firefox and WebKit with a single API.
TypeScript automationchromechromiume2e-testingelectronend-to-end-testingfirefoxjavascriptplaywrighttesttest-automationtestingtesting-toolswebwebkit
test(mcp): introduce eval tests (#38318)
Dmitry Gozman0a1d764126bb
diff --git a/package.json b/package.jsonindex e5289cf10b..0eb8f8930f 100644--- a/package.json+++ b/package.json@@ -30,7 +30,7 @@"test": "playwright test --config=tests/library/playwright.config.ts","test-mcp": "playwright test --config=tests/mcp/playwright.config.ts","ctest-mcp": "playwright test --config=tests/mcp/playwright.config.ts --project=chrome",- "etest-mcp": "playwright test --config=tests/mcp/playwright.config.extension.ts",+ "eval-mcp": "playwright test --config=tests/mcp/playwright.config.ts --project=eval","eslint": "eslint --cache","tsc": "tsc -p . && tsc -p packages/html-reporter/","doc": "node utils/doclint/cli.js",diff --git a/packages/playwright/src/agents/agent.ts b/packages/playwright/src/agents/agent.tsindex 923a674ae3..6510ffb134 100644--- a/packages/playwright/src/agents/agent.ts+++ b/packages/playwright/src/agents/agent.ts@@ -52,8 +52,9 @@ export class Agent<T extends z.ZodSchema<any>> {try {return await this.loop.run<z.output<T>>(`${prompt}\n\nTask:\n${task}\n\nParams:\n${JSON.stringify(params, null, 2)}`, {...options,- tools,- callTool,+ // TODO: fix types in tiny-loop+ tools: tools as any,+ callTool: callTool as any,resultSchema: this.resultSchema});} finally {diff --git a/packages/playwright/src/agents/performTask.ts b/packages/playwright/src/agents/performTask.tsindex df11ef6378..d425b5508e 100644--- a/packages/playwright/src/agents/performTask.ts+++ b/packages/playwright/src/agents/performTask.ts@@ -36,8 +36,9 @@ export async function performTask(context: playwright.BrowserContext, task: stritry {return await loop.run(task, {- tools: await backend.listTools(),- callTool,+ // TODO: fix types in tiny-loop+ tools: await backend.listTools() as any,+ callTool: callTool as any,logger,});} finally {diff --git a/tests/assets/evals/fill-form-submit.html b/tests/assets/evals/fill-form-submit.htmlnew file mode 100644index 0000000000..a115d6e941--- /dev/null+++ b/tests/assets/evals/fill-form-submit.html@@ -0,0 +1,369 @@+<!DOCTYPE html>+<html lang="en">+<head>+ <meta charset="UTF-8">+ <meta name="viewport" content="width=device-width, initial-scale=1.0">+ <title>Order Confirmation</title>+ <style>+ * {+ margin: 0;+ padding: 0;+ box-sizing: border-box;+ }++ body {+ font-family: 'Segoe UI', Tahoma, Geneva, Verdana, sans-serif;+ background: linear-gradient(135deg, #667eea 0%, #764ba2 100%);+ padding: 20px;+ min-height: 100vh;+ display: flex;+ justify-content: center;+ align-items: center;+ }++ .container {+ background: white;+ border-radius: 10px;+ box-shadow: 0 10px 40px rgba(0, 0, 0, 0.2);+ padding: 40px;+ max-width: 700px;+ width: 100%;+ }++ .success-icon {+ text-align: center;+ margin-bottom: 20px;+ }++ .success-icon svg {+ width: 80px;+ height: 80px;+ stroke: #4CAF50;+ stroke-width: 2;+ stroke-linecap: round;+ stroke-linejoin: round;+ fill: none;+ }++ h1 {+ color: #333;+ margin-bottom: 10px;+ text-align: center;+ font-size: 28px;+ }++ .subtitle {+ text-align: center;+ color: #666;+ margin-bottom: 30px;+ font-size: 16px;+ }++ .order-details {+ background: #f8f9fa;+ border-radius: 8px;+ padding: 25px;+ margin-bottom: 25px;+ }++ .detail-section {+ margin-bottom: 20px;+ }++ .detail-section:last-child {+ margin-bottom: 0;+ }++ .section-title {+ font-size: 14px;+ font-weight: 600;+ color: #888;+ text-transform: uppercase;+ margin-bottom: 10px;+ letter-spacing: 0.5px;+ }++ .detail-row {+ display: flex;+ justify-content: space-between;+ padding: 8px 0;+ border-bottom: 1px solid #e0e0e0;+ }++ .detail-row:last-child {+ border-bottom: none;+ }++ .detail-label {+ color: #555;+ font-weight: 500;+ }++ .detail-value {+ color: #333;+ font-weight: 400;+ text-align: right;+ }++ .price-section {+ background: linear-gradient(135deg, #667eea 0%, #764ba2 100%);+ color: white;+ border-radius: 8px;+ padding: 20px;+ margin-bottom: 25px;+ }++ .price-row {+ display: flex;+ justify-content: space-between;+ padding: 8px 0;+ }++ .total-price {+ font-size: 32px;+ font-weight: 700;+ margin-top: 10px;+ }++ .delivery-section {+ background: #e3f2fd;+ border-left: 4px solid #2196F3;+ padding: 20px;+ border-radius: 5px;+ margin-bottom: 25px;+ }++ .delivery-section h3 {+ color: #1976D2;+ margin-bottom: 10px;+ font-size: 18px;+ }++ .delivery-info {+ color: #555;+ line-height: 1.6;+ }++ .delivery-days {+ font-size: 24px;+ font-weight: 700;+ color: #1976D2;+ }++ .btn-back {+ width: 100%;+ padding: 15px;+ background: #e0e0e0;+ color: #333;+ border: none;+ border-radius: 5px;+ font-size: 16px;+ font-weight: 600;+ cursor: pointer;+ transition: all 0.2s;+ }++ .btn-back:hover {+ background: #d0d0d0;+ }++ .error-message {+ background: #ffebee;+ color: #c62828;+ padding: 20px;+ border-radius: 8px;+ border-left: 4px solid #c62828;+ text-align: center;+ }+ </style>+</head>+<body>+ <div class="container" id="content">+ <!-- Content will be generated by JavaScript -->+ </div>++ <script>+ // Product prices in USD+ const productPrices = {+ 'table': 299.99,+ 'chair': 89.99,+ 'sofa': 699.99,+ 'desk': 249.99,+ 'cabinet': 399.99+ };++ // Calculate delivery days based on ZIP code+ function calculateDeliveryDays(zipCode) {+ if (!zipCode) return 7;++ // Simple logic: use first digit of ZIP to determine delivery zone+ const firstDigit = parseInt(zipCode.toString().charAt(0));++ // ZIP codes 0-2: East Coast+ // ZIP codes 3-5: Central+ // ZIP codes 6-9: West Coast++ if (firstDigit >= 0 && firstDigit <= 2) {+ return 3;+ } else if (firstDigit >= 3 && firstDigit <= 5) {+ return 5;+ } else if (firstDigit >= 6 && firstDigit <= 9) {+ return 7;+ }++ return 7; // Default+ }++ // Get URL parameters+ function getUrlParams() {+ // Handle both URL search params and hash params+ let queryString = window.location.search;+ if (!queryString && window.location.hash) {+ // Extract query string from hash (format: #?param=value)+ queryString = window.location.hash.substring(2); // Remove '#?'+ }+ const params = new URLSearchParams(queryString);+ return {+ productType: params.get('productType'),+ color: params.get('color'),+ name: params.get('name'),+ email: params.get('email'),+ address: params.get('address'),+ city: params.get('city'),+ state: params.get('state'),+ zipCode: params.get('zipCode')+ };+ }++ // Format currency+ function formatCurrency(amount) {+ return new Intl.NumberFormat('en-US', {+ style: 'currency',+ currency: 'USD'+ }).format(amount);+ }++ // Capitalize first letter+ function capitalize(str) {+ if (!str) return '';+ return str.charAt(0).toUpperCase() + str.slice(1);+ }++ // Generate confirmation page+ function generateConfirmation() {+ const params = getUrlParams();+ const container = document.getElementById('content');++ // Check if we have required parameters+ if (!params.productType || !params.name) {+ container.innerHTML = `+ <div class="error-message">+ <h2>Order Information Missing</h2>+ <p>Unable to process order confirmation. Please return to the order form.</p>+ <button class="btn-back" onclick="window.history.back()">Go Back</button>+ </div>+ `;+ return;+ }++ // Calculate price and delivery+ const basePrice = productPrices[params.productType] || 0;+ const tax = basePrice * 0.08; // 8% tax+ const shipping = 29.99;+ const totalPrice = basePrice + tax + shipping;+ const deliveryDays = calculateDeliveryDays(params.zipCode);++ // Generate HTML+ container.innerHTML = `+ <div class="success-icon">+ <svg viewBox="0 0 52 52">+ <circle cx="26" cy="26" r="25"/>+ <path d="M14.1 27.2l7.1 7.2 16.7-16.8"/>+ </svg>+ </div>++ <h1>Order Confirmed!</h1>+ <p class="subtitle">Thank you for your purchase, ${params.name}</p>++ <div class="order-details">+ <div class="detail-section">+ <div class="section-title">Product Details</div>+ <div class="detail-row">+ <span class="detail-label">Product:</span>+ <span class="detail-value">${capitalize(params.productType)}</span>+ </div>+ <div class="detail-row">+ <span class="detail-label">Color:</span>+ <span class="detail-value">${capitalize(params.color)}</span>+ </div>+ </div>++ <div class="detail-section">+ <div class="section-title">Customer Information</div>+ <div class="detail-row">+ <span class="detail-label">Name:</span>+ <span class="detail-value">${params.name}</span>+ </div>+ <div class="detail-row">+ <span class="detail-label">Email:</span>+ <span class="detail-value">${params.email}</span>+ </div>+ </div>++ <div class="detail-section">+ <div class="section-title">Shipping Address</div>+ <div class="detail-row">+ <span class="detail-label">Address:</span>+ <span class="detail-value">${params.address}</span>+ </div>+ <div class="detail-row">+ <span class="detail-label">City:</span>+ <span class="detail-value">${params.city}</span>+ </div>+ <div class="detail-row">+ <span class="detail-label">State:</span>+ <span class="detail-value">${params.state}</span>+ </div>+ <div class="detail-row">+ <span class="detail-label">ZIP Code:</span>+ <span class="detail-value">${params.zipCode}</span>+ </div>+ </div>+ </div>++ <div class="price-section">+ <div class="price-row">+ <span>Product Price:</span>+ <span>${formatCurrency(basePrice)}</span>+ </div>+ <div class="price-row">+ <span>Tax (8%):</span>+ <span>${formatCurrency(tax)}</span>+ </div>+ <div class="price-row">+ <span>Shipping:</span>+ <span>${formatCurrency(shipping)}</span>+ </div>+ <div class="price-row total-price">+ <span>Total:</span>+ <span>${formatCurrency(totalPrice)}</span>+ </div>+ </div>++ <div class="delivery-section">+ <h3>📦 Estimated Delivery</h3>+ <div class="delivery-info">+ <p>Your order will arrive in approximately <span class="delivery-days">${deliveryDays} days</span></p>+ <p style="margin-top: 10px; font-size: 14px;">We'll send tracking information to ${params.email} once your order ships.</p>+ </div>+ </div>++ <button class="btn-back" onclick="window.location.href='./fill-form.html'">Place Another Order</button>+ `;+ }++ // Initialize page+ document.addEventListener('DOMContentLoaded', generateConfirmation);+ </script>+</body>+</html>diff --git a/tests/assets/evals/fill-form.html b/tests/assets/evals/fill-form.htmlnew file mode 100644index 0000000000..02de222e1e--- /dev/null+++ b/tests/assets/evals/fill-form.html@@ -0,0 +1,186 @@+<head>+ <meta charset="UTF-8">+ <meta name="viewport" content="width=device-width, initial-scale=1.0">+ <title>Order Form</title>+ <style>+ * {+ margin: 0;+ padding: 0;+ box-sizing: border-box;+ }++ body {+ font-family: 'Segoe UI', Tahoma, Geneva, Verdana, sans-serif;+ background: linear-gradient(135deg, #667eea 0%, #764ba2 100%);+ padding: 20px;+ min-height: 100vh;+ display: flex;+ justify-content: center;+ align-items: center;+ }++ .container {+ background: white;+ border-radius: 10px;+ box-shadow: 0 10px 40px rgba(0, 0, 0, 0.2);+ padding: 40px;+ max-width: 600px;+ width: 100%;+ }++ h1 {+ color: #333;+ margin-bottom: 30px;+ text-align: center;+ font-size: 28px;+ }++ .form-group {+ margin-bottom: 20px;+ }++ label {+ display: block;+ margin-bottom: 8px;+ color: #555;+ font-weight: 600;+ font-size: 14px;+ }++ input[type="text"],+ input[type="email"],+ select,+ textarea {+ width: 100%;+ padding: 12px;+ border: 2px solid #e0e0e0;+ border-radius: 5px;+ font-size: 14px;+ transition: border-color 0.3s;+ font-family: inherit;+ }++ input[type="text"]:focus,+ input[type="email"]:focus,+ select:focus,+ textarea:focus {+ outline: none;+ border-color: #667eea;+ }++ textarea {+ resize: vertical;+ min-height: 80px;+ }++ .btn-order {+ width: 100%;+ padding: 15px;+ background: linear-gradient(135deg, #667eea 0%, #764ba2 100%);+ color: white;+ border: none;+ border-radius: 5px;+ font-size: 16px;+ font-weight: 600;+ cursor: pointer;+ transition: transform 0.2s, box-shadow 0.2s;+ margin-top: 10px;+ }++ .btn-order:hover {+ transform: translateY(-2px);+ box-shadow: 0 5px 20px rgba(102, 126, 234, 0.4);+ }++ .btn-order:active {+ transform: translateY(0);+ }++ </style>+</head>+<body>+ <div class="container">+ <h1>Product Order Form</h1>+ <form id="orderForm">++ <div class="form-group">+ <label for="productType">Product Type *</label>+ <select id="productType" name="productType" required>+ <option value="">Select a product</option>+ <option value="chair">Chair</option>+ <option value="sofa">Sofa</option>+ <option value="desk">Desk</option>+ <option value="table">Table</option>+ <option value="cabinet">Cabinet</option>+ </select>+ </div>++ <div class="form-group">+ <label for="color">Color *</label>+ <select id="color" name="color" required>+ <option value="">Select a color</option>+ <option value="red">Red</option>+ <option value="green">Green</option>+ <option value="blue">Blue</option>+ <option value="black">Black</option>+ <option value="white">White</option>+ <option value="brown">Brown</option>+ </select>+ </div>++ <div class="form-group">+ <label for="name">Full Name *</label>+ <input type="text" id="name" name="name" placeholder="Enter your full name" required>+ </div>++ <div class="form-group">+ <label for="email">Email Address *</label>+ <input type="email" id="email" name="email" placeholder="[email protected]" required>+ </div>++ <div class="form-group">+ <label for="address">Street Address *</label>+ <input type="text" id="address" name="address" placeholder="123 Main Street" required>+ </div>++ <div class="form-group">+ <label for="city">City *</label>+ <input type="text" id="city" name="city" placeholder="City" required>+ </div>++ <div class="form-group">+ <label for="state">State/Province *</label>+ <input type="text" id="state" name="state" placeholder="State" required>+ </div>++ <div class="form-group">+ <label for="zipCode">ZIP/Postal Code *</label>+ <input type="text" id="zipCode" name="zipCode" placeholder="12345" required>+ </div>++ <button type="button" class="btn-order">Order</button>+ </form>+ </div>++ <script>+ const form = document.getElementById('orderForm');++ const submit = () => {+ location.href = './fill-form-submit.html#?' + new URLSearchParams(new FormData(form)).toString();+ };++ document.querySelector('.btn-order').addEventListener('click', () => {+ if (form.checkValidity()) {+ if (window.confirm(`Please confirm your order for a ${form.color.value} ${form.productType.value}.`))+ submit();+ } else {+ form.reportValidity();+ }+ });++ form.addEventListener('submit', e => {+ e.preventDefault();+ submit();+ });+ </script>+</body>diff --git a/tests/mcp/__cache__/copilot/mcp-browser-eval-fill-the-form-repeat0.json b/tests/mcp/__cache__/copilot/mcp-browser-eval-fill-the-form-repeat0.jsonnew file mode 100644index 0000000000..8b4e8b1238--- /dev/null+++ b/tests/mcp/__cache__/copilot/mcp-browser-eval-fill-the-form-repeat0.json@@ -0,0 +1,330 @@+{+ "facb950333d3cbebad4cb469c846c6a03a72dce8": {+ "result": {+ "role": "assistant",+ "content": "I'll help you navigate to the form, fill it out with the specified details, and report back the total price and delivery information.",+ "toolCalls": []+ },+ "usage": {+ "input": 4484,+ "output": 132+ }+ },+ "a8e0596eb3a8ab52837b1544d1c328f0886e26a1": {+ "result": {+ "role": "assistant",+ "content": "Navigate to the form page",+ "toolCalls": [+ {+ "name": "browser_navigate",+ "arguments": {+ "url": "http://localhost:<PORT>/evals/fill-form.html"+ },+ "id": "tooluse_GPuE3K9kQwSXLSPV2UAyEw"+ }+ ]+ },+ "usage": {+ "input": 4535,+ "output": 88+ }+ },+ "bbdb8dff944cf47be23aa17fa55a678a29f5cd4a": {+ "result": {+ "role": "assistant",+ "content": "Capture snapshot to see the form",+ "toolCalls": []+ },+ "usage": {+ "input": 5232,+ "output": 69+ }+ },+ "3b8c1b6c21376d9a91c21a9e33a35439a3990c54": {+ "result": {+ "role": "assistant",+ "content": "I'll fill out the form for the first item (blue table)",+ "toolCalls": []+ },+ "usage": {+ "input": 5263,+ "output": 130+ }+ },+ "40b838f47364c51fe3512a4fc385fa16763d53a4": {+ "result": {+ "role": "assistant",+ "content": "Fill form for first item - blue table",+ "toolCalls": [+ {+ "name": "browser_fill_form",+ "arguments": {+ "fields": [+ {+ "name": "Product Type",+ "ref": "e7",+ "type": "combobox",+ "value": "Table"+ },+ {+ "name": "Color",+ "ref": "e10",+ "type": "combobox",+ "value": "Blue"+ },+ {+ "name": "Full Name",+ "ref": "e13",+ "type": "textbox",+ "value": "John Doe"+ },+ {+ "name": "Email Address",+ "ref": "e16",+ "type": "textbox",+ "value": "[email protected]"+ },+ {+ "name": "Street Address",+ "ref": "e19",+ "type": "textbox",+ "value": "123 Main St"+ },+ {+ "name": "City",+ "ref": "e22",+ "type": "textbox",+ "value": "Anytown"+ },+ {+ "name": "State/Province",+ "ref": "e25",+ "type": "textbox",+ "value": "XYZ"+ },+ {+ "name": "ZIP/Postal Code",+ "ref": "e28",+ "type": "textbox",+ "value": "12345"+ }+ ]+ },+ "id": "tooluse_9iiHR97qRZWzf8YVVbkHmA"+ }+ ]+ },+ "usage": {+ "input": 5301,+ "output": 331+ }+ },+ "30b7c82f8a226b17b040f846903656852e7a6e36": {+ "result": {+ "role": "assistant",+ "content": "Click the Order button for the first item",+ "toolCalls": []+ },+ "usage": {+ "input": 5830,+ "output": 81+ }+ },+ "d4c841c1aa8400cc2aeaf0a4e4db2c4df625a320": {+ "result": {+ "role": "assistant",+ "content": "Click the Order button",+ "toolCalls": []+ },+ "usage": {+ "input": 5862,+ "output": 77+ }+ },+ "51ee9f20ae47d1660b4ed9dbdaf6d44b4b951339": {+ "result": {+ "role": "assistant",+ "content": "",+ "toolCalls": [+ {+ "name": "browser_click",+ "arguments": {+ "element": "Order button",+ "ref": "e29"+ },+ "id": "tooluse_WtpFqw9JScur-lAHZEsKRQ"+ }+ ]+ },+ "usage": {+ "input": 5890,+ "output": 72+ }+ },+ "4493b8b9ef4b1cfaff25033b937f9a3bc81de116": {+ "result": {+ "role": "assistant",+ "content": "",+ "toolCalls": [+ {+ "name": "browser_handle_dialog",+ "arguments": {+ "accept": true+ },+ "id": "tooluse_oUddkbggQ3aEEYKh8BU8wg"+ }+ ]+ },+ "usage": {+ "input": 6042,+ "output": 55+ }+ },+ "890e35cf92b660a3fa3ade73f150a929caf92a1d": {+ "result": {+ "role": "assistant",+ "content": "",+ "toolCalls": [+ {+ "name": "browser_click",+ "arguments": {+ "element": "Place Another Order button",+ "ref": "e58"+ },+ "id": "tooluse_AlBQSRtuQT2Yot3LPY5F-Q"+ }+ ]+ },+ "usage": {+ "input": 6903,+ "output": 74+ }+ },+ "9bee0bcd86f45f9622412b4c01bc0fa871054aa3": {+ "result": {+ "role": "assistant",+ "content": "",+ "toolCalls": [+ {+ "name": "browser_fill_form",+ "arguments": {+ "fields": [+ {+ "name": "Product Type",+ "ref": "e7",+ "type": "combobox",+ "value": "Desk"+ },+ {+ "name": "Color",+ "ref": "e10",+ "type": "combobox",+ "value": "Green"+ },+ {+ "name": "Full Name",+ "ref": "e13",+ "type": "textbox",+ "value": "John Doe"+ },+ {+ "name": "Email Address",+ "ref": "e16",+ "type": "textbox",+ "value": "[email protected]"+ },+ {+ "name": "Street Address",+ "ref": "e19",+ "type": "textbox",+ "value": "123 Main St"+ },+ {+ "name": "City",+ "ref": "e22",+ "type": "textbox",+ "value": "Anytown"+ },+ {+ "name": "State/Province",+ "ref": "e25",+ "type": "textbox",+ "value": "XYZ"+ },+ {+ "name": "ZIP/Postal Code",+ "ref": "e28",+ "type": "textbox",+ "value": "12345"+ }+ ]+ },+ "id": "tooluse_YGgfR5fAR6OJcd7Q9M4t7w"+ }+ ]+ },+ "usage": {+ "input": 7601,+ "output": 298+ }+ },+ "faae385281a92076a48da71d391fa0e477894e6d": {+ "result": {+ "role": "assistant",+ "content": "",+ "toolCalls": [+ {+ "name": "browser_click",+ "arguments": {+ "element": "Order button",+ "ref": "e29"+ },+ "id": "tooluse_qo1cC8s9SB-sPPPgTk36sA"+ }+ ]+ },+ "usage": {+ "input": 8123,+ "output": 72+ }+ },+ "eaab8ebfa4b63c74ba919e849ba983fcb874528f": {+ "result": {+ "role": "assistant",+ "content": "",+ "toolCalls": [+ {+ "name": "browser_handle_dialog",+ "arguments": {+ "accept": true+ },+ "id": "tooluse_EWFnsSPoTY6i4Qf7bpEwjg"+ }+ ]+ },+ "usage": {+ "input": 8274,+ "output": 55+ }+ },+ "2c31a0c54acc5a4c4f9b8e4c7acdbf94a81eebbb": {+ "result": {+ "role": "assistant",+ "content": "",+ "toolCalls": [+ {+ "name": "report_result",+ "arguments": {+ "deliveryDays": 3,+ "price": 653.96+ },+ "id": "tooluse_I1b-G92TS7qIuUsLtl020w"+ }+ ]+ },+ "usage": {+ "input": 9136,+ "output": 75+ }+ }+}\ No newline at end of filediff --git a/tests/mcp/__cache__/copilot/mcp-browser-eval-fill-the-form-repeat1.json b/tests/mcp/__cache__/copilot/mcp-browser-eval-fill-the-form-repeat1.jsonnew file mode 100644index 0000000000..d17add266f--- /dev/null+++ b/tests/mcp/__cache__/copilot/mcp-browser-eval-fill-the-form-repeat1.json@@ -0,0 +1,492 @@+{+ "facb950333d3cbebad4cb469c846c6a03a72dce8": {+ "result": {+ "role": "assistant",+ "content": "I'll help you navigate to the form, fill it out with the specified details, and complete the order. Let me start by navigating to the page.",+ "toolCalls": []+ },+ "usage": {+ "input": 4484,+ "output": 126+ }+ },+ "f65dd22f0f79ffd6398d42230797a4a53c72c9ab": {+ "result": {+ "role": "assistant",+ "content": "Navigate to the form page to start the ordering process",+ "toolCalls": [+ {+ "name": "browser_navigate",+ "arguments": {+ "url": "http://localhost:<PORT>/evals/fill-form.html"+ },+ "id": "tooluse_BCc-AvDnQWmc8O81Stj2YQ"+ }+ ]+ },+ "usage": {+ "input": 4540,+ "output": 93+ }+ },+ "eb63c3301417583c538a6501c36797d42e161a1a": {+ "result": {+ "role": "assistant",+ "content": "Take a snapshot to see the current page state",+ "toolCalls": [+ {+ "name": "browser_snapshot",+ "arguments": {},+ "id": "tooluse_EQPPTsKBT_G_YsVMHNZnAw"+ }+ ]+ },+ "usage": {+ "input": 5242,+ "output": 61+ }+ },+ "190598029fe374115a87b39e2d631689f712f8c5": {+ "result": {+ "role": "assistant",+ "content": "Fill out the form for the first item - blue table",+ "toolCalls": []+ },+ "usage": {+ "input": 5878,+ "output": 350+ }+ },+ "73845baea403b9e42331b19667593fbf028c3429": {+ "result": {+ "role": "assistant",+ "content": "Select Table as the product type",+ "toolCalls": [+ {+ "name": "browser_select_option",+ "arguments": {+ "element": "Product Type dropdown",+ "ref": "e7",+ "values": [+ "Table"+ ]+ },+ "id": "tooluse_WVHv7CI3R-WqyOj6O7LTKg"+ }+ ]+ },+ "usage": {+ "input": 5913,+ "output": 116+ }+ },+ "5cac5e0e4742f5674f0c4c7fd933d7e4c84f0513": {+ "result": {+ "role": "assistant",+ "content": "Select Blue as the color",+ "toolCalls": []+ },+ "usage": {+ "input": 6165,+ "output": 99+ }+ },+ "67a472d6a9e6e9c965eaa82a70a4bdbb7485eca6": {+ "result": {+ "role": "assistant",+ "content": "",+ "toolCalls": [+ {+ "name": "browser_select_option",+ "arguments": {+ "element": "Color dropdown",+ "ref": "e10",+ "values": [+ "Blue"+ ]+ },+ "id": "tooluse_f86G0fVES260pD2OGjYdCA"+ }+ ]+ },+ "usage": {+ "input": 6194,+ "output": 93+ }+ },+ "65f822bec8f70205e0d58e9a2ee09a3707ed6401": {+ "result": {+ "role": "assistant",+ "content": "Fill in customer information for the order",+ "toolCalls": [+ {+ "name": "browser_fill_form",+ "arguments": {+ "fields": [+ {+ "name": "Full Name",+ "ref": "e13",+ "type": "textbox",+ "value": "John Doe"+ },+ {+ "name": "Email Address",+ "ref": "e16",+ "type": "textbox",+ "value": "[email protected]"+ },+ {+ "name": "Street Address",+ "ref": "e19",+ "type": "textbox",+ "value": "123 Main St"+ },+ {+ "name": "City",+ "ref": "e22",+ "type": "textbox",+ "value": "Anytown"+ },+ {+ "name": "State/Province",+ "ref": "e25",+ "type": "textbox",+ "value": "XYZ"+ },+ {+ "name": "ZIP/Postal Code",+ "ref": "e28",+ "type": "textbox",+ "value": "12345"+ }+ ]+ },+ "id": "tooluse_ztCsxsUlTueuOPgHsO83dA"+ }+ ]+ },+ "usage": {+ "input": 6441,+ "output": 343+ }+ },+ "6b1f789361769bb90eb776cff6df695394e90ff5": {+ "result": {+ "role": "assistant",+ "content": "Click the Order button to submit the first order",+ "toolCalls": []+ },+ "usage": {+ "input": 6879,+ "output": 82+ }+ },+ "f036e1c5e71f07be6729fc6fc0ef5a7514564633": {+ "result": {+ "role": "assistant",+ "content": "",+ "toolCalls": [+ {+ "name": "browser_click",+ "arguments": {+ "element": "Order button",+ "ref": "e29"+ },+ "id": "tooluse_7Bw_du6XShCs0VM46yds3A"+ }+ ]+ },+ "usage": {+ "input": 6912,+ "output": 72+ }+ },+ "44e579cf6dfa27d84426d8afbffda3bb20e978b5": {+ "result": {+ "role": "assistant",+ "content": "Accept the order confirmation dialog",+ "toolCalls": []+ },+ "usage": {+ "input": 7064,+ "output": 61+ }+ },+ "b1043d04c5761a4273d3c6d358e18727ae53bedd": {+ "result": {+ "role": "assistant",+ "content": "",+ "toolCalls": [+ {+ "name": "browser_handle_dialog",+ "arguments": {+ "accept": true+ },+ "id": "tooluse_CBKRGvaMT8-0bxI3LmpgLQ"+ }+ ]+ },+ "usage": {+ "input": 7093,+ "output": 55+ }+ },+ "4a20e94c802689f0e5795cc01d0754593c95b8d9": {+ "result": {+ "role": "assistant",+ "content": "Click Place Another Order button to order the second item",+ "toolCalls": []+ },+ "usage": {+ "input": 7955,+ "output": 85+ }+ },+ "02d0f1acdf58e14bfacf4fb3d1a04973726b3f6a": {+ "result": {+ "role": "assistant",+ "content": "",+ "toolCalls": [+ {+ "name": "browser_click",+ "arguments": {+ "element": "Place Another Order button",+ "ref": "e58"+ },+ "id": "tooluse_mEVX8xrBQ-u7BPjnZ7PPKQ"+ }+ ]+ },+ "usage": {+ "input": 7989,+ "output": 74+ }+ },+ "19504162e60fbdd96072af88bd669b3e88860524": {+ "result": {+ "role": "assistant",+ "content": "Select Desk as the product type for the second order",+ "toolCalls": []+ },+ "usage": {+ "input": 8688,+ "output": 107+ }+ },+ "302dcb0c44750d90fdf69073db4a6533d657af9c": {+ "result": {+ "role": "assistant",+ "content": "",+ "toolCalls": [+ {+ "name": "browser_select_option",+ "arguments": {+ "element": "Product Type dropdown",+ "ref": "e7",+ "values": [+ "Desk"+ ]+ },+ "id": "tooluse_BaMRqzC-QbeyPBC-9RP0aw"+ }+ ]+ },+ "usage": {+ "input": 8723,+ "output": 95+ }+ },+ "b40f9a90caa41102d3b795a22a005ad9042b659e": {+ "result": {+ "role": "assistant",+ "content": "Select Green as the color for the desk",+ "toolCalls": []+ },+ "usage": {+ "input": 8971,+ "output": 11+ }+ },+ "9c502c32a6f3797f941b683c19473c89cec2a1b4": {+ "result": {+ "role": "assistant",+ "content": "",+ "toolCalls": [+ {+ "name": "browser_select_option",+ "arguments": {+ "element": "Color dropdown",+ "ref": "e10",+ "values": [+ "Green"+ ]+ },+ "id": "tooluse_bvxZZXldSu28CR0zYOoIng"+ }+ ]+ },+ "usage": {+ "input": 9003,+ "output": 93+ }+ },+ "ce3d6682dda9339c3efd384e5bd23e9aef4852fc": {+ "result": {+ "role": "assistant",+ "content": "Fill in the customer information for the second order",+ "toolCalls": []+ },+ "usage": {+ "input": 9250,+ "output": 250+ }+ },+ "92858cba54b40fc65bc94e7f1ffd16f427f3f1b0": {+ "result": {+ "role": "assistant",+ "content": "",+ "toolCalls": [+ {+ "name": "browser_fill_form",+ "arguments": {+ "fields": [+ {+ "name": "Full Name",+ "ref": "e13",+ "type": "textbox",+ "value": "John Doe"+ },+ {+ "name": "Email Address",+ "ref": "e16",+ "type": "textbox",+ "value": "[email protected]"+ },+ {+ "name": "Street Address",+ "ref": "e19",+ "type": "textbox",+ "value": "123 Main St"+ },+ {+ "name": "City",+ "ref": "e22",+ "type": "textbox",+ "value": "Anytown"+ },+ {+ "name": "State/Province",+ "ref": "e25",+ "type": "textbox",+ "value": "XYZ"+ },+ {+ "name": "ZIP/Postal Code",+ "ref": "e28",+ "type": "textbox",+ "value": "12345"+ }+ ]+ },+ "id": "tooluse_MiTMyTOaRoCmK0EapkxU7g"+ }+ ]+ },+ "usage": {+ "input": 9283,+ "output": 240+ }+ },+ "3c11d68c47519b7364a7dd5c1be26d6b7cd16a66": {+ "result": {+ "role": "assistant",+ "content": "Click the Order button to submit the second order",+ "toolCalls": []+ },+ "usage": {+ "input": 9714,+ "output": 12+ }+ },+ "334effd07d305facef120ed49d47eca9546b9ed2": {+ "result": {+ "role": "assistant",+ "content": "",+ "toolCalls": [+ {+ "name": "browser_click",+ "arguments": {+ "element": "Order button",+ "ref": "e29"+ },+ "id": "tooluse_jana5W7FSLiGM40Y58vxuA"+ }+ ]+ },+ "usage": {+ "input": 9747,+ "output": 72+ }+ },+ "3e70d82f485d3bb257053e605a888e391d8458f3": {+ "result": {+ "role": "assistant",+ "content": "Accept the order confirmation dialog for the second order",+ "toolCalls": []+ },+ "usage": {+ "input": 9899,+ "output": 12+ }+ },+ "f6df99806ce1301aedf58fd2ebfc77b3645f849b": {+ "result": {+ "role": "assistant",+ "content": "",+ "toolCalls": [+ {+ "name": "browser_handle_dialog",+ "arguments": {+ "accept": true+ },+ "id": "tooluse_j3MT0SvtSBuucN6YXXxfZg"+ }+ ]+ },+ "usage": {+ "input": 9932,+ "output": 55+ }+ },+ "c6d1b1ddc05aa572b48a0abfebaf84117d38aefd": {+ "result": {+ "role": "assistant",+ "content": "Report the total price and maximum estimated delivery time for both orders",+ "toolCalls": []+ },+ "usage": {+ "input": 10795,+ "output": 88+ }+ },+ "b2b70eb2a30bcd06f1cd7819b9a7d4e275a528b6": {+ "result": {+ "role": "assistant",+ "content": "",+ "toolCalls": [+ {+ "name": "report_result",+ "arguments": {+ "deliveryDays": 3,+ "price": 653.96+ },+ "id": "tooluse_CR5rFd09QRGnH5-Lt3-pGw"+ }+ ]+ },+ "usage": {+ "input": 10831,+ "output": 75+ }+ }+}\ No newline at end of filediff --git a/tests/mcp/__cache__/copilot/mcp-browser-eval-fill-the-form-repeat2.json b/tests/mcp/__cache__/copilot/mcp-browser-eval-fill-the-form-repeat2.jsonnew file mode 100644index 0000000000..d936c0aac0--- /dev/null+++ b/tests/mcp/__cache__/copilot/mcp-browser-eval-fill-the-form-repeat2.json@@ -0,0 +1,404 @@+{+ "facb950333d3cbebad4cb469c846c6a03a72dce8": {+ "result": {+ "role": "assistant",+ "content": "I'll help you navigate to the form, fill it out with the specified details, and report back the results. Let me start by navigating to the page.",+ "toolCalls": []+ },+ "usage": {+ "input": 4484,+ "output": 136+ }+ },+ "e9daf936478b73139df8f3ea61bc82a1a13c3cdb": {+ "result": {+ "role": "assistant",+ "content": "Navigate to the form page",+ "toolCalls": [+ {+ "name": "browser_navigate",+ "arguments": {+ "url": "http://localhost:<PORT>/evals/fill-form.html"+ },+ "id": "tooluse_2QTlFMmcQlqV0a1rVultZw"+ }+ ]+ },+ "usage": {+ "input": 4541,+ "output": 88+ }+ },+ "7dc1192eb36584ccdef322c804036a775f23f7a2": {+ "result": {+ "role": "assistant",+ "content": "Check the current state of the form before filling",+ "toolCalls": [+ {+ "name": "browser_snapshot",+ "arguments": {},+ "id": "tooluse_Xq-IYmyLTua6t1KRVizOew"+ }+ ]+ },+ "usage": {+ "input": 5238,+ "output": 61+ }+ },+ "e960fd42b63a75629d10e2b064a712cf4bb222c8": {+ "result": {+ "role": "assistant",+ "content": "Fill the first order form for a blue table",+ "toolCalls": []+ },+ "usage": {+ "input": 5874,+ "output": 126+ }+ },+ "ab8368051e5eb0335afc2627359409ff6e527b22": {+ "result": {+ "role": "assistant",+ "content": "Select Table as the product type for the first order",+ "toolCalls": [+ {+ "name": "browser_select_option",+ "arguments": {+ "element": "Product Type dropdown",+ "ref": "e7",+ "values": [+ "Table"+ ]+ },+ "id": "tooluse_qDJgPasgTZazcKmtzotHZg"+ }+ ]+ },+ "usage": {+ "input": 5907,+ "output": 120+ }+ },+ "eefa023ed6d14b88656c764ec83bf98bee091d0a": {+ "result": {+ "role": "assistant",+ "content": "Select Blue as the color",+ "toolCalls": []+ },+ "usage": {+ "input": 6163,+ "output": 99+ }+ },+ "c7ddf8898251a2d2a7fcadf36e226a14f2d32955": {+ "result": {+ "role": "assistant",+ "content": "",+ "toolCalls": [+ {+ "name": "browser_select_option",+ "arguments": {+ "element": "Color dropdown",+ "ref": "e10",+ "values": [+ "Blue"+ ]+ },+ "id": "tooluse_K_TcaZuDTaaPDZT-qhxH0g"+ }+ ]+ },+ "usage": {+ "input": 6192,+ "output": 93+ }+ },+ "5ccb74547a127df0533ca011b1ac02fb9df840c3": {+ "result": {+ "role": "assistant",+ "content": "Fill in all the customer details in the form",+ "toolCalls": [+ {+ "name": "browser_fill_form",+ "arguments": {+ "fields": [+ {+ "name": "Full Name",+ "ref": "e13",+ "type": "textbox",+ "value": "John Doe"+ },+ {+ "name": "Email Address",+ "ref": "e16",+ "type": "textbox",+ "value": "[email protected]"+ },+ {+ "name": "Street Address",+ "ref": "e19",+ "type": "textbox",+ "value": "123 Main St"+ },+ {+ "name": "City",+ "ref": "e22",+ "type": "textbox",+ "value": "Anytown"+ },+ {+ "name": "State/Province",+ "ref": "e25",+ "type": "textbox",+ "value": "XYZ"+ },+ {+ "name": "ZIP/Postal Code",+ "ref": "e28",+ "type": "textbox",+ "value": "12345"+ }+ ]+ },+ "id": "tooluse_fevjVnF0S2OhEUg_pdQNog"+ }+ ]+ },+ "usage": {+ "input": 6439,+ "output": 345+ }+ },+ "1ef5b1de2508c7962e6e4b4dcefe59c11179680d": {+ "result": {+ "role": "assistant",+ "content": "Submit the order form",+ "toolCalls": []+ },+ "usage": {+ "input": 6879,+ "output": 77+ }+ },+ "f096f02c459d6fa500047ad6ba5e65c0e5222e20": {+ "result": {+ "role": "assistant",+ "content": "",+ "toolCalls": [+ {+ "name": "browser_click",+ "arguments": {+ "element": "Order button",+ "ref": "e29"+ },+ "id": "tooluse_cDuf7VYITsOk3SPizGkdbw"+ }+ ]+ },+ "usage": {+ "input": 6907,+ "output": 72+ }+ },+ "a28b9ead7a286733ddbf6c52fd0c3ed1d5954725": {+ "result": {+ "role": "assistant",+ "content": "",+ "toolCalls": [+ {+ "name": "browser_handle_dialog",+ "arguments": {+ "accept": true+ },+ "id": "tooluse_A0g51QouRsWL9CsOTx5Ycg"+ }+ ]+ },+ "usage": {+ "input": 7059,+ "output": 55+ }+ },+ "fb6f4514207d7e27fa9a5c6da79320dd6d5ab572": {+ "result": {+ "role": "assistant",+ "content": "",+ "toolCalls": [+ {+ "name": "browser_click",+ "arguments": {+ "element": "Place Another Order button",+ "ref": "e58"+ },+ "id": "tooluse_4U7nwg7ZTMiom22WRl-FJw"+ }+ ]+ },+ "usage": {+ "input": 7920,+ "output": 74+ }+ },+ "4a73f42e2d9bdb78c9a2e5390bc7a964fbb343c0": {+ "result": {+ "role": "assistant",+ "content": "",+ "toolCalls": [+ {+ "name": "browser_select_option",+ "arguments": {+ "element": "Product Type dropdown",+ "ref": "e7",+ "values": [+ "Desk"+ ]+ },+ "id": "tooluse_67fzJrlgS06obb4sPHxk-Q"+ }+ ]+ },+ "usage": {+ "input": 8618,+ "output": 95+ }+ },+ "cf8bc3f7c495171c5541706d297582791b7c316f": {+ "result": {+ "role": "assistant",+ "content": "",+ "toolCalls": [+ {+ "name": "browser_select_option",+ "arguments": {+ "element": "Color dropdown",+ "ref": "e10",+ "values": [+ "Green"+ ]+ },+ "id": "tooluse_CroDBxgYS1akDERsrWsRAg"+ }+ ]+ },+ "usage": {+ "input": 8865,+ "output": 93+ }+ },+ "995ae7de03de31d0deaa9f9c415b682386ea0290": {+ "result": {+ "role": "assistant",+ "content": "",+ "toolCalls": [+ {+ "name": "browser_fill_form",+ "arguments": {+ "fields": [+ {+ "name": "Full Name",+ "ref": "e13",+ "type": "textbox",+ "value": "John Doe"+ },+ {+ "name": "Email Address",+ "ref": "e16",+ "type": "textbox",+ "value": "[email protected]"+ },+ {+ "name": "Street Address",+ "ref": "e19",+ "type": "textbox",+ "value": "123 Main St"+ },+ {+ "name": "City",+ "ref": "e22",+ "type": "textbox",+ "value": "Anytown"+ },+ {+ "name": "State/Province",+ "ref": "e25",+ "type": "textbox",+ "value": "XYZ"+ },+ {+ "name": "ZIP/Postal Code",+ "ref": "e28",+ "type": "textbox",+ "value": "12345"+ }+ ]+ },+ "id": "tooluse_DLFD9lwrQb2Oc_4NpnYOYg"+ }+ ]+ },+ "usage": {+ "input": 9111,+ "output": 240+ }+ },+ "6ec90af600847bd4dd2a0fa6b8ea5c08fd0e59d0": {+ "result": {+ "role": "assistant",+ "content": "",+ "toolCalls": [+ {+ "name": "browser_click",+ "arguments": {+ "element": "Order button",+ "ref": "e29"+ },+ "id": "tooluse_oNWDYmu7RTClOEuCTrn7Sg"+ }+ ]+ },+ "usage": {+ "input": 9541,+ "output": 72+ }+ },+ "c46467b8f90fdfbfdf5081dda99e2055b6f6152f": {+ "result": {+ "role": "assistant",+ "content": "",+ "toolCalls": [+ {+ "name": "browser_handle_dialog",+ "arguments": {+ "accept": true+ },+ "id": "tooluse_qBV8FnNVTMiBZ8oLuUVYOQ"+ }+ ]+ },+ "usage": {+ "input": 9692,+ "output": 55+ }+ },+ "c9ebdba51d456673a83fd1ec4e484fea864ef5b7": {+ "result": {+ "role": "assistant",+ "content": "",+ "toolCalls": [+ {+ "name": "report_result",+ "arguments": {+ "deliveryDays": 3,+ "price": 653.96+ },+ "id": "tooluse_Tsf2nC_CR6CMtZL91D0ovA"+ }+ ]+ },+ "usage": {+ "input": 10554,+ "output": 75+ }+ }+}\ No newline at end of filediff --git a/tests/mcp/browser.eval.ts b/tests/mcp/browser.eval.tsnew file mode 100644index 0000000000..c44bf25e92--- /dev/null+++ b/tests/mcp/browser.eval.ts@@ -0,0 +1,45 @@+/**+ * Copyright (c) Microsoft Corporation.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++import z from 'zod';+import { zodToJsonSchema } from 'zod-to-json-schema';++import { test, expect } from './fixtures';++import type * as tinyLoop from 'tiny-loop';++test.use({ mcpHeadless: async ({ headless }, use) => use(headless) });++test('fill the form', async ({ loop, client, server }) => {+ expect(await loop.run<{ price: number, deliveryDays: number }>(`+ Navigate to ${server.PREFIX + '/evals/fill-form.html'} via Playwright MCP.+ Order a blue table and a green desk.+ Use the following details for the order form:+ - Name: John Doe+ - Address: 123 Main St, Anytown, XYZ state+ - Zip Code: 12345+ - Email: [email protected]+ Report back total price for both items, and maximum estimated delivery.+ Do not take screenshots.+ `, {+ tools: (await client.listTools()).tools as tinyLoop.Tool[],+ callTool: (params => client.callTool(params)) as tinyLoop.ToolCallback,+ resultSchema: zodToJsonSchema(z.object({+ price: z.number().describe('Total price in USD.'),+ deliveryDays: z.number().describe('Maximum estimated delivery time in days.'),+ })) as tinyLoop.Schema,+ })).toEqual({ price: 653.96, deliveryDays: 3 });+});diff --git a/tests/mcp/fixtures.ts b/tests/mcp/fixtures.tsindex 9f55b0c643..ba92eaa84d 100644--- a/tests/mcp/fixtures.ts+++ b/tests/mcp/fixtures.ts@@ -17,6 +17,8 @@import fs from 'fs';import path from 'path';import { chromium } from 'playwright';+import { Loop } from 'tiny-loop';+import debug from 'debug';import { test as baseTest, expect as baseExpect } from '@playwright/test';import { StdioClientTransport } from '@modelcontextprotocol/sdk/client/stdio.js';@@ -65,6 +67,7 @@ type TestFixtures = {server: TestServer;httpsServer: TestServer;mcpHeadless: boolean;+ loop: Loop;};type WorkerFixtures = {@@ -184,6 +187,28 @@ export const test = serverTest.extend<TestFixtures & TestOptions, WorkerFixturesmcpBrowser: ['chrome', { option: true }],mcpServerType: ['mcp', { option: true }],++ loop: async ({ server }, use) => {+ const provider = 'copilot';+ const cacheFile = path.join(__dirname, '__cache__', provider, sanitizeFileName(test.info().titlePath.join(' ') + '-repeat' + test.info().repeatEachIndex) + '.json');+ const dataBefore = await fs.promises.readFile(cacheFile, 'utf-8').catch(() => '{}');+ let cache = {};+ try {+ cache = JSON.parse(dataBefore);+ } catch {+ }+ const caches = { before: cache, after: {} };+ await use(new Loop(provider, {+ caches,+ secrets: { PORT: String(server.PORT) },+ logger: (category, ...args) => debug(category)(...args),+ }));+ const dataAfter = JSON.stringify(caches.after, null, 2);+ if (test.info().status === 'passed' && dataBefore !== dataAfter) {+ await fs.promises.mkdir(path.dirname(cacheFile), { recursive: true });+ await fs.promises.writeFile(cacheFile, dataAfter);+ }+ },});async function createTransport(mcpServerType: TestOptions['mcpServerType'], options: { args: string[], env: NodeJS.ProcessEnv, cwd: string }): Promise<{@@ -312,3 +337,7 @@ export async function prepareDebugTest(startClient: StartClient, testFile?: striconst [, id] = listResult.content[0].text.match(/\[id=([^\]]+)\]/);return { client, id };}++function sanitizeFileName(name: string): string {+ return name.replace('.ts', '').replace(/[^a-zA-Z0-9_]+/g, '-');+}diff --git a/tests/mcp/playwright.config.extension.ts b/tests/mcp/playwright.config.extension.tsdeleted file mode 100644index 526fda2001..0000000000--- a/tests/mcp/playwright.config.extension.ts+++ /dev/null@@ -1,32 +0,0 @@-/**- * Copyright (c) Microsoft Corporation.- *- * Licensed under the Apache License, Version 2.0 (the "License");- * you may not use this file except in compliance with the License.- * You may obtain a copy of the License at- *- * http://www.apache.org/licenses/LICENSE-2.0- *- * Unless required by applicable law or agreed to in writing, software- * distributed under the License is distributed on an "AS IS" BASIS,- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.- * See the License for the specific language governing permissions and- * limitations under the License.- */--import { defineConfig } from '@playwright/test';--import type { TestOptions } from './fixtures';--export default defineConfig<TestOptions>({- testDir: './',- grep: /extension/,- fullyParallel: true,- forbidOnly: !!process.env.CI,- retries: process.env.CI ? 2 : 0,- workers: process.env.CI ? 1 : undefined,- reporter: 'list',- projects: [- { name: 'chromium', use: { mcpBrowser: 'chromium' } },- ],-});diff --git a/tests/mcp/playwright.config.ts b/tests/mcp/playwright.config.tsindex 11ab165bf8..965b9bb67f 100644--- a/tests/mcp/playwright.config.ts+++ b/tests/mcp/playwright.config.ts@@ -14,12 +14,15 @@* limitations under the License.*/import * as path from 'path';+import dotenv from 'dotenv';import { defineConfig } from '@playwright/test';import type { TestOptions } from './fixtures';import type { ReporterDescription } from '@playwright/test';+dotenv.config({ path: path.resolve(__dirname, '../../.env'), quiet: true });+const rootTestDir = path.join(__dirname, '..');const testDir = path.join(rootTestDir, 'mcp');const outputDir = path.join(__dirname, '..', '..', 'test-results');@@ -44,7 +47,6 @@ const metadata = {export default defineConfig<TestOptions>({testDir: rootTestDir,- grepInvert: /extension/,fullyParallel: true,forbidOnly: !!process.env.CI,workers: process.env.CI ? 2 : undefined,@@ -55,5 +57,14 @@ export default defineConfig<TestOptions>({{ name: 'firefox', use: { mcpBrowser: 'firefox' }, metadata: { ...metadata, browserName: 'firefox' }, testDir },{ name: 'webkit', use: { mcpBrowser: 'webkit' }, metadata: { ...metadata, browserName: 'webkit' }, testDir },... process.platform === 'win32' ? [{ name: 'msedge', use: { mcpBrowser: 'msedge' }, metadata: { ...metadata, browserName: 'chromium', channel: 'msedge' }, testDir }] : [],+ {+ name: 'eval',+ testDir,+ testMatch: /.*\.eval\.ts/,+ repeatEach: 3, // Generate three different trajectories per test.+ timeout: process.env.CI ? 30_000 : 180_000, // CI should be cache-only, but local use of LLM is slow.+ use: { mcpBrowser: 'chromium' },+ metadata: { ...metadata, browserName: 'chromium' },+ }],});