Skip to content

Playwright is a framework for Web Testing and Automation. It allows testing Chromium, Firefox and WebKit with a single API.

chore: add assert tools to mdd (#36437)

Dmitry Gozmana03dcc3aa366
diff --git a/packages/playwright-mdd/src/loop.ts b/packages/playwright-mdd/src/loop.ts
index 64d715d3b2..5b5b84bec2 100644
--- a/packages/playwright-mdd/src/loop.ts
+++ b/packages/playwright-mdd/src/loop.ts
@@ -69,25 +69,44 @@ async function runTask(openai: OpenAI, context: Context, task: string): Promise<
tool_calls: message.tool_calls
});
+ let hasPreviousError = false;
for (const toolCall of message.tool_calls) {
const functionCall = toolCall.function;
console.log('Call tool:', functionCall.name, functionCall.arguments);
- if (functionCall.name === 'done')
- return { taskCode };
-
const tool = context.tools.find(tool => tool.schema.name === functionCall.name);
if (!tool)
throw new Error('Unknown tool: ' + functionCall.name);
- const { code, content } = await context.run(tool, JSON.parse(functionCall.arguments));
- taskCode.push(...code);
+ if (hasPreviousError) {
+ messages.push({
+ role: 'tool',
+ tool_call_id: toolCall.id,
+ content: `This tool call is skipped due to previous error.`,
+ });
+ continue;
+ }
+
+ if (functionCall.name === 'done')
+ return { taskCode };
- messages.push({
- role: 'tool',
- tool_call_id: toolCall.id,
- content,
- });
+ try {
+ const { code, content } = await context.run(tool, JSON.parse(functionCall.arguments));
+ taskCode.push(...code);
+ messages.push({
+ role: 'tool',
+ tool_call_id: toolCall.id,
+ content,
+ });
+ } catch (error) {
+ hasPreviousError = true;
+ console.log('Tool error:', error);
+ messages.push({
+ role: 'tool',
+ tool_call_id: toolCall.id,
+ content: `Error while executing tool "${functionCall.name}": ${error instanceof Error ? error.message : String(error)}\n\nPlease try to recover and complete the task.`,
+ });
+ }
}
}
throw new Error('Failed to perform step, max attempts reached');
diff --git a/packages/playwright-mdd/src/program.ts b/packages/playwright-mdd/src/program.ts
index aebfae0f12..d85b24de7c 100644
--- a/packages/playwright-mdd/src/program.ts
+++ b/packages/playwright-mdd/src/program.ts
@@ -40,11 +40,21 @@ program
await context.close();
});
+// An example of a failing script.
+//
+// const script = [
+// 'Navigate to https://debs-obrien.github.io/playwright-movies-app/search?searchTerm=Twister&page=1',
+// 'Verify that the URL contains the search term "twisters"',
+// ];
+
const script = [
'Navigate to https://debs-obrien.github.io/playwright-movies-app',
'Click search icon',
'Type "Twister" in the search field and hit Enter',
+ 'Verify that the URL contains the search term "twister"',
+ 'Verify that the search results contain an image named "Twisters"',
'Click on the link for the movie "Twisters"',
+ 'Verify that the main heading on the movie page is "Twisters"',
];
export { program };
diff --git a/packages/playwright-mdd/src/tools.ts b/packages/playwright-mdd/src/tools.ts
index e4b7c9a6ce..5cbee6a3c1 100644
--- a/packages/playwright-mdd/src/tools.ts
+++ b/packages/playwright-mdd/src/tools.ts
@@ -14,6 +14,7 @@
* limitations under the License.
*/
+import assert from './tools/assert';
import snapshot from './tools/snapshot';
import done from './tools/done';
import navigate from './tools/navigate';
@@ -21,6 +22,7 @@ import navigate from './tools/navigate';
import type { Tool } from './tools/tool.js';
export const tools: Tool<any>[] = [
+ ...assert,
...navigate,
...snapshot,
...done,
diff --git a/packages/playwright-mdd/src/tools/assert.ts b/packages/playwright-mdd/src/tools/assert.ts
new file mode 100644
index 0000000000..38eae843ac
--- /dev/null
+++ b/packages/playwright-mdd/src/tools/assert.ts
@@ -0,0 +1,82 @@
+/**
+ * Copyright (c) Microsoft Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { z } from 'zod';
+
+import { defineTool } from './tool';
+import { generateLocator } from './utils';
+import { elementSchema } from './snapshot';
+
+const assertVisible = defineTool({
+ schema: {
+ name: 'browser_assert_visible',
+ description: 'Assert that an element is visible on the page',
+ inputSchema: elementSchema,
+ },
+
+ handle: async (context, params) => {
+ const locator = context.refLocator(params);
+
+ const code = [
+ `await expect(page.${await generateLocator(locator)}).toBeVisible();`
+ ];
+
+ return {
+ code,
+ action: async () => {
+ const isVisible = await locator.isVisible();
+ if (!isVisible)
+ throw new Error(`Expected ${params.element} to be visible, but it is hidden.`);
+ },
+ captureSnapshot: false,
+ waitForNetwork: false,
+ };
+ },
+});
+
+const assertURL = defineTool({
+ schema: {
+ name: 'browser_assert_url',
+ description: 'Assert that the current page URL matches the expected URL',
+ inputSchema: z.object({
+ url: z.string().describe('The expected URL regular expression, e.g. ".*example.com/path/.*"'),
+ ignoreCase: z.boolean().optional().default(true).describe('Whether to ignore case when matching the URL'),
+ }),
+ },
+
+ handle: async (context, params) => {
+ const code = [
+ `await expect(page).toHaveURL(/${params.url}/);`
+ ];
+
+ return {
+ code,
+ action: async () => {
+ const re = new RegExp(params.url, params.ignoreCase ? 'i' : '');
+ const url = context.page.url();
+ if (!re.test(url))
+ throw new Error(`Expected URL to match ${params.url}, but got ${url}.`);
+ },
+ captureSnapshot: false,
+ waitForNetwork: false,
+ };
+ },
+});
+
+export default [
+ assertVisible,
+ assertURL,
+];
diff --git a/packages/playwright-mdd/src/tools/snapshot.ts b/packages/playwright-mdd/src/tools/snapshot.ts
index e2a90ad852..b3979af095 100644
--- a/packages/playwright-mdd/src/tools/snapshot.ts
+++ b/packages/playwright-mdd/src/tools/snapshot.ts
@@ -36,7 +36,7 @@ const snapshot = defineTool({
},
});
-const elementSchema = z.object({
+export const elementSchema = z.object({
element: z.string().describe('Human-readable element description used to obtain permission to interact with the element'),
ref: z.string().describe('Exact target element reference from the page snapshot'),
});