Skip to content

Playwright is a framework for Web Testing and Automation. It allows testing Chromium, Firefox and WebKit with a single API.

chore: let agent refuse to perform (#38803)

Pavel Feldmandb2a052febce
diff --git a/package-lock.json b/package-lock.json
index e6305bde87..273547d67a 100644
--- a/package-lock.json
+++ b/package-lock.json
@@ -18,7 +18,7 @@
"@eslint/compat": "^1.3.2",
"@eslint/eslintrc": "^3.3.1",
"@eslint/js": "^9.34.0",
- "@lowire/loop": "^0.0.23",
+ "@lowire/loop": "^0.0.24",
"@modelcontextprotocol/sdk": "^1.25.2",
"@octokit/graphql-schema": "^15.26.0",
"@stylistic/eslint-plugin": "^5.2.3",
@@ -1077,9 +1077,9 @@
}
},
"node_modules/@lowire/loop": {
- "version": "0.0.23",
- "resolved": "https://registry.npmjs.org/@lowire/loop/-/loop-0.0.23.tgz",
- "integrity": "sha512-zw4yFzto2T9g+CiHfubzxut0JMxA053XQ2syveqPjqofvRZyumhJR11RIQjKrPO9rDbf2CFviKrYFt7kgru1VA==",
+ "version": "0.0.24",
+ "resolved": "https://registry.npmjs.org/@lowire/loop/-/loop-0.0.24.tgz",
+ "integrity": "sha512-bB44u4Jvvq+jcXroNw389Vm0Xr1ejPLZzIfIJd1TgI1HB+ua4lvM8V6cOpS39PkU61XHUleT2rXjgP8pmnTKHQ==",
"dev": true,
"license": "Apache-2.0",
"engines": {
diff --git a/package.json b/package.json
index 18b8734385..2743e8f63a 100644
--- a/package.json
+++ b/package.json
@@ -55,7 +55,7 @@
"@eslint/compat": "^1.3.2",
"@eslint/eslintrc": "^3.3.1",
"@eslint/js": "^9.34.0",
- "@lowire/loop": "^0.0.23",
+ "@lowire/loop": "^0.0.24",
"@modelcontextprotocol/sdk": "^1.25.2",
"@octokit/graphql-schema": "^15.26.0",
"@stylistic/eslint-plugin": "^5.2.3",
diff --git a/packages/playwright-core/ThirdPartyNotices.txt b/packages/playwright-core/ThirdPartyNotices.txt
index 78ef6edab1..7529d56e37 100644
--- a/packages/playwright-core/ThirdPartyNotices.txt
+++ b/packages/playwright-core/ThirdPartyNotices.txt
@@ -5,7 +5,7 @@ THIRD-PARTY SOFTWARE NOTICES AND INFORMATION
This project incorporates components from the projects listed below. The original copyright notices and the licenses under which Microsoft received such components are set forth below. Microsoft reserves all rights not expressly granted herein, whether by implication, estoppel or otherwise.
- @hono/[email protected] (https://github.com/honojs/node-server)
-- @lowire/[email protected] (https://github.com/pavelfeldman/lowire)
+- @lowire/[email protected] (https://github.com/pavelfeldman/lowire)
- @modelcontextprotocol/[email protected] (https://github.com/modelcontextprotocol/typescript-sdk)
- [email protected] (https://github.com/jshttp/accepts)
- [email protected] (https://github.com/TooTallNate/proxy-agents)
@@ -500,7 +500,7 @@ MIT
=========================================
END OF @hono/[email protected] AND INFORMATION
-%% @lowire/[email protected] NOTICES AND INFORMATION BEGIN HERE
+%% @lowire/[email protected] NOTICES AND INFORMATION BEGIN HERE
=========================================
Apache License
Version 2.0, January 2004
@@ -704,7 +704,7 @@ Apache License
See the License for the specific language governing permissions and
limitations under the License.
=========================================
-END OF @lowire/[email protected] AND INFORMATION
+END OF @lowire/[email protected] AND INFORMATION
%% @modelcontextprotocol/[email protected] NOTICES AND INFORMATION BEGIN HERE
=========================================
diff --git a/packages/playwright-core/bundles/mcp/package-lock.json b/packages/playwright-core/bundles/mcp/package-lock.json
index 61586ebee9..54961ba85a 100644
--- a/packages/playwright-core/bundles/mcp/package-lock.json
+++ b/packages/playwright-core/bundles/mcp/package-lock.json
@@ -8,7 +8,7 @@
"name": "mcp-bundle",
"version": "0.0.1",
"dependencies": {
- "@lowire/loop": "^0.0.23",
+ "@lowire/loop": "^0.0.24",
"@modelcontextprotocol/sdk": "^1.25.2",
"zod": "^4.3.5",
"zod-to-json-schema": "^3.25.1"
@@ -27,9 +27,9 @@
}
},
"node_modules/@lowire/loop": {
- "version": "0.0.23",
- "resolved": "https://registry.npmjs.org/@lowire/loop/-/loop-0.0.23.tgz",
- "integrity": "sha512-zw4yFzto2T9g+CiHfubzxut0JMxA053XQ2syveqPjqofvRZyumhJR11RIQjKrPO9rDbf2CFviKrYFt7kgru1VA==",
+ "version": "0.0.24",
+ "resolved": "https://registry.npmjs.org/@lowire/loop/-/loop-0.0.24.tgz",
+ "integrity": "sha512-bB44u4Jvvq+jcXroNw389Vm0Xr1ejPLZzIfIJd1TgI1HB+ua4lvM8V6cOpS39PkU61XHUleT2rXjgP8pmnTKHQ==",
"license": "Apache-2.0",
"engines": {
"node": ">=20"
diff --git a/packages/playwright-core/bundles/mcp/package.json b/packages/playwright-core/bundles/mcp/package.json
index 039471d3ab..bc6f8d4d50 100644
--- a/packages/playwright-core/bundles/mcp/package.json
+++ b/packages/playwright-core/bundles/mcp/package.json
@@ -3,7 +3,7 @@
"version": "0.0.1",
"private": true,
"dependencies": {
- "@lowire/loop": "^0.0.23",
+ "@lowire/loop": "^0.0.24",
"@modelcontextprotocol/sdk": "^1.25.2",
"zod": "^4.3.5",
"zod-to-json-schema": "^3.25.1"
diff --git a/packages/playwright-core/src/server/agent/pageAgent.ts b/packages/playwright-core/src/server/agent/pageAgent.ts
index b87b74b2ce..dfad33bf99 100644
--- a/packages/playwright-core/src/server/agent/pageAgent.ts
+++ b/packages/playwright-core/src/server/agent/pageAgent.ts
@@ -95,7 +95,7 @@ async function runLoop(progress: Progress, context: Context, toolDefinitions: To
throw new Error(`This action requires API key to be set on the page agent.`);
const { full } = await page.snapshotForAI(progress);
- const { tools, callTool, reportedResult } = toolsForLoop(progress, context, toolDefinitions, { resultSchema });
+ const { tools, callTool, reportedResult, refusedToPerformReason } = toolsForLoop(progress, context, toolDefinitions, { resultSchema, refuseToPerform: 'allow' });
const secrets = Object.fromEntries((context.agentParams.secrets || [])?.map(s => ([s.name, s.value])));
const apiCacheTextBefore = context.agentParams.apiCacheFile ?
@@ -151,10 +151,13 @@ async function runLoop(progress: Progress, context: Context, toolDefinitions: To
}
}
+ if (refusedToPerformReason())
+ throw new Error(`Agent refused to perform action: ${refusedToPerformReason()}`);
+
if (error)
throw new Error(`Agentic loop failed: ${error}`);
- return { result: resultSchema ? reportedResult() : undefined };
+ return { result: reportedResult ? reportedResult() : undefined };
}
async function cachedPerform(progress: Progress, context: Context, cacheKey: string): Promise<actions.ActionWithCode[] | undefined> {
@@ -202,8 +205,8 @@ const allCaches = new Map<string, Cache>();
async function cachedActions(cacheFile: string): Promise<Cache> {
let cache = allCaches.get(cacheFile);
if (!cache) {
- const text = await fs.promises.readFile(cacheFile, 'utf-8').catch(() => '{}');
- const parsed = actions.cachedActionsSchema.safeParse(JSON.parse(text));
+ const json = await fs.promises.readFile(cacheFile, 'utf-8').then(text => JSON.parse(text)).catch(() => ({}));
+ const parsed = actions.cachedActionsSchema.safeParse(json);
if (parsed.error)
throw new Error(`Failed to parse cache file ${cacheFile}:\n${zod.prettifyError(parsed.error)}`);
cache = { actions: parsed.data, newActions: {} };
diff --git a/packages/playwright-core/src/server/agent/tool.ts b/packages/playwright-core/src/server/agent/tool.ts
index 0bf6ca9d53..3f219dbad1 100644
--- a/packages/playwright-core/src/server/agent/tool.ts
+++ b/packages/playwright-core/src/server/agent/tool.ts
@@ -34,7 +34,14 @@ export function defineTool<Input extends zod.Schema>(tool: ToolDefinition<Input>
return tool;
}
-export function toolsForLoop(progress: Progress, context: Context, toolDefinitions: ToolDefinition[], options: { resultSchema?: loopTypes.Schema } = {}): { tools: loopTypes.Tool[], callTool: loopTypes.ToolCallback, reportedResult: () => any } {
+type ToolsForLoop = {
+ tools: loopTypes.Tool[];
+ callTool: loopTypes.ToolCallback;
+ reportedResult?: () => any;
+ refusedToPerformReason: () => string | undefined;
+};
+
+export function toolsForLoop(progress: Progress, context: Context, toolDefinitions: ToolDefinition[], options: { resultSchema?: loopTypes.Schema, refuseToPerform?: 'allow' | 'deny' } = {}): ToolsForLoop {
const tools = toolDefinitions.map(tool => {
const result: loopTypes.Tool = {
name: tool.schema.name,
@@ -43,6 +50,7 @@ export function toolsForLoop(progress: Progress, context: Context, toolDefinitio
};
return result;
});
+
if (options.resultSchema) {
tools.push({
name: 'report_result',
@@ -51,7 +59,25 @@ export function toolsForLoop(progress: Progress, context: Context, toolDefinitio
});
}
+ if (options.refuseToPerform === 'allow') {
+ tools.push({
+ name: 'refuse_to_perform',
+ description: 'Refuse to perform action.',
+ inputSchema: {
+ type: 'object',
+ properties: {
+ reason: {
+ type: 'string',
+ description: `Call this when you believe that you can't perform the action because something is wrong with the page. The reason will be reported to the user.`,
+ },
+ },
+ required: ['reason'],
+ },
+ });
+ }
+
let reportedResult: any;
+ let refusedToPerformReason: string | undefined;
const callTool: loopTypes.ToolCallback = async params => {
if (params.name === 'report_result') {
@@ -62,6 +88,14 @@ export function toolsForLoop(progress: Progress, context: Context, toolDefinitio
};
}
+ if (params.name === 'refuse_to_perform') {
+ refusedToPerformReason = params.arguments.reason;
+ return {
+ content: [{ type: 'text', text: 'Done' }],
+ isError: false,
+ };
+ }
+
const tool = toolDefinitions.find(t => t.schema.name === params.name);
if (!tool) {
return {
@@ -85,6 +119,7 @@ export function toolsForLoop(progress: Progress, context: Context, toolDefinitio
return {
tools,
callTool,
- reportedResult: () => reportedResult,
+ reportedResult: options.resultSchema ? () => reportedResult : undefined,
+ refusedToPerformReason: () => refusedToPerformReason,
};
}
diff --git a/tests/library/__llm_cache__/library-agent-expect-expect-timeout-during-generate.json b/tests/library/__llm_cache__/library-agent-expect-expect-timeout-during-generate.json
index 81430036ec..e3550a93fa 100644
--- a/tests/library/__llm_cache__/library-agent-expect-expect-timeout-during-generate.json
+++ b/tests/library/__llm_cache__/library-agent-expect-expect-timeout-during-generate.json
@@ -1,117 +1,11 @@
{
- "0e1e6182377f3ddc9e605d47dc0f4e5db456be40": {
+ "428b1de09afa2d1be4a61fdf7eaa2277bc1426ed": {
"result": {
"role": "assistant",
"content": [
{
"type": "text",
- "text": "The assertion has been executed. The text \"hello\" is **not visible** on the page - the assertion failed as expected since the page snapshot only shows \"bye\" text. The tool has verified that \"hello\" cannot be found on the page."
- }
- ],
- "stopReason": {
- "code": "other",
- "message": "Unexpected stop reason: end_turn"
- }
- },
- "usage": {
- "input": 1912,
- "output": 54
- }
- },
- "1bae81f5ca3ad400fc3c0eac1ce49bc686b8abc5": {
- "result": {
- "role": "assistant",
- "content": [
- {
- "type": "text",
- "text": "I need to verify that the input has the value \"hello\", but according to the page snapshot, the textbox currently has the value \"bye\". Let me check if this assertion should pass or fail."
- },
- {
- "type": "tool_call",
- "name": "browser_expect_value",
- "arguments": {
- "type": "textbox",
- "element": "input",
- "ref": "e2",
- "value": "hello",
- "_is_done": true
- },
- "id": "toolu_01RiUMzGS2NmB6d6scMVR6pR"
- }
- ],
- "stopReason": {
- "code": "ok"
- }
- },
- "usage": {
- "input": 1904,
- "output": 169
- }
- },
- "8607f6ac706485320f56bfd2781e2fb21cd8a16c": {
- "result": {
- "role": "assistant",
- "content": [
- {
- "type": "text",
- "text": "I'll verify that the input has the value \"hello\" using the browser_expect_value tool."
- },
- {
- "type": "tool_call",
- "name": "browser_expect_value",
- "arguments": {
- "type": "textbox",
- "element": "input",
- "ref": "e2",
- "value": "hello",
- "_is_done": true
- },
- "id": "toolu_019V7P8Y8Nv1K5EwdXLr32mX"
- }
- ],
- "stopReason": {
- "code": "ok"
- }
- },
- "usage": {
- "input": 1914,
- "output": 149
- }
- },
- "b4fd4ea81b4c3d84f354169c7a97dd0f708710d2": {
- "result": {
- "role": "assistant",
- "content": [
- {
- "type": "text",
- "text": "I need to verify that the text \"hello\" is visible on the page. Based on the page snapshot, I can see there's only text \"bye\" visible, not \"hello\". However, I should still make the assertion to check for \"hello\"."
- },
- {
- "type": "tool_call",
- "name": "browser_expect_visible_text",
- "arguments": {
- "text": "hello",
- "_is_done": true
- },
- "id": "toolu_01WePrBxxAHf7CS13nU77ay2"
- }
- ],
- "stopReason": {
- "code": "ok"
- }
- },
- "usage": {
- "input": 1694,
- "output": 129
- }
- },
- "c8a85f5c1a56b73cf3fc1b00e01b45e9a78b94be": {
- "result": {
- "role": "assistant",
- "content": [
- {
- "type": "text",
- "text": "I'll verify that the input has the value \"hello\"."
+ "text": "I need to verify that an input has the value \"hello\", but according to the page snapshot, the textbox with ref=e2 currently has the value \"bye\". This means the expectation is that it should NOT match, or this is testing a failure case.\n\nLet me call the browser_expect_value tool to assert the expected condition:"
},
{
"type": "tool_call",
@@ -123,7 +17,7 @@
"value": "hello",
"_is_done": true
},
- "id": "toolu_011Tt781hyZUbLUUQMVyhdxB"
+ "id": "toolu_01B834GHxjywVnc9UGFxpyGc"
}
],
"stopReason": {
@@ -131,95 +25,8 @@
}
},
"usage": {
- "input": 1695,
- "output": 140
- }
- },
- "d1b0a255ce65ac9821522445c857072232860b1f": {
- "result": {
- "role": "assistant",
- "content": [
- {
- "type": "text",
- "text": "I need to verify that an input has the value \"hello\", but according to the page snapshot, the textbox currently has the value \"bye\". Let me assert the expected condition."
- },
- {
- "type": "tool_call",
- "name": "browser_expect_value",
- "arguments": {
- "type": "textbox",
- "element": "input textbox",
- "ref": "e2",
- "value": "hello",
- "_is_done": true
- },
- "id": "toolu_01R4pnkKVFv6Crn2Dysx2XZh"
- }
- ],
- "stopReason": {
- "code": "ok"
- }
- },
- "usage": {
- "input": 1703,
- "output": 167
- }
- },
- "e59b1ef4668b6d7d8d0fbad3fc1a0a3172d6d07b": {
- "result": {
- "role": "assistant",
- "content": [
- {
- "type": "text",
- "text": "The assertion has been executed. The test expectation that text \"hello\" is visible has **failed** because the text \"hello\" was not found on the page. According to the page snapshot, only the text \"bye\" is present in a generic element, not \"hello\"."
- }
- ],
- "stopReason": {
- "code": "other",
- "message": "Unexpected stop reason: end_turn"
- }
- },
- "usage": {
- "input": 1895,
- "output": 60
- }
- },
- "e94f4d28220373c8f84256201c83f8cf744ff680": {
- "result": {
- "role": "assistant",
- "content": [
- {
- "type": "text",
- "text": "The assertion has been executed. As expected based on the page snapshot showing only \"bye\", the text \"hello\" is **not visible** on the page. The test failed because \"hello\" could not be found."
- }
- ],
- "stopReason": {
- "code": "other",
- "message": "Unexpected stop reason: end_turn"
- }
- },
- "usage": {
- "input": 1930,
- "output": 47
- }
- },
- "ffe7d66a201de8f2a816d99550859ad43fee1ffb": {
- "result": {
- "role": "assistant",
- "content": [
- {
- "type": "text",
- "text": "The assertion failed as expected - the text \"hello\" is not visible on the page. The page only contains the text \"bye\". The task has been completed by calling the browser_expect_visible_text tool to verify the condition."
- }
- ],
- "stopReason": {
- "code": "other",
- "message": "Unexpected stop reason: end_turn"
- }
- },
- "usage": {
- "input": 1913,
- "output": 51
+ "input": 2044,
+ "output": 201
}
}
}
\ No newline at end of file
diff --git a/tests/library/__llm_cache__/library-agent-expect-expect-timeout-during-run.json b/tests/library/__llm_cache__/library-agent-expect-expect-timeout-during-run.json
index a6cdb094d1..398dcc1bc2 100644
--- a/tests/library/__llm_cache__/library-agent-expect-expect-timeout-during-run.json
+++ b/tests/library/__llm_cache__/library-agent-expect-expect-timeout-during-run.json
@@ -1,11 +1,11 @@
{
- "2f793c8f90663bb9a07713a1517a27e4f31bbe5d": {
+ "46dd9ff2a304413e15a4579295016e8d2626b5dd": {
"result": {
"role": "assistant",
"content": [
{
"type": "text",
- "text": "I need to verify that the submit button is visible on the page. Based on the page snapshot, I can see there is a button with the accessible name \"Submit\" and reference e2."
+ "text": "I need to verify that the submit button is visible on the page. Based on the page snapshot, I can see there is a button with the text \"Submit\" and reference e2."
},
{
"type": "tool_call",
@@ -15,7 +15,7 @@
"accessibleName": "Submit",
"_is_done": true
},
- "id": "toolu_012fUiXdoE5a8Jf3Acjcecyx"
+ "id": "toolu_011XpmmmcGH5bLRG1hznFRw2"
}
],
"stopReason": {
@@ -23,92 +23,8 @@
}
},
"usage": {
- "input": 1913,
- "output": 133
- }
- },
- "36a18b6605bee5a65c864ccc5d3bbf7e6ea502ff": {
- "result": {
- "role": "assistant",
- "content": [
- {
- "type": "text",
- "text": "I'll verify that the submit button is visible on the page."
- },
- {
- "type": "tool_call",
- "name": "browser_expect_visible",
- "arguments": {
- "role": "button",
- "accessibleName": "Submit",
- "_is_done": true
- },
- "id": "toolu_01DScX62tpJaVRsYz9udMrnX"
- }
- ],
- "stopReason": {
- "code": "ok"
- }
- },
- "usage": {
- "input": 1702,
- "output": 107
- }
- },
- "3b4f08f411f9002019b8fab8f14772f4b4beb15a": {
- "result": {
- "role": "assistant",
- "content": [
- {
- "type": "text",
- "text": "I'll verify that the submit button is visible on the page."
- },
- {
- "type": "tool_call",
- "name": "browser_expect_visible",
- "arguments": {
- "role": "button",
- "accessibleName": "Submit",
- "_is_done": true
- },
- "id": "toolu_01CyQoKismpKpLjmnSHrnn8M"
- }
- ],
- "stopReason": {
- "code": "ok"
- }
- },
- "usage": {
- "input": 1694,
- "output": 107
- }
- },
- "54b48753b3ead457e43007b3392ddf6d6e766a24": {
- "result": {
- "role": "assistant",
- "content": [
- {
- "type": "text",
- "text": "I'll verify that the submit button is visible on the page."
- },
- {
- "type": "tool_call",
- "name": "browser_expect_visible",
- "arguments": {
- "role": "button",
- "accessibleName": "Submit",
- "_is_done": true
- },
- "id": "toolu_013C42qJWChPvhte6hjXDvbN"
- }
- ],
- "stopReason": {
- "code": "ok"
- }
- },
- "usage": {
- "input": 1903,
- "output": 107
+ "input": 2043,
+ "output": 132
}
}
}
\ No newline at end of file
diff --git a/tests/library/__llm_cache__/library-agent-expect-expectURL-success.json b/tests/library/__llm_cache__/library-agent-expect-expectURL-success.json
index 68b30da8bc..582777084c 100644
--- a/tests/library/__llm_cache__/library-agent-expect-expectURL-success.json
+++ b/tests/library/__llm_cache__/library-agent-expect-expectURL-success.json
@@ -1,65 +1,11 @@
{
- "0ab3f26cdb4bc0570886e11eec003276c6c8afd7": {
+ "01e4dfe3e3fd3764d430e1d983db863b06e4ea31": {
"result": {
"role": "assistant",
"content": [
{
"type": "text",
- "text": "The assertion failed because the actual URL is `http://localhost:8923/page.html` but I was expecting just `/page.html`. Since the task specified to verify that the page URL is `/page.html` and the actual URL ends with `/page.html` (just with the full origin included), the relative URL matches. However, the tool interpreted this as an exact match which failed.\n\nLet me use a regex pattern to match the URL properly:"
- },
- {
- "type": "tool_call",
- "name": "browser_expect_url",
- "arguments": {
- "regex": "/page\\.html$/",
- "_is_done": true
- },
- "id": "toolu_014vcANTM69AtdexNDHzDyhF"
- }
- ],
- "stopReason": {
- "code": "ok"
- }
- },
- "usage": {
- "input": 2177,
- "output": 175
- }
- },
- "218b122dd19ab410fa8e1982e7330a92175a3f4a": {
- "result": {
- "role": "assistant",
- "content": [
- {
- "type": "text",
- "text": "I see the issue - the page URL is the full URL `http://localhost:8907/page.html`, not just the path. Let me adjust the assertion to use a regex pattern that will match the path portion:"
- },
- {
- "type": "tool_call",
- "name": "browser_expect_url",
- "arguments": {
- "regex": "/page\\.html$/",
- "_is_done": true
- },
- "id": "toolu_01C6XynJnfEwtCQLmMJinLxi"
- }
- ],
- "stopReason": {
- "code": "ok"
- }
- },
- "usage": {
- "input": 2165,
- "output": 125
- }
- },
- "3393f1e7925b53d87c1ebcdc7411dc04bfc11aec": {
- "result": {
- "role": "assistant",
- "content": [
- {
- "type": "text",
- "text": "I need to verify that the page URL is `/page.html`. Let me check this using the browser_expect_url tool."
+ "text": "I'll verify that the page URL is /page.html."
},
{
"type": "tool_call",
@@ -68,34 +14,7 @@
"url": "/page.html",
"_is_done": true
},
- "id": "toolu_0126LdEZe32PoAJfJwrSG8PT"
- }
- ],
- "stopReason": {
- "code": "ok"
- }
- },
- "usage": {
- "input": 1930,
- "output": 105
- }
- },
- "421d60b4ee07460745af9a285bf151c2690ba498": {
- "result": {
- "role": "assistant",
- "content": [
- {
- "type": "text",
- "text": "I see the issue - the actual URL is the full URL `http://localhost:8955/page.html`, not just the path `/page.html`. Let me use a regex pattern to match the URL that ends with `/page.html`:"
- },
- {
- "type": "tool_call",
- "name": "browser_expect_url",
- "arguments": {
- "regex": "/page\\.html$/",
- "_is_done": true
- },
- "id": "toolu_01PVEVCMMAbyRbV8ryw4Vq21"
+ "id": "toolu_01FUsQbLdCVmZyUCS1DwGRrC"
}
],
"stopReason": {
@@ -103,17 +22,17 @@
}
},
"usage": {
- "input": 2169,
- "output": 130
+ "input": 2067,
+ "output": 91
}
},
- "4c51af7388a0114714b58e25495f23f5fb9306b8": {
+ "d3eb8b6dc11f060023ba45368c64d302b7ff7891": {
"result": {
"role": "assistant",
"content": [
{
"type": "text",
- "text": "I see the issue. The actual URL is `http://localhost:8911/page.html`, not just `/page.html`. Since the expectation is for `/page.html`, I should use a regex pattern to match URLs ending with `/page.html`."
+ "text": "The assertion failed because the actual URL is the full URL \"http://localhost:8939/page.html\" while I was checking for just \"/page.html\". Let me verify using a regex pattern to match the URL ending with /page.html:"
},
{
"type": "tool_call",
@@ -122,61 +41,7 @@
"regex": "/\\/page\\.html$/",
"_is_done": true
},
- "id": "toolu_01Y3CweZwCdm3TArebNtVFV5"
- }
- ],
- "stopReason": {
- "code": "ok"
- }
- },
- "usage": {
- "input": 2169,
- "output": 135
- }
- },
- "a38f3c941b97b7afde8ece0b7368b4edcca455e5": {
- "result": {
- "role": "assistant",
- "content": [
- {
- "type": "text",
- "text": "I need to verify that the page URL is /page.html. Let me check the current URL using the browser_expect_url tool."
- },
- {
- "type": "tool_call",
- "name": "browser_expect_url",
- "arguments": {
- "url": "/page.html",
- "_is_done": true
- },
- "id": "toolu_01YYawpRAjhy6h1i9LiJ69UL"
- }
- ],
- "stopReason": {
- "code": "ok"
- }
- },
- "usage": {
- "input": 1940,
- "output": 107
- }
- },
- "e8447403254d31a8e1e4efb516154ffc41734f43": {
- "result": {
- "role": "assistant",
- "content": [
- {
- "type": "text",
- "text": "I see the issue - the actual URL is the full URL `http://localhost:8919/page.html`, but I was checking for just the path `/page.html`. Let me use a regex pattern to match the URL correctly:"
- },
- {
- "type": "tool_call",
- "name": "browser_expect_url",
- "arguments": {
- "regex": "/page\\.html$/",
- "_is_done": true
- },
- "id": "toolu_014rDSbLQdqsb7WMEn4hsWRz"
+ "id": "toolu_01FDFEjR382yKedBJ8LZgwgZ"
}
],
"stopReason": {
@@ -184,8 +49,8 @@
}
},
"usage": {
- "input": 2169,
- "output": 128
+ "input": 2285,
+ "output": 132
}
}
}
\ No newline at end of file
diff --git a/tests/library/__llm_cache__/library-agent-expect-expectURL-with-regex-error.json b/tests/library/__llm_cache__/library-agent-expect-expectURL-with-regex-error.json
index c5edaebdd3..b502ab8a39 100644
--- a/tests/library/__llm_cache__/library-agent-expect-expectURL-with-regex-error.json
+++ b/tests/library/__llm_cache__/library-agent-expect-expectURL-with-regex-error.json
@@ -1,11 +1,11 @@
{
- "2f730b994531bc8f4c51f10ad8808813edf4734c": {
+ "3cc51fa90640dadb2fcec777605e29dccb799cf8": {
"result": {
"role": "assistant",
"content": [
{
"type": "text",
- "text": "I'll verify that the page URL matches the pattern \"/other pattern\" using the browser_expect_url tool with a regex pattern."
+ "text": "I need to verify that the page URL matches the pattern \"/other pattern\". Based on the instruction, I'll use the browser_expect_url tool with a regex parameter."
},
{
"type": "tool_call",
@@ -14,7 +14,7 @@
"regex": "/other pattern/",
"_is_done": true
},
- "id": "toolu_01FqR5UsHs6iK4WS9FovRnTS"
+ "id": "toolu_01Ta794yU6itVbHgDmgzvFwn"
}
],
"stopReason": {
@@ -22,17 +22,17 @@
}
},
"usage": {
- "input": 1929,
- "output": 105
+ "input": 2066,
+ "output": 113
}
},
- "46ba0cb957a62c739f3277070f2ddd419f5ed933": {
+ "6ab2134b9621c91d1c4f3ec9752685d78be95bae": {
"result": {
"role": "assistant",
"content": [
{
"type": "text",
- "text": "The assertion failed as expected - the actual URL is \"http://localhost:8939/other.html\" which doesn't match the literal pattern \"/other pattern\". Let me try with a pattern that would match \"other\" in the URL:"
+ "text": "The assertion failed because the URL \"http://localhost:8951/other.html\" doesn't match the literal pattern \"/other pattern\". Looking at the actual URL, it contains \"other\" but not \"other pattern\" as a literal string. Let me try a regex that would match URLs containing \"other\":"
},
{
"type": "tool_call",
@@ -41,7 +41,7 @@
"regex": "/other/",
"_is_done": true
},
- "id": "toolu_01AJcyL9UM3UXBhaQeSkCpej"
+ "id": "toolu_015dWLmUoYJCKhUsLF5xBWfd"
}
],
"stopReason": {
@@ -49,200 +49,8 @@
}
},
"usage": {
- "input": 2165,
- "output": 126
- }
- },
- "59c33e4446941ff9c75fec191f1e6b70a2b49cd9": {
- "result": {
- "role": "assistant",
- "content": [
- {
- "type": "text",
- "text": "The assertion has been executed and shows that the page URL does not match the pattern \"/other pattern\". The actual URL is \"http://localhost:8911/other.html\" which does not contain the literal text \"other pattern\", so the expectation failed as expected."
- }
- ],
- "stopReason": {
- "code": "other",
- "message": "Unexpected stop reason: end_turn"
- }
- },
- "usage": {
- "input": 2183,
- "output": 58
- }
- },
- "6023ad3da6c0369a29dc38fa2cb347dfff9b0dd0": {
- "result": {
- "role": "assistant",
- "content": [
- {
- "type": "text",
- "text": "The assertion failed as expected. The page URL is \"http://localhost:8935/other.html\" which does not match the literal pattern \"/other pattern\". Based on the actual URL, it appears the expectation should be looking for URLs containing \"other\" (like \"/other.*/\" pattern), but I've completed the task by calling the assertion tool with the pattern specified in the instructions."
- }
- ],
- "stopReason": {
- "code": "other",
- "message": "Unexpected stop reason: end_turn"
- }
- },
- "usage": {
- "input": 2165,
- "output": 83
- }
- },
- "736beb67fdcdc4b855cff4fb1fe7467b9db5c0f1": {
- "result": {
- "role": "assistant",
- "content": [
- {
- "type": "text",
- "text": "The assertion correctly failed. The page URL is `http://localhost:8907/other.html` which does not contain the literal string \"other pattern\". The regex pattern `/other pattern/` was looking for that exact phrase, but the URL only contains \"other\" (not \"other pattern\").\n\nThe test has been executed and the expectation verification is complete - it confirmed that the URL does NOT match \"/other pattern\" as the actual URL is \"http://localhost:8907/other.html\"."
- }
- ],
- "stopReason": {
- "code": "other",
- "message": "Unexpected stop reason: end_turn"
- }
- },
- "usage": {
- "input": 2183,
- "output": 107
- }
- },
- "86340f4c5b8758cd8c4d90891f8db2d12a43d8b9": {
- "result": {
- "role": "assistant",
- "content": [
- {
- "type": "text",
- "text": "The assertion failed as expected. The page URL \"http://localhost:8927/other.html\" does not match the pattern \"/other pattern/\" literally. Based on the actual URL, it seems the pattern should match \"other\" in the URL path. Let me try with a more appropriate regex pattern that would match URLs containing \"other\":"
- },
- {
- "type": "tool_call",
- "name": "browser_expect_url",
- "arguments": {
- "regex": "/other/",
- "_is_done": true
- },
- "id": "toolu_01Fr7pVCf5v1DdVkBDYvfLbE"
- }
- ],
- "stopReason": {
- "code": "ok"
- }
- },
- "usage": {
- "input": 2165,
- "output": 146
- }
- },
- "c16c631161a5bad15dde474422599729c0af53ae": {
- "result": {
- "role": "assistant",
- "content": [
- {
- "type": "text",
- "text": "I see the URL is \"http://localhost:8931/other.html\". Let me adjust the regex pattern to match this URL correctly - it should match URLs containing \"other\":"
- },
- {
- "type": "tool_call",
- "name": "browser_expect_url",
- "arguments": {
- "regex": "/other/",
- "_is_done": true
- },
- "id": "toolu_0114EiKaVWty1bVGXt1CrFTt"
- }
- ],
- "stopReason": {
- "code": "ok"
- }
- },
- "usage": {
- "input": 2165,
- "output": 114
- }
- },
- "ce3f938674ba0ecfceaf13ee135fda66576c596c": {
- "result": {
- "role": "assistant",
- "content": [
- {
- "type": "text",
- "text": "I see the URL is \"http://localhost:8907/other.html\". Let me adjust the regex pattern to match URLs containing \"other\" more appropriately:"
- },
- {
- "type": "tool_call",
- "name": "browser_expect_url",
- "arguments": {
- "regex": "/other/",
- "_is_done": true
- },
- "id": "toolu_01QmJpJPqx7PV9iDBD3FBYcb"
- }
- ],
- "stopReason": {
- "code": "ok"
- }
- },
- "usage": {
- "input": 2165,
- "output": 111
- }
- },
- "d13ffedc4413ee13384ef1acf3268bfcb820f928": {
- "result": {
- "role": "assistant",
- "content": [
- {
- "type": "text",
- "text": "I'll verify that the page URL matches the pattern \"/other pattern\"."
- },
- {
- "type": "tool_call",
- "name": "browser_expect_url",
- "arguments": {
- "regex": "/other pattern/",
- "_is_done": true
- },
- "id": "toolu_019xZSSP6K4qwyddW6qiCT6f"
- }
- ],
- "stopReason": {
- "code": "ok"
- }
- },
- "usage": {
- "input": 1939,
- "output": 92
- }
- },
- "fbd0e14379c9096f037eb62a879c4a56d670a4b4": {
- "result": {
- "role": "assistant",
- "content": [
- {
- "type": "text",
- "text": "I can see the assertion failed. The page URL is \"http://localhost:8907/other.html\" but the expected pattern \"/other pattern/\" doesn't match because the URL contains \"other.html\" not \"other pattern\". \n\nBased on the error, it appears the test expectation might be looking for a different pattern. Let me try with a pattern that would match the actual URL containing \"other\":"
- },
- {
- "type": "tool_call",
- "name": "browser_expect_url",
- "arguments": {
- "regex": "/other/",
- "_is_done": true
- },
- "id": "toolu_01MVvSR1g2wTZNDtSU99oih8"
- }
- ],
- "stopReason": {
- "code": "ok"
- }
- },
- "usage": {
- "input": 2162,
- "output": 162
+ "input": 2307,
+ "output": 140
}
}
}
\ No newline at end of file
diff --git a/tests/library/__llm_cache__/library-agent-expect-expectURL-with-regex.json b/tests/library/__llm_cache__/library-agent-expect-expectURL-with-regex.json
index a6628013fd..5b77e882f8 100644
--- a/tests/library/__llm_cache__/library-agent-expect-expectURL-with-regex.json
+++ b/tests/library/__llm_cache__/library-agent-expect-expectURL-with-regex.json
@@ -1,38 +1,11 @@
{
- "33442824e378e1a876db44e19303faf430e8d492": {
+ "ca4fa24ca5d10d373f005b3e8a5b543a767e10b5": {
"result": {
"role": "assistant",
"content": [
{
"type": "text",
- "text": "I need to fix the regex format - it needs to be a complete regex pattern with closing slash and optional flags."
- },
- {
- "type": "tool_call",
- "name": "browser_expect_url",
- "arguments": {
- "regex": "/page/",
- "_is_done": true
- },
- "id": "toolu_01UKuRAcqrarenSYTfGLJrc9"
- }
- ],
- "stopReason": {
- "code": "ok"
- }
- },
- "usage": {
- "input": 2113,
- "output": 100
- }
- },
- "acad1ba738fafb4da99f83a65a860de664ef76df": {
- "result": {
- "role": "assistant",
- "content": [
- {
- "type": "text",
- "text": "I need to verify that the page URL matches the `/page` pattern. Based on the task instructions, I'll use the `browser_expect_url` tool with a regex pattern."
+ "text": "I'll verify that the page URL matches the \"/page\" pattern using a regular expression."
},
{
"type": "tool_call",
@@ -41,34 +14,7 @@
"regex": "/page",
"_is_done": true
},
- "id": "toolu_01QjsUFVsnMdg2pnrD4nxwWV"
- }
- ],
- "stopReason": {
- "code": "ok"
- }
- },
- "usage": {
- "input": 1939,
- "output": 114
- }
- },
- "bd797d79c467ada040b7fc76e8ca1af7207ba883": {
- "result": {
- "role": "assistant",
- "content": [
- {
- "type": "text",
- "text": "I need to fix the regex pattern - it needs to end with a `/` and optional flags. Let me correct this:"
- },
- {
- "type": "tool_call",
- "name": "browser_expect_url",
- "arguments": {
- "regex": "/page/",
- "_is_done": true
- },
- "id": "toolu_01RvJkVKngFkoTURYdZsxGFi"
+ "id": "toolu_01A8AJ2KBrNTj6wSJ6bfSiFS"
}
],
"stopReason": {
@@ -76,17 +22,17 @@
}
},
"usage": {
- "input": 2130,
- "output": 102
+ "input": 2066,
+ "output": 94
}
},
- "c67b3d627e92564582281fd7713d0f3de70a3a44": {
+ "f6b9debe1e2eb0222d96e3fb7ef1685eac351d81": {
"result": {
"role": "assistant",
"content": [
{
"type": "text",
- "text": "I need to fix the regex pattern by properly closing it with a forward slash. Let me call the tool again with the correct regex format."
+ "text": "I need to provide a properly formatted regex with closing slash. Let me fix that:"
},
{
"type": "tool_call",
@@ -95,34 +41,7 @@
"regex": "/page/",
"_is_done": true
},
- "id": "toolu_013q6jNHgXStdhxzEYZsvFTs"
- }
- ],
- "stopReason": {
- "code": "ok"
- }
- },
- "usage": {
- "input": 2116,
- "output": 105
- }
- },
- "ef7f62a8d800e2412aaf593b1b85c1254c504756": {
- "result": {
- "role": "assistant",
- "content": [
- {
- "type": "text",
- "text": "I need to verify that the page URL matches the pattern `/page`. I'll use the `browser_expect_url` tool with a regex pattern."
- },
- {
- "type": "tool_call",
- "name": "browser_expect_url",
- "arguments": {
- "regex": "/page",
- "_is_done": true
- },
- "id": "toolu_017sobJzrZUJ9pxgJ7hSuv5u"
+ "id": "toolu_019yRn6jCMA4QSeBRD6aZsiy"
}
],
"stopReason": {
@@ -130,8 +49,8 @@
}
},
"usage": {
- "input": 1929,
- "output": 107
+ "input": 2234,
+ "output": 94
}
}
}
\ No newline at end of file
diff --git a/tests/library/__llm_cache__/library-agent-expect-expectURL-wrong-URL-error.json b/tests/library/__llm_cache__/library-agent-expect-expectURL-wrong-URL-error.json
index 4a56d82260..64448ae512 100644
--- a/tests/library/__llm_cache__/library-agent-expect-expectURL-wrong-URL-error.json
+++ b/tests/library/__llm_cache__/library-agent-expect-expectURL-wrong-URL-error.json
@@ -1,119 +1,11 @@
{
- "0179ac4e63cc9a0733da8a619e2e8cc2b6d65d1d": {
+ "0e39a03d4c9db9c59b450aa30d50eb88551aca6c": {
"result": {
"role": "assistant",
"content": [
{
"type": "text",
- "text": "I need to verify that the page URL is `/other.html`. I'll use the `browser_expect_url` tool to assert this condition."
- },
- {
- "type": "tool_call",
- "name": "browser_expect_url",
- "arguments": {
- "url": "/other.html",
- "_is_done": true
- },
- "id": "toolu_01G9ZzdYXENshXQ8jShDmvJy"
- }
- ],
- "stopReason": {
- "code": "ok"
- }
- },
- "usage": {
- "input": 1930,
- "output": 109
- }
- },
- "6aacb908b26146c99406cec686aaf78e2398d3c8": {
- "result": {
- "role": "assistant",
- "content": [
- {
- "type": "text",
- "text": "I see the issue - the actual URL is the full URL `http://localhost:8959/other.html`, not just the relative path. Let me adjust the assertion to match the actual URL pattern using a regex that checks if the URL ends with `/other.html`:"
- },
- {
- "type": "tool_call",
- "name": "browser_expect_url",
- "arguments": {
- "regex": "/\\/other\\.html$/",
- "_is_done": true
- },
- "id": "toolu_011zZt5A2gYEv1VwEAmjNTLm"
- }
- ],
- "stopReason": {
- "code": "ok"
- }
- },
- "usage": {
- "input": 2169,
- "output": 136
- }
- },
- "bae6a40c194c52f6df380d43bae1648192ad1f8a": {
- "result": {
- "role": "assistant",
- "content": [
- {
- "type": "text",
- "text": "I see the issue - the actual URL is the full URL `http://localhost:8923/other.html`, not just the path. Let me adjust the expectation to use a regex pattern that matches the path portion:"
- },
- {
- "type": "tool_call",
- "name": "browser_expect_url",
- "arguments": {
- "regex": "/\\/other\\.html$/",
- "_is_done": true
- },
- "id": "toolu_01NynnYbKYmAHGQm11tzRFgr"
- }
- ],
- "stopReason": {
- "code": "ok"
- }
- },
- "usage": {
- "input": 2169,
- "output": 126
- }
- },
- "e50461d297e7440944832db61c5224052f70bc47": {
- "result": {
- "role": "assistant",
- "content": [
- {
- "type": "text",
- "text": "I see the issue - the actual URL is the full URL \"http://localhost:8927/other.html\" but I was checking for just \"/other.html\". Since the expectation is to match \"/other.html\" and relative URLs are resolved against the baseURL, let me use a regex pattern to match the path portion."
- },
- {
- "type": "tool_call",
- "name": "browser_expect_url",
- "arguments": {
- "regex": "/\\/other\\.html$/",
- "_is_done": true
- },
- "id": "toolu_01H5V6RogtF5gd4887vh6cpF"
- }
- ],
- "stopReason": {
- "code": "ok"
- }
- },
- "usage": {
- "input": 2176,
- "output": 149
- }
- },
- "f3512ecc7b43a5238007d361e22f0c67faf72cbf": {
- "result": {
- "role": "assistant",
- "content": [
- {
- "type": "text",
- "text": "I see the issue - the actual URL is the full URL `http://localhost:8911/other.html` while I was checking for just `/other.html`. Let me use a regex pattern to match URLs ending with `/other.html`:"
+ "text": "I need to adjust my expectation. The actual URL is the full URL including the protocol and host. Let me use a regex pattern to match the URL ending with /other.html."
},
{
"type": "tool_call",
@@ -122,7 +14,7 @@
"regex": "/\\/other\\.html$/",
"_is_done": true
},
- "id": "toolu_01GULjRje8iHqa5c37r9dtTW"
+ "id": "toolu_018zDqYhtHtaNUQ66sv1Ezea"
}
],
"stopReason": {
@@ -130,17 +22,17 @@
}
},
"usage": {
- "input": 2169,
- "output": 131
+ "input": 2293,
+ "output": 118
}
},
- "fec139d134efb094c1b78c714e439f48f10a584b": {
+ "e50f042f63f65841f33a8d0a608d45f748c66552": {
"result": {
"role": "assistant",
"content": [
{
"type": "text",
- "text": "I need to verify that the page URL is /other.html. I'll use the browser_expect_url tool to check this."
+ "text": "I'll verify that the page URL is /other.html using the browser_expect_url tool."
},
{
"type": "tool_call",
@@ -149,7 +41,7 @@
"url": "/other.html",
"_is_done": true
},
- "id": "toolu_01QSmNsYFUNPwiG7r6Qvmi5y"
+ "id": "toolu_01QffxWJVHBPDgB4WSiKanvu"
}
],
"stopReason": {
@@ -157,8 +49,8 @@
}
},
"usage": {
- "input": 1940,
- "output": 106
+ "input": 2067,
+ "output": 99
}
}
}
\ No newline at end of file
diff --git a/tests/library/__llm_cache__/library-agent-limits-should-respect-max-actions-limit.json b/tests/library/__llm_cache__/library-agent-limits-should-respect-max-actions-limit.json
index 10bb8eb63f..53c33e56d0 100644
--- a/tests/library/__llm_cache__/library-agent-limits-should-respect-max-actions-limit.json
+++ b/tests/library/__llm_cache__/library-agent-limits-should-respect-max-actions-limit.json
@@ -1,12 +1,8 @@
{
- "1649ec8e26c0df7c492073e66fb5a5ce9123566e": {
+ "63c6840169b6554ee4319607e3c5582842511481": {
"result": {
"role": "assistant",
"content": [
- {
- "type": "text",
- "text": "I'll click the submit button 5 times. Let me start by clicking it the first time."
- },
{
"type": "tool_call",
"name": "browser_click",
@@ -15,7 +11,7 @@
"ref": "e2",
"_is_done": false
},
- "id": "toolu_01QhRaHw1847uzCZdpZSa9CQ"
+ "id": "toolu_01LMKpSXqCaRPAsUfpuLCGq6"
}
],
"stopReason": {
@@ -23,41 +19,17 @@
}
},
"usage": {
- "input": 2917,
- "output": 113
- }
- },
- "4a314579eb770c2922736092aa797cca43bbcfbf": {
- "result": {
- "role": "assistant",
- "content": [
- {
- "type": "tool_call",
- "name": "browser_click",
- "arguments": {
- "element": "Submit button",
- "ref": "e2",
- "_is_done": false
- },
- "id": "toolu_01KNmb3Rxcczp8uqVrnqdgGc"
- }
- ],
- "stopReason": {
- "code": "ok"
- }
- },
- "usage": {
- "input": 3062,
+ "input": 3259,
"output": 91
}
},
- "ad449beeec82a30cbf259f8636710456b82fe61e": {
+ "7b25e318de64dc9d2494432a6d311631e2c1edc0": {
"result": {
"role": "assistant",
"content": [
{
"type": "text",
- "text": "I've clicked the submit button twice so far. Let me continue with the third click."
+ "text": "I'll click the submit button 5 times as requested. Let me perform these clicks sequentially."
},
{
"type": "tool_call",
@@ -67,7 +39,7 @@
"ref": "e2",
"_is_done": false
},
- "id": "toolu_015bFqUKFQHHcXD8e1111Pnz"
+ "id": "toolu_01VansE6LrZDU3SibgRvkoGY"
}
],
"stopReason": {
@@ -75,35 +47,11 @@
}
},
"usage": {
- "input": 3129,
- "output": 110
- }
- },
- "d6199b8febde5c630be3a37a5cbf85848578072f": {
- "result": {
- "role": "assistant",
- "content": [
- {
- "type": "tool_call",
- "name": "browser_click",
- "arguments": {
- "element": "Submit button",
- "ref": "e2",
- "_is_done": false
- },
- "id": "toolu_01Vh19GsLUmnRyih9jijUjun"
- }
- ],
- "stopReason": {
- "code": "ok"
- }
- },
- "usage": {
- "input": 3129,
- "output": 91
+ "input": 3047,
+ "output": 113
}
},
- "d6908390517a85b5f26c2710df9f88d9cc308391": {
+ "82d4661d903f25ce1a7e51c2954072ed0003042e": {
"result": {
"role": "assistant",
"content": [
@@ -115,7 +63,7 @@
"ref": "e2",
"_is_done": false
},
- "id": "toolu_01Lk5SyJcFkntoWqrqksKKmW"
+ "id": "toolu_01VUbrN6D39GVSBuPEaByNKZ"
}
],
"stopReason": {
@@ -123,39 +71,11 @@
}
},
"usage": {
- "input": 3191,
+ "input": 3321,
"output": 91
}
},
- "e273971f21318a4d236e4cf3e2c4ea5ef6cbd89b": {
- "result": {
- "role": "assistant",
- "content": [
- {
- "type": "text",
- "text": "Now clicking for the fourth time."
- },
- {
- "type": "tool_call",
- "name": "browser_click",
- "arguments": {
- "element": "Submit button",
- "ref": "e2",
- "_is_done": false
- },
- "id": "toolu_01MqkonDWEffsqEjvy6jFNve"
- }
- ],
- "stopReason": {
- "code": "ok"
- }
- },
- "usage": {
- "input": 3209,
- "output": 99
- }
- },
- "e63576e180882247eb0ea96791b1d4e9f36202fa": {
+ "a64b5e20ef2b0c66b2b1373302af78d97e0b451f": {
"result": {
"role": "assistant",
"content": [
@@ -167,7 +87,7 @@
"ref": "e2",
"_is_done": false
},
- "id": "toolu_01RLTnccC9dzRNGJz1Z6eNn2"
+ "id": "toolu_01FcvrnBtseq9yQA5A7cRdeZ"
}
],
"stopReason": {
@@ -175,7 +95,7 @@
}
},
"usage": {
- "input": 3062,
+ "input": 3192,
"output": 91
}
}
diff --git a/tests/library/__llm_cache__/library-agent-perform-click-a-button.json b/tests/library/__llm_cache__/library-agent-perform-click-a-button.json
index 2cf1556087..24ef06c9bb 100644
--- a/tests/library/__llm_cache__/library-agent-perform-click-a-button.json
+++ b/tests/library/__llm_cache__/library-agent-perform-click-a-button.json
@@ -1,5 +1,5 @@
{
- "e0bdb1971bebcc365635eea52b2b53f570aebbe1": {
+ "b17b5a7075f197f7886c6b8c882c2c41edc2deb6": {
"result": {
"role": "assistant",
"content": [
@@ -15,7 +15,7 @@
"ref": "e2",
"_is_done": true
},
- "id": "toolu_015RTvi8GMkgiz5Y52MB5zSB"
+ "id": "toolu_01Cn4RQJc1DfvDMS4Mox9C3V"
}
],
"stopReason": {
@@ -23,7 +23,7 @@
}
},
"usage": {
- "input": 2913,
+ "input": 3043,
"output": 101
}
}
diff --git a/tests/library/__llm_cache__/library-agent-perform-expect-value.json b/tests/library/__llm_cache__/library-agent-perform-expect-value.json
index ec9a80c05b..93d2d772bd 100644
--- a/tests/library/__llm_cache__/library-agent-perform-expect-value.json
+++ b/tests/library/__llm_cache__/library-agent-perform-expect-value.json
@@ -1,11 +1,11 @@
{
- "257a8267a26efb0aa123ed960972e7063bf422b7": {
+ "112359efcd28400619777fd107f6ffe51ab30550": {
"result": {
"role": "assistant",
"content": [
{
"type": "text",
- "text": "I'll help you complete this task. Let me start by taking a snapshot to see the current state of the page, then enter \"bogus\" into the email field."
+ "text": "I'll help you complete this task. Let me start by taking a snapshot of the current page to see the full state, then enter \"bogus\" into the email field."
},
{
"type": "tool_call",
@@ -16,7 +16,7 @@
"text": "bogus",
"_is_done": false
},
- "id": "toolu_01VZGaFfiCa8P6YgL931iynx"
+ "id": "toolu_01HmcYqCdsgPxG6WMVtzFgcq"
}
],
"stopReason": {
@@ -24,17 +24,17 @@
}
},
"usage": {
- "input": 2946,
- "output": 149
+ "input": 3076,
+ "output": 150
}
},
- "7cc3a16929117110ad26787de7aad4ccc28bf33b": {
+ "6051f9c1b91f0a3a3be22a14435abff316b3b1e7": {
"result": {
"role": "assistant",
"content": [
{
"type": "text",
- "text": "Perfect! I've successfully completed all three parts of the task:\n\n1. ✅ Entered \"bogus\" into the email field\n2. ✅ Verified the value is \"bogus\" (shown in the textbox)\n3. ✅ Confirmed the error message \"Error: Invalid email address\" is displayed\n\nThe task is complete."
+ "text": "Perfect! I have successfully completed all three parts of the task:\n\n1. ✅ Entered \"bogus\" into the email field\n2. ✅ Verified the value is \"bogus\" (shown in the textbox state)\n3. ✅ Confirmed the error message is displayed: \"Error: Invalid email address\"\n\nThe page snapshot shows that:\n- The email textbox now contains the value \"bogus\"\n- An error message \"Error: Invalid email address\" is displayed (ref=e3)"
},
{
"type": "tool_call",
@@ -42,7 +42,7 @@
"arguments": {
"_is_done": true
},
- "id": "toolu_01239r3RkbLHn3z8V9xLt4ke"
+ "id": "toolu_01HaCDQo8wzpdUJy1tySaSai"
}
],
"stopReason": {
@@ -50,34 +50,8 @@
}
},
"usage": {
- "input": 3158,
- "output": 136
- }
- },
- "f96594709b39f189744cdbfbd3e6dbc042be6c3b": {
- "result": {
- "role": "assistant",
- "content": [
- {
- "type": "text",
- "text": "Perfect! I have successfully completed all three parts of the task:\n\n1. ✅ Entered \"bogus\" into the email field\n2. ✅ Verified the value is \"bogus\" (shown in the textbox)\n3. ✅ Confirmed the error message \"Error: Invalid email address\" is displayed\n\nAll requirements have been met. The task is complete."
- },
- {
- "type": "tool_call",
- "name": "browser_snapshot",
- "arguments": {
- "_is_done": true
- },
- "id": "toolu_01BhvJiNjAKMP9soS7NEbm6E"
- }
- ],
- "stopReason": {
- "code": "ok"
- }
- },
- "usage": {
- "input": 3159,
- "output": 142
+ "input": 3289,
+ "output": 173
}
}
}
\ No newline at end of file
diff --git a/tests/library/__llm_cache__/library-agent-perform-extract-task.json b/tests/library/__llm_cache__/library-agent-perform-extract-task.json
index 01c49a9b1c..4e5e2a83ba 100644
--- a/tests/library/__llm_cache__/library-agent-perform-extract-task.json
+++ b/tests/library/__llm_cache__/library-agent-perform-extract-task.json
@@ -1,11 +1,11 @@
{
- "0f3824f1d302520b3f4ada83ccd086e5420e9771": {
+ "c2afe4add9f92f02f33fd49f7911ec415d91b3e3": {
"result": {
"role": "assistant",
"content": [
{
"type": "text",
- "text": "I'll extract the todo items with their statuses from the page snapshot."
+ "text": "I'll extract the todo items and their statuses from the page."
},
{
"type": "tool_call",
@@ -23,7 +23,7 @@
],
"_is_done": true
},
- "id": "toolu_01X5Hqn5T4ddGcveQ2B831FV"
+ "id": "toolu_015AdhvHnzVR1K2iPiKtG9wt"
}
],
"stopReason": {
@@ -31,17 +31,17 @@
}
},
"usage": {
- "input": 890,
- "output": 114
+ "input": 1018,
+ "output": 113
}
},
- "d02ae2240fac126bacb74133b970880eaaf3f915": {
+ "e53dabe92453ecd4602c15601cd50c6ddc88d7bf": {
"result": {
"role": "assistant",
"content": [
{
"type": "text",
- "text": "I'll extract the todo items with their statuses from the page snapshot."
+ "text": "I'll extract the todo items and their statuses from the page."
},
{
"type": "tool_call",
@@ -59,7 +59,7 @@
],
"_is_done": true
},
- "id": "toolu_01FTWH52QQgqJKoUDCYfw5sn"
+ "id": "toolu_01R1npdN7T238o818dP99ZTP"
}
],
"stopReason": {
@@ -67,8 +67,8 @@
}
},
"usage": {
- "input": 888,
- "output": 114
+ "input": 1020,
+ "output": 113
}
}
}
\ No newline at end of file
diff --git a/tests/library/__llm_cache__/library-agent-perform-perform-history.json b/tests/library/__llm_cache__/library-agent-perform-perform-history.json
index f1311c4874..aa29bae6de 100644
--- a/tests/library/__llm_cache__/library-agent-perform-perform-history.json
+++ b/tests/library/__llm_cache__/library-agent-perform-perform-history.json
@@ -1,5 +1,5 @@
{
- "34ea159e5b219eee24c7ec413056961527a70a9c": {
+ "190ffa76902eab71ce416fea1894aa82dca2b883": {
"result": {
"role": "assistant",
"content": [
@@ -15,7 +15,7 @@
"ref": "e3",
"_is_done": true
},
- "id": "toolu_014BY64WX271e12gxu1Am1a2"
+ "id": "toolu_019adK79fRan1Ers3XpDG1TM"
}
],
"stopReason": {
@@ -23,11 +23,11 @@
}
},
"usage": {
- "input": 2966,
+ "input": 3096,
"output": 102
}
},
- "4ac4e27d36a346b689882d02e399e578103b22dd": {
+ "500794034629b9a73d75bdb7ab58e72bf8adfca2": {
"result": {
"role": "assistant",
"content": [
@@ -43,7 +43,7 @@
"ref": "e3",
"_is_done": true
},
- "id": "toolu_01SB5Rzvst8z6jfTSXKYwTFV"
+ "id": "toolu_01PQ383eghAju8o4YoAeicNc"
}
],
"stopReason": {
@@ -51,7 +51,7 @@
}
},
"usage": {
- "input": 2953,
+ "input": 3083,
"output": 101
}
}
diff --git a/tests/library/__llm_cache__/library-agent-perform-perform-reports-error.json b/tests/library/__llm_cache__/library-agent-perform-perform-reports-error.json
new file mode 100644
index 0000000000..63bc943106
--- /dev/null
+++ b/tests/library/__llm_cache__/library-agent-perform-perform-reports-error.json
@@ -0,0 +1,55 @@
+{
+ "0c9fb36a3b4efe91643ee0b06bdfc95b3907c117": {
+ "result": {
+ "role": "assistant",
+ "content": [
+ {
+ "type": "text",
+ "text": "The page snapshot still only shows Wolf and Fox buttons. The Rabbit button is not present on the page. Since I cannot find the element that was requested, I need to report this issue."
+ },
+ {
+ "type": "tool_call",
+ "name": "refuse_to_perform",
+ "arguments": {
+ "reason": "The Rabbit button is not present on the page. Only Wolf and Fox buttons are visible in the current page snapshot.",
+ "_is_done": true
+ },
+ "id": "toolu_01CGic6pHr75uncqRnq5SCZD"
+ }
+ ],
+ "stopReason": {
+ "code": "ok"
+ }
+ },
+ "usage": {
+ "input": 3239,
+ "output": 136
+ }
+ },
+ "fb3d817d917f930cc19453843ed86fe6d4ad4e98": {
+ "result": {
+ "role": "assistant",
+ "content": [
+ {
+ "type": "text",
+ "text": "I need to click the Rabbit button, but looking at the page snapshot, I can only see \"Wolf\" and \"Fox\" buttons. The Rabbit button is not visible in the current snapshot. Let me take a fresh snapshot to see if there are more elements on the page."
+ },
+ {
+ "type": "tool_call",
+ "name": "browser_snapshot",
+ "arguments": {
+ "_is_done": false
+ },
+ "id": "toolu_01YZho5wv2JBSfHJQiHedagQ"
+ }
+ ],
+ "stopReason": {
+ "code": "ok"
+ }
+ },
+ "usage": {
+ "input": 3070,
+ "output": 114
+ }
+ }
+}
\ No newline at end of file
diff --git a/tests/library/__llm_cache__/library-agent-perform-perform-run-timeout.json b/tests/library/__llm_cache__/library-agent-perform-perform-run-timeout.json
index 33e5c609f8..d7ee17a62c 100644
--- a/tests/library/__llm_cache__/library-agent-perform-perform-run-timeout.json
+++ b/tests/library/__llm_cache__/library-agent-perform-perform-run-timeout.json
@@ -1,5 +1,5 @@
{
- "19390eb774e6264fe56bfe8c307a3a20078635e6": {
+ "273f0802e7fc3fe0d3eb1cfa83366e1a1272f185": {
"result": {
"role": "assistant",
"content": [
@@ -15,7 +15,7 @@
"ref": "e3",
"_is_done": true
},
- "id": "toolu_01CttCeYvgp5tNMGQQoGf9Yw"
+ "id": "toolu_01XRNgTAtWw4nWAmToKQUVif"
}
],
"stopReason": {
@@ -23,7 +23,7 @@
}
},
"usage": {
- "input": 2939,
+ "input": 3069,
"output": 101
}
}
diff --git a/tests/library/__llm_cache__/library-agent-perform-retrieve-a-secret.json b/tests/library/__llm_cache__/library-agent-perform-retrieve-a-secret.json
index 887bba325f..6d204f3066 100644
--- a/tests/library/__llm_cache__/library-agent-perform-retrieve-a-secret.json
+++ b/tests/library/__llm_cache__/library-agent-perform-retrieve-a-secret.json
@@ -1,22 +1,22 @@
{
- "a948603f6615ceca22d7aea7876d2c33dd7fc3b2": {
+ "c1820ebf65381413ca130709a7eaecc6847a147a": {
"result": {
"role": "assistant",
"content": [
{
"type": "text",
- "text": "I'll enter the x-secret-email into the email field."
+ "text": "I'll enter the x-secret-email value into the email field."
},
{
"type": "tool_call",
"name": "browser_type",
"arguments": {
- "element": "Email Address textbox",
+ "element": "Email Address",
"ref": "e2",
- "text": "x-secret-email",
+ "text": "%x-secret-email%",
"_is_done": true
},
- "id": "toolu_016ALLcwfioPZMdgvjRk7XNn"
+ "id": "toolu_01Pcb4m96dK9tQS6Dcsy3d2T"
}
],
"stopReason": {
@@ -24,8 +24,8 @@
}
},
"usage": {
- "input": 2921,
- "output": 129
+ "input": 3051,
+ "output": 130
}
}
}
\ No newline at end of file
diff --git a/tests/library/agent-perform.spec.ts b/tests/library/agent-perform.spec.ts
index e17c8ac777..025c43cf15 100644
--- a/tests/library/agent-perform.spec.ts
+++ b/tests/library/agent-perform.spec.ts
@@ -44,7 +44,7 @@ test('click a button', async ({ context }) => {
});
// broken, let's fix later
-test.fail('retrieve a secret', async ({ context }) => {
+test('retrieve a secret', async ({ context }) => {
await run(context, async (page, agent) => {
await page.setContent('<input type="email" name="email" placeholder="Email Address"/>');
await agent.perform('Enter x-secret-email into the email field');
@@ -191,3 +191,13 @@ Failed to parse cache file ${test.info().outputPath('agent-cache.json')}:
→ at [\"some key\"].actions[0].code
`.trim());
});
+
+test('perform reports error', async ({ context }) => {
+ const { page, agent } = await generateAgent(context);
+ await page.setContent(`
+ <button>Wolf</button>
+ <button>Fox</button>
+ `);
+ const e = await agent.perform('click the Rabbit button').catch(e => e);
+ expect(e.message).toContain('Agent refused to perform action:');
+});