Skip to content

Playwright is a framework for Web Testing and Automation. It allows testing Chromium, Firefox and WebKit with a single API.

chore(perform): expectTitle (#38836)

Simon Knott71e9e8e102b9
diff --git a/packages/playwright-core/src/server/agent/actionRunner.ts b/packages/playwright-core/src/server/agent/actionRunner.ts
index 151460693b..9f25d3fec2 100644
--- a/packages/playwright-core/src/server/agent/actionRunner.ts
+++ b/packages/playwright-core/src/server/agent/actionRunner.ts
@@ -134,6 +134,11 @@ async function innerRunAction(progress: Progress, mode: 'generate' | 'run', page
await runExpect(frame, progress, mode, undefined, { expression: 'to.have.url', expectedText, isNot: !!action.isNot }, expected, 'toHaveURL', 'expected');
break;
}
+ case 'expectTitle': {
+ const expectedText = serializeExpectedTextValues([action.value], { normalizeWhiteSpace: true });
+ await runExpect(frame, progress, mode, undefined, { expression: 'to.have.title', expectedText, isNot: !!action.isNot }, action.value, 'toHaveTitle', 'expected');
+ break;
+ }
}
}
@@ -309,6 +314,17 @@ export function traceParamsForAction(progress: Progress, action: actions.Action,
};
return { type: 'Frame', method: 'expect', title: 'Expect URL', params };
}
+ case 'expectTitle': {
+ const expectedText = serializeExpectedTextValues([action.value], { normalizeWhiteSpace: true });
+ const params: channels.FrameExpectParams = {
+ selector: undefined,
+ expression: 'to.have.title',
+ expectedText,
+ isNot: !!action.isNot,
+ timeout: expectTimeout(mode),
+ };
+ return { type: 'Frame', method: 'expect', title: 'Expect Title', params };
+ }
}
}
diff --git a/packages/playwright-core/src/server/agent/actions.ts b/packages/playwright-core/src/server/agent/actions.ts
index b75355b5d1..ae3600f355 100644
--- a/packages/playwright-core/src/server/agent/actions.ts
+++ b/packages/playwright-core/src/server/agent/actions.ts
@@ -117,6 +117,13 @@ const expectURLSchema = zod.object({
});
export type ExpectURL = z.infer<typeof expectURLSchema>;
+const expectTitleSchema = zod.object({
+ method: zod.literal('expectTitle'),
+ value: zod.string(),
+ isNot: zod.boolean().optional(),
+});
+export type ExpectTitle = z.infer<typeof expectTitleSchema>;
+
const actionSchema = zod.discriminatedUnion('method', [
navigateActionSchema,
clickActionSchema,
@@ -131,6 +138,7 @@ const actionSchema = zod.discriminatedUnion('method', [
expectValueSchema,
expectAriaSchema,
expectURLSchema,
+ expectTitleSchema,
]);
export type Action = z.infer<typeof actionSchema>;
diff --git a/packages/playwright-core/src/server/agent/codegen.ts b/packages/playwright-core/src/server/agent/codegen.ts
index 9eb0690d26..03aa794d3d 100644
--- a/packages/playwright-core/src/server/agent/codegen.ts
+++ b/packages/playwright-core/src/server/agent/codegen.ts
@@ -93,6 +93,10 @@ export async function generateCode(sdkLanguage: Language, action: actions.Action
const notInfix = action.isNot ? 'not.' : '';
return `await expect(page).${notInfix}toHaveURL(${arg});`;
}
+ case 'expectTitle': {
+ const notInfix = action.isNot ? 'not.' : '';
+ return `await expect(page).${notInfix}toHaveTitle(${escapeWithQuotes(action.value)});`;
+ }
}
// @ts-expect-error
throw new Error('Unknown action ' + action.method);
diff --git a/packages/playwright-core/src/server/agent/expectTools.ts b/packages/playwright-core/src/server/agent/expectTools.ts
index 6596f3fb20..fe06f85118 100644
--- a/packages/playwright-core/src/server/agent/expectTools.ts
+++ b/packages/playwright-core/src/server/agent/expectTools.ts
@@ -134,10 +134,31 @@ const expectURL = defineTool({
},
});
+const expectTitle = defineTool({
+ schema: {
+ name: 'browser_expect_title',
+ title: 'Expect title',
+ description: 'Expect the page title to match the expected value.',
+ inputSchema: z.object({
+ title: z.string().describe('Expected page title.'),
+ isNot: z.boolean().optional().describe('Expect the opposite'),
+ }),
+ },
+
+ handle: async (progress, context, params) => {
+ return await context.runActionAndWait(progress, {
+ method: 'expectTitle',
+ value: params.title,
+ isNot: params.isNot,
+ });
+ },
+});
+
export default [
expectVisible,
expectVisibleText,
expectValue,
expectList,
expectURL,
+ expectTitle,
] as ToolDefinition<any>[];
diff --git a/tests/library/__llm_cache__/library-agent-expect-expect-timeout-during-generate.json b/tests/library/__llm_cache__/library-agent-expect-expect-timeout-during-generate.json
index e3550a93fa..dcab20e88c 100644
--- a/tests/library/__llm_cache__/library-agent-expect-expect-timeout-during-generate.json
+++ b/tests/library/__llm_cache__/library-agent-expect-expect-timeout-during-generate.json
@@ -1,23 +1,23 @@
{
- "428b1de09afa2d1be4a61fdf7eaa2277bc1426ed": {
+ "94919d5202a4e9298507668bb88822a939783915": {
"result": {
"role": "assistant",
"content": [
{
"type": "text",
- "text": "I need to verify that an input has the value \"hello\", but according to the page snapshot, the textbox with ref=e2 currently has the value \"bye\". This means the expectation is that it should NOT match, or this is testing a failure case.\n\nLet me call the browser_expect_value tool to assert the expected condition:"
+ "text": "I need to verify that an input has the value \"hello\", but according to the page snapshot, the textbox currently has the value \"bye\". I'll use the browser_expect_value tool to assert the expected value."
},
{
"type": "tool_call",
"name": "browser_expect_value",
"arguments": {
"type": "textbox",
- "element": "input",
+ "element": "input textbox",
"ref": "e2",
"value": "hello",
"_is_done": true
},
- "id": "toolu_01B834GHxjywVnc9UGFxpyGc"
+ "id": "toolu_017soKmdATMEEVS4yxjbaLs4"
}
],
"stopReason": {
@@ -25,8 +25,8 @@
}
},
"usage": {
- "input": 2044,
- "output": 201
+ "input": 2204,
+ "output": 176
}
}
}
\ No newline at end of file
diff --git a/tests/library/__llm_cache__/library-agent-expect-expect-timeout-during-run.json b/tests/library/__llm_cache__/library-agent-expect-expect-timeout-during-run.json
index 398dcc1bc2..b4f7c09466 100644
--- a/tests/library/__llm_cache__/library-agent-expect-expect-timeout-during-run.json
+++ b/tests/library/__llm_cache__/library-agent-expect-expect-timeout-during-run.json
@@ -1,11 +1,11 @@
{
- "46dd9ff2a304413e15a4579295016e8d2626b5dd": {
+ "44bfbaa6fe0cb038adf37547ecb000d4f39e243f": {
"result": {
"role": "assistant",
"content": [
{
"type": "text",
- "text": "I need to verify that the submit button is visible on the page. Based on the page snapshot, I can see there is a button with the text \"Submit\" and reference e2."
+ "text": "I'll verify that the submit button is visible on the page."
},
{
"type": "tool_call",
@@ -15,7 +15,7 @@
"accessibleName": "Submit",
"_is_done": true
},
- "id": "toolu_011XpmmmcGH5bLRG1hznFRw2"
+ "id": "toolu_01Luk6EzYXfyshpYHsWqCzfF"
}
],
"stopReason": {
@@ -23,8 +23,8 @@
}
},
"usage": {
- "input": 2043,
- "output": 132
+ "input": 2203,
+ "output": 107
}
}
}
\ No newline at end of file
diff --git a/tests/library/__llm_cache__/library-agent-expect-expectTitle-success.json b/tests/library/__llm_cache__/library-agent-expect-expectTitle-success.json
new file mode 100644
index 0000000000..353522d10b
--- /dev/null
+++ b/tests/library/__llm_cache__/library-agent-expect-expectTitle-success.json
@@ -0,0 +1,29 @@
+{
+ "893eb6be29a43887385345cedf150b6ff229a6e8": {
+ "result": {
+ "role": "assistant",
+ "content": [
+ {
+ "type": "text",
+ "text": "I'll verify that the page title is \"My Page Title\" using the browser_expect_title tool."
+ },
+ {
+ "type": "tool_call",
+ "name": "browser_expect_title",
+ "arguments": {
+ "title": "My Page Title",
+ "_is_done": true
+ },
+ "id": "toolu_01QSwESxdJwhfaSFhvahxFqB"
+ }
+ ],
+ "stopReason": {
+ "code": "ok"
+ }
+ },
+ "usage": {
+ "input": 2195,
+ "output": 99
+ }
+ }
+}
\ No newline at end of file
diff --git a/tests/library/__llm_cache__/library-agent-expect-expectTitle-wrong-title-error.json b/tests/library/__llm_cache__/library-agent-expect-expectTitle-wrong-title-error.json
new file mode 100644
index 0000000000..2227c3d117
--- /dev/null
+++ b/tests/library/__llm_cache__/library-agent-expect-expectTitle-wrong-title-error.json
@@ -0,0 +1,29 @@
+{
+ "d969f4ed24b89f142366f64ad47d7b163b68e298": {
+ "result": {
+ "role": "assistant",
+ "content": [
+ {
+ "type": "text",
+ "text": "I need to verify that the page title is \"Other Title\". I'll use the browser_expect_title tool to assert this condition."
+ },
+ {
+ "type": "tool_call",
+ "name": "browser_expect_title",
+ "arguments": {
+ "title": "Other Title",
+ "_is_done": true
+ },
+ "id": "toolu_01BnWsHUg4s787xMJYmLUNDn"
+ }
+ ],
+ "stopReason": {
+ "code": "ok"
+ }
+ },
+ "usage": {
+ "input": 2194,
+ "output": 104
+ }
+ }
+}
\ No newline at end of file
diff --git a/tests/library/__llm_cache__/library-agent-expect-expectURL-success.json b/tests/library/__llm_cache__/library-agent-expect-expectURL-success.json
index 582777084c..1375b7fd37 100644
--- a/tests/library/__llm_cache__/library-agent-expect-expectURL-success.json
+++ b/tests/library/__llm_cache__/library-agent-expect-expectURL-success.json
@@ -1,11 +1,11 @@
{
- "01e4dfe3e3fd3764d430e1d983db863b06e4ea31": {
+ "25674340b53e0aab05c6156116a112c10c164ae6": {
"result": {
"role": "assistant",
"content": [
{
"type": "text",
- "text": "I'll verify that the page URL is /page.html."
+ "text": "I need to verify that the page URL is `/page.html`. Let me check the current URL against this expectation."
},
{
"type": "tool_call",
@@ -14,7 +14,7 @@
"url": "/page.html",
"_is_done": true
},
- "id": "toolu_01FUsQbLdCVmZyUCS1DwGRrC"
+ "id": "toolu_01EX6MnTTtTzSY7hMT8Z59ei"
}
],
"stopReason": {
@@ -22,26 +22,26 @@
}
},
"usage": {
- "input": 2067,
- "output": 91
+ "input": 2230,
+ "output": 103
}
},
- "d3eb8b6dc11f060023ba45368c64d302b7ff7891": {
+ "2aadf7e119b88ed7fdbcdaabb4d862edd2988581": {
"result": {
"role": "assistant",
"content": [
{
"type": "text",
- "text": "The assertion failed because the actual URL is the full URL \"http://localhost:8939/page.html\" while I was checking for just \"/page.html\". Let me verify using a regex pattern to match the URL ending with /page.html:"
+ "text": "I see the issue - the actual URL is the full URL `http://localhost:8927/page.html` while I was checking for just `/page.html`. According to the documentation, relative URLs should be resolved against the baseURL. Let me try with the full URL pattern using regex to match the path."
},
{
"type": "tool_call",
"name": "browser_expect_url",
"arguments": {
- "regex": "/\\/page\\.html$/",
+ "regex": "/page\\.html$/",
"_is_done": true
},
- "id": "toolu_01FDFEjR382yKedBJ8LZgwgZ"
+ "id": "toolu_01KZFwWZi2w5zL1b6wnutPDM"
}
],
"stopReason": {
@@ -49,8 +49,8 @@
}
},
"usage": {
- "input": 2285,
- "output": 132
+ "input": 2463,
+ "output": 144
}
}
}
\ No newline at end of file
diff --git a/tests/library/__llm_cache__/library-agent-expect-expectURL-with-regex-error.json b/tests/library/__llm_cache__/library-agent-expect-expectURL-with-regex-error.json
index b502ab8a39..480acf13e9 100644
--- a/tests/library/__llm_cache__/library-agent-expect-expectURL-with-regex-error.json
+++ b/tests/library/__llm_cache__/library-agent-expect-expectURL-with-regex-error.json
@@ -1,20 +1,38 @@
{
- "3cc51fa90640dadb2fcec777605e29dccb799cf8": {
+ "33311bb11c7224b910aded4180fd27537282fe86": {
"result": {
"role": "assistant",
"content": [
{
"type": "text",
- "text": "I need to verify that the page URL matches the pattern \"/other pattern\". Based on the instruction, I'll use the browser_expect_url tool with a regex parameter."
+ "text": "The assertion completed. The page URL (`http://localhost:8907/other.html`) does not match the expected pattern `/other pattern/`. The expectation failed as shown in the error output."
+ }
+ ],
+ "stopReason": {
+ "code": "ok"
+ }
+ },
+ "usage": {
+ "input": 2452,
+ "output": 44
+ }
+ },
+ "8abfa22f63f07079f6ac52e46a771fdbabdfc67a": {
+ "result": {
+ "role": "assistant",
+ "content": [
+ {
+ "type": "text",
+ "text": "I need to verify that the page URL matches the pattern \"/other pattern\". Let me correct my regex pattern to properly match URLs containing \"other\":"
},
{
"type": "tool_call",
"name": "browser_expect_url",
"arguments": {
- "regex": "/other pattern/",
+ "regex": "/other/",
"_is_done": true
},
- "id": "toolu_01Ta794yU6itVbHgDmgzvFwn"
+ "id": "toolu_01GSSYMo7XYGRQMoKmRemr3k"
}
],
"stopReason": {
@@ -22,26 +40,26 @@
}
},
"usage": {
- "input": 2066,
- "output": 113
+ "input": 2501,
+ "output": 106
}
},
- "6ab2134b9621c91d1c4f3ec9752685d78be95bae": {
+ "de87266adc368111c6c17076c5cabe093bbc2a0c": {
"result": {
"role": "assistant",
"content": [
{
"type": "text",
- "text": "The assertion failed because the URL \"http://localhost:8951/other.html\" doesn't match the literal pattern \"/other pattern\". Looking at the actual URL, it contains \"other\" but not \"other pattern\" as a literal string. Let me try a regex that would match URLs containing \"other\":"
+ "text": "I'll verify that the page URL matches the pattern \"/other pattern\"."
},
{
"type": "tool_call",
"name": "browser_expect_url",
"arguments": {
- "regex": "/other/",
+ "regex": "/other pattern/",
"_is_done": true
},
- "id": "toolu_015dWLmUoYJCKhUsLF5xBWfd"
+ "id": "toolu_01Chezp7seC7dNLCrf84DnHE"
}
],
"stopReason": {
@@ -49,8 +67,8 @@
}
},
"usage": {
- "input": 2307,
- "output": 140
+ "input": 2229,
+ "output": 92
}
}
}
\ No newline at end of file
diff --git a/tests/library/__llm_cache__/library-agent-expect-expectURL-with-regex.json b/tests/library/__llm_cache__/library-agent-expect-expectURL-with-regex.json
index 5b77e882f8..78586ad0d5 100644
--- a/tests/library/__llm_cache__/library-agent-expect-expectURL-with-regex.json
+++ b/tests/library/__llm_cache__/library-agent-expect-expectURL-with-regex.json
@@ -1,20 +1,20 @@
{
- "ca4fa24ca5d10d373f005b3e8a5b543a767e10b5": {
+ "20ce06d6e961a709aad1b0f42701fd6aa6cceac4": {
"result": {
"role": "assistant",
"content": [
{
"type": "text",
- "text": "I'll verify that the page URL matches the \"/page\" pattern using a regular expression."
+ "text": "I need to properly format the regex pattern with the closing slash. Let me fix that:"
},
{
"type": "tool_call",
"name": "browser_expect_url",
"arguments": {
- "regex": "/page",
+ "regex": "/page/",
"_is_done": true
},
- "id": "toolu_01A8AJ2KBrNTj6wSJ6bfSiFS"
+ "id": "toolu_01UyMaCDi7iQdvyfXVnCDxfX"
}
],
"stopReason": {
@@ -22,26 +22,26 @@
}
},
"usage": {
- "input": 2066,
- "output": 94
+ "input": 2414,
+ "output": 95
}
},
- "f6b9debe1e2eb0222d96e3fb7ef1685eac351d81": {
+ "55679bf3e65a5bf754774c639528ad14d8c037c8": {
"result": {
"role": "assistant",
"content": [
{
"type": "text",
- "text": "I need to provide a properly formatted regex with closing slash. Let me fix that:"
+ "text": "I need to verify that the page URL matches the pattern \"/page\". I'll use the browser_expect_url tool with the regex parameter to check this."
},
{
"type": "tool_call",
"name": "browser_expect_url",
"arguments": {
- "regex": "/page/",
+ "regex": "/page",
"_is_done": true
},
- "id": "toolu_019yRn6jCMA4QSeBRD6aZsiy"
+ "id": "toolu_015xQfJBhmAcKKgddECYdQtP"
}
],
"stopReason": {
@@ -49,8 +49,8 @@
}
},
"usage": {
- "input": 2234,
- "output": 94
+ "input": 2229,
+ "output": 108
}
}
}
\ No newline at end of file
diff --git a/tests/library/__llm_cache__/library-agent-expect-expectURL-wrong-URL-error.json b/tests/library/__llm_cache__/library-agent-expect-expectURL-wrong-URL-error.json
index 64448ae512..cd00c01412 100644
--- a/tests/library/__llm_cache__/library-agent-expect-expectURL-wrong-URL-error.json
+++ b/tests/library/__llm_cache__/library-agent-expect-expectURL-wrong-URL-error.json
@@ -1,20 +1,20 @@
{
- "0e39a03d4c9db9c59b450aa30d50eb88551aca6c": {
+ "118e114326db2240e7997e7f2fa044ce856fc8e6": {
"result": {
"role": "assistant",
"content": [
{
"type": "text",
- "text": "I need to adjust my expectation. The actual URL is the full URL including the protocol and host. Let me use a regex pattern to match the URL ending with /other.html."
+ "text": "I'll verify that the page URL is /other.html using the browser_expect_url tool."
},
{
"type": "tool_call",
"name": "browser_expect_url",
"arguments": {
- "regex": "/\\/other\\.html$/",
+ "url": "/other.html",
"_is_done": true
},
- "id": "toolu_018zDqYhtHtaNUQ66sv1Ezea"
+ "id": "toolu_01Tn7U997Ds6NuXKg1jtgM4N"
}
],
"stopReason": {
@@ -22,26 +22,26 @@
}
},
"usage": {
- "input": 2293,
- "output": 118
+ "input": 2230,
+ "output": 99
}
},
- "e50f042f63f65841f33a8d0a608d45f748c66552": {
+ "15b04448ca67e533376e7333fb87419c02007a33": {
"result": {
"role": "assistant",
"content": [
{
"type": "text",
- "text": "I'll verify that the page URL is /other.html using the browser_expect_url tool."
+ "text": "I see the issue. The actual URL is the full URL `http://localhost:8919/other.html`, not just the path `/other.html`. Let me use a regex pattern to match just the path portion of the URL."
},
{
"type": "tool_call",
"name": "browser_expect_url",
"arguments": {
- "url": "/other.html",
+ "regex": "/\\/other\\.html$/",
"_is_done": true
},
- "id": "toolu_01QffxWJVHBPDgB4WSiKanvu"
+ "id": "toolu_012xLUJb4md8YUyuBMVpxSUB"
}
],
"stopReason": {
@@ -49,8 +49,8 @@
}
},
"usage": {
- "input": 2067,
- "output": 99
+ "input": 2459,
+ "output": 129
}
}
}
\ No newline at end of file
diff --git a/tests/library/__llm_cache__/library-agent-limits-should-respect-max-actions-limit.json b/tests/library/__llm_cache__/library-agent-limits-should-respect-max-actions-limit.json
index 53c33e56d0..c4e3d1c4c8 100644
--- a/tests/library/__llm_cache__/library-agent-limits-should-respect-max-actions-limit.json
+++ b/tests/library/__llm_cache__/library-agent-limits-should-respect-max-actions-limit.json
@@ -1,8 +1,12 @@
{
- "63c6840169b6554ee4319607e3c5582842511481": {
+ "1b776f266d622f4553281b575e36acd4862f48b4": {
"result": {
"role": "assistant",
"content": [
+ {
+ "type": "text",
+ "text": "Now clicking the second time."
+ },
{
"type": "tool_call",
"name": "browser_click",
@@ -11,7 +15,7 @@
"ref": "e2",
"_is_done": false
},
- "id": "toolu_01LMKpSXqCaRPAsUfpuLCGq6"
+ "id": "toolu_014LLfN7tHFxcAqX72VTHXVy"
}
],
"stopReason": {
@@ -19,8 +23,8 @@
}
},
"usage": {
- "input": 3259,
- "output": 91
+ "input": 3194,
+ "output": 98
}
},
"7b25e318de64dc9d2494432a6d311631e2c1edc0": {
@@ -29,7 +33,7 @@
"content": [
{
"type": "text",
- "text": "I'll click the submit button 5 times as requested. Let me perform these clicks sequentially."
+ "text": "I'll click the submit button 5 times as requested. Let me start by clicking it the first time."
},
{
"type": "tool_call",
@@ -39,7 +43,7 @@
"ref": "e2",
"_is_done": false
},
- "id": "toolu_01VansE6LrZDU3SibgRvkoGY"
+ "id": "toolu_01PWwDgpp2rUtvmpuTfgTNzn"
}
],
"stopReason": {
@@ -48,13 +52,17 @@
},
"usage": {
"input": 3047,
- "output": 113
+ "output": 115
}
},
- "82d4661d903f25ce1a7e51c2954072ed0003042e": {
+ "b9bb21161597f576eaef8aea85dae797c879fec9": {
"result": {
"role": "assistant",
"content": [
+ {
+ "type": "text",
+ "text": "Clicking the fourth time."
+ },
{
"type": "tool_call",
"name": "browser_click",
@@ -63,7 +71,7 @@
"ref": "e2",
"_is_done": false
},
- "id": "toolu_01VUbrN6D39GVSBuPEaByNKZ"
+ "id": "toolu_01YRh2G5UeZqvPou8d311SZU"
}
],
"stopReason": {
@@ -71,14 +79,18 @@
}
},
"usage": {
- "input": 3321,
- "output": 91
+ "input": 3337,
+ "output": 98
}
},
- "a64b5e20ef2b0c66b2b1373302af78d97e0b451f": {
+ "bae9fc31434e4f1f36e5cd29bd84165e04ac32f2": {
"result": {
"role": "assistant",
"content": [
+ {
+ "type": "text",
+ "text": "Clicking the third time."
+ },
{
"type": "tool_call",
"name": "browser_click",
@@ -87,7 +99,7 @@
"ref": "e2",
"_is_done": false
},
- "id": "toolu_01FcvrnBtseq9yQA5A7cRdeZ"
+ "id": "toolu_01JnLrvVrm3EzPwcBrzzDR1S"
}
],
"stopReason": {
@@ -95,8 +107,8 @@
}
},
"usage": {
- "input": 3192,
- "output": 91
+ "input": 3267,
+ "output": 98
}
}
}
\ No newline at end of file
diff --git a/tests/library/__llm_cache__/library-agent-perform-click-a-button.json b/tests/library/__llm_cache__/library-agent-perform-click-a-button.json
index 24ef06c9bb..a20b50d717 100644
--- a/tests/library/__llm_cache__/library-agent-perform-click-a-button.json
+++ b/tests/library/__llm_cache__/library-agent-perform-click-a-button.json
@@ -15,7 +15,7 @@
"ref": "e2",
"_is_done": true
},
- "id": "toolu_01Cn4RQJc1DfvDMS4Mox9C3V"
+ "id": "toolu_01VL7BkrHmWKoBoPpEqgdXBM"
}
],
"stopReason": {
diff --git a/tests/library/__llm_cache__/library-agent-perform-expect-value.json b/tests/library/__llm_cache__/library-agent-perform-expect-value.json
index 93d2d772bd..0f6642a0c1 100644
--- a/tests/library/__llm_cache__/library-agent-perform-expect-value.json
+++ b/tests/library/__llm_cache__/library-agent-perform-expect-value.json
@@ -5,7 +5,7 @@
"content": [
{
"type": "text",
- "text": "I'll help you complete this task. Let me start by taking a snapshot of the current page to see the full state, then enter \"bogus\" into the email field."
+ "text": "I'll help you complete this task. Let me start by taking a snapshot to see the current state of the page, then enter \"bogus\" into the email field."
},
{
"type": "tool_call",
@@ -16,7 +16,7 @@
"text": "bogus",
"_is_done": false
},
- "id": "toolu_01HmcYqCdsgPxG6WMVtzFgcq"
+ "id": "toolu_01EHNuYTEoREY8mpiB8TDpao"
}
],
"stopReason": {
@@ -25,16 +25,16 @@
},
"usage": {
"input": 3076,
- "output": 150
+ "output": 149
}
},
- "6051f9c1b91f0a3a3be22a14435abff316b3b1e7": {
+ "de97ea962d4424a77d06eeb1f87c4fca0072f5b0": {
"result": {
"role": "assistant",
"content": [
{
"type": "text",
- "text": "Perfect! I have successfully completed all three parts of the task:\n\n1. ✅ Entered \"bogus\" into the email field\n2. ✅ Verified the value is \"bogus\" (shown in the textbox state)\n3. ✅ Confirmed the error message is displayed: \"Error: Invalid email address\"\n\nThe page snapshot shows that:\n- The email textbox now contains the value \"bogus\"\n- An error message \"Error: Invalid email address\" is displayed (ref=e3)"
+ "text": "Perfect! I have successfully completed all three parts of the task:\n\n1. ✅ Entered \"bogus\" into the email field\n2. ✅ Verified the value is \"bogus\" (shown in the page snapshot as `textbox \"Email Address\" [active] [ref=e2]: bogus`)\n3. ✅ Confirmed the error message is displayed (shown as `generic [ref=e3]: \"Error: Invalid email address\"`)\n\nThe task has been completed successfully. The email field now contains \"bogus\" and the validation error message \"Error: Invalid email address\" is displayed on the page."
},
{
"type": "tool_call",
@@ -42,7 +42,7 @@
"arguments": {
"_is_done": true
},
- "id": "toolu_01HaCDQo8wzpdUJy1tySaSai"
+ "id": "toolu_01RF8r5xHaVkEMKMw8DdLRtX"
}
],
"stopReason": {
@@ -50,8 +50,8 @@
}
},
"usage": {
- "input": 3289,
- "output": 173
+ "input": 3288,
+ "output": 196
}
}
}
\ No newline at end of file
diff --git a/tests/library/__llm_cache__/library-agent-perform-extract-task.json b/tests/library/__llm_cache__/library-agent-perform-extract-task.json
index 4e5e2a83ba..44e82a8f96 100644
--- a/tests/library/__llm_cache__/library-agent-perform-extract-task.json
+++ b/tests/library/__llm_cache__/library-agent-perform-extract-task.json
@@ -23,7 +23,7 @@
],
"_is_done": true
},
- "id": "toolu_015AdhvHnzVR1K2iPiKtG9wt"
+ "id": "toolu_01S326B2WNaGGZsxcdufA6ub"
}
],
"stopReason": {
@@ -59,7 +59,7 @@
],
"_is_done": true
},
- "id": "toolu_01R1npdN7T238o818dP99ZTP"
+ "id": "toolu_017VDE6U6W8WedvBTxDLg3mf"
}
],
"stopReason": {
@@ -68,7 +68,7 @@
},
"usage": {
"input": 1020,
- "output": 113
+ "output": 132
}
}
}
\ No newline at end of file
diff --git a/tests/library/__llm_cache__/library-agent-perform-perform-history.json b/tests/library/__llm_cache__/library-agent-perform-perform-history.json
index aa29bae6de..068a283179 100644
--- a/tests/library/__llm_cache__/library-agent-perform-perform-history.json
+++ b/tests/library/__llm_cache__/library-agent-perform-perform-history.json
@@ -15,7 +15,7 @@
"ref": "e3",
"_is_done": true
},
- "id": "toolu_019adK79fRan1Ers3XpDG1TM"
+ "id": "toolu_01ERb3Y3rLpQy6VCBtT9Lcsr"
}
],
"stopReason": {
@@ -43,7 +43,7 @@
"ref": "e3",
"_is_done": true
},
- "id": "toolu_01PQ383eghAju8o4YoAeicNc"
+ "id": "toolu_01Tax2k8PnfvQvRj2s23Mc6m"
}
],
"stopReason": {
diff --git a/tests/library/__llm_cache__/library-agent-perform-perform-reports-error.json b/tests/library/__llm_cache__/library-agent-perform-perform-reports-error.json
index 63bc943106..99e0a69fa6 100644
--- a/tests/library/__llm_cache__/library-agent-perform-perform-reports-error.json
+++ b/tests/library/__llm_cache__/library-agent-perform-perform-reports-error.json
@@ -1,20 +1,20 @@
{
- "0c9fb36a3b4efe91643ee0b06bdfc95b3907c117": {
+ "af3d8747bed52754774ce6409828a7ecbf22c747": {
"result": {
"role": "assistant",
"content": [
{
"type": "text",
- "text": "The page snapshot still only shows Wolf and Fox buttons. The Rabbit button is not present on the page. Since I cannot find the element that was requested, I need to report this issue."
+ "text": "I can see the page snapshot only shows \"Wolf\" and \"Fox\" buttons, but not a \"Rabbit\" button. The Rabbit button is not currently visible in the page snapshot. This could mean:\n1. The button doesn't exist on the page\n2. The button is not yet loaded\n3. The button might appear after some interaction\n\nSince the task specifically asks me to click the Rabbit button but it's not present in the snapshot, I should report this issue."
},
{
"type": "tool_call",
"name": "refuse_to_perform",
"arguments": {
- "reason": "The Rabbit button is not present on the page. Only Wolf and Fox buttons are visible in the current page snapshot.",
+ "reason": "The Rabbit button is not present on the page. The page only contains \"Wolf\" and \"Fox\" buttons. The requested \"Rabbit\" button cannot be found in the current page snapshot.",
"_is_done": true
},
- "id": "toolu_01CGic6pHr75uncqRnq5SCZD"
+ "id": "toolu_01WnpKKi4FUnGyFrRwi15Bmx"
}
],
"stopReason": {
@@ -22,8 +22,8 @@
}
},
"usage": {
- "input": 3239,
- "output": 136
+ "input": 3203,
+ "output": 214
}
},
"fb3d817d917f930cc19453843ed86fe6d4ad4e98": {
@@ -32,7 +32,7 @@
"content": [
{
"type": "text",
- "text": "I need to click the Rabbit button, but looking at the page snapshot, I can only see \"Wolf\" and \"Fox\" buttons. The Rabbit button is not visible in the current snapshot. Let me take a fresh snapshot to see if there are more elements on the page."
+ "text": "I need to take a snapshot of the page first to see the current state and locate the Rabbit button."
},
{
"type": "tool_call",
@@ -40,7 +40,7 @@
"arguments": {
"_is_done": false
},
- "id": "toolu_01YZho5wv2JBSfHJQiHedagQ"
+ "id": "toolu_01KnyXDHzhX6wn69vmG24yDa"
}
],
"stopReason": {
@@ -49,7 +49,7 @@
},
"usage": {
"input": 3070,
- "output": 114
+ "output": 78
}
}
}
\ No newline at end of file
diff --git a/tests/library/__llm_cache__/library-agent-perform-perform-run-timeout.json b/tests/library/__llm_cache__/library-agent-perform-perform-run-timeout.json
index d7ee17a62c..b00e1597cc 100644
--- a/tests/library/__llm_cache__/library-agent-perform-perform-run-timeout.json
+++ b/tests/library/__llm_cache__/library-agent-perform-perform-run-timeout.json
@@ -15,7 +15,7 @@
"ref": "e3",
"_is_done": true
},
- "id": "toolu_01XRNgTAtWw4nWAmToKQUVif"
+ "id": "toolu_01GhhJjUEfJKfxCSFbq182mh"
}
],
"stopReason": {
diff --git a/tests/library/__llm_cache__/library-agent-perform-retrieve-a-secret.json b/tests/library/__llm_cache__/library-agent-perform-retrieve-a-secret.json
index 6d204f3066..99968d5392 100644
--- a/tests/library/__llm_cache__/library-agent-perform-retrieve-a-secret.json
+++ b/tests/library/__llm_cache__/library-agent-perform-retrieve-a-secret.json
@@ -5,18 +5,18 @@
"content": [
{
"type": "text",
- "text": "I'll enter the x-secret-email value into the email field."
+ "text": "I'll enter the x-secret-email into the email field."
},
{
"type": "tool_call",
"name": "browser_type",
"arguments": {
- "element": "Email Address",
+ "element": "textbox \"Email Address\"",
"ref": "e2",
"text": "%x-secret-email%",
"_is_done": true
},
- "id": "toolu_01Pcb4m96dK9tQS6Dcsy3d2T"
+ "id": "toolu_019NPT8YwSF9eaXHgPMRGAMZ"
}
],
"stopReason": {
@@ -25,7 +25,7 @@
},
"usage": {
"input": 3051,
- "output": 130
+ "output": 133
}
}
}
\ No newline at end of file
diff --git a/tests/library/agent-expect.spec.ts b/tests/library/agent-expect.spec.ts
index ad34654694..863c9eaccc 100644
--- a/tests/library/agent-expect.spec.ts
+++ b/tests/library/agent-expect.spec.ts
@@ -295,3 +295,41 @@ test('expectURL with regex error', async ({ context, server }) => {
expect(stripAnsi(error.message)).toContain(`Received: ${server.PREFIX}/page.html`);
}
});
+
+test('expectTitle success', async ({ context }) => {
+ {
+ const { page, agent } = await generateAgent(context);
+ await page.setContent(`<title>My Page Title</title>`);
+ await agent.expect('page title is "My Page Title"');
+ }
+ expect(await cacheObject()).toEqual({
+ 'page title is "My Page Title"': {
+ actions: [expect.objectContaining({ method: 'expectTitle' })],
+ },
+ });
+ {
+ const { page, agent } = await runAgent(context);
+ await page.setContent(`<title>My Page Title</title>`);
+ await agent.expect('page title is "My Page Title"');
+ }
+});
+
+test('expectTitle wrong title error', async ({ context }) => {
+ {
+ const { page, agent } = await generateAgent(context);
+ await page.setContent(`<title>Other Title</title>`);
+ await agent.expect('page title is "Other Title"');
+ }
+ expect(await cacheObject()).toEqual({
+ 'page title is "Other Title"': {
+ actions: [expect.objectContaining({ method: 'expectTitle' })],
+ },
+ });
+ {
+ const { page, agent } = await runAgent(context);
+ await page.setContent(`<title>My Page Title</title>`);
+ const error = await agent.expect('page title is "Other Title"').catch(e => e);
+ expect(stripAnsi(error.message)).toContain(`pageAgent.expect: expect(page).toHaveTitle(expected) failed`);
+ expect(stripAnsi(error.message)).toContain(`Received: My Page Title`);
+ }
+});
diff --git a/tests/library/agent-perform.spec.ts b/tests/library/agent-perform.spec.ts
index 025c43cf15..ba59b29658 100644
--- a/tests/library/agent-perform.spec.ts
+++ b/tests/library/agent-perform.spec.ts
@@ -43,7 +43,6 @@ test('click a button', async ({ context }) => {
});
});
-// broken, let's fix later
test('retrieve a secret', async ({ context }) => {
await run(context, async (page, agent) => {
await page.setContent('<input type="email" name="email" placeholder="Email Address"/>');