Skip to content

Playwright is a framework for Web Testing and Automation. It allows testing Chromium, Firefox and WebKit with a single API.

chore(perform): handle secrets in all value-based tools (#38963)

Dmitry Gozman8c99ff19b8e0
diff --git a/packages/playwright-core/src/server/agent/actionRunner.ts b/packages/playwright-core/src/server/agent/actionRunner.ts
index 9773c29b55..a644dd69df 100644
--- a/packages/playwright-core/src/server/agent/actionRunner.ts
+++ b/packages/playwright-core/src/server/agent/actionRunner.ts
@@ -23,6 +23,8 @@ import { parseAriaSnapshotUnsafe } from '../../utils/isomorphic/ariaSnapshot';
import { asLocatorDescription } from '../../utils/isomorphic/locatorGenerators';
import { yaml } from '../../utilsBundle';
import { serializeError } from '../errors';
+import { rewriteErrorMessage } from '../../utils/isomorphic/stackTrace';
+import { applySecrets, redactSecrets } from './context';
import type * as actions from './actions';
import type { Page } from '../page';
@@ -47,8 +49,10 @@ export async function runAction(progress: Progress, mode: 'generate' | 'run', pa
callMetadata.error = error ? serializeError(error) : undefined;
callMetadata.result = error ? undefined : result;
await frame.instrumentation.onAfterCall(frame, callMetadata);
- if (error)
+ if (error) {
+ rewriteErrorMessage(error, redactSecrets(error.message, secrets));
throw error;
+ }
return result;
}
@@ -78,21 +82,20 @@ async function innerRunAction(progress: Progress, mode: 'generate' | 'run', page
});
break;
case 'selectOption':
- await frame.selectOption(progress, action.selector, [], action.labels.map(a => ({ label: a })), { ...commonOptions });
+ const labels = action.labels.map(label => applySecrets(label, secrets));
+ await frame.selectOption(progress, action.selector, [], labels.map(a => ({ label: a })), { ...commonOptions });
break;
case 'pressKey':
await page.keyboard.press(progress, action.key);
break;
case 'pressSequentially': {
- const secret = secrets?.find(s => s.name === action.text)?.value ?? action.text;
- await frame.type(progress, action.selector, secret, { ...commonOptions });
+ await frame.type(progress, action.selector, applySecrets(action.text, secrets), { ...commonOptions });
if (action.submit)
await page.keyboard.press(progress, 'Enter');
break;
}
case 'fill': {
- const secret = secrets?.find(s => s.name === action.text)?.value ?? action.text;
- await frame.fill(progress, action.selector, secret, { ...commonOptions });
+ await frame.fill(progress, action.selector, applySecrets(action.text, secrets), { ...commonOptions });
if (action.submit)
await page.keyboard.press(progress, 'Enter');
break;
@@ -109,8 +112,9 @@ async function innerRunAction(progress: Progress, mode: 'generate' | 'run', page
}
case 'expectValue': {
if (action.type === 'textbox' || action.type === 'combobox' || action.type === 'slider') {
- const expectedText = serializeExpectedTextValues([action.value]);
- await runExpect(frame, progress, mode, action.selector, { expression: 'to.have.value', expectedText, isNot: !!action.isNot }, action.value, 'toHaveValue', 'expected');
+ const value = applySecrets(action.value, secrets);
+ const expectedText = serializeExpectedTextValues([value]);
+ await runExpect(frame, progress, mode, action.selector, { expression: 'to.have.value', expectedText, isNot: !!action.isNot }, value, 'toHaveValue', 'expected');
} else if (action.type === 'checkbox' || action.type === 'radio') {
const expectedValue = { checked: action.value === 'true' };
await runExpect(frame, progress, mode, action.selector, { selector: action.selector, expression: 'to.be.checked', expectedValue, isNot: !!action.isNot }, action.value ? 'checked' : 'unchecked', 'toBeChecked', '');
@@ -120,8 +124,9 @@ async function innerRunAction(progress: Progress, mode: 'generate' | 'run', page
break;
}
case 'expectAria': {
- const expectedValue = parseAriaSnapshotUnsafe(yaml, action.template);
- await runExpect(frame, progress, mode, 'body', { expression: 'to.match.aria', expectedValue, isNot: !!action.isNot }, '\n' + action.template, 'toMatchAriaSnapshot', 'expected');
+ const template = applySecrets(action.template, secrets);
+ const expectedValue = parseAriaSnapshotUnsafe(yaml, template);
+ await runExpect(frame, progress, mode, 'body', { expression: 'to.match.aria', expectedValue, isNot: !!action.isNot }, '\n' + template, 'toMatchAriaSnapshot', 'expected');
break;
}
case 'expectURL': {
@@ -135,8 +140,9 @@ async function innerRunAction(progress: Progress, mode: 'generate' | 'run', page
break;
}
case 'expectTitle': {
- const expectedText = serializeExpectedTextValues([action.value], { normalizeWhiteSpace: true });
- await runExpect(frame, progress, mode, undefined, { expression: 'to.have.title', expectedText, isNot: !!action.isNot }, action.value, 'toHaveTitle', 'expected');
+ const value = applySecrets(action.value, secrets);
+ const expectedText = serializeExpectedTextValues([value], { normalizeWhiteSpace: true });
+ await runExpect(frame, progress, mode, undefined, { expression: 'to.have.title', expectedText, isNot: !!action.isNot }, value, 'toHaveTitle', 'expected');
break;
}
}
diff --git a/packages/playwright-core/src/server/agent/context.ts b/packages/playwright-core/src/server/agent/context.ts
index 9b84009f67..7466ab1439 100644
--- a/packages/playwright-core/src/server/agent/context.ts
+++ b/packages/playwright-core/src/server/agent/context.ts
@@ -125,8 +125,9 @@ export class Context {
promises.push(request.response());
}
- await progress.race([...promises, progress.wait(5000)]);
- if (!promises.length)
+ if (promises.length)
+ await progress.race([...promises, progress.wait(5000)]);
+ else
await progress.wait(500);
return result;
@@ -134,12 +135,11 @@ export class Context {
async takeSnapshot(progress: Progress) {
const { full } = await this.page.snapshotForAI(progress, { doNotRenderActive: this.agentParams.doNotRenderActive });
- // TODO: it seems like redactText should be here.
- return full;
+ return redactSecrets(full, this.agentParams?.secrets);
}
async snapshotResult(progress: Progress, error?: Error): Promise<loopTypes.ToolResult> {
- const snapshot = this._redactText(await this.takeSnapshot(progress));
+ const snapshot = await this.takeSnapshot(progress);
const text: string[] = [];
if (error)
@@ -166,17 +166,23 @@ export class Context {
}));
}
- private _redactText(text: string): string {
- const secrets = this.agentParams?.secrets;
- if (!secrets)
- return text;
+}
- const redactText = (text: string) => {
- for (const { name, value } of secrets)
- text = text.replaceAll(value, `<secret>${name}</secret>`);
- return text;
- };
+export function redactSecrets(text: string, secrets: channels.NameValue[] | undefined): string {
+ if (!secrets)
+ return text;
+ for (const { name, value } of secrets)
+ text = text.replaceAll(value, `<secret>${name}</secret>`);
+ return text;
+}
- return redactText(text);
- }
+export function applySecrets(text: string, secrets: channels.NameValue[] | undefined): string {
+ if (!secrets)
+ return text;
+ const secret = secrets.find(s => s.name === text);
+ if (secret)
+ return secret.value;
+ for (const { name, value } of secrets)
+ text = text.replaceAll(`<secret>${name}</secret>`, value);
+ return text;
}
diff --git a/packages/playwright-core/src/server/agent/pageAgent.ts b/packages/playwright-core/src/server/agent/pageAgent.ts
index 7b22731aa5..5c1da041c8 100644
--- a/packages/playwright-core/src/server/agent/pageAgent.ts
+++ b/packages/playwright-core/src/server/agent/pageAgent.ts
@@ -45,7 +45,8 @@ export async function pageAgentPerform(progress: Progress, context: Context, use
const task = `
### Instructions
- Perform the following task on the page.
-- Your reply should be a tool call that performs action the page".
+- Your reply should be a tool call that performs action the page.
+- If you see text surrounded by <secret></secret>, it is a secret and you should preserve it as such. It will be replaced with the actual value before the tool call.
### Task
${userTask}
@@ -63,7 +64,7 @@ export async function pageAgentExpect(progress: Progress, context: Context, expe
const task = `
### Instructions
- Call one of the "browser_expect_*" tools to verify / assert the condition.
-- You can call exactly one tool and it can't be report_results, must be one of the assertion tools.
+- If you see text surrounded by <secret></secret>, it is a secret and you should preserve it as such. It will be replaced with the actual value before the tool call.
### Expectation
${expectation}
@@ -78,6 +79,7 @@ export async function pageAgentExtract(progress: Progress, context: Context, que
const task = `
### Instructions
Extract the following information from the page. Do not perform any actions, just extract the information.
+If you see text surrounded by <secret></secret>, it is a secret and you should preserve it as such. It will be replaced with the actual value before the tool call.
### Query
${query}`;
@@ -97,7 +99,6 @@ async function runLoop(progress: Progress, context: Context, toolDefinitions: To
const snapshot = await context.takeSnapshot(progress);
const { tools, callTool, reportedResult, refusedToPerformReason } = toolsForLoop(progress, context, toolDefinitions, { resultSchema, refuseToPerform: 'allow' });
- const secrets = Object.fromEntries((context.agentParams.secrets || [])?.map(s => ([s.name, s.value])));
const apiCacheTextBefore = context.agentParams.apiCacheFile ?
await fs.promises.readFile(context.agentParams.apiCacheFile, 'utf-8').catch(() => '{}') : '{}';
@@ -116,7 +117,6 @@ async function runLoop(progress: Progress, context: Context, toolDefinitions: To
debug,
callTool,
tools,
- secrets,
cache: apiCacheBefore,
...context.events,
});
diff --git a/tests/library/__llm_cache__/library-agent-expect-expect-timeout-during-generate.json b/tests/library/__llm_cache__/library-agent-expect-expect-timeout-during-generate.json
index ef50e269df..e53f97bc79 100644
--- a/tests/library/__llm_cache__/library-agent-expect-expect-timeout-during-generate.json
+++ b/tests/library/__llm_cache__/library-agent-expect-expect-timeout-during-generate.json
@@ -1,23 +1,23 @@
{
- "94919d5202a4e9298507668bb88822a939783915": {
+ "54f2adf81994f0dcc29b5165994b9ea09a0710c1": {
"result": {
"role": "assistant",
"content": [
{
"type": "text",
- "text": "I need to verify that an input has the value \"hello\", but according to the page snapshot, the textbox currently has the value \"bye\". I'll use the browser_expect_value tool to assert the expected value."
+ "text": "I need to verify that the input textbox has the value \"hello\", but according to the page snapshot, the textbox currently has the value \"bye\". Let me check this expectation."
},
{
"type": "tool_call",
"name": "browser_expect_value",
"arguments": {
"type": "textbox",
- "element": "input",
+ "element": "textbox",
"ref": "e2",
"value": "hello",
"_is_done": true
},
- "id": "toolu_015XEEa4SNqFf64DD2uXYyb8"
+ "id": "toolu_01FLk22B7JWYMXCnH4rgFiHS"
}
],
"stopReason": {
@@ -25,8 +25,8 @@
}
},
"usage": {
- "input": 2204,
- "output": 174
+ "input": 2217,
+ "output": 168
}
}
}
\ No newline at end of file
diff --git a/tests/library/__llm_cache__/library-agent-expect-expect-timeout-during-run-from-agent-options.json b/tests/library/__llm_cache__/library-agent-expect-expect-timeout-during-run-from-agent-options.json
index 5f3807f7c0..8ac04c1bfd 100644
--- a/tests/library/__llm_cache__/library-agent-expect-expect-timeout-during-run-from-agent-options.json
+++ b/tests/library/__llm_cache__/library-agent-expect-expect-timeout-during-run-from-agent-options.json
@@ -1,5 +1,5 @@
{
- "44bfbaa6fe0cb038adf37547ecb000d4f39e243f": {
+ "9c2cd8f6e72994d49af648ef822f48f208409acc": {
"result": {
"role": "assistant",
"content": [
@@ -15,7 +15,7 @@
"accessibleName": "Submit",
"_is_done": true
},
- "id": "toolu_0172x4DXyqgjDrUrBfPaLLZk"
+ "id": "toolu_01XyDgizei1g8JU7VdKe8TYo"
}
],
"stopReason": {
@@ -23,7 +23,7 @@
}
},
"usage": {
- "input": 2203,
+ "input": 2216,
"output": 107
}
}
diff --git a/tests/library/__llm_cache__/library-agent-expect-expect-timeout-during-run.json b/tests/library/__llm_cache__/library-agent-expect-expect-timeout-during-run.json
index 1b43b33307..11144c7775 100644
--- a/tests/library/__llm_cache__/library-agent-expect-expect-timeout-during-run.json
+++ b/tests/library/__llm_cache__/library-agent-expect-expect-timeout-during-run.json
@@ -1,11 +1,11 @@
{
- "44bfbaa6fe0cb038adf37547ecb000d4f39e243f": {
+ "9c2cd8f6e72994d49af648ef822f48f208409acc": {
"result": {
"role": "assistant",
"content": [
{
"type": "text",
- "text": "I need to verify that the submit button is visible on the page. Based on the page snapshot, I can see there is a button with the text \"Submit\" and reference `e2`."
+ "text": "I'll verify that the submit button is visible on the page."
},
{
"type": "tool_call",
@@ -15,7 +15,7 @@
"accessibleName": "Submit",
"_is_done": true
},
- "id": "toolu_01W7yygCD8L9Ppe5WGGfXGfZ"
+ "id": "toolu_01H6ErbqVqQTp3V3iTcGtkfj"
}
],
"stopReason": {
@@ -23,8 +23,8 @@
}
},
"usage": {
- "input": 2203,
- "output": 133
+ "input": 2216,
+ "output": 107
}
}
}
\ No newline at end of file
diff --git a/tests/library/__llm_cache__/library-agent-expect-expectTitle-success.json b/tests/library/__llm_cache__/library-agent-expect-expectTitle-success.json
index c9d778a14a..f36896bf56 100644
--- a/tests/library/__llm_cache__/library-agent-expect-expectTitle-success.json
+++ b/tests/library/__llm_cache__/library-agent-expect-expectTitle-success.json
@@ -1,11 +1,11 @@
{
- "893eb6be29a43887385345cedf150b6ff229a6e8": {
+ "4f7feecac5df193da7ccb5774663891e57273f80": {
"result": {
"role": "assistant",
"content": [
{
"type": "text",
- "text": "I need to verify that the page title is \"My Page Title\". I'll use the browser_expect_title tool to assert this condition."
+ "text": "I need to verify that the page title is \"My Page Title\". I'll use the browser_expect_title tool to check this."
},
{
"type": "tool_call",
@@ -14,7 +14,7 @@
"title": "My Page Title",
"_is_done": true
},
- "id": "toolu_01Xvf7Srkdda8vHPsMEbP2cC"
+ "id": "toolu_01P2u9QAXVAC9B3ARMBn44hz"
}
],
"stopReason": {
@@ -22,8 +22,8 @@
}
},
"usage": {
- "input": 2195,
- "output": 106
+ "input": 2208,
+ "output": 105
}
}
}
\ No newline at end of file
diff --git a/tests/library/__llm_cache__/library-agent-expect-expectTitle-wrong-title-error.json b/tests/library/__llm_cache__/library-agent-expect-expectTitle-wrong-title-error.json
index b44c9a1e18..7e3b4ff9e7 100644
--- a/tests/library/__llm_cache__/library-agent-expect-expectTitle-wrong-title-error.json
+++ b/tests/library/__llm_cache__/library-agent-expect-expectTitle-wrong-title-error.json
@@ -1,11 +1,11 @@
{
- "d969f4ed24b89f142366f64ad47d7b163b68e298": {
+ "3ffa506dc2ee3313201c176c0559552b960bde6f": {
"result": {
"role": "assistant",
"content": [
{
"type": "text",
- "text": "I need to verify that the page title is \"Other Title\". I'll use the browser_expect_title tool to assert this condition."
+ "text": "I need to verify that the page title is \"Other Title\". I'll use the browser_expect_title tool to check this."
},
{
"type": "tool_call",
@@ -14,7 +14,7 @@
"title": "Other Title",
"_is_done": true
},
- "id": "toolu_01CLLNmyLrSk8K98t85RQG8W"
+ "id": "toolu_0194o6G8uMhpoW7se3A8mfyb"
}
],
"stopReason": {
@@ -22,8 +22,8 @@
}
},
"usage": {
- "input": 2194,
- "output": 104
+ "input": 2207,
+ "output": 103
}
}
}
\ No newline at end of file
diff --git a/tests/library/__llm_cache__/library-agent-expect-expectURL-success.json b/tests/library/__llm_cache__/library-agent-expect-expectURL-success.json
index 9d221fb478..68aa243b7e 100644
--- a/tests/library/__llm_cache__/library-agent-expect-expectURL-success.json
+++ b/tests/library/__llm_cache__/library-agent-expect-expectURL-success.json
@@ -1,20 +1,20 @@
{
- "5b4a01d963d6ff8f48ac1d95c7cb61c279466901": {
+ "9fa986c2b86a0de020e4c2a78a1b3ddd30ccdc34": {
"result": {
"role": "assistant",
"content": [
{
"type": "text",
- "text": "I see the issue. The actual URL includes a server prefix. Let me use a regex pattern to match the URL that ends with /counter.html."
+ "text": "I'll verify that the page URL is /counter.html using the browser_expect_url tool."
},
{
"type": "tool_call",
"name": "browser_expect_url",
"arguments": {
- "regex": "/\\/counter\\.html$/",
+ "url": "/counter.html",
"_is_done": true
},
- "id": "toolu_012iHG67RxCNMvWiJ3b8XpAJ"
+ "id": "toolu_01M66r1LhnKBKaJeu9uNmbqp"
}
],
"stopReason": {
@@ -22,26 +22,26 @@
}
},
"usage": {
- "input": 2449,
- "output": 111
+ "input": 2253,
+ "output": 99
}
},
- "5fd07fcf484a378d71b68dc09227463b80797da0": {
+ "ead196d443fdc130c1e3131dee59fa13efafc8f0": {
"result": {
"role": "assistant",
"content": [
{
"type": "text",
- "text": "I need to verify that the page URL is /counter.html. I'll use the browser_expect_url tool to assert this condition."
+ "text": "I see the issue. The actual URL includes a server prefix. Let me use a regex pattern to match the URL ending with /counter.html."
},
{
"type": "tool_call",
"name": "browser_expect_url",
"arguments": {
- "url": "/counter.html",
+ "regex": "/\\/counter\\.html$/",
"_is_done": true
},
- "id": "toolu_01CHCwxsPcnbqiQ1mES884rT"
+ "id": "toolu_01Cr9RbuXDJdHwvQy15jTmiy"
}
],
"stopReason": {
@@ -49,8 +49,8 @@
}
},
"usage": {
- "input": 2240,
- "output": 107
+ "input": 2457,
+ "output": 110
}
}
}
\ No newline at end of file
diff --git a/tests/library/__llm_cache__/library-agent-expect-expectURL-with-regex-error.json b/tests/library/__llm_cache__/library-agent-expect-expectURL-with-regex-error.json
index 903146b512..881ceb10a7 100644
--- a/tests/library/__llm_cache__/library-agent-expect-expectURL-with-regex-error.json
+++ b/tests/library/__llm_cache__/library-agent-expect-expectURL-with-regex-error.json
@@ -1,5 +1,5 @@
{
- "f0db973fe7c770b8bdb9ae3c8999544a3305b4b3": {
+ "d122ec5e83cdbe0f76032e66a661e3d2f1ea89b1": {
"result": {
"role": "assistant",
"content": [
@@ -14,7 +14,7 @@
"regex": "/counter/",
"_is_done": true
},
- "id": "toolu_01KbFMW8c5VXyuzC9cHu9uPW"
+ "id": "toolu_01Dna2YzyJ1qd8UXSJe2aFmj"
}
],
"stopReason": {
@@ -22,7 +22,7 @@
}
},
"usage": {
- "input": 2239,
+ "input": 2252,
"output": 109
}
}
diff --git a/tests/library/__llm_cache__/library-agent-expect-expectURL-with-regex.json b/tests/library/__llm_cache__/library-agent-expect-expectURL-with-regex.json
index 36bae328ff..8cb13ff4e3 100644
--- a/tests/library/__llm_cache__/library-agent-expect-expectURL-with-regex.json
+++ b/tests/library/__llm_cache__/library-agent-expect-expectURL-with-regex.json
@@ -1,5 +1,5 @@
{
- "f0db973fe7c770b8bdb9ae3c8999544a3305b4b3": {
+ "d122ec5e83cdbe0f76032e66a661e3d2f1ea89b1": {
"result": {
"role": "assistant",
"content": [
@@ -14,7 +14,7 @@
"regex": "/counter/",
"_is_done": true
},
- "id": "toolu_0138rvBXbZ3q7JCYBVVHafSG"
+ "id": "toolu_01XNJbEmpYGXhFwqDNC7vphQ"
}
],
"stopReason": {
@@ -22,7 +22,7 @@
}
},
"usage": {
- "input": 2239,
+ "input": 2252,
"output": 102
}
}
diff --git a/tests/library/__llm_cache__/library-agent-expect-expectURL-wrong-URL-error.json b/tests/library/__llm_cache__/library-agent-expect-expectURL-wrong-URL-error.json
index a6eccd8bce..1d1ff1ae4a 100644
--- a/tests/library/__llm_cache__/library-agent-expect-expectURL-wrong-URL-error.json
+++ b/tests/library/__llm_cache__/library-agent-expect-expectURL-wrong-URL-error.json
@@ -1,20 +1,20 @@
{
- "0fb408b370e357bcf6be624699d1f2161564f2ba": {
+ "9fa986c2b86a0de020e4c2a78a1b3ddd30ccdc34": {
"result": {
"role": "assistant",
"content": [
{
"type": "text",
- "text": "I see the issue - the actual URL includes a server prefix. Let me use a regex pattern to match the URL that ends with /counter.html."
+ "text": "I'll verify that the page URL is /counter.html using the browser_expect_url tool."
},
{
"type": "tool_call",
"name": "browser_expect_url",
"arguments": {
- "regex": "/\\/counter\\.html$/",
+ "url": "/counter.html",
"_is_done": true
},
- "id": "toolu_014L3BQzJhnvoW9j4FeTEBUm"
+ "id": "toolu_01NuGwxGZzVX7eUdAxjFhcdp"
}
],
"stopReason": {
@@ -22,26 +22,26 @@
}
},
"usage": {
- "input": 2449,
- "output": 111
+ "input": 2253,
+ "output": 99
}
},
- "5fd07fcf484a378d71b68dc09227463b80797da0": {
+ "ed2e02072fbd34ab4e02136152b8d7a667ce0ddf": {
"result": {
"role": "assistant",
"content": [
{
"type": "text",
- "text": "I need to verify that the page URL is /counter.html. I'll use the browser_expect_url tool to assert this condition."
+ "text": "I see the issue. The actual URL includes a server prefix. Let me use a regex pattern to match the URL ending with /counter.html."
},
{
"type": "tool_call",
"name": "browser_expect_url",
"arguments": {
- "url": "/counter.html",
+ "regex": "/counter\\.html$/",
"_is_done": true
},
- "id": "toolu_01VQkp97y11LQjZD8pLzkqRz"
+ "id": "toolu_017JMMpUZjcix2q53NG9w5W3"
}
],
"stopReason": {
@@ -49,8 +49,8 @@
}
},
"usage": {
- "input": 2240,
- "output": 107
+ "input": 2457,
+ "output": 109
}
}
}
\ No newline at end of file
diff --git a/tests/library/__llm_cache__/library-agent-limits-should-respect-max-actions-limit.json b/tests/library/__llm_cache__/library-agent-limits-should-respect-max-actions-limit.json
index 9942fd543e..f42693851f 100644
--- a/tests/library/__llm_cache__/library-agent-limits-should-respect-max-actions-limit.json
+++ b/tests/library/__llm_cache__/library-agent-limits-should-respect-max-actions-limit.json
@@ -1,11 +1,11 @@
{
- "256715bc9f07732bc90dfd6d9cea24148e70518b": {
+ "02c08e7832aaa36c18ea5191d8576315d0b2de69": {
"result": {
"role": "assistant",
"content": [
{
"type": "text",
- "text": "Now clicking the submit button for the fourth time."
+ "text": "Now clicking the fourth time."
},
{
"type": "tool_call",
@@ -15,7 +15,7 @@
"ref": "e2",
"_is_done": false
},
- "id": "toolu_016zf6ndeWMLWtogdLUxMkeN"
+ "id": "toolu_01RaTy7Rx5THV9gfuXZDLpJc"
}
],
"stopReason": {
@@ -23,17 +23,17 @@
}
},
"usage": {
- "input": 3350,
- "output": 102
+ "input": 3372,
+ "output": 98
}
},
- "49310149fc683d654fbf987efc96e7b79a862a15": {
+ "3e5f433d3eb65d0452eb142cc6c9169ca743ad9c": {
"result": {
"role": "assistant",
"content": [
{
"type": "text",
- "text": "Now clicking the submit button for the second time."
+ "text": "I'll click the submit button 5 times as requested. Let me start by clicking it the first time."
},
{
"type": "tool_call",
@@ -43,7 +43,7 @@
"ref": "e2",
"_is_done": false
},
- "id": "toolu_01QeXTGs82oN5bSLP6nqbw5Y"
+ "id": "toolu_017Lc8Rcz2Pqo5bN2gnQ5xJr"
}
],
"stopReason": {
@@ -51,17 +51,17 @@
}
},
"usage": {
- "input": 3199,
- "output": 102
+ "input": 3085,
+ "output": 115
}
},
- "7b25e318de64dc9d2494432a6d311631e2c1edc0": {
+ "3eac3a4f34e9b3d5239f95864a043d84bcafb3cf": {
"result": {
"role": "assistant",
"content": [
{
"type": "text",
- "text": "I'll click the submit button 5 times as requested. Since each click needs to be done sequentially, I'll start with the first click."
+ "text": "Now clicking the second time."
},
{
"type": "tool_call",
@@ -71,7 +71,7 @@
"ref": "e2",
"_is_done": false
},
- "id": "toolu_019eyM9fuWSeXUuPxtPdyGyM"
+ "id": "toolu_01TRZkN9Q2PXgDbvNQo28WnV"
}
],
"stopReason": {
@@ -79,17 +79,17 @@
}
},
"usage": {
- "input": 3047,
- "output": 123
+ "input": 3229,
+ "output": 98
}
},
- "b43d5e2a22efa899d84137b945779e813636435a": {
+ "7da30a83cb82a113b53b02fd7ff3ca946a5bf2b0": {
"result": {
"role": "assistant",
"content": [
{
"type": "text",
- "text": "Now clicking the submit button for the third time."
+ "text": "Now clicking the third time."
},
{
"type": "tool_call",
@@ -99,7 +99,7 @@
"ref": "e2",
"_is_done": false
},
- "id": "toolu_01AV8B791DcfGqk3KqFt4zsk"
+ "id": "toolu_013WxjykEaqVN1aqpEqobvEx"
}
],
"stopReason": {
@@ -107,8 +107,8 @@
}
},
"usage": {
- "input": 3276,
- "output": 102
+ "input": 3302,
+ "output": 98
}
}
}
\ No newline at end of file
diff --git a/tests/library/__llm_cache__/library-agent-perform-click-a-button.json b/tests/library/__llm_cache__/library-agent-perform-click-a-button.json
index 41657148b7..77d2113e64 100644
--- a/tests/library/__llm_cache__/library-agent-perform-click-a-button.json
+++ b/tests/library/__llm_cache__/library-agent-perform-click-a-button.json
@@ -1,5 +1,5 @@
{
- "b17b5a7075f197f7886c6b8c882c2c41edc2deb6": {
+ "9326ee07d3e2e68abb13cb1efe5cda3578c9df1c": {
"result": {
"role": "assistant",
"content": [
@@ -15,7 +15,7 @@
"ref": "e2",
"_is_done": true
},
- "id": "toolu_01FRiTkXDHCSezCjbjiKwv1M"
+ "id": "toolu_01YUmruXpnPwjbUsFsueo7ES"
}
],
"stopReason": {
@@ -23,7 +23,7 @@
}
},
"usage": {
- "input": 3043,
+ "input": 3081,
"output": 101
}
}
diff --git a/tests/library/__llm_cache__/library-agent-perform-empty-cache-file-works.json b/tests/library/__llm_cache__/library-agent-perform-empty-cache-file-works.json
index 4105d11efc..4fc32ff3be 100644
--- a/tests/library/__llm_cache__/library-agent-perform-empty-cache-file-works.json
+++ b/tests/library/__llm_cache__/library-agent-perform-empty-cache-file-works.json
@@ -1,5 +1,5 @@
{
- "29a2028099170f3ab333459d1f9c78de2767e7b3": {
+ "4a2860c6790066a9e8107b817fbb9f8f9306571a": {
"result": {
"role": "assistant",
"content": [
@@ -15,7 +15,7 @@
"ref": "e2",
"_is_done": true
},
- "id": "toolu_01H4GJjgGimjjAYjRt3Nbs2W"
+ "id": "toolu_01H4Ejr8CtohnUGD27AbTcP4"
}
],
"stopReason": {
@@ -23,7 +23,7 @@
}
},
"usage": {
- "input": 3043,
+ "input": 3081,
"output": 101
}
}
diff --git a/tests/library/__llm_cache__/library-agent-perform-expect-value.json b/tests/library/__llm_cache__/library-agent-perform-expect-value.json
index 867c9ab9aa..5723b9cafe 100644
--- a/tests/library/__llm_cache__/library-agent-perform-expect-value.json
+++ b/tests/library/__llm_cache__/library-agent-perform-expect-value.json
@@ -1,37 +1,11 @@
{
- "112359efcd28400619777fd107f6ffe51ab30550": {
+ "7caf4c9ea50a2bfdff964a3a4e2749920efdd0ad": {
"result": {
"role": "assistant",
"content": [
{
"type": "text",
- "text": "I'll help you complete this task. Let me start by taking a snapshot of the page to see its current state, then enter \"bogus\" into the email field and verify the results."
- },
- {
- "type": "tool_call",
- "name": "browser_snapshot",
- "arguments": {
- "_is_done": false
- },
- "id": "toolu_0189HTF94bMQ43WjswLFdorx"
- }
- ],
- "stopReason": {
- "code": "ok"
- }
- },
- "usage": {
- "input": 3076,
- "output": 95
- }
- },
- "4e684686cbe26068e18ae9a999fce9ebc9fcc1e0": {
- "result": {
- "role": "assistant",
- "content": [
- {
- "type": "text",
- "text": "Perfect! I've successfully completed all three parts of the task:\n\n1. ✓ Entered \"bogus\" into the email field\n2. ✓ Verified the value is in fact \"bogus\" (shown in the snapshot as `textbox \"Email Address\" [ref=e2]: bogus`)\n3. ✓ Confirmed the error message is displayed (\"Error: Invalid email address\")"
+ "text": "Perfect! I have successfully completed all parts of the task:\n\n1. ✅ **Entered \"bogus\" into the email field** - The text has been typed into the email address textbox\n2. ✅ **Verified the value is \"bogus\"** - The page snapshot confirms the textbox contains \"bogus\"\n3. ✅ **Confirmed error message is displayed** - The error message \"Error: Invalid email address\" is now visible on the page\n\nAll task requirements have been met."
},
{
"type": "tool_call",
@@ -39,7 +13,7 @@
"arguments": {
"_is_done": true
},
- "id": "toolu_01XPDigBG7mfp98cGMJRU4jE"
+ "id": "toolu_01TZDE1jZhfNCCYpXw8NTwj3"
}
],
"stopReason": {
@@ -47,28 +21,28 @@
}
},
"usage": {
- "input": 3347,
- "output": 147
+ "input": 3324,
+ "output": 170
}
},
- "5c8d7623f599bfa85c020a625c8dddfd2ae285ac": {
+ "930612de8c1fc49854341e2f8b67ec5a69e4880f": {
"result": {
"role": "assistant",
"content": [
{
"type": "text",
- "text": "Now I'll enter \"bogus\" into the email field:"
+ "text": "I'll help you complete this task. Let me start by taking a snapshot of the current page to see the full state, then enter \"bogus\" into the email field."
},
{
"type": "tool_call",
"name": "browser_type",
"arguments": {
- "element": "Email Address textbox",
+ "element": "textbox \"Email Address\"",
"ref": "e2",
"text": "bogus",
"_is_done": false
},
- "id": "toolu_019cNZbsEFtgc3KABL28uGYX"
+ "id": "toolu_01GpawXwxFvNuv2fE3izZQw4"
}
],
"stopReason": {
@@ -76,8 +50,8 @@
}
},
"usage": {
- "input": 3202,
- "output": 125
+ "input": 3114,
+ "output": 150
}
}
}
\ No newline at end of file
diff --git a/tests/library/__llm_cache__/library-agent-perform-extract-task.json b/tests/library/__llm_cache__/library-agent-perform-extract-task.json
index a70ba9c15a..e03a1e2f69 100644
--- a/tests/library/__llm_cache__/library-agent-perform-extract-task.json
+++ b/tests/library/__llm_cache__/library-agent-perform-extract-task.json
@@ -1,11 +1,11 @@
{
- "c2afe4add9f92f02f33fd49f7911ec415d91b3e3": {
+ "750e0b9cc7fc409e41084beba26b787be1a03d2d": {
"result": {
"role": "assistant",
"content": [
{
"type": "text",
- "text": "I'll extract the todo items and their statuses from the page."
+ "text": "I'll extract the todo items with their statuses from the page."
},
{
"type": "tool_call",
@@ -23,7 +23,7 @@
],
"_is_done": true
},
- "id": "toolu_014zLDZkeeCQ3FaUiiATmEg1"
+ "id": "toolu_0125zCMU5LCT66P8oUVFpVka"
}
],
"stopReason": {
@@ -31,11 +31,11 @@
}
},
"usage": {
- "input": 1018,
+ "input": 1057,
"output": 113
}
},
- "e53dabe92453ecd4602c15601cd50c6ddc88d7bf": {
+ "934bde90819329f27493b394f563495f913da8dd": {
"result": {
"role": "assistant",
"content": [
@@ -59,7 +59,7 @@
],
"_is_done": true
},
- "id": "toolu_01WX3AiJbYeVKdJ8EiZSFP5q"
+ "id": "toolu_01Sq94TRGMakQC4pdGA5AFRw"
}
],
"stopReason": {
@@ -67,7 +67,7 @@
}
},
"usage": {
- "input": 1020,
+ "input": 1055,
"output": 113
}
}
diff --git a/tests/library/__llm_cache__/library-agent-perform-perform-history.json b/tests/library/__llm_cache__/library-agent-perform-perform-history.json
index 11294e061a..fbd133118e 100644
--- a/tests/library/__llm_cache__/library-agent-perform-perform-history.json
+++ b/tests/library/__llm_cache__/library-agent-perform-perform-history.json
@@ -1,11 +1,11 @@
{
- "35faa6e330f5198be604e1d5b81cd0c1c1627a63": {
+ "54afa1e29fe78452a29b389587b30f71e7b68876": {
"result": {
"role": "assistant",
"content": [
{
"type": "text",
- "text": "I'll click the Fox button again."
+ "text": "I'll click the Fox button again as requested."
},
{
"type": "tool_call",
@@ -15,7 +15,7 @@
"ref": "e3",
"_is_done": true
},
- "id": "toolu_014j2J9Q3i9dw1j6ZRtms6bR"
+ "id": "toolu_015TpKarPp9KvpT5K86f1fez"
}
],
"stopReason": {
@@ -23,17 +23,17 @@
}
},
"usage": {
- "input": 3093,
- "output": 100
+ "input": 3131,
+ "output": 102
}
},
- "d98ecb24c7732360ace82e5dabac9c9bf3266a71": {
+ "83a2685b1bdc1b947fbaa05e3e2ce3a7323ab216": {
"result": {
"role": "assistant",
"content": [
{
"type": "text",
- "text": "I'll click the Fox button for you."
+ "text": "I'll click the Fox button as requested."
},
{
"type": "tool_call",
@@ -43,7 +43,7 @@
"ref": "e3",
"_is_done": true
},
- "id": "toolu_011z2gTMSXGLrYHr5DtSXwG6"
+ "id": "toolu_01JnaV2Pgnyaz9oYDFkXzDMQ"
}
],
"stopReason": {
@@ -51,7 +51,7 @@
}
},
"usage": {
- "input": 3080,
+ "input": 3118,
"output": 101
}
}
diff --git a/tests/library/__llm_cache__/library-agent-perform-perform-reports-error.json b/tests/library/__llm_cache__/library-agent-perform-perform-reports-error.json
index c96bf228bc..e44caef465 100644
--- a/tests/library/__llm_cache__/library-agent-perform-perform-reports-error.json
+++ b/tests/library/__llm_cache__/library-agent-perform-perform-reports-error.json
@@ -1,11 +1,11 @@
{
- "4804d46618cdfe9201831d9757cc2478ba4d3909": {
+ "c7255b8d4cf1409c656d5b99ef1cf4c8e7100b46": {
"result": {
"role": "assistant",
"content": [
{
"type": "text",
- "text": "I'll help you click the Rabbit button. Let me first take a snapshot of the current page to locate it."
+ "text": "I need to click the Rabbit button, but I don't see it in the current page snapshot. The snapshot only shows \"Wolf\" and \"Fox\" buttons. Let me take a fresh snapshot to see if the Rabbit button is present on the page."
},
{
"type": "tool_call",
@@ -13,7 +13,7 @@
"arguments": {
"_is_done": false
},
- "id": "toolu_012uNHvhXieXRFhgLnVZhcm5"
+ "id": "toolu_01YRa3CAXt28CUBgHQak1c7P"
}
],
"stopReason": {
@@ -21,26 +21,26 @@
}
},
"usage": {
- "input": 3067,
- "output": 80
+ "input": 3105,
+ "output": 109
}
},
- "4f5fff9399106121f4cd50915a05ff4322f76e2c": {
+ "e0d9dd27b99a5a81c6eda46e666ddca703dab5ed": {
"result": {
"role": "assistant",
"content": [
{
"type": "text",
- "text": "I can see the page snapshot shows only \"Wolf\" and \"Fox\" buttons, but no \"Rabbit\" button is visible. The Rabbit button may not be present on the current page, or it might need to be revealed through some interaction. Let me check if there are any other elements or if scrolling might reveal more content."
+ "text": "The page snapshot shows only \"Wolf\" and \"Fox\" buttons, but no \"Rabbit\" button. The Rabbit button is not currently visible on the page. This could mean:\n1. The button doesn't exist on this page\n2. The button might appear after some interaction\n3. The page might need to be scrolled\n\nSince I cannot find the Rabbit button on the page, I cannot complete this task."
},
{
"type": "tool_call",
"name": "refuse_to_perform",
"arguments": {
- "reason": "The Rabbit button is not present in the current page snapshot. Only \"Wolf\" and \"Fox\" buttons are visible. The Rabbit button may not exist on this page, or it might require additional actions to make it appear.",
+ "reason": "The Rabbit button is not present on the page. The page only contains \"Wolf\" and \"Fox\" buttons. The requested Rabbit button cannot be found.",
"_is_done": true
},
- "id": "toolu_01BHYAisGb52ihyoJ9cwku7f"
+ "id": "toolu_018RL4xJTXhYz416fYa5fThY"
}
],
"stopReason": {
@@ -48,8 +48,8 @@
}
},
"usage": {
- "input": 3199,
- "output": 189
+ "input": 3266,
+ "output": 195
}
}
}
\ No newline at end of file
diff --git a/tests/library/__llm_cache__/library-agent-perform-perform-run-timeout-inherited-from-page.json b/tests/library/__llm_cache__/library-agent-perform-perform-run-timeout-inherited-from-page.json
index d215fb6b79..747b1201a9 100644
--- a/tests/library/__llm_cache__/library-agent-perform-perform-run-timeout-inherited-from-page.json
+++ b/tests/library/__llm_cache__/library-agent-perform-perform-run-timeout-inherited-from-page.json
@@ -1,5 +1,5 @@
{
- "554366043f6477d30a36655c10cbb4ec0eaa21d4": {
+ "5a8dd8595e3f169b3ef7d5cc8eeb0dc74cdbb6e5": {
"result": {
"role": "assistant",
"content": [
@@ -11,11 +11,11 @@
"type": "tool_call",
"name": "browser_click",
"arguments": {
- "element": "Fox button",
+ "element": "button \"Fox\"",
"ref": "e3",
"_is_done": true
},
- "id": "toolu_015YYjPVGNRetkKZUPDLRzdJ"
+ "id": "toolu_01ExgUecR6hzSb8QtpoBnCzo"
}
],
"stopReason": {
@@ -23,8 +23,8 @@
}
},
"usage": {
- "input": 3066,
- "output": 101
+ "input": 3104,
+ "output": 103
}
}
}
\ No newline at end of file
diff --git a/tests/library/__llm_cache__/library-agent-perform-perform-run-timeout.json b/tests/library/__llm_cache__/library-agent-perform-perform-run-timeout.json
index 6b925bc18f..64a01bbb18 100644
--- a/tests/library/__llm_cache__/library-agent-perform-perform-run-timeout.json
+++ b/tests/library/__llm_cache__/library-agent-perform-perform-run-timeout.json
@@ -1,11 +1,11 @@
{
- "554366043f6477d30a36655c10cbb4ec0eaa21d4": {
+ "5a8dd8595e3f169b3ef7d5cc8eeb0dc74cdbb6e5": {
"result": {
"role": "assistant",
"content": [
{
"type": "text",
- "text": "I'll click the Fox button for you."
+ "text": "I'll help you click the Fox button. Let me do that now."
},
{
"type": "tool_call",
@@ -15,7 +15,7 @@
"ref": "e3",
"_is_done": true
},
- "id": "toolu_01QM9zuuk8CZTzztnX77Zo48"
+ "id": "toolu_01XbDB7qqPFpbjYR8k7fgb6W"
}
],
"stopReason": {
@@ -23,8 +23,8 @@
}
},
"usage": {
- "input": 3066,
- "output": 101
+ "input": 3104,
+ "output": 107
}
}
}
\ No newline at end of file
diff --git a/tests/library/__llm_cache__/library-agent-perform-retrieve-a-secret.json b/tests/library/__llm_cache__/library-agent-perform-retrieve-a-secret.json
index 2477a9a157..dd3ddb8b0b 100644
--- a/tests/library/__llm_cache__/library-agent-perform-retrieve-a-secret.json
+++ b/tests/library/__llm_cache__/library-agent-perform-retrieve-a-secret.json
@@ -1,22 +1,142 @@
{
- "c1820ebf65381413ca130709a7eaecc6847a147a": {
+ "a026968099fa05b5b0ebc7c89a27adc6e9d496ad": {
"result": {
"role": "assistant",
"content": [
{
"type": "text",
- "text": "I'll enter the x-secret-email into the email field."
+ "text": "I need to verify that the list contains exactly two entries with the secret email value. I can see from the snapshot that there's a list with role \"list\" and accessible name \"list of emails\" containing two listitem elements, both with the secret email."
+ },
+ {
+ "type": "tool_call",
+ "name": "browser_expect_list_visible",
+ "arguments": {
+ "listRole": "list",
+ "listAccessibleName": "list of emails",
+ "itemRole": "listitem",
+ "items": [
+ "<secret>x-secret-email</secret>",
+ "<secret>x-secret-email</secret>"
+ ],
+ "_is_done": true
+ },
+ "id": "toolu_01Hpc12BGnTMa1HxaZp6AG3P"
+ }
+ ],
+ "stopReason": {
+ "code": "ok"
+ }
+ },
+ "usage": {
+ "input": 2359,
+ "output": 213
+ }
+ },
+ "a0f5cdc7923bde9a10b64a4d85509396b8e9f078": {
+ "result": {
+ "role": "assistant",
+ "content": [
+ {
+ "type": "text",
+ "text": "I'll type \"x-secret-email\" into the email field and press enter to submit it."
+ },
+ {
+ "type": "tool_call",
+ "name": "browser_type",
+ "arguments": {
+ "element": "textbox \"Email Address\"",
+ "ref": "e2",
+ "text": "x-secret-email",
+ "submit": true,
+ "_is_done": true
+ },
+ "id": "toolu_01Tx9AtcuXG1w4hDz1EtHYZB"
+ }
+ ],
+ "stopReason": {
+ "code": "ok"
+ }
+ },
+ "usage": {
+ "input": 3184,
+ "output": 155
+ }
+ },
+ "caa6dbc8b147dc1fa8dab4636c7f7a22a7596728": {
+ "result": {
+ "role": "assistant",
+ "content": [
+ {
+ "type": "text",
+ "text": "I'll enter \"x-secret-email\" into the email field."
},
{
"type": "tool_call",
"name": "browser_type",
"arguments": {
- "element": "Email Address textbox",
+ "element": "textbox \"Email Address\"",
"ref": "e2",
"text": "x-secret-email",
"_is_done": true
},
- "id": "toolu_01XDuSZZZ1qEJuhUHDoGouW4"
+ "id": "toolu_011b6YmkUakWPtHU7brLdPss"
+ }
+ ],
+ "stopReason": {
+ "code": "ok"
+ }
+ },
+ "usage": {
+ "input": 3108,
+ "output": 132
+ }
+ },
+ "f326cb8408f097d92c0df02b4ea808ff1330a8e5": {
+ "result": {
+ "role": "assistant",
+ "content": [
+ {
+ "type": "text",
+ "text": "I'll verify that the email field contains the secret email value."
+ },
+ {
+ "type": "tool_call",
+ "name": "browser_expect_value",
+ "arguments": {
+ "type": "textbox",
+ "element": "Email Address",
+ "ref": "e2",
+ "value": "<secret>x-secret-email</secret>",
+ "_is_done": true
+ },
+ "id": "toolu_01WciKv6y2zjsRXr2gr5K8dU"
+ }
+ ],
+ "stopReason": {
+ "code": "ok"
+ }
+ },
+ "usage": {
+ "input": 2272,
+ "output": 150
+ }
+ },
+ "fdb2848191adfed7e9f50ce3d2d465f434f3b8df": {
+ "result": {
+ "role": "assistant",
+ "content": [
+ {
+ "type": "text",
+ "text": "I'll press Enter to add the email to the list. The email field currently contains the secret email value, so pressing Enter should add it to the list."
+ },
+ {
+ "type": "tool_call",
+ "name": "browser_press_key",
+ "arguments": {
+ "key": "Enter",
+ "_is_done": true
+ },
+ "id": "toolu_011Yxd94HaHn2HF4UAkFCpze"
}
],
"stopReason": {
@@ -24,8 +144,8 @@
}
},
"usage": {
- "input": 3051,
- "output": 129
+ "input": 3150,
+ "output": 107
}
}
}
\ No newline at end of file
diff --git a/tests/library/agent-expect.spec.ts b/tests/library/agent-expect.spec.ts
index f301a6c5ab..988780dcb2 100644
--- a/tests/library/agent-expect.spec.ts
+++ b/tests/library/agent-expect.spec.ts
@@ -280,7 +280,7 @@ test('expectURL wrong URL error', async ({ context, server }) => {
await page.goto(server.PREFIX + '/empty.html');
const error = await agent.expect('page URL is /counter.html').catch(e => e);
expect(stripAnsi(error.message)).toContain(`pageAgent.expect: expect(page).toHaveURL(expected) failed`);
- expect(stripAnsi(error.message)).toContain(`Received: ${server.PREFIX}/empty.html`);
+ expect(stripAnsi(error.message)).toContain(`Received: <secret>SERVER</secret>/empty.html`);
}
});
@@ -324,7 +324,7 @@ test('expectURL with regex error', async ({ context, server }) => {
await page.goto(server.PREFIX + '/empty.html');
const error = await agent.expect('page URL matches /counter pattern').catch(e => e);
expect(stripAnsi(error.message)).toContain(`pageAgent.expect: expect(page).toHaveURL(expected) failed`);
- expect(stripAnsi(error.message)).toContain(`Received: ${server.PREFIX}/empty.html`);
+ expect(stripAnsi(error.message)).toContain(`Received: <secret>SERVER</secret>/empty.html`);
}
});
diff --git a/tests/library/agent-perform.spec.ts b/tests/library/agent-perform.spec.ts
index 9cdd91e34f..7bca002b8b 100644
--- a/tests/library/agent-perform.spec.ts
+++ b/tests/library/agent-perform.spec.ts
@@ -46,14 +46,34 @@ test('click a button', async ({ context }) => {
test('retrieve a secret', async ({ context }) => {
await run(context, async (page, agent) => {
- await page.setContent('<input type="email" name="email" placeholder="Email Address"/>');
+ await page.setContent(`
+ <input type="email" name="email" placeholder="Email Address"/>
+ <ul aria-label="list of emails"></ul>
+ <script>
+ const input = document.querySelector('input');
+ const list = document.querySelector('ul');
+ input.addEventListener('keydown', () => {
+ const item = document.createElement('li');
+ item.textContent = input.value;
+ list.appendChild(item);
+ input.value = '';
+ });
+ </script>
+ `);
await agent.perform('Enter x-secret-email into the email field');
+ await agent.expect('Check that email fields contains x-secret-email');
+ await agent.perform('Press enter to add the email to the list');
+ await agent.perform('Enter x-secret-email into the email field and press enter');
+ await agent.expect('Check that list of emails contains two entries with x-secret-email');
await expect(page.locator('body')).toMatchAriaSnapshot(`
- - textbox "Email Address": [email protected]
+ - textbox "Email Address"
+ - list "list of emails":
+ - listitem: [email protected]
+ - listitem: [email protected]
`);
}, { secrets: { 'x-secret-email': '[email protected]' } });
- expect(await cacheObject()).toEqual({
+ expect(await cacheObject()).toEqual(expect.objectContaining({
'Enter x-secret-email into the email field': {
actions: [{
code: `await page.getByRole('textbox', { name: 'Email Address' }).fill('x-secret-email');`,
@@ -62,7 +82,16 @@ test('retrieve a secret', async ({ context }) => {
text: 'x-secret-email',
}],
},
- });
+ 'Check that email fields contains x-secret-email': {
+ 'actions': [{
+ 'code': `await expect(page.getByRole('textbox', { name: 'Email Address' })).toHaveValue('<secret>x-secret-email</secret>');`,
+ 'method': 'expectValue',
+ 'selector': 'internal:role=textbox[name="Email Address"i]',
+ 'type': 'textbox',
+ 'value': '<secret>x-secret-email</secret>',
+ }],
+ },
+ }));
});
test('extract task', async ({ context }) => {