Skip to content

Playwright is a framework for Web Testing and Automation. It allows testing Chromium, Firefox and WebKit with a single API.

test: do not render [active] in snapshots to reuse cache across browsers (#38851)

Dmitry Gozman477cfe30a720
diff --git a/packages/injected/src/ariaSnapshot.ts b/packages/injected/src/ariaSnapshot.ts
index 6f7348115b..2ce324e81f 100644
--- a/packages/injected/src/ariaSnapshot.ts
+++ b/packages/injected/src/ariaSnapshot.ts
@@ -39,6 +39,7 @@ let lastRef = 0;
export type AriaTreeOptions = {
mode: 'ai' | 'expect' | 'codegen' | 'autoexpect';
refPrefix?: string;
+ doNotRenderActive?: boolean;
};
type InternalOptions = {
@@ -59,7 +60,7 @@ function toInternalOptions(options: AriaTreeOptions): InternalOptions {
refs: 'interactable',
refPrefix: options.refPrefix,
includeGenericRole: true,
- renderActive: true,
+ renderActive: !options.doNotRenderActive,
renderCursorPointer: true,
};
}
diff --git a/packages/playwright-core/src/client/page.ts b/packages/playwright-core/src/client/page.ts
index 4e3d827595..e7c027f2a8 100644
--- a/packages/playwright-core/src/client/page.ts
+++ b/packages/playwright-core/src/client/page.ts
@@ -854,6 +854,7 @@ export class Page extends ChannelOwner<channels.PageChannel> implements api.Page
apiKey: options.provider?.apiKey,
apiTimeout: options.provider?.apiTimeout,
apiCacheFile: (options.provider as any)?._apiCacheFile,
+ doNotRenderActive: (options as any)._doNotRenderActive,
model: options.provider?.model,
cacheFile: options.cache?.cacheFile,
cacheOutFile: options.cache?.cacheOutFile,
diff --git a/packages/playwright-core/src/protocol/validator.ts b/packages/playwright-core/src/protocol/validator.ts
index 4ed0e4af71..bc253a5a7b 100644
--- a/packages/playwright-core/src/protocol/validator.ts
+++ b/packages/playwright-core/src/protocol/validator.ts
@@ -1495,6 +1495,7 @@ scheme.PageAgentParams = tObject({
apiCacheFile: tOptional(tString),
cacheFile: tOptional(tString),
cacheOutFile: tOptional(tString),
+ doNotRenderActive: tOptional(tBoolean),
maxActions: tOptional(tInt),
maxActionRetries: tOptional(tInt),
maxTokens: tOptional(tInt),
diff --git a/packages/playwright-core/src/server/agent/context.ts b/packages/playwright-core/src/server/agent/context.ts
index 6bb1b618f1..41853f073d 100644
--- a/packages/playwright-core/src/server/agent/context.ts
+++ b/packages/playwright-core/src/server/agent/context.ts
@@ -132,9 +132,14 @@ export class Context {
return result;
}
+ async takeSnapshot(progress: Progress) {
+ const { full } = await this.page.snapshotForAI(progress, { doNotRenderActive: this.agentParams.doNotRenderActive });
+ // TODO: it seems like redactText should be here.
+ return full;
+ }
+
async snapshotResult(progress: Progress, error?: Error): Promise<loopTypes.ToolResult> {
- let { full } = await this.page.snapshotForAI(progress);
- full = this._redactText(full);
+ const snapshot = this._redactText(await this.takeSnapshot(progress));
const text: string[] = [];
if (error)
@@ -142,12 +147,12 @@ export class Context {
else
text.push(`# Success`);
- text.push(`# Page snapshot\n${full}`);
+ text.push(`# Page snapshot\n${snapshot}`);
return {
_meta: {
'dev.lowire/state': {
- 'Page snapshot': full
+ 'Page snapshot': snapshot
},
'dev.lowire/history': error ? [{
category: 'error',
diff --git a/packages/playwright-core/src/server/agent/pageAgent.ts b/packages/playwright-core/src/server/agent/pageAgent.ts
index dfad33bf99..cca7bb8639 100644
--- a/packages/playwright-core/src/server/agent/pageAgent.ts
+++ b/packages/playwright-core/src/server/agent/pageAgent.ts
@@ -88,13 +88,12 @@ ${query}`;
async function runLoop(progress: Progress, context: Context, toolDefinitions: ToolDefinition[], userTask: string, resultSchema: loopTypes.Schema | undefined, params: CallParams): Promise<{
result: any
}> {
- const { page } = context;
if (!context.agentParams.api || !context.agentParams.model)
throw new Error(`This action requires the API and API key to be set on the page agent. Did you mean to --run-agents=missing?`);
if (!context.agentParams.apiKey)
throw new Error(`This action requires API key to be set on the page agent.`);
- const { full } = await page.snapshotForAI(progress);
+ const snapshot = await context.takeSnapshot(progress);
const { tools, callTool, reportedResult, refusedToPerformReason } = toolsForLoop(progress, context, toolDefinitions, { resultSchema, refuseToPerform: 'allow' });
const secrets = Object.fromEntries((context.agentParams.secrets || [])?.map(s => ([s.name, s.value])));
@@ -136,7 +135,7 @@ async function runLoop(progress: Progress, context: Context, toolDefinitions: To
task.push('');
}
task.push('### Page snapshot');
- task.push(full);
+ task.push(snapshot);
task.push('');
const { error, usage } = await loop.run(task.join('\n'), { signal: progress.signal });
diff --git a/packages/playwright-core/src/server/page.ts b/packages/playwright-core/src/server/page.ts
index 1759727992..278b89a104 100644
--- a/packages/playwright-core/src/server/page.ts
+++ b/packages/playwright-core/src/server/page.ts
@@ -863,7 +863,7 @@ export class Page extends SdkObject<PageEventMap> {
await Promise.all(this.frames().map(frame => frame.hideHighlight().catch(() => {})));
}
- async snapshotForAI(progress: Progress, options: { track?: string } = {}): Promise<{ full: string, incremental?: string }> {
+ async snapshotForAI(progress: Progress, options: { track?: string, doNotRenderActive?: boolean } = {}): Promise<{ full: string, incremental?: string }> {
const snapshot = await snapshotFrameForAI(progress, this.mainFrame(), options);
return { full: snapshot.full.join('\n'), incremental: snapshot.incremental?.join('\n') };
}
@@ -986,7 +986,7 @@ export class InitScript {
}
-async function snapshotFrameForAI(progress: Progress, frame: frames.Frame, options: { track?: string }): Promise<{ full: string[], incremental?: string[] }> {
+async function snapshotFrameForAI(progress: Progress, frame: frames.Frame, options: { track?: string, doNotRenderActive?: boolean } = {}): Promise<{ full: string[], incremental?: string[] }> {
// Only await the topmost navigations, inner frames will be empty when racing.
const snapshot = await frame.retryWithProgressAndTimeouts(progress, [1000, 2000, 4000, 8000], async continuePolling => {
try {
@@ -997,7 +997,7 @@ async function snapshotFrameForAI(progress: Progress, frame: frames.Frame, optio
if (!node)
return true;
return injected.incrementalAriaSnapshot(node, { mode: 'ai', ...options });
- }, { refPrefix: frame.seq ? 'f' + frame.seq : '', track: options.track }));
+ }, { refPrefix: frame.seq ? 'f' + frame.seq : '', track: options.track, doNotRenderActive: options.doNotRenderActive }));
if (snapshotOrRetry === true)
return continuePolling;
return snapshotOrRetry;
diff --git a/packages/protocol/src/channels.d.ts b/packages/protocol/src/channels.d.ts
index a76dce8a90..86c22a6775 100644
--- a/packages/protocol/src/channels.d.ts
+++ b/packages/protocol/src/channels.d.ts
@@ -2590,6 +2590,7 @@ export type PageAgentParams = {
apiCacheFile?: string,
cacheFile?: string,
cacheOutFile?: string,
+ doNotRenderActive?: boolean,
maxActions?: number,
maxActionRetries?: number,
maxTokens?: number,
@@ -2605,6 +2606,7 @@ export type PageAgentOptions = {
apiCacheFile?: string,
cacheFile?: string,
cacheOutFile?: string,
+ doNotRenderActive?: boolean,
maxActions?: number,
maxActionRetries?: number,
maxTokens?: number,
diff --git a/packages/protocol/src/protocol.yml b/packages/protocol/src/protocol.yml
index d4af228ada..9a66cd1bee 100644
--- a/packages/protocol/src/protocol.yml
+++ b/packages/protocol/src/protocol.yml
@@ -2023,6 +2023,7 @@ Page:
apiCacheFile: string?
cacheFile: string?
cacheOutFile: string?
+ doNotRenderActive: boolean?
maxActions: int?
maxActionRetries: int?
maxTokens: int?
diff --git a/tests/library/__llm_cache__/library-agent-expect-expect-timeout-during-generate.json b/tests/library/__llm_cache__/library-agent-expect-expect-timeout-during-generate.json
index dcab20e88c..af59d69ae2 100644
--- a/tests/library/__llm_cache__/library-agent-expect-expect-timeout-during-generate.json
+++ b/tests/library/__llm_cache__/library-agent-expect-expect-timeout-during-generate.json
@@ -5,19 +5,19 @@
"content": [
{
"type": "text",
- "text": "I need to verify that an input has the value \"hello\", but according to the page snapshot, the textbox currently has the value \"bye\". I'll use the browser_expect_value tool to assert the expected value."
+ "text": "I need to verify that an input has the value \"hello\". Looking at the page snapshot, I can see there's a textbox with ref=e2 that currently shows \"bye\". I'll use the browser_expect_value tool to assert that this textbox should have the value \"hello\"."
},
{
"type": "tool_call",
"name": "browser_expect_value",
"arguments": {
"type": "textbox",
- "element": "input textbox",
+ "element": "textbox",
"ref": "e2",
"value": "hello",
"_is_done": true
},
- "id": "toolu_017soKmdATMEEVS4yxjbaLs4"
+ "id": "toolu_011sPLFthuXvN3mxhFhh4yQZ"
}
],
"stopReason": {
@@ -26,7 +26,7 @@
},
"usage": {
"input": 2204,
- "output": 176
+ "output": 191
}
}
}
\ No newline at end of file
diff --git a/tests/library/__llm_cache__/library-agent-expect-expect-timeout-during-run.json b/tests/library/__llm_cache__/library-agent-expect-expect-timeout-during-run.json
index b4f7c09466..790f520de6 100644
--- a/tests/library/__llm_cache__/library-agent-expect-expect-timeout-during-run.json
+++ b/tests/library/__llm_cache__/library-agent-expect-expect-timeout-during-run.json
@@ -5,7 +5,7 @@
"content": [
{
"type": "text",
- "text": "I'll verify that the submit button is visible on the page."
+ "text": "I need to verify that the submit button is visible on the page. Based on the page snapshot, I can see there's a button with the accessible name \"Submit\" with reference e2."
},
{
"type": "tool_call",
@@ -15,7 +15,7 @@
"accessibleName": "Submit",
"_is_done": true
},
- "id": "toolu_01Luk6EzYXfyshpYHsWqCzfF"
+ "id": "toolu_01AaRe6HUqqFdEuNGk3rVBQJ"
}
],
"stopReason": {
@@ -24,7 +24,7 @@
},
"usage": {
"input": 2203,
- "output": 107
+ "output": 133
}
}
}
\ No newline at end of file
diff --git a/tests/library/__llm_cache__/library-agent-expect-expectTitle-success.json b/tests/library/__llm_cache__/library-agent-expect-expectTitle-success.json
index 353522d10b..bb23b43b9d 100644
--- a/tests/library/__llm_cache__/library-agent-expect-expectTitle-success.json
+++ b/tests/library/__llm_cache__/library-agent-expect-expectTitle-success.json
@@ -5,7 +5,7 @@
"content": [
{
"type": "text",
- "text": "I'll verify that the page title is \"My Page Title\" using the browser_expect_title tool."
+ "text": "I need to verify that the page title is \"My Page Title\". I'll use the browser_expect_title tool to assert this condition."
},
{
"type": "tool_call",
@@ -14,7 +14,7 @@
"title": "My Page Title",
"_is_done": true
},
- "id": "toolu_01QSwESxdJwhfaSFhvahxFqB"
+ "id": "toolu_01VvWdf4VVFNQuDaEtTsAu3K"
}
],
"stopReason": {
@@ -23,7 +23,7 @@
},
"usage": {
"input": 2195,
- "output": 99
+ "output": 106
}
}
}
\ No newline at end of file
diff --git a/tests/library/__llm_cache__/library-agent-expect-expectTitle-wrong-title-error.json b/tests/library/__llm_cache__/library-agent-expect-expectTitle-wrong-title-error.json
index 2227c3d117..e14e18a7d2 100644
--- a/tests/library/__llm_cache__/library-agent-expect-expectTitle-wrong-title-error.json
+++ b/tests/library/__llm_cache__/library-agent-expect-expectTitle-wrong-title-error.json
@@ -5,7 +5,7 @@
"content": [
{
"type": "text",
- "text": "I need to verify that the page title is \"Other Title\". I'll use the browser_expect_title tool to assert this condition."
+ "text": "I'll verify that the page title is \"Other Title\" using the browser_expect_title tool."
},
{
"type": "tool_call",
@@ -14,7 +14,7 @@
"title": "Other Title",
"_is_done": true
},
- "id": "toolu_01BnWsHUg4s787xMJYmLUNDn"
+ "id": "toolu_017UT5LEe8JRoB53R1YGKrYD"
}
],
"stopReason": {
@@ -23,7 +23,7 @@
},
"usage": {
"input": 2194,
- "output": 104
+ "output": 97
}
}
}
\ No newline at end of file
diff --git a/tests/library/__llm_cache__/library-agent-expect-expectURL-success.json b/tests/library/__llm_cache__/library-agent-expect-expectURL-success.json
index 453653504d..c3f3b6c225 100644
--- a/tests/library/__llm_cache__/library-agent-expect-expectURL-success.json
+++ b/tests/library/__llm_cache__/library-agent-expect-expectURL-success.json
@@ -1,20 +1,20 @@
{
- "74935765e910e2883dc5925fd5f947bb528f174d": {
+ "5fd07fcf484a378d71b68dc09227463b80797da0": {
"result": {
"role": "assistant",
"content": [
{
"type": "text",
- "text": "I see the issue - the URL contains a server prefix. Let me use a regex pattern to match the URL ending with /counter.html."
+ "text": "I'll verify that the page URL is /counter.html using the browser_expect_url tool."
},
{
"type": "tool_call",
"name": "browser_expect_url",
"arguments": {
- "regex": "/counter\\.html$/",
+ "url": "/counter.html",
"_is_done": true
},
- "id": "toolu_018XaEFPSm6aLzqA1BBqRNhu"
+ "id": "toolu_017jkU3FcrQpAVJgtjrzYovK"
}
],
"stopReason": {
@@ -22,26 +22,26 @@
}
},
"usage": {
- "input": 2447,
- "output": 108
+ "input": 2240,
+ "output": 99
}
},
- "ee1b825529efd3c64fff821e9ce6f2e06433f6d3": {
+ "f6d9bc80fd5bad48f6a886d7ec77c8d5889521b1": {
"result": {
"role": "assistant",
"content": [
{
"type": "text",
- "text": "I'll verify that the page URL is /counter.html using the browser_expect_url tool."
+ "text": "I see the issue - the actual URL includes a server prefix. Let me use a regex pattern to match the URL ending with /counter.html."
},
{
"type": "tool_call",
"name": "browser_expect_url",
"arguments": {
- "url": "/counter.html",
+ "regex": "/counter\\.html$/",
"_is_done": true
},
- "id": "toolu_01PRTpTFD78UKubB5EzwTAii"
+ "id": "toolu_01JQuC1CJR7W6LiwAnvjaiDX"
}
],
"stopReason": {
@@ -49,8 +49,8 @@
}
},
"usage": {
- "input": 2243,
- "output": 99
+ "input": 2441,
+ "output": 109
}
}
}
\ No newline at end of file
diff --git a/tests/library/__llm_cache__/library-agent-expect-expectURL-with-regex-error.json b/tests/library/__llm_cache__/library-agent-expect-expectURL-with-regex-error.json
index 7e0cc5425c..f034545e13 100644
--- a/tests/library/__llm_cache__/library-agent-expect-expectURL-with-regex-error.json
+++ b/tests/library/__llm_cache__/library-agent-expect-expectURL-with-regex-error.json
@@ -1,38 +1,11 @@
{
- "37de212b569c4a2d8412505c647ce7ad1f824387": {
+ "f0db973fe7c770b8bdb9ae3c8999544a3305b4b3": {
"result": {
"role": "assistant",
"content": [
{
"type": "text",
- "text": "I need to verify that the page URL matches the `/counter` pattern. I'll use the `browser_expect_url` tool with a regex pattern."
- },
- {
- "type": "tool_call",
- "name": "browser_expect_url",
- "arguments": {
- "regex": "/counter",
- "_is_done": true
- },
- "id": "toolu_01NFN4Fz1V3Bek9pKTbuEsBg"
- }
- ],
- "stopReason": {
- "code": "ok"
- }
- },
- "usage": {
- "input": 2242,
- "output": 108
- }
- },
- "a786ca3af437fcb35753c1f7ceb7f9a90bc8deeb": {
- "result": {
- "role": "assistant",
- "content": [
- {
- "type": "text",
- "text": "I need to fix the regex pattern format. It should end with a `/` and optional flags."
+ "text": "I'll verify that the page URL matches the `/counter` pattern using the browser_expect_url tool with a regex pattern."
},
{
"type": "tool_call",
@@ -41,7 +14,7 @@
"regex": "/counter/",
"_is_done": true
},
- "id": "toolu_01SLaSETGg54hyKNATWckQVw"
+ "id": "toolu_01J38E1DkiNA7JsvQtGx3otW"
}
],
"stopReason": {
@@ -49,8 +22,8 @@
}
},
"usage": {
- "input": 2440,
- "output": 97
+ "input": 2239,
+ "output": 103
}
}
}
\ No newline at end of file
diff --git a/tests/library/__llm_cache__/library-agent-expect-expectURL-with-regex.json b/tests/library/__llm_cache__/library-agent-expect-expectURL-with-regex.json
index dd48e0d5fd..19a6a08b33 100644
--- a/tests/library/__llm_cache__/library-agent-expect-expectURL-with-regex.json
+++ b/tests/library/__llm_cache__/library-agent-expect-expectURL-with-regex.json
@@ -1,11 +1,11 @@
{
- "37de212b569c4a2d8412505c647ce7ad1f824387": {
+ "f0db973fe7c770b8bdb9ae3c8999544a3305b4b3": {
"result": {
"role": "assistant",
"content": [
{
"type": "text",
- "text": "I need to verify that the page URL matches the /counter pattern using a browser_expect assertion tool."
+ "text": "I need to verify that the page URL matches the `/counter` pattern. I'll use the `browser_expect_url` tool with a regex pattern."
},
{
"type": "tool_call",
@@ -14,7 +14,7 @@
"regex": "/counter/",
"_is_done": true
},
- "id": "toolu_015RtjMDCtGsTUpu36HUk5fC"
+ "id": "toolu_01Qen21mVwEVb2JsEVavTWbf"
}
],
"stopReason": {
@@ -22,8 +22,8 @@
}
},
"usage": {
- "input": 2242,
- "output": 98
+ "input": 2239,
+ "output": 109
}
}
}
\ No newline at end of file
diff --git a/tests/library/__llm_cache__/library-agent-expect-expectURL-wrong-URL-error.json b/tests/library/__llm_cache__/library-agent-expect-expectURL-wrong-URL-error.json
index 353b806e09..43b309b180 100644
--- a/tests/library/__llm_cache__/library-agent-expect-expectURL-wrong-URL-error.json
+++ b/tests/library/__llm_cache__/library-agent-expect-expectURL-wrong-URL-error.json
@@ -1,20 +1,20 @@
{
- "688127ac19026114a89d4e621b86bdeaa699418e": {
+ "5fd07fcf484a378d71b68dc09227463b80797da0": {
"result": {
"role": "assistant",
"content": [
{
"type": "text",
- "text": "I see the issue - the actual URL includes the server prefix. Let me use a regex pattern to match the URL ending with /counter.html."
+ "text": "I'll verify that the page URL is /counter.html using the browser_expect_url tool."
},
{
"type": "tool_call",
"name": "browser_expect_url",
"arguments": {
- "regex": "/\\/counter\\.html$/",
+ "url": "/counter.html",
"_is_done": true
},
- "id": "toolu_01V6sRugMvahkP21AHaffA11"
+ "id": "toolu_01B3LSMGJoDrDfCZKwa3neZ9"
}
],
"stopReason": {
@@ -22,26 +22,26 @@
}
},
"usage": {
- "input": 2447,
- "output": 110
+ "input": 2240,
+ "output": 99
}
},
- "ee1b825529efd3c64fff821e9ce6f2e06433f6d3": {
+ "99f45f7c71e64f612bfc0bcb148faf235ccfe006": {
"result": {
"role": "assistant",
"content": [
{
"type": "text",
- "text": "I'll verify that the page URL is /counter.html using the browser_expect_url tool."
+ "text": "I see the issue. The actual URL includes the server prefix. Let me use a regex pattern to match the URL ending with /counter.html."
},
{
"type": "tool_call",
"name": "browser_expect_url",
"arguments": {
- "url": "/counter.html",
+ "regex": "/counter\\.html$/",
"_is_done": true
},
- "id": "toolu_014UE4ajKxYwk8kM6MsURpks"
+ "id": "toolu_01Ufux7hKwV9Djp9jT1JgYwJ"
}
],
"stopReason": {
@@ -49,8 +49,8 @@
}
},
"usage": {
- "input": 2243,
- "output": 99
+ "input": 2441,
+ "output": 109
}
}
}
\ No newline at end of file
diff --git a/tests/library/__llm_cache__/library-agent-limits-should-respect-max-actions-limit.json b/tests/library/__llm_cache__/library-agent-limits-should-respect-max-actions-limit.json
index c4e3d1c4c8..a1046b1866 100644
--- a/tests/library/__llm_cache__/library-agent-limits-should-respect-max-actions-limit.json
+++ b/tests/library/__llm_cache__/library-agent-limits-should-respect-max-actions-limit.json
@@ -1,11 +1,11 @@
{
- "1b776f266d622f4553281b575e36acd4862f48b4": {
+ "37067acf73704df3b84beb06e5caed3eb9bf18ae": {
"result": {
"role": "assistant",
"content": [
{
"type": "text",
- "text": "Now clicking the second time."
+ "text": "Now clicking the submit button for the second time."
},
{
"type": "tool_call",
@@ -15,7 +15,7 @@
"ref": "e2",
"_is_done": false
},
- "id": "toolu_014LLfN7tHFxcAqX72VTHXVy"
+ "id": "toolu_01HrvXLTTXvo9BModqvnKT9X"
}
],
"stopReason": {
@@ -23,17 +23,17 @@
}
},
"usage": {
- "input": 3194,
- "output": 98
+ "input": 3191,
+ "output": 102
}
},
- "7b25e318de64dc9d2494432a6d311631e2c1edc0": {
+ "42f3f98abb30b1cd490e8afa7ed7f62dd63a0355": {
"result": {
"role": "assistant",
"content": [
{
"type": "text",
- "text": "I'll click the submit button 5 times as requested. Let me start by clicking it the first time."
+ "text": "Clicking the submit button for the fourth time."
},
{
"type": "tool_call",
@@ -43,7 +43,7 @@
"ref": "e2",
"_is_done": false
},
- "id": "toolu_01PWwDgpp2rUtvmpuTfgTNzn"
+ "id": "toolu_01Ac6wP23fB9JAMqdXJ4iHo9"
}
],
"stopReason": {
@@ -51,17 +51,17 @@
}
},
"usage": {
- "input": 3047,
- "output": 115
+ "input": 3342,
+ "output": 102
}
},
- "b9bb21161597f576eaef8aea85dae797c879fec9": {
+ "7b25e318de64dc9d2494432a6d311631e2c1edc0": {
"result": {
"role": "assistant",
"content": [
{
"type": "text",
- "text": "Clicking the fourth time."
+ "text": "I'll click the submit button 5 times as requested. Let me start by clicking it the first time."
},
{
"type": "tool_call",
@@ -71,7 +71,7 @@
"ref": "e2",
"_is_done": false
},
- "id": "toolu_01YRh2G5UeZqvPou8d311SZU"
+ "id": "toolu_013EzLkuM1hq5oKwFdwsfXgW"
}
],
"stopReason": {
@@ -79,17 +79,17 @@
}
},
"usage": {
- "input": 3337,
- "output": 98
+ "input": 3047,
+ "output": 115
}
},
- "bae9fc31434e4f1f36e5cd29bd84165e04ac32f2": {
+ "b0b9f5d4d75488a8be0bd393e38396f58e8bae3b": {
"result": {
"role": "assistant",
"content": [
{
"type": "text",
- "text": "Clicking the third time."
+ "text": "Clicking the submit button for the third time."
},
{
"type": "tool_call",
@@ -99,7 +99,7 @@
"ref": "e2",
"_is_done": false
},
- "id": "toolu_01JnLrvVrm3EzPwcBrzzDR1S"
+ "id": "toolu_01RC27VmiERhtpWN8DvhEXvg"
}
],
"stopReason": {
@@ -107,8 +107,8 @@
}
},
"usage": {
- "input": 3267,
- "output": 98
+ "input": 3268,
+ "output": 102
}
}
}
\ No newline at end of file
diff --git a/tests/library/__llm_cache__/library-agent-perform-click-a-button.json b/tests/library/__llm_cache__/library-agent-perform-click-a-button.json
index a20b50d717..1759105d6a 100644
--- a/tests/library/__llm_cache__/library-agent-perform-click-a-button.json
+++ b/tests/library/__llm_cache__/library-agent-perform-click-a-button.json
@@ -15,7 +15,7 @@
"ref": "e2",
"_is_done": true
},
- "id": "toolu_01VL7BkrHmWKoBoPpEqgdXBM"
+ "id": "toolu_011oz46csVKDo4fGJBzPX6Pu"
}
],
"stopReason": {
diff --git a/tests/library/__llm_cache__/library-agent-perform-expect-value.json b/tests/library/__llm_cache__/library-agent-perform-expect-value.json
index 0f6642a0c1..3045824187 100644
--- a/tests/library/__llm_cache__/library-agent-perform-expect-value.json
+++ b/tests/library/__llm_cache__/library-agent-perform-expect-value.json
@@ -16,7 +16,7 @@
"text": "bogus",
"_is_done": false
},
- "id": "toolu_01EHNuYTEoREY8mpiB8TDpao"
+ "id": "toolu_01Gpg2Zqam6FBo1hga4nU5m2"
}
],
"stopReason": {
@@ -28,13 +28,13 @@
"output": 149
}
},
- "de97ea962d4424a77d06eeb1f87c4fca0072f5b0": {
+ "30aa72b75cada58032f8e66e544580c2fa37b9f8": {
"result": {
"role": "assistant",
"content": [
{
"type": "text",
- "text": "Perfect! I have successfully completed all three parts of the task:\n\n1. ✅ Entered \"bogus\" into the email field\n2. ✅ Verified the value is \"bogus\" (shown in the page snapshot as `textbox \"Email Address\" [active] [ref=e2]: bogus`)\n3. ✅ Confirmed the error message is displayed (shown as `generic [ref=e3]: \"Error: Invalid email address\"`)\n\nThe task has been completed successfully. The email field now contains \"bogus\" and the validation error message \"Error: Invalid email address\" is displayed on the page."
+ "text": "Perfect! I have successfully completed all three parts of the task:\n\n1. ✓ Entered \"bogus\" into the email field\n2. ✓ Verified the value is \"bogus\" (shown in the snapshot as `textbox \"Email Address\" [ref=e2]: bogus`)\n3. ✓ Confirmed the error message is displayed (shown as `\"Error: Invalid email address\"`)\n\nAll requirements have been met. The task is complete."
},
{
"type": "tool_call",
@@ -42,7 +42,7 @@
"arguments": {
"_is_done": true
},
- "id": "toolu_01RF8r5xHaVkEMKMw8DdLRtX"
+ "id": "toolu_01K9CVYgsXmZxyvf3S8hvEag"
}
],
"stopReason": {
@@ -50,8 +50,8 @@
}
},
"usage": {
- "input": 3288,
- "output": 196
+ "input": 3285,
+ "output": 162
}
}
}
\ No newline at end of file
diff --git a/tests/library/__llm_cache__/library-agent-perform-extract-task.json b/tests/library/__llm_cache__/library-agent-perform-extract-task.json
index 44e82a8f96..23ad40342d 100644
--- a/tests/library/__llm_cache__/library-agent-perform-extract-task.json
+++ b/tests/library/__llm_cache__/library-agent-perform-extract-task.json
@@ -23,7 +23,7 @@
],
"_is_done": true
},
- "id": "toolu_01S326B2WNaGGZsxcdufA6ub"
+ "id": "toolu_01DvvdfNSormjy1wYd3tsX9B"
}
],
"stopReason": {
@@ -59,7 +59,7 @@
],
"_is_done": true
},
- "id": "toolu_017VDE6U6W8WedvBTxDLg3mf"
+ "id": "toolu_012Xfc5Au8c5bdhdNA9tmqFe"
}
],
"stopReason": {
@@ -68,7 +68,7 @@
},
"usage": {
"input": 1020,
- "output": 132
+ "output": 113
}
}
}
\ No newline at end of file
diff --git a/tests/library/__llm_cache__/library-agent-perform-perform-history.json b/tests/library/__llm_cache__/library-agent-perform-perform-history.json
index 068a283179..69362c3ece 100644
--- a/tests/library/__llm_cache__/library-agent-perform-perform-history.json
+++ b/tests/library/__llm_cache__/library-agent-perform-perform-history.json
@@ -1,5 +1,5 @@
{
- "190ffa76902eab71ce416fea1894aa82dca2b883": {
+ "35faa6e330f5198be604e1d5b81cd0c1c1627a63": {
"result": {
"role": "assistant",
"content": [
@@ -15,7 +15,7 @@
"ref": "e3",
"_is_done": true
},
- "id": "toolu_01ERb3Y3rLpQy6VCBtT9Lcsr"
+ "id": "toolu_01GmGw8peNBC1zWhPVjTM3UU"
}
],
"stopReason": {
@@ -23,11 +23,11 @@
}
},
"usage": {
- "input": 3096,
+ "input": 3093,
"output": 102
}
},
- "500794034629b9a73d75bdb7ab58e72bf8adfca2": {
+ "d98ecb24c7732360ace82e5dabac9c9bf3266a71": {
"result": {
"role": "assistant",
"content": [
@@ -43,7 +43,7 @@
"ref": "e3",
"_is_done": true
},
- "id": "toolu_01Tax2k8PnfvQvRj2s23Mc6m"
+ "id": "toolu_011PFYBHb9wVKpJLoMmNDNsz"
}
],
"stopReason": {
@@ -51,7 +51,7 @@
}
},
"usage": {
- "input": 3083,
+ "input": 3080,
"output": 101
}
}
diff --git a/tests/library/__llm_cache__/library-agent-perform-perform-reports-error.json b/tests/library/__llm_cache__/library-agent-perform-perform-reports-error.json
index 99e0a69fa6..33bc8a0341 100644
--- a/tests/library/__llm_cache__/library-agent-perform-perform-reports-error.json
+++ b/tests/library/__llm_cache__/library-agent-perform-perform-reports-error.json
@@ -1,20 +1,19 @@
{
- "af3d8747bed52754774ce6409828a7ecbf22c747": {
+ "4804d46618cdfe9201831d9757cc2478ba4d3909": {
"result": {
"role": "assistant",
"content": [
{
"type": "text",
- "text": "I can see the page snapshot only shows \"Wolf\" and \"Fox\" buttons, but not a \"Rabbit\" button. The Rabbit button is not currently visible in the page snapshot. This could mean:\n1. The button doesn't exist on the page\n2. The button is not yet loaded\n3. The button might appear after some interaction\n\nSince the task specifically asks me to click the Rabbit button but it's not present in the snapshot, I should report this issue."
+ "text": "I'll take a snapshot of the current page to locate the Rabbit button."
},
{
"type": "tool_call",
- "name": "refuse_to_perform",
+ "name": "browser_snapshot",
"arguments": {
- "reason": "The Rabbit button is not present on the page. The page only contains \"Wolf\" and \"Fox\" buttons. The requested \"Rabbit\" button cannot be found in the current page snapshot.",
- "_is_done": true
+ "_is_done": false
},
- "id": "toolu_01WnpKKi4FUnGyFrRwi15Bmx"
+ "id": "toolu_012JEEitpPzZfnUeD8Wfa9Z3"
}
],
"stopReason": {
@@ -22,25 +21,26 @@
}
},
"usage": {
- "input": 3203,
- "output": 214
+ "input": 3067,
+ "output": 72
}
},
- "fb3d817d917f930cc19453843ed86fe6d4ad4e98": {
+ "889157cd583283667b9fd5d9add230316106c3a4": {
"result": {
"role": "assistant",
"content": [
{
"type": "text",
- "text": "I need to take a snapshot of the page first to see the current state and locate the Rabbit button."
+ "text": "I can see the page snapshot only shows \"Wolf\" and \"Fox\" buttons, but there is no \"Rabbit\" button visible on the current page. Let me refuse to perform this action as the requested button is not available."
},
{
"type": "tool_call",
- "name": "browser_snapshot",
+ "name": "refuse_to_perform",
"arguments": {
- "_is_done": false
+ "reason": "The Rabbit button is not present on the page. Only Wolf and Fox buttons are visible in the current page snapshot.",
+ "_is_done": true
},
- "id": "toolu_01KnyXDHzhX6wn69vmG24yDa"
+ "id": "toolu_01RA82xhXBfG6hbJY8y1q278"
}
],
"stopReason": {
@@ -48,8 +48,8 @@
}
},
"usage": {
- "input": 3070,
- "output": 78
+ "input": 3191,
+ "output": 144
}
}
}
\ No newline at end of file
diff --git a/tests/library/__llm_cache__/library-agent-perform-perform-run-timeout.json b/tests/library/__llm_cache__/library-agent-perform-perform-run-timeout.json
index ecb06f8d8f..4eea493157 100644
--- a/tests/library/__llm_cache__/library-agent-perform-perform-run-timeout.json
+++ b/tests/library/__llm_cache__/library-agent-perform-perform-run-timeout.json
@@ -1,5 +1,5 @@
{
- "273f0802e7fc3fe0d3eb1cfa83366e1a1272f185": {
+ "554366043f6477d30a36655c10cbb4ec0eaa21d4": {
"result": {
"role": "assistant",
"content": [
@@ -15,7 +15,7 @@
"ref": "e3",
"_is_done": true
},
- "id": "toolu_01YVFbgqSfhDKsS843HKxwor"
+ "id": "toolu_01XMd7RXQygLwiLd7eiPB9iV"
}
],
"stopReason": {
@@ -23,7 +23,7 @@
}
},
"usage": {
- "input": 3069,
+ "input": 3066,
"output": 101
}
}
diff --git a/tests/library/__llm_cache__/library-agent-perform-retrieve-a-secret.json b/tests/library/__llm_cache__/library-agent-perform-retrieve-a-secret.json
index 99968d5392..3e44a2ca2b 100644
--- a/tests/library/__llm_cache__/library-agent-perform-retrieve-a-secret.json
+++ b/tests/library/__llm_cache__/library-agent-perform-retrieve-a-secret.json
@@ -5,7 +5,7 @@
"content": [
{
"type": "text",
- "text": "I'll enter the x-secret-email into the email field."
+ "text": "I'll help you enter the secret email into the email field. Let me type the value into the Email Address textbox."
},
{
"type": "tool_call",
@@ -16,7 +16,7 @@
"text": "%x-secret-email%",
"_is_done": true
},
- "id": "toolu_019NPT8YwSF9eaXHgPMRGAMZ"
+ "id": "toolu_01LxAKq2etJEHtoUe5pURHoa"
}
],
"stopReason": {
@@ -25,7 +25,7 @@
},
"usage": {
"input": 3051,
- "output": 133
+ "output": 144
}
}
}
\ No newline at end of file
diff --git a/tests/library/agent-helpers.ts b/tests/library/agent-helpers.ts
index b145b1882d..4ea2cd45f2 100644
--- a/tests/library/agent-helpers.ts
+++ b/tests/library/agent-helpers.ts
@@ -49,7 +49,8 @@ export async function generateAgent(context: BrowserContext, options: AgentOptio
...options,
cache: {
cacheFile: cacheFile(),
- }
+ },
+ ...{ _doNotRenderActive: true },
});
return { page, agent };
}
@@ -57,7 +58,8 @@ export async function generateAgent(context: BrowserContext, options: AgentOptio
export async function runAgent(context: BrowserContext, options: { secrets?: Record<string, string> } = {}) {
const page = await context.newPage();
const agent = await page.agent({
- cache: { cacheFile: cacheFile() }
+ cache: { cacheFile: cacheFile() },
+ ...{ _doNotRenderActive: true },
});
return { page, agent };
}