Skip to content

Playwright is a framework for Web Testing and Automation. It allows testing Chromium, Firefox and WebKit with a single API.

chore: perform agent turn event (#38535)

Pavel Feldman0ed249b8b9ca
diff --git a/docs/src/api/class-page.md b/docs/src/api/class-page.md
index 76e9d6b1f2..19aa0d94f5 100644
--- a/docs/src/api/class-page.md
+++ b/docs/src/api/class-page.md
@@ -544,6 +544,17 @@ sequence of events is `request`, `response` and `requestfinished`.
Emitted when [response] status and headers are received for a request. For a successful response, the sequence of events
is `request`, `response` and `requestfinished`.
+## event: Page.agentTurn
+* since: v1.58
+- argument: <[Object]>
+ - `role` <[string]>
+ - `message` <[string]>
+ - `usage` ?<[Object]>
+ - `inputTokens` <[int]>
+ - `outputTokens` <[int]>
+
+Emitted when the agent makes a turn.
+
## event: Page.webSocket
* since: v1.9
- argument: <[WebSocket]>
diff --git a/docs/src/api/params.md b/docs/src/api/params.md
index babbdaf3b0..807670d29a 100644
--- a/docs/src/api/params.md
+++ b/docs/src/api/params.md
@@ -375,7 +375,7 @@ Emulates consistent window screen size available inside web page via `window.scr
- `provider` <[string]> LLM provider to use.
- `model` <[string]> Model identifier within provider.
- `cacheFile` ?<[string]> Cache file to use/generate code for performed actions into. Cache is not used if not specified (default).
- - `cacheMode` ?<['force'|'ignore'|'auto']> Cache control, defaults to 'auto'.
+ - `cacheMode` ?<[CacheMode]<"force"|"ignore"|"update"|"auto">> Cache control, defaults to 'auto'.
- `secrets` ?<[Object]<[string], [string]>> Secrets to hide from the LLM.
- `maxTurns` ?<[int]> Maximum number of agentic turns to take per call. Defaults to 10.
- `maxTokens` ?<[int]> Maximum number of tokens to consume per call. The agentic loop will stop after input + output tokens exceed this value. Defaults on unlimited.
diff --git a/examples/todomvc/playwright.config.ts b/examples/todomvc/playwright.config.ts
index e2fbf59bb8..e21dd0f655 100644
--- a/examples/todomvc/playwright.config.ts
+++ b/examples/todomvc/playwright.config.ts
@@ -1,6 +1,9 @@
/* eslint-disable notice/notice */
import { defineConfig, devices } from '@playwright/test';
+import dotenv from 'dotenv';
+
+dotenv.config();
/**
* See https://playwright.dev/docs/test-configuration.
diff --git a/package-lock.json b/package-lock.json
index 623a2e8bf1..1c82ec5b7c 100644
--- a/package-lock.json
+++ b/package-lock.json
@@ -18,7 +18,7 @@
"@eslint/compat": "^1.3.2",
"@eslint/eslintrc": "^3.3.1",
"@eslint/js": "^9.34.0",
- "@lowire/loop": "^0.0.8",
+ "@lowire/loop": "^0.0.11",
"@modelcontextprotocol/sdk": "^1.17.5",
"@octokit/graphql-schema": "^15.26.0",
"@stylistic/eslint-plugin": "^5.2.3",
@@ -1064,9 +1064,9 @@
}
},
"node_modules/@lowire/loop": {
- "version": "0.0.8",
- "resolved": "https://registry.npmjs.org/@lowire/loop/-/loop-0.0.8.tgz",
- "integrity": "sha512-DuyxRnPnu8Bwsfwt77tbZAgWGYKhEN/0xpsvHNin/62OEBiVx/nvSbj9NMFmeP83NoqCTG5n7JZCkA5GIw1pCA==",
+ "version": "0.0.11",
+ "resolved": "https://registry.npmjs.org/@lowire/loop/-/loop-0.0.11.tgz",
+ "integrity": "sha512-kDfrYp6g6W4Nm/Owu3pP04UM7jbqR9IO9ITNJ0p52huoE3f7KcyrYOyyi7ITrmzLm2h+7vUUMZEzhGNVIelpRg==",
"dev": true,
"license": "Apache-2.0",
"engines": {
diff --git a/package.json b/package.json
index 9a507d71e1..5629dff786 100644
--- a/package.json
+++ b/package.json
@@ -56,7 +56,7 @@
"@eslint/compat": "^1.3.2",
"@eslint/eslintrc": "^3.3.1",
"@eslint/js": "^9.34.0",
- "@lowire/loop": "^0.0.8",
+ "@lowire/loop": "^0.0.11",
"@modelcontextprotocol/sdk": "^1.17.5",
"@octokit/graphql-schema": "^15.26.0",
"@stylistic/eslint-plugin": "^5.2.3",
diff --git a/packages/playwright-client/types/types.d.ts b/packages/playwright-client/types/types.d.ts
index d9917c1a1d..6788988908 100644
--- a/packages/playwright-client/types/types.d.ts
+++ b/packages/playwright-client/types/types.d.ts
@@ -1036,6 +1036,21 @@ export interface Page {
* @param options
*/
extract<Schema extends ZodTypeAny>(query: string, schema: Schema): Promise<ZodInfer<Schema>>;
+ /**
+ * Emitted when the agent makes a turn.
+ */
+ on(event: 'agentturn', listener: (data: {
+ role: string;
+
+ message: string;
+
+ usage?: {
+ inputTokens: number;
+
+ outputTokens: number;
+ };
+ }) => any): this;
+
/**
* Emitted when the page closes.
*/
@@ -1243,6 +1258,21 @@ export interface Page {
*/
on(event: 'worker', listener: (worker: Worker) => any): this;
+ /**
+ * Adds an event listener that will be automatically removed after it is triggered once. See `addListener` for more information about this event.
+ */
+ once(event: 'agentturn', listener: (data: {
+ role: string;
+
+ message: string;
+
+ usage?: {
+ inputTokens: number;
+
+ outputTokens: number;
+ };
+ }) => any): this;
+
/**
* Adds an event listener that will be automatically removed after it is triggered once. See `addListener` for more information about this event.
*/
@@ -1338,6 +1368,21 @@ export interface Page {
*/
once(event: 'worker', listener: (worker: Worker) => any): this;
+ /**
+ * Emitted when the agent makes a turn.
+ */
+ addListener(event: 'agentturn', listener: (data: {
+ role: string;
+
+ message: string;
+
+ usage?: {
+ inputTokens: number;
+
+ outputTokens: number;
+ };
+ }) => any): this;
+
/**
* Emitted when the page closes.
*/
@@ -1545,6 +1590,21 @@ export interface Page {
*/
addListener(event: 'worker', listener: (worker: Worker) => any): this;
+ /**
+ * Removes an event listener added by `on` or `addListener`.
+ */
+ removeListener(event: 'agentturn', listener: (data: {
+ role: string;
+
+ message: string;
+
+ usage?: {
+ inputTokens: number;
+
+ outputTokens: number;
+ };
+ }) => any): this;
+
/**
* Removes an event listener added by `on` or `addListener`.
*/
@@ -1640,6 +1700,21 @@ export interface Page {
*/
removeListener(event: 'worker', listener: (worker: Worker) => any): this;
+ /**
+ * Removes an event listener added by `on` or `addListener`.
+ */
+ off(event: 'agentturn', listener: (data: {
+ role: string;
+
+ message: string;
+
+ usage?: {
+ inputTokens: number;
+
+ outputTokens: number;
+ };
+ }) => any): this;
+
/**
* Removes an event listener added by `on` or `addListener`.
*/
@@ -1735,6 +1810,21 @@ export interface Page {
*/
off(event: 'worker', listener: (worker: Worker) => any): this;
+ /**
+ * Emitted when the agent makes a turn.
+ */
+ prependListener(event: 'agentturn', listener: (data: {
+ role: string;
+
+ message: string;
+
+ usage?: {
+ inputTokens: number;
+
+ outputTokens: number;
+ };
+ }) => any): this;
+
/**
* Emitted when the page closes.
*/
@@ -4790,6 +4880,41 @@ export interface Page {
height: number;
};
+ /**
+ * Emitted when the agent makes a turn.
+ */
+ waitForEvent(event: 'agentturn', optionsOrPredicate?: { predicate?: (data: {
+ role: string;
+
+ message: string;
+
+ usage?: {
+ inputTokens: number;
+
+ outputTokens: number;
+ };
+ }) => boolean | Promise<boolean>, timeout?: number } | ((data: {
+ role: string;
+
+ message: string;
+
+ usage?: {
+ inputTokens: number;
+
+ outputTokens: number;
+ };
+ }) => boolean | Promise<boolean>)): Promise<{
+ role: string;
+
+ message: string;
+
+ usage?: {
+ inputTokens: number;
+
+ outputTokens: number;
+ };
+ }>;
+
/**
* Emitted when the page closes.
*/
@@ -22118,7 +22243,7 @@ export interface BrowserContextOptions {
/**
* Cache control, defaults to 'auto'.
*/
- cacheMode?: 'force'|'ignore'|'auto';
+ cacheMode?: "force"|"ignore"|"update"|"auto";
/**
* Secrets to hide from the LLM.
diff --git a/packages/playwright-core/ThirdPartyNotices.txt b/packages/playwright-core/ThirdPartyNotices.txt
index c6d3cb6724..7ed4413e0b 100644
--- a/packages/playwright-core/ThirdPartyNotices.txt
+++ b/packages/playwright-core/ThirdPartyNotices.txt
@@ -4,7 +4,7 @@ THIRD-PARTY SOFTWARE NOTICES AND INFORMATION
This project incorporates components from the projects listed below. The original copyright notices and the licenses under which Microsoft received such components are set forth below. Microsoft reserves all rights not expressly granted herein, whether by implication, estoppel or otherwise.
-- @lowire/[email protected] (https://github.com/pavelfeldman/lowire)
+- @lowire/[email protected] (https://github.com/pavelfeldman/lowire)
- @modelcontextprotocol/[email protected] (https://github.com/modelcontextprotocol/typescript-sdk)
- [email protected] (https://github.com/jshttp/accepts)
- [email protected] (https://github.com/TooTallNate/proxy-agents)
@@ -135,7 +135,7 @@ This project incorporates components from the projects listed below. The origina
- [email protected] (https://github.com/StefanTerdell/zod-to-json-schema)
- [email protected] (https://github.com/colinhacks/zod)
-%% @lowire/[email protected] NOTICES AND INFORMATION BEGIN HERE
+%% @lowire/[email protected] NOTICES AND INFORMATION BEGIN HERE
=========================================
Apache License
Version 2.0, January 2004
@@ -339,7 +339,7 @@ Apache License
See the License for the specific language governing permissions and
limitations under the License.
=========================================
-END OF @lowire/[email protected] AND INFORMATION
+END OF @lowire/[email protected] AND INFORMATION
%% @modelcontextprotocol/[email protected] NOTICES AND INFORMATION BEGIN HERE
=========================================
diff --git a/packages/playwright-core/bundles/mcp/package-lock.json b/packages/playwright-core/bundles/mcp/package-lock.json
index 0e18bcbefe..2754245dd7 100644
--- a/packages/playwright-core/bundles/mcp/package-lock.json
+++ b/packages/playwright-core/bundles/mcp/package-lock.json
@@ -8,16 +8,16 @@
"name": "mcp-bundle",
"version": "0.0.1",
"dependencies": {
- "@lowire/loop": "^0.0.8",
+ "@lowire/loop": "^0.0.11",
"@modelcontextprotocol/sdk": "^1.24.0",
"zod": "^3.25.76",
"zod-to-json-schema": "^3.24.6"
}
},
"node_modules/@lowire/loop": {
- "version": "0.0.8",
- "resolved": "https://registry.npmjs.org/@lowire/loop/-/loop-0.0.8.tgz",
- "integrity": "sha512-DuyxRnPnu8Bwsfwt77tbZAgWGYKhEN/0xpsvHNin/62OEBiVx/nvSbj9NMFmeP83NoqCTG5n7JZCkA5GIw1pCA==",
+ "version": "0.0.11",
+ "resolved": "https://registry.npmjs.org/@lowire/loop/-/loop-0.0.11.tgz",
+ "integrity": "sha512-kDfrYp6g6W4Nm/Owu3pP04UM7jbqR9IO9ITNJ0p52huoE3f7KcyrYOyyi7ITrmzLm2h+7vUUMZEzhGNVIelpRg==",
"license": "Apache-2.0",
"engines": {
"node": ">=20"
diff --git a/packages/playwright-core/bundles/mcp/package.json b/packages/playwright-core/bundles/mcp/package.json
index 7e6f08a8e8..2c6d12e6f0 100644
--- a/packages/playwright-core/bundles/mcp/package.json
+++ b/packages/playwright-core/bundles/mcp/package.json
@@ -3,8 +3,8 @@
"version": "0.0.1",
"private": true,
"dependencies": {
+ "@lowire/loop": "^0.0.11",
"@modelcontextprotocol/sdk": "^1.24.0",
- "@lowire/loop": "^0.0.8",
"zod": "^3.25.76",
"zod-to-json-schema": "^3.24.6"
}
diff --git a/packages/playwright-core/src/client/events.ts b/packages/playwright-core/src/client/events.ts
index cede73640c..36a3c75093 100644
--- a/packages/playwright-core/src/client/events.ts
+++ b/packages/playwright-core/src/client/events.ts
@@ -55,6 +55,7 @@ export const Events = {
},
Page: {
+ AgentTurn: 'agentturn',
Close: 'close',
Crash: 'crash',
Console: 'console',
diff --git a/packages/playwright-core/src/client/page.ts b/packages/playwright-core/src/client/page.ts
index 4f36c15869..1c3cbe9e22 100644
--- a/packages/playwright-core/src/client/page.ts
+++ b/packages/playwright-core/src/client/page.ts
@@ -134,6 +134,7 @@ export class Page extends ChannelOwner<channels.PageChannel> implements api.Page
this._closed = initializer.isClosed;
this._opener = Page.fromNullable(initializer.opener);
+ this._channel.on('agentTurn', params => this.emit(Events.Page.AgentTurn, params));
this._channel.on('bindingCall', ({ binding }) => this._onBinding(BindingCall.from(binding)));
this._channel.on('close', () => this._onClose());
this._channel.on('crash', () => this._onCrash());
diff --git a/packages/playwright-core/src/protocol/validator.ts b/packages/playwright-core/src/protocol/validator.ts
index 2cf6e69c39..d75cb62722 100644
--- a/packages/playwright-core/src/protocol/validator.ts
+++ b/packages/playwright-core/src/protocol/validator.ts
@@ -606,7 +606,7 @@ scheme.BrowserTypeLaunchPersistentContextParams = tObject({
provider: tString,
model: tString,
cacheFile: tOptional(tString),
- cacheMode: tOptional(tEnum(['ignore', 'force', 'auto'])),
+ cacheMode: tOptional(tEnum(['ignore', 'force', 'update', 'auto'])),
secrets: tOptional(tArray(tType('NameValue'))),
maxTurns: tOptional(tInt),
maxTokens: tOptional(tInt),
@@ -707,7 +707,7 @@ scheme.BrowserNewContextParams = tObject({
provider: tString,
model: tString,
cacheFile: tOptional(tString),
- cacheMode: tOptional(tEnum(['ignore', 'force', 'auto'])),
+ cacheMode: tOptional(tEnum(['ignore', 'force', 'update', 'auto'])),
secrets: tOptional(tArray(tType('NameValue'))),
maxTurns: tOptional(tInt),
maxTokens: tOptional(tInt),
@@ -787,7 +787,7 @@ scheme.BrowserNewContextForReuseParams = tObject({
provider: tString,
model: tString,
cacheFile: tOptional(tString),
- cacheMode: tOptional(tEnum(['ignore', 'force', 'auto'])),
+ cacheMode: tOptional(tEnum(['ignore', 'force', 'update', 'auto'])),
secrets: tOptional(tArray(tType('NameValue'))),
maxTurns: tOptional(tInt),
maxTokens: tOptional(tInt),
@@ -912,7 +912,7 @@ scheme.BrowserContextInitializer = tObject({
provider: tString,
model: tString,
cacheFile: tOptional(tString),
- cacheMode: tOptional(tEnum(['ignore', 'force', 'auto'])),
+ cacheMode: tOptional(tEnum(['ignore', 'force', 'update', 'auto'])),
secrets: tOptional(tArray(tType('NameValue'))),
maxTurns: tOptional(tInt),
maxTokens: tOptional(tInt),
@@ -1180,6 +1180,14 @@ scheme.PageInitializer = tObject({
isClosed: tBoolean,
opener: tOptional(tChannel(['Page'])),
});
+scheme.PageAgentTurnEvent = tObject({
+ role: tString,
+ message: tString,
+ usage: tOptional(tObject({
+ inputTokens: tInt,
+ outputTokens: tInt,
+ })),
+});
scheme.PageBindingCallEvent = tObject({
binding: tChannel(['BindingCall']),
});
@@ -2832,7 +2840,7 @@ scheme.AndroidDeviceLaunchBrowserParams = tObject({
provider: tString,
model: tString,
cacheFile: tOptional(tString),
- cacheMode: tOptional(tEnum(['ignore', 'force', 'auto'])),
+ cacheMode: tOptional(tEnum(['ignore', 'force', 'update', 'auto'])),
secrets: tOptional(tArray(tType('NameValue'))),
maxTurns: tOptional(tInt),
maxTokens: tOptional(tInt),
diff --git a/packages/playwright-core/src/server/agent/agent.ts b/packages/playwright-core/src/server/agent/agent.ts
index 507d630e4c..57de45dcd2 100644
--- a/packages/playwright-core/src/server/agent/agent.ts
+++ b/packages/playwright-core/src/server/agent/agent.ts
@@ -21,10 +21,10 @@ import { debug } from '../../utilsBundle';
import { Loop } from '../../mcpBundle';
import { runAction } from './actionRunner';
import { Context } from './context';
+import { Page } from '../page';
import type { Progress } from '../progress';
import type * as channels from '@protocol/channels';
-import type { Page } from '../page';
import type * as loopTypes from '@lowire/loop';
import type * as actions from './actions';
@@ -72,6 +72,8 @@ async function perform(context: Context, userTask: string, resultSchema: loopTyp
const { full } = await page.snapshotForAI(progress);
const { tools, callTool } = toolsForLoop(context);
+ page.emit(Page.Events.AgentTurn, { role: 'user', message: userTask });
+
const limits = context.limits(options);
let turns = 0;
const loop = new Loop(browserContext._options.agent.provider as any, {
@@ -81,11 +83,23 @@ async function perform(context: Context, userTask: string, resultSchema: loopTyp
callTool,
tools,
...limits,
- beforeTurn: params => {
+ onAfterTurn: ({ assistantMessage, totalUsage }) => {
++turns;
- const lastReply = params.conversation.messages.findLast(m => m.role === 'assistant');
- const toolCall = lastReply?.content.find(c => c.type === 'tool_call');
- if (!resultSchema && toolCall && toolCall.arguments.thatShouldBeIt)
+ const usage = { inputTokens: totalUsage.input, outputTokens: totalUsage.output };
+ const intent = assistantMessage.content.filter(c => c.type === 'text').map(c => c.text).join('\n');
+ page.emit(Page.Events.AgentTurn, { role: 'assistant', message: intent, usage });
+ if (!assistantMessage.content.filter(c => c.type === 'tool_call').length)
+ page.emit(Page.Events.AgentTurn, { role: 'assistant', message: `no tool calls`, usage });
+ return 'continue';
+ },
+ onBeforeToolCall: ({ toolCall }) => {
+ page.emit(Page.Events.AgentTurn, { role: 'assistant', message: `call tool "${toolCall.name}"` });
+ return 'continue';
+ },
+ onAfterToolCall: ({ toolCall }) => {
+ const suffix = toolCall.result?.isError ? 'failed' : 'succeeded';
+ page.emit(Page.Events.AgentTurn, { role: 'user', message: `tool "${toolCall.name}" ${suffix}` });
+ if (toolCall.arguments.thatShouldBeIt)
return 'break';
return 'continue';
},
@@ -117,7 +131,7 @@ type CachedActions = Record<string, {
const allCaches = new Map<string, CachedActions>();
async function cachedPerform(context: Context, options: channels.PagePerformParams): Promise<boolean> {
- if (!context.options?.cacheFile || context.options.cacheMode === 'ignore')
+ if (!context.options?.cacheFile || context.options.cacheMode === 'ignore' || context.options.cacheMode === 'update')
return false;
const cache = await cachedActions(context.options.cacheFile);
diff --git a/packages/playwright-core/src/server/agent/tools.ts b/packages/playwright-core/src/server/agent/tools.ts
index 4b85eda6fb..f14671aef0 100644
--- a/packages/playwright-core/src/server/agent/tools.ts
+++ b/packages/playwright-core/src/server/agent/tools.ts
@@ -215,7 +215,7 @@ const fillForm = defineTool({
schema: {
name: 'browser_fill_form',
title: 'Fill form',
- description: 'Fill multiple form fields',
+ description: 'Fill multiple form fields. Always use this tool when you can fill more than one field at a time.',
inputSchema: baseSchema.extend({
fields: z.array(z.object({
name: z.string().describe('Human-readable field name'),
diff --git a/packages/playwright-core/src/server/dispatchers/pageDispatcher.ts b/packages/playwright-core/src/server/dispatchers/pageDispatcher.ts
index dcfc35b0ec..1ba01607c5 100644
--- a/packages/playwright-core/src/server/dispatchers/pageDispatcher.ts
+++ b/packages/playwright-core/src/server/dispatchers/pageDispatcher.ts
@@ -94,6 +94,7 @@ export class PageDispatcher extends Dispatcher<Page, channels.PageChannel, Brows
this._dispatchEvent('route', { route: new RouteDispatcher(RequestDispatcher.from(this.parentScope(), request), route) });
};
+ this.addObjectListener(Page.Events.AgentTurn, params => this._dispatchEvent('agentTurn', params));
this.addObjectListener(Page.Events.Close, () => {
this._dispatchEvent('close');
this._dispose();
diff --git a/packages/playwright-core/src/server/page.ts b/packages/playwright-core/src/server/page.ts
index 1759727992..2adb01e405 100644
--- a/packages/playwright-core/src/server/page.ts
+++ b/packages/playwright-core/src/server/page.ts
@@ -120,6 +120,7 @@ type ExpectScreenshotOptions = ImageComparatorOptions & ScreenshotOptions & {
};
const PageEvent = {
+ AgentTurn: 'agentturn',
Close: 'close',
Crash: 'crash',
Download: 'download',
@@ -136,6 +137,7 @@ const PageEvent = {
} as const;
export type PageEventMap = {
+ [PageEvent.AgentTurn]: [agentTurn: { role: string, message: string, usage?: { inputTokens: number, outputTokens: number } }];
[PageEvent.Close]: [];
[PageEvent.Crash]: [];
[PageEvent.Download]: [download: Download];
diff --git a/packages/playwright-core/types/types.d.ts b/packages/playwright-core/types/types.d.ts
index d9917c1a1d..6788988908 100644
--- a/packages/playwright-core/types/types.d.ts
+++ b/packages/playwright-core/types/types.d.ts
@@ -1036,6 +1036,21 @@ export interface Page {
* @param options
*/
extract<Schema extends ZodTypeAny>(query: string, schema: Schema): Promise<ZodInfer<Schema>>;
+ /**
+ * Emitted when the agent makes a turn.
+ */
+ on(event: 'agentturn', listener: (data: {
+ role: string;
+
+ message: string;
+
+ usage?: {
+ inputTokens: number;
+
+ outputTokens: number;
+ };
+ }) => any): this;
+
/**
* Emitted when the page closes.
*/
@@ -1243,6 +1258,21 @@ export interface Page {
*/
on(event: 'worker', listener: (worker: Worker) => any): this;
+ /**
+ * Adds an event listener that will be automatically removed after it is triggered once. See `addListener` for more information about this event.
+ */
+ once(event: 'agentturn', listener: (data: {
+ role: string;
+
+ message: string;
+
+ usage?: {
+ inputTokens: number;
+
+ outputTokens: number;
+ };
+ }) => any): this;
+
/**
* Adds an event listener that will be automatically removed after it is triggered once. See `addListener` for more information about this event.
*/
@@ -1338,6 +1368,21 @@ export interface Page {
*/
once(event: 'worker', listener: (worker: Worker) => any): this;
+ /**
+ * Emitted when the agent makes a turn.
+ */
+ addListener(event: 'agentturn', listener: (data: {
+ role: string;
+
+ message: string;
+
+ usage?: {
+ inputTokens: number;
+
+ outputTokens: number;
+ };
+ }) => any): this;
+
/**
* Emitted when the page closes.
*/
@@ -1545,6 +1590,21 @@ export interface Page {
*/
addListener(event: 'worker', listener: (worker: Worker) => any): this;
+ /**
+ * Removes an event listener added by `on` or `addListener`.
+ */
+ removeListener(event: 'agentturn', listener: (data: {
+ role: string;
+
+ message: string;
+
+ usage?: {
+ inputTokens: number;
+
+ outputTokens: number;
+ };
+ }) => any): this;
+
/**
* Removes an event listener added by `on` or `addListener`.
*/
@@ -1640,6 +1700,21 @@ export interface Page {
*/
removeListener(event: 'worker', listener: (worker: Worker) => any): this;
+ /**
+ * Removes an event listener added by `on` or `addListener`.
+ */
+ off(event: 'agentturn', listener: (data: {
+ role: string;
+
+ message: string;
+
+ usage?: {
+ inputTokens: number;
+
+ outputTokens: number;
+ };
+ }) => any): this;
+
/**
* Removes an event listener added by `on` or `addListener`.
*/
@@ -1735,6 +1810,21 @@ export interface Page {
*/
off(event: 'worker', listener: (worker: Worker) => any): this;
+ /**
+ * Emitted when the agent makes a turn.
+ */
+ prependListener(event: 'agentturn', listener: (data: {
+ role: string;
+
+ message: string;
+
+ usage?: {
+ inputTokens: number;
+
+ outputTokens: number;
+ };
+ }) => any): this;
+
/**
* Emitted when the page closes.
*/
@@ -4790,6 +4880,41 @@ export interface Page {
height: number;
};
+ /**
+ * Emitted when the agent makes a turn.
+ */
+ waitForEvent(event: 'agentturn', optionsOrPredicate?: { predicate?: (data: {
+ role: string;
+
+ message: string;
+
+ usage?: {
+ inputTokens: number;
+
+ outputTokens: number;
+ };
+ }) => boolean | Promise<boolean>, timeout?: number } | ((data: {
+ role: string;
+
+ message: string;
+
+ usage?: {
+ inputTokens: number;
+
+ outputTokens: number;
+ };
+ }) => boolean | Promise<boolean>)): Promise<{
+ role: string;
+
+ message: string;
+
+ usage?: {
+ inputTokens: number;
+
+ outputTokens: number;
+ };
+ }>;
+
/**
* Emitted when the page closes.
*/
@@ -22118,7 +22243,7 @@ export interface BrowserContextOptions {
/**
* Cache control, defaults to 'auto'.
*/
- cacheMode?: 'force'|'ignore'|'auto';
+ cacheMode?: "force"|"ignore"|"update"|"auto";
/**
* Secrets to hide from the LLM.
diff --git a/packages/playwright/src/runner/storage.ts b/packages/playwright/src/runner/storage.ts
index 46c897a6d2..90aace6faf 100644
--- a/packages/playwright/src/runner/storage.ts
+++ b/packages/playwright/src/runner/storage.ts
@@ -87,7 +87,7 @@ export class Storage {
}
private _writeFile(entries: Record<string, any>) {
- this._writeChain = this._writeChain.then(() => fs.promises.writeFile(this._fileName, JSON.stringify(entries, null, 2))).catch(() => {});
+ this._writeChain = this._writeChain.then(() => fs.promises.writeFile(this._fileName, JSON.stringify(entries, null, 2)));
return this._writeChain;
}
diff --git a/packages/protocol/src/channels.d.ts b/packages/protocol/src/channels.d.ts
index 3b2e83918e..190e39174e 100644
--- a/packages/protocol/src/channels.d.ts
+++ b/packages/protocol/src/channels.d.ts
@@ -1012,7 +1012,7 @@ export type BrowserTypeLaunchPersistentContextParams = {
provider: string,
model: string,
cacheFile?: string,
- cacheMode?: 'ignore' | 'force' | 'auto',
+ cacheMode?: 'ignore' | 'force' | 'update' | 'auto',
secrets?: NameValue[],
maxTurns?: number,
maxTokens?: number,
@@ -1103,7 +1103,7 @@ export type BrowserTypeLaunchPersistentContextOptions = {
provider: string,
model: string,
cacheFile?: string,
- cacheMode?: 'ignore' | 'force' | 'auto',
+ cacheMode?: 'ignore' | 'force' | 'update' | 'auto',
secrets?: NameValue[],
maxTurns?: number,
maxTokens?: number,
@@ -1233,7 +1233,7 @@ export type BrowserNewContextParams = {
provider: string,
model: string,
cacheFile?: string,
- cacheMode?: 'ignore' | 'force' | 'auto',
+ cacheMode?: 'ignore' | 'force' | 'update' | 'auto',
secrets?: NameValue[],
maxTurns?: number,
maxTokens?: number,
@@ -1310,7 +1310,7 @@ export type BrowserNewContextOptions = {
provider: string,
model: string,
cacheFile?: string,
- cacheMode?: 'ignore' | 'force' | 'auto',
+ cacheMode?: 'ignore' | 'force' | 'update' | 'auto',
secrets?: NameValue[],
maxTurns?: number,
maxTokens?: number,
@@ -1390,7 +1390,7 @@ export type BrowserNewContextForReuseParams = {
provider: string,
model: string,
cacheFile?: string,
- cacheMode?: 'ignore' | 'force' | 'auto',
+ cacheMode?: 'ignore' | 'force' | 'update' | 'auto',
secrets?: NameValue[],
maxTurns?: number,
maxTokens?: number,
@@ -1467,7 +1467,7 @@ export type BrowserNewContextForReuseOptions = {
provider: string,
model: string,
cacheFile?: string,
- cacheMode?: 'ignore' | 'force' | 'auto',
+ cacheMode?: 'ignore' | 'force' | 'update' | 'auto',
secrets?: NameValue[],
maxTurns?: number,
maxTokens?: number,
@@ -1611,7 +1611,7 @@ export type BrowserContextInitializer = {
provider: string,
model: string,
cacheFile?: string,
- cacheMode?: 'ignore' | 'force' | 'auto',
+ cacheMode?: 'ignore' | 'force' | 'update' | 'auto',
secrets?: NameValue[],
maxTurns?: number,
maxTokens?: number,
@@ -2076,6 +2076,7 @@ export type PageInitializer = {
opener?: PageChannel,
};
export interface PageEventTarget {
+ on(event: 'agentTurn', callback: (params: PageAgentTurnEvent) => void): this;
on(event: 'bindingCall', callback: (params: PageBindingCallEvent) => void): this;
on(event: 'close', callback: (params: PageCloseEvent) => void): this;
on(event: 'crash', callback: (params: PageCrashEvent) => void): this;
@@ -2135,6 +2136,14 @@ export interface PageChannel extends PageEventTarget, EventTargetChannel {
perform(params: PagePerformParams, progress?: Progress): Promise<PagePerformResult>;
extract(params: PageExtractParams, progress?: Progress): Promise<PageExtractResult>;
}
+export type PageAgentTurnEvent = {
+ role: string,
+ message: string,
+ usage?: {
+ inputTokens: number,
+ outputTokens: number,
+ },
+};
export type PageBindingCallEvent = {
binding: BindingCallChannel,
};
@@ -2669,6 +2678,7 @@ export type PageExtractResult = {
};
export interface PageEvents {
+ 'agentTurn': PageAgentTurnEvent;
'bindingCall': PageBindingCallEvent;
'close': PageCloseEvent;
'crash': PageCrashEvent;
@@ -4935,7 +4945,7 @@ export type AndroidDeviceLaunchBrowserParams = {
provider: string,
model: string,
cacheFile?: string,
- cacheMode?: 'ignore' | 'force' | 'auto',
+ cacheMode?: 'ignore' | 'force' | 'update' | 'auto',
secrets?: NameValue[],
maxTurns?: number,
maxTokens?: number,
@@ -5010,7 +5020,7 @@ export type AndroidDeviceLaunchBrowserOptions = {
provider: string,
model: string,
cacheFile?: string,
- cacheMode?: 'ignore' | 'force' | 'auto',
+ cacheMode?: 'ignore' | 'force' | 'update' | 'auto',
secrets?: NameValue[],
maxTurns?: number,
maxTokens?: number,
diff --git a/packages/protocol/src/protocol.yml b/packages/protocol/src/protocol.yml
index 20dcdf07b0..e2a194629d 100644
--- a/packages/protocol/src/protocol.yml
+++ b/packages/protocol/src/protocol.yml
@@ -600,6 +600,7 @@ ContextOptions:
literals:
- ignore
- force
+ - update
- auto
secrets:
type: array?
@@ -2052,6 +2053,16 @@ Page:
events:
+ agentTurn:
+ parameters:
+ role: string
+ message: string
+ usage:
+ type: object?
+ properties:
+ inputTokens: int
+ outputTokens: int
+
bindingCall:
parameters:
binding: BindingCall
diff --git a/tests/library/perform-task.spec.ts b/tests/library/perform-task.spec.ts
index 6ac69ea144..0edc7c5d29 100644
--- a/tests/library/perform-task.spec.ts
+++ b/tests/library/perform-task.spec.ts
@@ -21,9 +21,9 @@ import { browserTest as test, expect } from '../config/browserTest';
test.use({
agent: {
provider: 'github',
- model: 'claude-sonnet-4.5',
- cachePathTemplate: '{testFilePath}-cache.json',
- cacheMode: process.env.CI ? 'force' : 'auto',
+ model: 'gpt-4.1',
+ cachePathTemplate: '{testDir}/{testFilePath}-cache.json',
+ cacheMode: process.env.UPDATE_CACHE ? 'update' : process.env.CI ? 'force' : 'auto',
secrets: {
'x-secret-email': '[email protected]',
}
@@ -32,6 +32,9 @@ test.use({
test('page.perform', async ({ page, server }) => {
await page.goto(server.PREFIX + '/evals/fill-form.html');
+ page.on('agentturn', turn => {
+ console.log('agentturn', turn);
+ });
await page.perform('Fill out the form with the following details:\n' +
'Name: John Smith\n' +
'Address: 1045 La Avenida St, Mountain View, CA 94043\n' +
diff --git a/tests/library/perform-task.spec.ts-cache.json b/tests/library/perform-task.spec.ts-cache.json
index 528c77b738..ccd4d4a958 100644
--- a/tests/library/perform-task.spec.ts-cache.json
+++ b/tests/library/perform-task.spec.ts-cache.json
@@ -1,72 +1,72 @@
{
"Enter x-secret-email into the email field": {
- "timestamp": 1765482830501,
+ "timestamp": 1765673294749,
"actions": [
{
"method": "fill",
"selector": "internal:role=textbox[name=\"Email Address\"i]",
"text": "x-secret-email",
"code": "await page.getByRole('textbox', { name: 'Email Address' }).fill('x-secret-email');",
- "intent": "Enter 'x-secret-email' into the email address field"
+ "intent": "Enter 'x-secret-email' into the 'Email Address' textbox."
}
]
},
"Fill out the form with the following details:\nName: John Smith\nAddress: 1045 La Avenida St, Mountain View, CA 94043\nEmail: [email protected]": {
- "timestamp": 1765482831906,
+ "timestamp": 1765673328439,
"actions": [
{
"method": "fill",
"selector": "internal:role=textbox[name=\"Full Name *\"i]",
"text": "John Smith",
"code": "await page.getByRole('textbox', { name: 'Full Name *' }).fill('John Smith');",
- "intent": "Fill out the form with the provided details: Name (John Smith), Address (1045 La Avenida St, Mountain View, CA 94043), and Email ([email protected])"
+ "intent": "Fill in the name, address, and email fields according to the provided user information."
},
{
"method": "fill",
"selector": "internal:role=textbox[name=\"Email Address *\"i]",
"text": "[email protected]",
"code": "await page.getByRole('textbox', { name: 'Email Address *' }).fill('[email protected]');",
- "intent": "Fill out the form with the provided details: Name (John Smith), Address (1045 La Avenida St, Mountain View, CA 94043), and Email ([email protected])"
+ "intent": "Fill in the name, address, and email fields according to the provided user information."
},
{
"method": "fill",
"selector": "internal:role=textbox[name=\"Street Address *\"i]",
"text": "1045 La Avenida St",
"code": "await page.getByRole('textbox', { name: 'Street Address *' }).fill('1045 La Avenida St');",
- "intent": "Fill out the form with the provided details: Name (John Smith), Address (1045 La Avenida St, Mountain View, CA 94043), and Email ([email protected])"
+ "intent": "Fill in the name, address, and email fields according to the provided user information."
},
{
"method": "fill",
"selector": "internal:role=textbox[name=\"City *\"i]",
"text": "Mountain View",
"code": "await page.getByRole('textbox', { name: 'City *' }).fill('Mountain View');",
- "intent": "Fill out the form with the provided details: Name (John Smith), Address (1045 La Avenida St, Mountain View, CA 94043), and Email ([email protected])"
+ "intent": "Fill in the name, address, and email fields according to the provided user information."
},
{
"method": "fill",
"selector": "internal:role=textbox[name=\"State/Province *\"i]",
"text": "CA",
"code": "await page.getByRole('textbox', { name: 'State/Province *' }).fill('CA');",
- "intent": "Fill out the form with the provided details: Name (John Smith), Address (1045 La Avenida St, Mountain View, CA 94043), and Email ([email protected])"
+ "intent": "Fill in the name, address, and email fields according to the provided user information."
},
{
"method": "fill",
"selector": "internal:role=textbox[name=\"ZIP/Postal Code *\"i]",
"text": "94043",
"code": "await page.getByRole('textbox', { name: 'ZIP/Postal Code *' }).fill('94043');",
- "intent": "Fill out the form with the provided details: Name (John Smith), Address (1045 La Avenida St, Mountain View, CA 94043), and Email ([email protected])"
+ "intent": "Fill in the name, address, and email fields according to the provided user information."
}
]
},
"- Enter \"bogus\" into the email field\n - Check that the value is in fact \"bogus\"\n - Check that the error message is displayed": {
- "timestamp": 1765482849524,
+ "timestamp": 1765673309068,
"actions": [
{
"method": "fill",
"selector": "internal:role=textbox[name=\"Email Address\"i]",
"text": "bogus",
"code": "await page.getByRole('textbox', { name: 'Email Address' }).fill('bogus');",
- "intent": "Enter \"bogus\" into the email field"
+ "intent": "Enter 'bogus' into the email field to simulate invalid input and test validation."
},
{
"method": "expectValue",
@@ -74,14 +74,14 @@
"type": "textbox",
"value": "bogus",
"code": "await expect(page.getByRole('textbox', { name: 'Email Address' })).toHaveValue('bogus');",
- "intent": "Check that the email field value is \"bogus\""
+ "intent": "Check that the value in the email field is 'bogus'.\nCheck that the error message is displayed for the invalid email."
},
{
"method": "expectVisible",
- "selector": "internal:text=\"Error: Invalid email address\"i",
- "code": "await expect(page.getByText('Error: Invalid email address')).toBeVisible();",
- "intent": "Check that the error message \"Error: Invalid email address\" is displayed"
+ "selector": "internal:text=\"error\"i",
+ "code": "await expect(page.getByText('error')).toBeVisible();",
+ "intent": "Check that the value in the email field is 'bogus'.\nCheck that the error message is displayed for the invalid email."
}
]
}
-}
\ No newline at end of file
+}
diff --git a/tests/mcp/fixtures.ts b/tests/mcp/fixtures.ts
index 874f1d1ecb..6a3e609f09 100644
--- a/tests/mcp/fixtures.ts
+++ b/tests/mcp/fixtures.ts
@@ -202,6 +202,7 @@ export const test = serverTest.extend<TestFixtures & TestOptions, WorkerFixtures
messages: cache,
secrets: { PORT: String(server.PORT) },
},
+ maxTokens: 1_000_000,
debug,
});
await use(loop);