Skip to content

Playwright is a framework for Web Testing and Automation. It allows testing Chromium, Firefox and WebKit with a single API.

chore: update agent limits (#38800)

Pavel Feldmanf01cb2caceb2
diff --git a/docs/src/api/class-page.md b/docs/src/api/class-page.md
index 45a0e2dc1c..5c2b752197 100644
--- a/docs/src/api/class-page.md
+++ b/docs/src/api/class-page.md
@@ -722,15 +722,14 @@ Initialize page agent with the llm provider and cache.
- `cacheFile` ?<[string]> Cache file to use/generate code for performed actions into. Cache is not used if not specified (default).
- `cacheOutFile` ?<[string]> When specified, generated entries are written into the `cacheOutFile` instead of updating the `cacheFile`.
-### option: Page.agent.maxTurns
+### option: Page.agent.limits
* since: v1.58
-- `maxTurns` <[int]>
+- `limits` <[Object]>
+ - `maxTokens` ?<[int]> Maximum number of tokens to consume. The agentic loop will stop after input + output tokens exceed this value. Defaults to unlimited.
+ - `maxActions` ?<[int]> Maximum number of agentic actions to generate, defaults to 10.
+ - `maxActionRetries` ?<[int]> Maximum number retries per action, defaults to 3.
-Maximum number of agentic turns to take per call. Defaults to 10.
-
-### option: Page.agent.maxTokens
-* since: v1.58
-- `maxTokens` ?<[int]>
+Limits to use for the agentic loop.
### option: Page.agent.provider
* since: v1.58
@@ -738,6 +737,7 @@ Maximum number of agentic turns to take per call. Defaults to 10.
- `api` <[PageAgentAPI]<"openai"|"openai-compatible"|"anthropic"|"google">> API to use.
- `apiEndpoint` ?<[string]> Endpoint to use if different from default.
- `apiKey` <[string]> API key for the LLM provider.
+ - `apiTimeout` ?<[int]> Amount of time to wait for the provider to respond to each request.
- `model` <[string]> Model identifier within the provider. Required in non-cache mode.
### option: Page.agent.secrets
diff --git a/docs/src/api/params.md b/docs/src/api/params.md
index bd319b19ad..9b7bf8f3d4 100644
--- a/docs/src/api/params.md
+++ b/docs/src/api/params.md
@@ -384,11 +384,17 @@ By default, they are cached globally with the `task` as a key. This option allow
Maximum number of tokens to consume. The agentic loop will stop after input + output tokens exceed this value.
Defaults to context-wide value specified in `agent` property.
-## page-agent-max-turns
+## page-agent-max-actions
* since: v1.58
-- `maxTurns` <[int]>
+- `maxActions` <[int]>
-Maximum number of agentic turns during this call, defaults to context-wide value specified in `agent` property.
+Maximum number of agentic actions to generate, defaults to context-wide value specified in `agent` property.
+
+## page-agent-max-action-retries
+* since: v1.58
+- `maxActionRetries` <[int]>
+
+Maximum number of retries when generating each action, defaults to context-wide value specified in `agent` property.
## page-agent-timeout
* since: v1.58
@@ -399,7 +405,8 @@ Request timeout in milliseconds. Defaults to action timeout. Pass `0` to disable
## page-agent-call-options-v1.58
- %%-page-agent-cache-key-%%
- %%-page-agent-max-tokens-%%
-- %%-page-agent-max-turns-%%
+- %%-page-agent-max-actions-%%
+- %%-page-agent-max-action-retries-%%
- %%-page-agent-timeout-%%
## fetch-param-url
diff --git a/docs/src/test-api/class-testoptions.md b/docs/src/test-api/class-testoptions.md
index 194cd58616..4c60a5c896 100644
--- a/docs/src/test-api/class-testoptions.md
+++ b/docs/src/test-api/class-testoptions.md
@@ -49,14 +49,19 @@ export default defineConfig({
## property: TestOptions.agentOptions
* since: v1.58
- type: <[Object]>
- - `api` ?<[string]> LLM provider to use. Required in non-cache mode.
- - `apiKey` ?<[string]> Key for the LLM provider.
- - `apiEndpoint` ?<[string]> LLM provider endpoint.
- - `model` ?<[string]> Model identifier within the provider. Required in non-cache mode.
+ - `provider` <[Object]>
+ - `api` <[PageAgentAPI]<"openai"|"openai-compatible"|"anthropic"|"google">> API to use.
+ - `apiEndpoint` ?<[string]> Endpoint to use if different from default.
+ - `apiKey` <[string]> API key for the LLM provider.
+ - `apiTimeout` ?<[int]> Amount of time to wait for the provider to respond to each request.
+ - `model` <[string]> Model identifier within the provider. Required in non-cache mode.
- `cachePathTemplate` ?<[string]> Cache file template to use/generate code for performed actions into.
- - `maxTurns` ?<[int]> Maximum number of agentic turns to take per call. Defaults to 10.
- - `maxTokens` ?<[int]> Maximum number of tokens to consume per call. The agentic loop will stop after input + output tokens exceed this value. Defaults on unlimited.
+ - `limits` <[Object]>
+ - `maxTokens` ?<[int]> Maximum number of tokens to consume. The agentic loop will stop after input + output tokens exceed this value. Defaults to unlimited.
+ - `maxActions` ?<[int]> Maximum number of agentic actions to generate, defaults to 10.
+ - `maxActionRetries` ?<[int]> Maximum number retries per action, defaults to 3.
- `secrets` ?<[Object]<[string], [string]>> Secrets to hide from the LLM.
+ - `systemPrompt` <[string]> System prompt for the agent's loop.
## property: TestOptions.baseURL = %%-context-option-baseURL-%%
* since: v1.10
diff --git a/examples/todomvc/tests/fixtures.ts b/examples/todomvc/tests/fixtures.ts
index 0e317c4706..6226e718bf 100644
--- a/examples/todomvc/tests/fixtures.ts
+++ b/examples/todomvc/tests/fixtures.ts
@@ -6,10 +6,12 @@ export { expect } from '@playwright/test';
export const test = baseTest.extend({
agentOptions: {
- api: 'anthropic',
- apiKey: process.env.AZURE_SONNET_API_KEY!,
- apiEndpoint: process.env.AZURE_SONNET_ENDPOINT!,
- model: 'claude-sonnet-4-5',
+ provider: {
+ api: 'anthropic',
+ apiKey: process.env.AZURE_SONNET_API_KEY!,
+ apiEndpoint: process.env.AZURE_SONNET_ENDPOINT!,
+ model: 'claude-sonnet-4-5',
+ },
},
page: async ({ page }, use) => {
await page.goto('https://demo.playwright.dev/todomvc');
diff --git a/package-lock.json b/package-lock.json
index e3cd66d46e..5160eea979 100644
--- a/package-lock.json
+++ b/package-lock.json
@@ -18,7 +18,7 @@
"@eslint/compat": "^1.3.2",
"@eslint/eslintrc": "^3.3.1",
"@eslint/js": "^9.34.0",
- "@lowire/loop": "^0.0.19",
+ "@lowire/loop": "^0.0.23",
"@modelcontextprotocol/sdk": "^1.17.5",
"@octokit/graphql-schema": "^15.26.0",
"@stylistic/eslint-plugin": "^5.2.3",
@@ -1065,9 +1065,9 @@
}
},
"node_modules/@lowire/loop": {
- "version": "0.0.19",
- "resolved": "https://registry.npmjs.org/@lowire/loop/-/loop-0.0.19.tgz",
- "integrity": "sha512-uugjYtmA1mNSwHzbZEnvFedjB2EUX0s4NDbDbUcGuz6CH8fzZV2FFodY6fneik+r3GdLJtpvzpOP+cU/M0ba3w==",
+ "version": "0.0.23",
+ "resolved": "https://registry.npmjs.org/@lowire/loop/-/loop-0.0.23.tgz",
+ "integrity": "sha512-zw4yFzto2T9g+CiHfubzxut0JMxA053XQ2syveqPjqofvRZyumhJR11RIQjKrPO9rDbf2CFviKrYFt7kgru1VA==",
"dev": true,
"license": "Apache-2.0",
"engines": {
diff --git a/package.json b/package.json
index 8db7c35f51..8138c730d9 100644
--- a/package.json
+++ b/package.json
@@ -55,7 +55,7 @@
"@eslint/compat": "^1.3.2",
"@eslint/eslintrc": "^3.3.1",
"@eslint/js": "^9.34.0",
- "@lowire/loop": "^0.0.19",
+ "@lowire/loop": "^0.0.23",
"@modelcontextprotocol/sdk": "^1.17.5",
"@octokit/graphql-schema": "^15.26.0",
"@stylistic/eslint-plugin": "^5.2.3",
diff --git a/packages/playwright-client/types/types.d.ts b/packages/playwright-client/types/types.d.ts
index a9d3c3daff..51dd92fb29 100644
--- a/packages/playwright-client/types/types.d.ts
+++ b/packages/playwright-client/types/types.d.ts
@@ -2110,12 +2110,26 @@ export interface Page {
cacheOutFile?: string;
};
- maxTokens?: number;
-
/**
- * Maximum number of agentic turns to take per call. Defaults to 10.
+ * Limits to use for the agentic loop.
*/
- maxTurns?: number;
+ limits?: {
+ /**
+ * Maximum number of tokens to consume. The agentic loop will stop after input + output tokens exceed this value.
+ * Defaults to unlimited.
+ */
+ maxTokens?: number;
+
+ /**
+ * Maximum number of agentic actions to generate, defaults to 10.
+ */
+ maxActions?: number;
+
+ /**
+ * Maximum number retries per action, defaults to 3.
+ */
+ maxActionRetries?: number;
+ };
provider?: {
/**
@@ -2133,6 +2147,11 @@ export interface Page {
*/
apiKey: string;
+ /**
+ * Amount of time to wait for the provider to respond to each request.
+ */
+ apiTimeout?: number;
+
/**
* Model identifier within the provider. Required in non-cache mode.
*/
@@ -5402,15 +5421,21 @@ export interface PageAgent {
cacheKey?: string;
/**
- * Maximum number of tokens to consume. The agentic loop will stop after input + output tokens exceed this value.
- * Defaults to context-wide value specified in `agent` property.
+ * Maximum number of retries when generating each action, defaults to context-wide value specified in `agent`
+ * property.
*/
- maxTokens?: number;
+ maxActionRetries?: number;
+
+ /**
+ * Maximum number of agentic actions to generate, defaults to context-wide value specified in `agent` property.
+ */
+ maxActions?: number;
/**
- * Maximum number of agentic turns during this call, defaults to context-wide value specified in `agent` property.
+ * Maximum number of tokens to consume. The agentic loop will stop after input + output tokens exceed this value.
+ * Defaults to context-wide value specified in `agent` property.
*/
- maxTurns?: number;
+ maxTokens?: number;
/**
* Request timeout in milliseconds. Defaults to action timeout. Pass `0` to disable timeout.
@@ -5438,15 +5463,21 @@ export interface PageAgent {
cacheKey?: string;
/**
- * Maximum number of tokens to consume. The agentic loop will stop after input + output tokens exceed this value.
- * Defaults to context-wide value specified in `agent` property.
+ * Maximum number of retries when generating each action, defaults to context-wide value specified in `agent`
+ * property.
*/
- maxTokens?: number;
+ maxActionRetries?: number;
+
+ /**
+ * Maximum number of agentic actions to generate, defaults to context-wide value specified in `agent` property.
+ */
+ maxActions?: number;
/**
- * Maximum number of agentic turns during this call, defaults to context-wide value specified in `agent` property.
+ * Maximum number of tokens to consume. The agentic loop will stop after input + output tokens exceed this value.
+ * Defaults to context-wide value specified in `agent` property.
*/
- maxTurns?: number;
+ maxTokens?: number;
/**
* Request timeout in milliseconds. Defaults to action timeout. Pass `0` to disable timeout.
diff --git a/packages/playwright-core/ThirdPartyNotices.txt b/packages/playwright-core/ThirdPartyNotices.txt
index 71cefc3d7d..cf98b9ac14 100644
--- a/packages/playwright-core/ThirdPartyNotices.txt
+++ b/packages/playwright-core/ThirdPartyNotices.txt
@@ -4,7 +4,7 @@ THIRD-PARTY SOFTWARE NOTICES AND INFORMATION
This project incorporates components from the projects listed below. The original copyright notices and the licenses under which Microsoft received such components are set forth below. Microsoft reserves all rights not expressly granted herein, whether by implication, estoppel or otherwise.
-- @lowire/[email protected] (https://github.com/pavelfeldman/lowire)
+- @lowire/[email protected] (https://github.com/pavelfeldman/lowire)
- @modelcontextprotocol/[email protected] (https://github.com/modelcontextprotocol/typescript-sdk)
- [email protected] (https://github.com/jshttp/accepts)
- [email protected] (https://github.com/TooTallNate/proxy-agents)
@@ -135,7 +135,7 @@ This project incorporates components from the projects listed below. The origina
- [email protected] (https://github.com/StefanTerdell/zod-to-json-schema)
- [email protected] (https://github.com/colinhacks/zod)
-%% @lowire/[email protected] NOTICES AND INFORMATION BEGIN HERE
+%% @lowire/[email protected] NOTICES AND INFORMATION BEGIN HERE
=========================================
Apache License
Version 2.0, January 2004
@@ -339,7 +339,7 @@ Apache License
See the License for the specific language governing permissions and
limitations under the License.
=========================================
-END OF @lowire/[email protected] AND INFORMATION
+END OF @lowire/[email protected] AND INFORMATION
%% @modelcontextprotocol/[email protected] NOTICES AND INFORMATION BEGIN HERE
=========================================
diff --git a/packages/playwright-core/bundles/mcp/package-lock.json b/packages/playwright-core/bundles/mcp/package-lock.json
index 33a47d828b..bcc613bfc0 100644
--- a/packages/playwright-core/bundles/mcp/package-lock.json
+++ b/packages/playwright-core/bundles/mcp/package-lock.json
@@ -8,16 +8,16 @@
"name": "mcp-bundle",
"version": "0.0.1",
"dependencies": {
- "@lowire/loop": "^0.0.19",
+ "@lowire/loop": "^0.0.23",
"@modelcontextprotocol/sdk": "^1.24.0",
"zod": "^3.25.76",
"zod-to-json-schema": "^3.24.6"
}
},
"node_modules/@lowire/loop": {
- "version": "0.0.19",
- "resolved": "https://registry.npmjs.org/@lowire/loop/-/loop-0.0.19.tgz",
- "integrity": "sha512-uugjYtmA1mNSwHzbZEnvFedjB2EUX0s4NDbDbUcGuz6CH8fzZV2FFodY6fneik+r3GdLJtpvzpOP+cU/M0ba3w==",
+ "version": "0.0.23",
+ "resolved": "https://registry.npmjs.org/@lowire/loop/-/loop-0.0.23.tgz",
+ "integrity": "sha512-zw4yFzto2T9g+CiHfubzxut0JMxA053XQ2syveqPjqofvRZyumhJR11RIQjKrPO9rDbf2CFviKrYFt7kgru1VA==",
"license": "Apache-2.0",
"engines": {
"node": ">=20"
diff --git a/packages/playwright-core/bundles/mcp/package.json b/packages/playwright-core/bundles/mcp/package.json
index f3209bf1bc..fb90f525df 100644
--- a/packages/playwright-core/bundles/mcp/package.json
+++ b/packages/playwright-core/bundles/mcp/package.json
@@ -3,7 +3,7 @@
"version": "0.0.1",
"private": true,
"dependencies": {
- "@lowire/loop": "^0.0.19",
+ "@lowire/loop": "^0.0.23",
"@modelcontextprotocol/sdk": "^1.24.0",
"zod": "^3.25.76",
"zod-to-json-schema": "^3.24.6"
diff --git a/packages/playwright-core/src/client/page.ts b/packages/playwright-core/src/client/page.ts
index f464643d1f..4e3d827595 100644
--- a/packages/playwright-core/src/client/page.ts
+++ b/packages/playwright-core/src/client/page.ts
@@ -852,12 +852,14 @@ export class Page extends ChannelOwner<channels.PageChannel> implements api.Page
api: options.provider?.api,
apiEndpoint: options.provider?.apiEndpoint,
apiKey: options.provider?.apiKey,
+ apiTimeout: options.provider?.apiTimeout,
apiCacheFile: (options.provider as any)?._apiCacheFile,
model: options.provider?.model,
cacheFile: options.cache?.cacheFile,
cacheOutFile: options.cache?.cacheOutFile,
- maxTokens: options.maxTokens,
- maxTurns: options.maxTurns,
+ maxTokens: options.limits?.maxTokens,
+ maxActions: options.limits?.maxActions,
+ maxActionRetries: options.limits?.maxActionRetries,
secrets: options.secrets ? Object.entries(options.secrets).map(([name, value]) => ({ name, value })) : undefined,
systemPrompt: options.systemPrompt,
};
diff --git a/packages/playwright-core/src/protocol/validator.ts b/packages/playwright-core/src/protocol/validator.ts
index 62dca989ba..4ed0e4af71 100644
--- a/packages/playwright-core/src/protocol/validator.ts
+++ b/packages/playwright-core/src/protocol/validator.ts
@@ -1491,10 +1491,12 @@ scheme.PageAgentParams = tObject({
api: tOptional(tString),
apiKey: tOptional(tString),
apiEndpoint: tOptional(tString),
+ apiTimeout: tOptional(tInt),
apiCacheFile: tOptional(tString),
cacheFile: tOptional(tString),
cacheOutFile: tOptional(tString),
- maxTurns: tOptional(tInt),
+ maxActions: tOptional(tInt),
+ maxActionRetries: tOptional(tInt),
maxTokens: tOptional(tInt),
model: tOptional(tString),
secrets: tOptional(tArray(tType('NameValue'))),
@@ -2907,7 +2909,8 @@ scheme.PageAgentTurnEvent = tObject({
});
scheme.PageAgentPerformParams = tObject({
task: tString,
- maxTurns: tOptional(tInt),
+ maxActions: tOptional(tInt),
+ maxActionRetries: tOptional(tInt),
maxTokens: tOptional(tInt),
cacheKey: tOptional(tString),
timeout: tOptional(tInt),
@@ -2917,7 +2920,8 @@ scheme.PageAgentPerformResult = tObject({
});
scheme.PageAgentExpectParams = tObject({
expectation: tString,
- maxTurns: tOptional(tInt),
+ maxActions: tOptional(tInt),
+ maxActionRetries: tOptional(tInt),
maxTokens: tOptional(tInt),
cacheKey: tOptional(tString),
timeout: tOptional(tInt),
@@ -2928,7 +2932,8 @@ scheme.PageAgentExpectResult = tObject({
scheme.PageAgentExtractParams = tObject({
query: tString,
schema: tAny,
- maxTurns: tOptional(tInt),
+ maxActions: tOptional(tInt),
+ maxActionRetries: tOptional(tInt),
maxTokens: tOptional(tInt),
cacheKey: tOptional(tString),
timeout: tOptional(tInt),
diff --git a/packages/playwright-core/src/server/agent/context.ts b/packages/playwright-core/src/server/agent/context.ts
index 302b729341..12aebb4f03 100644
--- a/packages/playwright-core/src/server/agent/context.ts
+++ b/packages/playwright-core/src/server/agent/context.ts
@@ -38,12 +38,14 @@ export class Context {
readonly events: loopTypes.LoopEvents;
private _actions: actions.ActionWithCode[] = [];
private _history: HistoryItem[] = [];
+ private _budget: { tokens: number | undefined; };
- constructor(page: Page, agentParms: channels.PageAgentParams, events: loopTypes.LoopEvents) {
+ constructor(page: Page, agentParams: channels.PageAgentParams, events: loopTypes.LoopEvents) {
this.page = page;
- this.agentParams = agentParms;
+ this.agentParams = agentParams;
this.sdkLanguage = page.browserContext._browser.sdkLanguage();
this.events = events;
+ this._budget = { tokens: agentParams.maxTokens };
}
async runActionAndWait(progress: Progress, action: actions.Action) {
@@ -79,6 +81,16 @@ export class Context {
this._actions = [];
}
+ consumeTokens(tokens: number) {
+ if (this._budget.tokens === undefined)
+ return;
+ this._budget.tokens = Math.max(0, this._budget.tokens - tokens);
+ }
+
+ maxTokensRemaining(): number | undefined {
+ return this._budget.tokens;
+ }
+
async waitForCompletion<R>(progress: Progress, callback: () => Promise<R>, options?: { noWait?: boolean }): Promise<R> {
if (options?.noWait)
return await callback();
diff --git a/packages/playwright-core/src/server/agent/pageAgent.ts b/packages/playwright-core/src/server/agent/pageAgent.ts
index ff573aa9e7..2578bea30a 100644
--- a/packages/playwright-core/src/server/agent/pageAgent.ts
+++ b/packages/playwright-core/src/server/agent/pageAgent.ts
@@ -33,7 +33,8 @@ import type { Progress } from '../progress';
export type CallParams = {
cacheKey?: string;
maxTokens?: number;
- maxTurns?: number;
+ maxActions?: number;
+ maxActionRetries?: number;
};
export async function pageAgentPerform(progress: Progress, context: Context, userTask: string, callParams: CallParams) {
@@ -105,9 +106,11 @@ async function runLoop(progress: Progress, context: Context, toolDefinitions: To
api: context.agentParams.api as any,
apiEndpoint: context.agentParams.apiEndpoint,
apiKey: context.agentParams.apiKey,
+ apiTimeout: context.agentParams.apiTimeout ?? 0,
model: context.agentParams.model,
- maxTurns: params.maxTurns ?? context.agentParams.maxTurns,
- maxTokens: params.maxTokens ?? context.agentParams.maxTokens,
+ maxTokens: params.maxTokens ?? context.maxTokensRemaining(),
+ maxToolCalls: params.maxActions ?? context.agentParams.maxActions ?? 10,
+ maxToolCallRetries: params.maxActionRetries ?? context.agentParams.maxActionRetries ?? 3,
summarize: true,
debug,
callTool,
@@ -136,8 +139,8 @@ async function runLoop(progress: Progress, context: Context, toolDefinitions: To
task.push(full);
task.push('');
- await loop.run(task.join('\n'), { signal: progress.signal });
-
+ const { error, usage } = await loop.run(task.join('\n'), { signal: progress.signal });
+ context.consumeTokens(usage.input + usage.output);
if (context.agentParams.apiCacheFile) {
const apiCacheAfter = { ...apiCacheBefore, ...loop.cache() };
const sortedCache = Object.fromEntries(Object.entries(apiCacheAfter).sort(([a], [b]) => a.localeCompare(b)));
@@ -148,6 +151,9 @@ async function runLoop(progress: Progress, context: Context, toolDefinitions: To
}
}
+ if (error)
+ throw new Error(`Agentic loop failed: ${error}`);
+
return { result: resultSchema ? reportedResult() : undefined };
}
diff --git a/packages/playwright-core/types/types.d.ts b/packages/playwright-core/types/types.d.ts
index a9d3c3daff..51dd92fb29 100644
--- a/packages/playwright-core/types/types.d.ts
+++ b/packages/playwright-core/types/types.d.ts
@@ -2110,12 +2110,26 @@ export interface Page {
cacheOutFile?: string;
};
- maxTokens?: number;
-
/**
- * Maximum number of agentic turns to take per call. Defaults to 10.
+ * Limits to use for the agentic loop.
*/
- maxTurns?: number;
+ limits?: {
+ /**
+ * Maximum number of tokens to consume. The agentic loop will stop after input + output tokens exceed this value.
+ * Defaults to unlimited.
+ */
+ maxTokens?: number;
+
+ /**
+ * Maximum number of agentic actions to generate, defaults to 10.
+ */
+ maxActions?: number;
+
+ /**
+ * Maximum number retries per action, defaults to 3.
+ */
+ maxActionRetries?: number;
+ };
provider?: {
/**
@@ -2133,6 +2147,11 @@ export interface Page {
*/
apiKey: string;
+ /**
+ * Amount of time to wait for the provider to respond to each request.
+ */
+ apiTimeout?: number;
+
/**
* Model identifier within the provider. Required in non-cache mode.
*/
@@ -5402,15 +5421,21 @@ export interface PageAgent {
cacheKey?: string;
/**
- * Maximum number of tokens to consume. The agentic loop will stop after input + output tokens exceed this value.
- * Defaults to context-wide value specified in `agent` property.
+ * Maximum number of retries when generating each action, defaults to context-wide value specified in `agent`
+ * property.
*/
- maxTokens?: number;
+ maxActionRetries?: number;
+
+ /**
+ * Maximum number of agentic actions to generate, defaults to context-wide value specified in `agent` property.
+ */
+ maxActions?: number;
/**
- * Maximum number of agentic turns during this call, defaults to context-wide value specified in `agent` property.
+ * Maximum number of tokens to consume. The agentic loop will stop after input + output tokens exceed this value.
+ * Defaults to context-wide value specified in `agent` property.
*/
- maxTurns?: number;
+ maxTokens?: number;
/**
* Request timeout in milliseconds. Defaults to action timeout. Pass `0` to disable timeout.
@@ -5438,15 +5463,21 @@ export interface PageAgent {
cacheKey?: string;
/**
- * Maximum number of tokens to consume. The agentic loop will stop after input + output tokens exceed this value.
- * Defaults to context-wide value specified in `agent` property.
+ * Maximum number of retries when generating each action, defaults to context-wide value specified in `agent`
+ * property.
*/
- maxTokens?: number;
+ maxActionRetries?: number;
+
+ /**
+ * Maximum number of agentic actions to generate, defaults to context-wide value specified in `agent` property.
+ */
+ maxActions?: number;
/**
- * Maximum number of agentic turns during this call, defaults to context-wide value specified in `agent` property.
+ * Maximum number of tokens to consume. The agentic loop will stop after input + output tokens exceed this value.
+ * Defaults to context-wide value specified in `agent` property.
*/
- maxTurns?: number;
+ maxTokens?: number;
/**
* Request timeout in milliseconds. Defaults to action timeout. Pass `0` to disable timeout.
diff --git a/packages/playwright/src/index.ts b/packages/playwright/src/index.ts
index 1a52a43897..82b978fb8c 100644
--- a/packages/playwright/src/index.ts
+++ b/packages/playwright/src/index.ts
@@ -462,12 +462,7 @@ const playwrightFixtures: Fixtures<TestFixtures, WorkerFixtures> = ({
const cacheFile = testInfoImpl.config.runAgents === 'all' ? undefined : await testInfoImpl._cloneStorage(resolvedCacheFile);
const cacheOutFile = path.join(testInfoImpl.artifactsDir(), 'agent-cache-' + createGuid() + '.json');
- const provider = agentOptions?.api && testInfo.config.runAgents !== 'none' ? {
- api: agentOptions.api as any,
- apiEndpoint: agentOptions.apiEndpoint,
- apiKey: agentOptions.apiKey,
- model: agentOptions.model,
- } : undefined;
+ const provider = agentOptions?.provider && testInfo.config.runAgents !== 'none' ? agentOptions.provider : undefined;
if (provider)
testInfo.setTimeout(0);
@@ -479,9 +474,9 @@ const playwrightFixtures: Fixtures<TestFixtures, WorkerFixtures> = ({
const agent = await page.agent({
provider,
cache,
- maxTokens: agentOptions?.maxTokens,
- maxTurns: agentOptions?.maxTurns,
+ limits: agentOptions?.limits,
secrets: agentOptions?.secrets,
+ systemPrompt: agentOptions?.systemPrompt,
});
await use(agent);
diff --git a/packages/playwright/types/test.d.ts b/packages/playwright/types/test.d.ts
index 16b81c92f0..4119c6d1b1 100644
--- a/packages/playwright/types/test.d.ts
+++ b/packages/playwright/types/test.d.ts
@@ -6955,15 +6955,22 @@ export type ScreenshotMode = 'off' | 'on' | 'only-on-failure' | 'on-first-failur
export type TraceMode = 'off' | 'on' | 'retain-on-failure' | 'on-first-retry' | 'on-all-retries' | 'retain-on-first-failure';
export type VideoMode = 'off' | 'on' | 'retain-on-failure' | 'on-first-retry';
export type AgentOptions = {
- api: string;
- apiKey: string;
- apiEndpoint?: string;
- model: string;
+ provider?: {
+ api: 'openai' | 'openai-compatible' | 'anthropic' | 'google';
+ apiEndpoint?: string;
+ apiKey: string;
+ apiTimeout?: number;
+ model: string;
+ },
+ limits?: {
+ maxTokens?: number;
+ maxActions?: number;
+ maxActionRetries?: number;
+ };
cachePathTemplate?: string;
- maxTurns?: number;
- maxTokens?: number;
runAgents?: 'all' | 'missing' | 'none';
secrets?: { [key: string]: string };
+ systemPrompt?: string;
};
/**
diff --git a/packages/protocol/src/channels.d.ts b/packages/protocol/src/channels.d.ts
index 38e0009d0d..a76dce8a90 100644
--- a/packages/protocol/src/channels.d.ts
+++ b/packages/protocol/src/channels.d.ts
@@ -2586,10 +2586,12 @@ export type PageAgentParams = {
api?: string,
apiKey?: string,
apiEndpoint?: string,
+ apiTimeout?: number,
apiCacheFile?: string,
cacheFile?: string,
cacheOutFile?: string,
- maxTurns?: number,
+ maxActions?: number,
+ maxActionRetries?: number,
maxTokens?: number,
model?: string,
secrets?: NameValue[],
@@ -2599,10 +2601,12 @@ export type PageAgentOptions = {
api?: string,
apiKey?: string,
apiEndpoint?: string,
+ apiTimeout?: number,
apiCacheFile?: string,
cacheFile?: string,
cacheOutFile?: string,
- maxTurns?: number,
+ maxActions?: number,
+ maxActionRetries?: number,
maxTokens?: number,
model?: string,
secrets?: NameValue[],
@@ -5115,13 +5119,15 @@ export type PageAgentTurnEvent = {
};
export type PageAgentPerformParams = {
task: string,
- maxTurns?: number,
+ maxActions?: number,
+ maxActionRetries?: number,
maxTokens?: number,
cacheKey?: string,
timeout?: number,
};
export type PageAgentPerformOptions = {
- maxTurns?: number,
+ maxActions?: number,
+ maxActionRetries?: number,
maxTokens?: number,
cacheKey?: string,
timeout?: number,
@@ -5131,13 +5137,15 @@ export type PageAgentPerformResult = {
};
export type PageAgentExpectParams = {
expectation: string,
- maxTurns?: number,
+ maxActions?: number,
+ maxActionRetries?: number,
maxTokens?: number,
cacheKey?: string,
timeout?: number,
};
export type PageAgentExpectOptions = {
- maxTurns?: number,
+ maxActions?: number,
+ maxActionRetries?: number,
maxTokens?: number,
cacheKey?: string,
timeout?: number,
@@ -5148,13 +5156,15 @@ export type PageAgentExpectResult = {
export type PageAgentExtractParams = {
query: string,
schema: any,
- maxTurns?: number,
+ maxActions?: number,
+ maxActionRetries?: number,
maxTokens?: number,
cacheKey?: string,
timeout?: number,
};
export type PageAgentExtractOptions = {
- maxTurns?: number,
+ maxActions?: number,
+ maxActionRetries?: number,
maxTokens?: number,
cacheKey?: string,
timeout?: number,
diff --git a/packages/protocol/src/protocol.yml b/packages/protocol/src/protocol.yml
index edce8f286f..d4af228ada 100644
--- a/packages/protocol/src/protocol.yml
+++ b/packages/protocol/src/protocol.yml
@@ -2019,10 +2019,12 @@ Page:
api: string?
apiKey: string?
apiEndpoint: string?
+ apiTimeout: int?
apiCacheFile: string?
cacheFile: string?
cacheOutFile: string?
- maxTurns: int?
+ maxActions: int?
+ maxActionRetries: int?
maxTokens: int?
model: string?
secrets:
@@ -4374,7 +4376,8 @@ PageAgent:
PageAgentOptions:
type: mixin
properties:
- maxTurns: int?
+ maxActions: int?
+ maxActionRetries: int?
maxTokens: int?
cacheKey: string?
timeout: int?
diff --git a/tests/library/__llm_cache__/library-agent-limits-should-respect-max-actions-limit.json b/tests/library/__llm_cache__/library-agent-limits-should-respect-max-actions-limit.json
new file mode 100644
index 0000000000..0689191595
--- /dev/null
+++ b/tests/library/__llm_cache__/library-agent-limits-should-respect-max-actions-limit.json
@@ -0,0 +1,114 @@
+{
+ "02fe9016aefaf25bffe8ae4a623014e933282db1": {
+ "result": {
+ "role": "assistant",
+ "content": [
+ {
+ "type": "text",
+ "text": "Now clicking the second time."
+ },
+ {
+ "type": "tool_call",
+ "name": "browser_click",
+ "arguments": {
+ "element": "Submit button",
+ "ref": "e2",
+ "_is_done": false
+ },
+ "id": "toolu_01PVA67E6ey7gspaUHe6vKUb"
+ }
+ ],
+ "stopReason": {
+ "code": "ok"
+ }
+ },
+ "usage": {
+ "input": 3044,
+ "output": 98
+ }
+ },
+ "9406488d923a063c065ff21e2a4f78c47cd932aa": {
+ "result": {
+ "role": "assistant",
+ "content": [
+ {
+ "type": "text",
+ "text": "Now clicking the third time."
+ },
+ {
+ "type": "tool_call",
+ "name": "browser_click",
+ "arguments": {
+ "element": "Submit button",
+ "ref": "e2",
+ "_is_done": false
+ },
+ "id": "toolu_01HreVmfgh1urgtDcxbkDbsN"
+ }
+ ],
+ "stopReason": {
+ "code": "ok"
+ }
+ },
+ "usage": {
+ "input": 3117,
+ "output": 98
+ }
+ },
+ "a8a2db95880cad9f7a0ce1d3e409e2d273685eba": {
+ "result": {
+ "role": "assistant",
+ "content": [
+ {
+ "type": "text",
+ "text": "I'll click the submit button 5 times as requested. Let me start by clicking it the first time."
+ },
+ {
+ "type": "tool_call",
+ "name": "browser_click",
+ "arguments": {
+ "element": "Submit button",
+ "ref": "e2",
+ "_is_done": false
+ },
+ "id": "toolu_01Q2Bk335vL1wsswZGCsuaTe"
+ }
+ ],
+ "stopReason": {
+ "code": "ok"
+ }
+ },
+ "usage": {
+ "input": 2897,
+ "output": 115
+ }
+ },
+ "efe9f77d726c2eff73d5c51d2cbc30c3171d6294": {
+ "result": {
+ "role": "assistant",
+ "content": [
+ {
+ "type": "text",
+ "text": "Now clicking the fourth time."
+ },
+ {
+ "type": "tool_call",
+ "name": "browser_click",
+ "arguments": {
+ "element": "Submit button",
+ "ref": "e2",
+ "_is_done": false
+ },
+ "id": "toolu_01B29QmP39aFMi2uHevcZKU1"
+ }
+ ],
+ "stopReason": {
+ "code": "ok"
+ }
+ },
+ "usage": {
+ "input": 3187,
+ "output": 98
+ }
+ }
+}
\ No newline at end of file
diff --git a/tests/library/__llm_cache__/library-agent-perform-click-a-button.json b/tests/library/__llm_cache__/library-agent-perform-click-a-button.json
index 26f5b7cdc4..83a2310d62 100644
--- a/tests/library/__llm_cache__/library-agent-perform-click-a-button.json
+++ b/tests/library/__llm_cache__/library-agent-perform-click-a-button.json
@@ -1,5 +1,5 @@
{
- "0706eb777330056e04cc3abb4715ed3fa1c9d4d5": {
+ "8c5697c11562d27e025c16eaba3a3d333198bb56": {
"result": {
"role": "assistant",
"content": [
@@ -15,9 +15,12 @@
"ref": "e2",
"_is_done": true
},
- "id": "toolu_01W54FxjcWYV856RHP3V2xdj"
+ "id": "toolu_019UXybai8M1mCTeCBkEtzJc"
}
- ]
+ ],
+ "stopReason": {
+ "code": "ok"
+ }
},
"usage": {
"input": 2893,
diff --git a/tests/library/__llm_cache__/library-agent-perform-expect-value.json b/tests/library/__llm_cache__/library-agent-perform-expect-value.json
index c4cd9ce537..8081774369 100644
--- a/tests/library/__llm_cache__/library-agent-perform-expect-value.json
+++ b/tests/library/__llm_cache__/library-agent-perform-expect-value.json
@@ -1,5 +1,5 @@
{
- "77c0828e4bffb298a3850504550ffcac23e6692e": {
+ "46258723cbb8dc1fbc9c390304cd506187897079": {
"result": {
"role": "assistant",
"content": [
@@ -16,22 +16,25 @@
"text": "bogus",
"_is_done": false
},
- "id": "toolu_01FfLGgBK1CHZ8e54gQMukA6"
+ "id": "toolu_01V8GEC6fjiTt98e3k88ENyV"
}
- ]
+ ],
+ "stopReason": {
+ "code": "ok"
+ }
},
"usage": {
"input": 2926,
"output": 149
}
},
- "ca3ff77a849d4eacf88d299dc8727d63724a68aa": {
+ "5429c27bfbcae1d531a13010d3a5d4807251a7c4": {
"result": {
"role": "assistant",
"content": [
{
"type": "text",
- "text": "Perfect! I have successfully completed all three parts of the task:\n\n1. ✅ **Entered \"bogus\" into the email field** - The text has been typed into the textbox\n2. ✅ **Verified the value is \"bogus\"** - The page snapshot shows the textbox contains \"bogus\"\n3. ✅ **Confirmed error message is displayed** - The error message \"Error: Invalid email address\" is now visible on the page\n\nThe task is complete."
+ "text": "Perfect! I've successfully completed all three parts of the task:\n\n1. ✅ **Entered \"bogus\" into the email field** - The text has been typed into the Email Address textbox\n2. ✅ **Verified the value is \"bogus\"** - The snapshot confirms the textbox contains \"bogus\"\n3. ✅ **Confirmed error message is displayed** - The error message \"Error: Invalid email address\" is now visible on the page\n\nThe task has been completed successfully. The invalid email \"bogus\" triggered the expected validation error message."
},
{
"type": "tool_call",
@@ -39,13 +42,16 @@
"arguments": {
"_is_done": true
},
- "id": "toolu_018wjD4H2S5QHQ3rpE2sR7bM"
+ "id": "toolu_01WuNDf5vv1QM1SdgkAKnpVF"
}
- ]
+ ],
+ "stopReason": {
+ "code": "ok"
+ }
},
"usage": {
"input": 3138,
- "output": 167
+ "output": 184
}
}
}
\ No newline at end of file
diff --git a/tests/library/__llm_cache__/library-agent-perform-extract-task.json b/tests/library/__llm_cache__/library-agent-perform-extract-task.json
index 74f9d15fbb..46a3c383fc 100644
--- a/tests/library/__llm_cache__/library-agent-perform-extract-task.json
+++ b/tests/library/__llm_cache__/library-agent-perform-extract-task.json
@@ -1,5 +1,5 @@
{
- "b5f1ee58f1051afb059b3b33e8be41576a1324a4": {
+ "d02ae2240fac126bacb74133b970880eaaf3f915": {
"result": {
"role": "assistant",
"content": [
@@ -23,9 +23,12 @@
],
"_is_done": true
},
- "id": "toolu_01WvevmDTBxmKg79Ji6vQvpH"
+ "id": "toolu_01JtUAyrJqJSRwBsS7pn9fRY"
}
- ]
+ ],
+ "stopReason": {
+ "code": "ok"
+ }
},
"usage": {
"input": 888,
diff --git a/tests/library/__llm_cache__/library-agent-perform-perform-history.json b/tests/library/__llm_cache__/library-agent-perform-perform-history.json
index 00a85f8150..13be1898fc 100644
--- a/tests/library/__llm_cache__/library-agent-perform-perform-history.json
+++ b/tests/library/__llm_cache__/library-agent-perform-perform-history.json
@@ -1,5 +1,5 @@
{
- "4f9987bc22c6035501274532494c185d1270aba2": {
+ "7416c1b94dbb8a1f7405a276811783738c295295": {
"result": {
"role": "assistant",
"content": [
@@ -15,16 +15,19 @@
"ref": "e3",
"_is_done": true
},
- "id": "toolu_013sypFp7tiRCLaR3i3oWrei"
+ "id": "toolu_017c7pdkXdqSjQMiaF7UKwa8"
}
- ]
+ ],
+ "stopReason": {
+ "code": "ok"
+ }
},
"usage": {
"input": 2946,
"output": 102
}
},
- "79e2d58996075c4ebe8eb9e557bd0cdb8b04211a": {
+ "96cdcffad5cf3bf3d064acdeb05fbb4968f27c02": {
"result": {
"role": "assistant",
"content": [
@@ -40,9 +43,12 @@
"ref": "e3",
"_is_done": true
},
- "id": "toolu_01E6xBUYzQEdRrkWoobbUbFq"
+ "id": "toolu_01E1epU97Ub6RDTk4PeFuKke"
}
- ]
+ ],
+ "stopReason": {
+ "code": "ok"
+ }
},
"usage": {
"input": 2933,
diff --git a/tests/library/__llm_cache__/library-agent-perform-perform-run-timeout.json b/tests/library/__llm_cache__/library-agent-perform-perform-run-timeout.json
index 836cd0bc34..01648843e9 100644
--- a/tests/library/__llm_cache__/library-agent-perform-perform-run-timeout.json
+++ b/tests/library/__llm_cache__/library-agent-perform-perform-run-timeout.json
@@ -1,5 +1,5 @@
{
- "c2f54e956e3131255f03c5484a637b580f7d695f": {
+ "c3664a96a82cba8678b7ca68973bb25c1d1207a4": {
"result": {
"role": "assistant",
"content": [
@@ -15,9 +15,12 @@
"ref": "e3",
"_is_done": true
},
- "id": "toolu_01GipHDfLSETD8VyJt6D6ng5"
+ "id": "toolu_01FP2NCSpx8mnFeg16iX9gKi"
}
- ]
+ ],
+ "stopReason": {
+ "code": "ok"
+ }
},
"usage": {
"input": 2919,
diff --git a/tests/library/__llm_cache__/library-agent-perform-retrieve-a-secret.json b/tests/library/__llm_cache__/library-agent-perform-retrieve-a-secret.json
index f60e4427dd..3c0ad14395 100644
--- a/tests/library/__llm_cache__/library-agent-perform-retrieve-a-secret.json
+++ b/tests/library/__llm_cache__/library-agent-perform-retrieve-a-secret.json
@@ -1,5 +1,5 @@
{
- "c1d41afc231b64caf136132dd841ca8f45876674": {
+ "8bd748693a579b9f17e2fb127fcb3af9706934d4": {
"result": {
"role": "assistant",
"content": [
@@ -16,9 +16,12 @@
"text": "%x-secret-email%",
"_is_done": true
},
- "id": "toolu_018PD6jsENphY4ERa1XRF4ZN"
+ "id": "toolu_01YYGZnA96oGJMknVBJC4Cfj"
}
- ]
+ ],
+ "stopReason": {
+ "code": "ok"
+ }
},
"usage": {
"input": 2901,
diff --git a/tests/library/agent-helpers.ts b/tests/library/agent-helpers.ts
index b6d99de6a2..b145b1882d 100644
--- a/tests/library/agent-helpers.ts
+++ b/tests/library/agent-helpers.ts
@@ -32,7 +32,9 @@ export async function setCacheObject(object: any) {
await fs.promises.writeFile(cacheFile(), JSON.stringify(object, null, 2), 'utf8');
}
-export async function generateAgent(context: BrowserContext, options: { secrets?: Record<string, string> } = {}) {
+type AgentOptions = Parameters<Page['agent']>[0];
+
+export async function generateAgent(context: BrowserContext, options: AgentOptions = {}) {
const apiCacheFile = path.join(__dirname, '__llm_cache__', sanitizeFileName(test.info().titlePath.join(' ')) + '.json');
const page = await context.newPage();
diff --git a/tests/library/agent-limits.spec.ts b/tests/library/agent-limits.spec.ts
new file mode 100644
index 0000000000..07d3f30149
--- /dev/null
+++ b/tests/library/agent-limits.spec.ts
@@ -0,0 +1,54 @@
+/**
+ * Copyright (c) Microsoft Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { browserTest as test, expect } from '../config/browserTest';
+import { generateAgent } from './agent-helpers';
+
+test('should respect total max tokens limit', async ({ context }) => {
+ const { page, agent } = await generateAgent(context, {
+ limits: {
+ maxTokens: 123,
+ },
+ });
+ await page.setContent(`
+ <button>Submit</button>
+ `);
+ const e = await agent.perform('Click submit button').catch(e => e);
+ expect(e.message.toLowerCase()).toContain('budget');
+ expect(e.message).toContain('123');
+});
+
+test('should respect call max tokens limit', async ({ context }) => {
+ const { page, agent } = await generateAgent(context);
+ await page.setContent(`
+ <button>Submit</button>
+ `);
+ const e = await agent.perform('Click submit button', { maxTokens: 123 }).catch(e => e);
+ expect(e.message.toLowerCase()).toContain('budget');
+ expect(e.message).toContain('123');
+});
+
+test('should respect max actions limit', async ({ context }) => {
+ const { page, agent } = await generateAgent(context);
+ let clicked = 0;
+ await page.exposeFunction('clicked', () => ++clicked);
+ await page.setContent(`
+ <button onclick="clicked()">Submit</button>
+ `);
+ const e = await agent.perform('Click the submit button 5 times', { maxActions: 3 }).catch(e => e);
+ expect(e.message).toContain('Failed to perform step, max tool calls (3) reached');
+ expect(clicked).toBe(3);
+});
diff --git a/utils/generate_types/overrides-test.d.ts b/utils/generate_types/overrides-test.d.ts
index bf47bdab5a..3a0eb2eb2d 100644
--- a/utils/generate_types/overrides-test.d.ts
+++ b/utils/generate_types/overrides-test.d.ts
@@ -270,15 +270,22 @@ export type ScreenshotMode = 'off' | 'on' | 'only-on-failure' | 'on-first-failur
export type TraceMode = 'off' | 'on' | 'retain-on-failure' | 'on-first-retry' | 'on-all-retries' | 'retain-on-first-failure';
export type VideoMode = 'off' | 'on' | 'retain-on-failure' | 'on-first-retry';
export type AgentOptions = {
- api: string;
- apiKey: string;
- apiEndpoint?: string;
- model: string;
+ provider?: {
+ api: 'openai' | 'openai-compatible' | 'anthropic' | 'google';
+ apiEndpoint?: string;
+ apiKey: string;
+ apiTimeout?: number;
+ model: string;
+ },
+ limits?: {
+ maxTokens?: number;
+ maxActions?: number;
+ maxActionRetries?: number;
+ };
cachePathTemplate?: string;
- maxTurns?: number;
- maxTokens?: number;
runAgents?: 'all' | 'missing' | 'none';
secrets?: { [key: string]: string };
+ systemPrompt?: string;
};
export interface PlaywrightTestOptions {