Playwright is a framework for Web Testing and Automation. It allows testing Chromium, Firefox and WebKit with a single API.
TypeScript automationchromechromiume2e-testingelectronend-to-end-testingfirefoxjavascriptplaywrighttesttest-automationtestingtesting-toolswebwebkit
chore: update agent limits (#38800)
Pavel Feldmanf01cb2caceb2
diff --git a/docs/src/api/class-page.md b/docs/src/api/class-page.mdindex 45a0e2dc1c..5c2b752197 100644--- a/docs/src/api/class-page.md+++ b/docs/src/api/class-page.md@@ -722,15 +722,14 @@ Initialize page agent with the llm provider and cache.- `cacheFile` ?<[string]> Cache file to use/generate code for performed actions into. Cache is not used if not specified (default).- `cacheOutFile` ?<[string]> When specified, generated entries are written into the `cacheOutFile` instead of updating the `cacheFile`.-### option: Page.agent.maxTurns+### option: Page.agent.limits* since: v1.58-- `maxTurns` <[int]>+- `limits` <[Object]>+ - `maxTokens` ?<[int]> Maximum number of tokens to consume. The agentic loop will stop after input + output tokens exceed this value. Defaults to unlimited.+ - `maxActions` ?<[int]> Maximum number of agentic actions to generate, defaults to 10.+ - `maxActionRetries` ?<[int]> Maximum number retries per action, defaults to 3.-Maximum number of agentic turns to take per call. Defaults to 10.--### option: Page.agent.maxTokens-* since: v1.58-- `maxTokens` ?<[int]>+Limits to use for the agentic loop.### option: Page.agent.provider* since: v1.58@@ -738,6 +737,7 @@ Maximum number of agentic turns to take per call. Defaults to 10.- `api` <[PageAgentAPI]<"openai"|"openai-compatible"|"anthropic"|"google">> API to use.- `apiEndpoint` ?<[string]> Endpoint to use if different from default.- `apiKey` <[string]> API key for the LLM provider.+ - `apiTimeout` ?<[int]> Amount of time to wait for the provider to respond to each request.- `model` <[string]> Model identifier within the provider. Required in non-cache mode.### option: Page.agent.secretsdiff --git a/docs/src/api/params.md b/docs/src/api/params.mdindex bd319b19ad..9b7bf8f3d4 100644--- a/docs/src/api/params.md+++ b/docs/src/api/params.md@@ -384,11 +384,17 @@ By default, they are cached globally with the `task` as a key. This option allowMaximum number of tokens to consume. The agentic loop will stop after input + output tokens exceed this value.Defaults to context-wide value specified in `agent` property.-## page-agent-max-turns+## page-agent-max-actions* since: v1.58-- `maxTurns` <[int]>+- `maxActions` <[int]>-Maximum number of agentic turns during this call, defaults to context-wide value specified in `agent` property.+Maximum number of agentic actions to generate, defaults to context-wide value specified in `agent` property.++## page-agent-max-action-retries+* since: v1.58+- `maxActionRetries` <[int]>++Maximum number of retries when generating each action, defaults to context-wide value specified in `agent` property.## page-agent-timeout* since: v1.58@@ -399,7 +405,8 @@ Request timeout in milliseconds. Defaults to action timeout. Pass `0` to disable## page-agent-call-options-v1.58- %%-page-agent-cache-key-%%- %%-page-agent-max-tokens-%%-- %%-page-agent-max-turns-%%+- %%-page-agent-max-actions-%%+- %%-page-agent-max-action-retries-%%- %%-page-agent-timeout-%%## fetch-param-urldiff --git a/docs/src/test-api/class-testoptions.md b/docs/src/test-api/class-testoptions.mdindex 194cd58616..4c60a5c896 100644--- a/docs/src/test-api/class-testoptions.md+++ b/docs/src/test-api/class-testoptions.md@@ -49,14 +49,19 @@ export default defineConfig({## property: TestOptions.agentOptions* since: v1.58- type: <[Object]>- - `api` ?<[string]> LLM provider to use. Required in non-cache mode.- - `apiKey` ?<[string]> Key for the LLM provider.- - `apiEndpoint` ?<[string]> LLM provider endpoint.- - `model` ?<[string]> Model identifier within the provider. Required in non-cache mode.+ - `provider` <[Object]>+ - `api` <[PageAgentAPI]<"openai"|"openai-compatible"|"anthropic"|"google">> API to use.+ - `apiEndpoint` ?<[string]> Endpoint to use if different from default.+ - `apiKey` <[string]> API key for the LLM provider.+ - `apiTimeout` ?<[int]> Amount of time to wait for the provider to respond to each request.+ - `model` <[string]> Model identifier within the provider. Required in non-cache mode.- `cachePathTemplate` ?<[string]> Cache file template to use/generate code for performed actions into.- - `maxTurns` ?<[int]> Maximum number of agentic turns to take per call. Defaults to 10.- - `maxTokens` ?<[int]> Maximum number of tokens to consume per call. The agentic loop will stop after input + output tokens exceed this value. Defaults on unlimited.+ - `limits` <[Object]>+ - `maxTokens` ?<[int]> Maximum number of tokens to consume. The agentic loop will stop after input + output tokens exceed this value. Defaults to unlimited.+ - `maxActions` ?<[int]> Maximum number of agentic actions to generate, defaults to 10.+ - `maxActionRetries` ?<[int]> Maximum number retries per action, defaults to 3.- `secrets` ?<[Object]<[string], [string]>> Secrets to hide from the LLM.+ - `systemPrompt` <[string]> System prompt for the agent's loop.## property: TestOptions.baseURL = %%-context-option-baseURL-%%* since: v1.10diff --git a/examples/todomvc/tests/fixtures.ts b/examples/todomvc/tests/fixtures.tsindex 0e317c4706..6226e718bf 100644--- a/examples/todomvc/tests/fixtures.ts+++ b/examples/todomvc/tests/fixtures.ts@@ -6,10 +6,12 @@ export { expect } from '@playwright/test';export const test = baseTest.extend({agentOptions: {- api: 'anthropic',- apiKey: process.env.AZURE_SONNET_API_KEY!,- apiEndpoint: process.env.AZURE_SONNET_ENDPOINT!,- model: 'claude-sonnet-4-5',+ provider: {+ api: 'anthropic',+ apiKey: process.env.AZURE_SONNET_API_KEY!,+ apiEndpoint: process.env.AZURE_SONNET_ENDPOINT!,+ model: 'claude-sonnet-4-5',+ },},page: async ({ page }, use) => {await page.goto('https://demo.playwright.dev/todomvc');diff --git a/package-lock.json b/package-lock.jsonindex e3cd66d46e..5160eea979 100644--- a/package-lock.json+++ b/package-lock.json@@ -18,7 +18,7 @@"@eslint/compat": "^1.3.2","@eslint/eslintrc": "^3.3.1","@eslint/js": "^9.34.0",- "@lowire/loop": "^0.0.19",+ "@lowire/loop": "^0.0.23","@modelcontextprotocol/sdk": "^1.17.5","@octokit/graphql-schema": "^15.26.0","@stylistic/eslint-plugin": "^5.2.3",@@ -1065,9 +1065,9 @@}},"node_modules/@lowire/loop": {- "version": "0.0.19",- "resolved": "https://registry.npmjs.org/@lowire/loop/-/loop-0.0.19.tgz",- "integrity": "sha512-uugjYtmA1mNSwHzbZEnvFedjB2EUX0s4NDbDbUcGuz6CH8fzZV2FFodY6fneik+r3GdLJtpvzpOP+cU/M0ba3w==",+ "version": "0.0.23",+ "resolved": "https://registry.npmjs.org/@lowire/loop/-/loop-0.0.23.tgz",+ "integrity": "sha512-zw4yFzto2T9g+CiHfubzxut0JMxA053XQ2syveqPjqofvRZyumhJR11RIQjKrPO9rDbf2CFviKrYFt7kgru1VA==","dev": true,"license": "Apache-2.0","engines": {diff --git a/package.json b/package.jsonindex 8db7c35f51..8138c730d9 100644--- a/package.json+++ b/package.json@@ -55,7 +55,7 @@"@eslint/compat": "^1.3.2","@eslint/eslintrc": "^3.3.1","@eslint/js": "^9.34.0",- "@lowire/loop": "^0.0.19",+ "@lowire/loop": "^0.0.23","@modelcontextprotocol/sdk": "^1.17.5","@octokit/graphql-schema": "^15.26.0","@stylistic/eslint-plugin": "^5.2.3",diff --git a/packages/playwright-client/types/types.d.ts b/packages/playwright-client/types/types.d.tsindex a9d3c3daff..51dd92fb29 100644--- a/packages/playwright-client/types/types.d.ts+++ b/packages/playwright-client/types/types.d.ts@@ -2110,12 +2110,26 @@ export interface Page {cacheOutFile?: string;};- maxTokens?: number;-/**- * Maximum number of agentic turns to take per call. Defaults to 10.+ * Limits to use for the agentic loop.*/- maxTurns?: number;+ limits?: {+ /**+ * Maximum number of tokens to consume. The agentic loop will stop after input + output tokens exceed this value.+ * Defaults to unlimited.+ */+ maxTokens?: number;++ /**+ * Maximum number of agentic actions to generate, defaults to 10.+ */+ maxActions?: number;++ /**+ * Maximum number retries per action, defaults to 3.+ */+ maxActionRetries?: number;+ };provider?: {/**@@ -2133,6 +2147,11 @@ export interface Page {*/apiKey: string;+ /**+ * Amount of time to wait for the provider to respond to each request.+ */+ apiTimeout?: number;+/*** Model identifier within the provider. Required in non-cache mode.*/@@ -5402,15 +5421,21 @@ export interface PageAgent {cacheKey?: string;/**- * Maximum number of tokens to consume. The agentic loop will stop after input + output tokens exceed this value.- * Defaults to context-wide value specified in `agent` property.+ * Maximum number of retries when generating each action, defaults to context-wide value specified in `agent`+ * property.*/- maxTokens?: number;+ maxActionRetries?: number;++ /**+ * Maximum number of agentic actions to generate, defaults to context-wide value specified in `agent` property.+ */+ maxActions?: number;/**- * Maximum number of agentic turns during this call, defaults to context-wide value specified in `agent` property.+ * Maximum number of tokens to consume. The agentic loop will stop after input + output tokens exceed this value.+ * Defaults to context-wide value specified in `agent` property.*/- maxTurns?: number;+ maxTokens?: number;/*** Request timeout in milliseconds. Defaults to action timeout. Pass `0` to disable timeout.@@ -5438,15 +5463,21 @@ export interface PageAgent {cacheKey?: string;/**- * Maximum number of tokens to consume. The agentic loop will stop after input + output tokens exceed this value.- * Defaults to context-wide value specified in `agent` property.+ * Maximum number of retries when generating each action, defaults to context-wide value specified in `agent`+ * property.*/- maxTokens?: number;+ maxActionRetries?: number;++ /**+ * Maximum number of agentic actions to generate, defaults to context-wide value specified in `agent` property.+ */+ maxActions?: number;/**- * Maximum number of agentic turns during this call, defaults to context-wide value specified in `agent` property.+ * Maximum number of tokens to consume. The agentic loop will stop after input + output tokens exceed this value.+ * Defaults to context-wide value specified in `agent` property.*/- maxTurns?: number;+ maxTokens?: number;/*** Request timeout in milliseconds. Defaults to action timeout. Pass `0` to disable timeout.diff --git a/packages/playwright-core/ThirdPartyNotices.txt b/packages/playwright-core/ThirdPartyNotices.txtindex 71cefc3d7d..cf98b9ac14 100644--- a/packages/playwright-core/ThirdPartyNotices.txt+++ b/packages/playwright-core/ThirdPartyNotices.txt@@ -4,7 +4,7 @@ THIRD-PARTY SOFTWARE NOTICES AND INFORMATIONThis project incorporates components from the projects listed below. The original copyright notices and the licenses under which Microsoft received such components are set forth below. Microsoft reserves all rights not expressly granted herein, whether by implication, estoppel or otherwise.-- @lowire/[email protected] (https://github.com/pavelfeldman/lowire)+- @lowire/[email protected] (https://github.com/pavelfeldman/lowire)- @modelcontextprotocol/[email protected] (https://github.com/modelcontextprotocol/typescript-sdk)- [email protected] (https://github.com/jshttp/accepts)- [email protected] (https://github.com/TooTallNate/proxy-agents)@@ -135,7 +135,7 @@ This project incorporates components from the projects listed below. The origina- [email protected] (https://github.com/StefanTerdell/zod-to-json-schema)- [email protected] (https://github.com/colinhacks/zod)-%% @lowire/[email protected] NOTICES AND INFORMATION BEGIN HERE+%% @lowire/[email protected] NOTICES AND INFORMATION BEGIN HERE=========================================Apache LicenseVersion 2.0, January 2004@@ -339,7 +339,7 @@ Apache LicenseSee the License for the specific language governing permissions andlimitations under the License.=========================================-END OF @lowire/[email protected] AND INFORMATION+END OF @lowire/[email protected] AND INFORMATION%% @modelcontextprotocol/[email protected] NOTICES AND INFORMATION BEGIN HERE=========================================diff --git a/packages/playwright-core/bundles/mcp/package-lock.json b/packages/playwright-core/bundles/mcp/package-lock.jsonindex 33a47d828b..bcc613bfc0 100644--- a/packages/playwright-core/bundles/mcp/package-lock.json+++ b/packages/playwright-core/bundles/mcp/package-lock.json@@ -8,16 +8,16 @@"name": "mcp-bundle","version": "0.0.1","dependencies": {- "@lowire/loop": "^0.0.19",+ "@lowire/loop": "^0.0.23","@modelcontextprotocol/sdk": "^1.24.0","zod": "^3.25.76","zod-to-json-schema": "^3.24.6"}},"node_modules/@lowire/loop": {- "version": "0.0.19",- "resolved": "https://registry.npmjs.org/@lowire/loop/-/loop-0.0.19.tgz",- "integrity": "sha512-uugjYtmA1mNSwHzbZEnvFedjB2EUX0s4NDbDbUcGuz6CH8fzZV2FFodY6fneik+r3GdLJtpvzpOP+cU/M0ba3w==",+ "version": "0.0.23",+ "resolved": "https://registry.npmjs.org/@lowire/loop/-/loop-0.0.23.tgz",+ "integrity": "sha512-zw4yFzto2T9g+CiHfubzxut0JMxA053XQ2syveqPjqofvRZyumhJR11RIQjKrPO9rDbf2CFviKrYFt7kgru1VA==","license": "Apache-2.0","engines": {"node": ">=20"diff --git a/packages/playwright-core/bundles/mcp/package.json b/packages/playwright-core/bundles/mcp/package.jsonindex f3209bf1bc..fb90f525df 100644--- a/packages/playwright-core/bundles/mcp/package.json+++ b/packages/playwright-core/bundles/mcp/package.json@@ -3,7 +3,7 @@"version": "0.0.1","private": true,"dependencies": {- "@lowire/loop": "^0.0.19",+ "@lowire/loop": "^0.0.23","@modelcontextprotocol/sdk": "^1.24.0","zod": "^3.25.76","zod-to-json-schema": "^3.24.6"diff --git a/packages/playwright-core/src/client/page.ts b/packages/playwright-core/src/client/page.tsindex f464643d1f..4e3d827595 100644--- a/packages/playwright-core/src/client/page.ts+++ b/packages/playwright-core/src/client/page.ts@@ -852,12 +852,14 @@ export class Page extends ChannelOwner<channels.PageChannel> implements api.Pageapi: options.provider?.api,apiEndpoint: options.provider?.apiEndpoint,apiKey: options.provider?.apiKey,+ apiTimeout: options.provider?.apiTimeout,apiCacheFile: (options.provider as any)?._apiCacheFile,model: options.provider?.model,cacheFile: options.cache?.cacheFile,cacheOutFile: options.cache?.cacheOutFile,- maxTokens: options.maxTokens,- maxTurns: options.maxTurns,+ maxTokens: options.limits?.maxTokens,+ maxActions: options.limits?.maxActions,+ maxActionRetries: options.limits?.maxActionRetries,secrets: options.secrets ? Object.entries(options.secrets).map(([name, value]) => ({ name, value })) : undefined,systemPrompt: options.systemPrompt,};diff --git a/packages/playwright-core/src/protocol/validator.ts b/packages/playwright-core/src/protocol/validator.tsindex 62dca989ba..4ed0e4af71 100644--- a/packages/playwright-core/src/protocol/validator.ts+++ b/packages/playwright-core/src/protocol/validator.ts@@ -1491,10 +1491,12 @@ scheme.PageAgentParams = tObject({api: tOptional(tString),apiKey: tOptional(tString),apiEndpoint: tOptional(tString),+ apiTimeout: tOptional(tInt),apiCacheFile: tOptional(tString),cacheFile: tOptional(tString),cacheOutFile: tOptional(tString),- maxTurns: tOptional(tInt),+ maxActions: tOptional(tInt),+ maxActionRetries: tOptional(tInt),maxTokens: tOptional(tInt),model: tOptional(tString),secrets: tOptional(tArray(tType('NameValue'))),@@ -2907,7 +2909,8 @@ scheme.PageAgentTurnEvent = tObject({});scheme.PageAgentPerformParams = tObject({task: tString,- maxTurns: tOptional(tInt),+ maxActions: tOptional(tInt),+ maxActionRetries: tOptional(tInt),maxTokens: tOptional(tInt),cacheKey: tOptional(tString),timeout: tOptional(tInt),@@ -2917,7 +2920,8 @@ scheme.PageAgentPerformResult = tObject({});scheme.PageAgentExpectParams = tObject({expectation: tString,- maxTurns: tOptional(tInt),+ maxActions: tOptional(tInt),+ maxActionRetries: tOptional(tInt),maxTokens: tOptional(tInt),cacheKey: tOptional(tString),timeout: tOptional(tInt),@@ -2928,7 +2932,8 @@ scheme.PageAgentExpectResult = tObject({scheme.PageAgentExtractParams = tObject({query: tString,schema: tAny,- maxTurns: tOptional(tInt),+ maxActions: tOptional(tInt),+ maxActionRetries: tOptional(tInt),maxTokens: tOptional(tInt),cacheKey: tOptional(tString),timeout: tOptional(tInt),diff --git a/packages/playwright-core/src/server/agent/context.ts b/packages/playwright-core/src/server/agent/context.tsindex 302b729341..12aebb4f03 100644--- a/packages/playwright-core/src/server/agent/context.ts+++ b/packages/playwright-core/src/server/agent/context.ts@@ -38,12 +38,14 @@ export class Context {readonly events: loopTypes.LoopEvents;private _actions: actions.ActionWithCode[] = [];private _history: HistoryItem[] = [];+ private _budget: { tokens: number | undefined; };- constructor(page: Page, agentParms: channels.PageAgentParams, events: loopTypes.LoopEvents) {+ constructor(page: Page, agentParams: channels.PageAgentParams, events: loopTypes.LoopEvents) {this.page = page;- this.agentParams = agentParms;+ this.agentParams = agentParams;this.sdkLanguage = page.browserContext._browser.sdkLanguage();this.events = events;+ this._budget = { tokens: agentParams.maxTokens };}async runActionAndWait(progress: Progress, action: actions.Action) {@@ -79,6 +81,16 @@ export class Context {this._actions = [];}+ consumeTokens(tokens: number) {+ if (this._budget.tokens === undefined)+ return;+ this._budget.tokens = Math.max(0, this._budget.tokens - tokens);+ }++ maxTokensRemaining(): number | undefined {+ return this._budget.tokens;+ }+async waitForCompletion<R>(progress: Progress, callback: () => Promise<R>, options?: { noWait?: boolean }): Promise<R> {if (options?.noWait)return await callback();diff --git a/packages/playwright-core/src/server/agent/pageAgent.ts b/packages/playwright-core/src/server/agent/pageAgent.tsindex ff573aa9e7..2578bea30a 100644--- a/packages/playwright-core/src/server/agent/pageAgent.ts+++ b/packages/playwright-core/src/server/agent/pageAgent.ts@@ -33,7 +33,8 @@ import type { Progress } from '../progress';export type CallParams = {cacheKey?: string;maxTokens?: number;- maxTurns?: number;+ maxActions?: number;+ maxActionRetries?: number;};export async function pageAgentPerform(progress: Progress, context: Context, userTask: string, callParams: CallParams) {@@ -105,9 +106,11 @@ async function runLoop(progress: Progress, context: Context, toolDefinitions: Toapi: context.agentParams.api as any,apiEndpoint: context.agentParams.apiEndpoint,apiKey: context.agentParams.apiKey,+ apiTimeout: context.agentParams.apiTimeout ?? 0,model: context.agentParams.model,- maxTurns: params.maxTurns ?? context.agentParams.maxTurns,- maxTokens: params.maxTokens ?? context.agentParams.maxTokens,+ maxTokens: params.maxTokens ?? context.maxTokensRemaining(),+ maxToolCalls: params.maxActions ?? context.agentParams.maxActions ?? 10,+ maxToolCallRetries: params.maxActionRetries ?? context.agentParams.maxActionRetries ?? 3,summarize: true,debug,callTool,@@ -136,8 +139,8 @@ async function runLoop(progress: Progress, context: Context, toolDefinitions: Totask.push(full);task.push('');- await loop.run(task.join('\n'), { signal: progress.signal });-+ const { error, usage } = await loop.run(task.join('\n'), { signal: progress.signal });+ context.consumeTokens(usage.input + usage.output);if (context.agentParams.apiCacheFile) {const apiCacheAfter = { ...apiCacheBefore, ...loop.cache() };const sortedCache = Object.fromEntries(Object.entries(apiCacheAfter).sort(([a], [b]) => a.localeCompare(b)));@@ -148,6 +151,9 @@ async function runLoop(progress: Progress, context: Context, toolDefinitions: To}}+ if (error)+ throw new Error(`Agentic loop failed: ${error}`);+return { result: resultSchema ? reportedResult() : undefined };}diff --git a/packages/playwright-core/types/types.d.ts b/packages/playwright-core/types/types.d.tsindex a9d3c3daff..51dd92fb29 100644--- a/packages/playwright-core/types/types.d.ts+++ b/packages/playwright-core/types/types.d.ts@@ -2110,12 +2110,26 @@ export interface Page {cacheOutFile?: string;};- maxTokens?: number;-/**- * Maximum number of agentic turns to take per call. Defaults to 10.+ * Limits to use for the agentic loop.*/- maxTurns?: number;+ limits?: {+ /**+ * Maximum number of tokens to consume. The agentic loop will stop after input + output tokens exceed this value.+ * Defaults to unlimited.+ */+ maxTokens?: number;++ /**+ * Maximum number of agentic actions to generate, defaults to 10.+ */+ maxActions?: number;++ /**+ * Maximum number retries per action, defaults to 3.+ */+ maxActionRetries?: number;+ };provider?: {/**@@ -2133,6 +2147,11 @@ export interface Page {*/apiKey: string;+ /**+ * Amount of time to wait for the provider to respond to each request.+ */+ apiTimeout?: number;+/*** Model identifier within the provider. Required in non-cache mode.*/@@ -5402,15 +5421,21 @@ export interface PageAgent {cacheKey?: string;/**- * Maximum number of tokens to consume. The agentic loop will stop after input + output tokens exceed this value.- * Defaults to context-wide value specified in `agent` property.+ * Maximum number of retries when generating each action, defaults to context-wide value specified in `agent`+ * property.*/- maxTokens?: number;+ maxActionRetries?: number;++ /**+ * Maximum number of agentic actions to generate, defaults to context-wide value specified in `agent` property.+ */+ maxActions?: number;/**- * Maximum number of agentic turns during this call, defaults to context-wide value specified in `agent` property.+ * Maximum number of tokens to consume. The agentic loop will stop after input + output tokens exceed this value.+ * Defaults to context-wide value specified in `agent` property.*/- maxTurns?: number;+ maxTokens?: number;/*** Request timeout in milliseconds. Defaults to action timeout. Pass `0` to disable timeout.@@ -5438,15 +5463,21 @@ export interface PageAgent {cacheKey?: string;/**- * Maximum number of tokens to consume. The agentic loop will stop after input + output tokens exceed this value.- * Defaults to context-wide value specified in `agent` property.+ * Maximum number of retries when generating each action, defaults to context-wide value specified in `agent`+ * property.*/- maxTokens?: number;+ maxActionRetries?: number;++ /**+ * Maximum number of agentic actions to generate, defaults to context-wide value specified in `agent` property.+ */+ maxActions?: number;/**- * Maximum number of agentic turns during this call, defaults to context-wide value specified in `agent` property.+ * Maximum number of tokens to consume. The agentic loop will stop after input + output tokens exceed this value.+ * Defaults to context-wide value specified in `agent` property.*/- maxTurns?: number;+ maxTokens?: number;/*** Request timeout in milliseconds. Defaults to action timeout. Pass `0` to disable timeout.diff --git a/packages/playwright/src/index.ts b/packages/playwright/src/index.tsindex 1a52a43897..82b978fb8c 100644--- a/packages/playwright/src/index.ts+++ b/packages/playwright/src/index.ts@@ -462,12 +462,7 @@ const playwrightFixtures: Fixtures<TestFixtures, WorkerFixtures> = ({const cacheFile = testInfoImpl.config.runAgents === 'all' ? undefined : await testInfoImpl._cloneStorage(resolvedCacheFile);const cacheOutFile = path.join(testInfoImpl.artifactsDir(), 'agent-cache-' + createGuid() + '.json');- const provider = agentOptions?.api && testInfo.config.runAgents !== 'none' ? {- api: agentOptions.api as any,- apiEndpoint: agentOptions.apiEndpoint,- apiKey: agentOptions.apiKey,- model: agentOptions.model,- } : undefined;+ const provider = agentOptions?.provider && testInfo.config.runAgents !== 'none' ? agentOptions.provider : undefined;if (provider)testInfo.setTimeout(0);@@ -479,9 +474,9 @@ const playwrightFixtures: Fixtures<TestFixtures, WorkerFixtures> = ({const agent = await page.agent({provider,cache,- maxTokens: agentOptions?.maxTokens,- maxTurns: agentOptions?.maxTurns,+ limits: agentOptions?.limits,secrets: agentOptions?.secrets,+ systemPrompt: agentOptions?.systemPrompt,});await use(agent);diff --git a/packages/playwright/types/test.d.ts b/packages/playwright/types/test.d.tsindex 16b81c92f0..4119c6d1b1 100644--- a/packages/playwright/types/test.d.ts+++ b/packages/playwright/types/test.d.ts@@ -6955,15 +6955,22 @@ export type ScreenshotMode = 'off' | 'on' | 'only-on-failure' | 'on-first-failurexport type TraceMode = 'off' | 'on' | 'retain-on-failure' | 'on-first-retry' | 'on-all-retries' | 'retain-on-first-failure';export type VideoMode = 'off' | 'on' | 'retain-on-failure' | 'on-first-retry';export type AgentOptions = {- api: string;- apiKey: string;- apiEndpoint?: string;- model: string;+ provider?: {+ api: 'openai' | 'openai-compatible' | 'anthropic' | 'google';+ apiEndpoint?: string;+ apiKey: string;+ apiTimeout?: number;+ model: string;+ },+ limits?: {+ maxTokens?: number;+ maxActions?: number;+ maxActionRetries?: number;+ };cachePathTemplate?: string;- maxTurns?: number;- maxTokens?: number;runAgents?: 'all' | 'missing' | 'none';secrets?: { [key: string]: string };+ systemPrompt?: string;};/**diff --git a/packages/protocol/src/channels.d.ts b/packages/protocol/src/channels.d.tsindex 38e0009d0d..a76dce8a90 100644--- a/packages/protocol/src/channels.d.ts+++ b/packages/protocol/src/channels.d.ts@@ -2586,10 +2586,12 @@ export type PageAgentParams = {api?: string,apiKey?: string,apiEndpoint?: string,+ apiTimeout?: number,apiCacheFile?: string,cacheFile?: string,cacheOutFile?: string,- maxTurns?: number,+ maxActions?: number,+ maxActionRetries?: number,maxTokens?: number,model?: string,secrets?: NameValue[],@@ -2599,10 +2601,12 @@ export type PageAgentOptions = {api?: string,apiKey?: string,apiEndpoint?: string,+ apiTimeout?: number,apiCacheFile?: string,cacheFile?: string,cacheOutFile?: string,- maxTurns?: number,+ maxActions?: number,+ maxActionRetries?: number,maxTokens?: number,model?: string,secrets?: NameValue[],@@ -5115,13 +5119,15 @@ export type PageAgentTurnEvent = {};export type PageAgentPerformParams = {task: string,- maxTurns?: number,+ maxActions?: number,+ maxActionRetries?: number,maxTokens?: number,cacheKey?: string,timeout?: number,};export type PageAgentPerformOptions = {- maxTurns?: number,+ maxActions?: number,+ maxActionRetries?: number,maxTokens?: number,cacheKey?: string,timeout?: number,@@ -5131,13 +5137,15 @@ export type PageAgentPerformResult = {};export type PageAgentExpectParams = {expectation: string,- maxTurns?: number,+ maxActions?: number,+ maxActionRetries?: number,maxTokens?: number,cacheKey?: string,timeout?: number,};export type PageAgentExpectOptions = {- maxTurns?: number,+ maxActions?: number,+ maxActionRetries?: number,maxTokens?: number,cacheKey?: string,timeout?: number,@@ -5148,13 +5156,15 @@ export type PageAgentExpectResult = {export type PageAgentExtractParams = {query: string,schema: any,- maxTurns?: number,+ maxActions?: number,+ maxActionRetries?: number,maxTokens?: number,cacheKey?: string,timeout?: number,};export type PageAgentExtractOptions = {- maxTurns?: number,+ maxActions?: number,+ maxActionRetries?: number,maxTokens?: number,cacheKey?: string,timeout?: number,diff --git a/packages/protocol/src/protocol.yml b/packages/protocol/src/protocol.ymlindex edce8f286f..d4af228ada 100644--- a/packages/protocol/src/protocol.yml+++ b/packages/protocol/src/protocol.yml@@ -2019,10 +2019,12 @@ Page:api: string?apiKey: string?apiEndpoint: string?+ apiTimeout: int?apiCacheFile: string?cacheFile: string?cacheOutFile: string?- maxTurns: int?+ maxActions: int?+ maxActionRetries: int?maxTokens: int?model: string?secrets:@@ -4374,7 +4376,8 @@ PageAgent:PageAgentOptions:type: mixinproperties:- maxTurns: int?+ maxActions: int?+ maxActionRetries: int?maxTokens: int?cacheKey: string?timeout: int?diff --git a/tests/library/__llm_cache__/library-agent-limits-should-respect-max-actions-limit.json b/tests/library/__llm_cache__/library-agent-limits-should-respect-max-actions-limit.jsonnew file mode 100644index 0000000000..0689191595--- /dev/null+++ b/tests/library/__llm_cache__/library-agent-limits-should-respect-max-actions-limit.json@@ -0,0 +1,114 @@+{+ "02fe9016aefaf25bffe8ae4a623014e933282db1": {+ "result": {+ "role": "assistant",+ "content": [+ {+ "type": "text",+ "text": "Now clicking the second time."+ },+ {+ "type": "tool_call",+ "name": "browser_click",+ "arguments": {+ "element": "Submit button",+ "ref": "e2",+ "_is_done": false+ },+ "id": "toolu_01PVA67E6ey7gspaUHe6vKUb"+ }+ ],+ "stopReason": {+ "code": "ok"+ }+ },+ "usage": {+ "input": 3044,+ "output": 98+ }+ },+ "9406488d923a063c065ff21e2a4f78c47cd932aa": {+ "result": {+ "role": "assistant",+ "content": [+ {+ "type": "text",+ "text": "Now clicking the third time."+ },+ {+ "type": "tool_call",+ "name": "browser_click",+ "arguments": {+ "element": "Submit button",+ "ref": "e2",+ "_is_done": false+ },+ "id": "toolu_01HreVmfgh1urgtDcxbkDbsN"+ }+ ],+ "stopReason": {+ "code": "ok"+ }+ },+ "usage": {+ "input": 3117,+ "output": 98+ }+ },+ "a8a2db95880cad9f7a0ce1d3e409e2d273685eba": {+ "result": {+ "role": "assistant",+ "content": [+ {+ "type": "text",+ "text": "I'll click the submit button 5 times as requested. Let me start by clicking it the first time."+ },+ {+ "type": "tool_call",+ "name": "browser_click",+ "arguments": {+ "element": "Submit button",+ "ref": "e2",+ "_is_done": false+ },+ "id": "toolu_01Q2Bk335vL1wsswZGCsuaTe"+ }+ ],+ "stopReason": {+ "code": "ok"+ }+ },+ "usage": {+ "input": 2897,+ "output": 115+ }+ },+ "efe9f77d726c2eff73d5c51d2cbc30c3171d6294": {+ "result": {+ "role": "assistant",+ "content": [+ {+ "type": "text",+ "text": "Now clicking the fourth time."+ },+ {+ "type": "tool_call",+ "name": "browser_click",+ "arguments": {+ "element": "Submit button",+ "ref": "e2",+ "_is_done": false+ },+ "id": "toolu_01B29QmP39aFMi2uHevcZKU1"+ }+ ],+ "stopReason": {+ "code": "ok"+ }+ },+ "usage": {+ "input": 3187,+ "output": 98+ }+ }+}\ No newline at end of filediff --git a/tests/library/__llm_cache__/library-agent-perform-click-a-button.json b/tests/library/__llm_cache__/library-agent-perform-click-a-button.jsonindex 26f5b7cdc4..83a2310d62 100644--- a/tests/library/__llm_cache__/library-agent-perform-click-a-button.json+++ b/tests/library/__llm_cache__/library-agent-perform-click-a-button.json@@ -1,5 +1,5 @@{- "0706eb777330056e04cc3abb4715ed3fa1c9d4d5": {+ "8c5697c11562d27e025c16eaba3a3d333198bb56": {"result": {"role": "assistant","content": [@@ -15,9 +15,12 @@"ref": "e2","_is_done": true},- "id": "toolu_01W54FxjcWYV856RHP3V2xdj"+ "id": "toolu_019UXybai8M1mCTeCBkEtzJc"}- ]+ ],+ "stopReason": {+ "code": "ok"+ }},"usage": {"input": 2893,diff --git a/tests/library/__llm_cache__/library-agent-perform-expect-value.json b/tests/library/__llm_cache__/library-agent-perform-expect-value.jsonindex c4cd9ce537..8081774369 100644--- a/tests/library/__llm_cache__/library-agent-perform-expect-value.json+++ b/tests/library/__llm_cache__/library-agent-perform-expect-value.json@@ -1,5 +1,5 @@{- "77c0828e4bffb298a3850504550ffcac23e6692e": {+ "46258723cbb8dc1fbc9c390304cd506187897079": {"result": {"role": "assistant","content": [@@ -16,22 +16,25 @@"text": "bogus","_is_done": false},- "id": "toolu_01FfLGgBK1CHZ8e54gQMukA6"+ "id": "toolu_01V8GEC6fjiTt98e3k88ENyV"}- ]+ ],+ "stopReason": {+ "code": "ok"+ }},"usage": {"input": 2926,"output": 149}},- "ca3ff77a849d4eacf88d299dc8727d63724a68aa": {+ "5429c27bfbcae1d531a13010d3a5d4807251a7c4": {"result": {"role": "assistant","content": [{"type": "text",- "text": "Perfect! I have successfully completed all three parts of the task:\n\n1. ✅ **Entered \"bogus\" into the email field** - The text has been typed into the textbox\n2. ✅ **Verified the value is \"bogus\"** - The page snapshot shows the textbox contains \"bogus\"\n3. ✅ **Confirmed error message is displayed** - The error message \"Error: Invalid email address\" is now visible on the page\n\nThe task is complete."+ "text": "Perfect! I've successfully completed all three parts of the task:\n\n1. ✅ **Entered \"bogus\" into the email field** - The text has been typed into the Email Address textbox\n2. ✅ **Verified the value is \"bogus\"** - The snapshot confirms the textbox contains \"bogus\"\n3. ✅ **Confirmed error message is displayed** - The error message \"Error: Invalid email address\" is now visible on the page\n\nThe task has been completed successfully. The invalid email \"bogus\" triggered the expected validation error message."},{"type": "tool_call",@@ -39,13 +42,16 @@"arguments": {"_is_done": true},- "id": "toolu_018wjD4H2S5QHQ3rpE2sR7bM"+ "id": "toolu_01WuNDf5vv1QM1SdgkAKnpVF"}- ]+ ],+ "stopReason": {+ "code": "ok"+ }},"usage": {"input": 3138,- "output": 167+ "output": 184}}}\ No newline at end of filediff --git a/tests/library/__llm_cache__/library-agent-perform-extract-task.json b/tests/library/__llm_cache__/library-agent-perform-extract-task.jsonindex 74f9d15fbb..46a3c383fc 100644--- a/tests/library/__llm_cache__/library-agent-perform-extract-task.json+++ b/tests/library/__llm_cache__/library-agent-perform-extract-task.json@@ -1,5 +1,5 @@{- "b5f1ee58f1051afb059b3b33e8be41576a1324a4": {+ "d02ae2240fac126bacb74133b970880eaaf3f915": {"result": {"role": "assistant","content": [@@ -23,9 +23,12 @@],"_is_done": true},- "id": "toolu_01WvevmDTBxmKg79Ji6vQvpH"+ "id": "toolu_01JtUAyrJqJSRwBsS7pn9fRY"}- ]+ ],+ "stopReason": {+ "code": "ok"+ }},"usage": {"input": 888,diff --git a/tests/library/__llm_cache__/library-agent-perform-perform-history.json b/tests/library/__llm_cache__/library-agent-perform-perform-history.jsonindex 00a85f8150..13be1898fc 100644--- a/tests/library/__llm_cache__/library-agent-perform-perform-history.json+++ b/tests/library/__llm_cache__/library-agent-perform-perform-history.json@@ -1,5 +1,5 @@{- "4f9987bc22c6035501274532494c185d1270aba2": {+ "7416c1b94dbb8a1f7405a276811783738c295295": {"result": {"role": "assistant","content": [@@ -15,16 +15,19 @@"ref": "e3","_is_done": true},- "id": "toolu_013sypFp7tiRCLaR3i3oWrei"+ "id": "toolu_017c7pdkXdqSjQMiaF7UKwa8"}- ]+ ],+ "stopReason": {+ "code": "ok"+ }},"usage": {"input": 2946,"output": 102}},- "79e2d58996075c4ebe8eb9e557bd0cdb8b04211a": {+ "96cdcffad5cf3bf3d064acdeb05fbb4968f27c02": {"result": {"role": "assistant","content": [@@ -40,9 +43,12 @@"ref": "e3","_is_done": true},- "id": "toolu_01E6xBUYzQEdRrkWoobbUbFq"+ "id": "toolu_01E1epU97Ub6RDTk4PeFuKke"}- ]+ ],+ "stopReason": {+ "code": "ok"+ }},"usage": {"input": 2933,diff --git a/tests/library/__llm_cache__/library-agent-perform-perform-run-timeout.json b/tests/library/__llm_cache__/library-agent-perform-perform-run-timeout.jsonindex 836cd0bc34..01648843e9 100644--- a/tests/library/__llm_cache__/library-agent-perform-perform-run-timeout.json+++ b/tests/library/__llm_cache__/library-agent-perform-perform-run-timeout.json@@ -1,5 +1,5 @@{- "c2f54e956e3131255f03c5484a637b580f7d695f": {+ "c3664a96a82cba8678b7ca68973bb25c1d1207a4": {"result": {"role": "assistant","content": [@@ -15,9 +15,12 @@"ref": "e3","_is_done": true},- "id": "toolu_01GipHDfLSETD8VyJt6D6ng5"+ "id": "toolu_01FP2NCSpx8mnFeg16iX9gKi"}- ]+ ],+ "stopReason": {+ "code": "ok"+ }},"usage": {"input": 2919,diff --git a/tests/library/__llm_cache__/library-agent-perform-retrieve-a-secret.json b/tests/library/__llm_cache__/library-agent-perform-retrieve-a-secret.jsonindex f60e4427dd..3c0ad14395 100644--- a/tests/library/__llm_cache__/library-agent-perform-retrieve-a-secret.json+++ b/tests/library/__llm_cache__/library-agent-perform-retrieve-a-secret.json@@ -1,5 +1,5 @@{- "c1d41afc231b64caf136132dd841ca8f45876674": {+ "8bd748693a579b9f17e2fb127fcb3af9706934d4": {"result": {"role": "assistant","content": [@@ -16,9 +16,12 @@"text": "%x-secret-email%","_is_done": true},- "id": "toolu_018PD6jsENphY4ERa1XRF4ZN"+ "id": "toolu_01YYGZnA96oGJMknVBJC4Cfj"}- ]+ ],+ "stopReason": {+ "code": "ok"+ }},"usage": {"input": 2901,diff --git a/tests/library/agent-helpers.ts b/tests/library/agent-helpers.tsindex b6d99de6a2..b145b1882d 100644--- a/tests/library/agent-helpers.ts+++ b/tests/library/agent-helpers.ts@@ -32,7 +32,9 @@ export async function setCacheObject(object: any) {await fs.promises.writeFile(cacheFile(), JSON.stringify(object, null, 2), 'utf8');}-export async function generateAgent(context: BrowserContext, options: { secrets?: Record<string, string> } = {}) {+type AgentOptions = Parameters<Page['agent']>[0];++export async function generateAgent(context: BrowserContext, options: AgentOptions = {}) {const apiCacheFile = path.join(__dirname, '__llm_cache__', sanitizeFileName(test.info().titlePath.join(' ')) + '.json');const page = await context.newPage();diff --git a/tests/library/agent-limits.spec.ts b/tests/library/agent-limits.spec.tsnew file mode 100644index 0000000000..07d3f30149--- /dev/null+++ b/tests/library/agent-limits.spec.ts@@ -0,0 +1,54 @@+/**+ * Copyright (c) Microsoft Corporation.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++import { browserTest as test, expect } from '../config/browserTest';+import { generateAgent } from './agent-helpers';++test('should respect total max tokens limit', async ({ context }) => {+ const { page, agent } = await generateAgent(context, {+ limits: {+ maxTokens: 123,+ },+ });+ await page.setContent(`+ <button>Submit</button>+ `);+ const e = await agent.perform('Click submit button').catch(e => e);+ expect(e.message.toLowerCase()).toContain('budget');+ expect(e.message).toContain('123');+});++test('should respect call max tokens limit', async ({ context }) => {+ const { page, agent } = await generateAgent(context);+ await page.setContent(`+ <button>Submit</button>+ `);+ const e = await agent.perform('Click submit button', { maxTokens: 123 }).catch(e => e);+ expect(e.message.toLowerCase()).toContain('budget');+ expect(e.message).toContain('123');+});++test('should respect max actions limit', async ({ context }) => {+ const { page, agent } = await generateAgent(context);+ let clicked = 0;+ await page.exposeFunction('clicked', () => ++clicked);+ await page.setContent(`+ <button onclick="clicked()">Submit</button>+ `);+ const e = await agent.perform('Click the submit button 5 times', { maxActions: 3 }).catch(e => e);+ expect(e.message).toContain('Failed to perform step, max tool calls (3) reached');+ expect(clicked).toBe(3);+});diff --git a/utils/generate_types/overrides-test.d.ts b/utils/generate_types/overrides-test.d.tsindex bf47bdab5a..3a0eb2eb2d 100644--- a/utils/generate_types/overrides-test.d.ts+++ b/utils/generate_types/overrides-test.d.ts@@ -270,15 +270,22 @@ export type ScreenshotMode = 'off' | 'on' | 'only-on-failure' | 'on-first-failurexport type TraceMode = 'off' | 'on' | 'retain-on-failure' | 'on-first-retry' | 'on-all-retries' | 'retain-on-first-failure';export type VideoMode = 'off' | 'on' | 'retain-on-failure' | 'on-first-retry';export type AgentOptions = {- api: string;- apiKey: string;- apiEndpoint?: string;- model: string;+ provider?: {+ api: 'openai' | 'openai-compatible' | 'anthropic' | 'google';+ apiEndpoint?: string;+ apiKey: string;+ apiTimeout?: number;+ model: string;+ },+ limits?: {+ maxTokens?: number;+ maxActions?: number;+ maxActionRetries?: number;+ };cachePathTemplate?: string;- maxTurns?: number;- maxTokens?: number;runAgents?: 'all' | 'missing' | 'none';secrets?: { [key: string]: string };+ systemPrompt?: string;};export interface PlaywrightTestOptions {