Playwright is a framework for Web Testing and Automation. It allows testing Chromium, Firefox and WebKit with a single API.
TypeScript automationchromechromiume2e-testingelectronend-to-end-testingfirefoxjavascriptplaywrighttesttest-automationtestingtesting-toolswebwebkit
chore: report perform usage, cap tokens and turns (#38534)
Pavel Feldman22468c245e27
diff --git a/docs/src/api/class-browser.md b/docs/src/api/class-browser.mdindex 878bcda040..e61bf6930c 100644--- a/docs/src/api/class-browser.md+++ b/docs/src/api/class-browser.md@@ -267,7 +267,7 @@ await browser.CloseAsync();### option: Browser.newContext.storageStatePath = %%-csharp-java-context-option-storage-state-path-%%* since: v1.9-### option: Browser.newContext.agent = %%-js-context-option-agent-%%+### option: Browser.newContext.agent = %%-context-option-agent-%%* since: v1.58## async method: Browser.newPagediff --git a/docs/src/api/class-page.md b/docs/src/api/class-page.mdindex 7d37241513..76e9d6b1f2 100644--- a/docs/src/api/class-page.md+++ b/docs/src/api/class-page.md@@ -2026,8 +2026,12 @@ Callback function which will be called in Playwright's context.## async method: Page.extract* since: v1.58-* langs: js-- returns: <[any]>+- returns: <[Object]>+ - `result` <[any]>+ - `usage` <[Object]>+ - `turns` <[int]>+ - `inputTokens` <[int]>+ - `outputTokens` <[int]>Extract information from the page using the agentic loop, return it in a given Zod format.@@ -2050,11 +2054,18 @@ Task to perform using agentic loop.* since: v1.58- `schema` <[z.ZodSchema]>+### option: Page.extract.maxTokens+* since: v1.58+- `maxTokens` <[int]>++Maximum number of tokens to consume. The agentic loop will stop after input + output tokens exceed this value.+Defaults to context-wide value specified in `agent` property.+### option: Page.extract.maxTurns* since: v1.58- `maxTurns` <[int]>-Maximum number of agentic steps to take while extracting the information.+Maximum number of agentic turns during this call, defaults to context-wide value specified in `agent` property.## async method: Page.fill* since: v1.8@@ -3031,7 +3042,11 @@ Whether or not to embed the document outline into the PDF. Defaults to `false`.## async method: Page.perform* since: v1.58-* langs: js+- returns: <[Object]>+ - `usage` <[Object]>+ - `turns` <[int]>+ - `inputTokens` <[int]>+ - `outputTokens` <[int]>Perform action using agentic loop.@@ -3054,11 +3069,18 @@ Task to perform using agentic loop.All the agentic actions are converted to the Playwright calls and are cached.By default, they are cached globally with the `task` as a key. This option allows controlling the cache key explicitly.+### option: Page.perform.maxTokens+* since: v1.58+- `maxTokens` <[int]>++Maximum number of tokens to consume. The agentic loop will stop after input + output tokens exceed this value.+Defaults to context-wide value specified in `agent` property.+### option: Page.perform.maxTurns* since: v1.58- `maxTurns` <[int]>-Maximum number of agentic steps to take while performing this action.+Maximum number of agentic turns during this call, defaults to context-wide value specified in `agent` property.## async method: Page.pressdiff --git a/docs/src/api/params.md b/docs/src/api/params.mdindex 9220e12713..babbdaf3b0 100644--- a/docs/src/api/params.md+++ b/docs/src/api/params.md@@ -370,14 +370,15 @@ It makes the execution of the tests non-deterministic.Emulates consistent window screen size available inside web page via `window.screen`. Is only used when the[`option: viewport`] is set.-## js-context-option-agent-* langs: js+## context-option-agent- `agent` <[Object]>- `provider` <[string]> LLM provider to use.- `model` <[string]> Model identifier within provider.- `cacheFile` ?<[string]> Cache file to use/generate code for performed actions into. Cache is not used if not specified (default).- `cacheMode` ?<['force'|'ignore'|'auto']> Cache control, defaults to 'auto'.- `secrets` ?<[Object]<[string], [string]>> Secrets to hide from the LLM.+ - `maxTurns` ?<[int]> Maximum number of agentic turns to take per call. Defaults to 10.+ - `maxTokens` ?<[int]> Maximum number of tokens to consume per call. The agentic loop will stop after input + output tokens exceed this value. Defaults on unlimited.Agent settings for [`method: Page.perform`] and [`method: Page.extract`].diff --git a/docs/src/test-api/class-testoptions.md b/docs/src/test-api/class-testoptions.mdindex f00a272332..98cd67238e 100644--- a/docs/src/test-api/class-testoptions.md+++ b/docs/src/test-api/class-testoptions.md@@ -46,7 +46,7 @@ export default defineConfig({});```-## property: TestOptions.agent = %%-js-context-option-agent-%%+## property: TestOptions.agent = %%-context-option-agent-%%* since: v1.58diff --git a/packages/playwright-client/types/types.d.ts b/packages/playwright-client/types/types.d.tsindex 77196a4080..d9917c1a1d 100644--- a/packages/playwright-client/types/types.d.ts+++ b/packages/playwright-client/types/types.d.ts@@ -3839,10 +3839,24 @@ export interface Page {key?: string;/**- * Maximum number of agentic steps to take while performing this action.+ * Maximum number of tokens to consume. The agentic loop will stop after input + output tokens exceed this value.+ * Defaults to context-wide value specified in `agent` property.+ */+ maxTokens?: number;++ /**+ * Maximum number of agentic turns during this call, defaults to context-wide value specified in `agent` property.*/maxTurns?: number;- }): Promise<void>;+ }): Promise<{+ usage: {+ turns: number;++ inputTokens: number;++ outputTokens: number;+ };+ }>;/*** **NOTE** Use locator-based [locator.press(key[, options])](https://playwright.dev/docs/api/class-locator#locator-press)@@ -22110,6 +22124,17 @@ export interface BrowserContextOptions {* Secrets to hide from the LLM.*/secrets?: { [key: string]: string; };++ /**+ * Maximum number of agentic turns to take per call. Defaults to 10.+ */+ maxTurns?: number;++ /**+ * Maximum number of tokens to consume per call. The agentic loop will stop after input + output tokens exceed this+ * value. Defaults on unlimited.+ */+ maxTokens?: number;};/**diff --git a/packages/playwright-core/src/client/page.ts b/packages/playwright-core/src/client/page.tsindex c7c3dd1a86..4f36c15869 100644--- a/packages/playwright-core/src/client/page.ts+++ b/packages/playwright-core/src/client/page.ts@@ -846,13 +846,14 @@ export class Page extends ChannelOwner<channels.PageChannel> implements api.Pagereturn result.pdf;}- async perform(task: string, options: { key?: string, maxTurns?: number } = {}): Promise<void> {- await this._channel.perform({ task, ...options });+ async perform(task: string, options: { key?: string, maxTokens?: number, maxTurns?: number } = {}) {+ const result = await this._channel.perform({ task, ...options });+ return { usage: { ...result } };}- async extract<Schema extends z.ZodTypeAny>(query: string, schema: Schema, options: { maxTurns?: number } = {}): Promise<z.infer<Schema>> {- const { result } = await this._channel.extract({ query, schema: this._platform.zodToJsonSchema(schema), ...options });- return result;+ async extract<Schema extends z.ZodTypeAny>(query: string, schema: Schema, options: { maxTokens?: number, maxTurns?: number } = {}): Promise<z.infer<Schema>> {+ const { result, ...usage } = await this._channel.extract({ query, schema: this._platform.zodToJsonSchema(schema), ...options });+ return { result, usage };}async _snapshotForAI(options: TimeoutOptions & { track?: string } = {}): Promise<{ full: string, incremental?: string }> {diff --git a/packages/playwright-core/src/protocol/validator.ts b/packages/playwright-core/src/protocol/validator.tsindex d10e8f420e..2cf6e69c39 100644--- a/packages/playwright-core/src/protocol/validator.ts+++ b/packages/playwright-core/src/protocol/validator.ts@@ -608,6 +608,8 @@ scheme.BrowserTypeLaunchPersistentContextParams = tObject({cacheFile: tOptional(tString),cacheMode: tOptional(tEnum(['ignore', 'force', 'auto'])),secrets: tOptional(tArray(tType('NameValue'))),+ maxTurns: tOptional(tInt),+ maxTokens: tOptional(tInt),})),userDataDir: tString,slowMo: tOptional(tFloat),@@ -707,6 +709,8 @@ scheme.BrowserNewContextParams = tObject({cacheFile: tOptional(tString),cacheMode: tOptional(tEnum(['ignore', 'force', 'auto'])),secrets: tOptional(tArray(tType('NameValue'))),+ maxTurns: tOptional(tInt),+ maxTokens: tOptional(tInt),})),proxy: tOptional(tObject({server: tString,@@ -785,6 +789,8 @@ scheme.BrowserNewContextForReuseParams = tObject({cacheFile: tOptional(tString),cacheMode: tOptional(tEnum(['ignore', 'force', 'auto'])),secrets: tOptional(tArray(tType('NameValue'))),+ maxTurns: tOptional(tInt),+ maxTokens: tOptional(tInt),})),proxy: tOptional(tObject({server: tString,@@ -908,6 +914,8 @@ scheme.BrowserContextInitializer = tObject({cacheFile: tOptional(tString),cacheMode: tOptional(tEnum(['ignore', 'force', 'auto'])),secrets: tOptional(tArray(tType('NameValue'))),+ maxTurns: tOptional(tInt),+ maxTokens: tOptional(tInt),})),}),});@@ -1514,8 +1522,13 @@ scheme.PagePerformParams = tObject({task: tString,key: tOptional(tString),maxTurns: tOptional(tInt),+ maxTokens: tOptional(tInt),+});+scheme.PagePerformResult = tObject({+ turns: tInt,+ inputTokens: tInt,+ outputTokens: tInt,});-scheme.PagePerformResult = tOptional(tObject({}));scheme.PageExtractParams = tObject({query: tString,schema: tAny,@@ -1523,6 +1536,9 @@ scheme.PageExtractParams = tObject({});scheme.PageExtractResult = tObject({result: tAny,+ turns: tInt,+ inputTokens: tInt,+ outputTokens: tInt,});scheme.FrameInitializer = tObject({url: tString,@@ -2818,6 +2834,8 @@ scheme.AndroidDeviceLaunchBrowserParams = tObject({cacheFile: tOptional(tString),cacheMode: tOptional(tEnum(['ignore', 'force', 'auto'])),secrets: tOptional(tArray(tType('NameValue'))),+ maxTurns: tOptional(tInt),+ maxTokens: tOptional(tInt),})),pkg: tOptional(tString),args: tOptional(tArray(tString)),diff --git a/packages/playwright-core/src/server/agent/agent.ts b/packages/playwright-core/src/server/agent/agent.tsindex 05ca068fd2..507d630e4c 100644--- a/packages/playwright-core/src/server/agent/agent.ts+++ b/packages/playwright-core/src/server/agent/agent.ts@@ -28,17 +28,27 @@ import type { Page } from '../page';import type * as loopTypes from '@lowire/loop';import type * as actions from './actions';-export async function pagePerform(progress: Progress, page: Page, options: channels.PagePerformParams): Promise<void> {+type Usage = {+ turns: number,+ inputTokens: number,+ outputTokens: number,+};++export async function pagePerform(progress: Progress, page: Page, options: channels.PagePerformParams): Promise<Usage> {const context = new Context(progress, page);if (await cachedPerform(context, options))- return;+ return { turns: 0, inputTokens: 0, outputTokens: 0 };- await perform(context, options.task, undefined, options);+ const { usage } = await perform(context, options.task, undefined, options);await updateCache(context, options);+ return usage;}-export async function pageExtract(progress: Progress, page: Page, options: channels.PageExtractParams) {+export async function pageExtract(progress: Progress, page: Page, options: channels.PageExtractParams): Promise<{+ result: any,+ usage: Usage+}> {const context = new Context(progress, page);const task = `### Instructions@@ -46,10 +56,14 @@ Extract the following information from the page. Do not perform any actions, jus### Query${options.query}`;- return await perform(context, task, options.schema, options);+ const { result, usage } = await perform(context, task, options.schema, options);+ return { result, usage };}-async function perform(context: Context, userTask: string, resultSchema: loopTypes.Schema | undefined, options: { maxTurns?: number } = {}): Promise<any> {+async function perform(context: Context, userTask: string, resultSchema: loopTypes.Schema | undefined, options: { maxTurns?: number, maxTokens?: number } = {}): Promise<{+ result: any,+ usage: Usage+}> {const { progress, page } = context;const browserContext = page.browserContext;if (!browserContext._options.agent)@@ -58,13 +72,17 @@ async function perform(context: Context, userTask: string, resultSchema: loopTypconst { full } = await page.snapshotForAI(progress);const { tools, callTool } = toolsForLoop(context);+ const limits = context.limits(options);+ let turns = 0;const loop = new Loop(browserContext._options.agent.provider as any, {model: browserContext._options.agent.model,summarize: true,debug,callTool,tools,+ ...limits,beforeTurn: params => {+ ++turns;const lastReply = params.conversation.messages.findLast(m => m.role === 'assistant');const toolCall = lastReply?.content.find(c => c.type === 'tool_call');if (!resultSchema && toolCall && toolCall.arguments.thatShouldBeIt)@@ -80,8 +98,15 @@ async function perform(context: Context, userTask: string, resultSchema: loopTyp${full}`;- const { result } = await loop.run(task, { resultSchema });- return result;+ const { result, usage } = await loop.run(task, { resultSchema });+ return {+ result,+ usage: {+ turns,+ inputTokens: usage.input,+ outputTokens: usage.output,+ }+ };}type CachedActions = Record<string, {diff --git a/packages/playwright-core/src/server/agent/context.ts b/packages/playwright-core/src/server/agent/context.tsindex 42475eaae5..55cc4b25a3 100644--- a/packages/playwright-core/src/server/agent/context.ts+++ b/packages/playwright-core/src/server/agent/context.ts@@ -146,6 +146,13 @@ export class Context {}));}+ limits(options: { maxTurns?: number, maxTokens?: number } = {}): { maxTurns: number | undefined, maxTokens: number | undefined } {+ return {+ maxTurns: options.maxTurns ?? this.options?.maxTurns ?? 10,+ maxTokens: options.maxTokens ?? this.options?.maxTokens ?? undefined,+ };+ }+private _redactText(text: string): string {const secrets = this.options?.secrets;if (!secrets)diff --git a/packages/playwright-core/src/server/dispatchers/pageDispatcher.ts b/packages/playwright-core/src/server/dispatchers/pageDispatcher.tsindex 836d6bfffe..dcfc35b0ec 100644--- a/packages/playwright-core/src/server/dispatchers/pageDispatcher.ts+++ b/packages/playwright-core/src/server/dispatchers/pageDispatcher.ts@@ -322,11 +322,12 @@ export class PageDispatcher extends Dispatcher<Page, channels.PageChannel, Brows}async perform(params: channels.PagePerformParams, progress: Progress): Promise<channels.PagePerformResult> {- await pagePerform(progress, this._page, params);+ return await pagePerform(progress, this._page, params);}async extract(params: channels.PageExtractParams, progress: Progress): Promise<channels.PageExtractResult> {- return { result: await pageExtract(progress, this._page, params) };+ const { result, usage } = await pageExtract(progress, this._page, params);+ return { result, ...usage };}async requests(params: channels.PageRequestsParams, progress: Progress): Promise<channels.PageRequestsResult> {diff --git a/packages/playwright-core/types/types.d.ts b/packages/playwright-core/types/types.d.tsindex 77196a4080..d9917c1a1d 100644--- a/packages/playwright-core/types/types.d.ts+++ b/packages/playwright-core/types/types.d.ts@@ -3839,10 +3839,24 @@ export interface Page {key?: string;/**- * Maximum number of agentic steps to take while performing this action.+ * Maximum number of tokens to consume. The agentic loop will stop after input + output tokens exceed this value.+ * Defaults to context-wide value specified in `agent` property.+ */+ maxTokens?: number;++ /**+ * Maximum number of agentic turns during this call, defaults to context-wide value specified in `agent` property.*/maxTurns?: number;- }): Promise<void>;+ }): Promise<{+ usage: {+ turns: number;++ inputTokens: number;++ outputTokens: number;+ };+ }>;/*** **NOTE** Use locator-based [locator.press(key[, options])](https://playwright.dev/docs/api/class-locator#locator-press)@@ -22110,6 +22124,17 @@ export interface BrowserContextOptions {* Secrets to hide from the LLM.*/secrets?: { [key: string]: string; };++ /**+ * Maximum number of agentic turns to take per call. Defaults to 10.+ */+ maxTurns?: number;++ /**+ * Maximum number of tokens to consume per call. The agentic loop will stop after input + output tokens exceed this+ * value. Defaults on unlimited.+ */+ maxTokens?: number;};/**diff --git a/packages/protocol/src/channels.d.ts b/packages/protocol/src/channels.d.tsindex 1c74085748..3b2e83918e 100644--- a/packages/protocol/src/channels.d.ts+++ b/packages/protocol/src/channels.d.ts@@ -1014,6 +1014,8 @@ export type BrowserTypeLaunchPersistentContextParams = {cacheFile?: string,cacheMode?: 'ignore' | 'force' | 'auto',secrets?: NameValue[],+ maxTurns?: number,+ maxTokens?: number,},userDataDir: string,slowMo?: number,@@ -1103,6 +1105,8 @@ export type BrowserTypeLaunchPersistentContextOptions = {cacheFile?: string,cacheMode?: 'ignore' | 'force' | 'auto',secrets?: NameValue[],+ maxTurns?: number,+ maxTokens?: number,},slowMo?: number,};@@ -1231,6 +1235,8 @@ export type BrowserNewContextParams = {cacheFile?: string,cacheMode?: 'ignore' | 'force' | 'auto',secrets?: NameValue[],+ maxTurns?: number,+ maxTokens?: number,},proxy?: {server: string,@@ -1306,6 +1312,8 @@ export type BrowserNewContextOptions = {cacheFile?: string,cacheMode?: 'ignore' | 'force' | 'auto',secrets?: NameValue[],+ maxTurns?: number,+ maxTokens?: number,},proxy?: {server: string,@@ -1384,6 +1392,8 @@ export type BrowserNewContextForReuseParams = {cacheFile?: string,cacheMode?: 'ignore' | 'force' | 'auto',secrets?: NameValue[],+ maxTurns?: number,+ maxTokens?: number,},proxy?: {server: string,@@ -1459,6 +1469,8 @@ export type BrowserNewContextForReuseOptions = {cacheFile?: string,cacheMode?: 'ignore' | 'force' | 'auto',secrets?: NameValue[],+ maxTurns?: number,+ maxTokens?: number,},proxy?: {server: string,@@ -1601,6 +1613,8 @@ export type BrowserContextInitializer = {cacheFile?: string,cacheMode?: 'ignore' | 'force' | 'auto',secrets?: NameValue[],+ maxTurns?: number,+ maxTokens?: number,},},};@@ -2627,12 +2641,18 @@ export type PagePerformParams = {task: string,key?: string,maxTurns?: number,+ maxTokens?: number,};export type PagePerformOptions = {key?: string,maxTurns?: number,+ maxTokens?: number,+};+export type PagePerformResult = {+ turns: number,+ inputTokens: number,+ outputTokens: number,};-export type PagePerformResult = void;export type PageExtractParams = {query: string,schema: any,@@ -2643,6 +2663,9 @@ export type PageExtractOptions = {};export type PageExtractResult = {result: any,+ turns: number,+ inputTokens: number,+ outputTokens: number,};export interface PageEvents {@@ -4914,6 +4937,8 @@ export type AndroidDeviceLaunchBrowserParams = {cacheFile?: string,cacheMode?: 'ignore' | 'force' | 'auto',secrets?: NameValue[],+ maxTurns?: number,+ maxTokens?: number,},pkg?: string,args?: string[],@@ -4987,6 +5012,8 @@ export type AndroidDeviceLaunchBrowserOptions = {cacheFile?: string,cacheMode?: 'ignore' | 'force' | 'auto',secrets?: NameValue[],+ maxTurns?: number,+ maxTokens?: number,},pkg?: string,args?: string[],diff --git a/packages/protocol/src/protocol.yml b/packages/protocol/src/protocol.ymlindex 07c3dfad14..20dcdf07b0 100644--- a/packages/protocol/src/protocol.yml+++ b/packages/protocol/src/protocol.yml@@ -604,6 +604,8 @@ ContextOptions:secrets:type: array?items: NameValue+ maxTurns: int?+ maxTokens: int?LocalUtils:type: interface@@ -2030,6 +2032,11 @@ Page:task: stringkey: string?maxTurns: int?+ maxTokens: int?+ returns:+ turns: int+ inputTokens: int+ outputTokens: intextract:internal: true@@ -2039,6 +2046,9 @@ Page:maxTurns: int?returns:result: json+ turns: int+ inputTokens: int+ outputTokens: intevents: