Skip to content

Playwright is a framework for Web Testing and Automation. It allows testing Chromium, Firefox and WebKit with a single API.

Revert "chore: hide stuff" (#38917)

Simon Knott71c42408b29e
diff --git a/docs/src/api/class-page.md b/docs/src/api/class-page.md
index 13299d4646..55399a4d67 100644
--- a/docs/src/api/class-page.md
+++ b/docs/src/api/class-page.md
@@ -712,27 +712,23 @@ Raw CSS content to be injected into frame.
## async method: Page.agent
* since: v1.58
* langs: js
-* hidden
- returns: <[PageAgent]>
Initialize page agent with the llm provider and cache.
### option: Page.agent.cache
* since: v1.58
-* hidden
- `cache` <[Object]>
- `cacheFile` ?<[string]> Cache file to use/generate code for performed actions into. Cache is not used if not specified (default).
- `cacheOutFile` ?<[string]> When specified, generated entries are written into the `cacheOutFile` instead of updating the `cacheFile`.
### option: Page.agent.expect
* since: v1.58
-* hidden
- `expect` <[Object]>
- `timeout` ?<[int]> Default timeout for expect calls in milliseconds, defaults to 5000ms.
### option: Page.agent.limits
* since: v1.58
-* hidden
- `limits` <[Object]>
- `maxTokens` ?<[int]> Maximum number of tokens to consume. The agentic loop will stop after input + output tokens exceed this value. Defaults to unlimited.
- `maxActions` ?<[int]> Maximum number of agentic actions to generate, defaults to 10.
@@ -742,7 +738,6 @@ Limits to use for the agentic loop.
### option: Page.agent.provider
* since: v1.58
-* hidden
- `provider` <[Object]>
- `api` <[PageAgentAPI]<"openai"|"openai-compatible"|"anthropic"|"google">> API to use.
- `apiEndpoint` ?<[string]> Endpoint to use if different from default.
@@ -752,14 +747,12 @@ Limits to use for the agentic loop.
### option: Page.agent.secrets
* since: v1.58
-* hidden
- `secrets` ?<[Object]<[string], [string]>>
Secrets to hide from the LLM.
### option: Page.agent.systemPrompt
* since: v1.58
-* hidden
- `systemPrompt` <[string]>
System prompt for the agent's loop.
diff --git a/docs/src/api/class-pageagent.md b/docs/src/api/class-pageagent.md
index e1a9f4e520..0f04e2b04d 100644
--- a/docs/src/api/class-pageagent.md
+++ b/docs/src/api/class-pageagent.md
@@ -1,11 +1,9 @@
# class: PageAgent
* since: v1.58
* langs: js
-* hidden
## event: PageAgent.turn
* since: v1.58
-* hidden
- argument: <[Object]>
- `role` <[string]>
- `message` <[string]>
@@ -16,13 +14,11 @@
Emitted when the agent makes a turn.
## async method: PageAgent.dispose
-* hidden
* since: v1.58
Dispose this agent.
## async method: PageAgent.expect
-* hidden
* since: v1.58
Expect certain condition to be met.
@@ -35,14 +31,12 @@ await agent.expect('"0 items" to be reported');
### param: PageAgent.expect.expectation
* since: v1.58
-* hidden
- `expectation` <[string]>
Expectation to assert.
### option: PageAgent.expect.timeout
* since: v1.58
-* hidden
- `timeout` <[float]>
Expect timeout in milliseconds. Defaults to `5000`. The default value can be changed via `expect.timeout` option in the config, or by specifying the `expect` property of the [`option: Page.agent.expect`] option. Pass `0` to disable timeout.
@@ -52,7 +46,6 @@ Expect timeout in milliseconds. Defaults to `5000`. The default value can be cha
## async method: PageAgent.extract
* since: v1.58
-* hidden
- returns: <[Object]>
- `result` <[any]>
- `usage` <[Object]>
@@ -73,19 +66,16 @@ await agent.extract('List of items in the cart', z.object({
### param: PageAgent.extract.query
* since: v1.58
-* hidden
- `query` <[string]>
Task to perform using agentic loop.
### param: PageAgent.extract.schema
* since: v1.58
-* hidden
- `schema` <[z.ZodSchema]>
### option: PageAgent.extract.timeout
* since: v1.58
-* hidden
- `timeout` <[float]>
Extract timeout in milliseconds. Defaults to `5000`. The default value can be changed via `actionTimeout` option in the config, or by using the [`method: BrowserContext.setDefaultTimeout`] or
@@ -94,9 +84,9 @@ Extract timeout in milliseconds. Defaults to `5000`. The default value can be ch
### option: PageAgent.extract.-inline- = %%-page-agent-call-options-v1.58-%%
* since: v1.58
+
## async method: PageAgent.perform
* since: v1.58
-* hidden
- returns: <[Object]>
- `usage` <[Object]>
- `turns` <[int]>
@@ -113,14 +103,12 @@ await agent.perform('Click submit button');
### param: PageAgent.perform.task
* since: v1.58
-* hidden
- `task` <[string]>
Task to perform using agentic loop.
### option: PageAgent.perform.timeout
* since: v1.58
-* hidden
- `timeout` <[float]>
Perform timeout in milliseconds. Defaults to `5000`. The default value can be changed via `actionTimeout` option in the config, or by using the [`method: BrowserContext.setDefaultTimeout`] or
@@ -128,11 +116,9 @@ Perform timeout in milliseconds. Defaults to `5000`. The default value can be ch
### option: PageAgent.perform.-inline- = %%-page-agent-call-options-v1.58-%%
* since: v1.58
-* hidden
## async method: PageAgent.usage
* since: v1.58
-* hidden
- returns: <[Object]>
- `turns` <[int]>
- `inputTokens` <[int]>
diff --git a/docs/src/api/params.md b/docs/src/api/params.md
index 90c3b9e362..69257b93db 100644
--- a/docs/src/api/params.md
+++ b/docs/src/api/params.md
@@ -372,7 +372,6 @@ Emulates consistent window screen size available inside web page via `window.scr
## page-agent-cache-key
* since: v1.58
-* hidden
- `cacheKey` <[string]>
All the agentic actions are converted to the Playwright calls and are cached.
@@ -380,7 +379,6 @@ By default, they are cached globally with the `task` as a key. This option allow
## page-agent-max-tokens
* since: v1.58
-* hidden
- `maxTokens` <[int]>
Maximum number of tokens to consume. The agentic loop will stop after input + output tokens exceed this value.
@@ -388,14 +386,12 @@ Defaults to context-wide value specified in `agent` property.
## page-agent-max-actions
* since: v1.58
-* hidden
- `maxActions` <[int]>
Maximum number of agentic actions to generate, defaults to context-wide value specified in `agent` property.
## page-agent-max-action-retries
* since: v1.58
-* hidden
- `maxActionRetries` <[int]>
Maximum number of retries when generating each action, defaults to context-wide value specified in `agent` property.
diff --git a/docs/src/test-api/class-fixtures.md b/docs/src/test-api/class-fixtures.md
index 4704ff708a..9ec8e33f92 100644
--- a/docs/src/test-api/class-fixtures.md
+++ b/docs/src/test-api/class-fixtures.md
@@ -20,7 +20,6 @@ Playwright Test comes with builtin fixtures listed below, and you can add your o
## property: Fixtures.agent
* since: v1.58
-* hidden
- type: <[PageAgent]>
## property: Fixtures.browser
diff --git a/docs/src/test-api/class-fullconfig.md b/docs/src/test-api/class-fullconfig.md
index 411ba61585..1f3ee92df7 100644
--- a/docs/src/test-api/class-fullconfig.md
+++ b/docs/src/test-api/class-fullconfig.md
@@ -106,7 +106,6 @@ Base directory for all relative paths used in the reporters.
## property: FullConfig.runAgents
* since: v1.58
-* hidden
- type: <['RunAgentsMode]<"all"|"missing"|"none">>
Whether to run LLM agent for [PageAgent]:
diff --git a/docs/src/test-api/class-testconfig.md b/docs/src/test-api/class-testconfig.md
index a7b417484e..a01babad1f 100644
--- a/docs/src/test-api/class-testconfig.md
+++ b/docs/src/test-api/class-testconfig.md
@@ -516,7 +516,6 @@ export default defineConfig({
## property: TestConfig.runAgents
* since: v1.58
-* hidden
- type: ?<['RunAgentsMode]<"all"|"missing"|"none">>
Whether to run LLM agent for [PageAgent]:
diff --git a/docs/src/test-api/class-testoptions.md b/docs/src/test-api/class-testoptions.md
index 759fa86e55..4c60a5c896 100644
--- a/docs/src/test-api/class-testoptions.md
+++ b/docs/src/test-api/class-testoptions.md
@@ -48,7 +48,6 @@ export default defineConfig({
## property: TestOptions.agentOptions
* since: v1.58
-* hidden
- type: <[Object]>
- `provider` <[Object]>
- `api` <[PageAgentAPI]<"openai"|"openai-compatible"|"anthropic"|"google">> API to use.
diff --git a/packages/playwright-client/types/types.d.ts b/packages/playwright-client/types/types.d.ts
index f6e6165885..50dd490708 100644
--- a/packages/playwright-client/types/types.d.ts
+++ b/packages/playwright-client/types/types.d.ts
@@ -27,6 +27,13 @@ type ElementHandleWaitForSelectorOptionsNotHidden = ElementHandleWaitForSelector
state?: 'visible'|'attached';
};
+// @ts-ignore this will be any if zod is not installed
+import { ZodTypeAny, z } from 'zod';
+// @ts-ignore this will be any if zod is not installed
+import * as z3 from 'zod/v3';
+type ZodSchema = ZodTypeAny | z3.ZodTypeAny;
+type InferZodSchema<T extends ZodSchema> = T extends z3.ZodTypeAny ? z3.infer<T> : T extends ZodTypeAny ? z.infer<T> : never;
+
/**
* Page provides methods to interact with a single tab in a [Browser](https://playwright.dev/docs/api/class-browser),
* or an [extension background page](https://developer.chrome.com/extensions/background_pages) in Chromium. One
@@ -2088,6 +2095,89 @@ export interface Page {
url?: string;
}): Promise<ElementHandle>;
+ /**
+ * Initialize page agent with the llm provider and cache.
+ * @param options
+ */
+ agent(options?: {
+ cache?: {
+ /**
+ * Cache file to use/generate code for performed actions into. Cache is not used if not specified (default).
+ */
+ cacheFile?: string;
+
+ /**
+ * When specified, generated entries are written into the `cacheOutFile` instead of updating the `cacheFile`.
+ */
+ cacheOutFile?: string;
+ };
+
+ expect?: {
+ /**
+ * Default timeout for expect calls in milliseconds, defaults to 5000ms.
+ */
+ timeout?: number;
+ };
+
+ /**
+ * Limits to use for the agentic loop.
+ */
+ limits?: {
+ /**
+ * Maximum number of tokens to consume. The agentic loop will stop after input + output tokens exceed this value.
+ * Defaults to unlimited.
+ */
+ maxTokens?: number;
+
+ /**
+ * Maximum number of agentic actions to generate, defaults to 10.
+ */
+ maxActions?: number;
+
+ /**
+ * Maximum number retries per action, defaults to 3.
+ */
+ maxActionRetries?: number;
+ };
+
+ provider?: {
+ /**
+ * API to use.
+ */
+ api: "openai"|"openai-compatible"|"anthropic"|"google";
+
+ /**
+ * Endpoint to use if different from default.
+ */
+ apiEndpoint?: string;
+
+ /**
+ * API key for the LLM provider.
+ */
+ apiKey: string;
+
+ /**
+ * Amount of time to wait for the provider to respond to each request.
+ */
+ apiTimeout?: number;
+
+ /**
+ * Model identifier within the provider. Required in non-cache mode.
+ */
+ model: string;
+ };
+
+ /**
+ * Secrets to hide from the LLM.
+ */
+ secrets?: { [key: string]: string; };
+
+ /**
+ * System prompt for the agent's loop.
+ */
+ systemPrompt?: string;
+ }): Promise<PageAgent>;
+
/**
* Brings page to front (activates tab).
*/
@@ -5204,6 +5294,243 @@ export interface Page {
[Symbol.asyncDispose](): Promise<void>;
}
+/**
+ *
+ */
+export interface PageAgent {
+ /**
+ * Extract information from the page using the agentic loop, return it in a given Zod format.
+ *
+ * **Usage**
+ *
+ * ```js
+ * await agent.extract('List of items in the cart', z.object({
+ * title: z.string().describe('Item title to extract'),
+ * price: z.string().describe('Item price to extract'),
+ * }).array());
+ * ```
+ *
+ * @param query Task to perform using agentic loop.
+ * @param schema
+ * @param options
+ */
+ extract<Schema extends ZodSchema>(query: string, schema: Schema): Promise<{ result: InferZodSchema<Schema>, usage: { turns: number, inputTokens: number, outputTokens: number } }>;
+ /**
+ * Emitted when the agent makes a turn.
+ */
+ on(event: 'turn', listener: (data: {
+ role: string;
+
+ message: string;
+
+ usage?: {
+ inputTokens: number;
+
+ outputTokens: number;
+ };
+ }) => any): this;
+
+ /**
+ * Adds an event listener that will be automatically removed after it is triggered once. See `addListener` for more information about this event.
+ */
+ once(event: 'turn', listener: (data: {
+ role: string;
+
+ message: string;
+
+ usage?: {
+ inputTokens: number;
+
+ outputTokens: number;
+ };
+ }) => any): this;
+
+ /**
+ * Emitted when the agent makes a turn.
+ */
+ addListener(event: 'turn', listener: (data: {
+ role: string;
+
+ message: string;
+
+ usage?: {
+ inputTokens: number;
+
+ outputTokens: number;
+ };
+ }) => any): this;
+
+ /**
+ * Removes an event listener added by `on` or `addListener`.
+ */
+ removeListener(event: 'turn', listener: (data: {
+ role: string;
+
+ message: string;
+
+ usage?: {
+ inputTokens: number;
+
+ outputTokens: number;
+ };
+ }) => any): this;
+
+ /**
+ * Removes an event listener added by `on` or `addListener`.
+ */
+ off(event: 'turn', listener: (data: {
+ role: string;
+
+ message: string;
+
+ usage?: {
+ inputTokens: number;
+
+ outputTokens: number;
+ };
+ }) => any): this;
+
+ /**
+ * Emitted when the agent makes a turn.
+ */
+ prependListener(event: 'turn', listener: (data: {
+ role: string;
+
+ message: string;
+
+ usage?: {
+ inputTokens: number;
+
+ outputTokens: number;
+ };
+ }) => any): this;
+
+ /**
+ * Dispose this agent.
+ */
+ dispose(): Promise<void>;
+
+ /**
+ * Expect certain condition to be met.
+ *
+ * **Usage**
+ *
+ * ```js
+ * await agent.expect('"0 items" to be reported');
+ * ```
+ *
+ * @param expectation Expectation to assert.
+ * @param options
+ */
+ expect(expectation: string, options?: {
+ /**
+ * All the agentic actions are converted to the Playwright calls and are cached. By default, they are cached globally
+ * with the `task` as a key. This option allows controlling the cache key explicitly.
+ */
+ cacheKey?: string;
+
+ /**
+ * Maximum number of retries when generating each action, defaults to context-wide value specified in `agent`
+ * property.
+ */
+ maxActionRetries?: number;
+
+ /**
+ * Maximum number of agentic actions to generate, defaults to context-wide value specified in `agent` property.
+ */
+ maxActions?: number;
+
+ /**
+ * Maximum number of tokens to consume. The agentic loop will stop after input + output tokens exceed this value.
+ * Defaults to context-wide value specified in `agent` property.
+ */
+ maxTokens?: number;
+
+ /**
+ * Expect timeout in milliseconds. Defaults to `5000`. The default value can be changed via `expect.timeout` option in
+ * the config, or by specifying the `expect` property of the
+ * [`expect`](https://playwright.dev/docs/api/class-page#page-agent-option-expect) option. Pass `0` to disable
+ * timeout.
+ */
+ timeout?: number;
+ }): Promise<void>;
+
+ /**
+ * Perform action using agentic loop.
+ *
+ * **Usage**
+ *
+ * ```js
+ * await agent.perform('Click submit button');
+ * ```
+ *
+ * @param task Task to perform using agentic loop.
+ * @param options
+ */
+ perform(task: string, options?: {
+ /**
+ * All the agentic actions are converted to the Playwright calls and are cached. By default, they are cached globally
+ * with the `task` as a key. This option allows controlling the cache key explicitly.
+ */
+ cacheKey?: string;
+
+ /**
+ * Maximum number of retries when generating each action, defaults to context-wide value specified in `agent`
+ * property.
+ */
+ maxActionRetries?: number;
+
+ /**
+ * Maximum number of agentic actions to generate, defaults to context-wide value specified in `agent` property.
+ */
+ maxActions?: number;
+
+ /**
+ * Maximum number of tokens to consume. The agentic loop will stop after input + output tokens exceed this value.
+ * Defaults to context-wide value specified in `agent` property.
+ */
+ maxTokens?: number;
+
+ /**
+ * Perform timeout in milliseconds. Defaults to `5000`. The default value can be changed via `actionTimeout` option in
+ * the config, or by using the
+ * [browserContext.setDefaultTimeout(timeout)](https://playwright.dev/docs/api/class-browsercontext#browser-context-set-default-timeout)
+ * or [page.setDefaultTimeout(timeout)](https://playwright.dev/docs/api/class-page#page-set-default-timeout) methods.
+ * Pass `0` to disable timeout.
+ */
+ timeout?: number;
+ }): Promise<{
+ usage: {
+ turns: number;
+
+ inputTokens: number;
+
+ outputTokens: number;
+ };
+ }>;
+
+ /**
+ * Returns the current token usage for this agent.
+ *
+ * **Usage**
+ *
+ * ```js
+ * const usage = await agent.usage();
+ * console.log(`Tokens used: ${usage.inputTokens} in, ${usage.outputTokens} out`);
+ * ```
+ *
+ */
+ usage(): Promise<{
+ turns: number;
+
+ inputTokens: number;
+
+ outputTokens: number;
+ }>;
+
+ [Symbol.asyncDispose](): Promise<void>;
+}
+
/**
* At every point of time, page exposes its current frame tree via the
* [page.mainFrame()](https://playwright.dev/docs/api/class-page#page-main-frame) and
diff --git a/packages/playwright-core/src/client/page.ts b/packages/playwright-core/src/client/page.ts
index b8c2651ee9..adc7654de9 100644
--- a/packages/playwright-core/src/client/page.ts
+++ b/packages/playwright-core/src/client/page.ts
@@ -847,7 +847,6 @@ export class Page extends ChannelOwner<channels.PageChannel> implements api.Page
return result.pdf;
}
- // @ts-expect-error agents are hidden
async agent(options: Parameters<api.Page['agent']>[0] = {}) {
const params: channels.PageAgentParams = {
api: options.provider?.api,
@@ -862,7 +861,6 @@ export class Page extends ChannelOwner<channels.PageChannel> implements api.Page
maxTokens: options.limits?.maxTokens,
maxActions: options.limits?.maxActions,
maxActionRetries: options.limits?.maxActionRetries,
- // @ts-expect-error runAgents is hidden
secrets: options.secrets ? Object.entries(options.secrets).map(([name, value]) => ({ name, value })) : undefined,
systemPrompt: options.systemPrompt,
};
diff --git a/packages/playwright-core/src/client/pageAgent.ts b/packages/playwright-core/src/client/pageAgent.ts
index 960867f758..bcf0cdb839 100644
--- a/packages/playwright-core/src/client/pageAgent.ts
+++ b/packages/playwright-core/src/client/pageAgent.ts
@@ -22,7 +22,6 @@ import { Page } from './page';
import type * as api from '../../types/types';
import type * as channels from '@protocol/channels';
-// @ts-expect-error runAgents is hidden
export class PageAgent extends ChannelOwner<channels.PageAgentChannel> implements api.PageAgent {
private _page: Page;
_expectTimeout?: number;
diff --git a/packages/playwright-core/types/types.d.ts b/packages/playwright-core/types/types.d.ts
index f6e6165885..50dd490708 100644
--- a/packages/playwright-core/types/types.d.ts
+++ b/packages/playwright-core/types/types.d.ts
@@ -27,6 +27,13 @@ type ElementHandleWaitForSelectorOptionsNotHidden = ElementHandleWaitForSelector
state?: 'visible'|'attached';
};
+// @ts-ignore this will be any if zod is not installed
+import { ZodTypeAny, z } from 'zod';
+// @ts-ignore this will be any if zod is not installed
+import * as z3 from 'zod/v3';
+type ZodSchema = ZodTypeAny | z3.ZodTypeAny;
+type InferZodSchema<T extends ZodSchema> = T extends z3.ZodTypeAny ? z3.infer<T> : T extends ZodTypeAny ? z.infer<T> : never;
+
/**
* Page provides methods to interact with a single tab in a [Browser](https://playwright.dev/docs/api/class-browser),
* or an [extension background page](https://developer.chrome.com/extensions/background_pages) in Chromium. One
@@ -2088,6 +2095,89 @@ export interface Page {
url?: string;
}): Promise<ElementHandle>;
+ /**
+ * Initialize page agent with the llm provider and cache.
+ * @param options
+ */
+ agent(options?: {
+ cache?: {
+ /**
+ * Cache file to use/generate code for performed actions into. Cache is not used if not specified (default).
+ */
+ cacheFile?: string;
+
+ /**
+ * When specified, generated entries are written into the `cacheOutFile` instead of updating the `cacheFile`.
+ */
+ cacheOutFile?: string;
+ };
+
+ expect?: {
+ /**
+ * Default timeout for expect calls in milliseconds, defaults to 5000ms.
+ */
+ timeout?: number;
+ };
+
+ /**
+ * Limits to use for the agentic loop.
+ */
+ limits?: {
+ /**
+ * Maximum number of tokens to consume. The agentic loop will stop after input + output tokens exceed this value.
+ * Defaults to unlimited.
+ */
+ maxTokens?: number;
+
+ /**
+ * Maximum number of agentic actions to generate, defaults to 10.
+ */
+ maxActions?: number;
+
+ /**
+ * Maximum number retries per action, defaults to 3.
+ */
+ maxActionRetries?: number;
+ };
+
+ provider?: {
+ /**
+ * API to use.
+ */
+ api: "openai"|"openai-compatible"|"anthropic"|"google";
+
+ /**
+ * Endpoint to use if different from default.
+ */
+ apiEndpoint?: string;
+
+ /**
+ * API key for the LLM provider.
+ */
+ apiKey: string;
+
+ /**
+ * Amount of time to wait for the provider to respond to each request.
+ */
+ apiTimeout?: number;
+
+ /**
+ * Model identifier within the provider. Required in non-cache mode.
+ */
+ model: string;
+ };
+
+ /**
+ * Secrets to hide from the LLM.
+ */
+ secrets?: { [key: string]: string; };
+
+ /**
+ * System prompt for the agent's loop.
+ */
+ systemPrompt?: string;
+ }): Promise<PageAgent>;
+
/**
* Brings page to front (activates tab).
*/
@@ -5204,6 +5294,243 @@ export interface Page {
[Symbol.asyncDispose](): Promise<void>;
}
+/**
+ *
+ */
+export interface PageAgent {
+ /**
+ * Extract information from the page using the agentic loop, return it in a given Zod format.
+ *
+ * **Usage**
+ *
+ * ```js
+ * await agent.extract('List of items in the cart', z.object({
+ * title: z.string().describe('Item title to extract'),
+ * price: z.string().describe('Item price to extract'),
+ * }).array());
+ * ```
+ *
+ * @param query Task to perform using agentic loop.
+ * @param schema
+ * @param options
+ */
+ extract<Schema extends ZodSchema>(query: string, schema: Schema): Promise<{ result: InferZodSchema<Schema>, usage: { turns: number, inputTokens: number, outputTokens: number } }>;
+ /**
+ * Emitted when the agent makes a turn.
+ */
+ on(event: 'turn', listener: (data: {
+ role: string;
+
+ message: string;
+
+ usage?: {
+ inputTokens: number;
+
+ outputTokens: number;
+ };
+ }) => any): this;
+
+ /**
+ * Adds an event listener that will be automatically removed after it is triggered once. See `addListener` for more information about this event.
+ */
+ once(event: 'turn', listener: (data: {
+ role: string;
+
+ message: string;
+
+ usage?: {
+ inputTokens: number;
+
+ outputTokens: number;
+ };
+ }) => any): this;
+
+ /**
+ * Emitted when the agent makes a turn.
+ */
+ addListener(event: 'turn', listener: (data: {
+ role: string;
+
+ message: string;
+
+ usage?: {
+ inputTokens: number;
+
+ outputTokens: number;
+ };
+ }) => any): this;
+
+ /**
+ * Removes an event listener added by `on` or `addListener`.
+ */
+ removeListener(event: 'turn', listener: (data: {
+ role: string;
+
+ message: string;
+
+ usage?: {
+ inputTokens: number;
+
+ outputTokens: number;
+ };
+ }) => any): this;
+
+ /**
+ * Removes an event listener added by `on` or `addListener`.
+ */
+ off(event: 'turn', listener: (data: {
+ role: string;
+
+ message: string;
+
+ usage?: {
+ inputTokens: number;
+
+ outputTokens: number;
+ };
+ }) => any): this;
+
+ /**
+ * Emitted when the agent makes a turn.
+ */
+ prependListener(event: 'turn', listener: (data: {
+ role: string;
+
+ message: string;
+
+ usage?: {
+ inputTokens: number;
+
+ outputTokens: number;
+ };
+ }) => any): this;
+
+ /**
+ * Dispose this agent.
+ */
+ dispose(): Promise<void>;
+
+ /**
+ * Expect certain condition to be met.
+ *
+ * **Usage**
+ *
+ * ```js
+ * await agent.expect('"0 items" to be reported');
+ * ```
+ *
+ * @param expectation Expectation to assert.
+ * @param options
+ */
+ expect(expectation: string, options?: {
+ /**
+ * All the agentic actions are converted to the Playwright calls and are cached. By default, they are cached globally
+ * with the `task` as a key. This option allows controlling the cache key explicitly.
+ */
+ cacheKey?: string;
+
+ /**
+ * Maximum number of retries when generating each action, defaults to context-wide value specified in `agent`
+ * property.
+ */
+ maxActionRetries?: number;
+
+ /**
+ * Maximum number of agentic actions to generate, defaults to context-wide value specified in `agent` property.
+ */
+ maxActions?: number;
+
+ /**
+ * Maximum number of tokens to consume. The agentic loop will stop after input + output tokens exceed this value.
+ * Defaults to context-wide value specified in `agent` property.
+ */
+ maxTokens?: number;
+
+ /**
+ * Expect timeout in milliseconds. Defaults to `5000`. The default value can be changed via `expect.timeout` option in
+ * the config, or by specifying the `expect` property of the
+ * [`expect`](https://playwright.dev/docs/api/class-page#page-agent-option-expect) option. Pass `0` to disable
+ * timeout.
+ */
+ timeout?: number;
+ }): Promise<void>;
+
+ /**
+ * Perform action using agentic loop.
+ *
+ * **Usage**
+ *
+ * ```js
+ * await agent.perform('Click submit button');
+ * ```
+ *
+ * @param task Task to perform using agentic loop.
+ * @param options
+ */
+ perform(task: string, options?: {
+ /**
+ * All the agentic actions are converted to the Playwright calls and are cached. By default, they are cached globally
+ * with the `task` as a key. This option allows controlling the cache key explicitly.
+ */
+ cacheKey?: string;
+
+ /**
+ * Maximum number of retries when generating each action, defaults to context-wide value specified in `agent`
+ * property.
+ */
+ maxActionRetries?: number;
+
+ /**
+ * Maximum number of agentic actions to generate, defaults to context-wide value specified in `agent` property.
+ */
+ maxActions?: number;
+
+ /**
+ * Maximum number of tokens to consume. The agentic loop will stop after input + output tokens exceed this value.
+ * Defaults to context-wide value specified in `agent` property.
+ */
+ maxTokens?: number;
+
+ /**
+ * Perform timeout in milliseconds. Defaults to `5000`. The default value can be changed via `actionTimeout` option in
+ * the config, or by using the
+ * [browserContext.setDefaultTimeout(timeout)](https://playwright.dev/docs/api/class-browsercontext#browser-context-set-default-timeout)
+ * or [page.setDefaultTimeout(timeout)](https://playwright.dev/docs/api/class-page#page-set-default-timeout) methods.
+ * Pass `0` to disable timeout.
+ */
+ timeout?: number;
+ }): Promise<{
+ usage: {
+ turns: number;
+
+ inputTokens: number;
+
+ outputTokens: number;
+ };
+ }>;
+
+ /**
+ * Returns the current token usage for this agent.
+ *
+ * **Usage**
+ *
+ * ```js
+ * const usage = await agent.usage();
+ * console.log(`Tokens used: ${usage.inputTokens} in, ${usage.outputTokens} out`);
+ * ```
+ *
+ */
+ usage(): Promise<{
+ turns: number;
+
+ inputTokens: number;
+
+ outputTokens: number;
+ }>;
+
+ [Symbol.asyncDispose](): Promise<void>;
+}
+
/**
* At every point of time, page exposes its current frame tree via the
* [page.mainFrame()](https://playwright.dev/docs/api/class-page#page-main-frame) and
diff --git a/packages/playwright/src/common/config.ts b/packages/playwright/src/common/config.ts
index 1460935c07..82d88aafed 100644
--- a/packages/playwright/src/common/config.ts
+++ b/packages/playwright/src/common/config.ts
@@ -112,8 +112,7 @@ export class FullConfigInternal {
quiet: takeFirst(configCLIOverrides.quiet, userConfig.quiet, false),
reporter: takeFirst(configCLIOverrides.reporter, resolveReporters(userConfig.reporter, configDir), [[defaultReporter]]),
reportSlowTests: takeFirst(userConfig.reportSlowTests, { max: 5, threshold: 300_000 /* 5 minutes */ }),
- // @ts-expect-error runAgents is hidden
- runAgents: takeFirst(configCLIOverrides.runAgents, 'none'),
+ runAgents: takeFirst(configCLIOverrides.runAgents, userConfig.runAgents, 'none'),
shard: takeFirst(configCLIOverrides.shard, userConfig.shard, null),
tags: globalTags,
updateSnapshots: takeFirst(configCLIOverrides.updateSnapshots, userConfig.updateSnapshots, 'missing'),
@@ -197,7 +196,7 @@ export class FullProjectInternal {
testDir,
snapshotDir: takeFirst(pathResolve(configDir, projectConfig.snapshotDir), pathResolve(configDir, config.snapshotDir), testDir),
testIgnore: takeFirst(projectConfig.testIgnore, config.testIgnore, []),
- testMatch: takeFirst(projectConfig.testMatch, config.testMatch, '**/*.@(spec|test).?(c|m)[jt]s?(x)'),
+ testMatch: takeFirst(projectConfig.testMatch, config.testMatch, '**/*.@(spec|test).{md,?(c|m)[jt]s?(x)}'),
timeout: takeFirst(configCLIOverrides.debug ? 0 : undefined, configCLIOverrides.timeout, projectConfig.timeout, config.timeout, defaultTimeout),
use: mergeObjects(config.use, projectConfig.use, configCLIOverrides.use),
dependencies: projectConfig.dependencies || [],
diff --git a/packages/playwright/src/index.ts b/packages/playwright/src/index.ts
index e986e71b33..874ba50ef2 100644
--- a/packages/playwright/src/index.ts
+++ b/packages/playwright/src/index.ts
@@ -58,9 +58,6 @@ type TestFixtures = PlaywrightTestArgs & PlaywrightTestOptions & {
_setupContextOptions: void;
_setupArtifacts: void;
_contextFactory: (options?: BrowserContextOptions) => Promise<{ context: BrowserContext, close: () => Promise<void> }>;
-
- agent: {};
- agentOptions?: any;
};
type WorkerFixtures = PlaywrightWorkerArgs & PlaywrightWorkerOptions & {
@@ -155,7 +152,7 @@ const playwrightFixtures: Fixtures<TestFixtures, WorkerFixtures> = ({
}, { option: true, box: true }],
serviceWorkers: [({ contextOptions }, use) => use(contextOptions.serviceWorkers ?? 'allow'), { option: true, box: true }],
contextOptions: [{}, { option: true, box: true }],
- agentOptions: [undefined, { option: true, box: true }],
+ agentOptions: [({}, use) => use(undefined), { option: true, box: true }],
_combinedContextOptions: [async ({
acceptDownloads,
@@ -462,11 +459,9 @@ const playwrightFixtures: Fixtures<TestFixtures, WorkerFixtures> = ({
const testInfoImpl = testInfo as TestInfoImpl;
const cachePathTemplate = agentOptions?.cachePathTemplate ?? '{testDir}/{testFilePath}-cache.json';
const resolvedCacheFile = testInfoImpl._applyPathTemplate(cachePathTemplate, '', '.json');
- // @ts-expect-error runAgents is hidden
const cacheFile = testInfoImpl.config.runAgents === 'all' ? undefined : await testInfoImpl._cloneStorage(resolvedCacheFile);
const cacheOutFile = path.join(testInfoImpl.artifactsDir(), 'agent-cache-' + createGuid() + '.json');
- // @ts-expect-error runAgents is hidden
const provider = agentOptions?.provider && testInfo.config.runAgents !== 'none' ? agentOptions.provider : undefined;
if (provider)
testInfo.setTimeout(0);
@@ -476,7 +471,6 @@ const playwrightFixtures: Fixtures<TestFixtures, WorkerFixtures> = ({
cacheOutFile,
};
- // @ts-expect-error agent is hidden
const agent = await page.agent({
provider,
cache,
diff --git a/packages/playwright/src/isomorphic/teleReceiver.ts b/packages/playwright/src/isomorphic/teleReceiver.ts
index 458e177c16..b4829257f8 100644
--- a/packages/playwright/src/isomorphic/teleReceiver.ts
+++ b/packages/playwright/src/isomorphic/teleReceiver.ts
@@ -777,7 +777,6 @@ export const baseFullConfig: reporterTypes.FullConfig = {
tags: [],
updateSnapshots: 'missing',
updateSourceMethod: 'patch',
- // @ts-expect-error runAgents is hidden
runAgents: 'none',
version: '',
workers: 0,
diff --git a/packages/playwright/src/program.ts b/packages/playwright/src/program.ts
index e1fc9caf3c..da4ad0f648 100644
--- a/packages/playwright/src/program.ts
+++ b/packages/playwright/src/program.ts
@@ -424,6 +424,7 @@ const testOptions: [string, { description: string, choices?: string[], preset?:
['--repeat-each <N>', { description: `Run each test N times (default: 1)` }],
['--reporter <reporter>', { description: `Reporter to use, comma-separated, can be ${builtInReporters.map(name => `"${name}"`).join(', ')} (default: "${defaultReporter}")` }],
['--retries <retries>', { description: `Maximum retry count for flaky tests, zero for no retries (default: no retries)` }],
+ ['--run-agents <mode>', { description: `Run agents to generate the code for page.perform`, choices: ['missing', 'all', 'none'], preset: 'none' }],
['--shard <shard>', { description: `Shard tests and execute only the selected shard, specify in the form "current/all", 1-based, for example "3/5"` }],
['--test-list <file>', { description: `Path to a file containing a list of tests to run. See https://playwright.dev/docs/test-cli for more details.` }],
['--test-list-invert <file>', { description: `Path to a file containing a list of tests to skip. See https://playwright.dev/docs/test-cli for more details.` }],
diff --git a/packages/playwright/src/transform/transform.ts b/packages/playwright/src/transform/transform.ts
index 3cab05d844..862023cc5f 100644
--- a/packages/playwright/src/transform/transform.ts
+++ b/packages/playwright/src/transform/transform.ts
@@ -224,7 +224,7 @@ export function transformHook(originalCode: string, filename: string, moduleUrl?
// TODO: ideally, we would not transform before checking the cache. However, the source
// currently depends on the seed.md, so "originalCode" is not enough to produce a cache key.
let inputSourceMap: EncodedSourceMap | undefined;
- if (filename.endsWith('.md') && false) {
+ if (filename.endsWith('.md')) {
const transformed = transformMDToTS(originalCode, filename);
originalCode = transformed.code;
inputSourceMap = transformed.map;
diff --git a/packages/playwright/types/test.d.ts b/packages/playwright/types/test.d.ts
index 31caf199d7..33409b8445 100644
--- a/packages/playwright/types/test.d.ts
+++ b/packages/playwright/types/test.d.ts
@@ -15,7 +15,7 @@
* limitations under the License.
*/
-import type { APIRequestContext, Browser, BrowserContext, BrowserContextOptions, Page, LaunchOptions, ViewportSize, Geolocation, HTTPCredentials, Locator, APIResponse, PageScreenshotOptions } from 'playwright-core';
+import type { APIRequestContext, Browser, BrowserContext, BrowserContextOptions, Page, PageAgent, LaunchOptions, ViewportSize, Geolocation, HTTPCredentials, Locator, APIResponse, PageScreenshotOptions } from 'playwright-core';
export * from 'playwright-core';
export type BlobReporterOptions = { outputDir?: string, fileName?: string };
@@ -1611,6 +1611,14 @@ interface TestConfig<TestArgs = {}, WorkerArgs = {}> {
*/
retries?: number;
+ /**
+ * Whether to run LLM agent for [PageAgent](https://playwright.dev/docs/api/class-pageagent):
+ * - "all" disregards existing cache and performs all actions via LLM
+ * - "missing" only performs actions that don't have generated cache actions
+ * - "none" does not talk to LLM at all, relies on the cached actions (default)
+ */
+ runAgents?: "all"|"missing"|"none";
+
/**
* Shard tests and execute only the selected shard. Specify in the one-based form like `{ total: 5, current: 2 }`.
*
@@ -2067,6 +2075,14 @@ export interface FullConfig<TestArgs = {}, WorkerArgs = {}> {
*/
rootDir: string;
+ /**
+ * Whether to run LLM agent for [PageAgent](https://playwright.dev/docs/api/class-pageagent):
+ * - "all" disregards existing cache and performs all actions via LLM
+ * - "missing" only performs actions that don't have generated cache actions
+ * - "none" does not talk to LLM at all, relies on the cached actions (default)
+ */
+ runAgents: "all"|"missing"|"none";
+
/**
* See [testConfig.shard](https://playwright.dev/docs/api/class-testconfig#test-config-shard).
*/
@@ -6934,6 +6950,24 @@ export interface PlaywrightWorkerOptions {
export type ScreenshotMode = 'off' | 'on' | 'only-on-failure' | 'on-first-failure';
export type TraceMode = 'off' | 'on' | 'retain-on-failure' | 'on-first-retry' | 'on-all-retries' | 'retain-on-first-failure';
export type VideoMode = 'off' | 'on' | 'retain-on-failure' | 'on-first-retry';
+export type AgentOptions = {
+ provider?: {
+ api: 'openai' | 'openai-compatible' | 'anthropic' | 'google';
+ apiEndpoint?: string;
+ apiKey: string;
+ apiTimeout?: number;
+ model: string;
+ },
+ limits?: {
+ maxTokens?: number;
+ maxActions?: number;
+ maxActionRetries?: number;
+ };
+ cachePathTemplate?: string;
+ runAgents?: 'all' | 'missing' | 'none';
+ secrets?: { [key: string]: string };
+ systemPrompt?: string;
+};
/**
* Playwright Test provides many options to configure test environment,
@@ -6974,6 +7008,7 @@ export type VideoMode = 'off' | 'on' | 'retain-on-failure' | 'on-first-retry';
*
*/
export interface PlaywrightTestOptions {
+ agentOptions: AgentOptions | undefined;
/**
* Whether to automatically download all the attachments. Defaults to `true` where all the downloads are accepted.
*
@@ -7683,6 +7718,7 @@ export interface PlaywrightTestArgs {
*
*/
request: APIRequestContext;
+ agent: PageAgent;
}
type ExcludeProps<A, B> = {
diff --git a/tests/library/agent-helpers.ts b/tests/library/agent-helpers.ts
index a68988ed7f..5a718082df 100644
--- a/tests/library/agent-helpers.ts
+++ b/tests/library/agent-helpers.ts
@@ -18,7 +18,6 @@ import fs from 'fs';
import path from 'path';
import { browserTest as test } from '../config/browserTest';
-// @ts-expect-error
import type { BrowserContext, Page, PageAgent } from '@playwright/test';
export function cacheFile() {
@@ -33,14 +32,12 @@ export async function setCacheObject(object: any) {
await fs.promises.writeFile(cacheFile(), JSON.stringify(object, null, 2), 'utf8');
}
-// @ts-expect-error
type AgentOptions = Parameters<Page['agent']>[0];
export async function generateAgent(context: BrowserContext, options: AgentOptions = {}) {
const apiCacheFile = path.join(__dirname, '__llm_cache__', sanitizeFileName(test.info().titlePath.join(' ')) + '.json');
const page = await context.newPage();
- // @ts-expect-error
const agent = await page.agent({
provider: {
api: 'anthropic' as const,
@@ -60,7 +57,6 @@ export async function generateAgent(context: BrowserContext, options: AgentOptio
export async function runAgent(context: BrowserContext, options: AgentOptions = {}) {
const page = await context.newPage();
- // @ts-expect-error
const agent = await page.agent({
...options,
cache: { cacheFile: cacheFile() },
diff --git a/tests/library/agent-perform.spec.ts b/tests/library/agent-perform.spec.ts
index 2961178490..915915cd80 100644
--- a/tests/library/agent-perform.spec.ts
+++ b/tests/library/agent-perform.spec.ts
@@ -230,14 +230,12 @@ test('empty cache file works', async ({ context }) => {
});
test('missing apiKey throws a nice error', async ({ page }) => {
- // @ts-expect-error
const agent = await page.agent({ provider: { api: 'anthropic', model: 'some model' } as any });
const error = await agent.perform('click the Test button').catch(e => e);
expect(error.message).toContain(`This action requires API key to be set on the page agent`);
});
test('malformed apiEndpoint throws a nice error', async ({ page }) => {
- // @ts-expect-error
const agent = await page.agent({ provider: { api: 'anthropic', model: 'some model', apiKey: 'some key', apiEndpoint: 'foobar' } });
const error = await agent.perform('click the Test button').catch(e => e);
expect(error.message).toContain(`Agent API endpoint "foobar" is not a valid URL`);
diff --git a/utils/doclint/api_parser.js b/utils/doclint/api_parser.js
index 28e6711450..a4d504ea48 100644
--- a/utils/doclint/api_parser.js
+++ b/utils/doclint/api_parser.js
@@ -93,9 +93,6 @@ class ApiParser {
if (!match)
throw new Error('Invalid member: ' + spec.text);
const metainfo = extractMetainfo(spec);
- if (metainfo.hidden)
- return;
-
const name = match[3];
let returnType = null;
let optional = false;
@@ -128,6 +125,8 @@ class ApiParser {
const clazz = /** @type {docs.Class} */(this.classes.get(match[2]));
if (!clazz)
throw new Error(`Unknown class ${match[2]} for member: ` + spec.text);
+ if (metainfo.hidden)
+ return;
const existingMember = clazz.membersArray.find(m => m.name === name && m.kind === member.kind);
if (existingMember && isTypeOverride(existingMember, member)) {
@@ -147,9 +146,6 @@ class ApiParser {
const match = spec.text.match(/(param|option): (.*)/);
if (!match)
throw `Something went wrong with matching ${spec.text}`;
- const metainfo = extractMetainfo(spec);
- if (metainfo.hidden)
- return null;
// For "test.describe.only.title":
// - className is "test"
diff --git a/utils/doclint/missingDocs.js b/utils/doclint/missingDocs.js
index 0eca3a2cbe..856c4f0210 100644
--- a/utils/doclint/missingDocs.js
+++ b/utils/doclint/missingDocs.js
@@ -29,16 +29,12 @@ module.exports = function lint(documentation, jsSources, apiFileName) {
documentation.copyDocsFromSuperclasses(errors);
const apiMethods = listMethods(jsSources, apiFileName);
for (const [className, methods] of apiMethods) {
- if (className === 'PageAgent')
- continue;
const docClass = documentation.classes.get(className);
if (!docClass) {
errors.push(`Missing documentation for "${className}"`);
continue;
}
for (const [methodName, params] of methods) {
- if (className === 'Page' && methodName === 'agent')
- continue;
const members = docClass.membersArray.filter(m => m.alias === methodName && m.kind !== 'event');
if (!members.length) {
errors.push(`Missing documentation for "${className}.${methodName}"`);
diff --git a/utils/generate_types/overrides-test.d.ts b/utils/generate_types/overrides-test.d.ts
index 7b46b3402b..e71d6ebbf7 100644
--- a/utils/generate_types/overrides-test.d.ts
+++ b/utils/generate_types/overrides-test.d.ts
@@ -14,7 +14,7 @@
* limitations under the License.
*/
-import type { APIRequestContext, Browser, BrowserContext, BrowserContextOptions, Page, LaunchOptions, ViewportSize, Geolocation, HTTPCredentials, Locator, APIResponse, PageScreenshotOptions } from 'playwright-core';
+import type { APIRequestContext, Browser, BrowserContext, BrowserContextOptions, Page, PageAgent, LaunchOptions, ViewportSize, Geolocation, HTTPCredentials, Locator, APIResponse, PageScreenshotOptions } from 'playwright-core';
export * from 'playwright-core';
export type BlobReporterOptions = { outputDir?: string, fileName?: string };
@@ -265,8 +265,27 @@ export interface PlaywrightWorkerOptions {
export type ScreenshotMode = 'off' | 'on' | 'only-on-failure' | 'on-first-failure';
export type TraceMode = 'off' | 'on' | 'retain-on-failure' | 'on-first-retry' | 'on-all-retries' | 'retain-on-first-failure';
export type VideoMode = 'off' | 'on' | 'retain-on-failure' | 'on-first-retry';
+export type AgentOptions = {
+ provider?: {
+ api: 'openai' | 'openai-compatible' | 'anthropic' | 'google';
+ apiEndpoint?: string;
+ apiKey: string;
+ apiTimeout?: number;
+ model: string;
+ },
+ limits?: {
+ maxTokens?: number;
+ maxActions?: number;
+ maxActionRetries?: number;
+ };
+ cachePathTemplate?: string;
+ runAgents?: 'all' | 'missing' | 'none';
+ secrets?: { [key: string]: string };
+ systemPrompt?: string;
+};
export interface PlaywrightTestOptions {
+ agentOptions: AgentOptions | undefined;
acceptDownloads: boolean;
bypassCSP: boolean;
colorScheme: ColorScheme;
@@ -305,6 +324,7 @@ export interface PlaywrightTestArgs {
context: BrowserContext;
page: Page;
request: APIRequestContext;
+ agent: PageAgent;
}
type ExcludeProps<A, B> = {
diff --git a/utils/generate_types/overrides.d.ts b/utils/generate_types/overrides.d.ts
index a88c80ba6e..09b54a6dd7 100644
--- a/utils/generate_types/overrides.d.ts
+++ b/utils/generate_types/overrides.d.ts
@@ -26,6 +26,13 @@ type ElementHandleWaitForSelectorOptionsNotHidden = ElementHandleWaitForSelector
state?: 'visible'|'attached';
};
+// @ts-ignore this will be any if zod is not installed
+import { ZodTypeAny, z } from 'zod';
+// @ts-ignore this will be any if zod is not installed
+import * as z3 from 'zod/v3';
+type ZodSchema = ZodTypeAny | z3.ZodTypeAny;
+type InferZodSchema<T extends ZodSchema> = T extends z3.ZodTypeAny ? z3.infer<T> : T extends ZodTypeAny ? z.infer<T> : never;
+
export interface Page {
evaluate<R, Arg>(pageFunction: PageFunction<Arg, R>, arg: Arg): Promise<R>;
evaluate<R>(pageFunction: PageFunction<void, R>, arg?: any): Promise<R>;
@@ -74,6 +81,10 @@ export interface Page {
}): Promise<void>;
}
+export interface PageAgent {
+ extract<Schema extends ZodSchema>(query: string, schema: Schema): Promise<{ result: InferZodSchema<Schema>, usage: { turns: number, inputTokens: number, outputTokens: number } }>;
+}
+
export interface Frame {
evaluate<R, Arg>(pageFunction: PageFunction<Arg, R>, arg: Arg): Promise<R>;
evaluate<R>(pageFunction: PageFunction<void, R>, arg?: any): Promise<R>;