Playwright is a framework for Web Testing and Automation. It allows testing Chromium, Firefox and WebKit with a single API.
TypeScript automationchromechromiume2e-testingelectronend-to-end-testingfirefoxjavascriptplaywrighttesttest-automationtestingtesting-toolswebwebkit
chore: allow all/missing/none run agents modes (#38628)
Pavel Feldman1eba405f948b
diff --git a/docs/src/test-api/class-fullconfig.md b/docs/src/test-api/class-fullconfig.mdindex 4a72cf0556..2a97b248a8 100644--- a/docs/src/test-api/class-fullconfig.md+++ b/docs/src/test-api/class-fullconfig.md@@ -106,9 +106,12 @@ Base directory for all relative paths used in the reporters.## property: FullConfig.runAgents* since: v1.58-- type: <boolean>+- type: <['RunAgentsMode]<"all"|"missing"|"none">>-Run agents to generate the code for [`method: Page.perform`] and similar.+Whether to run LLM agent for [`method: Page.perform`]:+* "all" disregards existing cache and performs all actions via LLM+* "missing" only performs actions that don't have generated cache actions+* "none" does not talk to LLM at all, relies on the cached actions (default)## property: FullConfig.shard* since: v1.10diff --git a/docs/src/test-api/class-testconfig.md b/docs/src/test-api/class-testconfig.mdindex 5b24da84c2..d171115cd5 100644--- a/docs/src/test-api/class-testconfig.md+++ b/docs/src/test-api/class-testconfig.md@@ -516,9 +516,12 @@ export default defineConfig({## property: TestConfig.runAgents* since: v1.58-- type: ?<boolean>+- type: ?<['RunAgentsMode]<"all"|"missing"|"none">>-Run agents to generate the code for [`method: Page.perform`] and similar.+Whether to run LLM agent for [`method: Page.perform`]:+* "all" disregards existing cache and performs all actions via LLM+* "missing" only performs actions that don't have generated cache actions+* "none" does not talk to LLM at all, relies on the cached actions (default)## property: TestConfig.shard* since: v1.10diff --git a/packages/playwright/src/common/config.ts b/packages/playwright/src/common/config.tsindex b16632e904..4b6c466cd4 100644--- a/packages/playwright/src/common/config.ts+++ b/packages/playwright/src/common/config.ts@@ -112,7 +112,7 @@ export class FullConfigInternal {quiet: takeFirst(configCLIOverrides.quiet, userConfig.quiet, false),reporter: takeFirst(configCLIOverrides.reporter, resolveReporters(userConfig.reporter, configDir), [[defaultReporter]]),reportSlowTests: takeFirst(userConfig.reportSlowTests, { max: 5, threshold: 300_000 /* 5 minutes */ }),- runAgents: takeFirst(configCLIOverrides.runAgents, userConfig.runAgents, false),+ runAgents: takeFirst(configCLIOverrides.runAgents, userConfig.runAgents, 'none'),shard: takeFirst(configCLIOverrides.shard, userConfig.shard, null),tags: globalTags,updateSnapshots: takeFirst(configCLIOverrides.updateSnapshots, userConfig.updateSnapshots, 'missing'),diff --git a/packages/playwright/src/common/ipc.ts b/packages/playwright/src/common/ipc.tsindex ad2de5304f..b22fdc036f 100644--- a/packages/playwright/src/common/ipc.ts+++ b/packages/playwright/src/common/ipc.ts@@ -43,7 +43,7 @@ export type ConfigCLIOverrides = {ignoreSnapshots?: boolean;updateSnapshots?: 'all' | 'changed' | 'missing' | 'none';updateSourceMethod?: 'overwrite' | 'patch' | '3way';- runAgents?: boolean;+ runAgents?: 'all' | 'missing' | 'none';workers?: number | string;projects?: { name: string, use?: any }[],use?: any;diff --git a/packages/playwright/src/index.ts b/packages/playwright/src/index.tsindex d07e1662d5..fa369524b9 100644--- a/packages/playwright/src/index.ts+++ b/packages/playwright/src/index.ts@@ -716,9 +716,11 @@ class ArtifactsRecorder {this._agentCacheFile = this._testInfo._applyPathTemplate(cachePathTemplate, '', '.json');this._agentCacheOutFile = path.join(this._testInfo.artifactsDir(), 'agent-cache-' + createGuid() + '.json');- const cacheFile = this._testInfo.config.runAgents ? undefined : await this._testInfo._cloneStorage(this._agentCacheFile);+ const cacheFile = this._testInfo.config.runAgents === 'all' ? undefined : await this._testInfo._cloneStorage(this._agentCacheFile);options.agent = {...this._agent,+ provider: this._testInfo.config.runAgents !== 'none' ? this._agent.provider : undefined,+ model: this._testInfo.config.runAgents !== 'none' ? this._agent.model : undefined,cacheFile,cacheOutFile: this._agentCacheOutFile,};diff --git a/packages/playwright/src/isomorphic/teleReceiver.ts b/packages/playwright/src/isomorphic/teleReceiver.tsindex 859dd7ffdc..657664199e 100644--- a/packages/playwright/src/isomorphic/teleReceiver.ts+++ b/packages/playwright/src/isomorphic/teleReceiver.ts@@ -782,7 +782,7 @@ export const baseFullConfig: reporterTypes.FullConfig = {tags: [],updateSnapshots: 'missing',updateSourceMethod: 'patch',- runAgents: false,+ runAgents: 'none',version: '',workers: 0,webServer: null,diff --git a/packages/playwright/src/isomorphic/testServerInterface.ts b/packages/playwright/src/isomorphic/testServerInterface.tsindex e356678117..062b26958c 100644--- a/packages/playwright/src/isomorphic/testServerInterface.ts+++ b/packages/playwright/src/isomorphic/testServerInterface.ts@@ -97,7 +97,7 @@ export interface TestServerInterface {workers?: number | string;updateSnapshots?: 'all' | 'changed' | 'missing' | 'none';updateSourceMethod?: 'overwrite' | 'patch' | '3way';- runAgents?: boolean;+ runAgents?: 'all' | 'missing' | 'none';reporters?: string[],trace?: 'on' | 'off';video?: 'on' | 'off';diff --git a/packages/playwright/src/program.ts b/packages/playwright/src/program.tsindex 47c4adb633..c48a605a8f 100644--- a/packages/playwright/src/program.ts+++ b/packages/playwright/src/program.ts@@ -422,7 +422,7 @@ const testOptions: [string, { description: string, choices?: string[], preset?:['--repeat-each <N>', { description: `Run each test N times (default: 1)` }],['--reporter <reporter>', { description: `Reporter to use, comma-separated, can be ${builtInReporters.map(name => `"${name}"`).join(', ')} (default: "${defaultReporter}")` }],['--retries <retries>', { description: `Maximum retry count for flaky tests, zero for no retries (default: no retries)` }],- ['--run-agents', { description: `Run agents to generate the code for page.perform` }],+ ['--run-agents <mode>', { description: `Run agents to generate the code for page.perform`, choices: ['missing', 'all', 'none'], preset: 'none' }],['--shard <shard>', { description: `Shard tests and execute only the selected shard, specify in the form "current/all", 1-based, for example "3/5"` }],['--shard-weights <weights>', { description: `Weights for each shard, colon-separated, for example "2:1:1" for 3 shards where the first shard should be allocated half of the work` }],['--test-list <file>', { description: `Path to a file containing a list of tests to run. See https://playwright.dev/docs/test-cli for more details.` }],diff --git a/packages/playwright/src/runner/testRunner.ts b/packages/playwright/src/runner/testRunner.tsindex 5639735add..c75f75a815 100644--- a/packages/playwright/src/runner/testRunner.ts+++ b/packages/playwright/src/runner/testRunner.ts@@ -68,7 +68,7 @@ export type RunTestsParams = {workers?: number | string;updateSnapshots?: 'all' | 'changed' | 'missing' | 'none';updateSourceMethod?: 'overwrite' | 'patch' | '3way';- runAgents?: boolean;+ runAgents?: 'all' | 'missing' | 'none';reporters?: string[],trace?: 'on' | 'off';video?: 'on' | 'off';diff --git a/packages/playwright/types/test.d.ts b/packages/playwright/types/test.d.tsindex 20735502d4..e3ce87e8a3 100644--- a/packages/playwright/types/test.d.ts+++ b/packages/playwright/types/test.d.ts@@ -1612,10 +1612,13 @@ interface TestConfig<TestArgs = {}, WorkerArgs = {}> {retries?: number;/**- * Run agents to generate the code for- * [page.perform(task[, options])](https://playwright.dev/docs/api/class-page#page-perform) and similar.+ * Whether to run LLM agent for+ * [page.perform(task[, options])](https://playwright.dev/docs/api/class-page#page-perform):+ * - "all" disregards existing cache and performs all actions via LLM+ * - "missing" only performs actions that don't have generated cache actions+ * - "none" does not talk to LLM at all, relies on the cached actions (default)*/- runAgents?: boolean;+ runAgents?: "all"|"missing"|"none";/*** Shard tests and execute only the selected shard. Specify in the one-based form like `{ total: 5, current: 2 }`.@@ -2074,10 +2077,13 @@ export interface FullConfig<TestArgs = {}, WorkerArgs = {}> {rootDir: string;/**- * Run agents to generate the code for- * [page.perform(task[, options])](https://playwright.dev/docs/api/class-page#page-perform) and similar.+ * Whether to run LLM agent for+ * [page.perform(task[, options])](https://playwright.dev/docs/api/class-page#page-perform):+ * - "all" disregards existing cache and performs all actions via LLM+ * - "missing" only performs actions that don't have generated cache actions+ * - "none" does not talk to LLM at all, relies on the cached actions (default)*/- runAgents: boolean;+ runAgents: "all"|"missing"|"none";/*** See [testConfig.shard](https://playwright.dev/docs/api/class-testconfig#test-config-shard).@@ -6952,7 +6958,7 @@ export type Agent = {cachePathTemplate?: string;maxTurns?: number;maxTokens?: number;- runAgents?: boolean;+ runAgents?: 'all' | 'missing' | 'none';secrets?: { [key: string]: string };};diff --git a/utils/generate_types/overrides-test.d.ts b/utils/generate_types/overrides-test.d.tsindex 9cf5e4291c..cab28cc257 100644--- a/utils/generate_types/overrides-test.d.ts+++ b/utils/generate_types/overrides-test.d.ts@@ -271,7 +271,7 @@ export type Agent = {cachePathTemplate?: string;maxTurns?: number;maxTokens?: number;- runAgents?: boolean;+ runAgents?: 'all' | 'missing' | 'none';secrets?: { [key: string]: string };};