Skip to content

Playwright is a framework for Web Testing and Automation. It allows testing Chromium, Firefox and WebKit with a single API.

chore: allow all/missing/none run agents modes (#38628)

Pavel Feldman1eba405f948b
diff --git a/docs/src/test-api/class-fullconfig.md b/docs/src/test-api/class-fullconfig.md
index 4a72cf0556..2a97b248a8 100644
--- a/docs/src/test-api/class-fullconfig.md
+++ b/docs/src/test-api/class-fullconfig.md
@@ -106,9 +106,12 @@ Base directory for all relative paths used in the reporters.
## property: FullConfig.runAgents
* since: v1.58
-- type: <boolean>
+- type: <['RunAgentsMode]<"all"|"missing"|"none">>
-Run agents to generate the code for [`method: Page.perform`] and similar.
+Whether to run LLM agent for [`method: Page.perform`]:
+* "all" disregards existing cache and performs all actions via LLM
+* "missing" only performs actions that don't have generated cache actions
+* "none" does not talk to LLM at all, relies on the cached actions (default)
## property: FullConfig.shard
* since: v1.10
diff --git a/docs/src/test-api/class-testconfig.md b/docs/src/test-api/class-testconfig.md
index 5b24da84c2..d171115cd5 100644
--- a/docs/src/test-api/class-testconfig.md
+++ b/docs/src/test-api/class-testconfig.md
@@ -516,9 +516,12 @@ export default defineConfig({
## property: TestConfig.runAgents
* since: v1.58
-- type: ?<boolean>
+- type: ?<['RunAgentsMode]<"all"|"missing"|"none">>
-Run agents to generate the code for [`method: Page.perform`] and similar.
+Whether to run LLM agent for [`method: Page.perform`]:
+* "all" disregards existing cache and performs all actions via LLM
+* "missing" only performs actions that don't have generated cache actions
+* "none" does not talk to LLM at all, relies on the cached actions (default)
## property: TestConfig.shard
* since: v1.10
diff --git a/packages/playwright/src/common/config.ts b/packages/playwright/src/common/config.ts
index b16632e904..4b6c466cd4 100644
--- a/packages/playwright/src/common/config.ts
+++ b/packages/playwright/src/common/config.ts
@@ -112,7 +112,7 @@ export class FullConfigInternal {
quiet: takeFirst(configCLIOverrides.quiet, userConfig.quiet, false),
reporter: takeFirst(configCLIOverrides.reporter, resolveReporters(userConfig.reporter, configDir), [[defaultReporter]]),
reportSlowTests: takeFirst(userConfig.reportSlowTests, { max: 5, threshold: 300_000 /* 5 minutes */ }),
- runAgents: takeFirst(configCLIOverrides.runAgents, userConfig.runAgents, false),
+ runAgents: takeFirst(configCLIOverrides.runAgents, userConfig.runAgents, 'none'),
shard: takeFirst(configCLIOverrides.shard, userConfig.shard, null),
tags: globalTags,
updateSnapshots: takeFirst(configCLIOverrides.updateSnapshots, userConfig.updateSnapshots, 'missing'),
diff --git a/packages/playwright/src/common/ipc.ts b/packages/playwright/src/common/ipc.ts
index ad2de5304f..b22fdc036f 100644
--- a/packages/playwright/src/common/ipc.ts
+++ b/packages/playwright/src/common/ipc.ts
@@ -43,7 +43,7 @@ export type ConfigCLIOverrides = {
ignoreSnapshots?: boolean;
updateSnapshots?: 'all' | 'changed' | 'missing' | 'none';
updateSourceMethod?: 'overwrite' | 'patch' | '3way';
- runAgents?: boolean;
+ runAgents?: 'all' | 'missing' | 'none';
workers?: number | string;
projects?: { name: string, use?: any }[],
use?: any;
diff --git a/packages/playwright/src/index.ts b/packages/playwright/src/index.ts
index d07e1662d5..fa369524b9 100644
--- a/packages/playwright/src/index.ts
+++ b/packages/playwright/src/index.ts
@@ -716,9 +716,11 @@ class ArtifactsRecorder {
this._agentCacheFile = this._testInfo._applyPathTemplate(cachePathTemplate, '', '.json');
this._agentCacheOutFile = path.join(this._testInfo.artifactsDir(), 'agent-cache-' + createGuid() + '.json');
- const cacheFile = this._testInfo.config.runAgents ? undefined : await this._testInfo._cloneStorage(this._agentCacheFile);
+ const cacheFile = this._testInfo.config.runAgents === 'all' ? undefined : await this._testInfo._cloneStorage(this._agentCacheFile);
options.agent = {
...this._agent,
+ provider: this._testInfo.config.runAgents !== 'none' ? this._agent.provider : undefined,
+ model: this._testInfo.config.runAgents !== 'none' ? this._agent.model : undefined,
cacheFile,
cacheOutFile: this._agentCacheOutFile,
};
diff --git a/packages/playwright/src/isomorphic/teleReceiver.ts b/packages/playwright/src/isomorphic/teleReceiver.ts
index 859dd7ffdc..657664199e 100644
--- a/packages/playwright/src/isomorphic/teleReceiver.ts
+++ b/packages/playwright/src/isomorphic/teleReceiver.ts
@@ -782,7 +782,7 @@ export const baseFullConfig: reporterTypes.FullConfig = {
tags: [],
updateSnapshots: 'missing',
updateSourceMethod: 'patch',
- runAgents: false,
+ runAgents: 'none',
version: '',
workers: 0,
webServer: null,
diff --git a/packages/playwright/src/isomorphic/testServerInterface.ts b/packages/playwright/src/isomorphic/testServerInterface.ts
index e356678117..062b26958c 100644
--- a/packages/playwright/src/isomorphic/testServerInterface.ts
+++ b/packages/playwright/src/isomorphic/testServerInterface.ts
@@ -97,7 +97,7 @@ export interface TestServerInterface {
workers?: number | string;
updateSnapshots?: 'all' | 'changed' | 'missing' | 'none';
updateSourceMethod?: 'overwrite' | 'patch' | '3way';
- runAgents?: boolean;
+ runAgents?: 'all' | 'missing' | 'none';
reporters?: string[],
trace?: 'on' | 'off';
video?: 'on' | 'off';
diff --git a/packages/playwright/src/program.ts b/packages/playwright/src/program.ts
index 47c4adb633..c48a605a8f 100644
--- a/packages/playwright/src/program.ts
+++ b/packages/playwright/src/program.ts
@@ -422,7 +422,7 @@ const testOptions: [string, { description: string, choices?: string[], preset?:
['--repeat-each <N>', { description: `Run each test N times (default: 1)` }],
['--reporter <reporter>', { description: `Reporter to use, comma-separated, can be ${builtInReporters.map(name => `"${name}"`).join(', ')} (default: "${defaultReporter}")` }],
['--retries <retries>', { description: `Maximum retry count for flaky tests, zero for no retries (default: no retries)` }],
- ['--run-agents', { description: `Run agents to generate the code for page.perform` }],
+ ['--run-agents <mode>', { description: `Run agents to generate the code for page.perform`, choices: ['missing', 'all', 'none'], preset: 'none' }],
['--shard <shard>', { description: `Shard tests and execute only the selected shard, specify in the form "current/all", 1-based, for example "3/5"` }],
['--shard-weights <weights>', { description: `Weights for each shard, colon-separated, for example "2:1:1" for 3 shards where the first shard should be allocated half of the work` }],
['--test-list <file>', { description: `Path to a file containing a list of tests to run. See https://playwright.dev/docs/test-cli for more details.` }],
diff --git a/packages/playwright/src/runner/testRunner.ts b/packages/playwright/src/runner/testRunner.ts
index 5639735add..c75f75a815 100644
--- a/packages/playwright/src/runner/testRunner.ts
+++ b/packages/playwright/src/runner/testRunner.ts
@@ -68,7 +68,7 @@ export type RunTestsParams = {
workers?: number | string;
updateSnapshots?: 'all' | 'changed' | 'missing' | 'none';
updateSourceMethod?: 'overwrite' | 'patch' | '3way';
- runAgents?: boolean;
+ runAgents?: 'all' | 'missing' | 'none';
reporters?: string[],
trace?: 'on' | 'off';
video?: 'on' | 'off';
diff --git a/packages/playwright/types/test.d.ts b/packages/playwright/types/test.d.ts
index 20735502d4..e3ce87e8a3 100644
--- a/packages/playwright/types/test.d.ts
+++ b/packages/playwright/types/test.d.ts
@@ -1612,10 +1612,13 @@ interface TestConfig<TestArgs = {}, WorkerArgs = {}> {
retries?: number;
/**
- * Run agents to generate the code for
- * [page.perform(task[, options])](https://playwright.dev/docs/api/class-page#page-perform) and similar.
+ * Whether to run LLM agent for
+ * [page.perform(task[, options])](https://playwright.dev/docs/api/class-page#page-perform):
+ * - "all" disregards existing cache and performs all actions via LLM
+ * - "missing" only performs actions that don't have generated cache actions
+ * - "none" does not talk to LLM at all, relies on the cached actions (default)
*/
- runAgents?: boolean;
+ runAgents?: "all"|"missing"|"none";
/**
* Shard tests and execute only the selected shard. Specify in the one-based form like `{ total: 5, current: 2 }`.
@@ -2074,10 +2077,13 @@ export interface FullConfig<TestArgs = {}, WorkerArgs = {}> {
rootDir: string;
/**
- * Run agents to generate the code for
- * [page.perform(task[, options])](https://playwright.dev/docs/api/class-page#page-perform) and similar.
+ * Whether to run LLM agent for
+ * [page.perform(task[, options])](https://playwright.dev/docs/api/class-page#page-perform):
+ * - "all" disregards existing cache and performs all actions via LLM
+ * - "missing" only performs actions that don't have generated cache actions
+ * - "none" does not talk to LLM at all, relies on the cached actions (default)
*/
- runAgents: boolean;
+ runAgents: "all"|"missing"|"none";
/**
* See [testConfig.shard](https://playwright.dev/docs/api/class-testconfig#test-config-shard).
@@ -6952,7 +6958,7 @@ export type Agent = {
cachePathTemplate?: string;
maxTurns?: number;
maxTokens?: number;
- runAgents?: boolean;
+ runAgents?: 'all' | 'missing' | 'none';
secrets?: { [key: string]: string };
};
diff --git a/utils/generate_types/overrides-test.d.ts b/utils/generate_types/overrides-test.d.ts
index 9cf5e4291c..cab28cc257 100644
--- a/utils/generate_types/overrides-test.d.ts
+++ b/utils/generate_types/overrides-test.d.ts
@@ -271,7 +271,7 @@ export type Agent = {
cachePathTemplate?: string;
maxTurns?: number;
maxTokens?: number;
- runAgents?: boolean;
+ runAgents?: 'all' | 'missing' | 'none';
secrets?: { [key: string]: string };
};