Skip to content

Playwright is a framework for Web Testing and Automation. It allows testing Chromium, Firefox and WebKit with a single API.

feat(test): add retryStrategy option to defer retries (#41297)

Dmitry Gozmana9736eef2c0d
diff --git a/docs/src/test-api/class-testconfig.md b/docs/src/test-api/class-testconfig.md
index 88b059a35a..169b8c943f 100644
--- a/docs/src/test-api/class-testconfig.md
+++ b/docs/src/test-api/class-testconfig.md
@@ -515,6 +515,27 @@ export default defineConfig({
});
```
+## property: TestConfig.retryStrategy
+* since: v1.62
+- type: ?<[RetryStrategy]<"immediate"|"deferred">>
+
+Controls when failed tests are retried. Defaults to `'immediate'`.
+* `'immediate'` - A failed test is retried as soon as a worker is available, interleaved with the rest of the run. This is the default.
+* `'deferred'` - Retries are run only after all tests have had their first attempt, in parallel up to the configured number of [workers](#test-config-workers).
+
+Learn more about [test retries](../test-retries.md#retries).
+
+**Usage**
+
+```js title="playwright.config.ts"
+import { defineConfig } from '@playwright/test';
+
+export default defineConfig({
+ retries: 2,
+ retryStrategy: 'deferred',
+});
+```
+
## property: TestConfig.shard
* since: v1.10
- type: ?<[null]|[Object]>
diff --git a/packages/playwright/src/common/config.ts b/packages/playwright/src/common/config.ts
index e521d61c49..4dd22b11d8 100644
--- a/packages/playwright/src/common/config.ts
+++ b/packages/playwright/src/common/config.ts
@@ -48,6 +48,7 @@ export class FullConfigInternal {
readonly projects: FullProjectInternal[] = [];
readonly singleTSConfigPath?: string;
readonly captureGitInfo: Config['captureGitInfo'];
+ readonly retryStrategy: 'immediate' | 'deferred';
defineConfigWasUsed = false;
globalSetups: string[] = [];
@@ -67,6 +68,7 @@ export class FullConfigInternal {
this.plugins = (privateConfiguration?.plugins || []).map((p: any) => ({ factory: p }));
this.singleTSConfigPath = pathResolve(configDir, userConfig.tsconfig);
this.captureGitInfo = userConfig.captureGitInfo;
+ this.retryStrategy = takeFirst(userConfig.retryStrategy, 'immediate');
this.globalSetups = (Array.isArray(userConfig.globalSetup) ? userConfig.globalSetup : [userConfig.globalSetup]).map(s => resolveScript(s, configDir)).filter(script => script !== undefined);
this.globalTeardowns = (Array.isArray(userConfig.globalTeardown) ? userConfig.globalTeardown : [userConfig.globalTeardown]).map(s => resolveScript(s, configDir)).filter(script => script !== undefined);
diff --git a/packages/playwright/src/common/configLoader.ts b/packages/playwright/src/common/configLoader.ts
index 32f3a56217..9beee268ca 100644
--- a/packages/playwright/src/common/configLoader.ts
+++ b/packages/playwright/src/common/configLoader.ts
@@ -243,6 +243,11 @@ function validateConfig(file: string, config: Config) {
throw errorWithFile(file, `config.updateSnapshots must be one of "all", "changed", "missing" or "none"`);
}
+ if ('retryStrategy' in config && config.retryStrategy !== undefined) {
+ if (typeof config.retryStrategy !== 'string' || !['immediate', 'deferred'].includes(config.retryStrategy))
+ throw errorWithFile(file, `config.retryStrategy must be one of "immediate" or "deferred"`);
+ }
+
if ('tsconfig' in config && config.tsconfig !== undefined) {
if (typeof config.tsconfig !== 'string')
throw errorWithFile(file, `config.tsconfig must be a string`);
diff --git a/packages/playwright/src/runner/dispatcher.ts b/packages/playwright/src/runner/dispatcher.ts
index 29f4f4e84e..77cb0aa86c 100644
--- a/packages/playwright/src/runner/dispatcher.ts
+++ b/packages/playwright/src/runner/dispatcher.ts
@@ -152,7 +152,10 @@ export class Dispatcher {
// 5. Possibly queue a new job with leftover tests and/or retries.
if (!this._isStopped && result.newJob) {
- this._queue.unshift(result.newJob);
+ if (this._testRun.config.retryStrategy === 'deferred')
+ this._queue.push(result.newJob);
+ else
+ this._queue.unshift(result.newJob);
this._updateCounterForWorkerHash(result.newJob.workerHash, +1);
}
}
diff --git a/packages/playwright/types/test.d.ts b/packages/playwright/types/test.d.ts
index c2037da8f8..ab8160cd39 100644
--- a/packages/playwright/types/test.d.ts
+++ b/packages/playwright/types/test.d.ts
@@ -1640,6 +1640,30 @@ interface TestConfig<TestArgs = {}, WorkerArgs = {}> {
*/
retries?: number;
+ /**
+ * Controls when failed tests are retried. Defaults to `'immediate'`.
+ * - `'immediate'` - A failed test is retried as soon as a worker is available, interleaved with the rest of the
+ * run. This is the default.
+ * - `'deferred'` - Retries are run only after all tests have had their first attempt, in parallel up to the
+ * configured number of [workers](#test-config-workers).
+ *
+ * Learn more about [test retries](https://playwright.dev/docs/test-retries#retries).
+ *
+ * **Usage**
+ *
+ * ```js
+ * // playwright.config.ts
+ * import { defineConfig } from '@playwright/test';
+ *
+ * export default defineConfig({
+ * retries: 2,
+ * retryStrategy: 'deferred',
+ * });
+ * ```
+ *
+ */
+ retryStrategy?: "immediate"|"deferred";
+
/**
* Shard tests and execute only the selected shard. Specify in the one-based form like `{ total: 5, current: 2 }`.
*
diff --git a/tests/playwright-test/retry.spec.ts b/tests/playwright-test/retry.spec.ts
index c82a61b20f..1800eaded0 100644
--- a/tests/playwright-test/retry.spec.ts
+++ b/tests/playwright-test/retry.spec.ts
@@ -262,3 +262,39 @@ test('failed and skipped on retry should be marked as flaky', async ({ runInline
expect(result.output).toContain('Failed on first run');
expect(result.report.suites[0].specs[0].tests[0].annotations).toEqual([{ type: 'skip', description: 'Skipped on first retry', location: expect.anything() }]);
});
+
+test('should defer retries to the end of the run', async ({ runInlineTest }) => {
+ const result = await runInlineTest({
+ 'playwright.config.js': `
+ module.exports = { retries: 3, retryStrategy: 'deferred' };
+ `,
+ 'a.test.js': `
+ import { test, expect } from '@playwright/test';
+ test('a', ({}, testInfo) => {
+ console.log('\\n%%a-' + testInfo.retry);
+ expect(testInfo.retry).toBe(3);
+ });
+ `,
+ 'b.test.js': `
+ import { test, expect } from '@playwright/test';
+ test.describe.configure({ retries: 2 });
+ test('b', ({}, testInfo) => {
+ console.log('\\n%%b-' + testInfo.retry);
+ expect(testInfo.retry).toBe(2);
+ });
+ `,
+ }, { workers: 1 });
+ expect(result.exitCode).toBe(0);
+ expect(result.flaky).toBe(2);
+ expect(result.results.length).toBe(7);
+ // First attempts run before any retry, and each retry round is interleaved.
+ expect(result.outputLines).toEqual([
+ 'a-0',
+ 'b-0',
+ 'a-1',
+ 'b-1',
+ 'a-2',
+ 'b-2',
+ 'a-3',
+ ]);
+});