Playwright is a framework for Web Testing and Automation. It allows testing Chromium, Firefox and WebKit with a single API.
TypeScript automationchromechromiume2e-testingelectronend-to-end-testingfirefoxjavascriptplaywrighttesttest-automationtestingtesting-toolswebwebkit
feat(test): add retryStrategy option to defer retries (#41297)
Dmitry Gozmana9736eef2c0d
diff --git a/docs/src/test-api/class-testconfig.md b/docs/src/test-api/class-testconfig.mdindex 88b059a35a..169b8c943f 100644--- a/docs/src/test-api/class-testconfig.md+++ b/docs/src/test-api/class-testconfig.md@@ -515,6 +515,27 @@ export default defineConfig({});```+## property: TestConfig.retryStrategy+* since: v1.62+- type: ?<[RetryStrategy]<"immediate"|"deferred">>++Controls when failed tests are retried. Defaults to `'immediate'`.+* `'immediate'` - A failed test is retried as soon as a worker is available, interleaved with the rest of the run. This is the default.+* `'deferred'` - Retries are run only after all tests have had their first attempt, in parallel up to the configured number of [workers](#test-config-workers).++Learn more about [test retries](../test-retries.md#retries).++**Usage**++```js title="playwright.config.ts"+import { defineConfig } from '@playwright/test';++export default defineConfig({+ retries: 2,+ retryStrategy: 'deferred',+});+```+## property: TestConfig.shard* since: v1.10- type: ?<[null]|[Object]>diff --git a/packages/playwright/src/common/config.ts b/packages/playwright/src/common/config.tsindex e521d61c49..4dd22b11d8 100644--- a/packages/playwright/src/common/config.ts+++ b/packages/playwright/src/common/config.ts@@ -48,6 +48,7 @@ export class FullConfigInternal {readonly projects: FullProjectInternal[] = [];readonly singleTSConfigPath?: string;readonly captureGitInfo: Config['captureGitInfo'];+ readonly retryStrategy: 'immediate' | 'deferred';defineConfigWasUsed = false;globalSetups: string[] = [];@@ -67,6 +68,7 @@ export class FullConfigInternal {this.plugins = (privateConfiguration?.plugins || []).map((p: any) => ({ factory: p }));this.singleTSConfigPath = pathResolve(configDir, userConfig.tsconfig);this.captureGitInfo = userConfig.captureGitInfo;+ this.retryStrategy = takeFirst(userConfig.retryStrategy, 'immediate');this.globalSetups = (Array.isArray(userConfig.globalSetup) ? userConfig.globalSetup : [userConfig.globalSetup]).map(s => resolveScript(s, configDir)).filter(script => script !== undefined);this.globalTeardowns = (Array.isArray(userConfig.globalTeardown) ? userConfig.globalTeardown : [userConfig.globalTeardown]).map(s => resolveScript(s, configDir)).filter(script => script !== undefined);diff --git a/packages/playwright/src/common/configLoader.ts b/packages/playwright/src/common/configLoader.tsindex 32f3a56217..9beee268ca 100644--- a/packages/playwright/src/common/configLoader.ts+++ b/packages/playwright/src/common/configLoader.ts@@ -243,6 +243,11 @@ function validateConfig(file: string, config: Config) {throw errorWithFile(file, `config.updateSnapshots must be one of "all", "changed", "missing" or "none"`);}+ if ('retryStrategy' in config && config.retryStrategy !== undefined) {+ if (typeof config.retryStrategy !== 'string' || !['immediate', 'deferred'].includes(config.retryStrategy))+ throw errorWithFile(file, `config.retryStrategy must be one of "immediate" or "deferred"`);+ }+if ('tsconfig' in config && config.tsconfig !== undefined) {if (typeof config.tsconfig !== 'string')throw errorWithFile(file, `config.tsconfig must be a string`);diff --git a/packages/playwright/src/runner/dispatcher.ts b/packages/playwright/src/runner/dispatcher.tsindex 29f4f4e84e..77cb0aa86c 100644--- a/packages/playwright/src/runner/dispatcher.ts+++ b/packages/playwright/src/runner/dispatcher.ts@@ -152,7 +152,10 @@ export class Dispatcher {// 5. Possibly queue a new job with leftover tests and/or retries.if (!this._isStopped && result.newJob) {- this._queue.unshift(result.newJob);+ if (this._testRun.config.retryStrategy === 'deferred')+ this._queue.push(result.newJob);+ else+ this._queue.unshift(result.newJob);this._updateCounterForWorkerHash(result.newJob.workerHash, +1);}}diff --git a/packages/playwright/types/test.d.ts b/packages/playwright/types/test.d.tsindex c2037da8f8..ab8160cd39 100644--- a/packages/playwright/types/test.d.ts+++ b/packages/playwright/types/test.d.ts@@ -1640,6 +1640,30 @@ interface TestConfig<TestArgs = {}, WorkerArgs = {}> {*/retries?: number;+ /**+ * Controls when failed tests are retried. Defaults to `'immediate'`.+ * - `'immediate'` - A failed test is retried as soon as a worker is available, interleaved with the rest of the+ * run. This is the default.+ * - `'deferred'` - Retries are run only after all tests have had their first attempt, in parallel up to the+ * configured number of [workers](#test-config-workers).+ *+ * Learn more about [test retries](https://playwright.dev/docs/test-retries#retries).+ *+ * **Usage**+ *+ * ```js+ * // playwright.config.ts+ * import { defineConfig } from '@playwright/test';+ *+ * export default defineConfig({+ * retries: 2,+ * retryStrategy: 'deferred',+ * });+ * ```+ *+ */+ retryStrategy?: "immediate"|"deferred";+/*** Shard tests and execute only the selected shard. Specify in the one-based form like `{ total: 5, current: 2 }`.*diff --git a/tests/playwright-test/retry.spec.ts b/tests/playwright-test/retry.spec.tsindex c82a61b20f..1800eaded0 100644--- a/tests/playwright-test/retry.spec.ts+++ b/tests/playwright-test/retry.spec.ts@@ -262,3 +262,39 @@ test('failed and skipped on retry should be marked as flaky', async ({ runInlineexpect(result.output).toContain('Failed on first run');expect(result.report.suites[0].specs[0].tests[0].annotations).toEqual([{ type: 'skip', description: 'Skipped on first retry', location: expect.anything() }]);});++test('should defer retries to the end of the run', async ({ runInlineTest }) => {+ const result = await runInlineTest({+ 'playwright.config.js': `+ module.exports = { retries: 3, retryStrategy: 'deferred' };+ `,+ 'a.test.js': `+ import { test, expect } from '@playwright/test';+ test('a', ({}, testInfo) => {+ console.log('\\n%%a-' + testInfo.retry);+ expect(testInfo.retry).toBe(3);+ });+ `,+ 'b.test.js': `+ import { test, expect } from '@playwright/test';+ test.describe.configure({ retries: 2 });+ test('b', ({}, testInfo) => {+ console.log('\\n%%b-' + testInfo.retry);+ expect(testInfo.retry).toBe(2);+ });+ `,+ }, { workers: 1 });+ expect(result.exitCode).toBe(0);+ expect(result.flaky).toBe(2);+ expect(result.results.length).toBe(7);+ // First attempts run before any retry, and each retry round is interleaved.+ expect(result.outputLines).toEqual([+ 'a-0',+ 'b-0',+ 'a-1',+ 'b-1',+ 'a-2',+ 'b-2',+ 'a-3',+ ]);+});