Playwright is a framework for Web Testing and Automation. It allows testing Chromium, Firefox and WebKit with a single API.
TypeScript automationchromechromiume2e-testingelectronend-to-end-testingfirefoxjavascriptplaywrighttesttest-automationtestingtesting-toolswebwebkit
fix(shard): fail when no tests match across all shards (#43173)
Pavel Feldman4dd2a65fe881
diff --git a/packages/playwright/src/runner/loadUtils.ts b/packages/playwright/src/runner/loadUtils.tsindex 80553cb8e2..55759c8ca4 100644--- a/packages/playwright/src/runner/loadUtils.ts+++ b/packages/playwright/src/runner/loadUtils.ts@@ -155,6 +155,7 @@ export async function createRootSuite(testRun: TestRun, errors: TestError[], shorootSuite._addSuite(buildProjectSuite(project, filteredProjectSuites.get(project)!));}}+ testRun.testCountBeforeSharding = rootSuite.allTests().length;// Complain about only.if (config.config.forbidOnly) {diff --git a/packages/playwright/src/runner/tasks.ts b/packages/playwright/src/runner/tasks.tsindex 1361860eab..462dfa5a05 100644--- a/packages/playwright/src/runner/tasks.ts+++ b/packages/playwright/src/runner/tasks.ts@@ -91,6 +91,7 @@ export class TestRun {topLevelProjects: commonConfig.FullProjectInternal[] = [];hasWorkerErrors = false;failedTestCount = 0;+ testCountBeforeSharding = 0;readonly loadFileFilters: Matcher[] = [];readonly preOnlyTestFilters: TestCaseFilter[] = [];readonly postShardTestFilters: TestCaseFilter[] = [];@@ -335,9 +336,10 @@ export function createLoadTask(mode: 'out-of-process' | 'in-process', options: {await createRootSuite(testRun, options.failOnLoadErrors ? errors : softErrors, !!options.filterOnly);// Fail when no tests.- if (options.failOnLoadErrors && !testRun.rootSuite?.allTests().length+ const testCount = testRun.config.config.shard ? testRun.testCountBeforeSharding : testRun.rootSuite?.allTests().length;+ if (options.failOnLoadErrors && !testCount&& !testRun.options.passWithNoTests- && !testRun.config.config.shard && !testRun.options.onlyChanged+ && !testRun.options.onlyChanged&& !testRun.options.testList && !testRun.options.testListInvert) {if (testRun.options.locations?.length) {throw new Error([diff --git a/tests/playwright-test/reporter-preprocess.spec.ts b/tests/playwright-test/reporter-preprocess.spec.tsindex 66a84ba4fb..6ff4e2c790 100644--- a/tests/playwright-test/reporter-preprocess.spec.ts+++ b/tests/playwright-test/reporter-preprocess.spec.ts@@ -369,6 +369,30 @@ test('skipSharding disables the built-in shard filter; preprocess sees the fullexpect(result.outputLines).toContain('begin: t0,t2');});+test('custom sharding may produce an empty shard', async ({ runInlineTest }) => {+ const result = await runInlineTest({+ 'reporter.ts': `+ class R {+ async preprocess({ suite, testRun }) {+ testRun.skipSharding();+ for (const t of suite.allTests())+ testRun.exclude(t);+ }+ }+ module.exports = R;+ `,+ 'playwright.config.ts': `module.exports = { reporter: './reporter.ts', shard: { current: 2, total: 2 } };`,+ 'a.test.ts': `+ import { test } from '@playwright/test';+ test('t0', async () => {});+ `,+ }, { reporter: '', workers: 1 });++ expect(result.exitCode).toBe(0);+ expect(result.passed).toBe(0);+ expect(result.output).not.toContain('No tests found');+});+test('multiple reporters declaring custom sharding throws', async ({ runInlineTest }) => {const result = await runInlineTest({'reporter-a.ts': `diff --git a/tests/playwright-test/shard.spec.ts b/tests/playwright-test/shard.spec.tsindex 15774e8e37..a700812e0b 100644--- a/tests/playwright-test/shard.spec.ts+++ b/tests/playwright-test/shard.spec.ts@@ -156,6 +156,25 @@ test('should not produce skipped tests for zero-sized shards', async ({ runInlinexpect(result.outputLines).toEqual([]);});+test('should fail when no tests match across all shards', async ({ runInlineTest }) => {+ const result = await runInlineTest(tests, { shard: '1/2', grep: 'does-not-match' });+ expect(result.exitCode).toBe(1);+ expect(result.output).toContain('No tests found');+});++test('should fail when no test files match across all shards', async ({ runInlineTest }) => {+ const result = await runInlineTest(tests, { shard: '1/2' }, {}, { additionalArgs: ['does-not-match'] });+ expect(result.exitCode).toBe(1);+ expect(result.output).toContain('No tests found');+ expect(result.output).toContain('Make sure that arguments are regular expressions matching test files');+});++test('should pass with no tests across all shards and --pass-with-no-tests', async ({ runInlineTest }) => {+ const result = await runInlineTest(tests, { 'shard': '1/2', 'grep': 'does-not-match', 'pass-with-no-tests': true });+ expect(result.exitCode).toBe(0);+ expect(result.passed).toBe(0);+});+test('should respect shard=1/2 in config', async ({ runInlineTest }) => {const result = await runInlineTest({...tests,