Skip to content

Playwright is a framework for Web Testing and Automation. It allows testing Chromium, Firefox and WebKit with a single API.

fix: report unknown test IDs in serial mode when test titles change (#39376)

Yury Semikhatskyc99918d013f3
diff --git a/packages/playwright/src/worker/workerMain.ts b/packages/playwright/src/worker/workerMain.ts
index c2d1a93858..bdd46abaf0 100644
--- a/packages/playwright/src/worker/workerMain.ts
+++ b/packages/playwright/src/worker/workerMain.ts
@@ -225,25 +225,32 @@ export class WorkerMain extends ProcessRunner {
const suite = bindFileSuiteToProject(this._project, fileSuite);
if (this._params.repeatEachIndex)
applyRepeatEachIndex(this._project, suite, this._params.repeatEachIndex);
- const hasEntries = filterTestsRemoveEmptySuites(suite, test => entries.has(test.id));
- if (hasEntries) {
- this._poolBuilder.buildPools(suite);
- this._activeSuites = new Map();
- this._didRunFullCleanup = false;
- const tests = suite.allTests();
- for (let i = 0; i < tests.length; i++) {
- // Do not run tests after full cleanup, because we are entirely done.
- if (this._isStopped && this._didRunFullCleanup)
- break;
- const entry = entries.get(tests[i].id)!;
- entries.delete(tests[i].id);
- debugTest(`test started "${tests[i].title}"`);
- await this._runTest(tests[i], entry.retry, tests[i + 1]);
- debugTest(`test finished "${tests[i].title}"`);
- }
- } else {
- fatalUnknownTestIds = runPayload.entries.map(e => e.testId);
+ filterTestsRemoveEmptySuites(suite, test => entries.has(test.id));
+ const tests = suite.allTests();
+
+ // Collect test IDs that were not found in the worker
+ // (e.g. test titles changed between runner and worker).
+ const unknownTestIds = new Set(entries.keys());
+ for (const test of tests)
+ unknownTestIds.delete(test.id);
+ if (unknownTestIds.size) {
+ fatalUnknownTestIds = [...unknownTestIds];
void this._stop();
+ return;
+ }
+
+ this._poolBuilder.buildPools(suite);
+ this._activeSuites = new Map();
+ this._didRunFullCleanup = false;
+ for (let i = 0; i < tests.length; i++) {
+ // Do not run tests after full cleanup, because we are entirely done.
+ if (this._isStopped && this._didRunFullCleanup)
+ break;
+ const entry = entries.get(tests[i].id)!;
+ entries.delete(tests[i].id);
+ debugTest(`test started "${tests[i].title}"`);
+ await this._runTest(tests[i], entry.retry, tests[i + 1]);
+ debugTest(`test finished "${tests[i].title}"`);
}
} catch (e) {
// In theory, we should run above code without any errors.
diff --git a/tests/playwright-test/runner.spec.ts b/tests/playwright-test/runner.spec.ts
index da07cfdf09..7c43691184 100644
--- a/tests/playwright-test/runner.spec.ts
+++ b/tests/playwright-test/runner.spec.ts
@@ -368,6 +368,35 @@ test('should not hang if test suites in worker are inconsistent with runner', as
expect(countTimes(result.output, expectedError)).toBe(2); // Once per each test that was missing.
});
+test('should throw in serial mode if test suites in worker are inconsistent with runner', async ({ runInlineTest }) => {
+ const result = await runInlineTest({
+ 'playwright.config.ts': `
+ module.exports = { name: 'project-name' };
+ `,
+ 'a.spec.js': `
+ import { test, expect } from "@playwright/test";
+
+ test.describe.configure({ mode: "serial" });
+
+ const inWorker = process.env.TEST_WORKER_INDEX !== undefined;
+
+ test("first test", async () => {});
+
+ test("second test", async () => {
+ expect(false).toBeTruthy();
+ });
+
+ test("test - " + inWorker, async () => {});
+ `,
+ }, { 'workers': 1 }, { TEST_WORKER_INDEX: undefined });
+ expect.soft(result.exitCode).toBe(1);
+ expect.soft(result.passed).toBe(0);
+ expect.soft(result.failed).toBe(1);
+ expect.soft(result.skipped).toBe(0);
+ const expectedError = 'Test not found in the worker process. Make sure test title does not change.';
+ expect.soft(result.output).toContain(expectedError);
+});
+
test('sigint should stop global setup', async ({ interactWithTestRunner }) => {
test.skip(process.platform === 'win32', 'No sending SIGINT on Windows');