Skip to content

Playwright is a framework for Web Testing and Automation. It allows testing Chromium, Firefox and WebKit with a single API.

chore(bidi): filter tests via preprocessSuite (#41710)

Simon Knotte6a5512bacab
diff --git a/tests/bidi/expectationReporter.ts b/tests/bidi/expectationReporter.ts
index d88875f031..d58c6be80d 100644
--- a/tests/bidi/expectationReporter.ts
+++ b/tests/bidi/expectationReporter.ts
@@ -18,8 +18,9 @@ import type {
FullConfig, FullResult, Reporter, Suite, TestCase
} from '@playwright/test/reporter';
import fs from 'fs';
-import { parseBidiExpectations as parseExpectations, projectExpectationPath } from './expectationUtil';
-import type { TestExpectation } from './expectationUtil';
+import path from 'path';
+
+type TestExpectation = 'unknown' | 'flaky' | 'pass' | 'fail' | 'timeout';
type ReporterOptions = {
rebase?: boolean;
@@ -34,6 +35,21 @@ class ExpectationReporter implements Reporter {
this._options = options;
}
+ async preprocessSuite(config: FullConfig, suite: Suite) {
+ if (!process.env.PWTEST_USE_BIDI_EXPECTATIONS)
+ return;
+ if (this._options.rebase)
+ return;
+ for (const project of suite.suites) {
+ const expectations = await parseExpectations(project.title);
+ for (const test of project.allTests()) {
+ const expectation = expectations.get(expectationKey(test));
+ if (expectation && ['flaky', 'fail', 'timeout'].includes(expectation))
+ test.fixme(`marked as ${expectation} in bidi expectations`);
+ }
+ }
+ }
+
onBegin(config: FullConfig, suite: Suite) {
this._suite = suite;
}
@@ -54,8 +70,7 @@ class ExpectationReporter implements Reporter {
const expectations = await parseExpectations(project.title);
for (const test of project.allTests()) {
const outcome = getOutcome(test);
- // Strip root and project names.
- const key = test.titlePath().slice(2).join(' › ');
+ const key = expectationKey(test);
if (outcome === 'timeout')
expectations.set(key, outcome);
else if (expectations.has(key) && test.outcome() !== 'skipped')
@@ -87,4 +102,31 @@ function getOutcome(test: TestCase): TestExpectation {
return 'unknown';
}
+function expectationKey(test: TestCase): string {
+ return test.titlePath().slice(2).join(' › ');
+}
+
+async function parseExpectations(projectName: string): Promise<Map<string, TestExpectation>> {
+ const filePath = projectExpectationPath(projectName);
+ try {
+ await fs.promises.access(filePath);
+ } catch (e) {
+ return new Map();
+ }
+ const content = await fs.promises.readFile(filePath);
+ const pairs = content.toString().split('\n').map(line => {
+ const match = /(?<titlePath>.+) \[(?<expectation>[^\]]+)\]$/.exec(line);
+ if (!match) {
+ console.error('Bad expectation line: ' + line);
+ return undefined;
+ }
+ return [match.groups!.titlePath, match.groups!.expectation];
+ }).filter(Boolean) as [string, TestExpectation][];
+ return new Map(pairs);
+}
+
+function projectExpectationPath(project: string): string {
+ return path.join(__dirname, 'expectations', project + '.txt');
+}
+
export default ExpectationReporter;
diff --git a/tests/bidi/expectationUtil.ts b/tests/bidi/expectationUtil.ts
deleted file mode 100644
index ab9f36ff84..0000000000
--- a/tests/bidi/expectationUtil.ts
+++ /dev/null
@@ -1,57 +0,0 @@
-/**
- * Copyright (c) Microsoft Corporation.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-import fs from 'fs';
-import path from 'path';
-import type { TestInfo } from 'playwright/test';
-
-export type TestExpectation = 'unknown' | 'flaky' | 'pass' | 'fail' | 'timeout';
-
-type ShouldSkipPredicate = (info: TestInfo) => boolean;
-
-export async function createSkipTestPredicate(projectName: string): Promise<ShouldSkipPredicate> {
- if (!process.env.PWTEST_USE_BIDI_EXPECTATIONS)
- return () => false;
- const expectationsMap = await parseBidiExpectations(projectName);
- return (info: TestInfo) => {
- const key = info.titlePath.join(' › ');
- const expectation = expectationsMap.get(key);
- return ['flaky', 'fail', 'timeout'].includes(expectation);
- };
-}
-
-export async function parseBidiExpectations(projectName: string): Promise<Map<string, TestExpectation>> {
- const filePath = projectExpectationPath(projectName);
- try {
- await fs.promises.access(filePath);
- } catch (e) {
- return new Map();
- }
- const content = await fs.promises.readFile(filePath);
- const pairs = content.toString().split('\n').map(line => {
- const match = /(?<titlePath>.+) \[(?<expectation>[^\]]+)\]$/.exec(line);
- if (!match) {
- console.error('Bad expectation line: ' + line);
- return undefined;
- }
- return [match.groups!.titlePath, match.groups!.expectation];
- }).filter(Boolean) as [string, TestExpectation][];
- return new Map(pairs);
-}
-
-export function projectExpectationPath(project: string): string {
- return path.join(__dirname, 'expectations', project + '.txt');
-}
diff --git a/tests/bidi/playwright.config.ts b/tests/bidi/playwright.config.ts
index 5ee75a3c02..3dfe1d6786 100644
--- a/tests/bidi/playwright.config.ts
+++ b/tests/bidi/playwright.config.ts
@@ -48,6 +48,7 @@ const reporters = () => {
['blob'],
['../config/parquetReporter.ts'],
['./csvReporter', { outputFile: path.join(outputDir, 'report.csv') }],
+ ['./expectationReporter', { rebase: false }],
] : [
['html', { open: 'on-failure' }],
['./csvReporter', { outputFile: path.join(outputDir, 'report.csv') }],
diff --git a/tests/config/browserTest.ts b/tests/config/browserTest.ts
index 2ed18b3c70..c350201b64 100644
--- a/tests/config/browserTest.ts
+++ b/tests/config/browserTest.ts
@@ -21,12 +21,10 @@ import { baseTest } from './baseTest';
import { RunServer, RemoteServer } from './remoteServer';
import { utils } from '../../packages/playwright-core/lib/coreBundle';
import { isBidiChannel, parseHar } from '../config/utils';
-import { createSkipTestPredicate } from '../bidi/expectationUtil';
import type { PageTestFixtures, PageWorkerFixtures } from '../page/pageTestApi';
import type { RemoteServerOptions, PlaywrightServer } from './remoteServer';
import type { BrowserContext, BrowserContextOptions, BrowserType, Page } from 'playwright-core';
import type { Log } from '../../packages/trace/src/har';
-import type { TestInfo } from '@playwright/test';
const { removeFolders, hostPlatform } = utils;
@@ -41,7 +39,6 @@ export type BrowserTestWorkerFixtures = PageWorkerFixtures & {
isHeadlessShell: boolean;
isFrozenWebkit: boolean;
isBidi: boolean;
- bidiTestSkipPredicate: (info: TestInfo) => boolean;
};
interface StartRemoteServer {
@@ -55,7 +52,6 @@ type BrowserTestTestFixtures = PageTestFixtures & {
startRemoteServer: StartRemoteServer;
contextFactory: (options?: BrowserContextOptions) => Promise<BrowserContext>;
pageWithHar(options?: { outputPath?: string, content?: 'embed' | 'attach' | 'omit', omitContent?: boolean }): Promise<{ context: BrowserContext, page: Page, getLog: () => Promise<Log>, getZip: () => Promise<Map<string, Buffer>> }>
- autoSkipBidiTest: void;
};
type ContextFactory = (options?: BrowserContextOptions) => Promise<{ context: BrowserContext, close: () => Promise<void> }>;
@@ -214,16 +210,6 @@ const test = baseTest.extend<BrowserTestTestFixtures & { _contextFactory: Contex
};
await use(pageWithHar);
},
-
- bidiTestSkipPredicate: [async ({ }, run) => {
- const filter = await createSkipTestPredicate(test.info().project.name);
- await run(filter);
- }, { scope: 'worker' }],
-
- autoSkipBidiTest: [async ({ bidiTestSkipPredicate }, run) => {
- test.fixme(bidiTestSkipPredicate(test.info()), 'marked as timeout in bidi expectations');
- await run();
- }, { auto: true, scope: 'test' }],
});
export const playwrightTest = test;