Playwright is a framework for Web Testing and Automation. It allows testing Chromium, Firefox and WebKit with a single API.
TypeScript automationchromechromiume2e-testingelectronend-to-end-testingfirefoxjavascriptplaywrighttesttest-automationtestingtesting-toolswebwebkit
chore(bidi): filter tests via preprocessSuite (#41710)
Simon Knotte6a5512bacab
diff --git a/tests/bidi/expectationReporter.ts b/tests/bidi/expectationReporter.tsindex d88875f031..d58c6be80d 100644--- a/tests/bidi/expectationReporter.ts+++ b/tests/bidi/expectationReporter.ts@@ -18,8 +18,9 @@ import type {FullConfig, FullResult, Reporter, Suite, TestCase} from '@playwright/test/reporter';import fs from 'fs';-import { parseBidiExpectations as parseExpectations, projectExpectationPath } from './expectationUtil';-import type { TestExpectation } from './expectationUtil';+import path from 'path';++type TestExpectation = 'unknown' | 'flaky' | 'pass' | 'fail' | 'timeout';type ReporterOptions = {rebase?: boolean;@@ -34,6 +35,21 @@ class ExpectationReporter implements Reporter {this._options = options;}+ async preprocessSuite(config: FullConfig, suite: Suite) {+ if (!process.env.PWTEST_USE_BIDI_EXPECTATIONS)+ return;+ if (this._options.rebase)+ return;+ for (const project of suite.suites) {+ const expectations = await parseExpectations(project.title);+ for (const test of project.allTests()) {+ const expectation = expectations.get(expectationKey(test));+ if (expectation && ['flaky', 'fail', 'timeout'].includes(expectation))+ test.fixme(`marked as ${expectation} in bidi expectations`);+ }+ }+ }+onBegin(config: FullConfig, suite: Suite) {this._suite = suite;}@@ -54,8 +70,7 @@ class ExpectationReporter implements Reporter {const expectations = await parseExpectations(project.title);for (const test of project.allTests()) {const outcome = getOutcome(test);- // Strip root and project names.- const key = test.titlePath().slice(2).join(' › ');+ const key = expectationKey(test);if (outcome === 'timeout')expectations.set(key, outcome);else if (expectations.has(key) && test.outcome() !== 'skipped')@@ -87,4 +102,31 @@ function getOutcome(test: TestCase): TestExpectation {return 'unknown';}+function expectationKey(test: TestCase): string {+ return test.titlePath().slice(2).join(' › ');+}++async function parseExpectations(projectName: string): Promise<Map<string, TestExpectation>> {+ const filePath = projectExpectationPath(projectName);+ try {+ await fs.promises.access(filePath);+ } catch (e) {+ return new Map();+ }+ const content = await fs.promises.readFile(filePath);+ const pairs = content.toString().split('\n').map(line => {+ const match = /(?<titlePath>.+) \[(?<expectation>[^\]]+)\]$/.exec(line);+ if (!match) {+ console.error('Bad expectation line: ' + line);+ return undefined;+ }+ return [match.groups!.titlePath, match.groups!.expectation];+ }).filter(Boolean) as [string, TestExpectation][];+ return new Map(pairs);+}++function projectExpectationPath(project: string): string {+ return path.join(__dirname, 'expectations', project + '.txt');+}+export default ExpectationReporter;diff --git a/tests/bidi/expectationUtil.ts b/tests/bidi/expectationUtil.tsdeleted file mode 100644index ab9f36ff84..0000000000--- a/tests/bidi/expectationUtil.ts+++ /dev/null@@ -1,57 +0,0 @@-/**- * Copyright (c) Microsoft Corporation.- *- * Licensed under the Apache License, Version 2.0 (the "License");- * you may not use this file except in compliance with the License.- * You may obtain a copy of the License at- *- * http://www.apache.org/licenses/LICENSE-2.0- *- * Unless required by applicable law or agreed to in writing, software- * distributed under the License is distributed on an "AS IS" BASIS,- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.- * See the License for the specific language governing permissions and- * limitations under the License.- */--import fs from 'fs';-import path from 'path';-import type { TestInfo } from 'playwright/test';--export type TestExpectation = 'unknown' | 'flaky' | 'pass' | 'fail' | 'timeout';--type ShouldSkipPredicate = (info: TestInfo) => boolean;--export async function createSkipTestPredicate(projectName: string): Promise<ShouldSkipPredicate> {- if (!process.env.PWTEST_USE_BIDI_EXPECTATIONS)- return () => false;- const expectationsMap = await parseBidiExpectations(projectName);- return (info: TestInfo) => {- const key = info.titlePath.join(' › ');- const expectation = expectationsMap.get(key);- return ['flaky', 'fail', 'timeout'].includes(expectation);- };-}--export async function parseBidiExpectations(projectName: string): Promise<Map<string, TestExpectation>> {- const filePath = projectExpectationPath(projectName);- try {- await fs.promises.access(filePath);- } catch (e) {- return new Map();- }- const content = await fs.promises.readFile(filePath);- const pairs = content.toString().split('\n').map(line => {- const match = /(?<titlePath>.+) \[(?<expectation>[^\]]+)\]$/.exec(line);- if (!match) {- console.error('Bad expectation line: ' + line);- return undefined;- }- return [match.groups!.titlePath, match.groups!.expectation];- }).filter(Boolean) as [string, TestExpectation][];- return new Map(pairs);-}--export function projectExpectationPath(project: string): string {- return path.join(__dirname, 'expectations', project + '.txt');-}diff --git a/tests/bidi/playwright.config.ts b/tests/bidi/playwright.config.tsindex 5ee75a3c02..3dfe1d6786 100644--- a/tests/bidi/playwright.config.ts+++ b/tests/bidi/playwright.config.ts@@ -48,6 +48,7 @@ const reporters = () => {['blob'],['../config/parquetReporter.ts'],['./csvReporter', { outputFile: path.join(outputDir, 'report.csv') }],+ ['./expectationReporter', { rebase: false }],] : [['html', { open: 'on-failure' }],['./csvReporter', { outputFile: path.join(outputDir, 'report.csv') }],diff --git a/tests/config/browserTest.ts b/tests/config/browserTest.tsindex 2ed18b3c70..c350201b64 100644--- a/tests/config/browserTest.ts+++ b/tests/config/browserTest.ts@@ -21,12 +21,10 @@ import { baseTest } from './baseTest';import { RunServer, RemoteServer } from './remoteServer';import { utils } from '../../packages/playwright-core/lib/coreBundle';import { isBidiChannel, parseHar } from '../config/utils';-import { createSkipTestPredicate } from '../bidi/expectationUtil';import type { PageTestFixtures, PageWorkerFixtures } from '../page/pageTestApi';import type { RemoteServerOptions, PlaywrightServer } from './remoteServer';import type { BrowserContext, BrowserContextOptions, BrowserType, Page } from 'playwright-core';import type { Log } from '../../packages/trace/src/har';-import type { TestInfo } from '@playwright/test';const { removeFolders, hostPlatform } = utils;@@ -41,7 +39,6 @@ export type BrowserTestWorkerFixtures = PageWorkerFixtures & {isHeadlessShell: boolean;isFrozenWebkit: boolean;isBidi: boolean;- bidiTestSkipPredicate: (info: TestInfo) => boolean;};interface StartRemoteServer {@@ -55,7 +52,6 @@ type BrowserTestTestFixtures = PageTestFixtures & {startRemoteServer: StartRemoteServer;contextFactory: (options?: BrowserContextOptions) => Promise<BrowserContext>;pageWithHar(options?: { outputPath?: string, content?: 'embed' | 'attach' | 'omit', omitContent?: boolean }): Promise<{ context: BrowserContext, page: Page, getLog: () => Promise<Log>, getZip: () => Promise<Map<string, Buffer>> }>- autoSkipBidiTest: void;};type ContextFactory = (options?: BrowserContextOptions) => Promise<{ context: BrowserContext, close: () => Promise<void> }>;@@ -214,16 +210,6 @@ const test = baseTest.extend<BrowserTestTestFixtures & { _contextFactory: Contex};await use(pageWithHar);},-- bidiTestSkipPredicate: [async ({ }, run) => {- const filter = await createSkipTestPredicate(test.info().project.name);- await run(filter);- }, { scope: 'worker' }],-- autoSkipBidiTest: [async ({ bidiTestSkipPredicate }, run) => {- test.fixme(bidiTestSkipPredicate(test.info()), 'marked as timeout in bidi expectations');- await run();- }, { auto: true, scope: 'test' }],});export const playwrightTest = test;