Playwright is a framework for Web Testing and Automation. It allows testing Chromium, Firefox and WebKit with a single API.
TypeScript automationchromechromiume2e-testingelectronend-to-end-testingfirefoxjavascriptplaywrighttesttest-automationtestingtesting-toolswebwebkit
devops(bidi): publish BiDi reports to a git branch instead of Azure (#42760)
Yury Semikhatsky63919674b13b
diff --git a/.github/workflows/tests_bidi.yml b/.github/workflows/tests_bidi.ymlindex 342207e7ab..1e68db76ef 100644--- a/.github/workflows/tests_bidi.yml+++ b/.github/workflows/tests_bidi.yml@@ -24,10 +24,8 @@ env:jobs:test_bidi:name: BiDi- environment: ${{ github.event_name == 'push' && 'allow-uploading-flakiness-results' || null }}runs-on: ubuntu-24.04permissions:- id-token: write # This is required for OIDC login (azure/login) to succeedcontents: read # This is required for actions/checkout to succeedstrategy:fail-fast: false@@ -76,19 +74,53 @@ jobs:with:job_name: ${{ matrix.channel }}- - name: Azure Login- if: ${{ !cancelled() && github.ref == 'refs/heads/main' }}- uses: azure/login@7ddb5af1ef8758cf1353cf3b42f940aee27ba21c # v3.0.2- with:- client-id: ${{ secrets.AZURE_BLOB_REPORTS_CLIENT_ID }}- tenant-id: ${{ secrets.AZURE_BLOB_REPORTS_TENANT_ID }}- subscription-id: ${{ secrets.AZURE_BLOB_REPORTS_SUBSCRIPTION_ID }}+ - name: Add report to the job summary+ if: ${{ !cancelled() && hashFiles('test-results/report.md') != '' }}+ run: cat test-results/report.md >> "$GITHUB_STEP_SUMMARY"- - name: Upload report.csv to Azure- if: ${{ !cancelled() && github.ref == 'refs/heads/main' }}+ publish_reports:+ name: Publish reports+ needs: test_bidi+ if: ${{ !cancelled() && github.ref == 'refs/heads/main' && github.repository == 'microsoft/playwright' }}+ runs-on: ubuntu-24.04+ permissions:+ contents: write # This is required to push to the bidi-reports branch+ steps:+ - name: Download csv reports+ uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1+ with:+ pattern: csv-report-*+ path: csv-reports+ - name: Publish the reports to the bidi-reports branchrun: |- REPORT_DIR='bidi-reports'- azcopy cp "./test-results/report.csv" "https://mspwblobreport.blob.core.windows.net/\$web/$REPORT_DIR/${{ matrix.channel }}.csv"- echo "Report url: https://mspwblobreport.z1.web.core.windows.net/$REPORT_DIR/${{ matrix.channel }}.csv"+ git init -q "$RUNNER_TEMP/reports"+ cd "$RUNNER_TEMP/reports"+ git remote add origin "https://x-access-token:[email protected]/$GITHUB_REPOSITORY.git"+ # Continue the branch history, or start it on the very first run. An+ # initial commit with no fetched parent is what makes the branch orphan.+ git fetch --depth=1 origin bidi-reports && git reset --hard FETCH_HEAD || true+ echo 'Nightly BiDi test reports, published by .github/workflows/tests_bidi.yml.' > README.md+ channels=()+ for dir in "$GITHUB_WORKSPACE"/csv-reports/csv-report-*/; do+ channel=${dir%/}+ channel=${channel##*/csv-report-}+ cp "$dir/report.csv" "$channel.csv"+ channels+=("$channel")+ done+ git add -A+ if git diff --cached --quiet; then+ echo "No changes since the last run."+ exit 0+ fi+ git commit -q -m "bidi: results for ${GITHUB_SHA:0:9}"+ git push origin HEAD:bidi-reports+ for channel in "${channels[@]}"; do+ echo "Report url: https://github.com/$GITHUB_REPOSITORY/blob/bidi-reports/$channel.csv"+ echo "Raw report url: https://raw.githubusercontent.com/$GITHUB_REPOSITORY/bidi-reports/$channel.csv"+ doneenv:- AZCOPY_AUTO_LOGIN_TYPE: AZCLI+ GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}+ GIT_AUTHOR_NAME: microsoft-playwright-automation[bot]+ GIT_AUTHOR_EMAIL: 203992400+microsoft-playwright-automation[bot]@users.noreply.github.com+ GIT_COMMITTER_NAME: microsoft-playwright-automation[bot]+ GIT_COMMITTER_EMAIL: 203992400+microsoft-playwright-automation[bot]@users.noreply.github.comdiff --git a/tests/bidi/csvReporter.ts b/tests/bidi/csvReporter.tsindex b84ca9e81d..d5ef17cdf0 100644--- a/tests/bidi/csvReporter.ts+++ b/tests/bidi/csvReporter.ts@@ -24,9 +24,12 @@ import path from 'path';type ReporterOptions = {outputFile?: string,+ markdownFile?: string,configDir: string,};+const header = ['Test Name', 'Expected Status', 'Status', 'Error Message'];+class CsvReporter implements Reporter {private _suite: Suite;private _options: ReporterOptions;@@ -41,7 +44,7 @@ class CsvReporter implements Reporter {}onEnd(result: FullResult) {- const rows = [['Test Name', 'Expected Status', 'Status', 'Error Message']];+ const rows: string[][] = [];for (const project of this._suite.suites) {for (const file of project.suites) {for (const test of file.allTests()) {@@ -51,7 +54,7 @@ class CsvReporter implements Reporter {continue;const row = [];const [, , , ...titles] = test.titlePath();- row.push(csvEscape(`${file.title} › ${titles.join(' › ')}`));+ row.push(`${file.title} › ${titles.join(' › ')}`);row.push(test.expectedStatus);row.push(test.outcome());if (fixme) {@@ -60,11 +63,11 @@ class CsvReporter implements Reporter {const result = test.results.find(r => r.error);if (result) {const errorMessage = stripAnsi(result.error?.message.replace(/\s+/g, ' ').trim().substring(0, 1024) ?? '');- row.push(csvEscape(errorMessage));+ row.push(errorMessage);} else {const fail = test.annotations.find(a => a.type === 'fail');if (fail)- row.push(csvEscape(`Should have failed: ${fail.description}`));+ row.push(`Should have failed: ${fail.description}`);elserow.push('');}@@ -73,11 +76,16 @@ class CsvReporter implements Reporter {}}}- const csv = rows.map(r => r.join(',')).join('\n');const reportFile = path.resolve(this._options.configDir, this._options.outputFile || 'test-results.csv');+ const markdownFile = this._options.markdownFile && path.resolve(this._options.configDir, this._options.markdownFile);this._pendingWrite = (async () => {await fs.promises.mkdir(path.dirname(reportFile), { recursive: true });+ const csv = [header, ...rows].map(r => r.map(csvEscape).join(',')).join('\n');await fs.promises.writeFile(reportFile, csv);+ if (markdownFile) {+ await fs.promises.mkdir(path.dirname(markdownFile), { recursive: true });+ await fs.promises.writeFile(markdownFile, markdownTable(rows));+ }})();}@@ -96,4 +104,24 @@ function csvEscape(str) {return str;}+// GitHub job summaries are capped at 1MiB, so keep the rendered table bounded.+const maxMarkdownRows = 500;++function markdownTable(rows: string[][]): string {+ const lines = [+ `### ${rows.length} failing tests`,+ '',+ `| ${header.join(' | ')} |`,+ `| ${header.map(() => '---').join(' | ')} |`,+ ...rows.slice(0, maxMarkdownRows).map(row => `| ${row.map(markdownEscape).join(' | ')} |`),+ ];+ if (rows.length > maxMarkdownRows)+ lines.push('', `_...and ${rows.length - maxMarkdownRows} more, see the csv report._`);+ return lines.join('\n') + '\n';+}++function markdownEscape(str: string): string {+ return str.replace(/[\\|`<>]/g, c => '\\' + c);+}+export default CsvReporter;diff --git a/tests/bidi/playwright.config.ts b/tests/bidi/playwright.config.tsindex afa43752e1..83996f2bde 100644--- a/tests/bidi/playwright.config.ts+++ b/tests/bidi/playwright.config.ts@@ -46,7 +46,7 @@ const reporters = () => {hasDebugOutput ? ['list'] : ['dot'],['blob'],['../config/parquetReporter.ts'],- ['./csvReporter', { outputFile: path.join(outputDir, 'report.csv') }],+ ['./csvReporter', { outputFile: path.join(outputDir, 'report.csv'), markdownFile: path.join(outputDir, 'report.md') }],['./expectationReporter', { rebase: false }],] : [['html', { open: 'on-failure' }],