Skip to content

Playwright is a framework for Web Testing and Automation. It allows testing Chromium, Firefox and WebKit with a single API.

devops(bidi): publish BiDi reports to a git branch instead of Azure (#42760)

Yury Semikhatsky63919674b13b
diff --git a/.github/workflows/tests_bidi.yml b/.github/workflows/tests_bidi.yml
index 342207e7ab..1e68db76ef 100644
--- a/.github/workflows/tests_bidi.yml
+++ b/.github/workflows/tests_bidi.yml
@@ -24,10 +24,8 @@ env:
jobs:
test_bidi:
name: BiDi
- environment: ${{ github.event_name == 'push' && 'allow-uploading-flakiness-results' || null }}
runs-on: ubuntu-24.04
permissions:
- id-token: write # This is required for OIDC login (azure/login) to succeed
contents: read # This is required for actions/checkout to succeed
strategy:
fail-fast: false
@@ -76,19 +74,53 @@ jobs:
with:
job_name: ${{ matrix.channel }}
- - name: Azure Login
- if: ${{ !cancelled() && github.ref == 'refs/heads/main' }}
- uses: azure/login@7ddb5af1ef8758cf1353cf3b42f940aee27ba21c # v3.0.2
- with:
- client-id: ${{ secrets.AZURE_BLOB_REPORTS_CLIENT_ID }}
- tenant-id: ${{ secrets.AZURE_BLOB_REPORTS_TENANT_ID }}
- subscription-id: ${{ secrets.AZURE_BLOB_REPORTS_SUBSCRIPTION_ID }}
+ - name: Add report to the job summary
+ if: ${{ !cancelled() && hashFiles('test-results/report.md') != '' }}
+ run: cat test-results/report.md >> "$GITHUB_STEP_SUMMARY"
- - name: Upload report.csv to Azure
- if: ${{ !cancelled() && github.ref == 'refs/heads/main' }}
+ publish_reports:
+ name: Publish reports
+ needs: test_bidi
+ if: ${{ !cancelled() && github.ref == 'refs/heads/main' && github.repository == 'microsoft/playwright' }}
+ runs-on: ubuntu-24.04
+ permissions:
+ contents: write # This is required to push to the bidi-reports branch
+ steps:
+ - name: Download csv reports
+ uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
+ with:
+ pattern: csv-report-*
+ path: csv-reports
+ - name: Publish the reports to the bidi-reports branch
run: |
- REPORT_DIR='bidi-reports'
- azcopy cp "./test-results/report.csv" "https://mspwblobreport.blob.core.windows.net/\$web/$REPORT_DIR/${{ matrix.channel }}.csv"
- echo "Report url: https://mspwblobreport.z1.web.core.windows.net/$REPORT_DIR/${{ matrix.channel }}.csv"
+ git init -q "$RUNNER_TEMP/reports"
+ cd "$RUNNER_TEMP/reports"
+ git remote add origin "https://x-access-token:[email protected]/$GITHUB_REPOSITORY.git"
+ # Continue the branch history, or start it on the very first run. An
+ # initial commit with no fetched parent is what makes the branch orphan.
+ git fetch --depth=1 origin bidi-reports && git reset --hard FETCH_HEAD || true
+ echo 'Nightly BiDi test reports, published by .github/workflows/tests_bidi.yml.' > README.md
+ channels=()
+ for dir in "$GITHUB_WORKSPACE"/csv-reports/csv-report-*/; do
+ channel=${dir%/}
+ channel=${channel##*/csv-report-}
+ cp "$dir/report.csv" "$channel.csv"
+ channels+=("$channel")
+ done
+ git add -A
+ if git diff --cached --quiet; then
+ echo "No changes since the last run."
+ exit 0
+ fi
+ git commit -q -m "bidi: results for ${GITHUB_SHA:0:9}"
+ git push origin HEAD:bidi-reports
+ for channel in "${channels[@]}"; do
+ echo "Report url: https://github.com/$GITHUB_REPOSITORY/blob/bidi-reports/$channel.csv"
+ echo "Raw report url: https://raw.githubusercontent.com/$GITHUB_REPOSITORY/bidi-reports/$channel.csv"
+ done
env:
- AZCOPY_AUTO_LOGIN_TYPE: AZCLI
+ GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+ GIT_AUTHOR_NAME: microsoft-playwright-automation[bot]
+ GIT_AUTHOR_EMAIL: 203992400+microsoft-playwright-automation[bot]@users.noreply.github.com
+ GIT_COMMITTER_NAME: microsoft-playwright-automation[bot]
+ GIT_COMMITTER_EMAIL: 203992400+microsoft-playwright-automation[bot]@users.noreply.github.com
diff --git a/tests/bidi/csvReporter.ts b/tests/bidi/csvReporter.ts
index b84ca9e81d..d5ef17cdf0 100644
--- a/tests/bidi/csvReporter.ts
+++ b/tests/bidi/csvReporter.ts
@@ -24,9 +24,12 @@ import path from 'path';
type ReporterOptions = {
outputFile?: string,
+ markdownFile?: string,
configDir: string,
};
+const header = ['Test Name', 'Expected Status', 'Status', 'Error Message'];
+
class CsvReporter implements Reporter {
private _suite: Suite;
private _options: ReporterOptions;
@@ -41,7 +44,7 @@ class CsvReporter implements Reporter {
}
onEnd(result: FullResult) {
- const rows = [['Test Name', 'Expected Status', 'Status', 'Error Message']];
+ const rows: string[][] = [];
for (const project of this._suite.suites) {
for (const file of project.suites) {
for (const test of file.allTests()) {
@@ -51,7 +54,7 @@ class CsvReporter implements Reporter {
continue;
const row = [];
const [, , , ...titles] = test.titlePath();
- row.push(csvEscape(`${file.title} › ${titles.join(' › ')}`));
+ row.push(`${file.title} › ${titles.join(' › ')}`);
row.push(test.expectedStatus);
row.push(test.outcome());
if (fixme) {
@@ -60,11 +63,11 @@ class CsvReporter implements Reporter {
const result = test.results.find(r => r.error);
if (result) {
const errorMessage = stripAnsi(result.error?.message.replace(/\s+/g, ' ').trim().substring(0, 1024) ?? '');
- row.push(csvEscape(errorMessage));
+ row.push(errorMessage);
} else {
const fail = test.annotations.find(a => a.type === 'fail');
if (fail)
- row.push(csvEscape(`Should have failed: ${fail.description}`));
+ row.push(`Should have failed: ${fail.description}`);
else
row.push('');
}
@@ -73,11 +76,16 @@ class CsvReporter implements Reporter {
}
}
}
- const csv = rows.map(r => r.join(',')).join('\n');
const reportFile = path.resolve(this._options.configDir, this._options.outputFile || 'test-results.csv');
+ const markdownFile = this._options.markdownFile && path.resolve(this._options.configDir, this._options.markdownFile);
this._pendingWrite = (async () => {
await fs.promises.mkdir(path.dirname(reportFile), { recursive: true });
+ const csv = [header, ...rows].map(r => r.map(csvEscape).join(',')).join('\n');
await fs.promises.writeFile(reportFile, csv);
+ if (markdownFile) {
+ await fs.promises.mkdir(path.dirname(markdownFile), { recursive: true });
+ await fs.promises.writeFile(markdownFile, markdownTable(rows));
+ }
})();
}
@@ -96,4 +104,24 @@ function csvEscape(str) {
return str;
}
+// GitHub job summaries are capped at 1MiB, so keep the rendered table bounded.
+const maxMarkdownRows = 500;
+
+function markdownTable(rows: string[][]): string {
+ const lines = [
+ `### ${rows.length} failing tests`,
+ '',
+ `| ${header.join(' | ')} |`,
+ `| ${header.map(() => '---').join(' | ')} |`,
+ ...rows.slice(0, maxMarkdownRows).map(row => `| ${row.map(markdownEscape).join(' | ')} |`),
+ ];
+ if (rows.length > maxMarkdownRows)
+ lines.push('', `_...and ${rows.length - maxMarkdownRows} more, see the csv report._`);
+ return lines.join('\n') + '\n';
+}
+
+function markdownEscape(str: string): string {
+ return str.replace(/[\\|`<>]/g, c => '\\' + c);
+}
+
export default CsvReporter;
diff --git a/tests/bidi/playwright.config.ts b/tests/bidi/playwright.config.ts
index afa43752e1..83996f2bde 100644
--- a/tests/bidi/playwright.config.ts
+++ b/tests/bidi/playwright.config.ts
@@ -46,7 +46,7 @@ const reporters = () => {
hasDebugOutput ? ['list'] : ['dot'],
['blob'],
['../config/parquetReporter.ts'],
- ['./csvReporter', { outputFile: path.join(outputDir, 'report.csv') }],
+ ['./csvReporter', { outputFile: path.join(outputDir, 'report.csv'), markdownFile: path.join(outputDir, 'report.md') }],
['./expectationReporter', { rebase: false }],
] : [
['html', { open: 'on-failure' }],