Skip to content

Playwright is a framework for Web Testing and Automation. It allows testing Chromium, Firefox and WebKit with a single API.

devops: add GitHub actions markdown reporter (#35406)

Yury Semikhatskye2c8163b145d
diff --git a/.github/actions/upload-blob-report/action.yml b/.github/actions/upload-blob-report/action.yml
index 72a084142a..e407e67573 100644
--- a/.github/actions/upload-blob-report/action.yml
+++ b/.github/actions/upload-blob-report/action.yml
@@ -22,13 +22,3 @@ runs:
name: blob-report-${{ inputs.job_name }}
path: ${{ inputs.report_dir }}/**
retention-days: 7
- - name: Write triggering pull request number in a file
- if: ${{ !cancelled() && github.event_name == 'pull_request' }}
- shell: bash
- run: echo '${{ github.event.number }}' > pull_request_number.txt;
- - name: Upload artifact with the pull request number
- if: ${{ !cancelled() && github.event_name == 'pull_request' }}
- uses: actions/upload-artifact@v4
- with:
- name: pull-request-${{ inputs.job_name }}
- path: pull_request_number.txt
\ No newline at end of file
diff --git a/.github/workflows/create_test_report.yml b/.github/workflows/create_test_report.yml
index dfc6961c2c..8393ef446a 100644
--- a/.github/workflows/create_test_report.yml
+++ b/.github/workflows/create_test_report.yml
@@ -35,7 +35,9 @@ jobs:
run: |
npx playwright merge-reports --config .github/workflows/merge.config.ts ./all-blob-reports
env:
+ GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
NODE_OPTIONS: --max-old-space-size=8192
+ HTML_REPORT_URL: 'https://mspwblobreport.z1.web.core.windows.net/run-${{ github.event.workflow_run.id }}-${{ github.event.workflow_run.run_attempt }}-${{ github.sha }}/index.html'
- name: Azure Login
uses: azure/login@v2
@@ -51,73 +53,3 @@ jobs:
echo "Report url: https://mspwblobreport.z1.web.core.windows.net/$REPORT_DIR/index.html"
env:
AZCOPY_AUTO_LOGIN_TYPE: AZCLI
-
- - name: Read pull request number
- uses: ./.github/actions/download-artifact
- with:
- namePrefix: 'pull-request'
- path: '.'
-
- - name: Comment on PR
- uses: actions/github-script@v7
- with:
- github-token: ${{ secrets.GITHUB_TOKEN }}
- script: |
- const fs = require('fs');
- let prNumber;
- if (context.payload.workflow_run.event === 'pull_request') {
- const prs = context.payload.workflow_run.pull_requests;
- if (prs.length) {
- prNumber = prs[0].number;
- } else {
- prNumber = parseInt(fs.readFileSync('pull_request_number.txt').toString());
- console.log('Read pull request number from file: ' + prNumber);
- }
- } else {
- core.error('Unsupported workflow trigger event: ' + context.payload.workflow_run.event);
- return;
- }
- if (!prNumber) {
- core.error('No pull request found for commit ' + context.sha + ' and workflow triggered by: ' + context.payload.workflow_run.event);
- return;
- }
- {
- // Mark previous comments as outdated by minimizing them.
- const { data: comments } = await github.rest.issues.listComments({
- ...context.repo,
- issue_number: prNumber,
- });
- for (const comment of comments) {
- if (comment.user.login === 'github-actions[bot]' && /\[Test results\]\(https:\/\/.+?\) for "${{ github.event.workflow_run.name }}"/.test(comment.body)) {
- await github.graphql(`
- mutation {
- minimizeComment(input: {subjectId: "${comment.node_id}", classifier: OUTDATED}) {
- clientMutationId
- }
- }
- `);
- }
- }
- }
- const reportDir = 'run-${{ github.event.workflow_run.id }}-${{ github.event.workflow_run.run_attempt }}-${{ github.sha }}';
- const reportUrl = `https://mspwblobreport.z1.web.core.windows.net/${reportDir}/index.html#?q=s%3Afailed%20s%3Aflaky`;
- core.notice('Report url: ' + reportUrl);
- const mergeWorkflowUrl = `${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`;
- const reportMd = await fs.promises.readFile('report.md', 'utf8');
- function formatComment(lines) {
- let body = lines.join('\n');
- if (body.length > 65535)
- body = body.substring(0, 65000) + `... ${body.length - 65000} more characters`;
- return body;
- }
- const { data: response } = await github.rest.issues.createComment({
- ...context.repo,
- issue_number: prNumber,
- body: formatComment([
- `### [Test results](${reportUrl}) for "${{ github.event.workflow_run.name }}"`,
- reportMd,
- '',
- `Merge [workflow run](${mergeWorkflowUrl}).`
- ]),
- });
- core.info('Posted comment: ' + response.html_url);
diff --git a/.github/workflows/merge.config.ts b/.github/workflows/merge.config.ts
index e8582ed521..0ef418b3b2 100644
--- a/.github/workflows/merge.config.ts
+++ b/.github/workflows/merge.config.ts
@@ -1,4 +1,4 @@
export default {
testDir: '../../tests',
- reporter: [[require.resolve('../../packages/playwright/lib/reporters/markdown')], ['html']]
+ reporter: [[require.resolve('../../tests/config/ghaMarkdownReporter.ts')], ['html']]
};
\ No newline at end of file
diff --git a/package-lock.json b/package-lock.json
index d7b06db081..f17a65797e 100644
--- a/package-lock.json
+++ b/package-lock.json
@@ -13,6 +13,7 @@
],
"devDependencies": {
"@actions/core": "^1.10.0",
+ "@actions/github": "^6.0.0",
"@babel/cli": "^7.26.4",
"@babel/code-frame": "^7.26.2",
"@babel/plugin-transform-class-properties": "^7.25.9",
@@ -96,6 +97,19 @@
"@actions/io": "^1.0.1"
}
},
+ "node_modules/@actions/github": {
+ "version": "6.0.0",
+ "resolved": "https://registry.npmjs.org/@actions/github/-/github-6.0.0.tgz",
+ "integrity": "sha512-alScpSVnYmjNEXboZjarjukQEzgCRmjMv6Xj47fsdnqGS73bjJNDpiiXmp8jr0UZLdUB6d9jW63IcmddUP+l0g==",
+ "dev": true,
+ "license": "MIT",
+ "dependencies": {
+ "@actions/http-client": "^2.2.0",
+ "@octokit/core": "^5.0.1",
+ "@octokit/plugin-paginate-rest": "^9.0.0",
+ "@octokit/plugin-rest-endpoint-methods": "^10.0.0"
+ }
+ },
"node_modules/@actions/http-client": {
"version": "2.2.3",
"resolved": "https://registry.npmjs.org/@actions/http-client/-/http-client-2.2.3.tgz",
@@ -1523,6 +1537,178 @@
"node": ">= 8"
}
},
+ "node_modules/@octokit/auth-token": {
+ "version": "4.0.0",
+ "resolved": "https://registry.npmjs.org/@octokit/auth-token/-/auth-token-4.0.0.tgz",
+ "integrity": "sha512-tY/msAuJo6ARbK6SPIxZrPBms3xPbfwBrulZe0Wtr/DIY9lje2HeV1uoebShn6mx7SjCHif6EjMvoREj+gZ+SA==",
+ "dev": true,
+ "license": "MIT",
+ "engines": {
+ "node": ">= 18"
+ }
+ },
+ "node_modules/@octokit/core": {
+ "version": "5.2.1",
+ "resolved": "https://registry.npmjs.org/@octokit/core/-/core-5.2.1.tgz",
+ "integrity": "sha512-dKYCMuPO1bmrpuogcjQ8z7ICCH3FP6WmxpwC03yjzGfZhj9fTJg6+bS1+UAplekbN2C+M61UNllGOOoAfGCrdQ==",
+ "dev": true,
+ "license": "MIT",
+ "dependencies": {
+ "@octokit/auth-token": "^4.0.0",
+ "@octokit/graphql": "^7.1.0",
+ "@octokit/request": "^8.4.1",
+ "@octokit/request-error": "^5.1.1",
+ "@octokit/types": "^13.0.0",
+ "before-after-hook": "^2.2.0",
+ "universal-user-agent": "^6.0.0"
+ },
+ "engines": {
+ "node": ">= 18"
+ }
+ },
+ "node_modules/@octokit/endpoint": {
+ "version": "9.0.6",
+ "resolved": "https://registry.npmjs.org/@octokit/endpoint/-/endpoint-9.0.6.tgz",
+ "integrity": "sha512-H1fNTMA57HbkFESSt3Y9+FBICv+0jFceJFPWDePYlR/iMGrwM5ph+Dd4XRQs+8X+PUFURLQgX9ChPfhJ/1uNQw==",
+ "dev": true,
+ "license": "MIT",
+ "dependencies": {
+ "@octokit/types": "^13.1.0",
+ "universal-user-agent": "^6.0.0"
+ },
+ "engines": {
+ "node": ">= 18"
+ }
+ },
+ "node_modules/@octokit/graphql": {
+ "version": "7.1.1",
+ "resolved": "https://registry.npmjs.org/@octokit/graphql/-/graphql-7.1.1.tgz",
+ "integrity": "sha512-3mkDltSfcDUoa176nlGoA32RGjeWjl3K7F/BwHwRMJUW/IteSa4bnSV8p2ThNkcIcZU2umkZWxwETSSCJf2Q7g==",
+ "dev": true,
+ "license": "MIT",
+ "dependencies": {
+ "@octokit/request": "^8.4.1",
+ "@octokit/types": "^13.0.0",
+ "universal-user-agent": "^6.0.0"
+ },
+ "engines": {
+ "node": ">= 18"
+ }
+ },
+ "node_modules/@octokit/openapi-types": {
+ "version": "24.2.0",
+ "resolved": "https://registry.npmjs.org/@octokit/openapi-types/-/openapi-types-24.2.0.tgz",
+ "integrity": "sha512-9sIH3nSUttelJSXUrmGzl7QUBFul0/mB8HRYl3fOlgHbIWG+WnYDXU3v/2zMtAvuzZ/ed00Ei6on975FhBfzrg==",
+ "dev": true,
+ "license": "MIT"
+ },
+ "node_modules/@octokit/plugin-paginate-rest": {
+ "version": "9.2.2",
+ "resolved": "https://registry.npmjs.org/@octokit/plugin-paginate-rest/-/plugin-paginate-rest-9.2.2.tgz",
+ "integrity": "sha512-u3KYkGF7GcZnSD/3UP0S7K5XUFT2FkOQdcfXZGZQPGv3lm4F2Xbf71lvjldr8c1H3nNbF+33cLEkWYbokGWqiQ==",
+ "dev": true,
+ "license": "MIT",
+ "dependencies": {
+ "@octokit/types": "^12.6.0"
+ },
+ "engines": {
+ "node": ">= 18"
+ },
+ "peerDependencies": {
+ "@octokit/core": "5"
+ }
+ },
+ "node_modules/@octokit/plugin-paginate-rest/node_modules/@octokit/openapi-types": {
+ "version": "20.0.0",
+ "resolved": "https://registry.npmjs.org/@octokit/openapi-types/-/openapi-types-20.0.0.tgz",
+ "integrity": "sha512-EtqRBEjp1dL/15V7WiX5LJMIxxkdiGJnabzYx5Apx4FkQIFgAfKumXeYAqqJCj1s+BMX4cPFIFC4OLCR6stlnA==",
+ "dev": true,
+ "license": "MIT"
+ },
+ "node_modules/@octokit/plugin-paginate-rest/node_modules/@octokit/types": {
+ "version": "12.6.0",
+ "resolved": "https://registry.npmjs.org/@octokit/types/-/types-12.6.0.tgz",
+ "integrity": "sha512-1rhSOfRa6H9w4YwK0yrf5faDaDTb+yLyBUKOCV4xtCDB5VmIPqd/v9yr9o6SAzOAlRxMiRiCic6JVM1/kunVkw==",
+ "dev": true,
+ "license": "MIT",
+ "dependencies": {
+ "@octokit/openapi-types": "^20.0.0"
+ }
+ },
+ "node_modules/@octokit/plugin-rest-endpoint-methods": {
+ "version": "10.4.1",
+ "resolved": "https://registry.npmjs.org/@octokit/plugin-rest-endpoint-methods/-/plugin-rest-endpoint-methods-10.4.1.tgz",
+ "integrity": "sha512-xV1b+ceKV9KytQe3zCVqjg+8GTGfDYwaT1ATU5isiUyVtlVAO3HNdzpS4sr4GBx4hxQ46s7ITtZrAsxG22+rVg==",
+ "dev": true,
+ "license": "MIT",
+ "dependencies": {
+ "@octokit/types": "^12.6.0"
+ },
+ "engines": {
+ "node": ">= 18"
+ },
+ "peerDependencies": {
+ "@octokit/core": "5"
+ }
+ },
+ "node_modules/@octokit/plugin-rest-endpoint-methods/node_modules/@octokit/openapi-types": {
+ "version": "20.0.0",
+ "resolved": "https://registry.npmjs.org/@octokit/openapi-types/-/openapi-types-20.0.0.tgz",
+ "integrity": "sha512-EtqRBEjp1dL/15V7WiX5LJMIxxkdiGJnabzYx5Apx4FkQIFgAfKumXeYAqqJCj1s+BMX4cPFIFC4OLCR6stlnA==",
+ "dev": true,
+ "license": "MIT"
+ },
+ "node_modules/@octokit/plugin-rest-endpoint-methods/node_modules/@octokit/types": {
+ "version": "12.6.0",
+ "resolved": "https://registry.npmjs.org/@octokit/types/-/types-12.6.0.tgz",
+ "integrity": "sha512-1rhSOfRa6H9w4YwK0yrf5faDaDTb+yLyBUKOCV4xtCDB5VmIPqd/v9yr9o6SAzOAlRxMiRiCic6JVM1/kunVkw==",
+ "dev": true,
+ "license": "MIT",
+ "dependencies": {
+ "@octokit/openapi-types": "^20.0.0"
+ }
+ },
+ "node_modules/@octokit/request": {
+ "version": "8.4.1",
+ "resolved": "https://registry.npmjs.org/@octokit/request/-/request-8.4.1.tgz",
+ "integrity": "sha512-qnB2+SY3hkCmBxZsR/MPCybNmbJe4KAlfWErXq+rBKkQJlbjdJeS85VI9r8UqeLYLvnAenU8Q1okM/0MBsAGXw==",
+ "dev": true,
+ "license": "MIT",
+ "dependencies": {
+ "@octokit/endpoint": "^9.0.6",
+ "@octokit/request-error": "^5.1.1",
+ "@octokit/types": "^13.1.0",
+ "universal-user-agent": "^6.0.0"
+ },
+ "engines": {
+ "node": ">= 18"
+ }
+ },
+ "node_modules/@octokit/request-error": {
+ "version": "5.1.1",
+ "resolved": "https://registry.npmjs.org/@octokit/request-error/-/request-error-5.1.1.tgz",
+ "integrity": "sha512-v9iyEQJH6ZntoENr9/yXxjuezh4My67CBSu9r6Ve/05Iu5gNgnisNWOsoJHTP6k0Rr0+HQIpnH+kyammu90q/g==",
+ "dev": true,
+ "license": "MIT",
+ "dependencies": {
+ "@octokit/types": "^13.1.0",
+ "deprecation": "^2.0.0",
+ "once": "^1.4.0"
+ },
+ "engines": {
+ "node": ">= 18"
+ }
+ },
+ "node_modules/@octokit/types": {
+ "version": "13.10.0",
+ "resolved": "https://registry.npmjs.org/@octokit/types/-/types-13.10.0.tgz",
+ "integrity": "sha512-ifLaO34EbbPj0Xgro4G5lP5asESjwHracYJvVaPIyXMuiuXLlhic3S47cBdTb+jfODkTE5YtGCLt3Ay3+J97sA==",
+ "dev": true,
+ "license": "MIT",
+ "dependencies": {
+ "@octokit/openapi-types": "^24.2.0"
+ }
+ },
"node_modules/@playwright/browser-chromium": {
"resolved": "packages/playwright-browser-chromium",
"link": true
@@ -2804,6 +2990,13 @@
"dev": true,
"license": "MIT"
},
+ "node_modules/before-after-hook": {
+ "version": "2.2.3",
+ "resolved": "https://registry.npmjs.org/before-after-hook/-/before-after-hook-2.2.3.tgz",
+ "integrity": "sha512-NzUnlZexiaH/46WDhANlyR2bXRopNg4F/zuSA3OpZnllCUgRaOF2znDioDWrmbNVsuZk6l9pMquQB38cfBZwkQ==",
+ "dev": true,
+ "license": "Apache-2.0"
+ },
"node_modules/binary-extensions": {
"version": "2.3.0",
"resolved": "https://registry.npmjs.org/binary-extensions/-/binary-extensions-2.3.0.tgz",
@@ -3434,6 +3627,13 @@
"url": "https://github.com/sponsors/ljharb"
}
},
+ "node_modules/deprecation": {
+ "version": "2.3.1",
+ "resolved": "https://registry.npmjs.org/deprecation/-/deprecation-2.3.1.tgz",
+ "integrity": "sha512-xmHIy4F3scKVwMsQ4WnVaS8bHOx0DmVwRywosKhaILI0ywMDWPtBSku2HNxRvF7jtwDRsoEwYQSfbxj8b7RlJQ==",
+ "dev": true,
+ "license": "ISC"
+ },
"node_modules/detect-node": {
"version": "2.1.0",
"resolved": "https://registry.npmjs.org/detect-node/-/detect-node-2.1.0.tgz",
@@ -7662,6 +7862,13 @@
"devOptional": true,
"license": "MIT"
},
+ "node_modules/universal-user-agent": {
+ "version": "6.0.1",
+ "resolved": "https://registry.npmjs.org/universal-user-agent/-/universal-user-agent-6.0.1.tgz",
+ "integrity": "sha512-yCzhz6FN2wU1NiiQRogkTQszlQSlpWaw8SvVegAc+bDxbzHgh1vX8uIe8OYyMH6DwH+sdTJsgMl36+mSMdRJIQ==",
+ "dev": true,
+ "license": "ISC"
+ },
"node_modules/universalify": {
"version": "0.1.2",
"resolved": "https://registry.npmjs.org/universalify/-/universalify-0.1.2.tgz",
diff --git a/package.json b/package.json
index e97dd26745..d20a27e554 100644
--- a/package.json
+++ b/package.json
@@ -52,6 +52,7 @@
],
"devDependencies": {
"@actions/core": "^1.10.0",
+ "@actions/github": "^6.0.0",
"@babel/cli": "^7.26.4",
"@babel/code-frame": "^7.26.2",
"@babel/plugin-transform-class-properties": "^7.25.9",
diff --git a/packages/playwright/src/internalsForTest.ts b/packages/playwright/src/internalsForTest.ts
index 129f9e858d..1cdd2cedb4 100644
--- a/packages/playwright/src/internalsForTest.ts
+++ b/packages/playwright/src/internalsForTest.ts
@@ -17,6 +17,7 @@
import path from 'path';
import { fileDependenciesForTest } from './transform/compilationCache';
+export { default as MarkdownReporter } from './reporters/markdown';
export function fileDependencies() {
return Object.fromEntries([...fileDependenciesForTest().entries()].map(entry => (
diff --git a/packages/playwright/src/reporters/markdown.ts b/packages/playwright/src/reporters/markdown.ts
index 8b2ea93dac..5a2d1d4203 100644
--- a/packages/playwright/src/reporters/markdown.ts
+++ b/packages/playwright/src/reporters/markdown.ts
@@ -69,9 +69,13 @@ class MarkdownReporter extends TerminalReporter {
lines.push(`:heavy_check_mark::heavy_check_mark::heavy_check_mark:`);
lines.push(``);
+ await this.publishReport(lines.join('\n'));
+ }
+
+ protected async publishReport(report: string): Promise<void> {
const reportFile = resolveReporterOutputPath('report.md', this._options.configDir, this._options.outputFile);
await fs.promises.mkdir(path.dirname(reportFile), { recursive: true });
- await fs.promises.writeFile(reportFile, lines.join('\n'));
+ await fs.promises.writeFile(reportFile, report);
}
private _printTestList(prefix: string, tests: TestCase[], lines: string[], suffix?: string) {
diff --git a/tests/config/ghaMarkdownReporter.ts b/tests/config/ghaMarkdownReporter.ts
new file mode 100644
index 0000000000..37ddcccd63
--- /dev/null
+++ b/tests/config/ghaMarkdownReporter.ts
@@ -0,0 +1,103 @@
+/**
+ * Copyright (c) Microsoft Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { MarkdownReporter } from 'playwright/lib/internalsForTest';
+import { context, getOctokit } from '@actions/github';
+import * as core from '@actions/core';
+
+import type { MetadataWithCommitInfo } from 'playwright/src/isomorphic/types';
+
+function getGithubToken() {
+ const token = process.env.GITHUB_TOKEN || core.getInput('github-token');
+ if (!token) {
+ core.setFailed('Missing "github-token" input');
+ throw new Error('Missing "github-token" input');
+ }
+ return token;
+}
+
+const octokit = getOctokit(getGithubToken());
+
+const magicComment = '<!-- Generated by Playwright markdown reporter -->';
+
+class GHAMarkdownReporter extends MarkdownReporter {
+ // declare config: FullConfig;
+
+ async publishReport(report: string) {
+ core.info('Publishing report to PR.');
+ const { prNumber, prHref } = this.pullRequestFromMetadata();
+ if (!prNumber) {
+ core.info(`No PR number found, skipping GHA comment. PR href: ${prHref}`);
+ return;
+ }
+ core.info(`Posting comment to PR ${prHref}`);
+
+ await this.collapsePreviousComments(prNumber);
+ await this.addNewReportComment(prNumber, report);
+ }
+
+ private async collapsePreviousComments(prNumber: number) {
+ const { data: comments } = await octokit.rest.issues.listComments({
+ ...context.repo,
+ issue_number: prNumber,
+ });
+ for (const comment of comments) {
+ if (comment.user.login === 'github-actions[bot]' && comment.body.includes(magicComment)) {
+ core.info(`Minimizing comment: ${comment.html_url}`);
+ await octokit.graphql(`
+ mutation {
+ minimizeComment(input: {subjectId: "${comment.node_id}", classifier: OUTDATED}) {
+ clientMutationId
+ }
+ }
+ `);
+ }
+ }
+ }
+
+ private async addNewReportComment(prNumber: number, report: string) {
+ const reportUrl = process.env.HTML_REPORT_URL;
+ const mergeWorkflowUrl = `${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`;
+
+ const { data: response } = await octokit.rest.issues.createComment({
+ ...context.repo,
+ issue_number: prNumber,
+ body: formatComment([
+ magicComment,
+ `### [Test results](${reportUrl}) for "${context.payload.workflow_run?.name}"`,
+ report,
+ '',
+ `Merge [workflow run](${mergeWorkflowUrl}).`
+ ]),
+ });
+ core.info(`Posted comment: ${response.html_url}`);
+ }
+
+ private pullRequestFromMetadata() {
+ const metadata = this.config.metadata as MetadataWithCommitInfo;
+ const prHref = metadata.ci?.prHref;
+ return { prNumber: parseInt(prHref?.split('/').pop(), 10), prHref };
+ }
+}
+
+function formatComment(lines) {
+ let body = lines.join('\n');
+ if (body.length > 65535)
+ body = body.substring(0, 65000) + `... ${body.length - 65000} more characters`;
+ return body;
+}
+
+export default GHAMarkdownReporter;