Playwright is a framework for Web Testing and Automation. It allows testing Chromium, Firefox and WebKit with a single API.
TypeScript automationchromechromiume2e-testingelectronend-to-end-testingfirefoxjavascriptplaywrighttesttest-automationtestingtesting-toolswebwebkit
devops: add GitHub actions markdown reporter (#35406)
Yury Semikhatskye2c8163b145d
diff --git a/.github/actions/upload-blob-report/action.yml b/.github/actions/upload-blob-report/action.ymlindex 72a084142a..e407e67573 100644--- a/.github/actions/upload-blob-report/action.yml+++ b/.github/actions/upload-blob-report/action.yml@@ -22,13 +22,3 @@ runs:name: blob-report-${{ inputs.job_name }}path: ${{ inputs.report_dir }}/**retention-days: 7- - name: Write triggering pull request number in a file- if: ${{ !cancelled() && github.event_name == 'pull_request' }}- shell: bash- run: echo '${{ github.event.number }}' > pull_request_number.txt;- - name: Upload artifact with the pull request number- if: ${{ !cancelled() && github.event_name == 'pull_request' }}- uses: actions/upload-artifact@v4- with:- name: pull-request-${{ inputs.job_name }}- path: pull_request_number.txt\ No newline at end of filediff --git a/.github/workflows/create_test_report.yml b/.github/workflows/create_test_report.ymlindex dfc6961c2c..8393ef446a 100644--- a/.github/workflows/create_test_report.yml+++ b/.github/workflows/create_test_report.yml@@ -35,7 +35,9 @@ jobs:run: |npx playwright merge-reports --config .github/workflows/merge.config.ts ./all-blob-reportsenv:+ GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}NODE_OPTIONS: --max-old-space-size=8192+ HTML_REPORT_URL: 'https://mspwblobreport.z1.web.core.windows.net/run-${{ github.event.workflow_run.id }}-${{ github.event.workflow_run.run_attempt }}-${{ github.sha }}/index.html'- name: Azure Loginuses: azure/login@v2@@ -51,73 +53,3 @@ jobs:echo "Report url: https://mspwblobreport.z1.web.core.windows.net/$REPORT_DIR/index.html"env:AZCOPY_AUTO_LOGIN_TYPE: AZCLI-- - name: Read pull request number- uses: ./.github/actions/download-artifact- with:- namePrefix: 'pull-request'- path: '.'-- - name: Comment on PR- uses: actions/github-script@v7- with:- github-token: ${{ secrets.GITHUB_TOKEN }}- script: |- const fs = require('fs');- let prNumber;- if (context.payload.workflow_run.event === 'pull_request') {- const prs = context.payload.workflow_run.pull_requests;- if (prs.length) {- prNumber = prs[0].number;- } else {- prNumber = parseInt(fs.readFileSync('pull_request_number.txt').toString());- console.log('Read pull request number from file: ' + prNumber);- }- } else {- core.error('Unsupported workflow trigger event: ' + context.payload.workflow_run.event);- return;- }- if (!prNumber) {- core.error('No pull request found for commit ' + context.sha + ' and workflow triggered by: ' + context.payload.workflow_run.event);- return;- }- {- // Mark previous comments as outdated by minimizing them.- const { data: comments } = await github.rest.issues.listComments({- ...context.repo,- issue_number: prNumber,- });- for (const comment of comments) {- if (comment.user.login === 'github-actions[bot]' && /\[Test results\]\(https:\/\/.+?\) for "${{ github.event.workflow_run.name }}"/.test(comment.body)) {- await github.graphql(`- mutation {- minimizeComment(input: {subjectId: "${comment.node_id}", classifier: OUTDATED}) {- clientMutationId- }- }- `);- }- }- }- const reportDir = 'run-${{ github.event.workflow_run.id }}-${{ github.event.workflow_run.run_attempt }}-${{ github.sha }}';- const reportUrl = `https://mspwblobreport.z1.web.core.windows.net/${reportDir}/index.html#?q=s%3Afailed%20s%3Aflaky`;- core.notice('Report url: ' + reportUrl);- const mergeWorkflowUrl = `${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`;- const reportMd = await fs.promises.readFile('report.md', 'utf8');- function formatComment(lines) {- let body = lines.join('\n');- if (body.length > 65535)- body = body.substring(0, 65000) + `... ${body.length - 65000} more characters`;- return body;- }- const { data: response } = await github.rest.issues.createComment({- ...context.repo,- issue_number: prNumber,- body: formatComment([- `### [Test results](${reportUrl}) for "${{ github.event.workflow_run.name }}"`,- reportMd,- '',- `Merge [workflow run](${mergeWorkflowUrl}).`- ]),- });- core.info('Posted comment: ' + response.html_url);diff --git a/.github/workflows/merge.config.ts b/.github/workflows/merge.config.tsindex e8582ed521..0ef418b3b2 100644--- a/.github/workflows/merge.config.ts+++ b/.github/workflows/merge.config.ts@@ -1,4 +1,4 @@export default {testDir: '../../tests',- reporter: [[require.resolve('../../packages/playwright/lib/reporters/markdown')], ['html']]+ reporter: [[require.resolve('../../tests/config/ghaMarkdownReporter.ts')], ['html']]};\ No newline at end of filediff --git a/package-lock.json b/package-lock.jsonindex d7b06db081..f17a65797e 100644--- a/package-lock.json+++ b/package-lock.json@@ -13,6 +13,7 @@],"devDependencies": {"@actions/core": "^1.10.0",+ "@actions/github": "^6.0.0","@babel/cli": "^7.26.4","@babel/code-frame": "^7.26.2","@babel/plugin-transform-class-properties": "^7.25.9",@@ -96,6 +97,19 @@"@actions/io": "^1.0.1"}},+ "node_modules/@actions/github": {+ "version": "6.0.0",+ "resolved": "https://registry.npmjs.org/@actions/github/-/github-6.0.0.tgz",+ "integrity": "sha512-alScpSVnYmjNEXboZjarjukQEzgCRmjMv6Xj47fsdnqGS73bjJNDpiiXmp8jr0UZLdUB6d9jW63IcmddUP+l0g==",+ "dev": true,+ "license": "MIT",+ "dependencies": {+ "@actions/http-client": "^2.2.0",+ "@octokit/core": "^5.0.1",+ "@octokit/plugin-paginate-rest": "^9.0.0",+ "@octokit/plugin-rest-endpoint-methods": "^10.0.0"+ }+ },"node_modules/@actions/http-client": {"version": "2.2.3","resolved": "https://registry.npmjs.org/@actions/http-client/-/http-client-2.2.3.tgz",@@ -1523,6 +1537,178 @@"node": ">= 8"}},+ "node_modules/@octokit/auth-token": {+ "version": "4.0.0",+ "resolved": "https://registry.npmjs.org/@octokit/auth-token/-/auth-token-4.0.0.tgz",+ "integrity": "sha512-tY/msAuJo6ARbK6SPIxZrPBms3xPbfwBrulZe0Wtr/DIY9lje2HeV1uoebShn6mx7SjCHif6EjMvoREj+gZ+SA==",+ "dev": true,+ "license": "MIT",+ "engines": {+ "node": ">= 18"+ }+ },+ "node_modules/@octokit/core": {+ "version": "5.2.1",+ "resolved": "https://registry.npmjs.org/@octokit/core/-/core-5.2.1.tgz",+ "integrity": "sha512-dKYCMuPO1bmrpuogcjQ8z7ICCH3FP6WmxpwC03yjzGfZhj9fTJg6+bS1+UAplekbN2C+M61UNllGOOoAfGCrdQ==",+ "dev": true,+ "license": "MIT",+ "dependencies": {+ "@octokit/auth-token": "^4.0.0",+ "@octokit/graphql": "^7.1.0",+ "@octokit/request": "^8.4.1",+ "@octokit/request-error": "^5.1.1",+ "@octokit/types": "^13.0.0",+ "before-after-hook": "^2.2.0",+ "universal-user-agent": "^6.0.0"+ },+ "engines": {+ "node": ">= 18"+ }+ },+ "node_modules/@octokit/endpoint": {+ "version": "9.0.6",+ "resolved": "https://registry.npmjs.org/@octokit/endpoint/-/endpoint-9.0.6.tgz",+ "integrity": "sha512-H1fNTMA57HbkFESSt3Y9+FBICv+0jFceJFPWDePYlR/iMGrwM5ph+Dd4XRQs+8X+PUFURLQgX9ChPfhJ/1uNQw==",+ "dev": true,+ "license": "MIT",+ "dependencies": {+ "@octokit/types": "^13.1.0",+ "universal-user-agent": "^6.0.0"+ },+ "engines": {+ "node": ">= 18"+ }+ },+ "node_modules/@octokit/graphql": {+ "version": "7.1.1",+ "resolved": "https://registry.npmjs.org/@octokit/graphql/-/graphql-7.1.1.tgz",+ "integrity": "sha512-3mkDltSfcDUoa176nlGoA32RGjeWjl3K7F/BwHwRMJUW/IteSa4bnSV8p2ThNkcIcZU2umkZWxwETSSCJf2Q7g==",+ "dev": true,+ "license": "MIT",+ "dependencies": {+ "@octokit/request": "^8.4.1",+ "@octokit/types": "^13.0.0",+ "universal-user-agent": "^6.0.0"+ },+ "engines": {+ "node": ">= 18"+ }+ },+ "node_modules/@octokit/openapi-types": {+ "version": "24.2.0",+ "resolved": "https://registry.npmjs.org/@octokit/openapi-types/-/openapi-types-24.2.0.tgz",+ "integrity": "sha512-9sIH3nSUttelJSXUrmGzl7QUBFul0/mB8HRYl3fOlgHbIWG+WnYDXU3v/2zMtAvuzZ/ed00Ei6on975FhBfzrg==",+ "dev": true,+ "license": "MIT"+ },+ "node_modules/@octokit/plugin-paginate-rest": {+ "version": "9.2.2",+ "resolved": "https://registry.npmjs.org/@octokit/plugin-paginate-rest/-/plugin-paginate-rest-9.2.2.tgz",+ "integrity": "sha512-u3KYkGF7GcZnSD/3UP0S7K5XUFT2FkOQdcfXZGZQPGv3lm4F2Xbf71lvjldr8c1H3nNbF+33cLEkWYbokGWqiQ==",+ "dev": true,+ "license": "MIT",+ "dependencies": {+ "@octokit/types": "^12.6.0"+ },+ "engines": {+ "node": ">= 18"+ },+ "peerDependencies": {+ "@octokit/core": "5"+ }+ },+ "node_modules/@octokit/plugin-paginate-rest/node_modules/@octokit/openapi-types": {+ "version": "20.0.0",+ "resolved": "https://registry.npmjs.org/@octokit/openapi-types/-/openapi-types-20.0.0.tgz",+ "integrity": "sha512-EtqRBEjp1dL/15V7WiX5LJMIxxkdiGJnabzYx5Apx4FkQIFgAfKumXeYAqqJCj1s+BMX4cPFIFC4OLCR6stlnA==",+ "dev": true,+ "license": "MIT"+ },+ "node_modules/@octokit/plugin-paginate-rest/node_modules/@octokit/types": {+ "version": "12.6.0",+ "resolved": "https://registry.npmjs.org/@octokit/types/-/types-12.6.0.tgz",+ "integrity": "sha512-1rhSOfRa6H9w4YwK0yrf5faDaDTb+yLyBUKOCV4xtCDB5VmIPqd/v9yr9o6SAzOAlRxMiRiCic6JVM1/kunVkw==",+ "dev": true,+ "license": "MIT",+ "dependencies": {+ "@octokit/openapi-types": "^20.0.0"+ }+ },+ "node_modules/@octokit/plugin-rest-endpoint-methods": {+ "version": "10.4.1",+ "resolved": "https://registry.npmjs.org/@octokit/plugin-rest-endpoint-methods/-/plugin-rest-endpoint-methods-10.4.1.tgz",+ "integrity": "sha512-xV1b+ceKV9KytQe3zCVqjg+8GTGfDYwaT1ATU5isiUyVtlVAO3HNdzpS4sr4GBx4hxQ46s7ITtZrAsxG22+rVg==",+ "dev": true,+ "license": "MIT",+ "dependencies": {+ "@octokit/types": "^12.6.0"+ },+ "engines": {+ "node": ">= 18"+ },+ "peerDependencies": {+ "@octokit/core": "5"+ }+ },+ "node_modules/@octokit/plugin-rest-endpoint-methods/node_modules/@octokit/openapi-types": {+ "version": "20.0.0",+ "resolved": "https://registry.npmjs.org/@octokit/openapi-types/-/openapi-types-20.0.0.tgz",+ "integrity": "sha512-EtqRBEjp1dL/15V7WiX5LJMIxxkdiGJnabzYx5Apx4FkQIFgAfKumXeYAqqJCj1s+BMX4cPFIFC4OLCR6stlnA==",+ "dev": true,+ "license": "MIT"+ },+ "node_modules/@octokit/plugin-rest-endpoint-methods/node_modules/@octokit/types": {+ "version": "12.6.0",+ "resolved": "https://registry.npmjs.org/@octokit/types/-/types-12.6.0.tgz",+ "integrity": "sha512-1rhSOfRa6H9w4YwK0yrf5faDaDTb+yLyBUKOCV4xtCDB5VmIPqd/v9yr9o6SAzOAlRxMiRiCic6JVM1/kunVkw==",+ "dev": true,+ "license": "MIT",+ "dependencies": {+ "@octokit/openapi-types": "^20.0.0"+ }+ },+ "node_modules/@octokit/request": {+ "version": "8.4.1",+ "resolved": "https://registry.npmjs.org/@octokit/request/-/request-8.4.1.tgz",+ "integrity": "sha512-qnB2+SY3hkCmBxZsR/MPCybNmbJe4KAlfWErXq+rBKkQJlbjdJeS85VI9r8UqeLYLvnAenU8Q1okM/0MBsAGXw==",+ "dev": true,+ "license": "MIT",+ "dependencies": {+ "@octokit/endpoint": "^9.0.6",+ "@octokit/request-error": "^5.1.1",+ "@octokit/types": "^13.1.0",+ "universal-user-agent": "^6.0.0"+ },+ "engines": {+ "node": ">= 18"+ }+ },+ "node_modules/@octokit/request-error": {+ "version": "5.1.1",+ "resolved": "https://registry.npmjs.org/@octokit/request-error/-/request-error-5.1.1.tgz",+ "integrity": "sha512-v9iyEQJH6ZntoENr9/yXxjuezh4My67CBSu9r6Ve/05Iu5gNgnisNWOsoJHTP6k0Rr0+HQIpnH+kyammu90q/g==",+ "dev": true,+ "license": "MIT",+ "dependencies": {+ "@octokit/types": "^13.1.0",+ "deprecation": "^2.0.0",+ "once": "^1.4.0"+ },+ "engines": {+ "node": ">= 18"+ }+ },+ "node_modules/@octokit/types": {+ "version": "13.10.0",+ "resolved": "https://registry.npmjs.org/@octokit/types/-/types-13.10.0.tgz",+ "integrity": "sha512-ifLaO34EbbPj0Xgro4G5lP5asESjwHracYJvVaPIyXMuiuXLlhic3S47cBdTb+jfODkTE5YtGCLt3Ay3+J97sA==",+ "dev": true,+ "license": "MIT",+ "dependencies": {+ "@octokit/openapi-types": "^24.2.0"+ }+ },"node_modules/@playwright/browser-chromium": {"resolved": "packages/playwright-browser-chromium","link": true@@ -2804,6 +2990,13 @@"dev": true,"license": "MIT"},+ "node_modules/before-after-hook": {+ "version": "2.2.3",+ "resolved": "https://registry.npmjs.org/before-after-hook/-/before-after-hook-2.2.3.tgz",+ "integrity": "sha512-NzUnlZexiaH/46WDhANlyR2bXRopNg4F/zuSA3OpZnllCUgRaOF2znDioDWrmbNVsuZk6l9pMquQB38cfBZwkQ==",+ "dev": true,+ "license": "Apache-2.0"+ },"node_modules/binary-extensions": {"version": "2.3.0","resolved": "https://registry.npmjs.org/binary-extensions/-/binary-extensions-2.3.0.tgz",@@ -3434,6 +3627,13 @@"url": "https://github.com/sponsors/ljharb"}},+ "node_modules/deprecation": {+ "version": "2.3.1",+ "resolved": "https://registry.npmjs.org/deprecation/-/deprecation-2.3.1.tgz",+ "integrity": "sha512-xmHIy4F3scKVwMsQ4WnVaS8bHOx0DmVwRywosKhaILI0ywMDWPtBSku2HNxRvF7jtwDRsoEwYQSfbxj8b7RlJQ==",+ "dev": true,+ "license": "ISC"+ },"node_modules/detect-node": {"version": "2.1.0","resolved": "https://registry.npmjs.org/detect-node/-/detect-node-2.1.0.tgz",@@ -7662,6 +7862,13 @@"devOptional": true,"license": "MIT"},+ "node_modules/universal-user-agent": {+ "version": "6.0.1",+ "resolved": "https://registry.npmjs.org/universal-user-agent/-/universal-user-agent-6.0.1.tgz",+ "integrity": "sha512-yCzhz6FN2wU1NiiQRogkTQszlQSlpWaw8SvVegAc+bDxbzHgh1vX8uIe8OYyMH6DwH+sdTJsgMl36+mSMdRJIQ==",+ "dev": true,+ "license": "ISC"+ },"node_modules/universalify": {"version": "0.1.2","resolved": "https://registry.npmjs.org/universalify/-/universalify-0.1.2.tgz",diff --git a/package.json b/package.jsonindex e97dd26745..d20a27e554 100644--- a/package.json+++ b/package.json@@ -52,6 +52,7 @@],"devDependencies": {"@actions/core": "^1.10.0",+ "@actions/github": "^6.0.0","@babel/cli": "^7.26.4","@babel/code-frame": "^7.26.2","@babel/plugin-transform-class-properties": "^7.25.9",diff --git a/packages/playwright/src/internalsForTest.ts b/packages/playwright/src/internalsForTest.tsindex 129f9e858d..1cdd2cedb4 100644--- a/packages/playwright/src/internalsForTest.ts+++ b/packages/playwright/src/internalsForTest.ts@@ -17,6 +17,7 @@import path from 'path';import { fileDependenciesForTest } from './transform/compilationCache';+export { default as MarkdownReporter } from './reporters/markdown';export function fileDependencies() {return Object.fromEntries([...fileDependenciesForTest().entries()].map(entry => (diff --git a/packages/playwright/src/reporters/markdown.ts b/packages/playwright/src/reporters/markdown.tsindex 8b2ea93dac..5a2d1d4203 100644--- a/packages/playwright/src/reporters/markdown.ts+++ b/packages/playwright/src/reporters/markdown.ts@@ -69,9 +69,13 @@ class MarkdownReporter extends TerminalReporter {lines.push(`:heavy_check_mark::heavy_check_mark::heavy_check_mark:`);lines.push(``);+ await this.publishReport(lines.join('\n'));+ }++ protected async publishReport(report: string): Promise<void> {const reportFile = resolveReporterOutputPath('report.md', this._options.configDir, this._options.outputFile);await fs.promises.mkdir(path.dirname(reportFile), { recursive: true });- await fs.promises.writeFile(reportFile, lines.join('\n'));+ await fs.promises.writeFile(reportFile, report);}private _printTestList(prefix: string, tests: TestCase[], lines: string[], suffix?: string) {diff --git a/tests/config/ghaMarkdownReporter.ts b/tests/config/ghaMarkdownReporter.tsnew file mode 100644index 0000000000..37ddcccd63--- /dev/null+++ b/tests/config/ghaMarkdownReporter.ts@@ -0,0 +1,103 @@+/**+ * Copyright (c) Microsoft Corporation.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++import { MarkdownReporter } from 'playwright/lib/internalsForTest';+import { context, getOctokit } from '@actions/github';+import * as core from '@actions/core';++import type { MetadataWithCommitInfo } from 'playwright/src/isomorphic/types';++function getGithubToken() {+ const token = process.env.GITHUB_TOKEN || core.getInput('github-token');+ if (!token) {+ core.setFailed('Missing "github-token" input');+ throw new Error('Missing "github-token" input');+ }+ return token;+}++const octokit = getOctokit(getGithubToken());++const magicComment = '<!-- Generated by Playwright markdown reporter -->';++class GHAMarkdownReporter extends MarkdownReporter {+ // declare config: FullConfig;++ async publishReport(report: string) {+ core.info('Publishing report to PR.');+ const { prNumber, prHref } = this.pullRequestFromMetadata();+ if (!prNumber) {+ core.info(`No PR number found, skipping GHA comment. PR href: ${prHref}`);+ return;+ }+ core.info(`Posting comment to PR ${prHref}`);++ await this.collapsePreviousComments(prNumber);+ await this.addNewReportComment(prNumber, report);+ }++ private async collapsePreviousComments(prNumber: number) {+ const { data: comments } = await octokit.rest.issues.listComments({+ ...context.repo,+ issue_number: prNumber,+ });+ for (const comment of comments) {+ if (comment.user.login === 'github-actions[bot]' && comment.body.includes(magicComment)) {+ core.info(`Minimizing comment: ${comment.html_url}`);+ await octokit.graphql(`+ mutation {+ minimizeComment(input: {subjectId: "${comment.node_id}", classifier: OUTDATED}) {+ clientMutationId+ }+ }+ `);+ }+ }+ }++ private async addNewReportComment(prNumber: number, report: string) {+ const reportUrl = process.env.HTML_REPORT_URL;+ const mergeWorkflowUrl = `${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`;++ const { data: response } = await octokit.rest.issues.createComment({+ ...context.repo,+ issue_number: prNumber,+ body: formatComment([+ magicComment,+ `### [Test results](${reportUrl}) for "${context.payload.workflow_run?.name}"`,+ report,+ '',+ `Merge [workflow run](${mergeWorkflowUrl}).`+ ]),+ });+ core.info(`Posted comment: ${response.html_url}`);+ }++ private pullRequestFromMetadata() {+ const metadata = this.config.metadata as MetadataWithCommitInfo;+ const prHref = metadata.ci?.prHref;+ return { prNumber: parseInt(prHref?.split('/').pop(), 10), prHref };+ }+}++function formatComment(lines) {+ let body = lines.join('\n');+ if (body.length > 65535)+ body = body.substring(0, 65000) + `... ${body.length - 65000} more characters`;+ return body;+}++export default GHAMarkdownReporter;