Playwright is a framework for Web Testing and Automation. It allows testing Chromium, Firefox and WebKit with a single API.
TypeScript automationchromechromiume2e-testingelectronend-to-end-testingfirefoxjavascriptplaywrighttesttest-automationtestingtesting-toolswebwebkit
devops: publish test report as a browser-openable GitHub artifact (#41520)
Dmitry Gozman800400f816d7
diff --git a/.github/workflows/create_test_report.yml b/.github/workflows/create_test_report.ymlindex ddf32c64b1..7850e4f1c0 100644--- a/.github/workflows/create_test_report.yml+++ b/.github/workflows/create_test_report.yml@@ -10,10 +10,11 @@ jobs:pull-requests: writechecks: writestatuses: write- id-token: write # This is required for OIDC login (azure/login) to succeedcontents: read # This is required for actions/checkout to succeedif: ${{ github.event.workflow_run.event == 'pull_request' || github.event.workflow_run.event == 'push' }}runs-on: ubuntu-latest+ env:+ MARKDOWN_OUTPUT_FILE: ${{ github.workspace }}/report.mdsteps:- uses: actions/checkout@v6- uses: actions/setup-node@v6@@ -32,28 +33,48 @@ jobs:run: |npx playwright merge-reports --config .github/workflows/merge.config.ts ./all-blob-reportsenv:- GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}NODE_OPTIONS: --max-old-space-size=8192- HTML_REPORT_URL: 'https://mspwblobreport.z1.web.core.windows.net/run-${{ github.event.workflow_run.id }}-${{ github.event.workflow_run.run_attempt }}-${{ github.sha }}/index.html'WORKFLOW_RUN_CONCLUSION: ${{ github.event.workflow_run.conclusion }}- - name: Azure Login- uses: azure/login@v3+ - name: Upload HTML report+ id: upload-report+ uses: actions/upload-artifact@v7with:- client-id: ${{ secrets.AZURE_BLOB_REPORTS_CLIENT_ID }}- tenant-id: ${{ secrets.AZURE_BLOB_REPORTS_TENANT_ID }}- subscription-id: ${{ secrets.AZURE_BLOB_REPORTS_SUBSCRIPTION_ID }}+ path: playwright-report/index.html+ archive: false # Upload as a single, browser-openable file (no zip)+ retention-days: 30- - name: Upload HTML report to Azure- id: upload-report+ # The triggering workflow ran on a pull_request event, but workflow_run.pull_requests is+ # empty for PRs from forks, so resolve the number from the head repo + branch instead.+ - name: Resolve PR number+ if: ${{ github.event.workflow_run.event == 'pull_request' }}+ id: pr+ env:+ GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}run: |- REPORT_DIR='run-${{ github.event.workflow_run.id }}-${{ github.event.workflow_run.run_attempt }}-${{ github.sha }}'- azcopy cp --recursive "./playwright-report/*" "https://mspwblobreport.blob.core.windows.net/\$web/$REPORT_DIR"- REPORT_URL="https://mspwblobreport.z1.web.core.windows.net/$REPORT_DIR/index.html"- echo "Report url: $REPORT_URL"- echo "reportUrl=$REPORT_URL" >> "$GITHUB_OUTPUT"+ NUMBER=$(gh pr view --repo "${{ github.repository }}" \+ "${{ github.event.workflow_run.head_repository.owner.login }}:${{ github.event.workflow_run.head_branch }}" \+ --json number --jq '.number' 2>/dev/null || true)+ echo "number=$NUMBER" >> "$GITHUB_OUTPUT"++ - name: Post report comment to PR+ if: ${{ steps.pr.outputs.number }}+ uses: actions/github-script@v8env:- AZCOPY_AUTO_LOGIN_TYPE: AZCLI+ HTML_REPORT_URL: ${{ steps.upload-report.outputs.artifact-url }}+ PR_NUMBER: ${{ steps.pr.outputs.number }}+ with:+ github-token: ${{ secrets.GITHUB_TOKEN }}+ script: |+ const { postReportComment } = require('./tests/config/postReportComment');+ await postReportComment({+ github,+ context,+ core,+ reportFile: process.env.MARKDOWN_OUTPUT_FILE,+ prNumber: +process.env.PR_NUMBER,+ reportUrl: process.env.HTML_REPORT_URL,+ });- name: Publish Report URL as Commit Statususes: actions/github-script@v9@@ -64,6 +85,6 @@ jobs:repo: context.repo.repo,sha: '${{ github.event.workflow_run.head_sha }}',state: 'success',- target_url: '${{ steps.upload-report.outputs.reportUrl }}',+ target_url: '${{ steps.upload-report.outputs.artifact-url }}',context: 'HTML Report (${{ github.event.workflow_run.name }})',});diff --git a/.github/workflows/merge.config.ts b/.github/workflows/merge.config.tsindex 222b728f7c..5d4588b2a4 100644--- a/.github/workflows/merge.config.ts+++ b/.github/workflows/merge.config.ts@@ -1,4 +1,4 @@export default {testDir: '../../tests',- reporter: [[require.resolve('../../tests/config/ghaMarkdownReporter')], ['html']]-};\ No newline at end of file+ reporter: [[require.resolve('../../tests/config/markdownReporter')], ['html']],+};diff --git a/tests/config/ghaMarkdownReporter.ts b/tests/config/markdownReporter.tssimilarity index 50%rename from tests/config/ghaMarkdownReporter.tsrename to tests/config/markdownReporter.tsindex 548e2a2452..3d477e4a97 100644--- a/tests/config/ghaMarkdownReporter.ts+++ b/tests/config/markdownReporter.ts@@ -17,19 +17,19 @@import fs from 'fs';import path from 'path';-import type { MetadataWithCommitInfo } from '@testIsomorphic/types';-import type { IssueCommentEdge, Repository } from '@octokit/graphql-schema';import type { FullConfig, FullResult, Reporter, Suite, TestCase, TestError } from '@playwright/test/reporter';type MarkdownReporterOptions = {configDir: string, // TODO: make it public?- outputFile?: string;};+// Produces the markdown summary of a merged report and writes it to a file. The actual PR+// comment is posted by a separate workflow step (see postReportComment.js) once the HTML+// report has been uploaded as an artifact and its URL is known.class MarkdownReporter implements Reporter {private _options: MarkdownReporterOptions;private _fatalErrors: TestError[] = [];- protected _config!: FullConfig;+ private _config!: FullConfig;private _suite!: Suite;constructor(options: MarkdownReporterOptions) {@@ -82,24 +82,24 @@ class MarkdownReporter implements Reporter {lines.push(`**${summary.expected} passed${skipped}${didNotRun}**`);lines.push(``);- await this.publishReport(lines.join('\n'));+ await this._writeReport(lines.join('\n'));}- protected async publishReport(report: string): Promise<void> {- const maybeRelativeFile = this._options.outputFile || 'report.md';+ private async _writeReport(report: string): Promise<void> {+ const maybeRelativeFile = process.env.MARKDOWN_OUTPUT_FILE || 'report.md';const reportFile = path.resolve(this._options.configDir, maybeRelativeFile);await fs.promises.mkdir(path.dirname(reportFile), { recursive: true });await fs.promises.writeFile(reportFile, report);}- protected _incompleteRunWarning(): string | undefined {+ private _incompleteRunWarning(): string | undefined {const conclusion = process.env.WORKFLOW_RUN_CONCLUSION;if (!conclusion || conclusion === 'success' || conclusion === 'failure')return undefined;return `> [!WARNING]\n> The triggering workflow run ended with status \`${conclusion}\`. Results below may be incomplete — blob reports from cancelled or timed-out shards are missing, so passing/failing counts do not reflect the full test suite.`;}- protected _generateSummary() {+ private _generateSummary() {let didNotRun = 0;let skipped = 0;let expected = 0;@@ -158,134 +158,4 @@ function formatTestTitle(rootDir: string, test: TestCase): string {return `${testTitle}${extraTags.length ? ' ' + formattedTags : ''}`;}-class GHAMarkdownReporter extends MarkdownReporter {- private octokit: ReturnType<typeof import('@actions/github').getOctokit>;- private context: typeof import('@actions/github').context;- private core: typeof import('@actions/core');-- override async publishReport(report: string) {- this.core = await import('@actions/core');- const token = process.env.GITHUB_TOKEN || this.core.getInput('github-token');- if (!token) {- this.core.setFailed('Missing "github-token" input');- throw new Error('Missing "github-token" input');- }- const { context, getOctokit } = await import('@actions/github');- this.context = context;- this.octokit = getOctokit(token);-- this.core.info('Publishing report to PR.');- const { prNumber, prHref } = this.pullRequestFromMetadata();- if (!prNumber) {- this.core.info(`No PR number found, skipping GHA comment. PR href: ${prHref}`);- return;- }- this.core.info(`Posting comment to PR ${prHref}`);-- const prNodeId = await this.collapsePreviousComments(prNumber);- if (!prNodeId) {- this.core.warning(`No PR node ID found, skipping GHA comment. PR href: ${prHref}`);- return;- }- await this.addNewReportComment(prNodeId, report);- }-- private async collapsePreviousComments(prNumber: number) {- const { owner, repo } = this.context.repo;- const data = await this.octokit.graphql<{ repository: Repository }>(`- query($owner: String!, $repo: String!, $prNumber: Int!) {- repository(owner: $owner, name: $repo) {- pullRequest(number: $prNumber) {- id- comments(last: 100) {- nodes {- id- body- author {- __typename- login- }- }- }- }- }- }- `, { owner, repo, prNumber });- const comments = data.repository.pullRequest?.comments.nodes?.filter(comment =>- comment?.author?.__typename === 'Bot' &&- comment?.author?.login === 'github-actions' &&- comment.body?.includes(this._magicComment()));- const prId = data.repository.pullRequest?.id;- if (!comments?.length)- return prId;- const variableDecls = comments.map((_, i) => `$id${i}: ID!`).join(', ');- const mutations = comments.map((_, i) =>- `m${i}: minimizeComment(input: { subjectId: $id${i}, classifier: OUTDATED }) { clientMutationId }`);- const subjectIds = Object.fromEntries(comments.map((comment, i) => [`id${i}`, comment!.id]));- await this.octokit.graphql(`- mutation(${variableDecls}) {- ${mutations.join('\n')}- }- `, subjectIds);- return prId;- }-- private _magicComment() {- return `<!-- Generated by Playwright markdown reporter for ${this._workflowRunName()} in job ${process.env.GITHUB_JOB} -->`;- }-- private _workflowRunName() {- // When used via 'workflow_run' event.- const workflowRunName = this.context.payload.workflow_run?.name;- if (workflowRunName)- return workflowRunName;- // When used via 'pull_request'/'push' event.- // This is the name of the workflow file, e.g. 'ci.yml' or name if set.- return process.env.GITHUB_WORKFLOW;- }-- private async addNewReportComment(prNodeId: string, report: string) {- const reportUrl = process.env.HTML_REPORT_URL;- const mergeWorkflowUrl = `${this.context.serverUrl}/${this.context.repo.owner}/${this.context.repo.repo}/actions/runs/${this.context.runId}`;-- const body = formatComment([- this._magicComment(),- `### ${reportUrl ? `[Test results](${reportUrl})` : 'Test results'} for "${this._workflowRunName()}"`,- report,- '',- '---',- '',- `Merge [workflow run](${mergeWorkflowUrl}).`- ]);-- const response = await this.octokit.graphql<{ addComment: { commentEdge: IssueCommentEdge } }>(`- mutation($subjectId: ID!, $body: String!) {- addComment(input: {subjectId: $subjectId, body: $body}) {- commentEdge {- node {- ... on IssueComment {- url- }- }- }- }- }- `, { subjectId: prNodeId, body });- this.core.info(`Posted comment: ${response.addComment.commentEdge.node?.url}`);- }-- private pullRequestFromMetadata() {- const metadata = this._config.metadata as MetadataWithCommitInfo;- const prHref = metadata.ci?.prHref;- return { prNumber: parseInt(prHref?.split('/').pop() ?? '', 10), prHref };- }-}--function formatComment(lines: string[]) {- let body = lines.join('\n');- if (body.length > 65535)- body = body.substring(0, 65000) + `... ${body.length - 65000} more characters`;- return body;-}--export default GHAMarkdownReporter;+export default MarkdownReporter;diff --git a/tests/config/postReportComment.js b/tests/config/postReportComment.jsnew file mode 100644index 0000000000..55378d3a18--- /dev/null+++ b/tests/config/postReportComment.js@@ -0,0 +1,129 @@+/**+ * Copyright (c) Microsoft Corporation.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++const fs = require('fs');+const path = require('path');++/**+ * @param {{ github: any, context: any, core: any, reportFile: string, prNumber: number, reportUrl?: string }} params+ */+async function postReportComment({ github, context, core, reportFile, prNumber, reportUrl }) {+ if (!prNumber) {+ core.info('No PR number provided, skipping GHA comment.');+ return;+ }+ const report = fs.readFileSync(path.resolve(process.cwd(), reportFile), 'utf8');+ core.info(`Posting comment to PR #${prNumber}`);++ const prNodeId = await collapsePreviousComments(github, context, prNumber);+ if (!prNodeId) {+ core.warning(`No PR node ID found for #${prNumber}, skipping GHA comment.`);+ return;+ }+ const url = await addNewReportComment(github, context, prNodeId, report, reportUrl);+ core.info(`Posted comment: ${url}`);+}++function workflowRunName(context) {+ // When used via 'workflow_run' event.+ const name = context.payload.workflow_run?.name;+ if (name)+ return name;+ // When used via 'pull_request'/'push' event.+ return process.env.GITHUB_WORKFLOW;+}++function magicComment(context) {+ return `<!-- Generated by Playwright markdown reporter for ${workflowRunName(context)} in job ${process.env.GITHUB_JOB} -->`;+}++async function collapsePreviousComments(github, context, prNumber) {+ const { owner, repo } = context.repo;+ const data = await github.graphql(`+ query($owner: String!, $repo: String!, $prNumber: Int!) {+ repository(owner: $owner, name: $repo) {+ pullRequest(number: $prNumber) {+ id+ comments(last: 100) {+ nodes {+ id+ body+ author {+ __typename+ login+ }+ }+ }+ }+ }+ }+ `, { owner, repo, prNumber });+ const comments = data.repository.pullRequest?.comments.nodes?.filter(comment =>+ comment?.author?.__typename === 'Bot' &&+ comment?.author?.login === 'github-actions' &&+ comment.body?.includes(magicComment(context)));+ const prId = data.repository.pullRequest?.id;+ if (!comments?.length)+ return prId;+ const variableDecls = comments.map((_, i) => `$id${i}: ID!`).join(', ');+ const mutations = comments.map((_, i) =>+ `m${i}: minimizeComment(input: { subjectId: $id${i}, classifier: OUTDATED }) { clientMutationId }`);+ const subjectIds = Object.fromEntries(comments.map((comment, i) => [`id${i}`, comment.id]));+ await github.graphql(`+ mutation(${variableDecls}) {+ ${mutations.join('\n')}+ }+ `, subjectIds);+ return prId;+}++async function addNewReportComment(github, context, prNodeId, report, reportUrl) {+ const mergeWorkflowUrl = `${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`;++ const body = formatComment([+ magicComment(context),+ `### ${reportUrl ? `[Test results](${reportUrl})` : 'Test results'} for "${workflowRunName(context)}"`,+ report,+ '',+ '---',+ '',+ `Merge [workflow run](${mergeWorkflowUrl}).`+ ]);++ const response = await github.graphql(`+ mutation($subjectId: ID!, $body: String!) {+ addComment(input: {subjectId: $subjectId, body: $body}) {+ commentEdge {+ node {+ ... on IssueComment {+ url+ }+ }+ }+ }+ }+ `, { subjectId: prNodeId, body });+ return response.addComment.commentEdge.node?.url;+}++function formatComment(lines) {+ let body = lines.join('\n');+ if (body.length > 65535)+ body = body.substring(0, 65000) + `... ${body.length - 65000} more characters`;+ return body;+}++module.exports = { postReportComment };