Skip to content

Playwright is a framework for Web Testing and Automation. It allows testing Chromium, Firefox and WebKit with a single API.

devops: publish test report as a browser-openable GitHub artifact (#41520)

Dmitry Gozman800400f816d7
diff --git a/.github/workflows/create_test_report.yml b/.github/workflows/create_test_report.yml
index ddf32c64b1..7850e4f1c0 100644
--- a/.github/workflows/create_test_report.yml
+++ b/.github/workflows/create_test_report.yml
@@ -10,10 +10,11 @@ jobs:
pull-requests: write
checks: write
statuses: write
- id-token: write # This is required for OIDC login (azure/login) to succeed
contents: read # This is required for actions/checkout to succeed
if: ${{ github.event.workflow_run.event == 'pull_request' || github.event.workflow_run.event == 'push' }}
runs-on: ubuntu-latest
+ env:
+ MARKDOWN_OUTPUT_FILE: ${{ github.workspace }}/report.md
steps:
- uses: actions/checkout@v6
- uses: actions/setup-node@v6
@@ -32,28 +33,48 @@ jobs:
run: |
npx playwright merge-reports --config .github/workflows/merge.config.ts ./all-blob-reports
env:
- GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
NODE_OPTIONS: --max-old-space-size=8192
- HTML_REPORT_URL: 'https://mspwblobreport.z1.web.core.windows.net/run-${{ github.event.workflow_run.id }}-${{ github.event.workflow_run.run_attempt }}-${{ github.sha }}/index.html'
WORKFLOW_RUN_CONCLUSION: ${{ github.event.workflow_run.conclusion }}
- - name: Azure Login
- uses: azure/login@v3
+ - name: Upload HTML report
+ id: upload-report
+ uses: actions/upload-artifact@v7
with:
- client-id: ${{ secrets.AZURE_BLOB_REPORTS_CLIENT_ID }}
- tenant-id: ${{ secrets.AZURE_BLOB_REPORTS_TENANT_ID }}
- subscription-id: ${{ secrets.AZURE_BLOB_REPORTS_SUBSCRIPTION_ID }}
+ path: playwright-report/index.html
+ archive: false # Upload as a single, browser-openable file (no zip)
+ retention-days: 30
- - name: Upload HTML report to Azure
- id: upload-report
+ # The triggering workflow ran on a pull_request event, but workflow_run.pull_requests is
+ # empty for PRs from forks, so resolve the number from the head repo + branch instead.
+ - name: Resolve PR number
+ if: ${{ github.event.workflow_run.event == 'pull_request' }}
+ id: pr
+ env:
+ GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
run: |
- REPORT_DIR='run-${{ github.event.workflow_run.id }}-${{ github.event.workflow_run.run_attempt }}-${{ github.sha }}'
- azcopy cp --recursive "./playwright-report/*" "https://mspwblobreport.blob.core.windows.net/\$web/$REPORT_DIR"
- REPORT_URL="https://mspwblobreport.z1.web.core.windows.net/$REPORT_DIR/index.html"
- echo "Report url: $REPORT_URL"
- echo "reportUrl=$REPORT_URL" >> "$GITHUB_OUTPUT"
+ NUMBER=$(gh pr view --repo "${{ github.repository }}" \
+ "${{ github.event.workflow_run.head_repository.owner.login }}:${{ github.event.workflow_run.head_branch }}" \
+ --json number --jq '.number' 2>/dev/null || true)
+ echo "number=$NUMBER" >> "$GITHUB_OUTPUT"
+
+ - name: Post report comment to PR
+ if: ${{ steps.pr.outputs.number }}
+ uses: actions/github-script@v8
env:
- AZCOPY_AUTO_LOGIN_TYPE: AZCLI
+ HTML_REPORT_URL: ${{ steps.upload-report.outputs.artifact-url }}
+ PR_NUMBER: ${{ steps.pr.outputs.number }}
+ with:
+ github-token: ${{ secrets.GITHUB_TOKEN }}
+ script: |
+ const { postReportComment } = require('./tests/config/postReportComment');
+ await postReportComment({
+ github,
+ context,
+ core,
+ reportFile: process.env.MARKDOWN_OUTPUT_FILE,
+ prNumber: +process.env.PR_NUMBER,
+ reportUrl: process.env.HTML_REPORT_URL,
+ });
- name: Publish Report URL as Commit Status
uses: actions/github-script@v9
@@ -64,6 +85,6 @@ jobs:
repo: context.repo.repo,
sha: '${{ github.event.workflow_run.head_sha }}',
state: 'success',
- target_url: '${{ steps.upload-report.outputs.reportUrl }}',
+ target_url: '${{ steps.upload-report.outputs.artifact-url }}',
context: 'HTML Report (${{ github.event.workflow_run.name }})',
});
diff --git a/.github/workflows/merge.config.ts b/.github/workflows/merge.config.ts
index 222b728f7c..5d4588b2a4 100644
--- a/.github/workflows/merge.config.ts
+++ b/.github/workflows/merge.config.ts
@@ -1,4 +1,4 @@
export default {
testDir: '../../tests',
- reporter: [[require.resolve('../../tests/config/ghaMarkdownReporter')], ['html']]
-};
\ No newline at end of file
+ reporter: [[require.resolve('../../tests/config/markdownReporter')], ['html']],
+};
diff --git a/tests/config/ghaMarkdownReporter.ts b/tests/config/markdownReporter.ts
similarity index 50%
rename from tests/config/ghaMarkdownReporter.ts
rename to tests/config/markdownReporter.ts
index 548e2a2452..3d477e4a97 100644
--- a/tests/config/ghaMarkdownReporter.ts
+++ b/tests/config/markdownReporter.ts
@@ -17,19 +17,19 @@
import fs from 'fs';
import path from 'path';
-import type { MetadataWithCommitInfo } from '@testIsomorphic/types';
-import type { IssueCommentEdge, Repository } from '@octokit/graphql-schema';
import type { FullConfig, FullResult, Reporter, Suite, TestCase, TestError } from '@playwright/test/reporter';
type MarkdownReporterOptions = {
configDir: string, // TODO: make it public?
- outputFile?: string;
};
+// Produces the markdown summary of a merged report and writes it to a file. The actual PR
+// comment is posted by a separate workflow step (see postReportComment.js) once the HTML
+// report has been uploaded as an artifact and its URL is known.
class MarkdownReporter implements Reporter {
private _options: MarkdownReporterOptions;
private _fatalErrors: TestError[] = [];
- protected _config!: FullConfig;
+ private _config!: FullConfig;
private _suite!: Suite;
constructor(options: MarkdownReporterOptions) {
@@ -82,24 +82,24 @@ class MarkdownReporter implements Reporter {
lines.push(`**${summary.expected} passed${skipped}${didNotRun}**`);
lines.push(``);
- await this.publishReport(lines.join('\n'));
+ await this._writeReport(lines.join('\n'));
}
- protected async publishReport(report: string): Promise<void> {
- const maybeRelativeFile = this._options.outputFile || 'report.md';
+ private async _writeReport(report: string): Promise<void> {
+ const maybeRelativeFile = process.env.MARKDOWN_OUTPUT_FILE || 'report.md';
const reportFile = path.resolve(this._options.configDir, maybeRelativeFile);
await fs.promises.mkdir(path.dirname(reportFile), { recursive: true });
await fs.promises.writeFile(reportFile, report);
}
- protected _incompleteRunWarning(): string | undefined {
+ private _incompleteRunWarning(): string | undefined {
const conclusion = process.env.WORKFLOW_RUN_CONCLUSION;
if (!conclusion || conclusion === 'success' || conclusion === 'failure')
return undefined;
return `> [!WARNING]\n> The triggering workflow run ended with status \`${conclusion}\`. Results below may be incomplete — blob reports from cancelled or timed-out shards are missing, so passing/failing counts do not reflect the full test suite.`;
}
- protected _generateSummary() {
+ private _generateSummary() {
let didNotRun = 0;
let skipped = 0;
let expected = 0;
@@ -158,134 +158,4 @@ function formatTestTitle(rootDir: string, test: TestCase): string {
return `${testTitle}${extraTags.length ? ' ' + formattedTags : ''}`;
}
-class GHAMarkdownReporter extends MarkdownReporter {
- private octokit: ReturnType<typeof import('@actions/github').getOctokit>;
- private context: typeof import('@actions/github').context;
- private core: typeof import('@actions/core');
-
- override async publishReport(report: string) {
- this.core = await import('@actions/core');
- const token = process.env.GITHUB_TOKEN || this.core.getInput('github-token');
- if (!token) {
- this.core.setFailed('Missing "github-token" input');
- throw new Error('Missing "github-token" input');
- }
- const { context, getOctokit } = await import('@actions/github');
- this.context = context;
- this.octokit = getOctokit(token);
-
- this.core.info('Publishing report to PR.');
- const { prNumber, prHref } = this.pullRequestFromMetadata();
- if (!prNumber) {
- this.core.info(`No PR number found, skipping GHA comment. PR href: ${prHref}`);
- return;
- }
- this.core.info(`Posting comment to PR ${prHref}`);
-
- const prNodeId = await this.collapsePreviousComments(prNumber);
- if (!prNodeId) {
- this.core.warning(`No PR node ID found, skipping GHA comment. PR href: ${prHref}`);
- return;
- }
- await this.addNewReportComment(prNodeId, report);
- }
-
- private async collapsePreviousComments(prNumber: number) {
- const { owner, repo } = this.context.repo;
- const data = await this.octokit.graphql<{ repository: Repository }>(`
- query($owner: String!, $repo: String!, $prNumber: Int!) {
- repository(owner: $owner, name: $repo) {
- pullRequest(number: $prNumber) {
- id
- comments(last: 100) {
- nodes {
- id
- body
- author {
- __typename
- login
- }
- }
- }
- }
- }
- }
- `, { owner, repo, prNumber });
- const comments = data.repository.pullRequest?.comments.nodes?.filter(comment =>
- comment?.author?.__typename === 'Bot' &&
- comment?.author?.login === 'github-actions' &&
- comment.body?.includes(this._magicComment()));
- const prId = data.repository.pullRequest?.id;
- if (!comments?.length)
- return prId;
- const variableDecls = comments.map((_, i) => `$id${i}: ID!`).join(', ');
- const mutations = comments.map((_, i) =>
- `m${i}: minimizeComment(input: { subjectId: $id${i}, classifier: OUTDATED }) { clientMutationId }`);
- const subjectIds = Object.fromEntries(comments.map((comment, i) => [`id${i}`, comment!.id]));
- await this.octokit.graphql(`
- mutation(${variableDecls}) {
- ${mutations.join('\n')}
- }
- `, subjectIds);
- return prId;
- }
-
- private _magicComment() {
- return `<!-- Generated by Playwright markdown reporter for ${this._workflowRunName()} in job ${process.env.GITHUB_JOB} -->`;
- }
-
- private _workflowRunName() {
- // When used via 'workflow_run' event.
- const workflowRunName = this.context.payload.workflow_run?.name;
- if (workflowRunName)
- return workflowRunName;
- // When used via 'pull_request'/'push' event.
- // This is the name of the workflow file, e.g. 'ci.yml' or name if set.
- return process.env.GITHUB_WORKFLOW;
- }
-
- private async addNewReportComment(prNodeId: string, report: string) {
- const reportUrl = process.env.HTML_REPORT_URL;
- const mergeWorkflowUrl = `${this.context.serverUrl}/${this.context.repo.owner}/${this.context.repo.repo}/actions/runs/${this.context.runId}`;
-
- const body = formatComment([
- this._magicComment(),
- `### ${reportUrl ? `[Test results](${reportUrl})` : 'Test results'} for "${this._workflowRunName()}"`,
- report,
- '',
- '---',
- '',
- `Merge [workflow run](${mergeWorkflowUrl}).`
- ]);
-
- const response = await this.octokit.graphql<{ addComment: { commentEdge: IssueCommentEdge } }>(`
- mutation($subjectId: ID!, $body: String!) {
- addComment(input: {subjectId: $subjectId, body: $body}) {
- commentEdge {
- node {
- ... on IssueComment {
- url
- }
- }
- }
- }
- }
- `, { subjectId: prNodeId, body });
- this.core.info(`Posted comment: ${response.addComment.commentEdge.node?.url}`);
- }
-
- private pullRequestFromMetadata() {
- const metadata = this._config.metadata as MetadataWithCommitInfo;
- const prHref = metadata.ci?.prHref;
- return { prNumber: parseInt(prHref?.split('/').pop() ?? '', 10), prHref };
- }
-}
-
-function formatComment(lines: string[]) {
- let body = lines.join('\n');
- if (body.length > 65535)
- body = body.substring(0, 65000) + `... ${body.length - 65000} more characters`;
- return body;
-}
-
-export default GHAMarkdownReporter;
+export default MarkdownReporter;
diff --git a/tests/config/postReportComment.js b/tests/config/postReportComment.js
new file mode 100644
index 0000000000..55378d3a18
--- /dev/null
+++ b/tests/config/postReportComment.js
@@ -0,0 +1,129 @@
+/**
+ * Copyright (c) Microsoft Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+const fs = require('fs');
+const path = require('path');
+
+/**
+ * @param {{ github: any, context: any, core: any, reportFile: string, prNumber: number, reportUrl?: string }} params
+ */
+async function postReportComment({ github, context, core, reportFile, prNumber, reportUrl }) {
+ if (!prNumber) {
+ core.info('No PR number provided, skipping GHA comment.');
+ return;
+ }
+ const report = fs.readFileSync(path.resolve(process.cwd(), reportFile), 'utf8');
+ core.info(`Posting comment to PR #${prNumber}`);
+
+ const prNodeId = await collapsePreviousComments(github, context, prNumber);
+ if (!prNodeId) {
+ core.warning(`No PR node ID found for #${prNumber}, skipping GHA comment.`);
+ return;
+ }
+ const url = await addNewReportComment(github, context, prNodeId, report, reportUrl);
+ core.info(`Posted comment: ${url}`);
+}
+
+function workflowRunName(context) {
+ // When used via 'workflow_run' event.
+ const name = context.payload.workflow_run?.name;
+ if (name)
+ return name;
+ // When used via 'pull_request'/'push' event.
+ return process.env.GITHUB_WORKFLOW;
+}
+
+function magicComment(context) {
+ return `<!-- Generated by Playwright markdown reporter for ${workflowRunName(context)} in job ${process.env.GITHUB_JOB} -->`;
+}
+
+async function collapsePreviousComments(github, context, prNumber) {
+ const { owner, repo } = context.repo;
+ const data = await github.graphql(`
+ query($owner: String!, $repo: String!, $prNumber: Int!) {
+ repository(owner: $owner, name: $repo) {
+ pullRequest(number: $prNumber) {
+ id
+ comments(last: 100) {
+ nodes {
+ id
+ body
+ author {
+ __typename
+ login
+ }
+ }
+ }
+ }
+ }
+ }
+ `, { owner, repo, prNumber });
+ const comments = data.repository.pullRequest?.comments.nodes?.filter(comment =>
+ comment?.author?.__typename === 'Bot' &&
+ comment?.author?.login === 'github-actions' &&
+ comment.body?.includes(magicComment(context)));
+ const prId = data.repository.pullRequest?.id;
+ if (!comments?.length)
+ return prId;
+ const variableDecls = comments.map((_, i) => `$id${i}: ID!`).join(', ');
+ const mutations = comments.map((_, i) =>
+ `m${i}: minimizeComment(input: { subjectId: $id${i}, classifier: OUTDATED }) { clientMutationId }`);
+ const subjectIds = Object.fromEntries(comments.map((comment, i) => [`id${i}`, comment.id]));
+ await github.graphql(`
+ mutation(${variableDecls}) {
+ ${mutations.join('\n')}
+ }
+ `, subjectIds);
+ return prId;
+}
+
+async function addNewReportComment(github, context, prNodeId, report, reportUrl) {
+ const mergeWorkflowUrl = `${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`;
+
+ const body = formatComment([
+ magicComment(context),
+ `### ${reportUrl ? `[Test results](${reportUrl})` : 'Test results'} for "${workflowRunName(context)}"`,
+ report,
+ '',
+ '---',
+ '',
+ `Merge [workflow run](${mergeWorkflowUrl}).`
+ ]);
+
+ const response = await github.graphql(`
+ mutation($subjectId: ID!, $body: String!) {
+ addComment(input: {subjectId: $subjectId, body: $body}) {
+ commentEdge {
+ node {
+ ... on IssueComment {
+ url
+ }
+ }
+ }
+ }
+ }
+ `, { subjectId: prNodeId, body });
+ return response.addComment.commentEdge.node?.url;
+}
+
+function formatComment(lines) {
+ let body = lines.join('\n');
+ if (body.length > 65535)
+ body = body.substring(0, 65000) + `... ${body.length - 65000} more characters`;
+ return body;
+}
+
+module.exports = { postReportComment };