Skip to content

Playwright is a framework for Web Testing and Automation. It allows testing Chromium, Firefox and WebKit with a single API.

6.6 KB
Raw Blame
/**
 * Copyright (c) Microsoft Corporation.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

import { test, expect } from './playwright-test-fixtures';
import path from 'path';

function relativeFilePath(file: string): string {
  if (!path.isAbsolute(file))
    return file;
  return path.relative(process.cwd(), file);
}

for (const useIntermediateMergeReport of [false, true] as const) {
  test.describe(`${useIntermediateMergeReport ? 'merged' : 'created'}`, () => {
    test.use({ useIntermediateMergeReport });

    test('print GitHub annotations for success', async ({ runInlineTest }) => {
      const result = await runInlineTest({
        'a.test.js': `
          import { test, expect } from '@playwright/test';
          test('example1', async ({}) => {
            expect(1 + 1).toBe(2);
          });
        `
      }, { reporter: 'github' });
      const text = result.output;
      expect(text).not.toContain('::error');
      expect(text).toContain('::notice title=🎭 Playwright Run Summary::  1 passed');
      expect(result.exitCode).toBe(0);
    });

    test('print GitHub annotations for failed tests', {
      annotation: { type: 'issue', description: 'https://github.com/microsoft/playwright/issues/43461' },
    }, async ({ runInlineTest }, testInfo) => {
      const result = await runInlineTest({
        'a.test.js': `
          const { test, expect } = require('@playwright/test');
          test('example', async ({}) => {
            expect(1 + 1).toBe(3);
          });
          test('login, then 100% off', async ({}) => {
            throw new Error('GET /a%0Ab failed\\r\\nagain');
          });
        `
      }, { retries: 3, reporter: 'github' }, { GITHUB_WORKSPACE: process.cwd() });
      const text = result.output;
      const testPath = relativeFilePath(testInfo.outputPath('a.test.js'));
      expect(text).toContain(`::error file=${testPath},title=a.test.js%3A3%3A11 β€Ί example,line=4,col=27::  1) a.test.js:3:11 β€Ί example ──────────────────────────────────────────────────────────────────────%0A%0A    Retry #1`);
      expect(text).toContain(`::error file=${testPath},title=a.test.js%3A3%3A11 β€Ί example,line=4,col=27::  1) a.test.js:3:11 β€Ί example ──────────────────────────────────────────────────────────────────────%0A%0A    Retry #2`);
      expect(text).toContain(`::error file=${testPath},title=a.test.js%3A3%3A11 β€Ί example,line=4,col=27::  1) a.test.js:3:11 β€Ί example ──────────────────────────────────────────────────────────────────────%0A%0A    Retry #3`);
      // Commas, percent signs and line breaks are escaped the way the GitHub Actions runner unescapes them.
      // The column of a thrown error differs between Node and Bun, so do not pin it.
      expect(text).toContain(`::error file=${testPath},title=a.test.js%3A6%3A11 β€Ί login%2C then 100%25 off,line=7,col=`);
      expect(text).toContain(`::  2) a.test.js:6:11 β€Ί login, then 100%25 off `);
      expect(text).toContain(`Error: GET /a%250Ab failed%0D%0A    again`);
      expect(result.exitCode).toBe(1);
    });

    test('print GitHub annotations for slow tests', async ({ runInlineTest }) => {
      const result = await runInlineTest({
        'playwright.config.ts': `
          export default {
            reportSlowTests: { max: 0, threshold: 100 }
          };
        `,
        'a.test.js': `
          import { test, expect } from '@playwright/test';
          test('slow test', async ({}) => {
            await new Promise(f => setTimeout(f, 200));
          });
        `
      }, { retries: 3, reporter: 'github' }, { GITHUB_WORKSPACE: '' });
      const text = result.output;
      expect(text).toContain('::warning title=Slow Test,file=a.test.js::a.test.js took');
      expect(text).toContain('::notice title=🎭 Playwright Run Summary::  1 passed');
      expect(result.exitCode).toBe(0);
    });

    test('print GitHub annotations for global error', async ({ runInlineTest }) => {
      const result = await runInlineTest({
        'a.test.ts': `
          import { test as base, expect } from '@playwright/test';
          const test = base.extend({
            w: [async ({}, use) => {
              await use();
              throw new Error('Oh my!');
            }, { scope: 'worker' }],
          });
          test('passes but...', ({w}) => {
          });
        `,
      }, { reporter: 'github' });
      const text = result.output;
      expect(text).toContain('::error ::Error: Oh my!%0A%0A');
      expect(result.exitCode).toBe(1);
    });

    test('emit GitHub failure annotation before run summary', async ({ runInlineTest }) => {
      const result = await runInlineTest({
        'a.test.js': `
          const { test, expect } = require('@playwright/test');
          test('failing', async ({}) => {
            expect(1 + 1).toBe(3);
          });
          test.skip('marker', async ({}) => {});
        `
      }, { workers: 1, reporter: 'github' }, { GITHUB_WORKSPACE: process.cwd() });
      const text = result.output;
      const errorIndex = text.indexOf('::error');
      const summaryIndex = text.indexOf('🎭 Playwright Run Summary');
      expect(errorIndex).toBeGreaterThanOrEqual(0);
      expect(summaryIndex).toBeGreaterThan(errorIndex);
      expect(result.exitCode).toBe(1);
    });

    test('starts GitHub failure annotation on a new line with dot reporter', async ({ runInlineTest }) => {
      const result = await runInlineTest({
        'a.test.js': `
          const { test, expect } = require('@playwright/test');
          test('failing', async ({}) => {
            expect(1 + 1).toBe(3);
          });
        `
      }, { workers: 1, reporter: 'dot,github' }, { GITHUB_WORKSPACE: process.cwd() });
      expect(result.output).toContain('F\n::error file=');
      expect(result.exitCode).toBe(1);
    });
  });
}