Playwright is a framework for Web Testing and Automation. It allows testing Chromium, Firefox and WebKit with a single API.
TypeScript automationchromechromiume2e-testingelectronend-to-end-testingfirefoxjavascriptplaywrighttesttest-automationtestingtesting-toolswebwebkit
feat(prompt): link to error prompt in terminal (#35341)
Simon Knott2f3fe8f11353
diff --git a/packages/html-reporter/src/testErrorView.tsx b/packages/html-reporter/src/testErrorView.tsxindex 722917c191..881c420c22 100644--- a/packages/html-reporter/src/testErrorView.tsx+++ b/packages/html-reporter/src/testErrorView.tsx@@ -19,11 +19,12 @@ import * as React from 'react';import './testErrorView.css';import type { ImageDiff } from '@web/shared/imageDiffView';import { ImageDiffView } from '@web/shared/imageDiffView';+import { TestAttachment } from './types';export const TestErrorView: React.FC<{error: string;testId?: string;- prompt?: string;+ prompt?: TestAttachment;}> = ({ error, testId, prompt }) => {return (<CodeSnippet code={error} testId={testId}>@@ -46,13 +47,14 @@ export const CodeSnippet = ({ code, children, testId }: React.PropsWithChildren<);};-const PromptButton: React.FC<{ prompt: string }> = ({ prompt }) => {+const PromptButton: React.FC<{ prompt: TestAttachment }> = ({ prompt }) => {const [copied, setCopied] = React.useState(false);return <buttonclassName='button'style={{ minWidth: 100 }}onClick={async () => {- await navigator.clipboard.writeText(prompt);+ const text = prompt.body ? prompt.body : await fetch(prompt.path!).then(r => r.text());+ await navigator.clipboard.writeText(text);setCopied(true);setTimeout(() => {setCopied(false);diff --git a/packages/html-reporter/src/testResultView.tsx b/packages/html-reporter/src/testResultView.tsxindex ff979ffd4f..6a3175471a 100644--- a/packages/html-reporter/src/testResultView.tsx+++ b/packages/html-reporter/src/testResultView.tsx@@ -165,7 +165,7 @@ function classifyErrors(testErrors: string[], diffs: ImageDiff[], attachments: T}}- const prompt = attachments.find(a => a.name === `_prompt-${i}`)?.body;+ const prompt = attachments.find(a => a.name === `_prompt-${i}`);return { type: 'regular', error, prompt };});}diff --git a/packages/playwright/src/prompt.ts b/packages/playwright/src/prompt.tsindex 82be19bbd2..9c323f59b9 100644--- a/packages/playwright/src/prompt.ts+++ b/packages/playwright/src/prompt.ts@@ -14,7 +14,7 @@* limitations under the License.*/-import * as fs from 'fs';+import * as fs from 'fs/promises';import * as path from 'path';import { parseErrorStack } from 'playwright-core/lib/utils';@@ -38,17 +38,21 @@ export async function attachErrorPrompts(testInfo: TestInfo, sourceCache: Map<st}}- for (const [index, error] of testInfo.errors.entries()) {+ const errors = [...testInfo.errors.entries()].filter(([, error]) => {if (!error.message)- return;- if (testInfo.attachments.find(a => a.name === `_prompt-${index}`))- continue;+ return false;// Skip errors that are just a single line - they are likely to already be the error message.if (!error.message.includes('\n') && !meaningfulSingleLineErrors.has(error.message))- continue;+ return false;++ return true;+ });+ for (const [index, error] of errors) {const metadata = testInfo.config.metadata as MetadataWithCommitInfo;+ if (testInfo.attachments.find(a => a.name === `_prompt-${index}`))+ continue;const promptParts = [`# Instructions`,@@ -119,10 +123,13 @@ export async function attachErrorPrompts(testInfo: TestInfo, sourceCache: Map<st);}+ const promptPath = testInfo.outputPath(errors.length === 1 ? `prompt.md` : `prompt-${index}.md`);+ await fs.writeFile(promptPath, promptParts.join('\n'), 'utf8');+(testInfo as TestInfoImpl)._attach({name: `_prompt-${index}`,contentType: 'text/markdown',- body: Buffer.from(promptParts.join('\n')),+ path: promptPath,}, undefined);}}@@ -144,7 +151,7 @@ async function loadSource(file: string, sourceCache: Map<string, string>) {let source = sourceCache.get(file);if (!source) {// A mild race is Ok here.- source = await fs.promises.readFile(file, 'utf8');+ source = await fs.readFile(file, 'utf8');sourceCache.set(file, source);}return source;diff --git a/packages/playwright/src/reporters/base.ts b/packages/playwright/src/reporters/base.tsindex c8326ece31..58b98a9574 100644--- a/packages/playwright/src/reporters/base.ts+++ b/packages/playwright/src/reporters/base.ts@@ -350,15 +350,18 @@ export function formatFailure(screen: Screen, config: FullConfig, test: TestCaseconst errors = formatResultFailure(screen, test, result, ' ');if (!errors.length)continue;- const retryLines = [];if (result.retry) {- retryLines.push('');- retryLines.push(screen.colors.gray(separator(screen, ` Retry #${result.retry}`)));+ resultLines.push('');+ resultLines.push(screen.colors.gray(separator(screen, ` Retry #${result.retry}`)));}- resultLines.push(...retryLines);resultLines.push(...errors.map(error => '\n' + error.message));for (let i = 0; i < result.attachments.length; ++i) {const attachment = result.attachments[i];+ if (attachment.name.startsWith('_prompt') && attachment.path) {+ resultLines.push('');+ resultLines.push(screen.colors.dim(` Error Prompt: ${relativeFilePath(screen, config, attachment.path)}`));+ continue;+ }if (attachment.name.startsWith('_'))continue;const hasPrintableContent = attachment.contentType.startsWith('text/');diff --git a/tests/playwright-test/playwright.artifacts.spec.ts b/tests/playwright-test/playwright.artifacts.spec.tsindex 0460be88b4..5f19139e46 100644--- a/tests/playwright-test/playwright.artifacts.spec.ts+++ b/tests/playwright-test/playwright.artifacts.spec.ts@@ -130,7 +130,7 @@ test('should work with screenshot: on', async ({ runInlineTest }, testInfo) => {'playwright.config.ts': `module.exports = { use: { screenshot: 'on' } };`,- }, { workers: 1 });+ }, { workers: 1 }, { PLAYWRIGHT_NO_COPY_PROMPT: 'true' });expect(result.exitCode).toBe(1);expect(result.passed).toBe(5);@@ -168,7 +168,7 @@ test('should work with screenshot: only-on-failure', async ({ runInlineTest }, t'playwright.config.ts': `module.exports = { use: { screenshot: 'only-on-failure' } };`,- }, { workers: 1 });+ }, { workers: 1 }, { PLAYWRIGHT_NO_COPY_PROMPT: 'true' });expect(result.exitCode).toBe(1);expect(result.passed).toBe(5);@@ -204,7 +204,7 @@ test('should work with screenshot: on-first-failure', async ({ runInlineTest },use: { screenshot: 'on-first-failure' }};`,- }, { workers: 1 });+ }, { workers: 1 }, { PLAYWRIGHT_NO_COPY_PROMPT: 'true' });expect(result.exitCode).toBe(1);expect(result.passed).toBe(0);@@ -230,7 +230,7 @@ test('should work with screenshot: only-on-failure & fullPage', async ({ runInli'playwright.config.ts': `module.exports = { use: { screenshot: { mode: 'only-on-failure', fullPage: true } } };`,- }, { workers: 1 });+ }, { workers: 1 }, { PLAYWRIGHT_NO_COPY_PROMPT: 'true' });expect(result.exitCode).toBe(1);expect(result.passed).toBe(0);expect(result.failed).toBe(1);@@ -263,7 +263,7 @@ test('should capture a single screenshot on failure when afterAll fails', asyncawait page.setContent('this is test');});`,- }, { workers: 1 });+ }, { workers: 1 }, { PLAYWRIGHT_NO_COPY_PROMPT: 'true' });expect(result.exitCode).toBe(1);expect(result.passed).toBe(0);@@ -282,7 +282,7 @@ test('should work with trace: on', async ({ runInlineTest }, testInfo) => {'playwright.config.ts': `module.exports = { use: { trace: 'on' } };`,- }, { workers: 1 });+ }, { workers: 1 }, { PLAYWRIGHT_NO_COPY_PROMPT: 'true' });expect(result.exitCode).toBe(1);expect(result.passed).toBe(5);@@ -318,7 +318,7 @@ test('should work with trace: retain-on-failure', async ({ runInlineTest }, test'playwright.config.ts': `module.exports = { use: { trace: 'retain-on-failure' } };`,- }, { workers: 1 });+ }, { workers: 1 }, { PLAYWRIGHT_NO_COPY_PROMPT: 'true' });expect(result.exitCode).toBe(1);expect(result.passed).toBe(5);@@ -344,7 +344,7 @@ test('should work with trace: on-first-retry', async ({ runInlineTest }, testInf'playwright.config.ts': `module.exports = { use: { trace: 'on-first-retry' } };`,- }, { workers: 1, retries: 1 });+ }, { workers: 1, retries: 1 }, { PLAYWRIGHT_NO_COPY_PROMPT: 'true' });expect(result.exitCode).toBe(1);expect(result.passed).toBe(5);@@ -370,7 +370,7 @@ test('should work with trace: on-all-retries', async ({ runInlineTest }, testInf'playwright.config.ts': `module.exports = { use: { trace: 'on-all-retries' } };`,- }, { workers: 1, retries: 2 });+ }, { workers: 1, retries: 2 }, { PLAYWRIGHT_NO_COPY_PROMPT: 'true' });expect(result.exitCode).toBe(1);expect(result.passed).toBe(5);@@ -406,7 +406,7 @@ test('should work with trace: retain-on-first-failure', async ({ runInlineTest }'playwright.config.ts': `module.exports = { use: { trace: 'retain-on-first-failure' } };`,- }, { workers: 1, retries: 2 });+ }, { workers: 1, retries: 2 }, { PLAYWRIGHT_NO_COPY_PROMPT: 'true' });expect(result.exitCode).toBe(1);expect(result.passed).toBe(5);@@ -442,7 +442,7 @@ test('should take screenshot when page is closed in afterEach', async ({ runInliexpect(1).toBe(2);});`,- }, { workers: 1 });+ }, { workers: 1 }, { PLAYWRIGHT_NO_COPY_PROMPT: 'true' });expect(result.exitCode).toBe(1);expect(result.failed).toBe(1);diff --git a/tests/playwright-test/playwright.spec.ts b/tests/playwright-test/playwright.spec.tsindex 5e85619b65..f75aba94a8 100644--- a/tests/playwright-test/playwright.spec.ts+++ b/tests/playwright-test/playwright.spec.ts@@ -510,7 +510,7 @@ test('should work with video: on-first-retry', async ({ runInlineTest }) => {expect(fs.existsSync(dirPass)).toBeFalsy();const dirFail = test.info().outputPath('test-results', 'a-fail-chromium');- expect(fs.existsSync(dirFail)).toBeFalsy();+ expect(fs.readdirSync(dirFail)).toEqual(['prompt.md']);const dirRetry = test.info().outputPath('test-results', 'a-fail-chromium-retry1');const videoFailRetry = fs.readdirSync(dirRetry).find(file => file.endsWith('webm'));diff --git a/tests/playwright-test/reporter-line.spec.ts b/tests/playwright-test/reporter-line.spec.tsindex 9c69fd9cdc..777e771210 100644--- a/tests/playwright-test/reporter-line.spec.ts+++ b/tests/playwright-test/reporter-line.spec.ts@@ -14,6 +14,7 @@* limitations under the License.*/+import path from 'path';import { test, expect } from './playwright-test-fixtures';for (const useIntermediateMergeReport of [false, true] as const) {@@ -188,5 +189,22 @@ for (const useIntermediateMergeReport of [false, true] as const) {expect(text).toContain('1) a.test.ts:3:15 › passes ──');expect(result.exitCode).toBe(1);});++ test('should show error prompt with relative path', async ({ runInlineTest, useIntermediateMergeReport }) => {+ const result = await runInlineTest({+ 'a.test.js': `+ const { test, expect } = require('@playwright/test');+ test('one', async ({}) => {+ expect(1).toBe(0);+ });+ `,+ }, { reporter: 'line' });+ const text = result.output;+ if (useIntermediateMergeReport)+ expect(text).toContain(`Error Prompt: ${path.join('blob-report', 'resources')}`);+ else+ expect(text).toContain(`Error Prompt: ${path.join('test-results', 'a-one', 'prompt.md')}`);+ expect(result.exitCode).toBe(1);+ });});}