Skip to content

Playwright is a framework for Web Testing and Automation. It allows testing Chromium, Firefox and WebKit with a single API.

feat(prompt): link to error prompt in terminal (#35341)

Simon Knott2f3fe8f11353
diff --git a/packages/html-reporter/src/testErrorView.tsx b/packages/html-reporter/src/testErrorView.tsx
index 722917c191..881c420c22 100644
--- a/packages/html-reporter/src/testErrorView.tsx
+++ b/packages/html-reporter/src/testErrorView.tsx
@@ -19,11 +19,12 @@ import * as React from 'react';
import './testErrorView.css';
import type { ImageDiff } from '@web/shared/imageDiffView';
import { ImageDiffView } from '@web/shared/imageDiffView';
+import { TestAttachment } from './types';
export const TestErrorView: React.FC<{
error: string;
testId?: string;
- prompt?: string;
+ prompt?: TestAttachment;
}> = ({ error, testId, prompt }) => {
return (
<CodeSnippet code={error} testId={testId}>
@@ -46,13 +47,14 @@ export const CodeSnippet = ({ code, children, testId }: React.PropsWithChildren<
);
};
-const PromptButton: React.FC<{ prompt: string }> = ({ prompt }) => {
+const PromptButton: React.FC<{ prompt: TestAttachment }> = ({ prompt }) => {
const [copied, setCopied] = React.useState(false);
return <button
className='button'
style={{ minWidth: 100 }}
onClick={async () => {
- await navigator.clipboard.writeText(prompt);
+ const text = prompt.body ? prompt.body : await fetch(prompt.path!).then(r => r.text());
+ await navigator.clipboard.writeText(text);
setCopied(true);
setTimeout(() => {
setCopied(false);
diff --git a/packages/html-reporter/src/testResultView.tsx b/packages/html-reporter/src/testResultView.tsx
index ff979ffd4f..6a3175471a 100644
--- a/packages/html-reporter/src/testResultView.tsx
+++ b/packages/html-reporter/src/testResultView.tsx
@@ -165,7 +165,7 @@ function classifyErrors(testErrors: string[], diffs: ImageDiff[], attachments: T
}
}
- const prompt = attachments.find(a => a.name === `_prompt-${i}`)?.body;
+ const prompt = attachments.find(a => a.name === `_prompt-${i}`);
return { type: 'regular', error, prompt };
});
}
diff --git a/packages/playwright/src/prompt.ts b/packages/playwright/src/prompt.ts
index 82be19bbd2..9c323f59b9 100644
--- a/packages/playwright/src/prompt.ts
+++ b/packages/playwright/src/prompt.ts
@@ -14,7 +14,7 @@
* limitations under the License.
*/
-import * as fs from 'fs';
+import * as fs from 'fs/promises';
import * as path from 'path';
import { parseErrorStack } from 'playwright-core/lib/utils';
@@ -38,17 +38,21 @@ export async function attachErrorPrompts(testInfo: TestInfo, sourceCache: Map<st
}
}
- for (const [index, error] of testInfo.errors.entries()) {
+ const errors = [...testInfo.errors.entries()].filter(([, error]) => {
if (!error.message)
- return;
- if (testInfo.attachments.find(a => a.name === `_prompt-${index}`))
- continue;
+ return false;
// Skip errors that are just a single line - they are likely to already be the error message.
if (!error.message.includes('\n') && !meaningfulSingleLineErrors.has(error.message))
- continue;
+ return false;
+
+ return true;
+ });
+ for (const [index, error] of errors) {
const metadata = testInfo.config.metadata as MetadataWithCommitInfo;
+ if (testInfo.attachments.find(a => a.name === `_prompt-${index}`))
+ continue;
const promptParts = [
`# Instructions`,
@@ -119,10 +123,13 @@ export async function attachErrorPrompts(testInfo: TestInfo, sourceCache: Map<st
);
}
+ const promptPath = testInfo.outputPath(errors.length === 1 ? `prompt.md` : `prompt-${index}.md`);
+ await fs.writeFile(promptPath, promptParts.join('\n'), 'utf8');
+
(testInfo as TestInfoImpl)._attach({
name: `_prompt-${index}`,
contentType: 'text/markdown',
- body: Buffer.from(promptParts.join('\n')),
+ path: promptPath,
}, undefined);
}
}
@@ -144,7 +151,7 @@ async function loadSource(file: string, sourceCache: Map<string, string>) {
let source = sourceCache.get(file);
if (!source) {
// A mild race is Ok here.
- source = await fs.promises.readFile(file, 'utf8');
+ source = await fs.readFile(file, 'utf8');
sourceCache.set(file, source);
}
return source;
diff --git a/packages/playwright/src/reporters/base.ts b/packages/playwright/src/reporters/base.ts
index c8326ece31..58b98a9574 100644
--- a/packages/playwright/src/reporters/base.ts
+++ b/packages/playwright/src/reporters/base.ts
@@ -350,15 +350,18 @@ export function formatFailure(screen: Screen, config: FullConfig, test: TestCase
const errors = formatResultFailure(screen, test, result, ' ');
if (!errors.length)
continue;
- const retryLines = [];
if (result.retry) {
- retryLines.push('');
- retryLines.push(screen.colors.gray(separator(screen, ` Retry #${result.retry}`)));
+ resultLines.push('');
+ resultLines.push(screen.colors.gray(separator(screen, ` Retry #${result.retry}`)));
}
- resultLines.push(...retryLines);
resultLines.push(...errors.map(error => '\n' + error.message));
for (let i = 0; i < result.attachments.length; ++i) {
const attachment = result.attachments[i];
+ if (attachment.name.startsWith('_prompt') && attachment.path) {
+ resultLines.push('');
+ resultLines.push(screen.colors.dim(` Error Prompt: ${relativeFilePath(screen, config, attachment.path)}`));
+ continue;
+ }
if (attachment.name.startsWith('_'))
continue;
const hasPrintableContent = attachment.contentType.startsWith('text/');
diff --git a/tests/playwright-test/playwright.artifacts.spec.ts b/tests/playwright-test/playwright.artifacts.spec.ts
index 0460be88b4..5f19139e46 100644
--- a/tests/playwright-test/playwright.artifacts.spec.ts
+++ b/tests/playwright-test/playwright.artifacts.spec.ts
@@ -130,7 +130,7 @@ test('should work with screenshot: on', async ({ runInlineTest }, testInfo) => {
'playwright.config.ts': `
module.exports = { use: { screenshot: 'on' } };
`,
- }, { workers: 1 });
+ }, { workers: 1 }, { PLAYWRIGHT_NO_COPY_PROMPT: 'true' });
expect(result.exitCode).toBe(1);
expect(result.passed).toBe(5);
@@ -168,7 +168,7 @@ test('should work with screenshot: only-on-failure', async ({ runInlineTest }, t
'playwright.config.ts': `
module.exports = { use: { screenshot: 'only-on-failure' } };
`,
- }, { workers: 1 });
+ }, { workers: 1 }, { PLAYWRIGHT_NO_COPY_PROMPT: 'true' });
expect(result.exitCode).toBe(1);
expect(result.passed).toBe(5);
@@ -204,7 +204,7 @@ test('should work with screenshot: on-first-failure', async ({ runInlineTest },
use: { screenshot: 'on-first-failure' }
};
`,
- }, { workers: 1 });
+ }, { workers: 1 }, { PLAYWRIGHT_NO_COPY_PROMPT: 'true' });
expect(result.exitCode).toBe(1);
expect(result.passed).toBe(0);
@@ -230,7 +230,7 @@ test('should work with screenshot: only-on-failure & fullPage', async ({ runInli
'playwright.config.ts': `
module.exports = { use: { screenshot: { mode: 'only-on-failure', fullPage: true } } };
`,
- }, { workers: 1 });
+ }, { workers: 1 }, { PLAYWRIGHT_NO_COPY_PROMPT: 'true' });
expect(result.exitCode).toBe(1);
expect(result.passed).toBe(0);
expect(result.failed).toBe(1);
@@ -263,7 +263,7 @@ test('should capture a single screenshot on failure when afterAll fails', async
await page.setContent('this is test');
});
`,
- }, { workers: 1 });
+ }, { workers: 1 }, { PLAYWRIGHT_NO_COPY_PROMPT: 'true' });
expect(result.exitCode).toBe(1);
expect(result.passed).toBe(0);
@@ -282,7 +282,7 @@ test('should work with trace: on', async ({ runInlineTest }, testInfo) => {
'playwright.config.ts': `
module.exports = { use: { trace: 'on' } };
`,
- }, { workers: 1 });
+ }, { workers: 1 }, { PLAYWRIGHT_NO_COPY_PROMPT: 'true' });
expect(result.exitCode).toBe(1);
expect(result.passed).toBe(5);
@@ -318,7 +318,7 @@ test('should work with trace: retain-on-failure', async ({ runInlineTest }, test
'playwright.config.ts': `
module.exports = { use: { trace: 'retain-on-failure' } };
`,
- }, { workers: 1 });
+ }, { workers: 1 }, { PLAYWRIGHT_NO_COPY_PROMPT: 'true' });
expect(result.exitCode).toBe(1);
expect(result.passed).toBe(5);
@@ -344,7 +344,7 @@ test('should work with trace: on-first-retry', async ({ runInlineTest }, testInf
'playwright.config.ts': `
module.exports = { use: { trace: 'on-first-retry' } };
`,
- }, { workers: 1, retries: 1 });
+ }, { workers: 1, retries: 1 }, { PLAYWRIGHT_NO_COPY_PROMPT: 'true' });
expect(result.exitCode).toBe(1);
expect(result.passed).toBe(5);
@@ -370,7 +370,7 @@ test('should work with trace: on-all-retries', async ({ runInlineTest }, testInf
'playwright.config.ts': `
module.exports = { use: { trace: 'on-all-retries' } };
`,
- }, { workers: 1, retries: 2 });
+ }, { workers: 1, retries: 2 }, { PLAYWRIGHT_NO_COPY_PROMPT: 'true' });
expect(result.exitCode).toBe(1);
expect(result.passed).toBe(5);
@@ -406,7 +406,7 @@ test('should work with trace: retain-on-first-failure', async ({ runInlineTest }
'playwright.config.ts': `
module.exports = { use: { trace: 'retain-on-first-failure' } };
`,
- }, { workers: 1, retries: 2 });
+ }, { workers: 1, retries: 2 }, { PLAYWRIGHT_NO_COPY_PROMPT: 'true' });
expect(result.exitCode).toBe(1);
expect(result.passed).toBe(5);
@@ -442,7 +442,7 @@ test('should take screenshot when page is closed in afterEach', async ({ runInli
expect(1).toBe(2);
});
`,
- }, { workers: 1 });
+ }, { workers: 1 }, { PLAYWRIGHT_NO_COPY_PROMPT: 'true' });
expect(result.exitCode).toBe(1);
expect(result.failed).toBe(1);
diff --git a/tests/playwright-test/playwright.spec.ts b/tests/playwright-test/playwright.spec.ts
index 5e85619b65..f75aba94a8 100644
--- a/tests/playwright-test/playwright.spec.ts
+++ b/tests/playwright-test/playwright.spec.ts
@@ -510,7 +510,7 @@ test('should work with video: on-first-retry', async ({ runInlineTest }) => {
expect(fs.existsSync(dirPass)).toBeFalsy();
const dirFail = test.info().outputPath('test-results', 'a-fail-chromium');
- expect(fs.existsSync(dirFail)).toBeFalsy();
+ expect(fs.readdirSync(dirFail)).toEqual(['prompt.md']);
const dirRetry = test.info().outputPath('test-results', 'a-fail-chromium-retry1');
const videoFailRetry = fs.readdirSync(dirRetry).find(file => file.endsWith('webm'));
diff --git a/tests/playwright-test/reporter-line.spec.ts b/tests/playwright-test/reporter-line.spec.ts
index 9c69fd9cdc..777e771210 100644
--- a/tests/playwright-test/reporter-line.spec.ts
+++ b/tests/playwright-test/reporter-line.spec.ts
@@ -14,6 +14,7 @@
* limitations under the License.
*/
+import path from 'path';
import { test, expect } from './playwright-test-fixtures';
for (const useIntermediateMergeReport of [false, true] as const) {
@@ -188,5 +189,22 @@ for (const useIntermediateMergeReport of [false, true] as const) {
expect(text).toContain('1) a.test.ts:3:15 › passes ──');
expect(result.exitCode).toBe(1);
});
+
+ test('should show error prompt with relative path', async ({ runInlineTest, useIntermediateMergeReport }) => {
+ const result = await runInlineTest({
+ 'a.test.js': `
+ const { test, expect } = require('@playwright/test');
+ test('one', async ({}) => {
+ expect(1).toBe(0);
+ });
+ `,
+ }, { reporter: 'line' });
+ const text = result.output;
+ if (useIntermediateMergeReport)
+ expect(text).toContain(`Error Prompt: ${path.join('blob-report', 'resources')}`);
+ else
+ expect(text).toContain(`Error Prompt: ${path.join('test-results', 'a-one', 'prompt.md')}`);
+ expect(result.exitCode).toBe(1);
+ });
});
}