Skip to content

Playwright is a framework for Web Testing and Automation. It allows testing Chromium, Firefox and WebKit with a single API.

feat(html): "copy prompt" button (#34670)

Simon Knott2f8d448dbbcc
diff --git a/packages/html-reporter/src/metadataView.tsx b/packages/html-reporter/src/metadataView.tsx
index 8cc66571ab..ed050f4480 100644
--- a/packages/html-reporter/src/metadataView.tsx
+++ b/packages/html-reporter/src/metadataView.tsx
@@ -26,9 +26,25 @@ import { linkifyText } from '@web/renderUtils';
type MetadataEntries = [string, unknown][];
-export function filterMetadata(metadata: Metadata): MetadataEntries {
- // TODO: do not plumb actualWorkers through metadata.
- return Object.entries(metadata).filter(([key]) => key !== 'actualWorkers');
+export const MetadataContext = React.createContext<MetadataEntries>([]);
+
+export function MetadataProvider({ metadata, children }: React.PropsWithChildren<{ metadata: Metadata }>) {
+ const entries = React.useMemo(() => {
+ // TODO: do not plumb actualWorkers through metadata.
+
+ return Object.entries(metadata).filter(([key]) => key !== 'actualWorkers');
+ }, [metadata]);
+
+ return <MetadataContext.Provider value={entries}>{children}</MetadataContext.Provider>;
+}
+
+export function useMetadata() {
+ return React.useContext(MetadataContext);
+}
+
+export function useGitCommitInfo() {
+ const metadataEntries = useMetadata();
+ return metadataEntries.find(([key]) => key === 'git.commit.info')?.[1] as GitCommitInfo | undefined;
}
class ErrorBoundary extends React.Component<React.PropsWithChildren<{}>, { error: Error | null, errorInfo: React.ErrorInfo | null }> {
@@ -57,12 +73,13 @@ class ErrorBoundary extends React.Component<React.PropsWithChildren<{}>, { error
}
}
-export const MetadataView: React.FC<{ metadataEntries: MetadataEntries }> = ({ metadataEntries }) => {
- return <ErrorBoundary><InnerMetadataView metadataEntries={metadataEntries}/></ErrorBoundary>;
+export const MetadataView = () => {
+ return <ErrorBoundary><InnerMetadataView/></ErrorBoundary>;
};
-const InnerMetadataView: React.FC<{ metadataEntries: MetadataEntries }> = ({ metadataEntries }) => {
- const gitCommitInfo = metadataEntries.find(([key]) => key === 'git.commit.info')?.[1] as GitCommitInfo | undefined;
+const InnerMetadataView = () => {
+ const metadataEntries = useMetadata();
+ const gitCommitInfo = useGitCommitInfo();
const entries = metadataEntries.filter(([key]) => key !== 'git.commit.info');
if (!gitCommitInfo && !entries.length)
return null;
diff --git a/packages/html-reporter/src/reportView.tsx b/packages/html-reporter/src/reportView.tsx
index e48064201c..baf85f32a8 100644
--- a/packages/html-reporter/src/reportView.tsx
+++ b/packages/html-reporter/src/reportView.tsx
@@ -26,6 +26,7 @@ import './reportView.css';
import { TestCaseView } from './testCaseView';
import { TestFilesHeader, TestFilesView } from './testFilesView';
import './theme.css';
+import { MetadataProvider } from './metadataView';
declare global {
interface Window {
@@ -72,7 +73,7 @@ export const ReportView: React.FC<{
return result;
}, [report, filter]);
- return <div className='htmlreport vbox px-4 pb-4'>
+ return <MetadataProvider metadata={report?.json().metadata ?? {}}><div className='htmlreport vbox px-4 pb-4'>
<main>
{report?.json() && <HeaderView stats={report.json().stats} filterText={filterText} setFilterText={setFilterText}></HeaderView>}
<Route predicate={testFilesRoutePredicate}>
@@ -88,7 +89,7 @@ export const ReportView: React.FC<{
{!!report && <TestCaseViewLoader report={report} tests={filteredTests.tests} testIdToFileIdMap={testIdToFileIdMap} />}
</Route>
</main>
- </div>;
+ </div></MetadataProvider>;
};
const TestCaseViewLoader: React.FC<{
diff --git a/packages/html-reporter/src/testErrorView.css b/packages/html-reporter/src/testErrorView.css
index e29ea2a18b..7cc9e88a79 100644
--- a/packages/html-reporter/src/testErrorView.css
+++ b/packages/html-reporter/src/testErrorView.css
@@ -16,18 +16,47 @@
@import '@web/third_party/vscode/colors.css';
-.test-error-view {
+.test-error-container {
white-space: pre;
overflow: auto;
flex: none;
padding: 0;
background-color: var(--color-canvas-subtle);
border-radius: 6px;
- padding: 16px;
line-height: initial;
margin-bottom: 6px;
}
+.test-error-view {
+ padding: 16px;
+}
+
.test-error-text {
font-family: monospace;
}
+
+.prompt-button {
+ flex: none;
+ height: 24px;
+ width: 80px;
+ border: 1px solid var(--color-btn-border);
+ outline: none;
+ color: var(--color-btn-text);
+ background: var(--color-btn-bg);
+ padding: 4px;
+ cursor: pointer;
+ display: inline-flex;
+ align-items: center;
+ justify-content: center;
+ border-radius: 4px;
+}
+
+.prompt-button svg {
+ color: var(--color-fg-subtle);
+}
+
+.prompt-button:not(:disabled):hover {
+ border-color: var(--color-btn-hover-border);
+ background-color: var(--color-btn-hover-bg);
+}
+
diff --git a/packages/html-reporter/src/testErrorView.tsx b/packages/html-reporter/src/testErrorView.tsx
index ea402106a8..11f8e4b474 100644
--- a/packages/html-reporter/src/testErrorView.tsx
+++ b/packages/html-reporter/src/testErrorView.tsx
@@ -17,15 +17,57 @@
import { ansi2html } from '@web/ansi2html';
import * as React from 'react';
import './testErrorView.css';
+import * as icons from './icons';
import type { ImageDiff } from '@web/shared/imageDiffView';
import { ImageDiffView } from '@web/shared/imageDiffView';
+import type { TestResult } from './types';
+import { fixTestPrompt } from '@web/components/prompts';
+import { useGitCommitInfo } from './metadataView';
-export const TestErrorView: React.FC<{
+export const TestErrorView: React.FC<{ error: string; testId?: string; result?: TestResult }> = ({ error, testId, result }) => {
+ return (
+ <CodeSnippet code={error} testId={testId}>
+ <div style={{ float: 'right', padding: '5px' }}>
+ <PromptButton error={error} result={result} />
+ </div>
+ </CodeSnippet>
+ );
+};
+
+export const CodeSnippet = ({ code, children, testId }: React.PropsWithChildren<{ code: string; testId?: string; }>) => {
+ const html = React.useMemo(() => ansiErrorToHtml(code), [code]);
+ return (
+ <div className='test-error-container test-error-text' data-testid={testId}>
+ {children}
+ <div className='test-error-view' dangerouslySetInnerHTML={{ __html: html || '' }}></div>
+ </div>
+ );
+};
+
+const PromptButton: React.FC<{
error: string;
- testId?: string;
-}> = ({ error, testId }) => {
- const html = React.useMemo(() => ansiErrorToHtml(error), [error]);
- return <div className='test-error-view test-error-text' data-testid={testId} dangerouslySetInnerHTML={{ __html: html || '' }}></div>;
+ result?: TestResult;
+}> = ({ error, result }) => {
+ const gitCommitInfo = useGitCommitInfo();
+ const prompt = React.useMemo(() => fixTestPrompt(
+ error,
+ gitCommitInfo?.['pull.diff'] ?? gitCommitInfo?.['revision.diff'],
+ result?.attachments.find(a => a.name === 'pageSnapshot')?.body
+ ), [gitCommitInfo, result, error]);
+
+ const [copied, setCopied] = React.useState(false);
+
+ return <button
+ className='prompt-button'
+ onClick={async () => {
+ await navigator.clipboard.writeText(prompt);
+ setCopied(true);
+ setTimeout(() => {
+ setCopied(false);
+ }, 3000);
+ }}>
+ {copied ? <span className='prompt-button-copied'>Copied <icons.copy/></span> : 'Fix with AI'}
+ </button>;
};
export const TestScreenshotErrorView: React.FC<{
diff --git a/packages/html-reporter/src/testFilesView.tsx b/packages/html-reporter/src/testFilesView.tsx
index bcb0696946..dcac5e8cea 100644
--- a/packages/html-reporter/src/testFilesView.tsx
+++ b/packages/html-reporter/src/testFilesView.tsx
@@ -22,7 +22,7 @@ import { msToString } from './utils';
import { AutoChip } from './chip';
import { TestErrorView } from './testErrorView';
import * as icons from './icons';
-import { filterMetadata, MetadataView } from './metadataView';
+import { MetadataView, useMetadata } from './metadataView';
export const TestFilesView: React.FC<{
tests: TestFileSummary[],
@@ -67,9 +67,9 @@ export const TestFilesHeader: React.FC<{
metadataVisible: boolean,
toggleMetadataVisible: () => void,
}> = ({ report, filteredStats, metadataVisible, toggleMetadataVisible }) => {
+ const metadataEntries = useMetadata();
if (!report)
return;
- const metadataEntries = filterMetadata(report.metadata || {});
return <>
<div className='mx-1' style={{ display: 'flex', marginTop: 10 }}>
{metadataEntries.length > 0 && <div className='metadata-toggle' role='button' onClick={toggleMetadataVisible} title={metadataVisible ? 'Hide metadata' : 'Show metadata'}>
@@ -81,7 +81,7 @@ export const TestFilesHeader: React.FC<{
<div data-testid='overall-time' style={{ color: 'var(--color-fg-subtle)', marginRight: '10px' }}>{report ? new Date(report.startTime).toLocaleString() : ''}</div>
<div data-testid='overall-duration' style={{ color: 'var(--color-fg-subtle)' }}>Total time: {msToString(report.duration ?? 0)}</div>
</div>
- {metadataVisible && <MetadataView metadataEntries={metadataEntries}/>}
+ {metadataVisible && <MetadataView/>}
{!!report.errors.length && <AutoChip header='Errors' dataTestId='report-errors'>
{report.errors.map((error, index) => <TestErrorView key={'test-report-error-message-' + index} error={error}></TestErrorView>)}
</AutoChip>}
diff --git a/packages/html-reporter/src/testResultView.tsx b/packages/html-reporter/src/testResultView.tsx
index 681f4b507a..3243b2bcb9 100644
--- a/packages/html-reporter/src/testResultView.tsx
+++ b/packages/html-reporter/src/testResultView.tsx
@@ -24,7 +24,7 @@ import { Anchor, AttachmentLink, generateTraceUrl, testResultHref } from './link
import { statusIcon } from './statusIcon';
import type { ImageDiff } from '@web/shared/imageDiffView';
import { ImageDiffView } from '@web/shared/imageDiffView';
-import { TestErrorView, TestScreenshotErrorView } from './testErrorView';
+import { CodeSnippet, TestErrorView, TestScreenshotErrorView } from './testErrorView';
import * as icons from './icons';
import './testResultView.css';
@@ -90,7 +90,7 @@ export const TestResultView: React.FC<{
{errors.map((error, index) => {
if (error.type === 'screenshot')
return <TestScreenshotErrorView key={'test-result-error-message-' + index} errorPrefix={error.errorPrefix} diff={error.diff!} errorSuffix={error.errorSuffix}></TestScreenshotErrorView>;
- return <TestErrorView key={'test-result-error-message-' + index} error={error.error!}></TestErrorView>;
+ return <TestErrorView key={'test-result-error-message-' + index} error={error.error!} result={result}></TestErrorView>;
})}
</AutoChip>}
{!!result.steps.length && <AutoChip header='Test Steps'>
@@ -182,7 +182,7 @@ const StepTreeItem: React.FC<{
{step.count > 1 && <> ✕ <span className='test-result-counter'>{step.count}</span></>}
{step.location && <span className='test-result-path'>— {step.location.file}:{step.location.line}</span>}
</span>} loadChildren={step.steps.length || step.snippet ? () => {
- const snippet = step.snippet ? [<TestErrorView testId='test-snippet' key='line' error={step.snippet}/>] : [];
+ const snippet = step.snippet ? [<CodeSnippet testId='test-snippet' key='line' code={step.snippet} />] : [];
const steps = step.steps.map((s, i) => <StepTreeItem key={i} step={s} depth={depth + 1} result={result} test={test} />);
return snippet.concat(steps);
} : undefined} depth={depth}/>;
diff --git a/packages/playwright/src/reporters/html.ts b/packages/playwright/src/reporters/html.ts
index 557ad580b7..2600b51797 100644
--- a/packages/playwright/src/reporters/html.ts
+++ b/packages/playwright/src/reporters/html.ts
@@ -449,6 +449,17 @@ class HtmlBuilder {
return a;
}
+ if (a.name === 'pageSnapshot') {
+ try {
+ const body = fs.readFileSync(a.path!, { encoding: 'utf-8' });
+ return {
+ name: 'pageSnapshot',
+ contentType: a.contentType,
+ body,
+ };
+ } catch {}
+ }
+
if (a.path) {
let fileName = a.path;
try {
diff --git a/packages/web/src/components/prompts.ts b/packages/web/src/components/prompts.ts
new file mode 100644
index 0000000000..88a086815e
--- /dev/null
+++ b/packages/web/src/components/prompts.ts
@@ -0,0 +1,48 @@
+/**
+ * Copyright (c) Microsoft Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+const ansiRegex = new RegExp('([\\u001B\\u009B][[\\]()#;?]*(?:(?:(?:[a-zA-Z\\d]*(?:;[-a-zA-Z\\d\\/#&.:=?%@~_]*)*)?\\u0007)|(?:(?:\\d{1,4}(?:;\\d{0,4})*)?[\\dA-PR-TZcf-ntqry=><~])))', 'g');
+function stripAnsiEscapes(str: string): string {
+ return str.replace(ansiRegex, '');
+}
+
+export function fixTestPrompt(error: string, diff?: string, pageSnapshot?: string) {
+ const promptParts = [
+ 'This test failed, suggest how to fix it. Please be correct, concise and keep Playwright best practices in mind.',
+ 'Here is the error:',
+ '\n',
+ stripAnsiEscapes(error),
+ '\n',
+ ];
+
+ if (pageSnapshot) {
+ promptParts.push(
+ 'This is how the page looked at the end of the test:',
+ pageSnapshot,
+ '\n'
+ );
+ }
+
+ if (diff) {
+ promptParts.push(
+ 'And this is the code diff:',
+ diff,
+ '\n'
+ );
+ }
+
+ return promptParts.join('\n');
+}
diff --git a/tests/playwright-test/reporter-html.spec.ts b/tests/playwright-test/reporter-html.spec.ts
index d5cc7ed50f..acb54f3b31 100644
--- a/tests/playwright-test/reporter-html.spec.ts
+++ b/tests/playwright-test/reporter-html.spec.ts
@@ -2694,6 +2694,59 @@ for (const useIntermediateMergeReport of [true, false] as const) {
await page.getByText('my test').click();
await expect(page.locator('.tree-item', { hasText: 'stdout' })).toHaveCount(1);
});
+
+ test('should show AI prompt', async ({ runInlineTest, writeFiles, showReport, page }) => {
+ const files = {
+ 'uncommitted.txt': `uncommitted file`,
+ 'playwright.config.ts': `
+ export default {
+ populateGitInfo: true,
+ metadata: { foo: 'value1', bar: { prop: 'value2' }, baz: ['value3', 123] }
+ };
+ `,
+ 'example.spec.ts': `
+ import { test, expect } from '@playwright/test';
+ test('sample', async ({}) => { expect(2).toBe(3); });
+ `,
+ };
+ const baseDir = await writeFiles(files);
+
+ const execGit = async (args: string[]) => {
+ const { code, stdout, stderr } = await spawnAsync('git', args, { stdio: 'pipe', cwd: baseDir });
+ if (!!code)
+ throw new Error(`Non-zero exit of:\n$ git ${args.join(' ')}\nConsole:\nstdout:\n${stdout}\n\nstderr:\n${stderr}\n\n`);
+ return;
+ };
+
+ await execGit(['init']);
+ await execGit(['config', '--local', 'user.email', '[email protected]']);
+ await execGit(['config', '--local', 'user.name', 'William']);
+ await execGit(['add', 'playwright.config.ts']);
+ await execGit(['commit', '-m', 'init']);
+ await execGit(['add', '*.ts']);
+ await execGit(['commit', '-m', 'chore(html): make this test look nice']);
+
+ const result = await runInlineTest(files, { reporter: 'dot,html' }, {
+ PLAYWRIGHT_HTML_OPEN: 'never',
+ GITHUB_REPOSITORY: 'microsoft/playwright-example-for-test',
+ GITHUB_RUN_ID: 'example-run-id',
+ GITHUB_SERVER_URL: 'https://playwright.dev',
+ GITHUB_SHA: 'example-sha',
+ GITHUB_REF_NAME: '42/merge',
+ GITHUB_BASE_REF: 'HEAD~1',
+ });
+
+ expect(result.exitCode).toBe(1);
+ await showReport();
+
+ await page.context().grantPermissions(['clipboard-read', 'clipboard-write']);
+
+ await page.getByRole('link', { name: 'sample' }).click();
+ await page.getByRole('button', { name: 'Fix with AI' }).click();
+ const prompt = await page.evaluate(() => navigator.clipboard.readText());
+ expect(prompt, 'contains error').toContain('expect(received).toBe(expected)');
+ expect(prompt, 'contains diff').toContain(`+ test('sample', async ({}) => { expect(2).toBe(3); });`);
+ });
});
}