Playwright is a framework for Web Testing and Automation. It allows testing Chromium, Firefox and WebKit with a single API.
TypeScript automationchromechromiume2e-testingelectronend-to-end-testingfirefoxjavascriptplaywrighttesttest-automationtestingtesting-toolswebwebkit
chore: refactor error context (#35613)
Simon Knottcb2d94e4679c
diff --git a/packages/html-reporter/src/testErrorView.tsx b/packages/html-reporter/src/testErrorView.tsxindex 881c420c22..d7a979a9d9 100644--- a/packages/html-reporter/src/testErrorView.tsx+++ b/packages/html-reporter/src/testErrorView.tsx@@ -20,17 +20,18 @@ import './testErrorView.css';import type { ImageDiff } from '@web/shared/imageDiffView';import { ImageDiffView } from '@web/shared/imageDiffView';import { TestAttachment } from './types';+import { fixTestInstructions } from '@web/prompts';export const TestErrorView: React.FC<{error: string;testId?: string;- prompt?: TestAttachment;-}> = ({ error, testId, prompt }) => {+ context?: TestAttachment;+}> = ({ error, testId, context }) => {return (<CodeSnippet code={error} testId={testId}>- {prompt && (+ {context && (<div style={{ position: 'absolute', right: 0, padding: '10px' }}>- <PromptButton prompt={prompt} />+ <PromptButton context={context} /></div>)}</CodeSnippet>@@ -47,14 +48,14 @@ export const CodeSnippet = ({ code, children, testId }: React.PropsWithChildren<);};-const PromptButton: React.FC<{ prompt: TestAttachment }> = ({ prompt }) => {+const PromptButton: React.FC<{ context: TestAttachment }> = ({ context }) => {const [copied, setCopied] = React.useState(false);return <buttonclassName='button'style={{ minWidth: 100 }}onClick={async () => {- const text = prompt.body ? prompt.body : await fetch(prompt.path!).then(r => r.text());- await navigator.clipboard.writeText(text);+ const text = context.body ? context.body : await fetch(context.path!).then(r => r.text());+ await navigator.clipboard.writeText(fixTestInstructions + text);setCopied(true);setTimeout(() => {setCopied(false);diff --git a/packages/html-reporter/src/testResultView.tsx b/packages/html-reporter/src/testResultView.tsxindex 6a3175471a..34b4312be5 100644--- a/packages/html-reporter/src/testResultView.tsx+++ b/packages/html-reporter/src/testResultView.tsx@@ -90,7 +90,7 @@ export const TestResultView: React.FC<{{errors.map((error, index) => {if (error.type === 'screenshot')return <TestScreenshotErrorView key={'test-result-error-message-' + index} errorPrefix={error.errorPrefix} diff={error.diff!} errorSuffix={error.errorSuffix}></TestScreenshotErrorView>;- return <TestErrorView key={'test-result-error-message-' + index} error={error.error!} prompt={error.prompt}></TestErrorView>;+ return <TestErrorView key={'test-result-error-message-' + index} error={error.error!} context={error.context}></TestErrorView>;})}</AutoChip>}{!!result.steps.length && <AutoChip header='Test Steps'>@@ -165,8 +165,8 @@ function classifyErrors(testErrors: string[], diffs: ImageDiff[], attachments: T}}- const prompt = attachments.find(a => a.name === `_prompt-${i}`);- return { type: 'regular', error, prompt };+ const context = attachments.find(a => a.name === `_error-context-${i}`);+ return { type: 'regular', error, context };});}diff --git a/packages/playwright/src/DEPS.list b/packages/playwright/src/DEPS.listindex 25ca70ff18..e9b4b2f493 100644--- a/packages/playwright/src/DEPS.list+++ b/packages/playwright/src/DEPS.list@@ -14,5 +14,5 @@ common/[internalsForTest.ts]**-[prompt.ts]+[errorContext.ts]./transform/babelBundle.tsdiff --git a/packages/playwright/src/prompt.ts b/packages/playwright/src/errorContext.tssimilarity index 75%rename from packages/playwright/src/prompt.tsrename to packages/playwright/src/errorContext.tsindex 9c323f59b9..651258c77b 100644--- a/packages/playwright/src/prompt.ts+++ b/packages/playwright/src/errorContext.ts@@ -22,13 +22,24 @@ import { parseErrorStack } from 'playwright-core/lib/utils';import { stripAnsiEscapes } from './util';import { codeFrameColumns } from './transform/babelBundle';-import type { TestInfo } from '../types/test';import type { MetadataWithCommitInfo } from './isomorphic/types';import type { TestInfoImpl } from './worker/testInfo';-export async function attachErrorPrompts(testInfo: TestInfo, sourceCache: Map<string, string>, ariaSnapshot: string | undefined) {- if (process.env.PLAYWRIGHT_NO_COPY_PROMPT)+export async function attachErrorContext(testInfo: TestInfoImpl, format: 'markdown' | 'json', sourceCache: Map<string, string>, ariaSnapshot: string | undefined) {+ if (format === 'json') {+ if (!ariaSnapshot)+ return;++ testInfo._attach({+ name: `_error-context`,+ contentType: 'application/json',+ body: Buffer.from(JSON.stringify({+ pageSnapshot: ariaSnapshot,+ })),+ }, undefined);+return;+ }const meaningfulSingleLineErrors = new Set(testInfo.errors.filter(e => e.message && !e.message.includes('\n')).map(e => e.message!));for (const error of testInfo.errors) {@@ -51,16 +62,10 @@ export async function attachErrorPrompts(testInfo: TestInfo, sourceCache: Map<stfor (const [index, error] of errors) {const metadata = testInfo.config.metadata as MetadataWithCommitInfo;- if (testInfo.attachments.find(a => a.name === `_prompt-${index}`))+ if (testInfo.attachments.find(a => a.name === `_error-context-${index}`))continue;- const promptParts = [- `# Instructions`,- '',- `- Following Playwright test failed.`,- `- Explain why, be concise, respect Playwright best practices.`,- `- Provide a snippet of code with the fix, if possible.`,- '',+ const lines = [`# Test info`,'',`- Name: ${testInfo.titlePath.slice(1).join(' >> ')}`,@@ -74,7 +79,7 @@ export async function attachErrorPrompts(testInfo: TestInfo, sourceCache: Map<st];if (ariaSnapshot) {- promptParts.push(+ lines.push('','# Page snapshot','',@@ -103,7 +108,7 @@ export async function attachErrorPrompts(testInfo: TestInfo, sourceCache: Map<stmessage: inlineMessage || undefined,});- promptParts.push(+ lines.push('','# Test source','',@@ -113,7 +118,7 @@ export async function attachErrorPrompts(testInfo: TestInfo, sourceCache: Map<st);if (metadata.gitDiff) {- promptParts.push(+ lines.push('','# Local changes','',@@ -123,30 +128,17 @@ export async function attachErrorPrompts(testInfo: TestInfo, sourceCache: Map<st);}- const promptPath = testInfo.outputPath(errors.length === 1 ? `prompt.md` : `prompt-${index}.md`);- await fs.writeFile(promptPath, promptParts.join('\n'), 'utf8');+ const filePath = testInfo.outputPath(errors.length === 1 ? `error-context.md` : `error-context-${index}.md`);+ await fs.writeFile(filePath, lines.join('\n'), 'utf8');(testInfo as TestInfoImpl)._attach({- name: `_prompt-${index}`,+ name: `_error-context-${index}`,contentType: 'text/markdown',- path: promptPath,+ path: filePath,}, undefined);}}-export async function attachErrorContext(testInfo: TestInfo, ariaSnapshot: string | undefined) {- if (!ariaSnapshot)- return;-- (testInfo as TestInfoImpl)._attach({- name: `_error-context`,- contentType: 'application/json',- body: Buffer.from(JSON.stringify({- pageSnapshot: ariaSnapshot,- })),- }, undefined);-}-async function loadSource(file: string, sourceCache: Map<string, string>) {let source = sourceCache.get(file);if (!source) {diff --git a/packages/playwright/src/index.ts b/packages/playwright/src/index.tsindex 0a7d121146..8370382bb3 100644--- a/packages/playwright/src/index.ts+++ b/packages/playwright/src/index.ts@@ -22,7 +22,7 @@ import { setBoxedStackPrefixes, asLocator, createGuid, currentZone, debugMode, iimport { currentTestInfo } from './common/globals';import { rootTestType } from './common/testType';-import { attachErrorContext, attachErrorPrompts } from './prompt';+import { attachErrorContext } from './errorContext';import type { Fixtures, PlaywrightTestArgs, PlaywrightTestOptions, PlaywrightWorkerArgs, PlaywrightWorkerOptions, ScreenshotMode, TestInfo, TestType, VideoMode } from '../types/test';import type { ContextReuseMode } from './common/config';@@ -55,13 +55,15 @@ type TestFixtures = PlaywrightTestArgs & PlaywrightTestOptions & {_contextFactory: (options?: BrowserContextOptions) => Promise<BrowserContext>;};+type ErrorContextOption = { format: 'json' | 'markdown' } | undefined;+type WorkerFixtures = PlaywrightWorkerArgs & PlaywrightWorkerOptions & {playwright: PlaywrightImpl;_browserOptions: LaunchOptions;_optionContextReuseMode: ContextReuseMode,_optionConnectOptions: PlaywrightWorkerOptions['connectOptions'],_reuseContext: boolean,- _optionAttachErrorContext: boolean,+ _optionErrorContext: ErrorContextOption,};const playwrightFixtures: Fixtures<TestFixtures, WorkerFixtures> = ({@@ -245,13 +247,13 @@ const playwrightFixtures: Fixtures<TestFixtures, WorkerFixtures> = ({playwright._defaultContextNavigationTimeout = undefined;}, { auto: 'all-hooks-included', title: 'context configuration', box: true } as any],- _setupArtifacts: [async ({ playwright, screenshot, _optionAttachErrorContext }, use, testInfo) => {+ _setupArtifacts: [async ({ playwright, screenshot, _optionErrorContext }, use, testInfo) => {// This fixture has a separate zero-timeout slot to ensure that artifact collection// happens even after some fixtures or hooks time out.// Now that default test timeout is known, we can replace zero with an actual value.testInfo.setTimeout(testInfo.project.timeout);- const artifactsRecorder = new ArtifactsRecorder(playwright, tracing().artifactsDir(), screenshot, _optionAttachErrorContext);+ const artifactsRecorder = new ArtifactsRecorder(playwright, tracing().artifactsDir(), screenshot, _optionErrorContext);await artifactsRecorder.willStartTest(testInfo as TestInfoImpl);const tracingGroupSteps: TestStepInternal[] = [];@@ -393,7 +395,7 @@ const playwrightFixtures: Fixtures<TestFixtures, WorkerFixtures> = ({_optionContextReuseMode: ['none', { scope: 'worker', option: true }],_optionConnectOptions: [undefined, { scope: 'worker', option: true }],- _optionAttachErrorContext: [false, { scope: 'worker', option: true }],+ _optionErrorContext: [process.env.PLAYWRIGHT_NO_COPY_PROMPT ? undefined : { format: 'markdown' }, { scope: 'worker', option: true }],_reuseContext: [async ({ video, _optionContextReuseMode }, use) => {let mode = _optionContextReuseMode;@@ -622,12 +624,12 @@ class ArtifactsRecorder {private _screenshotRecorder: SnapshotRecorder;private _pageSnapshot: string | undefined;private _sourceCache: Map<string, string> = new Map();- private _attachErrorContext: boolean;+ private _errorContext: ErrorContextOption;- constructor(playwright: PlaywrightImpl, artifactsDir: string, screenshot: ScreenshotOption, attachErrorContext: boolean) {+ constructor(playwright: PlaywrightImpl, artifactsDir: string, screenshot: ScreenshotOption, errorContext: ErrorContextOption) {this._playwright = playwright;this._artifactsDir = artifactsDir;- this._attachErrorContext = attachErrorContext;+ this._errorContext = errorContext;const screenshotOptions = typeof screenshot === 'string' ? undefined : screenshot;this._startedCollectingArtifacts = Symbol('startedCollectingArtifacts');@@ -671,7 +673,7 @@ class ArtifactsRecorder {}private async _takePageSnapshot(context: BrowserContext) {- if (process.env.PLAYWRIGHT_NO_COPY_PROMPT)+ if (!this._errorContext)return;if (this._testInfo.errors.length === 0)return;@@ -719,10 +721,8 @@ class ArtifactsRecorder {if (context)await this._takePageSnapshot(context);- if (this._attachErrorContext)- await attachErrorContext(this._testInfo, this._pageSnapshot);- else- await attachErrorPrompts(this._testInfo, this._sourceCache, this._pageSnapshot);+ if (this._errorContext)+ await attachErrorContext(this._testInfo, this._errorContext.format, this._sourceCache, this._pageSnapshot);}private async _startTraceChunkOnContextCreation(tracing: Tracing) {diff --git a/packages/playwright/src/isomorphic/testServerInterface.ts b/packages/playwright/src/isomorphic/testServerInterface.tsindex e327b8b11a..5849168542 100644--- a/packages/playwright/src/isomorphic/testServerInterface.ts+++ b/packages/playwright/src/isomorphic/testServerInterface.ts@@ -102,7 +102,7 @@ export interface TestServerInterface {projects?: string[];reuseContext?: boolean;connectWsEndpoint?: string;- attachErrorContext?: boolean;+ errorContext?: { format: 'json' | 'markdown' };}): Promise<{status: reporterTypes.FullResult['status'];}>;diff --git a/packages/playwright/src/reporters/base.ts b/packages/playwright/src/reporters/base.tsindex 75807c1b67..d7f6a2a825 100644--- a/packages/playwright/src/reporters/base.ts+++ b/packages/playwright/src/reporters/base.ts@@ -337,9 +337,9 @@ export function formatFailure(screen: Screen, config: FullConfig, test: TestCase// }for (let i = 0; i < result.attachments.length; ++i) {const attachment = result.attachments[i];- if (attachment.name.startsWith('_prompt') && attachment.path) {+ if (attachment.name.startsWith('_error-context') && attachment.path) {resultLines.push('');- resultLines.push(screen.colors.dim(` Error Prompt: ${relativeFilePath(screen, config, attachment.path)}`));+ resultLines.push(screen.colors.dim(` Error Context: ${relativeFilePath(screen, config, attachment.path)}`));continue;}if (attachment.name.startsWith('_'))diff --git a/packages/playwright/src/runner/testServer.ts b/packages/playwright/src/runner/testServer.tsindex 4ff1dd55ce..a6f8b0b7ca 100644--- a/packages/playwright/src/runner/testServer.ts+++ b/packages/playwright/src/runner/testServer.ts@@ -314,7 +314,7 @@ export class TestServerDispatcher implements TestServerInterface {...(params.headed !== undefined ? { headless: !params.headed } : {}),_optionContextReuseMode: params.reuseContext ? 'when-possible' : undefined,_optionConnectOptions: params.connectWsEndpoint ? { wsEndpoint: params.connectWsEndpoint } : undefined,- _optionAttachErrorContext: params.attachErrorContext,+ _optionErrorContext: params.errorContext,},...(params.updateSnapshots ? { updateSnapshots: params.updateSnapshots } : {}),...(params.updateSourceMethod ? { updateSourceMethod: params.updateSourceMethod } : {}),diff --git a/packages/trace-viewer/src/ui/errorsTab.tsx b/packages/trace-viewer/src/ui/errorsTab.tsxindex 7ade3e2dbf..a5ea191e1f 100644--- a/packages/trace-viewer/src/ui/errorsTab.tsx+++ b/packages/trace-viewer/src/ui/errorsTab.tsx@@ -26,6 +26,7 @@ import { ToolbarButton } from '@web/components/toolbarButton';import { useIsLLMAvailable, useLLMChat } from './llm';import { useAsyncMemo } from '@web/uiUtils';import { attachmentURL } from './attachmentsTab';+import { fixTestInstructions } from '@web/prompts';const CopyPromptButton: React.FC<{ prompt: string }> = ({ prompt }) => {return (@@ -67,10 +68,10 @@ function Error({ message, error, errorId, sdkLanguage, revealInSource }: { messa}const prompt = useAsyncMemo(async () => {- if (!error.prompt)+ if (!error.context)return;- const response = await fetch(attachmentURL(error.prompt));- return await response.text();+ const response = await fetch(attachmentURL(error.context));+ return fixTestInstructions + await response.text();}, [error], undefined);return <div style={{ display: 'flex', flexDirection: 'column', overflowX: 'clip' }}>diff --git a/packages/trace-viewer/src/ui/modelUtil.ts b/packages/trace-viewer/src/ui/modelUtil.tsindex c75c722419..ea8a15676d 100644--- a/packages/trace-viewer/src/ui/modelUtil.ts+++ b/packages/trace-viewer/src/ui/modelUtil.ts@@ -56,7 +56,7 @@ export type ErrorDescription = {action?: ActionTraceEventInContext;stack?: StackFrame[];message: string;- prompt?: trace.AfterActionTraceEventAttachment & { traceUrl: string };+ context?: trace.AfterActionTraceEventAttachment & { traceUrl: string };};export type Attachment = trace.AfterActionTraceEventAttachment & { traceUrl: string };@@ -141,7 +141,7 @@ export class MultiTraceModel {return this.errors.filter(e => !!e.message).map((error, i) => ({stack: error.stack,message: error.message,- prompt: this.attachments.find(a => a.name === `_prompt-${i}`),+ context: this.attachments.find(a => a.name === `_error-context-${i}`),}));}}diff --git a/packages/web/src/prompts.ts b/packages/web/src/prompts.tsnew file mode 100644index 0000000000..ef0b5bf78b--- /dev/null+++ b/packages/web/src/prompts.ts@@ -0,0 +1,23 @@+/**+ * Copyright (c) Microsoft Corporation.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++export const fixTestInstructions = `+# Instructions++- Following Playwright test failed.+- Explain why, be concise, respect Playwright best practices.+- Provide a snippet of code with the fix, if possible.+`.trimStart();diff --git a/tests/playwright-test/playwright.connect.spec.ts b/tests/playwright-test/playwright.connect.spec.tsindex e9d29dfe0e..e1644bb8ea 100644--- a/tests/playwright-test/playwright.connect.spec.ts+++ b/tests/playwright-test/playwright.connect.spec.ts@@ -167,7 +167,7 @@ test('should print debug log when failed to connect', async ({ runInlineTest })expect(result.exitCode).toBe(1);expect(result.failed).toBe(1);expect(result.output).toContain('b-debug-log-string');- expect(result.results[0].attachments).toEqual([expect.objectContaining({ name: '_prompt-0' })]);+ expect(result.results[0].attachments).toEqual([expect.objectContaining({ name: '_error-context-0' })]);});test('should record trace', async ({ runInlineTest }) => {@@ -223,7 +223,7 @@ test('should record trace', async ({ runInlineTest }) => {'After Hooks','fixture: page','fixture: context',- '_attach "_prompt-0"',+ '_attach "_error-context-0"','Worker Cleanup','fixture: browser',]);diff --git a/tests/playwright-test/playwright.spec.ts b/tests/playwright-test/playwright.spec.tsindex f75aba94a8..875133e7d4 100644--- a/tests/playwright-test/playwright.spec.ts+++ b/tests/playwright-test/playwright.spec.ts@@ -510,13 +510,13 @@ test('should work with video: on-first-retry', async ({ runInlineTest }) => {expect(fs.existsSync(dirPass)).toBeFalsy();const dirFail = test.info().outputPath('test-results', 'a-fail-chromium');- expect(fs.readdirSync(dirFail)).toEqual(['prompt.md']);+ expect(fs.readdirSync(dirFail)).toEqual(['error-context.md']);const dirRetry = test.info().outputPath('test-results', 'a-fail-chromium-retry1');const videoFailRetry = fs.readdirSync(dirRetry).find(file => file.endsWith('webm'));expect(videoFailRetry).toBeTruthy();- const errorPrompt = expect.objectContaining({ name: '_prompt-0' });+ const errorPrompt = expect.objectContaining({ name: '_error-context-0' });expect(result.report.suites[0].specs[1].tests[0].results[0].attachments).toEqual([errorPrompt]);expect(result.report.suites[0].specs[1].tests[0].results[1].attachments).toEqual([{name: 'video',diff --git a/tests/playwright-test/reporter-json.spec.ts b/tests/playwright-test/reporter-json.spec.tsindex 377e20aa68..b2cdbf44d8 100644--- a/tests/playwright-test/reporter-json.spec.ts+++ b/tests/playwright-test/reporter-json.spec.ts@@ -359,7 +359,7 @@ test('should report parallelIndex', async ({ runInlineTest }, testInfo) => {test('attaches error context', async ({ runInlineTest }) => {const result = await runInlineTest({'playwright.config.ts': `- export default { use: { _optionAttachErrorContext: true } };+ export default { use: { _optionErrorContext: { format: 'json' } } };`,'a.test.js': `const { test, expect } = require('@playwright/test');diff --git a/tests/playwright-test/reporter-line.spec.ts b/tests/playwright-test/reporter-line.spec.tsindex 777e771210..b3a84ac1dc 100644--- a/tests/playwright-test/reporter-line.spec.ts+++ b/tests/playwright-test/reporter-line.spec.ts@@ -190,7 +190,7 @@ for (const useIntermediateMergeReport of [false, true] as const) {expect(result.exitCode).toBe(1);});- test('should show error prompt with relative path', async ({ runInlineTest, useIntermediateMergeReport }) => {+ test('should show error context with relative path', async ({ runInlineTest, useIntermediateMergeReport }) => {const result = await runInlineTest({'a.test.js': `const { test, expect } = require('@playwright/test');@@ -201,9 +201,9 @@ for (const useIntermediateMergeReport of [false, true] as const) {}, { reporter: 'line' });const text = result.output;if (useIntermediateMergeReport)- expect(text).toContain(`Error Prompt: ${path.join('blob-report', 'resources')}`);+ expect(text).toContain(`Error Context: ${path.join('blob-report', 'resources')}`);else- expect(text).toContain(`Error Prompt: ${path.join('test-results', 'a-one', 'prompt.md')}`);+ expect(text).toContain(`Error Context: ${path.join('test-results', 'a-one', 'error-context.md')}`);expect(result.exitCode).toBe(1);});});