Playwright is a framework for Web Testing and Automation. It allows testing Chromium, Firefox and WebKit with a single API.
TypeScript automationchromechromiume2e-testingelectronend-to-end-testingfirefoxjavascriptplaywrighttesttest-automationtestingtesting-toolswebwebkit
chore: improve prompt to use code frame and inline error (#35032)
Pavel Feldman78fa91175614
diff --git a/examples/todomvc/playwright.config.ts b/examples/todomvc/playwright.config.tsindex 3d76370d4d..831aa90355 100644--- a/examples/todomvc/playwright.config.ts+++ b/examples/todomvc/playwright.config.ts@@ -12,6 +12,8 @@ export default defineConfig({/* Maximum time one test can run for. */timeout: 15_000,+ captureGitInfo: { commit: true, diff: true },+expect: {/**diff --git a/packages/playwright-core/src/utils/isomorphic/stackTrace.ts b/packages/playwright-core/src/utils/isomorphic/stackTrace.tsindex f3d5a54da0..dca83848e9 100644--- a/packages/playwright-core/src/utils/isomorphic/stackTrace.ts+++ b/packages/playwright-core/src/utils/isomorphic/stackTrace.ts@@ -134,6 +134,34 @@ export function splitErrorMessage(message: string): { name: string, message: str};}+export function parseErrorStack(stack: string, pathSeparator: string, showInternalStackFrames: boolean = false): {+ message: string;+ stackLines: string[];+ location?: StackFrame;+} {+ const lines = stack.split('\n');+ let firstStackLine = lines.findIndex(line => line.startsWith(' at '));+ if (firstStackLine === -1)+ firstStackLine = lines.length;+ const message = lines.slice(0, firstStackLine).join('\n');+ const stackLines = lines.slice(firstStackLine);+ let location: StackFrame | undefined;+ for (const line of stackLines) {+ const frame = parseStackFrame(line, pathSeparator, showInternalStackFrames);+ if (!frame || !frame.file)+ continue;+ if (belongsToNodeModules(frame.file, pathSeparator))+ continue;+ location = { file: frame.file, column: frame.column || 0, line: frame.line || 0 };+ break;+ }+ return { message, stackLines, location };+}++function belongsToNodeModules(file: string, pathSeparator: string) {+ return file.includes(`${pathSeparator}node_modules${pathSeparator}`);+}+const re = new RegExp('^' +// Sometimes we strip out the ' at' because it's noisy'(?:\\s*at )?' +diff --git a/packages/playwright/src/DEPS.list b/packages/playwright/src/DEPS.listindex 200d75de1a..25ca70ff18 100644--- a/packages/playwright/src/DEPS.list+++ b/packages/playwright/src/DEPS.list@@ -8,7 +8,11 @@ common/[index.ts]@testIsomorphic/**+./prompt.ts./worker/testTracing.ts[internalsForTest.ts]**++[prompt.ts]+./transform/babelBundle.tsdiff --git a/packages/playwright/src/index.ts b/packages/playwright/src/index.tsindex cc5a906c4c..a8c128694e 100644--- a/packages/playwright/src/index.ts+++ b/packages/playwright/src/index.ts@@ -22,15 +22,15 @@ import { setBoxedStackPrefixes, asLocator, createGuid, currentZone, debugMode, iimport { currentTestInfo } from './common/globals';import { rootTestType } from './common/testType';-import { stripAnsiEscapes } from './util';+import { attachErrorPrompts } from './prompt';-import type { MetadataWithCommitInfo } from './isomorphic/types';import type { Fixtures, PlaywrightTestArgs, PlaywrightTestOptions, PlaywrightWorkerArgs, PlaywrightWorkerOptions, ScreenshotMode, TestInfo, TestType, VideoMode } from '../types/test';import type { ContextReuseMode } from './common/config';import type { TestInfoImpl, TestStepInternal } from './worker/testInfo';import type { ApiCallData, ClientInstrumentation, ClientInstrumentationListener } from '../../playwright-core/src/client/clientInstrumentation';import type { Playwright as PlaywrightImpl } from '../../playwright-core/src/client/playwright';import type { APIRequestContext, Browser, BrowserContext, BrowserContextOptions, LaunchOptions, Page, Tracing, Video } from 'playwright-core';+export { expect } from './matchers/expect';export const _baseTest: TestType<{}, {}> = rootTestType.test;@@ -619,6 +619,7 @@ class ArtifactsRecorder {private _screenshotRecorder: SnapshotRecorder;private _pageSnapshot: string | undefined;+ private _sourceCache: Map<string, string> = new Map();constructor(playwright: PlaywrightImpl, artifactsDir: string, screenshot: ScreenshotOption) {this._playwright = playwright;@@ -701,71 +702,7 @@ class ArtifactsRecorder {})));await this._screenshotRecorder.persistTemporary();- await this._attachErrorPrompts();- }-- private async _attachErrorPrompts() {- if (process.env.PLAYWRIGHT_NO_COPY_PROMPT)- return;-- if (this._testInfo.errors.length === 0)- return;-- const testSources = await fs.promises.readFile(this._testInfo.file, 'utf-8');- for (const [index, error] of this._testInfo.errors.entries()) {- if (this._testInfo.attachments.find(a => a.name === `_prompt-${index}`))- continue;-- const metadata = this._testInfo.config.metadata as MetadataWithCommitInfo;-- const promptParts = [- `My Playwright test failed.`,- `Explain why, be concise, respect Playwright best practices.`,- '',- `Failed test: ${this._testInfo.titlePath.join(' >> ')}`,- '',- 'Error:',- '',- '```',- stripAnsiEscapes(error.stack || error.message || ''),- '```',- ];-- if (this._pageSnapshot) {- promptParts.push(- '',- 'Page snapshot:',- '```yaml',- this._pageSnapshot,- '```',- );- }-- if (metadata.gitDiff) {- promptParts.push(- '',- 'Local changes:',- '```diff',- metadata.gitDiff,- '```',- );- }-- promptParts.push(- '',- 'Test file:',- '```ts',- `// ${this._testInfo.file}`,- testSources,- '```',- );-- this._testInfo._attach({- name: `_prompt-${index}`,- contentType: 'text/markdown',- body: Buffer.from(promptParts.join('\n')),- }, undefined);- }+ await attachErrorPrompts(this._testInfo, this._sourceCache, this._pageSnapshot);}private async _startTraceChunkOnContextCreation(tracing: Tracing) {diff --git a/packages/playwright/src/prompt.ts b/packages/playwright/src/prompt.tsnew file mode 100644index 0000000000..d115f7187a--- /dev/null+++ b/packages/playwright/src/prompt.ts@@ -0,0 +1,124 @@+/**+ * Copyright (c) Microsoft Corporation.+ *+ * Licensed under the Apache License, Version 2.0 (the 'License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++import * as fs from 'fs';+import * as path from 'path';++import { parseErrorStack } from 'playwright-core/lib/utils';++import { stripAnsiEscapes } from './util';+import { codeFrameColumns } from './transform/babelBundle';++import type { TestInfo } from '../types/test';+import type { MetadataWithCommitInfo } from './isomorphic/types';+import type { TestInfoImpl } from './worker/testInfo';++export async function attachErrorPrompts(testInfo: TestInfo, sourceCache: Map<string, string>, ariaSnapshot: string | undefined) {+ if (process.env.PLAYWRIGHT_NO_COPY_PROMPT)+ return;++ for (const [index, error] of testInfo.errors.entries()) {+ if (testInfo.attachments.find(a => a.name === `_prompt-${index}`))+ continue;++ const metadata = testInfo.config.metadata as MetadataWithCommitInfo;++ const promptParts = [+ `# Instructions`,+ '',+ `- Following Playwright test failed.`,+ `- Explain why, be concise, respect Playwright best practices.`,+ `- Provide a snippet of code with the fix is possible.`,+ '',+ `# Test info`,+ '',+ `- Name: ${testInfo.titlePath.slice(1).join(' >> ')}`,+ `- Location: ${testInfo.file}:${testInfo.line}:${testInfo.column}`,+ '',+ '# Error details',+ '',+ '```',+ stripAnsiEscapes(error.stack || error.message || ''),+ '```',+ ];++ if (ariaSnapshot) {+ promptParts.push(+ '',+ '# Page snapshot',+ '',+ '```yaml',+ ariaSnapshot,+ '```',+ );+ }++ const parsedError = error.stack ? parseErrorStack(error.stack, path.sep) : undefined;+ const inlineMessage = stripAnsiEscapes(parsedError?.message || error.message || '').split('\n')[0];+ const location = parsedError?.location || { file: testInfo.file, line: testInfo.line, column: testInfo.column };+ const source = await loadSource(location.file, sourceCache);+ const codeFrame = codeFrameColumns(+ source,+ {+ start: {+ line: location.line,+ column: location.column+ },+ },+ {+ highlightCode: false,+ linesAbove: 100,+ linesBelow: 100,+ message: inlineMessage || undefined,+ }+ );+ promptParts.push(+ '',+ '# Test source',+ '',+ '```ts',+ codeFrame,+ '```',+ );++ if (metadata.gitDiff) {+ promptParts.push(+ '',+ '# Local changes',+ '',+ '```diff',+ metadata.gitDiff,+ '```',+ );+ }++ (testInfo as TestInfoImpl)._attach({+ name: `_prompt-${index}`,+ contentType: 'text/markdown',+ body: Buffer.from(promptParts.join('\n')),+ }, undefined);+ }+}++async function loadSource(file: string, sourceCache: Map<string, string>) {+ let source = sourceCache.get(file);+ if (!source) {+ // A mild race is Ok here.+ source = await fs.promises.readFile(file, 'utf8');+ sourceCache.set(file, source);+ }+ return source;+}diff --git a/packages/playwright/src/reporters/base.ts b/packages/playwright/src/reporters/base.tsindex b8a78105bc..30534d7e8f 100644--- a/packages/playwright/src/reporters/base.ts+++ b/packages/playwright/src/reporters/base.ts@@ -16,8 +16,7 @@import path from 'path';-import { getPackageManagerExecCommand } from 'playwright-core/lib/utils';-import { parseStackFrame } from 'playwright-core/lib/utils';+import { getPackageManagerExecCommand, parseErrorStack } from 'playwright-core/lib/utils';import { ms as milliseconds } from 'playwright-core/lib/utilsBundle';import { colors as realColors, noColors } from 'playwright-core/lib/utils';@@ -540,23 +539,7 @@ export function prepareErrorStack(stack: string): {stackLines: string[];location?: Location;} {- const lines = stack.split('\n');- let firstStackLine = lines.findIndex(line => line.startsWith(' at '));- if (firstStackLine === -1)- firstStackLine = lines.length;- const message = lines.slice(0, firstStackLine).join('\n');- const stackLines = lines.slice(firstStackLine);- let location: Location | undefined;- for (const line of stackLines) {- const frame = parseStackFrame(line, path.sep, !!process.env.PWDEBUGIMPL);- if (!frame || !frame.file)- continue;- if (belongsToNodeModules(frame.file))- continue;- location = { file: frame.file, column: frame.column || 0, line: frame.line || 0 };- break;- }- return { message, stackLines, location };+ return parseErrorStack(stack, path.sep, !!process.env.PWDEBUGIMPL);}function characterWidth(c: string) {@@ -611,10 +594,6 @@ export function fitToWidth(line: string, width: number, prefix?: string): stringreturn taken.reverse().join('');}-function belongsToNodeModules(file: string) {- return file.includes(`${path.sep}node_modules${path.sep}`);-}-function resolveFromEnv(name: string): string | undefined {const value = process.env[name];if (value)diff --git a/tests/playwright-test/ui-mode-llm.spec.ts b/tests/playwright-test/ui-mode-llm.spec.tsindex b639f52e37..27f5c91aeb 100644--- a/tests/playwright-test/ui-mode-llm.spec.ts+++ b/tests/playwright-test/ui-mode-llm.spec.ts@@ -18,7 +18,7 @@ import { test, expect, retries } from './ui-mode-fixtures';test.describe.configure({ mode: 'parallel', retries });-test('openai', async ({ runUITest, server }) => {+test.fixme('openai', async ({ runUITest, server }) => {server.setRoute('/v1/chat/completions', async (req, res) => {res.setHeader('Access-Control-Allow-Origin', '*');res.setHeader('Access-Control-Allow-Headers', '*');@@ -59,7 +59,7 @@ test('openai', async ({ runUITest, server }) => {`);});-test('anthropic', async ({ runUITest, server }) => {+test.fixme('anthropic', async ({ runUITest, server }) => {server.setRoute('/v1/messages', async (req, res) => {res.setHeader('Access-Control-Allow-Origin', '*');res.setHeader('Access-Control-Allow-Headers', '*');diff --git a/tests/playwright-test/ui-mode-trace.spec.ts b/tests/playwright-test/ui-mode-trace.spec.tsindex 9cf0f8ac1e..47662adf97 100644--- a/tests/playwright-test/ui-mode-trace.spec.ts+++ b/tests/playwright-test/ui-mode-trace.spec.ts@@ -519,9 +519,11 @@ test('fails', async ({ page }) => {const prompt = await page.evaluate(() => navigator.clipboard.readText());expect(prompt, 'contains error').toContain('expect(received).toBe(expected)');expect(prompt.replaceAll('\r\n', '\n'), 'contains test sources').toContain(`-test('fails', async ({ page }) => {- await page.setContent('<button>Submit</button>');- expect(1).toBe(2);-});+ 1 | import { test, expect } from '@playwright/test';+ 2 | test('fails', async ({ page }) => {+ 3 | await page.setContent('<button>Submit</button>');+> 4 | expect(1).toBe(2);+ | ^ Error: expect(received).toBe(expected) // Object.is equality+ 5 | });`.trim());});