Playwright is a framework for Web Testing and Automation. It allows testing Chromium, Firefox and WebKit with a single API.
TypeScript automationchromechromiume2e-testingelectronend-to-end-testingfirefoxjavascriptplaywrighttesttest-automationtestingtesting-toolswebwebkit
chore: manage timeouts for perform expect (#38579)
Pavel Feldman6731dbf4463a
diff --git a/examples/todomvc/tests/adding-new-todos/add-multiple-todos-2.spec.ts b/examples/todomvc/tests/adding-new-todos/add-multiple-todos-2.spec.tsnew file mode 100644index 0000000000..779aae1cfa--- /dev/null+++ b/examples/todomvc/tests/adding-new-todos/add-multiple-todos-2.spec.ts@@ -0,0 +1,25 @@+import { test } from '../fixtures';++test.use({+ agent: {+ provider: 'github',+ model: 'gpt-4.1',+ cachePathTemplate: '{testDir}/{testFilePath}-cache.json',+ cacheMode: process.env.UPDATE_CACHE ? 'update' : process.env.CI ? 'force' : 'auto',+ }+});++test.describe('Adding New Todos', () => {+ test('should add multiple todos', async ({ page }) => {+ page.on('agentturn', turn => {+ console.log('agentturn', turn.role, turn.message);+ })+ await page.perform(`Add "Buy groceries", todo item`);+ await page.perform(`Add "Walk the dog" todo item`);+ await page.perform(`Add "Read a book" todo item`);++ await page.perform(`Ensure each todo has an unchecked checkbox`);+ await page.perform(`Ensure counter shows "3 items left" (plural)`);+ await page.perform(`Ensure input field is cleared`);+ });+});diff --git a/examples/todomvc/tests/adding-new-todos/add-multiple-todos-2.spec.ts-cache.json b/examples/todomvc/tests/adding-new-todos/add-multiple-todos-2.spec.ts-cache.jsonnew file mode 100644index 0000000000..620c26dfde--- /dev/null+++ b/examples/todomvc/tests/adding-new-todos/add-multiple-todos-2.spec.ts-cache.json@@ -0,0 +1,112 @@+{+ "Add \"Buy groceries\", todo item": {+ "timestamp": 1765848997203,+ "actions": [+ {+ "method": "fill",+ "selector": "internal:role=textbox[name=\"What needs to be done?\"i]",+ "text": "Buy groceries",+ "submit": true,+ "code": "await page.getByRole('textbox', { name: 'What needs to be done?' }).fill('Buy groceries');\nawait page.keyboard.press('Enter');",+ "intent": "Add the new todo item 'Buy groceries' by typing into the input field and submitting."+ },+ {+ "method": "expectVisible",+ "selector": "internal:text=\"Buy groceries\"i",+ "code": "await expect(page.getByText('Buy groceries')).toBeVisible();",+ "intent": "Verify that the todo item 'Buy groceries' appears in the list, confirming addition."+ }+ ]+ },+ "Add \"Read a book\" todo item": {+ "timestamp": 1765849025987,+ "actions": [+ {+ "method": "fill",+ "selector": "internal:role=textbox[name=\"What needs to be done?\"i]",+ "text": "Read a book",+ "submit": true,+ "code": "await page.getByRole('textbox', { name: 'What needs to be done?' }).fill('Read a book');\nawait page.keyboard.press('Enter');",+ "intent": "Add a new todo item called 'Read a book' by typing it into the input and submitting."+ },+ {+ "method": "expectVisible",+ "selector": "internal:text=\"Read a book\"i",+ "code": "await expect(page.getByText('Read a book')).toBeVisible();",+ "intent": "Verify that the todo item 'Read a book' is now visible on the list to confirm it was added successfully."+ }+ ]+ },+ "Add \"Walk the dog\" todo item": {+ "timestamp": 1765849011629,+ "actions": [+ {+ "method": "fill",+ "selector": "internal:role=textbox[name=\"What needs to be done?\"i]",+ "text": "Walk the dog",+ "submit": true,+ "code": "await page.getByRole('textbox', { name: 'What needs to be done?' }).fill('Walk the dog');\nawait page.keyboard.press('Enter');",+ "intent": "Add a new todo item 'Walk the dog' by typing into the input field and submitting."+ },+ {+ "method": "expectVisible",+ "selector": "internal:text=\"Walk the dog\"i",+ "code": "await expect(page.getByText('Walk the dog')).toBeVisible();",+ "intent": "Verify that 'Walk the dog' appears in the list of todo items, confirming the item was added successfully."+ }+ ]+ },+ "Ensure each todo has an unchecked checkbox": {+ "timestamp": 1765908604120,+ "actions": [+ {+ "method": "expectValue",+ "selector": "internal:role=listitem >> internal:has-text=\"Buy groceries\"i >> internal:label=\"Toggle Todo\"i",+ "type": "checkbox",+ "value": "false",+ "code": "await expect(page.getByRole('listitem').filter({ hasText: 'Buy groceries' }).getByLabel('Toggle Todo')).toBeChecked({ checked: false });",+ "intent": "Verify that the first todo's checkbox is unchecked.\nVerify that the second todo's checkbox is unchecked.\nVerify that the third todo's checkbox is unchecked."+ },+ {+ "method": "expectValue",+ "selector": "internal:role=listitem >> internal:has-text=\"Walk the dog\"i >> internal:label=\"Toggle Todo\"i",+ "type": "checkbox",+ "value": "false",+ "code": "await expect(page.getByRole('listitem').filter({ hasText: 'Walk the dog' }).getByLabel('Toggle Todo')).toBeChecked({ checked: false });",+ "intent": "Verify that the first todo's checkbox is unchecked.\nVerify that the second todo's checkbox is unchecked.\nVerify that the third todo's checkbox is unchecked."+ },+ {+ "method": "expectValue",+ "selector": "internal:role=listitem >> internal:has-text=\"Read a book\"i >> internal:label=\"Toggle Todo\"i",+ "type": "checkbox",+ "value": "false",+ "code": "await expect(page.getByRole('listitem').filter({ hasText: 'Read a book' }).getByLabel('Toggle Todo')).toBeChecked({ checked: false });",+ "intent": "Verify that the first todo's checkbox is unchecked.\nVerify that the second todo's checkbox is unchecked.\nVerify that the third todo's checkbox is unchecked."+ }+ ]+ },+ "Ensure counter shows \"3 items left\" (plural)": {+ "timestamp": 1765908755394,+ "actions": [+ {+ "method": "expectVisible",+ "selector": "internal:text=\"3 items left\"i",+ "code": "await expect(page.getByText('3 items left')).toBeVisible();",+ "intent": "Verify that the counter displays '3 items left' to ensure proper pluralization for 3 items remaining in the todo list."+ }+ ]+ },+ "Ensure input field is cleared": {+ "timestamp": 1765908762823,+ "actions": [+ {+ "method": "expectValue",+ "selector": "internal:role=textbox[name=\"What needs to be done?\"i]",+ "type": "textbox",+ "value": "",+ "code": "await expect(page.getByRole('textbox', { name: 'What needs to be done?' })).toHaveValue('');",+ "intent": "Check that the input field is cleared (empty)."+ }+ ]+ }+}\ No newline at end of filediff --git a/packages/playwright-core/src/server/agent/actionRunner.ts b/packages/playwright-core/src/server/agent/actionRunner.tsindex b572f0e1bf..efe7f4d072 100644--- a/packages/playwright-core/src/server/agent/actionRunner.ts+++ b/packages/playwright-core/src/server/agent/actionRunner.ts@@ -15,16 +15,27 @@*/import { serializeExpectedTextValues } from '../utils/expectUtils';-import { serializePlainValue } from '../../protocol/serializers';+import { monotonicTime } from '../../utils/isomorphic/time';+import { ProgressController } from '../progress';import type * as actions from './actions';-import type * as channels from '@protocol/channels';import type { Page } from '../page';import type { Progress } from '../progress';import type { NameValue } from '@protocol/channels';import type { ExpectResult } from '../frames';-export async function runAction(progress: Progress, page: Page, action: actions.Action, secrets: NameValue[]) {+export async function runAction(parentProgress: Progress, mode: 'generate' | 'run', page: Page, action: actions.Action, secrets: NameValue[]) {+ const timeout = mode === 'generate' ? generateActionTimeout(action) : performActionTimeout(action);+ const mt = monotonicTime();+ const deadline = mt + timeout;+ const minDeadline = parentProgress.deadline ? Math.min(parentProgress.deadline, deadline) : deadline;+ const pc = new ProgressController();+ return await pc.run(async progress => {+ return await innerRunAction(progress, page, action, secrets);+ }, minDeadline - mt);+}++async function innerRunAction(progress: Progress, page: Page, action: actions.Action, secrets: NameValue[]) {const frame = page.mainFrame();switch (action.method) {case 'click':@@ -63,7 +74,7 @@ export async function runAction(progress: Progress, page: Page, action: actions.await frame.uncheck(progress, action.selector, { ...strictTrue });break;case 'expectVisible': {- const result = await frame.expect(progress, action.selector, { expression: 'to.be.visible', isNot: false }, 5000);+ const result = await frame.expect(progress, action.selector, { expression: 'to.be.visible', isNot: false });if (result.errorMessage)throw new Error(result.errorMessage);break;@@ -72,25 +83,52 @@ export async function runAction(progress: Progress, page: Page, action: actions.let result: ExpectResult;if (action.type === 'textbox' || action.type === 'combobox' || action.type === 'slider') {const expectedText = serializeExpectedTextValues([action.value]);- result = await frame.expect(progress, action.selector, { expression: 'to.have.value', expectedText, isNot: false }, 5000);+ result = await frame.expect(progress, action.selector, { expression: 'to.have.value', expectedText, isNot: false });} else if (action.type === 'checkbox' || action.type === 'radio') {- const expectedValue = serializeArgument({ checked: true });- result = await frame.expect(progress, action.selector, { expression: 'to.be.checked', expectedValue, isNot: false }, 5000);+ const expectedValue = { checked: action.value === 'true' };+ result = await frame.expect(progress, action.selector, { selector: action.selector, expression: 'to.be.checked', expectedValue, isNot: false });} else {throw new Error(`Unsupported element type: ${action.type}`);}- if (result.errorMessage)+ if (!result.matches)throw new Error(result.errorMessage);break;}}}-export function serializeArgument(arg: any): channels.SerializedArgument {- return {- value: serializePlainValue(arg),- handles: []- };+export function generateActionTimeout(action: actions.Action): number {+ switch (action.method) {+ case 'click':+ case 'drag':+ case 'hover':+ case 'selectOption':+ case 'pressKey':+ case 'pressSequentially':+ case 'fill':+ case 'setChecked':+ return 5000;+ case 'expectVisible':+ case 'expectValue':+ return 1; // one shot+ }+}++export function performActionTimeout(action: actions.Action): number {+ switch (action.method) {+ case 'click':+ case 'drag':+ case 'hover':+ case 'selectOption':+ case 'pressKey':+ case 'pressSequentially':+ case 'fill':+ case 'setChecked':+ return 0; // no timeout+ case 'expectVisible':+ case 'expectValue':+ return 5000; // default expect timeout.+ }}const strictTrue = { strict: true };diff --git a/packages/playwright-core/src/server/agent/agent.ts b/packages/playwright-core/src/server/agent/agent.tsindex 57de45dcd2..d2d92f3088 100644--- a/packages/playwright-core/src/server/agent/agent.ts+++ b/packages/playwright-core/src/server/agent/agent.ts@@ -37,10 +37,10 @@ type Usage = {export async function pagePerform(progress: Progress, page: Page, options: channels.PagePerformParams): Promise<Usage> {const context = new Context(progress, page);- if (await cachedPerform(context, options))+ if (await cachedPerform(progress, context, options))return { turns: 0, inputTokens: 0, outputTokens: 0 };- const { usage } = await perform(context, options.task, undefined, options);+ const { usage } = await perform(progress, context, options.task, undefined, options);await updateCache(context, options);return usage;}@@ -56,15 +56,15 @@ Extract the following information from the page. Do not perform any actions, jus### Query${options.query}`;- const { result, usage } = await perform(context, task, options.schema, options);+ const { result, usage } = await perform(progress, context, task, options.schema, options);return { result, usage };}-async function perform(context: Context, userTask: string, resultSchema: loopTypes.Schema | undefined, options: { maxTurns?: number, maxTokens?: number } = {}): Promise<{+async function perform(progress: Progress, context: Context, userTask: string, resultSchema: loopTypes.Schema | undefined, options: { maxTurns?: number, maxTokens?: number } = {}): Promise<{result: any,usage: Usage}> {- const { progress, page } = context;+ const { page } = context;const browserContext = page.browserContext;if (!browserContext._options.agent)throw new Error(`page.perform() and page.extract() require the agent to be set on the browser context`);@@ -83,6 +83,11 @@ async function perform(context: Context, userTask: string, resultSchema: loopTypcallTool,tools,...limits,+ onBeforeTurn: ({ conversation }) => {+ const userMessage = conversation.messages.find(m => m.role === 'user');+ page.emit(Page.Events.AgentTurn, { role: 'user', message: userMessage?.content ?? '' });+ return 'continue';+ },onAfterTurn: ({ assistantMessage, totalUsage }) => {++turns;const usage = { inputTokens: totalUsage.input, outputTokens: totalUsage.output };@@ -103,6 +108,10 @@ async function perform(context: Context, userTask: string, resultSchema: loopTypreturn 'break';return 'continue';},+ onToolCallError: ({ toolCall, error }) => {+ page.emit(Page.Events.AgentTurn, { role: 'user', message: `tool "${toolCall.name}" failed: ${error.message}` });+ return 'continue';+ },...options});@@ -130,7 +139,7 @@ type CachedActions = Record<string, {const allCaches = new Map<string, CachedActions>();-async function cachedPerform(context: Context, options: channels.PagePerformParams): Promise<boolean> {+async function cachedPerform(progress: Progress, context: Context, options: channels.PagePerformParams): Promise<boolean> {if (!context.options?.cacheFile || context.options.cacheMode === 'ignore' || context.options.cacheMode === 'update')return false;@@ -144,7 +153,7 @@ async function cachedPerform(context: Context, options: channels.PagePerformPara}for (const action of entry.actions)- await runAction(context.progress, context.page, action, context.options.secrets ?? []);+ await runAction(progress, 'run', context.page, action, context.options.secrets ?? []);return true;}@@ -158,7 +167,9 @@ async function updateCache(context: Context, options: channels.PagePerformParamstimestamp: Date.now(),actions: context.actions,};- await fs.promises.writeFile(cacheFile, JSON.stringify(cache, undefined, 2));+ const entries = Object.entries(cache);+ entries.sort((e1, e2) => e1[0].localeCompare(e2[0]));+ await fs.promises.writeFile(cacheFile, JSON.stringify(Object.fromEntries(entries), undefined, 2));}async function cachedActions(cacheFile: string): Promise<CachedActions> {diff --git a/packages/playwright-core/src/server/agent/codegen.ts b/packages/playwright-core/src/server/agent/codegen.tsindex 363c29a1e7..187ff70d04 100644--- a/packages/playwright-core/src/server/agent/codegen.ts+++ b/packages/playwright-core/src/server/agent/codegen.ts@@ -69,6 +69,8 @@ export async function generateCode(sdkLanguage: Language, action: actions.Action}case 'expectValue': {const locator = asLocator(sdkLanguage, action.selector);+ if (action.type === 'checkbox' || action.type === 'radio')+ return `await expect(page.${locator}).toBeChecked({ checked: ${action.value === 'true'} });`;return `await expect(page.${locator}).toHaveValue(${escapeWithQuotes(action.value)});`;}}diff --git a/packages/playwright-core/src/server/agent/context.ts b/packages/playwright-core/src/server/agent/context.tsindex 55cc4b25a3..2102af4ac2 100644--- a/packages/playwright-core/src/server/agent/context.ts+++ b/packages/playwright-core/src/server/agent/context.ts@@ -31,14 +31,14 @@ type AgentOptions = BrowserContextOptions['agent'];export class Context {readonly options: AgentOptions;- readonly progress: Progress;readonly page: Page;readonly actions: actions.ActionWithCode[] = [];readonly sdkLanguage: Language;+ private _progress: Progress;private _callIntent: string | undefined;- constructor(progress: Progress, page: Page) {- this.progress = progress;+ constructor(apiCallProgress: Progress, page: Page) {+ this._progress = apiCallProgress;this.page = page;this.options = page.browserContext._options.agent;this.sdkLanguage = page.browserContext._browser.sdkLanguage();@@ -58,18 +58,15 @@ export class Context {}async runActionsAndWait(action: actions.Action[]) {- try {- await this.waitForCompletion(async () => {- for (const a of action) {- await runAction(this.progress, this.page, a, this.options?.secrets ?? []);- const code = await generateCode(this.sdkLanguage, a);- this.actions.push({ ...a, code, intent: this._callIntent });- }- });- return await this.snapshotResult();- } catch (e) {- return await this.snapshotResult(e);- }+ const error = await this.waitForCompletion(async () => {+ for (const a of action) {+ await runAction(this._progress, 'generate', this.page, a, this.options?.secrets ?? []);+ const code = await generateCode(this.sdkLanguage, a);+ this.actions.push({ ...a, code, intent: this._callIntent });+ }+ return undefined;+ }).catch((error: Error) => error);+ return await this.snapshotResult(error);}async waitForCompletion<R>(callback: () => Promise<R>): Promise<R> {@@ -83,14 +80,14 @@ export class Context {let result: R;try {result = await callback();- await this.progress.wait(500);+ await this._progress.wait(500);} finally {disposeListeners();}const requestedNavigation = requests.some(request => request.isNavigationRequest());if (requestedNavigation) {- await this.page.mainFrame().waitForLoadState(this.progress, 'load');+ await this.page.mainFrame().waitForLoadState(this._progress, 'load');return result;}@@ -101,15 +98,15 @@ export class Context {elsepromises.push(request.response());}- await this.progress.race(promises, { timeout: 5000 });+ await this._progress.race(promises, { timeout: 5000 });if (requests.length)- await this.progress.wait(500);+ await this._progress.wait(500);return result;}async snapshotResult(error?: Error): Promise<loopTypes.ToolResult> {- let { full } = await this.page.snapshotForAI(this.progress);+ let { full } = await this.page.snapshotForAI(this._progress);full = this._redactText(full);const text: string[] = [];@@ -138,7 +135,7 @@ export class Context {async refSelectors(params: { element: string, ref: string }[]): Promise<string[]> {return Promise.all(params.map(async param => {try {- const { resolvedSelector } = await this.page.mainFrame().resolveSelector(this.progress, `aria-ref=${param.ref}`);+ const { resolvedSelector } = await this.page.mainFrame().resolveSelector(this._progress, `aria-ref=${param.ref}`);return resolvedSelector;} catch (e) {throw new Error(`Ref ${param.ref} not found in the current page snapshot. Try capturing new snapshot.`);diff --git a/packages/playwright-core/src/server/progress.ts b/packages/playwright-core/src/server/progress.tsindex e80a1404e8..46900117d8 100644--- a/packages/playwright-core/src/server/progress.ts+++ b/packages/playwright-core/src/server/progress.ts@@ -15,7 +15,7 @@*/import { TimeoutError } from './errors';-import { assert } from '../utils';+import { assert, monotonicTime } from '../utils';import { ManualPromise } from '../utils/isomorphic/manualPromise';import type { Progress } from '@protocol/progress';@@ -47,10 +47,12 @@ export class ProgressController {}async run<T>(task: (progress: Progress) => Promise<T>, timeout?: number): Promise<T> {+ const deadline = timeout ? monotonicTime() + timeout : 0;assert(this._state === 'before');this._state = 'running';const progress: Progress = {+ deadline,log: message => {if (this._state === 'running')this.metadata.log.push(message);@@ -60,10 +62,13 @@ export class ProgressController {metadata: this.metadata,race: <T>(promise: Promise<T> | Promise<T>[], options?: { timeout?: number }) => {const promises = Array.isArray(promise) ? promise : [promise];- const timerPromise = options?.timeout ? new Promise<void>(f => setTimeout(f, options.timeout)) : null;+ const mt = monotonicTime();+ const dl = options?.timeout ? mt + options.timeout : 0;+ const timerPromise = dl && (!deadline || dl < deadline) ? new Promise<void>(f => setTimeout(f, dl - mt)) : null;return Promise.race([...promises, ...(timerPromise ? [timerPromise] : []), this._forceAbortPromise]);},wait: async (timeout: number) => {+ // Timeout = 0 here means nowait. Counter to what it typically is (wait forever).let timer: NodeJS.Timeout;const promise = new Promise<void>(f => timer = setTimeout(f, timeout));return progress.race(promise).finally(() => clearTimeout(timer));@@ -71,7 +76,7 @@ export class ProgressController {};let timer: NodeJS.Timeout | undefined;- if (timeout) {+ if (deadline) {const timeoutError = new TimeoutError(`Timeout ${timeout}ms exceeded.`);timer = setTimeout(() => {if (this.metadata.pauseStartTime && !this.metadata.pauseEndTime)@@ -81,7 +86,7 @@ export class ProgressController {this._state = { error: timeoutError };this._forceAbortPromise.reject(timeoutError);}- }, timeout);+ }, deadline - monotonicTime());}try {diff --git a/packages/protocol/src/progress.d.ts b/packages/protocol/src/progress.d.tsindex 7f1734627b..96e0a33b97 100644--- a/packages/protocol/src/progress.d.ts+++ b/packages/protocol/src/progress.d.ts@@ -35,6 +35,7 @@ import type { CallMetadata } from './callMetadata';// should be used to cleanup. Whether the cleanup is awaited or not depends on the method details.// For the trickiest cases, look at `raceUncancellableOperationWithCleanup()` helper method.export interface Progress {+ deadline: number;log(message: string): void;race<T>(promise: Promise<T> | Promise<T>[], options?: { timeout?: number }): Promise<T>;wait(timeout: number): Promise<void>;