Skip to content

Playwright is a framework for Web Testing and Automation. It allows testing Chromium, Firefox and WebKit with a single API.

chore: manage timeouts for perform expect (#38579)

Pavel Feldman6731dbf4463a
diff --git a/examples/todomvc/tests/adding-new-todos/add-multiple-todos-2.spec.ts b/examples/todomvc/tests/adding-new-todos/add-multiple-todos-2.spec.ts
new file mode 100644
index 0000000000..779aae1cfa
--- /dev/null
+++ b/examples/todomvc/tests/adding-new-todos/add-multiple-todos-2.spec.ts
@@ -0,0 +1,25 @@
+import { test } from '../fixtures';
+
+test.use({
+ agent: {
+ provider: 'github',
+ model: 'gpt-4.1',
+ cachePathTemplate: '{testDir}/{testFilePath}-cache.json',
+ cacheMode: process.env.UPDATE_CACHE ? 'update' : process.env.CI ? 'force' : 'auto',
+ }
+});
+
+test.describe('Adding New Todos', () => {
+ test('should add multiple todos', async ({ page }) => {
+ page.on('agentturn', turn => {
+ console.log('agentturn', turn.role, turn.message);
+ })
+ await page.perform(`Add "Buy groceries", todo item`);
+ await page.perform(`Add "Walk the dog" todo item`);
+ await page.perform(`Add "Read a book" todo item`);
+
+ await page.perform(`Ensure each todo has an unchecked checkbox`);
+ await page.perform(`Ensure counter shows "3 items left" (plural)`);
+ await page.perform(`Ensure input field is cleared`);
+ });
+});
diff --git a/examples/todomvc/tests/adding-new-todos/add-multiple-todos-2.spec.ts-cache.json b/examples/todomvc/tests/adding-new-todos/add-multiple-todos-2.spec.ts-cache.json
new file mode 100644
index 0000000000..620c26dfde
--- /dev/null
+++ b/examples/todomvc/tests/adding-new-todos/add-multiple-todos-2.spec.ts-cache.json
@@ -0,0 +1,112 @@
+{
+ "Add \"Buy groceries\", todo item": {
+ "timestamp": 1765848997203,
+ "actions": [
+ {
+ "method": "fill",
+ "selector": "internal:role=textbox[name=\"What needs to be done?\"i]",
+ "text": "Buy groceries",
+ "submit": true,
+ "code": "await page.getByRole('textbox', { name: 'What needs to be done?' }).fill('Buy groceries');\nawait page.keyboard.press('Enter');",
+ "intent": "Add the new todo item 'Buy groceries' by typing into the input field and submitting."
+ },
+ {
+ "method": "expectVisible",
+ "selector": "internal:text=\"Buy groceries\"i",
+ "code": "await expect(page.getByText('Buy groceries')).toBeVisible();",
+ "intent": "Verify that the todo item 'Buy groceries' appears in the list, confirming addition."
+ }
+ ]
+ },
+ "Add \"Read a book\" todo item": {
+ "timestamp": 1765849025987,
+ "actions": [
+ {
+ "method": "fill",
+ "selector": "internal:role=textbox[name=\"What needs to be done?\"i]",
+ "text": "Read a book",
+ "submit": true,
+ "code": "await page.getByRole('textbox', { name: 'What needs to be done?' }).fill('Read a book');\nawait page.keyboard.press('Enter');",
+ "intent": "Add a new todo item called 'Read a book' by typing it into the input and submitting."
+ },
+ {
+ "method": "expectVisible",
+ "selector": "internal:text=\"Read a book\"i",
+ "code": "await expect(page.getByText('Read a book')).toBeVisible();",
+ "intent": "Verify that the todo item 'Read a book' is now visible on the list to confirm it was added successfully."
+ }
+ ]
+ },
+ "Add \"Walk the dog\" todo item": {
+ "timestamp": 1765849011629,
+ "actions": [
+ {
+ "method": "fill",
+ "selector": "internal:role=textbox[name=\"What needs to be done?\"i]",
+ "text": "Walk the dog",
+ "submit": true,
+ "code": "await page.getByRole('textbox', { name: 'What needs to be done?' }).fill('Walk the dog');\nawait page.keyboard.press('Enter');",
+ "intent": "Add a new todo item 'Walk the dog' by typing into the input field and submitting."
+ },
+ {
+ "method": "expectVisible",
+ "selector": "internal:text=\"Walk the dog\"i",
+ "code": "await expect(page.getByText('Walk the dog')).toBeVisible();",
+ "intent": "Verify that 'Walk the dog' appears in the list of todo items, confirming the item was added successfully."
+ }
+ ]
+ },
+ "Ensure each todo has an unchecked checkbox": {
+ "timestamp": 1765908604120,
+ "actions": [
+ {
+ "method": "expectValue",
+ "selector": "internal:role=listitem >> internal:has-text=\"Buy groceries\"i >> internal:label=\"Toggle Todo\"i",
+ "type": "checkbox",
+ "value": "false",
+ "code": "await expect(page.getByRole('listitem').filter({ hasText: 'Buy groceries' }).getByLabel('Toggle Todo')).toBeChecked({ checked: false });",
+ "intent": "Verify that the first todo's checkbox is unchecked.\nVerify that the second todo's checkbox is unchecked.\nVerify that the third todo's checkbox is unchecked."
+ },
+ {
+ "method": "expectValue",
+ "selector": "internal:role=listitem >> internal:has-text=\"Walk the dog\"i >> internal:label=\"Toggle Todo\"i",
+ "type": "checkbox",
+ "value": "false",
+ "code": "await expect(page.getByRole('listitem').filter({ hasText: 'Walk the dog' }).getByLabel('Toggle Todo')).toBeChecked({ checked: false });",
+ "intent": "Verify that the first todo's checkbox is unchecked.\nVerify that the second todo's checkbox is unchecked.\nVerify that the third todo's checkbox is unchecked."
+ },
+ {
+ "method": "expectValue",
+ "selector": "internal:role=listitem >> internal:has-text=\"Read a book\"i >> internal:label=\"Toggle Todo\"i",
+ "type": "checkbox",
+ "value": "false",
+ "code": "await expect(page.getByRole('listitem').filter({ hasText: 'Read a book' }).getByLabel('Toggle Todo')).toBeChecked({ checked: false });",
+ "intent": "Verify that the first todo's checkbox is unchecked.\nVerify that the second todo's checkbox is unchecked.\nVerify that the third todo's checkbox is unchecked."
+ }
+ ]
+ },
+ "Ensure counter shows \"3 items left\" (plural)": {
+ "timestamp": 1765908755394,
+ "actions": [
+ {
+ "method": "expectVisible",
+ "selector": "internal:text=\"3 items left\"i",
+ "code": "await expect(page.getByText('3 items left')).toBeVisible();",
+ "intent": "Verify that the counter displays '3 items left' to ensure proper pluralization for 3 items remaining in the todo list."
+ }
+ ]
+ },
+ "Ensure input field is cleared": {
+ "timestamp": 1765908762823,
+ "actions": [
+ {
+ "method": "expectValue",
+ "selector": "internal:role=textbox[name=\"What needs to be done?\"i]",
+ "type": "textbox",
+ "value": "",
+ "code": "await expect(page.getByRole('textbox', { name: 'What needs to be done?' })).toHaveValue('');",
+ "intent": "Check that the input field is cleared (empty)."
+ }
+ ]
+ }
+}
\ No newline at end of file
diff --git a/packages/playwright-core/src/server/agent/actionRunner.ts b/packages/playwright-core/src/server/agent/actionRunner.ts
index b572f0e1bf..efe7f4d072 100644
--- a/packages/playwright-core/src/server/agent/actionRunner.ts
+++ b/packages/playwright-core/src/server/agent/actionRunner.ts
@@ -15,16 +15,27 @@
*/
import { serializeExpectedTextValues } from '../utils/expectUtils';
-import { serializePlainValue } from '../../protocol/serializers';
+import { monotonicTime } from '../../utils/isomorphic/time';
+import { ProgressController } from '../progress';
import type * as actions from './actions';
-import type * as channels from '@protocol/channels';
import type { Page } from '../page';
import type { Progress } from '../progress';
import type { NameValue } from '@protocol/channels';
import type { ExpectResult } from '../frames';
-export async function runAction(progress: Progress, page: Page, action: actions.Action, secrets: NameValue[]) {
+export async function runAction(parentProgress: Progress, mode: 'generate' | 'run', page: Page, action: actions.Action, secrets: NameValue[]) {
+ const timeout = mode === 'generate' ? generateActionTimeout(action) : performActionTimeout(action);
+ const mt = monotonicTime();
+ const deadline = mt + timeout;
+ const minDeadline = parentProgress.deadline ? Math.min(parentProgress.deadline, deadline) : deadline;
+ const pc = new ProgressController();
+ return await pc.run(async progress => {
+ return await innerRunAction(progress, page, action, secrets);
+ }, minDeadline - mt);
+}
+
+async function innerRunAction(progress: Progress, page: Page, action: actions.Action, secrets: NameValue[]) {
const frame = page.mainFrame();
switch (action.method) {
case 'click':
@@ -63,7 +74,7 @@ export async function runAction(progress: Progress, page: Page, action: actions.
await frame.uncheck(progress, action.selector, { ...strictTrue });
break;
case 'expectVisible': {
- const result = await frame.expect(progress, action.selector, { expression: 'to.be.visible', isNot: false }, 5000);
+ const result = await frame.expect(progress, action.selector, { expression: 'to.be.visible', isNot: false });
if (result.errorMessage)
throw new Error(result.errorMessage);
break;
@@ -72,25 +83,52 @@ export async function runAction(progress: Progress, page: Page, action: actions.
let result: ExpectResult;
if (action.type === 'textbox' || action.type === 'combobox' || action.type === 'slider') {
const expectedText = serializeExpectedTextValues([action.value]);
- result = await frame.expect(progress, action.selector, { expression: 'to.have.value', expectedText, isNot: false }, 5000);
+ result = await frame.expect(progress, action.selector, { expression: 'to.have.value', expectedText, isNot: false });
} else if (action.type === 'checkbox' || action.type === 'radio') {
- const expectedValue = serializeArgument({ checked: true });
- result = await frame.expect(progress, action.selector, { expression: 'to.be.checked', expectedValue, isNot: false }, 5000);
+ const expectedValue = { checked: action.value === 'true' };
+ result = await frame.expect(progress, action.selector, { selector: action.selector, expression: 'to.be.checked', expectedValue, isNot: false });
} else {
throw new Error(`Unsupported element type: ${action.type}`);
}
- if (result.errorMessage)
+ if (!result.matches)
throw new Error(result.errorMessage);
break;
}
}
}
-export function serializeArgument(arg: any): channels.SerializedArgument {
- return {
- value: serializePlainValue(arg),
- handles: []
- };
+export function generateActionTimeout(action: actions.Action): number {
+ switch (action.method) {
+ case 'click':
+ case 'drag':
+ case 'hover':
+ case 'selectOption':
+ case 'pressKey':
+ case 'pressSequentially':
+ case 'fill':
+ case 'setChecked':
+ return 5000;
+ case 'expectVisible':
+ case 'expectValue':
+ return 1; // one shot
+ }
+}
+
+export function performActionTimeout(action: actions.Action): number {
+ switch (action.method) {
+ case 'click':
+ case 'drag':
+ case 'hover':
+ case 'selectOption':
+ case 'pressKey':
+ case 'pressSequentially':
+ case 'fill':
+ case 'setChecked':
+ return 0; // no timeout
+ case 'expectVisible':
+ case 'expectValue':
+ return 5000; // default expect timeout.
+ }
}
const strictTrue = { strict: true };
diff --git a/packages/playwright-core/src/server/agent/agent.ts b/packages/playwright-core/src/server/agent/agent.ts
index 57de45dcd2..d2d92f3088 100644
--- a/packages/playwright-core/src/server/agent/agent.ts
+++ b/packages/playwright-core/src/server/agent/agent.ts
@@ -37,10 +37,10 @@ type Usage = {
export async function pagePerform(progress: Progress, page: Page, options: channels.PagePerformParams): Promise<Usage> {
const context = new Context(progress, page);
- if (await cachedPerform(context, options))
+ if (await cachedPerform(progress, context, options))
return { turns: 0, inputTokens: 0, outputTokens: 0 };
- const { usage } = await perform(context, options.task, undefined, options);
+ const { usage } = await perform(progress, context, options.task, undefined, options);
await updateCache(context, options);
return usage;
}
@@ -56,15 +56,15 @@ Extract the following information from the page. Do not perform any actions, jus
### Query
${options.query}`;
- const { result, usage } = await perform(context, task, options.schema, options);
+ const { result, usage } = await perform(progress, context, task, options.schema, options);
return { result, usage };
}
-async function perform(context: Context, userTask: string, resultSchema: loopTypes.Schema | undefined, options: { maxTurns?: number, maxTokens?: number } = {}): Promise<{
+async function perform(progress: Progress, context: Context, userTask: string, resultSchema: loopTypes.Schema | undefined, options: { maxTurns?: number, maxTokens?: number } = {}): Promise<{
result: any,
usage: Usage
}> {
- const { progress, page } = context;
+ const { page } = context;
const browserContext = page.browserContext;
if (!browserContext._options.agent)
throw new Error(`page.perform() and page.extract() require the agent to be set on the browser context`);
@@ -83,6 +83,11 @@ async function perform(context: Context, userTask: string, resultSchema: loopTyp
callTool,
tools,
...limits,
+ onBeforeTurn: ({ conversation }) => {
+ const userMessage = conversation.messages.find(m => m.role === 'user');
+ page.emit(Page.Events.AgentTurn, { role: 'user', message: userMessage?.content ?? '' });
+ return 'continue';
+ },
onAfterTurn: ({ assistantMessage, totalUsage }) => {
++turns;
const usage = { inputTokens: totalUsage.input, outputTokens: totalUsage.output };
@@ -103,6 +108,10 @@ async function perform(context: Context, userTask: string, resultSchema: loopTyp
return 'break';
return 'continue';
},
+ onToolCallError: ({ toolCall, error }) => {
+ page.emit(Page.Events.AgentTurn, { role: 'user', message: `tool "${toolCall.name}" failed: ${error.message}` });
+ return 'continue';
+ },
...options
});
@@ -130,7 +139,7 @@ type CachedActions = Record<string, {
const allCaches = new Map<string, CachedActions>();
-async function cachedPerform(context: Context, options: channels.PagePerformParams): Promise<boolean> {
+async function cachedPerform(progress: Progress, context: Context, options: channels.PagePerformParams): Promise<boolean> {
if (!context.options?.cacheFile || context.options.cacheMode === 'ignore' || context.options.cacheMode === 'update')
return false;
@@ -144,7 +153,7 @@ async function cachedPerform(context: Context, options: channels.PagePerformPara
}
for (const action of entry.actions)
- await runAction(context.progress, context.page, action, context.options.secrets ?? []);
+ await runAction(progress, 'run', context.page, action, context.options.secrets ?? []);
return true;
}
@@ -158,7 +167,9 @@ async function updateCache(context: Context, options: channels.PagePerformParams
timestamp: Date.now(),
actions: context.actions,
};
- await fs.promises.writeFile(cacheFile, JSON.stringify(cache, undefined, 2));
+ const entries = Object.entries(cache);
+ entries.sort((e1, e2) => e1[0].localeCompare(e2[0]));
+ await fs.promises.writeFile(cacheFile, JSON.stringify(Object.fromEntries(entries), undefined, 2));
}
async function cachedActions(cacheFile: string): Promise<CachedActions> {
diff --git a/packages/playwright-core/src/server/agent/codegen.ts b/packages/playwright-core/src/server/agent/codegen.ts
index 363c29a1e7..187ff70d04 100644
--- a/packages/playwright-core/src/server/agent/codegen.ts
+++ b/packages/playwright-core/src/server/agent/codegen.ts
@@ -69,6 +69,8 @@ export async function generateCode(sdkLanguage: Language, action: actions.Action
}
case 'expectValue': {
const locator = asLocator(sdkLanguage, action.selector);
+ if (action.type === 'checkbox' || action.type === 'radio')
+ return `await expect(page.${locator}).toBeChecked({ checked: ${action.value === 'true'} });`;
return `await expect(page.${locator}).toHaveValue(${escapeWithQuotes(action.value)});`;
}
}
diff --git a/packages/playwright-core/src/server/agent/context.ts b/packages/playwright-core/src/server/agent/context.ts
index 55cc4b25a3..2102af4ac2 100644
--- a/packages/playwright-core/src/server/agent/context.ts
+++ b/packages/playwright-core/src/server/agent/context.ts
@@ -31,14 +31,14 @@ type AgentOptions = BrowserContextOptions['agent'];
export class Context {
readonly options: AgentOptions;
- readonly progress: Progress;
readonly page: Page;
readonly actions: actions.ActionWithCode[] = [];
readonly sdkLanguage: Language;
+ private _progress: Progress;
private _callIntent: string | undefined;
- constructor(progress: Progress, page: Page) {
- this.progress = progress;
+ constructor(apiCallProgress: Progress, page: Page) {
+ this._progress = apiCallProgress;
this.page = page;
this.options = page.browserContext._options.agent;
this.sdkLanguage = page.browserContext._browser.sdkLanguage();
@@ -58,18 +58,15 @@ export class Context {
}
async runActionsAndWait(action: actions.Action[]) {
- try {
- await this.waitForCompletion(async () => {
- for (const a of action) {
- await runAction(this.progress, this.page, a, this.options?.secrets ?? []);
- const code = await generateCode(this.sdkLanguage, a);
- this.actions.push({ ...a, code, intent: this._callIntent });
- }
- });
- return await this.snapshotResult();
- } catch (e) {
- return await this.snapshotResult(e);
- }
+ const error = await this.waitForCompletion(async () => {
+ for (const a of action) {
+ await runAction(this._progress, 'generate', this.page, a, this.options?.secrets ?? []);
+ const code = await generateCode(this.sdkLanguage, a);
+ this.actions.push({ ...a, code, intent: this._callIntent });
+ }
+ return undefined;
+ }).catch((error: Error) => error);
+ return await this.snapshotResult(error);
}
async waitForCompletion<R>(callback: () => Promise<R>): Promise<R> {
@@ -83,14 +80,14 @@ export class Context {
let result: R;
try {
result = await callback();
- await this.progress.wait(500);
+ await this._progress.wait(500);
} finally {
disposeListeners();
}
const requestedNavigation = requests.some(request => request.isNavigationRequest());
if (requestedNavigation) {
- await this.page.mainFrame().waitForLoadState(this.progress, 'load');
+ await this.page.mainFrame().waitForLoadState(this._progress, 'load');
return result;
}
@@ -101,15 +98,15 @@ export class Context {
else
promises.push(request.response());
}
- await this.progress.race(promises, { timeout: 5000 });
+ await this._progress.race(promises, { timeout: 5000 });
if (requests.length)
- await this.progress.wait(500);
+ await this._progress.wait(500);
return result;
}
async snapshotResult(error?: Error): Promise<loopTypes.ToolResult> {
- let { full } = await this.page.snapshotForAI(this.progress);
+ let { full } = await this.page.snapshotForAI(this._progress);
full = this._redactText(full);
const text: string[] = [];
@@ -138,7 +135,7 @@ export class Context {
async refSelectors(params: { element: string, ref: string }[]): Promise<string[]> {
return Promise.all(params.map(async param => {
try {
- const { resolvedSelector } = await this.page.mainFrame().resolveSelector(this.progress, `aria-ref=${param.ref}`);
+ const { resolvedSelector } = await this.page.mainFrame().resolveSelector(this._progress, `aria-ref=${param.ref}`);
return resolvedSelector;
} catch (e) {
throw new Error(`Ref ${param.ref} not found in the current page snapshot. Try capturing new snapshot.`);
diff --git a/packages/playwright-core/src/server/progress.ts b/packages/playwright-core/src/server/progress.ts
index e80a1404e8..46900117d8 100644
--- a/packages/playwright-core/src/server/progress.ts
+++ b/packages/playwright-core/src/server/progress.ts
@@ -15,7 +15,7 @@
*/
import { TimeoutError } from './errors';
-import { assert } from '../utils';
+import { assert, monotonicTime } from '../utils';
import { ManualPromise } from '../utils/isomorphic/manualPromise';
import type { Progress } from '@protocol/progress';
@@ -47,10 +47,12 @@ export class ProgressController {
}
async run<T>(task: (progress: Progress) => Promise<T>, timeout?: number): Promise<T> {
+ const deadline = timeout ? monotonicTime() + timeout : 0;
assert(this._state === 'before');
this._state = 'running';
const progress: Progress = {
+ deadline,
log: message => {
if (this._state === 'running')
this.metadata.log.push(message);
@@ -60,10 +62,13 @@ export class ProgressController {
metadata: this.metadata,
race: <T>(promise: Promise<T> | Promise<T>[], options?: { timeout?: number }) => {
const promises = Array.isArray(promise) ? promise : [promise];
- const timerPromise = options?.timeout ? new Promise<void>(f => setTimeout(f, options.timeout)) : null;
+ const mt = monotonicTime();
+ const dl = options?.timeout ? mt + options.timeout : 0;
+ const timerPromise = dl && (!deadline || dl < deadline) ? new Promise<void>(f => setTimeout(f, dl - mt)) : null;
return Promise.race([...promises, ...(timerPromise ? [timerPromise] : []), this._forceAbortPromise]);
},
wait: async (timeout: number) => {
+ // Timeout = 0 here means nowait. Counter to what it typically is (wait forever).
let timer: NodeJS.Timeout;
const promise = new Promise<void>(f => timer = setTimeout(f, timeout));
return progress.race(promise).finally(() => clearTimeout(timer));
@@ -71,7 +76,7 @@ export class ProgressController {
};
let timer: NodeJS.Timeout | undefined;
- if (timeout) {
+ if (deadline) {
const timeoutError = new TimeoutError(`Timeout ${timeout}ms exceeded.`);
timer = setTimeout(() => {
if (this.metadata.pauseStartTime && !this.metadata.pauseEndTime)
@@ -81,7 +86,7 @@ export class ProgressController {
this._state = { error: timeoutError };
this._forceAbortPromise.reject(timeoutError);
}
- }, timeout);
+ }, deadline - monotonicTime());
}
try {
diff --git a/packages/protocol/src/progress.d.ts b/packages/protocol/src/progress.d.ts
index 7f1734627b..96e0a33b97 100644
--- a/packages/protocol/src/progress.d.ts
+++ b/packages/protocol/src/progress.d.ts
@@ -35,6 +35,7 @@ import type { CallMetadata } from './callMetadata';
// should be used to cleanup. Whether the cleanup is awaited or not depends on the method details.
// For the trickiest cases, look at `raceUncancellableOperationWithCleanup()` helper method.
export interface Progress {
+ deadline: number;
log(message: string): void;
race<T>(promise: Promise<T> | Promise<T>[], options?: { timeout?: number }): Promise<T>;
wait(timeout: number): Promise<void>;