Playwright is a framework for Web Testing and Automation. It allows testing Chromium, Firefox and WebKit with a single API.
TypeScript automationchromechromiume2e-testingelectronend-to-end-testingfirefoxjavascriptplaywrighttesttest-automationtestingtesting-toolswebwebkit
fix(perform): ensure proper timeouts for expect tools (#38810)
Dmitry Gozman28f2825c8715
diff --git a/packages/injected/src/injectedScript.ts b/packages/injected/src/injectedScript.tsindex 014466f493..ce18bf46aa 100644--- a/packages/injected/src/injectedScript.ts+++ b/packages/injected/src/injectedScript.ts@@ -47,7 +47,12 @@ import type { ElementText, TextMatcher } from './selectorUtils';import type { Builtins } from './utilityScript';-export type FrameExpectParams = Omit<channels.FrameExpectParams, 'expectedValue' | 'timeout'> & { expectedValue?: any };+export type FrameExpectParams = Omit<channels.FrameExpectParams, 'expectedValue' | 'timeout'> & {+ expectedValue?: any;+ timeoutForLogs?: number;+ explicitTimeout?: number;+ noPreChecks?: boolean;+};export type ElementState = 'visible' | 'hidden' | 'enabled' | 'disabled' | 'editable' | 'checked' | 'unchecked' | 'indeterminate' | 'stable';export type ElementStateWithoutStable = Exclude<ElementState, 'stable'>;diff --git a/packages/playwright-core/src/server/agent/actionRunner.ts b/packages/playwright-core/src/server/agent/actionRunner.tsindex 0d57a3cc0d..b54ae40454 100644--- a/packages/playwright-core/src/server/agent/actionRunner.ts+++ b/packages/playwright-core/src/server/agent/actionRunner.ts@@ -34,7 +34,7 @@ import type { FrameExpectParams } from '@injected/injectedScript';export async function runAction(progress: Progress, mode: 'generate' | 'run', page: Page, action: actions.Action, secrets: NameValue[]) {const parentMetadata = progress.metadata;const frame = page.mainFrame();- const callMetadata = callMetadataForAction(progress, frame, action);+ const callMetadata = callMetadataForAction(progress, frame, action, mode);callMetadata.log = parentMetadata.log;progress.metadata = callMetadata;@@ -52,6 +52,7 @@ export async function runAction(progress: Progress, mode: 'generate' | 'run', paasync function innerRunAction(progress: Progress, mode: 'generate' | 'run', page: Page, action: actions.Action, secrets: NameValue[]) {const frame = page.mainFrame();+ // Disable auto-waiting to avoid timeouts, model has seen the snapshot anyway.const commonOptions = { strict: true, noAutoWaiting: mode === 'generate' };switch (action.method) {case 'navigate':@@ -101,16 +102,16 @@ async function innerRunAction(progress: Progress, mode: 'generate' | 'run', pageawait frame.uncheck(progress, action.selector, { ...commonOptions });break;case 'expectVisible': {- await runExpect(frame, progress, action.selector, { expression: 'to.be.visible', isNot: !!action.isNot }, 'visible', 'toBeVisible', '');+ await runExpect(frame, progress, mode, action.selector, { expression: 'to.be.visible', isNot: !!action.isNot }, 'visible', 'toBeVisible', '');break;}case 'expectValue': {if (action.type === 'textbox' || action.type === 'combobox' || action.type === 'slider') {const expectedText = serializeExpectedTextValues([action.value]);- await runExpect(frame, progress, action.selector, { expression: 'to.have.value', expectedText, isNot: !!action.isNot }, action.value, 'toHaveValue', 'expected');+ await runExpect(frame, progress, mode, action.selector, { expression: 'to.have.value', expectedText, isNot: !!action.isNot }, action.value, 'toHaveValue', 'expected');} else if (action.type === 'checkbox' || action.type === 'radio') {const expectedValue = { checked: action.value === 'true' };- await runExpect(frame, progress, action.selector, { selector: action.selector, expression: 'to.be.checked', expectedValue, isNot: !!action.isNot }, action.value ? 'checked' : 'unchecked', 'toBeChecked', '');+ await runExpect(frame, progress, mode, action.selector, { selector: action.selector, expression: 'to.be.checked', expectedValue, isNot: !!action.isNot }, action.value ? 'checked' : 'unchecked', 'toBeChecked', '');} else {throw new Error(`Unsupported element type: ${action.type}`);}@@ -118,14 +119,22 @@ async function innerRunAction(progress: Progress, mode: 'generate' | 'run', page}case 'expectAria': {const expectedValue = parseAriaSnapshotUnsafe(yaml, action.template);- await runExpect(frame, progress, 'body', { expression: 'to.match.aria', expectedValue, isNot: !!action.isNot }, '\n' + action.template, 'toMatchAriaSnapshot', 'expected');+ await runExpect(frame, progress, mode, 'body', { expression: 'to.match.aria', expectedValue, isNot: !!action.isNot }, '\n' + action.template, 'toMatchAriaSnapshot', 'expected');break;}}}-async function runExpect(frame: Frame, progress: Progress, selector: string | undefined, options: FrameExpectParams, expected: string, matcherName: string, expectation: string) {- const result = await frame.expect(progress, selector, options);+async function runExpect(frame: Frame, progress: Progress, mode: 'generate' | 'run', selector: string | undefined, options: FrameExpectParams, expected: string, matcherName: string, expectation: string) {+ // Pass explicit timeout to limit the single expect action inside the overall "agentic expect" multi-step progress.+ const timeout = expectTimeout(mode);+ const result = await frame.expect(progress, selector, {+ ...options,+ timeoutForLogs: timeout,+ explicitTimeout: timeout,+ // Disable pre-checks to avoid them timing out, model has seen the snapshot anyway.+ noPreChecks: mode === 'generate',+ });if (!result.matches === !options.isNot) {const received = matcherName === 'toMatchAriaSnapshot' ? '\n' + result.received.raw : result.received;throw new Error(formatMatcherMessage(simpleMatcherUtils, {@@ -134,7 +143,7 @@ async function runExpect(frame: Frame, progress: Progress, selector: string | unexpectation,locator: selector ? asLocatorDescription('javascript', selector) : undefined,timedOut: result.timedOut,- timeout: progress.timeout,+ timeout,printedExpected: options.isNot ? `Expected: not ${expected}` : `Expected: ${expected}`,printedReceived: result.errorMessage ? '' : `Received: ${received}`,errorMessage: result.errorMessage,@@ -144,7 +153,7 @@ async function runExpect(frame: Frame, progress: Progress, selector: string | un}}-export function traceParamsForAction(progress: Progress, action: actions.Action): { title?: string, type: string, method: string, params: any } {+export function traceParamsForAction(progress: Progress, action: actions.Action, mode: 'generate' | 'run'): { title?: string, type: string, method: string, params: any } {const timeout = progress.timeout;switch (action.method) {case 'navigate': {@@ -238,7 +247,7 @@ export function traceParamsForAction(progress: Progress, action: actions.Action)expression: 'to.have.value',expectedText,isNot: !!action.isNot,- timeout: kDefaultTimeout,+ timeout: expectTimeout(mode),};return { type: 'Frame', method: 'expect', title: 'Expect Value', params };} else if (action.type === 'checkbox' || action.type === 'radio') {@@ -247,7 +256,7 @@ export function traceParamsForAction(progress: Progress, action: actions.Action)selector: action.selector,expression: 'to.be.checked',isNot: !!action.isNot,- timeout: kDefaultTimeout,+ timeout: expectTimeout(mode),};return { type: 'Frame', method: 'expect', title: 'Expect Checked', params };} else {@@ -259,7 +268,7 @@ export function traceParamsForAction(progress: Progress, action: actions.Action)selector: action.selector,expression: 'to.be.visible',isNot: !!action.isNot,- timeout: kDefaultTimeout,+ timeout: expectTimeout(mode),};return { type: 'Frame', method: 'expect', title: 'Expect Visible', params };}@@ -270,14 +279,14 @@ export function traceParamsForAction(progress: Progress, action: actions.Action)expression: 'to.match.snapshot',expectedText: [],isNot: !!action.isNot,- timeout: kDefaultTimeout,+ timeout: expectTimeout(mode),};return { type: 'Frame', method: 'expect', title: 'Expect Aria Snapshot', params };}}}-function callMetadataForAction(progress: Progress, frame: Frame, action: actions.Action): CallMetadata {+function callMetadataForAction(progress: Progress, frame: Frame, action: actions.Action, mode: 'generate' | 'run'): CallMetadata {const callMetadata: CallMetadata = {id: `call@${createGuid()}`,objectId: frame.guid,@@ -286,9 +295,11 @@ function callMetadataForAction(progress: Progress, frame: Frame, action: actionsstartTime: monotonicTime(),endTime: 0,log: [],- ...traceParamsForAction(progress, action),+ ...traceParamsForAction(progress, action, mode),};return callMetadata;}-const kDefaultTimeout = 5000;+function expectTimeout(mode: 'generate' | 'run') {+ return mode === 'generate' ? 0 : 5000;+}diff --git a/packages/playwright-core/src/server/agent/context.ts b/packages/playwright-core/src/server/agent/context.tsindex 12aebb4f03..64be25de9d 100644--- a/packages/playwright-core/src/server/agent/context.ts+++ b/packages/playwright-core/src/server/agent/context.ts@@ -124,7 +124,7 @@ export class Context {promises.push(request.response());}- await progress.race(promises, { timeout: 5000 });+ await progress.race([...promises, progress.wait(5000)]);if (!promises.length)await progress.wait(500);diff --git a/packages/playwright-core/src/server/dispatchers/frameDispatcher.ts b/packages/playwright-core/src/server/dispatchers/frameDispatcher.tsindex 6f9e692473..dc64740578 100644--- a/packages/playwright-core/src/server/dispatchers/frameDispatcher.ts+++ b/packages/playwright-core/src/server/dispatchers/frameDispatcher.ts@@ -266,7 +266,7 @@ export class FrameDispatcher extends Dispatcher<Frame, channels.FrameChannel, Brlet expectedValue = params.expectedValue ? parseArgument(params.expectedValue) : undefined;if (params.expression === 'to.match.aria' && expectedValue)expectedValue = parseAriaSnapshotUnsafe(yaml, expectedValue);- const result = await this._frame.expect(progress, params.selector, { ...params, expectedValue }, params.timeout);+ const result = await this._frame.expect(progress, params.selector, { ...params, expectedValue, timeoutForLogs: params.timeout });if (result.received !== undefined)result.received = serializeResult(result.received);return result;diff --git a/packages/playwright-core/src/server/frames.ts b/packages/playwright-core/src/server/frames.tsindex 619d4973f2..5431ea6a40 100644--- a/packages/playwright-core/src/server/frames.ts+++ b/packages/playwright-core/src/server/frames.ts@@ -1380,8 +1380,8 @@ export class Frame extends SdkObject<FrameEventMap> {return await this._retryWithProgressIfNotConnected(progress, selector, { strict: true, performActionPreChecks: true }, handle => progress.race(handle.ariaSnapshot()));}- async expect(progress: Progress, selector: string | undefined, options: FrameExpectParams, timeout?: number): Promise<ExpectResult> {- progress.log(`${renderTitleForCall(progress.metadata)}${timeout ? ` with timeout ${timeout}ms` : ''}`);+ async expect(progress: Progress, selector: string | undefined, options: FrameExpectParams): Promise<ExpectResult> {+ progress.log(`${renderTitleForCall(progress.metadata)}${options.timeoutForLogs ? ` with timeout ${options.timeoutForLogs}ms` : ''}`);const lastIntermediateResult: { received?: any, isSet: boolean, errorMessage?: string } = { isSet: false };const fixupMetadataError = (result: ExpectResult) => {// Library mode special case for the expect errors which are return values, not exceptions.@@ -1389,10 +1389,15 @@ export class Frame extends SdkObject<FrameEventMap> {progress.metadata.error = { error: { name: 'Expect', message: 'Expect failed' } };};try {+ // When explicit timeout is passed, constrain expect by it, in addition to regular progress abort.+ const timeoutPromise = options.explicitTimeout !== undefined ? progress.wait(options.explicitTimeout).then(() => { throw new TimeoutError(`Timed out after ${options.explicitTimeout}ms`); }) : undefined;+ timeoutPromise?.catch(() => { /* Prevent unhandled promise rejection */ });+// Step 1: perform locator handlers checkpoint with a specified timeout.if (selector)progress.log(`waiting for ${this._asLocator(selector)}`);- await this._page.performActionPreChecks(progress);+ if (!options.noPreChecks)+ await this._page.performActionPreChecks(progress);// Step 2: perform one-shot expect check without a timeout.// Supports the case of `expect(locator).toBeVisible({ timeout: 1 })`@@ -1409,8 +1414,9 @@ export class Frame extends SdkObject<FrameEventMap> {// Step 3: auto-retry expect with increasing timeouts. Bounded by the total remaining time.const result = await this.retryWithProgressAndTimeouts(progress, [100, 250, 500, 1000], async continuePolling => {- await this._page.performActionPreChecks(progress);- const { matches, received } = await this._expectInternal(progress, selector, options, lastIntermediateResult, false);+ if (!options.noPreChecks)+ await this._page.performActionPreChecks(progress);+ const { matches, received } = await this._expectInternal(progress, selector, options, lastIntermediateResult, false, timeoutPromise);if (matches === options.isNot) {// Keep waiting in these cases:// expect(locator).conditionThatDoesNotMatch@@ -1440,9 +1446,9 @@ export class Frame extends SdkObject<FrameEventMap> {}}- private async _expectInternal(progress: Progress, selector: string | undefined, options: FrameExpectParams, lastIntermediateResult: { received?: any, isSet: boolean, errorMessage?: string }, noAbort: boolean) {+ private async _expectInternal(progress: Progress, selector: string | undefined, options: FrameExpectParams, lastIntermediateResult: { received?: any, isSet: boolean, errorMessage?: string }, noAbort: boolean, timeoutPromise?: Promise<never>) {// The first expect check, a.k.a. one-shot, always finishes - even when progress is aborted.- const race = <T>(p: Promise<T>) => noAbort ? p : progress.race(p);+ const race = <T>(p: Promise<T>) => noAbort ? p : (timeoutPromise ? progress.race([p, timeoutPromise]) : progress.race(p));const selectorInFrame = selector ? await race(this.selectors.resolveFrameForSelector(selector, { strict: true })) : undefined;const { frame, info } = selectorInFrame || { frame: this, info: undefined };diff --git a/packages/playwright-core/src/server/progress.ts b/packages/playwright-core/src/server/progress.tsindex 39155d6767..b6cb8201fb 100644--- a/packages/playwright-core/src/server/progress.ts+++ b/packages/playwright-core/src/server/progress.ts@@ -72,14 +72,11 @@ export class ProgressController {this._onCallLog?.(message);},metadata: this.metadata,- race: <T>(promise: Promise<T> | Promise<T>[], options?: { timeout?: number }) => {+ race: <T>(promise: Promise<T> | Promise<T>[]) => {const promises = Array.isArray(promise) ? promise : [promise];if (!promises.length)return Promise.resolve();- const mt = monotonicTime();- const dl = options?.timeout ? mt + options.timeout : 0;- const timerPromise = dl && (!deadline || dl < deadline) ? new Promise<void>(f => setTimeout(f, dl - mt)) : null;- return Promise.race([...promises, ...(timerPromise ? [timerPromise] : []), this._forceAbortPromise]);+ return Promise.race([...promises, this._forceAbortPromise]);},wait: async (timeout: number) => {// Timeout = 0 here means nowait. Counter to what it typically is (wait forever).@@ -97,7 +94,6 @@ export class ProgressController {if (this.metadata.pauseStartTime && !this.metadata.pauseEndTime)return;if (this._state === 'running') {- (timeoutError as any)[kAbortErrorSymbol] = true;this._state = { error: timeoutError };this._forceAbortPromise.reject(timeoutError);this._controller.abort(timeoutError);@@ -122,7 +118,7 @@ export class ProgressController {const kAbortErrorSymbol = Symbol('kAbortError');export function isAbortError(error: Error): boolean {- return !!(error as any)[kAbortErrorSymbol];+ return error instanceof TimeoutError || !!(error as any)[kAbortErrorSymbol];}// Use this method to race some external operation that you really want to undodiff --git a/packages/protocol/src/progress.d.ts b/packages/protocol/src/progress.d.tsindex f5748ad421..8e389e0c31 100644--- a/packages/protocol/src/progress.d.ts+++ b/packages/protocol/src/progress.d.ts@@ -38,8 +38,8 @@ export interface Progress {timeout: number;deadline: number;log(message: string): void;- race<T>(promise: Promise<T> | Promise<T>[], options?: { timeout?: number }): Promise<T>;- wait(timeout: number): Promise<void>;+ race<T>(promise: Promise<T> | Promise<T>[]): Promise<T>;+ wait(timeout: number): Promise<void>; // timeout = 0 here means "wait 0 ms", not forever.signal: AbortSignal;metadata: CallMetadata;}diff --git a/tests/library/__llm_cache__/library-agent-expect-expect-timeout-during-generate.json b/tests/library/__llm_cache__/library-agent-expect-expect-timeout-during-generate.jsonnew file mode 100644index 0000000000..39cf77ebbf--- /dev/null+++ b/tests/library/__llm_cache__/library-agent-expect-expect-timeout-during-generate.json@@ -0,0 +1,135 @@+{+ "0e1e6182377f3ddc9e605d47dc0f4e5db456be40": {+ "result": {+ "role": "assistant",+ "content": [+ {+ "type": "text",+ "text": "The assertion has been executed. The text \"hello\" is **not visible** on the page - the assertion failed as expected since the page snapshot only shows \"bye\" text. The tool has verified that \"hello\" cannot be found on the page."+ }+ ],+ "stopReason": {+ "code": "other",+ "message": "Unexpected stop reason: end_turn"+ }+ },+ "usage": {+ "input": 1912,+ "output": 54+ }+ },+ "b4fd4ea81b4c3d84f354169c7a97dd0f708710d2": {+ "result": {+ "role": "assistant",+ "content": [+ {+ "type": "text",+ "text": "I need to verify that the text \"hello\" is visible on the page. Based on the page snapshot, I can see there's only text \"bye\" visible, not \"hello\". However, I should still make the assertion to check for \"hello\"."+ },+ {+ "type": "tool_call",+ "name": "browser_expect_visible_text",+ "arguments": {+ "text": "hello",+ "_is_done": true+ },+ "id": "toolu_01WePrBxxAHf7CS13nU77ay2"+ }+ ],+ "stopReason": {+ "code": "ok"+ }+ },+ "usage": {+ "input": 1694,+ "output": 129+ }+ },+ "c8a85f5c1a56b73cf3fc1b00e01b45e9a78b94be": {+ "result": {+ "role": "assistant",+ "content": [+ {+ "type": "text",+ "text": "I'll verify that the input has the value \"hello\"."+ },+ {+ "type": "tool_call",+ "name": "browser_expect_value",+ "arguments": {+ "type": "textbox",+ "element": "input",+ "ref": "e2",+ "value": "hello",+ "_is_done": true+ },+ "id": "toolu_011Tt781hyZUbLUUQMVyhdxB"+ }+ ],+ "stopReason": {+ "code": "ok"+ }+ },+ "usage": {+ "input": 1695,+ "output": 140+ }+ },+ "e59b1ef4668b6d7d8d0fbad3fc1a0a3172d6d07b": {+ "result": {+ "role": "assistant",+ "content": [+ {+ "type": "text",+ "text": "The assertion has been executed. The test expectation that text \"hello\" is visible has **failed** because the text \"hello\" was not found on the page. According to the page snapshot, only the text \"bye\" is present in a generic element, not \"hello\"."+ }+ ],+ "stopReason": {+ "code": "other",+ "message": "Unexpected stop reason: end_turn"+ }+ },+ "usage": {+ "input": 1895,+ "output": 60+ }+ },+ "e94f4d28220373c8f84256201c83f8cf744ff680": {+ "result": {+ "role": "assistant",+ "content": [+ {+ "type": "text",+ "text": "The assertion has been executed. As expected based on the page snapshot showing only \"bye\", the text \"hello\" is **not visible** on the page. The test failed because \"hello\" could not be found."+ }+ ],+ "stopReason": {+ "code": "other",+ "message": "Unexpected stop reason: end_turn"+ }+ },+ "usage": {+ "input": 1930,+ "output": 47+ }+ },+ "ffe7d66a201de8f2a816d99550859ad43fee1ffb": {+ "result": {+ "role": "assistant",+ "content": [+ {+ "type": "text",+ "text": "The assertion failed as expected - the text \"hello\" is not visible on the page. The page only contains the text \"bye\". The task has been completed by calling the browser_expect_visible_text tool to verify the condition."+ }+ ],+ "stopReason": {+ "code": "other",+ "message": "Unexpected stop reason: end_turn"+ }+ },+ "usage": {+ "input": 1913,+ "output": 51+ }+ }+}\ No newline at end of filediff --git a/tests/library/__llm_cache__/library-agent-expect-expect-timeout-during-run.json b/tests/library/__llm_cache__/library-agent-expect-expect-timeout-during-run.jsonnew file mode 100644index 0000000000..67c2d896df--- /dev/null+++ b/tests/library/__llm_cache__/library-agent-expect-expect-timeout-during-run.json@@ -0,0 +1,30 @@+{+ "3b4f08f411f9002019b8fab8f14772f4b4beb15a": {+ "result": {+ "role": "assistant",+ "content": [+ {+ "type": "text",+ "text": "I'll verify that the submit button is visible on the page."+ },+ {+ "type": "tool_call",+ "name": "browser_expect_visible",+ "arguments": {+ "role": "button",+ "accessibleName": "Submit",+ "_is_done": true+ },+ "id": "toolu_01CyQoKismpKpLjmnSHrnn8M"+ }+ ],+ "stopReason": {+ "code": "ok"+ }+ },+ "usage": {+ "input": 1694,+ "output": 107+ }+ }+}\ No newline at end of filediff --git a/tests/library/agent-expect.spec.ts b/tests/library/agent-expect.spec.tsindex d141a0d7fb..9dff7ff1af 100644--- a/tests/library/agent-expect.spec.ts+++ b/tests/library/agent-expect.spec.ts@@ -16,7 +16,7 @@import { stripAnsi } from '../config/utils';import { browserTest as test, expect } from '../config/browserTest';-import { setCacheObject, runAgent } from './agent-helpers';+import { setCacheObject, runAgent, generateAgent, cacheObject } from './agent-helpers';// LOWIRE_NO_CACHE=1 to generate api caches// LOWIRE_FORCE_CACHE=1 to force api caches@@ -33,16 +33,16 @@ test('expectVisible not found error', async ({ context }) => {});const { page, agent } = await runAgent(context);await page.setContent(`<button hidden>Submit</button>`);- const error = await agent.expect('submit button is visible', { timeout: 1000 }).catch(e => e);+ const error = await agent.expect('submit button is visible').catch(e => e);expect(stripAnsi(error.message)).toContain(`pageAgent.expect: expect(locator).toBeVisible() failedLocator: getByRole('button')Expected: visible-Timeout: 1000ms+Timeout: 5000msError: element(s) not foundCall log:- - Expect Visible+ - Expect Visible with timeout 5000ms- waiting for getByRole('button')`);});@@ -58,16 +58,16 @@ test('expectVisible not visible error', async ({ context }) => {});const { page, agent } = await runAgent(context);await page.setContent(`<button hidden>Submit</button>`);- const error = await agent.expect('submit button is visible', { timeout: 1000 }).catch(e => e);+ const error = await agent.expect('submit button is visible').catch(e => e);expect(stripAnsi(error.message)).toContain(`pageAgent.expect: expect(locator).toBeVisible() failedLocator: locator('button')Expected: visibleReceived: hidden-Timeout: 1000ms+Timeout: 5000msCall log:- - Expect Visible+ - Expect Visible with timeout 5000ms- waiting for locator('button')`);});@@ -84,16 +84,16 @@ test('not expectVisible visible error', async ({ context }) => {});const { page, agent } = await runAgent(context);await page.setContent(`<button>Submit</button>`);- const error = await agent.expect('submit button is not visible', { timeout: 1000 }).catch(e => e);+ const error = await agent.expect('submit button is not visible').catch(e => e);expect(stripAnsi(error.message)).toContain(`pageAgent.expect: expect(locator).not.toBeVisible() failedLocator: locator('button')Expected: not visibleReceived: visible-Timeout: 1000ms+Timeout: 5000msCall log:- - Expect Visible+ - Expect Visible with timeout 5000ms- waiting for locator('button')`);});@@ -111,16 +111,16 @@ test('expectChecked not checked error', async ({ context }) => {});const { page, agent } = await runAgent(context);await page.setContent(`<input type=checkbox>`);- const error = await agent.expect('checkbox is checked', { timeout: 1000 }).catch(e => e);+ const error = await agent.expect('checkbox is checked').catch(e => e);expect(stripAnsi(error.message)).toContain(`pageAgent.expect: expect(locator).toBeChecked() failedLocator: locator('input')Expected: checkedReceived: unchecked-Timeout: 1000ms+Timeout: 5000msCall log:- - Expect Checked+ - Expect Checked with timeout 5000ms- waiting for locator('input')`);});@@ -138,16 +138,16 @@ test('expectValue wrong value error', async ({ context }) => {});const { page, agent } = await runAgent(context);await page.setContent(`<input type=text value="world">`);- const error = await agent.expect('input has value "hello"', { timeout: 1000 }).catch(e => e);+ const error = await agent.expect('input has value "hello"').catch(e => e);expect(stripAnsi(error.message)).toContain(`pageAgent.expect: expect(locator).toHaveValue(expected) failedLocator: locator('input')Expected: helloReceived: world-Timeout: 1000ms+Timeout: 5000msCall log:- - Expect Value+ - Expect Value with timeout 5000ms- waiting for locator('input')`);});@@ -163,7 +163,7 @@ test('expectAria wrong snapshot error', async ({ context }) => {});const { page, agent } = await runAgent(context);await page.setContent(`<ul><li>one</li><li style="display:none">two</li></ul>`);- const error = await agent.expect('two items are visible', { timeout: 1000 }).catch(e => e);+ const error = await agent.expect('two items are visible').catch(e => e);const errorMessage = `pageAgent.expect: expect(locator).toMatchAriaSnapshot(expected) failedLocator: locator('body')@@ -174,10 +174,48 @@ Expected:Received:- list:- listitem: one-Timeout: 1000ms+Timeout: 5000msCall log:- - Expect Aria Snapshot+ - Expect Aria Snapshot with timeout 5000ms- waiting for locator('body')`.replace('Expected:', 'Expected: ').replace('Received:', 'Received: ');expect(stripAnsi(error.message)).toContain(errorMessage);});++test('expect timeout during run', async ({ context }) => {+ {+ const { page, agent } = await generateAgent(context);+ await page.setContent(`<button>Submit</button>`);+ await agent.expect('submit button is visible');+ }+ expect(await cacheObject()).toEqual({+ 'submit button is visible': {+ actions: [expect.objectContaining({ method: 'expectVisible' })],+ },+ });+ {+ const { page, agent } = await runAgent(context);+ await page.setContent(`<button hidden>Submit</button>`);+ const error = await agent.expect('submit button is visible', { timeout: 10000 }).catch(e => e);+ expect(stripAnsi(error.message)).toContain(`pageAgent.expect: expect(locator).toBeVisible() failed++Locator: getByRole('button', { name: 'Submit' })+Expected: visible+Timeout: 5000ms+Error: element(s) not found++Call log:+ - Expect Visible with timeout 5000ms`);+ }+});++test('expect timeout during generate', async ({ context }) => {+ const { page, agent } = await generateAgent(context, { limits: { maxActionRetries: 0 } });+ await page.setContent(`<input type=text value="bye">`);+ const error = await agent.expect('input has value "hello"').catch(e => e);+ expect(stripAnsi(error.message)).toContain(`pageAgent.expect: Agentic loop failed: Failed to perform action after 0 tool call retries+Call log:+ - Expect Value+ - waiting for getByRole('textbox')`);+ expect(stripAnsi(error.message)).toContain(`- unexpected value "bye"`);+});