Skip to content

Playwright is a framework for Web Testing and Automation. It allows testing Chromium, Firefox and WebKit with a single API.

fix(perform): ensure proper timeouts for expect tools (#38810)

Dmitry Gozman28f2825c8715
diff --git a/packages/injected/src/injectedScript.ts b/packages/injected/src/injectedScript.ts
index 014466f493..ce18bf46aa 100644
--- a/packages/injected/src/injectedScript.ts
+++ b/packages/injected/src/injectedScript.ts
@@ -47,7 +47,12 @@ import type { ElementText, TextMatcher } from './selectorUtils';
import type { Builtins } from './utilityScript';
-export type FrameExpectParams = Omit<channels.FrameExpectParams, 'expectedValue' | 'timeout'> & { expectedValue?: any };
+export type FrameExpectParams = Omit<channels.FrameExpectParams, 'expectedValue' | 'timeout'> & {
+ expectedValue?: any;
+ timeoutForLogs?: number;
+ explicitTimeout?: number;
+ noPreChecks?: boolean;
+};
export type ElementState = 'visible' | 'hidden' | 'enabled' | 'disabled' | 'editable' | 'checked' | 'unchecked' | 'indeterminate' | 'stable';
export type ElementStateWithoutStable = Exclude<ElementState, 'stable'>;
diff --git a/packages/playwright-core/src/server/agent/actionRunner.ts b/packages/playwright-core/src/server/agent/actionRunner.ts
index 0d57a3cc0d..b54ae40454 100644
--- a/packages/playwright-core/src/server/agent/actionRunner.ts
+++ b/packages/playwright-core/src/server/agent/actionRunner.ts
@@ -34,7 +34,7 @@ import type { FrameExpectParams } from '@injected/injectedScript';
export async function runAction(progress: Progress, mode: 'generate' | 'run', page: Page, action: actions.Action, secrets: NameValue[]) {
const parentMetadata = progress.metadata;
const frame = page.mainFrame();
- const callMetadata = callMetadataForAction(progress, frame, action);
+ const callMetadata = callMetadataForAction(progress, frame, action, mode);
callMetadata.log = parentMetadata.log;
progress.metadata = callMetadata;
@@ -52,6 +52,7 @@ export async function runAction(progress: Progress, mode: 'generate' | 'run', pa
async function innerRunAction(progress: Progress, mode: 'generate' | 'run', page: Page, action: actions.Action, secrets: NameValue[]) {
const frame = page.mainFrame();
+ // Disable auto-waiting to avoid timeouts, model has seen the snapshot anyway.
const commonOptions = { strict: true, noAutoWaiting: mode === 'generate' };
switch (action.method) {
case 'navigate':
@@ -101,16 +102,16 @@ async function innerRunAction(progress: Progress, mode: 'generate' | 'run', page
await frame.uncheck(progress, action.selector, { ...commonOptions });
break;
case 'expectVisible': {
- await runExpect(frame, progress, action.selector, { expression: 'to.be.visible', isNot: !!action.isNot }, 'visible', 'toBeVisible', '');
+ await runExpect(frame, progress, mode, action.selector, { expression: 'to.be.visible', isNot: !!action.isNot }, 'visible', 'toBeVisible', '');
break;
}
case 'expectValue': {
if (action.type === 'textbox' || action.type === 'combobox' || action.type === 'slider') {
const expectedText = serializeExpectedTextValues([action.value]);
- await runExpect(frame, progress, action.selector, { expression: 'to.have.value', expectedText, isNot: !!action.isNot }, action.value, 'toHaveValue', 'expected');
+ await runExpect(frame, progress, mode, action.selector, { expression: 'to.have.value', expectedText, isNot: !!action.isNot }, action.value, 'toHaveValue', 'expected');
} else if (action.type === 'checkbox' || action.type === 'radio') {
const expectedValue = { checked: action.value === 'true' };
- await runExpect(frame, progress, action.selector, { selector: action.selector, expression: 'to.be.checked', expectedValue, isNot: !!action.isNot }, action.value ? 'checked' : 'unchecked', 'toBeChecked', '');
+ await runExpect(frame, progress, mode, action.selector, { selector: action.selector, expression: 'to.be.checked', expectedValue, isNot: !!action.isNot }, action.value ? 'checked' : 'unchecked', 'toBeChecked', '');
} else {
throw new Error(`Unsupported element type: ${action.type}`);
}
@@ -118,14 +119,22 @@ async function innerRunAction(progress: Progress, mode: 'generate' | 'run', page
}
case 'expectAria': {
const expectedValue = parseAriaSnapshotUnsafe(yaml, action.template);
- await runExpect(frame, progress, 'body', { expression: 'to.match.aria', expectedValue, isNot: !!action.isNot }, '\n' + action.template, 'toMatchAriaSnapshot', 'expected');
+ await runExpect(frame, progress, mode, 'body', { expression: 'to.match.aria', expectedValue, isNot: !!action.isNot }, '\n' + action.template, 'toMatchAriaSnapshot', 'expected');
break;
}
}
}
-async function runExpect(frame: Frame, progress: Progress, selector: string | undefined, options: FrameExpectParams, expected: string, matcherName: string, expectation: string) {
- const result = await frame.expect(progress, selector, options);
+async function runExpect(frame: Frame, progress: Progress, mode: 'generate' | 'run', selector: string | undefined, options: FrameExpectParams, expected: string, matcherName: string, expectation: string) {
+ // Pass explicit timeout to limit the single expect action inside the overall "agentic expect" multi-step progress.
+ const timeout = expectTimeout(mode);
+ const result = await frame.expect(progress, selector, {
+ ...options,
+ timeoutForLogs: timeout,
+ explicitTimeout: timeout,
+ // Disable pre-checks to avoid them timing out, model has seen the snapshot anyway.
+ noPreChecks: mode === 'generate',
+ });
if (!result.matches === !options.isNot) {
const received = matcherName === 'toMatchAriaSnapshot' ? '\n' + result.received.raw : result.received;
throw new Error(formatMatcherMessage(simpleMatcherUtils, {
@@ -134,7 +143,7 @@ async function runExpect(frame: Frame, progress: Progress, selector: string | un
expectation,
locator: selector ? asLocatorDescription('javascript', selector) : undefined,
timedOut: result.timedOut,
- timeout: progress.timeout,
+ timeout,
printedExpected: options.isNot ? `Expected: not ${expected}` : `Expected: ${expected}`,
printedReceived: result.errorMessage ? '' : `Received: ${received}`,
errorMessage: result.errorMessage,
@@ -144,7 +153,7 @@ async function runExpect(frame: Frame, progress: Progress, selector: string | un
}
}
-export function traceParamsForAction(progress: Progress, action: actions.Action): { title?: string, type: string, method: string, params: any } {
+export function traceParamsForAction(progress: Progress, action: actions.Action, mode: 'generate' | 'run'): { title?: string, type: string, method: string, params: any } {
const timeout = progress.timeout;
switch (action.method) {
case 'navigate': {
@@ -238,7 +247,7 @@ export function traceParamsForAction(progress: Progress, action: actions.Action)
expression: 'to.have.value',
expectedText,
isNot: !!action.isNot,
- timeout: kDefaultTimeout,
+ timeout: expectTimeout(mode),
};
return { type: 'Frame', method: 'expect', title: 'Expect Value', params };
} else if (action.type === 'checkbox' || action.type === 'radio') {
@@ -247,7 +256,7 @@ export function traceParamsForAction(progress: Progress, action: actions.Action)
selector: action.selector,
expression: 'to.be.checked',
isNot: !!action.isNot,
- timeout: kDefaultTimeout,
+ timeout: expectTimeout(mode),
};
return { type: 'Frame', method: 'expect', title: 'Expect Checked', params };
} else {
@@ -259,7 +268,7 @@ export function traceParamsForAction(progress: Progress, action: actions.Action)
selector: action.selector,
expression: 'to.be.visible',
isNot: !!action.isNot,
- timeout: kDefaultTimeout,
+ timeout: expectTimeout(mode),
};
return { type: 'Frame', method: 'expect', title: 'Expect Visible', params };
}
@@ -270,14 +279,14 @@ export function traceParamsForAction(progress: Progress, action: actions.Action)
expression: 'to.match.snapshot',
expectedText: [],
isNot: !!action.isNot,
- timeout: kDefaultTimeout,
+ timeout: expectTimeout(mode),
};
return { type: 'Frame', method: 'expect', title: 'Expect Aria Snapshot', params };
}
}
}
-function callMetadataForAction(progress: Progress, frame: Frame, action: actions.Action): CallMetadata {
+function callMetadataForAction(progress: Progress, frame: Frame, action: actions.Action, mode: 'generate' | 'run'): CallMetadata {
const callMetadata: CallMetadata = {
id: `call@${createGuid()}`,
objectId: frame.guid,
@@ -286,9 +295,11 @@ function callMetadataForAction(progress: Progress, frame: Frame, action: actions
startTime: monotonicTime(),
endTime: 0,
log: [],
- ...traceParamsForAction(progress, action),
+ ...traceParamsForAction(progress, action, mode),
};
return callMetadata;
}
-const kDefaultTimeout = 5000;
+function expectTimeout(mode: 'generate' | 'run') {
+ return mode === 'generate' ? 0 : 5000;
+}
diff --git a/packages/playwright-core/src/server/agent/context.ts b/packages/playwright-core/src/server/agent/context.ts
index 12aebb4f03..64be25de9d 100644
--- a/packages/playwright-core/src/server/agent/context.ts
+++ b/packages/playwright-core/src/server/agent/context.ts
@@ -124,7 +124,7 @@ export class Context {
promises.push(request.response());
}
- await progress.race(promises, { timeout: 5000 });
+ await progress.race([...promises, progress.wait(5000)]);
if (!promises.length)
await progress.wait(500);
diff --git a/packages/playwright-core/src/server/dispatchers/frameDispatcher.ts b/packages/playwright-core/src/server/dispatchers/frameDispatcher.ts
index 6f9e692473..dc64740578 100644
--- a/packages/playwright-core/src/server/dispatchers/frameDispatcher.ts
+++ b/packages/playwright-core/src/server/dispatchers/frameDispatcher.ts
@@ -266,7 +266,7 @@ export class FrameDispatcher extends Dispatcher<Frame, channels.FrameChannel, Br
let expectedValue = params.expectedValue ? parseArgument(params.expectedValue) : undefined;
if (params.expression === 'to.match.aria' && expectedValue)
expectedValue = parseAriaSnapshotUnsafe(yaml, expectedValue);
- const result = await this._frame.expect(progress, params.selector, { ...params, expectedValue }, params.timeout);
+ const result = await this._frame.expect(progress, params.selector, { ...params, expectedValue, timeoutForLogs: params.timeout });
if (result.received !== undefined)
result.received = serializeResult(result.received);
return result;
diff --git a/packages/playwright-core/src/server/frames.ts b/packages/playwright-core/src/server/frames.ts
index 619d4973f2..5431ea6a40 100644
--- a/packages/playwright-core/src/server/frames.ts
+++ b/packages/playwright-core/src/server/frames.ts
@@ -1380,8 +1380,8 @@ export class Frame extends SdkObject<FrameEventMap> {
return await this._retryWithProgressIfNotConnected(progress, selector, { strict: true, performActionPreChecks: true }, handle => progress.race(handle.ariaSnapshot()));
}
- async expect(progress: Progress, selector: string | undefined, options: FrameExpectParams, timeout?: number): Promise<ExpectResult> {
- progress.log(`${renderTitleForCall(progress.metadata)}${timeout ? ` with timeout ${timeout}ms` : ''}`);
+ async expect(progress: Progress, selector: string | undefined, options: FrameExpectParams): Promise<ExpectResult> {
+ progress.log(`${renderTitleForCall(progress.metadata)}${options.timeoutForLogs ? ` with timeout ${options.timeoutForLogs}ms` : ''}`);
const lastIntermediateResult: { received?: any, isSet: boolean, errorMessage?: string } = { isSet: false };
const fixupMetadataError = (result: ExpectResult) => {
// Library mode special case for the expect errors which are return values, not exceptions.
@@ -1389,10 +1389,15 @@ export class Frame extends SdkObject<FrameEventMap> {
progress.metadata.error = { error: { name: 'Expect', message: 'Expect failed' } };
};
try {
+ // When explicit timeout is passed, constrain expect by it, in addition to regular progress abort.
+ const timeoutPromise = options.explicitTimeout !== undefined ? progress.wait(options.explicitTimeout).then(() => { throw new TimeoutError(`Timed out after ${options.explicitTimeout}ms`); }) : undefined;
+ timeoutPromise?.catch(() => { /* Prevent unhandled promise rejection */ });
+
// Step 1: perform locator handlers checkpoint with a specified timeout.
if (selector)
progress.log(`waiting for ${this._asLocator(selector)}`);
- await this._page.performActionPreChecks(progress);
+ if (!options.noPreChecks)
+ await this._page.performActionPreChecks(progress);
// Step 2: perform one-shot expect check without a timeout.
// Supports the case of `expect(locator).toBeVisible({ timeout: 1 })`
@@ -1409,8 +1414,9 @@ export class Frame extends SdkObject<FrameEventMap> {
// Step 3: auto-retry expect with increasing timeouts. Bounded by the total remaining time.
const result = await this.retryWithProgressAndTimeouts(progress, [100, 250, 500, 1000], async continuePolling => {
- await this._page.performActionPreChecks(progress);
- const { matches, received } = await this._expectInternal(progress, selector, options, lastIntermediateResult, false);
+ if (!options.noPreChecks)
+ await this._page.performActionPreChecks(progress);
+ const { matches, received } = await this._expectInternal(progress, selector, options, lastIntermediateResult, false, timeoutPromise);
if (matches === options.isNot) {
// Keep waiting in these cases:
// expect(locator).conditionThatDoesNotMatch
@@ -1440,9 +1446,9 @@ export class Frame extends SdkObject<FrameEventMap> {
}
}
- private async _expectInternal(progress: Progress, selector: string | undefined, options: FrameExpectParams, lastIntermediateResult: { received?: any, isSet: boolean, errorMessage?: string }, noAbort: boolean) {
+ private async _expectInternal(progress: Progress, selector: string | undefined, options: FrameExpectParams, lastIntermediateResult: { received?: any, isSet: boolean, errorMessage?: string }, noAbort: boolean, timeoutPromise?: Promise<never>) {
// The first expect check, a.k.a. one-shot, always finishes - even when progress is aborted.
- const race = <T>(p: Promise<T>) => noAbort ? p : progress.race(p);
+ const race = <T>(p: Promise<T>) => noAbort ? p : (timeoutPromise ? progress.race([p, timeoutPromise]) : progress.race(p));
const selectorInFrame = selector ? await race(this.selectors.resolveFrameForSelector(selector, { strict: true })) : undefined;
const { frame, info } = selectorInFrame || { frame: this, info: undefined };
diff --git a/packages/playwright-core/src/server/progress.ts b/packages/playwright-core/src/server/progress.ts
index 39155d6767..b6cb8201fb 100644
--- a/packages/playwright-core/src/server/progress.ts
+++ b/packages/playwright-core/src/server/progress.ts
@@ -72,14 +72,11 @@ export class ProgressController {
this._onCallLog?.(message);
},
metadata: this.metadata,
- race: <T>(promise: Promise<T> | Promise<T>[], options?: { timeout?: number }) => {
+ race: <T>(promise: Promise<T> | Promise<T>[]) => {
const promises = Array.isArray(promise) ? promise : [promise];
if (!promises.length)
return Promise.resolve();
- const mt = monotonicTime();
- const dl = options?.timeout ? mt + options.timeout : 0;
- const timerPromise = dl && (!deadline || dl < deadline) ? new Promise<void>(f => setTimeout(f, dl - mt)) : null;
- return Promise.race([...promises, ...(timerPromise ? [timerPromise] : []), this._forceAbortPromise]);
+ return Promise.race([...promises, this._forceAbortPromise]);
},
wait: async (timeout: number) => {
// Timeout = 0 here means nowait. Counter to what it typically is (wait forever).
@@ -97,7 +94,6 @@ export class ProgressController {
if (this.metadata.pauseStartTime && !this.metadata.pauseEndTime)
return;
if (this._state === 'running') {
- (timeoutError as any)[kAbortErrorSymbol] = true;
this._state = { error: timeoutError };
this._forceAbortPromise.reject(timeoutError);
this._controller.abort(timeoutError);
@@ -122,7 +118,7 @@ export class ProgressController {
const kAbortErrorSymbol = Symbol('kAbortError');
export function isAbortError(error: Error): boolean {
- return !!(error as any)[kAbortErrorSymbol];
+ return error instanceof TimeoutError || !!(error as any)[kAbortErrorSymbol];
}
// Use this method to race some external operation that you really want to undo
diff --git a/packages/protocol/src/progress.d.ts b/packages/protocol/src/progress.d.ts
index f5748ad421..8e389e0c31 100644
--- a/packages/protocol/src/progress.d.ts
+++ b/packages/protocol/src/progress.d.ts
@@ -38,8 +38,8 @@ export interface Progress {
timeout: number;
deadline: number;
log(message: string): void;
- race<T>(promise: Promise<T> | Promise<T>[], options?: { timeout?: number }): Promise<T>;
- wait(timeout: number): Promise<void>;
+ race<T>(promise: Promise<T> | Promise<T>[]): Promise<T>;
+ wait(timeout: number): Promise<void>; // timeout = 0 here means "wait 0 ms", not forever.
signal: AbortSignal;
metadata: CallMetadata;
}
diff --git a/tests/library/__llm_cache__/library-agent-expect-expect-timeout-during-generate.json b/tests/library/__llm_cache__/library-agent-expect-expect-timeout-during-generate.json
new file mode 100644
index 0000000000..39cf77ebbf
--- /dev/null
+++ b/tests/library/__llm_cache__/library-agent-expect-expect-timeout-during-generate.json
@@ -0,0 +1,135 @@
+{
+ "0e1e6182377f3ddc9e605d47dc0f4e5db456be40": {
+ "result": {
+ "role": "assistant",
+ "content": [
+ {
+ "type": "text",
+ "text": "The assertion has been executed. The text \"hello\" is **not visible** on the page - the assertion failed as expected since the page snapshot only shows \"bye\" text. The tool has verified that \"hello\" cannot be found on the page."
+ }
+ ],
+ "stopReason": {
+ "code": "other",
+ "message": "Unexpected stop reason: end_turn"
+ }
+ },
+ "usage": {
+ "input": 1912,
+ "output": 54
+ }
+ },
+ "b4fd4ea81b4c3d84f354169c7a97dd0f708710d2": {
+ "result": {
+ "role": "assistant",
+ "content": [
+ {
+ "type": "text",
+ "text": "I need to verify that the text \"hello\" is visible on the page. Based on the page snapshot, I can see there's only text \"bye\" visible, not \"hello\". However, I should still make the assertion to check for \"hello\"."
+ },
+ {
+ "type": "tool_call",
+ "name": "browser_expect_visible_text",
+ "arguments": {
+ "text": "hello",
+ "_is_done": true
+ },
+ "id": "toolu_01WePrBxxAHf7CS13nU77ay2"
+ }
+ ],
+ "stopReason": {
+ "code": "ok"
+ }
+ },
+ "usage": {
+ "input": 1694,
+ "output": 129
+ }
+ },
+ "c8a85f5c1a56b73cf3fc1b00e01b45e9a78b94be": {
+ "result": {
+ "role": "assistant",
+ "content": [
+ {
+ "type": "text",
+ "text": "I'll verify that the input has the value \"hello\"."
+ },
+ {
+ "type": "tool_call",
+ "name": "browser_expect_value",
+ "arguments": {
+ "type": "textbox",
+ "element": "input",
+ "ref": "e2",
+ "value": "hello",
+ "_is_done": true
+ },
+ "id": "toolu_011Tt781hyZUbLUUQMVyhdxB"
+ }
+ ],
+ "stopReason": {
+ "code": "ok"
+ }
+ },
+ "usage": {
+ "input": 1695,
+ "output": 140
+ }
+ },
+ "e59b1ef4668b6d7d8d0fbad3fc1a0a3172d6d07b": {
+ "result": {
+ "role": "assistant",
+ "content": [
+ {
+ "type": "text",
+ "text": "The assertion has been executed. The test expectation that text \"hello\" is visible has **failed** because the text \"hello\" was not found on the page. According to the page snapshot, only the text \"bye\" is present in a generic element, not \"hello\"."
+ }
+ ],
+ "stopReason": {
+ "code": "other",
+ "message": "Unexpected stop reason: end_turn"
+ }
+ },
+ "usage": {
+ "input": 1895,
+ "output": 60
+ }
+ },
+ "e94f4d28220373c8f84256201c83f8cf744ff680": {
+ "result": {
+ "role": "assistant",
+ "content": [
+ {
+ "type": "text",
+ "text": "The assertion has been executed. As expected based on the page snapshot showing only \"bye\", the text \"hello\" is **not visible** on the page. The test failed because \"hello\" could not be found."
+ }
+ ],
+ "stopReason": {
+ "code": "other",
+ "message": "Unexpected stop reason: end_turn"
+ }
+ },
+ "usage": {
+ "input": 1930,
+ "output": 47
+ }
+ },
+ "ffe7d66a201de8f2a816d99550859ad43fee1ffb": {
+ "result": {
+ "role": "assistant",
+ "content": [
+ {
+ "type": "text",
+ "text": "The assertion failed as expected - the text \"hello\" is not visible on the page. The page only contains the text \"bye\". The task has been completed by calling the browser_expect_visible_text tool to verify the condition."
+ }
+ ],
+ "stopReason": {
+ "code": "other",
+ "message": "Unexpected stop reason: end_turn"
+ }
+ },
+ "usage": {
+ "input": 1913,
+ "output": 51
+ }
+ }
+}
\ No newline at end of file
diff --git a/tests/library/__llm_cache__/library-agent-expect-expect-timeout-during-run.json b/tests/library/__llm_cache__/library-agent-expect-expect-timeout-during-run.json
new file mode 100644
index 0000000000..67c2d896df
--- /dev/null
+++ b/tests/library/__llm_cache__/library-agent-expect-expect-timeout-during-run.json
@@ -0,0 +1,30 @@
+{
+ "3b4f08f411f9002019b8fab8f14772f4b4beb15a": {
+ "result": {
+ "role": "assistant",
+ "content": [
+ {
+ "type": "text",
+ "text": "I'll verify that the submit button is visible on the page."
+ },
+ {
+ "type": "tool_call",
+ "name": "browser_expect_visible",
+ "arguments": {
+ "role": "button",
+ "accessibleName": "Submit",
+ "_is_done": true
+ },
+ "id": "toolu_01CyQoKismpKpLjmnSHrnn8M"
+ }
+ ],
+ "stopReason": {
+ "code": "ok"
+ }
+ },
+ "usage": {
+ "input": 1694,
+ "output": 107
+ }
+ }
+}
\ No newline at end of file
diff --git a/tests/library/agent-expect.spec.ts b/tests/library/agent-expect.spec.ts
index d141a0d7fb..9dff7ff1af 100644
--- a/tests/library/agent-expect.spec.ts
+++ b/tests/library/agent-expect.spec.ts
@@ -16,7 +16,7 @@
import { stripAnsi } from '../config/utils';
import { browserTest as test, expect } from '../config/browserTest';
-import { setCacheObject, runAgent } from './agent-helpers';
+import { setCacheObject, runAgent, generateAgent, cacheObject } from './agent-helpers';
// LOWIRE_NO_CACHE=1 to generate api caches
// LOWIRE_FORCE_CACHE=1 to force api caches
@@ -33,16 +33,16 @@ test('expectVisible not found error', async ({ context }) => {
});
const { page, agent } = await runAgent(context);
await page.setContent(`<button hidden>Submit</button>`);
- const error = await agent.expect('submit button is visible', { timeout: 1000 }).catch(e => e);
+ const error = await agent.expect('submit button is visible').catch(e => e);
expect(stripAnsi(error.message)).toContain(`pageAgent.expect: expect(locator).toBeVisible() failed
Locator: getByRole('button')
Expected: visible
-Timeout: 1000ms
+Timeout: 5000ms
Error: element(s) not found
Call log:
- - Expect Visible
+ - Expect Visible with timeout 5000ms
- waiting for getByRole('button')`);
});
@@ -58,16 +58,16 @@ test('expectVisible not visible error', async ({ context }) => {
});
const { page, agent } = await runAgent(context);
await page.setContent(`<button hidden>Submit</button>`);
- const error = await agent.expect('submit button is visible', { timeout: 1000 }).catch(e => e);
+ const error = await agent.expect('submit button is visible').catch(e => e);
expect(stripAnsi(error.message)).toContain(`pageAgent.expect: expect(locator).toBeVisible() failed
Locator: locator('button')
Expected: visible
Received: hidden
-Timeout: 1000ms
+Timeout: 5000ms
Call log:
- - Expect Visible
+ - Expect Visible with timeout 5000ms
- waiting for locator('button')`);
});
@@ -84,16 +84,16 @@ test('not expectVisible visible error', async ({ context }) => {
});
const { page, agent } = await runAgent(context);
await page.setContent(`<button>Submit</button>`);
- const error = await agent.expect('submit button is not visible', { timeout: 1000 }).catch(e => e);
+ const error = await agent.expect('submit button is not visible').catch(e => e);
expect(stripAnsi(error.message)).toContain(`pageAgent.expect: expect(locator).not.toBeVisible() failed
Locator: locator('button')
Expected: not visible
Received: visible
-Timeout: 1000ms
+Timeout: 5000ms
Call log:
- - Expect Visible
+ - Expect Visible with timeout 5000ms
- waiting for locator('button')`);
});
@@ -111,16 +111,16 @@ test('expectChecked not checked error', async ({ context }) => {
});
const { page, agent } = await runAgent(context);
await page.setContent(`<input type=checkbox>`);
- const error = await agent.expect('checkbox is checked', { timeout: 1000 }).catch(e => e);
+ const error = await agent.expect('checkbox is checked').catch(e => e);
expect(stripAnsi(error.message)).toContain(`pageAgent.expect: expect(locator).toBeChecked() failed
Locator: locator('input')
Expected: checked
Received: unchecked
-Timeout: 1000ms
+Timeout: 5000ms
Call log:
- - Expect Checked
+ - Expect Checked with timeout 5000ms
- waiting for locator('input')`);
});
@@ -138,16 +138,16 @@ test('expectValue wrong value error', async ({ context }) => {
});
const { page, agent } = await runAgent(context);
await page.setContent(`<input type=text value="world">`);
- const error = await agent.expect('input has value "hello"', { timeout: 1000 }).catch(e => e);
+ const error = await agent.expect('input has value "hello"').catch(e => e);
expect(stripAnsi(error.message)).toContain(`pageAgent.expect: expect(locator).toHaveValue(expected) failed
Locator: locator('input')
Expected: hello
Received: world
-Timeout: 1000ms
+Timeout: 5000ms
Call log:
- - Expect Value
+ - Expect Value with timeout 5000ms
- waiting for locator('input')`);
});
@@ -163,7 +163,7 @@ test('expectAria wrong snapshot error', async ({ context }) => {
});
const { page, agent } = await runAgent(context);
await page.setContent(`<ul><li>one</li><li style="display:none">two</li></ul>`);
- const error = await agent.expect('two items are visible', { timeout: 1000 }).catch(e => e);
+ const error = await agent.expect('two items are visible').catch(e => e);
const errorMessage = `pageAgent.expect: expect(locator).toMatchAriaSnapshot(expected) failed
Locator: locator('body')
@@ -174,10 +174,48 @@ Expected:
Received:
- list:
- listitem: one
-Timeout: 1000ms
+Timeout: 5000ms
Call log:
- - Expect Aria Snapshot
+ - Expect Aria Snapshot with timeout 5000ms
- waiting for locator('body')`.replace('Expected:', 'Expected: ').replace('Received:', 'Received: ');
expect(stripAnsi(error.message)).toContain(errorMessage);
});
+
+test('expect timeout during run', async ({ context }) => {
+ {
+ const { page, agent } = await generateAgent(context);
+ await page.setContent(`<button>Submit</button>`);
+ await agent.expect('submit button is visible');
+ }
+ expect(await cacheObject()).toEqual({
+ 'submit button is visible': {
+ actions: [expect.objectContaining({ method: 'expectVisible' })],
+ },
+ });
+ {
+ const { page, agent } = await runAgent(context);
+ await page.setContent(`<button hidden>Submit</button>`);
+ const error = await agent.expect('submit button is visible', { timeout: 10000 }).catch(e => e);
+ expect(stripAnsi(error.message)).toContain(`pageAgent.expect: expect(locator).toBeVisible() failed
+
+Locator: getByRole('button', { name: 'Submit' })
+Expected: visible
+Timeout: 5000ms
+Error: element(s) not found
+
+Call log:
+ - Expect Visible with timeout 5000ms`);
+ }
+});
+
+test('expect timeout during generate', async ({ context }) => {
+ const { page, agent } = await generateAgent(context, { limits: { maxActionRetries: 0 } });
+ await page.setContent(`<input type=text value="bye">`);
+ const error = await agent.expect('input has value "hello"').catch(e => e);
+ expect(stripAnsi(error.message)).toContain(`pageAgent.expect: Agentic loop failed: Failed to perform action after 0 tool call retries
+Call log:
+ - Expect Value
+ - waiting for getByRole('textbox')`);
+ expect(stripAnsi(error.message)).toContain(`- unexpected value "bye"`);
+});