Playwright is a framework for Web Testing and Automation. It allows testing Chromium, Firefox and WebKit with a single API.
TypeScript automationchromechromiume2e-testingelectronend-to-end-testingfirefoxjavascriptplaywrighttesttest-automationtestingtesting-toolswebwebkit
chore: page.agent timeouts sorted (#38839)
Dmitry Gozmanc6a7c4fcbdbc
diff --git a/docs/src/api/class-page.md b/docs/src/api/class-page.mdindex 5c2b752197..55399a4d67 100644--- a/docs/src/api/class-page.md+++ b/docs/src/api/class-page.md@@ -722,6 +722,11 @@ Initialize page agent with the llm provider and cache.- `cacheFile` ?<[string]> Cache file to use/generate code for performed actions into. Cache is not used if not specified (default).- `cacheOutFile` ?<[string]> When specified, generated entries are written into the `cacheOutFile` instead of updating the `cacheFile`.+### option: Page.agent.expect+* since: v1.58+- `expect` <[Object]>+ - `timeout` ?<[int]> Default timeout for expect calls in milliseconds, defaults to 5000ms.+### option: Page.agent.limits* since: v1.58- `limits` <[Object]>diff --git a/docs/src/api/class-pageagent.md b/docs/src/api/class-pageagent.mdindex 26242e8c44..0f04e2b04d 100644--- a/docs/src/api/class-pageagent.md+++ b/docs/src/api/class-pageagent.md@@ -35,6 +35,12 @@ await agent.expect('"0 items" to be reported');Expectation to assert.+### option: PageAgent.expect.timeout+* since: v1.58+- `timeout` <[float]>++Expect timeout in milliseconds. Defaults to `5000`. The default value can be changed via `expect.timeout` option in the config, or by specifying the `expect` property of the [`option: Page.agent.expect`] option. Pass `0` to disable timeout.+### option: PageAgent.expect.-inline- = %%-page-agent-call-options-v1.58-%%* since: v1.58@@ -68,6 +74,13 @@ Task to perform using agentic loop.* since: v1.58- `schema` <[z.ZodSchema]>+### option: PageAgent.extract.timeout+* since: v1.58+- `timeout` <[float]>++Extract timeout in milliseconds. Defaults to `5000`. The default value can be changed via `actionTimeout` option in the config, or by using the [`method: BrowserContext.setDefaultTimeout`] or+[`method: Page.setDefaultTimeout`] methods. Pass `0` to disable timeout.+### option: PageAgent.extract.-inline- = %%-page-agent-call-options-v1.58-%%* since: v1.58@@ -94,6 +107,13 @@ await agent.perform('Click submit button');Task to perform using agentic loop.+### option: PageAgent.perform.timeout+* since: v1.58+- `timeout` <[float]>++Perform timeout in milliseconds. Defaults to `5000`. The default value can be changed via `actionTimeout` option in the config, or by using the [`method: BrowserContext.setDefaultTimeout`] or+[`method: Page.setDefaultTimeout`] methods. Pass `0` to disable timeout.+### option: PageAgent.perform.-inline- = %%-page-agent-call-options-v1.58-%%* since: v1.58diff --git a/docs/src/api/params.md b/docs/src/api/params.mdindex 9b7bf8f3d4..69257b93db 100644--- a/docs/src/api/params.md+++ b/docs/src/api/params.md@@ -396,18 +396,11 @@ Maximum number of agentic actions to generate, defaults to context-wide value spMaximum number of retries when generating each action, defaults to context-wide value specified in `agent` property.-## page-agent-timeout-* since: v1.58-- `timeout` <[int]>--Request timeout in milliseconds. Defaults to action timeout. Pass `0` to disable timeout.-## page-agent-call-options-v1.58- %%-page-agent-cache-key-%%- %%-page-agent-max-tokens-%%- %%-page-agent-max-actions-%%- %%-page-agent-max-action-retries-%%-- %%-page-agent-timeout-%%## fetch-param-url- `url` <[string]>diff --git a/packages/injected/src/injectedScript.ts b/packages/injected/src/injectedScript.tsindex ce18bf46aa..2be0082e4d 100644--- a/packages/injected/src/injectedScript.ts+++ b/packages/injected/src/injectedScript.ts@@ -50,8 +50,7 @@ import type { Builtins } from './utilityScript';export type FrameExpectParams = Omit<channels.FrameExpectParams, 'expectedValue' | 'timeout'> & {expectedValue?: any;timeoutForLogs?: number;- explicitTimeout?: number;- noPreChecks?: boolean;+ noAutoWaiting?: boolean;};export type ElementState = 'visible' | 'hidden' | 'enabled' | 'disabled' | 'editable' | 'checked' | 'unchecked' | 'indeterminate' | 'stable';diff --git a/packages/playwright-client/types/types.d.ts b/packages/playwright-client/types/types.d.tsindex 6e2a58a63d..50dd490708 100644--- a/packages/playwright-client/types/types.d.ts+++ b/packages/playwright-client/types/types.d.ts@@ -2112,6 +2112,13 @@ export interface Page {cacheOutFile?: string;};+ expect?: {+ /**+ * Default timeout for expect calls in milliseconds, defaults to 5000ms.+ */+ timeout?: number;+ };+/*** Limits to use for the agentic loop.*/@@ -5440,7 +5447,10 @@ export interface PageAgent {maxTokens?: number;/**- * Request timeout in milliseconds. Defaults to action timeout. Pass `0` to disable timeout.+ * Expect timeout in milliseconds. Defaults to `5000`. The default value can be changed via `expect.timeout` option in+ * the config, or by specifying the `expect` property of the+ * [`expect`](https://playwright.dev/docs/api/class-page#page-agent-option-expect) option. Pass `0` to disable+ * timeout.*/timeout?: number;}): Promise<void>;@@ -5482,7 +5492,11 @@ export interface PageAgent {maxTokens?: number;/**- * Request timeout in milliseconds. Defaults to action timeout. Pass `0` to disable timeout.+ * Perform timeout in milliseconds. Defaults to `5000`. The default value can be changed via `actionTimeout` option in+ * the config, or by using the+ * [browserContext.setDefaultTimeout(timeout)](https://playwright.dev/docs/api/class-browsercontext#browser-context-set-default-timeout)+ * or [page.setDefaultTimeout(timeout)](https://playwright.dev/docs/api/class-page#page-set-default-timeout) methods.+ * Pass `0` to disable timeout.*/timeout?: number;}): Promise<{diff --git a/packages/playwright-core/src/client/page.ts b/packages/playwright-core/src/client/page.tsindex e7c027f2a8..adc7654de9 100644--- a/packages/playwright-core/src/client/page.ts+++ b/packages/playwright-core/src/client/page.ts@@ -865,7 +865,9 @@ export class Page extends ChannelOwner<channels.PageChannel> implements api.PagesystemPrompt: options.systemPrompt,};const { agent } = await this._channel.agent(params);- return PageAgent.from(agent);+ const pageAgent = PageAgent.from(agent);+ pageAgent._expectTimeout = options?.expect?.timeout;+ return pageAgent;}async _snapshotForAI(options: TimeoutOptions & { track?: string } = {}): Promise<{ full: string, incremental?: string }> {diff --git a/packages/playwright-core/src/client/pageAgent.ts b/packages/playwright-core/src/client/pageAgent.tsindex 2837f5bbb1..ec6a16cc9a 100644--- a/packages/playwright-core/src/client/pageAgent.ts+++ b/packages/playwright-core/src/client/pageAgent.ts@@ -22,14 +22,9 @@ import { Page } from './page';import type * as api from '../../types/types';import type * as channels from '@protocol/channels';-type PageAgentOptions = {- maxTokens?: number;- maxTurns?: number;- cacheKey?: string;-};-export class PageAgent extends ChannelOwner<channels.PageAgentChannel> implements api.PageAgent {private _page: Page;+ _expectTimeout?: number;static from(channel: channels.PageAgentChannel): PageAgent {return (channel as any)._object;@@ -41,17 +36,20 @@ export class PageAgent extends ChannelOwner<channels.PageAgentChannel> implementthis._channel.on('turn', params => this.emit(Events.Page.AgentTurn, params));}- async expect(expectation: string, options: PageAgentOptions = {}) {- await this._channel.expect({ expectation, ...options });+ async expect(expectation: string, options: channels.PageAgentExpectOptions = {}) {+ const timeout = options.timeout ?? this._expectTimeout ?? 5000;+ await this._channel.expect({ expectation, ...options, timeout });}- async perform(task: string, options: PageAgentOptions = {}) {- const { usage } = await this._channel.perform({ task, ...options });+ async perform(task: string, options: channels.PageAgentPerformOptions = {}) {+ const timeout = this._page._timeoutSettings.timeout(options);+ const { usage } = await this._channel.perform({ task, ...options, timeout });return { usage };}- async extract<Schema extends any>(query: string, schema: Schema, options: PageAgentOptions = {}): Promise<{ result: any, usage: channels.AgentUsage }> {- const { result, usage } = await this._channel.extract({ query, schema: this._page._platform.zodToJsonSchema(schema), ...options });+ async extract<Schema extends any>(query: string, schema: Schema, options: channels.PageAgentExtractOptions = {}): Promise<{ result: any, usage: channels.AgentUsage }> {+ const timeout = this._page._timeoutSettings.timeout(options);+ const { result, usage } = await this._channel.extract({ query, schema: this._page._platform.zodToJsonSchema(schema), ...options, timeout });return { result, usage };}diff --git a/packages/playwright-core/src/server/agent/actionRunner.ts b/packages/playwright-core/src/server/agent/actionRunner.tsindex 9f25d3fec2..9773c29b55 100644--- a/packages/playwright-core/src/server/agent/actionRunner.ts+++ b/packages/playwright-core/src/server/agent/actionRunner.ts@@ -143,14 +143,11 @@ async function innerRunAction(progress: Progress, mode: 'generate' | 'run', page}async function runExpect(frame: Frame, progress: Progress, mode: 'generate' | 'run', selector: string | undefined, options: FrameExpectParams, expected: string | RegExp, matcherName: string, expectation: string) {- // Pass explicit timeout to limit the single expect action inside the overall "agentic expect" multi-step progress.- const timeout = expectTimeout(mode);const result = await frame.expect(progress, selector, {...options,- timeoutForLogs: timeout,- explicitTimeout: timeout,- // Disable pre-checks to avoid them timing out, model has seen the snapshot anyway.- noPreChecks: mode === 'generate',+ // When generating, we want the expect to pass or fail immediately and give feedback to the model.+ noAutoWaiting: mode === 'generate',+ timeoutForLogs: mode === 'generate' ? undefined : progress.timeout,});if (!result.matches === !options.isNot) {const received = matcherName === 'toMatchAriaSnapshot' ? '\n' + result.received.raw : result.received;@@ -162,7 +159,7 @@ async function runExpect(frame: Frame, progress: Progress, mode: 'generate' | 'rexpectation,locator: selector ? asLocatorDescription('javascript', selector) : undefined,timedOut: result.timedOut,- timeout,+ timeout: mode === 'generate' ? undefined : progress.timeout,printedExpected: options.isNot ? `Expected${expectedSuffix}: not ${expectedDisplay}` : `Expected${expectedSuffix}: ${expectedDisplay}`,printedReceived: result.errorMessage ? '' : `Received: ${received}`,errorMessage: result.errorMessage,@@ -266,7 +263,7 @@ export function traceParamsForAction(progress: Progress, action: actions.Action,expression: 'to.have.value',expectedText,isNot: !!action.isNot,- timeout: expectTimeout(mode),+ timeout,};return { type: 'Frame', method: 'expect', title: 'Expect Value', params };} else if (action.type === 'checkbox' || action.type === 'radio') {@@ -275,7 +272,7 @@ export function traceParamsForAction(progress: Progress, action: actions.Action,selector: action.selector,expression: 'to.be.checked',isNot: !!action.isNot,- timeout: expectTimeout(mode),+ timeout,};return { type: 'Frame', method: 'expect', title: 'Expect Checked', params };} else {@@ -287,7 +284,7 @@ export function traceParamsForAction(progress: Progress, action: actions.Action,selector: action.selector,expression: 'to.be.visible',isNot: !!action.isNot,- timeout: expectTimeout(mode),+ timeout,};return { type: 'Frame', method: 'expect', title: 'Expect Visible', params };}@@ -298,7 +295,7 @@ export function traceParamsForAction(progress: Progress, action: actions.Action,expression: 'to.match.snapshot',expectedText: [],isNot: !!action.isNot,- timeout: expectTimeout(mode),+ timeout,};return { type: 'Frame', method: 'expect', title: 'Expect Aria Snapshot', params };}@@ -310,7 +307,7 @@ export function traceParamsForAction(progress: Progress, action: actions.Action,expression: 'to.have.url',expectedText,isNot: !!action.isNot,- timeout: expectTimeout(mode),+ timeout,};return { type: 'Frame', method: 'expect', title: 'Expect URL', params };}@@ -321,7 +318,7 @@ export function traceParamsForAction(progress: Progress, action: actions.Action,expression: 'to.have.title',expectedText,isNot: !!action.isNot,- timeout: expectTimeout(mode),+ timeout,};return { type: 'Frame', method: 'expect', title: 'Expect Title', params };}@@ -341,7 +338,3 @@ function callMetadataForAction(progress: Progress, frame: Frame, action: actions};return callMetadata;}--function expectTimeout(mode: 'generate' | 'run') {- return mode === 'generate' ? 0 : 5000;-}diff --git a/packages/playwright-core/src/server/agent/pageAgent.ts b/packages/playwright-core/src/server/agent/pageAgent.tsindex 120f611390..7b22731aa5 100644--- a/packages/playwright-core/src/server/agent/pageAgent.ts+++ b/packages/playwright-core/src/server/agent/pageAgent.ts@@ -50,7 +50,7 @@ export async function pageAgentPerform(progress: Progress, context: Context, use### Task${userTask}`;-+ progress.disableTimeout();await runLoop(progress, context, performTools, task, undefined, callParams);await updateCache(context, cacheKey);}@@ -68,7 +68,7 @@ export async function pageAgentExpect(progress: Progress, context: Context, expe### Expectation${expectation}`;-+ progress.disableTimeout();await runLoop(progress, context, expectTools, task, undefined, callParams);await updateCache(context, cacheKey);}@@ -101,7 +101,7 @@ async function runLoop(progress: Progress, context: Context, toolDefinitions: Toconst apiCacheTextBefore = context.agentParams.apiCacheFile ?await fs.promises.readFile(context.agentParams.apiCacheFile, 'utf-8').catch(() => '{}') : '{}';- const apiCacheBefore = JSON.parse(apiCacheTextBefore);+ const apiCacheBefore = JSON.parse(apiCacheTextBefore || '{}');const loop = new Loop({api: context.agentParams.api as any,diff --git a/packages/playwright-core/src/server/frames.ts b/packages/playwright-core/src/server/frames.tsindex 5431ea6a40..a09b41d050 100644--- a/packages/playwright-core/src/server/frames.ts+++ b/packages/playwright-core/src/server/frames.ts@@ -1389,14 +1389,10 @@ export class Frame extends SdkObject<FrameEventMap> {progress.metadata.error = { error: { name: 'Expect', message: 'Expect failed' } };};try {- // When explicit timeout is passed, constrain expect by it, in addition to regular progress abort.- const timeoutPromise = options.explicitTimeout !== undefined ? progress.wait(options.explicitTimeout).then(() => { throw new TimeoutError(`Timed out after ${options.explicitTimeout}ms`); }) : undefined;- timeoutPromise?.catch(() => { /* Prevent unhandled promise rejection */ });-// Step 1: perform locator handlers checkpoint with a specified timeout.if (selector)progress.log(`waiting for ${this._asLocator(selector)}`);- if (!options.noPreChecks)+ if (!options.noAutoWaiting)await this._page.performActionPreChecks(progress);// Step 2: perform one-shot expect check without a timeout.@@ -1404,19 +1400,19 @@ export class Frame extends SdkObject<FrameEventMap> {// that should succeed when the locator is already visible.try {const resultOneShot = await this._expectInternal(progress, selector, options, lastIntermediateResult, true);- if (resultOneShot.matches !== options.isNot)+ if (options.noAutoWaiting || resultOneShot.matches !== options.isNot)return resultOneShot;} catch (e) {- if (this.isNonRetriableError(e))+ if (options.noAutoWaiting || this.isNonRetriableError(e))throw e;// Ignore any other errors from one-shot, we'll handle them during retries.}// Step 3: auto-retry expect with increasing timeouts. Bounded by the total remaining time.const result = await this.retryWithProgressAndTimeouts(progress, [100, 250, 500, 1000], async continuePolling => {- if (!options.noPreChecks)+ if (!options.noAutoWaiting)await this._page.performActionPreChecks(progress);- const { matches, received } = await this._expectInternal(progress, selector, options, lastIntermediateResult, false, timeoutPromise);+ const { matches, received } = await this._expectInternal(progress, selector, options, lastIntermediateResult, false);if (matches === options.isNot) {// Keep waiting in these cases:// expect(locator).conditionThatDoesNotMatch@@ -1446,9 +1442,9 @@ export class Frame extends SdkObject<FrameEventMap> {}}- private async _expectInternal(progress: Progress, selector: string | undefined, options: FrameExpectParams, lastIntermediateResult: { received?: any, isSet: boolean, errorMessage?: string }, noAbort: boolean, timeoutPromise?: Promise<never>) {+ private async _expectInternal(progress: Progress, selector: string | undefined, options: FrameExpectParams, lastIntermediateResult: { received?: any, isSet: boolean, errorMessage?: string }, noAbort: boolean) {// The first expect check, a.k.a. one-shot, always finishes - even when progress is aborted.- const race = <T>(p: Promise<T>) => noAbort ? p : (timeoutPromise ? progress.race([p, timeoutPromise]) : progress.race(p));+ const race = <T>(p: Promise<T>) => noAbort ? p : progress.race(p);const selectorInFrame = selector ? await race(this.selectors.resolveFrameForSelector(selector, { strict: true })) : undefined;const { frame, info } = selectorInFrame || { frame: this, info: undefined };diff --git a/packages/playwright-core/src/server/progress.ts b/packages/playwright-core/src/server/progress.tsindex b6cb8201fb..07b580bff6 100644--- a/packages/playwright-core/src/server/progress.ts+++ b/packages/playwright-core/src/server/progress.ts@@ -61,10 +61,14 @@ export class ProgressController {const deadline = timeout ? monotonicTime() + timeout : 0;assert(this._state === 'before');this._state = 'running';+ let timer: NodeJS.Timeout | undefined;const progress: Progress = {timeout: timeout ?? 0,deadline,+ disableTimeout: () => {+ clearTimeout(timer);+ },log: message => {if (this._state === 'running')this.metadata.log.push(message);@@ -87,10 +91,10 @@ export class ProgressController {signal: this._controller.signal,};- let timer: NodeJS.Timeout | undefined;if (deadline) {const timeoutError = new TimeoutError(`Timeout ${timeout}ms exceeded.`);timer = setTimeout(() => {+ // TODO: migrate this to "progress.disableTimeout()".if (this.metadata.pauseStartTime && !this.metadata.pauseEndTime)return;if (this._state === 'running') {diff --git a/packages/playwright-core/types/types.d.ts b/packages/playwright-core/types/types.d.tsindex 6e2a58a63d..50dd490708 100644--- a/packages/playwright-core/types/types.d.ts+++ b/packages/playwright-core/types/types.d.ts@@ -2112,6 +2112,13 @@ export interface Page {cacheOutFile?: string;};+ expect?: {+ /**+ * Default timeout for expect calls in milliseconds, defaults to 5000ms.+ */+ timeout?: number;+ };+/*** Limits to use for the agentic loop.*/@@ -5440,7 +5447,10 @@ export interface PageAgent {maxTokens?: number;/**- * Request timeout in milliseconds. Defaults to action timeout. Pass `0` to disable timeout.+ * Expect timeout in milliseconds. Defaults to `5000`. The default value can be changed via `expect.timeout` option in+ * the config, or by specifying the `expect` property of the+ * [`expect`](https://playwright.dev/docs/api/class-page#page-agent-option-expect) option. Pass `0` to disable+ * timeout.*/timeout?: number;}): Promise<void>;@@ -5482,7 +5492,11 @@ export interface PageAgent {maxTokens?: number;/**- * Request timeout in milliseconds. Defaults to action timeout. Pass `0` to disable timeout.+ * Perform timeout in milliseconds. Defaults to `5000`. The default value can be changed via `actionTimeout` option in+ * the config, or by using the+ * [browserContext.setDefaultTimeout(timeout)](https://playwright.dev/docs/api/class-browsercontext#browser-context-set-default-timeout)+ * or [page.setDefaultTimeout(timeout)](https://playwright.dev/docs/api/class-page#page-set-default-timeout) methods.+ * Pass `0` to disable timeout.*/timeout?: number;}): Promise<{diff --git a/packages/playwright/src/index.ts b/packages/playwright/src/index.tsindex 82b978fb8c..874ba50ef2 100644--- a/packages/playwright/src/index.ts+++ b/packages/playwright/src/index.ts@@ -477,6 +477,9 @@ const playwrightFixtures: Fixtures<TestFixtures, WorkerFixtures> = ({limits: agentOptions?.limits,secrets: agentOptions?.secrets,systemPrompt: agentOptions?.systemPrompt,+ expect: {+ timeout: testInfoImpl._projectInternal.expect?.timeout,+ },});await use(agent);diff --git a/packages/protocol/src/progress.d.ts b/packages/protocol/src/progress.d.tsindex 8e389e0c31..f7abd096ac 100644--- a/packages/protocol/src/progress.d.ts+++ b/packages/protocol/src/progress.d.ts@@ -37,6 +37,7 @@ import type { CallMetadata } from './callMetadata';export interface Progress {timeout: number;deadline: number;+ disableTimeout(): void;log(message: string): void;race<T>(promise: Promise<T> | Promise<T>[]): Promise<T>;wait(timeout: number): Promise<void>; // timeout = 0 here means "wait 0 ms", not forever.diff --git a/tests/library/__llm_cache__/library-agent-expect-expect-timeout-during-run-from-agent-options.json b/tests/library/__llm_cache__/library-agent-expect-expect-timeout-during-run-from-agent-options.jsonnew file mode 100644index 0000000000..1109e7715d--- /dev/null+++ b/tests/library/__llm_cache__/library-agent-expect-expect-timeout-during-run-from-agent-options.json@@ -0,0 +1,30 @@+{+ "44bfbaa6fe0cb038adf37547ecb000d4f39e243f": {+ "result": {+ "role": "assistant",+ "content": [+ {+ "type": "text",+ "text": "I'll verify that the submit button is visible on the page."+ },+ {+ "type": "tool_call",+ "name": "browser_expect_visible",+ "arguments": {+ "role": "button",+ "accessibleName": "Submit",+ "_is_done": true+ },+ "id": "toolu_012RgUS8buCjrQRgt4w2sYFf"+ }+ ],+ "stopReason": {+ "code": "ok"+ }+ },+ "usage": {+ "input": 2203,+ "output": 107+ }+ }+}\ No newline at end of filediff --git a/tests/library/__llm_cache__/library-agent-expect-expectTitle-success.json b/tests/library/__llm_cache__/library-agent-expect-expectTitle-success.jsonindex bb23b43b9d..0f0ff37ab6 100644--- a/tests/library/__llm_cache__/library-agent-expect-expectTitle-success.json+++ b/tests/library/__llm_cache__/library-agent-expect-expectTitle-success.json@@ -5,7 +5,7 @@"content": [{"type": "text",- "text": "I need to verify that the page title is \"My Page Title\". I'll use the browser_expect_title tool to assert this condition."+ "text": "I need to verify that the page title is \"My Page Title\". I'll use the browser_expect_title tool to check this."},{"type": "tool_call",@@ -14,7 +14,7 @@"title": "My Page Title","_is_done": true},- "id": "toolu_01VvWdf4VVFNQuDaEtTsAu3K"+ "id": "toolu_01R1QyAwfxk2KSz3tsAx9RCA"}],"stopReason": {@@ -23,7 +23,7 @@},"usage": {"input": 2195,- "output": 106+ "output": 105}}}\ No newline at end of filediff --git a/tests/library/__llm_cache__/library-agent-expect-expectTitle-wrong-title-error.json b/tests/library/__llm_cache__/library-agent-expect-expectTitle-wrong-title-error.jsonindex e14e18a7d2..a4ae141362 100644--- a/tests/library/__llm_cache__/library-agent-expect-expectTitle-wrong-title-error.json+++ b/tests/library/__llm_cache__/library-agent-expect-expectTitle-wrong-title-error.json@@ -5,7 +5,7 @@"content": [{"type": "text",- "text": "I'll verify that the page title is \"Other Title\" using the browser_expect_title tool."+ "text": "I need to verify that the page title is \"Other Title\" using one of the browser_expect_* tools."},{"type": "tool_call",@@ -14,7 +14,7 @@"title": "Other Title","_is_done": true},- "id": "toolu_017UT5LEe8JRoB53R1YGKrYD"+ "id": "toolu_017ZvXqNbcab9Uyjzw3VuByK"}],"stopReason": {@@ -23,7 +23,7 @@},"usage": {"input": 2194,- "output": 97+ "output": 100}}}\ No newline at end of filediff --git a/tests/library/__llm_cache__/library-agent-expect-expectURL-success.json b/tests/library/__llm_cache__/library-agent-expect-expectURL-success.jsonindex c3f3b6c225..56fadb43dc 100644--- a/tests/library/__llm_cache__/library-agent-expect-expectURL-success.json+++ b/tests/library/__llm_cache__/library-agent-expect-expectURL-success.json@@ -5,7 +5,7 @@"content": [{"type": "text",- "text": "I'll verify that the page URL is /counter.html using the browser_expect_url tool."+ "text": "I'll verify that the page URL is /counter.html."},{"type": "tool_call",@@ -14,7 +14,7 @@"url": "/counter.html","_is_done": true},- "id": "toolu_017jkU3FcrQpAVJgtjrzYovK"+ "id": "toolu_01PkjsZaM5eEGtjLtMixyJUk"}],"stopReason": {@@ -23,25 +23,25 @@},"usage": {"input": 2240,- "output": 99+ "output": 91}},- "f6d9bc80fd5bad48f6a886d7ec77c8d5889521b1": {+ "71a7a4f51b3994a200f239eda9b505454825a79f": {"result": {"role": "assistant","content": [{"type": "text",- "text": "I see the issue - the actual URL includes a server prefix. Let me use a regex pattern to match the URL ending with /counter.html."+ "text": "I need to use a regex pattern to match the URL since it includes a server prefix. Let me update the expectation to match the actual URL pattern."},{"type": "tool_call","name": "browser_expect_url","arguments": {- "regex": "/counter\\.html$/",+ "regex": "/\\/counter\\.html$/","_is_done": true},- "id": "toolu_01JQuC1CJR7W6LiwAnvjaiDX"+ "id": "toolu_01Wwq7Pbe2DthqpmskgGfCzk"}],"stopReason": {@@ -49,8 +49,8 @@}},"usage": {- "input": 2441,- "output": 109+ "input": 2433,+ "output": 111}}}\ No newline at end of filediff --git a/tests/library/__llm_cache__/library-agent-expect-expectURL-with-regex-error.json b/tests/library/__llm_cache__/library-agent-expect-expectURL-with-regex-error.jsonindex f034545e13..3f76a92de9 100644--- a/tests/library/__llm_cache__/library-agent-expect-expectURL-with-regex-error.json+++ b/tests/library/__llm_cache__/library-agent-expect-expectURL-with-regex-error.json@@ -5,7 +5,7 @@"content": [{"type": "text",- "text": "I'll verify that the page URL matches the `/counter` pattern using the browser_expect_url tool with a regex pattern."+ "text": "I'll verify that the page URL matches the /counter pattern using the browser_expect_url tool with a regex pattern."},{"type": "tool_call",@@ -14,7 +14,7 @@"regex": "/counter/","_is_done": true},- "id": "toolu_01J38E1DkiNA7JsvQtGx3otW"+ "id": "toolu_01JxsK661QmxpScYeTMCfWTZ"}],"stopReason": {@@ -23,7 +23,7 @@},"usage": {"input": 2239,- "output": 103+ "output": 102}}}\ No newline at end of filediff --git a/tests/library/__llm_cache__/library-agent-expect-expectURL-with-regex.json b/tests/library/__llm_cache__/library-agent-expect-expectURL-with-regex.jsonindex 19a6a08b33..c8ae66692e 100644--- a/tests/library/__llm_cache__/library-agent-expect-expectURL-with-regex.json+++ b/tests/library/__llm_cache__/library-agent-expect-expectURL-with-regex.json@@ -14,7 +14,7 @@"regex": "/counter/","_is_done": true},- "id": "toolu_01Qen21mVwEVb2JsEVavTWbf"+ "id": "toolu_01YVkbby1j3FBXk2o5Bi4cjP"}],"stopReason": {diff --git a/tests/library/__llm_cache__/library-agent-expect-expectURL-wrong-URL-error.json b/tests/library/__llm_cache__/library-agent-expect-expectURL-wrong-URL-error.jsonindex 43b309b180..b88a2b2d35 100644--- a/tests/library/__llm_cache__/library-agent-expect-expectURL-wrong-URL-error.json+++ b/tests/library/__llm_cache__/library-agent-expect-expectURL-wrong-URL-error.json@@ -14,7 +14,7 @@"url": "/counter.html","_is_done": true},- "id": "toolu_01B3LSMGJoDrDfCZKwa3neZ9"+ "id": "toolu_01BrgLhGzD2dYWeu75KGgxKv"}],"stopReason": {@@ -26,22 +26,22 @@"output": 99}},- "99f45f7c71e64f612bfc0bcb148faf235ccfe006": {+ "8315bc81a0dfc915cb8be72ad00102f627e370dd": {"result": {"role": "assistant","content": [{"type": "text",- "text": "I see the issue. The actual URL includes the server prefix. Let me use a regex pattern to match the URL ending with /counter.html."+ "text": "I see the issue. The actual URL includes the server portion. Let me use a regex pattern to match the URL ending with /counter.html."},{"type": "tool_call","name": "browser_expect_url","arguments": {- "regex": "/counter\\.html$/",+ "regex": "/\\/counter\\.html$/","_is_done": true},- "id": "toolu_01Ufux7hKwV9Djp9jT1JgYwJ"+ "id": "toolu_01KtWVz2EdJjx2V9viYeqs71"}],"stopReason": {@@ -50,7 +50,7 @@},"usage": {"input": 2441,- "output": 109+ "output": 110}}}\ No newline at end of filediff --git a/tests/library/__llm_cache__/library-agent-perform-perform-run-timeout-inherited-from-page.json b/tests/library/__llm_cache__/library-agent-perform-perform-run-timeout-inherited-from-page.jsonnew file mode 100644index 0000000000..1cd1ae7633--- /dev/null+++ b/tests/library/__llm_cache__/library-agent-perform-perform-run-timeout-inherited-from-page.json@@ -0,0 +1,30 @@+{+ "554366043f6477d30a36655c10cbb4ec0eaa21d4": {+ "result": {+ "role": "assistant",+ "content": [+ {+ "type": "text",+ "text": "I'll click the Fox button for you."+ },+ {+ "type": "tool_call",+ "name": "browser_click",+ "arguments": {+ "element": "Fox button",+ "ref": "e3",+ "_is_done": true+ },+ "id": "toolu_01TBMe2oPqGUpNuPnFnDeZJe"+ }+ ],+ "stopReason": {+ "code": "ok"+ }+ },+ "usage": {+ "input": 3066,+ "output": 101+ }+ }+}\ No newline at end of filediff --git a/tests/library/agent-expect.spec.ts b/tests/library/agent-expect.spec.tsindex c7befd887a..f301a6c5ab 100644--- a/tests/library/agent-expect.spec.ts+++ b/tests/library/agent-expect.spec.ts@@ -196,16 +196,36 @@ test('expect timeout during run', async ({ context }) => {{const { page, agent } = await runAgent(context);await page.setContent(`<button hidden>Submit</button>`);- const error = await agent.expect('submit button is visible', { timeout: 10000 }).catch(e => e);+ const error = await agent.expect('submit button is visible', { timeout: 3000 }).catch(e => e);expect(stripAnsi(error.message)).toContain(`pageAgent.expect: expect(locator).toBeVisible() failedLocator: getByRole('button', { name: 'Submit' })Expected: visible-Timeout: 5000ms+Timeout: 3000msError: element(s) not foundCall log:- - Expect Visible with timeout 5000ms`);+ - Expect Visible with timeout 3000ms`);+ }+});++test('expect timeout during run from agent options', async ({ context }) => {+ {+ const { page, agent } = await generateAgent(context);+ await page.setContent(`<button>Submit</button>`);+ await agent.expect('submit button is visible');+ }+ expect(await cacheObject()).toEqual({+ 'submit button is visible': {+ actions: [expect.objectContaining({ method: 'expectVisible' })],+ },+ });+ {+ const { page, agent } = await runAgent(context, { expect: { timeout: 3000 } });+ await page.setContent(`<button hidden>Submit</button>`);+ const error = await agent.expect('submit button is visible').catch(e => e);+ expect(stripAnsi(error.message)).toContain(`pageAgent.expect: expect(locator).toBeVisible() failed`);+ expect(stripAnsi(error.message)).toContain(`Expect Visible with timeout 3000ms`);}});diff --git a/tests/library/agent-helpers.ts b/tests/library/agent-helpers.tsindex 88492bca96..5a718082df 100644--- a/tests/library/agent-helpers.ts+++ b/tests/library/agent-helpers.ts@@ -55,9 +55,10 @@ export async function generateAgent(context: BrowserContext, options: AgentOptioreturn { page, agent };}-export async function runAgent(context: BrowserContext, options: { secrets?: Record<string, string> } = {}) {+export async function runAgent(context: BrowserContext, options: AgentOptions = {}) {const page = await context.newPage();const agent = await page.agent({+ ...options,cache: { cacheFile: cacheFile() },...{ _doNotRenderActive: true },});diff --git a/tests/library/agent-perform.spec.ts b/tests/library/agent-perform.spec.tsindex c7f80dd3ec..9a14b409a2 100644--- a/tests/library/agent-perform.spec.ts+++ b/tests/library/agent-perform.spec.ts@@ -174,6 +174,28 @@ test('perform run timeout', async ({ context }) => {}});+test('perform run timeout inherited from page', async ({ context }) => {+ {+ const { page, agent } = await generateAgent(context);+ await page.setContent(`+ <button>Wolf</button>+ <button>Fox</button>+ `);+ await agent.perform('click the Fox button');+ }+ {+ const { page, agent } = await runAgent(context);+ await page.setContent(`+ <button>Wolf</button>+ <button>Rabbit</button>+ `);+ page.setDefaultTimeout(3000);+ const error = await agent.perform('click the Fox button').catch(e => e);+ expect(error.message).toContain('Timeout 3000ms exceeded.');+ expect(error.message).toContain(`waiting for getByRole('button', { name: 'Fox' })`);+ }+});+test('invalid cache file throws error', async ({ context }) => {await setCacheObject({'some key': {