Skip to content

Playwright is a framework for Web Testing and Automation. It allows testing Chromium, Firefox and WebKit with a single API.

chore: page.agent timeouts sorted (#38839)

Dmitry Gozmanc6a7c4fcbdbc
diff --git a/docs/src/api/class-page.md b/docs/src/api/class-page.md
index 5c2b752197..55399a4d67 100644
--- a/docs/src/api/class-page.md
+++ b/docs/src/api/class-page.md
@@ -722,6 +722,11 @@ Initialize page agent with the llm provider and cache.
- `cacheFile` ?<[string]> Cache file to use/generate code for performed actions into. Cache is not used if not specified (default).
- `cacheOutFile` ?<[string]> When specified, generated entries are written into the `cacheOutFile` instead of updating the `cacheFile`.
+### option: Page.agent.expect
+* since: v1.58
+- `expect` <[Object]>
+ - `timeout` ?<[int]> Default timeout for expect calls in milliseconds, defaults to 5000ms.
+
### option: Page.agent.limits
* since: v1.58
- `limits` <[Object]>
diff --git a/docs/src/api/class-pageagent.md b/docs/src/api/class-pageagent.md
index 26242e8c44..0f04e2b04d 100644
--- a/docs/src/api/class-pageagent.md
+++ b/docs/src/api/class-pageagent.md
@@ -35,6 +35,12 @@ await agent.expect('"0 items" to be reported');
Expectation to assert.
+### option: PageAgent.expect.timeout
+* since: v1.58
+- `timeout` <[float]>
+
+Expect timeout in milliseconds. Defaults to `5000`. The default value can be changed via `expect.timeout` option in the config, or by specifying the `expect` property of the [`option: Page.agent.expect`] option. Pass `0` to disable timeout.
+
### option: PageAgent.expect.-inline- = %%-page-agent-call-options-v1.58-%%
* since: v1.58
@@ -68,6 +74,13 @@ Task to perform using agentic loop.
* since: v1.58
- `schema` <[z.ZodSchema]>
+### option: PageAgent.extract.timeout
+* since: v1.58
+- `timeout` <[float]>
+
+Extract timeout in milliseconds. Defaults to `5000`. The default value can be changed via `actionTimeout` option in the config, or by using the [`method: BrowserContext.setDefaultTimeout`] or
+[`method: Page.setDefaultTimeout`] methods. Pass `0` to disable timeout.
+
### option: PageAgent.extract.-inline- = %%-page-agent-call-options-v1.58-%%
* since: v1.58
@@ -94,6 +107,13 @@ await agent.perform('Click submit button');
Task to perform using agentic loop.
+### option: PageAgent.perform.timeout
+* since: v1.58
+- `timeout` <[float]>
+
+Perform timeout in milliseconds. Defaults to `5000`. The default value can be changed via `actionTimeout` option in the config, or by using the [`method: BrowserContext.setDefaultTimeout`] or
+[`method: Page.setDefaultTimeout`] methods. Pass `0` to disable timeout.
+
### option: PageAgent.perform.-inline- = %%-page-agent-call-options-v1.58-%%
* since: v1.58
diff --git a/docs/src/api/params.md b/docs/src/api/params.md
index 9b7bf8f3d4..69257b93db 100644
--- a/docs/src/api/params.md
+++ b/docs/src/api/params.md
@@ -396,18 +396,11 @@ Maximum number of agentic actions to generate, defaults to context-wide value sp
Maximum number of retries when generating each action, defaults to context-wide value specified in `agent` property.
-## page-agent-timeout
-* since: v1.58
-- `timeout` <[int]>
-
-Request timeout in milliseconds. Defaults to action timeout. Pass `0` to disable timeout.
-
## page-agent-call-options-v1.58
- %%-page-agent-cache-key-%%
- %%-page-agent-max-tokens-%%
- %%-page-agent-max-actions-%%
- %%-page-agent-max-action-retries-%%
-- %%-page-agent-timeout-%%
## fetch-param-url
- `url` <[string]>
diff --git a/packages/injected/src/injectedScript.ts b/packages/injected/src/injectedScript.ts
index ce18bf46aa..2be0082e4d 100644
--- a/packages/injected/src/injectedScript.ts
+++ b/packages/injected/src/injectedScript.ts
@@ -50,8 +50,7 @@ import type { Builtins } from './utilityScript';
export type FrameExpectParams = Omit<channels.FrameExpectParams, 'expectedValue' | 'timeout'> & {
expectedValue?: any;
timeoutForLogs?: number;
- explicitTimeout?: number;
- noPreChecks?: boolean;
+ noAutoWaiting?: boolean;
};
export type ElementState = 'visible' | 'hidden' | 'enabled' | 'disabled' | 'editable' | 'checked' | 'unchecked' | 'indeterminate' | 'stable';
diff --git a/packages/playwright-client/types/types.d.ts b/packages/playwright-client/types/types.d.ts
index 6e2a58a63d..50dd490708 100644
--- a/packages/playwright-client/types/types.d.ts
+++ b/packages/playwright-client/types/types.d.ts
@@ -2112,6 +2112,13 @@ export interface Page {
cacheOutFile?: string;
};
+ expect?: {
+ /**
+ * Default timeout for expect calls in milliseconds, defaults to 5000ms.
+ */
+ timeout?: number;
+ };
+
/**
* Limits to use for the agentic loop.
*/
@@ -5440,7 +5447,10 @@ export interface PageAgent {
maxTokens?: number;
/**
- * Request timeout in milliseconds. Defaults to action timeout. Pass `0` to disable timeout.
+ * Expect timeout in milliseconds. Defaults to `5000`. The default value can be changed via `expect.timeout` option in
+ * the config, or by specifying the `expect` property of the
+ * [`expect`](https://playwright.dev/docs/api/class-page#page-agent-option-expect) option. Pass `0` to disable
+ * timeout.
*/
timeout?: number;
}): Promise<void>;
@@ -5482,7 +5492,11 @@ export interface PageAgent {
maxTokens?: number;
/**
- * Request timeout in milliseconds. Defaults to action timeout. Pass `0` to disable timeout.
+ * Perform timeout in milliseconds. Defaults to `5000`. The default value can be changed via `actionTimeout` option in
+ * the config, or by using the
+ * [browserContext.setDefaultTimeout(timeout)](https://playwright.dev/docs/api/class-browsercontext#browser-context-set-default-timeout)
+ * or [page.setDefaultTimeout(timeout)](https://playwright.dev/docs/api/class-page#page-set-default-timeout) methods.
+ * Pass `0` to disable timeout.
*/
timeout?: number;
}): Promise<{
diff --git a/packages/playwright-core/src/client/page.ts b/packages/playwright-core/src/client/page.ts
index e7c027f2a8..adc7654de9 100644
--- a/packages/playwright-core/src/client/page.ts
+++ b/packages/playwright-core/src/client/page.ts
@@ -865,7 +865,9 @@ export class Page extends ChannelOwner<channels.PageChannel> implements api.Page
systemPrompt: options.systemPrompt,
};
const { agent } = await this._channel.agent(params);
- return PageAgent.from(agent);
+ const pageAgent = PageAgent.from(agent);
+ pageAgent._expectTimeout = options?.expect?.timeout;
+ return pageAgent;
}
async _snapshotForAI(options: TimeoutOptions & { track?: string } = {}): Promise<{ full: string, incremental?: string }> {
diff --git a/packages/playwright-core/src/client/pageAgent.ts b/packages/playwright-core/src/client/pageAgent.ts
index 2837f5bbb1..ec6a16cc9a 100644
--- a/packages/playwright-core/src/client/pageAgent.ts
+++ b/packages/playwright-core/src/client/pageAgent.ts
@@ -22,14 +22,9 @@ import { Page } from './page';
import type * as api from '../../types/types';
import type * as channels from '@protocol/channels';
-type PageAgentOptions = {
- maxTokens?: number;
- maxTurns?: number;
- cacheKey?: string;
-};
-
export class PageAgent extends ChannelOwner<channels.PageAgentChannel> implements api.PageAgent {
private _page: Page;
+ _expectTimeout?: number;
static from(channel: channels.PageAgentChannel): PageAgent {
return (channel as any)._object;
@@ -41,17 +36,20 @@ export class PageAgent extends ChannelOwner<channels.PageAgentChannel> implement
this._channel.on('turn', params => this.emit(Events.Page.AgentTurn, params));
}
- async expect(expectation: string, options: PageAgentOptions = {}) {
- await this._channel.expect({ expectation, ...options });
+ async expect(expectation: string, options: channels.PageAgentExpectOptions = {}) {
+ const timeout = options.timeout ?? this._expectTimeout ?? 5000;
+ await this._channel.expect({ expectation, ...options, timeout });
}
- async perform(task: string, options: PageAgentOptions = {}) {
- const { usage } = await this._channel.perform({ task, ...options });
+ async perform(task: string, options: channels.PageAgentPerformOptions = {}) {
+ const timeout = this._page._timeoutSettings.timeout(options);
+ const { usage } = await this._channel.perform({ task, ...options, timeout });
return { usage };
}
- async extract<Schema extends any>(query: string, schema: Schema, options: PageAgentOptions = {}): Promise<{ result: any, usage: channels.AgentUsage }> {
- const { result, usage } = await this._channel.extract({ query, schema: this._page._platform.zodToJsonSchema(schema), ...options });
+ async extract<Schema extends any>(query: string, schema: Schema, options: channels.PageAgentExtractOptions = {}): Promise<{ result: any, usage: channels.AgentUsage }> {
+ const timeout = this._page._timeoutSettings.timeout(options);
+ const { result, usage } = await this._channel.extract({ query, schema: this._page._platform.zodToJsonSchema(schema), ...options, timeout });
return { result, usage };
}
diff --git a/packages/playwright-core/src/server/agent/actionRunner.ts b/packages/playwright-core/src/server/agent/actionRunner.ts
index 9f25d3fec2..9773c29b55 100644
--- a/packages/playwright-core/src/server/agent/actionRunner.ts
+++ b/packages/playwright-core/src/server/agent/actionRunner.ts
@@ -143,14 +143,11 @@ async function innerRunAction(progress: Progress, mode: 'generate' | 'run', page
}
async function runExpect(frame: Frame, progress: Progress, mode: 'generate' | 'run', selector: string | undefined, options: FrameExpectParams, expected: string | RegExp, matcherName: string, expectation: string) {
- // Pass explicit timeout to limit the single expect action inside the overall "agentic expect" multi-step progress.
- const timeout = expectTimeout(mode);
const result = await frame.expect(progress, selector, {
...options,
- timeoutForLogs: timeout,
- explicitTimeout: timeout,
- // Disable pre-checks to avoid them timing out, model has seen the snapshot anyway.
- noPreChecks: mode === 'generate',
+ // When generating, we want the expect to pass or fail immediately and give feedback to the model.
+ noAutoWaiting: mode === 'generate',
+ timeoutForLogs: mode === 'generate' ? undefined : progress.timeout,
});
if (!result.matches === !options.isNot) {
const received = matcherName === 'toMatchAriaSnapshot' ? '\n' + result.received.raw : result.received;
@@ -162,7 +159,7 @@ async function runExpect(frame: Frame, progress: Progress, mode: 'generate' | 'r
expectation,
locator: selector ? asLocatorDescription('javascript', selector) : undefined,
timedOut: result.timedOut,
- timeout,
+ timeout: mode === 'generate' ? undefined : progress.timeout,
printedExpected: options.isNot ? `Expected${expectedSuffix}: not ${expectedDisplay}` : `Expected${expectedSuffix}: ${expectedDisplay}`,
printedReceived: result.errorMessage ? '' : `Received: ${received}`,
errorMessage: result.errorMessage,
@@ -266,7 +263,7 @@ export function traceParamsForAction(progress: Progress, action: actions.Action,
expression: 'to.have.value',
expectedText,
isNot: !!action.isNot,
- timeout: expectTimeout(mode),
+ timeout,
};
return { type: 'Frame', method: 'expect', title: 'Expect Value', params };
} else if (action.type === 'checkbox' || action.type === 'radio') {
@@ -275,7 +272,7 @@ export function traceParamsForAction(progress: Progress, action: actions.Action,
selector: action.selector,
expression: 'to.be.checked',
isNot: !!action.isNot,
- timeout: expectTimeout(mode),
+ timeout,
};
return { type: 'Frame', method: 'expect', title: 'Expect Checked', params };
} else {
@@ -287,7 +284,7 @@ export function traceParamsForAction(progress: Progress, action: actions.Action,
selector: action.selector,
expression: 'to.be.visible',
isNot: !!action.isNot,
- timeout: expectTimeout(mode),
+ timeout,
};
return { type: 'Frame', method: 'expect', title: 'Expect Visible', params };
}
@@ -298,7 +295,7 @@ export function traceParamsForAction(progress: Progress, action: actions.Action,
expression: 'to.match.snapshot',
expectedText: [],
isNot: !!action.isNot,
- timeout: expectTimeout(mode),
+ timeout,
};
return { type: 'Frame', method: 'expect', title: 'Expect Aria Snapshot', params };
}
@@ -310,7 +307,7 @@ export function traceParamsForAction(progress: Progress, action: actions.Action,
expression: 'to.have.url',
expectedText,
isNot: !!action.isNot,
- timeout: expectTimeout(mode),
+ timeout,
};
return { type: 'Frame', method: 'expect', title: 'Expect URL', params };
}
@@ -321,7 +318,7 @@ export function traceParamsForAction(progress: Progress, action: actions.Action,
expression: 'to.have.title',
expectedText,
isNot: !!action.isNot,
- timeout: expectTimeout(mode),
+ timeout,
};
return { type: 'Frame', method: 'expect', title: 'Expect Title', params };
}
@@ -341,7 +338,3 @@ function callMetadataForAction(progress: Progress, frame: Frame, action: actions
};
return callMetadata;
}
-
-function expectTimeout(mode: 'generate' | 'run') {
- return mode === 'generate' ? 0 : 5000;
-}
diff --git a/packages/playwright-core/src/server/agent/pageAgent.ts b/packages/playwright-core/src/server/agent/pageAgent.ts
index 120f611390..7b22731aa5 100644
--- a/packages/playwright-core/src/server/agent/pageAgent.ts
+++ b/packages/playwright-core/src/server/agent/pageAgent.ts
@@ -50,7 +50,7 @@ export async function pageAgentPerform(progress: Progress, context: Context, use
### Task
${userTask}
`;
-
+ progress.disableTimeout();
await runLoop(progress, context, performTools, task, undefined, callParams);
await updateCache(context, cacheKey);
}
@@ -68,7 +68,7 @@ export async function pageAgentExpect(progress: Progress, context: Context, expe
### Expectation
${expectation}
`;
-
+ progress.disableTimeout();
await runLoop(progress, context, expectTools, task, undefined, callParams);
await updateCache(context, cacheKey);
}
@@ -101,7 +101,7 @@ async function runLoop(progress: Progress, context: Context, toolDefinitions: To
const apiCacheTextBefore = context.agentParams.apiCacheFile ?
await fs.promises.readFile(context.agentParams.apiCacheFile, 'utf-8').catch(() => '{}') : '{}';
- const apiCacheBefore = JSON.parse(apiCacheTextBefore);
+ const apiCacheBefore = JSON.parse(apiCacheTextBefore || '{}');
const loop = new Loop({
api: context.agentParams.api as any,
diff --git a/packages/playwright-core/src/server/frames.ts b/packages/playwright-core/src/server/frames.ts
index 5431ea6a40..a09b41d050 100644
--- a/packages/playwright-core/src/server/frames.ts
+++ b/packages/playwright-core/src/server/frames.ts
@@ -1389,14 +1389,10 @@ export class Frame extends SdkObject<FrameEventMap> {
progress.metadata.error = { error: { name: 'Expect', message: 'Expect failed' } };
};
try {
- // When explicit timeout is passed, constrain expect by it, in addition to regular progress abort.
- const timeoutPromise = options.explicitTimeout !== undefined ? progress.wait(options.explicitTimeout).then(() => { throw new TimeoutError(`Timed out after ${options.explicitTimeout}ms`); }) : undefined;
- timeoutPromise?.catch(() => { /* Prevent unhandled promise rejection */ });
-
// Step 1: perform locator handlers checkpoint with a specified timeout.
if (selector)
progress.log(`waiting for ${this._asLocator(selector)}`);
- if (!options.noPreChecks)
+ if (!options.noAutoWaiting)
await this._page.performActionPreChecks(progress);
// Step 2: perform one-shot expect check without a timeout.
@@ -1404,19 +1400,19 @@ export class Frame extends SdkObject<FrameEventMap> {
// that should succeed when the locator is already visible.
try {
const resultOneShot = await this._expectInternal(progress, selector, options, lastIntermediateResult, true);
- if (resultOneShot.matches !== options.isNot)
+ if (options.noAutoWaiting || resultOneShot.matches !== options.isNot)
return resultOneShot;
} catch (e) {
- if (this.isNonRetriableError(e))
+ if (options.noAutoWaiting || this.isNonRetriableError(e))
throw e;
// Ignore any other errors from one-shot, we'll handle them during retries.
}
// Step 3: auto-retry expect with increasing timeouts. Bounded by the total remaining time.
const result = await this.retryWithProgressAndTimeouts(progress, [100, 250, 500, 1000], async continuePolling => {
- if (!options.noPreChecks)
+ if (!options.noAutoWaiting)
await this._page.performActionPreChecks(progress);
- const { matches, received } = await this._expectInternal(progress, selector, options, lastIntermediateResult, false, timeoutPromise);
+ const { matches, received } = await this._expectInternal(progress, selector, options, lastIntermediateResult, false);
if (matches === options.isNot) {
// Keep waiting in these cases:
// expect(locator).conditionThatDoesNotMatch
@@ -1446,9 +1442,9 @@ export class Frame extends SdkObject<FrameEventMap> {
}
}
- private async _expectInternal(progress: Progress, selector: string | undefined, options: FrameExpectParams, lastIntermediateResult: { received?: any, isSet: boolean, errorMessage?: string }, noAbort: boolean, timeoutPromise?: Promise<never>) {
+ private async _expectInternal(progress: Progress, selector: string | undefined, options: FrameExpectParams, lastIntermediateResult: { received?: any, isSet: boolean, errorMessage?: string }, noAbort: boolean) {
// The first expect check, a.k.a. one-shot, always finishes - even when progress is aborted.
- const race = <T>(p: Promise<T>) => noAbort ? p : (timeoutPromise ? progress.race([p, timeoutPromise]) : progress.race(p));
+ const race = <T>(p: Promise<T>) => noAbort ? p : progress.race(p);
const selectorInFrame = selector ? await race(this.selectors.resolveFrameForSelector(selector, { strict: true })) : undefined;
const { frame, info } = selectorInFrame || { frame: this, info: undefined };
diff --git a/packages/playwright-core/src/server/progress.ts b/packages/playwright-core/src/server/progress.ts
index b6cb8201fb..07b580bff6 100644
--- a/packages/playwright-core/src/server/progress.ts
+++ b/packages/playwright-core/src/server/progress.ts
@@ -61,10 +61,14 @@ export class ProgressController {
const deadline = timeout ? monotonicTime() + timeout : 0;
assert(this._state === 'before');
this._state = 'running';
+ let timer: NodeJS.Timeout | undefined;
const progress: Progress = {
timeout: timeout ?? 0,
deadline,
+ disableTimeout: () => {
+ clearTimeout(timer);
+ },
log: message => {
if (this._state === 'running')
this.metadata.log.push(message);
@@ -87,10 +91,10 @@ export class ProgressController {
signal: this._controller.signal,
};
- let timer: NodeJS.Timeout | undefined;
if (deadline) {
const timeoutError = new TimeoutError(`Timeout ${timeout}ms exceeded.`);
timer = setTimeout(() => {
+ // TODO: migrate this to "progress.disableTimeout()".
if (this.metadata.pauseStartTime && !this.metadata.pauseEndTime)
return;
if (this._state === 'running') {
diff --git a/packages/playwright-core/types/types.d.ts b/packages/playwright-core/types/types.d.ts
index 6e2a58a63d..50dd490708 100644
--- a/packages/playwright-core/types/types.d.ts
+++ b/packages/playwright-core/types/types.d.ts
@@ -2112,6 +2112,13 @@ export interface Page {
cacheOutFile?: string;
};
+ expect?: {
+ /**
+ * Default timeout for expect calls in milliseconds, defaults to 5000ms.
+ */
+ timeout?: number;
+ };
+
/**
* Limits to use for the agentic loop.
*/
@@ -5440,7 +5447,10 @@ export interface PageAgent {
maxTokens?: number;
/**
- * Request timeout in milliseconds. Defaults to action timeout. Pass `0` to disable timeout.
+ * Expect timeout in milliseconds. Defaults to `5000`. The default value can be changed via `expect.timeout` option in
+ * the config, or by specifying the `expect` property of the
+ * [`expect`](https://playwright.dev/docs/api/class-page#page-agent-option-expect) option. Pass `0` to disable
+ * timeout.
*/
timeout?: number;
}): Promise<void>;
@@ -5482,7 +5492,11 @@ export interface PageAgent {
maxTokens?: number;
/**
- * Request timeout in milliseconds. Defaults to action timeout. Pass `0` to disable timeout.
+ * Perform timeout in milliseconds. Defaults to `5000`. The default value can be changed via `actionTimeout` option in
+ * the config, or by using the
+ * [browserContext.setDefaultTimeout(timeout)](https://playwright.dev/docs/api/class-browsercontext#browser-context-set-default-timeout)
+ * or [page.setDefaultTimeout(timeout)](https://playwright.dev/docs/api/class-page#page-set-default-timeout) methods.
+ * Pass `0` to disable timeout.
*/
timeout?: number;
}): Promise<{
diff --git a/packages/playwright/src/index.ts b/packages/playwright/src/index.ts
index 82b978fb8c..874ba50ef2 100644
--- a/packages/playwright/src/index.ts
+++ b/packages/playwright/src/index.ts
@@ -477,6 +477,9 @@ const playwrightFixtures: Fixtures<TestFixtures, WorkerFixtures> = ({
limits: agentOptions?.limits,
secrets: agentOptions?.secrets,
systemPrompt: agentOptions?.systemPrompt,
+ expect: {
+ timeout: testInfoImpl._projectInternal.expect?.timeout,
+ },
});
await use(agent);
diff --git a/packages/protocol/src/progress.d.ts b/packages/protocol/src/progress.d.ts
index 8e389e0c31..f7abd096ac 100644
--- a/packages/protocol/src/progress.d.ts
+++ b/packages/protocol/src/progress.d.ts
@@ -37,6 +37,7 @@ import type { CallMetadata } from './callMetadata';
export interface Progress {
timeout: number;
deadline: number;
+ disableTimeout(): void;
log(message: string): void;
race<T>(promise: Promise<T> | Promise<T>[]): Promise<T>;
wait(timeout: number): Promise<void>; // timeout = 0 here means "wait 0 ms", not forever.
diff --git a/tests/library/__llm_cache__/library-agent-expect-expect-timeout-during-run-from-agent-options.json b/tests/library/__llm_cache__/library-agent-expect-expect-timeout-during-run-from-agent-options.json
new file mode 100644
index 0000000000..1109e7715d
--- /dev/null
+++ b/tests/library/__llm_cache__/library-agent-expect-expect-timeout-during-run-from-agent-options.json
@@ -0,0 +1,30 @@
+{
+ "44bfbaa6fe0cb038adf37547ecb000d4f39e243f": {
+ "result": {
+ "role": "assistant",
+ "content": [
+ {
+ "type": "text",
+ "text": "I'll verify that the submit button is visible on the page."
+ },
+ {
+ "type": "tool_call",
+ "name": "browser_expect_visible",
+ "arguments": {
+ "role": "button",
+ "accessibleName": "Submit",
+ "_is_done": true
+ },
+ "id": "toolu_012RgUS8buCjrQRgt4w2sYFf"
+ }
+ ],
+ "stopReason": {
+ "code": "ok"
+ }
+ },
+ "usage": {
+ "input": 2203,
+ "output": 107
+ }
+ }
+}
\ No newline at end of file
diff --git a/tests/library/__llm_cache__/library-agent-expect-expectTitle-success.json b/tests/library/__llm_cache__/library-agent-expect-expectTitle-success.json
index bb23b43b9d..0f0ff37ab6 100644
--- a/tests/library/__llm_cache__/library-agent-expect-expectTitle-success.json
+++ b/tests/library/__llm_cache__/library-agent-expect-expectTitle-success.json
@@ -5,7 +5,7 @@
"content": [
{
"type": "text",
- "text": "I need to verify that the page title is \"My Page Title\". I'll use the browser_expect_title tool to assert this condition."
+ "text": "I need to verify that the page title is \"My Page Title\". I'll use the browser_expect_title tool to check this."
},
{
"type": "tool_call",
@@ -14,7 +14,7 @@
"title": "My Page Title",
"_is_done": true
},
- "id": "toolu_01VvWdf4VVFNQuDaEtTsAu3K"
+ "id": "toolu_01R1QyAwfxk2KSz3tsAx9RCA"
}
],
"stopReason": {
@@ -23,7 +23,7 @@
},
"usage": {
"input": 2195,
- "output": 106
+ "output": 105
}
}
}
\ No newline at end of file
diff --git a/tests/library/__llm_cache__/library-agent-expect-expectTitle-wrong-title-error.json b/tests/library/__llm_cache__/library-agent-expect-expectTitle-wrong-title-error.json
index e14e18a7d2..a4ae141362 100644
--- a/tests/library/__llm_cache__/library-agent-expect-expectTitle-wrong-title-error.json
+++ b/tests/library/__llm_cache__/library-agent-expect-expectTitle-wrong-title-error.json
@@ -5,7 +5,7 @@
"content": [
{
"type": "text",
- "text": "I'll verify that the page title is \"Other Title\" using the browser_expect_title tool."
+ "text": "I need to verify that the page title is \"Other Title\" using one of the browser_expect_* tools."
},
{
"type": "tool_call",
@@ -14,7 +14,7 @@
"title": "Other Title",
"_is_done": true
},
- "id": "toolu_017UT5LEe8JRoB53R1YGKrYD"
+ "id": "toolu_017ZvXqNbcab9Uyjzw3VuByK"
}
],
"stopReason": {
@@ -23,7 +23,7 @@
},
"usage": {
"input": 2194,
- "output": 97
+ "output": 100
}
}
}
\ No newline at end of file
diff --git a/tests/library/__llm_cache__/library-agent-expect-expectURL-success.json b/tests/library/__llm_cache__/library-agent-expect-expectURL-success.json
index c3f3b6c225..56fadb43dc 100644
--- a/tests/library/__llm_cache__/library-agent-expect-expectURL-success.json
+++ b/tests/library/__llm_cache__/library-agent-expect-expectURL-success.json
@@ -5,7 +5,7 @@
"content": [
{
"type": "text",
- "text": "I'll verify that the page URL is /counter.html using the browser_expect_url tool."
+ "text": "I'll verify that the page URL is /counter.html."
},
{
"type": "tool_call",
@@ -14,7 +14,7 @@
"url": "/counter.html",
"_is_done": true
},
- "id": "toolu_017jkU3FcrQpAVJgtjrzYovK"
+ "id": "toolu_01PkjsZaM5eEGtjLtMixyJUk"
}
],
"stopReason": {
@@ -23,25 +23,25 @@
},
"usage": {
"input": 2240,
- "output": 99
+ "output": 91
}
},
- "f6d9bc80fd5bad48f6a886d7ec77c8d5889521b1": {
+ "71a7a4f51b3994a200f239eda9b505454825a79f": {
"result": {
"role": "assistant",
"content": [
{
"type": "text",
- "text": "I see the issue - the actual URL includes a server prefix. Let me use a regex pattern to match the URL ending with /counter.html."
+ "text": "I need to use a regex pattern to match the URL since it includes a server prefix. Let me update the expectation to match the actual URL pattern."
},
{
"type": "tool_call",
"name": "browser_expect_url",
"arguments": {
- "regex": "/counter\\.html$/",
+ "regex": "/\\/counter\\.html$/",
"_is_done": true
},
- "id": "toolu_01JQuC1CJR7W6LiwAnvjaiDX"
+ "id": "toolu_01Wwq7Pbe2DthqpmskgGfCzk"
}
],
"stopReason": {
@@ -49,8 +49,8 @@
}
},
"usage": {
- "input": 2441,
- "output": 109
+ "input": 2433,
+ "output": 111
}
}
}
\ No newline at end of file
diff --git a/tests/library/__llm_cache__/library-agent-expect-expectURL-with-regex-error.json b/tests/library/__llm_cache__/library-agent-expect-expectURL-with-regex-error.json
index f034545e13..3f76a92de9 100644
--- a/tests/library/__llm_cache__/library-agent-expect-expectURL-with-regex-error.json
+++ b/tests/library/__llm_cache__/library-agent-expect-expectURL-with-regex-error.json
@@ -5,7 +5,7 @@
"content": [
{
"type": "text",
- "text": "I'll verify that the page URL matches the `/counter` pattern using the browser_expect_url tool with a regex pattern."
+ "text": "I'll verify that the page URL matches the /counter pattern using the browser_expect_url tool with a regex pattern."
},
{
"type": "tool_call",
@@ -14,7 +14,7 @@
"regex": "/counter/",
"_is_done": true
},
- "id": "toolu_01J38E1DkiNA7JsvQtGx3otW"
+ "id": "toolu_01JxsK661QmxpScYeTMCfWTZ"
}
],
"stopReason": {
@@ -23,7 +23,7 @@
},
"usage": {
"input": 2239,
- "output": 103
+ "output": 102
}
}
}
\ No newline at end of file
diff --git a/tests/library/__llm_cache__/library-agent-expect-expectURL-with-regex.json b/tests/library/__llm_cache__/library-agent-expect-expectURL-with-regex.json
index 19a6a08b33..c8ae66692e 100644
--- a/tests/library/__llm_cache__/library-agent-expect-expectURL-with-regex.json
+++ b/tests/library/__llm_cache__/library-agent-expect-expectURL-with-regex.json
@@ -14,7 +14,7 @@
"regex": "/counter/",
"_is_done": true
},
- "id": "toolu_01Qen21mVwEVb2JsEVavTWbf"
+ "id": "toolu_01YVkbby1j3FBXk2o5Bi4cjP"
}
],
"stopReason": {
diff --git a/tests/library/__llm_cache__/library-agent-expect-expectURL-wrong-URL-error.json b/tests/library/__llm_cache__/library-agent-expect-expectURL-wrong-URL-error.json
index 43b309b180..b88a2b2d35 100644
--- a/tests/library/__llm_cache__/library-agent-expect-expectURL-wrong-URL-error.json
+++ b/tests/library/__llm_cache__/library-agent-expect-expectURL-wrong-URL-error.json
@@ -14,7 +14,7 @@
"url": "/counter.html",
"_is_done": true
},
- "id": "toolu_01B3LSMGJoDrDfCZKwa3neZ9"
+ "id": "toolu_01BrgLhGzD2dYWeu75KGgxKv"
}
],
"stopReason": {
@@ -26,22 +26,22 @@
"output": 99
}
},
- "99f45f7c71e64f612bfc0bcb148faf235ccfe006": {
+ "8315bc81a0dfc915cb8be72ad00102f627e370dd": {
"result": {
"role": "assistant",
"content": [
{
"type": "text",
- "text": "I see the issue. The actual URL includes the server prefix. Let me use a regex pattern to match the URL ending with /counter.html."
+ "text": "I see the issue. The actual URL includes the server portion. Let me use a regex pattern to match the URL ending with /counter.html."
},
{
"type": "tool_call",
"name": "browser_expect_url",
"arguments": {
- "regex": "/counter\\.html$/",
+ "regex": "/\\/counter\\.html$/",
"_is_done": true
},
- "id": "toolu_01Ufux7hKwV9Djp9jT1JgYwJ"
+ "id": "toolu_01KtWVz2EdJjx2V9viYeqs71"
}
],
"stopReason": {
@@ -50,7 +50,7 @@
},
"usage": {
"input": 2441,
- "output": 109
+ "output": 110
}
}
}
\ No newline at end of file
diff --git a/tests/library/__llm_cache__/library-agent-perform-perform-run-timeout-inherited-from-page.json b/tests/library/__llm_cache__/library-agent-perform-perform-run-timeout-inherited-from-page.json
new file mode 100644
index 0000000000..1cd1ae7633
--- /dev/null
+++ b/tests/library/__llm_cache__/library-agent-perform-perform-run-timeout-inherited-from-page.json
@@ -0,0 +1,30 @@
+{
+ "554366043f6477d30a36655c10cbb4ec0eaa21d4": {
+ "result": {
+ "role": "assistant",
+ "content": [
+ {
+ "type": "text",
+ "text": "I'll click the Fox button for you."
+ },
+ {
+ "type": "tool_call",
+ "name": "browser_click",
+ "arguments": {
+ "element": "Fox button",
+ "ref": "e3",
+ "_is_done": true
+ },
+ "id": "toolu_01TBMe2oPqGUpNuPnFnDeZJe"
+ }
+ ],
+ "stopReason": {
+ "code": "ok"
+ }
+ },
+ "usage": {
+ "input": 3066,
+ "output": 101
+ }
+ }
+}
\ No newline at end of file
diff --git a/tests/library/agent-expect.spec.ts b/tests/library/agent-expect.spec.ts
index c7befd887a..f301a6c5ab 100644
--- a/tests/library/agent-expect.spec.ts
+++ b/tests/library/agent-expect.spec.ts
@@ -196,16 +196,36 @@ test('expect timeout during run', async ({ context }) => {
{
const { page, agent } = await runAgent(context);
await page.setContent(`<button hidden>Submit</button>`);
- const error = await agent.expect('submit button is visible', { timeout: 10000 }).catch(e => e);
+ const error = await agent.expect('submit button is visible', { timeout: 3000 }).catch(e => e);
expect(stripAnsi(error.message)).toContain(`pageAgent.expect: expect(locator).toBeVisible() failed
Locator: getByRole('button', { name: 'Submit' })
Expected: visible
-Timeout: 5000ms
+Timeout: 3000ms
Error: element(s) not found
Call log:
- - Expect Visible with timeout 5000ms`);
+ - Expect Visible with timeout 3000ms`);
+ }
+});
+
+test('expect timeout during run from agent options', async ({ context }) => {
+ {
+ const { page, agent } = await generateAgent(context);
+ await page.setContent(`<button>Submit</button>`);
+ await agent.expect('submit button is visible');
+ }
+ expect(await cacheObject()).toEqual({
+ 'submit button is visible': {
+ actions: [expect.objectContaining({ method: 'expectVisible' })],
+ },
+ });
+ {
+ const { page, agent } = await runAgent(context, { expect: { timeout: 3000 } });
+ await page.setContent(`<button hidden>Submit</button>`);
+ const error = await agent.expect('submit button is visible').catch(e => e);
+ expect(stripAnsi(error.message)).toContain(`pageAgent.expect: expect(locator).toBeVisible() failed`);
+ expect(stripAnsi(error.message)).toContain(`Expect Visible with timeout 3000ms`);
}
});
diff --git a/tests/library/agent-helpers.ts b/tests/library/agent-helpers.ts
index 88492bca96..5a718082df 100644
--- a/tests/library/agent-helpers.ts
+++ b/tests/library/agent-helpers.ts
@@ -55,9 +55,10 @@ export async function generateAgent(context: BrowserContext, options: AgentOptio
return { page, agent };
}
-export async function runAgent(context: BrowserContext, options: { secrets?: Record<string, string> } = {}) {
+export async function runAgent(context: BrowserContext, options: AgentOptions = {}) {
const page = await context.newPage();
const agent = await page.agent({
+ ...options,
cache: { cacheFile: cacheFile() },
...{ _doNotRenderActive: true },
});
diff --git a/tests/library/agent-perform.spec.ts b/tests/library/agent-perform.spec.ts
index c7f80dd3ec..9a14b409a2 100644
--- a/tests/library/agent-perform.spec.ts
+++ b/tests/library/agent-perform.spec.ts
@@ -174,6 +174,28 @@ test('perform run timeout', async ({ context }) => {
}
});
+test('perform run timeout inherited from page', async ({ context }) => {
+ {
+ const { page, agent } = await generateAgent(context);
+ await page.setContent(`
+ <button>Wolf</button>
+ <button>Fox</button>
+ `);
+ await agent.perform('click the Fox button');
+ }
+ {
+ const { page, agent } = await runAgent(context);
+ await page.setContent(`
+ <button>Wolf</button>
+ <button>Rabbit</button>
+ `);
+ page.setDefaultTimeout(3000);
+ const error = await agent.perform('click the Fox button').catch(e => e);
+ expect(error.message).toContain('Timeout 3000ms exceeded.');
+ expect(error.message).toContain(`waiting for getByRole('button', { name: 'Fox' })`);
+ }
+});
+
test('invalid cache file throws error', async ({ context }) => {
await setCacheObject({
'some key': {