Skip to content

armgit / orca

Public Imported from GitHub
0

Orca is the place for working with a fleet of agents (e.g. Grok Bot, Claude Code). Run any coding agent with your own subscription. Available on desktop, mobile and remote runtime.

fix(browser): reject a ref from another tab's snapshot after a tab switch (#36251)

Kelvin Amoaba2b6e63ee2f87
diff --git a/src/main/browser/agent-browser-batch-command.ts b/src/main/browser/agent-browser-batch-command.ts
index b5f745bf0ee..5712b879ce8 100644
--- a/src/main/browser/agent-browser-batch-command.ts
+++ b/src/main/browser/agent-browser-batch-command.ts
@@ -34,7 +34,17 @@ function withoutDaemonLifecycle(envelope: AgentBrowserEnvelope): AgentBrowserEnv
return { ...envelope, data: rest }
}
-/** "Unknown ref" for a ref never issued; 0.39+ also says "Stale ref for role=…" once its element is gone. */
+/** Every snapshot form (plain, --full, --delta) answers with `snapshot`; annotated screenshots with `annotations`. */
+export function agentBrowserResultMintsRefs(data: unknown): boolean {
+ if (!data || typeof data !== 'object') {
+ return false
+ }
+ return 'snapshot' in data || ('annotations' in data && Array.isArray(data.annotations))
+}
+
+/** "Unknown ref" for a ref never issued; 0.39+ says "Stale ref for role=…" or "Could not locate element with role=…" once its element is gone. */
export function isAgentBrowserStaleRefError(message: string): boolean {
- return /unknown ref|stale ref|ref not found|element not found: @e/i.test(message)
+ return /unknown ref|stale ref|ref not found|element not found: @e|could not locate element with role=/i.test(
+ message
+ )
}
diff --git a/src/main/browser/agent-browser-bridge-command-transport.test.ts b/src/main/browser/agent-browser-bridge-command-transport.test.ts
index 3e838bdaf11..565f3f03200 100644
--- a/src/main/browser/agent-browser-bridge-command-transport.test.ts
+++ b/src/main/browser/agent-browser-bridge-command-transport.test.ts
@@ -210,11 +210,12 @@ describe('AgentBrowserBridge', () => {
await expect(bridge.click('@e2')).resolves.toEqual({ clicked: '@e2' })
})
- it('reports a ref whose element is gone as stale', async () => {
- // agent-browser 0.39.2+ wording when it cannot re-resolve a ref by role and name.
- failWith(
- 'Stale ref for role=button name=Approve cannot be recovered: several elements now match. Run snapshot again and use a fresh ref.'
- )
+ // agent-browser 0.39.2+ wording when it cannot re-resolve a ref by role and name.
+ it.each([
+ 'Stale ref for role=button name=Approve cannot be recovered: several elements now match. Run snapshot again and use a fresh ref.',
+ 'Could not locate element with role=link name=Back to basic'
+ ])('reports a ref whose element is gone as stale: %s', async (message) => {
+ failWith(message)
await expect(bridge.click('@e2')).rejects.toMatchObject({ code: 'browser_stale_ref' })
})
diff --git a/src/main/browser/agent-browser-bridge-core-commands.ts b/src/main/browser/agent-browser-bridge-core-commands.ts
index 1ae29d05de8..53f3185c1b4 100644
--- a/src/main/browser/agent-browser-bridge-core-commands.ts
+++ b/src/main/browser/agent-browser-bridge-core-commands.ts
@@ -34,9 +34,14 @@ export abstract class AgentBrowserBridgeCoreCommands extends AgentBrowserBridgeQ
worktreeId?: string,
browserPageId?: string
): Promise<BrowserClickResult> {
- return this.enqueueTargetedCommand(worktreeId, browserPageId, async (sessionName) => {
- return (await this.execAgentBrowser(sessionName, ['click', element])) as BrowserClickResult
- })
+ return this.enqueueTargetedCommand(
+ worktreeId,
+ browserPageId,
+ async (sessionName) => {
+ return (await this.execAgentBrowser(sessionName, ['click', element])) as BrowserClickResult
+ },
+ { elements: [element] }
+ )
}
async dblclick(
@@ -44,9 +49,18 @@ export abstract class AgentBrowserBridgeCoreCommands extends AgentBrowserBridgeQ
worktreeId?: string,
browserPageId?: string
): Promise<BrowserClickResult> {
- return this.enqueueTargetedCommand(worktreeId, browserPageId, async (sessionName) => {
- return (await this.execAgentBrowser(sessionName, ['dblclick', element])) as BrowserClickResult
- })
+ return this.enqueueTargetedCommand(
+ worktreeId,
+ browserPageId,
+ async (sessionName) => {
+ // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: agent-browser's dblclick --json answers the BrowserClickResult shape.
+ return (await this.execAgentBrowser(sessionName, [
+ 'dblclick',
+ element
+ ])) as BrowserClickResult
+ },
+ { elements: [element] }
+ )
}
async goto(url: string, worktreeId?: string, browserPageId?: string): Promise<BrowserGotoResult> {
@@ -162,7 +176,7 @@ export abstract class AgentBrowserBridgeCoreCommands extends AgentBrowserBridgeQ
await this.fillExplicitContentEditable(sessionName, element, value)
return { filled: element } as BrowserFillResult
},
- { requireScopedTarget: true }
+ { requireScopedTarget: true, elements: [element] }
)
}
}
diff --git a/src/main/browser/agent-browser-bridge-execution.ts b/src/main/browser/agent-browser-bridge-execution.ts
index e7e5cb22a92..7faee8ef652 100644
--- a/src/main/browser/agent-browser-bridge-execution.ts
+++ b/src/main/browser/agent-browser-bridge-execution.ts
@@ -10,7 +10,11 @@ import {
pageUnavailableMessageForSession
} from './agent-browser-bridge-process'
import { translateResult } from './agent-browser-bridge-result'
-import { AGENT_BROWSER_BATCH_ARGS, agentBrowserBatchInput } from './agent-browser-batch-command'
+import {
+ AGENT_BROWSER_BATCH_ARGS,
+ agentBrowserBatchInput,
+ agentBrowserResultMintsRefs
+} from './agent-browser-batch-command'
import { AgentBrowserBridgeTabs } from './agent-browser-bridge-tabs'
import { assertRemoteCliBrowserAccess } from '../runtime/desktop-browser-bridge/remote-cli-browser-scope'
import { ORCA_TAB_SESSION_PREFIX } from './agent-browser-orphan-sweep'
@@ -114,6 +118,13 @@ export abstract class AgentBrowserBridgeExecution extends AgentBrowserBridgeTabs
)
}
+ if (agentBrowserResultMintsRefs(translated.result)) {
+ const browserPageId = this.resolveTabIdSafe(session.webContentsId)
+ if (browserPageId) {
+ this.recordRefPage(browserPageId)
+ }
+ }
+
if (needsInit) {
session.initialized = true
diff --git a/src/main/browser/agent-browser-bridge-input-commands.ts b/src/main/browser/agent-browser-bridge-input-commands.ts
index 83bbcf9a8fa..242fdbc7b99 100644
--- a/src/main/browser/agent-browser-bridge-input-commands.ts
+++ b/src/main/browser/agent-browser-bridge-input-commands.ts
@@ -37,13 +37,18 @@ export abstract class AgentBrowserBridgeInputCommands extends AgentBrowserBridge
worktreeId?: string,
browserPageId?: string
): Promise<BrowserSelectResult> {
- return this.enqueueTargetedCommand(worktreeId, browserPageId, async (sessionName) => {
- return (await this.execAgentBrowser(sessionName, [
- 'select',
- element,
- value
- ])) as BrowserSelectResult
- })
+ return this.enqueueTargetedCommand(
+ worktreeId,
+ browserPageId,
+ async (sessionName) => {
+ return (await this.execAgentBrowser(sessionName, [
+ 'select',
+ element,
+ value
+ ])) as BrowserSelectResult
+ },
+ { elements: [element] }
+ )
}
async scroll(
@@ -66,9 +71,14 @@ export abstract class AgentBrowserBridgeInputCommands extends AgentBrowserBridge
worktreeId?: string,
browserPageId?: string
): Promise<unknown> {
- return this.enqueueTargetedCommand(worktreeId, browserPageId, async (sessionName) => {
- return await this.execAgentBrowser(sessionName, ['scrollintoview', element])
- })
+ return this.enqueueTargetedCommand(
+ worktreeId,
+ browserPageId,
+ async (sessionName) => {
+ return await this.execAgentBrowser(sessionName, ['scrollintoview', element])
+ },
+ { elements: [element] }
+ )
}
async get(
@@ -77,13 +87,18 @@ export abstract class AgentBrowserBridgeInputCommands extends AgentBrowserBridge
worktreeId?: string,
browserPageId?: string
): Promise<unknown> {
- return this.enqueueTargetedCommand(worktreeId, browserPageId, async (sessionName) => {
- const args = ['get', what]
- if (selector) {
- args.push(selector)
- }
- return await this.execAgentBrowser(sessionName, args)
- })
+ return this.enqueueTargetedCommand(
+ worktreeId,
+ browserPageId,
+ async (sessionName) => {
+ const args = ['get', what]
+ if (selector) {
+ args.push(selector)
+ }
+ return await this.execAgentBrowser(sessionName, args)
+ },
+ { elements: [selector] }
+ )
}
async is(
@@ -92,9 +107,14 @@ export abstract class AgentBrowserBridgeInputCommands extends AgentBrowserBridge
worktreeId?: string,
browserPageId?: string
): Promise<unknown> {
- return this.enqueueTargetedCommand(worktreeId, browserPageId, async (sessionName) => {
- return await this.execAgentBrowser(sessionName, ['is', what, selector])
- })
+ return this.enqueueTargetedCommand(
+ worktreeId,
+ browserPageId,
+ async (sessionName) => {
+ return await this.execAgentBrowser(sessionName, ['is', what, selector])
+ },
+ { elements: [selector] }
+ )
}
async keyboardInsertText(
diff --git a/src/main/browser/agent-browser-bridge-interaction-commands.ts b/src/main/browser/agent-browser-bridge-interaction-commands.ts
index fa35e97ffd8..53c5cf4dc74 100644
--- a/src/main/browser/agent-browser-bridge-interaction-commands.ts
+++ b/src/main/browser/agent-browser-bridge-interaction-commands.ts
@@ -28,9 +28,14 @@ export abstract class AgentBrowserBridgeInteractionCommands extends AgentBrowser
worktreeId?: string,
browserPageId?: string
): Promise<BrowserHoverResult> {
- return this.enqueueTargetedCommand(worktreeId, browserPageId, async (sessionName) => {
- return (await this.execAgentBrowser(sessionName, ['hover', element])) as BrowserHoverResult
- })
+ return this.enqueueTargetedCommand(
+ worktreeId,
+ browserPageId,
+ async (sessionName) => {
+ return (await this.execAgentBrowser(sessionName, ['hover', element])) as BrowserHoverResult
+ },
+ { elements: [element] }
+ )
}
async drag(
@@ -39,9 +44,14 @@ export abstract class AgentBrowserBridgeInteractionCommands extends AgentBrowser
worktreeId?: string,
browserPageId?: string
): Promise<BrowserDragResult> {
- return this.enqueueTargetedCommand(worktreeId, browserPageId, async (sessionName) => {
- return (await this.execAgentBrowser(sessionName, ['drag', from, to])) as BrowserDragResult
- })
+ return this.enqueueTargetedCommand(
+ worktreeId,
+ browserPageId,
+ async (sessionName) => {
+ return (await this.execAgentBrowser(sessionName, ['drag', from, to])) as BrowserDragResult
+ },
+ { elements: [from, to] }
+ )
}
async upload(
@@ -50,13 +60,18 @@ export abstract class AgentBrowserBridgeInteractionCommands extends AgentBrowser
worktreeId?: string,
browserPageId?: string
): Promise<BrowserUploadResult> {
- return this.enqueueTargetedCommand(worktreeId, browserPageId, async (sessionName) => {
- return (await this.execAgentBrowser(sessionName, [
- 'upload',
- element,
- ...filePaths
- ])) as BrowserUploadResult
- })
+ return this.enqueueTargetedCommand(
+ worktreeId,
+ browserPageId,
+ async (sessionName) => {
+ return (await this.execAgentBrowser(sessionName, [
+ 'upload',
+ element,
+ ...filePaths
+ ])) as BrowserUploadResult
+ },
+ { elements: [element] }
+ )
}
async wait(
@@ -64,22 +79,27 @@ export abstract class AgentBrowserBridgeInteractionCommands extends AgentBrowser
worktreeId?: string,
browserPageId?: string
): Promise<BrowserWaitResult> {
- return this.enqueueTargetedCommand(worktreeId, browserPageId, async (sessionName) => {
- const { args, conditionTimeoutMs, processTimeoutMs } = agentBrowserWaitArguments(
- options ?? {}
- )
- // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: agent-browser's wait --json answers the BrowserWaitResult shape.
- return (await this.execAgentBrowser(sessionName, args, {
- timeoutMs: processTimeoutMs,
- timeoutError:
- conditionTimeoutMs === undefined
- ? undefined
- : new BrowserError(
- 'browser_timeout',
- `Timed out waiting for browser condition after ${conditionTimeoutMs}ms.`
- )
- })) as BrowserWaitResult
- })
+ return this.enqueueTargetedCommand(
+ worktreeId,
+ browserPageId,
+ async (sessionName) => {
+ const { args, conditionTimeoutMs, processTimeoutMs } = agentBrowserWaitArguments(
+ options ?? {}
+ )
+ // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: agent-browser's wait --json answers the BrowserWaitResult shape.
+ return (await this.execAgentBrowser(sessionName, args, {
+ timeoutMs: processTimeoutMs,
+ timeoutError:
+ conditionTimeoutMs === undefined
+ ? undefined
+ : new BrowserError(
+ 'browser_timeout',
+ `Timed out waiting for browser condition after ${conditionTimeoutMs}ms.`
+ )
+ })) as BrowserWaitResult
+ },
+ { elements: [options?.selector] }
+ )
}
async check(
@@ -88,10 +108,15 @@ export abstract class AgentBrowserBridgeInteractionCommands extends AgentBrowser
worktreeId?: string,
browserPageId?: string
): Promise<BrowserCheckResult> {
- return this.enqueueTargetedCommand(worktreeId, browserPageId, async (sessionName) => {
- const args = checked ? ['check', element] : ['uncheck', element]
- return (await this.execAgentBrowser(sessionName, args)) as BrowserCheckResult
- })
+ return this.enqueueTargetedCommand(
+ worktreeId,
+ browserPageId,
+ async (sessionName) => {
+ const args = checked ? ['check', element] : ['uncheck', element]
+ return (await this.execAgentBrowser(sessionName, args)) as BrowserCheckResult
+ },
+ { elements: [element] }
+ )
}
async focus(
@@ -99,9 +124,14 @@ export abstract class AgentBrowserBridgeInteractionCommands extends AgentBrowser
worktreeId?: string,
browserPageId?: string
): Promise<BrowserFocusResult> {
- return this.enqueueTargetedCommand(worktreeId, browserPageId, async (sessionName) => {
- return (await this.execAgentBrowser(sessionName, ['focus', element])) as BrowserFocusResult
- })
+ return this.enqueueTargetedCommand(
+ worktreeId,
+ browserPageId,
+ async (sessionName) => {
+ return (await this.execAgentBrowser(sessionName, ['focus', element])) as BrowserFocusResult
+ },
+ { elements: [element] }
+ )
}
async clear(
@@ -122,7 +152,7 @@ export abstract class AgentBrowserBridgeInteractionCommands extends AgentBrowser
await this.fillExplicitContentEditable(sessionName, element, '')
return { cleared: element }
},
- { requireScopedTarget: true }
+ { requireScopedTarget: true, elements: [element] }
)
}
@@ -131,13 +161,18 @@ export abstract class AgentBrowserBridgeInteractionCommands extends AgentBrowser
worktreeId?: string,
browserPageId?: string
): Promise<BrowserSelectAllResult> {
- return this.enqueueTargetedCommand(worktreeId, browserPageId, async (sessionName) => {
- await this.execAgentBrowser(sessionName, ['focus', element])
- return (await this.execAgentBrowser(sessionName, [
- 'press',
- 'Control+a'
- ])) as BrowserSelectAllResult
- })
+ return this.enqueueTargetedCommand(
+ worktreeId,
+ browserPageId,
+ async (sessionName) => {
+ await this.execAgentBrowser(sessionName, ['focus', element])
+ return (await this.execAgentBrowser(sessionName, [
+ 'press',
+ 'Control+a'
+ ])) as BrowserSelectAllResult
+ },
+ { elements: [element] }
+ )
}
async keypress(
diff --git a/src/main/browser/agent-browser-bridge-queue.ts b/src/main/browser/agent-browser-bridge-queue.ts
index 3617b129c80..f2bc07dec8f 100644
--- a/src/main/browser/agent-browser-bridge-queue.ts
+++ b/src/main/browser/agent-browser-bridge-queue.ts
@@ -70,6 +70,9 @@ export abstract class AgentBrowserBridgeQueue extends AgentBrowserBridgeShutdown
browserPageId,
options.requireScopedTarget
)
+ if (!browserPageId && options.elements) {
+ this.assertRefsFromPage(pageId, options.elements)
+ }
const sessionName = `${ORCA_TAB_SESSION_PREFIX}${pageId}`
const agentCommand = this.browserManager.beginAgentCommand(pageId)
diff --git a/src/main/browser/agent-browser-bridge-state-commands.ts b/src/main/browser/agent-browser-bridge-state-commands.ts
index 01e12337f17..1eaa0ba4168 100644
--- a/src/main/browser/agent-browser-bridge-state-commands.ts
+++ b/src/main/browser/agent-browser-bridge-state-commands.ts
@@ -263,10 +263,39 @@ export abstract class AgentBrowserBridgeStateCommands extends AgentBrowserBridge
}
async exec(command: string, worktreeId?: string, browserPageId?: string): Promise<unknown> {
- return this.enqueueTargetedCommand(worktreeId, browserPageId, async (sessionName) => {
- // Passthrough arguments must not override Orca's selected page or CDP proxy.
- const args = stripAgentBrowserTargetArgs(parseShellArgs(command.trim()))
- return await this.execAgentBrowser(sessionName, args, { cliArgv: true })
- })
+ // Passthrough arguments must not override Orca's selected page or CDP proxy.
+ const args = stripAgentBrowserTargetArgs(parseShellArgs(command.trim()))
+ return this.enqueueTargetedCommand(
+ worktreeId,
+ browserPageId,
+ async (sessionName) => await this.execAgentBrowser(sessionName, args, { cliArgv: true }),
+ { elements: passthroughElementArgs(args) }
+ )
}
}
+
+// Commands whose first operand is an element, where agent-browser reads a bare `e2` as a ref.
+const ELEMENT_FIRST_COMMANDS = new Set([
+ 'click',
+ 'dblclick',
+ 'hover',
+ 'focus',
+ 'check',
+ 'uncheck',
+ 'fill',
+ 'type',
+ 'select',
+ 'scrollintoview',
+ 'upload',
+ 'highlight',
+ 'download',
+ 'drag'
+])
+
+// Free text such as `keyboard type e2` is not a ref; explicit `@e2`/`ref=e2` are refs anywhere.
+function passthroughElementArgs([command, ...operands]: string[]): string[] {
+ const explicit = operands.filter((arg) => arg.startsWith('@') || arg.startsWith('ref='))
+ return command && ELEMENT_FIRST_COMMANDS.has(command) && operands[0]
+ ? [operands[0], ...explicit]
+ : explicit
+}
diff --git a/src/main/browser/agent-browser-bridge-state.ts b/src/main/browser/agent-browser-bridge-state.ts
index da3fcdd4079..0d9c8d2cd2f 100644
--- a/src/main/browser/agent-browser-bridge-state.ts
+++ b/src/main/browser/agent-browser-bridge-state.ts
@@ -15,6 +15,8 @@ export abstract class AgentBrowserBridgeState {
protected activeWebContentsId: number | null = null
// A person's tab pick waiting for the agent's tab to go idle; the undefined key is the global target.
protected readonly heldPersonSelections = new Map<string | undefined, number>()
+ // Each page has its own ref map; a worktree's unpinned refs belong to the page that last minted refs.
+ protected readonly refPageByWorktree = new Map<string | undefined, string>()
protected readonly sessions = new Map<string, SessionState>()
protected readonly commandQueues = new Map<string, QueuedCommand[]>()
protected readonly processingQueues = new Set<string>()
diff --git a/src/main/browser/agent-browser-bridge-tab-routing.test.ts b/src/main/browser/agent-browser-bridge-tab-routing.test.ts
index 5911889c06e..df9db318fb3 100644
--- a/src/main/browser/agent-browser-bridge-tab-routing.test.ts
+++ b/src/main/browser/agent-browser-bridge-tab-routing.test.ts
@@ -339,6 +339,48 @@ describe('AgentBrowserBridge', () => {
])
})
+ it('rejects an unpinned ref from another tab snapshot until the current tab is snapshotted', async () => {
+ const tabs = new Map([
+ ['tab-a', 1],
+ ['tab-b', 2]
+ ])
+ const worktrees = new Map([
+ ['tab-a', 'wt-1'],
+ ['tab-b', 'wt-1']
+ ])
+ const wc1 = mockWebContents(1)
+ const wc2 = mockWebContents(2)
+ webContentsFromIdMock.mockImplementation((id: number) => (id === 1 ? wc1 : wc2))
+ const b = new AgentBrowserBridge(mockBrowserManager(tabs, worktrees))
+ b.setActiveTab(1, 'wt-1')
+ const clickSessions: string[] = []
+ answerAgentBrowserCommands(execFileMock, stdinWrites, (command, cb, args) => {
+ if (command[0] === 'click') {
+ clickSessions.push(args[args.indexOf('--session') + 1])
+ }
+ const data = command[0] === 'snapshot' ? { snapshot: '' } : {}
+ cb(null, JSON.stringify({ success: true, data }), '')
+ })
+
+ await b.snapshot('wt-1')
+ await b.tabSwitch(undefined, 'wt-1', 'tab-b')
+
+ await expect(b.click('@e2', 'wt-1')).rejects.toMatchObject({
+ code: 'browser_stale_ref',
+ message: expect.stringContaining(
+ 'Run snapshot to get refs for page tab-b, or pass --page tab-a'
+ )
+ })
+ await expect(b.exec('click e2', 'wt-1')).rejects.toMatchObject({ code: 'browser_stale_ref' })
+ await b.exec('keyboard type e2', 'wt-1')
+ expect(clickSessions).toEqual([])
+
+ await b.click('@e2', undefined, 'tab-a')
+ await b.exec('snapshot -i', 'wt-1')
+ await b.click('@e2', 'wt-1')
+ expect(clickSessions).toEqual(['orca-tab-tab-a', 'orca-tab-tab-b'])
+ })
+
it('queues tabSwitch behind in-flight commands on the current session', async () => {
const tabs = new Map([
['tab-a', 1],
diff --git a/src/main/browser/agent-browser-bridge-tabs.ts b/src/main/browser/agent-browser-bridge-tabs.ts
index 047b7fd978b..61881d32abe 100644
--- a/src/main/browser/agent-browser-bridge-tabs.ts
+++ b/src/main/browser/agent-browser-bridge-tabs.ts
@@ -53,6 +53,38 @@ export abstract class AgentBrowserBridgeTabs extends AgentBrowserBridgeState {
}
}
+ protected recordRefPage(browserPageId: string): void {
+ this.refPageByWorktree.set(
+ this.browserManager.getWorktreeIdForTab(browserPageId),
+ browserPageId
+ )
+ }
+
+ // Refs are numbered per page, so another page's ref would silently match an element here.
+ protected assertRefsFromPage(
+ browserPageId: string,
+ elements: readonly (string | undefined)[]
+ ): void {
+ const ref = elements.find((element) => element !== undefined && isSnapshotRef(element))
+ if (!ref) {
+ return
+ }
+ const snapshotPageId = this.refPageByWorktree.get(
+ this.browserManager.getWorktreeIdForTab(browserPageId)
+ )
+ if (!snapshotPageId || snapshotPageId === browserPageId) {
+ return
+ }
+ // The latest snapshot may have been a pinned look elsewhere, so the snapshot hint comes first.
+ const pinnedRetry = this.browserManager.getWebContentsIdByTabId().has(snapshotPageId)
+ ? `, or pass --page ${snapshotPageId} if ${ref} came from that page's snapshot`
+ : ''
+ throw new BrowserError(
+ 'browser_stale_ref',
+ `The latest snapshot was of page ${snapshotPageId}, not the current tab (page ${browserPageId}), so ${ref} may point at the wrong element. Run snapshot to get refs for page ${browserPageId}${pinnedRetry}.`
+ )
+ }
+
private isAgentUsingTab(webContentsId: number): boolean {
const browserPageId = this.resolveTabIdSafe(webContentsId)
return browserPageId !== null && this.browserManager.isAgentUsingPage(browserPageId)
@@ -296,3 +328,8 @@ export abstract class AgentBrowserBridgeTabs extends AgentBrowserBridgeState {
function routingKeys(worktreeId: string | undefined): (string | undefined)[] {
return worktreeId ? [undefined, worktreeId] : [undefined]
}
+
+// agent-browser reads `@e3`, `ref=e3` and bare `e3` as snapshot refs.
+function isSnapshotRef(element: string): boolean {
+ return /^(?:@|ref=)?e\d+$/.test(element)
+}
diff --git a/src/main/browser/agent-browser-bridge-types.ts b/src/main/browser/agent-browser-bridge-types.ts
index 0c77be5831d..f2834e34857 100644
--- a/src/main/browser/agent-browser-bridge-types.ts
+++ b/src/main/browser/agent-browser-bridge-types.ts
@@ -65,6 +65,8 @@ export type EnqueueTargetedCommandOptions = {
ensureSession?: boolean
// Text mutations must not fall back to a global tab from another worktree.
requireScopedTarget?: boolean
+ // Element arguments that may be snapshot refs.
+ elements?: readonly (string | undefined)[]
}
export type AgentBrowserBridgeOptions = {
diff --git a/src/main/browser/agent-browser-bridge-utility-commands.ts b/src/main/browser/agent-browser-bridge-utility-commands.ts
index e36dd0c76d1..ab4326587cb 100644
--- a/src/main/browser/agent-browser-bridge-utility-commands.ts
+++ b/src/main/browser/agent-browser-bridge-utility-commands.ts
@@ -113,45 +113,55 @@ export abstract class AgentBrowserBridgeUtilityCommands extends AgentBrowserBrid
throw new BrowserError('invalid_argument', `--path must be an absolute path: ${savePath}`)
}
const replyDeadline = Date.now() + AGENT_DOWNLOAD_REPLY_DEADLINE_MS
- return this.enqueueTargetedCommand(worktreeId, browserPageId, async (sessionName, target) => {
- await mkdir(path.dirname(savePath), { recursive: true })
- const claim = this.browserManager.claimNextDownload(target.webContentsId, savePath)
- try {
- await this.execAgentBrowser(sessionName, ['click', selector])
- if (!(await withTimeout(claim.started, AGENT_DOWNLOAD_START_TIMEOUT_MS, false))) {
- throw new BrowserError(
- 'browser_download_not_started',
- `Clicking ${selector} did not start a download within ${AGENT_DOWNLOAD_START_TIMEOUT_MS / 1000}s.`
- )
- }
- const finished = await withTimeout(
- claim.finished,
- Math.max(0, replyDeadline - Date.now()),
- null
- )
- if (!finished) {
- throw new BrowserError(
- 'browser_download_in_progress',
- `The download is still in progress and will be saved to ${savePath}.`
- )
- }
- if (finished.status !== 'completed') {
- throw new BrowserError(
- 'browser_download_failed',
- finished.error ?? `The download was ${finished.status}.`
+ return this.enqueueTargetedCommand(
+ worktreeId,
+ browserPageId,
+ async (sessionName, target) => {
+ await mkdir(path.dirname(savePath), { recursive: true })
+ const claim = this.browserManager.claimNextDownload(target.webContentsId, savePath)
+ try {
+ await this.execAgentBrowser(sessionName, ['click', selector])
+ if (!(await withTimeout(claim.started, AGENT_DOWNLOAD_START_TIMEOUT_MS, false))) {
+ throw new BrowserError(
+ 'browser_download_not_started',
+ `Clicking ${selector} did not start a download within ${AGENT_DOWNLOAD_START_TIMEOUT_MS / 1000}s.`
+ )
+ }
+ const finished = await withTimeout(
+ claim.finished,
+ Math.max(0, replyDeadline - Date.now()),
+ null
)
+ if (!finished) {
+ throw new BrowserError(
+ 'browser_download_in_progress',
+ `The download is still in progress and will be saved to ${savePath}.`
+ )
+ }
+ if (finished.status !== 'completed') {
+ throw new BrowserError(
+ 'browser_download_failed',
+ finished.error ?? `The download was ${finished.status}.`
+ )
+ }
+ return { path: savePath }
+ } finally {
+ claim.release()
}
- return { path: savePath }
- } finally {
- claim.release()
- }
- })
+ },
+ { elements: [selector] }
+ )
}
async highlight(selector: string, worktreeId?: string, browserPageId?: string): Promise<unknown> {
- return this.enqueueTargetedCommand(worktreeId, browserPageId, async (sessionName) => {
- return await this.execAgentBrowser(sessionName, ['highlight', selector])
- })
+ return this.enqueueTargetedCommand(
+ worktreeId,
+ browserPageId,
+ async (sessionName) => {
+ return await this.execAgentBrowser(sessionName, ['highlight', selector])
+ },
+ { elements: [selector] }
+ )
}
async back(worktreeId?: string, browserPageId?: string): Promise<BrowserBackResult> {