Open-source AI browser agent for Chrome and Firefox (monorepo) π§
JavaScript ai-agentai-agentsai-toolsbrowser-aibrowser-automationbrowser-extensionchrome-aichrome-devtoolschrome-extensionchrome-extensionsfirefox-addonfirefox-addonsfirefox-extensionfirefox-extensionsgemmagemma4minimaxoffline-aioffline-llmqwen
Bound indicator hide requests by capture deadline and abort signal
Barack Sokullu5fc8fd119107
diff --git a/src/chrome/src/agent/agent.js b/src/chrome/src/agent/agent.jsindex b27be85c8..14e16b535 100644--- a/src/chrome/src/agent/agent.js+++ b/src/chrome/src/agent/agent.js@@ -19544,27 +19544,40 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d* still hides, which a hidden indicator ignores but a page that* navigated in between needs, and the indicator comes back once, when* the last call ends, instead of blinking around every capture.+ *+ * The hide is bounded by the capture deadline and run abort signal so an+ * unresponsive renderer or stop request cannot leave the run hung.*/async _withIndicatorsHidden(tabId, fn) {const holds = (this._indicatorHolds ??= new Map());const hold = holds.get(tabId) || { count: 0, hidden: false };hold.count += 1;holds.set(tabId, hold);+ let hideFailed = false;try {try {- const reply = await chrome.tabs.sendMessage(tabId, { type: 'WB_HIDE_FOR_TOOL_USE' });+ const sendHide = () => chrome.tabs.sendMessage(tabId, { type: 'WB_HIDE_FOR_TOOL_USE' });+ const reply = typeof this._visionBounded === 'function'+ ? await this._visionBounded(tabId, 'screenshot', sendHide)+ : await sendHide();hold.hidden = true;// One paint frame for Chrome to apply the display:none the// content script just set, before CDP grabs the surface (none// when it was hidden already).if (reply?.changed !== false) await new Promise((r) => setTimeout(r, 16));- } catch { /* content script absent β that's fine, no UI to hide */ }+ } catch (error) {+ if (error?.name === 'AbortError' || error?.code === 'vision_capture_timeout' || this._runAbortSignal?.(tabId)?.aborted) {+ hideFailed = true;+ throw error;+ }+ /* content script absent β that's fine, no UI to hide */+ }return await fn();} finally {hold.count -= 1;if (hold.count === 0) {holds.delete(tabId);- if (hold.hidden) {+ if (hold.hidden || hideFailed) {try {chrome.tabs.sendMessage(tabId, { type: 'WB_SHOW_AFTER_TOOL_USE' }).catch(() => {});} catch { /* ignore */ }diff --git a/src/chrome/src/agent/vision-runtime.js b/src/chrome/src/agent/vision-runtime.jsindex cf9cd6d1a..d83517c7a 100644--- a/src/chrome/src/agent/vision-runtime.js+++ b/src/chrome/src/agent/vision-runtime.js@@ -749,6 +749,7 @@ export function installVisionRuntime(AgentClass, { trace = null } = {}) {*/proto._visionCaptureSettled = async function _visionCaptureSettled(tabId, state) {await sleep(SETTLE_INITIAL_DELAY_MS);+ this._throwIfAborted?.(this._runAbortSignal?.(tabId));return this._withIndicatorsHidden(tabId, async () => {let frame = await this._visionCaptureFrame(tabId, state);for (let attempt = 0; attempt < SETTLE_MAX_EXTRA_CAPTURES; attempt++) {diff --git a/src/firefox/src/agent/agent.js b/src/firefox/src/agent/agent.jsindex 72ba562fe..ac159283f 100644--- a/src/firefox/src/agent/agent.js+++ b/src/firefox/src/agent/agent.js@@ -18623,27 +18623,40 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d* still hides, which a hidden indicator ignores but a page that* navigated in between needs, and the indicator comes back once, when* the last call ends, instead of blinking around every capture.+ *+ * The hide is bounded by the capture deadline and run abort signal so an+ * unresponsive renderer or stop request cannot leave the run hung.*/async _withIndicatorsHidden(tabId, fn) {const holds = (this._indicatorHolds ??= new Map());const hold = holds.get(tabId) || { count: 0, hidden: false };hold.count += 1;holds.set(tabId, hold);+ let hideFailed = false;try {try {- const reply = await browser.tabs.sendMessage(tabId, { type: 'WB_HIDE_FOR_TOOL_USE' });+ const sendHide = () => browser.tabs.sendMessage(tabId, { type: 'WB_HIDE_FOR_TOOL_USE' });+ const reply = typeof this._visionBounded === 'function'+ ? await this._visionBounded(tabId, 'screenshot', sendHide)+ : await sendHide();hold.hidden = true;// Give the renderer one paint frame to apply the display:none the// content script just set, before we capture (none when it was// hidden already).if (reply?.changed !== false) await new Promise((r) => setTimeout(r, 16));- } catch { /* content script absent β capture without hiding */ }+ } catch (error) {+ if (error?.name === 'AbortError' || error?.code === 'vision_capture_timeout' || this._runAbortSignal?.(tabId)?.aborted) {+ hideFailed = true;+ throw error;+ }+ /* content script absent β capture without hiding */+ }return await fn();} finally {hold.count -= 1;if (hold.count === 0) {holds.delete(tabId);- if (hold.hidden) {+ if (hold.hidden || hideFailed) {try {browser.tabs.sendMessage(tabId, { type: 'WB_SHOW_AFTER_TOOL_USE' }).catch(() => {});} catch { /* ignore */ }diff --git a/src/firefox/src/agent/vision-runtime.js b/src/firefox/src/agent/vision-runtime.jsindex cf9cd6d1a..d83517c7a 100644--- a/src/firefox/src/agent/vision-runtime.js+++ b/src/firefox/src/agent/vision-runtime.js@@ -749,6 +749,7 @@ export function installVisionRuntime(AgentClass, { trace = null } = {}) {*/proto._visionCaptureSettled = async function _visionCaptureSettled(tabId, state) {await sleep(SETTLE_INITIAL_DELAY_MS);+ this._throwIfAborted?.(this._runAbortSignal?.(tabId));return this._withIndicatorsHidden(tabId, async () => {let frame = await this._visionCaptureFrame(tabId, state);for (let attempt = 0; attempt < SETTLE_MAX_EXTRA_CAPTURES; attempt++) {diff --git a/test/vision-mode-agent.mjs b/test/vision-mode-agent.mjsindex cc52e3abc..64152455c 100644--- a/test/vision-mode-agent.mjs+++ b/test/vision-mode-agent.mjs@@ -1410,6 +1410,60 @@ for (const build of (process.env.VISION_BUILDS || 'chrome,firefox').split(','))assert.equal(agent._indicatorHolds?.has(tab), false);});+ test(`${build}: indicator hide respects capture deadline and abort, cleaning up holds`, async () => {+ const scene = tickingScene({ x: 300, y: 200, w: 400, h: 300 });+ const { agent } = runAgent(() => ({ content: 'done' }), { scene });+ const tab = nextTab++;+ const state = agent._visionRunState(tab, 'act');+ const tabs = (build === 'chrome' ? globalThis.chrome : globalThis.browser).tabs;+ const send = tabs.sendMessage;+ const log = [];+ tabs.sendMessage = async (id, message) => {+ if (id !== tab) return send(id, message);+ log.push(message.type);+ if (message.type === 'WB_HIDE_FOR_TOOL_USE') {+ return new Promise(() => {});+ }+ return { ok: true };+ };+ agent._visionCaptureDeadlineMs = 50;+ try {+ await assert.rejects(+ () => agent._visionCaptureSettled(tab, state),+ (err) => err.code === 'vision_capture_timeout'+ );+ assert.equal(agent._indicatorHolds?.has(tab), false, 'hold cleaned up on timeout');+ assert.ok(log.includes('WB_SHOW_AFTER_TOOL_USE'), 'show attempted on timeout');+ } finally {+ tabs.sendMessage = send;+ }++ const abortTab = nextTab++;+ const abortState = agent._visionRunState(abortTab, 'act');+ const abortLog = [];+ const controller = new AbortController();+ agent._runAbortSignal = (id) => (id === abortTab ? controller.signal : null);+ tabs.sendMessage = async (id, message) => {+ if (id !== abortTab) return send(id, message);+ abortLog.push(message.type);+ if (message.type === 'WB_HIDE_FOR_TOOL_USE') {+ controller.abort();+ return new Promise(() => {});+ }+ return { ok: true };+ };+ try {+ await assert.rejects(+ () => agent._visionCaptureSettled(abortTab, abortState),+ (err) => err.name === 'AbortError'+ );+ assert.equal(agent._indicatorHolds?.has(abortTab), false, 'hold cleaned up on abort');+ assert.ok(abortLog.includes('WB_SHOW_AFTER_TOOL_USE'), 'show attempted on abort');+ } finally {+ tabs.sendMessage = send;+ }+ });+if (build === 'chrome') {test('chrome: a vision capture reads the tab surface as it is and crops it; only a zoom needing more detail than the screen has, or a pinch-zoomed page, uses a clip', async () => {const { agent } = runAgent(() => ({ content: 'done' }));