Skip to content

Open-source AI browser agent for Chrome and Firefox (monorepo) 🧠

Bound indicator hide requests by capture deadline and abort signal

Barack Sokullu5fc8fd119107
diff --git a/src/chrome/src/agent/agent.js b/src/chrome/src/agent/agent.js
index b27be85c8..14e16b535 100644
--- a/src/chrome/src/agent/agent.js
+++ b/src/chrome/src/agent/agent.js
@@ -19544,27 +19544,40 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d
* still hides, which a hidden indicator ignores but a page that
* navigated in between needs, and the indicator comes back once, when
* the last call ends, instead of blinking around every capture.
+ *
+ * The hide is bounded by the capture deadline and run abort signal so an
+ * unresponsive renderer or stop request cannot leave the run hung.
*/
async _withIndicatorsHidden(tabId, fn) {
const holds = (this._indicatorHolds ??= new Map());
const hold = holds.get(tabId) || { count: 0, hidden: false };
hold.count += 1;
holds.set(tabId, hold);
+ let hideFailed = false;
try {
try {
- const reply = await chrome.tabs.sendMessage(tabId, { type: 'WB_HIDE_FOR_TOOL_USE' });
+ const sendHide = () => chrome.tabs.sendMessage(tabId, { type: 'WB_HIDE_FOR_TOOL_USE' });
+ const reply = typeof this._visionBounded === 'function'
+ ? await this._visionBounded(tabId, 'screenshot', sendHide)
+ : await sendHide();
hold.hidden = true;
// One paint frame for Chrome to apply the display:none the
// content script just set, before CDP grabs the surface (none
// when it was hidden already).
if (reply?.changed !== false) await new Promise((r) => setTimeout(r, 16));
- } catch { /* content script absent β€” that's fine, no UI to hide */ }
+ } catch (error) {
+ if (error?.name === 'AbortError' || error?.code === 'vision_capture_timeout' || this._runAbortSignal?.(tabId)?.aborted) {
+ hideFailed = true;
+ throw error;
+ }
+ /* content script absent β€” that's fine, no UI to hide */
+ }
return await fn();
} finally {
hold.count -= 1;
if (hold.count === 0) {
holds.delete(tabId);
- if (hold.hidden) {
+ if (hold.hidden || hideFailed) {
try {
chrome.tabs.sendMessage(tabId, { type: 'WB_SHOW_AFTER_TOOL_USE' }).catch(() => {});
} catch { /* ignore */ }
diff --git a/src/chrome/src/agent/vision-runtime.js b/src/chrome/src/agent/vision-runtime.js
index cf9cd6d1a..d83517c7a 100644
--- a/src/chrome/src/agent/vision-runtime.js
+++ b/src/chrome/src/agent/vision-runtime.js
@@ -749,6 +749,7 @@ export function installVisionRuntime(AgentClass, { trace = null } = {}) {
*/
proto._visionCaptureSettled = async function _visionCaptureSettled(tabId, state) {
await sleep(SETTLE_INITIAL_DELAY_MS);
+ this._throwIfAborted?.(this._runAbortSignal?.(tabId));
return this._withIndicatorsHidden(tabId, async () => {
let frame = await this._visionCaptureFrame(tabId, state);
for (let attempt = 0; attempt < SETTLE_MAX_EXTRA_CAPTURES; attempt++) {
diff --git a/src/firefox/src/agent/agent.js b/src/firefox/src/agent/agent.js
index 72ba562fe..ac159283f 100644
--- a/src/firefox/src/agent/agent.js
+++ b/src/firefox/src/agent/agent.js
@@ -18623,27 +18623,40 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d
* still hides, which a hidden indicator ignores but a page that
* navigated in between needs, and the indicator comes back once, when
* the last call ends, instead of blinking around every capture.
+ *
+ * The hide is bounded by the capture deadline and run abort signal so an
+ * unresponsive renderer or stop request cannot leave the run hung.
*/
async _withIndicatorsHidden(tabId, fn) {
const holds = (this._indicatorHolds ??= new Map());
const hold = holds.get(tabId) || { count: 0, hidden: false };
hold.count += 1;
holds.set(tabId, hold);
+ let hideFailed = false;
try {
try {
- const reply = await browser.tabs.sendMessage(tabId, { type: 'WB_HIDE_FOR_TOOL_USE' });
+ const sendHide = () => browser.tabs.sendMessage(tabId, { type: 'WB_HIDE_FOR_TOOL_USE' });
+ const reply = typeof this._visionBounded === 'function'
+ ? await this._visionBounded(tabId, 'screenshot', sendHide)
+ : await sendHide();
hold.hidden = true;
// Give the renderer one paint frame to apply the display:none the
// content script just set, before we capture (none when it was
// hidden already).
if (reply?.changed !== false) await new Promise((r) => setTimeout(r, 16));
- } catch { /* content script absent β€” capture without hiding */ }
+ } catch (error) {
+ if (error?.name === 'AbortError' || error?.code === 'vision_capture_timeout' || this._runAbortSignal?.(tabId)?.aborted) {
+ hideFailed = true;
+ throw error;
+ }
+ /* content script absent β€” capture without hiding */
+ }
return await fn();
} finally {
hold.count -= 1;
if (hold.count === 0) {
holds.delete(tabId);
- if (hold.hidden) {
+ if (hold.hidden || hideFailed) {
try {
browser.tabs.sendMessage(tabId, { type: 'WB_SHOW_AFTER_TOOL_USE' }).catch(() => {});
} catch { /* ignore */ }
diff --git a/src/firefox/src/agent/vision-runtime.js b/src/firefox/src/agent/vision-runtime.js
index cf9cd6d1a..d83517c7a 100644
--- a/src/firefox/src/agent/vision-runtime.js
+++ b/src/firefox/src/agent/vision-runtime.js
@@ -749,6 +749,7 @@ export function installVisionRuntime(AgentClass, { trace = null } = {}) {
*/
proto._visionCaptureSettled = async function _visionCaptureSettled(tabId, state) {
await sleep(SETTLE_INITIAL_DELAY_MS);
+ this._throwIfAborted?.(this._runAbortSignal?.(tabId));
return this._withIndicatorsHidden(tabId, async () => {
let frame = await this._visionCaptureFrame(tabId, state);
for (let attempt = 0; attempt < SETTLE_MAX_EXTRA_CAPTURES; attempt++) {
diff --git a/test/vision-mode-agent.mjs b/test/vision-mode-agent.mjs
index cc52e3abc..64152455c 100644
--- a/test/vision-mode-agent.mjs
+++ b/test/vision-mode-agent.mjs
@@ -1410,6 +1410,60 @@ for (const build of (process.env.VISION_BUILDS || 'chrome,firefox').split(','))
assert.equal(agent._indicatorHolds?.has(tab), false);
});
+ test(`${build}: indicator hide respects capture deadline and abort, cleaning up holds`, async () => {
+ const scene = tickingScene({ x: 300, y: 200, w: 400, h: 300 });
+ const { agent } = runAgent(() => ({ content: 'done' }), { scene });
+ const tab = nextTab++;
+ const state = agent._visionRunState(tab, 'act');
+ const tabs = (build === 'chrome' ? globalThis.chrome : globalThis.browser).tabs;
+ const send = tabs.sendMessage;
+ const log = [];
+ tabs.sendMessage = async (id, message) => {
+ if (id !== tab) return send(id, message);
+ log.push(message.type);
+ if (message.type === 'WB_HIDE_FOR_TOOL_USE') {
+ return new Promise(() => {});
+ }
+ return { ok: true };
+ };
+ agent._visionCaptureDeadlineMs = 50;
+ try {
+ await assert.rejects(
+ () => agent._visionCaptureSettled(tab, state),
+ (err) => err.code === 'vision_capture_timeout'
+ );
+ assert.equal(agent._indicatorHolds?.has(tab), false, 'hold cleaned up on timeout');
+ assert.ok(log.includes('WB_SHOW_AFTER_TOOL_USE'), 'show attempted on timeout');
+ } finally {
+ tabs.sendMessage = send;
+ }
+
+ const abortTab = nextTab++;
+ const abortState = agent._visionRunState(abortTab, 'act');
+ const abortLog = [];
+ const controller = new AbortController();
+ agent._runAbortSignal = (id) => (id === abortTab ? controller.signal : null);
+ tabs.sendMessage = async (id, message) => {
+ if (id !== abortTab) return send(id, message);
+ abortLog.push(message.type);
+ if (message.type === 'WB_HIDE_FOR_TOOL_USE') {
+ controller.abort();
+ return new Promise(() => {});
+ }
+ return { ok: true };
+ };
+ try {
+ await assert.rejects(
+ () => agent._visionCaptureSettled(abortTab, abortState),
+ (err) => err.name === 'AbortError'
+ );
+ assert.equal(agent._indicatorHolds?.has(abortTab), false, 'hold cleaned up on abort');
+ assert.ok(abortLog.includes('WB_SHOW_AFTER_TOOL_USE'), 'show attempted on abort');
+ } finally {
+ tabs.sendMessage = send;
+ }
+ });
+
if (build === 'chrome') {
test('chrome: a vision capture reads the tab surface as it is and crops it; only a zoom needing more detail than the screen has, or a pinch-zoomed page, uses a clip', async () => {
const { agent } = runAgent(() => ({ content: 'done' }));