Skip to content

Open-source AI browser agent for Chrome and Firefox (monorepo) 🧠

Keep a stalled screenshot from hiding the page UI for good

Barack Sokullu07f0f22ab939
diff --git a/src/chrome/src/agent/agent.js b/src/chrome/src/agent/agent.js
index 14e16b535..55bc07bb8 100644
--- a/src/chrome/src/agent/agent.js
+++ b/src/chrome/src/agent/agent.js
@@ -19539,53 +19539,52 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d
* we capture as-is. Same on tabs the user opened before the extension
* had a chance to inject.
*
- * Calls for a tab nest: a vision screen update holds one around all of
- * its captures (vision-runtime.js _visionCaptureSettled). Every call
- * still hides, which a hidden indicator ignores but a page that
- * navigated in between needs, and the indicator comes back once, when
- * the last call ends, instead of blinking around every capture.
+ * A vision screen update keeps the indicator hidden across all of its
+ * captures (vision-runtime.js _visionCaptureSettled) and passes them its
+ * `hold`. Each capture still hides, which a hidden indicator ignores but
+ * a page that navigated in between needs, and leaves the show to the
+ * update, which sends it once at its end instead of blinking around
+ * every capture. Nothing is counted per tab: a capture that outlives its
+ * update (one _visionBounded gave up on) shows by itself if it ever
+ * finishes, and one that never finishes holds nothing back.
*
* The hide is bounded by the capture deadline and run abort signal so an
* unresponsive renderer or stop request cannot leave the run hung.
*/
- async _withIndicatorsHidden(tabId, fn) {
- const holds = (this._indicatorHolds ??= new Map());
- const hold = holds.get(tabId) || { count: 0, hidden: false };
- hold.count += 1;
- holds.set(tabId, hold);
- let hideFailed = false;
+ async _withIndicatorsHidden(tabId, fn, { hold = null } = {}) {
+ let hidden = false;
try {
try {
const sendHide = () => chrome.tabs.sendMessage(tabId, { type: 'WB_HIDE_FOR_TOOL_USE' });
const reply = typeof this._visionBounded === 'function'
? await this._visionBounded(tabId, 'screenshot', sendHide)
: await sendHide();
- hold.hidden = true;
+ hidden = true;
// One paint frame for Chrome to apply the display:none the
// content script just set, before CDP grabs the surface (none
// when it was hidden already).
if (reply?.changed !== false) await new Promise((r) => setTimeout(r, 16));
} catch (error) {
if (error?.name === 'AbortError' || error?.code === 'vision_capture_timeout' || this._runAbortSignal?.(tabId)?.aborted) {
- hideFailed = true;
+ // The hide may still arrive, so a show must follow it.
+ hidden = true;
throw error;
}
/* content script absent — that's fine, no UI to hide */
}
return await fn();
} finally {
- hold.count -= 1;
- if (hold.count === 0) {
- holds.delete(tabId);
- if (hold.hidden || hideFailed) {
- try {
- chrome.tabs.sendMessage(tabId, { type: 'WB_SHOW_AFTER_TOOL_USE' }).catch(() => {});
- } catch { /* ignore */ }
- }
- }
+ if (hidden && (!hold || hold.ended)) this._showIndicatorsAfterToolUse(tabId);
}
}
+ /** Brings back what _withIndicatorsHidden hid; with nothing hidden it changes nothing. */
+ _showIndicatorsAfterToolUse(tabId) {
+ try {
+ chrome.tabs.sendMessage(tabId, { type: 'WB_SHOW_AFTER_TOOL_USE' }).catch(() => {});
+ } catch { /* ignore */ }
+ }
+
_showAgentTarget(tabId, rect, source = 'interaction') {
if (!rect) return;
try {
@@ -21215,7 +21214,7 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d
* outWidth wide. Whatever size actually comes back is resampled to the
* plan by the runtime.
*/
- async _visionPlatformCapture(tabId, { probe, cssRegion = null, outWidth, outHeight }) {
+ async _visionPlatformCapture(tabId, { probe, cssRegion = null, outWidth, outHeight, hold = null }) {
await cdpClient.attach(tabId);
await cdpClient.sendCommand(tabId, 'Page.enable').catch(() => {});
await this._preparePageForCapture(tabId);
@@ -21234,7 +21233,7 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d
format: 'jpeg',
quality: 90,
fromSurface: true,
- }));
+ }), { hold });
if (!shot?.data) return null;
return this._visionCropImage(`data:image/jpeg;base64,${shot.data}`,
{ x: region.x * dpr, y: region.y * dpr, width: width * dpr, height: height * dpr }, outWidth, outHeight);
@@ -21252,7 +21251,7 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d
height: height * zoom,
scale,
},
- }));
+ }), { hold });
return shot?.data ? `data:image/jpeg;base64,${shot.data}` : null;
}
diff --git a/src/chrome/src/agent/vision-runtime.js b/src/chrome/src/agent/vision-runtime.js
index d83517c7a..0c7c4a4d1 100644
--- a/src/chrome/src/agent/vision-runtime.js
+++ b/src/chrome/src/agent/vision-runtime.js
@@ -599,14 +599,16 @@ export function installVisionRuntime(AgentClass, { trace = null } = {}) {
/**
* Capture the viewport at the planned model size. The platform capture may
* return any size (device pixels, emulated DPR, zoom); rasterizeFrame
- * resamples to the plan so the coordinate map is exact.
+ * resamples to the plan so the coordinate map is exact. `hold` is a screen
+ * update's (_visionCaptureSettled), which shows WebBrain's page UI again
+ * after its last capture.
*/
- proto._visionCaptureFrame = async function _visionCaptureFrame(tabId, state, { redactionRetry = true } = {}) {
+ proto._visionCaptureFrame = async function _visionCaptureFrame(tabId, state, { redactionRetry = true, hold = null } = {}) {
const probe = await this._visionBounded(tabId, 'viewport measurement', () => this._visionPlatformProbe(tabId));
if (!probe || !(probe.cssWidth > 0) || !(probe.cssHeight > 0)) throw new Error('The page viewport could not be measured.');
const profile = VISION_PROFILES[state.profileId] || VISION_PROFILES.default;
const size = planVisionImageSize({ cssWidth: probe.cssWidth, cssHeight: probe.cssHeight, profile, quality: state.quality });
- let dataUrl = await this._visionBounded(tabId, 'screenshot', () => this._visionPlatformCapture(tabId, { probe, outWidth: size.width, outHeight: size.height }));
+ let dataUrl = await this._visionBounded(tabId, 'screenshot', () => this._visionPlatformCapture(tabId, { probe, outWidth: size.width, outHeight: size.height, hold }));
if (!dataUrl) throw new Error('The browser returned no screenshot.');
let redaction = null;
if (this.screenshotRedaction && typeof this._redactScreenshotDataUrl === 'function') {
@@ -628,7 +630,7 @@ export function installVisionRuntime(AgentClass, { trace = null } = {}) {
// scan and the image show the same page.
if (outcome.status === 'scan_unavailable' && redactionRetry && await this._visionTabLoading?.(tabId) === true) {
await this._visionAwaitRedactionScan(tabId);
- return this._visionCaptureFrame(tabId, state, { redactionRetry: false });
+ return this._visionCaptureFrame(tabId, state, { redactionRetry: false, hold });
}
redaction = outcome.status || 'error';
}
@@ -743,26 +745,33 @@ export function installVisionRuntime(AgentClass, { trace = null } = {}) {
* Capture until two consecutive frames match (bounded). Micro changes (a
* caret, a clock, a spinner) count as a match: they never stop, and waiting
* for them only cost the full settle time on every step. WebBrain's own
- * page UI (border, cursor, Stop button) is hidden once for all of the
- * captures, not around each (agent _withIndicatorsHidden), so it does not
- * blink on every step.
+ * page UI (border, cursor, Stop button) stays hidden from the first
+ * capture to the last (`hold`, agent _withIndicatorsHidden) and is shown
+ * again once at the end, also when a capture fails or is given up on, so
+ * it does not blink on every step.
*/
proto._visionCaptureSettled = async function _visionCaptureSettled(tabId, state) {
await sleep(SETTLE_INITIAL_DELAY_MS);
this._throwIfAborted?.(this._runAbortSignal?.(tabId));
- return this._withIndicatorsHidden(tabId, async () => {
- let frame = await this._visionCaptureFrame(tabId, state);
+ const hold = { ended: false };
+ try {
+ let frame = await this._visionCaptureFrame(tabId, state, { hold });
for (let attempt = 0; attempt < SETTLE_MAX_EXTRA_CAPTURES; attempt++) {
await sleep(SETTLE_POLL_MS);
let next;
- try { next = await this._visionCaptureFrame(tabId, state); } catch { break; }
+ try { next = await this._visionCaptureFrame(tabId, state, { hold }); } catch { break; }
const sameShape = next.size.width === frame.size.width && next.size.height === frame.size.height;
const moving = !sameShape || !isMicroScreenChange(diffFrames(frame.raster.pixels, next.raster.pixels, next.size.width, next.size.height));
frame = next;
if (!moving) break;
}
return frame;
- });
+ } finally {
+ // Sent after every hide of the update, even one still unanswered:
+ // the page handles them in order.
+ hold.ended = true;
+ this._showIndicatorsAfterToolUse?.(tabId);
+ }
};
/**
diff --git a/src/firefox/src/agent/agent.js b/src/firefox/src/agent/agent.js
index ac159283f..9d7c76906 100644
--- a/src/firefox/src/agent/agent.js
+++ b/src/firefox/src/agent/agent.js
@@ -17519,7 +17519,7 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d
* the whole content area, classic scrollbars included, while the probe's
* visual viewport excludes them, and points would drift by the scrollbar.
*/
- async _visionPlatformCapture(tabId, { probe, cssRegion = null, outWidth, outHeight }) {
+ async _visionPlatformCapture(tabId, { probe, cssRegion = null, outWidth, outHeight, hold = null }) {
const region = cssRegion || { x: 0, y: 0, width: probe.cssWidth, height: probe.cssHeight };
const scale = Math.max(0.05, Math.min(4, Number(outWidth) / Math.max(1, region.width)));
const options = {
@@ -17533,7 +17533,7 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d
height: Math.max(1, region.height),
},
};
- return this._withIndicatorsHidden(tabId, () => browser.tabs.captureTab(tabId, options));
+ return this._withIndicatorsHidden(tabId, () => browser.tabs.captureTab(tabId, options), { hold });
}
/**
@@ -18618,53 +18618,52 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d
* pages, or pre-existing tabs that loaded before the extension), the
* sendMessage rejects and we capture as-is.
*
- * Calls for a tab nest: a vision screen update holds one around all of
- * its captures (vision-runtime.js _visionCaptureSettled). Every call
- * still hides, which a hidden indicator ignores but a page that
- * navigated in between needs, and the indicator comes back once, when
- * the last call ends, instead of blinking around every capture.
+ * A vision screen update keeps the indicator hidden across all of its
+ * captures (vision-runtime.js _visionCaptureSettled) and passes them its
+ * `hold`. Each capture still hides, which a hidden indicator ignores but
+ * a page that navigated in between needs, and leaves the show to the
+ * update, which sends it once at its end instead of blinking around
+ * every capture. Nothing is counted per tab: a capture that outlives its
+ * update (one _visionBounded gave up on) shows by itself if it ever
+ * finishes, and one that never finishes holds nothing back.
*
* The hide is bounded by the capture deadline and run abort signal so an
* unresponsive renderer or stop request cannot leave the run hung.
*/
- async _withIndicatorsHidden(tabId, fn) {
- const holds = (this._indicatorHolds ??= new Map());
- const hold = holds.get(tabId) || { count: 0, hidden: false };
- hold.count += 1;
- holds.set(tabId, hold);
- let hideFailed = false;
+ async _withIndicatorsHidden(tabId, fn, { hold = null } = {}) {
+ let hidden = false;
try {
try {
const sendHide = () => browser.tabs.sendMessage(tabId, { type: 'WB_HIDE_FOR_TOOL_USE' });
const reply = typeof this._visionBounded === 'function'
? await this._visionBounded(tabId, 'screenshot', sendHide)
: await sendHide();
- hold.hidden = true;
+ hidden = true;
// Give the renderer one paint frame to apply the display:none the
// content script just set, before we capture (none when it was
// hidden already).
if (reply?.changed !== false) await new Promise((r) => setTimeout(r, 16));
} catch (error) {
if (error?.name === 'AbortError' || error?.code === 'vision_capture_timeout' || this._runAbortSignal?.(tabId)?.aborted) {
- hideFailed = true;
+ // The hide may still arrive, so a show must follow it.
+ hidden = true;
throw error;
}
/* content script absent — capture without hiding */
}
return await fn();
} finally {
- hold.count -= 1;
- if (hold.count === 0) {
- holds.delete(tabId);
- if (hold.hidden || hideFailed) {
- try {
- browser.tabs.sendMessage(tabId, { type: 'WB_SHOW_AFTER_TOOL_USE' }).catch(() => {});
- } catch { /* ignore */ }
- }
- }
+ if (hidden && (!hold || hold.ended)) this._showIndicatorsAfterToolUse(tabId);
}
}
+ /** Brings back what _withIndicatorsHidden hid; with nothing hidden it changes nothing. */
+ _showIndicatorsAfterToolUse(tabId) {
+ try {
+ browser.tabs.sendMessage(tabId, { type: 'WB_SHOW_AFTER_TOOL_USE' }).catch(() => {});
+ } catch { /* ignore */ }
+ }
+
async _captureViewportProbe(tabId) {
try {
const probeCode = `
diff --git a/src/firefox/src/agent/vision-runtime.js b/src/firefox/src/agent/vision-runtime.js
index d83517c7a..0c7c4a4d1 100644
--- a/src/firefox/src/agent/vision-runtime.js
+++ b/src/firefox/src/agent/vision-runtime.js
@@ -599,14 +599,16 @@ export function installVisionRuntime(AgentClass, { trace = null } = {}) {
/**
* Capture the viewport at the planned model size. The platform capture may
* return any size (device pixels, emulated DPR, zoom); rasterizeFrame
- * resamples to the plan so the coordinate map is exact.
+ * resamples to the plan so the coordinate map is exact. `hold` is a screen
+ * update's (_visionCaptureSettled), which shows WebBrain's page UI again
+ * after its last capture.
*/
- proto._visionCaptureFrame = async function _visionCaptureFrame(tabId, state, { redactionRetry = true } = {}) {
+ proto._visionCaptureFrame = async function _visionCaptureFrame(tabId, state, { redactionRetry = true, hold = null } = {}) {
const probe = await this._visionBounded(tabId, 'viewport measurement', () => this._visionPlatformProbe(tabId));
if (!probe || !(probe.cssWidth > 0) || !(probe.cssHeight > 0)) throw new Error('The page viewport could not be measured.');
const profile = VISION_PROFILES[state.profileId] || VISION_PROFILES.default;
const size = planVisionImageSize({ cssWidth: probe.cssWidth, cssHeight: probe.cssHeight, profile, quality: state.quality });
- let dataUrl = await this._visionBounded(tabId, 'screenshot', () => this._visionPlatformCapture(tabId, { probe, outWidth: size.width, outHeight: size.height }));
+ let dataUrl = await this._visionBounded(tabId, 'screenshot', () => this._visionPlatformCapture(tabId, { probe, outWidth: size.width, outHeight: size.height, hold }));
if (!dataUrl) throw new Error('The browser returned no screenshot.');
let redaction = null;
if (this.screenshotRedaction && typeof this._redactScreenshotDataUrl === 'function') {
@@ -628,7 +630,7 @@ export function installVisionRuntime(AgentClass, { trace = null } = {}) {
// scan and the image show the same page.
if (outcome.status === 'scan_unavailable' && redactionRetry && await this._visionTabLoading?.(tabId) === true) {
await this._visionAwaitRedactionScan(tabId);
- return this._visionCaptureFrame(tabId, state, { redactionRetry: false });
+ return this._visionCaptureFrame(tabId, state, { redactionRetry: false, hold });
}
redaction = outcome.status || 'error';
}
@@ -743,26 +745,33 @@ export function installVisionRuntime(AgentClass, { trace = null } = {}) {
* Capture until two consecutive frames match (bounded). Micro changes (a
* caret, a clock, a spinner) count as a match: they never stop, and waiting
* for them only cost the full settle time on every step. WebBrain's own
- * page UI (border, cursor, Stop button) is hidden once for all of the
- * captures, not around each (agent _withIndicatorsHidden), so it does not
- * blink on every step.
+ * page UI (border, cursor, Stop button) stays hidden from the first
+ * capture to the last (`hold`, agent _withIndicatorsHidden) and is shown
+ * again once at the end, also when a capture fails or is given up on, so
+ * it does not blink on every step.
*/
proto._visionCaptureSettled = async function _visionCaptureSettled(tabId, state) {
await sleep(SETTLE_INITIAL_DELAY_MS);
this._throwIfAborted?.(this._runAbortSignal?.(tabId));
- return this._withIndicatorsHidden(tabId, async () => {
- let frame = await this._visionCaptureFrame(tabId, state);
+ const hold = { ended: false };
+ try {
+ let frame = await this._visionCaptureFrame(tabId, state, { hold });
for (let attempt = 0; attempt < SETTLE_MAX_EXTRA_CAPTURES; attempt++) {
await sleep(SETTLE_POLL_MS);
let next;
- try { next = await this._visionCaptureFrame(tabId, state); } catch { break; }
+ try { next = await this._visionCaptureFrame(tabId, state, { hold }); } catch { break; }
const sameShape = next.size.width === frame.size.width && next.size.height === frame.size.height;
const moving = !sameShape || !isMicroScreenChange(diffFrames(frame.raster.pixels, next.raster.pixels, next.size.width, next.size.height));
frame = next;
if (!moving) break;
}
return frame;
- });
+ } finally {
+ // Sent after every hide of the update, even one still unanswered:
+ // the page handles them in order.
+ hold.ended = true;
+ this._showIndicatorsAfterToolUse?.(tabId);
+ }
};
/**
diff --git a/test/vision-mode-agent.mjs b/test/vision-mode-agent.mjs
index 64152455c..79e1b45de 100644
--- a/test/vision-mode-agent.mjs
+++ b/test/vision-mode-agent.mjs
@@ -1396,7 +1396,7 @@ for (const build of (process.env.VISION_BUILDS || 'chrome,firefox').split(','))
log.push(`capture, hidden: ${hidden}`);
if (log.filter(entry => entry.startsWith('capture')).length === 2) hidden = false;
return capture.call(this, tabId, options);
- });
+ }, { hold: options?.hold });
};
try {
await agent._visionCaptureSettled(tab, state);
@@ -1407,10 +1407,52 @@ for (const build of (process.env.VISION_BUILDS || 'chrome,firefox').split(','))
assert.deepEqual(captures, Array(4).fill('capture, hidden: true'), log.join('\n'));
assert.equal(log.filter(entry => entry === 'WB_SHOW_AFTER_TOOL_USE').length, 1, log.join('\n'));
assert.equal(log.at(-1), 'WB_SHOW_AFTER_TOOL_USE', 'shown again after the last capture');
- assert.equal(agent._indicatorHolds?.has(tab), false);
});
- test(`${build}: indicator hide respects capture deadline and abort, cleaning up holds`, async () => {
+ test(`${build}: a screenshot that never answers does not keep WebBrain's page UI hidden`, async () => {
+ const scene = tickingScene({ x: 300, y: 200, w: 400, h: 300 });
+ const { agent } = runAgent(() => ({ content: 'done' }), { scene });
+ const tab = nextTab++;
+ const state = agent._visionRunState(tab, 'act');
+ const tabs = (build === 'chrome' ? globalThis.chrome : globalThis.browser).tabs;
+ const send = tabs.sendMessage;
+ let hidden = false;
+ tabs.sendMessage = async (id, message) => {
+ if (id !== tab) return send(id, message);
+ if (message.type === 'WB_HIDE_FOR_TOOL_USE') {
+ const changed = !hidden;
+ hidden = true;
+ return { ok: true, changed };
+ }
+ if (message.type === 'WB_SHOW_AFTER_TOOL_USE') hidden = false;
+ return { ok: true };
+ };
+ // The hide answers, then the first screenshot never does (a stalled
+ // debugger), and the capture deadline gives up on it.
+ const capture = agent._visionPlatformCapture;
+ let stalled = false;
+ agent._visionPlatformCapture = function (tabId, options) {
+ return this._withIndicatorsHidden(tabId, () => {
+ if (stalled) return capture.call(this, tabId, options);
+ stalled = true;
+ return new Promise(() => {});
+ }, { hold: options?.hold });
+ };
+ agent._visionCaptureDeadlineMs = 50;
+ try {
+ await assert.rejects(() => agent._visionCaptureSettled(tab, state), (err) => err.code === 'vision_capture_timeout');
+ assert.equal(hidden, false, 'shown again when the screen update gave up');
+ agent._visionCaptureDeadlineMs = 0;
+ await agent._visionCaptureSettled(tab, state);
+ assert.equal(hidden, false, 'shown again after the next screen update');
+ await agent._visionCaptureFrame(tab, state);
+ assert.equal(hidden, false, 'shown again after a capture of its own');
+ } finally {
+ tabs.sendMessage = send;
+ }
+ });
+
+ test(`${build}: indicator hide respects capture deadline and abort, and the page UI is shown again`, async () => {
const scene = tickingScene({ x: 300, y: 200, w: 400, h: 300 });
const { agent } = runAgent(() => ({ content: 'done' }), { scene });
const tab = nextTab++;
@@ -1426,13 +1468,17 @@ for (const build of (process.env.VISION_BUILDS || 'chrome,firefox').split(','))
}
return { ok: true };
};
+ // Each capture hides around itself, as both platform layers do.
+ const capture = agent._visionPlatformCapture;
+ agent._visionPlatformCapture = function (tabId, options) {
+ return this._withIndicatorsHidden(tabId, () => capture.call(this, tabId, options), { hold: options?.hold });
+ };
agent._visionCaptureDeadlineMs = 50;
try {
await assert.rejects(
() => agent._visionCaptureSettled(tab, state),
(err) => err.code === 'vision_capture_timeout'
);
- assert.equal(agent._indicatorHolds?.has(tab), false, 'hold cleaned up on timeout');
assert.ok(log.includes('WB_SHOW_AFTER_TOOL_USE'), 'show attempted on timeout');
} finally {
tabs.sendMessage = send;
@@ -1457,7 +1503,6 @@ for (const build of (process.env.VISION_BUILDS || 'chrome,firefox').split(','))
() => agent._visionCaptureSettled(abortTab, abortState),
(err) => err.name === 'AbortError'
);
- assert.equal(agent._indicatorHolds?.has(abortTab), false, 'hold cleaned up on abort');
assert.ok(abortLog.includes('WB_SHOW_AFTER_TOOL_USE'), 'show attempted on abort');
} finally {
tabs.sendMessage = send;