Open-source AI browser agent for Chrome and Firefox (monorepo) ðŸ§
JavaScript ai-agentai-agentsai-toolsbrowser-aibrowser-automationbrowser-extensionchrome-aichrome-devtoolschrome-extensionchrome-extensionsfirefox-addonfirefox-addonsfirefox-extensionfirefox-extensionsgemmagemma4minimaxoffline-aioffline-llmqwen
Keep a stalled screenshot from hiding the page UI for good
Barack Sokullu07f0f22ab939
diff --git a/src/chrome/src/agent/agent.js b/src/chrome/src/agent/agent.jsindex 14e16b535..55bc07bb8 100644--- a/src/chrome/src/agent/agent.js+++ b/src/chrome/src/agent/agent.js@@ -19539,53 +19539,52 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d* we capture as-is. Same on tabs the user opened before the extension* had a chance to inject.*- * Calls for a tab nest: a vision screen update holds one around all of- * its captures (vision-runtime.js _visionCaptureSettled). Every call- * still hides, which a hidden indicator ignores but a page that- * navigated in between needs, and the indicator comes back once, when- * the last call ends, instead of blinking around every capture.+ * A vision screen update keeps the indicator hidden across all of its+ * captures (vision-runtime.js _visionCaptureSettled) and passes them its+ * `hold`. Each capture still hides, which a hidden indicator ignores but+ * a page that navigated in between needs, and leaves the show to the+ * update, which sends it once at its end instead of blinking around+ * every capture. Nothing is counted per tab: a capture that outlives its+ * update (one _visionBounded gave up on) shows by itself if it ever+ * finishes, and one that never finishes holds nothing back.** The hide is bounded by the capture deadline and run abort signal so an* unresponsive renderer or stop request cannot leave the run hung.*/- async _withIndicatorsHidden(tabId, fn) {- const holds = (this._indicatorHolds ??= new Map());- const hold = holds.get(tabId) || { count: 0, hidden: false };- hold.count += 1;- holds.set(tabId, hold);- let hideFailed = false;+ async _withIndicatorsHidden(tabId, fn, { hold = null } = {}) {+ let hidden = false;try {try {const sendHide = () => chrome.tabs.sendMessage(tabId, { type: 'WB_HIDE_FOR_TOOL_USE' });const reply = typeof this._visionBounded === 'function'? await this._visionBounded(tabId, 'screenshot', sendHide): await sendHide();- hold.hidden = true;+ hidden = true;// One paint frame for Chrome to apply the display:none the// content script just set, before CDP grabs the surface (none// when it was hidden already).if (reply?.changed !== false) await new Promise((r) => setTimeout(r, 16));} catch (error) {if (error?.name === 'AbortError' || error?.code === 'vision_capture_timeout' || this._runAbortSignal?.(tabId)?.aborted) {- hideFailed = true;+ // The hide may still arrive, so a show must follow it.+ hidden = true;throw error;}/* content script absent — that's fine, no UI to hide */}return await fn();} finally {- hold.count -= 1;- if (hold.count === 0) {- holds.delete(tabId);- if (hold.hidden || hideFailed) {- try {- chrome.tabs.sendMessage(tabId, { type: 'WB_SHOW_AFTER_TOOL_USE' }).catch(() => {});- } catch { /* ignore */ }- }- }+ if (hidden && (!hold || hold.ended)) this._showIndicatorsAfterToolUse(tabId);}}+ /** Brings back what _withIndicatorsHidden hid; with nothing hidden it changes nothing. */+ _showIndicatorsAfterToolUse(tabId) {+ try {+ chrome.tabs.sendMessage(tabId, { type: 'WB_SHOW_AFTER_TOOL_USE' }).catch(() => {});+ } catch { /* ignore */ }+ }+_showAgentTarget(tabId, rect, source = 'interaction') {if (!rect) return;try {@@ -21215,7 +21214,7 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d* outWidth wide. Whatever size actually comes back is resampled to the* plan by the runtime.*/- async _visionPlatformCapture(tabId, { probe, cssRegion = null, outWidth, outHeight }) {+ async _visionPlatformCapture(tabId, { probe, cssRegion = null, outWidth, outHeight, hold = null }) {await cdpClient.attach(tabId);await cdpClient.sendCommand(tabId, 'Page.enable').catch(() => {});await this._preparePageForCapture(tabId);@@ -21234,7 +21233,7 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout dformat: 'jpeg',quality: 90,fromSurface: true,- }));+ }), { hold });if (!shot?.data) return null;return this._visionCropImage(`data:image/jpeg;base64,${shot.data}`,{ x: region.x * dpr, y: region.y * dpr, width: width * dpr, height: height * dpr }, outWidth, outHeight);@@ -21252,7 +21251,7 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout dheight: height * zoom,scale,},- }));+ }), { hold });return shot?.data ? `data:image/jpeg;base64,${shot.data}` : null;}diff --git a/src/chrome/src/agent/vision-runtime.js b/src/chrome/src/agent/vision-runtime.jsindex d83517c7a..0c7c4a4d1 100644--- a/src/chrome/src/agent/vision-runtime.js+++ b/src/chrome/src/agent/vision-runtime.js@@ -599,14 +599,16 @@ export function installVisionRuntime(AgentClass, { trace = null } = {}) {/*** Capture the viewport at the planned model size. The platform capture may* return any size (device pixels, emulated DPR, zoom); rasterizeFrame- * resamples to the plan so the coordinate map is exact.+ * resamples to the plan so the coordinate map is exact. `hold` is a screen+ * update's (_visionCaptureSettled), which shows WebBrain's page UI again+ * after its last capture.*/- proto._visionCaptureFrame = async function _visionCaptureFrame(tabId, state, { redactionRetry = true } = {}) {+ proto._visionCaptureFrame = async function _visionCaptureFrame(tabId, state, { redactionRetry = true, hold = null } = {}) {const probe = await this._visionBounded(tabId, 'viewport measurement', () => this._visionPlatformProbe(tabId));if (!probe || !(probe.cssWidth > 0) || !(probe.cssHeight > 0)) throw new Error('The page viewport could not be measured.');const profile = VISION_PROFILES[state.profileId] || VISION_PROFILES.default;const size = planVisionImageSize({ cssWidth: probe.cssWidth, cssHeight: probe.cssHeight, profile, quality: state.quality });- let dataUrl = await this._visionBounded(tabId, 'screenshot', () => this._visionPlatformCapture(tabId, { probe, outWidth: size.width, outHeight: size.height }));+ let dataUrl = await this._visionBounded(tabId, 'screenshot', () => this._visionPlatformCapture(tabId, { probe, outWidth: size.width, outHeight: size.height, hold }));if (!dataUrl) throw new Error('The browser returned no screenshot.');let redaction = null;if (this.screenshotRedaction && typeof this._redactScreenshotDataUrl === 'function') {@@ -628,7 +630,7 @@ export function installVisionRuntime(AgentClass, { trace = null } = {}) {// scan and the image show the same page.if (outcome.status === 'scan_unavailable' && redactionRetry && await this._visionTabLoading?.(tabId) === true) {await this._visionAwaitRedactionScan(tabId);- return this._visionCaptureFrame(tabId, state, { redactionRetry: false });+ return this._visionCaptureFrame(tabId, state, { redactionRetry: false, hold });}redaction = outcome.status || 'error';}@@ -743,26 +745,33 @@ export function installVisionRuntime(AgentClass, { trace = null } = {}) {* Capture until two consecutive frames match (bounded). Micro changes (a* caret, a clock, a spinner) count as a match: they never stop, and waiting* for them only cost the full settle time on every step. WebBrain's own- * page UI (border, cursor, Stop button) is hidden once for all of the- * captures, not around each (agent _withIndicatorsHidden), so it does not- * blink on every step.+ * page UI (border, cursor, Stop button) stays hidden from the first+ * capture to the last (`hold`, agent _withIndicatorsHidden) and is shown+ * again once at the end, also when a capture fails or is given up on, so+ * it does not blink on every step.*/proto._visionCaptureSettled = async function _visionCaptureSettled(tabId, state) {await sleep(SETTLE_INITIAL_DELAY_MS);this._throwIfAborted?.(this._runAbortSignal?.(tabId));- return this._withIndicatorsHidden(tabId, async () => {- let frame = await this._visionCaptureFrame(tabId, state);+ const hold = { ended: false };+ try {+ let frame = await this._visionCaptureFrame(tabId, state, { hold });for (let attempt = 0; attempt < SETTLE_MAX_EXTRA_CAPTURES; attempt++) {await sleep(SETTLE_POLL_MS);let next;- try { next = await this._visionCaptureFrame(tabId, state); } catch { break; }+ try { next = await this._visionCaptureFrame(tabId, state, { hold }); } catch { break; }const sameShape = next.size.width === frame.size.width && next.size.height === frame.size.height;const moving = !sameShape || !isMicroScreenChange(diffFrames(frame.raster.pixels, next.raster.pixels, next.size.width, next.size.height));frame = next;if (!moving) break;}return frame;- });+ } finally {+ // Sent after every hide of the update, even one still unanswered:+ // the page handles them in order.+ hold.ended = true;+ this._showIndicatorsAfterToolUse?.(tabId);+ }};/**diff --git a/src/firefox/src/agent/agent.js b/src/firefox/src/agent/agent.jsindex ac159283f..9d7c76906 100644--- a/src/firefox/src/agent/agent.js+++ b/src/firefox/src/agent/agent.js@@ -17519,7 +17519,7 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d* the whole content area, classic scrollbars included, while the probe's* visual viewport excludes them, and points would drift by the scrollbar.*/- async _visionPlatformCapture(tabId, { probe, cssRegion = null, outWidth, outHeight }) {+ async _visionPlatformCapture(tabId, { probe, cssRegion = null, outWidth, outHeight, hold = null }) {const region = cssRegion || { x: 0, y: 0, width: probe.cssWidth, height: probe.cssHeight };const scale = Math.max(0.05, Math.min(4, Number(outWidth) / Math.max(1, region.width)));const options = {@@ -17533,7 +17533,7 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout dheight: Math.max(1, region.height),},};- return this._withIndicatorsHidden(tabId, () => browser.tabs.captureTab(tabId, options));+ return this._withIndicatorsHidden(tabId, () => browser.tabs.captureTab(tabId, options), { hold });}/**@@ -18618,53 +18618,52 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d* pages, or pre-existing tabs that loaded before the extension), the* sendMessage rejects and we capture as-is.*- * Calls for a tab nest: a vision screen update holds one around all of- * its captures (vision-runtime.js _visionCaptureSettled). Every call- * still hides, which a hidden indicator ignores but a page that- * navigated in between needs, and the indicator comes back once, when- * the last call ends, instead of blinking around every capture.+ * A vision screen update keeps the indicator hidden across all of its+ * captures (vision-runtime.js _visionCaptureSettled) and passes them its+ * `hold`. Each capture still hides, which a hidden indicator ignores but+ * a page that navigated in between needs, and leaves the show to the+ * update, which sends it once at its end instead of blinking around+ * every capture. Nothing is counted per tab: a capture that outlives its+ * update (one _visionBounded gave up on) shows by itself if it ever+ * finishes, and one that never finishes holds nothing back.** The hide is bounded by the capture deadline and run abort signal so an* unresponsive renderer or stop request cannot leave the run hung.*/- async _withIndicatorsHidden(tabId, fn) {- const holds = (this._indicatorHolds ??= new Map());- const hold = holds.get(tabId) || { count: 0, hidden: false };- hold.count += 1;- holds.set(tabId, hold);- let hideFailed = false;+ async _withIndicatorsHidden(tabId, fn, { hold = null } = {}) {+ let hidden = false;try {try {const sendHide = () => browser.tabs.sendMessage(tabId, { type: 'WB_HIDE_FOR_TOOL_USE' });const reply = typeof this._visionBounded === 'function'? await this._visionBounded(tabId, 'screenshot', sendHide): await sendHide();- hold.hidden = true;+ hidden = true;// Give the renderer one paint frame to apply the display:none the// content script just set, before we capture (none when it was// hidden already).if (reply?.changed !== false) await new Promise((r) => setTimeout(r, 16));} catch (error) {if (error?.name === 'AbortError' || error?.code === 'vision_capture_timeout' || this._runAbortSignal?.(tabId)?.aborted) {- hideFailed = true;+ // The hide may still arrive, so a show must follow it.+ hidden = true;throw error;}/* content script absent — capture without hiding */}return await fn();} finally {- hold.count -= 1;- if (hold.count === 0) {- holds.delete(tabId);- if (hold.hidden || hideFailed) {- try {- browser.tabs.sendMessage(tabId, { type: 'WB_SHOW_AFTER_TOOL_USE' }).catch(() => {});- } catch { /* ignore */ }- }- }+ if (hidden && (!hold || hold.ended)) this._showIndicatorsAfterToolUse(tabId);}}+ /** Brings back what _withIndicatorsHidden hid; with nothing hidden it changes nothing. */+ _showIndicatorsAfterToolUse(tabId) {+ try {+ browser.tabs.sendMessage(tabId, { type: 'WB_SHOW_AFTER_TOOL_USE' }).catch(() => {});+ } catch { /* ignore */ }+ }+async _captureViewportProbe(tabId) {try {const probeCode = `diff --git a/src/firefox/src/agent/vision-runtime.js b/src/firefox/src/agent/vision-runtime.jsindex d83517c7a..0c7c4a4d1 100644--- a/src/firefox/src/agent/vision-runtime.js+++ b/src/firefox/src/agent/vision-runtime.js@@ -599,14 +599,16 @@ export function installVisionRuntime(AgentClass, { trace = null } = {}) {/*** Capture the viewport at the planned model size. The platform capture may* return any size (device pixels, emulated DPR, zoom); rasterizeFrame- * resamples to the plan so the coordinate map is exact.+ * resamples to the plan so the coordinate map is exact. `hold` is a screen+ * update's (_visionCaptureSettled), which shows WebBrain's page UI again+ * after its last capture.*/- proto._visionCaptureFrame = async function _visionCaptureFrame(tabId, state, { redactionRetry = true } = {}) {+ proto._visionCaptureFrame = async function _visionCaptureFrame(tabId, state, { redactionRetry = true, hold = null } = {}) {const probe = await this._visionBounded(tabId, 'viewport measurement', () => this._visionPlatformProbe(tabId));if (!probe || !(probe.cssWidth > 0) || !(probe.cssHeight > 0)) throw new Error('The page viewport could not be measured.');const profile = VISION_PROFILES[state.profileId] || VISION_PROFILES.default;const size = planVisionImageSize({ cssWidth: probe.cssWidth, cssHeight: probe.cssHeight, profile, quality: state.quality });- let dataUrl = await this._visionBounded(tabId, 'screenshot', () => this._visionPlatformCapture(tabId, { probe, outWidth: size.width, outHeight: size.height }));+ let dataUrl = await this._visionBounded(tabId, 'screenshot', () => this._visionPlatformCapture(tabId, { probe, outWidth: size.width, outHeight: size.height, hold }));if (!dataUrl) throw new Error('The browser returned no screenshot.');let redaction = null;if (this.screenshotRedaction && typeof this._redactScreenshotDataUrl === 'function') {@@ -628,7 +630,7 @@ export function installVisionRuntime(AgentClass, { trace = null } = {}) {// scan and the image show the same page.if (outcome.status === 'scan_unavailable' && redactionRetry && await this._visionTabLoading?.(tabId) === true) {await this._visionAwaitRedactionScan(tabId);- return this._visionCaptureFrame(tabId, state, { redactionRetry: false });+ return this._visionCaptureFrame(tabId, state, { redactionRetry: false, hold });}redaction = outcome.status || 'error';}@@ -743,26 +745,33 @@ export function installVisionRuntime(AgentClass, { trace = null } = {}) {* Capture until two consecutive frames match (bounded). Micro changes (a* caret, a clock, a spinner) count as a match: they never stop, and waiting* for them only cost the full settle time on every step. WebBrain's own- * page UI (border, cursor, Stop button) is hidden once for all of the- * captures, not around each (agent _withIndicatorsHidden), so it does not- * blink on every step.+ * page UI (border, cursor, Stop button) stays hidden from the first+ * capture to the last (`hold`, agent _withIndicatorsHidden) and is shown+ * again once at the end, also when a capture fails or is given up on, so+ * it does not blink on every step.*/proto._visionCaptureSettled = async function _visionCaptureSettled(tabId, state) {await sleep(SETTLE_INITIAL_DELAY_MS);this._throwIfAborted?.(this._runAbortSignal?.(tabId));- return this._withIndicatorsHidden(tabId, async () => {- let frame = await this._visionCaptureFrame(tabId, state);+ const hold = { ended: false };+ try {+ let frame = await this._visionCaptureFrame(tabId, state, { hold });for (let attempt = 0; attempt < SETTLE_MAX_EXTRA_CAPTURES; attempt++) {await sleep(SETTLE_POLL_MS);let next;- try { next = await this._visionCaptureFrame(tabId, state); } catch { break; }+ try { next = await this._visionCaptureFrame(tabId, state, { hold }); } catch { break; }const sameShape = next.size.width === frame.size.width && next.size.height === frame.size.height;const moving = !sameShape || !isMicroScreenChange(diffFrames(frame.raster.pixels, next.raster.pixels, next.size.width, next.size.height));frame = next;if (!moving) break;}return frame;- });+ } finally {+ // Sent after every hide of the update, even one still unanswered:+ // the page handles them in order.+ hold.ended = true;+ this._showIndicatorsAfterToolUse?.(tabId);+ }};/**diff --git a/test/vision-mode-agent.mjs b/test/vision-mode-agent.mjsindex 64152455c..79e1b45de 100644--- a/test/vision-mode-agent.mjs+++ b/test/vision-mode-agent.mjs@@ -1396,7 +1396,7 @@ for (const build of (process.env.VISION_BUILDS || 'chrome,firefox').split(','))log.push(`capture, hidden: ${hidden}`);if (log.filter(entry => entry.startsWith('capture')).length === 2) hidden = false;return capture.call(this, tabId, options);- });+ }, { hold: options?.hold });};try {await agent._visionCaptureSettled(tab, state);@@ -1407,10 +1407,52 @@ for (const build of (process.env.VISION_BUILDS || 'chrome,firefox').split(','))assert.deepEqual(captures, Array(4).fill('capture, hidden: true'), log.join('\n'));assert.equal(log.filter(entry => entry === 'WB_SHOW_AFTER_TOOL_USE').length, 1, log.join('\n'));assert.equal(log.at(-1), 'WB_SHOW_AFTER_TOOL_USE', 'shown again after the last capture');- assert.equal(agent._indicatorHolds?.has(tab), false);});- test(`${build}: indicator hide respects capture deadline and abort, cleaning up holds`, async () => {+ test(`${build}: a screenshot that never answers does not keep WebBrain's page UI hidden`, async () => {+ const scene = tickingScene({ x: 300, y: 200, w: 400, h: 300 });+ const { agent } = runAgent(() => ({ content: 'done' }), { scene });+ const tab = nextTab++;+ const state = agent._visionRunState(tab, 'act');+ const tabs = (build === 'chrome' ? globalThis.chrome : globalThis.browser).tabs;+ const send = tabs.sendMessage;+ let hidden = false;+ tabs.sendMessage = async (id, message) => {+ if (id !== tab) return send(id, message);+ if (message.type === 'WB_HIDE_FOR_TOOL_USE') {+ const changed = !hidden;+ hidden = true;+ return { ok: true, changed };+ }+ if (message.type === 'WB_SHOW_AFTER_TOOL_USE') hidden = false;+ return { ok: true };+ };+ // The hide answers, then the first screenshot never does (a stalled+ // debugger), and the capture deadline gives up on it.+ const capture = agent._visionPlatformCapture;+ let stalled = false;+ agent._visionPlatformCapture = function (tabId, options) {+ return this._withIndicatorsHidden(tabId, () => {+ if (stalled) return capture.call(this, tabId, options);+ stalled = true;+ return new Promise(() => {});+ }, { hold: options?.hold });+ };+ agent._visionCaptureDeadlineMs = 50;+ try {+ await assert.rejects(() => agent._visionCaptureSettled(tab, state), (err) => err.code === 'vision_capture_timeout');+ assert.equal(hidden, false, 'shown again when the screen update gave up');+ agent._visionCaptureDeadlineMs = 0;+ await agent._visionCaptureSettled(tab, state);+ assert.equal(hidden, false, 'shown again after the next screen update');+ await agent._visionCaptureFrame(tab, state);+ assert.equal(hidden, false, 'shown again after a capture of its own');+ } finally {+ tabs.sendMessage = send;+ }+ });++ test(`${build}: indicator hide respects capture deadline and abort, and the page UI is shown again`, async () => {const scene = tickingScene({ x: 300, y: 200, w: 400, h: 300 });const { agent } = runAgent(() => ({ content: 'done' }), { scene });const tab = nextTab++;@@ -1426,13 +1468,17 @@ for (const build of (process.env.VISION_BUILDS || 'chrome,firefox').split(','))}return { ok: true };};+ // Each capture hides around itself, as both platform layers do.+ const capture = agent._visionPlatformCapture;+ agent._visionPlatformCapture = function (tabId, options) {+ return this._withIndicatorsHidden(tabId, () => capture.call(this, tabId, options), { hold: options?.hold });+ };agent._visionCaptureDeadlineMs = 50;try {await assert.rejects(() => agent._visionCaptureSettled(tab, state),(err) => err.code === 'vision_capture_timeout');- assert.equal(agent._indicatorHolds?.has(tab), false, 'hold cleaned up on timeout');assert.ok(log.includes('WB_SHOW_AFTER_TOOL_USE'), 'show attempted on timeout');} finally {tabs.sendMessage = send;@@ -1457,7 +1503,6 @@ for (const build of (process.env.VISION_BUILDS || 'chrome,firefox').split(','))() => agent._visionCaptureSettled(abortTab, abortState),(err) => err.name === 'AbortError');- assert.equal(agent._indicatorHolds?.has(abortTab), false, 'hold cleaned up on abort');assert.ok(abortLog.includes('WB_SHOW_AFTER_TOOL_USE'), 'show attempted on abort');} finally {tabs.sendMessage = send;