Skip to content

The agent that grows with you

fix(browser): bring a hidden tab to the front only on a cloud browser

Teknium9c6d155a160f
diff --git a/tests/tools/test_browser_use_shot.py b/tests/tools/test_browser_use_shot.py
index 4950f02d229..a220104e741 100644
--- a/tests/tools/test_browser_use_shot.py
+++ b/tests/tools/test_browser_use_shot.py
@@ -102,20 +102,33 @@ def run_exec(monkeypatch, tmp_path):
return run
[email protected]("activate_works", [True, False], ids=["activated", "activation-unsupported"])
-def test_hidden_tab_screenshot_returns_a_png_instead_of_hanging(run_exec, activate_works):
[email protected]("as_path", [False, True], ids=["str", "pathlib"])
+def test_hidden_tab_screenshot_on_the_users_chrome_never_switches_their_tab(run_exec, as_path):
# a pathlib.Path on the jpeg fallback, where the extension decides the re-encode
- code = "print(capture_screenshot({shot!r}))" if activate_works else \
- "from pathlib import Path\nprint(capture_screenshot(Path({shot!r})))"
- result, shot, calls = run_exec(activate_works, code=code)
+ code = "from pathlib import Path\nprint(capture_screenshot(Path({shot!r})))" if as_path else \
+ "print(capture_screenshot({shot!r}))"
+ result, shot, calls = run_exec(True, code=code)
assert result.get("success") is True, result.get("stderr") or result
with Image.open(shot) as img:
assert img.format == "PNG" # the path says .png, so a jpeg fallback is re-encoded
+ # the user's visible tab stays put: one bounded png, then jpeg, no activation
+ assert "Target.activateTarget" not in [c["method"] for c in calls], calls
+ shots = [c for c in calls if c["method"] == "Page.captureScreenshot"]
+ assert [s["format"] for s in shots] == ["png", "jpeg"] and shots[0]["timeout"] <= 10, shots
+
+
[email protected]("activate_works", [True, False], ids=["activated", "activation-unsupported"])
+def test_hidden_tab_screenshot_on_a_cloud_browser_is_brought_to_front(run_exec, activate_works):
+ result, shot, calls = run_exec(activate_works, cloud=True)
+
+ assert result.get("success") is True, result.get("stderr") or result
+ with Image.open(shot) as img:
+ assert img.format == "PNG"
shots = [c for c in calls if c["method"] == "Page.captureScreenshot"]
assert shots[0]["format"] == "png" and shots[0]["timeout"] <= 10, shots
+ assert [c["method"] for c in calls].index("Target.activateTarget") < calls.index(shots[0])
if activate_works: # the tab was brought to the front, so png answered first time
- assert [c["method"] for c in calls].index("Target.activateTarget") < calls.index(shots[0])
assert len(shots) == 1
else: # png never comes for a hidden tab: one bounded png, then jpeg
assert [s["format"] for s in shots] == ["png", "jpeg"]
@@ -130,3 +143,11 @@ def test_new_tab_is_brought_to_front_only_on_a_cloud_browser(run_exec, cloud):
activations = [c for c in calls if c["method"] == "Target.activateTarget"]
# nobody watches a cloud browser; on the user's own Chrome the visible tab stays put
assert activations == ([{"method": "Target.activateTarget", "timeout": 5.0, "targetId": "T2"}] if cloud else [])
+
+
+def test_cloud_new_tab_survives_a_provider_rejecting_activation(run_exec):
+ result, _, calls = run_exec(False, cloud=True, code="print('TAB', new_tab())")
+
+ assert result.get("success") is True, result.get("stderr") or result
+ assert "TAB T2" in result.get("output", "") + result.get("stdout", ""), result
+ assert [c["targetId"] for c in calls if c["method"] == "Target.activateTarget"] == ["T2"]
diff --git a/tools/browser_use_cli.py b/tools/browser_use_cli.py
index 5cb71427a04..7948e45fe44 100644
--- a/tools/browser_use_cli.py
+++ b/tools/browser_use_cli.py
@@ -725,8 +725,8 @@ def browser_exec(code: str, session: str = "", timeout_s: int = _DEFAULT_TIMEOUT
private_browser = env.pop(_PRIVATE_BROWSER_SENTINEL, None) # always pop: never exported to the CLI
if session and not private_browser:
code = _OWN_TAB_PREAMBLE + code
- # Every backend: capture_screenshot() brings a hidden tab to the front and falls back to jpeg instead of
- # hanging on a png a hidden tab never renders (tools/browser_use_shot.py).
+ # Every backend: capture_screenshot() falls back to jpeg instead of hanging on a png a hidden tab never
+ # renders; only on a cloud browser does it (and new_tab) bring the tab to the front (tools/browser_use_shot.py).
from tools.browser_use_shot import harness_command
cmd = harness_command(cmd, activate_new_tabs=bool(env.pop(_CLOUD_BROWSER_SENTINEL, None)))
diff --git a/tools/browser_use_shot.py b/tools/browser_use_shot.py
index bb0f283fcfa..976d75f2a25 100644
--- a/tools/browser_use_shot.py
+++ b/tools/browser_use_shot.py
@@ -12,6 +12,7 @@ from __future__ import annotations
import base64
import io
+import logging
import os
import tempfile
from typing import Any, Callable
@@ -21,6 +22,7 @@ from typing import Any, Callable
PNG_DEADLINE_S = 10.0
JPEG_DEADLINE_S = 60.0
_HARNESS_MODULE = "browser_harness.helpers"
+_log = logging.getLogger("hermes_browser_use_shot")
def _default_shot_path() -> str:
@@ -55,13 +57,17 @@ def _write_shot(data: bytes, path: str, fmt: str) -> None:
f.write(data)
-def make_capture_screenshot(cdp: Callable[..., Any], current_tab: Callable[[], dict]) -> Callable[..., str]:
+def make_capture_screenshot(cdp: Callable[..., Any], current_tab: Callable[[], dict],
+ activate_hidden: bool = False) -> Callable[..., str]:
+ """``activate_hidden`` only on a browser no human watches (cloud): on the user's own or connected
+ Chrome, activating the agent's tab would switch the tab they are looking at. Every backend keeps the
+ bounded png attempt and the jpeg fallback."""
def capture_screenshot(path=None, full=False, max_dim=None):
- """Save a PNG of the current viewport (bringing a hidden tab to the front first). Set
- max_dim=1800 on a 2x display to keep the file under the 2000px-per-side limit some image-aware
- LLMs enforce."""
+ """Save a PNG of the current viewport. Set max_dim=1800 on a 2x display to keep the file under
+ the 2000px-per-side limit some image-aware LLMs enforce."""
path = os.fspath(path or _default_shot_path()) # a pathlib.Path works too
- _bring_to_front_if_hidden(cdp, current_tab)
+ if activate_hidden:
+ _bring_to_front_if_hidden(cdp, current_tab)
fmt = "png"
try:
r = cdp("Page.captureScreenshot", _response_timeout=PNG_DEADLINE_S,
@@ -91,7 +97,11 @@ def make_new_tab(new_tab: Callable[..., Any], cdp: Callable[..., Any]) -> Callab
"""Open (or reuse a blank) tab, navigate to url, and bring it to the front."""
tid = new_tab(url)
if tid:
- cdp("Target.activateTarget", targetId=tid)
+ try:
+ cdp("Target.activateTarget", targetId=tid)
+ # The tab exists either way; a provider that rejects activation leaves it in the background.
+ except (RuntimeError, OSError) as e:
+ _log.warning("new_tab: Target.activateTarget failed, tab left in background: %s", e)
return tid
return new_tab_in_front
@@ -106,9 +116,11 @@ def install(helpers: Any, activate_new_tabs: bool = False) -> None:
"""Rebind the helpers on the ``browser_harness.helpers`` module before ``browser_harness.run`` star-imports
them, so the agent's code, the harness's step tracer and recorder all see these. ``activate_new_tabs`` is
for browsers no human is watching (cloud providers): there a background tab only costs throttling and
- blank screenshots, while on a user's own Chrome the harness deliberately leaves the visible tab alone."""
+ blank screenshots, so new tabs and hidden tabs being captured are brought to the front; on a user's own
+ Chrome the harness deliberately leaves the visible tab alone."""
if _is_harness_helper(helpers.capture_screenshot):
- helpers.capture_screenshot = make_capture_screenshot(helpers.cdp, helpers.current_tab)
+ helpers.capture_screenshot = make_capture_screenshot(helpers.cdp, helpers.current_tab,
+ activate_hidden=activate_new_tabs)
if activate_new_tabs and _is_harness_helper(helpers.new_tab):
helpers.new_tab = make_new_tab(helpers.new_tab, helpers.cdp)