Skip to content

The agent that grows with you

fix(browser): capture_screenshot() no longer hangs 60s on a hidden tab

Teknium5abbff9d0f1b
diff --git a/tests/tools/test_browser_use_shot.py b/tests/tools/test_browser_use_shot.py
new file mode 100644
index 00000000000..2e8ba832701
--- /dev/null
+++ b/tests/tools/test_browser_use_shot.py
@@ -0,0 +1,113 @@
+"""browser_exec ``capture_screenshot()`` on a hidden tab (tools/browser_use_shot.py).
+
+Chrome draws no frame for a hidden tab, and Browser Use cloud never answers a png
+``Page.captureScreenshot`` for one, so the harness helper hung for its 60 s IPC timeout.
+``browser_exec`` below runs the real harness CLI it launches in production; only the daemon transport
+is faked.
+"""
+import json
+import os
+import sys
+import textwrap
+
+import pytest
+from PIL import Image
+
+import tools.browser_use_cli as bu_cli
+from tools import browser_session as bs
+from tools import browser_session_cloud as bs_cloud
+
+# sitecustomize on the harness's PYTHONPATH: the real browser_harness CLI runs, only its daemon transport is
+# faked. A png Page.captureScreenshot on a hidden target raises the harness's own IPC timeout.
+_FAKE_TRANSPORT = textwrap.dedent("""\
+ import base64, io, json, os
+ from PIL import Image
+ import browser_harness.helpers as h
+
+ LOG = os.environ["FAKE_CDP_LOG"]
+ state = {"hidden": True, "activate_works": os.environ["FAKE_ACTIVATE_WORKS"] == "1"}
+
+ def log(entry):
+ with open(LOG, "a") as f:
+ f.write(json.dumps(entry) + "\\n")
+
+ def image(fmt):
+ buf = io.BytesIO()
+ Image.new("RGB", (8, 6), "red").save(buf, format=fmt.upper())
+ return base64.b64encode(buf.getvalue()).decode()
+
+ def cdp(method, session_id=None, _response_timeout=5.0, **params):
+ log({"method": method, "timeout": _response_timeout, **params})
+ if method == "Runtime.evaluate":
+ return {"result": {"value": "hidden" if state["hidden"] else "visible"}}
+ if method == "Target.activateTarget":
+ if not state["activate_works"]:
+ raise RuntimeError("Target.activateTarget wasn't found")
+ state["hidden"] = False
+ return {}
+ if method == "Page.captureScreenshot":
+ if params.get("format") == "png" and state["hidden"]:
+ raise h._IPCResponseTimeout(f"{method} timed out after {_response_timeout:g}s")
+ return {"data": image(params["format"])}
+ raise AssertionError(method)
+
+ h.cdp = cdp
+ h.current_tab = lambda: {"targetId": "T1", "target_id": "T1", "url": "about:blank", "title": ""}
+ import browser_harness.admin as admin, browser_harness.telemetry as telemetry
+ admin.ensure_daemon = admin.print_update_banner = lambda *a, **k: None
+ admin.daemon_alive = lambda *a, **k: True
+ telemetry.is_enabled = lambda: False
+ telemetry.capture_cli_event = lambda *a, **k: None
+""")
+
+
+def run_exec(monkeypatch, tmp_path):
+ """browser_exec on the local route; the real harness CLI with the fake transport above."""
+ monkeypatch.delenv("BU_NAME", raising=False)
+ monkeypatch.delenv("BU_AUTOSPAWN", raising=False)
+ monkeypatch.setattr("tools.browser_session_cdp.get_cdp_override", lambda: "")
+ monkeypatch.setattr("tools.browser_session_cdp.get_cdp_override_raw", lambda: "")
+ monkeypatch.setattr(bs_cloud, "get_cloud_provider", lambda: None)
+ monkeypatch.setattr(bs, "get_session_info",
+ lambda key: {"cdp_url": f"ws://127.0.0.1:47000/devtools/browser/{key}",
+ "features": {"local": True}})
+ monkeypatch.setattr(bu_cli, "_attach_vault_supervisor", lambda env, task_id: None)
+ monkeypatch.setattr(bu_cli, "_exec_route_is_sidecar", {}, raising=False)
+ monkeypatch.setattr(bu_cli, "_driven_daemons", {})
+ fake_dir = tmp_path / "fake_transport"
+ fake_dir.mkdir()
+ (fake_dir / "sitecustomize.py").write_text(_FAKE_TRANSPORT, encoding="utf-8")
+ site_dir = bu_cli._harness_site_dir()
+ monkeypatch.setattr(bu_cli, "_harness_site_dir", lambda: os.pathsep.join([str(fake_dir), site_dir]))
+ # tests/tools/conftest.py hides the CLI; this is _find_cli()'s production command
+ monkeypatch.setattr(bu_cli, "_find_cli", lambda: [sys.executable, "-m", "browser_harness.run"])
+ log = tmp_path / "cdp.jsonl"
+
+ def run(activate_works: bool):
+ monkeypatch.setenv("FAKE_CDP_LOG", str(log))
+ monkeypatch.setenv("FAKE_ACTIVATE_WORKS", "1" if activate_works else "0")
+ shot = tmp_path / "shot.png"
+ result = bu_cli.browser_exec(f"print(capture_screenshot({str(shot)!r}))", task_id="t", timeout_s=60)
+ if not isinstance(result, str): # a native screenshot result (image attached)
+ result = json.dumps({"success": True, "screenshot_path": str(shot)})
+ calls = [json.loads(line) for line in log.read_text().splitlines()] if log.exists() else []
+ return json.loads(result), shot, calls
+
+ return run
+
+
[email protected]("activate_works", [True, False], ids=["activated", "activation-unsupported"])
+def test_hidden_tab_screenshot_returns_a_png_instead_of_hanging(run_exec, activate_works):
+ result, shot, calls = run_exec(activate_works)
+
+ assert result.get("success") is True, result.get("stderr") or result
+ with Image.open(shot) as img:
+ assert img.format == "PNG" # the path says .png, so a jpeg fallback is re-encoded
+ shots = [c for c in calls if c["method"] == "Page.captureScreenshot"]
+ assert shots[0]["format"] == "png" and shots[0]["timeout"] <= 10, shots
+ if activate_works: # the tab was brought to the front, so png answered first time
+ assert [c["method"] for c in calls].index("Target.activateTarget") < calls.index(shots[0])
+ assert len(shots) == 1
+ else: # png never comes for a hidden tab: one bounded png, then jpeg
+ assert [s["format"] for s in shots] == ["png", "jpeg"]
diff --git a/tools/browser_use_cli.py b/tools/browser_use_cli.py
index 271437d642b..5cb71427a04 100644
--- a/tools/browser_use_cli.py
+++ b/tools/browser_use_cli.py
@@ -46,6 +46,9 @@ _BOT_DESKTOP_BROWSER_SENTINEL = "_HERMES_BU_BOT_DESKTOP_BROWSER"
_REAL_PROFILE_FALLBACK_SENTINEL = "_HERMES_BU_REAL_PROFILE_FALLBACK"
# local=True whose real profile could not load: run on the local sidecar browser. Popped before launch.
_LOCAL_FALLBACK_SENTINEL = "_HERMES_BU_LOCAL_FALLBACK"
+# The resolved browser is a cloud provider's (no human watching it): new tabs are brought to the front.
+# Popped before launch.
+_CLOUD_BROWSER_SENTINEL = "_HERMES_BU_CLOUD_BROWSER"
# Prepended to the model's code for named sessions on SHARED browsers (a /browser connect CDP override): the
# harness daemon attaches to the first existing page at startup, so two fresh named daemons can land on the
@@ -431,6 +434,7 @@ def _resolve_backend_cdp(env: dict, task_id: Optional[str], session_name: str =
provider_key = str(getattr(provider, "name", "") or "").strip().lower()
if provider_key == _BROWSER_USE_PROVIDER and not _use_gateway(_read_browser_cfg()) and _browser_use_api_key_set():
env[_PRIVATE_BROWSER_SENTINEL] = "1" # named BU cloud browsers are exclusive to their daemon
+ env[_CLOUD_BROWSER_SENTINEL] = "1"
return None
provider_name = type(provider).__name__
@@ -456,6 +460,8 @@ def _resolve_backend_cdp(env: dict, task_id: Optional[str], session_name: str =
if err is None:
_mark_local_browser(env, session, key)
return err
+ if err is None:
+ env[_CLOUD_BROWSER_SENTINEL] = "1"
# A provider browser keyed bu-named-<name> is exclusive to this session — the
# own-tab preamble would just leak a blank tab into it.
if err is None and session_name:
@@ -719,6 +725,10 @@ def browser_exec(code: str, session: str = "", timeout_s: int = _DEFAULT_TIMEOUT
private_browser = env.pop(_PRIVATE_BROWSER_SENTINEL, None) # always pop: never exported to the CLI
if session and not private_browser:
code = _OWN_TAB_PREAMBLE + code
+ # Every backend: capture_screenshot() brings a hidden tab to the front and falls back to jpeg instead of
+ # hanging on a png a hidden tab never renders (tools/browser_use_shot.py).
+ from tools.browser_use_shot import harness_command
+ cmd = harness_command(cmd, activate_new_tabs=bool(env.pop(_CLOUD_BROWSER_SENTINEL, None)))
workspace = _workspace_dir(task_id)
if workspace:
diff --git a/tools/browser_use_shot.py b/tools/browser_use_shot.py
new file mode 100644
index 00000000000..eb746774bf7
--- /dev/null
+++ b/tools/browser_use_shot.py
@@ -0,0 +1,132 @@
+"""Screenshot and tab helpers that browser_exec installs over browser-harness's own.
+
+Chrome draws no compositor frame for a hidden tab, and some remotes (Browser Use cloud) never answer
+``Page.captureScreenshot`` with ``format: "png"`` for one, so the harness's ``capture_screenshot()`` hangs
+for its full 60 s IPC timeout. Its ``new_tab()`` and its daemon both create tabs with ``background=True``,
+so the tab the agent drives is often hidden. browser_harness is a pinned third-party dependency; this
+module is loaded by file path inside the harness process (``harness_command()``), so it imports nothing
+from Hermes and touches the harness only through the helpers module it is given.
+"""
+
+from __future__ import annotations
+
+import base64
+import io
+import os
+import tempfile
+from typing import Any, Callable
+
+# A png that has not arrived by then will not: a visible tab answers in well under a second, a hidden
+# one on Browser Use cloud never does. jpeg on the same hidden tab answers in a few seconds.
+PNG_DEADLINE_S = 10.0
+JPEG_DEADLINE_S = 60.0
+_HARNESS_MODULE = "browser_harness.helpers"
+
+
+def _default_shot_path() -> str:
+ try:
+ from browser_harness import _ipc # the harness's own per-user temp dir
+ return str(_ipc._TMP / "shot.png")
+ except (ImportError, AttributeError):
+ return os.path.join(tempfile.gettempdir(), "shot.png")
+
+
+def _bring_to_front_if_hidden(cdp: Callable[..., Any], current_tab: Callable[[], dict]) -> None:
+ """Activate the attached tab when the page reports it is hidden. A visible tab is left alone, so a
+ capture never takes window focus it does not need."""
+ try:
+ state = cdp("Runtime.evaluate", expression="document.visibilityState", returnByValue=True)
+ if (state.get("result") or {}).get("value") != "hidden":
+ return
+ cdp("Target.activateTarget", targetId=current_tab()["targetId"])
+ except Exception: # an engine without Target.activateTarget (Lightpanda) still captures below
+ return
+
+
+def _write_shot(data: bytes, path: str, fmt: str) -> None:
+ """Write ``data`` (encoded as ``fmt``) to ``path``, re-encoding a jpeg as png when the path says png:
+ callers and the screenshot embed trust the extension."""
+ if fmt == "jpeg" and path.lower().endswith(".png"):
+ from PIL import Image
+ Image.open(io.BytesIO(data)).save(path, format="PNG")
+ return
+ with open(path, "wb") as f:
+ f.write(data)
+
+
+def make_capture_screenshot(cdp: Callable[..., Any], current_tab: Callable[[], dict]) -> Callable[..., str]:
+ def capture_screenshot(path=None, full=False, max_dim=None):
+ """Save a PNG of the current viewport (bringing a hidden tab to the front first). Set
+ max_dim=1800 on a 2x display to keep the file under the 2000px-per-side limit some image-aware
+ LLMs enforce."""
+ path = path or _default_shot_path()
+ _bring_to_front_if_hidden(cdp, current_tab)
+ fmt = "png"
+ try:
+ r = cdp("Page.captureScreenshot", _response_timeout=PNG_DEADLINE_S,
+ format="png", captureBeyondViewport=full)
+ except TimeoutError:
+ fmt = "jpeg"
+ try:
+ r = cdp("Page.captureScreenshot", _response_timeout=JPEG_DEADLINE_S,
+ format="jpeg", quality=90, captureBeyondViewport=full)
+ except TimeoutError as e:
+ raise RuntimeError(
+ f"Page.captureScreenshot timed out (png after {PNG_DEADLINE_S:g}s, "
+ f"jpeg after {JPEG_DEADLINE_S:g}s)") from e
+ _write_shot(base64.b64decode(r["data"]), path, fmt)
+ if max_dim:
+ from PIL import Image
+ img = Image.open(path)
+ if max(img.size) > max_dim:
+ img.thumbnail((max_dim, max_dim))
+ img.save(path)
+ return path
+ return capture_screenshot
+
+
+def make_new_tab(new_tab: Callable[..., Any], cdp: Callable[..., Any]) -> Callable[..., Any]:
+ def new_tab_in_front(url="about:blank"):
+ """Open (or reuse a blank) tab, navigate to url, and bring it to the front."""
+ tid = new_tab(url)
+ if tid:
+ cdp("Target.activateTarget", targetId=tid)
+ return tid
+ return new_tab_in_front
+
+
+def _is_harness_helper(fn: Any) -> bool:
+ """True for the harness's own helper; a same-named function from the agent's workspace
+ ``agent_helpers.py`` (loaded into the helpers module at import) is the agent's choice."""
+ return callable(fn) and getattr(fn, "__module__", "") == _HARNESS_MODULE
+
+
+def install(helpers: Any, activate_new_tabs: bool = False) -> None:
+ """Rebind the helpers on the ``browser_harness.helpers`` module before ``browser_harness.run`` star-imports
+ them, so the agent's code, the harness's step tracer and recorder all see these. ``activate_new_tabs`` is
+ for browsers no human is watching (cloud providers): there a background tab only costs throttling and
+ blank screenshots, while on a user's own Chrome the harness deliberately leaves the visible tab alone."""
+ if _is_harness_helper(helpers.capture_screenshot):
+ helpers.capture_screenshot = make_capture_screenshot(helpers.cdp, helpers.current_tab)
+ if activate_new_tabs and _is_harness_helper(helpers.new_tab):
+ helpers.new_tab = make_new_tab(helpers.new_tab, helpers.cdp)
+
+
+_HARNESS_MODULE_ARGS = ["-m", "browser_harness.run"]
+_LAUNCHER = """\
+import importlib.util, runpy
+spec = importlib.util.spec_from_file_location('hermes_browser_use_shot', {path!r})
+shot = importlib.util.module_from_spec(spec); spec.loader.exec_module(shot)
+import browser_harness.helpers
+shot.install(browser_harness.helpers, activate_new_tabs={activate!r})
+runpy.run_module('browser_harness.run', run_name='__main__', alter_sys=True)
+"""
+
+
+def harness_command(cmd: list[str], activate_new_tabs: bool) -> list[str]:
+ """``python -m browser_harness.run`` as a launcher that installs these helpers first. The model's code
+ still arrives untouched on stdin (no preamble shifting its traceback line numbers). This file is loaded
+ by path: the harness interpreter has no Hermes checkout on sys.path."""
+ if cmd[1:] != _HARNESS_MODULE_ARGS:
+ return cmd
+ return [cmd[0], "-c", _LAUNCHER.format(path=os.path.abspath(__file__), activate=bool(activate_new_tabs))]