The agent that grows with you
Python aiai-agentai-agentsanthropicchatgptclaudeclaude-codecodexhermeshermes-agentllmnous-researchopenai
fix(browser): capture_screenshot() no longer hangs 60s on a hidden tab
Teknium5abbff9d0f1b
diff --git a/tests/tools/test_browser_use_shot.py b/tests/tools/test_browser_use_shot.pynew file mode 100644index 00000000000..2e8ba832701--- /dev/null+++ b/tests/tools/test_browser_use_shot.py@@ -0,0 +1,113 @@+"""browser_exec ``capture_screenshot()`` on a hidden tab (tools/browser_use_shot.py).++Chrome draws no frame for a hidden tab, and Browser Use cloud never answers a png+``Page.captureScreenshot`` for one, so the harness helper hung for its 60 s IPC timeout.+``browser_exec`` below runs the real harness CLI it launches in production; only the daemon transport+is faked.+"""+import json+import os+import sys+import textwrap++import pytest+from PIL import Image++import tools.browser_use_cli as bu_cli+from tools import browser_session as bs+from tools import browser_session_cloud as bs_cloud++# sitecustomize on the harness's PYTHONPATH: the real browser_harness CLI runs, only its daemon transport is+# faked. A png Page.captureScreenshot on a hidden target raises the harness's own IPC timeout.+_FAKE_TRANSPORT = textwrap.dedent("""\+ import base64, io, json, os+ from PIL import Image+ import browser_harness.helpers as h++ LOG = os.environ["FAKE_CDP_LOG"]+ state = {"hidden": True, "activate_works": os.environ["FAKE_ACTIVATE_WORKS"] == "1"}++ def log(entry):+ with open(LOG, "a") as f:+ f.write(json.dumps(entry) + "\\n")++ def image(fmt):+ buf = io.BytesIO()+ Image.new("RGB", (8, 6), "red").save(buf, format=fmt.upper())+ return base64.b64encode(buf.getvalue()).decode()++ def cdp(method, session_id=None, _response_timeout=5.0, **params):+ log({"method": method, "timeout": _response_timeout, **params})+ if method == "Runtime.evaluate":+ return {"result": {"value": "hidden" if state["hidden"] else "visible"}}+ if method == "Target.activateTarget":+ if not state["activate_works"]:+ raise RuntimeError("Target.activateTarget wasn't found")+ state["hidden"] = False+ return {}+ if method == "Page.captureScreenshot":+ if params.get("format") == "png" and state["hidden"]:+ raise h._IPCResponseTimeout(f"{method} timed out after {_response_timeout:g}s")+ return {"data": image(params["format"])}+ raise AssertionError(method)++ h.cdp = cdp+ h.current_tab = lambda: {"targetId": "T1", "target_id": "T1", "url": "about:blank", "title": ""}+ import browser_harness.admin as admin, browser_harness.telemetry as telemetry+ admin.ensure_daemon = admin.print_update_banner = lambda *a, **k: None+ admin.daemon_alive = lambda *a, **k: True+ telemetry.is_enabled = lambda: False+ telemetry.capture_cli_event = lambda *a, **k: None+""")+++def run_exec(monkeypatch, tmp_path):+ """browser_exec on the local route; the real harness CLI with the fake transport above."""+ monkeypatch.delenv("BU_NAME", raising=False)+ monkeypatch.delenv("BU_AUTOSPAWN", raising=False)+ monkeypatch.setattr("tools.browser_session_cdp.get_cdp_override", lambda: "")+ monkeypatch.setattr("tools.browser_session_cdp.get_cdp_override_raw", lambda: "")+ monkeypatch.setattr(bs_cloud, "get_cloud_provider", lambda: None)+ monkeypatch.setattr(bs, "get_session_info",+ lambda key: {"cdp_url": f"ws://127.0.0.1:47000/devtools/browser/{key}",+ "features": {"local": True}})+ monkeypatch.setattr(bu_cli, "_attach_vault_supervisor", lambda env, task_id: None)+ monkeypatch.setattr(bu_cli, "_exec_route_is_sidecar", {}, raising=False)+ monkeypatch.setattr(bu_cli, "_driven_daemons", {})+ fake_dir = tmp_path / "fake_transport"+ fake_dir.mkdir()+ (fake_dir / "sitecustomize.py").write_text(_FAKE_TRANSPORT, encoding="utf-8")+ site_dir = bu_cli._harness_site_dir()+ monkeypatch.setattr(bu_cli, "_harness_site_dir", lambda: os.pathsep.join([str(fake_dir), site_dir]))+ # tests/tools/conftest.py hides the CLI; this is _find_cli()'s production command+ monkeypatch.setattr(bu_cli, "_find_cli", lambda: [sys.executable, "-m", "browser_harness.run"])+ log = tmp_path / "cdp.jsonl"++ def run(activate_works: bool):+ monkeypatch.setenv("FAKE_CDP_LOG", str(log))+ monkeypatch.setenv("FAKE_ACTIVATE_WORKS", "1" if activate_works else "0")+ shot = tmp_path / "shot.png"+ result = bu_cli.browser_exec(f"print(capture_screenshot({str(shot)!r}))", task_id="t", timeout_s=60)+ if not isinstance(result, str): # a native screenshot result (image attached)+ result = json.dumps({"success": True, "screenshot_path": str(shot)})+ calls = [json.loads(line) for line in log.read_text().splitlines()] if log.exists() else []+ return json.loads(result), shot, calls++ return run++[email protected]("activate_works", [True, False], ids=["activated", "activation-unsupported"])+def test_hidden_tab_screenshot_returns_a_png_instead_of_hanging(run_exec, activate_works):+ result, shot, calls = run_exec(activate_works)++ assert result.get("success") is True, result.get("stderr") or result+ with Image.open(shot) as img:+ assert img.format == "PNG" # the path says .png, so a jpeg fallback is re-encoded+ shots = [c for c in calls if c["method"] == "Page.captureScreenshot"]+ assert shots[0]["format"] == "png" and shots[0]["timeout"] <= 10, shots+ if activate_works: # the tab was brought to the front, so png answered first time+ assert [c["method"] for c in calls].index("Target.activateTarget") < calls.index(shots[0])+ assert len(shots) == 1+ else: # png never comes for a hidden tab: one bounded png, then jpeg+ assert [s["format"] for s in shots] == ["png", "jpeg"]diff --git a/tools/browser_use_cli.py b/tools/browser_use_cli.pyindex 271437d642b..5cb71427a04 100644--- a/tools/browser_use_cli.py+++ b/tools/browser_use_cli.py@@ -46,6 +46,9 @@ _BOT_DESKTOP_BROWSER_SENTINEL = "_HERMES_BU_BOT_DESKTOP_BROWSER"_REAL_PROFILE_FALLBACK_SENTINEL = "_HERMES_BU_REAL_PROFILE_FALLBACK"# local=True whose real profile could not load: run on the local sidecar browser. Popped before launch._LOCAL_FALLBACK_SENTINEL = "_HERMES_BU_LOCAL_FALLBACK"+# The resolved browser is a cloud provider's (no human watching it): new tabs are brought to the front.+# Popped before launch.+_CLOUD_BROWSER_SENTINEL = "_HERMES_BU_CLOUD_BROWSER"# Prepended to the model's code for named sessions on SHARED browsers (a /browser connect CDP override): the# harness daemon attaches to the first existing page at startup, so two fresh named daemons can land on the@@ -431,6 +434,7 @@ def _resolve_backend_cdp(env: dict, task_id: Optional[str], session_name: str =provider_key = str(getattr(provider, "name", "") or "").strip().lower()if provider_key == _BROWSER_USE_PROVIDER and not _use_gateway(_read_browser_cfg()) and _browser_use_api_key_set():env[_PRIVATE_BROWSER_SENTINEL] = "1" # named BU cloud browsers are exclusive to their daemon+ env[_CLOUD_BROWSER_SENTINEL] = "1"return Noneprovider_name = type(provider).__name__@@ -456,6 +460,8 @@ def _resolve_backend_cdp(env: dict, task_id: Optional[str], session_name: str =if err is None:_mark_local_browser(env, session, key)return err+ if err is None:+ env[_CLOUD_BROWSER_SENTINEL] = "1"# A provider browser keyed bu-named-<name> is exclusive to this session — the# own-tab preamble would just leak a blank tab into it.if err is None and session_name:@@ -719,6 +725,10 @@ def browser_exec(code: str, session: str = "", timeout_s: int = _DEFAULT_TIMEOUTprivate_browser = env.pop(_PRIVATE_BROWSER_SENTINEL, None) # always pop: never exported to the CLIif session and not private_browser:code = _OWN_TAB_PREAMBLE + code+ # Every backend: capture_screenshot() brings a hidden tab to the front and falls back to jpeg instead of+ # hanging on a png a hidden tab never renders (tools/browser_use_shot.py).+ from tools.browser_use_shot import harness_command+ cmd = harness_command(cmd, activate_new_tabs=bool(env.pop(_CLOUD_BROWSER_SENTINEL, None)))workspace = _workspace_dir(task_id)if workspace:diff --git a/tools/browser_use_shot.py b/tools/browser_use_shot.pynew file mode 100644index 00000000000..eb746774bf7--- /dev/null+++ b/tools/browser_use_shot.py@@ -0,0 +1,132 @@+"""Screenshot and tab helpers that browser_exec installs over browser-harness's own.++Chrome draws no compositor frame for a hidden tab, and some remotes (Browser Use cloud) never answer+``Page.captureScreenshot`` with ``format: "png"`` for one, so the harness's ``capture_screenshot()`` hangs+for its full 60 s IPC timeout. Its ``new_tab()`` and its daemon both create tabs with ``background=True``,+so the tab the agent drives is often hidden. browser_harness is a pinned third-party dependency; this+module is loaded by file path inside the harness process (``harness_command()``), so it imports nothing+from Hermes and touches the harness only through the helpers module it is given.+"""++from __future__ import annotations++import base64+import io+import os+import tempfile+from typing import Any, Callable++# A png that has not arrived by then will not: a visible tab answers in well under a second, a hidden+# one on Browser Use cloud never does. jpeg on the same hidden tab answers in a few seconds.+PNG_DEADLINE_S = 10.0+JPEG_DEADLINE_S = 60.0+_HARNESS_MODULE = "browser_harness.helpers"+++def _default_shot_path() -> str:+ try:+ from browser_harness import _ipc # the harness's own per-user temp dir+ return str(_ipc._TMP / "shot.png")+ except (ImportError, AttributeError):+ return os.path.join(tempfile.gettempdir(), "shot.png")+++def _bring_to_front_if_hidden(cdp: Callable[..., Any], current_tab: Callable[[], dict]) -> None:+ """Activate the attached tab when the page reports it is hidden. A visible tab is left alone, so a+ capture never takes window focus it does not need."""+ try:+ state = cdp("Runtime.evaluate", expression="document.visibilityState", returnByValue=True)+ if (state.get("result") or {}).get("value") != "hidden":+ return+ cdp("Target.activateTarget", targetId=current_tab()["targetId"])+ except Exception: # an engine without Target.activateTarget (Lightpanda) still captures below+ return+++def _write_shot(data: bytes, path: str, fmt: str) -> None:+ """Write ``data`` (encoded as ``fmt``) to ``path``, re-encoding a jpeg as png when the path says png:+ callers and the screenshot embed trust the extension."""+ if fmt == "jpeg" and path.lower().endswith(".png"):+ from PIL import Image+ Image.open(io.BytesIO(data)).save(path, format="PNG")+ return+ with open(path, "wb") as f:+ f.write(data)+++def make_capture_screenshot(cdp: Callable[..., Any], current_tab: Callable[[], dict]) -> Callable[..., str]:+ def capture_screenshot(path=None, full=False, max_dim=None):+ """Save a PNG of the current viewport (bringing a hidden tab to the front first). Set+ max_dim=1800 on a 2x display to keep the file under the 2000px-per-side limit some image-aware+ LLMs enforce."""+ path = path or _default_shot_path()+ _bring_to_front_if_hidden(cdp, current_tab)+ fmt = "png"+ try:+ r = cdp("Page.captureScreenshot", _response_timeout=PNG_DEADLINE_S,+ format="png", captureBeyondViewport=full)+ except TimeoutError:+ fmt = "jpeg"+ try:+ r = cdp("Page.captureScreenshot", _response_timeout=JPEG_DEADLINE_S,+ format="jpeg", quality=90, captureBeyondViewport=full)+ except TimeoutError as e:+ raise RuntimeError(+ f"Page.captureScreenshot timed out (png after {PNG_DEADLINE_S:g}s, "+ f"jpeg after {JPEG_DEADLINE_S:g}s)") from e+ _write_shot(base64.b64decode(r["data"]), path, fmt)+ if max_dim:+ from PIL import Image+ img = Image.open(path)+ if max(img.size) > max_dim:+ img.thumbnail((max_dim, max_dim))+ img.save(path)+ return path+ return capture_screenshot+++def make_new_tab(new_tab: Callable[..., Any], cdp: Callable[..., Any]) -> Callable[..., Any]:+ def new_tab_in_front(url="about:blank"):+ """Open (or reuse a blank) tab, navigate to url, and bring it to the front."""+ tid = new_tab(url)+ if tid:+ cdp("Target.activateTarget", targetId=tid)+ return tid+ return new_tab_in_front+++def _is_harness_helper(fn: Any) -> bool:+ """True for the harness's own helper; a same-named function from the agent's workspace+ ``agent_helpers.py`` (loaded into the helpers module at import) is the agent's choice."""+ return callable(fn) and getattr(fn, "__module__", "") == _HARNESS_MODULE+++def install(helpers: Any, activate_new_tabs: bool = False) -> None:+ """Rebind the helpers on the ``browser_harness.helpers`` module before ``browser_harness.run`` star-imports+ them, so the agent's code, the harness's step tracer and recorder all see these. ``activate_new_tabs`` is+ for browsers no human is watching (cloud providers): there a background tab only costs throttling and+ blank screenshots, while on a user's own Chrome the harness deliberately leaves the visible tab alone."""+ if _is_harness_helper(helpers.capture_screenshot):+ helpers.capture_screenshot = make_capture_screenshot(helpers.cdp, helpers.current_tab)+ if activate_new_tabs and _is_harness_helper(helpers.new_tab):+ helpers.new_tab = make_new_tab(helpers.new_tab, helpers.cdp)+++_HARNESS_MODULE_ARGS = ["-m", "browser_harness.run"]+_LAUNCHER = """\+import importlib.util, runpy+spec = importlib.util.spec_from_file_location('hermes_browser_use_shot', {path!r})+shot = importlib.util.module_from_spec(spec); spec.loader.exec_module(shot)+import browser_harness.helpers+shot.install(browser_harness.helpers, activate_new_tabs={activate!r})+runpy.run_module('browser_harness.run', run_name='__main__', alter_sys=True)+"""+++def harness_command(cmd: list[str], activate_new_tabs: bool) -> list[str]:+ """``python -m browser_harness.run`` as a launcher that installs these helpers first. The model's code+ still arrives untouched on stdin (no preamble shifting its traceback line numbers). This file is loaded+ by path: the harness interpreter has no Hermes checkout on sys.path."""+ if cmd[1:] != _HARNESS_MODULE_ARGS:+ return cmd+ return [cmd[0], "-c", _LAUNCHER.format(path=os.path.abspath(__file__), activate=bool(activate_new_tabs))]