The agent that grows with you
Python aiai-agentai-agentsanthropicchatgptclaudeclaude-codecodexhermeshermes-agentllmnous-researchopenai
fix(browser): bring a hidden tab to the front only on a cloud browser
Teknium9c6d155a160f
diff --git a/tests/tools/test_browser_use_shot.py b/tests/tools/test_browser_use_shot.pyindex 4950f02d229..a220104e741 100644--- a/tests/tools/test_browser_use_shot.py+++ b/tests/tools/test_browser_use_shot.py@@ -102,20 +102,33 @@ def run_exec(monkeypatch, tmp_path):return run[email protected]("activate_works", [True, False], ids=["activated", "activation-unsupported"])-def test_hidden_tab_screenshot_returns_a_png_instead_of_hanging(run_exec, activate_works):[email protected]("as_path", [False, True], ids=["str", "pathlib"])+def test_hidden_tab_screenshot_on_the_users_chrome_never_switches_their_tab(run_exec, as_path):# a pathlib.Path on the jpeg fallback, where the extension decides the re-encode- code = "print(capture_screenshot({shot!r}))" if activate_works else \- "from pathlib import Path\nprint(capture_screenshot(Path({shot!r})))"- result, shot, calls = run_exec(activate_works, code=code)+ code = "from pathlib import Path\nprint(capture_screenshot(Path({shot!r})))" if as_path else \+ "print(capture_screenshot({shot!r}))"+ result, shot, calls = run_exec(True, code=code)assert result.get("success") is True, result.get("stderr") or resultwith Image.open(shot) as img:assert img.format == "PNG" # the path says .png, so a jpeg fallback is re-encoded+ # the user's visible tab stays put: one bounded png, then jpeg, no activation+ assert "Target.activateTarget" not in [c["method"] for c in calls], calls+ shots = [c for c in calls if c["method"] == "Page.captureScreenshot"]+ assert [s["format"] for s in shots] == ["png", "jpeg"] and shots[0]["timeout"] <= 10, shots++[email protected]("activate_works", [True, False], ids=["activated", "activation-unsupported"])+def test_hidden_tab_screenshot_on_a_cloud_browser_is_brought_to_front(run_exec, activate_works):+ result, shot, calls = run_exec(activate_works, cloud=True)++ assert result.get("success") is True, result.get("stderr") or result+ with Image.open(shot) as img:+ assert img.format == "PNG"shots = [c for c in calls if c["method"] == "Page.captureScreenshot"]assert shots[0]["format"] == "png" and shots[0]["timeout"] <= 10, shots+ assert [c["method"] for c in calls].index("Target.activateTarget") < calls.index(shots[0])if activate_works: # the tab was brought to the front, so png answered first time- assert [c["method"] for c in calls].index("Target.activateTarget") < calls.index(shots[0])assert len(shots) == 1else: # png never comes for a hidden tab: one bounded png, then jpegassert [s["format"] for s in shots] == ["png", "jpeg"]@@ -130,3 +143,11 @@ def test_new_tab_is_brought_to_front_only_on_a_cloud_browser(run_exec, cloud):activations = [c for c in calls if c["method"] == "Target.activateTarget"]# nobody watches a cloud browser; on the user's own Chrome the visible tab stays putassert activations == ([{"method": "Target.activateTarget", "timeout": 5.0, "targetId": "T2"}] if cloud else [])+++def test_cloud_new_tab_survives_a_provider_rejecting_activation(run_exec):+ result, _, calls = run_exec(False, cloud=True, code="print('TAB', new_tab())")++ assert result.get("success") is True, result.get("stderr") or result+ assert "TAB T2" in result.get("output", "") + result.get("stdout", ""), result+ assert [c["targetId"] for c in calls if c["method"] == "Target.activateTarget"] == ["T2"]diff --git a/tools/browser_use_cli.py b/tools/browser_use_cli.pyindex 5cb71427a04..7948e45fe44 100644--- a/tools/browser_use_cli.py+++ b/tools/browser_use_cli.py@@ -725,8 +725,8 @@ def browser_exec(code: str, session: str = "", timeout_s: int = _DEFAULT_TIMEOUTprivate_browser = env.pop(_PRIVATE_BROWSER_SENTINEL, None) # always pop: never exported to the CLIif session and not private_browser:code = _OWN_TAB_PREAMBLE + code- # Every backend: capture_screenshot() brings a hidden tab to the front and falls back to jpeg instead of- # hanging on a png a hidden tab never renders (tools/browser_use_shot.py).+ # Every backend: capture_screenshot() falls back to jpeg instead of hanging on a png a hidden tab never+ # renders; only on a cloud browser does it (and new_tab) bring the tab to the front (tools/browser_use_shot.py).from tools.browser_use_shot import harness_commandcmd = harness_command(cmd, activate_new_tabs=bool(env.pop(_CLOUD_BROWSER_SENTINEL, None)))diff --git a/tools/browser_use_shot.py b/tools/browser_use_shot.pyindex bb0f283fcfa..976d75f2a25 100644--- a/tools/browser_use_shot.py+++ b/tools/browser_use_shot.py@@ -12,6 +12,7 @@ from __future__ import annotationsimport base64import io+import loggingimport osimport tempfilefrom typing import Any, Callable@@ -21,6 +22,7 @@ from typing import Any, CallablePNG_DEADLINE_S = 10.0JPEG_DEADLINE_S = 60.0_HARNESS_MODULE = "browser_harness.helpers"+_log = logging.getLogger("hermes_browser_use_shot")def _default_shot_path() -> str:@@ -55,13 +57,17 @@ def _write_shot(data: bytes, path: str, fmt: str) -> None:f.write(data)-def make_capture_screenshot(cdp: Callable[..., Any], current_tab: Callable[[], dict]) -> Callable[..., str]:+def make_capture_screenshot(cdp: Callable[..., Any], current_tab: Callable[[], dict],+ activate_hidden: bool = False) -> Callable[..., str]:+ """``activate_hidden`` only on a browser no human watches (cloud): on the user's own or connected+ Chrome, activating the agent's tab would switch the tab they are looking at. Every backend keeps the+ bounded png attempt and the jpeg fallback."""def capture_screenshot(path=None, full=False, max_dim=None):- """Save a PNG of the current viewport (bringing a hidden tab to the front first). Set- max_dim=1800 on a 2x display to keep the file under the 2000px-per-side limit some image-aware- LLMs enforce."""+ """Save a PNG of the current viewport. Set max_dim=1800 on a 2x display to keep the file under+ the 2000px-per-side limit some image-aware LLMs enforce."""path = os.fspath(path or _default_shot_path()) # a pathlib.Path works too- _bring_to_front_if_hidden(cdp, current_tab)+ if activate_hidden:+ _bring_to_front_if_hidden(cdp, current_tab)fmt = "png"try:r = cdp("Page.captureScreenshot", _response_timeout=PNG_DEADLINE_S,@@ -91,7 +97,11 @@ def make_new_tab(new_tab: Callable[..., Any], cdp: Callable[..., Any]) -> Callab"""Open (or reuse a blank) tab, navigate to url, and bring it to the front."""tid = new_tab(url)if tid:- cdp("Target.activateTarget", targetId=tid)+ try:+ cdp("Target.activateTarget", targetId=tid)+ # The tab exists either way; a provider that rejects activation leaves it in the background.+ except (RuntimeError, OSError) as e:+ _log.warning("new_tab: Target.activateTarget failed, tab left in background: %s", e)return tidreturn new_tab_in_front@@ -106,9 +116,11 @@ def install(helpers: Any, activate_new_tabs: bool = False) -> None:"""Rebind the helpers on the ``browser_harness.helpers`` module before ``browser_harness.run`` star-importsthem, so the agent's code, the harness's step tracer and recorder all see these. ``activate_new_tabs`` isfor browsers no human is watching (cloud providers): there a background tab only costs throttling and- blank screenshots, while on a user's own Chrome the harness deliberately leaves the visible tab alone."""+ blank screenshots, so new tabs and hidden tabs being captured are brought to the front; on a user's own+ Chrome the harness deliberately leaves the visible tab alone."""if _is_harness_helper(helpers.capture_screenshot):- helpers.capture_screenshot = make_capture_screenshot(helpers.cdp, helpers.current_tab)+ helpers.capture_screenshot = make_capture_screenshot(helpers.cdp, helpers.current_tab,+ activate_hidden=activate_new_tabs)if activate_new_tabs and _is_harness_helper(helpers.new_tab):helpers.new_tab = make_new_tab(helpers.new_tab, helpers.cdp)