The agent that grows with you
Python aiai-agentai-agentsanthropicchatgptclaudeclaude-codecodexhermeshermes-agentllmnous-researchopenai
"""Screenshot and tab helpers that browser_exec installs over browser-harness's own.
Chrome draws no compositor frame for a hidden tab, and some remotes (Browser Use cloud) never answer
``Page.captureScreenshot`` with ``format: "png"`` for one, so the harness's ``capture_screenshot()`` hangs
for its full 60 s IPC timeout. Its ``new_tab()`` and its daemon both create tabs with ``background=True``,
so the tab the agent drives is often hidden. browser_harness is a pinned third-party dependency; this
module is loaded by file path inside the harness process (``harness_command()``), so it imports nothing
from Hermes and touches the harness only through the helpers module it is given.
"""
from __future__ import annotations
import base64
import io
import logging
import os
import tempfile
from typing import Any, Callable
# A png that has not arrived by then will not: a visible tab answers in well under a second, a hidden
# one on Browser Use cloud never does. jpeg on the same hidden tab answers in a few seconds.
PNG_DEADLINE_S = 10.0
JPEG_DEADLINE_S = 60.0
_HARNESS_MODULE = "browser_harness.helpers"
_log = logging.getLogger("hermes_browser_use_shot")
def _default_shot_path() -> str:
try:
from browser_harness import _ipc # the harness's own per-user temp dir
return str(_ipc._TMP / "shot.png")
except (ImportError, AttributeError):
return os.path.join(tempfile.gettempdir(), "shot.png")
def _bring_to_front_if_hidden(cdp: Callable[..., Any], current_tab: Callable[[], dict]) -> None:
"""Activate the attached tab when the page reports it is hidden. A visible tab is left alone, so a
capture never takes window focus it does not need."""
try:
state = cdp("Runtime.evaluate", expression="document.visibilityState", returnByValue=True)
if (state.get("result") or {}).get("value") != "hidden":
return
cdp("Target.activateTarget", targetId=current_tab()["targetId"])
# A CDP error (Lightpanda has no Target.activateTarget) or a slow IPC round trip: capture anyway.
except (RuntimeError, OSError, KeyError):
return
def _write_shot(data: bytes, path: str, fmt: str) -> None:
"""Write ``data`` (encoded as ``fmt``) to ``path``, re-encoding a jpeg as png when the path says png:
callers and the screenshot embed trust the extension."""
if fmt == "jpeg" and path.lower().endswith(".png"):
from PIL import Image
Image.open(io.BytesIO(data)).save(path, format="PNG")
return
with open(path, "wb") as f:
f.write(data)
def make_capture_screenshot(cdp: Callable[..., Any], current_tab: Callable[[], dict],
activate_hidden: Callable[[], bool] = lambda: False) -> Callable[..., str]:
"""``activate_hidden()`` is true only on a browser no human watches (cloud): on the user's own or
connected Chrome, activating the agent's tab would switch the tab they are looking at. Every backend
keeps the bounded png attempt and the jpeg fallback."""
def capture_screenshot(path=None, full=False, max_dim=None):
"""Save a PNG of the current viewport. Set max_dim=1800 on a 2x display to keep the file under
the 2000px-per-side limit some image-aware LLMs enforce."""
path = os.fspath(path or _default_shot_path()) # a pathlib.Path works too
if activate_hidden():
_bring_to_front_if_hidden(cdp, current_tab)
fmt = "png"
try:
r = cdp("Page.captureScreenshot", _response_timeout=PNG_DEADLINE_S,
format="png", captureBeyondViewport=full)
except TimeoutError:
fmt = "jpeg"
try:
r = cdp("Page.captureScreenshot", _response_timeout=JPEG_DEADLINE_S,
format="jpeg", quality=90, captureBeyondViewport=full)
except TimeoutError as e:
raise RuntimeError(
f"Page.captureScreenshot timed out (png after {PNG_DEADLINE_S:g}s, "
f"jpeg after {JPEG_DEADLINE_S:g}s)") from e
_write_shot(base64.b64decode(r["data"]), path, fmt)
if max_dim:
from PIL import Image
img = Image.open(path)
if max(img.size) > max_dim:
img.thumbnail((max_dim, max_dim))
img.save(path)
return path
return capture_screenshot
def make_new_tab(new_tab: Callable[..., Any], cdp: Callable[..., Any],
activate: Callable[[], bool] = lambda: True) -> Callable[..., Any]:
def new_tab_in_front(url="about:blank"):
"""Open (or reuse a blank) tab, navigate to url, and bring it to the front on a cloud browser."""
tid = new_tab(url)
if tid and activate():
try:
cdp("Target.activateTarget", targetId=tid)
# The tab exists either way; a provider that rejects activation leaves it in the background.
except (RuntimeError, OSError) as e:
_log.warning("new_tab: Target.activateTarget failed, tab left in background: %s", e)
return tid
return new_tab_in_front
def _is_harness_helper(fn: Any) -> bool:
"""True for the harness's own helper; a same-named function from the agent's workspace
``agent_helpers.py`` (loaded into the helpers module at import) is the agent's choice."""
return callable(fn) and getattr(fn, "__module__", "") == _HARNESS_MODULE
# Which browser the tabs may be activated on, decided by browser_exec from the route it resolved.
ACTIVATE_NEVER = "never" # local, connected or real-profile Chrome: a human may be looking at it
ACTIVATE_ALWAYS = "always" # a cloud provider's session whose CDP endpoint Hermes itself resolved
ACTIVATE_IF_CLOUD = "if-cloud" # BU_AUTOSPAWN: the harness picks the browser (a listening local Chrome wins)
def _harness_daemon_is_cloud() -> bool:
"""True only when the live harness daemon says it drives a Browser Use cloud browser (it was started
with that browser's id). Asked per call, after the harness has chosen or replaced its daemon; an
unreachable or older daemon counts as not cloud."""
from browser_harness import admin
return admin.daemon_browser_kind() == "cloud"
def _activation_check(mode: str) -> Callable[[], bool]:
if mode == ACTIVATE_IF_CLOUD:
return _harness_daemon_is_cloud
return (lambda: True) if mode == ACTIVATE_ALWAYS else (lambda: False)
def install(helpers: Any, activate: str = ACTIVATE_NEVER) -> None:
"""Rebind the helpers on the ``browser_harness.helpers`` module before ``browser_harness.run`` star-imports
them, so the agent's code, the harness's step tracer and recorder all see these. Activation is for
browsers no human is watching (cloud): there a background tab only costs throttling and blank
screenshots, so new tabs and hidden tabs being captured are brought to the front; on a user's own
Chrome the harness deliberately leaves the visible tab alone. ``activate`` is an ``ACTIVATE_*`` mode."""
check = _activation_check(activate)
if _is_harness_helper(helpers.capture_screenshot):
helpers.capture_screenshot = make_capture_screenshot(helpers.cdp, helpers.current_tab,
activate_hidden=check)
if activate != ACTIVATE_NEVER and _is_harness_helper(helpers.new_tab):
helpers.new_tab = make_new_tab(helpers.new_tab, helpers.cdp, activate=check)
_HARNESS_MODULE_ARGS = ["-m", "browser_harness.run"]
_LAUNCHER = """\
import importlib.util, runpy
spec = importlib.util.spec_from_file_location('hermes_browser_use_shot', {path!r})
shot = importlib.util.module_from_spec(spec); spec.loader.exec_module(shot)
import browser_harness.helpers
shot.install(browser_harness.helpers, activate={activate!r})
runpy.run_module('browser_harness.run', run_name='__main__', alter_sys=True)
"""
def harness_command(cmd: list[str], activate: str) -> list[str]:
"""``python -m browser_harness.run`` as a launcher that installs these helpers first. The model's code
still arrives untouched on stdin (no preamble shifting its traceback line numbers). This file is loaded
by path: the harness interpreter has no Hermes checkout on sys.path."""
if cmd[1:] != _HARNESS_MODULE_ARGS:
return cmd
return [cmd[0], "-c", _LAUNCHER.format(path=os.path.abspath(__file__), activate=activate)]