The agent that grows with you
Python aiai-agentai-agentsanthropicchatgptclaudeclaude-codecodexhermeshermes-agentllmnous-researchopenai
fix(browser): browser_exec says how screenshots reach the model instead of calling it blind
Tekniumbb216b95bd2e
diff --git a/tests/tools/test_browser_use_cli.py b/tests/tools/test_browser_use_cli.pyindex 7b6395b45d2..9bd2fb574d2 100644--- a/tests/tools/test_browser_use_cli.py+++ b/tests/tools/test_browser_use_cli.py@@ -730,25 +730,25 @@ class TestHeaderVariants:monkeypatch.setattr("tools.vision_tools._should_use_native_vision_fast_path", lambda: False)+ monkeypatch.setattr("tools.vision_tools.check_vision_requirements", lambda: False)monkeypatch.setattr("tools.browser_lightpanda.lightpanda_engine_status", lambda: (False, ""))assert bu_cli._description_header() == bu_cli._HEADER_BASE + bu_cli._HEADER_TEXT_ONLY- def test_text_only_header_claims_nothing_about_model_vision(self, monkeypatch):- """The text-only gate measures whether screenshots ride back in TOOL RESULTS, not whether- the model has vision; a vision-capable model on a provider without tool-result media must- not be told it "cannot view images" (#131568: it then skipped a user-attached image)."""- monkeypatch.setattr(- "tools.vision_tools._should_use_native_vision_fast_path", lambda: False- )- monkeypatch.setattr(- "tools.browser_tool_lightpanda_fallback.lightpanda_engine_status", lambda: (False, "")- )- header = bu_cli._description_header().lower()- assert "your model cannot" not in header- assert "cannot view images" not in header- assert "tool's results" in header and "text-first" in header+ def test_non_native_header_never_calls_the_model_blind(self, monkeypatch):+ """The gate measures whether screenshots ride back in THIS tool's results, not whether the+ model sees (#131568: "your model cannot view images" made a model skip a user image). With a+ vision route (an explicit auxiliary.vision pin) the header names it; without one it says so."""+ monkeypatch.setattr("tools.vision_tools._should_use_native_vision_fast_path", lambda: False)+ monkeypatch.setattr("tools.browser_lightpanda.lightpanda_engine_status", lambda: (False, ""))+ for aux_route in (True, False):+ monkeypatch.setattr("tools.vision_tools.check_vision_requirements", lambda: aux_route)+ header = bu_cli._description_header().lower()+ assert "cannot view images" not in header and "your model cannot" not in header+ assert "described below" not in header # no screenshot section follows the header+ assert "not attached to this tool's results" in header and "text-first" in header+ assert ("vision_analyze" in header) is aux_routeclass TestSkillTextDescription:diff --git a/tools/browser_use_cli.py b/tools/browser_use_cli.pyindex 159de1b3d90..1c18831f1c9 100644--- a/tools/browser_use_cli.py+++ b/tools/browser_use_cli.py@@ -808,25 +808,32 @@ _HEADER_VISION = ("with your own vision — never send browser screenshots to a separate vision tool.")-# Chosen when screenshots cannot ride back inside THIS tool's results (provider/model-# rejects tool-result media, or routing is not ``native``). That is a transport fact-# about tool results, not a verdict on the model's vision: a vision-capable model still-# sees images the user attaches to the conversation, so the header must never say-# "your model cannot view images" — models quote that against themselves (#131568).-_HEADER_TEXT_ONLY = (- " Screenshots will not reach you through this tool's results in this setup, so work text-first: "+_TEXT_FIRST = ("page_info() for state, js() for reading/extracting DOM text, fill_input(selector, text) for inputs, and "- "js(\"document.querySelector('…').click()\") for clicks — skip the screenshot-driven workflow described below. "- "This says nothing about images the user attaches to the conversation."+ "js(\"document.querySelector('…').click()\") for clicks."+)++# The non-native headers state a transport fact (screenshots do not ride back inside THIS+# tool's results: routing is not ``native`` — e.g. an explicit ``auxiliary.vision`` pin — or+# the route rejects tool-result media), never a verdict on the model's own vision: models+# quote "your model cannot view images" against themselves and skip user images (#131568).+_HEADER_VISION_TOOL = (+ " Screenshots are not attached to this tool's results; to inspect one, pass its capture_screenshot() "+ "path to vision_analyze. Otherwise work text-first: " + _TEXT_FIRST+)++_HEADER_TEXT_ONLY = (+ " Screenshots are not attached to this tool's results and no vision route is configured, so work "+ "text-first: " + _TEXT_FIRST)-# Appended when the local engine is Lightpanda: no graphical renderer, and one CDP-# connection holds one page — a second Target.createTarget fails with-# TargetAlreadyLoaded (drop the new_tab() sentence once lightpanda-io/browser#1962 lands).+# Lightpanda: no graphical renderer, and one CDP connection holds one page — a second+# Target.createTarget fails with TargetAlreadyLoaded (drop the new_tab() sentence once+# lightpanda-io/browser#1962 lands)._HEADER_LIGHTPANDA = (" The local engine is Lightpanda (no graphical renderer, one page per session): capture_screenshot() "- "is unavailable, so work text-first; navigate with new_tab(url) exactly once, then goto_url(url) for "- "every later navigation — a second new_tab() fails with TargetAlreadyLoaded."+ "is unavailable, so work text-first: " + _TEXT_FIRST + " Navigate with new_tab(url) exactly once, then "+ "goto_url(url) for every later navigation — a second new_tab() fails with TargetAlreadyLoaded.")# Pinned quick-reference for the CLI's pre-imported helpers, replacing the live@@ -847,12 +854,15 @@ _HELPERS_DIGEST = (def _description_header() -> str:- """Header tailored to whether the active model can see images natively"""+ """Header tailored to how a screenshot reaches the model: attached natively, via+ vision_analyze (the same gate that registers that tool), or not at all."""if _lazy_call("tools.browser_lightpanda", "lightpanda_engine_status", (False, ""),"lightpanda engine status unavailable")[0]: # no screenshots, whatever the model sees- return _HEADER_BASE + _HEADER_TEXT_ONLY + _HEADER_LIGHTPANDA- vision = _lazy_call("tools.vision_tools", "_should_use_native_vision_fast_path", False, "")- return _HEADER_BASE + (_HEADER_VISION if vision else _HEADER_TEXT_ONLY)+ return _HEADER_BASE + _HEADER_LIGHTPANDA+ if _lazy_call("tools.vision_tools", "_should_use_native_vision_fast_path", False, ""):+ return _HEADER_BASE + _HEADER_VISION+ vision_tool = _lazy_call("tools.vision_tools", "check_vision_requirements", False, "vision route lookup failed")+ return _HEADER_BASE + (_HEADER_VISION_TOOL if vision_tool else _HEADER_TEXT_ONLY)def _dynamic_schema_overrides() -> dict: