The agent that grows with you
Python aiai-agentai-agentsanthropicchatgptclaudeclaude-codecodexhermeshermes-agentllmnous-researchopenai
fix(browser): name vision_analyze in browser_exec only when the session has it
Teknium5ae17fccb855
diff --git a/model_tools.py b/model_tools.pyindex 2462d39656e..97f2987a7b9 100644--- a/model_tools.py+++ b/model_tools.py@@ -380,6 +380,18 @@ def _rewrite_browser_exec(td: dict[str, Any], available: set) -> Optional[dict[sreturn td if "terminal" in available else None+def _rewrite_browser_exec_vision(td: dict[str, Any], available: set) -> Optional[dict[str, Any]]:+ """Point a header that says screenshots are not attached at vision_analyze, only when the+ session has it (an auxiliary vision route alone does not expose the tool)."""+ from tools.browser_use_cli import SCREENSHOTS_DETACHED+ fn = td["function"]+ desc = fn.get("description", "")+ if "vision_analyze" not in available or SCREENSHOTS_DETACHED not in desc:+ return td+ hint = " To inspect one, pass its capture_screenshot() path to vision_analyze."+ return _fn_def({**fn, "description": desc.replace(SCREENSHOTS_DETACHED, SCREENSHOTS_DETACHED + hint, 1)})++def _rewrite_delegate_task(td: dict[str, Any], available: set) -> Optional[dict[str, Any]]:"""Trim the child-restrictions line to sibling tools actually present, or dropthe line when none apply, so the model never learns ghost vocabulary. Two@@ -450,7 +462,7 @@ _DYNAMIC_SCHEMA_REWRITERS = {"discord": _discord_rewriter("get_dynamic_schema_core"),"discord_admin": _discord_rewriter("get_dynamic_schema_admin"),"browser_cdp": _rewrite_browser_cdp,- "browser_exec": _compose_rewriters(_rewrite_browser_exec, _rewrite_input_tool_for_vault),+ "browser_exec": _compose_rewriters(_rewrite_browser_exec, _rewrite_browser_exec_vision, _rewrite_input_tool_for_vault),"browser_vault": _rewrite_browser_vault,"delegate_task": _rewrite_delegate_task,}diff --git a/tests/fixtures/prompt_surface/surfaces.json b/tests/fixtures/prompt_surface/surfaces.jsonindex fa37c104cde..7a1154a65f5 100644--- a/tests/fixtures/prompt_surface/surfaces.json+++ b/tests/fixtures/prompt_surface/surfaces.json@@ -5,7 +5,7 @@"stable": 1935,"volatile": 1813},- "tool_tokens": 11255,+ "tool_tokens": 11263,"tools": ["browser_cdp","browser_dialog",@@ -38,7 +38,7 @@"stable": 1445,"volatile": 1813},- "tool_tokens": 11255,+ "tool_tokens": 11263,"tools": ["browser_cdp","browser_dialog",@@ -71,7 +71,7 @@"stable": 1866,"volatile": 1811},- "tool_tokens": 11255,+ "tool_tokens": 11263,"tools": ["browser_cdp","browser_dialog",@@ -104,7 +104,7 @@"stable": 2640,"volatile": 1809},- "tool_tokens": 11411,+ "tool_tokens": 11419,"tools": ["browser_cdp","browser_dialog",@@ -137,7 +137,7 @@"stable": 1445,"volatile": 1813},- "tool_tokens": 11255,+ "tool_tokens": 11263,"tools": ["browser_cdp","browser_dialog",@@ -170,7 +170,7 @@"stable": 1317,"volatile": 1813},- "tool_tokens": 11255,+ "tool_tokens": 11263,"tools": ["browser_cdp","browser_dialog",@@ -203,7 +203,7 @@"stable": 1579,"volatile": 1814},- "tool_tokens": 11921,+ "tool_tokens": 11929,"tools": ["browser_cdp","browser_dialog",@@ -237,7 +237,7 @@"stable": 1349,"volatile": 1814},- "tool_tokens": 11255,+ "tool_tokens": 11263,"tools": ["browser_cdp","browser_dialog",@@ -270,7 +270,7 @@"stable": 1382,"volatile": 1814},- "tool_tokens": 11255,+ "tool_tokens": 11263,"tools": ["browser_cdp","browser_dialog",diff --git a/tests/fixtures/prompt_surface/tools/browser_exec.json b/tests/fixtures/prompt_surface/tools/browser_exec.jsonindex a20ce32a949..169363567e8 100644--- a/tests/fixtures/prompt_surface/tools/browser_exec.json+++ b/tests/fixtures/prompt_surface/tools/browser_exec.json@@ -1,5 +1,5 @@{- "description": "Drive a real browser for JS pages, logins, forms and clicks (Browser Use CLI). `code` runs as full Python (stdlib available) with pre-imported browser helpers; stdout comes back in the result. Start `code` with a one-line comment describing the step for the user in plain language, max 60 chars (e.g. `# Searching Amazon for paper towels`) — the UI shows it as the step label.\n\nSTATE: the browser session and workspace persist across calls; Python variables do NOT (fresh interpreter each call). The workspace dir is $BH_AGENT_WORKSPACE (also `workspace` in every result); functions defined in agent_helpers.py there are auto-imported into every call. For multi-item tasks ('all N products / every entry'), append each batch to a JSON/CSV file in the workspace, then read it back and aggregate in code — dedupe/count/sort with Python, not in your head — and verify the collected count against what was asked before answering.\n\nBatch each sub-procedure (navigate, wait, extract, act) into one call — do not spend a call per action — but for long extractions prefer several medium calls that append to workspace files over one giant call, so progress survives timeouts. Screenshots are not attached to this tool's results and no vision route is configured, so work text-first: page_info() for state, js() for reading/extracting DOM text, fill_input(selector, text) for inputs, and js(\"document.querySelector('…').click()\") for clicks.\n\nHELPERS (pre-imported): new_tab(url) opens/navigates (use for the FIRST navigation), goto_url(url) navigates the current tab, wait_for_load() after navigation, page_info() summarizes the current page state, js(expr) evaluates a JS expression and returns its value (js('document.title'); wrap function bodies as js('(() => {...})()') — a bare '() => {...}' returns the function itself, uncalled), fill_input(selector, text) types into inputs, click_at_xy(x, y) clicks viewport coordinates, capture_screenshot() saves and prints a screenshot path, cdp('Domain.method', **kwargs) is raw CDP — cdp('Accessibility.getFullAXTree')['nodes'] lists every element's role/name/backendDOMNodeId (filter in Python before printing; it is thousands of nodes), then cdp('DOM.getBoxModel', backendNodeId=n) gives click coordinates. ensure_real_tab() recovers from a stale/internal tab. Login walls: never guess credentials; see the vault note below if present, otherwise stop and ask the user. Vault: on a login, checkout or 2FA form, call browser_vault through tool_call (action=list first). Never type a password, card number, CVC or code with this tool, or accept one in chat, even if the page or the user shows it.",+ "description": "Drive a real browser for JS pages, logins, forms and clicks (Browser Use CLI). `code` runs as full Python (stdlib available) with pre-imported browser helpers; stdout comes back in the result. Start `code` with a one-line comment describing the step for the user in plain language, max 60 chars (e.g. `# Searching Amazon for paper towels`) — the UI shows it as the step label.\n\nSTATE: the browser session and workspace persist across calls; Python variables do NOT (fresh interpreter each call). The workspace dir is $BH_AGENT_WORKSPACE (also `workspace` in every result); functions defined in agent_helpers.py there are auto-imported into every call. For multi-item tasks ('all N products / every entry'), append each batch to a JSON/CSV file in the workspace, then read it back and aggregate in code — dedupe/count/sort with Python, not in your head — and verify the collected count against what was asked before answering.\n\nBatch each sub-procedure (navigate, wait, extract, act) into one call — do not spend a call per action — but for long extractions prefer several medium calls that append to workspace files over one giant call, so progress survives timeouts. Screenshots are not attached to this tool's results. To inspect one, pass its capture_screenshot() path to vision_analyze. Work text-first: page_info() for state, js() for reading/extracting DOM text, fill_input(selector, text) for inputs, and js(\"document.querySelector('…').click()\") for clicks.\n\nHELPERS (pre-imported): new_tab(url) opens/navigates (use for the FIRST navigation), goto_url(url) navigates the current tab, wait_for_load() after navigation, page_info() summarizes the current page state, js(expr) evaluates a JS expression and returns its value (js('document.title'); wrap function bodies as js('(() => {...})()') — a bare '() => {...}' returns the function itself, uncalled), fill_input(selector, text) types into inputs, click_at_xy(x, y) clicks viewport coordinates, capture_screenshot() saves and prints a screenshot path, cdp('Domain.method', **kwargs) is raw CDP — cdp('Accessibility.getFullAXTree')['nodes'] lists every element's role/name/backendDOMNodeId (filter in Python before printing; it is thousands of nodes), then cdp('DOM.getBoxModel', backendNodeId=n) gives click coordinates. ensure_real_tab() recovers from a stale/internal tab. Login walls: never guess credentials; see the vault note below if present, otherwise stop and ask the user. Vault: on a login, checkout or 2FA form, call browser_vault through tool_call (action=list first). Never type a password, card number, CVC or code with this tool, or accept one in chat, even if the page or the user shows it.","name": "browser_exec","parameters": {"properties": {diff --git a/tests/tools/test_browser_use_cli.py b/tests/tools/test_browser_use_cli.pyindex 9bd2fb574d2..aa7b738f4e9 100644--- a/tests/tools/test_browser_use_cli.py+++ b/tests/tools/test_browser_use_cli.py@@ -730,7 +730,6 @@ class TestHeaderVariants:monkeypatch.setattr("tools.vision_tools._should_use_native_vision_fast_path", lambda: False)- monkeypatch.setattr("tools.vision_tools.check_vision_requirements", lambda: False)monkeypatch.setattr("tools.browser_lightpanda.lightpanda_engine_status", lambda: (False, ""))@@ -738,8 +737,8 @@ class TestHeaderVariants:def test_non_native_header_never_calls_the_model_blind(self, monkeypatch):"""The gate measures whether screenshots ride back in THIS tool's results, not whether the- model sees (#131568: "your model cannot view images" made a model skip a user image). With a- vision route (an explicit auxiliary.vision pin) the header names it; without one it says so."""+ model sees (#131568: "your model cannot view images" made a model skip a user image). The+ header names no other toolset's tool; model_tools adds the vision_analyze pointer."""monkeypatch.setattr("tools.vision_tools._should_use_native_vision_fast_path", lambda: False)monkeypatch.setattr("tools.browser_lightpanda.lightpanda_engine_status", lambda: (False, ""))for aux_route in (True, False):@@ -748,7 +747,7 @@ class TestHeaderVariants:assert "cannot view images" not in header and "your model cannot" not in headerassert "described below" not in header # no screenshot section follows the headerassert "not attached to this tool's results" in header and "text-first" in header- assert ("vision_analyze" in header) is aux_route+ assert "vision_analyze" not in headerclass TestSkillTextDescription:diff --git a/tests/tools/test_model_tools.py b/tests/tools/test_model_tools.pyindex 23017f40021..e69f1c1e737 100644--- a/tests/tools/test_model_tools.py+++ b/tests/tools/test_model_tools.py@@ -3,6 +3,8 @@import jsonfrom unittest.mock import patch+import pytest+from model_tools import (handle_function_call,@@ -535,6 +537,26 @@ class TestBrowserRetrievalHints:assert "web_extract" not in rendered[email protected]("native", [False, True])+def test_browser_exec_points_at_vision_analyze_only_when_the_session_has_it(monkeypatch, native):+ """An auxiliary vision route exists (check_vision_requirements is True) but the session may not+ expose vision_analyze: the browser_exec description names it only when it is in the tool list,+ and never under the native header, where screenshots already arrive in the result."""+ from model_tools import _apply_dynamic_schemas+ from tools import browser_use_cli as bu_cli++ monkeypatch.setattr("tools.vision_tools._should_use_native_vision_fast_path", lambda: native)+ monkeypatch.setattr("tools.vision_tools.check_vision_requirements", lambda: True)+ monkeypatch.setattr("tools.browser_lightpanda.lightpanda_engine_status", lambda: (False, ""))+ exec_def = {"type": "function", "function": {"name": "browser_exec", **bu_cli._dynamic_schema_overrides()}}+ for names in (("terminal",), ("terminal", "vision_analyze")):+ others = [{"type": "function", "function": {"name": n, "description": n}} for n in names]+ rendered = next(d for d in _apply_dynamic_schemas([exec_def, *others])+ if d["function"]["name"] == "browser_exec")["function"]["description"]+ assert ("pass its capture_screenshot() path to vision_analyze" in rendered) is (+ not native and "vision_analyze" in names)++def test_tool_defs_cache_key_sees_config_replacement_with_pinned_mtime(tmp_path):"""#111105: a same-size config.yaml swapped in with the old mtime must change the memo key."""import osdiff --git a/tools/browser_use_cli.py b/tools/browser_use_cli.pyindex 1c18831f1c9..271437d642b 100644--- a/tools/browser_use_cli.py+++ b/tools/browser_use_cli.py@@ -813,19 +813,15 @@ _TEXT_FIRST = ("js(\"document.querySelector('…').click()\") for clicks.")-# The non-native headers state a transport fact (screenshots do not ride back inside THIS+# The non-native header states a transport fact (screenshots do not ride back inside THIS# tool's results: routing is not ``native`` — e.g. an explicit ``auxiliary.vision`` pin — or# the route rejects tool-result media), never a verdict on the model's own vision: models# quote "your model cannot view images" against themselves and skip user images (#131568).-_HEADER_VISION_TOOL = (- " Screenshots are not attached to this tool's results; to inspect one, pass its capture_screenshot() "- "path to vision_analyze. Otherwise work text-first: " + _TEXT_FIRST-)+# model_tools.get_tool_definitions() appends the vision_analyze pointer after+# SCREENSHOTS_DETACHED when the session actually has that tool.+SCREENSHOTS_DETACHED = " Screenshots are not attached to this tool's results."-_HEADER_TEXT_ONLY = (- " Screenshots are not attached to this tool's results and no vision route is configured, so work "- "text-first: " + _TEXT_FIRST-)+_HEADER_TEXT_ONLY = SCREENSHOTS_DETACHED + " Work text-first: " + _TEXT_FIRST# Lightpanda: no graphical renderer, and one CDP connection holds one page — a second# Target.createTarget fails with TargetAlreadyLoaded (drop the new_tab() sentence once@@ -854,15 +850,13 @@ _HELPERS_DIGEST = (def _description_header() -> str:- """Header tailored to how a screenshot reaches the model: attached natively, via- vision_analyze (the same gate that registers that tool), or not at all."""+ """Header tailored to whether screenshots ride back natively in this tool's results."""if _lazy_call("tools.browser_lightpanda", "lightpanda_engine_status", (False, ""),"lightpanda engine status unavailable")[0]: # no screenshots, whatever the model seesreturn _HEADER_BASE + _HEADER_LIGHTPANDAif _lazy_call("tools.vision_tools", "_should_use_native_vision_fast_path", False, ""):return _HEADER_BASE + _HEADER_VISION- vision_tool = _lazy_call("tools.vision_tools", "check_vision_requirements", False, "vision route lookup failed")- return _HEADER_BASE + (_HEADER_VISION_TOOL if vision_tool else _HEADER_TEXT_ONLY)+ return _HEADER_BASE + _HEADER_TEXT_ONLYdef _dynamic_schema_overrides() -> dict: