Agent Zero AI framework
Consolidate vision tool prompts
Alessandro0354324e1340
diff --git a/extensions/python/system_prompt/_11_tools_prompt.py b/extensions/python/system_prompt/_11_tools_prompt.pyindex a395d6c3b..d9086e503 100644--- a/extensions/python/system_prompt/_11_tools_prompt.py+++ b/extensions/python/system_prompt/_11_tools_prompt.py@@ -57,9 +57,10 @@ async def build_prompt(agent: Agent) -> str:)chat_cfg = get_chat_model_config(agent)- if use_vision_sidecar(agent):- prompt += "\n\n" + agent.read_prompt("agent.system.tools_vision_sidecar.md")- elif chat_cfg.get("vision", False):- prompt += "\n\n" + agent.read_prompt("agent.system.tools_vision.md")+ sidecar = use_vision_sidecar(agent)+ if sidecar or chat_cfg.get("vision", False):+ prompt += "\n\n" + agent.read_prompt(+ "agent.system.tools_vision.md", sidecar=sidecar+ )return promptdiff --git a/helpers/responses_tools.py b/helpers/responses_tools.pyindex 877f03d79..28b86ed28 100644--- a/helpers/responses_tools.py+++ b/helpers/responses_tools.py@@ -119,11 +119,10 @@ def _vision_tool_prompt(agent: Any) -> str:use_vision_sidecar,)- if use_vision_sidecar(agent):- return agent.read_prompt("agent.system.tools_vision_sidecar.md")- if not get_chat_model_config(agent).get("vision", False):+ sidecar = use_vision_sidecar(agent)+ if not (sidecar or get_chat_model_config(agent).get("vision", False)):return ""- return agent.read_prompt("agent.system.tools_vision.md")+ return agent.read_prompt("agent.system.tools_vision.md", sidecar=sidecar)except Exception:return ""diff --git a/helpers/responses_tools.py.dox.md b/helpers/responses_tools.py.dox.mdindex 041919146..3a7edc4aa 100644--- a/helpers/responses_tools.py.dox.md+++ b/helpers/responses_tools.py.dox.md@@ -12,7 +12,7 @@## Local Contracts-- Build local function tools from enabled `agent.system.tool.*.md` prompt files and include `vision_load` when either Main native vision or the effective preset's Vision Model enables the matching prompt.+- Build local function tools from enabled `agent.system.tool.*.md` prompt files and include `vision_load` when either Main native vision or the effective preset's Vision Model enables the canonical vision prompt.- Discover local prompt files through `helpers.subagents.get_paths`; this moduleowns the Responses-specific prompt-name compatibility rules.- Local prompt-derived function names use existing bullet declarations that pair a backticked name with `arg` or `args` for multi-tool prompt files, otherwise prefer explicit `"tool_name"` examples, then the first prompt heading, and finally the prompt filename.diff --git a/prompts/AGENTS.md b/prompts/AGENTS.mdindex 3119674eb..132fa30a2 100644--- a/prompts/AGENTS.md+++ b/prompts/AGENTS.md@@ -25,7 +25,7 @@- Read the rendering path before changing placeholders or filenames.- Prefer small prompt additions over broad rewrites when fixing a specific behavior.- Keep document/OCR routing explicit: image files, screenshots, scans, charts, photos, and diagrams should prefer vision tools when available, while `document_query` is for documents, large text-heavy files, and fallback OCR.-- Keep native and sidecar vision prompts synchronized with `vision_load`: related images belong in one call, while a sidecar result is text-only unless Main explicitly requests its native raw route.+- Keep the single `vision_load` prompt accurate for both native and sidecar routing: related images belong in one call, while a sidecar result is text-only unless Main explicitly requests its native raw route.- Update tests or snapshots when prompt budget, required sections, or generated system content changes.## Verificationdiff --git a/prompts/agent.system.tools_vision.md b/prompts/agent.system.tools_vision.mdindex ee50376ad..9cbeb91e9 100644--- a/prompts/agent.system.tools_vision.md+++ b/prompts/agent.system.tools_vision.md@@ -1,11 +1,41 @@## multimodal vision tools### vision_load-load images into the model for visual reasoning-args: `paths` list of absolute image paths or tool-returned ephemeral image refs+{{if sidecar}}analyze images with the preset's separate Vision Model and return a text result{{endif}}+{{if not sidecar}}load images into Main for visual reasoning{{endif}}+args: `paths` list of absolute image paths or ephemeral image refs{{if sidecar}}, `query` optional focused instruction, `raw` optional boolean{{endif}}+{{if sidecar}}+Input schema for tool_args:+```json+{+ "type": "object",+ "properties": {+ "paths": {+ "type": "array",+ "items": {"type": "string"},+ "description": "Absolute image paths or ephemeral image refs."+ },+ "query": {+ "type": "string",+ "description": "What the Vision Model should inspect, compare, locate, or read."+ },+ "raw": {+ "type": "boolean",+ "description": "Use Main's native vision instead of the separate Vision Model, when Main supports vision."+ }+ },+ "required": ["paths"],+ "additionalProperties": false+}+```+{{endif}}rules:-- load all relevant images in one call when comparing screenshots or pages+- put all images needed for one comparison or visual task in the same `paths` array- use when the task depends on screenshots, diagrams, scanned documents, charts, or photos+{{if sidecar}}+- when the separate Vision Model is active, use a focused `query`; the result is a text capsule and Main does not receive raw images+- use `raw=true` only when Main supports vision and must inspect the pixels itself+{{endif}}- only bitmaps are supported; convert other formats first if needed- the tool result includes loaded/skipped image totals and the corresponding path listsexample:@@ -14,10 +44,12 @@ example:"thoughts": ["I need to inspect the screenshot before answering."],- "headline": "Loading screenshot for visual analysis",+ "headline": "Comparing screenshots","tool_name": "vision_load","tool_args": {- "paths": ["/path/to/screenshot.png"]+ "paths": ["/path/to/before.png", "/path/to/after.png"]{{if sidecar}},+ "query": "Compare the error banners and describe what changed."+ {{endif}}}}```diff --git a/prompts/agent.system.tools_vision_sidecar.md b/prompts/agent.system.tools_vision_sidecar.mddeleted file mode 100644index 73a84ee23..000000000--- a/prompts/agent.system.tools_vision_sidecar.md+++ /dev/null@@ -1,48 +0,0 @@-## multimodal vision tools--### vision_load-analyze images with the separate Vision Model and return a text result-args: `paths` list of absolute image paths or ephemeral image refs, `query` optional focused instruction, `raw` optional boolean-Input schema for tool_args:-```json-{- "type": "object",- "properties": {- "paths": {- "type": "array",- "items": {"type": "string"},- "description": "Absolute image paths or ephemeral image refs."- },- "query": {- "type": "string",- "description": "What the Vision Model should inspect, compare, locate, or read."- },- "raw": {- "type": "boolean",- "description": "Use the Main model's native vision instead, when Main supports vision."- }- },- "required": ["paths"],- "additionalProperties": false-}-```-rules:-- put all images needed for one comparison or visual task in the same `paths` array; they are sent in one Vision Model call-- use a focused `query`; if omitted, the Vision Model returns a concise general description-- the result is a text capsule; the Main model does not receive the raw images-- use `raw=true` only when Main supports vision and must inspect the pixels itself-- only bitmaps are supported-example:-```json-{- "thoughts": [- "I need to compare both screenshots before answering."- ],- "headline": "Comparing screenshots",- "tool_name": "vision_load",- "tool_args": {- "paths": ["/path/to/before.png", "/path/to/after.png"],- "query": "Compare the error banners and describe what changed."- }-}-```diff --git a/tests/test_tool_policy.py b/tests/test_tool_policy.pyindex 3d3447325..f64231136 100644--- a/tests/test_tool_policy.py+++ b/tests/test_tool_policy.py@@ -8,7 +8,7 @@ from types import SimpleNamespaceimport pytestfrom extensions.python.system_prompt import _11_tools_prompt, _13_skills_prompt-from helpers import mcp_handler, responses_tools, tool_policy+from helpers import files, mcp_handler, responses_tools, tool_policyfrom helpers.errors import RepairableExceptionfrom plugins._tool_access.extensions.python.tool_execute_before._10_enforce_tool_policy import (EnforceToolPolicy,@@ -547,19 +547,14 @@ async def test_vision_tool_follows_chat_config_not_profile_policy(@pytest.mark.asyncio-async def test_vision_sidecar_prompt_replaces_native_vision_prompt(+async def test_vision_sidecar_uses_canonical_vision_prompt(monkeypatch, tmp_path: Path) -> None:_write_prompt(tmp_path, "agent.system.tools.md", "TOOLS\n{{tools}}")_write_prompt(tmp_path,"agent.system.tools_vision.md",- "### vision_load\nnative pixels\nargs: `paths`",- )- _write_prompt(- tmp_path,- "agent.system.tools_vision_sidecar.md",- "### vision_load\nsidecar capsule\nargs: `paths`, `query`",+ "### vision_load\ncanonical vision\nargs: `paths`, `query`",)monkeypatch.setattr(tool_policy.subagents, "get_paths", _prompt_paths(tmp_path))monkeypatch.setattr(@@ -576,19 +571,20 @@ async def test_vision_sidecar_prompt_replaces_native_vision_prompt(prompt = await _11_tools_prompt.build_prompt(agent)schemas, _name_map = responses_tools.build_responses_function_tools(agent)- assert "sidecar capsule" in prompt- assert "native pixels" not in prompt+ assert prompt.count("canonical vision") == 1assert schemas[0]["name"] == "vision_load"- assert schemas[0]["description"] == "sidecar capsule"+ assert schemas[0]["description"] == "canonical vision"-def test_vision_sidecar_prompt_declares_multi_image_native_schema() -> None:- prompt = (+def test_vision_prompt_declares_multi_image_native_schema() -> None:+ source = (Path(__file__).resolve().parents[1]/ "prompts"- / "agent.system.tools_vision_sidecar.md"+ / "agent.system.tools_vision.md").read_text(encoding="utf-8")- schema = responses_tools._schema_from_prompt(prompt)+ sidecar_prompt = files.evaluate_text_conditions(source, sidecar=True)+ native_prompt = files.evaluate_text_conditions(source, sidecar=False)+ schema = responses_tools._schema_from_prompt(sidecar_prompt)assert schema["required"] == ["paths"]assert schema["properties"]["paths"] == {@@ -597,6 +593,13 @@ def test_vision_sidecar_prompt_declares_multi_image_native_schema() -> None:"description": "Absolute image paths or ephemeral image refs.",}assert {"query", "raw"} <= schema["properties"].keys()+ assert "separate Vision Model" in sidecar_prompt+ assert "separate Vision Model" not in native_prompt+ assert responses_tools._schema_from_prompt(native_prompt) == {+ "type": "object",+ "properties": {},+ "additionalProperties": True,+ }def test_mcp_prompt_and_native_schema_omit_blocked_tool(