Skip to content

Agent Zero AI framework

Consolidate vision tool prompts

Alessandro0354324e1340
diff --git a/extensions/python/system_prompt/_11_tools_prompt.py b/extensions/python/system_prompt/_11_tools_prompt.py
index a395d6c3b..d9086e503 100644
--- a/extensions/python/system_prompt/_11_tools_prompt.py
+++ b/extensions/python/system_prompt/_11_tools_prompt.py
@@ -57,9 +57,10 @@ async def build_prompt(agent: Agent) -> str:
)
chat_cfg = get_chat_model_config(agent)
- if use_vision_sidecar(agent):
- prompt += "\n\n" + agent.read_prompt("agent.system.tools_vision_sidecar.md")
- elif chat_cfg.get("vision", False):
- prompt += "\n\n" + agent.read_prompt("agent.system.tools_vision.md")
+ sidecar = use_vision_sidecar(agent)
+ if sidecar or chat_cfg.get("vision", False):
+ prompt += "\n\n" + agent.read_prompt(
+ "agent.system.tools_vision.md", sidecar=sidecar
+ )
return prompt
diff --git a/helpers/responses_tools.py b/helpers/responses_tools.py
index 877f03d79..28b86ed28 100644
--- a/helpers/responses_tools.py
+++ b/helpers/responses_tools.py
@@ -119,11 +119,10 @@ def _vision_tool_prompt(agent: Any) -> str:
use_vision_sidecar,
)
- if use_vision_sidecar(agent):
- return agent.read_prompt("agent.system.tools_vision_sidecar.md")
- if not get_chat_model_config(agent).get("vision", False):
+ sidecar = use_vision_sidecar(agent)
+ if not (sidecar or get_chat_model_config(agent).get("vision", False)):
return ""
- return agent.read_prompt("agent.system.tools_vision.md")
+ return agent.read_prompt("agent.system.tools_vision.md", sidecar=sidecar)
except Exception:
return ""
diff --git a/helpers/responses_tools.py.dox.md b/helpers/responses_tools.py.dox.md
index 041919146..3a7edc4aa 100644
--- a/helpers/responses_tools.py.dox.md
+++ b/helpers/responses_tools.py.dox.md
@@ -12,7 +12,7 @@
## Local Contracts
-- Build local function tools from enabled `agent.system.tool.*.md` prompt files and include `vision_load` when either Main native vision or the effective preset's Vision Model enables the matching prompt.
+- Build local function tools from enabled `agent.system.tool.*.md` prompt files and include `vision_load` when either Main native vision or the effective preset's Vision Model enables the canonical vision prompt.
- Discover local prompt files through `helpers.subagents.get_paths`; this module
owns the Responses-specific prompt-name compatibility rules.
- Local prompt-derived function names use existing bullet declarations that pair a backticked name with `arg` or `args` for multi-tool prompt files, otherwise prefer explicit `"tool_name"` examples, then the first prompt heading, and finally the prompt filename.
diff --git a/prompts/AGENTS.md b/prompts/AGENTS.md
index 3119674eb..132fa30a2 100644
--- a/prompts/AGENTS.md
+++ b/prompts/AGENTS.md
@@ -25,7 +25,7 @@
- Read the rendering path before changing placeholders or filenames.
- Prefer small prompt additions over broad rewrites when fixing a specific behavior.
- Keep document/OCR routing explicit: image files, screenshots, scans, charts, photos, and diagrams should prefer vision tools when available, while `document_query` is for documents, large text-heavy files, and fallback OCR.
-- Keep native and sidecar vision prompts synchronized with `vision_load`: related images belong in one call, while a sidecar result is text-only unless Main explicitly requests its native raw route.
+- Keep the single `vision_load` prompt accurate for both native and sidecar routing: related images belong in one call, while a sidecar result is text-only unless Main explicitly requests its native raw route.
- Update tests or snapshots when prompt budget, required sections, or generated system content changes.
## Verification
diff --git a/prompts/agent.system.tools_vision.md b/prompts/agent.system.tools_vision.md
index ee50376ad..9cbeb91e9 100644
--- a/prompts/agent.system.tools_vision.md
+++ b/prompts/agent.system.tools_vision.md
@@ -1,11 +1,41 @@
## multimodal vision tools
### vision_load
-load images into the model for visual reasoning
-args: `paths` list of absolute image paths or tool-returned ephemeral image refs
+{{if sidecar}}analyze images with the preset's separate Vision Model and return a text result{{endif}}
+{{if not sidecar}}load images into Main for visual reasoning{{endif}}
+args: `paths` list of absolute image paths or ephemeral image refs{{if sidecar}}, `query` optional focused instruction, `raw` optional boolean{{endif}}
+{{if sidecar}}
+Input schema for tool_args:
+```json
+{
+ "type": "object",
+ "properties": {
+ "paths": {
+ "type": "array",
+ "items": {"type": "string"},
+ "description": "Absolute image paths or ephemeral image refs."
+ },
+ "query": {
+ "type": "string",
+ "description": "What the Vision Model should inspect, compare, locate, or read."
+ },
+ "raw": {
+ "type": "boolean",
+ "description": "Use Main's native vision instead of the separate Vision Model, when Main supports vision."
+ }
+ },
+ "required": ["paths"],
+ "additionalProperties": false
+}
+```
+{{endif}}
rules:
-- load all relevant images in one call when comparing screenshots or pages
+- put all images needed for one comparison or visual task in the same `paths` array
- use when the task depends on screenshots, diagrams, scanned documents, charts, or photos
+{{if sidecar}}
+- when the separate Vision Model is active, use a focused `query`; the result is a text capsule and Main does not receive raw images
+- use `raw=true` only when Main supports vision and must inspect the pixels itself
+{{endif}}
- only bitmaps are supported; convert other formats first if needed
- the tool result includes loaded/skipped image totals and the corresponding path lists
example:
@@ -14,10 +44,12 @@ example:
"thoughts": [
"I need to inspect the screenshot before answering."
],
- "headline": "Loading screenshot for visual analysis",
+ "headline": "Comparing screenshots",
"tool_name": "vision_load",
"tool_args": {
- "paths": ["/path/to/screenshot.png"]
+ "paths": ["/path/to/before.png", "/path/to/after.png"]{{if sidecar}},
+ "query": "Compare the error banners and describe what changed."
+ {{endif}}
}
}
```
diff --git a/prompts/agent.system.tools_vision_sidecar.md b/prompts/agent.system.tools_vision_sidecar.md
deleted file mode 100644
index 73a84ee23..000000000
--- a/prompts/agent.system.tools_vision_sidecar.md
+++ /dev/null
@@ -1,48 +0,0 @@
-## multimodal vision tools
-
-### vision_load
-analyze images with the separate Vision Model and return a text result
-args: `paths` list of absolute image paths or ephemeral image refs, `query` optional focused instruction, `raw` optional boolean
-Input schema for tool_args:
-```json
-{
- "type": "object",
- "properties": {
- "paths": {
- "type": "array",
- "items": {"type": "string"},
- "description": "Absolute image paths or ephemeral image refs."
- },
- "query": {
- "type": "string",
- "description": "What the Vision Model should inspect, compare, locate, or read."
- },
- "raw": {
- "type": "boolean",
- "description": "Use the Main model's native vision instead, when Main supports vision."
- }
- },
- "required": ["paths"],
- "additionalProperties": false
-}
-```
-rules:
-- put all images needed for one comparison or visual task in the same `paths` array; they are sent in one Vision Model call
-- use a focused `query`; if omitted, the Vision Model returns a concise general description
-- the result is a text capsule; the Main model does not receive the raw images
-- use `raw=true` only when Main supports vision and must inspect the pixels itself
-- only bitmaps are supported
-example:
-```json
-{
- "thoughts": [
- "I need to compare both screenshots before answering."
- ],
- "headline": "Comparing screenshots",
- "tool_name": "vision_load",
- "tool_args": {
- "paths": ["/path/to/before.png", "/path/to/after.png"],
- "query": "Compare the error banners and describe what changed."
- }
-}
-```
diff --git a/tests/test_tool_policy.py b/tests/test_tool_policy.py
index 3d3447325..f64231136 100644
--- a/tests/test_tool_policy.py
+++ b/tests/test_tool_policy.py
@@ -8,7 +8,7 @@ from types import SimpleNamespace
import pytest
from extensions.python.system_prompt import _11_tools_prompt, _13_skills_prompt
-from helpers import mcp_handler, responses_tools, tool_policy
+from helpers import files, mcp_handler, responses_tools, tool_policy
from helpers.errors import RepairableException
from plugins._tool_access.extensions.python.tool_execute_before._10_enforce_tool_policy import (
EnforceToolPolicy,
@@ -547,19 +547,14 @@ async def test_vision_tool_follows_chat_config_not_profile_policy(
@pytest.mark.asyncio
-async def test_vision_sidecar_prompt_replaces_native_vision_prompt(
+async def test_vision_sidecar_uses_canonical_vision_prompt(
monkeypatch, tmp_path: Path
) -> None:
_write_prompt(tmp_path, "agent.system.tools.md", "TOOLS\n{{tools}}")
_write_prompt(
tmp_path,
"agent.system.tools_vision.md",
- "### vision_load\nnative pixels\nargs: `paths`",
- )
- _write_prompt(
- tmp_path,
- "agent.system.tools_vision_sidecar.md",
- "### vision_load\nsidecar capsule\nargs: `paths`, `query`",
+ "### vision_load\ncanonical vision\nargs: `paths`, `query`",
)
monkeypatch.setattr(tool_policy.subagents, "get_paths", _prompt_paths(tmp_path))
monkeypatch.setattr(
@@ -576,19 +571,20 @@ async def test_vision_sidecar_prompt_replaces_native_vision_prompt(
prompt = await _11_tools_prompt.build_prompt(agent)
schemas, _name_map = responses_tools.build_responses_function_tools(agent)
- assert "sidecar capsule" in prompt
- assert "native pixels" not in prompt
+ assert prompt.count("canonical vision") == 1
assert schemas[0]["name"] == "vision_load"
- assert schemas[0]["description"] == "sidecar capsule"
+ assert schemas[0]["description"] == "canonical vision"
-def test_vision_sidecar_prompt_declares_multi_image_native_schema() -> None:
- prompt = (
+def test_vision_prompt_declares_multi_image_native_schema() -> None:
+ source = (
Path(__file__).resolve().parents[1]
/ "prompts"
- / "agent.system.tools_vision_sidecar.md"
+ / "agent.system.tools_vision.md"
).read_text(encoding="utf-8")
- schema = responses_tools._schema_from_prompt(prompt)
+ sidecar_prompt = files.evaluate_text_conditions(source, sidecar=True)
+ native_prompt = files.evaluate_text_conditions(source, sidecar=False)
+ schema = responses_tools._schema_from_prompt(sidecar_prompt)
assert schema["required"] == ["paths"]
assert schema["properties"]["paths"] == {
@@ -597,6 +593,13 @@ def test_vision_sidecar_prompt_declares_multi_image_native_schema() -> None:
"description": "Absolute image paths or ephemeral image refs.",
}
assert {"query", "raw"} <= schema["properties"].keys()
+ assert "separate Vision Model" in sidecar_prompt
+ assert "separate Vision Model" not in native_prompt
+ assert responses_tools._schema_from_prompt(native_prompt) == {
+ "type": "object",
+ "properties": {},
+ "additionalProperties": True,
+ }
def test_mcp_prompt_and_native_schema_omit_blocked_tool(