Skip to content

Agent Zero AI framework

Restore focused vision queries

GreifMax54534ba6368b
diff --git a/prompts/agent.system.tools_vision.md b/prompts/agent.system.tools_vision.md
index ee50376ad..99db1d49e 100644
--- a/prompts/agent.system.tools_vision.md
+++ b/prompts/agent.system.tools_vision.md
@@ -2,9 +2,10 @@
### vision_load
load images into the model for visual reasoning
-args: `paths` list of absolute image paths or tool-returned ephemeral image refs
+args: `paths` list of absolute image paths or tool-returned ephemeral image refs, optional `query` for focused inspection
rules:
- load all relevant images in one call when comparing screenshots or pages
+- add `query` when the visual task is narrower than the user's request or derived during the work
- use when the task depends on screenshots, diagrams, scanned documents, charts, or photos
- only bitmaps are supported; convert other formats first if needed
- the tool result includes loaded/skipped image totals and the corresponding path lists
@@ -12,12 +13,13 @@ example:
```json
{
"thoughts": [
- "I need to inspect the screenshot before answering."
+ "I need to compare the screenshots."
],
- "headline": "Loading screenshot for visual analysis",
+ "headline": "Comparing screenshots",
"tool_name": "vision_load",
"tool_args": {
- "paths": ["/path/to/screenshot.png"]
+ "paths": ["/path/to/before.png", "/path/to/after.png"],
+ "query": "Compare the error-banner alignment."
}
}
```
diff --git a/prompts/fw.vision_load.md b/prompts/fw.vision_load.md
index 1c7320e2f..dd9ecab4e 100644
--- a/prompts/fw.vision_load.md
+++ b/prompts/fw.vision_load.md
@@ -1,4 +1,9 @@
-Analyze the attached image(s) for the current request. Return only relevant visible details, text, and layout.
+Analyze the attached image(s) for the current request, focusing on the visual query when provided. Return only relevant visible details, text, and layout.
Current request:
{{request}}
+
+{{if query}}
+Visual query:
+{{query}}
+{{endif}}
diff --git a/tests/test_tool_policy.py b/tests/test_tool_policy.py
index f64526e5b..6dbf9da0e 100644
--- a/tests/test_tool_policy.py
+++ b/tests/test_tool_policy.py
@@ -8,7 +8,7 @@ from types import SimpleNamespace
import pytest
from extensions.python.system_prompt import _11_tools_prompt, _13_skills_prompt
-from helpers import mcp_handler, responses_tools, tool_policy
+from helpers import files, mcp_handler, responses_tools, tool_policy
from helpers.errors import RepairableException
from plugins._tool_access.extensions.python.tool_execute_before._10_enforce_tool_policy import (
EnforceToolPolicy,
@@ -596,7 +596,29 @@ def test_vision_prompt_stays_route_agnostic_and_batches_paths() -> None:
assert "load all relevant images in one call" in source
assert "Input schema for tool_args" not in source
assert "Vision Model" not in source
- assert "query" not in source
+ assert "sidecar" not in source.lower()
+ assert "optional `query`" in source
+
+
+def test_vision_framework_prompt_renders_optional_query() -> None:
+ prompt_dir = str(Path(__file__).resolve().parents[1] / "prompts")
+
+ without_query = files.read_prompt_file(
+ "fw.vision_load.md",
+ _directories=[prompt_dir],
+ request="Review these screenshots.",
+ query="",
+ )
+ with_query = files.read_prompt_file(
+ "fw.vision_load.md",
+ _directories=[prompt_dir],
+ request="Review these screenshots.",
+ query="Compare the error banners.",
+ )
+
+ assert "Visual query:" not in without_query
+ assert "Current request:\nReview these screenshots." in with_query
+ assert "Visual query:\nCompare the error banners." in with_query
def test_mcp_prompt_and_native_schema_omit_blocked_tool(
diff --git a/tests/test_vision_load_image_refs.py b/tests/test_vision_load_image_refs.py
index 09ff771bb..e38ed316d 100644
--- a/tests/test_vision_load_image_refs.py
+++ b/tests/test_vision_load_image_refs.py
@@ -126,11 +126,15 @@ async def test_vision_load_materializes_local_image_to_chat_artifact(monkeypatch
invalid = await tool.execute(paths=None)
assert invalid.message == "vision_load error: `paths` must be a string or an array."
- response = await tool.execute(paths=str(image_path))
+ response = await tool.execute(
+ paths=str(image_path),
+ query="Read the footer text.",
+ )
image_path.unlink()
await tool.after_execution(response)
raw_message = messages[0][1]["content"]
+ assert [item["type"] for item in raw_message["raw_content"]] == ["image_url"]
stored_ref = raw_message["raw_content"][0]["image_url"]["url"]
assert stored_ref.startswith("/a0/usr/chats/ctx-vision/images/vision-load/sample-image-")
stored_path = tmp_path / stored_ref.removeprefix("/a0/")
@@ -206,9 +210,11 @@ async def test_vision_model_sends_multiple_images_once_and_keeps_history_text_on
context=SimpleNamespace(id=""),
agent_name="Agent 0",
last_user_message=SimpleNamespace(
- output_text=lambda: "Compare the login errors."
+ output_text=lambda: "Review these UI screenshots."
+ ),
+ read_prompt=lambda _name, request, query: (
+ f"Current request: {request}\n\nVisual query: {query}"
),
- read_prompt=lambda _name, request: f"Analyze: {request}",
hist_add_tool_result=lambda *args, **kwargs: tool_results.append((args, kwargs)),
hist_add_message=lambda *args, **kwargs: raw_messages.append((args, kwargs)),
)
@@ -222,7 +228,10 @@ async def test_vision_model_sends_multiple_images_once_and_keeps_history_text_on
)
tool.log = SimpleNamespace(id="vision-log", update=lambda **kwargs: None)
- response = await tool.execute(paths=[str(path) for path in image_paths])
+ response = await tool.execute(
+ paths=[str(path) for path in image_paths],
+ query="Compare the login error banners.",
+ )
response.additional = {"_responses_output_item": {"output": response.message}}
await tool.after_execution(response)
@@ -230,7 +239,10 @@ async def test_vision_model_sends_multiple_images_once_and_keeps_history_text_on
content = calls[0]["messages"][0].content
assert content[0] == {
"type": "text",
- "text": "Analyze: Compare the login errors.",
+ "text": (
+ "Current request: Review these UI screenshots.\n\n"
+ "Visual query: Compare the login error banners."
+ ),
}
assert [item["type"] for item in content].count("image_url") == 2
assert "max_tokens" not in calls[0]
@@ -265,7 +277,7 @@ async def test_vision_model_empty_response_is_reported_as_error(monkeypatch):
agent = SimpleNamespace(
context=SimpleNamespace(id="", get_data=lambda _key: ""),
last_user_message=SimpleNamespace(output_text=lambda: "Inspect the image."),
- read_prompt=lambda _name, request: request,
+ read_prompt=lambda _name, request, query: f"{request}\n{query}",
)
tool = vision_load_module.VisionLoad(
agent=agent,
@@ -398,7 +410,7 @@ async def test_independent_vision_model_calls_can_run_concurrently(monkeypatch,
last_user_message=SimpleNamespace(
output_text=lambda: f"inspection {index}"
),
- read_prompt=lambda _name, request: request,
+ read_prompt=lambda _name, request, query: f"{request}\n{query}",
)
return vision_load_module.VisionLoad(
agent=agent,
diff --git a/tools/vision_load.py b/tools/vision_load.py
index 61b22a6fb..423f61ea1 100644
--- a/tools/vision_load.py
+++ b/tools/vision_load.py
@@ -23,7 +23,12 @@ TOKENS_ESTIMATE = 1500
class VisionLoad(Tool):
- async def execute(self, paths: list[str] | str = [], **kwargs) -> Response:
+ async def execute(
+ self,
+ paths: list[str] | str = [],
+ query: str = "",
+ **kwargs,
+ ) -> Response:
self.images_dict = {}
self.loaded_paths: list[str] = []
@@ -87,7 +92,10 @@ class VisionLoad(Tool):
message = self._summary() if self.images_dict or self.skipped_paths else "No images processed"
if self.vision_config and self.images_dict:
try:
- capsule = await self._call_vision_model(list(self.images_dict.values()))
+ capsule = await self._call_vision_model(
+ list(self.images_dict.values()),
+ query,
+ )
message = (
f"Analyzed {len(self.images_dict)} image(s)"
f"; {len(self.skipped_paths)} skipped.\n\n{capsule.strip()}"
@@ -110,14 +118,18 @@ class VisionLoad(Tool):
)
return str(parent_id or getattr(context, "id", "") or "").strip()
- async def _call_vision_model(self, image_paths: list[str]) -> str:
+ async def _call_vision_model(self, image_paths: list[str], query: str) -> str:
user_message = getattr(self.agent, "last_user_message", None)
output_text = getattr(user_message, "output_text", None)
request = str(output_text() if callable(output_text) else "").strip()
content = [
{
"type": "text",
- "text": self.agent.read_prompt("fw.vision_load.md", request=request),
+ "text": self.agent.read_prompt(
+ "fw.vision_load.md",
+ request=request,
+ query=str(query or "").strip(),
+ ),
}
]
content.extend(
diff --git a/tools/vision_load.py.dox.md b/tools/vision_load.py.dox.md
index 2960e6833..14bf0e6ef 100644
--- a/tools/vision_load.py.dox.md
+++ b/tools/vision_load.py.dox.md
@@ -12,7 +12,7 @@
- `vision_load.py.dox.md` owns durable notes about responsibilities, contracts, side effects, and verification for that implementation.
- Classes:
- `VisionLoad` (`Tool`)
- - `async execute(self, paths, **kwargs) -> Response`
+ - `async execute(self, paths, query="", **kwargs) -> Response`
- `async after_execution(self, response: Response, **kwargs)`
- Notable constants/configuration names: `TOKENS_ESTIMATE`.
@@ -21,7 +21,8 @@
- Tool modules must define `helpers.tool.Tool` subclasses and return `helpers.tool.Response` from `execute(...)`.
- One call may contain multiple paths; a bare string is treated as one path. The Vision Model route sends every selected path in one request and returns one textual capsule.
- Model configuration exposes a Vision Model only when the effective preset selects that route; otherwise this tool follows Main's native vision path.
-- The public tool contract is route-agnostic. A Vision Model receives the current user request through `fw.vision_load.md`; direct parallel workers inherit that request from their parent.
+- The public tool contract is route-agnostic and accepts an optional focused `query`. A Vision Model receives both that query and the current user request through `fw.vision_load.md`; direct parallel workers inherit the request from their parent.
+- Native Main vision already retains the query in its authored tool-call transcript, so native raw history remains image-only and does not repeat model-authored instructions as user content.
- Delegation completes during `execute(...)` so native Responses function output contains the real capsule before `after_execution(...)` persists it.
- Delegated history contains the text capsule only. Native history contains the tool result followed by one raw message holding all loaded image blocks.
- In a direct parallel worker, native image content is queued for the parent and promoted immediately after the outer `parallel` result; the disposable worker never owns the only copy of model-visible pixels.