Skip to content

Agent Zero AI framework

Simplify Vision Model routing and execution

Alessandro2c09a4ed7241
diff --git a/extensions/python/system_prompt/_11_tools_prompt.py b/extensions/python/system_prompt/_11_tools_prompt.py
index d9086e503..bb6de55e6 100644
--- a/extensions/python/system_prompt/_11_tools_prompt.py
+++ b/extensions/python/system_prompt/_11_tools_prompt.py
@@ -53,14 +53,11 @@ async def build_prompt(agent: Agent) -> str:
# vision support
from plugins._model_config.helpers.model_config import (
get_chat_model_config,
- use_vision_sidecar,
+ get_vision_model_config,
)
chat_cfg = get_chat_model_config(agent)
- sidecar = use_vision_sidecar(agent)
- if sidecar or chat_cfg.get("vision", False):
- prompt += "\n\n" + agent.read_prompt(
- "agent.system.tools_vision.md", sidecar=sidecar
- )
+ if get_vision_model_config(agent) or chat_cfg.get("vision", False):
+ prompt += "\n\n" + agent.read_prompt("agent.system.tools_vision.md")
return prompt
diff --git a/helpers/responses_tools.py b/helpers/responses_tools.py
index 28b86ed28..ffe570773 100644
--- a/helpers/responses_tools.py
+++ b/helpers/responses_tools.py
@@ -116,13 +116,15 @@ def _vision_tool_prompt(agent: Any) -> str:
try:
from plugins._model_config.helpers.model_config import (
get_chat_model_config,
- use_vision_sidecar,
+ get_vision_model_config,
)
- sidecar = use_vision_sidecar(agent)
- if not (sidecar or get_chat_model_config(agent).get("vision", False)):
+ if not (
+ get_vision_model_config(agent)
+ or get_chat_model_config(agent).get("vision", False)
+ ):
return ""
- return agent.read_prompt("agent.system.tools_vision.md", sidecar=sidecar)
+ return agent.read_prompt("agent.system.tools_vision.md")
except Exception:
return ""
diff --git a/plugins/_model_config/AGENTS.md b/plugins/_model_config/AGENTS.md
index 28ec60ee9..f6350ea11 100644
--- a/plugins/_model_config/AGENTS.md
+++ b/plugins/_model_config/AGENTS.md
@@ -23,7 +23,7 @@
- Coordinate OAuth-backed providers with `_oauth` instead of hardcoding provider-specific auth here.
- `model_config_get` exposes `model_configured` as a derived chat-model readiness flag from provider, model name, and API-key availability.
- Non-default presets may inherit omitted main, utility, or embedding slots and durable tuning from `Default`, but must replace or clear per-slot `kwargs` so provider-specific extra params never leak across model providers.
-- The optional `vision` slot is strictly per preset and never inherited from `Default`; an empty slot disables sidecar vision for that preset.
+- The optional `vision` slot is strictly per preset and never inherited from `Default`; an empty slot disables the separate Vision Model for that preset.
- Main native vision wins by default. A configured Vision Model handles `vision_load` when Main lacks vision, or when that preset explicitly enables `override_main`.
- Keep the optional Vision provider/model selector inside the Main Model card and flush with Main's field alignment, without a nested left inset. Show it only while Main vision is disabled or `override_main` is enabled; do not render a standalone Vision Model card.
- Show `Use separate Vision Model` immediately below `Supports Vision` while Main vision is enabled, not inside Advanced Settings; describe the disabled state as using Main's native vision.
diff --git a/plugins/_model_config/helpers/model_config.py b/plugins/_model_config/helpers/model_config.py
index 3f6c36f93..003d7be7d 100644
--- a/plugins/_model_config/helpers/model_config.py
+++ b/plugins/_model_config/helpers/model_config.py
@@ -759,20 +759,17 @@ def get_chat_model_config(agent=None) -> dict:
def get_vision_model_config(agent=None) -> dict:
- """Get the optional, strictly per-preset Vision Model config."""
- return get_effective_config(agent).get("vision_model", {})
-
-
-def use_vision_sidecar(agent=None) -> bool:
- """Return whether vision_load should delegate to the preset's Vision Model."""
- vision_cfg = get_vision_model_config(agent)
- if not (
- str(vision_cfg.get("provider") or "").strip()
- and str(vision_cfg.get("name") or "").strip()
- ):
- return False
- chat_cfg = get_chat_model_config(agent)
- return not bool(chat_cfg.get("vision")) or bool(vision_cfg.get("override_main"))
+ """Get the active Vision Model config after applying Main-first routing."""
+ cfg = get_effective_config(agent)
+ vision_cfg = cfg.get("vision_model", {})
+ if not _slot_has_identity(vision_cfg):
+ return {}
+ chat_cfg = cfg.get("chat_model", {})
+ return (
+ vision_cfg
+ if not chat_cfg.get("vision") or vision_cfg.get("override_main")
+ else {}
+ )
def get_utility_model_config(agent=None) -> dict:
@@ -928,8 +925,9 @@ def get_missing_api_key_providers(agent=None) -> list[dict]:
("Utility Model", cfg.get("utility_model", {})),
("Embedding Model", get_embedding_model_config(agent)),
]
- if use_vision_sidecar(agent):
- checks.insert(1, ("Vision Model", cfg.get("vision_model", {})))
+ vision_cfg = get_vision_model_config(agent)
+ if vision_cfg:
+ checks.insert(1, ("Vision Model", vision_cfg))
for label, model_cfg in checks:
provider = model_cfg.get("provider", "")
diff --git a/prompts/AGENTS.md b/prompts/AGENTS.md
index 132fa30a2..07c4d5282 100644
--- a/prompts/AGENTS.md
+++ b/prompts/AGENTS.md
@@ -25,7 +25,7 @@
- Read the rendering path before changing placeholders or filenames.
- Prefer small prompt additions over broad rewrites when fixing a specific behavior.
- Keep document/OCR routing explicit: image files, screenshots, scans, charts, photos, and diagrams should prefer vision tools when available, while `document_query` is for documents, large text-heavy files, and fallback OCR.
-- Keep the single `vision_load` prompt accurate for both native and sidecar routing: related images belong in one call, while a sidecar result is text-only unless Main explicitly requests its native raw route.
+- Keep the single `vision_load` prompt accurate for both native and separate Vision Model routing: related images belong in one call, and a Vision Model result is text-only.
- Update tests or snapshots when prompt budget, required sections, or generated system content changes.
## Verification
diff --git a/prompts/agent.system.tools_vision.md b/prompts/agent.system.tools_vision.md
index 9cbeb91e9..b766699cf 100644
--- a/prompts/agent.system.tools_vision.md
+++ b/prompts/agent.system.tools_vision.md
@@ -1,10 +1,8 @@
## multimodal vision tools
### vision_load
-{{if sidecar}}analyze images with the preset's separate Vision Model and return a text result{{endif}}
-{{if not sidecar}}load images into Main for visual reasoning{{endif}}
-args: `paths` list of absolute image paths or ephemeral image refs{{if sidecar}}, `query` optional focused instruction, `raw` optional boolean{{endif}}
-{{if sidecar}}
+load or analyze images for visual reasoning
+args: `paths` list of absolute image paths or ephemeral image refs, `query` optional focused instruction
Input schema for tool_args:
```json
{
@@ -12,32 +10,20 @@ Input schema for tool_args:
"properties": {
"paths": {
"type": "array",
- "items": {"type": "string"},
- "description": "Absolute image paths or ephemeral image refs."
+ "items": {"type": "string"}
},
- "query": {
- "type": "string",
- "description": "What the Vision Model should inspect, compare, locate, or read."
- },
- "raw": {
- "type": "boolean",
- "description": "Use Main's native vision instead of the separate Vision Model, when Main supports vision."
- }
+ "query": {"type": "string"}
},
"required": ["paths"],
"additionalProperties": false
}
```
-{{endif}}
rules:
- put all images needed for one comparison or visual task in the same `paths` array
- use when the task depends on screenshots, diagrams, scanned documents, charts, or photos
-{{if sidecar}}
-- when the separate Vision Model is active, use a focused `query`; the result is a text capsule and Main does not receive raw images
-- use `raw=true` only when Main supports vision and must inspect the pixels itself
-{{endif}}
+- use a focused `query` when asking the configured Vision Model to inspect, compare, locate, or read something
- only bitmaps are supported; convert other formats first if needed
-- the tool result includes loaded/skipped image totals and the corresponding path lists
+- the tool result reports loaded and skipped image counts
example:
```json
{
@@ -47,9 +33,8 @@ example:
"headline": "Comparing screenshots",
"tool_name": "vision_load",
"tool_args": {
- "paths": ["/path/to/before.png", "/path/to/after.png"]{{if sidecar}},
+ "paths": ["/path/to/before.png", "/path/to/after.png"],
"query": "Compare the error banners and describe what changed."
- {{endif}}
}
}
```
diff --git a/tests/test_browser_agent_regressions.py b/tests/test_browser_agent_regressions.py
index a924ec00b..9198aff38 100644
--- a/tests/test_browser_agent_regressions.py
+++ b/tests/test_browser_agent_regressions.py
@@ -63,6 +63,15 @@ class _TestTool:
self.message = message
self.loop_data = loop_data
+ async def after_execution(self, response, **kwargs):
+ self.agent.hist_add_tool_result(
+ self.name,
+ response.message.strip(),
+ id=self.log.id,
+ **(response.additional or {}),
+ )
+ self.log.update(content=response.message.strip())
+
class _TestWsHandler:
def __init__(self, *args, **kwargs):
@@ -102,7 +111,6 @@ _model_config_stub.get_presets = lambda: []
_model_config_stub.get_preset_by_name = lambda name: None
_model_config_stub.get_chat_model_config = lambda agent=None: {}
_model_config_stub.get_vision_model_config = lambda agent=None: {}
-_model_config_stub.use_vision_sidecar = lambda agent=None: False
_model_config_stub.build_vision_model = lambda agent=None: None
sys.modules.setdefault("plugins._model_config.helpers.model_config", _model_config_stub)
@@ -4307,7 +4315,6 @@ async def test_vision_load_materializes_ephemeral_browser_refs(monkeypatch, tmp_
monkeypatch.setattr(vision_load_module.chat_media.files, "normalize_a0_path", fake_normalize_a0_path)
monkeypatch.setattr(vision_load_module, "get_chat_model_config", lambda _agent: {"vision": True, "max_embeds": 10})
monkeypatch.setattr(vision_load_module, "get_vision_model_config", lambda _agent: {})
- monkeypatch.setattr(vision_load_module, "use_vision_sidecar", lambda _agent: False)
tool_results = []
messages = []
@@ -4344,7 +4351,7 @@ async def test_vision_load_materializes_ephemeral_browser_refs(monkeypatch, tmp_
assert stored_ref.startswith("/a0/usr/chats/ctx-vision/screenshots/browser/browser-shot-")
stored_path = tmp_path / stored_ref.removeprefix("/a0/")
assert stored_path.read_bytes() == __import__("base64").b64decode(SMALL_JPEG_10X10)
- assert updates[-1]["result"] == "1 images loaded, 0 skipped"
+ assert updates[-1]["content"] == response.message
@pytest.mark.anyio
diff --git a/tests/test_model_config_api_keys.py b/tests/test_model_config_api_keys.py
index f2e7a0cbc..ee9c1eda5 100644
--- a/tests/test_model_config_api_keys.py
+++ b/tests/test_model_config_api_keys.py
@@ -93,7 +93,7 @@ def test_missing_api_key_checks_only_the_active_vision_model(monkeypatch):
from plugins._model_config.helpers import model_config
config = {
- "chat_model": {"provider": "ollama", "name": "text-main"},
+ "chat_model": {"provider": "ollama", "name": "text-main", "vision": True},
"vision_model": {"provider": "openai", "name": "vision"},
"utility_model": {"provider": "ollama", "name": "utility"},
"embedding_model": {
@@ -101,19 +101,17 @@ def test_missing_api_key_checks_only_the_active_vision_model(monkeypatch):
"name": "sentence-transformers/all-MiniLM-L6-v2",
},
}
- active = False
monkeypatch.setattr(model_config, "get_effective_config", lambda _agent=None: config)
monkeypatch.setattr(
model_config,
"get_embedding_model_config",
lambda _agent=None: config["embedding_model"],
)
- monkeypatch.setattr(model_config, "use_vision_sidecar", lambda _agent=None: active)
monkeypatch.setattr(model_config, "has_provider_api_key", lambda *args: False)
assert model_config.get_missing_api_key_providers() == []
- active = True
+ config["chat_model"]["vision"] = False
assert model_config.get_missing_api_key_providers() == [
{"model_type": "Vision Model", "provider": "openai"}
]
diff --git a/tests/test_tool_policy.py b/tests/test_tool_policy.py
index f64231136..96f0c594d 100644
--- a/tests/test_tool_policy.py
+++ b/tests/test_tool_policy.py
@@ -8,7 +8,7 @@ from types import SimpleNamespace
import pytest
from extensions.python.system_prompt import _11_tools_prompt, _13_skills_prompt
-from helpers import files, mcp_handler, responses_tools, tool_policy
+from helpers import mcp_handler, responses_tools, tool_policy
from helpers.errors import RepairableException
from plugins._tool_access.extensions.python.tool_execute_before._10_enforce_tool_policy import (
EnforceToolPolicy,
@@ -528,6 +528,10 @@ async def test_vision_tool_follows_chat_config_not_profile_policy(
"plugins._model_config.helpers.model_config.get_chat_model_config",
lambda agent: {"vision": True},
)
+ monkeypatch.setattr(
+ "plugins._model_config.helpers.model_config.get_vision_model_config",
+ lambda agent: {},
+ )
monkeypatch.setattr(
tool_policy,
"get_policy",
@@ -547,7 +551,7 @@ async def test_vision_tool_follows_chat_config_not_profile_policy(
@pytest.mark.asyncio
-async def test_vision_sidecar_uses_canonical_vision_prompt(
+async def test_active_vision_model_uses_canonical_vision_prompt(
monkeypatch, tmp_path: Path
) -> None:
_write_prompt(tmp_path, "agent.system.tools.md", "TOOLS\n{{tools}}")
@@ -559,11 +563,11 @@ async def test_vision_sidecar_uses_canonical_vision_prompt(
monkeypatch.setattr(tool_policy.subagents, "get_paths", _prompt_paths(tmp_path))
monkeypatch.setattr(
"plugins._model_config.helpers.model_config.get_chat_model_config",
- lambda agent: {"vision": True},
+ lambda agent: {"vision": False},
)
monkeypatch.setattr(
- "plugins._model_config.helpers.model_config.use_vision_sidecar",
- lambda agent: True,
+ "plugins._model_config.helpers.model_config.get_vision_model_config",
+ lambda agent: {"provider": "test", "name": "vision"},
)
monkeypatch.setattr(responses_tools, "_mcp_tools", lambda agent: [])
agent = _Agent(tmp_path)
@@ -582,24 +586,16 @@ def test_vision_prompt_declares_multi_image_native_schema() -> None:
/ "prompts"
/ "agent.system.tools_vision.md"
).read_text(encoding="utf-8")
- sidecar_prompt = files.evaluate_text_conditions(source, sidecar=True)
- native_prompt = files.evaluate_text_conditions(source, sidecar=False)
- schema = responses_tools._schema_from_prompt(sidecar_prompt)
+ schema = responses_tools._schema_from_prompt(source)
assert schema["required"] == ["paths"]
assert schema["properties"]["paths"] == {
"type": "array",
"items": {"type": "string"},
- "description": "Absolute image paths or ephemeral image refs.",
- }
- assert {"query", "raw"} <= schema["properties"].keys()
- assert "separate Vision Model" in sidecar_prompt
- assert "separate Vision Model" not in native_prompt
- assert responses_tools._schema_from_prompt(native_prompt) == {
- "type": "object",
- "properties": {},
- "additionalProperties": True,
}
+ assert "query" in schema["properties"]
+ assert "raw" not in schema["properties"]
+ assert "Vision Model" in source
def test_mcp_prompt_and_native_schema_omit_blocked_tool(
diff --git a/tests/test_vision_load_image_refs.py b/tests/test_vision_load_image_refs.py
index 61ba5bbf3..6213335b1 100644
--- a/tests/test_vision_load_image_refs.py
+++ b/tests/test_vision_load_image_refs.py
@@ -41,6 +41,15 @@ class _TestTool:
self.message = message
self.loop_data = loop_data
+ async def after_execution(self, response, **kwargs):
+ self.agent.hist_add_tool_result(
+ self.name,
+ response.message.strip(),
+ id=self.log.id,
+ **(response.additional or {}),
+ )
+ self.log.update(content=response.message.strip())
+
def _install_tool_stub(monkeypatch):
tool_stub = types.ModuleType("helpers.tool")
@@ -82,7 +91,6 @@ async def test_vision_load_materializes_local_image_to_chat_artifact(monkeypatch
monkeypatch.setattr(vision_load_module.chat_media.files, "normalize_a0_path", fake_normalize_a0_path)
monkeypatch.setattr(vision_load_module, "get_chat_model_config", lambda _agent: {"vision": True, "max_embeds": 10})
monkeypatch.setattr(vision_load_module, "get_vision_model_config", lambda _agent: {})
- monkeypatch.setattr(vision_load_module, "use_vision_sidecar", lambda _agent: False)
async def direct_call(func, *args, **kwargs):
return func(*args, **kwargs)
@@ -115,6 +123,9 @@ async def test_vision_load_materializes_local_image_to_chat_artifact(monkeypatch
)
tool.log = SimpleNamespace(id="vision-log", update=lambda **kwargs: updates.append(kwargs))
+ invalid = await tool.execute(paths=None)
+ assert invalid.message == "vision_load error: `paths` must be an array."
+
response = await tool.execute(paths=[str(image_path)])
image_path.unlink()
await tool.after_execution(response)
@@ -124,10 +135,10 @@ async def test_vision_load_materializes_local_image_to_chat_artifact(monkeypatch
assert stored_ref.startswith("/a0/usr/chats/ctx-vision/images/vision-load/sample-image-")
stored_path = tmp_path / stored_ref.removeprefix("/a0/")
assert stored_path.read_bytes() == b"png-data"
- assert updates[-1]["result"] == "1 images loaded, 0 skipped"
+ assert updates[-1]["content"] == response.message
-def test_vision_sidecar_route_matrix_prefers_main_native_vision(monkeypatch):
+def test_active_vision_model_route_prefers_main_native_vision(monkeypatch):
from plugins._model_config.helpers import model_config
cases = [
@@ -149,11 +160,11 @@ def test_vision_sidecar_route_matrix_prefers_main_native_vision(monkeypatch):
"vision_model": vision,
},
)
- assert model_config.use_vision_sidecar() is expected
+ assert bool(model_config.get_vision_model_config()) is expected
@pytest.mark.anyio
-async def test_vision_sidecar_sends_multiple_images_once_and_keeps_history_text_only(
+async def test_vision_model_sends_multiple_images_once_and_keeps_history_text_only(
monkeypatch,
tmp_path,
):
@@ -182,7 +193,6 @@ async def test_vision_sidecar_sends_multiple_images_once_and_keeps_history_text_
"get_vision_model_config",
lambda _agent: {"provider": "test", "name": "vision", "max_embeds": 5},
)
- monkeypatch.setattr(vision_load_module, "use_vision_sidecar", lambda _agent: True)
image_paths = [tmp_path / "before.png", tmp_path / "after.png"]
for path in image_paths:
@@ -214,7 +224,7 @@ async def test_vision_sidecar_sends_multiple_images_once_and_keeps_history_text_
await tool.after_execution(response)
assert len(calls) == 1
- content = calls[0]["messages"][1].content
+ content = calls[0]["messages"][0].content
assert content[0] == {"type": "text", "text": "Compare the login errors."}
assert [item["type"] for item in content].count("image_url") == 2
assert "max_tokens" not in calls[0]
@@ -240,29 +250,12 @@ async def test_parallel_worker_consumes_parent_ephemeral_image(monkeypatch, tmp_
monkeypatch.setattr(vision_load_module.chat_media.files, "get_abs_path", fake_get_abs_path)
monkeypatch.setattr(vision_load_module.chat_media.files, "normalize_a0_path", fake_normalize_a0_path)
parent_id = "parent-vision"
- parent_agent = SimpleNamespace(
- context=SimpleNamespace(id=parent_id),
- agent_name="Parent Agent",
- )
- parent_context = SimpleNamespace(agent0=parent_agent)
- agent_stub = types.ModuleType("agent")
- agent_stub.AgentContext = SimpleNamespace(
- get=lambda context_id: parent_context if context_id == parent_id else None
- )
- monkeypatch.setitem(sys.modules, "agent", agent_stub)
- config_owners = []
-
- def get_chat_config(owner):
- config_owners.append(owner)
- return {"vision": True, "max_embeds": 10}
-
monkeypatch.setattr(
vision_load_module,
"get_chat_model_config",
- get_chat_config,
+ lambda _agent: {"vision": True, "max_embeds": 10},
)
monkeypatch.setattr(vision_load_module, "get_vision_model_config", lambda _agent: {})
- monkeypatch.setattr(vision_load_module, "use_vision_sidecar", lambda _agent: False)
ref = vision_load_module.ephemeral_images.put_image_bytes(
context_id=parent_id,
@@ -288,8 +281,6 @@ async def test_parallel_worker_consumes_parent_ephemeral_image(monkeypatch, tmp_
await tool.execute(paths=[ref])
- assert tool._config_owner is parent_agent
- assert config_owners and all(owner is parent_agent for owner in config_owners)
assert tool._context_id() == parent_id
assert tool.loaded_paths == ["shot.png"]
assert vision_load_module.ephemeral_images.get_image(ref, context_id=parent_id) is None
@@ -298,7 +289,7 @@ async def test_parallel_worker_consumes_parent_ephemeral_image(monkeypatch, tmp_
@pytest.mark.anyio
-async def test_independent_vision_sidecar_calls_can_run_concurrently(monkeypatch, tmp_path):
+async def test_independent_vision_model_calls_can_run_concurrently(monkeypatch, tmp_path):
_install_tool_stub(monkeypatch)
import tools.vision_load as vision_load_module
@@ -327,7 +318,6 @@ async def test_independent_vision_sidecar_calls_can_run_concurrently(monkeypatch
"get_vision_model_config",
lambda _agent: {"provider": "test", "name": "vision", "max_embeds": 10},
)
- monkeypatch.setattr(vision_load_module, "use_vision_sidecar", lambda _agent: True)
image_paths = [tmp_path / "one.png", tmp_path / "two.png"]
for path in image_paths:
diff --git a/tools/vision_load.py b/tools/vision_load.py
index 6cba5d2c0..b14b6fa5d 100644
--- a/tools/vision_load.py
+++ b/tools/vision_load.py
@@ -1,24 +1,18 @@
-import json
from mimetypes import guess_type
-from langchain_core.messages import HumanMessage, SystemMessage
+from langchain_core.messages import HumanMessage
from helpers import chat_media, ephemeral_images, files, history, images, runtime
-from helpers.parallel_tools import PARALLEL_WORKER_PARENT_CONTEXT_KEY, coerce_bool
-from helpers.print_style import PrintStyle
+from helpers.parallel_tools import PARALLEL_WORKER_PARENT_CONTEXT_KEY
from helpers.tool import Response, Tool
from plugins._model_config.helpers.model_config import (
build_vision_model,
get_chat_model_config,
get_vision_model_config,
- use_vision_sidecar,
)
+# image token estimation for context window
TOKENS_ESTIMATE = 1500
-VISION_SYSTEM_PROMPT = (
- "You are a precise vision analyst. Answer only what was asked about the images. "
- "Be concise and factual. Preserve exact visible text when asked to read it."
-)
DEFAULT_VISION_QUERY = (
"Describe the images precisely, including the key objects, visible text, and layout."
)
@@ -26,172 +20,99 @@ DEFAULT_VISION_QUERY = (
class VisionLoad(Tool):
async def execute(
- self,
- paths: list[str] | str | None = None,
- query: str = "",
- raw: bool = False,
- **kwargs,
+ self, paths: list[str] = [], query: str = "", **kwargs
) -> Response:
- self.images_dict: dict[str, str] = {}
+
+ self.images_dict = {}
self.loaded_paths: list[str] = []
self.skipped_paths: list[str] = []
- self._config_owner = self._config_agent()
- self._main_has_vision = bool(
- get_chat_model_config(self._config_owner).get("vision", False)
- )
- self._delegated = use_vision_sidecar(self._config_owner) and not (
- coerce_bool(raw, False) and self._main_has_vision
- )
- self._max_embeds = self._get_max_embeds()
-
- normalized = self._normalize_paths(paths)
- if isinstance(normalized, str):
- self._history_result = normalized
- return Response(message=normalized, break_loop=False)
+ self.vision_config = get_vision_model_config(self.agent)
+ if not isinstance(paths, list):
+ return Response(
+ message="vision_load error: `paths` must be an array.",
+ break_loop=False,
+ )
+ max_embeds = self._get_max_embeds()
requested = [
- (path.strip(), self._display_input_path(path.strip(), index + 1))
- for index, path in enumerate(normalized)
+ (str(path or "").strip(), self._display_input_path(str(path or "").strip(), idx + 1))
+ for idx, path in enumerate(paths)
]
- limited = requested if self._max_embeds <= 0 else requested[-self._max_embeds :]
- if self._max_embeds > 0 and len(requested) > self._max_embeds:
- self.skipped_paths = [display for _, display in requested[: -self._max_embeds]]
+ limited_paths = requested if max_embeds <= 0 else requested[-max_embeds:]
+ self.skipped_paths = (
+ [display for _, display in requested[:-max_embeds]]
+ if max_embeds > 0 and len(requested) > max_embeds
+ else []
+ )
- for index, (path, display_path) in enumerate(limited):
+ for idx, (path, display_path) in enumerate(limited_paths):
if not path:
continue
if ephemeral_images.is_ref(path):
- image = ephemeral_images.consume_image(path, context_id=self._context_id())
+ image = ephemeral_images.consume_image(
+ path,
+ context_id=self._context_id(),
+ )
if image is None:
continue
- display_path = image.display_name or display_path
+ display = image.display_name or display_path
stored_ref = self._store_ephemeral_image(image)
- elif self._is_data_image_url(path):
- stored_ref = self._store_data_url(
- path, preferred_name=f"vision-load-{index + 1}.png"
- )
- elif await runtime.call_development_function(files.exists, path):
- mime_type, _ = guess_type(path)
- if not mime_type or not mime_type.startswith("image/"):
- continue
- try:
- stored_ref = self._store_local_image(
- path, preferred_name=files.basename(path)
- )
- except (FileNotFoundError, OSError, ValueError):
- continue
- else:
+ if stored_ref:
+ self.images_dict[display] = stored_ref
+ self.loaded_paths.append(display)
+ continue
+ if self._is_data_image_url(path):
+ stored_ref = self._store_data_url(path, preferred_name=f"vision-load-{idx + 1}.png")
+ if stored_ref:
+ self.images_dict[display_path] = stored_ref
+ self.loaded_paths.append(display_path)
+ continue
+ if not await runtime.call_development_function(files.exists, str(path)):
continue
- if stored_ref:
- self.images_dict[display_path] = stored_ref
- self.loaded_paths.append(display_path)
-
- summary = self._summary()
- if self._delegated and self.images_dict:
+ if path not in self.images_dict:
+ mime_type, _ = guess_type(str(path))
+ if mime_type and mime_type.startswith("image/"):
+ try:
+ stored_ref = self._store_local_image(path, preferred_name=files.basename(path))
+ self.images_dict[display_path] = stored_ref
+ self.loaded_paths.append(display_path)
+ except (FileNotFoundError, OSError, ValueError):
+ continue
+
+ message = self._summary() if self.images_dict or self.skipped_paths else "No images processed"
+ if self.vision_config and self.images_dict:
try:
capsule = await self._call_vision_model(
- list(self.images_dict.values()),
- str(query or "").strip() or DEFAULT_VISION_QUERY,
+ list(self.images_dict.values()), query
)
message = (
f"Vision Model analyzed {len(self.images_dict)} image(s)"
f"; {len(self.skipped_paths)} skipped.\n\n{capsule.strip()}"
)
- self._history_result = message
- return Response(message=message, break_loop=False)
except Exception as exc:
message = f"Vision Model error: {str(exc)[:1000]}"
- self._history_result = f"{summary}\n\n{message}"
- return Response(message=message, break_loop=False)
-
- if self.images_dict and not self._main_has_vision:
- summary += (
- "\n\nImages were not injected because neither Main native vision nor "
- "a usable Vision Model is active."
- )
- self._history_result = (
- summary if self.images_dict or self.skipped_paths else "No images processed"
- )
- message = (
- "No images processed"
- if not self.images_dict and not self.skipped_paths
- else f"{len(self.images_dict)} images loaded, {len(self.skipped_paths)} skipped"
- )
return Response(message=message, break_loop=False)
- async def after_execution(self, response: Response, **kwargs):
- log_id = str(getattr(getattr(self, "log", None), "id", "") or "")
- self.agent.hist_add_tool_result(
- self.name,
- self._history_result,
- id=log_id,
- **(response.additional or {}),
- )
-
- if self.images_dict and self._main_has_vision and not self._delegated:
- content = [
- {"type": "image_url", "image_url": {"url": image_path}}
- for image_path in self.images_dict.values()
- ]
- self.agent.hist_add_message(
- False,
- content=history.RawMessage(
- raw_content=content,
- preview="<Image attachments loaded by path>",
- ),
- tokens=TOKENS_ESTIMATE * len(content),
- )
-
- PrintStyle(
- font_color="#1B4F72", background_color="white", padding=True, bold=True
- ).print(f"{self.agent.agent_name}: Response from tool '{self.name}'")
- PrintStyle(font_color="#85C1E9").print(response.message)
- if getattr(self, "log", None):
- self.log.update(result=response.message)
-
def _get_max_embeds(self) -> int:
- cfg = (
- get_vision_model_config(self._config_owner)
- if self._delegated
- else get_chat_model_config(self._config_owner)
- )
- try:
- return int(cfg.get("max_embeds", 10) or 0)
- except (TypeError, ValueError):
- return 10
+ cfg = self.vision_config or get_chat_model_config(self.agent)
+ return int(cfg.get("max_embeds", 10) or 0)
def _context_id(self) -> str:
- context = getattr(self._config_owner, "context", None)
- return str(getattr(context, "id", "") or "").strip()
-
- def _config_agent(self):
context = getattr(self.agent, "context", None)
get_data = getattr(context, "get_data", None)
parent_id = get_data(PARALLEL_WORKER_PARENT_CONTEXT_KEY) if get_data else ""
- if parent_id:
- from agent import AgentContext
-
- parent = AgentContext.get(str(parent_id))
- if parent:
- return parent.agent0
- return self.agent
+ return str(parent_id or getattr(context, "id", "") or "").strip()
async def _call_vision_model(self, image_paths: list[str], query: str) -> str:
- model = build_vision_model(self._config_owner)
- content = [{"type": "text", "text": query or DEFAULT_VISION_QUERY}]
+ content = [{"type": "text", "text": str(query or "").strip() or DEFAULT_VISION_QUERY}]
content.extend(
{"type": "image_url", "image_url": {"url": path}}
for path in image_paths
)
- response, _ = await model.unified_call(
- messages=[
- SystemMessage(content=VISION_SYSTEM_PROMPT),
- HumanMessage(content=content),
- ],
+ response, _ = await build_vision_model(self.agent).unified_call(
+ messages=[HumanMessage(content=content)],
)
- if not str(response or "").strip():
- raise RuntimeError("Vision Model returned an empty response.")
return str(response)
def _store_ephemeral_image(self, image: ephemeral_images.EphemeralImage) -> str:
@@ -199,11 +120,12 @@ class VisionLoad(Tool):
if not context_id:
return image.data_url
source = chat_media.infer_source(image.ref, image.display_name)
+ category = chat_media.category_for_source(source)
saved = chat_media.save_image_base64(
context_id=context_id,
data=image.data,
mime_type=image.mime,
- category=chat_media.category_for_source(source),
+ category=category,
source=source,
preferred_name=image.display_name,
)
@@ -214,10 +136,11 @@ class VisionLoad(Tool):
if not context_id:
return data_url
source = chat_media.infer_source(data_url, preferred_name)
+ category = chat_media.category_for_source(source)
saved = chat_media.save_image_data_url(
context_id=context_id,
data_url=data_url,
- category=chat_media.category_for_source(source),
+ category=category,
source=source,
preferred_name=preferred_name,
)
@@ -239,23 +162,9 @@ class VisionLoad(Tool):
skipped = "\n".join(self.skipped_paths) if self.skipped_paths else "none"
return (
f"Loaded images ({len(self.loaded_paths)}):\n{loaded}\n\n"
- f"Skipped images ({len(self.skipped_paths)}, max {self._max_embeds}):\n{skipped}"
+ f"Skipped images ({len(self.skipped_paths)}, max {self._get_max_embeds()}):\n{skipped}"
)
- @staticmethod
- def _normalize_paths(paths: list[str] | str | None) -> list[str] | str:
- if isinstance(paths, str):
- try:
- decoded = json.loads(paths)
- except json.JSONDecodeError:
- decoded = paths
- paths = decoded if isinstance(decoded, list) else [paths]
- if paths is None:
- return []
- if not isinstance(paths, (list, tuple)):
- return "vision_load error: `paths` must be an array of image paths."
- return [str(path or "").strip() for path in paths]
-
@staticmethod
def _is_data_image_url(value: str) -> bool:
normalized = str(value or "").strip().lower()
@@ -266,5 +175,22 @@ class VisionLoad(Tool):
if ephemeral_images.is_ref(value):
return ephemeral_images.display_ref(value)
if cls._is_data_image_url(value):
- return f"{value.split(',', 1)[0]},<ephemeral-image-{index}>"
+ prefix = value.split(",", 1)[0]
+ return f"{prefix},<ephemeral-image-{index}>"
return value
+
+ async def after_execution(self, response: Response, **kwargs):
+ await super().after_execution(response, **kwargs)
+ if self.images_dict and not self.vision_config:
+ content = [
+ {"type": "image_url", "image_url": {"url": image_path}}
+ for image_path in self.images_dict.values()
+ ]
+ self.agent.hist_add_message(
+ False,
+ content=history.RawMessage(
+ raw_content=content,
+ preview="<Image attachments loaded by path>",
+ ),
+ tokens=TOKENS_ESTIMATE * len(content),
+ )
diff --git a/tools/vision_load.py.dox.md b/tools/vision_load.py.dox.md
index 82445f699..2001fca05 100644
--- a/tools/vision_load.py.dox.md
+++ b/tools/vision_load.py.dox.md
@@ -12,29 +12,29 @@
- `vision_load.py.dox.md` owns durable notes about responsibilities, contracts, side effects, and verification for that implementation.
- Classes:
- `VisionLoad` (`Tool`)
- - `async execute(self, paths, query="", raw=False, **kwargs) -> Response`
+ - `async execute(self, paths, query="", **kwargs) -> Response`
- `async after_execution(self, response: Response, **kwargs)`
- Notable constants/configuration names: `TOKENS_ESTIMATE`.
## Runtime Contracts
- Tool modules must define `helpers.tool.Tool` subclasses and return `helpers.tool.Response` from `execute(...)`.
-- One call may contain multiple paths. The delegated route sends every selected path in one Vision Model request and returns one textual capsule.
-- Main native vision wins unless the effective preset selects the sidecar route; `raw=true` returns to Main native vision only when Main supports it.
+- One call may contain multiple paths. The Vision Model route sends every selected path in one request and returns one textual capsule.
+- Model configuration exposes a Vision Model only when the effective preset selects that route; otherwise this tool follows Main's native vision path.
- Delegation completes during `execute(...)` so native Responses function output contains the real capsule before `after_execution(...)` persists it.
- Delegated history contains the text capsule only. Native history contains the tool result followed by one raw message holding all loaded image blocks.
-- Direct parallel workers resolve ephemeral refs, model routing, and durable chat-media storage against their recorded parent context; independent vision jobs remain generic parallel jobs.
+- Direct parallel workers inherit the parent's model override generically. This tool uses their recorded parent context only to resolve ephemeral refs and durable chat media.
- `max_embeds` comes from the model that actually receives the images.
- Vision Model calls use the selected model's Advanced `kwargs`; this tool does not impose a separate timeout or output-token limit.
- Update this file whenever tool arguments, output shape, `break_loop` behavior, intervention handling, prompt instructions, or side effects change.
- `VisionLoad` is a `Tool`.
- `VisionLoad` defines `execute(...)`.
- Observed side-effect areas: filesystem writes, model calls, plugin state, settings/state persistence, secret handling.
-- Imported dependency areas include: `helpers`, `helpers.print_style`, `helpers.tool`, `mimetypes`.
+- Imported dependency areas include: `helpers`, `helpers.tool`, `langchain_core.messages`, `mimetypes`, and `_model_config`.
## Key Concepts
-- Important called helpers/classes observed in the source: `build_vision_model`, `use_vision_sidecar`, `self._get_max_embeds`, `Response`, `self._context_id`, `chat_media.save_image_base64`, `chat_media.save_image_data_url`, `chat_media.materialize_image_ref`, `ephemeral_images.consume_image`, `images.to_data_url`, `self.agent.hist_add_tool_result`, `history.RawMessage`, `model.unified_call`.
+- Important called helpers/classes observed in the source: `build_vision_model`, `get_vision_model_config`, `self._get_max_embeds`, `Response`, `self._context_id`, `chat_media.save_image_base64`, `chat_media.save_image_data_url`, `chat_media.materialize_image_ref`, `ephemeral_images.consume_image`, `images.to_data_url`, `history.RawMessage`, `super().after_execution`, `model.unified_call`.
- Keep request/response, tool, or helper semantics documented here at the same time as source changes.
## Work Guidance