Agent Zero AI framework
Preserve native vision output from parallel workers
Alessandro66cc11096c7e
diff --git a/helpers/parallel_tools.py b/helpers/parallel_tools.pyindex dbc79ef9f..ebb8e8315 100644--- a/helpers/parallel_tools.py+++ b/helpers/parallel_tools.py@@ -65,6 +65,7 @@ class ParallelJob:log_id: str = field(default_factory=lambda: str(uuid.uuid4()))log_item: "LogItem | None" = field(default=None, repr=False)deferred_task: DeferredTask | None = field(default=None, repr=False)+ parent_history: list[tuple[Any, int]] = field(default_factory=list, repr=False)def elapsed(self) -> float:end = self.completed_at or time.time()@@ -170,6 +171,26 @@ def is_parallel_worker(agent: "Agent | None") -> bool:return _parallel_worker_kind(agent) == "tool"+def queue_parallel_parent_history(+ agent: "Agent",+ *,+ content: Any,+ tokens: int = 0,+) -> bool:+ if not is_parallel_worker(agent):+ return False+ context = agent.context+ parent_context_id = str(+ context.get_data(PARALLEL_WORKER_PARENT_CONTEXT_KEY) or ""+ )+ job_id = str(context.get_data(PARALLEL_WORKER_JOB_KEY) or "")+ job = _get_job(parent_context_id, job_id)+ if not job or job.kind != "tool":+ return False+ job.parent_history.append((content, tokens))+ return True++def _jobs_for_context(context: "AgentContext") -> dict[str, ParallelJob]:jobs = context.get_data(PARALLEL_JOBS_KEY)if not isinstance(jobs, dict):@@ -242,7 +263,6 @@ async def await_parallel_jobs(raise ValueError("No `job_ids` were provided to await.")deadline = time.time() + timeout- known_job_ids = set(job_ids)wait_timed_out_job_ids: set[str] = set()while True:await refresh_parallel_jobs(agent)@@ -271,11 +291,7 @@ async def await_parallel_jobs(snapshots.append(snapshot)if collect:- for job_id in known_job_ids:- job = _jobs_for_context(agent.context).get(job_id)- if job and job.state in TERMINAL_STATES:- await cleanup_parallel_job(agent, job)- _jobs_for_context(agent.context).pop(job_id, None)+ await collect_parallel_jobs(agent, job_ids)return snapshots@@ -326,6 +342,25 @@ async def cleanup_parallel_job(agent: "Agent", job: ParallelJob) -> None:await _remove_context(job.worker_context_id)+async def collect_parallel_jobs(+ agent: "Agent",+ job_ids: list[str],+ *,+ promote_parent_history: bool = False,+) -> None:+ jobs = _jobs_for_context(agent.context)+ for job_id in dict.fromkeys(job_ids):+ job = jobs.get(job_id)+ if not job or job.state not in TERMINAL_STATES:+ continue+ if promote_parent_history:+ for content, tokens in job.parent_history:+ agent.hist_add_message(False, content=content, tokens=tokens)+ job.parent_history.clear()+ await cleanup_parallel_job(agent, job)+ jobs.pop(job_id, None)++async def build_parallel_jobs_extras(agent: "Agent") -> str:await refresh_parallel_jobs(agent)jobs = [diff --git a/helpers/parallel_tools.py.dox.md b/helpers/parallel_tools.py.dox.mdindex 9fb3f6b5f..924c81025 100644--- a/helpers/parallel_tools.py.dox.md+++ b/helpers/parallel_tools.py.dox.md@@ -38,6 +38,7 @@- Child tool logs mirror normal tool-call visible args; job ids remain available through wrapper results and prompt extras rather than visible process-step args.- Wrapped tool child logs use each tool's native `get_log_object()` output when available, preserving special log rendering (for example: `code_execution_tool` uses `code_exe`, `wait` uses `progress`, MCP tools use `mcp`, and regular tools use `tool`).- Direct parallel worker execution reuses the parent-visible child log item so tool `before_execution()` cannot create a second generic worker log or lose the native badge type.+- Direct tools may explicitly queue model-visible history for their parent. Terminal collection records the outer `parallel` result first, promotes queued messages in job order, and only then removes disposable worker state; background jobs retain queued history until they are collected.- Job IDs are stable handles for later await, collect, or cancel operations.- Prompt extras must stay bounded and expose only job IDs, tool names, status, and compact result/error summaries.@@ -47,6 +48,7 @@- `wait=True` starts jobs and awaits them before returning until all requested jobs finish or the wait timeout is reached; the timeout stops waiting but does not cancel running jobs.- `collect` returns already-finished job results without waiting; `await` waits for requested job IDs.- Canceled jobs should be marked terminal and should stop their background `DeferredTask` when cancellation is possible.+- `queue_parallel_parent_history(...)` accepts messages only from registered direct tool workers. `collect_parallel_jobs(...)` optionally promotes those messages while collecting terminal jobs; arbitrary worker history is never copied.## Work Guidancediff --git a/tests/test_parallel_tool.py b/tests/test_parallel_tool.pyindex 1e612e43f..578df4051 100644--- a/tests/test_parallel_tool.py+++ b/tests/test_parallel_tool.py@@ -1046,6 +1046,61 @@ async def test_parallel_tool_keeps_wrapper_out_of_visible_log() -> None:assert agent.tool_results == [("parallel", "done", {"extra": "value"})]+async def test_parallel_collect_promotes_history_after_wrapper_result() -> None:+ from tools.parallel import ParallelTool++ class HistoryAgent(_FakeAgent):+ def __init__(self) -> None:+ super().__init__()+ self.history_events = []++ def hist_add_tool_result(self, tool_name, tool_result, **kwargs):+ self.history_events.append(("tool", tool_name, tool_result, kwargs))++ def hist_add_message(self, ai, content, tokens=0, **kwargs):+ self.history_events.append(("message", ai, content, tokens, kwargs))++ agent = HistoryAgent()+ job = parallel_tools.ParallelJob(+ id="vision-ready",+ parent_context_id=agent.context.id,+ index=0,+ tool_name="vision_load",+ tool_args={"paths": ["/image.png"]},+ kind="tool",+ state="success",+ result="Loaded images (1)",+ parent_history=[("raw-image-message", 1500)],+ )+ agent.context.set_data(parallel_tools.PARALLEL_JOBS_KEY, {job.id: job})+ tool = ParallelTool(+ agent, # type: ignore[arg-type]+ "parallel",+ None,+ {"action": "collect", "job_ids": [job.id]},+ "",+ None,+ )++ response = await tool.execute(**tool.args)++ assert job.id in agent.context.get_data(parallel_tools.PARALLEL_JOBS_KEY)+ assert agent.history_events == []++ await tool.after_execution(response)++ assert agent.history_events[0][0] == "tool"+ assert agent.history_events[1] == (+ "message",+ False,+ "raw-image-message",+ 1500,+ {},+ )+ assert job.id not in agent.context.get_data(parallel_tools.PARALLEL_JOBS_KEY)++@pytest.mark.asyncioasync def test_parallel_child_contexts_are_chats_not_tasks(monkeypatch) -> None:from agent import AgentContextdiff --git a/tests/test_vision_load_image_refs.py b/tests/test_vision_load_image_refs.pyindex 43540b11b..09ff771bb 100644--- a/tests/test_vision_load_image_refs.py+++ b/tests/test_vision_load_image_refs.py@@ -108,7 +108,7 @@ async def test_vision_load_materializes_local_image_to_chat_artifact(monkeypatchmessages = []updates = []agent = SimpleNamespace(- context=SimpleNamespace(id="ctx-vision"),+ context=SimpleNamespace(id="ctx-vision", get_data=lambda *_args, **_kwargs: None),agent_name="Agent 0",hist_add_tool_result=lambda *args, **kwargs: tool_results.append((args, kwargs)),hist_add_message=lambda *args, **kwargs: messages.append((args, kwargs)),@@ -303,6 +303,12 @@ async def test_parallel_worker_consumes_parent_ephemeral_image(monkeypatch, tmp_lambda _agent: {"vision": True, "max_embeds": 10},)monkeypatch.setattr(vision_load_module, "get_vision_model_config", lambda _agent: {})+ queued = []+ monkeypatch.setattr(+ vision_load_module.parallel_tools,+ "queue_parallel_parent_history",+ lambda _agent, **message: queued.append(message) or True,+ )ref = vision_load_module.ephemeral_images.put_image_bytes(context_id=parent_id,@@ -313,10 +319,17 @@ async def test_parallel_worker_consumes_parent_ephemeral_image(monkeypatch, tmp_context = SimpleNamespace(id="parallel-worker",get_data=lambda key: parent_id- if key == vision_load_module.PARALLEL_WORKER_PARENT_CONTEXT_KEY+ if key == vision_load_module.parallel_tools.PARALLEL_WORKER_PARENT_CONTEXT_KEYelse None,)- agent = SimpleNamespace(context=context, agent_name="Agent 0")+ tool_results = []+ local_messages = []+ agent = SimpleNamespace(+ context=context,+ agent_name="Agent 0",+ hist_add_tool_result=lambda *args, **kwargs: tool_results.append((args, kwargs)),+ hist_add_message=lambda *args, **kwargs: local_messages.append((args, kwargs)),+ )tool = vision_load_module.VisionLoad(agent=agent,name="vision_load",@@ -325,14 +338,22 @@ async def test_parallel_worker_consumes_parent_ephemeral_image(monkeypatch, tmp_message="",loop_data=None,)+ tool.log = SimpleNamespace(id="vision-log", update=lambda **kwargs: None)- await tool.execute(paths=[ref])+ response = await tool.execute(paths=[ref])+ await tool.after_execution(response)assert tool._context_id() == parent_idassert tool.loaded_paths == ["shot.png"]assert vision_load_module.ephemeral_images.get_image(ref, context_id=parent_id) is Nonestored_ref = tool.images_dict["shot.png"]assert stored_ref.startswith("/a0/usr/chats/parent-vision/images/vision-load/shot-")+ assert local_messages == []+ assert queued[0]["tokens"] == vision_load_module.TOKENS_ESTIMATE+ raw_content = queued[0]["content"]["raw_content"]+ assert raw_content == [+ {"type": "image_url", "image_url": {"url": stored_ref}}+ ]@pytest.mark.anyiodiff --git a/tools/parallel.py b/tools/parallel.pyindex b10b9580f..cf83bc823 100644--- a/tools/parallel.py+++ b/tools/parallel.py@@ -14,8 +14,14 @@ class ParallelTool(Tool):text,**(response.additional or {}),)+ await parallel_tools.collect_parallel_jobs(+ self.agent,+ getattr(self, "_collect_job_ids", []),+ promote_parent_history=True,+ )async def execute(self, **kwargs) -> Response:+ self._collect_job_ids = []args = {**self.args, **kwargs}action = str(args.get("action") or "").strip().lower()@@ -68,9 +74,10 @@ class ParallelTool(Tool):self.agent,all_job_ids,timeout=timeout,- collect=True,+ collect=False,wait=False,)+ self._collect_job_ids = [result["job_id"] for result in results]return Response(message=parallel_tools.format_parallel_results(results),break_loop=False,@@ -80,9 +87,10 @@ class ParallelTool(Tool):self.agent,all_job_ids,timeout=timeout,- collect=True,+ collect=False,wait=True,)+ self._collect_job_ids = [result["job_id"] for result in results]return Response(message=parallel_tools.format_parallel_results(results),break_loop=False,diff --git a/tools/parallel.py.dox.md b/tools/parallel.py.dox.mdindex e7696d7f9..f60a6d3a3 100644--- a/tools/parallel.py.dox.md+++ b/tools/parallel.py.dox.md@@ -31,6 +31,7 @@- Recursive use of `parallel` from inside a direct background tool worker is blocked before execution; numbered subordinate child chats can use normal child-chat tools, including `parallel`, to create their next-level descendants.- Wrapped `call_subordinate` uses the same lifecycle as a top-level call. `job_id` identifies one parallel invocation, while its returned `context_id` identifies the reusable child agent for later `reset=false` calls.- The wrapper tool does not create its own visible process-step log; each wrapped child call owns the visible log row, and the wrapper result is recorded only in model history.+- Terminal direct jobs are collected after the wrapper result enters model history. Any explicitly queued parent-history messages are appended next, preserving result-before-content ordering for native multimodal tools.## Key Conceptsdiff --git a/tools/vision_load.py b/tools/vision_load.pyindex 791a4dfce..61b22a6fb 100644--- a/tools/vision_load.py+++ b/tools/vision_load.py@@ -2,8 +2,15 @@ from mimetypes import guess_typefrom langchain_core.messages import HumanMessage-from helpers import chat_media, ephemeral_images, files, history, images, runtime-from helpers.parallel_tools import PARALLEL_WORKER_PARENT_CONTEXT_KEY+from helpers import (+ chat_media,+ ephemeral_images,+ files,+ history,+ images,+ parallel_tools,+ runtime,+)from helpers.tool import Response, Toolfrom plugins._model_config.helpers.model_config import (build_vision_model,@@ -96,7 +103,11 @@ class VisionLoad(Tool):def _context_id(self) -> str:context = getattr(self.agent, "context", None)get_data = getattr(context, "get_data", None)- parent_id = get_data(PARALLEL_WORKER_PARENT_CONTEXT_KEY) if get_data else ""+ parent_id = (+ get_data(parallel_tools.PARALLEL_WORKER_PARENT_CONTEXT_KEY)+ if get_data+ else ""+ )return str(parent_id or getattr(context, "id", "") or "").strip()async def _call_vision_model(self, image_paths: list[str]) -> str:@@ -191,11 +202,18 @@ class VisionLoad(Tool):{"type": "image_url", "image_url": {"url": image_path}}for image_path in self.images_dict.values()]- self.agent.hist_add_message(- False,- content=history.RawMessage(- raw_content=content,- preview="<Image attachments loaded by path>",- ),- tokens=TOKENS_ESTIMATE * len(content),+ raw_message = history.RawMessage(+ raw_content=content,+ preview="<Image attachments loaded by path>",)+ tokens = TOKENS_ESTIMATE * len(content)+ if not parallel_tools.queue_parallel_parent_history(+ self.agent,+ content=raw_message,+ tokens=tokens,+ ):+ self.agent.hist_add_message(+ False,+ content=raw_message,+ tokens=tokens,+ )diff --git a/tools/vision_load.py.dox.md b/tools/vision_load.py.dox.mdindex ef09102e0..2960e6833 100644--- a/tools/vision_load.py.dox.md+++ b/tools/vision_load.py.dox.md@@ -24,6 +24,7 @@- The public tool contract is route-agnostic. A Vision Model receives the current user request through `fw.vision_load.md`; direct parallel workers inherit that request from their parent.- Delegation completes during `execute(...)` so native Responses function output contains the real capsule before `after_execution(...)` persists it.- Delegated history contains the text capsule only. Native history contains the tool result followed by one raw message holding all loaded image blocks.+- In a direct parallel worker, native image content is queued for the parent and promoted immediately after the outer `parallel` result; the disposable worker never owns the only copy of model-visible pixels.- Direct parallel workers inherit the parent's model override generically. This tool uses their recorded parent context only to resolve ephemeral refs and durable chat media.- `max_embeds` comes from the model that actually receives the images.- Vision Model calls use the selected model's Advanced `kwargs`; this tool does not impose a separate timeout or output-token limit.