Skip to content

Agent Zero AI framework

Preserve native vision output from parallel workers

Alessandro66cc11096c7e
diff --git a/helpers/parallel_tools.py b/helpers/parallel_tools.py
index dbc79ef9f..ebb8e8315 100644
--- a/helpers/parallel_tools.py
+++ b/helpers/parallel_tools.py
@@ -65,6 +65,7 @@ class ParallelJob:
log_id: str = field(default_factory=lambda: str(uuid.uuid4()))
log_item: "LogItem | None" = field(default=None, repr=False)
deferred_task: DeferredTask | None = field(default=None, repr=False)
+ parent_history: list[tuple[Any, int]] = field(default_factory=list, repr=False)
def elapsed(self) -> float:
end = self.completed_at or time.time()
@@ -170,6 +171,26 @@ def is_parallel_worker(agent: "Agent | None") -> bool:
return _parallel_worker_kind(agent) == "tool"
+def queue_parallel_parent_history(
+ agent: "Agent",
+ *,
+ content: Any,
+ tokens: int = 0,
+) -> bool:
+ if not is_parallel_worker(agent):
+ return False
+ context = agent.context
+ parent_context_id = str(
+ context.get_data(PARALLEL_WORKER_PARENT_CONTEXT_KEY) or ""
+ )
+ job_id = str(context.get_data(PARALLEL_WORKER_JOB_KEY) or "")
+ job = _get_job(parent_context_id, job_id)
+ if not job or job.kind != "tool":
+ return False
+ job.parent_history.append((content, tokens))
+ return True
+
+
def _jobs_for_context(context: "AgentContext") -> dict[str, ParallelJob]:
jobs = context.get_data(PARALLEL_JOBS_KEY)
if not isinstance(jobs, dict):
@@ -242,7 +263,6 @@ async def await_parallel_jobs(
raise ValueError("No `job_ids` were provided to await.")
deadline = time.time() + timeout
- known_job_ids = set(job_ids)
wait_timed_out_job_ids: set[str] = set()
while True:
await refresh_parallel_jobs(agent)
@@ -271,11 +291,7 @@ async def await_parallel_jobs(
snapshots.append(snapshot)
if collect:
- for job_id in known_job_ids:
- job = _jobs_for_context(agent.context).get(job_id)
- if job and job.state in TERMINAL_STATES:
- await cleanup_parallel_job(agent, job)
- _jobs_for_context(agent.context).pop(job_id, None)
+ await collect_parallel_jobs(agent, job_ids)
return snapshots
@@ -326,6 +342,25 @@ async def cleanup_parallel_job(agent: "Agent", job: ParallelJob) -> None:
await _remove_context(job.worker_context_id)
+async def collect_parallel_jobs(
+ agent: "Agent",
+ job_ids: list[str],
+ *,
+ promote_parent_history: bool = False,
+) -> None:
+ jobs = _jobs_for_context(agent.context)
+ for job_id in dict.fromkeys(job_ids):
+ job = jobs.get(job_id)
+ if not job or job.state not in TERMINAL_STATES:
+ continue
+ if promote_parent_history:
+ for content, tokens in job.parent_history:
+ agent.hist_add_message(False, content=content, tokens=tokens)
+ job.parent_history.clear()
+ await cleanup_parallel_job(agent, job)
+ jobs.pop(job_id, None)
+
+
async def build_parallel_jobs_extras(agent: "Agent") -> str:
await refresh_parallel_jobs(agent)
jobs = [
diff --git a/helpers/parallel_tools.py.dox.md b/helpers/parallel_tools.py.dox.md
index 9fb3f6b5f..924c81025 100644
--- a/helpers/parallel_tools.py.dox.md
+++ b/helpers/parallel_tools.py.dox.md
@@ -38,6 +38,7 @@
- Child tool logs mirror normal tool-call visible args; job ids remain available through wrapper results and prompt extras rather than visible process-step args.
- Wrapped tool child logs use each tool's native `get_log_object()` output when available, preserving special log rendering (for example: `code_execution_tool` uses `code_exe`, `wait` uses `progress`, MCP tools use `mcp`, and regular tools use `tool`).
- Direct parallel worker execution reuses the parent-visible child log item so tool `before_execution()` cannot create a second generic worker log or lose the native badge type.
+- Direct tools may explicitly queue model-visible history for their parent. Terminal collection records the outer `parallel` result first, promotes queued messages in job order, and only then removes disposable worker state; background jobs retain queued history until they are collected.
- Job IDs are stable handles for later await, collect, or cancel operations.
- Prompt extras must stay bounded and expose only job IDs, tool names, status, and compact result/error summaries.
@@ -47,6 +48,7 @@
- `wait=True` starts jobs and awaits them before returning until all requested jobs finish or the wait timeout is reached; the timeout stops waiting but does not cancel running jobs.
- `collect` returns already-finished job results without waiting; `await` waits for requested job IDs.
- Canceled jobs should be marked terminal and should stop their background `DeferredTask` when cancellation is possible.
+- `queue_parallel_parent_history(...)` accepts messages only from registered direct tool workers. `collect_parallel_jobs(...)` optionally promotes those messages while collecting terminal jobs; arbitrary worker history is never copied.
## Work Guidance
diff --git a/tests/test_parallel_tool.py b/tests/test_parallel_tool.py
index 1e612e43f..578df4051 100644
--- a/tests/test_parallel_tool.py
+++ b/tests/test_parallel_tool.py
@@ -1046,6 +1046,61 @@ async def test_parallel_tool_keeps_wrapper_out_of_visible_log() -> None:
assert agent.tool_results == [("parallel", "done", {"extra": "value"})]
+async def test_parallel_collect_promotes_history_after_wrapper_result() -> None:
+ from tools.parallel import ParallelTool
+
+ class HistoryAgent(_FakeAgent):
+ def __init__(self) -> None:
+ super().__init__()
+ self.history_events = []
+
+ def hist_add_tool_result(self, tool_name, tool_result, **kwargs):
+ self.history_events.append(("tool", tool_name, tool_result, kwargs))
+
+ def hist_add_message(self, ai, content, tokens=0, **kwargs):
+ self.history_events.append(("message", ai, content, tokens, kwargs))
+
+ agent = HistoryAgent()
+ job = parallel_tools.ParallelJob(
+ id="vision-ready",
+ parent_context_id=agent.context.id,
+ index=0,
+ tool_name="vision_load",
+ tool_args={"paths": ["/image.png"]},
+ kind="tool",
+ state="success",
+ result="Loaded images (1)",
+ parent_history=[("raw-image-message", 1500)],
+ )
+ agent.context.set_data(parallel_tools.PARALLEL_JOBS_KEY, {job.id: job})
+ tool = ParallelTool(
+ agent, # type: ignore[arg-type]
+ "parallel",
+ None,
+ {"action": "collect", "job_ids": [job.id]},
+ "",
+ None,
+ )
+
+ response = await tool.execute(**tool.args)
+
+ assert job.id in agent.context.get_data(parallel_tools.PARALLEL_JOBS_KEY)
+ assert agent.history_events == []
+
+ await tool.after_execution(response)
+
+ assert agent.history_events[0][0] == "tool"
+ assert agent.history_events[1] == (
+ "message",
+ False,
+ "raw-image-message",
+ 1500,
+ {},
+ )
+ assert job.id not in agent.context.get_data(parallel_tools.PARALLEL_JOBS_KEY)
+
+
@pytest.mark.asyncio
async def test_parallel_child_contexts_are_chats_not_tasks(monkeypatch) -> None:
from agent import AgentContext
diff --git a/tests/test_vision_load_image_refs.py b/tests/test_vision_load_image_refs.py
index 43540b11b..09ff771bb 100644
--- a/tests/test_vision_load_image_refs.py
+++ b/tests/test_vision_load_image_refs.py
@@ -108,7 +108,7 @@ async def test_vision_load_materializes_local_image_to_chat_artifact(monkeypatch
messages = []
updates = []
agent = SimpleNamespace(
- context=SimpleNamespace(id="ctx-vision"),
+ context=SimpleNamespace(id="ctx-vision", get_data=lambda *_args, **_kwargs: None),
agent_name="Agent 0",
hist_add_tool_result=lambda *args, **kwargs: tool_results.append((args, kwargs)),
hist_add_message=lambda *args, **kwargs: messages.append((args, kwargs)),
@@ -303,6 +303,12 @@ async def test_parallel_worker_consumes_parent_ephemeral_image(monkeypatch, tmp_
lambda _agent: {"vision": True, "max_embeds": 10},
)
monkeypatch.setattr(vision_load_module, "get_vision_model_config", lambda _agent: {})
+ queued = []
+ monkeypatch.setattr(
+ vision_load_module.parallel_tools,
+ "queue_parallel_parent_history",
+ lambda _agent, **message: queued.append(message) or True,
+ )
ref = vision_load_module.ephemeral_images.put_image_bytes(
context_id=parent_id,
@@ -313,10 +319,17 @@ async def test_parallel_worker_consumes_parent_ephemeral_image(monkeypatch, tmp_
context = SimpleNamespace(
id="parallel-worker",
get_data=lambda key: parent_id
- if key == vision_load_module.PARALLEL_WORKER_PARENT_CONTEXT_KEY
+ if key == vision_load_module.parallel_tools.PARALLEL_WORKER_PARENT_CONTEXT_KEY
else None,
)
- agent = SimpleNamespace(context=context, agent_name="Agent 0")
+ tool_results = []
+ local_messages = []
+ agent = SimpleNamespace(
+ context=context,
+ agent_name="Agent 0",
+ hist_add_tool_result=lambda *args, **kwargs: tool_results.append((args, kwargs)),
+ hist_add_message=lambda *args, **kwargs: local_messages.append((args, kwargs)),
+ )
tool = vision_load_module.VisionLoad(
agent=agent,
name="vision_load",
@@ -325,14 +338,22 @@ async def test_parallel_worker_consumes_parent_ephemeral_image(monkeypatch, tmp_
message="",
loop_data=None,
)
+ tool.log = SimpleNamespace(id="vision-log", update=lambda **kwargs: None)
- await tool.execute(paths=[ref])
+ response = await tool.execute(paths=[ref])
+ await tool.after_execution(response)
assert tool._context_id() == parent_id
assert tool.loaded_paths == ["shot.png"]
assert vision_load_module.ephemeral_images.get_image(ref, context_id=parent_id) is None
stored_ref = tool.images_dict["shot.png"]
assert stored_ref.startswith("/a0/usr/chats/parent-vision/images/vision-load/shot-")
+ assert local_messages == []
+ assert queued[0]["tokens"] == vision_load_module.TOKENS_ESTIMATE
+ raw_content = queued[0]["content"]["raw_content"]
+ assert raw_content == [
+ {"type": "image_url", "image_url": {"url": stored_ref}}
+ ]
@pytest.mark.anyio
diff --git a/tools/parallel.py b/tools/parallel.py
index b10b9580f..cf83bc823 100644
--- a/tools/parallel.py
+++ b/tools/parallel.py
@@ -14,8 +14,14 @@ class ParallelTool(Tool):
text,
**(response.additional or {}),
)
+ await parallel_tools.collect_parallel_jobs(
+ self.agent,
+ getattr(self, "_collect_job_ids", []),
+ promote_parent_history=True,
+ )
async def execute(self, **kwargs) -> Response:
+ self._collect_job_ids = []
args = {**self.args, **kwargs}
action = str(args.get("action") or "").strip().lower()
@@ -68,9 +74,10 @@ class ParallelTool(Tool):
self.agent,
all_job_ids,
timeout=timeout,
- collect=True,
+ collect=False,
wait=False,
)
+ self._collect_job_ids = [result["job_id"] for result in results]
return Response(
message=parallel_tools.format_parallel_results(results),
break_loop=False,
@@ -80,9 +87,10 @@ class ParallelTool(Tool):
self.agent,
all_job_ids,
timeout=timeout,
- collect=True,
+ collect=False,
wait=True,
)
+ self._collect_job_ids = [result["job_id"] for result in results]
return Response(
message=parallel_tools.format_parallel_results(results),
break_loop=False,
diff --git a/tools/parallel.py.dox.md b/tools/parallel.py.dox.md
index e7696d7f9..f60a6d3a3 100644
--- a/tools/parallel.py.dox.md
+++ b/tools/parallel.py.dox.md
@@ -31,6 +31,7 @@
- Recursive use of `parallel` from inside a direct background tool worker is blocked before execution; numbered subordinate child chats can use normal child-chat tools, including `parallel`, to create their next-level descendants.
- Wrapped `call_subordinate` uses the same lifecycle as a top-level call. `job_id` identifies one parallel invocation, while its returned `context_id` identifies the reusable child agent for later `reset=false` calls.
- The wrapper tool does not create its own visible process-step log; each wrapped child call owns the visible log row, and the wrapper result is recorded only in model history.
+- Terminal direct jobs are collected after the wrapper result enters model history. Any explicitly queued parent-history messages are appended next, preserving result-before-content ordering for native multimodal tools.
## Key Concepts
diff --git a/tools/vision_load.py b/tools/vision_load.py
index 791a4dfce..61b22a6fb 100644
--- a/tools/vision_load.py
+++ b/tools/vision_load.py
@@ -2,8 +2,15 @@ from mimetypes import guess_type
from langchain_core.messages import HumanMessage
-from helpers import chat_media, ephemeral_images, files, history, images, runtime
-from helpers.parallel_tools import PARALLEL_WORKER_PARENT_CONTEXT_KEY
+from helpers import (
+ chat_media,
+ ephemeral_images,
+ files,
+ history,
+ images,
+ parallel_tools,
+ runtime,
+)
from helpers.tool import Response, Tool
from plugins._model_config.helpers.model_config import (
build_vision_model,
@@ -96,7 +103,11 @@ class VisionLoad(Tool):
def _context_id(self) -> str:
context = getattr(self.agent, "context", None)
get_data = getattr(context, "get_data", None)
- parent_id = get_data(PARALLEL_WORKER_PARENT_CONTEXT_KEY) if get_data else ""
+ parent_id = (
+ get_data(parallel_tools.PARALLEL_WORKER_PARENT_CONTEXT_KEY)
+ if get_data
+ else ""
+ )
return str(parent_id or getattr(context, "id", "") or "").strip()
async def _call_vision_model(self, image_paths: list[str]) -> str:
@@ -191,11 +202,18 @@ class VisionLoad(Tool):
{"type": "image_url", "image_url": {"url": image_path}}
for image_path in self.images_dict.values()
]
- self.agent.hist_add_message(
- False,
- content=history.RawMessage(
- raw_content=content,
- preview="<Image attachments loaded by path>",
- ),
- tokens=TOKENS_ESTIMATE * len(content),
+ raw_message = history.RawMessage(
+ raw_content=content,
+ preview="<Image attachments loaded by path>",
)
+ tokens = TOKENS_ESTIMATE * len(content)
+ if not parallel_tools.queue_parallel_parent_history(
+ self.agent,
+ content=raw_message,
+ tokens=tokens,
+ ):
+ self.agent.hist_add_message(
+ False,
+ content=raw_message,
+ tokens=tokens,
+ )
diff --git a/tools/vision_load.py.dox.md b/tools/vision_load.py.dox.md
index ef09102e0..2960e6833 100644
--- a/tools/vision_load.py.dox.md
+++ b/tools/vision_load.py.dox.md
@@ -24,6 +24,7 @@
- The public tool contract is route-agnostic. A Vision Model receives the current user request through `fw.vision_load.md`; direct parallel workers inherit that request from their parent.
- Delegation completes during `execute(...)` so native Responses function output contains the real capsule before `after_execution(...)` persists it.
- Delegated history contains the text capsule only. Native history contains the tool result followed by one raw message holding all loaded image blocks.
+- In a direct parallel worker, native image content is queued for the parent and promoted immediately after the outer `parallel` result; the disposable worker never owns the only copy of model-visible pixels.
- Direct parallel workers inherit the parent's model override generically. This tool uses their recorded parent context only to resolve ephemeral refs and durable chat media.
- `max_embeds` comes from the model that actually receives the images.
- Vision Model calls use the selected model's Advanced `kwargs`; this tool does not impose a separate timeout or output-token limit.