Skip to content

Self-improving AI Employees for everyone

Make connected tools and sub-agents usable by the Vertex managed agent

trohitg129d1e4e112c
diff --git a/docs-internal/agent_delegation.md b/docs-internal/agent_delegation.md
index 6ef574df..aec64047 100644
--- a/docs-internal/agent_delegation.md
+++ b/docs-internal/agent_delegation.md
@@ -402,6 +402,18 @@ The parent LLM receives this immediate response from the tool call:
The parent does NOT wait for the child's result. The child's output is broadcast to the UI via WebSocket (`broadcaster.update_node_status`).
+### Blocking Delegation for Bridged Cloud Agents (delegation_wait_seconds)
+
+Fire-and-forget assumes the parent loop can poll cheaply — a local LLM turn costs milliseconds. A bridged cloud agent (`vertex_managed_agent`) pays a full Interactions API round trip per poll, so it opts into a blocking wait instead:
+
+- The bridge sets `config["delegation_wait_seconds"]` (node param, default 600s, clamped to the remaining Temporal activity budget) on the dispatch config for agent-type tools only.
+- `_execute_delegated_agent` then awaits the spawned child via `wait_for_delegation` (`asyncio.wait_for(asyncio.shield(task))` — a timeout cancels the WAITER, never the child) and returns the child's REAL result as the function result, shaped by `_delegation_result_reply`.
+- The reply carries `delegation_lifecycle: True`; `execute_tool` pops it and skips its terminal `success` broadcast (the child's `run_child_agent` already owns the terminal status — an awaited `error` result must not be stomped by a `success` glow).
+- On timeout the child keeps running in the background and the parent receives the standard `status: "delegated"` ack; the vertex bridge declares `check_delegated_tasks` alongside any `delegate_to_*` tool (mirroring the native auto-injection) so the cloud agent can retrieve the result on a later turn. An identical re-call after a timeout hits the `_active_delegations` dedupe and awaits the existing in-flight task instead of duplicating work.
+- Absent `delegation_wait_seconds` (all native agents), behavior is byte-identical fire-and-forget.
+
+Contract locked by `TestDelegationWait` (`tests/nodes/test_ai_tools.py`) and the vertex bridge tests (`tests/nodes/test_vertex_agents.py`).
+
### Result Retrieval (check_delegated_tasks Tool)
When a parent agent has delegation tools, a `check_delegated_tasks` tool is automatically injected. The parent LLM can call this tool to check on child status and retrieve results:
diff --git a/server/nodes/agent/_vertex.py b/server/nodes/agent/_vertex.py
index 0d1614e7..01dd34d1 100644
--- a/server/nodes/agent/_vertex.py
+++ b/server/nodes/agent/_vertex.py
@@ -49,6 +49,11 @@ TERMINAL_STATUSES = frozenset(
_POLL_INTERVAL_SECONDS = 3.0
+# Set on a client instance after the surface rejects a streaming create
+# (400 "Precondition check failed" on the enterprise agent surface) so
+# later turns in the same run skip straight to the background+poll path.
+_STREAM_UNSUPPORTED_ATTR = "_machina_stream_unsupported"
+
def is_genai_error(exc: BaseException) -> bool:
"""True for any google-genai SDK error, across its two hierarchies.
@@ -81,6 +86,22 @@ def is_expired_environment_error(exc: BaseException) -> bool:
)
+def is_precondition_failure(exc: BaseException) -> bool:
+ """400 'Precondition check failed' from the enterprise surface.
+
+ Live-verified trigger: creating with ``previous_interaction_id``
+ pointing at an interaction that is not in a resumable state (e.g.
+ still ``in_progress`` — an orphaned background interaction keeps
+ running server-side after a client-side cancel/reload). A chain
+ wedged on such a target fails every subsequent run, so callers
+ treat this like an expired chain: wipe the stored ids and retry
+ fresh.
+ """
+ if not is_genai_error(exc):
+ return False
+ return "precondition check failed" in str(exc).lower()
+
+
def build_genai_client(api_key: str, project_id: str, location: str = DEFAULT_LOCATION):
"""Build the google-genai client for the managed-agent surfaces."""
from google import genai
@@ -168,9 +189,20 @@ async def stream_interaction(
non-stream ``interactions.get`` and polled to a terminal status as a
safety net.
- If the streaming create itself is rejected by the surface, falls back
- to the plain background+poll path so the node still works.
+ If the streaming create itself is rejected by the surface (e.g. the
+ enterprise agent surface answers 400 "Precondition check failed"),
+ falls back to the plain background+poll path so the node still works
+ — and latches the rejection on the client so the remaining turns of
+ the run skip the doomed streaming attempt instead of paying a failed
+ round trip + a warning per turn. A fresh run builds a fresh client,
+ so streaming is re-probed next run.
"""
+ # Identity check on purpose: only an explicit latch counts (mock
+ # clients auto-create truthy attributes).
+ if getattr(client, _STREAM_UNSUPPORTED_ATTR, False) is True:
+ return await create_interaction_and_wait(
+ client, poll_interval=poll_interval, **kwargs
+ )
try:
stream = await client.aio.interactions.create(
background=True, stream=True, **kwargs
@@ -178,8 +210,10 @@ async def stream_interaction(
except Exception as exc: # noqa: BLE001 — fall back, then error-map there
if not is_genai_error(exc):
raise
+ setattr(client, _STREAM_UNSUPPORTED_ATTR, True)
logger.warning(
- "[Vertex] streaming create rejected (%s) — falling back to poll",
+ "[Vertex] streaming create rejected (%s) — falling back to poll "
+ "for the rest of this run",
str(exc).replace("\n", " ")[:200],
)
return await create_interaction_and_wait(
diff --git a/server/nodes/agent/vertex_agent_admin/__init__.py b/server/nodes/agent/vertex_agent_admin/__init__.py
index 885da74e..f72e6271 100644
--- a/server/nodes/agent/vertex_agent_admin/__init__.py
+++ b/server/nodes/agent/vertex_agent_admin/__init__.py
@@ -104,7 +104,11 @@ class VertexAgentAdminNode(ActionNode):
type = "vertex_agent_admin"
display_name = "Vertex Agent Admin"
subtitle = "Agent Lifecycle"
- group = ("tool",)
+ # Operator-facing lifecycle CRUD — belongs with the Vertex agents in
+ # the AI Agents palette. ("tool",) would auto-derive a bogus
+ # isConfigNode uiHint and imply input-tools wiring this node has no
+ # output-tool handle to satisfy.
+ group = ("agent",)
description = (
"Create, list, inspect, and delete custom managed agents on the "
"Gemini Enterprise Agent Platform."
diff --git a/server/nodes/agent/vertex_managed_agent/__init__.py b/server/nodes/agent/vertex_managed_agent/__init__.py
index 0e1d654e..1c2889ba 100644
--- a/server/nodes/agent/vertex_managed_agent/__init__.py
+++ b/server/nodes/agent/vertex_managed_agent/__init__.py
@@ -40,6 +40,7 @@ from .._vertex import (
DEFAULT_MANAGED_AGENT,
build_genai_client,
is_expired_environment_error,
+ is_precondition_failure,
raise_as_user_error,
resolve_api_key_from_context,
stream_interaction,
@@ -59,6 +60,12 @@ _CLOUD_STEP_LABELS = {
}
_CLOUD_NOISE_NAMES = frozenset({"provision_sandbox"})
+# Final statuses whose interaction id may be chained onto with
+# previous_interaction_id on a later run (live-verified: completed and
+# requires_action resume fine; failed/stuck targets 400 with
+# "Precondition check failed").
+_RESUMABLE_STATUSES = frozenset({"completed", "requires_action"})
+
class VertexManagedAgentParams(BaseModel):
"""Managed-agent tuning surface (aiAgent-parity minimal).
@@ -111,6 +118,18 @@ class VertexManagedAgentParams(BaseModel):
description="Cap on requires_action tool round-trips per run.",
json_schema_extra={"group": "options"},
)
+ delegation_wait_seconds: int = Field(
+ default=600,
+ ge=0,
+ le=1500,
+ description=(
+ "How long a delegate_to_* call blocks waiting for the "
+ "sub-agent's real answer before falling back to a task_id + "
+ "check_delegated_tasks polling contract. 0 disables the wait "
+ "(fire-and-forget, native behavior)."
+ ),
+ json_schema_extra={"group": "options"},
+ )
visualize_cloud_tools: bool = Field(
default=True,
description=(
@@ -144,6 +163,9 @@ class VertexManagedAgentOutput(BaseModel):
cloud_tools_used: Optional[List[str]] = None
usage: Optional[Dict[str, Any]] = None
timestamp: Optional[str] = None
+ # Populated when the requires_action loop stops without executing a
+ # tool (undeclared call names) — surfaces "why didn't my tool run".
+ warnings: Optional[List[str]] = None
model_config = ConfigDict(extra="allow")
@@ -218,7 +240,7 @@ class VertexManagedAgentNode(ActionNode):
# ---- tool bridge: connected tool nodes -> function declarations
await phase("building_tools")
- declared_tools, tool_configs = await self._build_function_tools(tool_data)
+ declared_tools, tool_configs = await self._build_function_tools(tool_data, node_id)
# ---- live visibility state (shared with the post-turn sweep)
live_nodes: Dict[str, str] = {} # cloud_tool_key -> canvas node id
@@ -248,8 +270,29 @@ class VertexManagedAgentNode(ActionNode):
"agent": params.agent,
"store": True,
}
- if params.system_instruction:
- create_kwargs["system_instruction"] = params.system_instruction
+ system_instruction = params.system_instruction or ""
+ if "check_delegated_tasks" in tool_configs:
+ from constants import AI_AGENT_TYPES
+
+ delegate_names = sorted(n for n, c in tool_configs.items() if c.get("node_type") in AI_AGENT_TYPES)
+ # Inverts the fire-and-forget wording baked into the shared
+ # DelegateToAgentSchema docstring: bridged delegation blocks
+ # and returns the real result unless the wait times out.
+ system_instruction += (
+ "\n\n## Agent Delegation\n"
+ "When delegating to sub-agents, use 'task' for the mission "
+ "directive (role and goal) and 'context' for input data.\n"
+ "Delegation calls WAIT for the sub-agent and return its "
+ "final result directly in the function result. If a result "
+ "instead reports status='delegated' or 'ALREADY_DELEGATED', "
+ "the agent is still working: do NOT re-call the delegate "
+ "tool for the same task; continue other work, and call "
+ "'check_delegated_tasks' with the task_id to retrieve the "
+ "result.\n"
+ f"Available agents: {', '.join(delegate_names)}"
+ )
+ if system_instruction:
+ create_kwargs["system_instruction"] = system_instruction
if declared_tools:
create_kwargs["tools"] = declared_tools
@@ -275,35 +318,58 @@ class VertexManagedAgentNode(ActionNode):
)
except NodeUserError as exc:
cause = exc.__cause__
- if prev_interaction_id and cause is not None and is_expired_environment_error(cause):
+ # Unresumable chain: expired environment (7-day TTL) OR a
+ # 400 precondition failure — live-verified when the stored
+ # previous_interaction_id points at an interaction that is
+ # not resumable (e.g. an orphaned background interaction
+ # still in_progress after a client-side cancel/reload).
+ # Either way the stored ids are useless: wipe and go fresh.
+ if (
+ prev_interaction_id
+ and cause is not None
+ and (is_expired_environment_error(cause) or is_precondition_failure(cause))
+ ):
logger.warning(
- "[Vertex Agent] stale interaction chain for %s — retrying fresh", node_id
+ "[Vertex Agent] unresumable interaction chain for %s (%s) — retrying fresh",
+ node_id,
+ str(cause).replace("\n", " ")[:120],
)
if memory_node_id:
await self._save_chain_ids(database, memory_node_id, None, None)
prev_interaction_id = None
+ environment = "remote"
interaction = await run_turn(input=prompt, environment="remote")
else:
raise
# ---- requires_action loop: answer pending local function calls
+ run_warnings: List[str] = []
while (
getattr(interaction, "status", None) == "requires_action"
and turn < params.max_turns
):
results = self._pending_function_results_needed(interaction, tool_configs)
if not results:
- logger.warning(
- "[Vertex Agent] requires_action with no answerable "
- "function calls (node=%s) — stopping",
- node_id,
+ unmatched = self._unanswered_call_names(interaction, tool_configs)
+ msg = (
+ "requires_action with no answerable function calls: "
+ f"pending={unmatched or ['<none>']} "
+ f"declared={sorted(tool_configs) or ['<none>']}"
)
+ logger.warning("[Vertex Agent] %s (node=%s) — stopping", msg, node_id)
+ run_warnings.append(msg)
break
+ # Delegation waits must fit the Temporal activity budget:
+ # leave margin for the closing turn + memory persist + sweep.
+ remaining = int(AI_START_TO_CLOSE.total_seconds() - (time.time() - start_time)) - 120
+ wait = max(0, min(params.delegation_wait_seconds, remaining))
function_results = []
for call in results:
await phase("executing_tool", tool_name=call["name"])
function_results.append(
- await self._execute_bridged_call(call, tool_configs, ctx)
+ await self._execute_bridged_call(
+ call, tool_configs, ctx, delegation_wait_seconds=wait
+ )
)
turn += 1
await broadcaster.broadcast_agent_progress(
@@ -357,11 +423,21 @@ class VertexManagedAgentNode(ActionNode):
# ---- memory persist
if memory_node_id:
await phase("saving_memory")
+ # Only a RESUMABLE final status may overwrite the stored chain
+ # ids: persisting a failed/stuck interaction id wedges every
+ # later run on 400 Precondition (chaining onto an unresumable
+ # target — live-verified). Otherwise keep the last good pair.
+ if status in _RESUMABLE_STATUSES:
+ chain_ix = getattr(interaction, "id", None)
+ chain_env = environment_id
+ else:
+ chain_ix = prev_interaction_id
+ chain_env = environment if environment != "remote" else None
await self._save_chain_ids(
database,
memory_node_id,
- getattr(interaction, "id", None),
- environment_id,
+ chain_ix,
+ chain_env,
human=prompt,
assistant=response_text,
window_size=int((memory_data or {}).get("window_size") or 10),
@@ -392,6 +468,7 @@ class VertexManagedAgentNode(ActionNode):
"message": f"Vertex agent {status} in {turn} turn(s)",
"status": status,
"turns": turn,
+ **({"warnings": run_warnings} if run_warnings else {}),
},
workflow_id=workflow_id,
)
@@ -414,6 +491,7 @@ class VertexManagedAgentNode(ActionNode):
"cloud_tools_used": sorted(cloud_tools.values()) or None,
"usage": usage,
"timestamp": datetime.now().isoformat(),
+ "warnings": run_warnings or None,
}
# ------------------------------------------------------------------
@@ -430,44 +508,114 @@ class VertexManagedAgentNode(ActionNode):
return str(input_data) if input_data else ""
return str(input_data) if input_data else ""
+ @staticmethod
+ def _is_declarable_tool_type(node_type: str) -> bool:
+ """Mirror of the native rebind gate (services/ai.py) and
+ ``iter_tool_node_classes``: ToolNodes, agents (delegation), and
+ dual-purpose ``usable_as_tool`` plugins declare; display-only
+ plugins (vertexCloudTool — minted by ``_ops`` with a persisted
+ ``input-tools`` edge) don't. Unknown types (androidTool
+ aggregator, pseudo-types) default-allow to preserve behavior.
+ """
+ from services.node_registry import get_node_class
+
+ cls = get_node_class(node_type)
+ if cls is None:
+ return True
+ kind = getattr(cls, "component_kind", "")
+ if kind in ("tool", "agent"):
+ return True
+ return bool(getattr(cls, "usable_as_tool", False))
+
@staticmethod
async def _build_function_tools(
tool_data: List[Dict[str, Any]],
+ agent_node_id: str,
) -> Tuple[List[Dict[str, Any]], Dict[str, Dict[str, Any]]]:
"""Convert connected tool nodes into Interactions function tools.
Reuses ``AIService._build_tool_from_node`` so DB ToolSchema
overrides, plugin ClassVar names, and the delegation schema gate
- all apply exactly as they do for the local agent loop.
+ all apply exactly as they do for the local agent loop. When any
+ connected node is an agent, ``check_delegated_tasks`` is also
+ declared — the timeout fallback of the blocking-delegation
+ contract (mirrors the native auto-injection in
+ ``AIService.execute_agent``).
"""
if not tool_data:
return [], {}
+ from constants import AI_AGENT_TYPES
from services.plugin.deps import get_ai_service
+ from services.plugin.tool import inline_schema_refs
ai_service = get_ai_service()
declared: List[Dict[str, Any]] = []
configs: Dict[str, Dict[str, Any]] = {}
- for tool_info in tool_data:
- structured, config = await ai_service._build_tool_from_node(tool_info)
- if structured is None:
- continue
+
+ def _declare(structured: Any, config: Optional[Dict[str, Any]]) -> None:
if structured.args_schema is not None:
- schema = structured.args_schema.model_json_schema()
+ # Function-calling APIs reject schema indirection —
+ # inline $defs/$ref (nested BaseModel / Enum Params
+ # fields) instead of stripping and leaving dangling refs.
+ schema = inline_schema_refs(structured.args_schema.model_json_schema())
else:
schema = {"type": "object", "properties": {}}
- # Function-calling APIs reject schema indirection; the plugin
- # contract already forbids $defs in tool Params schemas.
- schema.pop("$defs", None)
- schema.pop("definitions", None)
+ # Deterministic de-dup: first node keeps the base name, later
+ # nodes (edge order) get _2, _3, ... — declared list and
+ # dispatch configs stay in sync. Dispatch routes by the
+ # config's node_type/node_id, so a suffixed name is safe.
+ name = structured.name
+ if name in configs:
+ n = 2
+ while f"{name}_{n}" in configs:
+ n += 1
+ logger.warning(
+ "[Vertex Agent] duplicate tool name %r (node %s) — declared as %r",
+ name,
+ (config or {}).get("node_id"),
+ f"{name}_{n}",
+ )
+ name = f"{name}_{n}"
declared.append(
{
"type": "function",
- "name": structured.name,
- "description": structured.description or structured.name,
+ "name": name,
+ "description": structured.description or name,
"parameters": schema,
}
)
- configs[structured.name] = config or {}
+ configs[name] = config or {}
+
+ for tool_info in tool_data:
+ node_type = tool_info.get("node_type", "")
+ if not VertexManagedAgentNode._is_declarable_tool_type(node_type):
+ logger.debug(
+ "[Vertex Agent] skipping non-declarable tool node type=%s node=%s",
+ node_type,
+ tool_info.get("node_id"),
+ )
+ continue
+ structured, config = await ai_service._build_tool_from_node(tool_info)
+ if structured is None:
+ continue
+ _declare(structured, config)
+
+ # Sub-agent support: delegation may time out and fall back to a
+ # task_id — the cloud agent then needs check_delegated_tasks to
+ # retrieve the result. MUST happen here (not later): the live
+ # handler snapshots declared_names from these configs, and a
+ # streamed check call would otherwise mint a bogus cloud-tool node.
+ if any(cfg.get("node_type") in AI_AGENT_TYPES for cfg in configs.values()):
+ check_info = {
+ "node_type": "_builtin_check_delegated_tasks",
+ "node_id": f"{agent_node_id}_check_tasks",
+ "parameters": {},
+ "label": "Check Delegated Tasks",
+ }
+ structured, config = await ai_service._build_tool_from_node(check_info)
+ if structured is not None:
+ _declare(structured, config)
+
return declared, configs
@staticmethod
@@ -506,15 +654,45 @@ class VertexManagedAgentNode(ActionNode):
)
return pending
+ @staticmethod
+ def _unanswered_call_names(
+ interaction: Any,
+ tool_configs: Dict[str, Dict[str, Any]],
+ ) -> List[str]:
+ """Unanswered function_call names we did NOT declare (noise excluded).
+
+ Diagnostic companion to ``_pending_function_results_needed``:
+ when the loop stops because nothing is answerable, these names
+ explain WHY (cloud-side calls, or a declaration mismatch).
+ """
+ steps = list(getattr(interaction, "steps", None) or [])
+ answered = {
+ getattr(s, "call_id", None)
+ for s in steps
+ if getattr(s, "type", "") == "function_result"
+ }
+ return sorted(
+ {
+ (getattr(s, "name", "") or "")
+ for s in steps
+ if getattr(s, "type", "") == "function_call"
+ and getattr(s, "id", None) not in answered
+ and (getattr(s, "name", "") or "") not in tool_configs
+ and (getattr(s, "name", "") or "") not in _CLOUD_NOISE_NAMES
+ }
+ )
+
@staticmethod
async def _execute_bridged_call(
call: Dict[str, Any],
tool_configs: Dict[str, Dict[str, Any]],
ctx: NodeContext,
+ delegation_wait_seconds: int = 0,
) -> Dict[str, Any]:
"""Dispatch one bridged function call through execute_tool."""
from pydantic_core import to_jsonable_python
+ from constants import AI_AGENT_TYPES
from services.handlers.tools import execute_tool
from services.plugin.deps import get_ai_service, get_database
@@ -526,6 +704,11 @@ class VertexManagedAgentNode(ActionNode):
for key in ("nodes", "edges", "workspace_dir", "execution_id"):
if ctx.raw.get(key) is not None:
config[key] = ctx.raw[key]
+ # Blocking delegation (agent tools only): the cloud agent cannot
+ # poll cheaply, so the bridged call awaits the child inline and
+ # falls back to task_id + check_delegated_tasks on timeout.
+ if delegation_wait_seconds and config.get("node_type") in AI_AGENT_TYPES:
+ config["delegation_wait_seconds"] = delegation_wait_seconds
from ._ops import record_tool_output
diff --git a/server/services/handlers/tools.py b/server/services/handlers/tools.py
index a757ce88..c2c1e85a 100644
--- a/server/services/handlers/tools.py
+++ b/server/services/handlers/tools.py
@@ -119,7 +119,14 @@ async def execute_tool(tool_name: str, tool_args: Dict[str, Any], config: Dict[s
)
raise
- handler_owns_lifecycle = isinstance(result, dict) and result.get("status") in ("delegated", "ALREADY_DELEGATED")
+ # Awaited-delegation results (delegation_wait_seconds > 0) carry a
+ # delegation_lifecycle marker: run_child_agent already broadcast the
+ # child node's terminal status, so a duplicate `success` here would
+ # stomp an `error` glow. pop() keeps the marker out of the payload
+ # the LLM sees.
+ handler_owns_lifecycle = isinstance(result, dict) and (
+ result.get("status") in ("delegated", "ALREADY_DELEGATED") or result.pop("delegation_lifecycle", False)
+ )
if node_id and broadcaster and not handler_owns_lifecycle:
await broadcaster.update_node_status(
node_id,
@@ -331,6 +338,14 @@ async def _execute_delegated_agent(args: Dict[str, Any], config: Dict[str, Any])
# Get parent node ID for duplicate tracking
parent_node_id = config.get("parent_node_id", "")
+ # Opt-in blocking wait (delegation_wait_seconds in config): bridged
+ # cloud agents (vertex_managed_agent) cannot poll cheaply — a
+ # check_delegated_tasks round trip costs a full Interactions API turn
+ # — so they await the child inline and fall back to the polling
+ # contract only on timeout. Absent key = fire-and-forget (native
+ # agent loop, unchanged).
+ wait_seconds = float(config.get("delegation_wait_seconds") or 0)
+
# Generate hash of task to detect duplicate delegation attempts
task_hash = hashlib.md5(f"{task_description}:{task_context}".encode()).hexdigest()[:16]
delegation_key = (parent_node_id, node_id, task_hash)
@@ -339,6 +354,12 @@ async def _execute_delegated_agent(args: Dict[str, Any], config: Dict[str, Any])
existing_task_id = _active_delegations.get(delegation_key)
if existing_task_id:
logger.warning(f"[Delegated Agent] Duplicate delegation detected: task_hash={task_hash}, existing_task_id={existing_task_id}")
+ if wait_seconds > 0:
+ # Re-call after a timed-out wait: await the existing in-flight
+ # task instead of duplicating work.
+ resolved = await wait_for_delegation(existing_task_id, timeout=wait_seconds, database=database)
+ if resolved is not None:
+ return _delegation_result_reply(existing_task_id, resolved)
return {
"success": True,
"status": "ALREADY_DELEGATED",
@@ -612,6 +633,25 @@ async def _execute_delegated_agent(args: Dict[str, Any], config: Dict[str, Any])
task = asyncio.create_task(run_child_agent())
_delegated_tasks[task_id] = task
+ if wait_seconds > 0:
+ resolved = await wait_for_delegation(task_id, timeout=wait_seconds, task=task, database=database)
+ if resolved is not None:
+ return _delegation_result_reply(task_id, resolved)
+ return {
+ "success": True,
+ "status": "delegated",
+ "task_id": task_id,
+ "agent_node_id": node_id,
+ "agent_name": agent_label,
+ "message": (
+ f"'{agent_label}' is STILL WORKING after {int(wait_seconds)}s "
+ f"(task_id: {task_id}). It continues in the background. "
+ f"Do NOT call this tool again for this task. Use "
+ f"'check_delegated_tasks' with task_id='{task_id}' to "
+ f"retrieve the result later."
+ ),
+ }
+
# Return immediately - Parent agent continues working
return {
"success": True,
@@ -628,6 +668,56 @@ async def _execute_delegated_agent(args: Dict[str, Any], config: Dict[str, Any])
}
+def _delegation_result_reply(task_id: str, entry: Dict[str, Any]) -> Dict[str, Any]:
+ """Shape a terminal delegation entry as an awaited tool result.
+
+ ``delegation_lifecycle`` tells :func:`execute_tool` that
+ ``run_child_agent`` already owns the child node's terminal broadcast.
+ """
+ completed = entry.get("status") == "completed"
+ return {
+ "success": completed,
+ "status": entry.get("status"), # "completed" | "error" | "not_found"
+ "task_id": task_id,
+ "agent_name": entry.get("agent_name"),
+ "result": entry.get("result"),
+ "error": entry.get("error"),
+ "delegation_lifecycle": True,
+ }
+
+
+async def wait_for_delegation(
+ task_id: str,
+ *,
+ timeout: float,
+ task: Optional[asyncio.Task] = None,
+ database=None,
+) -> Optional[Dict[str, Any]]:
+ """Block until a delegated task reaches a terminal state, or timeout.
+
+ Returns the ``_delegation_results``-shaped entry on completion/error,
+ or ``None`` when the child is still running after ``timeout``. The
+ child is NEVER cancelled — ``asyncio.shield`` cancels only the waiter,
+ so ``_delegation_results`` caching / taskTrigger events still fire
+ later and the result stays retrievable via ``check_delegated_tasks``.
+ """
+ live = task if task is not None else _delegated_tasks.get(task_id)
+ if live is not None:
+ try:
+ await asyncio.wait_for(asyncio.shield(live), timeout=timeout)
+ except asyncio.TimeoutError:
+ return None # still working in the background
+ except Exception: # noqa: BLE001 — run_child_agent caches its own errors
+ pass
+ cached = _delegation_results.get(task_id)
+ if cached:
+ return cached
+ # Cleanup race / cross-restart: 3-layer lookup (live -> cache -> DB).
+ status = await get_delegated_task_status(task_ids=[task_id], database=database)
+ entry = status["tasks"][0]
+ return None if entry.get("status") == "running" else entry
+
+
async def get_delegated_task_status(task_ids: Optional[List[str]] = None, database=None) -> Dict[str, Any]:
"""Check status and retrieve results of delegated tasks.
diff --git a/server/services/plugin/tool.py b/server/services/plugin/tool.py
index 29b8cec2..fd848387 100644
--- a/server/services/plugin/tool.py
+++ b/server/services/plugin/tool.py
@@ -9,9 +9,39 @@ from __future__ import annotations
from typing import Any, ClassVar, Dict, Optional
+from core.logging import get_logger
from services.plugin.base import BaseNode
from services.plugin.scaling import TaskQueue
+logger = get_logger(__name__)
+
+
+def inline_schema_refs(schema: Dict[str, Any]) -> Dict[str, Any]:
+ """Inline $defs/$ref indirection for LLM function-calling surfaces.
+
+ Function-calling APIs reject schema indirection, but a Params model
+ with nested BaseModel or Enum fields emits ``$defs`` + ``$ref`` under
+ Pydantic v2 — stripping ``$defs`` alone would leave dangling refs.
+ Circular refs degrade to a permissive ``{}`` at the cycle point
+ (``dereference_refs`` contract); a dereference failure (out-of-document
+ ref, malformed schema) falls back to a permissive object schema.
+ """
+ # Lazy import: langchain_core is heavyweight and this module loads at
+ # plugin-registration time (cold-start rule).
+ from langchain_core.utils.json_schema import dereference_refs
+
+ try:
+ inlined = dereference_refs(schema)
+ except Exception as e: # noqa: BLE001 — KeyError/ValueError from _retrieve_ref
+ logger.warning(
+ "inline_schema_refs: dereference failed (%s) — falling back to permissive schema",
+ e,
+ )
+ return {"type": "object", "properties": {}}
+ inlined.pop("$defs", None)
+ inlined.pop("definitions", None)
+ return inlined
+
class ToolNode(BaseNode, abstract=True):
"""Base class for AI-Agent tool nodes (calculatorTool, currentTimeTool)."""
@@ -30,10 +60,9 @@ class ToolNode(BaseNode, abstract=True):
def as_tool_schema(cls) -> Dict[str, Any]:
"""LLM-visible schema: ``{name, description, parameters}`` where
``parameters`` is the Pydantic JSON schema of :class:`Params`."""
- schema = cls.Params.model_json_schema()
- # Strip $defs / definitions — LLM function-calling doesn't cope.
- schema.pop("$defs", None)
- schema.pop("definitions", None)
+ # Inline $defs / $ref — LLM function-calling doesn't cope with
+ # indirection, and a bare strip would leave dangling $ref.
+ schema = inline_schema_refs(cls.Params.model_json_schema())
return {
"name": cls.type,
"description": cls.description or cls.display_name or cls.type,
diff --git a/server/tests/nodes/test_ai_tools.py b/server/tests/nodes/test_ai_tools.py
index 50cbeacd..dd7d1a15 100644
--- a/server/tests/nodes/test_ai_tools.py
+++ b/server/tests/nodes/test_ai_tools.py
@@ -525,3 +525,183 @@ class TestWriteTodos:
assert result["success"] is True
assert get_todo_service().get("wf") == [{"content": "x", "status": "pending"}]
+
+
+# ============================================================================
+# delegation_wait_seconds (blocking delegation for bridged cloud agents)
+# ============================================================================
+
+
+class TestDelegationWait:
+ """Opt-in blocking wait in _execute_delegated_agent.
+
+ ``config["delegation_wait_seconds"] > 0`` awaits the child inline
+ (asyncio.shield -- the child is never cancelled) and returns its real
+ result; timeout falls back to the task_id + check_delegated_tasks
+ polling contract. Absent key = fire-and-forget (native agent loop).
+ """
+
+ @pytest.fixture(autouse=True)
+ def _reset_registry(self):
+ """Snapshot + restore all delegation module state around each test."""
+ from services.handlers import tools as tools_mod
+
+ backups = {
+ name: dict(getattr(tools_mod, name))
+ for name in (
+ "_delegated_tasks",
+ "_delegation_results",
+ "_active_delegations",
+ "_active_delegated_nodes",
+ )
+ }
+ for name in backups:
+ getattr(tools_mod, name).clear()
+ try:
+ yield tools_mod
+ finally:
+ for name, backup in backups.items():
+ getattr(tools_mod, name).clear()
+ getattr(tools_mod, name).update(backup)
+
+ def _make_env(self, child_result=None, child_delay=0.0):
+ """Fake plugin + mocked services for _execute_delegated_agent."""
+ calls = {"execute": 0}
+ result = child_result if child_result is not None else {"success": True, "result": {"response": "hi"}}
+
+ class FakeAgentPlugin:
+ async def execute(self, node_id, params, ctx):
+ calls["execute"] += 1
+ if child_delay:
+ import asyncio
+
+ await asyncio.sleep(child_delay)
+ return result
+
+ database = MagicMock()
+ database.get_node_parameters = AsyncMock(return_value={"api_key": "k", "model": "m", "label": "child_agent"})
+ database.save_node_output = AsyncMock(return_value=None)
+ database.get_node_output_by_session = AsyncMock(return_value=None)
+
+ broadcaster = MagicMock()
+ broadcaster.update_node_status = AsyncMock(return_value=None)
+
+ config = {
+ "node_id": "child-1",
+ "node_type": "aiAgent",
+ "workflow_id": "wf-1",
+ "parent_node_id": "vertex-1",
+ "parameters": {"label": "child_agent"},
+ "ai_service": MagicMock(),
+ "database": database,
+ "nodes": [],
+ "edges": [],
+ }
+ return FakeAgentPlugin, broadcaster, config, calls
+
+ def _patches(self, plugin_cls, broadcaster):
+ return (
+ patch("services.status_broadcaster.get_status_broadcaster", return_value=broadcaster),
+ patch("services.node_registry.get_node_class", return_value=plugin_cls),
+ patch("nodes.agent._events.broadcast_agent_task_completed", new=AsyncMock()),
+ patch("nodes.agent._events.broadcast_agent_task_failed", new=AsyncMock()),
+ )
+
+ async def test_wait_returns_child_result_inline(self, _reset_registry):
+ tools_mod = _reset_registry
+ plugin_cls, broadcaster, config, calls = self._make_env()
+ config["delegation_wait_seconds"] = 5
+
+ p1, p2, p3, p4 = self._patches(plugin_cls, broadcaster)
+ with p1, p2, p3, p4:
+ result = await tools_mod._execute_delegated_agent({"task": "do it"}, config)
+
+ assert result["status"] == "completed"
+ assert result["success"] is True
+ assert result["result"] == "hi"
+ assert result["delegation_lifecycle"] is True
+ assert calls["execute"] == 1
+ assert tools_mod._active_delegations == {}
+
+ async def test_wait_timeout_keeps_child_running(self, _reset_registry):
+ import asyncio
+
+ tools_mod = _reset_registry
+ plugin_cls, broadcaster, config, calls = self._make_env(child_delay=0.3)
+ config["delegation_wait_seconds"] = 0.05
+
+ p1, p2, p3, p4 = self._patches(plugin_cls, broadcaster)
+ with p1, p2, p3, p4:
+ result = await tools_mod._execute_delegated_agent({"task": "slow"}, config)
+ task_id = result["task_id"]
+
+ assert result["status"] == "delegated"
+ assert "STILL WORKING" in result["message"]
+ assert "check_delegated_tasks" in result["message"]
+
+ # The shield must NOT have cancelled the child: it completes
+ # in the background and caches its result.
+ await asyncio.sleep(0.5)
+
+ assert tools_mod._delegation_results[task_id]["status"] == "completed"
+ assert tools_mod._delegation_results[task_id]["result"] == "hi"
+
+ async def test_already_delegated_with_wait_awaits_existing(self, _reset_registry):
+ tools_mod = _reset_registry
+ plugin_cls, broadcaster, config, calls = self._make_env(child_delay=0.2)
+ config["delegation_wait_seconds"] = 0.05
+
+ p1, p2, p3, p4 = self._patches(plugin_cls, broadcaster)
+ with p1, p2, p3, p4:
+ first = await tools_mod._execute_delegated_agent({"task": "same"}, config)
+ assert first["status"] == "delegated"
+
+ # Identical re-call while the child is in flight: md5 dedupe
+ # converts it into "await the existing task".
+ config["delegation_wait_seconds"] = 5
+ second = await tools_mod._execute_delegated_agent({"task": "same"}, config)
+
+ assert second["status"] == "completed"
+ assert second["task_id"] == first["task_id"]
+ assert second["result"] == "hi"
+ assert calls["execute"] == 1
+
+ async def test_no_wait_preserves_fire_and_forget(self, _reset_registry):
+ import asyncio
+
+ tools_mod = _reset_registry
+ plugin_cls, broadcaster, config, calls = self._make_env()
+ # No delegation_wait_seconds key at all.
+
+ p1, p2, p3, p4 = self._patches(plugin_cls, broadcaster)
+ with p1, p2, p3, p4:
+ result = await tools_mod._execute_delegated_agent({"task": "bg"}, config)
+
+ assert result["status"] == "delegated"
+ assert "SUCCESS: Task delegated" in result["message"]
+ assert "delegation_lifecycle" not in result
+ # Child had no chance to run before the ack returned.
+ assert calls["execute"] == 0
+
+ await asyncio.sleep(0.05) # let the background task finish
+
+ assert calls["execute"] == 1
+
+ async def test_execute_tool_no_success_broadcast_on_awaited_error(self, _reset_registry):
+ tools_mod = _reset_registry
+ plugin_cls, broadcaster, config, calls = self._make_env(child_result={"success": False, "error": "boom"})
+ config["delegation_wait_seconds"] = 5
+
+ p1, p2, p3, p4 = self._patches(plugin_cls, broadcaster)
+ with p1, p2, p3, p4:
+ result = await tools_mod.execute_tool("delegate_to_ai_agent", {"task": "fail"}, config)
+
+ assert result["status"] == "error"
+ assert result["error"] == "boom"
+ # Marker consumed by execute_tool, not leaked to the LLM payload.
+ assert "delegation_lifecycle" not in result
+ # run_child_agent owns the child's terminal broadcast; execute_tool
+ # must not stomp the error glow with a success broadcast.
+ statuses = [call.args[1] for call in broadcaster.update_node_status.await_args_list]
+ assert "error" in statuses
+ assert "success" not in statuses
diff --git a/server/tests/nodes/test_vertex_agents.py b/server/tests/nodes/test_vertex_agents.py
index 6643bd76..2df58df2 100644
--- a/server/tests/nodes/test_vertex_agents.py
+++ b/server/tests/nodes/test_vertex_agents.py
@@ -319,6 +319,475 @@ class TestVertexManagedAgent:
assert rec.args[1]["result"] == {"answer": 42}
assert rec.args[1]["is_error"] is False
+ async def test_requires_action_awaits_delegated_agent_and_answers_real_result(self, harness):
+ """Sub-agent bridging: the delegate call is dispatched with the
+ blocking-wait contract (delegation_wait_seconds) and the child's
+ REAL answer — not a fire-and-forget task_id ack — is sent back
+ to the cloud agent as the function_result."""
+ agent_id = "vx-1"
+ child_id = "child-1"
+
+ delegate_tool = SimpleNamespace(name="delegate_to_ai_agent", description="d", args_schema=None)
+ check_tool = SimpleNamespace(name="check_delegated_tasks", description="check", args_schema=None)
+
+ async def build_side_effect(tool_info):
+ if tool_info["node_type"] == "_builtin_check_delegated_tasks":
+ return (check_tool, {"node_type": "_builtin_check_delegated_tasks", "node_id": tool_info["node_id"], "parameters": {}})
+ return (delegate_tool, {"node_type": "aiAgent", "node_id": child_id, "parameters": {}, "label": "child"})
+
+ ai_service = MagicMock(name="AIService")
+ ai_service._build_tool_from_node = AsyncMock(side_effect=build_side_effect)
+ execute_tool_mock = AsyncMock(
+ return_value={
+ "success": True,
+ "status": "completed",
+ "task_id": "delegated_x",
+ "agent_name": "child",
+ "result": "child answer",
+ }
+ )
+
+ turns = [
+ _interaction(
+ status="requires_action",
+ interaction_id="ix-1",
+ steps=[_fc_step("delegate_to_ai_agent", "c1", {"task": "do it"})],
+ ),
+ _interaction(status="completed", interaction_id="ix-2"),
+ ]
+ client_patch, create_patch = _patched_interactions(turns)
+ with (
+ patched_container(auth_api_keys={}),
+ patched_broadcaster() as bc,
+ client_patch,
+ create_patch as create_mock,
+ patch("services.plugin.deps.get_ai_service", return_value=ai_service),
+ patch("services.handlers.tools.execute_tool", execute_tool_mock),
+ patch(f"{_NODE_MODULE}._ops.record_tool_output", AsyncMock()),
+ ):
+ _wire_async_broadcasts(bc)
+ result = await harness.execute(
+ "vertex_managed_agent",
+ {"prompt": "delegate", "project_id": "test-proj"},
+ node_id=agent_id,
+ nodes=[
+ _node(agent_id, "vertex_managed_agent"),
+ _node(child_id, "aiAgent"),
+ ],
+ edges=[_edge(child_id, agent_id, "input-tools")],
+ )
+
+ harness.assert_envelope(result, success=True)
+ assert result["result"]["turns"] == 2
+
+ # The bridged dispatch carried the blocking-wait contract.
+ execute_tool_mock.assert_awaited_once()
+ name_arg, args_arg, config_arg = execute_tool_mock.await_args.args
+ assert name_arg == "delegate_to_ai_agent"
+ assert config_arg["delegation_wait_seconds"] == 600
+
+ # The cloud agent got the child's real answer, not a task_id ack.
+ second_kwargs = create_mock.call_args_list[1].kwargs
+ fr = second_kwargs["input"][0]
+ assert fr["type"] == "function_result"
+ assert fr["call_id"] == "c1"
+ assert fr["result"]["result"] == "child answer"
+ assert fr["result"]["status"] == "completed"
+
+ async def test_check_tool_injected_when_agent_connected(self, harness):
+ agent_id = "vx-1"
+ child_id = "child-1"
+
+ delegate_tool = SimpleNamespace(name="delegate_to_ai_agent", description="d", args_schema=None)
+ check_tool = SimpleNamespace(name="check_delegated_tasks", description="check", args_schema=None)
+
+ async def build_side_effect(tool_info):
+ if tool_info["node_type"] == "_builtin_check_delegated_tasks":
+ return (check_tool, {"node_type": "_builtin_check_delegated_tasks", "node_id": tool_info["node_id"], "parameters": {}})
+ return (delegate_tool, {"node_type": "aiAgent", "node_id": child_id, "parameters": {}, "label": "child"})
+
+ ai_service = MagicMock(name="AIService")
+ ai_service._build_tool_from_node = AsyncMock(side_effect=build_side_effect)
+
+ client_patch, create_patch = _patched_interactions([_interaction()])
+ with (
+ patched_container(auth_api_keys={}),
+ patched_broadcaster() as bc,
+ client_patch,
+ create_patch as create_mock,
+ patch("services.plugin.deps.get_ai_service", return_value=ai_service),
+ ):
+ _wire_async_broadcasts(bc)
+ result = await harness.execute(
+ "vertex_managed_agent",
+ {"prompt": "p", "project_id": "test-proj"},
+ node_id=agent_id,
+ nodes=[
+ _node(agent_id, "vertex_managed_agent"),
+ _node(child_id, "aiAgent"),
+ ],
+ edges=[_edge(child_id, agent_id, "input-tools")],
+ )
+
+ harness.assert_envelope(result, success=True)
+ kwargs = create_mock.call_args.kwargs
+ declared_names = [t["name"] for t in kwargs["tools"]]
+ assert declared_names == ["delegate_to_ai_agent", "check_delegated_tasks"]
+ # Delegation guidance injected even with no user system_instruction.
+ assert "## Agent Delegation" in kwargs["system_instruction"]
+ assert "delegate_to_ai_agent" in kwargs["system_instruction"]
+
+ async def test_check_tool_not_injected_for_plain_tools(self, harness):
+ agent_id = "vx-1"
+ tool_id = "tool-1"
+
+ fake_tool = SimpleNamespace(name="fake_tool", description="d", args_schema=None)
+ ai_service = MagicMock(name="AIService")
+ ai_service._build_tool_from_node = AsyncMock(
+ return_value=(
+ fake_tool,
+ {"node_type": "duckduckgoSearch", "node_id": tool_id, "parameters": {}, "label": "ddg"},
+ )
+ )
+
+ client_patch, create_patch = _patched_interactions([_interaction()])
+ with (
+ patched_container(auth_api_keys={}),
+ patched_broadcaster() as bc,
+ client_patch,
+ create_patch as create_mock,
+ patch("services.plugin.deps.get_ai_service", return_value=ai_service),
+ ):
+ _wire_async_broadcasts(bc)
+ result = await harness.execute(
+ "vertex_managed_agent",
+ {"prompt": "p", "project_id": "test-proj"},
+ node_id=agent_id,
+ nodes=[_node(agent_id, "vertex_managed_agent"), _node(tool_id, "duckduckgoSearch")],
+ edges=[_edge(tool_id, agent_id, "input-tools")],
+ )
+
+ harness.assert_envelope(result, success=True)
+ kwargs = create_mock.call_args.kwargs
+ assert [t["name"] for t in kwargs["tools"]] == ["fake_tool"]
+ assert "system_instruction" not in kwargs
+
+ async def test_delegation_wait_clamped_to_activity_budget(self, harness):
+ """A delegation wait must never blow the Temporal activity
+ deadline: with ~29 min already elapsed, the wait clamps to 0 and
+ the dispatch falls back to the fire-and-forget contract."""
+ agent_id = "vx-1"
+ child_id = "child-1"
+
+ delegate_tool = SimpleNamespace(name="delegate_to_ai_agent", description="d", args_schema=None)
+ check_tool = SimpleNamespace(name="check_delegated_tasks", description="check", args_schema=None)
+
+ async def build_side_effect(tool_info):
+ if tool_info["node_type"] == "_builtin_check_delegated_tasks":
+ return (check_tool, {"node_type": "_builtin_check_delegated_tasks", "node_id": tool_info["node_id"], "parameters": {}})
+ return (delegate_tool, {"node_type": "aiAgent", "node_id": child_id, "parameters": {}, "label": "child"})
+
+ ai_service = MagicMock(name="AIService")
+ ai_service._build_tool_from_node = AsyncMock(side_effect=build_side_effect)
+ execute_tool_mock = AsyncMock(return_value={"success": True, "status": "delegated", "task_id": "t1"})
+
+ turns = [
+ _interaction(
+ status="requires_action",
+ interaction_id="ix-1",
+ steps=[_fc_step("delegate_to_ai_agent", "c1", {"task": "t"})],
+ ),
+ _interaction(status="completed", interaction_id="ix-2"),
+ ]
+ client_patch, create_patch = _patched_interactions(turns)
+
+ # First time.time() call is start_time (0.0); every later call
+ # reports 1740s elapsed -> remaining budget is negative.
+ ticks = iter([0.0])
+ fake_time = MagicMock()
+ fake_time.time = lambda: next(ticks, 1740.0)
+
+ with (
+ patched_container(auth_api_keys={}),
+ patched_broadcaster() as bc,
+ client_patch,
+ create_patch,
+ patch("services.plugin.deps.get_ai_service", return_value=ai_service),
+ patch("services.handlers.tools.execute_tool", execute_tool_mock),
+ patch(f"{_NODE_MODULE}._ops.record_tool_output", AsyncMock()),
+ patch(f"{_NODE_MODULE}.time", fake_time),
+ ):
+ _wire_async_broadcasts(bc)
+ result = await harness.execute(
+ "vertex_managed_agent",
+ {"prompt": "p", "project_id": "test-proj"},
+ node_id=agent_id,
+ nodes=[_node(agent_id, "vertex_managed_agent"), _node(child_id, "aiAgent")],
+ edges=[_edge(child_id, agent_id, "input-tools")],
+ )
+
+ harness.assert_envelope(result, success=True)
+ execute_tool_mock.assert_awaited_once()
+ config_arg = execute_tool_mock.await_args.args[2]
+ assert "delegation_wait_seconds" not in config_arg
+
+ async def test_declared_schema_inlines_nested_refs(self, harness):
+ """Nested BaseModel / Enum Params fields emit $defs + $ref under
+ Pydantic v2 — the declaration must inline them, not strip $defs
+ and leave dangling refs."""
+ import json
+ from enum import Enum
+
+ from pydantic import BaseModel as PydanticBase
+
+ class Color(str, Enum):
+ RED = "red"
+ BLUE = "blue"
+
+ class Inner(PydanticBase):
+ name: str
+
+ class NestedToolParams(PydanticBase):
+ nested: Inner
+ color: Color = Color.RED
+
+ agent_id = "vx-1"
+ tool_id = "tool-1"
+ fake_tool = SimpleNamespace(name="fake_tool", description="d", args_schema=NestedToolParams)
+ ai_service = MagicMock(name="AIService")
+ ai_service._build_tool_from_node = AsyncMock(
+ return_value=(
+ fake_tool,
+ {"node_type": "duckduckgoSearch", "node_id": tool_id, "parameters": {}, "label": "ddg"},
+ )
+ )
+
+ client_patch, create_patch = _patched_interactions([_interaction()])
+ with (
+ patched_container(auth_api_keys={}),
+ patched_broadcaster() as bc,
+ client_patch,
+ create_patch as create_mock,
+ patch("services.plugin.deps.get_ai_service", return_value=ai_service),
+ ):
+ _wire_async_broadcasts(bc)
+ result = await harness.execute(
+ "vertex_managed_agent",
+ {"prompt": "p", "project_id": "test-proj"},
+ node_id=agent_id,
+ nodes=[_node(agent_id, "vertex_managed_agent"), _node(tool_id, "duckduckgoSearch")],
+ edges=[_edge(tool_id, agent_id, "input-tools")],
+ )
+
+ harness.assert_envelope(result, success=True)
+ parameters = create_mock.call_args.kwargs["tools"][0]["parameters"]
+ dumped = json.dumps(parameters)
+ assert "$defs" not in parameters
+ assert '"$ref"' not in dumped
+ assert parameters["properties"]["nested"]["type"] == "object"
+ assert "red" in json.dumps(parameters["properties"]["color"])
+
+ async def test_recursive_params_degrade_safely(self, harness):
+ import json
+ from typing import List as TypingList
+
+ from pydantic import BaseModel as PydanticBase
+
+ class TreeParams(PydanticBase):
+ label: str
+ children: TypingList["TreeParams"] = []
+
+ agent_id = "vx-1"
+ tool_id = "tool-1"
+ fake_tool = SimpleNamespace(name="fake_tool", description="d", args_schema=TreeParams)
+ ai_service = MagicMock(name="AIService")
+ ai_service._build_tool_from_node = AsyncMock(
+ return_value=(
+ fake_tool,
+ {"node_type": "duckduckgoSearch", "node_id": tool_id, "parameters": {}, "label": "ddg"},
+ )
+ )
+
+ client_patch, create_patch = _patched_interactions([_interaction()])
+ with (
+ patched_container(auth_api_keys={}),
+ patched_broadcaster() as bc,
+ client_patch,
+ create_patch as create_mock,
+ patch("services.plugin.deps.get_ai_service", return_value=ai_service),
+ ):
+ _wire_async_broadcasts(bc)
+ result = await harness.execute(
+ "vertex_managed_agent",
+ {"prompt": "p", "project_id": "test-proj"},
+ node_id=agent_id,
+ nodes=[_node(agent_id, "vertex_managed_agent"), _node(tool_id, "duckduckgoSearch")],
+ edges=[_edge(tool_id, agent_id, "input-tools")],
+ )
+
+ harness.assert_envelope(result, success=True)
+ parameters = create_mock.call_args.kwargs["tools"][0]["parameters"]
+ dumped = json.dumps(parameters) # must stay JSON-serializable
+ assert "$defs" not in parameters
+ assert '"$ref"' not in dumped
+
+ async def test_minted_cloud_tool_nodes_not_redeclared(self, harness):
+ """Display-only vertexCloudTool nodes (minted with a persisted
+ input-tools edge) must not be re-declared as junk function tools
+ on later runs."""
+ agent_id = "vx-1"
+ tool_id = "tool-1"
+ minted_id = "vertexCloudTool-1-aaa"
+
+ fake_tool = SimpleNamespace(name="fake_tool", description="d", args_schema=None)
+ ai_service = MagicMock(name="AIService")
+ ai_service._build_tool_from_node = AsyncMock(
+ return_value=(
+ fake_tool,
+ {"node_type": "duckduckgoSearch", "node_id": tool_id, "parameters": {}, "label": "ddg"},
+ )
+ )
+
+ client_patch, create_patch = _patched_interactions([_interaction()])
+ with (
+ patched_container(auth_api_keys={}),
+ patched_broadcaster() as bc,
+ client_patch,
+ create_patch as create_mock,
+ patch("services.plugin.deps.get_ai_service", return_value=ai_service),
+ ):
+ _wire_async_broadcasts(bc)
+ result = await harness.execute(
+ "vertex_managed_agent",
+ {"prompt": "p", "project_id": "test-proj"},
+ node_id=agent_id,
+ nodes=[
+ _node(agent_id, "vertex_managed_agent"),
+ _node(tool_id, "duckduckgoSearch"),
+ _node(minted_id, "vertexCloudTool", label="run_command"),
+ ],
+ edges=[
+ _edge(tool_id, agent_id, "input-tools"),
+ _edge(minted_id, agent_id, "input-tools"),
+ ],
+ )
+
+ harness.assert_envelope(result, success=True)
+ # Only the real tool was built + declared; the display node was
+ # gated out before the DB round-trip.
+ ai_service._build_tool_from_node.assert_awaited_once()
+ assert [t["name"] for t in create_mock.call_args.kwargs["tools"]] == ["fake_tool"]
+
+ async def test_duplicate_tool_names_deduped(self, harness):
+ """Two nodes resolving to the same tool name get deterministic
+ _2 suffixes; the suffixed name dispatches with the SECOND node's
+ config."""
+ agent_id = "vx-1"
+
+ def make_tool(node_id):
+ return (
+ SimpleNamespace(name="fake_tool", description="d", args_schema=None),
+ {"node_type": "duckduckgoSearch", "node_id": node_id, "parameters": {}, "label": node_id},
+ )
+
+ ai_service = MagicMock(name="AIService")
+ ai_service._build_tool_from_node = AsyncMock(side_effect=[make_tool("tool-1"), make_tool("tool-2")])
+ execute_tool_mock = AsyncMock(return_value={"answer": 1})
+
+ turns = [
+ _interaction(
+ status="requires_action",
+ interaction_id="ix-1",
+ steps=[_fc_step("fake_tool_2", "c1", {"query": "x"})],
+ ),
+ _interaction(status="completed", interaction_id="ix-2"),
+ ]
+ client_patch, create_patch = _patched_interactions(turns)
+ with (
+ patched_container(auth_api_keys={}),
+ patched_broadcaster() as bc,
+ client_patch,
+ create_patch as create_mock,
+ patch("services.plugin.deps.get_ai_service", return_value=ai_service),
+ patch("services.handlers.tools.execute_tool", execute_tool_mock),
+ patch(f"{_NODE_MODULE}._ops.record_tool_output", AsyncMock()),
+ ):
+ _wire_async_broadcasts(bc)
+ result = await harness.execute(
+ "vertex_managed_agent",
+ {"prompt": "p", "project_id": "test-proj"},
+ node_id=agent_id,
+ nodes=[
+ _node(agent_id, "vertex_managed_agent"),
+ _node("tool-1", "duckduckgoSearch"),
+ _node("tool-2", "duckduckgoSearch"),
+ ],
+ edges=[
+ _edge("tool-1", agent_id, "input-tools"),
+ _edge("tool-2", agent_id, "input-tools"),
+ ],
+ )
+
+ harness.assert_envelope(result, success=True)
+ declared_names = [t["name"] for t in create_mock.call_args_list[0].kwargs["tools"]]
+ assert declared_names == ["fake_tool", "fake_tool_2"]
+ execute_tool_mock.assert_awaited_once()
+ config_arg = execute_tool_mock.await_args.args[2]
+ assert config_arg["node_id"] == "tool-2"
+
+ async def test_unanswerable_requires_action_surfaces_warning(self, harness):
+ """When requires_action only carries calls we never declared, the
+ loop stops AND says why — in the log, the output payload, and the
+ node-status details."""
+ agent_id = "vx-1"
+ tool_id = "tool-1"
+
+ fake_tool = SimpleNamespace(name="fake_tool", description="d", args_schema=None)
+ ai_service = MagicMock(name="AIService")
+ ai_service._build_tool_from_node = AsyncMock(
+ return_value=(
+ fake_tool,
+ {"node_type": "duckduckgoSearch", "node_id": tool_id, "parameters": {}, "label": "ddg"},
+ )
+ )
+
+ turns = [
+ _interaction(
+ status="requires_action",
+ interaction_id="ix-1",
+ steps=[
+ _fc_step("provision_sandbox", "c0"),
+ _fc_step("mystery_fn", "c1"),
+ ],
+ ),
+ ]
+ client_patch, create_patch = _patched_interactions(turns)
+ with (
+ patched_container(auth_api_keys={}),
+ patched_broadcaster() as bc,
+ client_patch,
+ create_patch as create_mock,
+ patch("services.plugin.deps.get_ai_service", return_value=ai_service),
+ ):
+ _wire_async_broadcasts(bc)
+ result = await harness.execute(
+ "vertex_managed_agent",
+ {"prompt": "p", "project_id": "test-proj"},
+ node_id=agent_id,
+ nodes=[_node(agent_id, "vertex_managed_agent"), _node(tool_id, "duckduckgoSearch")],
+ edges=[_edge(tool_id, agent_id, "input-tools")],
+ )
+
+ harness.assert_envelope(result, success=True)
+ assert create_mock.await_count == 1 # loop stopped, no follow-up turn
+ payload = result["result"]
+ assert payload["status"] == "requires_action"
+ warning = payload["warnings"][0]
+ assert "mystery_fn" in warning
+ assert "fake_tool" in warning
+ # Noise names are excluded from the diagnostic.
+ assert "provision_sandbox" not in warning
+
async def test_stale_chain_wipes_and_retries_fresh(self, harness):
agent_id = "vx-1"
mem_id = "mem-1"
@@ -365,6 +834,114 @@ class TestVertexManagedAgent:
assert calls[1]["environment"] == "remote"
assert result["result"]["interaction_id"] == "ix-fresh"
+ async def test_precondition_failure_wipes_chain_and_retries_fresh(self, harness):
+ """Live-verified wedge: previous_interaction_id pointing at an
+ unresumable interaction 400s with 'Precondition check failed'.
+ Turn 1 must wipe the stored ids and retry fresh instead of
+ failing every subsequent run."""
+ agent_id = "vx-1"
+ mem_id = "mem-1"
+
+ harness.database.get_node_parameters = AsyncMock(
+ return_value={
+ "vertex_interaction_id": "ix-wedged",
+ "vertex_environment_id": "env_wedged",
+ "memory_content": "# Conversation History\n",
+ }
+ )
+ saved: dict = {}
+
+ async def capture_save(node_id, params):
+ saved[node_id] = params
+ return True
+
+ harness.database.save_node_parameters = AsyncMock(side_effect=capture_save)
+
+ calls: list = []
+
+ async def create_side_effect(client, **kwargs):
+ calls.append(kwargs)
+ if len(calls) == 1:
+ raise FakeGenaiError(
+ "Error code: 400 - {'error': {'message': 'Precondition check failed.', 'code': 'invalid_request'}}"
+ )
+ return _interaction(interaction_id="ix-fresh", environment_id="env_fresh")
+
+ client_patch = patch(f"{_NODE_MODULE}.build_genai_client", return_value=MagicMock())
+ create_patch = patch(
+ f"{_NODE_MODULE}.stream_interaction",
+ AsyncMock(side_effect=create_side_effect),
+ )
+ with (
+ patched_container(auth_api_keys={}),
+ patched_broadcaster() as bc,
+ client_patch,
+ create_patch,
+ ):
+ _wire_async_broadcasts(bc)
+ result = await harness.execute(
+ "vertex_managed_agent",
+ {"prompt": "p", "project_id": "test-proj"},
+ node_id=agent_id,
+ nodes=[_node(agent_id, "vertex_managed_agent"), _node(mem_id, "simpleMemory")],
+ edges=[_edge(mem_id, agent_id, "input-memory")],
+ )
+
+ harness.assert_envelope(result, success=True)
+ assert calls[0]["previous_interaction_id"] == "ix-wedged"
+ assert "previous_interaction_id" not in calls[1]
+ assert calls[1]["environment"] == "remote"
+ assert result["result"]["interaction_id"] == "ix-fresh"
+ # The fresh chain got persisted (completed status is resumable).
+ assert saved[mem_id]["vertex_interaction_id"] == "ix-fresh"
+ assert saved[mem_id]["vertex_environment_id"] == "env_fresh"
+
+ async def test_unresumable_final_status_keeps_previous_chain_ids(self, harness):
+ """A run whose final interaction is failed/stuck must NOT persist
+ that id — chaining onto it wedges every later run. Keep the last
+ good pair instead."""
+ agent_id = "vx-1"
+ mem_id = "mem-1"
+
+ harness.database.get_node_parameters = AsyncMock(
+ return_value={
+ "vertex_interaction_id": "ix-good",
+ "vertex_environment_id": "env_good",
+ "memory_content": "# Conversation History\n",
+ }
+ )
+ saved: dict = {}
+
+ async def capture_save(node_id, params):
+ saved[node_id] = params
+ return True
+
+ harness.database.save_node_parameters = AsyncMock(side_effect=capture_save)
+
+ client_patch, create_patch = _patched_interactions(
+ [_interaction(status="failed", interaction_id="ix-bad", environment_id="env_bad")]
+ )
+ with (
+ patched_container(auth_api_keys={}),
+ patched_broadcaster() as bc,
+ client_patch,
+ create_patch,
+ ):
+ _wire_async_broadcasts(bc)
+ result = await harness.execute(
+ "vertex_managed_agent",
+ {"prompt": "p", "project_id": "test-proj"},
+ node_id=agent_id,
+ nodes=[_node(agent_id, "vertex_managed_agent"), _node(mem_id, "simpleMemory")],
+ edges=[_edge(mem_id, agent_id, "input-memory")],
+ )
+
+ harness.assert_envelope(result, success=True)
+ assert result["result"]["status"] == "failed"
+ # The wedging id was NOT persisted; the last good chain survives.
+ assert saved[mem_id]["vertex_interaction_id"] == "ix-good"
+ assert saved[mem_id]["vertex_environment_id"] == "env_good"
+
async def test_cloud_tool_usage_feeds_minting(self, harness):
agent_id = "vx-1"
steps = [
@@ -588,6 +1165,33 @@ class TestStreamInteraction:
assert second.get("stream") is not True
assert second["background"] is True
+ async def test_streaming_rejection_latches_poll_for_the_client(self):
+ """A rejected streaming create (enterprise 400 'Precondition
+ check failed') must not be re-attempted on later turns of the
+ same run: the client is latched to the poll path."""
+ from nodes.agent._vertex import stream_interaction
+
+ final = _interaction(interaction_id="ix-2")
+ client = MagicMock()
+ client.aio.interactions.create = AsyncMock(
+ side_effect=[
+ FakeGenaiError("Error code: 400 - Precondition check failed."),
+ final, # turn 1 poll fallback
+ final, # turn 2 goes straight to poll
+ ]
+ )
+ client.aio.interactions.get = AsyncMock(return_value=final)
+
+ await stream_interaction(client, agent="a", input="p")
+ await stream_interaction(client, agent="a", input="p2")
+
+ creates = client.aio.interactions.create.await_args_list
+ assert len(creates) == 3
+ # Only the very first create attempted streaming.
+ assert creates[0].kwargs.get("stream") is True
+ assert creates[1].kwargs.get("stream") is not True
+ assert creates[2].kwargs.get("stream") is not True
+
# ============================================================================
# ensure_cloud_tool_nodes (minting helper)
@@ -755,6 +1359,15 @@ class TestCloudToolMinting:
class TestVertexAgentAdmin:
+ def test_palette_group_is_agent_not_tool(self):
+ """("tool",) put the admin node in the AI Tools palette and
+ auto-derived a bogus isConfigNode hint — but it has no
+ output-tool handle, so it can never wire into input-tools."""
+ from nodes.agent.vertex_agent_admin import VertexAgentAdminNode
+
+ assert VertexAgentAdminNode.group == ("agent",)
+ assert VertexAgentAdminNode.usable_as_tool is False
+
def _client_with_agents(self):
client = MagicMock()
client.aio.agents.create = AsyncMock(
diff --git a/server/tests/services/test_schema_inline.py b/server/tests/services/test_schema_inline.py
new file mode 100644
index 00000000..4cc4ab2b
--- /dev/null
+++ b/server/tests/services/test_schema_inline.py
@@ -0,0 +1,85 @@
+"""Unit tests for ``inline_schema_refs`` — the LLM function-calling
+schema flattener used by ``ToolNode.as_tool_schema`` and the Vertex
+managed-agent tool bridge.
+
+The contract: declared tool parameter schemas must never contain
+``$defs`` / ``definitions`` / dangling ``$ref`` — nested models and
+Enums inline; pathological schemas degrade to a permissive object
+schema instead of raising.
+"""
+
+from __future__ import annotations
+
+import json
+from enum import Enum
+from typing import List, Optional
+
+from pydantic import BaseModel
+
+from services.plugin.tool import inline_schema_refs
+
+
+class Color(str, Enum):
+ RED = "red"
+ BLUE = "blue"
+
+
+class Inner(BaseModel):
+ name: str
+ weight: float = 1.0
+
+
+class NestedParams(BaseModel):
+ items: List[Inner]
+ color: Color = Color.RED
+
+
+class TreeNode(BaseModel):
+ label: str
+ children: List["TreeNode"] = []
+
+
+class TestInlineSchemaRefs:
+ def test_nested_model_inlined(self):
+ schema = inline_schema_refs(NestedParams.model_json_schema())
+ dumped = json.dumps(schema)
+ assert "$defs" not in schema
+ assert "definitions" not in schema
+ assert '"$ref"' not in dumped
+ item_schema = schema["properties"]["items"]["items"]
+ assert item_schema["type"] == "object"
+ assert set(item_schema["properties"]) == {"name", "weight"}
+
+ def test_str_enum_inlined(self):
+ schema = inline_schema_refs(NestedParams.model_json_schema())
+ color = schema["properties"]["color"]
+ # Pydantic may wrap defaulted enum refs in allOf; dereference_refs
+ # inlines the target either way — assert the enum values survive
+ # somewhere inside the property with no $ref indirection.
+ dumped = json.dumps(color)
+ assert '"$ref"' not in dumped
+ assert "red" in dumped and "blue" in dumped
+
+ def test_recursive_model_degrades_safely(self):
+ schema = inline_schema_refs(TreeNode.model_json_schema())
+ dumped = json.dumps(schema) # must stay JSON-serializable
+ assert "$defs" not in schema
+ assert '"$ref"' not in dumped
+
+ def test_flat_schema_passthrough(self):
+ class Flat(BaseModel):
+ query: str
+ limit: int = 5
+
+ original = Flat.model_json_schema()
+ schema = inline_schema_refs(original)
+ assert schema["properties"] == original["properties"]
+ assert schema["required"] == original["required"]
+
+ def test_broken_ref_falls_back_permissive(self):
+ broken = {
+ "type": "object",
+ "properties": {"x": {"$ref": "#/$defs/DoesNotExist"}},
+ }
+ schema = inline_schema_refs(broken)
+ assert schema == {"type": "object", "properties": {}}
diff --git a/server/tests/test_plugin_contract.py b/server/tests/test_plugin_contract.py
index 976bf35e..5b7bc1d0 100644
--- a/server/tests/test_plugin_contract.py
+++ b/server/tests/test_plugin_contract.py
@@ -172,6 +172,8 @@ class TestToolSchemaGeneration:
"""ToolNode.Params must produce LLM-compatible JSON schema."""
def test_tool_schema_has_no_refs(self):
+ import json
+
from services.plugin.tool import ToolNode
for cls in _all_plugin_classes():
@@ -181,6 +183,10 @@ class TestToolSchemaGeneration:
params = schema["parameters"]
assert "$defs" not in params, f"{cls.__qualname__} tool schema contains $defs — LLM " "function-calling rejects $ref"
assert "definitions" not in params
+ # A stripped $defs with a surviving $ref is worse than $defs
+ # itself — the declaration points at a definition that no
+ # longer exists (nested BaseModel / Enum Params fields).
+ assert '"$ref"' not in json.dumps(params), f"{cls.__qualname__} tool schema contains a dangling $ref — " "nested refs must be inlined, not stripped"
assert schema.get("name") == cls.type