Self-improving AI Employees for everyone
Python ai-agentsai-employeeai-employeesassistantautonomous-agentsco-employeescompanygrok-botinstinctloop-agentsmuseown-your-datapersonalself-hosted
Live cloud-tool visibility for the Vertex managed agent (SSE streaming)
trohitgc0a4665d3ce7
diff --git a/server/nodes/agent/_vertex.py b/server/nodes/agent/_vertex.pyindex 51f6a3d2..0d1614e7 100644--- a/server/nodes/agent/_vertex.py+++ b/server/nodes/agent/_vertex.py@@ -148,3 +148,67 @@ async def create_interaction_and_wait(await on_poll()interaction = await client.aio.interactions.get(interaction.id)return interaction+++async def stream_interaction(+ client: Any,+ *,+ on_event: Optional[Callable[[Any], Awaitable[None]]] = None,+ poll_interval: float = _POLL_INTERVAL_SECONDS,+ **kwargs: Any,+) -> Any:+ """Create a streamed background interaction; return the FULL resource.++ The SSE stream (``create(stream=True, background=True)``) exists purely+ for LIVE visibility — each event is handed to ``on_event`` best-effort+ (an exception there is logged, never fatal to the turn). The+ ``interaction.completed`` event carries only a PARTIAL interaction+ (no ``output_text`` / ``environment_id``; ``steps`` optional), so after+ the stream closes the authoritative resource is fetched with a+ non-stream ``interactions.get`` and polled to a terminal status as a+ safety net.++ If the streaming create itself is rejected by the surface, falls back+ to the plain background+poll path so the node still works.+ """+ try:+ stream = await client.aio.interactions.create(+ background=True, stream=True, **kwargs+ )+ except Exception as exc: # noqa: BLE001 — fall back, then error-map there+ if not is_genai_error(exc):+ raise+ logger.warning(+ "[Vertex] streaming create rejected (%s) — falling back to poll",+ str(exc).replace("\n", " ")[:200],+ )+ return await create_interaction_and_wait(+ client, poll_interval=poll_interval, **kwargs+ )++ interaction_id: Optional[str] = None+ async for event in stream:+ if interaction_id is None:+ inner = getattr(event, "interaction", None)+ candidate = getattr(inner, "id", None) or getattr(+ event, "interaction_id", None+ )+ if candidate:+ interaction_id = candidate+ if on_event is not None:+ try:+ await on_event(event)+ except Exception: # noqa: BLE001 — visibility must never kill the turn+ logger.exception("[Vertex] on_event handler failed (ignored)")++ if interaction_id is None:+ raise NodeUserError(+ "Vertex managed agent stream ended without an interaction id — "+ "no interaction.created event was received."+ )++ interaction = await client.aio.interactions.get(interaction_id)+ while getattr(interaction, "status", None) not in TERMINAL_STATUSES:+ await asyncio.sleep(poll_interval)+ interaction = await client.aio.interactions.get(interaction_id)+ return interactiondiff --git a/server/nodes/agent/vertex_managed_agent/__init__.py b/server/nodes/agent/vertex_managed_agent/__init__.pyindex 6a3ee864..dbe28aeb 100644--- a/server/nodes/agent/vertex_managed_agent/__init__.py+++ b/server/nodes/agent/vertex_managed_agent/__init__.py@@ -13,11 +13,12 @@ Interactions API and bridges the MachinaOS canvas into it:node: ``vertex_interaction_id`` / ``vertex_environment_id`` are storedin its params and the turn is appended to ``memory_content``.- Cloud-side tool usage (sandbox commands, google_search, ...) is- surfaced post-turn as dynamic ``vertexCloudTool`` canvas nodes via the- workflow-ops protocol (agentBuilder pattern) — see ``_ops.py``.--No SSE streaming in v1: the enterprise surface requires background-interactions, so each turn is create(background=True) + poll.+ surfaced LIVE as dynamic ``vertexCloudTool`` canvas nodes via the+ workflow-ops protocol (agentBuilder pattern) — see ``_ops.py``: each+ turn streams SSE step events (``stream_interaction``), minting and+ glowing nodes while the cloud agent is still working, with a post-turn+ sweep as catch-all. The stream is visibility-only; the authoritative+ result always comes from the final non-stream ``interactions.get``."""from __future__ import annotations@@ -38,10 +39,10 @@ from .._vertex import (DEFAULT_LOCATION,DEFAULT_MANAGED_AGENT,build_genai_client,- create_interaction_and_wait,is_expired_environment_error,raise_as_user_error,resolve_api_key_from_context,+ stream_interaction,)logger = get_logger(__name__)@@ -219,6 +220,20 @@ class VertexManagedAgentNode(ActionNode):await phase("building_tools")declared_tools, tool_configs = await self._build_function_tools(tool_data)+ # ---- live visibility state (shared with the post-turn sweep)+ live_nodes: Dict[str, str] = {} # cloud_tool_key -> canvas node id+ open_calls: Dict[str, str] = {} # step id -> canvas node id+ on_event = None+ if params.visualize_cloud_tools:+ on_event = self._make_live_handler(+ broadcaster=broadcaster,+ workflow_id=workflow_id,+ agent_node_id=node_id,+ declared_names=frozenset(tool_configs),+ live_nodes=live_nodes,+ open_calls=open_calls,+ )+# ---- memory bridge: interaction/environment chain idsprev_interaction_id: Optional[str] = Noneenvironment: Any = "remote"@@ -240,7 +255,9 @@ class VertexManagedAgentNode(ActionNode):async def run_turn(**kw: Any) -> Any:try:- return await create_interaction_and_wait(client, **create_kwargs, **kw)+ return await stream_interaction(+ client, on_event=on_event, **create_kwargs, **kw+ )except Exception as exc: # noqa: BLE001 — mapped belowraise_as_user_error(exc, what="Vertex managed agent interaction")@@ -303,17 +320,36 @@ class VertexManagedAgentNode(ActionNode):environment_id = getattr(interaction, "environment_id", None)usage = self._usage_dict(interaction)- # ---- cloud tool visualization (post-turn, from steps)- cloud_tools = self._collect_cloud_tool_usage(interaction, tool_configs)- if cloud_tools and params.visualize_cloud_tools and workflow_id:- from ._ops import ensure_cloud_tool_nodes+ # ---- cloud tool visualization: post-turn catch-all sweep.+ # Live streaming already minted+pulsed most usage; this covers+ # steps the stream missed and closes any still-glowing calls.+ final_used = self._collect_cloud_tool_usage(interaction, tool_configs)+ cloud_tools = {+ **{key: self._label_for_key(key) for key in live_nodes},+ **final_used,+ }+ if params.visualize_cloud_tools and workflow_id:+ from ._ops import ensure_cloud_tool_nodes, pulse_nodetry:- await ensure_cloud_tool_nodes(- workflow_id=workflow_id,- agent_node_id=node_id,- used=cloud_tools,- )+ sweep = {+ key: label+ for key, label in final_used.items()+ if key not in live_nodes+ }+ if sweep:+ resolved = await ensure_cloud_tool_nodes(+ workflow_id=workflow_id,+ agent_node_id=node_id,+ used=sweep,+ )+ for swept_id in resolved.values():+ await pulse_node(swept_id, "executing", workflow_id=workflow_id)+ await pulse_node(swept_id, "success", workflow_id=workflow_id)+ # Calls whose result step never streamed: don't leave them glowing.+ for dangling_id in set(open_calls.values()):+ await pulse_node(dangling_id, "success", workflow_id=workflow_id)+ open_calls.clear()except Exception: # noqa: BLE001 — visualization is best-effortlogger.exception("[Vertex Agent] cloud-tool minting failed")@@ -528,6 +564,113 @@ class VertexManagedAgentNode(ActionNode):used[f"fn:{name}"] = namereturn used+ @staticmethod+ def _label_for_key(key: str) -> str:+ """Recover the display label from a cloud_tool_key."""+ if key.startswith("type:"):+ return _CLOUD_STEP_LABELS.get(key[5:], key[5:])+ if key.startswith("fn:"):+ return key[3:]+ return key++ def _make_live_handler(+ self,+ *,+ broadcaster: Any,+ workflow_id: Optional[str],+ agent_node_id: str,+ declared_names: frozenset,+ live_nodes: Dict[str, str],+ open_calls: Dict[str, str],+ ):+ """Build the per-event SSE callback for live tool visibility.++ Mints a vertexCloudTool node the FIRST time each cloud-side tool+ appears (memoized in ``live_nodes`` — one DB round-trip per key+ per run), pulses it ``executing`` on the call step and ``success``+ on the matching result step (``call_id`` join via ``open_calls``).+ Declared local tools only get an agent phase broadcast here —+ their real execution still happens at requires_action, where+ ``execute_tool`` owns their canvas animation.++ Exceptions are swallowed by ``stream_interaction`` — visibility+ must never kill the turn.+ """+ from ._ops import ensure_cloud_tool_nodes, pulse_node++ async def phase_tool(label: str) -> None:+ await broadcaster.update_node_status(+ agent_node_id,+ "executing",+ {"phase": "executing_tool", "agent_type": "vertex_managed", "tool_name": label},+ workflow_id=workflow_id,+ )++ async def on_event(event: Any) -> None:+ event_type = getattr(event, "event_type", "") or ""++ if event_type == "interaction.status_update":+ if getattr(event, "status", None) == "requires_action":+ await phase_tool("local tools")+ return+ if event_type == "error":+ error = getattr(event, "error", None)+ logger.warning(+ "[Vertex Agent] stream error event: %s",+ getattr(error, "message", None) or error,+ )+ return+ if event_type != "step.start":+ return++ step = getattr(event, "step", None)+ if step is None:+ return+ step_type = getattr(step, "type", "") or ""++ # Result steps close the matching call's glow.+ if step_type == "function_result" or step_type.endswith("_result"):+ call_id = getattr(step, "call_id", None)+ node_id = open_calls.pop(call_id, None)+ if node_id and workflow_id:+ await pulse_node(node_id, "success", workflow_id=workflow_id)+ return++ name = getattr(step, "name", "") or ""+ if step_type in _CLOUD_STEP_LABELS:+ key, label = f"type:{step_type}", _CLOUD_STEP_LABELS[step_type]+ elif step_type == "function_call":+ if name in declared_names:+ # Ours — bridged at requires_action; phase only.+ await phase_tool(name)+ return+ if not name or name in _CLOUD_NOISE_NAMES:+ return+ key, label = f"fn:{name}", name+ else:+ return++ await phase_tool(label)+ if not workflow_id:+ return+ node_id = live_nodes.get(key)+ if node_id is None:+ resolved = await ensure_cloud_tool_nodes(+ workflow_id=workflow_id,+ agent_node_id=agent_node_id,+ used={key: label},+ )+ node_id = resolved.get(key)+ if node_id is None:+ return+ live_nodes[key] = node_id+ await pulse_node(node_id, "executing", workflow_id=workflow_id)+ step_id = getattr(step, "id", None)+ if step_id:+ open_calls[step_id] = node_id++ return on_event+@staticmethoddef _usage_dict(interaction: Any) -> Optional[Dict[str, Any]]:usage = getattr(interaction, "usage", None)diff --git a/server/nodes/agent/vertex_managed_agent/_ops.py b/server/nodes/agent/vertex_managed_agent/_ops.pyindex 38c1f580..54f02bca 100644--- a/server/nodes/agent/vertex_managed_agent/_ops.py+++ b/server/nodes/agent/vertex_managed_agent/_ops.py@@ -5,8 +5,9 @@ distinct cloud-side tool the managed agent used, wire them to theagent's ``input-tools`` handle, persist into ``workflow.data`` FIRST(DB is source of truth on the next run) and then broadcast the opsbatch on the ``workflow_ops_apply`` wire key so an open canvas applies-them live. Finally each used node gets a one-shot executing->success-pulse (the frontend's 500ms minimum glow makes it visible).+them live. Pulsing (executing/success glow via ``pulse_node``) is the+caller's concern: the live SSE handler pulses per call/result step,+the post-turn sweep pulses once per missed key."""from __future__ import annotations@@ -31,23 +32,42 @@ def _mint_node_id(prefix: str) -> str:return f"{prefix}-{int(time.time() * 1000)}-{secrets.token_hex(3)}"+async def pulse_node(+ node_id: str,+ status: str,+ *,+ workflow_id: str,+ message: str = "Used by Vertex agent",+) -> None:+ """One status broadcast on a display node (executing / success)."""+ await get_status_broadcaster().update_node_status(+ node_id,+ status,+ {"message": message},+ workflow_id=workflow_id,+ )++async def ensure_cloud_tool_nodes(*,workflow_id: str,agent_node_id: str,used: Dict[str, str],-) -> List[str]:+) -> Dict[str, str]:"""Mint/dedupe display nodes for ``{cloud_tool_key: label}`` usage.- Returns the canvas node ids that were pulsed (existing + minted).+ Mint + persist + broadcast only — pulsing is the caller's concern+ (live streaming pulses per call/result; the post-turn sweep pulses+ once). Returns ``{cloud_tool_key: canvas_node_id}`` for every+ requested key (existing + minted)."""if not used:- return []+ return {}database = get_database()workflow = await database.get_workflow(workflow_id)if workflow is None:- return []+ return {}data = dict(workflow.data or {})nodes = list(data.get("nodes") or [])@@ -69,9 +89,9 @@ async def ensure_cloud_tool_nodes(existing[label] = node.get("id")operations: List[dict] = []- pulse_ids: List[str] = [- existing[label] for label in used.values() if label in existing- ]+ resolved: Dict[str, str] = {+ key: existing[label] for key, label in used.items() if label in existing+ }fan_index = 0for key, label in used.items():@@ -131,12 +151,11 @@ async def ensure_cloud_tool_nodes(await database.save_node_parameters(minted_id, {"label": label, "cloud_tool_key": key})- pulse_ids.append(minted_id)+ resolved[key] = minted_idfan_index += 1- broadcaster = get_status_broadcaster()-if operations:+ broadcaster = get_status_broadcaster()data["nodes"] = nodesdata["edges"] = edges# Persist-then-broadcast: the DB write must land before any@@ -164,18 +183,4 @@ async def ensure_cloud_tool_nodes(agent_node_id,)- for node_id in pulse_ids:- await broadcaster.update_node_status(- node_id,- "executing",- {"message": "Used by Vertex agent"},- workflow_id=workflow_id,- )- await broadcaster.update_node_status(- node_id,- "success",- {"message": "Used by Vertex agent"},- workflow_id=workflow_id,- )-- return pulse_ids+ return resolveddiff --git a/server/tests/nodes/test_vertex_agents.py b/server/tests/nodes/test_vertex_agents.pyindex 8e16c893..068c1153 100644--- a/server/tests/nodes/test_vertex_agents.py+++ b/server/tests/nodes/test_vertex_agents.py@@ -74,11 +74,15 @@ _ADMIN_MODULE = "nodes.agent.vertex_agent_admin"def _patched_interactions(side_effect):- """Patch the node's SDK seam: fake client + canned interactions."""+ """Patch the node's SDK seam: fake client + canned interactions.++ The node now calls ``stream_interaction`` (SSE-backed); non-live+ tests mock it to return final interactions, ignoring ``on_event``.+ """return (patch(f"{_NODE_MODULE}.build_genai_client", return_value=MagicMock()),patch(- f"{_NODE_MODULE}.create_interaction_and_wait",+ f"{_NODE_MODULE}.stream_interaction",AsyncMock(side_effect=side_effect),),)@@ -327,7 +331,7 @@ class TestVertexManagedAgent:client_patch = patch(f"{_NODE_MODULE}.build_genai_client", return_value=MagicMock())create_patch = patch(- f"{_NODE_MODULE}.create_interaction_and_wait",+ f"{_NODE_MODULE}.stream_interaction",AsyncMock(side_effect=create_side_effect),)with (@@ -359,13 +363,15 @@ class TestVertexManagedAgent:SimpleNamespace(type="google_search_call", id="c2", name=None),]client_patch, create_patch = _patched_interactions([_interaction(steps=steps)])- mint_mock = AsyncMock(return_value=[])+ mint_mock = AsyncMock(return_value={})+ pulse_mock = AsyncMock()with (patched_container(auth_api_keys={}),patched_broadcaster() as bc,client_patch,create_patch,patch(f"{_NODE_MODULE}._ops.ensure_cloud_tool_nodes", mint_mock),+ patch(f"{_NODE_MODULE}._ops.pulse_node", pulse_mock),):_wire_async_broadcasts(bc)result = await harness.execute(@@ -381,6 +387,8 @@ class TestVertexManagedAgent:)harness.assert_envelope(result, success=True)+ # Stream mock never invoked on_event, so everything lands in the+ # post-turn sweep.mint_mock.assert_awaited_once()used = mint_mock.await_args.kwargs["used"]assert used == {@@ -413,6 +421,148 @@ class TestVertexManagedAgent:harness.assert_envelope(result, success=True)mint_mock.assert_not_awaited()+ async def test_live_stream_mints_and_pulses_mid_turn(self, harness):+ """Cloud tool nodes appear WHILE streaming: mint + executing pulse+ on the call step, success pulse on the call_id-matched result+ step; noise names and non-step events are ignored; the live+ union feeds cloud_tools_used even when the final resource+ carries no steps."""+ agent_id = "vx-1"++ async def fake_stream(client, *, on_event=None, **kwargs):+ assert on_event is not None+ await on_event(+ SimpleNamespace(+ event_type="step.start",+ index=0,+ step=SimpleNamespace(type="code_execution_call", id="c1", name=None),+ )+ )+ await on_event(SimpleNamespace(event_type="step.delta", index=0))+ await on_event(+ SimpleNamespace(+ event_type="step.start",+ index=1,+ step=SimpleNamespace(type="code_execution_result", call_id="c1"),+ )+ )+ await on_event(+ SimpleNamespace(+ event_type="step.start",+ index=2,+ step=SimpleNamespace(+ type="function_call", id="c9", name="provision_sandbox"+ ),+ )+ )+ return _interaction(steps=[]) # final resource: no steps++ mint_mock = AsyncMock(return_value={"type:code_execution_call": "vct-1"})+ pulse_mock = AsyncMock()+ with (+ patched_container(auth_api_keys={}),+ patched_broadcaster() as bc,+ patch(f"{_NODE_MODULE}.build_genai_client", return_value=MagicMock()),+ patch(+ f"{_NODE_MODULE}.stream_interaction",+ AsyncMock(side_effect=fake_stream),+ ),+ patch(f"{_NODE_MODULE}._ops.ensure_cloud_tool_nodes", mint_mock),+ patch(f"{_NODE_MODULE}._ops.pulse_node", pulse_mock),+ ):+ _wire_async_broadcasts(bc)+ result = await harness.execute(+ "vertex_managed_agent",+ {"prompt": "p", "project_id": "test-proj"},+ node_id=agent_id,+ context={+ "workflow_id": "wf-1",+ "nodes": [_node(agent_id, "vertex_managed_agent")],+ "edges": [],+ "outputs": {},+ },+ )++ harness.assert_envelope(result, success=True)+ # Minted once, LIVE (noise name skipped; empty final steps = no sweep).+ mint_mock.assert_awaited_once()+ assert mint_mock.await_args.kwargs["used"] == {+ "type:code_execution_call": "Code Execution"+ }+ pulses = [(c.args[0], c.args[1]) for c in pulse_mock.await_args_list]+ assert ("vct-1", "executing") in pulses+ assert ("vct-1", "success") in pulses+ assert result["result"]["cloud_tools_used"] == ["Code Execution"]+++# ============================================================================+# stream_interaction (SSE helper)+# ============================================================================+++class _FakeStream:+ def __init__(self, events):+ self._events = list(events)++ def __aiter__(self):+ return self++ async def __anext__(self):+ if not self._events:+ raise StopAsyncIteration+ return self._events.pop(0)+++class TestStreamInteraction:+ async def test_on_event_errors_swallowed_and_full_resource_fetched(self):+ from nodes.agent._vertex import stream_interaction++ events = [+ SimpleNamespace(+ event_type="interaction.created",+ interaction=SimpleNamespace(id="ix-9"),+ ),+ SimpleNamespace(+ event_type="step.start",+ step=SimpleNamespace(type="model_output"),+ ),+ ]+ client = MagicMock()+ client.aio.interactions.create = AsyncMock(return_value=_FakeStream(events))+ final = _interaction(interaction_id="ix-9")+ client.aio.interactions.get = AsyncMock(return_value=final)++ async def bad_handler(event):+ raise RuntimeError("boom")++ interaction = await stream_interaction(+ client, on_event=bad_handler, agent="a", input="p"+ )++ assert interaction is final+ client.aio.interactions.get.assert_awaited_with("ix-9")+ create_kwargs = client.aio.interactions.create.await_args.kwargs+ assert create_kwargs["stream"] is True+ assert create_kwargs["background"] is True++ async def test_falls_back_to_poll_when_streaming_rejected(self):+ from nodes.agent._vertex import stream_interaction++ final = _interaction(interaction_id="ix-2")+ client = MagicMock()+ client.aio.interactions.create = AsyncMock(+ side_effect=[FakeGenaiError("streaming not supported"), final]+ )+ client.aio.interactions.get = AsyncMock(return_value=final)++ interaction = await stream_interaction(client, agent="a", input="p")++ assert interaction is final+ # Second create is the non-stream background fallback.+ second = client.aio.interactions.create.await_args_list[1].kwargs+ assert second.get("stream") is not True+ assert second["background"] is True+# ============================================================================# ensure_cloud_tool_nodes (minting helper)@@ -428,7 +578,7 @@ class TestCloudToolMinting:description=None,)- async def test_mints_persists_then_broadcasts_and_pulses(self):+ async def test_mints_persists_then_broadcasts(self):from nodes.agent.vertex_managed_agent import _opsdatabase = MagicMock()@@ -443,14 +593,14 @@ class TestCloudToolMinting:patch.object(_ops, "get_database", return_value=database),patch.object(_ops, "get_status_broadcaster", return_value=broadcaster),):- pulsed = await _ops.ensure_cloud_tool_nodes(+ resolved = await _ops.ensure_cloud_tool_nodes(workflow_id="wf-1",agent_node_id="vx-1",used={"fn:run_command": "run_command"},)- assert len(pulsed) == 1- minted_id = pulsed[0]+ assert list(resolved) == ["fn:run_command"]+ minted_id = resolved["fn:run_command"]assert minted_id.startswith("vertexCloudTool-")# Persisted node + edge with the minted id.@@ -475,9 +625,21 @@ class TestCloudToolMinting:assert ops[1]["source_handle"] == "output-tool"assert ops[1]["target_handle"] == "input-tools"- # executing -> success pulse on the minted node.- statuses = [c.args[1] for c in broadcaster.update_node_status.await_args_list]- assert statuses == ["executing", "success"]+ # Pulsing is the caller's concern now — mint emits no node_status.+ broadcaster.update_node_status.assert_not_awaited()++ async def test_pulse_node_broadcasts_status(self):+ from nodes.agent.vertex_managed_agent import _ops++ broadcaster = MagicMock()+ broadcaster.update_node_status = AsyncMock()+ with patch.object(_ops, "get_status_broadcaster", return_value=broadcaster):+ await _ops.pulse_node("vct-1", "executing", workflow_id="wf-1")++ args = broadcaster.update_node_status.await_args+ assert args.args[0] == "vct-1"+ assert args.args[1] == "executing"+ assert args.kwargs["workflow_id"] == "wf-1"async def test_dedupes_existing_node_by_label(self):from nodes.agent.vertex_managed_agent import _ops@@ -510,17 +672,16 @@ class TestCloudToolMinting:patch.object(_ops, "get_database", return_value=database),patch.object(_ops, "get_status_broadcaster", return_value=broadcaster),):- pulsed = await _ops.ensure_cloud_tool_nodes(+ resolved = await _ops.ensure_cloud_tool_nodes(workflow_id="wf-1",agent_node_id="vx-1",used={"fn:run_command": "run_command"},)- assert pulsed == ["vertexCloudTool-1-aaa"]+ assert resolved == {"fn:run_command": "vertexCloudTool-1-aaa"}database.save_workflow.assert_not_awaited()broadcaster.broadcast.assert_not_awaited()- # Existing node still pulses.- assert broadcaster.update_node_status.await_count == 2+ broadcaster.update_node_status.assert_not_awaited()# ============================================================================