Self-improving AI Employees for everyone
Python ai-agentsai-employeeai-employeesassistantautonomous-agentsco-employeescompanygrok-botinstinctloop-agentsmuseown-your-datapersonalself-hosted
fix(agents): preserve recovery hints and blocked tool turns
trohitg511b627c8606
diff --git a/client/src/components/__tests__/OutputPanel.test.tsx b/client/src/components/__tests__/OutputPanel.test.tsxindex fb17b8ca..1312cf80 100644--- a/client/src/components/__tests__/OutputPanel.test.tsx+++ b/client/src/components/__tests__/OutputPanel.test.tsx@@ -104,6 +104,15 @@ describe('OutputPanel response routing', () => {* hint-driven, so a ref reaching this panel from anywhere still renders.*/describe('OutputPanel — audio', () => {+ it('shows the node error recovery hint beside the failure', () => {+ renderWithProviders(<OutputPanel results={[{+ ...makeResult({ hint: 'Review billing before resuming.' })[0],+ success: false, error: 'Spending cap reached.',+ }]} selectedNode={NODE} />);+ expect(screen.getByRole('alert')).toHaveTextContent('Spending cap reached.');+ expect(screen.getByRole('alert')).toHaveTextContent('Review billing before resuming.');+ });+const SPEECH_NODE = { id: 'tts-1', type: 'textToSpeech' } as any;const audioRef = (overrides: Record<string, any> = {}) => ({diff --git a/client/src/components/output/OutputPanel.tsx b/client/src/components/output/OutputPanel.tsxindex 5cd019fb..f350310f 100644--- a/client/src/components/output/OutputPanel.tsx+++ b/client/src/components/output/OutputPanel.tsx@@ -13,6 +13,7 @@ import remarkBreaks from 'remark-breaks';import JsonView from '@uiw/react-json-view';import { Node } from 'reactflow';import { Badge } from '@/components/ui/badge';+import { Alert, AlertDescription } from '@/components/ui/alert';import { Button } from '@/components/ui/button';import {Collapsible,@@ -148,9 +149,12 @@ export default function OutputPanel({ results, onClear, selectedNode }: Props) {{/* Content */}<div className="flex-1 overflow-auto p-4">{latest.error && (- <pre className="mb-3 overflow-auto rounded-md border border-destructive/40 border-l-4 border-l-destructive bg-destructive/5 p-3 font-mono text-sm whitespace-pre-wrap break-words text-destructive">- {latest.error}- </pre>+ <Alert variant="destructive" className="mb-3">+ <AlertDescription>+ <p className="whitespace-pre-wrap break-words">{latest.error}</p>+ {typeof raw?.hint === 'string' && <p className="whitespace-pre-wrap break-words">{raw.hint}</p>}+ </AlertDescription>+ </Alert>)}<div className="space-y-0">diff --git a/docs-internal/plugin_system.md b/docs-internal/plugin_system.mdindex 428a3e0f..989b0625 100644--- a/docs-internal/plugin_system.md+++ b/docs-internal/plugin_system.md@@ -115,6 +115,8 @@ Defense in depth below the plugin layer: the SQLAlchemy engine in `core/database**Error convention — `NodeUserError` vs bare `Exception`.** For any failure the user (or the calling LLM) can correct — missing required field, unknown enum value, bad path, service not running — raise `NodeUserError("...")` (exported from `services.plugin`). `BaseNode.execute()` catches it specially: one WARN line in the operator log (no traceback) plus a structured `{success: False, error_type: "NodeUserError", error: ...}` envelope, and the shared Temporal retry policy treats it as **non-retryable** (`services/plugin/scaling.py`) so a bad input doesn't burn three activity attempts. Reserve bare `Exception` / `RuntimeError` for genuine server bugs — those keep the full `logger.exception` traceback.+Use `NodeUserError(message, hint="How to recover", requires_user_action=True)` for an owner-only blocker such as exhausted billing or an unusable provider credential. Keep both texts safe for public display. The framework preserves `hint`, `requires_user_action`, and `retryable: false` through tool results. Both agent loops stop before another model turn or compaction, and a controlled trigger deployment pauses on its first blocked run. Ordinary input errors leave `requires_user_action` false so the model can correct them; other run failures retain the configured circuit-breaker threshold. Temporal saves outstanding tool results before stopping so Resume has a complete tool turn. Home and the node output panel display the recovery hint.+### Auto-derived uiHints`BaseNode._metadata_dict` (`server/services/plugin/base.py`) callsdiff --git a/server/services/agent_context/conversation.py b/server/services/agent_context/conversation.pyindex fe70a74d..64420eaa 100644--- a/server/services/agent_context/conversation.py+++ b/server/services/agent_context/conversation.py@@ -8,6 +8,8 @@ The whole contract:turn, serialized per key by an in-process lock. Best-effort callers(the LLM-step activity) must catch — persistence never fails a run.- ``clear_conversation`` — delete rows (panel clear, workflow cleanup).+- ``append_tool_results`` — close unanswered calls when a blocked run stops+ before another model request; retries do not append duplicates.- ``list_conversations`` — metadata for the Context panel.Key = ``(workflow_id, generation, agent_node_id)``. Generation is part of@@ -140,6 +142,50 @@ async def save_conversation()+async def append_tool_results(+ database: Any, *, workflow_id: str, generation: int, agent_node_id: str,+ results: List[Dict[str, Any]],+) -> bool:+ """Close a saved tool turn without paying for another model request.++ Only unanswered calls in the latest assistant turn are eligible. Activity+ retries are idempotent; a stale result cannot attach to a different turn.+ """+ async with _lock(workflow_id, generation, agent_node_id):+ async with database.get_session() as session:+ result = await session.execute(select(AgentConversation).where(+ AgentConversation.workflow_id == workflow_id,+ AgentConversation.generation == generation,+ AgentConversation.agent_node_id == agent_node_id,+ ))+ row = result.scalar_one_or_none()+ if row is None:+ return False+ messages = list(row.messages or [])+ latest = next((i for i in range(len(messages) - 1, -1, -1) if messages[i].get("role") == "assistant"), None)+ if latest is None:+ return False+ pending = {call.get("id") for call in (messages[latest].get("tool_calls") or []) if call.get("id")}+ pending -= {m.get("tool_call_id") for m in messages[latest + 1:] if m.get("role") == "tool"}+ additions = []+ for item in results:+ call_id = item.get("tool_call_id")+ if item.get("role") == "tool" and call_id in pending:+ additions.append(dict(item))+ pending.remove(call_id)+ if not additions:+ return False+ row.messages = _stamp_messages(messages + additions, messages, datetime.now(timezone.utc).isoformat())+ row.updated_at = datetime.now(timezone.utc)+ session.add(row)+ await session.commit()+ await notify_conversation_saved(+ workflow_id=workflow_id, generation=generation, agent_node_id=agent_node_id,+ message_count=len(messages) + len(additions),+ )+ return True++async def clear_conversation(database: Any,*,diff --git a/server/services/temporal/agent_activities.py b/server/services/temporal/agent_activities.pyindex f2a1057c..57a338fb 100644--- a/server/services/temporal/agent_activities.py+++ b/server/services/temporal/agent_activities.py@@ -113,7 +113,12 @@ def _unsaved_endpoint_error(provider: str) -> Optional[ApplicationError]:message = unconfigured_endpoint_message(provider)if not message:return None- return ApplicationError(message, type="MissingAgentProviderCredential", non_retryable=True)+ return ApplicationError(+ message,+ {"hint": "Choose and save a provider credential in Settings > Connectors, then resume.",+ "requires_user_action": True, "retryable": False},+ type="MissingAgentProviderCredential", non_retryable=True,+ )async def _resolve_activity_api_key(payload: Dict[str, Any]) -> str:@@ -150,6 +155,8 @@ async def _resolve_activity_api_key(payload: Dict[str, Any]) -> str:if not api_key:raise _unsaved_endpoint_error(provider) or ApplicationError(f"API key for provider {provider!r} is not configured",+ {"hint": "Connect the provider in Settings > Connectors, then resume.",+ "requires_user_action": True, "retryable": False},type="MissingAgentProviderCredential",non_retryable=True,)@@ -375,6 +382,8 @@ async def _execute_native_llm_step(payload: Dict[str, Any]) -> Dict[str, Any]:detail=f"LLM step waiting: {payload.get('model')}",)except LLMError as error:+ if not error.retryable:+ await _save_failed_tool_results(payload, payload.get("messages") or [])attempt = _llm_activity_attempt()failure = _as_temporal_llm_error(error, attempt=attempt)if error.retryable:@@ -542,10 +551,31 @@ async def execute_llm_step(payload: Dict[str, Any]) -> Dict[str, Any]:return result+async def _save_failed_tool_results(payload: Dict[str, Any], results: List[Dict[str, Any]]) -> bool:+ key = payload.get("conversation_key")+ if not isinstance(key, dict) or not any(m.get("role") == "tool" for m in results):+ return False+ from core.container import container+ from services.agent_context.conversation import append_tool_results++ try:+ return await append_tool_results(+ container.database(), workflow_id=str(key.get("workflow_id") or ""),+ generation=int(key.get("generation") or 0),+ agent_node_id=str(key.get("agent_node_id") or ""), results=results,+ )+ except Exception:+ activity.logger.warning("Could not save failed tool turn", exc_info=True)+ return False++@activity.defn(name="agent.persist_turn")async def persist_agent_turn(payload: Dict[str, Any]) -> Dict[str, Any]:"""Append one ``(human, assistant)`` pair to the connected memory.+ A blocked tool turn supplies ``conversation_key`` and ``tool_results``+ instead: finish the already-saved Context turn without another LLM call.+Per user decision in plan §15 (F4.B), memory appends per turn sofailure mid-loop doesn't lose progress. Uses the same markdownhelpers ``services/ai.py`` calls today via@@ -565,6 +595,8 @@ async def persist_agent_turn(payload: Dict[str, Any]) -> Dict[str, Any]:No-op when ``memory_node_id`` is empty (the agent has no memoryconnected)."""+ if "tool_results" in payload:+ return {"appended": await _save_failed_tool_results(payload, payload["tool_results"]), "trimmed_count": 0}memory_node_id = payload.get("memory_node_id") or ""if not memory_node_id:return {"appended": False, "trimmed_count": 0}@@ -929,8 +961,11 @@ async def prepare_agent_payload(context: Dict[str, Any]) -> Dict[str, Any]:endpoint_error = _unsaved_endpoint_error(provider)if endpoint_error:raise endpoint_error- raise RuntimeError(- f"API key for provider {provider!r} required for AgentWorkflow " f"node {node_id!r}; configure it in the Credentials Modal."+ raise ApplicationError(+ f"API key for provider {provider!r} is not configured",+ {"hint": "Connect the provider in Settings > Connectors, then resume.",+ "requires_user_action": True, "retryable": False},+ type="MissingAgentProviderCredential", non_retryable=True,)max_tokens = resolve_max_tokens(flattened, model, provider)diff --git a/server/services/temporal/agent_workflow.py b/server/services/temporal/agent_workflow.pyindex 7222d8de..cc6dee8d 100644--- a/server/services/temporal/agent_workflow.py+++ b/server/services/temporal/agent_workflow.py@@ -923,7 +923,7 @@ class AgentWorkflow:getattr(cause, "message", "") or "").strip()recovery = {}- if cause_type.startswith("LLMError."):+ if cause_type.startswith("LLMError.") or cause_type == "MissingAgentProviderCredential":diagnostics = getattr(cause, "details", ())if diagnostics and isinstance(diagnostics[0], dict):recovery = {@@ -931,6 +931,8 @@ class AgentWorkflow:for key in ("hint", "requires_user_action", "retryable")if diagnostics[0].get(key) is not None}+ if recovery:+ recovery["error_type"] = "NodeUserError"safe_activity_types = {"MissingAgentProviderCredential","EmptyAgentPrompt",@@ -2166,6 +2168,15 @@ class AgentWorkflow:if blocked_error:# Complete/persist the current tool turn, then stop before# another model request or paid compaction can be scheduled.+ if conversation_key and workflow.patched("agent-blocked-tool-turn-save-v1"):+ await workflow.execute_activity(+ "agent.persist_turn",+ args=[{"conversation_key": conversation_key,+ "tool_results": [m for m in messages[turn_start:] if m.get("role") == "tool"]}],+ activity_id="persist-blocked-tool-turn",+ start_to_close_timeout=PERSIST_TURN_TIMEOUT,+ retry_policy=AGENT_ACTIVITY_RETRY,+ )await workflow.execute_activity("agent.skill.clear",args=[{"workflow_id": payload.get("workflow_id"),diff --git a/server/services/temporal/workflow.py b/server/services/temporal/workflow.pyindex f1da1d6d..8692f837 100644--- a/server/services/temporal/workflow.py+++ b/server/services/temporal/workflow.py@@ -874,6 +874,26 @@ class MachinaWorkflow:queue = cls.task_queue if worker_pool_enabled else Nonereturn f"node.{cls.type}.v{cls.version}", queue+ @staticmethod+ def _node_failure(error: Exception) -> Dict[str, Any]:+ """Keep safe recovery metadata through child/activity wrappers."""+ if workflow.patched("workflow-node-user-action-error-v1"):+ cause = error+ for _ in range(10):+ kind = str(getattr(cause, "type", ""))+ if kind.startswith("LLMError.") or kind == "MissingAgentProviderCredential":+ details = getattr(cause, "details", ())+ if details and isinstance(details[0], dict) and details[0].get("requires_user_action") is True:+ return {+ "success": False, "error_type": "NodeUserError",+ "error": cause.message, "hint": details[0].get("hint"),+ "requires_user_action": True, "retryable": False,+ }+ cause = getattr(cause, "cause", None)+ if cause is None:+ break+ return {"success": False, "error": str(error)}+async def _wait_any_complete(self, running: Dict[str, Any]) -> tuple:"""Wait for any activity to complete, return (node_id, result).@@ -890,7 +910,7 @@ class MachinaWorkflow:result = await handlereturn node_id, resultexcept Exception as e:- return node_id, {"success": False, "error": str(e)}+ return node_id, self._node_failure(e)# Wait for first completion using Temporal's waitawait workflow.wait_condition(lambda: any(h.done() for _, h in items))@@ -903,7 +923,7 @@ class MachinaWorkflow:result = await handlereturn node_id, resultexcept Exception as e:- return node_id, {"success": False, "error": str(e)}+ return node_id, self._node_failure(e)# Should not reach hereraise RuntimeError("No activity completed after wait")diff --git a/server/tests/services/agent_context/test_conversation.py b/server/tests/services/agent_context/test_conversation.pyindex fbdc5ae5..e14f0b45 100644--- a/server/tests/services/agent_context/test_conversation.py+++ b/server/tests/services/agent_context/test_conversation.py@@ -95,6 +95,25 @@ _KEY = {}+async def test_blocked_tool_turn_is_saved_once_and_stale_results_are_ignored(conversation_database, isolated_listeners):+ from services.agent_context.conversation import append_tool_results+ from services.llm.protocol import ToolCall++ assistant = dict(message_to_wire(Message(role="assistant", tool_calls=[ToolCall(id="call-1", name="model", args={})])))+ tool_result = dict(message_to_wire(Message(role="tool", tool_call_id="call-1", name="model", content='{"error":"blocked","hint":"Fix billing"}')))+ await save_conversation(conversation_database, **_KEY, messages=[_wire("user", "go"), assistant])+ assert await append_tool_results(conversation_database, **_KEY, results=[tool_result])+ assert not await append_tool_results(conversation_database, **_KEY, results=[tool_result])+ stored = await load_conversation(conversation_database, **_KEY)+ assert [m["role"] for m in stored] == ["user", "assistant", "tool"]+ assert stored[-1]["content"] == tool_result["content"]++ await save_conversation(conversation_database, **_KEY, messages=[_wire("assistant", "A later turn")])+ assert not await append_tool_results(conversation_database, **_KEY, results=[tool_result])+ assert len(await load_conversation(conversation_database, **_KEY)) == 1++def _sans_ts(messages: list[dict]) -> list[dict]:return [{k: v for k, v in m.items() if k != "ts"} for m in messages]diff --git a/server/tests/services/test_tool_input_security.py b/server/tests/services/test_tool_input_security.pyindex cc84310c..2b905d40 100644--- a/server/tests/services/test_tool_input_security.py+++ b/server/tests/services/test_tool_input_security.py@@ -87,6 +87,22 @@ def test_tool_input_defaults_to_params():assert CurrentTimeToolNode.tool_input_model() is CurrentTimeToolNode.Params+async def test_node_error_recovery_metadata_reaches_agent_tools():+ from services.plugin import NodeUserError++ class BlockedTool(_ConfiguredToolNode, abstract=True):+ @Operation("run")+ async def run(self, ctx, params):+ raise NodeUserError("Spending cap reached", hint="Review billing", requires_user_action=True)++ result = await BlockedTool().execute_as_tool(+ {"endpoint": "test"}, {"endpoint": "test"},+ NodeContext(node_id="tool-1", node_type=BlockedTool.type, raw={}),+ )+ assert result == {"error": "Spending cap reached", "hint": "Review billing", "requires_user_action": True, "retryable": False}++def test_native_validation_does_not_materialize_omitted_defaults():spec = AgentToolSpec(definition=ToolDef(diff --git a/server/tests/temporal/test_agent_llm_contract.py b/server/tests/temporal/test_agent_llm_contract.pyindex 313d4176..8e0edd3d 100644--- a/server/tests/temporal/test_agent_llm_contract.py+++ b/server/tests/temporal/test_agent_llm_contract.py@@ -100,6 +100,9 @@ class TestWorkflowLoopContract:prepared["compaction_threshold"] = 1calls = []phases = []+ checkpoints = []+ if from_tool:+ prepared["conversation_key"] = {"workflow_id": "graph-1", "generation": 1, "agent_node_id": "agent-1"}async def execute(name, *, args, **_kwargs):calls.append(name)@@ -117,6 +120,9 @@ class TestWorkflowLoopContract:if name == "agent.broadcast_progress":phases.append(args[0])return {}+ if name == "agent.persist_turn":+ checkpoints.append(args[0])+ return {}if name in {"agent.skill.clear", "agent.persist_turn"}:return {}raise AssertionError(f"Unexpected activity: {name}")@@ -131,6 +137,9 @@ class TestWorkflowLoopContract:failure = next(p for p in phases if p.get("status") == "error")assert failure["hint"] == error.hintassert "secret" not in str(failure)+ if from_tool:+ assert len(checkpoints) == 1+ assert checkpoints[0]["tool_results"][0]["tool_call_id"] == "call-1"@pytest.mark.asyncioasync def test_llm_step_payload_shape_and_heartbeat(diff --git a/server/tests/temporal/test_durability_hardening.py b/server/tests/temporal/test_durability_hardening.pyindex 408484c3..ce96ed6a 100644--- a/server/tests/temporal/test_durability_hardening.py+++ b/server/tests/temporal/test_durability_hardening.py@@ -1248,6 +1248,31 @@ async def test_machina_failure_schedules_the_circuit_breaker_activity(monkeypatcassert captured == {}+async def test_prepare_failure_keeps_owner_recovery_through_child_wrappers(monkeypatch):+ import asyncio+ from temporalio.exceptions import ApplicationError+ from services.temporal import workflow as module++ monkeypatch.setattr(module.workflow, "patched", lambda _name: True)+ error = RuntimeError("Child workflow failed")+ error.cause = ApplicationError(+ "Provider credential missing", {"hint": "Connect the provider", "requires_user_action": True},+ type="MissingAgentProviderCredential", non_retryable=True,+ )+ future = asyncio.get_running_loop().create_future()+ future.set_exception(error)+ node, result = await module.MachinaWorkflow()._wait_any_complete({"agent": future})+ assert node == "agent"+ assert result["requires_user_action"] is True+ assert result["error"] == "Provider credential missing"+ assert result["hint"] == "Connect the provider"++ # Histories recorded before the propagation change keep their result.+ monkeypatch.setattr(module.workflow, "patched", lambda _name: False)+ assert module.MachinaWorkflow._node_failure(error) == {"success": False, "error": "Child workflow failed"}++def test_can_edit_capability_is_server_owned():"""Canvas editability is a capability the backend computes — the FErenders it and never re-derives the rule from state strings. Paused