Skip to content

Self-improving AI Employees for everyone

fix(agents): preserve recovery hints and blocked tool turns

trohitg511b627c8606
diff --git a/client/src/components/__tests__/OutputPanel.test.tsx b/client/src/components/__tests__/OutputPanel.test.tsx
index fb17b8ca..1312cf80 100644
--- a/client/src/components/__tests__/OutputPanel.test.tsx
+++ b/client/src/components/__tests__/OutputPanel.test.tsx
@@ -104,6 +104,15 @@ describe('OutputPanel response routing', () => {
* hint-driven, so a ref reaching this panel from anywhere still renders.
*/
describe('OutputPanel — audio', () => {
+ it('shows the node error recovery hint beside the failure', () => {
+ renderWithProviders(<OutputPanel results={[{
+ ...makeResult({ hint: 'Review billing before resuming.' })[0],
+ success: false, error: 'Spending cap reached.',
+ }]} selectedNode={NODE} />);
+ expect(screen.getByRole('alert')).toHaveTextContent('Spending cap reached.');
+ expect(screen.getByRole('alert')).toHaveTextContent('Review billing before resuming.');
+ });
+
const SPEECH_NODE = { id: 'tts-1', type: 'textToSpeech' } as any;
const audioRef = (overrides: Record<string, any> = {}) => ({
diff --git a/client/src/components/output/OutputPanel.tsx b/client/src/components/output/OutputPanel.tsx
index 5cd019fb..f350310f 100644
--- a/client/src/components/output/OutputPanel.tsx
+++ b/client/src/components/output/OutputPanel.tsx
@@ -13,6 +13,7 @@ import remarkBreaks from 'remark-breaks';
import JsonView from '@uiw/react-json-view';
import { Node } from 'reactflow';
import { Badge } from '@/components/ui/badge';
+import { Alert, AlertDescription } from '@/components/ui/alert';
import { Button } from '@/components/ui/button';
import {
Collapsible,
@@ -148,9 +149,12 @@ export default function OutputPanel({ results, onClear, selectedNode }: Props) {
{/* Content */}
<div className="flex-1 overflow-auto p-4">
{latest.error && (
- <pre className="mb-3 overflow-auto rounded-md border border-destructive/40 border-l-4 border-l-destructive bg-destructive/5 p-3 font-mono text-sm whitespace-pre-wrap break-words text-destructive">
- {latest.error}
- </pre>
+ <Alert variant="destructive" className="mb-3">
+ <AlertDescription>
+ <p className="whitespace-pre-wrap break-words">{latest.error}</p>
+ {typeof raw?.hint === 'string' && <p className="whitespace-pre-wrap break-words">{raw.hint}</p>}
+ </AlertDescription>
+ </Alert>
)}
<div className="space-y-0">
diff --git a/docs-internal/plugin_system.md b/docs-internal/plugin_system.md
index 428a3e0f..989b0625 100644
--- a/docs-internal/plugin_system.md
+++ b/docs-internal/plugin_system.md
@@ -115,6 +115,8 @@ Defense in depth below the plugin layer: the SQLAlchemy engine in `core/database
**Error convention — `NodeUserError` vs bare `Exception`.** For any failure the user (or the calling LLM) can correct — missing required field, unknown enum value, bad path, service not running — raise `NodeUserError("...")` (exported from `services.plugin`). `BaseNode.execute()` catches it specially: one WARN line in the operator log (no traceback) plus a structured `{success: False, error_type: "NodeUserError", error: ...}` envelope, and the shared Temporal retry policy treats it as **non-retryable** (`services/plugin/scaling.py`) so a bad input doesn't burn three activity attempts. Reserve bare `Exception` / `RuntimeError` for genuine server bugs — those keep the full `logger.exception` traceback.
+Use `NodeUserError(message, hint="How to recover", requires_user_action=True)` for an owner-only blocker such as exhausted billing or an unusable provider credential. Keep both texts safe for public display. The framework preserves `hint`, `requires_user_action`, and `retryable: false` through tool results. Both agent loops stop before another model turn or compaction, and a controlled trigger deployment pauses on its first blocked run. Ordinary input errors leave `requires_user_action` false so the model can correct them; other run failures retain the configured circuit-breaker threshold. Temporal saves outstanding tool results before stopping so Resume has a complete tool turn. Home and the node output panel display the recovery hint.
+
### Auto-derived uiHints
`BaseNode._metadata_dict` (`server/services/plugin/base.py`) calls
diff --git a/server/services/agent_context/conversation.py b/server/services/agent_context/conversation.py
index fe70a74d..64420eaa 100644
--- a/server/services/agent_context/conversation.py
+++ b/server/services/agent_context/conversation.py
@@ -8,6 +8,8 @@ The whole contract:
turn, serialized per key by an in-process lock. Best-effort callers
(the LLM-step activity) must catch — persistence never fails a run.
- ``clear_conversation`` — delete rows (panel clear, workflow cleanup).
+- ``append_tool_results`` — close unanswered calls when a blocked run stops
+ before another model request; retries do not append duplicates.
- ``list_conversations`` — metadata for the Context panel.
Key = ``(workflow_id, generation, agent_node_id)``. Generation is part of
@@ -140,6 +142,50 @@ async def save_conversation(
)
+async def append_tool_results(
+ database: Any, *, workflow_id: str, generation: int, agent_node_id: str,
+ results: List[Dict[str, Any]],
+) -> bool:
+ """Close a saved tool turn without paying for another model request.
+
+ Only unanswered calls in the latest assistant turn are eligible. Activity
+ retries are idempotent; a stale result cannot attach to a different turn.
+ """
+ async with _lock(workflow_id, generation, agent_node_id):
+ async with database.get_session() as session:
+ result = await session.execute(select(AgentConversation).where(
+ AgentConversation.workflow_id == workflow_id,
+ AgentConversation.generation == generation,
+ AgentConversation.agent_node_id == agent_node_id,
+ ))
+ row = result.scalar_one_or_none()
+ if row is None:
+ return False
+ messages = list(row.messages or [])
+ latest = next((i for i in range(len(messages) - 1, -1, -1) if messages[i].get("role") == "assistant"), None)
+ if latest is None:
+ return False
+ pending = {call.get("id") for call in (messages[latest].get("tool_calls") or []) if call.get("id")}
+ pending -= {m.get("tool_call_id") for m in messages[latest + 1:] if m.get("role") == "tool"}
+ additions = []
+ for item in results:
+ call_id = item.get("tool_call_id")
+ if item.get("role") == "tool" and call_id in pending:
+ additions.append(dict(item))
+ pending.remove(call_id)
+ if not additions:
+ return False
+ row.messages = _stamp_messages(messages + additions, messages, datetime.now(timezone.utc).isoformat())
+ row.updated_at = datetime.now(timezone.utc)
+ session.add(row)
+ await session.commit()
+ await notify_conversation_saved(
+ workflow_id=workflow_id, generation=generation, agent_node_id=agent_node_id,
+ message_count=len(messages) + len(additions),
+ )
+ return True
+
+
async def clear_conversation(
database: Any,
*,
diff --git a/server/services/temporal/agent_activities.py b/server/services/temporal/agent_activities.py
index f2a1057c..57a338fb 100644
--- a/server/services/temporal/agent_activities.py
+++ b/server/services/temporal/agent_activities.py
@@ -113,7 +113,12 @@ def _unsaved_endpoint_error(provider: str) -> Optional[ApplicationError]:
message = unconfigured_endpoint_message(provider)
if not message:
return None
- return ApplicationError(message, type="MissingAgentProviderCredential", non_retryable=True)
+ return ApplicationError(
+ message,
+ {"hint": "Choose and save a provider credential in Settings > Connectors, then resume.",
+ "requires_user_action": True, "retryable": False},
+ type="MissingAgentProviderCredential", non_retryable=True,
+ )
async def _resolve_activity_api_key(payload: Dict[str, Any]) -> str:
@@ -150,6 +155,8 @@ async def _resolve_activity_api_key(payload: Dict[str, Any]) -> str:
if not api_key:
raise _unsaved_endpoint_error(provider) or ApplicationError(
f"API key for provider {provider!r} is not configured",
+ {"hint": "Connect the provider in Settings > Connectors, then resume.",
+ "requires_user_action": True, "retryable": False},
type="MissingAgentProviderCredential",
non_retryable=True,
)
@@ -375,6 +382,8 @@ async def _execute_native_llm_step(payload: Dict[str, Any]) -> Dict[str, Any]:
detail=f"LLM step waiting: {payload.get('model')}",
)
except LLMError as error:
+ if not error.retryable:
+ await _save_failed_tool_results(payload, payload.get("messages") or [])
attempt = _llm_activity_attempt()
failure = _as_temporal_llm_error(error, attempt=attempt)
if error.retryable:
@@ -542,10 +551,31 @@ async def execute_llm_step(payload: Dict[str, Any]) -> Dict[str, Any]:
return result
+async def _save_failed_tool_results(payload: Dict[str, Any], results: List[Dict[str, Any]]) -> bool:
+ key = payload.get("conversation_key")
+ if not isinstance(key, dict) or not any(m.get("role") == "tool" for m in results):
+ return False
+ from core.container import container
+ from services.agent_context.conversation import append_tool_results
+
+ try:
+ return await append_tool_results(
+ container.database(), workflow_id=str(key.get("workflow_id") or ""),
+ generation=int(key.get("generation") or 0),
+ agent_node_id=str(key.get("agent_node_id") or ""), results=results,
+ )
+ except Exception:
+ activity.logger.warning("Could not save failed tool turn", exc_info=True)
+ return False
+
+
@activity.defn(name="agent.persist_turn")
async def persist_agent_turn(payload: Dict[str, Any]) -> Dict[str, Any]:
"""Append one ``(human, assistant)`` pair to the connected memory.
+ A blocked tool turn supplies ``conversation_key`` and ``tool_results``
+ instead: finish the already-saved Context turn without another LLM call.
+
Per user decision in plan §15 (F4.B), memory appends per turn so
failure mid-loop doesn't lose progress. Uses the same markdown
helpers ``services/ai.py`` calls today via
@@ -565,6 +595,8 @@ async def persist_agent_turn(payload: Dict[str, Any]) -> Dict[str, Any]:
No-op when ``memory_node_id`` is empty (the agent has no memory
connected).
"""
+ if "tool_results" in payload:
+ return {"appended": await _save_failed_tool_results(payload, payload["tool_results"]), "trimmed_count": 0}
memory_node_id = payload.get("memory_node_id") or ""
if not memory_node_id:
return {"appended": False, "trimmed_count": 0}
@@ -929,8 +961,11 @@ async def prepare_agent_payload(context: Dict[str, Any]) -> Dict[str, Any]:
endpoint_error = _unsaved_endpoint_error(provider)
if endpoint_error:
raise endpoint_error
- raise RuntimeError(
- f"API key for provider {provider!r} required for AgentWorkflow " f"node {node_id!r}; configure it in the Credentials Modal."
+ raise ApplicationError(
+ f"API key for provider {provider!r} is not configured",
+ {"hint": "Connect the provider in Settings > Connectors, then resume.",
+ "requires_user_action": True, "retryable": False},
+ type="MissingAgentProviderCredential", non_retryable=True,
)
max_tokens = resolve_max_tokens(flattened, model, provider)
diff --git a/server/services/temporal/agent_workflow.py b/server/services/temporal/agent_workflow.py
index 7222d8de..cc6dee8d 100644
--- a/server/services/temporal/agent_workflow.py
+++ b/server/services/temporal/agent_workflow.py
@@ -923,7 +923,7 @@ class AgentWorkflow:
getattr(cause, "message", "") or ""
).strip()
recovery = {}
- if cause_type.startswith("LLMError."):
+ if cause_type.startswith("LLMError.") or cause_type == "MissingAgentProviderCredential":
diagnostics = getattr(cause, "details", ())
if diagnostics and isinstance(diagnostics[0], dict):
recovery = {
@@ -931,6 +931,8 @@ class AgentWorkflow:
for key in ("hint", "requires_user_action", "retryable")
if diagnostics[0].get(key) is not None
}
+ if recovery:
+ recovery["error_type"] = "NodeUserError"
safe_activity_types = {
"MissingAgentProviderCredential",
"EmptyAgentPrompt",
@@ -2166,6 +2168,15 @@ class AgentWorkflow:
if blocked_error:
# Complete/persist the current tool turn, then stop before
# another model request or paid compaction can be scheduled.
+ if conversation_key and workflow.patched("agent-blocked-tool-turn-save-v1"):
+ await workflow.execute_activity(
+ "agent.persist_turn",
+ args=[{"conversation_key": conversation_key,
+ "tool_results": [m for m in messages[turn_start:] if m.get("role") == "tool"]}],
+ activity_id="persist-blocked-tool-turn",
+ start_to_close_timeout=PERSIST_TURN_TIMEOUT,
+ retry_policy=AGENT_ACTIVITY_RETRY,
+ )
await workflow.execute_activity(
"agent.skill.clear",
args=[{"workflow_id": payload.get("workflow_id"),
diff --git a/server/services/temporal/workflow.py b/server/services/temporal/workflow.py
index f1da1d6d..8692f837 100644
--- a/server/services/temporal/workflow.py
+++ b/server/services/temporal/workflow.py
@@ -874,6 +874,26 @@ class MachinaWorkflow:
queue = cls.task_queue if worker_pool_enabled else None
return f"node.{cls.type}.v{cls.version}", queue
+ @staticmethod
+ def _node_failure(error: Exception) -> Dict[str, Any]:
+ """Keep safe recovery metadata through child/activity wrappers."""
+ if workflow.patched("workflow-node-user-action-error-v1"):
+ cause = error
+ for _ in range(10):
+ kind = str(getattr(cause, "type", ""))
+ if kind.startswith("LLMError.") or kind == "MissingAgentProviderCredential":
+ details = getattr(cause, "details", ())
+ if details and isinstance(details[0], dict) and details[0].get("requires_user_action") is True:
+ return {
+ "success": False, "error_type": "NodeUserError",
+ "error": cause.message, "hint": details[0].get("hint"),
+ "requires_user_action": True, "retryable": False,
+ }
+ cause = getattr(cause, "cause", None)
+ if cause is None:
+ break
+ return {"success": False, "error": str(error)}
+
async def _wait_any_complete(self, running: Dict[str, Any]) -> tuple:
"""Wait for any activity to complete, return (node_id, result).
@@ -890,7 +910,7 @@ class MachinaWorkflow:
result = await handle
return node_id, result
except Exception as e:
- return node_id, {"success": False, "error": str(e)}
+ return node_id, self._node_failure(e)
# Wait for first completion using Temporal's wait
await workflow.wait_condition(lambda: any(h.done() for _, h in items))
@@ -903,7 +923,7 @@ class MachinaWorkflow:
result = await handle
return node_id, result
except Exception as e:
- return node_id, {"success": False, "error": str(e)}
+ return node_id, self._node_failure(e)
# Should not reach here
raise RuntimeError("No activity completed after wait")
diff --git a/server/tests/services/agent_context/test_conversation.py b/server/tests/services/agent_context/test_conversation.py
index fbdc5ae5..e14f0b45 100644
--- a/server/tests/services/agent_context/test_conversation.py
+++ b/server/tests/services/agent_context/test_conversation.py
@@ -95,6 +95,25 @@ _KEY = {
}
+async def test_blocked_tool_turn_is_saved_once_and_stale_results_are_ignored(conversation_database, isolated_listeners):
+ from services.agent_context.conversation import append_tool_results
+ from services.llm.protocol import ToolCall
+
+ assistant = dict(message_to_wire(Message(role="assistant", tool_calls=[ToolCall(id="call-1", name="model", args={})])))
+ tool_result = dict(message_to_wire(Message(role="tool", tool_call_id="call-1", name="model", content='{"error":"blocked","hint":"Fix billing"}')))
+ await save_conversation(conversation_database, **_KEY, messages=[_wire("user", "go"), assistant])
+ assert await append_tool_results(conversation_database, **_KEY, results=[tool_result])
+ assert not await append_tool_results(conversation_database, **_KEY, results=[tool_result])
+ stored = await load_conversation(conversation_database, **_KEY)
+ assert [m["role"] for m in stored] == ["user", "assistant", "tool"]
+ assert stored[-1]["content"] == tool_result["content"]
+
+ await save_conversation(conversation_database, **_KEY, messages=[_wire("assistant", "A later turn")])
+ assert not await append_tool_results(conversation_database, **_KEY, results=[tool_result])
+ assert len(await load_conversation(conversation_database, **_KEY)) == 1
+
+
def _sans_ts(messages: list[dict]) -> list[dict]:
return [{k: v for k, v in m.items() if k != "ts"} for m in messages]
diff --git a/server/tests/services/test_tool_input_security.py b/server/tests/services/test_tool_input_security.py
index cc84310c..2b905d40 100644
--- a/server/tests/services/test_tool_input_security.py
+++ b/server/tests/services/test_tool_input_security.py
@@ -87,6 +87,22 @@ def test_tool_input_defaults_to_params():
assert CurrentTimeToolNode.tool_input_model() is CurrentTimeToolNode.Params
+async def test_node_error_recovery_metadata_reaches_agent_tools():
+ from services.plugin import NodeUserError
+
+ class BlockedTool(_ConfiguredToolNode, abstract=True):
+ @Operation("run")
+ async def run(self, ctx, params):
+ raise NodeUserError("Spending cap reached", hint="Review billing", requires_user_action=True)
+
+ result = await BlockedTool().execute_as_tool(
+ {"endpoint": "test"}, {"endpoint": "test"},
+ NodeContext(node_id="tool-1", node_type=BlockedTool.type, raw={}),
+ )
+ assert result == {"error": "Spending cap reached", "hint": "Review billing", "requires_user_action": True, "retryable": False}
+
+
def test_native_validation_does_not_materialize_omitted_defaults():
spec = AgentToolSpec(
definition=ToolDef(
diff --git a/server/tests/temporal/test_agent_llm_contract.py b/server/tests/temporal/test_agent_llm_contract.py
index 313d4176..8e0edd3d 100644
--- a/server/tests/temporal/test_agent_llm_contract.py
+++ b/server/tests/temporal/test_agent_llm_contract.py
@@ -100,6 +100,9 @@ class TestWorkflowLoopContract:
prepared["compaction_threshold"] = 1
calls = []
phases = []
+ checkpoints = []
+ if from_tool:
+ prepared["conversation_key"] = {"workflow_id": "graph-1", "generation": 1, "agent_node_id": "agent-1"}
async def execute(name, *, args, **_kwargs):
calls.append(name)
@@ -117,6 +120,9 @@ class TestWorkflowLoopContract:
if name == "agent.broadcast_progress":
phases.append(args[0])
return {}
+ if name == "agent.persist_turn":
+ checkpoints.append(args[0])
+ return {}
if name in {"agent.skill.clear", "agent.persist_turn"}:
return {}
raise AssertionError(f"Unexpected activity: {name}")
@@ -131,6 +137,9 @@ class TestWorkflowLoopContract:
failure = next(p for p in phases if p.get("status") == "error")
assert failure["hint"] == error.hint
assert "secret" not in str(failure)
+ if from_tool:
+ assert len(checkpoints) == 1
+ assert checkpoints[0]["tool_results"][0]["tool_call_id"] == "call-1"
@pytest.mark.asyncio
async def test_llm_step_payload_shape_and_heartbeat(
diff --git a/server/tests/temporal/test_durability_hardening.py b/server/tests/temporal/test_durability_hardening.py
index 408484c3..ce96ed6a 100644
--- a/server/tests/temporal/test_durability_hardening.py
+++ b/server/tests/temporal/test_durability_hardening.py
@@ -1248,6 +1248,31 @@ async def test_machina_failure_schedules_the_circuit_breaker_activity(monkeypatc
assert captured == {}
+async def test_prepare_failure_keeps_owner_recovery_through_child_wrappers(monkeypatch):
+ import asyncio
+ from temporalio.exceptions import ApplicationError
+ from services.temporal import workflow as module
+
+ monkeypatch.setattr(module.workflow, "patched", lambda _name: True)
+ error = RuntimeError("Child workflow failed")
+ error.cause = ApplicationError(
+ "Provider credential missing", {"hint": "Connect the provider", "requires_user_action": True},
+ type="MissingAgentProviderCredential", non_retryable=True,
+ )
+ future = asyncio.get_running_loop().create_future()
+ future.set_exception(error)
+ node, result = await module.MachinaWorkflow()._wait_any_complete({"agent": future})
+ assert node == "agent"
+ assert result["requires_user_action"] is True
+ assert result["error"] == "Provider credential missing"
+ assert result["hint"] == "Connect the provider"
+
+ # Histories recorded before the propagation change keep their result.
+ monkeypatch.setattr(module.workflow, "patched", lambda _name: False)
+ assert module.MachinaWorkflow._node_failure(error) == {"success": False, "error": "Child workflow failed"}
+
+
def test_can_edit_capability_is_server_owned():
"""Canvas editability is a capability the backend computes — the FE
renders it and never re-derives the rule from state strings. Paused