Self-improving AI Employees for everyone
Python ai-agentsai-employeeai-employeesassistantautonomous-agentsco-employeescompanygrok-botinstinctloop-agentsmuseown-your-datapersonalself-hosted
fix(agents): pause on provider blockers and show recovery hints
trohitg9195cf035a29
diff --git a/client/src/contexts/WebSocketContext.tsx b/client/src/contexts/WebSocketContext.tsxindex d92ab9ca..70dd0520 100644--- a/client/src/contexts/WebSocketContext.tsx+++ b/client/src/contexts/WebSocketContext.tsx@@ -176,6 +176,8 @@ export interface WorkflowControlStatus {/** Server-owned canvas-editability capability (backend SSOT — the FE* renders it, never re-derives it from state strings). */can_edit: boolean;+ pause_reason?: string | null;+ pause_detail?: string | null;started_at?: string | null;updated_at?: string | null;error?: string | null;diff --git a/client/src/features/home/__tests__/employeeView.test.tsx b/client/src/features/home/__tests__/employeeView.test.tsxindex a275cca2..dd5ff824 100644--- a/client/src/features/home/__tests__/employeeView.test.tsx+++ b/client/src/features/home/__tests__/employeeView.test.tsx@@ -130,6 +130,13 @@ describe('EmployeeView', () => {expect(screen.getByRole('button', { name: 'Resume' })).toBeEnabled();});+ it('keeps the recovery hint visible for a failure-paused employee after reopening', () => {+ const why = 'Spending cap reached. Review billing before resuming.';+ renderPage(summary({ status: 'attention', talk: TALK_ON }, { state: 'paused', pause_reason: 'failures', pause_detail: why }));+ expect(screen.getByRole('alert')).toHaveTextContent(why);+ expect(screen.getByRole('button', { name: 'Resume' })).toBeEnabled();+ });+it('says to connect an AI model, and opens that dialog, when Start is refused for want of one', async () => {useHomeStore.setState({ connectAIOpen: false });actions.startEmployee.mockRejectedValue(new Error('needs_ai'));diff --git a/client/src/features/home/__tests__/talk.test.tsx b/client/src/features/home/__tests__/talk.test.tsxindex fb54c613..7ead1120 100644--- a/client/src/features/home/__tests__/talk.test.tsx+++ b/client/src/features/home/__tests__/talk.test.tsx@@ -129,6 +129,20 @@ describe('the thread', () => {});describe('sending', () => {+ it('shows the node failure and hint without waiting for another model answer', async () => {+ renderTalk(employee());+ const box = await screen.findByRole('textbox', { name: 'Message Maya' });+ fireEvent.change(box, { target: { value: 'Hello' } });+ fireEvent.keyDown(box, { key: 'Enter' });+ await screen.findByText('Hello');+ act(() => useNodeStatusStore.getState().setStatus('w1', AGENT, {+ status: 'error', data: { error: 'Spending cap reached.', hint: 'Review billing before resuming.' },+ }));+ expect(screen.getByRole('alert')).toHaveTextContent('Spending cap reached. Review billing before resuming.');+ expect(screen.queryByText('Working…')).not.toBeInTheDocument();+ expect(screen.queryByText('No answer from Maya yet.')).not.toBeInTheDocument();+ });+it('shows the message at once and thinks until the answer lands', async () => {renderTalk(employee());const box = await screen.findByRole('textbox', { name: 'Message Maya' });@@ -271,6 +285,17 @@ describe('useReplyWait', () => {vi.useRealTimers();});+ it('ignores an old failure when a new message begins', () => {+ act(() => useNodeStatusStore.getState().setStatus('w1', AGENT, { status: 'error', data: { error: 'Old failure' } }));+ const { result } = renderHook(() => useReplyWait('w1', AGENT, []));+ act(() => result.current.begin());+ expect(result.current).toMatchObject({ waiting: true, failure: null });+ act(() => useNodeStatusStore.getState().setStatus('w1', AGENT, { status: 'error', data: { error: 'New failure', hint: 'Fix the key' } }));+ expect(result.current).toMatchObject({ waiting: false, unanswered: false, failure: { error: 'New failure', hint: 'Fix the key' } });+ act(() => result.current.begin());+ expect(result.current.failure).toBeNull();+ });+it('gives up when the agent never picks the message up', () => {const { result } = renderHook(() => useReplyWait('w1', AGENT, []));act(() => result.current.begin());diff --git a/client/src/features/home/data/talk.ts b/client/src/features/home/data/talk.tsindex 35a60e23..4d09f5d6 100644--- a/client/src/features/home/data/talk.ts+++ b/client/src/features/home/data/talk.ts@@ -193,6 +193,8 @@ export function useSendTalkMessage(workflowId: string) {interface Wait {since: number;+ /** Ignore an error left over from a previous message. */+ initialStatus: object | undefined;/** The newest answer when the message went out; another one answers it. */repliedTo: string | null;/** The agent has been seen working. */@@ -206,6 +208,7 @@ export interface ReplyWait {waiting: boolean;/** The last wait ended with no answer. */unanswered: boolean;+ failure: { error: string; hint?: string } | null;/** A message just went to the agent. */begin: () => void;/** It did not after all (refused, or waiting for Resume). */@@ -219,26 +222,37 @@ export function useReplyWait(): ReplyWait {const [wait, setWait] = useState<Wait | null>(null);const [unanswered, setUnanswered] = useState(false);- const status = useNodeStatusStore(+ const [failure, setFailure] = useState<ReplyWait['failure']>(null);+ const nodeStatus = useNodeStatusStore(useCallback(- (state) => (agentNodeId ? state.allStatuses[workflowId]?.[agentNodeId]?.status : undefined),+ (state) => (agentNodeId ? state.allStatuses[workflowId]?.[agentNodeId] : undefined),[workflowId, agentNodeId],),);+ const status = nodeStatus?.status;const replyId = latestReplyId(messages);// An answer ends the wait. The agent working, then stopping, settles it.useEffect(() => {+ if (status === 'executing') setFailure(null);if (!wait) return;if (replyId !== wait.repliedTo) {setWait(null);setUnanswered(false);+ setFailure(null);+ } else if (status === 'error' && nodeStatus !== wait.initialStatus && typeof nodeStatus?.data?.error === 'string') {+ setFailure({+ error: nodeStatus.data.error,+ hint: typeof nodeStatus.data.hint === 'string' ? nodeStatus.data.hint : undefined,+ });+ setWait(null);+ setUnanswered(false);} else if (status === 'executing') {if (!wait.working || wait.settling) setWait({ ...wait, working: true, settling: false });} else if (wait.working && !wait.settling) {setWait({ ...wait, settling: true });}- }, [wait, status, replyId]);+ }, [wait, status, nodeStatus, replyId]);// Never picked up, never finished, or finished with nothing to say.useEffect(() => {@@ -254,11 +268,13 @@ export function useReplyWait(}, [wait]);const begin = useCallback(() => {- setWait({ since: Date.now(), repliedTo: latestReplyId(messages), working: false, settling: false });+ const initialStatus = agentNodeId ? useNodeStatusStore.getState().allStatuses[workflowId]?.[agentNodeId] : undefined;+ setWait({ since: Date.now(), initialStatus, repliedTo: latestReplyId(messages), working: false, settling: false });setUnanswered(false);- }, [messages]);+ setFailure(null);+ }, [messages, workflowId, agentNodeId]);const cancel = useCallback(() => setWait(null), []);- return { waiting: wait !== null, unanswered, begin, cancel };+ return { waiting: wait !== null, unanswered, failure, begin, cancel };}// ----- Turn on Talk, Apply -----diff --git a/client/src/features/home/employee/EmployeeTalk.tsx b/client/src/features/home/employee/EmployeeTalk.tsxindex 4f769f30..8bc2497f 100644--- a/client/src/features/home/employee/EmployeeTalk.tsx+++ b/client/src/features/home/employee/EmployeeTalk.tsx@@ -23,6 +23,7 @@ import { ArrowUp, Monitor } from 'lucide-react';import { Suspense, lazy, useEffect, useLayoutEffect, useRef, useState, type KeyboardEvent, type ReactNode } from 'react';import { useQueryClient } from '@tanstack/react-query';import { ActionButton } from '@/components/ui/action-button';+import { Alert, AlertDescription } from '@/components/ui/alert';import { Button } from '@/components/ui/button';import { Skeleton } from '@/components/ui/skeleton';import { Textarea } from '@/components/ui/textarea';@@ -163,7 +164,7 @@ function Conversation({ employee, control, drafts }: { employee: EmployeeSummaryconst thread = useTalkThread(workflowId);const messages = thread.data;const send = useSendTalkMessage(workflowId);- const { waiting, unanswered, begin, cancel } = useReplyWait(workflowId, employee.talk.agent_node_id, messages);+ const { waiting, unanswered, failure, begin, cancel } = useReplyWait(workflowId, employee.talk.agent_node_id, messages);const mode = talkMode(employee.control);const [draft, setDraft] = useState('');const [queued, setQueued] = useState(false);@@ -220,6 +221,9 @@ function Conversation({ employee, control, drafts }: { employee: EmployeeSummaryconst rows = messages ? threadRows(messages) : [];const notice = talkNoticeText(mode, name, queued);+ const failureText = mode === 'queue' && employee.control.pause_reason === 'failures'+ ? employee.control.pause_detail || (employee.task?.label === 'Paused' ? employee.task.text : null)+ : failure ? [failure.error, failure.hint].filter(Boolean).join(' ') : null;const now = new Date();return (@@ -255,11 +259,12 @@ function Conversation({ employee, control, drafts }: { employee: EmployeeSummary<p className={cn('m-0', BUBBLE, THEIRS, 'text-fg-muted')}>Working…</p></div>)}- {unanswered && !waiting && <p className={cn('m-0 text-xs text-fg-muted', PAST_AVATAR)}>No answer from {name} yet.</p>}+ {unanswered && !waiting && !failureText && <p className={cn('m-0 text-xs text-fg-muted', PAST_AVATAR)}>No answer from {name} yet.</p>}</div>{drafts}</div><div className="sticky bottom-0 z-10 mt-auto flex flex-col gap-2.5 bg-bg-app pt-4 pb-3">+ {failureText && <Alert variant="destructive"><AlertDescription>{failureText}</AlertDescription></Alert>}{employee.pending_changes && <PendingChangesNotice employee={employee} />}{employee.browser_request && <BrowserNotice employee={employee} />}{notice && (diff --git a/server/services/agent_runtime.py b/server/services/agent_runtime.pyindex db8a56d8..4ca82dde 100644--- a/server/services/agent_runtime.py+++ b/server/services/agent_runtime.py@@ -149,9 +149,7 @@ async def run_native_llm_step(if not error.retryable or attempt + 1 >= attempts:if not translate_errors:raise- from services.plugin import NodeUserError-- raise NodeUserError(error.user_message) from error+ raise error.as_node_error() from errordelay = (error.retry_afterif error.retry_after is not None@@ -340,6 +338,7 @@ async def run_native_agent_loop(specs = _tool_specs_by_name(current_tools)iteration_new_tools: List[AgentToolSpec] = []+ blocked_error = Nonefor call_index, call in enumerate(calls, start=1):if call.name not in specs:result: Any = {@@ -377,7 +376,12 @@ async def run_native_agent_loop(call.name,exc,)- result = {"error": str(exc)}+ from services.plugin import NodeUserError++ result = exc.as_dict() if isinstance(exc, NodeUserError) else {"error": str(exc)}++ if isinstance(result, dict) and result.get("error") and result.get("requires_user_action") is True:+ blocked_error = blocked_error or resultif (rebind_from_operations is not None@@ -420,6 +424,16 @@ async def run_native_agent_loop(# One save per turn covers every tool result appended above.await save_now()+ if blocked_error:+ # All tool-call/result pairs are saved, but no extra LLM request+ # (including compaction) is spent on an owner-only repair.+ from services.plugin import NodeUserError++ raise NodeUserError(+ blocked_error["error"], hint=blocked_error.get("hint"),+ requires_user_action=True,+ )+if iteration_new_tools:current_tools.extend(iteration_new_tools)logger.info(diff --git a/server/services/deployment/handlers.py b/server/services/deployment/handlers.pyindex 7b933f35..f845e5af 100644--- a/server/services/deployment/handlers.py+++ b/server/services/deployment/handlers.py@@ -2315,7 +2315,10 @@ async def _clear_failure_streak(database, control) -> None:logger.debug(f"Failed to clear failure streak: {exc}")-async def pause_generation_on_failure(*, workflow_id: str, reason: str) -> Dict[str, Any]:+async def pause_generation_on_failure(+ *, workflow_id: str, reason: str, requires_user_action: bool = False,+ hint: Optional[str] = None, generation: Optional[int] = None,+) -> Dict[str, Any]:"""Circuit breaker: pause a controlled deployment after failed runs.Called by the ``workflow_control.pause_on_failure.v1`` activity that@@ -2346,7 +2349,11 @@ async def pause_generation_on_failure(*, workflow_id: str, reason: str) -> Dict["reason": "not_running","status": getattr(control, "status", None),}- threshold = _pause_on_failure_threshold()+ if generation is not None and int(generation) != control.generation:+ return {"paused": False, "reason": "stale_generation"}+ # Account/configuration blockers cannot heal by spending two more runs.+ # Ordinary node/input failures retain the existing circuit breaker.+ threshold = 1 if requires_user_action else _pause_on_failure_threshold()if threshold > 1:streak = await _bump_failure_streak(service.database,@@ -2381,7 +2388,10 @@ async def pause_generation_on_failure(*, workflow_id: str, reason: str) -> Dict["expected_revision": control.revision,"idempotency_key": f"pause-on-failure:{control.id}:{control.revision}","_pause_reason": PAUSE_REASON_FAILURES,- "_pause_detail": _failure_pause_detail(reason),+ "_pause_detail": (+ " ".join(f"{reason} {hint or ''}".split())[:1000]+ if requires_user_action else _failure_pause_detail(reason)+ ),},None,)diff --git a/server/services/llm/protocol.py b/server/services/llm/protocol.pyindex ccb1bc8c..5bb2b4f3 100644--- a/server/services/llm/protocol.py+++ b/server/services/llm/protocol.py@@ -308,6 +308,7 @@ class LLMErrorCategory(str, Enum):AUTHENTICATION = "authentication"PERMISSION = "permission"+ BILLING = "billing"RATE_LIMIT = "rate_limit"INVALID_REQUEST = "invalid_request"NOT_FOUND = "not_found"@@ -402,6 +403,9 @@ class LLMError(Exception):f"{provider} denied this request. ""Check account and model access."),+ LLMErrorCategory.BILLING.value: (+ f"{provider} blocked this request because a spending limit or available credits were exhausted."+ ),LLMErrorCategory.RATE_LIMIT.value: (f"{provider} is rate-limiting requests. ""Retry after a short delay."@@ -430,6 +434,35 @@ class LLMError(Exception):}return messages.get(category, messages[LLMErrorCategory.UNKNOWN.value])+ @property+ def hint(self) -> Optional[str]:+ """Fixed recovery advice; never copy the provider's raw response."""+ return {+ LLMErrorCategory.BILLING: "Review the provider project's billing, spending cap, and credits. Resume after access is restored; repeating the request will not fix it.",+ LLMErrorCategory.AUTHENTICATION: "Update the provider credential in Settings > Connectors, then resume.",+ LLMErrorCategory.PERMISSION: "Check the credential's project and model permissions, then resume after access is restored.",+ LLMErrorCategory.NOT_FOUND: "Choose a model available to this provider and credential, then apply the change and resume.",+ LLMErrorCategory.PROTOCOL: "Correct the provider's base URL and API configuration, then resume.",+ LLMErrorCategory.CONTEXT_LENGTH: "Reduce or clear the agent's context before trying again.",+ LLMErrorCategory.INVALID_REQUEST: "Check the model's supported request settings before trying again.",+ }.get(self.category)++ @property+ def requires_user_action(self) -> bool:+ return not self.retryable and self.category in {+ LLMErrorCategory.BILLING, LLMErrorCategory.AUTHENTICATION,+ LLMErrorCategory.PERMISSION, LLMErrorCategory.NOT_FOUND,+ LLMErrorCategory.PROTOCOL,+ }++ def as_node_error(self):+ from services.plugin import NodeUserError++ return NodeUserError(+ self.user_message, hint=self.hint,+ requires_user_action=self.requires_user_action,+ )+@classmethoddef from_exception(cls, provider: str, exc: BaseException) -> "LLMError":status = _optional_int(@@ -699,6 +732,15 @@ def _classify_error() -> LLMErrorCategory:name = type(exc).__name__.lower()message = str(exc).lower()+ # A quota/credit exhaustion can arrive as 429, but backoff cannot fix it.+ # Match narrow billing signatures before the generic HTTP classifiers.+ if status in {400, 402, 403, 429} and (+ status == 402 or any(marker in message for marker in (+ "spend cap breached", "insufficient_quota", "insufficient credits",+ "credit balance is too low", "billing_hard_limit_reached",+ ))+ ):+ return LLMErrorCategory.BILLINGif status == 401 or "authentication" in name or "api key" in message:return LLMErrorCategory.AUTHENTICATIONif status == 403 or "permission" in name:diff --git a/server/services/llm/unifier.py b/server/services/llm/unifier.pyindex 3b14814a..d378472c 100644--- a/server/services/llm/unifier.py+++ b/server/services/llm/unifier.py@@ -129,13 +129,13 @@ class ChatUnifier:self._log_failure("LLM provider request failed", error, entry, provider)if not translate_errors:raise- raise NodeUserError(error.user_message) from error+ raise error.as_node_error() from errorexcept spec.sdk_exception_types as e:error = LLMError.from_exception(provider, e)self._log_failure("LLM provider request failed", error, entry, provider)if not translate_errors:raise error from e- raise NodeUserError(error.user_message) from error+ raise error.as_node_error() from errorexcept (ValueError, TypeError, OSError) as e:# Only normalize generic configuration/transport failures raised# while constructing a provider client. Once a client exists,@@ -162,7 +162,7 @@ class ChatUnifier:)if not translate_errors:raise error from e- raise NodeUserError(error.user_message) from error+ raise error.as_node_error() from errorfinally:if entry is not None:await self._release_client(entry)@@ -185,11 +185,11 @@ class ChatUnifier:models = await entry.client.fetch_models(api_key)except LLMError as error:self._log_failure("LLM model-list request failed", error, entry, provider)- raise NodeUserError(error.user_message) from error+ raise error.as_node_error() from errorexcept spec.sdk_exception_types as e:error = LLMError.from_exception(provider, e)self._log_failure("LLM model-list request failed", error, entry, provider)- raise NodeUserError(error.user_message) from error+ raise error.as_node_error() from errorexcept (ValueError, TypeError, OSError) as e:if entry is not None:raise@@ -210,7 +210,7 @@ class ChatUnifier:category=error.category.value,retryable=error.retryable,)- raise NodeUserError(error.user_message) from error+ raise error.as_node_error() from errorfinally:if entry is not None:await self._release_client(entry)diff --git a/server/services/plugin/base.py b/server/services/plugin/base.pyindex fb2b80d9..ba82786e 100644--- a/server/services/plugin/base.py+++ b/server/services/plugin/base.py@@ -60,6 +60,25 @@ class NodeUserError(Exception):unexpected failures that warrant a stacktrace in the operator log."""+ def __init__(self, message: str, *, hint: Optional[str] = None, requires_user_action: bool = False):+ super().__init__(message)+ self.hint = hint+ self.requires_user_action = requires_user_action++ def as_dict(self) -> Dict[str, Any]:+ """Public recovery information, shared by nodes and agent tools.++ Ordinary input errors remain correctable by the model. Account or+ configuration blockers explicitly require the owner, so another+ model turn cannot fix them. Callers must supply safe text only.+ """+ return {+ "error": str(self),+ "error_type": "NodeUserError",+ **({"hint": self.hint} if self.hint else {}),+ **({"requires_user_action": True, "retryable": False} if self.requires_user_action else {}),+ }+NODE_WAIT_INTERRUPTED = "NodeWaitInterrupted"@@ -639,7 +658,7 @@ class BaseNode:# LLM gets the message in the structured response and can# retry with corrected input.logger.warning("[%s] %s op %s: %s", self.type, op_name, type(e).__name__, e)- return self._wrap_error(start_time=start_time, error=str(e), error_type="NodeUserError")+ return self._wrap_error(start_time=start_time, error=str(e), extra=e.as_dict())except NodeWaitInterrupted as e:# Not a failure: the process is going away mid-wait. The activity# wrapper turns this envelope into a retryable error.@@ -781,7 +800,7 @@ class BaseNode:def _tool_error_payload(self, envelope: Dict[str, Any]) -> Dict[str, Any]:"""Copy only explicitly declared public metadata from a tool error."""result = {"error": envelope.get("error", "tool execution failed")}- for name in self.tool_error_fields:+ for name in self.tool_error_fields | {"hint", "requires_user_action", "retryable"}:if name != "error" and name in envelope and envelope[name] is not None:result[name] = envelope[name]return result@@ -1174,7 +1193,8 @@ class BaseNode:await broadcaster.update_node_status(node_id,"error",- {"error": error, "execution_id": execution_id},+ {"error": error, "execution_id": execution_id,+ **{key: result[key] for key in ("hint", "requires_user_action", "error_type") if key in result}},workflow_id=workflow_id,)diff --git a/server/services/temporal/activities.py b/server/services/temporal/activities.pyindex a9e674d0..64339bd1 100644--- a/server/services/temporal/activities.py+++ b/server/services/temporal/activities.py@@ -614,6 +614,11 @@ async def pause_workflow_on_failure_activity(payload: Dict[str, Any]) -> Dict[streturn await pause_generation_on_failure(workflow_id=str(payload.get("workflow_id") or ""),reason=str(payload.get("reason") or "run_failed"),+ **({+ "requires_user_action": True,+ "hint": payload.get("hint"),+ "generation": payload.get("generation"),+ } if payload.get("requires_user_action") is True else {}),)except Exception as exc: # noqa: BLE001 — non-fatalactivity.logger.warning(f"pause_workflow_on_failure_activity failed for " f"workflow={payload.get('workflow_id')!r}: {exc}")diff --git a/server/services/temporal/agent_activities.py b/server/services/temporal/agent_activities.pyindex 2a6f4f1a..629751db 100644--- a/server/services/temporal/agent_activities.py+++ b/server/services/temporal/agent_activities.py@@ -183,6 +183,8 @@ def _as_temporal_llm_error(error: Any):"request_id": getattr(error, "request_id", None),"retry_after": getattr(error, "retry_after", None),"retry_after_raw": getattr(error, "retry_after_raw", None),+ **({"hint": error.hint} if getattr(error, "hint", None) else {}),+ **({"requires_user_action": True} if getattr(error, "requires_user_action", False) else {}),}# Honor the provider's own pacing: a 429 with Retry-After should wait# exactly that long before the next attempt instead of the policy's@@ -623,6 +625,12 @@ async def broadcast_agent_progress(payload: Dict[str, Any]) -> Dict[str, Any]:**({"phase": phase} if phase else {}),**capability_data,}+ if status == "error":+ lifecycle_data.update({+ key: payload[key]+ for key in ("error", "error_type", "hint", "requires_user_action")+ if payload.get(key) is not None+ })if phase == "starting":lifecycle_data.update({"active_skills": [], "last_skills": [], "last_tool_name": None, "last_capability": None})await broadcaster.update_node_status(diff --git a/server/services/temporal/agent_workflow.py b/server/services/temporal/agent_workflow.pyindex ae3e1330..b478159a 100644--- a/server/services/temporal/agent_workflow.py+++ b/server/services/temporal/agent_workflow.py@@ -919,6 +919,15 @@ class AgentWorkflow:cause_message = str(getattr(cause, "message", "") or "").strip()+ recovery = {}+ if cause_type.startswith("LLMError."):+ diagnostics = getattr(cause, "details", ())+ if diagnostics and isinstance(diagnostics[0], dict):+ recovery = {+ key: diagnostics[0][key]+ for key in ("hint", "requires_user_action", "retryable")+ if diagnostics[0].get(key) is not None+ }safe_activity_types = {"MissingAgentProviderCredential","EmptyAgentPrompt",@@ -965,11 +974,14 @@ class AgentWorkflow:max_iterations,phase="failed",status="error",+ extra={"error": detail, "error_type": "NodeUserError", **recovery}+ if recovery else None,)return {"success": False,"error": f"LLM step failed: {detail}","error_type": "LLMStepError",+ **recovery,"result": {"iterations": iteration + 1,"usage": usage_total,@@ -1605,6 +1617,7 @@ class AgentWorkflow:await _cleanup_cancelled_delegations()raise+ blocked_error = Nonefor call_index, call in enumerate(calls):if call.get("parse_error"):# Native adapters retain malformed arguments rather than@@ -1860,12 +1873,18 @@ class AgentWorkflow:call_index,None,)+ child_recovery = {+ key: tool_result[key]+ for key in ("hint", "requires_user_action", "retryable")+ if isinstance(tool_result, dict) and key in tool_result+ }tool_result = {"success": child_succeeded,"status": "submitted" if child_succeeded else "failed","result": child_response,"usage": child_summary.get("usage"),**({"error": child_error} if child_error else {}),+ **child_recovery,}if next_delegation_to_start < len(delegation_call_indices):await _start_delegation(@@ -1912,6 +1931,19 @@ class AgentWorkflow:"delegation_result": delegated["result"],"delegation_usage": delegated.get("usage"),}+ if (+ isinstance(tool_result, dict)+ and tool_result.get("error")+ and tool_result.get("requires_user_action") is True+ and workflow.patched("agent-user-action-error-v1")+ ):+ blocked_error = blocked_error or {+ "error": tool_result["error"],+ "error_type": "NodeUserError",+ "hint": tool_result.get("hint"),+ "requires_user_action": True,+ "retryable": False,+ }tool_content = _serialise_tool_result(tool_result)if (tool_output_limit@@ -2128,6 +2160,25 @@ class AgentWorkflow:)# ---- Transcript pressure and compaction ----------------------+ if blocked_error:+ # Complete/persist the current tool turn, then stop before+ # another model request or paid compaction can be scheduled.+ await workflow.execute_activity(+ "agent.skill.clear",+ args=[{"workflow_id": payload.get("workflow_id"),+ "execution_id": task_scope_execution_id,+ "agent_node_id": agent_node_id}],+ activity_id="clear-active-skills-blocked",+ start_to_close_timeout=PERSIST_TURN_TIMEOUT,+ retry_policy=AGENT_ACTIVITY_RETRY,+ )+ await self._emit_phase(+ agent_node_id, agent_workflow_id, iteration, max_iterations,+ phase="failed", status="error", extra=blocked_error,+ )+ return {"success": False, **blocked_error,+ "result": {"iterations": iteration + 1, "usage": usage_total}}+# Summarize, then swap messages: no memory-node gate, no# checkpoint machinery. A later rollover resumes from the# compacted state for free, because the next LLM turn sends anddiff --git a/server/services/temporal/workflow.py b/server/services/temporal/workflow.pyindex 8f4dfbe1..f1da1d6d 100644--- a/server/services/temporal/workflow.py+++ b/server/services/temporal/workflow.py@@ -246,12 +246,21 @@ class MachinaWorkflow:spawned_by_trigger = any(node.get("_pre_executed") for node in nodes)if not workflow_id or not spawned_by_trigger:return+ failure = errors[0] or {}+ recovery = {}+ if failure.get("requires_user_action") is True:+ recovery = {+ "requires_user_action": True,+ "hint": failure.get("hint"),+ "generation": workflow_data.get("generation"),+ }try:await workflow.execute_activity("workflow_control.pause_on_failure",{"workflow_id": workflow_id,"reason": str((errors[0] or {}).get("error", "run_failed"))[:500],+ **recovery,},start_to_close_timeout=timedelta(seconds=30),retry_policy=QUICK_ACTIVITY_RETRY,@@ -700,6 +709,7 @@ class MachinaWorkflow:error_info = {"node_id": done_id,"error": result.get("error", "Unknown error"),+ **{key: result[key] for key in ("hint", "requires_user_action", "retryable") if key in result},}errors.append(error_info)workflow.logger.error(f"Node failed: {done_id} - {error_info['error']}")diff --git a/server/tests/llm/test_protocol_codec.py b/server/tests/llm/test_protocol_codec.pyindex 55bd902d..eb2e2bc8 100644--- a/server/tests/llm/test_protocol_codec.py+++ b/server/tests/llm/test_protocol_codec.py@@ -179,6 +179,26 @@ def test_structured_error_exposes_only_category_based_user_message():assert "Bearer secret" not in error.user_message[email protected](("status", "message"), [+ (403, "Spend cap breached for project: projects/private-id for service: aiplatform.googleapis.com"),+ (429, "insufficient_quota: private account details"),+ (400, "Your credit balance is too low: secret"),+ (402, "Payment required: secret"),+])+def test_billing_errors_need_owner_action_instead_of_retry(status, message):+ exc = Exception(message)+ exc.status_code = status+ error = LLMError.from_exception("gemini", exc)+ assert error.category == LLMErrorCategory.BILLING+ assert not error.retryable+ node_error = error.as_node_error()+ assert node_error.requires_user_action+ assert "spending" in node_error.hint+ assert "private" not in str(node_error)+ assert "secret" not in str(node_error)+ assert node_error.as_dict()["retryable"] is False++@pytest.mark.parametrize(("provider", "not_found", "context_length"),[diff --git a/server/tests/services/test_native_agent_runtime.py b/server/tests/services/test_native_agent_runtime.pyindex 975b95cd..8a95ee78 100644--- a/server/tests/services/test_native_agent_runtime.py+++ b/server/tests/services/test_native_agent_runtime.py@@ -49,6 +49,34 @@ class _Database:return None[email protected]("needs_owner", [True, False])+async def test_tool_errors_only_stop_the_loop_when_the_owner_must_fix_them(needs_owner):+ response = LLMResponse(tool_calls=[ToolCall(id="call-1", name="model", args={})])+ unifier = _FakeUnifier([response, LLMResponse(content="fixed")])+ save = AsyncMock()+ async def execute(_name, _args):+ raise NodeUserError("Blocked", hint="Fix the account" if needs_owner else "Supply a value", requires_user_action=needs_owner)++ kwargs = dict(provider="gemini", api_key="test", model="test", temperature=0,+ max_tokens=100, initial_messages=[Message(role="user", content="go")],+ tools=[AgentToolSpec(definition=ToolDef(name="model", description="model", parameters={}))],+ tool_executor=execute, conversation_saver=save)+ if needs_owner:+ with pytest.raises(NodeUserError) as caught:+ await run_native_agent_loop(unifier, **kwargs)+ assert caught.value.requires_user_action+ assert caught.value.hint == "Fix the account"+ assert len(unifier.calls) == 1+ # The failed tool result is saved with its matching call, without+ # spending a second model turn to explain the error.+ assert save.await_args.args[0][-1].role == "tool"+ else:+ result = await run_native_agent_loop(unifier, **kwargs)+ assert result["response"].content == "fixed"+ assert len(unifier.calls) == 2++def test_in_process_compaction_usage_joins_execution_wide_total():from services.ai import _accumulate_compaction_usagediff --git a/server/tests/temporal/test_agent_llm_contract.py b/server/tests/temporal/test_agent_llm_contract.pyindex 3346f631..7bdbb7de 100644--- a/server/tests/temporal/test_agent_llm_contract.py+++ b/server/tests/temporal/test_agent_llm_contract.py@@ -87,6 +87,50 @@ def patched_workflow(monkeypatch):class TestWorkflowLoopContract:+ @pytest.mark.asyncio+ @pytest.mark.parametrize("from_tool", [False, True])+ async def test_owner_action_error_stops_before_another_model_or_compaction(self, monkeypatch, patched_workflow, from_tool):+ from services.llm.protocol import LLMError, LLMErrorCategory+ from services.temporal.agent_activities import _as_temporal_llm_error+ from services.temporal.agent_workflow import AgentWorkflow++ error = LLMError("secret provider body", "gemini", category=LLMErrorCategory.BILLING)+ prepared = _payload()+ prepared["compaction_threshold"] = 1+ calls = []+ phases = []++ async def execute(name, *, args, **_kwargs):+ calls.append(name)+ if name == "agent.prepare_payload":+ return prepared+ if name == "agent.execute_llm_step":+ if from_tool:+ return {"kind": "tool_calls", "calls": [{"id": "call-1", "name": "write_todos", "args": {}}],+ "usage": {"input_tokens": 10, "output_tokens": 2}}+ failure = RuntimeError("activity failed")+ failure.cause = _as_temporal_llm_error(error)+ raise failure+ if name == "node.writeTodos.v1":+ return {"success": False, **error.as_node_error().as_dict()}+ if name == "agent.broadcast_progress":+ phases.append(args[0])+ return {}+ if name in {"agent.skill.clear", "agent.persist_turn"}:+ return {}+ raise AssertionError(f"Unexpected activity: {name}")++ monkeypatch.setattr(patched_workflow, "execute_activity", execute)+ result = await AgentWorkflow().run({"node_id": "agent-1", "execution_id": "root-run-1"})+ assert result["success"] is False+ assert result["requires_user_action"] is True+ assert result["hint"] == error.hint+ assert calls.count("agent.execute_llm_step") == 1+ assert "agent.compact_context" not in calls+ failure = next(p for p in phases if p.get("status") == "error")+ assert failure["hint"] == error.hint+ assert "secret" not in str(failure)+@pytest.mark.asyncioasync def test_llm_step_payload_shape_and_heartbeat(self,diff --git a/server/tests/temporal/test_durability_hardening.py b/server/tests/temporal/test_durability_hardening.pyindex 2c327768..408484c3 100644--- a/server/tests/temporal/test_durability_hardening.py+++ b/server/tests/temporal/test_durability_hardening.py@@ -1118,6 +1118,30 @@ async def test_pause_on_failure_pauses_immediately_at_threshold_one(monkeypatch)assert request["expected_revision"] == running.revision+async def test_owner_action_blocker_pauses_once_without_spending_threshold_runs(monkeypatch):+ from services.deployment import handlers++ running = _boot_control("running")+ _breaker_settings(monkeypatch, threshold=3)+ monkeypatch.setattr(handlers, "_control_service", lambda: SimpleNamespace(database=_streak_database(running, {})))+ pause = AsyncMock(return_value={"success": True})+ monkeypatch.setattr(handlers, "handle_pause_workflow", pause)+ result = await handlers.pause_generation_on_failure(+ workflow_id="wf", reason="Spending cap reached", hint="Review billing before resuming.",+ requires_user_action=True, generation=running.generation,+ )+ assert result["paused"] is True+ pause.assert_awaited_once()+ assert pause.await_args.args[0]["_pause_detail"] == "Spending cap reached Review billing before resuming."+ pause.reset_mock()+ result = await handlers.pause_generation_on_failure(+ workflow_id="wf", reason="Old failure", requires_user_action=True, generation=running.generation - 1,+ )+ assert result["reason"] == "stale_generation"+ pause.assert_not_awaited()++@pytest.mark.asyncioasync def test_single_node_failure_does_not_trip_the_breaker(monkeypatch):"""The reported regression: one telegramSend NodeUserError on one