Skip to content

Self-improving AI Employees for everyone

fix(agents): pause on provider blockers and show recovery hints

trohitg9195cf035a29
diff --git a/client/src/contexts/WebSocketContext.tsx b/client/src/contexts/WebSocketContext.tsx
index d92ab9ca..70dd0520 100644
--- a/client/src/contexts/WebSocketContext.tsx
+++ b/client/src/contexts/WebSocketContext.tsx
@@ -176,6 +176,8 @@ export interface WorkflowControlStatus {
/** Server-owned canvas-editability capability (backend SSOT — the FE
* renders it, never re-derives it from state strings). */
can_edit: boolean;
+ pause_reason?: string | null;
+ pause_detail?: string | null;
started_at?: string | null;
updated_at?: string | null;
error?: string | null;
diff --git a/client/src/features/home/__tests__/employeeView.test.tsx b/client/src/features/home/__tests__/employeeView.test.tsx
index a275cca2..dd5ff824 100644
--- a/client/src/features/home/__tests__/employeeView.test.tsx
+++ b/client/src/features/home/__tests__/employeeView.test.tsx
@@ -130,6 +130,13 @@ describe('EmployeeView', () => {
expect(screen.getByRole('button', { name: 'Resume' })).toBeEnabled();
});
+ it('keeps the recovery hint visible for a failure-paused employee after reopening', () => {
+ const why = 'Spending cap reached. Review billing before resuming.';
+ renderPage(summary({ status: 'attention', talk: TALK_ON }, { state: 'paused', pause_reason: 'failures', pause_detail: why }));
+ expect(screen.getByRole('alert')).toHaveTextContent(why);
+ expect(screen.getByRole('button', { name: 'Resume' })).toBeEnabled();
+ });
+
it('says to connect an AI model, and opens that dialog, when Start is refused for want of one', async () => {
useHomeStore.setState({ connectAIOpen: false });
actions.startEmployee.mockRejectedValue(new Error('needs_ai'));
diff --git a/client/src/features/home/__tests__/talk.test.tsx b/client/src/features/home/__tests__/talk.test.tsx
index fb54c613..7ead1120 100644
--- a/client/src/features/home/__tests__/talk.test.tsx
+++ b/client/src/features/home/__tests__/talk.test.tsx
@@ -129,6 +129,20 @@ describe('the thread', () => {
});
describe('sending', () => {
+ it('shows the node failure and hint without waiting for another model answer', async () => {
+ renderTalk(employee());
+ const box = await screen.findByRole('textbox', { name: 'Message Maya' });
+ fireEvent.change(box, { target: { value: 'Hello' } });
+ fireEvent.keyDown(box, { key: 'Enter' });
+ await screen.findByText('Hello');
+ act(() => useNodeStatusStore.getState().setStatus('w1', AGENT, {
+ status: 'error', data: { error: 'Spending cap reached.', hint: 'Review billing before resuming.' },
+ }));
+ expect(screen.getByRole('alert')).toHaveTextContent('Spending cap reached. Review billing before resuming.');
+ expect(screen.queryByText('Working…')).not.toBeInTheDocument();
+ expect(screen.queryByText('No answer from Maya yet.')).not.toBeInTheDocument();
+ });
+
it('shows the message at once and thinks until the answer lands', async () => {
renderTalk(employee());
const box = await screen.findByRole('textbox', { name: 'Message Maya' });
@@ -271,6 +285,17 @@ describe('useReplyWait', () => {
vi.useRealTimers();
});
+ it('ignores an old failure when a new message begins', () => {
+ act(() => useNodeStatusStore.getState().setStatus('w1', AGENT, { status: 'error', data: { error: 'Old failure' } }));
+ const { result } = renderHook(() => useReplyWait('w1', AGENT, []));
+ act(() => result.current.begin());
+ expect(result.current).toMatchObject({ waiting: true, failure: null });
+ act(() => useNodeStatusStore.getState().setStatus('w1', AGENT, { status: 'error', data: { error: 'New failure', hint: 'Fix the key' } }));
+ expect(result.current).toMatchObject({ waiting: false, unanswered: false, failure: { error: 'New failure', hint: 'Fix the key' } });
+ act(() => result.current.begin());
+ expect(result.current.failure).toBeNull();
+ });
+
it('gives up when the agent never picks the message up', () => {
const { result } = renderHook(() => useReplyWait('w1', AGENT, []));
act(() => result.current.begin());
diff --git a/client/src/features/home/data/talk.ts b/client/src/features/home/data/talk.ts
index 35a60e23..4d09f5d6 100644
--- a/client/src/features/home/data/talk.ts
+++ b/client/src/features/home/data/talk.ts
@@ -193,6 +193,8 @@ export function useSendTalkMessage(workflowId: string) {
interface Wait {
since: number;
+ /** Ignore an error left over from a previous message. */
+ initialStatus: object | undefined;
/** The newest answer when the message went out; another one answers it. */
repliedTo: string | null;
/** The agent has been seen working. */
@@ -206,6 +208,7 @@ export interface ReplyWait {
waiting: boolean;
/** The last wait ended with no answer. */
unanswered: boolean;
+ failure: { error: string; hint?: string } | null;
/** A message just went to the agent. */
begin: () => void;
/** It did not after all (refused, or waiting for Resume). */
@@ -219,26 +222,37 @@ export function useReplyWait(
): ReplyWait {
const [wait, setWait] = useState<Wait | null>(null);
const [unanswered, setUnanswered] = useState(false);
- const status = useNodeStatusStore(
+ const [failure, setFailure] = useState<ReplyWait['failure']>(null);
+ const nodeStatus = useNodeStatusStore(
useCallback(
- (state) => (agentNodeId ? state.allStatuses[workflowId]?.[agentNodeId]?.status : undefined),
+ (state) => (agentNodeId ? state.allStatuses[workflowId]?.[agentNodeId] : undefined),
[workflowId, agentNodeId],
),
);
+ const status = nodeStatus?.status;
const replyId = latestReplyId(messages);
// An answer ends the wait. The agent working, then stopping, settles it.
useEffect(() => {
+ if (status === 'executing') setFailure(null);
if (!wait) return;
if (replyId !== wait.repliedTo) {
setWait(null);
setUnanswered(false);
+ setFailure(null);
+ } else if (status === 'error' && nodeStatus !== wait.initialStatus && typeof nodeStatus?.data?.error === 'string') {
+ setFailure({
+ error: nodeStatus.data.error,
+ hint: typeof nodeStatus.data.hint === 'string' ? nodeStatus.data.hint : undefined,
+ });
+ setWait(null);
+ setUnanswered(false);
} else if (status === 'executing') {
if (!wait.working || wait.settling) setWait({ ...wait, working: true, settling: false });
} else if (wait.working && !wait.settling) {
setWait({ ...wait, settling: true });
}
- }, [wait, status, replyId]);
+ }, [wait, status, nodeStatus, replyId]);
// Never picked up, never finished, or finished with nothing to say.
useEffect(() => {
@@ -254,11 +268,13 @@ export function useReplyWait(
}, [wait]);
const begin = useCallback(() => {
- setWait({ since: Date.now(), repliedTo: latestReplyId(messages), working: false, settling: false });
+ const initialStatus = agentNodeId ? useNodeStatusStore.getState().allStatuses[workflowId]?.[agentNodeId] : undefined;
+ setWait({ since: Date.now(), initialStatus, repliedTo: latestReplyId(messages), working: false, settling: false });
setUnanswered(false);
- }, [messages]);
+ setFailure(null);
+ }, [messages, workflowId, agentNodeId]);
const cancel = useCallback(() => setWait(null), []);
- return { waiting: wait !== null, unanswered, begin, cancel };
+ return { waiting: wait !== null, unanswered, failure, begin, cancel };
}
// ----- Turn on Talk, Apply -----
diff --git a/client/src/features/home/employee/EmployeeTalk.tsx b/client/src/features/home/employee/EmployeeTalk.tsx
index 4f769f30..8bc2497f 100644
--- a/client/src/features/home/employee/EmployeeTalk.tsx
+++ b/client/src/features/home/employee/EmployeeTalk.tsx
@@ -23,6 +23,7 @@ import { ArrowUp, Monitor } from 'lucide-react';
import { Suspense, lazy, useEffect, useLayoutEffect, useRef, useState, type KeyboardEvent, type ReactNode } from 'react';
import { useQueryClient } from '@tanstack/react-query';
import { ActionButton } from '@/components/ui/action-button';
+import { Alert, AlertDescription } from '@/components/ui/alert';
import { Button } from '@/components/ui/button';
import { Skeleton } from '@/components/ui/skeleton';
import { Textarea } from '@/components/ui/textarea';
@@ -163,7 +164,7 @@ function Conversation({ employee, control, drafts }: { employee: EmployeeSummary
const thread = useTalkThread(workflowId);
const messages = thread.data;
const send = useSendTalkMessage(workflowId);
- const { waiting, unanswered, begin, cancel } = useReplyWait(workflowId, employee.talk.agent_node_id, messages);
+ const { waiting, unanswered, failure, begin, cancel } = useReplyWait(workflowId, employee.talk.agent_node_id, messages);
const mode = talkMode(employee.control);
const [draft, setDraft] = useState('');
const [queued, setQueued] = useState(false);
@@ -220,6 +221,9 @@ function Conversation({ employee, control, drafts }: { employee: EmployeeSummary
const rows = messages ? threadRows(messages) : [];
const notice = talkNoticeText(mode, name, queued);
+ const failureText = mode === 'queue' && employee.control.pause_reason === 'failures'
+ ? employee.control.pause_detail || (employee.task?.label === 'Paused' ? employee.task.text : null)
+ : failure ? [failure.error, failure.hint].filter(Boolean).join(' ') : null;
const now = new Date();
return (
@@ -255,11 +259,12 @@ function Conversation({ employee, control, drafts }: { employee: EmployeeSummary
<p className={cn('m-0', BUBBLE, THEIRS, 'text-fg-muted')}>Working…</p>
</div>
)}
- {unanswered && !waiting && <p className={cn('m-0 text-xs text-fg-muted', PAST_AVATAR)}>No answer from {name} yet.</p>}
+ {unanswered && !waiting && !failureText && <p className={cn('m-0 text-xs text-fg-muted', PAST_AVATAR)}>No answer from {name} yet.</p>}
</div>
{drafts}
</div>
<div className="sticky bottom-0 z-10 mt-auto flex flex-col gap-2.5 bg-bg-app pt-4 pb-3">
+ {failureText && <Alert variant="destructive"><AlertDescription>{failureText}</AlertDescription></Alert>}
{employee.pending_changes && <PendingChangesNotice employee={employee} />}
{employee.browser_request && <BrowserNotice employee={employee} />}
{notice && (
diff --git a/server/services/agent_runtime.py b/server/services/agent_runtime.py
index db8a56d8..4ca82dde 100644
--- a/server/services/agent_runtime.py
+++ b/server/services/agent_runtime.py
@@ -149,9 +149,7 @@ async def run_native_llm_step(
if not error.retryable or attempt + 1 >= attempts:
if not translate_errors:
raise
- from services.plugin import NodeUserError
-
- raise NodeUserError(error.user_message) from error
+ raise error.as_node_error() from error
delay = (
error.retry_after
if error.retry_after is not None
@@ -340,6 +338,7 @@ async def run_native_agent_loop(
specs = _tool_specs_by_name(current_tools)
iteration_new_tools: List[AgentToolSpec] = []
+ blocked_error = None
for call_index, call in enumerate(calls, start=1):
if call.name not in specs:
result: Any = {
@@ -377,7 +376,12 @@ async def run_native_agent_loop(
call.name,
exc,
)
- result = {"error": str(exc)}
+ from services.plugin import NodeUserError
+
+ result = exc.as_dict() if isinstance(exc, NodeUserError) else {"error": str(exc)}
+
+ if isinstance(result, dict) and result.get("error") and result.get("requires_user_action") is True:
+ blocked_error = blocked_error or result
if (
rebind_from_operations is not None
@@ -420,6 +424,16 @@ async def run_native_agent_loop(
# One save per turn covers every tool result appended above.
await save_now()
+ if blocked_error:
+ # All tool-call/result pairs are saved, but no extra LLM request
+ # (including compaction) is spent on an owner-only repair.
+ from services.plugin import NodeUserError
+
+ raise NodeUserError(
+ blocked_error["error"], hint=blocked_error.get("hint"),
+ requires_user_action=True,
+ )
+
if iteration_new_tools:
current_tools.extend(iteration_new_tools)
logger.info(
diff --git a/server/services/deployment/handlers.py b/server/services/deployment/handlers.py
index 7b933f35..f845e5af 100644
--- a/server/services/deployment/handlers.py
+++ b/server/services/deployment/handlers.py
@@ -2315,7 +2315,10 @@ async def _clear_failure_streak(database, control) -> None:
logger.debug(f"Failed to clear failure streak: {exc}")
-async def pause_generation_on_failure(*, workflow_id: str, reason: str) -> Dict[str, Any]:
+async def pause_generation_on_failure(
+ *, workflow_id: str, reason: str, requires_user_action: bool = False,
+ hint: Optional[str] = None, generation: Optional[int] = None,
+) -> Dict[str, Any]:
"""Circuit breaker: pause a controlled deployment after failed runs.
Called by the ``workflow_control.pause_on_failure.v1`` activity that
@@ -2346,7 +2349,11 @@ async def pause_generation_on_failure(*, workflow_id: str, reason: str) -> Dict[
"reason": "not_running",
"status": getattr(control, "status", None),
}
- threshold = _pause_on_failure_threshold()
+ if generation is not None and int(generation) != control.generation:
+ return {"paused": False, "reason": "stale_generation"}
+ # Account/configuration blockers cannot heal by spending two more runs.
+ # Ordinary node/input failures retain the existing circuit breaker.
+ threshold = 1 if requires_user_action else _pause_on_failure_threshold()
if threshold > 1:
streak = await _bump_failure_streak(
service.database,
@@ -2381,7 +2388,10 @@ async def pause_generation_on_failure(*, workflow_id: str, reason: str) -> Dict[
"expected_revision": control.revision,
"idempotency_key": f"pause-on-failure:{control.id}:{control.revision}",
"_pause_reason": PAUSE_REASON_FAILURES,
- "_pause_detail": _failure_pause_detail(reason),
+ "_pause_detail": (
+ " ".join(f"{reason} {hint or ''}".split())[:1000]
+ if requires_user_action else _failure_pause_detail(reason)
+ ),
},
None,
)
diff --git a/server/services/llm/protocol.py b/server/services/llm/protocol.py
index ccb1bc8c..5bb2b4f3 100644
--- a/server/services/llm/protocol.py
+++ b/server/services/llm/protocol.py
@@ -308,6 +308,7 @@ class LLMErrorCategory(str, Enum):
AUTHENTICATION = "authentication"
PERMISSION = "permission"
+ BILLING = "billing"
RATE_LIMIT = "rate_limit"
INVALID_REQUEST = "invalid_request"
NOT_FOUND = "not_found"
@@ -402,6 +403,9 @@ class LLMError(Exception):
f"{provider} denied this request. "
"Check account and model access."
),
+ LLMErrorCategory.BILLING.value: (
+ f"{provider} blocked this request because a spending limit or available credits were exhausted."
+ ),
LLMErrorCategory.RATE_LIMIT.value: (
f"{provider} is rate-limiting requests. "
"Retry after a short delay."
@@ -430,6 +434,35 @@ class LLMError(Exception):
}
return messages.get(category, messages[LLMErrorCategory.UNKNOWN.value])
+ @property
+ def hint(self) -> Optional[str]:
+ """Fixed recovery advice; never copy the provider's raw response."""
+ return {
+ LLMErrorCategory.BILLING: "Review the provider project's billing, spending cap, and credits. Resume after access is restored; repeating the request will not fix it.",
+ LLMErrorCategory.AUTHENTICATION: "Update the provider credential in Settings > Connectors, then resume.",
+ LLMErrorCategory.PERMISSION: "Check the credential's project and model permissions, then resume after access is restored.",
+ LLMErrorCategory.NOT_FOUND: "Choose a model available to this provider and credential, then apply the change and resume.",
+ LLMErrorCategory.PROTOCOL: "Correct the provider's base URL and API configuration, then resume.",
+ LLMErrorCategory.CONTEXT_LENGTH: "Reduce or clear the agent's context before trying again.",
+ LLMErrorCategory.INVALID_REQUEST: "Check the model's supported request settings before trying again.",
+ }.get(self.category)
+
+ @property
+ def requires_user_action(self) -> bool:
+ return not self.retryable and self.category in {
+ LLMErrorCategory.BILLING, LLMErrorCategory.AUTHENTICATION,
+ LLMErrorCategory.PERMISSION, LLMErrorCategory.NOT_FOUND,
+ LLMErrorCategory.PROTOCOL,
+ }
+
+ def as_node_error(self):
+ from services.plugin import NodeUserError
+
+ return NodeUserError(
+ self.user_message, hint=self.hint,
+ requires_user_action=self.requires_user_action,
+ )
+
@classmethod
def from_exception(cls, provider: str, exc: BaseException) -> "LLMError":
status = _optional_int(
@@ -699,6 +732,15 @@ def _classify_error(
) -> LLMErrorCategory:
name = type(exc).__name__.lower()
message = str(exc).lower()
+ # A quota/credit exhaustion can arrive as 429, but backoff cannot fix it.
+ # Match narrow billing signatures before the generic HTTP classifiers.
+ if status in {400, 402, 403, 429} and (
+ status == 402 or any(marker in message for marker in (
+ "spend cap breached", "insufficient_quota", "insufficient credits",
+ "credit balance is too low", "billing_hard_limit_reached",
+ ))
+ ):
+ return LLMErrorCategory.BILLING
if status == 401 or "authentication" in name or "api key" in message:
return LLMErrorCategory.AUTHENTICATION
if status == 403 or "permission" in name:
diff --git a/server/services/llm/unifier.py b/server/services/llm/unifier.py
index 3b14814a..d378472c 100644
--- a/server/services/llm/unifier.py
+++ b/server/services/llm/unifier.py
@@ -129,13 +129,13 @@ class ChatUnifier:
self._log_failure("LLM provider request failed", error, entry, provider)
if not translate_errors:
raise
- raise NodeUserError(error.user_message) from error
+ raise error.as_node_error() from error
except spec.sdk_exception_types as e:
error = LLMError.from_exception(provider, e)
self._log_failure("LLM provider request failed", error, entry, provider)
if not translate_errors:
raise error from e
- raise NodeUserError(error.user_message) from error
+ raise error.as_node_error() from error
except (ValueError, TypeError, OSError) as e:
# Only normalize generic configuration/transport failures raised
# while constructing a provider client. Once a client exists,
@@ -162,7 +162,7 @@ class ChatUnifier:
)
if not translate_errors:
raise error from e
- raise NodeUserError(error.user_message) from error
+ raise error.as_node_error() from error
finally:
if entry is not None:
await self._release_client(entry)
@@ -185,11 +185,11 @@ class ChatUnifier:
models = await entry.client.fetch_models(api_key)
except LLMError as error:
self._log_failure("LLM model-list request failed", error, entry, provider)
- raise NodeUserError(error.user_message) from error
+ raise error.as_node_error() from error
except spec.sdk_exception_types as e:
error = LLMError.from_exception(provider, e)
self._log_failure("LLM model-list request failed", error, entry, provider)
- raise NodeUserError(error.user_message) from error
+ raise error.as_node_error() from error
except (ValueError, TypeError, OSError) as e:
if entry is not None:
raise
@@ -210,7 +210,7 @@ class ChatUnifier:
category=error.category.value,
retryable=error.retryable,
)
- raise NodeUserError(error.user_message) from error
+ raise error.as_node_error() from error
finally:
if entry is not None:
await self._release_client(entry)
diff --git a/server/services/plugin/base.py b/server/services/plugin/base.py
index fb2b80d9..ba82786e 100644
--- a/server/services/plugin/base.py
+++ b/server/services/plugin/base.py
@@ -60,6 +60,25 @@ class NodeUserError(Exception):
unexpected failures that warrant a stacktrace in the operator log.
"""
+ def __init__(self, message: str, *, hint: Optional[str] = None, requires_user_action: bool = False):
+ super().__init__(message)
+ self.hint = hint
+ self.requires_user_action = requires_user_action
+
+ def as_dict(self) -> Dict[str, Any]:
+ """Public recovery information, shared by nodes and agent tools.
+
+ Ordinary input errors remain correctable by the model. Account or
+ configuration blockers explicitly require the owner, so another
+ model turn cannot fix them. Callers must supply safe text only.
+ """
+ return {
+ "error": str(self),
+ "error_type": "NodeUserError",
+ **({"hint": self.hint} if self.hint else {}),
+ **({"requires_user_action": True, "retryable": False} if self.requires_user_action else {}),
+ }
+
NODE_WAIT_INTERRUPTED = "NodeWaitInterrupted"
@@ -639,7 +658,7 @@ class BaseNode:
# LLM gets the message in the structured response and can
# retry with corrected input.
logger.warning("[%s] %s op %s: %s", self.type, op_name, type(e).__name__, e)
- return self._wrap_error(start_time=start_time, error=str(e), error_type="NodeUserError")
+ return self._wrap_error(start_time=start_time, error=str(e), extra=e.as_dict())
except NodeWaitInterrupted as e:
# Not a failure: the process is going away mid-wait. The activity
# wrapper turns this envelope into a retryable error.
@@ -781,7 +800,7 @@ class BaseNode:
def _tool_error_payload(self, envelope: Dict[str, Any]) -> Dict[str, Any]:
"""Copy only explicitly declared public metadata from a tool error."""
result = {"error": envelope.get("error", "tool execution failed")}
- for name in self.tool_error_fields:
+ for name in self.tool_error_fields | {"hint", "requires_user_action", "retryable"}:
if name != "error" and name in envelope and envelope[name] is not None:
result[name] = envelope[name]
return result
@@ -1174,7 +1193,8 @@ class BaseNode:
await broadcaster.update_node_status(
node_id,
"error",
- {"error": error, "execution_id": execution_id},
+ {"error": error, "execution_id": execution_id,
+ **{key: result[key] for key in ("hint", "requires_user_action", "error_type") if key in result}},
workflow_id=workflow_id,
)
diff --git a/server/services/temporal/activities.py b/server/services/temporal/activities.py
index a9e674d0..64339bd1 100644
--- a/server/services/temporal/activities.py
+++ b/server/services/temporal/activities.py
@@ -614,6 +614,11 @@ async def pause_workflow_on_failure_activity(payload: Dict[str, Any]) -> Dict[st
return await pause_generation_on_failure(
workflow_id=str(payload.get("workflow_id") or ""),
reason=str(payload.get("reason") or "run_failed"),
+ **({
+ "requires_user_action": True,
+ "hint": payload.get("hint"),
+ "generation": payload.get("generation"),
+ } if payload.get("requires_user_action") is True else {}),
)
except Exception as exc: # noqa: BLE001 — non-fatal
activity.logger.warning(f"pause_workflow_on_failure_activity failed for " f"workflow={payload.get('workflow_id')!r}: {exc}")
diff --git a/server/services/temporal/agent_activities.py b/server/services/temporal/agent_activities.py
index 2a6f4f1a..629751db 100644
--- a/server/services/temporal/agent_activities.py
+++ b/server/services/temporal/agent_activities.py
@@ -183,6 +183,8 @@ def _as_temporal_llm_error(error: Any):
"request_id": getattr(error, "request_id", None),
"retry_after": getattr(error, "retry_after", None),
"retry_after_raw": getattr(error, "retry_after_raw", None),
+ **({"hint": error.hint} if getattr(error, "hint", None) else {}),
+ **({"requires_user_action": True} if getattr(error, "requires_user_action", False) else {}),
}
# Honor the provider's own pacing: a 429 with Retry-After should wait
# exactly that long before the next attempt instead of the policy's
@@ -623,6 +625,12 @@ async def broadcast_agent_progress(payload: Dict[str, Any]) -> Dict[str, Any]:
**({"phase": phase} if phase else {}),
**capability_data,
}
+ if status == "error":
+ lifecycle_data.update({
+ key: payload[key]
+ for key in ("error", "error_type", "hint", "requires_user_action")
+ if payload.get(key) is not None
+ })
if phase == "starting":
lifecycle_data.update({"active_skills": [], "last_skills": [], "last_tool_name": None, "last_capability": None})
await broadcaster.update_node_status(
diff --git a/server/services/temporal/agent_workflow.py b/server/services/temporal/agent_workflow.py
index ae3e1330..b478159a 100644
--- a/server/services/temporal/agent_workflow.py
+++ b/server/services/temporal/agent_workflow.py
@@ -919,6 +919,15 @@ class AgentWorkflow:
cause_message = str(
getattr(cause, "message", "") or ""
).strip()
+ recovery = {}
+ if cause_type.startswith("LLMError."):
+ diagnostics = getattr(cause, "details", ())
+ if diagnostics and isinstance(diagnostics[0], dict):
+ recovery = {
+ key: diagnostics[0][key]
+ for key in ("hint", "requires_user_action", "retryable")
+ if diagnostics[0].get(key) is not None
+ }
safe_activity_types = {
"MissingAgentProviderCredential",
"EmptyAgentPrompt",
@@ -965,11 +974,14 @@ class AgentWorkflow:
max_iterations,
phase="failed",
status="error",
+ extra={"error": detail, "error_type": "NodeUserError", **recovery}
+ if recovery else None,
)
return {
"success": False,
"error": f"LLM step failed: {detail}",
"error_type": "LLMStepError",
+ **recovery,
"result": {
"iterations": iteration + 1,
"usage": usage_total,
@@ -1605,6 +1617,7 @@ class AgentWorkflow:
await _cleanup_cancelled_delegations()
raise
+ blocked_error = None
for call_index, call in enumerate(calls):
if call.get("parse_error"):
# Native adapters retain malformed arguments rather than
@@ -1860,12 +1873,18 @@ class AgentWorkflow:
call_index,
None,
)
+ child_recovery = {
+ key: tool_result[key]
+ for key in ("hint", "requires_user_action", "retryable")
+ if isinstance(tool_result, dict) and key in tool_result
+ }
tool_result = {
"success": child_succeeded,
"status": "submitted" if child_succeeded else "failed",
"result": child_response,
"usage": child_summary.get("usage"),
**({"error": child_error} if child_error else {}),
+ **child_recovery,
}
if next_delegation_to_start < len(delegation_call_indices):
await _start_delegation(
@@ -1912,6 +1931,19 @@ class AgentWorkflow:
"delegation_result": delegated["result"],
"delegation_usage": delegated.get("usage"),
}
+ if (
+ isinstance(tool_result, dict)
+ and tool_result.get("error")
+ and tool_result.get("requires_user_action") is True
+ and workflow.patched("agent-user-action-error-v1")
+ ):
+ blocked_error = blocked_error or {
+ "error": tool_result["error"],
+ "error_type": "NodeUserError",
+ "hint": tool_result.get("hint"),
+ "requires_user_action": True,
+ "retryable": False,
+ }
tool_content = _serialise_tool_result(tool_result)
if (
tool_output_limit
@@ -2128,6 +2160,25 @@ class AgentWorkflow:
)
# ---- Transcript pressure and compaction ----------------------
+ if blocked_error:
+ # Complete/persist the current tool turn, then stop before
+ # another model request or paid compaction can be scheduled.
+ await workflow.execute_activity(
+ "agent.skill.clear",
+ args=[{"workflow_id": payload.get("workflow_id"),
+ "execution_id": task_scope_execution_id,
+ "agent_node_id": agent_node_id}],
+ activity_id="clear-active-skills-blocked",
+ start_to_close_timeout=PERSIST_TURN_TIMEOUT,
+ retry_policy=AGENT_ACTIVITY_RETRY,
+ )
+ await self._emit_phase(
+ agent_node_id, agent_workflow_id, iteration, max_iterations,
+ phase="failed", status="error", extra=blocked_error,
+ )
+ return {"success": False, **blocked_error,
+ "result": {"iterations": iteration + 1, "usage": usage_total}}
+
# Summarize, then swap messages: no memory-node gate, no
# checkpoint machinery. A later rollover resumes from the
# compacted state for free, because the next LLM turn sends and
diff --git a/server/services/temporal/workflow.py b/server/services/temporal/workflow.py
index 8f4dfbe1..f1da1d6d 100644
--- a/server/services/temporal/workflow.py
+++ b/server/services/temporal/workflow.py
@@ -246,12 +246,21 @@ class MachinaWorkflow:
spawned_by_trigger = any(node.get("_pre_executed") for node in nodes)
if not workflow_id or not spawned_by_trigger:
return
+ failure = errors[0] or {}
+ recovery = {}
+ if failure.get("requires_user_action") is True:
+ recovery = {
+ "requires_user_action": True,
+ "hint": failure.get("hint"),
+ "generation": workflow_data.get("generation"),
+ }
try:
await workflow.execute_activity(
"workflow_control.pause_on_failure",
{
"workflow_id": workflow_id,
"reason": str((errors[0] or {}).get("error", "run_failed"))[:500],
+ **recovery,
},
start_to_close_timeout=timedelta(seconds=30),
retry_policy=QUICK_ACTIVITY_RETRY,
@@ -700,6 +709,7 @@ class MachinaWorkflow:
error_info = {
"node_id": done_id,
"error": result.get("error", "Unknown error"),
+ **{key: result[key] for key in ("hint", "requires_user_action", "retryable") if key in result},
}
errors.append(error_info)
workflow.logger.error(f"Node failed: {done_id} - {error_info['error']}")
diff --git a/server/tests/llm/test_protocol_codec.py b/server/tests/llm/test_protocol_codec.py
index 55bd902d..eb2e2bc8 100644
--- a/server/tests/llm/test_protocol_codec.py
+++ b/server/tests/llm/test_protocol_codec.py
@@ -179,6 +179,26 @@ def test_structured_error_exposes_only_category_based_user_message():
assert "Bearer secret" not in error.user_message
[email protected](("status", "message"), [
+ (403, "Spend cap breached for project: projects/private-id for service: aiplatform.googleapis.com"),
+ (429, "insufficient_quota: private account details"),
+ (400, "Your credit balance is too low: secret"),
+ (402, "Payment required: secret"),
+])
+def test_billing_errors_need_owner_action_instead_of_retry(status, message):
+ exc = Exception(message)
+ exc.status_code = status
+ error = LLMError.from_exception("gemini", exc)
+ assert error.category == LLMErrorCategory.BILLING
+ assert not error.retryable
+ node_error = error.as_node_error()
+ assert node_error.requires_user_action
+ assert "spending" in node_error.hint
+ assert "private" not in str(node_error)
+ assert "secret" not in str(node_error)
+ assert node_error.as_dict()["retryable"] is False
+
+
@pytest.mark.parametrize(
("provider", "not_found", "context_length"),
[
diff --git a/server/tests/services/test_native_agent_runtime.py b/server/tests/services/test_native_agent_runtime.py
index 975b95cd..8a95ee78 100644
--- a/server/tests/services/test_native_agent_runtime.py
+++ b/server/tests/services/test_native_agent_runtime.py
@@ -49,6 +49,34 @@ class _Database:
return None
[email protected]("needs_owner", [True, False])
+async def test_tool_errors_only_stop_the_loop_when_the_owner_must_fix_them(needs_owner):
+ response = LLMResponse(tool_calls=[ToolCall(id="call-1", name="model", args={})])
+ unifier = _FakeUnifier([response, LLMResponse(content="fixed")])
+ save = AsyncMock()
+ async def execute(_name, _args):
+ raise NodeUserError("Blocked", hint="Fix the account" if needs_owner else "Supply a value", requires_user_action=needs_owner)
+
+ kwargs = dict(provider="gemini", api_key="test", model="test", temperature=0,
+ max_tokens=100, initial_messages=[Message(role="user", content="go")],
+ tools=[AgentToolSpec(definition=ToolDef(name="model", description="model", parameters={}))],
+ tool_executor=execute, conversation_saver=save)
+ if needs_owner:
+ with pytest.raises(NodeUserError) as caught:
+ await run_native_agent_loop(unifier, **kwargs)
+ assert caught.value.requires_user_action
+ assert caught.value.hint == "Fix the account"
+ assert len(unifier.calls) == 1
+ # The failed tool result is saved with its matching call, without
+ # spending a second model turn to explain the error.
+ assert save.await_args.args[0][-1].role == "tool"
+ else:
+ result = await run_native_agent_loop(unifier, **kwargs)
+ assert result["response"].content == "fixed"
+ assert len(unifier.calls) == 2
+
+
def test_in_process_compaction_usage_joins_execution_wide_total():
from services.ai import _accumulate_compaction_usage
diff --git a/server/tests/temporal/test_agent_llm_contract.py b/server/tests/temporal/test_agent_llm_contract.py
index 3346f631..7bdbb7de 100644
--- a/server/tests/temporal/test_agent_llm_contract.py
+++ b/server/tests/temporal/test_agent_llm_contract.py
@@ -87,6 +87,50 @@ def patched_workflow(monkeypatch):
class TestWorkflowLoopContract:
+ @pytest.mark.asyncio
+ @pytest.mark.parametrize("from_tool", [False, True])
+ async def test_owner_action_error_stops_before_another_model_or_compaction(self, monkeypatch, patched_workflow, from_tool):
+ from services.llm.protocol import LLMError, LLMErrorCategory
+ from services.temporal.agent_activities import _as_temporal_llm_error
+ from services.temporal.agent_workflow import AgentWorkflow
+
+ error = LLMError("secret provider body", "gemini", category=LLMErrorCategory.BILLING)
+ prepared = _payload()
+ prepared["compaction_threshold"] = 1
+ calls = []
+ phases = []
+
+ async def execute(name, *, args, **_kwargs):
+ calls.append(name)
+ if name == "agent.prepare_payload":
+ return prepared
+ if name == "agent.execute_llm_step":
+ if from_tool:
+ return {"kind": "tool_calls", "calls": [{"id": "call-1", "name": "write_todos", "args": {}}],
+ "usage": {"input_tokens": 10, "output_tokens": 2}}
+ failure = RuntimeError("activity failed")
+ failure.cause = _as_temporal_llm_error(error)
+ raise failure
+ if name == "node.writeTodos.v1":
+ return {"success": False, **error.as_node_error().as_dict()}
+ if name == "agent.broadcast_progress":
+ phases.append(args[0])
+ return {}
+ if name in {"agent.skill.clear", "agent.persist_turn"}:
+ return {}
+ raise AssertionError(f"Unexpected activity: {name}")
+
+ monkeypatch.setattr(patched_workflow, "execute_activity", execute)
+ result = await AgentWorkflow().run({"node_id": "agent-1", "execution_id": "root-run-1"})
+ assert result["success"] is False
+ assert result["requires_user_action"] is True
+ assert result["hint"] == error.hint
+ assert calls.count("agent.execute_llm_step") == 1
+ assert "agent.compact_context" not in calls
+ failure = next(p for p in phases if p.get("status") == "error")
+ assert failure["hint"] == error.hint
+ assert "secret" not in str(failure)
+
@pytest.mark.asyncio
async def test_llm_step_payload_shape_and_heartbeat(
self,
diff --git a/server/tests/temporal/test_durability_hardening.py b/server/tests/temporal/test_durability_hardening.py
index 2c327768..408484c3 100644
--- a/server/tests/temporal/test_durability_hardening.py
+++ b/server/tests/temporal/test_durability_hardening.py
@@ -1118,6 +1118,30 @@ async def test_pause_on_failure_pauses_immediately_at_threshold_one(monkeypatch)
assert request["expected_revision"] == running.revision
+async def test_owner_action_blocker_pauses_once_without_spending_threshold_runs(monkeypatch):
+ from services.deployment import handlers
+
+ running = _boot_control("running")
+ _breaker_settings(monkeypatch, threshold=3)
+ monkeypatch.setattr(handlers, "_control_service", lambda: SimpleNamespace(database=_streak_database(running, {})))
+ pause = AsyncMock(return_value={"success": True})
+ monkeypatch.setattr(handlers, "handle_pause_workflow", pause)
+ result = await handlers.pause_generation_on_failure(
+ workflow_id="wf", reason="Spending cap reached", hint="Review billing before resuming.",
+ requires_user_action=True, generation=running.generation,
+ )
+ assert result["paused"] is True
+ pause.assert_awaited_once()
+ assert pause.await_args.args[0]["_pause_detail"] == "Spending cap reached Review billing before resuming."
+ pause.reset_mock()
+ result = await handlers.pause_generation_on_failure(
+ workflow_id="wf", reason="Old failure", requires_user_action=True, generation=running.generation - 1,
+ )
+ assert result["reason"] == "stale_generation"
+ pause.assert_not_awaited()
+
+
@pytest.mark.asyncio
async def test_single_node_failure_does_not_trip_the_breaker(monkeypatch):
"""The reported regression: one telegramSend NodeUserError on one