Skip to content

AutoGPT is the vision of accessible AI for everyone, to use and to build on. Our mission is to provide the tools, so that you can focus on what matters.

feat(backend): Make agent graph execution retriable and its failure visible (#10518)

Zamil Majdy8331dabf6a7b
diff --git a/autogpt_platform/backend/backend/executor/manager.py b/autogpt_platform/backend/backend/executor/manager.py
index e443602713..1b6d86db11 100644
--- a/autogpt_platform/backend/backend/executor/manager.py
+++ b/autogpt_platform/backend/backend/executor/manager.py
@@ -664,6 +664,7 @@ class Executor:
execution_stats.cost += cost
@classmethod
+ @func_retry
@time_measured
def _on_graph_execution(
cls,
diff --git a/autogpt_platform/backend/backend/util/service.py b/autogpt_platform/backend/backend/util/service.py
index 35206889b3..c725a76700 100644
--- a/autogpt_platform/backend/backend/util/service.py
+++ b/autogpt_platform/backend/backend/util/service.py
@@ -105,6 +105,7 @@ EXCEPTION_MAPPING = {
e.__name__: e
for e in [
ValueError,
+ RuntimeError,
TimeoutError,
ConnectionError,
*[
@@ -286,7 +287,7 @@ def get_service_client(
return retry(
reraise=True,
stop=stop_after_attempt(api_comm_retry),
- wait=wait_exponential_jitter(max=4.0),
+ wait=wait_exponential_jitter(max=5.0),
retry=retry_if_not_exception_type(
(
# Don't retry these specific exceptions that won't be fixed by retrying
diff --git a/autogpt_platform/backend/backend/util/service_test.py b/autogpt_platform/backend/backend/util/service_test.py
index 8b9e295bb9..00fd206179 100644
--- a/autogpt_platform/backend/backend/util/service_test.py
+++ b/autogpt_platform/backend/backend/util/service_test.py
@@ -19,6 +19,7 @@ TEST_SERVICE_PORT = 8765
class ServiceTest(AppService):
def __init__(self):
super().__init__()
+ self.fail_count = 0
def cleanup(self):
pass
@@ -42,6 +43,19 @@ class ServiceTest(AppService):
return self.run_and_wait(add_async(a, b))
+ @expose
+ def failing_add(self, a: int, b: int) -> int:
+ """Method that fails 2 times then succeeds - for testing retry logic"""
+ self.fail_count += 1
+ if self.fail_count <= 2:
+ raise RuntimeError("Database connection failed")
+ return a + b
+
+ @expose
+ def always_failing_add(self, a: int, b: int) -> int:
+ """Method that always fails - for testing no retry when disabled"""
+ raise RuntimeError("Database connection failed")
+
class ServiceTestClient(AppServiceClient):
@classmethod
@@ -51,6 +65,8 @@ class ServiceTestClient(AppServiceClient):
add = ServiceTest.add
subtract = ServiceTest.subtract
fun_with_async = ServiceTest.fun_with_async
+ failing_add = ServiceTest.failing_add
+ always_failing_add = ServiceTest.always_failing_add
add_async = endpoint_to_async(ServiceTest.add)
subtract_async = endpoint_to_async(ServiceTest.subtract)
@@ -313,3 +329,25 @@ def test_cached_property_behavior():
resource3 = obj.expensive_resource
assert creation_count == 2 # New creation
assert resource1 != resource3
+
+
+def test_service_with_runtime_error_retries(server):
+ """Test a real service method that throws RuntimeError and gets retried"""
+ with ServiceTest():
+ # Get client with retry enabled
+ client = get_service_client(ServiceTestClient, request_retry=True)
+
+ # This should succeed after retries (fails 2 times, succeeds on 3rd try)
+ result = client.failing_add(5, 3)
+ assert result == 8
+
+
+def test_service_no_retry_when_disabled(server):
+ """Test that retry doesn't happen when disabled"""
+ with ServiceTest():
+ # Get client with retry disabled
+ client = get_service_client(ServiceTestClient, request_retry=False)
+
+ # This should fail immediately without retry
+ with pytest.raises(RuntimeError, match="Database connection failed"):
+ client.always_failing_add(5, 3)
diff --git a/autogpt_platform/backend/backend/util/settings.py b/autogpt_platform/backend/backend/util/settings.py
index c7685bfc79..c17f8fba42 100644
--- a/autogpt_platform/backend/backend/util/settings.py
+++ b/autogpt_platform/backend/backend/util/settings.py
@@ -68,7 +68,7 @@ class Config(UpdateTrackingModel["Config"], BaseSettings):
description="The default timeout in seconds, for Pyro client connections.",
)
pyro_client_comm_retry: int = Field(
- default=3,
+ default=5,
description="The default number of retries for Pyro client connections.",
)
rpc_client_call_timeout: int = Field(
diff --git a/autogpt_platform/frontend/src/components/Flow.tsx b/autogpt_platform/frontend/src/components/Flow.tsx
index 5dbf56e943..6ea5cf098d 100644
--- a/autogpt_platform/frontend/src/components/Flow.tsx
+++ b/autogpt_platform/frontend/src/components/Flow.tsx
@@ -116,6 +116,7 @@ const FlowEditor: React.FC<{
isRunning,
isStopping,
isScheduling,
+ graphExecutionError,
nodes,
setNodes,
edges,
@@ -764,6 +765,7 @@ const FlowEditor: React.FC<{
ref={runnerUIRef}
graph={savedAgent}
nodes={nodes}
+ graphExecutionError={graphExecutionError}
createRunSchedule={createRunSchedule}
saveAndRun={saveAndRun}
/>
diff --git a/autogpt_platform/frontend/src/components/RunnerUIWrapper.tsx b/autogpt_platform/frontend/src/components/RunnerUIWrapper.tsx
index deae14d0d6..9370c1c211 100644
--- a/autogpt_platform/frontend/src/components/RunnerUIWrapper.tsx
+++ b/autogpt_platform/frontend/src/components/RunnerUIWrapper.tsx
@@ -19,6 +19,7 @@ import RunnerOutputUI, {
interface RunnerUIWrapperProps {
graph: GraphMeta;
nodes: Node<CustomNodeData>[];
+ graphExecutionError?: string | null;
saveAndRun: (
inputs: Record<string, any>,
credentialsInputs: Record<string, CredentialsMetaInput>,
@@ -38,7 +39,10 @@ export interface RunnerUIWrapperRef {
}
const RunnerUIWrapper = forwardRef<RunnerUIWrapperRef, RunnerUIWrapperProps>(
- ({ graph, nodes, saveAndRun, createRunSchedule }, ref) => {
+ (
+ { graph, nodes, graphExecutionError, saveAndRun, createRunSchedule },
+ ref,
+ ) => {
const [isRunInputDialogOpen, setIsRunInputDialogOpen] = useState(false);
const [isRunnerOutputOpen, setIsRunnerOutputOpen] = useState(false);
@@ -103,6 +107,7 @@ const RunnerUIWrapper = forwardRef<RunnerUIWrapperRef, RunnerUIWrapperProps>(
isOpen={isRunnerOutputOpen}
doClose={() => setIsRunnerOutputOpen(false)}
outputs={graphOutputs}
+ graphExecutionError={graphExecutionError}
/>
</>
);
diff --git a/autogpt_platform/frontend/src/components/agents/agent-run-details-view.tsx b/autogpt_platform/frontend/src/components/agents/agent-run-details-view.tsx
index 422aa24939..de3cf28b60 100644
--- a/autogpt_platform/frontend/src/components/agents/agent-run-details-view.tsx
+++ b/autogpt_platform/frontend/src/components/agents/agent-run-details-view.tsx
@@ -235,6 +235,15 @@ export default function AgentRunDetailsView({
</div>
))}
</div>
+ {run.status === "FAILED" && (
+ <div className="mt-4 rounded-md border border-red-200 bg-red-50 p-3 dark:border-red-800 dark:bg-red-900/20">
+ <p className="text-sm text-red-800 dark:text-red-200">
+ <strong>Error:</strong>{" "}
+ {run.stats?.error ||
+ "The execution failed due to an internal error. You can re-run the agent to retry."}
+ </p>
+ </div>
+ )}
</CardContent>
</Card>
diff --git a/autogpt_platform/frontend/src/components/runner-ui/RunnerOutputUI.tsx b/autogpt_platform/frontend/src/components/runner-ui/RunnerOutputUI.tsx
index 6f9316c476..b8f03cdc64 100644
--- a/autogpt_platform/frontend/src/components/runner-ui/RunnerOutputUI.tsx
+++ b/autogpt_platform/frontend/src/components/runner-ui/RunnerOutputUI.tsx
@@ -25,6 +25,7 @@ interface OutputModalProps {
isOpen: boolean;
doClose: () => void;
outputs: OutputNodeInfo[];
+ graphExecutionError?: string | null;
}
const formatOutput = (output: any): string => {
@@ -47,7 +48,12 @@ const formatOutput = (output: any): string => {
return String(output);
};
-export function RunnerOutputUI({ isOpen, doClose, outputs }: OutputModalProps) {
+export function RunnerOutputUI({
+ isOpen,
+ doClose,
+ outputs,
+ graphExecutionError,
+}: OutputModalProps) {
const { toast } = useToast();
const copyOutput = (name: string, output: any) => {
@@ -80,6 +86,13 @@ export function RunnerOutputUI({ isOpen, doClose, outputs }: OutputModalProps) {
<div className="flex-grow overflow-y-auto px-2 py-2">
<ScrollArea className="h-full overflow-auto pr-4">
<div className="space-y-4">
+ {graphExecutionError && (
+ <div className="rounded-md border border-red-200 bg-red-50 p-3 dark:border-red-800 dark:bg-red-900/20">
+ <p className="text-sm text-red-800 dark:text-red-200">
+ <strong>Error:</strong> {graphExecutionError}
+ </p>
+ </div>
+ )}
{outputs && outputs.length > 0 ? (
outputs.map((output, i) => (
<div key={i} className="space-y-1">
diff --git a/autogpt_platform/frontend/src/hooks/useAgentGraph.tsx b/autogpt_platform/frontend/src/hooks/useAgentGraph.tsx
index 3e8cd9c9e2..e8801c7b38 100644
--- a/autogpt_platform/frontend/src/hooks/useAgentGraph.tsx
+++ b/autogpt_platform/frontend/src/hooks/useAgentGraph.tsx
@@ -59,6 +59,9 @@ export default function useAgentGraph(
const [isStopping, setIsStopping] = useState(false);
const [activeExecutionID, setActiveExecutionID] =
useState<GraphExecutionID | null>(null);
+ const [graphExecutionError, setGraphExecutionError] = useState<string | null>(
+ null,
+ );
const [xyNodes, setXYNodes] = useState<CustomNode[]>([]);
const [xyEdges, setXYEdges] = useState<CustomEdge[]>([]);
const { state, completeStep, incrementRuns } = useOnboarding();
@@ -461,6 +464,15 @@ export default function useAgentGraph(
flowID,
flowExecutionID,
);
+
+ // Set graph execution error from the initial fetch
+ if (execution.status === "FAILED") {
+ setGraphExecutionError(
+ execution.stats?.error ||
+ "The execution failed due to an internal error. You can re-run the agent to retry.",
+ );
+ }
+
if (
(execution.status === "QUEUED" || execution.status === "RUNNING") &&
!isRunning
@@ -479,32 +491,48 @@ export default function useAgentGraph(
if (graphExec.id != flowExecutionID) {
return;
}
- if (
- graphExec.status === "FAILED" &&
- graphExec?.stats?.error
- ?.toLowerCase()
- ?.includes("insufficient balance")
- ) {
- // Show no credits toast if user has low credits
- toast({
- variant: "destructive",
- title: "Credits low",
- description: (
- <div>
- Agent execution failed due to insufficient credits.
- <br />
- Go to the{" "}
- <NextLink
- className="text-purple-300"
- href="/marketplace/credits"
- >
- Credits
- </NextLink>{" "}
- page to top up.
- </div>
- ),
- duration: 5000,
- });
+
+ // Update graph execution error state and show toast
+ if (graphExec.status === "FAILED") {
+ const errorMessage =
+ graphExec.stats?.error ||
+ "The execution failed due to an internal error. You can re-run the agent to retry.";
+ setGraphExecutionError(errorMessage);
+
+ if (
+ graphExec.stats?.error
+ ?.toLowerCase()
+ .includes("insufficient balance")
+ ) {
+ // Show no credits toast if user has low credits
+ toast({
+ variant: "destructive",
+ title: "Credits low",
+ description: (
+ <div>
+ Agent execution failed due to insufficient credits.
+ <br />
+ Go to the{" "}
+ <NextLink
+ className="text-purple-300"
+ href="/profile/credits"
+ >
+ Credits
+ </NextLink>{" "}
+ page to top up.
+ </div>
+ ),
+ duration: 5000,
+ });
+ } else {
+ // Show general graph execution error
+ toast({
+ variant: "destructive",
+ title: "Agent execution failed",
+ description: errorMessage,
+ duration: 8000,
+ });
+ }
}
if (
graphExec.status === "COMPLETED" ||
@@ -999,6 +1027,7 @@ export default function useAgentGraph(
isRunning,
isStopping,
isScheduling,
+ graphExecutionError,
nodes: xyNodes,
setNodes: setXYNodes,
edges: xyEdges,