Skip to content

Open-source LLM knowledge platform: turn raw documents into a queryable RAG, an autonomous reasoning agent, and a self-maintaining Wiki.

fix(agent): recover tool panics instead of crashing the server

wizardchenef6f28c1cafe
diff --git a/internal/agent/act.go b/internal/agent/act.go
index 6933c4fd3..3ea6dfb7f 100644
--- a/internal/agent/act.go
+++ b/internal/agent/act.go
@@ -5,6 +5,7 @@ import (
"encoding/json"
"errors"
"fmt"
+ "runtime/debug"
"sort"
"sync"
"time"
@@ -315,6 +316,18 @@ func (e *AgentEngine) executeToolCallsParallel(
}
readCtx := gCtx
g.Go(func() error {
+ // The registry recovers panics inside tool execution; this guards
+ // the bookkeeping around it, which would otherwise crash the
+ // process from a goroutine nothing above can recover.
+ defer func() {
+ if r := recover(); r != nil {
+ logger.Errorf(ctx, "[Agent][Round-%d] Tool call %s panicked: %v\n%s",
+ round, tc.Function.Name, r, debug.Stack())
+ mu.Lock()
+ results[i] = panickedToolCall(tc, i)
+ mu.Unlock()
+ }
+ }()
toolCall := e.runToolCall(readCtx, tc, i, iteration, round, sessionID, assistantMessageID)
mu.Lock()
results[i] = toolCall
@@ -332,6 +345,21 @@ func (e *AgentEngine) executeToolCallsParallel(
}
}
+// panickedToolCall is the failed record for a call whose handling panicked, so
+// the round still returns one result per tool call the model issued.
+func panickedToolCall(tc types.LLMToolCall, i int) types.ToolCall {
+ return types.ToolCall{
+ ID: agenttools.NormalizeToolCallID(tc.ID, tc.Function.Name, i),
+ Name: tc.Function.Name,
+ Args: map[string]any{"_raw": tc.Function.Arguments},
+ ProviderMetadata: tc.ProviderMetadata,
+ Result: &types.ToolResult{
+ Success: false,
+ Error: fmt.Sprintf("tool %s failed with an internal error", tc.Function.Name),
+ },
+ }
+}
+
// emitToolOutcome publishes the result and action events for one finished tool
// call. Every path that produces a ToolCall goes through here — sequential,
// parallel, and refused-as-truncated — so the UI sees one event shape.
diff --git a/internal/agent/act_order_test.go b/internal/agent/act_order_test.go
index 11205c6c1..bd6a95b97 100644
--- a/internal/agent/act_order_test.go
+++ b/internal/agent/act_order_test.go
@@ -88,3 +88,28 @@ func TestParallelReadsRespectMutationBarriers(t *testing.T) {
require.Equal(t, names[i], call.Name, "result events retain original order")
}
}
+
+func TestParallelToolPanicFailsOnlyThatCall(t *testing.T) {
+ engine := newTestEngine(t, &mockChat{})
+ engine.toolRegistry = tools.NewToolRegistry()
+ engine.toolRegistry.RegisterTool(&orderedTestTool{
+ BaseTool: tools.NewBaseTool(tools.ToolSearchKnowledge, "", json.RawMessage(`{"type":"object"}`)),
+ run: func(context.Context) *types.ToolResult { panic("tool bug") },
+ })
+ engine.toolRegistry.RegisterTool(&orderedTestTool{
+ BaseTool: tools.NewBaseTool(tools.ToolReadDocument, "", json.RawMessage(`{"type":"object"}`)),
+ run: func(context.Context) *types.ToolResult { return &types.ToolResult{Success: true, Output: "ok"} },
+ })
+ response := &types.ChatResponse{ToolCalls: []types.LLMToolCall{
+ {ID: "a", Function: types.FunctionCall{Name: tools.ToolSearchKnowledge, Arguments: `{}`}},
+ {ID: "b", Function: types.FunctionCall{Name: tools.ToolReadDocument, Arguments: `{}`}},
+ }}
+ step := &types.AgentStep{}
+
+ engine.executeToolCallsParallel(context.Background(), response, step, 0, "session", "message")
+
+ require.Len(t, step.ToolCalls, 2)
+ require.False(t, step.ToolCalls[0].Result.Success)
+ require.Contains(t, step.ToolCalls[0].Result.Error, "internal error")
+ require.True(t, step.ToolCalls[1].Result.Success)
+}
diff --git a/internal/agent/tools/registry.go b/internal/agent/tools/registry.go
index 439f36121..9003e3131 100644
--- a/internal/agent/tools/registry.go
+++ b/internal/agent/tools/registry.go
@@ -4,6 +4,7 @@ import (
"context"
"encoding/json"
"fmt"
+ "runtime/debug"
"sort"
"strings"
"unicode/utf8"
@@ -247,7 +248,7 @@ func (r *ToolRegistry) execute(ctx context.Context, tool types.Tool, args json.R
maxOutput = toolLimit
}
}
- result, execErr := tool.Execute(WithOutputBudget(ctx, maxOutput), args)
+ result, execErr := executeRecovered(WithOutputBudget(ctx, maxOutput), tool, args)
if result == nil {
result = &types.ToolResult{Success: false, Error: "tool returned no result"}
}
@@ -290,6 +291,25 @@ func (r *ToolRegistry) execute(ctx context.Context, tool types.Tool, args json.R
return result, execErr
}
+// executeRecovered runs one tool and turns a panic into a failed result. Tools
+// run on errgroup goroutines when a round executes in parallel, and errgroup
+// does not carry a panic back to Wait: an unrecovered one from any tool (MCP
+// clients, sandboxes, third-party SDKs) would take down the whole server.
+func executeRecovered(
+ ctx context.Context, tool types.Tool, args json.RawMessage,
+) (result *types.ToolResult, err error) {
+ defer func() {
+ if r := recover(); r != nil {
+ logger.Errorf(ctx, "[ToolRegistry] Tool %s panicked: %v\n%s", tool.Name(), r, debug.Stack())
+ // The panic value can carry internals (paths, addresses); the
+ // model and the user only need to know the call did not complete.
+ err = fmt.Errorf("tool %s failed with an internal error", tool.Name())
+ result = &types.ToolResult{Success: false, Error: err.Error()}
+ }
+ }()
+ return tool.Execute(ctx, args)
+}
+
// Cleanup cleans up all registered tools that implement the types.Cleanable interface.
// This is called at the end of agent sessions to release tool-specific resources.
func (r *ToolRegistry) Cleanup(ctx context.Context) {
diff --git a/internal/agent/tools/registry_outcome_test.go b/internal/agent/tools/registry_outcome_test.go
index 25ad7b9cb..710836ccc 100644
--- a/internal/agent/tools/registry_outcome_test.go
+++ b/internal/agent/tools/registry_outcome_test.go
@@ -61,3 +61,23 @@ func TestRegistryNormalizesMissingAndContradictoryResults(t *testing.T) {
require.NotEmpty(t, result.Error)
}
}
+
+type panickingTool struct{ BaseTool }
+
+func (t *panickingTool) Execute(context.Context, json.RawMessage) (*types.ToolResult, error) {
+ var items []int
+ _ = items[len(items)] // index out of range: a runtime panic like a buggy tool would raise
+ return nil, nil
+}
+
+func TestRegistryRecoversToolPanic(t *testing.T) {
+ registry := NewToolRegistry()
+ registry.RegisterTool(&panickingTool{BaseTool: NewBaseTool("crashy", "", json.RawMessage(`{"type":"object"}`))})
+
+ result, err := registry.ExecuteTool(context.Background(), "crashy", json.RawMessage(`{}`))
+ require.Error(t, err)
+ require.NotNil(t, result)
+ require.False(t, result.Success)
+ require.Equal(t, "tool crashy failed with an internal error", result.Error)
+ require.NotContains(t, result.Error, "index out of range")
+}