Open-source LLM knowledge platform: turn raw documents into a queryable RAG, an autonomous reasoning agent, and a self-maintaining Wiki.
Go agentagenticaichatbotdsh-pluginembeddingsevaluationgenerative-aigolangknowledge-basellmmulti-tenantollamaopenaiquestion-answeringragrerankingsemantic-searchvector-searchwiki
fix(agent): recover tool panics instead of crashing the server
wizardchenef6f28c1cafe
diff --git a/internal/agent/act.go b/internal/agent/act.goindex 6933c4fd3..3ea6dfb7f 100644--- a/internal/agent/act.go+++ b/internal/agent/act.go@@ -5,6 +5,7 @@ import ("encoding/json""errors""fmt"+ "runtime/debug""sort""sync""time"@@ -315,6 +316,18 @@ func (e *AgentEngine) executeToolCallsParallel(}readCtx := gCtxg.Go(func() error {+ // The registry recovers panics inside tool execution; this guards+ // the bookkeeping around it, which would otherwise crash the+ // process from a goroutine nothing above can recover.+ defer func() {+ if r := recover(); r != nil {+ logger.Errorf(ctx, "[Agent][Round-%d] Tool call %s panicked: %v\n%s",+ round, tc.Function.Name, r, debug.Stack())+ mu.Lock()+ results[i] = panickedToolCall(tc, i)+ mu.Unlock()+ }+ }()toolCall := e.runToolCall(readCtx, tc, i, iteration, round, sessionID, assistantMessageID)mu.Lock()results[i] = toolCall@@ -332,6 +345,21 @@ func (e *AgentEngine) executeToolCallsParallel(}}+// panickedToolCall is the failed record for a call whose handling panicked, so+// the round still returns one result per tool call the model issued.+func panickedToolCall(tc types.LLMToolCall, i int) types.ToolCall {+ return types.ToolCall{+ ID: agenttools.NormalizeToolCallID(tc.ID, tc.Function.Name, i),+ Name: tc.Function.Name,+ Args: map[string]any{"_raw": tc.Function.Arguments},+ ProviderMetadata: tc.ProviderMetadata,+ Result: &types.ToolResult{+ Success: false,+ Error: fmt.Sprintf("tool %s failed with an internal error", tc.Function.Name),+ },+ }+}+// emitToolOutcome publishes the result and action events for one finished tool// call. Every path that produces a ToolCall goes through here — sequential,// parallel, and refused-as-truncated — so the UI sees one event shape.diff --git a/internal/agent/act_order_test.go b/internal/agent/act_order_test.goindex 11205c6c1..bd6a95b97 100644--- a/internal/agent/act_order_test.go+++ b/internal/agent/act_order_test.go@@ -88,3 +88,28 @@ func TestParallelReadsRespectMutationBarriers(t *testing.T) {require.Equal(t, names[i], call.Name, "result events retain original order")}}++func TestParallelToolPanicFailsOnlyThatCall(t *testing.T) {+ engine := newTestEngine(t, &mockChat{})+ engine.toolRegistry = tools.NewToolRegistry()+ engine.toolRegistry.RegisterTool(&orderedTestTool{+ BaseTool: tools.NewBaseTool(tools.ToolSearchKnowledge, "", json.RawMessage(`{"type":"object"}`)),+ run: func(context.Context) *types.ToolResult { panic("tool bug") },+ })+ engine.toolRegistry.RegisterTool(&orderedTestTool{+ BaseTool: tools.NewBaseTool(tools.ToolReadDocument, "", json.RawMessage(`{"type":"object"}`)),+ run: func(context.Context) *types.ToolResult { return &types.ToolResult{Success: true, Output: "ok"} },+ })+ response := &types.ChatResponse{ToolCalls: []types.LLMToolCall{+ {ID: "a", Function: types.FunctionCall{Name: tools.ToolSearchKnowledge, Arguments: `{}`}},+ {ID: "b", Function: types.FunctionCall{Name: tools.ToolReadDocument, Arguments: `{}`}},+ }}+ step := &types.AgentStep{}++ engine.executeToolCallsParallel(context.Background(), response, step, 0, "session", "message")++ require.Len(t, step.ToolCalls, 2)+ require.False(t, step.ToolCalls[0].Result.Success)+ require.Contains(t, step.ToolCalls[0].Result.Error, "internal error")+ require.True(t, step.ToolCalls[1].Result.Success)+}diff --git a/internal/agent/tools/registry.go b/internal/agent/tools/registry.goindex 439f36121..9003e3131 100644--- a/internal/agent/tools/registry.go+++ b/internal/agent/tools/registry.go@@ -4,6 +4,7 @@ import ("context""encoding/json""fmt"+ "runtime/debug""sort""strings""unicode/utf8"@@ -247,7 +248,7 @@ func (r *ToolRegistry) execute(ctx context.Context, tool types.Tool, args json.RmaxOutput = toolLimit}}- result, execErr := tool.Execute(WithOutputBudget(ctx, maxOutput), args)+ result, execErr := executeRecovered(WithOutputBudget(ctx, maxOutput), tool, args)if result == nil {result = &types.ToolResult{Success: false, Error: "tool returned no result"}}@@ -290,6 +291,25 @@ func (r *ToolRegistry) execute(ctx context.Context, tool types.Tool, args json.Rreturn result, execErr}+// executeRecovered runs one tool and turns a panic into a failed result. Tools+// run on errgroup goroutines when a round executes in parallel, and errgroup+// does not carry a panic back to Wait: an unrecovered one from any tool (MCP+// clients, sandboxes, third-party SDKs) would take down the whole server.+func executeRecovered(+ ctx context.Context, tool types.Tool, args json.RawMessage,+) (result *types.ToolResult, err error) {+ defer func() {+ if r := recover(); r != nil {+ logger.Errorf(ctx, "[ToolRegistry] Tool %s panicked: %v\n%s", tool.Name(), r, debug.Stack())+ // The panic value can carry internals (paths, addresses); the+ // model and the user only need to know the call did not complete.+ err = fmt.Errorf("tool %s failed with an internal error", tool.Name())+ result = &types.ToolResult{Success: false, Error: err.Error()}+ }+ }()+ return tool.Execute(ctx, args)+}+// Cleanup cleans up all registered tools that implement the types.Cleanable interface.// This is called at the end of agent sessions to release tool-specific resources.func (r *ToolRegistry) Cleanup(ctx context.Context) {diff --git a/internal/agent/tools/registry_outcome_test.go b/internal/agent/tools/registry_outcome_test.goindex 25ad7b9cb..710836ccc 100644--- a/internal/agent/tools/registry_outcome_test.go+++ b/internal/agent/tools/registry_outcome_test.go@@ -61,3 +61,23 @@ func TestRegistryNormalizesMissingAndContradictoryResults(t *testing.T) {require.NotEmpty(t, result.Error)}}++type panickingTool struct{ BaseTool }++func (t *panickingTool) Execute(context.Context, json.RawMessage) (*types.ToolResult, error) {+ var items []int+ _ = items[len(items)] // index out of range: a runtime panic like a buggy tool would raise+ return nil, nil+}++func TestRegistryRecoversToolPanic(t *testing.T) {+ registry := NewToolRegistry()+ registry.RegisterTool(&panickingTool{BaseTool: NewBaseTool("crashy", "", json.RawMessage(`{"type":"object"}`))})++ result, err := registry.ExecuteTool(context.Background(), "crashy", json.RawMessage(`{}`))+ require.Error(t, err)+ require.NotNil(t, result)+ require.False(t, result.Success)+ require.Equal(t, "tool crashy failed with an internal error", result.Error)+ require.NotContains(t, result.Error, "index out of range")+}