Skip to content

Open-source LLM knowledge platform: turn raw documents into a queryable RAG, an autonomous reasoning agent, and a self-maintaining Wiki.

fix(agent): 工具自起 goroutine 的 panic 自行兜底,MCP 预热不再带走进程 (#3846)

linus-liu-webe394c7adecbe
diff --git a/internal/agent/think.go b/internal/agent/think.go
index e7a5ad109..1e71edd98 100644
--- a/internal/agent/think.go
+++ b/internal/agent/think.go
@@ -227,7 +227,10 @@ func watchStreamStall(
stalled := &atomic.Bool{}
done := make(chan struct{})
- go func() {
+ // This goroutine cancels a stream in flight: a panic here (a non-positive
+ // stall timeout reaches time.NewTicker) must not end the process, and no
+ // unit of work needs a failure recorded.
+ agenttools.GoRecovered(ctx, func() {
// Poll well inside the window so the detected gap stays close to the
// configured timeout instead of rounding up to twice it.
ticker := time.NewTicker(stallTimeout / 4)
@@ -249,7 +252,7 @@ func watchStreamStall(
return
}
}
- }()
+ })
var once sync.Once
return stalled, func() { once.Do(func() { close(done) }) }
diff --git a/internal/agent/tool_goroutine_panic_test.go b/internal/agent/tool_goroutine_panic_test.go
new file mode 100644
index 000000000..498872f43
--- /dev/null
+++ b/internal/agent/tool_goroutine_panic_test.go
@@ -0,0 +1,71 @@
+package agent
+
+import (
+ "context"
+ "encoding/json"
+ "sync/atomic"
+ "testing"
+
+ agenttools "github.com/Tencent/WeKnora/internal/agent/tools"
+ "github.com/Tencent/WeKnora/internal/types"
+)
+
+// A tool that hands adapter work to a goroutine of its own must contain the
+// panic there: executeRecovered (registry.go) only covers the goroutine that
+// calls tool.Execute, so before the child barrier a panic from an MCP client,
+// a retrieval adapter or an SDK killed the whole process instead of failing
+// one tool call.
+type goroutineBarrierPanicTool struct {
+ agenttools.BaseTool
+}
+
+func (t *goroutineBarrierPanicTool) Execute(ctx context.Context, _ json.RawMessage) (*types.ToolResult, error) {
+ var panicked atomic.Bool
+ done := make(chan struct{})
+ go func() {
+ defer close(done)
+ // RecoverGoroutine runs before close(done) and marks the failed unit of
+ // work; this is the convention tools use in production.
+ defer agenttools.RecoverGoroutine(ctx, "test adapter", func() { panicked.Store(true) })
+ panic("test: tool adapter panicked on a child goroutine")
+ }()
+ <-done
+ if panicked.Load() {
+ return &types.ToolResult{Success: false, Error: "adapter failed with an internal error"}, nil
+ }
+ return &types.ToolResult{Success: true}, nil
+}
+
+func runAgentToolCall(t *testing.T, tool types.Tool) *types.AgentStep {
+ t.Helper()
+ engine := newTestEngine(t, &mockChat{})
+ engine.toolRegistry = agenttools.NewToolRegistry()
+ engine.toolRegistry.RegisterTool(tool)
+ response := &types.ChatResponse{ToolCalls: []types.LLMToolCall{
+ {ID: "a", Function: types.FunctionCall{Name: agenttools.ToolSearchKnowledge, Arguments: `{}`}},
+ }}
+ step := &types.AgentStep{}
+ engine.executeToolCallsParallel(context.Background(), response, step, 0, "session", "message")
+ return step
+}
+
+// A child-goroutine panic is degraded into a failed tool result and the test
+// process survives; removing the barrier makes this binary exit with the panic.
+func TestToolChildGoroutinePanicDegradesToFailedResult(t *testing.T) {
+ tool := &goroutineBarrierPanicTool{
+ BaseTool: agenttools.NewBaseTool(
+ agenttools.ToolSearchKnowledge, "", json.RawMessage(`{"type":"object"}`),
+ ),
+ }
+ step := runAgentToolCall(t, tool)
+ if len(step.ToolCalls) != 1 {
+ t.Fatalf("expected 1 tool call record, got %d", len(step.ToolCalls))
+ }
+ result := step.ToolCalls[0].Result
+ if result == nil || result.Success {
+ t.Fatalf("expected a failed tool result, got %+v", result)
+ }
+ if result.Error == "" {
+ t.Fatal("failed result must carry an error the model can act on")
+ }
+}
diff --git a/internal/agent/tools/goroutine.go b/internal/agent/tools/goroutine.go
new file mode 100644
index 000000000..0896a48b5
--- /dev/null
+++ b/internal/agent/tools/goroutine.go
@@ -0,0 +1,50 @@
+package tools
+
+import (
+ "context"
+ "runtime/debug"
+
+ "github.com/Tencent/WeKnora/internal/logger"
+)
+
+// GoRecovered runs fn on a new goroutine and contains a panic raised there.
+//
+// recover only covers the goroutine that calls it, so the barrier in
+// executeRecovered (registry.go) protects the call to tool.Execute alone: a
+// goroutine a tool starts for itself — an MCP client, a sandbox, a third-party
+// SDK, a retrieval adapter — must install its own barrier, or one panic escapes
+// every recover site and takes the whole server (and every conversation,
+// upload and sync in flight) down with it.
+//
+// Deferred calls inside fn still run while a panic unwinds, so wait groups and
+// cleanup are released. A caller that must also degrade the failed work into an
+// error result uses RecoverGoroutine directly and records it in onPanic.
+func GoRecovered(ctx context.Context, fn func()) {
+ go func() {
+ defer RecoverGoroutine(ctx, "", nil)
+ fn()
+ }()
+}
+
+// RecoverGoroutine is deferred by a goroutine that must not let a panic escape
+// (either directly or through GoRecovered). It logs the panic with its stack,
+// then calls onPanic — when set — on that same goroutine so the caller can mark
+// the failed unit of work: one MCP service, one knowledge base, one retrieval
+// call. label names that work in the log; an empty one leaves it unnamed.
+//
+// onPanic deliberately receives no panic value: a recovered value carries
+// internals (paths, addresses) that must not reach the model or the user.
+func RecoverGoroutine(ctx context.Context, label string, onPanic func()) {
+ r := recover()
+ if r == nil {
+ return
+ }
+ where := "[ToolGoroutine]"
+ if label != "" {
+ where += " " + label
+ }
+ logger.Errorf(ctx, "%s panicked: %v\n%s", where, r, debug.Stack())
+ if onPanic != nil {
+ onPanic()
+ }
+}
diff --git a/internal/agent/tools/goroutine_test.go b/internal/agent/tools/goroutine_test.go
new file mode 100644
index 000000000..b7d5a7e3f
--- /dev/null
+++ b/internal/agent/tools/goroutine_test.go
@@ -0,0 +1,43 @@
+package tools
+
+import (
+ "testing"
+ "time"
+)
+
+// GoRecovered must contain a panic raised on the goroutine it starts and still
+// run the deferred calls fn installed. Without the barrier this test binary
+// dies with "test: adapter panicked on a helper goroutine".
+func TestGoRecoveredContainsPanicAndRunsDefers(t *testing.T) {
+ cleaned := make(chan struct{})
+ GoRecovered(catalogTestContext(), func() {
+ defer close(cleaned)
+ panic("test: adapter panicked on a helper goroutine")
+ })
+ select {
+ case <-cleaned:
+ case <-time.After(5 * time.Second):
+ t.Fatal("GoRecovered did not run fn")
+ }
+}
+
+// RecoverGoroutine is the deferred form: it reports the failure through onPanic
+// on the panicking goroutine, before the caller's own deferred hooks run.
+func TestRecoverGoroutineReportsPanicToCaller(t *testing.T) {
+ ctx := catalogTestContext()
+ var marked bool
+ done := make(chan struct{})
+ go func() {
+ defer close(done)
+ defer RecoverGoroutine(ctx, "test retrieval", func() { marked = true })
+ panic("test: adapter panicked on a caller-owned goroutine")
+ }()
+ select {
+ case <-done:
+ case <-time.After(5 * time.Second):
+ t.Fatal("RecoverGoroutine did not contain the panic")
+ }
+ if !marked {
+ t.Fatal("RecoverGoroutine did not report the panic to onPanic")
+ }
+}
diff --git a/internal/agent/tools/mcp_exposure.go b/internal/agent/tools/mcp_exposure.go
index 8213171f9..f756f76cc 100644
--- a/internal/agent/tools/mcp_exposure.go
+++ b/internal/agent/tools/mcp_exposure.go
@@ -120,6 +120,10 @@ func (r *ToolRegistry) prepareMCPToolsWithMode(ctx context.Context, grace time.D
c.preloadDone = make(chan struct{})
go func() {
defer close(c.preloadDone)
+ // prepareMCPToolsWithMode selects on preloadDone, so a panic in the
+ // fan-out below must not skip the close; the barrier also records
+ // every service this pass left unfinished.
+ defer RecoverGoroutine(ctx, "MCP catalog preload", func() { c.markPreloadFailed() })
loadCtx, cancel := context.WithTimeout(
context.WithValue(ctx, execCtxKey{}, (*ToolExecContext)(nil)),
mcpCatalogLoadTimeout,
@@ -137,9 +141,18 @@ func (r *ToolRegistry) prepareMCPToolsWithMode(ctx context.Context, grace time.D
for i := 0; i < min(8, len(ids)); i++ {
workers.Add(1)
go func() {
+ // Two barriers: this one keeps a panic in the loop itself
+ // (channel bookkeeping, Done) from escaping; the one inside
+ // knows which service the MCP client panicked on.
defer workers.Done()
+ defer RecoverGoroutine(loadCtx, "MCP catalog preload worker", nil)
for id := range jobs {
- _, _, _ = c.snapshot(loadCtx, id, false)
+ func() {
+ defer RecoverGoroutine(loadCtx, "MCP catalog preload for service "+id, func() {
+ c.markPreloadFailed(id)
+ })
+ _, _, _ = c.snapshot(loadCtx, id, false)
+ }()
}
}()
}
@@ -165,6 +178,34 @@ func (r *ToolRegistry) prepareMCPToolsWithMode(ctx context.Context, grace time.D
r.RefreshMCPTools(ctx)
}
+// markPreloadFailed records a preload pass that a panic ended, reusing the
+// catalog's existing store/status mechanism and the same "error" status
+// snapshot() stores when a load returns an error. A service whose goroutine is
+// gone must not stay "loading", which would read as a preload still in flight.
+// With no id every service that never reached "ready" is marked; a ready
+// service loaded fine and keeps its snapshot.
+func (c *MCPCatalog) markPreloadFailed(ids ...string) {
+ if len(ids) == 0 {
+ ids = make([]string, 0, len(c.servers))
+ for id := range c.servers {
+ ids = append(ids, id)
+ }
+ }
+ for _, id := range ids {
+ entry := c.servers[id]
+ if entry == nil {
+ continue
+ }
+ entry.mu.Lock()
+ service, status := entry.service, entry.status
+ entry.mu.Unlock()
+ if status == "ready" {
+ continue
+ }
+ entry.store(service, nil, "error")
+ }
+}
+
// RememberMCPHistory republishes tools this session already described or called
// so a new engine does not require another describe round.
func (r *ToolRegistry) RememberMCPHistory(messages []chat.Message) {
diff --git a/internal/agent/tools/mcp_exposure_panic_test.go b/internal/agent/tools/mcp_exposure_panic_test.go
new file mode 100644
index 000000000..d41ec6f17
--- /dev/null
+++ b/internal/agent/tools/mcp_exposure_panic_test.go
@@ -0,0 +1,80 @@
+package tools
+
+import (
+ "context"
+ "testing"
+ "time"
+
+ "github.com/Tencent/WeKnora/internal/types"
+ "github.com/stretchr/testify/require"
+)
+
+// A panic raised by the MCP client on a catalog preload worker used to kill the
+// whole process: the preload goroutines in mcp_exposure.go had no recover, and
+// the recover sites in registry.go and act.go only cover their own goroutine.
+// Before the fix this test never reaches its assertions — the test binary dies
+// on prepareMCPToolsWithMode.func1.1.1 (mcp_exposure.go:139 -> snapshot -> load).
+func TestMCPPreloadWorkerPanicIsContained(t *testing.T) {
+ ctx := catalogTestContext()
+ service := &types.MCPService{ID: "panic-mcp", Name: "Panic", Enabled: true}
+ catalog := newMCPCatalog(
+ ctx,
+ []*types.MCPService{service},
+ nil,
+ func(context.Context, *types.MCPService, bool) ([]*MCPTool, error) {
+ // The loader is the seam snapshot() reaches the MCP client through.
+ panic("test: MCP client panicked on a preload worker")
+ },
+ nil,
+ )
+ registry := NewToolRegistry()
+ installMCPCatalog(registry, catalog)
+
+ returned := make(chan struct{})
+ go func() {
+ defer close(returned)
+ registry.PrepareMCPTools(ctx)
+ }()
+ select {
+ case <-returned:
+ case <-time.After(10 * time.Second):
+ t.Fatal("PrepareMCPTools did not return after a preload worker panicked")
+ }
+
+ entry := catalog.servers[service.ID]
+ require.NotNil(t, entry)
+ entry.mu.Lock()
+ status, tools := entry.status, entry.tools
+ entry.mu.Unlock()
+ require.Equal(t, "error", status, "a panicking service must be marked failed, not left loading")
+ require.Empty(t, tools, "a panicking service must not keep a partial snapshot")
+}
+
+// The coordinator barrier records the services a panicking preload pass never
+// finished and leaves a ready snapshot alone. Both reuse the existing "error"
+// status snapshot() stores when a load fails; no new state is introduced.
+func TestMCPPreloadFailureMarkingKeepsReadySnapshots(t *testing.T) {
+ ctx := catalogTestContext()
+ ready := &types.MCPService{ID: "ready-mcp", Name: "Ready", Enabled: true}
+ loading := &types.MCPService{ID: "loading-mcp", Name: "Loading", Enabled: true}
+ catalog := newMCPCatalog(ctx, []*types.MCPService{ready, loading}, nil, nil, nil)
+ tool := NewMCPTool(ready, &types.MCPTool{Name: "ping"}, nil, nil, 0)
+ catalog.servers[ready.ID].store(ready, []*MCPTool{tool}, "ready")
+ catalog.servers[loading.ID].store(loading, nil, "loading")
+
+ catalog.markPreloadFailed()
+
+ readyEntry := catalog.servers[ready.ID]
+ readyEntry.mu.Lock()
+ readyStatus, readyTools := readyEntry.status, readyEntry.tools
+ readyEntry.mu.Unlock()
+ require.Equal(t, "ready", readyStatus, "a service that loaded fine keeps its snapshot")
+ require.Len(t, readyTools, 1)
+
+ loadingEntry := catalog.servers[loading.ID]
+ loadingEntry.mu.Lock()
+ loadingStatus, loadingTools := loadingEntry.status, loadingEntry.tools
+ loadingEntry.mu.Unlock()
+ require.Equal(t, "error", loadingStatus)
+ require.Empty(t, loadingTools)
+}
diff --git a/internal/agent/tools/registry.go b/internal/agent/tools/registry.go
index 9003e3131..267d24764 100644
--- a/internal/agent/tools/registry.go
+++ b/internal/agent/tools/registry.go
@@ -295,6 +295,10 @@ func (r *ToolRegistry) execute(ctx context.Context, tool types.Tool, args json.R
// run on errgroup goroutines when a round executes in parallel, and errgroup
// does not carry a panic back to Wait: an unrecovered one from any tool (MCP
// clients, sandboxes, third-party SDKs) would take down the whole server.
+//
+// recover only covers the goroutine that calls it, so this guards the
+// tool.Execute call alone: a goroutine a tool starts for itself must install
+// its own barrier (GoRecovered / RecoverGoroutine in goroutine.go).
func executeRecovered(
ctx context.Context, tool types.Tool, args json.RawMessage,
) (result *types.ToolResult, err error) {
diff --git a/internal/agent/tools/search_knowledge.go b/internal/agent/tools/search_knowledge.go
index a43ceee33..a113c5bad 100644
--- a/internal/agent/tools/search_knowledge.go
+++ b/internal/agent/tools/search_knowledge.go
@@ -3,6 +3,7 @@ package tools
import (
"context"
"encoding/json"
+ "errors"
"fmt"
"math"
"regexp"
@@ -32,6 +33,11 @@ const (
searchKnowledgeMaxLimit = 30
)
+// errSearchPanicked reports a retrieval adapter that panicked. The barrier
+// logs the stack and the knowledge base is reported as failed, exactly like a
+// search that returned an error; the panic value itself is never surfaced.
+var errSearchPanicked = errors.New("knowledge base search failed with an internal error")
+
var searchKnowledgeTool = BaseTool{
name: ToolSearchKnowledge,
description: "Search the knowledge bases in scope and return the most relevant chunks with their content.\n" +
@@ -591,6 +597,16 @@ func (t *SearchKnowledgeTool) concurrentSearchByTargets(
wg.Add(1)
go func(modelKey string, targets []*types.SearchTarget) {
defer wg.Done()
+ // The embedding adapter runs here, before any target is searched: a
+ // panic from it is contained and every target still pending in the
+ // group is reported as failed instead of ending the process.
+ defer RecoverGoroutine(ctx, "search knowledge embedding group", func() {
+ ids := make([]string, 0, len(targets))
+ for _, st := range targets {
+ ids = append(ids, st.KnowledgeBaseID)
+ }
+ fail(ids, errSearchPanicked)
+ })
needsEmbedding := false
for _, st := range targets {
@@ -655,6 +671,9 @@ func (t *SearchKnowledgeTool) concurrentSearchByTargets(
innerWg.Add(1)
go func(usedMode string, fullKBIDs []string) {
defer innerWg.Done()
+ defer RecoverGoroutine(ctx, "search knowledge combined retrieval", func() {
+ fail(fullKBIDs, errSearchPanicked)
+ })
attempt()
kbResults, err := t.knowledgeBaseService.HybridSearch(ctx, fullKBIDs[0], types.SearchParams{
QueryText: query,
@@ -686,6 +705,9 @@ func (t *SearchKnowledgeTool) concurrentSearchByTargets(
innerWg.Add(1)
go func() {
defer innerWg.Done()
+ defer RecoverGoroutine(ctx, "search knowledge retrieval", func() {
+ fail([]string{st.KnowledgeBaseID}, errSearchPanicked)
+ })
attempt()
usedMode, _ := groupModeFor(st.KnowledgeBaseID)
stVectorThreshold, stKeywordThreshold := st.RecallThresholds(vectorThreshold, keywordThreshold)