Open-source LLM knowledge platform: turn raw documents into a queryable RAG, an autonomous reasoning agent, and a self-maintaining Wiki.
Go agentagenticaichatbotdsh-pluginembeddingsevaluationgenerative-aigolangknowledge-basellmmulti-tenantollamaopenaiquestion-answeringragrerankingsemantic-searchvector-searchwiki
fix(agent): 工具自起 goroutine 的 panic 自行兜底,MCP 预热不再带走进程 (#3846)
linus-liu-webe394c7adecbe
diff --git a/internal/agent/think.go b/internal/agent/think.goindex e7a5ad109..1e71edd98 100644--- a/internal/agent/think.go+++ b/internal/agent/think.go@@ -227,7 +227,10 @@ func watchStreamStall(stalled := &atomic.Bool{}done := make(chan struct{})- go func() {+ // This goroutine cancels a stream in flight: a panic here (a non-positive+ // stall timeout reaches time.NewTicker) must not end the process, and no+ // unit of work needs a failure recorded.+ agenttools.GoRecovered(ctx, func() {// Poll well inside the window so the detected gap stays close to the// configured timeout instead of rounding up to twice it.ticker := time.NewTicker(stallTimeout / 4)@@ -249,7 +252,7 @@ func watchStreamStall(return}}- }()+ })var once sync.Oncereturn stalled, func() { once.Do(func() { close(done) }) }diff --git a/internal/agent/tool_goroutine_panic_test.go b/internal/agent/tool_goroutine_panic_test.gonew file mode 100644index 000000000..498872f43--- /dev/null+++ b/internal/agent/tool_goroutine_panic_test.go@@ -0,0 +1,71 @@+package agent++import (+ "context"+ "encoding/json"+ "sync/atomic"+ "testing"++ agenttools "github.com/Tencent/WeKnora/internal/agent/tools"+ "github.com/Tencent/WeKnora/internal/types"+)++// A tool that hands adapter work to a goroutine of its own must contain the+// panic there: executeRecovered (registry.go) only covers the goroutine that+// calls tool.Execute, so before the child barrier a panic from an MCP client,+// a retrieval adapter or an SDK killed the whole process instead of failing+// one tool call.+type goroutineBarrierPanicTool struct {+ agenttools.BaseTool+}++func (t *goroutineBarrierPanicTool) Execute(ctx context.Context, _ json.RawMessage) (*types.ToolResult, error) {+ var panicked atomic.Bool+ done := make(chan struct{})+ go func() {+ defer close(done)+ // RecoverGoroutine runs before close(done) and marks the failed unit of+ // work; this is the convention tools use in production.+ defer agenttools.RecoverGoroutine(ctx, "test adapter", func() { panicked.Store(true) })+ panic("test: tool adapter panicked on a child goroutine")+ }()+ <-done+ if panicked.Load() {+ return &types.ToolResult{Success: false, Error: "adapter failed with an internal error"}, nil+ }+ return &types.ToolResult{Success: true}, nil+}++func runAgentToolCall(t *testing.T, tool types.Tool) *types.AgentStep {+ t.Helper()+ engine := newTestEngine(t, &mockChat{})+ engine.toolRegistry = agenttools.NewToolRegistry()+ engine.toolRegistry.RegisterTool(tool)+ response := &types.ChatResponse{ToolCalls: []types.LLMToolCall{+ {ID: "a", Function: types.FunctionCall{Name: agenttools.ToolSearchKnowledge, Arguments: `{}`}},+ }}+ step := &types.AgentStep{}+ engine.executeToolCallsParallel(context.Background(), response, step, 0, "session", "message")+ return step+}++// A child-goroutine panic is degraded into a failed tool result and the test+// process survives; removing the barrier makes this binary exit with the panic.+func TestToolChildGoroutinePanicDegradesToFailedResult(t *testing.T) {+ tool := &goroutineBarrierPanicTool{+ BaseTool: agenttools.NewBaseTool(+ agenttools.ToolSearchKnowledge, "", json.RawMessage(`{"type":"object"}`),+ ),+ }+ step := runAgentToolCall(t, tool)+ if len(step.ToolCalls) != 1 {+ t.Fatalf("expected 1 tool call record, got %d", len(step.ToolCalls))+ }+ result := step.ToolCalls[0].Result+ if result == nil || result.Success {+ t.Fatalf("expected a failed tool result, got %+v", result)+ }+ if result.Error == "" {+ t.Fatal("failed result must carry an error the model can act on")+ }+}diff --git a/internal/agent/tools/goroutine.go b/internal/agent/tools/goroutine.gonew file mode 100644index 000000000..0896a48b5--- /dev/null+++ b/internal/agent/tools/goroutine.go@@ -0,0 +1,50 @@+package tools++import (+ "context"+ "runtime/debug"++ "github.com/Tencent/WeKnora/internal/logger"+)++// GoRecovered runs fn on a new goroutine and contains a panic raised there.+//+// recover only covers the goroutine that calls it, so the barrier in+// executeRecovered (registry.go) protects the call to tool.Execute alone: a+// goroutine a tool starts for itself — an MCP client, a sandbox, a third-party+// SDK, a retrieval adapter — must install its own barrier, or one panic escapes+// every recover site and takes the whole server (and every conversation,+// upload and sync in flight) down with it.+//+// Deferred calls inside fn still run while a panic unwinds, so wait groups and+// cleanup are released. A caller that must also degrade the failed work into an+// error result uses RecoverGoroutine directly and records it in onPanic.+func GoRecovered(ctx context.Context, fn func()) {+ go func() {+ defer RecoverGoroutine(ctx, "", nil)+ fn()+ }()+}++// RecoverGoroutine is deferred by a goroutine that must not let a panic escape+// (either directly or through GoRecovered). It logs the panic with its stack,+// then calls onPanic — when set — on that same goroutine so the caller can mark+// the failed unit of work: one MCP service, one knowledge base, one retrieval+// call. label names that work in the log; an empty one leaves it unnamed.+//+// onPanic deliberately receives no panic value: a recovered value carries+// internals (paths, addresses) that must not reach the model or the user.+func RecoverGoroutine(ctx context.Context, label string, onPanic func()) {+ r := recover()+ if r == nil {+ return+ }+ where := "[ToolGoroutine]"+ if label != "" {+ where += " " + label+ }+ logger.Errorf(ctx, "%s panicked: %v\n%s", where, r, debug.Stack())+ if onPanic != nil {+ onPanic()+ }+}diff --git a/internal/agent/tools/goroutine_test.go b/internal/agent/tools/goroutine_test.gonew file mode 100644index 000000000..b7d5a7e3f--- /dev/null+++ b/internal/agent/tools/goroutine_test.go@@ -0,0 +1,43 @@+package tools++import (+ "testing"+ "time"+)++// GoRecovered must contain a panic raised on the goroutine it starts and still+// run the deferred calls fn installed. Without the barrier this test binary+// dies with "test: adapter panicked on a helper goroutine".+func TestGoRecoveredContainsPanicAndRunsDefers(t *testing.T) {+ cleaned := make(chan struct{})+ GoRecovered(catalogTestContext(), func() {+ defer close(cleaned)+ panic("test: adapter panicked on a helper goroutine")+ })+ select {+ case <-cleaned:+ case <-time.After(5 * time.Second):+ t.Fatal("GoRecovered did not run fn")+ }+}++// RecoverGoroutine is the deferred form: it reports the failure through onPanic+// on the panicking goroutine, before the caller's own deferred hooks run.+func TestRecoverGoroutineReportsPanicToCaller(t *testing.T) {+ ctx := catalogTestContext()+ var marked bool+ done := make(chan struct{})+ go func() {+ defer close(done)+ defer RecoverGoroutine(ctx, "test retrieval", func() { marked = true })+ panic("test: adapter panicked on a caller-owned goroutine")+ }()+ select {+ case <-done:+ case <-time.After(5 * time.Second):+ t.Fatal("RecoverGoroutine did not contain the panic")+ }+ if !marked {+ t.Fatal("RecoverGoroutine did not report the panic to onPanic")+ }+}diff --git a/internal/agent/tools/mcp_exposure.go b/internal/agent/tools/mcp_exposure.goindex 8213171f9..f756f76cc 100644--- a/internal/agent/tools/mcp_exposure.go+++ b/internal/agent/tools/mcp_exposure.go@@ -120,6 +120,10 @@ func (r *ToolRegistry) prepareMCPToolsWithMode(ctx context.Context, grace time.Dc.preloadDone = make(chan struct{})go func() {defer close(c.preloadDone)+ // prepareMCPToolsWithMode selects on preloadDone, so a panic in the+ // fan-out below must not skip the close; the barrier also records+ // every service this pass left unfinished.+ defer RecoverGoroutine(ctx, "MCP catalog preload", func() { c.markPreloadFailed() })loadCtx, cancel := context.WithTimeout(context.WithValue(ctx, execCtxKey{}, (*ToolExecContext)(nil)),mcpCatalogLoadTimeout,@@ -137,9 +141,18 @@ func (r *ToolRegistry) prepareMCPToolsWithMode(ctx context.Context, grace time.Dfor i := 0; i < min(8, len(ids)); i++ {workers.Add(1)go func() {+ // Two barriers: this one keeps a panic in the loop itself+ // (channel bookkeeping, Done) from escaping; the one inside+ // knows which service the MCP client panicked on.defer workers.Done()+ defer RecoverGoroutine(loadCtx, "MCP catalog preload worker", nil)for id := range jobs {- _, _, _ = c.snapshot(loadCtx, id, false)+ func() {+ defer RecoverGoroutine(loadCtx, "MCP catalog preload for service "+id, func() {+ c.markPreloadFailed(id)+ })+ _, _, _ = c.snapshot(loadCtx, id, false)+ }()}}()}@@ -165,6 +178,34 @@ func (r *ToolRegistry) prepareMCPToolsWithMode(ctx context.Context, grace time.Dr.RefreshMCPTools(ctx)}+// markPreloadFailed records a preload pass that a panic ended, reusing the+// catalog's existing store/status mechanism and the same "error" status+// snapshot() stores when a load returns an error. A service whose goroutine is+// gone must not stay "loading", which would read as a preload still in flight.+// With no id every service that never reached "ready" is marked; a ready+// service loaded fine and keeps its snapshot.+func (c *MCPCatalog) markPreloadFailed(ids ...string) {+ if len(ids) == 0 {+ ids = make([]string, 0, len(c.servers))+ for id := range c.servers {+ ids = append(ids, id)+ }+ }+ for _, id := range ids {+ entry := c.servers[id]+ if entry == nil {+ continue+ }+ entry.mu.Lock()+ service, status := entry.service, entry.status+ entry.mu.Unlock()+ if status == "ready" {+ continue+ }+ entry.store(service, nil, "error")+ }+}+// RememberMCPHistory republishes tools this session already described or called// so a new engine does not require another describe round.func (r *ToolRegistry) RememberMCPHistory(messages []chat.Message) {diff --git a/internal/agent/tools/mcp_exposure_panic_test.go b/internal/agent/tools/mcp_exposure_panic_test.gonew file mode 100644index 000000000..d41ec6f17--- /dev/null+++ b/internal/agent/tools/mcp_exposure_panic_test.go@@ -0,0 +1,80 @@+package tools++import (+ "context"+ "testing"+ "time"++ "github.com/Tencent/WeKnora/internal/types"+ "github.com/stretchr/testify/require"+)++// A panic raised by the MCP client on a catalog preload worker used to kill the+// whole process: the preload goroutines in mcp_exposure.go had no recover, and+// the recover sites in registry.go and act.go only cover their own goroutine.+// Before the fix this test never reaches its assertions — the test binary dies+// on prepareMCPToolsWithMode.func1.1.1 (mcp_exposure.go:139 -> snapshot -> load).+func TestMCPPreloadWorkerPanicIsContained(t *testing.T) {+ ctx := catalogTestContext()+ service := &types.MCPService{ID: "panic-mcp", Name: "Panic", Enabled: true}+ catalog := newMCPCatalog(+ ctx,+ []*types.MCPService{service},+ nil,+ func(context.Context, *types.MCPService, bool) ([]*MCPTool, error) {+ // The loader is the seam snapshot() reaches the MCP client through.+ panic("test: MCP client panicked on a preload worker")+ },+ nil,+ )+ registry := NewToolRegistry()+ installMCPCatalog(registry, catalog)++ returned := make(chan struct{})+ go func() {+ defer close(returned)+ registry.PrepareMCPTools(ctx)+ }()+ select {+ case <-returned:+ case <-time.After(10 * time.Second):+ t.Fatal("PrepareMCPTools did not return after a preload worker panicked")+ }++ entry := catalog.servers[service.ID]+ require.NotNil(t, entry)+ entry.mu.Lock()+ status, tools := entry.status, entry.tools+ entry.mu.Unlock()+ require.Equal(t, "error", status, "a panicking service must be marked failed, not left loading")+ require.Empty(t, tools, "a panicking service must not keep a partial snapshot")+}++// The coordinator barrier records the services a panicking preload pass never+// finished and leaves a ready snapshot alone. Both reuse the existing "error"+// status snapshot() stores when a load fails; no new state is introduced.+func TestMCPPreloadFailureMarkingKeepsReadySnapshots(t *testing.T) {+ ctx := catalogTestContext()+ ready := &types.MCPService{ID: "ready-mcp", Name: "Ready", Enabled: true}+ loading := &types.MCPService{ID: "loading-mcp", Name: "Loading", Enabled: true}+ catalog := newMCPCatalog(ctx, []*types.MCPService{ready, loading}, nil, nil, nil)+ tool := NewMCPTool(ready, &types.MCPTool{Name: "ping"}, nil, nil, 0)+ catalog.servers[ready.ID].store(ready, []*MCPTool{tool}, "ready")+ catalog.servers[loading.ID].store(loading, nil, "loading")++ catalog.markPreloadFailed()++ readyEntry := catalog.servers[ready.ID]+ readyEntry.mu.Lock()+ readyStatus, readyTools := readyEntry.status, readyEntry.tools+ readyEntry.mu.Unlock()+ require.Equal(t, "ready", readyStatus, "a service that loaded fine keeps its snapshot")+ require.Len(t, readyTools, 1)++ loadingEntry := catalog.servers[loading.ID]+ loadingEntry.mu.Lock()+ loadingStatus, loadingTools := loadingEntry.status, loadingEntry.tools+ loadingEntry.mu.Unlock()+ require.Equal(t, "error", loadingStatus)+ require.Empty(t, loadingTools)+}diff --git a/internal/agent/tools/registry.go b/internal/agent/tools/registry.goindex 9003e3131..267d24764 100644--- a/internal/agent/tools/registry.go+++ b/internal/agent/tools/registry.go@@ -295,6 +295,10 @@ func (r *ToolRegistry) execute(ctx context.Context, tool types.Tool, args json.R// run on errgroup goroutines when a round executes in parallel, and errgroup// does not carry a panic back to Wait: an unrecovered one from any tool (MCP// clients, sandboxes, third-party SDKs) would take down the whole server.+//+// recover only covers the goroutine that calls it, so this guards the+// tool.Execute call alone: a goroutine a tool starts for itself must install+// its own barrier (GoRecovered / RecoverGoroutine in goroutine.go).func executeRecovered(ctx context.Context, tool types.Tool, args json.RawMessage,) (result *types.ToolResult, err error) {diff --git a/internal/agent/tools/search_knowledge.go b/internal/agent/tools/search_knowledge.goindex a43ceee33..a113c5bad 100644--- a/internal/agent/tools/search_knowledge.go+++ b/internal/agent/tools/search_knowledge.go@@ -3,6 +3,7 @@ package toolsimport ("context""encoding/json"+ "errors""fmt""math""regexp"@@ -32,6 +33,11 @@ const (searchKnowledgeMaxLimit = 30)+// errSearchPanicked reports a retrieval adapter that panicked. The barrier+// logs the stack and the knowledge base is reported as failed, exactly like a+// search that returned an error; the panic value itself is never surfaced.+var errSearchPanicked = errors.New("knowledge base search failed with an internal error")+var searchKnowledgeTool = BaseTool{name: ToolSearchKnowledge,description: "Search the knowledge bases in scope and return the most relevant chunks with their content.\n" +@@ -591,6 +597,16 @@ func (t *SearchKnowledgeTool) concurrentSearchByTargets(wg.Add(1)go func(modelKey string, targets []*types.SearchTarget) {defer wg.Done()+ // The embedding adapter runs here, before any target is searched: a+ // panic from it is contained and every target still pending in the+ // group is reported as failed instead of ending the process.+ defer RecoverGoroutine(ctx, "search knowledge embedding group", func() {+ ids := make([]string, 0, len(targets))+ for _, st := range targets {+ ids = append(ids, st.KnowledgeBaseID)+ }+ fail(ids, errSearchPanicked)+ })needsEmbedding := falsefor _, st := range targets {@@ -655,6 +671,9 @@ func (t *SearchKnowledgeTool) concurrentSearchByTargets(innerWg.Add(1)go func(usedMode string, fullKBIDs []string) {defer innerWg.Done()+ defer RecoverGoroutine(ctx, "search knowledge combined retrieval", func() {+ fail(fullKBIDs, errSearchPanicked)+ })attempt()kbResults, err := t.knowledgeBaseService.HybridSearch(ctx, fullKBIDs[0], types.SearchParams{QueryText: query,@@ -686,6 +705,9 @@ func (t *SearchKnowledgeTool) concurrentSearchByTargets(innerWg.Add(1)go func() {defer innerWg.Done()+ defer RecoverGoroutine(ctx, "search knowledge retrieval", func() {+ fail([]string{st.KnowledgeBaseID}, errSearchPanicked)+ })attempt()usedMode, _ := groupModeFor(st.KnowledgeBaseID)stVectorThreshold, stKeywordThreshold := st.RecallThresholds(vectorThreshold, keywordThreshold)