RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
fix: recover panics in executeMemoryTask so a poison memory task cannot kill the ingestor worker (#18687)
Jack68552b6429cd
diff --git a/internal/ingestion/service/execute_memory_task_panic_test.go b/internal/ingestion/service/execute_memory_task_panic_test.gonew file mode 100644index 0000000000..9a4be0556a--- /dev/null+++ b/internal/ingestion/service/execute_memory_task_panic_test.go@@ -0,0 +1,130 @@+package service++import (+ "context"+ "testing"+ "time"++ "ragflow/internal/common"+ "ragflow/internal/entity"+ taskpkg "ragflow/internal/ingestion/task"+ "ragflow/internal/ingestion/testutil"+ servicepkg "ragflow/internal/service"+)++// TestExecuteMemoryTask_PanicDoesNotPropagate verifies that a panic raised inside+// the memory-extraction path (HandleSaveToMemoryTask) is recovered by+// executeMemoryTask. Before the fix, executeMemoryTask had no recover guard+// (unlike the ingestion path's settleMessage), so a panic crashed the worker+// goroutine. With max_concurrent_workers defaults that can be as low as 1, a+// single panicking memory task permanently removed the only worker, after which+// every normal document parse (PDF etc.) stalled in the queue — exactly the+// "parsing works at first, then breaks after some memory tasks" symptom.+//+// The test injects a panic through the runMemoryTask seam (mirrors runDocumentTask)+// so it does not depend on a live DB or a real MemoryMessageService. The contract+// asserted:+// - executeMemoryTask returns normally (no panic propagates to the caller/worker)+// - the MQ message is Nacked so the broker redelivers it (never silently dropped)+func TestExecuteMemoryTask_PanicDoesNotPropagate(t *testing.T) {+ _ = testutil.SetupTestDB(t) // ensure DB helpers are initialized; not exercised here++ ingestor := NewIngestor("test", 1, []string{"pdf"})+ // Real (non-nil) memory service so executeMemoryTask reaches the+ // runMemoryTask dispatch path instead of early-Acking on nil service.+ ingestor.SetMemoryMessageService(servicepkg.NewMemoryMessageService(servicepkg.NewMemoryService()))+ // Inject a panicking memory runner through the same seam used in production+ // (defaultRunMemoryTask calls memorySvc.HandleSaveToMemoryTask).+ ingestor.runMemoryTask = func(_ context.Context, _ map[string]any) error {+ panic("simulated memory extraction panic")+ }++ handle := &fakeTaskHandle{msg: common.TaskMessage{TaskID: "mem-panic-1", TaskType: common.TaskTypeMemory}}+ taskCtx := taskpkg.NewMemoryTaskContextForScheduling(context.Background(), map[string]any{+ "id": "mem-panic-1", "task_type": "memory", "memory_id": "mem-p", "source_id": 1,+ "message_dict": map[string]any{"user_id": "u", "agent_id": "a", "session_id": "s"},+ }, handle)++ // If the panic is not recovered, this call itself will panic and the test+ // fails. We also guard with a recover here only to convert a propagated+ // panic into a clear test failure (the real recovery belongs in production).+ func() {+ defer func() {+ if r := recover(); r != nil {+ t.Fatalf("executeMemoryTask let a memory panic propagate: %v", r)+ }+ }()+ ingestor.executeMemoryTask(context.Background(), taskCtx)+ }()++ if handle.nacks.Load() != 1 || handle.acks.Load() != 0 {+ t.Fatalf("panicking memory task: expected 0 Ack/1 Nack, got acks=%d nacks=%d", handle.acks.Load(), handle.nacks.Load())+ }+}++// TestWorkerLoop_SurvivesMemoryPanicAndKeepsServing proves the production+// failure mode end-to-end: a panicking memory task must NOT kill the worker+// goroutine, so a subsequent normal document parse is still processed. This is+// the exact "parsing works at first, then breaks after some memory tasks"+// symptom — at max_concurrent_workers=1 the only worker is lost on the first+// panicking memory task, and every later PDF stalls in the queue.+//+// The test drives the real workerLoop (not executeMemoryTask directly): it+// starts one worker, enqueues (1) a panicking memory task, then (2) a normal+// ingestion task with a stubbed runDocumentTask, and asserts the worker both+// Nacked the poison memory task and Acked the document task before exiting.+func TestWorkerLoop_SurvivesMemoryPanicAndKeepsServing(t *testing.T) {+ db := testutil.SetupTestDB(t)+ cleanup := testutil.ReplaceDBForTest(t, db)+ defer cleanup()++ _, _, docID, taskID := testutil.SeedTestData(t, db, testutil.WithPipelineID("flow-1"))++ ingestor := NewIngestor("test", 1, []string{"pdf"})+ ingestor.SetMemoryMessageService(servicepkg.NewMemoryMessageService(servicepkg.NewMemoryService()))+ ingestor.runMemoryTask = func(_ context.Context, _ map[string]any) error {+ panic("simulated memory extraction panic")+ }+ ingestor.runDocumentTask = func(ctx context.Context, _ *entity.IngestionTask) error {+ return nil+ }++ memHandle := &fakeTaskHandle{msg: common.TaskMessage{TaskID: "mem-panic-2", TaskType: common.TaskTypeMemory}}+ memCtx := taskpkg.NewMemoryTaskContextForScheduling(context.Background(), map[string]any{+ "id": "mem-panic-2", "task_type": "memory", "memory_id": "mem-p2", "source_id": 1,+ "message_dict": map[string]any{"user_id": "u", "agent_id": "a", "session_id": "s"},+ }, memHandle)++ docHandle := &fakeTaskHandle{}+ docCtx := newAckTaskCtx(context.Background(), taskID, docID, docHandle)++ ingestor.startWorkerPool()+ // Feed the poison memory task first, then a normal document task.+ ingestor.taskChan <- memCtx+ ingestor.taskChan <- docCtx++ // Wait for the worker to drain both tasks (poison Nacked, document Acked)+ // before cancelling, so the loop does not exit on ctx.Done() first.+ deadline := time.After(5 * time.Second)+ for memHandle.nacks.Load() != 1 || docHandle.acks.Load() != 1 {+ select {+ case <-deadline:+ t.Fatalf("worker did not drain tasks in time: mem(acks=%d nacks=%d) doc(acks=%d nacks=%d)",+ memHandle.acks.Load(), memHandle.nacks.Load(), docHandle.acks.Load(), docHandle.nacks.Load())+ default:+ time.Sleep(5 * time.Millisecond)+ }+ }++ // Cancel the ingestor context so the worker loop exits; then wait for the+ // worker goroutine to finish cleanly (proves it was not crashed by the panic).+ ingestor.cancel()+ ingestor.workerWg.Wait()++ if memHandle.nacks.Load() != 1 || memHandle.acks.Load() != 0 {+ t.Fatalf("poison memory task: expected 0 Ack/1 Nack, got acks=%d nacks=%d", memHandle.acks.Load(), memHandle.nacks.Load())+ }+ if docHandle.acks.Load() != 1 || docHandle.nacks.Load() != 0 {+ t.Fatalf("document task after memory panic: expected 1 Ack/0 Nack (worker survived), got acks=%d nacks=%d", docHandle.acks.Load(), docHandle.nacks.Load())+ }+}diff --git a/internal/ingestion/service/ingestion_service.go b/internal/ingestion/service/ingestion_service.goindex 71983a9f49..cd78abc129 100644--- a/internal/ingestion/service/ingestion_service.go+++ b/internal/ingestion/service/ingestion_service.go@@ -101,6 +101,12 @@ type Ingestor struct {// the full downstream stack.runDocumentTask func(ctx context.Context, ingestionTask *entity.IngestionTask) error+ // runMemoryTask dispatches one async memory-extraction task. Tests may+ // override this to inject a panicking/failing runner without a live DB or+ // real MemoryMessageService. Defaults to defaultRunMemoryTask, which calls+ // memorySvc.HandleSaveToMemoryTask.+ runMemoryTask func(ctx context.Context, payload map[string]any) error+// cancelCheck is polled periodically (every 3s) during task execution.// When it returns true the task's context is cancelled, which causes the// pipeline to stop at the next ctx.Err() check. Defaults to a Redis@@ -135,6 +141,7 @@ func NewIngestor(name string, maxConcurrency int32, supportedTypes []string) *InheartbeatInterval: defaultHeartbeatInterval,}ingestor.runDocumentTask = ingestor.defaultRunDocumentTask+ ingestor.runMemoryTask = ingestor.defaultRunMemoryTaskingestor.cancelCheck = ingestor.defaultCancelCheckingestor.checkpointExists = canvas.RedisCheckpointExistsingestor.kcConcurrency = maxConcurrency // parallel dataset-level compile workers default to the task width@@ -478,8 +485,8 @@ func (e *Ingestor) workerLoop(id int32) {// executeMemoryTask runs one async memory-extraction task (TaskKindMemory) on// a worker of the shared pool. Unlike ingestion tasks, memory tasks have no-// ingestion_task row / state machine: HandleSaveToMemoryTask persists the-// extracted messages and settles task progress on the way out.+// ingestion_task row / state machine: the runner persists the extracted+// messages and settles task progress on the way out.//// Settlement is error-category aware:// - Terminal failure (task row absent, already-failed, or progress=-1 already@@ -488,11 +495,32 @@ func (e *Ingestor) workerLoop(id int32) {// - Transient failure (a task-load DB error before any durable marker, or an// LLM/network failure that did not reach progress=-1) is Nacked so the// message is redelivered and retried instead of being silently dropped.+//+// A panic in the memory path is recovered so the worker goroutine survives:+// the ingestion path already does this via settleMessage's recover, but a+// panic here previously crashed the worker. With max_concurrent_workers as low+// as 1, one panicking memory task could permanently remove the only worker and+// stall every subsequent document parse. The recovered panic is treated as a+// transient failure and Nacked for redelivery.func (e *Ingestor) executeMemoryTask(ctx context.Context, taskCtx *taskpkg.TaskContext) {taskID, _ := taskCtx.MemoryPayload["id"].(string)if taskID == "" {taskID, _ = taskCtx.MemoryPayload["task_id"].(string)}++ // Recover a panic so a single poison memory task never crashes the worker+ // (and, at max_concurrent_workers=1, the whole ingestor's only slot).+ defer func() {+ if r := recover(); r != nil {+ common.Error(fmt.Sprintf("memory task %s panicked: %v", taskID, r), fmt.Errorf("%v", r))+ if taskCtx != nil && taskCtx.Handle != nil {+ if nackErr := taskCtx.Handle.Nack(); nackErr != nil {+ common.Error(fmt.Sprintf("nack memory task %s after panic", taskID), nackErr)+ }+ }+ }+ }()+common.Info(fmt.Sprintf("Starting memory task %s", taskID))if taskCtx.Handle == nil {common.Warn("memory task handle is nil, skip")@@ -505,8 +533,8 @@ func (e *Ingestor) executeMemoryTask(ctx context.Context, taskCtx *taskpkg.TaskC}return}- if err := e.memorySvc.HandleSaveToMemoryTask(ctx, taskCtx.MemoryPayload); err != nil {- // HandleSaveToMemoryTask wraps terminal outcomes in ErrMemoryTaskTerminal+ if err := e.runMemoryTask(ctx, taskCtx.MemoryPayload); err != nil {+ // defaultRunMemoryTask wraps terminal outcomes in ErrMemoryTaskTerminal// (durable progress=-1 written, or no row to retry). Everything else is// transient and must be redelivered rather than dropped.if errors.Is(err, servicepkg.ErrMemoryTaskTerminal) {@@ -528,6 +556,13 @@ func (e *Ingestor) executeMemoryTask(ctx context.Context, taskCtx *taskpkg.TaskC}}+// defaultRunMemoryTask is the production memory-task runner. It is held behind+// the runMemoryTask field so tests can substitute a panicking/failing runner+// without a live DB or real MemoryMessageService.+func (e *Ingestor) defaultRunMemoryTask(ctx context.Context, payload map[string]any) error {+ return e.memorySvc.HandleSaveToMemoryTask(ctx, payload)+}+func (e *Ingestor) executeTask(ctx context.Context, taskCtx *taskpkg.TaskContext) {task := taskCtx.IngestionTaskcommon.Info(fmt.Sprintf("Starting task %s", task.ID))