Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

fix: recover panics in executeMemoryTask so a poison memory task cannot kill the ingestor worker (#18687)

Jack68552b6429cd
diff --git a/internal/ingestion/service/execute_memory_task_panic_test.go b/internal/ingestion/service/execute_memory_task_panic_test.go
new file mode 100644
index 0000000000..9a4be0556a
--- /dev/null
+++ b/internal/ingestion/service/execute_memory_task_panic_test.go
@@ -0,0 +1,130 @@
+package service
+
+import (
+ "context"
+ "testing"
+ "time"
+
+ "ragflow/internal/common"
+ "ragflow/internal/entity"
+ taskpkg "ragflow/internal/ingestion/task"
+ "ragflow/internal/ingestion/testutil"
+ servicepkg "ragflow/internal/service"
+)
+
+// TestExecuteMemoryTask_PanicDoesNotPropagate verifies that a panic raised inside
+// the memory-extraction path (HandleSaveToMemoryTask) is recovered by
+// executeMemoryTask. Before the fix, executeMemoryTask had no recover guard
+// (unlike the ingestion path's settleMessage), so a panic crashed the worker
+// goroutine. With max_concurrent_workers defaults that can be as low as 1, a
+// single panicking memory task permanently removed the only worker, after which
+// every normal document parse (PDF etc.) stalled in the queue — exactly the
+// "parsing works at first, then breaks after some memory tasks" symptom.
+//
+// The test injects a panic through the runMemoryTask seam (mirrors runDocumentTask)
+// so it does not depend on a live DB or a real MemoryMessageService. The contract
+// asserted:
+// - executeMemoryTask returns normally (no panic propagates to the caller/worker)
+// - the MQ message is Nacked so the broker redelivers it (never silently dropped)
+func TestExecuteMemoryTask_PanicDoesNotPropagate(t *testing.T) {
+ _ = testutil.SetupTestDB(t) // ensure DB helpers are initialized; not exercised here
+
+ ingestor := NewIngestor("test", 1, []string{"pdf"})
+ // Real (non-nil) memory service so executeMemoryTask reaches the
+ // runMemoryTask dispatch path instead of early-Acking on nil service.
+ ingestor.SetMemoryMessageService(servicepkg.NewMemoryMessageService(servicepkg.NewMemoryService()))
+ // Inject a panicking memory runner through the same seam used in production
+ // (defaultRunMemoryTask calls memorySvc.HandleSaveToMemoryTask).
+ ingestor.runMemoryTask = func(_ context.Context, _ map[string]any) error {
+ panic("simulated memory extraction panic")
+ }
+
+ handle := &fakeTaskHandle{msg: common.TaskMessage{TaskID: "mem-panic-1", TaskType: common.TaskTypeMemory}}
+ taskCtx := taskpkg.NewMemoryTaskContextForScheduling(context.Background(), map[string]any{
+ "id": "mem-panic-1", "task_type": "memory", "memory_id": "mem-p", "source_id": 1,
+ "message_dict": map[string]any{"user_id": "u", "agent_id": "a", "session_id": "s"},
+ }, handle)
+
+ // If the panic is not recovered, this call itself will panic and the test
+ // fails. We also guard with a recover here only to convert a propagated
+ // panic into a clear test failure (the real recovery belongs in production).
+ func() {
+ defer func() {
+ if r := recover(); r != nil {
+ t.Fatalf("executeMemoryTask let a memory panic propagate: %v", r)
+ }
+ }()
+ ingestor.executeMemoryTask(context.Background(), taskCtx)
+ }()
+
+ if handle.nacks.Load() != 1 || handle.acks.Load() != 0 {
+ t.Fatalf("panicking memory task: expected 0 Ack/1 Nack, got acks=%d nacks=%d", handle.acks.Load(), handle.nacks.Load())
+ }
+}
+
+// TestWorkerLoop_SurvivesMemoryPanicAndKeepsServing proves the production
+// failure mode end-to-end: a panicking memory task must NOT kill the worker
+// goroutine, so a subsequent normal document parse is still processed. This is
+// the exact "parsing works at first, then breaks after some memory tasks"
+// symptom — at max_concurrent_workers=1 the only worker is lost on the first
+// panicking memory task, and every later PDF stalls in the queue.
+//
+// The test drives the real workerLoop (not executeMemoryTask directly): it
+// starts one worker, enqueues (1) a panicking memory task, then (2) a normal
+// ingestion task with a stubbed runDocumentTask, and asserts the worker both
+// Nacked the poison memory task and Acked the document task before exiting.
+func TestWorkerLoop_SurvivesMemoryPanicAndKeepsServing(t *testing.T) {
+ db := testutil.SetupTestDB(t)
+ cleanup := testutil.ReplaceDBForTest(t, db)
+ defer cleanup()
+
+ _, _, docID, taskID := testutil.SeedTestData(t, db, testutil.WithPipelineID("flow-1"))
+
+ ingestor := NewIngestor("test", 1, []string{"pdf"})
+ ingestor.SetMemoryMessageService(servicepkg.NewMemoryMessageService(servicepkg.NewMemoryService()))
+ ingestor.runMemoryTask = func(_ context.Context, _ map[string]any) error {
+ panic("simulated memory extraction panic")
+ }
+ ingestor.runDocumentTask = func(ctx context.Context, _ *entity.IngestionTask) error {
+ return nil
+ }
+
+ memHandle := &fakeTaskHandle{msg: common.TaskMessage{TaskID: "mem-panic-2", TaskType: common.TaskTypeMemory}}
+ memCtx := taskpkg.NewMemoryTaskContextForScheduling(context.Background(), map[string]any{
+ "id": "mem-panic-2", "task_type": "memory", "memory_id": "mem-p2", "source_id": 1,
+ "message_dict": map[string]any{"user_id": "u", "agent_id": "a", "session_id": "s"},
+ }, memHandle)
+
+ docHandle := &fakeTaskHandle{}
+ docCtx := newAckTaskCtx(context.Background(), taskID, docID, docHandle)
+
+ ingestor.startWorkerPool()
+ // Feed the poison memory task first, then a normal document task.
+ ingestor.taskChan <- memCtx
+ ingestor.taskChan <- docCtx
+
+ // Wait for the worker to drain both tasks (poison Nacked, document Acked)
+ // before cancelling, so the loop does not exit on ctx.Done() first.
+ deadline := time.After(5 * time.Second)
+ for memHandle.nacks.Load() != 1 || docHandle.acks.Load() != 1 {
+ select {
+ case <-deadline:
+ t.Fatalf("worker did not drain tasks in time: mem(acks=%d nacks=%d) doc(acks=%d nacks=%d)",
+ memHandle.acks.Load(), memHandle.nacks.Load(), docHandle.acks.Load(), docHandle.nacks.Load())
+ default:
+ time.Sleep(5 * time.Millisecond)
+ }
+ }
+
+ // Cancel the ingestor context so the worker loop exits; then wait for the
+ // worker goroutine to finish cleanly (proves it was not crashed by the panic).
+ ingestor.cancel()
+ ingestor.workerWg.Wait()
+
+ if memHandle.nacks.Load() != 1 || memHandle.acks.Load() != 0 {
+ t.Fatalf("poison memory task: expected 0 Ack/1 Nack, got acks=%d nacks=%d", memHandle.acks.Load(), memHandle.nacks.Load())
+ }
+ if docHandle.acks.Load() != 1 || docHandle.nacks.Load() != 0 {
+ t.Fatalf("document task after memory panic: expected 1 Ack/0 Nack (worker survived), got acks=%d nacks=%d", docHandle.acks.Load(), docHandle.nacks.Load())
+ }
+}
diff --git a/internal/ingestion/service/ingestion_service.go b/internal/ingestion/service/ingestion_service.go
index 71983a9f49..cd78abc129 100644
--- a/internal/ingestion/service/ingestion_service.go
+++ b/internal/ingestion/service/ingestion_service.go
@@ -101,6 +101,12 @@ type Ingestor struct {
// the full downstream stack.
runDocumentTask func(ctx context.Context, ingestionTask *entity.IngestionTask) error
+ // runMemoryTask dispatches one async memory-extraction task. Tests may
+ // override this to inject a panicking/failing runner without a live DB or
+ // real MemoryMessageService. Defaults to defaultRunMemoryTask, which calls
+ // memorySvc.HandleSaveToMemoryTask.
+ runMemoryTask func(ctx context.Context, payload map[string]any) error
+
// cancelCheck is polled periodically (every 3s) during task execution.
// When it returns true the task's context is cancelled, which causes the
// pipeline to stop at the next ctx.Err() check. Defaults to a Redis
@@ -135,6 +141,7 @@ func NewIngestor(name string, maxConcurrency int32, supportedTypes []string) *In
heartbeatInterval: defaultHeartbeatInterval,
}
ingestor.runDocumentTask = ingestor.defaultRunDocumentTask
+ ingestor.runMemoryTask = ingestor.defaultRunMemoryTask
ingestor.cancelCheck = ingestor.defaultCancelCheck
ingestor.checkpointExists = canvas.RedisCheckpointExists
ingestor.kcConcurrency = maxConcurrency // parallel dataset-level compile workers default to the task width
@@ -478,8 +485,8 @@ func (e *Ingestor) workerLoop(id int32) {
// executeMemoryTask runs one async memory-extraction task (TaskKindMemory) on
// a worker of the shared pool. Unlike ingestion tasks, memory tasks have no
-// ingestion_task row / state machine: HandleSaveToMemoryTask persists the
-// extracted messages and settles task progress on the way out.
+// ingestion_task row / state machine: the runner persists the extracted
+// messages and settles task progress on the way out.
//
// Settlement is error-category aware:
// - Terminal failure (task row absent, already-failed, or progress=-1 already
@@ -488,11 +495,32 @@ func (e *Ingestor) workerLoop(id int32) {
// - Transient failure (a task-load DB error before any durable marker, or an
// LLM/network failure that did not reach progress=-1) is Nacked so the
// message is redelivered and retried instead of being silently dropped.
+//
+// A panic in the memory path is recovered so the worker goroutine survives:
+// the ingestion path already does this via settleMessage's recover, but a
+// panic here previously crashed the worker. With max_concurrent_workers as low
+// as 1, one panicking memory task could permanently remove the only worker and
+// stall every subsequent document parse. The recovered panic is treated as a
+// transient failure and Nacked for redelivery.
func (e *Ingestor) executeMemoryTask(ctx context.Context, taskCtx *taskpkg.TaskContext) {
taskID, _ := taskCtx.MemoryPayload["id"].(string)
if taskID == "" {
taskID, _ = taskCtx.MemoryPayload["task_id"].(string)
}
+
+ // Recover a panic so a single poison memory task never crashes the worker
+ // (and, at max_concurrent_workers=1, the whole ingestor's only slot).
+ defer func() {
+ if r := recover(); r != nil {
+ common.Error(fmt.Sprintf("memory task %s panicked: %v", taskID, r), fmt.Errorf("%v", r))
+ if taskCtx != nil && taskCtx.Handle != nil {
+ if nackErr := taskCtx.Handle.Nack(); nackErr != nil {
+ common.Error(fmt.Sprintf("nack memory task %s after panic", taskID), nackErr)
+ }
+ }
+ }
+ }()
+
common.Info(fmt.Sprintf("Starting memory task %s", taskID))
if taskCtx.Handle == nil {
common.Warn("memory task handle is nil, skip")
@@ -505,8 +533,8 @@ func (e *Ingestor) executeMemoryTask(ctx context.Context, taskCtx *taskpkg.TaskC
}
return
}
- if err := e.memorySvc.HandleSaveToMemoryTask(ctx, taskCtx.MemoryPayload); err != nil {
- // HandleSaveToMemoryTask wraps terminal outcomes in ErrMemoryTaskTerminal
+ if err := e.runMemoryTask(ctx, taskCtx.MemoryPayload); err != nil {
+ // defaultRunMemoryTask wraps terminal outcomes in ErrMemoryTaskTerminal
// (durable progress=-1 written, or no row to retry). Everything else is
// transient and must be redelivered rather than dropped.
if errors.Is(err, servicepkg.ErrMemoryTaskTerminal) {
@@ -528,6 +556,13 @@ func (e *Ingestor) executeMemoryTask(ctx context.Context, taskCtx *taskpkg.TaskC
}
}
+// defaultRunMemoryTask is the production memory-task runner. It is held behind
+// the runMemoryTask field so tests can substitute a panicking/failing runner
+// without a live DB or real MemoryMessageService.
+func (e *Ingestor) defaultRunMemoryTask(ctx context.Context, payload map[string]any) error {
+ return e.memorySvc.HandleSaveToMemoryTask(ctx, payload)
+}
+
func (e *Ingestor) executeTask(ctx context.Context, taskCtx *taskpkg.TaskContext) {
task := taskCtx.IngestionTask
common.Info(fmt.Sprintf("Starting task %s", task.ID))