RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
refactor(ingestion): use worker-driven task pull dispatcher (#19314)
jay777211ed2fc26f019
diff --git a/cmd/ragflow_server.go b/cmd/ragflow_server.goindex b3f2071090..641ca7cbdf 100644--- a/cmd/ragflow_server.go+++ b/cmd/ragflow_server.go@@ -593,7 +593,7 @@ func runIngestor(ctx context.Context, cancel context.CancelFunc, args *serverArg// Search and UpsertDoc can embed queries/summaries automatically.nav.SetNavService(nlp.NewNavService(service.NewNavEmbedder(service.NewModelProviderService(), "")))// Memory extraction runs on the Ingestor's shared NATS consumer + worker- // pool (task_type="memory" dispatched by processMessage -> executeMemoryTask),+ // pool (task_type="memory" dispatched by handleAndExecute -> executeMemoryTask),// so there is no longer a dedicated Redis memory consumer to start.ingestor.SetMemoryMessageService(service.NewMemoryMessageService(service.NewMemoryService()))diff --git a/internal/admin/handler.go b/internal/admin/handler.goindex abf47d18cc..3393c779e9 100644--- a/internal/admin/handler.go+++ b/internal/admin/handler.go@@ -17,6 +17,7 @@package adminimport (+ "context""encoding/json""errors""fmt"@@ -923,7 +924,7 @@ func (h *Handler) PublishMessageToQueue(c *gin.Context) {}msgQueueEngine := engine.GetMessageQueueEngine()- err = msgQueueEngine.PublishTask("tasks.RAGFLOW", taskMessageStr)+ err = msgQueueEngine.PublishTask(common.TaskSubject, taskMessageStr)if err != nil {common.ErrorWithCode(c, common.CodeBadRequest, err.Error())return@@ -933,7 +934,7 @@ func (h *Handler) PublishMessageToQueue(c *gin.Context) {}type PullMessageFromQueueRequest struct {- MessageCount int `json:"message_count" binding:"required"`+ MessageCount int `json:"message_count" binding:"required,gt=0"`AckPolicy string `json:"ack_policy" binding:"required"`}@@ -943,14 +944,20 @@ func (h *Handler) PullMessageFromQueue(c *gin.Context) {common.ErrorWithCode(c, common.CodeBadRequest, fmt.Sprintf("Message count error: %s", err.Error()))return}+ if req.MessageCount > common.MaxManualPullMessages {+ common.ErrorWithCode(c, common.CodeBadRequest,+ fmt.Sprintf("message count must be between 1 and %d", common.MaxManualPullMessages))+ return+ }msgQueueEngine := engine.GetMessageQueueEngine()- err := msgQueueEngine.InitConsumer("tasks.RAGFLOW")- if err != nil {+ if err := msgQueueEngine.InitConsumer(common.TaskSubject); err != nil {common.ErrorWithCode(c, common.CodeBadRequest, err.Error())return}- messages, err := msgQueueEngine.GetMessages(req.MessageCount)+ pullCtx, cancel := context.WithTimeout(c.Request.Context(), time.Second)+ defer cancel()+ messages, err := msgQueueEngine.PullMessages(pullCtx, req.MessageCount)if err != nil {common.ErrorWithCode(c, common.CodeBadRequest, err.Error())return@@ -978,6 +985,7 @@ func (h *Handler) PullMessageFromQueue(c *gin.Context) {"id": taskMessage.TaskID,"type": taskMessage.TaskType,}+ err = message.Nack()if err == nil {resultMessage["nack"] = "true"} else {diff --git a/internal/admin/handler_queue_test.go b/internal/admin/handler_queue_test.gonew file mode 100644index 0000000000..80a3a15266--- /dev/null+++ b/internal/admin/handler_queue_test.go@@ -0,0 +1,118 @@+//+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.+//++package admin++import (+ "bytes"+ "encoding/json"+ "fmt"+ "net/http"+ "net/http/httptest"+ "testing"++ "ragflow/internal/common"+ "ragflow/internal/engine"+ "ragflow/internal/ingestion/testutil"++ "github.com/gin-gonic/gin"+)++// TestPullMessageFromQueueInitializesSharedConsumer preserves the admin+// endpoint's ability to pull messages before the ingestor starts. The endpoint+// must initialize the existing durable consumer rather than creating a+// dispatcher-specific consumer.+func TestPullMessageFromQueueInitializesSharedConsumer(t *testing.T) {+ gin.SetMode(gin.TestMode)+ queue := testutil.SetupNatsEngine(t)+ previousQueue := engine.GetMessageQueueEngine()+ engine.SetMessageQueueEngine(queue)+ t.Cleanup(func() { engine.SetMessageQueueEngine(previousQueue) })++ payload, err := json.Marshal(common.TaskMessage{+ TaskID: "admin-pull-before-ingestor",+ TaskType: common.TaskTypeIngestionTask,+ })+ if err != nil {+ t.Fatalf("marshal task: %v", err)+ }+ if err := queue.PublishTask(common.TaskSubject, payload); err != nil {+ t.Fatalf("publish task: %v", err)+ }++ recorder := httptest.NewRecorder()+ ctx, _ := gin.CreateTestContext(recorder)+ ctx.Request = httptest.NewRequest(+ http.MethodPost,+ "/",+ bytes.NewBufferString(`{"message_count":1,"ack_policy":"ACK"}`),+ )+ ctx.Request.Header.Set("Content-Type", "application/json")++ (&Handler{}).PullMessageFromQueue(ctx)++ var response struct {+ Code int `json:"code"`+ Data []struct {+ ID string `json:"id"`+ Ack string `json:"ack"`+ } `json:"data"`+ }+ if err := json.Unmarshal(recorder.Body.Bytes(), &response); err != nil {+ t.Fatalf("decode response: %v", err)+ }+ if response.Code != int(common.CodeSuccess) {+ t.Fatalf("response code = %d, want %d; body = %s", response.Code, common.CodeSuccess, recorder.Body.String())+ }+ if len(response.Data) != 1 || response.Data[0].ID != "admin-pull-before-ingestor" || response.Data[0].Ack != "true" {+ t.Fatalf("pull response = %+v, want acked admin-pull-before-ingestor", response.Data)+ }+}++func TestPullMessageFromQueueRejectsOutOfRangeMessageCount(t *testing.T) {+ gin.SetMode(gin.TestMode)+ for _, testCase := range []struct {+ name string+ messageCount int+ }{+ {name: "negative", messageCount: -1},+ {name: "zero", messageCount: 0},+ {name: "above limit", messageCount: 101},+ } {+ t.Run(testCase.name, func(t *testing.T) {+ recorder := httptest.NewRecorder()+ ctx, _ := gin.CreateTestContext(recorder)+ ctx.Request = httptest.NewRequest(+ http.MethodPost,+ "/",+ bytes.NewBufferString(fmt.Sprintf(`{"message_count":%d,"ack_policy":"ACK"}`, testCase.messageCount)),+ )+ ctx.Request.Header.Set("Content-Type", "application/json")++ (&Handler{}).PullMessageFromQueue(ctx)++ var response struct {+ Code int `json:"code"`+ }+ if err := json.Unmarshal(recorder.Body.Bytes(), &response); err != nil {+ t.Fatalf("decode response: %v", err)+ }+ if response.Code != int(common.CodeBadRequest) {+ t.Fatalf("response code = %d, want %d; body = %s", response.Code, common.CodeBadRequest, recorder.Body.String())+ }+ })+ }+}diff --git a/internal/cli/admin_parser.go b/internal/cli/admin_parser.goindex f3a6d18054..ca93a8c409 100644--- a/internal/cli/admin_parser.go+++ b/internal/cli/admin_parser.go@@ -2714,8 +2714,8 @@ func (p *Parser) parseMessageQueueCommand() (*Command, error) {p.nextToken() // consume NUMBER}- if messageCount <= 0 || messageCount > 100 {- return nil, fmt.Errorf("message count cannot be less than 0 or greater than 100")+ if messageCount <= 0 || messageCount > common.MaxManualPullMessages {+ return nil, fmt.Errorf("message count must be between 1 and %d", common.MaxManualPullMessages)}cmd = NewCommand("user_pull_message_command")diff --git a/internal/common/task.go b/internal/common/task.goindex d6b1237e9e..c8c2447cde 100644--- a/internal/common/task.go+++ b/internal/common/task.go@@ -24,6 +24,9 @@ const (// symbol so the routing contract cannot diverge (mirrors the RAGFLOW_TASKS// JetStream subject in internal/engine/nats).TaskSubject = "tasks.RAGFLOW"+ // MaxManualPullMessages is the largest task batch the administrative queue+ // pull endpoint accepts.+ MaxManualPullMessages = 100TaskTypeIngestionTask = "ingestion_task"TaskTypeIngestionTest = "ingestion_test"@@ -31,7 +34,7 @@ const (TaskTypeSyncer = "syncer"// TaskTypeMemory is the async memory-extraction task type. Memory tasks// share the tasks.RAGFLOW subject and the Ingestor's consumer + worker- // pool with ingestion tasks; processMessage dispatches them by TaskType.+ // pool with ingestion tasks; handleAndExecute dispatches them by TaskType.// The memory-specific payload (message_dict/memory_id/source_id) is// carried in TaskMessage.Payload.TaskTypeMemory = "memory"diff --git a/internal/engine/engine.go b/internal/engine/engine.goindex a23fa1dfb5..9b1af9b44c 100644--- a/internal/engine/engine.go+++ b/internal/engine/engine.go@@ -113,7 +113,8 @@ type MessageQueue interface {Type() stringInitConsumer(subject string) errorPublishTask(subject string, payload []byte) error- GetMessages(messageCount int) ([]common.TaskHandle, error)+ PullMessages(ctx context.Context, messageCount int) ([]common.TaskHandle, error)+ PullMessage(ctx context.Context) (common.TaskHandle, error)ListMessages(messageType string, pending bool) ([]map[string]string, error)ShowMessageQueue() (map[string]string, error)CheckStatus() stringdiff --git a/internal/engine/global.go b/internal/engine/global.goindex 1058d7b085..d701d4457c 100644--- a/internal/engine/global.go+++ b/internal/engine/global.go@@ -111,7 +111,10 @@ func InitMessageQueue() error {switch messageQueueType {case "nats":natsConfig := globalConfig.GetNATSConfig()- messageQueueEngine = nats.NewNatsEngine(natsConfig.Host, natsConfig.Port)+ messageQueueEngine = nats.NewNatsEngine(+ natsConfig.Host,+ natsConfig.Port,+ )err := messageQueueEngine.Init()if err != nil {return errdiff --git a/internal/engine/nats/nats.go b/internal/engine/nats/nats.goindex 7fceec5727..2657e4cc48 100644--- a/internal/engine/nats/nats.go+++ b/internal/engine/nats/nats.go@@ -121,7 +121,7 @@ func (n *NatsEngine) PublishTask(subject string, payload []byte) error {// Duplicate delivery is instead made safe at the consumer level:// StartRunning's CREATED/SCHEDULED→RUNNING CAS plus the in-process claim guard// prevent a second copy from executing while the first owner is active (see- // Ingestor.processMessage).+ // Ingestor.handleAndExecute).ack, err := n.jetStream.Publish(ctx, subject, payload)if err != nil {return err@@ -256,22 +256,54 @@ func (n *NatsEngine) InitConsumer(subject string) error {}return nil}-func (n *NatsEngine) GetMessages(messageCount int) ([]common.TaskHandle, error) {++// PullMessages fetches up to messageCount messages before ctx expires.+func (n *NatsEngine) PullMessages(ctx context.Context, messageCount int) ([]common.TaskHandle, error) {+ if messageCount < 1 || messageCount > common.MaxManualPullMessages {+ return nil, fmt.Errorf("message count must be between 1 and %d", common.MaxManualPullMessages)+ }if n.consumer == nil {return nil, errors.New("NATS consumer is nil, engine not properly initialized")}+ if _, ok := ctx.Deadline(); !ok {+ return nil, errors.New("pull messages context must have a deadline")+ }- resultMessages := make([]common.TaskHandle, 0)- messages, err := n.consumer.Fetch(messageCount, jetstream.FetchMaxWait(1*time.Second))+ resultMessages := make([]common.TaskHandle, 0, messageCount)+ messages, err := n.consumer.Fetch(messageCount, jetstream.FetchContext(ctx))if err != nil {return nil, fmt.Errorf("failed to fetch messages: %w", err)}- for msg := range messages.Messages() {- resultMessages = append(resultMessages, NewNatsMessageHandle(msg))+ for message := range messages.Messages() {+ resultMessages = append(resultMessages, NewNatsMessageHandle(message))+ }+ if batchErr := messages.Error(); batchErr != nil {+ if errors.Is(batchErr, context.DeadlineExceeded) {+ return resultMessages, nil+ }+ for _, message := range resultMessages {+ if nackErr := message.Nack(); nackErr != nil {+ common.Error("nack message after failed pull", nackErr)+ }+ }+ return nil, fmt.Errorf("failed to fetch messages: %w", batchErr)}return resultMessages, nil}+// PullMessage returns one task handle from PullMessages. A nil handle+// with a nil error means the pull expired without an available task.+func (n *NatsEngine) PullMessage(ctx context.Context) (common.TaskHandle, error) {+ messages, err := n.PullMessages(ctx, 1)+ if err != nil {+ return nil, err+ }+ if len(messages) == 0 {+ return nil, nil+ }+ return messages[0], nil+}+func (n *NatsEngine) CheckStatus() string {if n.nc == nil {return "NATS connection is nil, engine not properly initialized"diff --git a/internal/engine/nats/nats_uninitialized_test.go b/internal/engine/nats/nats_uninitialized_test.goindex 2aade2f5a3..5533fc64b2 100644--- a/internal/engine/nats/nats_uninitialized_test.go+++ b/internal/engine/nats/nats_uninitialized_test.go@@ -19,8 +19,8 @@ func TestUninitializedEngineMethodsReturnErrors(t *testing.T) {t.Fatalf("PublishTask on uninitialized engine: err = %v, want 'not properly initialized'", err)}- if _, err := e.GetMessages(1); err == nil || !strings.Contains(err.Error(), "not properly initialized") {- t.Fatalf("GetMessages on uninitialized engine: err = %v, want 'not properly initialized'", err)+ if _, err := e.PullMessages(t.Context(), 1); err == nil || !strings.Contains(err.Error(), "not properly initialized") {+ t.Fatalf("PullMessages on uninitialized engine: err = %v, want 'not properly initialized'", err)}if _, err := e.ShowMessageQueue(); err == nil || !strings.Contains(err.Error(), "not properly initialized") {diff --git a/internal/engine/nats/pull_messages_test.go b/internal/engine/nats/pull_messages_test.gonew file mode 100644index 0000000000..56f41b74ad--- /dev/null+++ b/internal/engine/nats/pull_messages_test.go@@ -0,0 +1,289 @@+//+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.+//++package nats++import (+ "context"+ "encoding/json"+ "errors"+ "strings"+ "testing"+ "time"++ "ragflow/internal/common"++ "github.com/nats-io/nats.go/jetstream"+)++func newTaskQueue(t *testing.T) *NatsEngine {+ t.Helper()+ host, port := newEmbeddedNatsServer(t)+ queue := NewNatsEngine(host, port)+ if err := queue.Init(); err != nil {+ t.Fatalf("Init: %v", err)+ }+ if err := queue.InitConsumer(common.TaskSubject); err != nil {+ t.Fatalf("InitConsumer: %v", err)+ }+ return queue+}++func taskPullContext(t *testing.T) (context.Context, context.CancelFunc) {+ t.Helper()+ return context.WithTimeout(t.Context(), time.Second)+}++func publishTask(t *testing.T, queue *NatsEngine, taskID string) {+ t.Helper()+ payload, err := json.Marshal(common.TaskMessage{+ TaskID: taskID,+ TaskType: common.TaskTypeIngestionTask,+ })+ if err != nil {+ t.Fatalf("marshal task: %v", err)+ }+ if err := queue.PublishTask(common.TaskSubject, payload); err != nil {+ t.Fatalf("publish task: %v", err)+ }+}++func TestPullMessageFetchesOneMessage(t *testing.T) {+ queue := newTaskQueue(t)+ publishTask(t, queue, "stream-one")+ publishTask(t, queue, "stream-two")++ ctx, cancel := taskPullContext(t)+ defer cancel()+ handle, err := queue.PullMessage(ctx)+ if err != nil {+ t.Fatalf("PullMessage: %v", err)+ }+ if handle == nil || handle.GetMessage().TaskID != "stream-one" {+ t.Fatalf("stream handle = %v, want stream-one", handle)+ }++ handles, err := queue.PullMessages(ctx, 1)+ if err != nil {+ t.Fatalf("PullMessages: %v", err)+ }+ if len(handles) != 1 || handles[0].GetMessage().TaskID != "stream-two" {+ t.Fatalf("remaining messages = %+v, want stream-two", handles)+ }+}++func TestPullMessageRequiresDeadline(t *testing.T) {+ queue := newTaskQueue(t)+ _, err := queue.PullMessage(t.Context())+ if err == nil {+ t.Fatal("PullMessage without a deadline succeeded")+ }+ if !strings.Contains(err.Error(), "deadline") {+ t.Fatalf("PullMessage error = %v, want deadline error", err)+ }+}++func TestPullMessageReturnsNilForEmptyQueue(t *testing.T) {+ queue := newTaskQueue(t)+ ctx, cancel := context.WithTimeout(t.Context(), 100*time.Millisecond)+ defer cancel()++ handle, err := queue.PullMessage(ctx)+ if err != nil {+ t.Fatalf("PullMessage: %v", err)+ }+ if handle != nil {+ t.Fatalf("empty pull handle = %v, want nil", handle)+ }+}++func TestPullMessageReturnsCancellation(t *testing.T) {+ queue := newTaskQueue(t)+ ctx, cancel := taskPullContext(t)+ cancel()++ _, err := queue.PullMessage(ctx)+ if !errors.Is(err, context.Canceled) {+ t.Fatalf("PullMessage error = %v, want cancellation", err)+ }+}++func TestPullMessageReportsMaxWaiting(t *testing.T) {+ queue := newTaskQueue(t)+ ctx, cancel := taskPullContext(t)+ defer cancel()+ if err := queue.stream.DeleteConsumer(ctx, "RAGFLOW_CONSUMER"); err != nil {+ t.Fatalf("delete default consumer: %v", err)+ }+ consumer, err := queue.stream.CreateOrUpdateConsumer(ctx, jetstream.ConsumerConfig{+ Name: "RAGFLOW_CONSUMER",+ AckPolicy: jetstream.AckExplicitPolicy,+ FilterSubject: "tasks.>",+ MaxWaiting: 1,+ })+ if err != nil {+ t.Fatalf("create limited consumer: %v", err)+ }+ queue.consumer = consumer++ firstCtx, cancelFirst := taskPullContext(t)+ defer cancelFirst()+ firstResult := make(chan error, 1)+ go func() {+ _, err := queue.PullMessage(firstCtx)+ firstResult <- err+ }()++ deadline := time.Now().Add(time.Second)+ for time.Now().Before(deadline) {+ info, err := queue.consumer.Info(t.Context())+ if err != nil {+ t.Fatalf("consumer info: %v", err)+ }+ if info.NumWaiting == 1 {+ break+ }+ time.Sleep(10 * time.Millisecond)+ }+ info, err := queue.consumer.Info(t.Context())+ if err != nil {+ t.Fatalf("consumer info: %v", err)+ }+ if info.NumWaiting != 1 {+ t.Fatalf("waiting pulls = %d, want 1", info.NumWaiting)+ }++ secondCtx, cancelSecond := taskPullContext(t)+ defer cancelSecond()+ _, err = queue.PullMessage(secondCtx)+ if err == nil || errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) {+ t.Fatalf("MaxWaiting error = %v, want capacity rejection", err)+ }++ cancelFirst()+ select {+ case <-firstResult:+ case <-time.After(250 * time.Millisecond):+ t.Fatal("first pending pull did not exit after cancellation")+ }+}++func TestPullMessagesFetchesMessages(t *testing.T) {+ queue := newTaskQueue(t)+ publishTask(t, queue, "admin-direct-pull-1")+ publishTask(t, queue, "admin-direct-pull-2")++ ctx, cancel := taskPullContext(t)+ defer cancel()+ handles, err := queue.PullMessages(ctx, 2)+ if err != nil {+ t.Fatalf("PullMessages: %v", err)+ }+ if len(handles) != 2 {+ t.Fatalf("len(handles) = %d, want 2", len(handles))+ }+ for index, taskID := range []string{"admin-direct-pull-1", "admin-direct-pull-2"} {+ if got := handles[index].GetMessage().TaskID; got != taskID {+ t.Fatalf("task id = %s, want %s", got, taskID)+ }+ }+}++func TestPullMessagesReturnsPartialBatchOnDeadline(t *testing.T) {+ queue := newTaskQueue(t)+ publishTask(t, queue, "partial-batch")++ ctx, cancel := context.WithTimeout(t.Context(), 100*time.Millisecond)+ defer cancel()+ handles, err := queue.PullMessages(ctx, 2)+ if err != nil {+ t.Fatalf("PullMessages: %v", err)+ }+ if len(handles) != 1 || handles[0].GetMessage().TaskID != "partial-batch" {+ t.Fatalf("partial handles = %+v, want partial-batch", handles)+ }+}++func TestPullMessagesRejectsOutOfRangeMessageCount(t *testing.T) {+ queue := newTaskQueue(t)+ ctx, cancel := taskPullContext(t)+ defer cancel()++ for _, testCase := range []struct {+ name string+ messageCount int+ }{+ {name: "zero", messageCount: 0},+ {name: "negative", messageCount: -1},+ {name: "above limit", messageCount: 101},+ } {+ t.Run(testCase.name, func(t *testing.T) {+ _, err := queue.PullMessages(ctx, testCase.messageCount)+ if err == nil {+ t.Fatal("PullMessages succeeded for an out-of-range message count")+ }+ })+ }+}++func TestPullMessagesReportsBatchError(t *testing.T) {+ queue := newTaskQueue(t)+ ctx, cancel := taskPullContext(t)+ defer cancel()+ if err := queue.stream.DeleteConsumer(ctx, "RAGFLOW_CONSUMER"); err != nil {+ t.Fatalf("delete default consumer: %v", err)+ }+ consumer, err := queue.stream.CreateOrUpdateConsumer(ctx, jetstream.ConsumerConfig{+ Name: "RAGFLOW_CONSUMER",+ AckPolicy: jetstream.AckExplicitPolicy,+ FilterSubject: "tasks.>",+ MaxWaiting: 1,+ })+ if err != nil {+ t.Fatalf("create limited consumer: %v", err)+ }+ queue.consumer = consumer++ occupiedCtx, cancelOccupied := taskPullContext(t)+ defer cancelOccupied()+ occupied := make(chan error, 1)+ go func() {+ _, err := queue.PullMessage(occupiedCtx)+ occupied <- err+ }()++ deadline := time.Now().Add(time.Second)+ for time.Now().Before(deadline) {+ info, err := queue.consumer.Info(t.Context())+ if err != nil {+ t.Fatalf("consumer info: %v", err)+ }+ if info.NumWaiting == 1 {+ break+ }+ time.Sleep(10 * time.Millisecond)+ }++ if _, err := queue.PullMessages(ctx, 1); err == nil {+ t.Fatal("PullMessages succeeded after the consumer rejected its pull")+ }+ cancelOccupied()+ select {+ case <-occupied:+ case <-time.After(250 * time.Millisecond):+ t.Fatal("occupied pull did not exit after cancellation")+ }+}diff --git a/internal/ingestion/service/burst_backpressure_test.go b/internal/ingestion/service/burst_backpressure_test.godeleted file mode 100644index 3728e1fd77..0000000000--- a/internal/ingestion/service/burst_backpressure_test.go+++ /dev/null@@ -1,258 +0,0 @@-//-// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.-//-// Licensed under the Apache License, Version 2.0 (the "License");-// you may not use this file except in compliance with the License.-// You may obtain a copy of the License at-//-// http://www.apache.org/licenses/LICENSE-2.0-//-// Unless required by applicable law or agreed to in writing, software-// distributed under the License is distributed on an "AS IS" BASIS,-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.-// See the License for the specific language governing permissions and-// limitations under the License.-//-//-// This test reproduces the "20+ files selected, only some parse, the rest-// get stuck forever" report against the Go ingestor (canvas / pipeline path).-//-// Root cause it guards against: processMessage applies backpressure by-// NACKing the message when taskChan is full. The NATS consumer is configured-// with MaxDeliver: 16 and NO dead-letter subject (internal/engine/nats/nats.go),-// so a message NACKed 16 times is permanently dropped. By that point-// StartRunning has already flipped the task (and its document) to RUNNING in-// the DB, and the ingestor has no scan-and-re-enqueue path on completion — so-// the dropped document is stuck in RUNNING forever.-//-// The test drives the REAL processMessage + REAL worker pool, but substitutes-// a deterministic "broker" that mirrors NATS' contract: bounded redelivery-// (MaxDeliver) with no dead-letter. A slow worker models a saturated pipeline-// (e.g. LLM calls in a canvas). The invariant under test is that NO task is-// ever lost: all N delivered tasks eventually reach COMPLETED.-//-// Under the buggy (NACK-on-backpressure) code, the burst overflow is dropped-// after MaxDeliver and the test FAILS. After the fix (blocking send so the-// consume loop applies backpressure instead of dropping), every task is-// processed and the test PASSES.--package service--import (- "context"- "fmt"- "sync"- "testing"- "time"-- "ragflow/internal/common"- "ragflow/internal/entity"- taskpkg "ragflow/internal/ingestion/task"- "ragflow/internal/ingestion/testutil"-- "gorm.io/gorm"-)--const (- burstTaskCount = 20- burstMaxDeliver = 16 // mirrors NATS MaxDeliver in internal/engine/nats/nats.go- burstWorkerMs = 50 // slow worker models a saturated pipeline-)--// burstMsg models one in-flight broker message: the handle processMessage-// settles, plus how many times it has been delivered (1 = initial delivery).-type burstMsg struct {- handle *fakeTaskHandle- deliveries int-}--// TestProcessMessage_BurstNoTaskLossUnderBackpressure drives a 20-file burst-// through the real processMessage + worker pool with a simulated NATS broker-// (bounded redelivery, no dead-letter). It asserts that every task completes-// and none are dropped.-func TestProcessMessage_BurstNoTaskLossUnderBackpressure(t *testing.T) {- db := testutil.SetupTestDB(t)- cleanup := testutil.ReplaceDBForTest(t, db)- defer cleanup()-- taskIDs := seedBurstTasks(t, db, burstTaskCount)-- ingestor := newUnitIngestor("test", 1, []string{"pdf"})- // Slow worker: model a saturated pipeline so the channel is full during- // the burst, forcing the backpressure path.- ingestor.runDocumentTask = func(ctx context.Context, _ *entity.IngestionTask) error {- time.Sleep(burstWorkerMs * time.Millisecond)- return nil- }- ingestor.startWorkerPool()- defer ingestor.Stop(context.Background())-- // Simulated NATS broker: pending holds messages waiting to be delivered.- var mu sync.Mutex- var pending []*burstMsg- dropped := 0-- seedBroker := func() {- mu.Lock()- defer mu.Unlock()- for _, id := range taskIDs {- pending = append(pending, &burstMsg{- handle: newFakeHandle(id, common.TaskTypeIngestionTask),- deliveries: 1,- })- }- }- seedBroker()-- // Simulated consume loop: mirrors consumeLoop -> GetMessages(4) ->- // processMessage, with NATS-style bounded redelivery on Nack.- stop := make(chan struct{})- var wg sync.WaitGroup- wg.Add(1)- go func() {- defer wg.Done()- for {- select {- case <-stop:- return- default:- }- mu.Lock()- if len(pending) == 0 {- mu.Unlock()- time.Sleep(5 * time.Millisecond) // idle backoff- continue- }- batchSize := 4- if len(pending) < batchSize {- batchSize = len(pending)- }- batch := pending[:batchSize]- pending = pending[batchSize:]- mu.Unlock()-- for _, bm := range batch {- ingestor.processMessage(bm.handle)- if bm.handle.nacks.Load() > 0 {- mu.Lock()- bm.deliveries++- if bm.deliveries < burstMaxDeliver {- // Redeliver (no delay — worst case for loss).- pending = append(pending, bm)- } else {- // NATS drops after MaxDeliver with no dead-letter:- // the task is permanently lost.- dropped++- }- mu.Unlock()- }- }- }- }()-- // Wait until all tasks reach COMPLETED, or time out.- deadline := time.Now().Add(30 * time.Second)- completed := 0- for time.Now().Before(deadline) {- completed = countTasksWithStatus(t, db, taskIDs, common.COMPLETED)- if completed == burstTaskCount {- break- }- time.Sleep(50 * time.Millisecond)- }- close(stop)- wg.Wait()-- if dropped > 0 {- t.Fatalf("BUG: %d/%d task(s) were permanently dropped (lost) under burst backpressure "+- "with MaxDeliver=%d and no dead-letter; only %d/%d completed. "+- "This is the 'files get stuck after the first few parse' defect.",- dropped, burstTaskCount, burstMaxDeliver, completed, burstTaskCount)- }- if completed != burstTaskCount {- t.Fatalf("expected all %d tasks to complete, got %d (stuck in RUNNING)",- burstTaskCount, completed)- }-}--// TestFetchBudget_NeverExceedsChannelCapacity: the consume loop must never-// pull more messages than it can enqueue immediately. The fetch budget is-// min(free channel slots, maxConcurrency) with a floor of 1 so a fully-// saturated channel still polls (and lets blocked sends drain) instead of-// stalling until AckWait expires on in-flight messages.-func TestFetchBudget_NeverExceedsChannelCapacity(t *testing.T) {- ingestor := newUnitIngestor("test", 2, []string{"pdf"}) // channel cap = 4- defer ingestor.Stop(context.Background())-- if got := ingestor.fetchBudget(); got != 2 {- t.Fatalf("empty channel: fetchBudget = %d, want 2 (capped by maxConcurrency)", got)- }-- for i := 0; i < cap(ingestor.taskChan)-1; i++ {- ingestor.taskChan <- taskpkg.NewTaskContextForScheduling(t.Context(), &entity.IngestionTask{ID: "filler"})- }- if got := ingestor.fetchBudget(); got != 1 {- t.Fatalf("3/4-full channel: fetchBudget = %d, want 1 (one free slot)", got)- }-- ingestor.taskChan <- taskpkg.NewTaskContextForScheduling(t.Context(), &entity.IngestionTask{ID: "filler"})- if got := ingestor.fetchBudget(); got != 1 {- t.Fatalf("full channel: fetchBudget = %d, want 1 (floor: must never fetch 0)", got)- }-- // A wide pool is bounded by the channel, not the worker count.- wide := newUnitIngestor("test-wide", 8, []string{"pdf"}) // channel cap = 16- defer wide.Stop(context.Background())- if got := wide.fetchBudget(); got != 8 {- t.Fatalf("wide empty channel: fetchBudget = %d, want 8 (capped by maxConcurrency)", got)- }-}--// seedBurstTasks creates one tenant/kb plus N (document, ingestion_task) pairs-// in CREATED status, mirroring how CreateAndEnqueue publishes a message for a-// freshly-created task. Returns the ingestion task IDs.-func seedBurstTasks(t *testing.T, db *gorm.DB, n int) []string {- t.Helper()- const tenantID, kbID = "burst-tenant", "burst-kb"- if err := db.Create(&entity.Tenant{ID: tenantID, LLMID: "gpt-4", Status: testutil.StrPtr("1")}).Error; err != nil {- t.Fatalf("create tenant: %v", err)- }- if err := db.Create(&entity.Knowledgebase{- ID: kbID, TenantID: tenantID, EmbdID: "embd-1", Status: testutil.StrPtr("1"), ParserConfig: entity.JSONMap{},- }).Error; err != nil {- t.Fatalf("create kb: %v", err)- }-- ids := make([]string, 0, n)- for i := 0; i < n; i++ {- docID := fmt.Sprintf("burst-doc-%d", i)- taskID := fmt.Sprintf("burst-task-%d", i)- loc := "doc_store/" + docID- if err := db.Create(&entity.Document{- ID: docID, KbID: kbID, Name: &docID, ParserID: "naive",- ParserConfig: entity.JSONMap{}, PipelineID: testutil.StrPtr("flow-1"),- Status: testutil.StrPtr("1"), Type: "pdf", Location: &loc,- }).Error; err != nil {- t.Fatalf("create doc %s: %v", docID, err)- }- if err := db.Create(&entity.IngestionTask{- ID: taskID, UserID: "u1", DocumentID: docID, DatasetID: kbID, Status: common.CREATED,- }).Error; err != nil {- t.Fatalf("create ingestion task %s: %v", taskID, err)- }- ids = append(ids, taskID)- }- return ids-}--// countTasksWithStatus returns how many of the given task IDs have the given status.-func countTasksWithStatus(t *testing.T, db *gorm.DB, ids []string, status string) int {- t.Helper()- var count int64- if err := db.Model(&entity.IngestionTask{}).- Where("id IN ? AND status = ?", ids, status).- Count(&count).Error; err != nil {- t.Fatalf("count tasks: %v", err)- }- return int(count)-}diff --git a/internal/ingestion/service/execute_memory_task_panic_test.go b/internal/ingestion/service/execute_memory_task_panic_test.goindex ecb21e6806..45cb9ac82d 100644--- a/internal/ingestion/service/execute_memory_task_panic_test.go+++ b/internal/ingestion/service/execute_memory_task_panic_test.go@@ -3,7 +3,6 @@ package serviceimport ("context""testing"- "time""ragflow/internal/common""ragflow/internal/entity"@@ -62,23 +61,15 @@ func TestExecuteMemoryTask_PanicDoesNotPropagate(t *testing.T) {}}-// TestWorkerLoop_SurvivesMemoryPanicAndKeepsServing proves the production-// failure mode end-to-end: a panicking memory task must NOT kill the worker-// goroutine, so a subsequent normal document parse is still processed. This is-// the exact "parsing works at first, then breaks after some memory tasks"-// symptom — at max_concurrent_workers=1 the only worker is lost on the first-// panicking memory task, and every later PDF stalls in the queue.-//-// The test drives the real workerLoop (not executeMemoryTask directly): it-// starts one worker, enqueues (1) a panicking memory task, then (2) a normal-// ingestion task with a stubbed runDocumentTask, and asserts the worker both-// Nacked the poison memory task and Acked the document task before exiting.-func TestWorkerLoop_SurvivesMemoryPanicAndKeepsServing(t *testing.T) {+// TestHandleAndExecute_SurvivesMemoryPanicAndKeepsServing proves a recovered+// memory panic does not prevent the same worker from processing its next+// hand-off.+func TestHandleAndExecute_SurvivesMemoryPanicAndKeepsServing(t *testing.T) {db := testutil.SetupTestDB(t)cleanup := testutil.ReplaceDBForTest(t, db)defer cleanup()- _, _, docID, taskID := testutil.SeedTestData(t, db, testutil.WithPipelineID("flow-1"))+ _, _, _, taskID := testutil.SeedTestData(t, db, testutil.WithPipelineID("flow-1"))ingestor := NewIngestor("test", 1, []string{"pdf"})ingestor.SetMemoryMessageService(servicepkg.NewMemoryMessageService(servicepkg.NewMemoryService()))@@ -89,37 +80,16 @@ func TestWorkerLoop_SurvivesMemoryPanicAndKeepsServing(t *testing.T) {return nil}- memHandle := &fakeTaskHandle{msg: common.TaskMessage{TaskID: "mem-panic-2", TaskType: common.TaskTypeMemory}}- memCtx := taskpkg.NewMemoryTaskContextForScheduling(context.Background(), "mem-panic-2", map[string]any{- "memory_id": "mem-p2", "source_id": 1,- "message_dict": map[string]any{"user_id": "u", "agent_id": "a", "session_id": "s"},- }, memHandle)-- docHandle := &fakeTaskHandle{}- docCtx := newAckTaskCtx(context.Background(), taskID, docID, docHandle)+ memHandle := &fakeTaskHandle{msg: common.TaskMessage{+ TaskID: "mem-panic-2",+ TaskType: common.TaskTypeMemory,+ Payload: []byte(`{"memory_id":"mem-p2","source_id":1,"message_dict":{"user_id":"u","agent_id":"a","session_id":"s"}}`),+ }}- ingestor.startWorkerPool()- // Feed the poison memory task first, then a normal document task.- ingestor.taskChan <- memCtx- ingestor.taskChan <- docCtx-- // Wait for the worker to drain both tasks (poison Nacked, document Acked)- // before cancelling, so the loop does not exit on ctx.Done() first.- deadline := time.After(5 * time.Second)- for memHandle.nacks.Load() != 1 || docHandle.acks.Load() != 1 {- select {- case <-deadline:- t.Fatalf("worker did not drain tasks in time: mem(acks=%d nacks=%d) doc(acks=%d nacks=%d)",- memHandle.acks.Load(), memHandle.nacks.Load(), docHandle.acks.Load(), docHandle.nacks.Load())- default:- time.Sleep(5 * time.Millisecond)- }- }+ ingestor.handleAndExecute(memHandle)- // Cancel the ingestor context so the worker loop exits; then wait for the- // worker goroutine to finish cleanly (proves it was not crashed by the panic).- ingestor.cancel()- ingestor.workerWg.Wait()+ docHandle := &fakeTaskHandle{msg: common.TaskMessage{TaskID: taskID, TaskType: common.TaskTypeIngestionTask}}+ ingestor.handleAndExecute(docHandle)if memHandle.nacks.Load() != 1 || memHandle.acks.Load() != 0 {t.Fatalf("poison memory task: expected 0 Ack/1 Nack, got acks=%d nacks=%d", memHandle.acks.Load(), memHandle.nacks.Load())diff --git a/internal/ingestion/service/execute_task_ack_test.go b/internal/ingestion/service/execute_task_ack_test.goindex e658601591..aef6513249 100644--- a/internal/ingestion/service/execute_task_ack_test.go+++ b/internal/ingestion/service/execute_task_ack_test.go@@ -315,8 +315,10 @@ func TestSettleMessage_AckOnTerminal(t *testing.T) {handle := &fakeTaskHandle{}ctx := t.Context()taskCtx := newAckTaskCtx(t.Context(), "task-1", "doc-1", handle)+ hb := NewHeartbeat(taskCtx.ID(), taskCtx.Handle, ingestor.heartbeatInterval).WithContext(taskCtx.Ctx)+ hb.Start()- ingestor.settleMessage(ctx, taskCtx, func(ctx context.Context) bool { return true })+ ingestor.settleMessage(ctx, taskCtx, hb, func(ctx context.Context) bool { return true })if handle.acks.Load() != 1 || handle.nacks.Load() != 0 {t.Fatalf("body=true: expected 1 Ack/0 Nack, got acks=%d nacks=%d", handle.acks.Load(), handle.nacks.Load())@@ -329,8 +331,10 @@ func TestSettleMessage_NackOnNonTerminal(t *testing.T) {handle := &fakeTaskHandle{}ctx := t.Context()taskCtx := newAckTaskCtx(t.Context(), "task-1", "doc-1", handle)+ hb := NewHeartbeat(taskCtx.ID(), taskCtx.Handle, ingestor.heartbeatInterval).WithContext(taskCtx.Ctx)+ hb.Start()- ingestor.settleMessage(ctx, taskCtx, func(ctx context.Context) bool { return false })+ ingestor.settleMessage(ctx, taskCtx, hb, func(ctx context.Context) bool { return false })if handle.nacks.Load() != 1 || handle.acks.Load() != 0 {t.Fatalf("body=false: expected 1 Nack/0 Ack, got acks=%d nacks=%d", handle.acks.Load(), handle.nacks.Load())@@ -355,6 +359,8 @@ func TestSettleMessage_RecoversPanicAndAcksWhenTaskTerminal(t *testing.T) {handle := &fakeTaskHandle{}ctx := t.Context()taskCtx := newAckTaskCtx(t.Context(), taskID, docID, handle)+ hb := NewHeartbeat(taskCtx.ID(), taskCtx.Handle, ingestor.heartbeatInterval).WithContext(taskCtx.Ctx)+ hb.Start()panicked := falsefunc() {@@ -363,7 +369,7 @@ func TestSettleMessage_RecoversPanicAndAcksWhenTaskTerminal(t *testing.T) {panicked = true}}()- ingestor.settleMessage(ctx, taskCtx, func(ctx context.Context) bool {+ ingestor.settleMessage(ctx, taskCtx, hb, func(ctx context.Context) bool {panic("boom")})}()@@ -451,7 +457,9 @@ func TestSettleMessage_DBTruthOverridesBodyReturn(t *testing.T) {}ctx := t.Context()- ingestor.settleMessage(ctx, taskCtx, body)+ hb := NewHeartbeat(taskCtx.ID(), taskCtx.Handle, ingestor.heartbeatInterval).WithContext(taskCtx.Ctx)+ hb.Start()+ ingestor.settleMessage(ctx, taskCtx, hb, body)// DB shows FAILED → terminal → Ack, overriding body's false.if handle.acks.Load() != 1 || handle.nacks.Load() != 0 {diff --git a/internal/ingestion/service/handle_and_execute_test.go b/internal/ingestion/service/handle_and_execute_test.gonew file mode 100644index 0000000000..4df6dd4944--- /dev/null+++ b/internal/ingestion/service/handle_and_execute_test.go@@ -0,0 +1,366 @@+//+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.+//++package service++import (+ "context"+ "testing"+ "time"++ "ragflow/internal/common"+ "ragflow/internal/entity"+ "ragflow/internal/ingestion/testutil"+ servicepkg "ragflow/internal/service"+)++// TestHandleAndExecute_MalformedMemoryPayloadAcks verifies that a memory task with+// an unparseable payload is acked and skipped without executing the runner.+func TestHandleAndExecute_MalformedMemoryPayloadAcks(t *testing.T) {+ ingestor := newUnitIngestor("test-mem-malformed", 1, nil)+ ingestor.SetMemoryMessageService(servicepkg.NewMemoryMessageService(servicepkg.NewMemoryService()))++ runnerCalled := false+ ingestor.runMemoryTask = func(ctx context.Context, taskID string, payload map[string]any) error {+ runnerCalled = true+ return nil+ }++ handle := &fakeTaskHandle{msg: common.TaskMessage{+ TaskID: "mem-bad-payload",+ TaskType: common.TaskTypeMemory,+ Payload: []byte(`{invalid-json`),+ }}++ ingestor.handleAndExecute(handle)++ if runnerCalled {+ t.Fatal("expected runMemoryTask to not be called for malformed payload")+ }+ if handle.acks.Load() != 1 || handle.nacks.Load() != 0 {+ t.Fatalf("expected 1 Ack/0 Nack, got acks=%d nacks=%d", handle.acks.Load(), handle.nacks.Load())+ }+}++// TestHandleAndExecute_MemoryExtractorDisabledAcks verifies that a memory task+// received when memory extractor is disabled is ack-skipped.+func TestHandleAndExecute_MemoryExtractorDisabledAcks(t *testing.T) {+ ingestor := newUnitIngestor("test-mem-disabled", 1, nil)+ // memorySvc is nil++ handle := &fakeTaskHandle{msg: common.TaskMessage{+ TaskID: "mem-disabled",+ TaskType: common.TaskTypeMemory,+ Payload: []byte(`{"memory_id":"m1"}`),+ }}++ ingestor.handleAndExecute(handle)++ if handle.acks.Load() != 1 || handle.nacks.Load() != 0 {+ t.Fatalf("expected 1 Ack/0 Nack, got acks=%d nacks=%d", handle.acks.Load(), handle.nacks.Load())+ }+}++// TestHandleAndExecute_MemoryEmptyTaskIDAcks verifies that a memory task with+// an empty task ID is ack-skipped.+func TestHandleAndExecute_MemoryEmptyTaskIDAcks(t *testing.T) {+ ingestor := newUnitIngestor("test-mem-empty-id", 1, nil)+ ingestor.SetMemoryMessageService(servicepkg.NewMemoryMessageService(servicepkg.NewMemoryService()))++ handle := &fakeTaskHandle{msg: common.TaskMessage{+ TaskID: "",+ TaskType: common.TaskTypeMemory,+ Payload: []byte(`{"memory_id":"m1"}`),+ }}++ ingestor.handleAndExecute(handle)++ if handle.acks.Load() != 1 || handle.nacks.Load() != 0 {+ t.Fatalf("expected 1 Ack/0 Nack, got acks=%d nacks=%d", handle.acks.Load(), handle.nacks.Load())+ }+}++// TestHandleAndExecute_UnknownTaskTypeAcks verifies that a message with an+// unknown task type is ack-skipped without touching the ingestion task service.+func TestHandleAndExecute_UnknownTaskTypeAcks(t *testing.T) {+ ingestor := newUnitIngestor("test-unknown-type", 1, nil)++ handle := &fakeTaskHandle{msg: common.TaskMessage{+ TaskID: "unknown-task-1",+ TaskType: "completely_unknown_type",+ }}++ ingestor.handleAndExecute(handle)++ if handle.acks.Load() != 1 || handle.nacks.Load() != 0 {+ t.Fatalf("expected 1 Ack/0 Nack for unknown type, got acks=%d nacks=%d", handle.acks.Load(), handle.nacks.Load())+ }+}++// TestHandleAndExecute_TaskNotFoundAcks verifies that when StartRunning returns+// ErrTaskNotFound, the message is Acked to prevent indefinite redeliveries.+func TestHandleAndExecute_TaskNotFoundAcks(t *testing.T) {+ db := testutil.SetupTestDB(t)+ cleanup := testutil.ReplaceDBForTest(t, db)+ defer cleanup()++ ingestor := newUnitIngestor("test-task-not-found", 1, []string{"pdf"})+ handle := &fakeTaskHandle{msg: common.TaskMessage{+ TaskID: "non-existent-task-id",+ TaskType: common.TaskTypeIngestionTask,+ }}++ ingestor.handleAndExecute(handle)++ if handle.acks.Load() != 1 || handle.nacks.Load() != 0 {+ t.Fatalf("expected 1 Ack/0 Nack for not-found task, got acks=%d nacks=%d", handle.acks.Load(), handle.nacks.Load())+ }+}++// TestHandleAndExecute_StartRunningTransientErrorNacks verifies that a transient+// database error during StartRunning causes the message to be Nacked for redelivery.+func TestHandleAndExecute_StartRunningTransientErrorNacks(t *testing.T) {+ // Close/nil DB to simulate transient database failure+ db := testutil.SetupTestDB(t)+ cleanup := testutil.ReplaceDBForTest(t, db)+ defer cleanup()++ _, _, _, taskID := testutil.SeedTestData(t, db)++ // Close underlying sql db to force query errors+ sqlDB, err := db.DB()+ if err == nil {+ _ = sqlDB.Close()+ }++ ingestor := newUnitIngestor("test-start-running-err", 1, []string{"pdf"})+ handle := &fakeTaskHandle{msg: common.TaskMessage{+ TaskID: taskID,+ TaskType: common.TaskTypeIngestionTask,+ }}++ ingestor.handleAndExecute(handle)++ if handle.nacks.Load() != 1 || handle.acks.Load() != 0 {+ t.Fatalf("expected 0 Ack/1 Nack on transient StartRunning error, got acks=%d nacks=%d", handle.acks.Load(), handle.nacks.Load())+ }+}++// TestHandleAndExecute_AlreadyTerminalTaskAcks verifies that tasks already in a+// terminal state (COMPLETED, STOPPED, FAILED) are Ack-skipped without re-execution.+func TestHandleAndExecute_AlreadyTerminalTaskAcks(t *testing.T) {+ statuses := []string{common.COMPLETED, common.STOPPED, common.FAILED}+ for _, status := range statuses {+ t.Run(status, func(t *testing.T) {+ db := testutil.SetupTestDB(t)+ cleanup := testutil.ReplaceDBForTest(t, db)+ defer cleanup()++ _, _, _, taskID := testutil.SeedTestData(t, db)+ if err := db.Model(&entity.IngestionTask{}).Where("id = ?", taskID).+ Update("status", status).Error; err != nil {+ t.Fatalf("set status %s: %v", status, err)+ }++ ingestor := newUnitIngestor("test-terminal-"+status, 1, []string{"pdf"})+ pipelineRan := false+ ingestor.runDocumentTask = func(ctx context.Context, task *entity.IngestionTask) error {+ pipelineRan = true+ return nil+ }++ handle := &fakeTaskHandle{msg: common.TaskMessage{+ TaskID: taskID,+ TaskType: common.TaskTypeIngestionTask,+ }}++ ingestor.handleAndExecute(handle)++ if pipelineRan {+ t.Fatalf("pipeline should not run for already %s task", status)+ }+ if handle.acks.Load() != 1 || handle.nacks.Load() != 0 {+ t.Fatalf("status %s: expected 1 Ack/0 Nack, got acks=%d nacks=%d", status, handle.acks.Load(), handle.nacks.Load())+ }+ })+ }+}++// TestHandleAndExecute_StoppingTaskConvergedAndAcks verifies that a task in STOPPING+// status is transitioned to STOPPED by StartRunning and then Acked.+func TestHandleAndExecute_StoppingTaskConvergedAndAcks(t *testing.T) {+ db := testutil.SetupTestDB(t)+ cleanup := testutil.ReplaceDBForTest(t, db)+ defer cleanup()++ _, _, _, taskID := testutil.SeedTestData(t, db)+ if err := db.Model(&entity.IngestionTask{}).Where("id = ?", taskID).+ Update("status", common.STOPPING).Error; err != nil {+ t.Fatalf("set status STOPPING: %v", err)+ }++ ingestor := newUnitIngestor("test-stopping-converge", 1, []string{"pdf"})+ handle := &fakeTaskHandle{msg: common.TaskMessage{+ TaskID: taskID,+ TaskType: common.TaskTypeIngestionTask,+ }}++ ingestor.handleAndExecute(handle)++ if handle.acks.Load() != 1 || handle.nacks.Load() != 0 {+ t.Fatalf("expected 1 Ack/0 Nack for STOPPING task, got acks=%d nacks=%d", handle.acks.Load(), handle.nacks.Load())+ }++ var task entity.IngestionTask+ if err := db.Where("id = ?", taskID).First(&task).Error; err != nil {+ t.Fatalf("reload task: %v", err)+ }+ if task.Status != common.STOPPED {+ t.Fatalf("task status = %s, want STOPPED", task.Status)+ }+}++// TestHandleAndExecute_DocumentDuplicateClaimRenewsLease verifies that a redelivered+// document task while the worker is still processing renews its lease via InProgress+// and does not Ack or start a second execution.+func TestHandleAndExecute_DocumentDuplicateClaimRenewsLease(t *testing.T) {+ db := testutil.SetupTestDB(t)+ cleanup := testutil.ReplaceDBForTest(t, db)+ defer cleanup()++ _, _, _, taskID := testutil.SeedTestData(t, db)+ if err := db.Model(&entity.IngestionTask{}).Where("id = ?", taskID).+ Update("status", common.RUNNING).Error; err != nil {+ t.Fatalf("set status RUNNING: %v", err)+ }++ ingestor := newUnitIngestor("test-doc-dup", 1, []string{"pdf"})+ // Simulate active claim by another worker+ if !ingestor.claimTask(taskID) {+ t.Fatal("first claim should succeed")+ }++ pipelineRan := false+ ingestor.runDocumentTask = func(ctx context.Context, task *entity.IngestionTask) error {+ pipelineRan = true+ return nil+ }++ handle := &fakeTaskHandle{msg: common.TaskMessage{+ TaskID: taskID,+ TaskType: common.TaskTypeIngestionTask,+ }}++ ingestor.handleAndExecute(handle)++ if pipelineRan {+ t.Fatal("duplicate delivery should not execute pipeline")+ }+ if handle.inProgress.Load() != 1 {+ t.Fatalf("expected InProgress = 1 for duplicate delivery, got %d", handle.inProgress.Load())+ }+ if handle.acks.Load() != 0 || handle.nacks.Load() != 0 {+ t.Fatalf("expected 0 Ack/0 Nack for duplicate delivery, got acks=%d nacks=%d", handle.acks.Load(), handle.nacks.Load())+ }+}++// TestHandleAndExecute_MemoryDuplicateClaimRenewsLease verifies duplicate delivery+// protection for memory extraction tasks.+func TestHandleAndExecute_MemoryDuplicateClaimRenewsLease(t *testing.T) {+ ingestor := newUnitIngestor("test-mem-dup", 1, nil)+ ingestor.SetMemoryMessageService(servicepkg.NewMemoryMessageService(servicepkg.NewMemoryService()))++ taskID := "mem-dup-1"+ if !ingestor.claimTask(taskID) {+ t.Fatal("first claim should succeed")+ }++ handle := &fakeTaskHandle{msg: common.TaskMessage{+ TaskID: taskID,+ TaskType: common.TaskTypeMemory,+ Payload: []byte(`{"memory_id":"m1","source_id":1,"message_dict":{"user_id":"u"}}`),+ }}++ ingestor.handleAndExecute(handle)++ if handle.inProgress.Load() != 1 {+ t.Fatalf("expected InProgress = 1 for duplicate memory delivery, got %d", handle.inProgress.Load())+ }+ if handle.acks.Load() != 0 || handle.nacks.Load() != 0 {+ t.Fatalf("expected 0 Ack/0 Nack, got acks=%d nacks=%d", handle.acks.Load(), handle.nacks.Load())+ }+}++// TestHandleAndExecute_SlowAdmissionHeartbeatsUnderLeaseProtection verifies that+// when admission (StartRunning) is slow (e.g. database lag exceeding ack deadline),+// the heartbeat started upon receiving the handle renews the broker lease before+// StartRunning finishes.+func TestHandleAndExecute_SlowAdmissionHeartbeatsUnderLeaseProtection(t *testing.T) {+ db := testutil.SetupTestDB(t)+ cleanup := testutil.ReplaceDBForTest(t, db)+ defer cleanup()++ _, _, _, taskID := testutil.SeedTestData(t, db)+ if err := db.Model(&entity.IngestionTask{}).Where("id = ?", taskID).+ Update("status", common.RUNNING).Error; err != nil {+ t.Fatalf("set status RUNNING: %v", err)+ }++ ingestor := newUnitIngestor("test-slow-admission", 1, []string{"pdf"})+ ingestor.heartbeatInterval = 10 * time.Millisecond++ // Delay document pipeline execution briefly so heartbeats fire during run+ ingestor.runDocumentTask = func(ctx context.Context, task *entity.IngestionTask) error {+ time.Sleep(50 * time.Millisecond)+ return nil+ }++ handle := &fakeTaskHandle{msg: common.TaskMessage{+ TaskID: taskID,+ TaskType: common.TaskTypeIngestionTask,+ }}++ ingestor.handleAndExecute(handle)++ if handle.inProgress.Load() == 0 {+ t.Fatal("expected InProgress heartbeats during task execution, got 0")+ }+ if handle.acks.Load() != 1 {+ t.Fatalf("expected 1 Ack on completion, got %d", handle.acks.Load())+ }+}++// TestWorkerDispatcher_WaitAfterPullErrorCancelsCleanly verifies that waitAfterPullError+// unblocks promptly when dispatch context is cancelled, without hanging on the sleep.+func TestWorkerDispatcher_WaitAfterPullErrorCancelsCleanly(t *testing.T) {+ ingestor := newUnitIngestor("test-pull-backoff-cancel", 1, nil)++ done := make(chan struct{})+ go func() {+ ingestor.waitAfterPullError()+ close(done)+ }()++ // Cancel dispatch context+ ingestor.dispatchCancel()++ select {+ case <-done:+ // unblocked promptly+ case <-time.After(200 * time.Millisecond):+ t.Fatal("waitAfterPullError did not cancel promptly on dispatchCtx.Done")+ }+}diff --git a/internal/ingestion/service/heartbeat.go b/internal/ingestion/service/heartbeat.goindex 5ef6ff6383..796716a2dd 100644--- a/internal/ingestion/service/heartbeat.go+++ b/internal/ingestion/service/heartbeat.go@@ -40,6 +40,7 @@ type Heartbeat struct {handle common.TaskHandleinterval time.Durationctx context.Context+ onError func(error)mu sync.Mutexstarted bool@@ -61,6 +62,14 @@ func NewHeartbeat(id string, handle common.TaskHandle, interval time.Duration) *}}+// WithOnError sets an optional error callback invoked when handle.InProgress fails.+func (h *Heartbeat) WithOnError(fn func(error)) *Heartbeat {+ if h != nil {+ h.onError = fn+ }+ return h+}+// WithContext binds an external cancellation signal (e.g. ingestor shutdown) so// the renewal goroutine also exits when ctx is done. It must be called before// Start.@@ -119,6 +128,9 @@ func (h *Heartbeat) loop() {case <-ticker.C:if err := h.handle.InProgress(); err != nil {common.Error(fmt.Sprintf("heartbeat task %s", h.id), err)+ if h.onError != nil {+ h.onError(err)+ }}case <-h.stopCh:returndiff --git a/internal/ingestion/service/ingestion_service.go b/internal/ingestion/service/ingestion_service.goindex b0e1d10234..f82719fe94 100644--- a/internal/ingestion/service/ingestion_service.go+++ b/internal/ingestion/service/ingestion_service.go@@ -41,6 +41,7 @@ import ("ragflow/internal/utility""github.com/cenkalti/backoff/v5"+ "go.uber.org/zap")// defaultHeartbeatInterval paces the InProgress() working pulse that keeps an@@ -56,9 +57,12 @@ const defaultHeartbeatInterval = 2 * time.Secondtype Ingestor struct {id stringname string- ctx context.Context+ ctx context.Context // execution context for handles already owned by workerscancel context.CancelFunc+ dispatchCtx context.Context // worker queue registration and Pull lifecycle+ dispatchCancel context.CancelFunc+// ConfigurationmaxConcurrency int32supportedDocTypes []string@@ -69,16 +73,21 @@ type Ingestor struct {currentTasks map[string]struct{} // set of task IDs currently claimed by a workertasksMu sync.RWMutexactiveWorkers atomic.Int32 // number of worker goroutines currently in workerLoop+ activeLeases map[*Heartbeat]*activeLease+ leasesMu sync.Mutex+ stopLeases atomic.Bool// Shutdown channel - receive on this to trigger graceful shutdownShutdownCh chan struct{}// Worker pool- taskChan chan *taskpkg.TaskContext- workerWg sync.WaitGroup- startOnce sync.Once- workerOnce sync.Once // guards startWorkerPool; must NOT be startOnce (Start wraps start() in startOnce, and start() calls startWorkerPool -> re-entry deadlock)- stopOnce sync.Once // guards close(ShutdownCh) against double-close on repeated Stop+ workerQueue chan *worker+ dispatcherWg sync.WaitGroup+ workerWg sync.WaitGroup+ startOnce sync.Once+ startErr error+ workerOnce sync.Once // guards startWorkerPool; must NOT be startOnce (Start wraps start() in startOnce, and start() calls startWorkerPool -> re-entry deadlock)+ stopOnce sync.Once // guards close(ShutdownCh) against double-close on repeated StopingestionTaskSvc *servicepkg.IngestionTaskServicedocState *docStateUpdater@@ -122,27 +131,38 @@ type Ingestor struct {// override this to simulate cancel without Redis.cancelCheck func(ctx context.Context, taskID string) bool- // checkpointExists distinguishes a fresh run from a checkpoint resume.- // Tests inject this dependency so unit tests do not require Redis.checkpointExists func(ctx context.Context, taskID string) (bool, error)}+type worker struct {+ id int32+ inbox chan common.TaskHandle+}++type activeLease struct {+ abandoned atomic.Bool+}+func NewIngestor(name string, maxConcurrency int32, supportedTypes []string) *Ingestor {if maxConcurrency <= 0 {maxConcurrency = int32(runtime.NumCPU())}ctx, cancel := context.WithCancel(context.Background())+ dispatchCtx, dispatchCancel := context.WithCancel(context.Background())id := utility.GenerateUUID()ingestor := &Ingestor{id: id,name: name,ctx: ctx,cancel: cancel,+ dispatchCtx: dispatchCtx,+ dispatchCancel: dispatchCancel,maxConcurrency: maxConcurrency,supportedDocTypes: supportedTypes,version: "1.0.0",currentTasks: make(map[string]struct{}),- taskChan: make(chan *taskpkg.TaskContext, maxConcurrency*2),+ activeLeases: make(map[*Heartbeat]*activeLease),+ workerQueue: make(chan *worker, maxConcurrency),ShutdownCh: make(chan struct{}, 1),ingestionTaskSvc: servicepkg.NewIngestionTaskService(),docState: newDocStateUpdater(),@@ -160,18 +180,19 @@ func (e *Ingestor) ID() string {return e.id}-// consumeErrorBackoff paces the consume loop when GetMessages returns an-// error, so a persistent MQ failure does not pin a CPU. The backoff is-// cancellable so a shutdown during backoff returns promptly.+// consumeErrorBackoff paces failed Pull requests so a persistent MQ failure+// does not pin a CPU. The backoff is cancellable and does not block another+// idle worker from making its own Pull request.const consumeErrorBackoff = 1 * time.Second+const taskPullRequestTimeout = 1 * time.Second+func (e *Ingestor) Start() error {common.Info(fmt.Sprintf("Ingestor %s initialized", e.id))- var startErr errore.startOnce.Do(func() {- startErr = e.start()+ e.startErr = e.start()})- return startErr+ return e.startErr}// start runs the full startup sequence. It is invoked at most once (guarded by@@ -199,66 +220,88 @@ func (e *Ingestor) start() error {e.startWorkerPool()e.startDatasetKnowledgeCompile()- // Run the main tasks.RAGFLOW consume loop off the caller's goroutine so- // Start returns promptly; it is joined by Stop via workerWg.- e.workerWg.Add(1)+ // Run the main tasks.RAGFLOW dispatcher off the caller's goroutine so Start+ // returns promptly. Stop joins it before cancelling task execution.+ e.dispatcherWg.Add(1)go e.consumeLoop()return nil}-// fetchBudget caps each Fetch batch by the worker channel's free capacity (and-// the worker width) so the consumer never pulls more messages than it can-// enqueue immediately: messages beyond the budget would sit claimed in-// taskChan's senders' hands or force blocking sends that stall the consume-// loop while their AckWait runs out.-func (e *Ingestor) fetchBudget() int {- budget := cap(e.taskChan) - len(e.taskChan)- if budget > int(e.maxConcurrency) {- budget = int(e.maxConcurrency)- }- if budget < 1 {- budget = 1- }- return budget-}--// consumeLoop is the main tasks.RAGFLOW consume loop. It runs until e.ctx is-// cancelled (graceful shutdown); per-message failures are settled by-// processMessage and never terminate the consumer. Dataset-level compile work-// is owned by the kcConcurrency compileLoop workers (each running the-// Consumer's Run loop), so this loop stays focused on tasks.RAGFLOW and is-// never held up by compile work.+// consumeLoop is the worker dispatcher for tasks.RAGFLOW. It waits for an+// available worker before synchronously pulling one task, preventing prefetch+// and limiting this ingestor to one outstanding broker request.func (e *Ingestor) consumeLoop() {- defer e.workerWg.Done()+ defer e.dispatcherWg.Done()msgQueueEngine := engine.GetMessageQueueEngine()for {- // Graceful shutdown is the only condition under which the consume- // loop exits. Per-message processing failures never terminate the- // consumer: processMessage settles (ack/nack) each message itself.- if err := e.ctx.Err(); err != nil {+ var w *worker+ select {+ case <-e.dispatchCtx.Done():+ return+ case w = <-e.workerQueue:+ }+ if e.dispatchCtx.Err() != nil {+ e.returnWorker(w)return}- taskHandles, err := msgQueueEngine.GetMessages(e.fetchBudget())++ pullStart := time.Now()+ pullCtx, cancel := context.WithTimeout(e.dispatchCtx, taskPullRequestTimeout)+ handle, err := msgQueueEngine.PullMessage(pullCtx)+ cancel()if err != nil {- common.Error("error consuming message", err)- select {- case <-time.After(consumeErrorBackoff):- case <-e.ctx.Done():- return+ e.returnWorker(w)+ e.logPullError(err)+ if !errors.Is(err, context.Canceled) && !errors.Is(err, context.DeadlineExceeded) {+ e.waitAfterPullError()}continue}- if len(taskHandles) > 0 {- for _, taskHandle := range taskHandles {- e.processMessage(taskHandle)- }+ if handle == nil {+ e.returnWorker(w)continue}- // tasks.RAGFLOW idle: dataset-level compile work is owned by the- // kcConcurrency compileLoop workers (started in- // startDatasetKnowledgeCompile), so the task loop stays focused on- // tasks.RAGFLOW and is never held up by compile work.++ recvTime := time.Now()+ latency := time.Since(pullStart)+ common.Debug(fmt.Sprintf("Pull delivered handle in %v", latency),+ zap.Duration("pull_first_handle_latency", latency),+ )+ select {+ case w.inbox <- handle:+ handoffDuration := time.Since(recvTime)+ common.Debug(fmt.Sprintf("Handed off handle to worker %d in %v", w.id, handoffDuration),+ zap.Int32("worker_id", w.id),+ zap.Duration("handoff_duration", handoffDuration),+ )+ case <-e.dispatchCtx.Done():+ if err := handle.Nack(); err != nil {+ common.Error("nack task after shutdown interrupted handoff", err)+ }+ return+ }+ }+}++func (e *Ingestor) logPullError(err error) {+ if err == nil || errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) {+ return+ }+ common.Error("error pulling task message", err)+}++func (e *Ingestor) waitAfterPullError() {+ select {+ case <-time.After(consumeErrorBackoff):+ case <-e.dispatchCtx.Done():+ }+}++func (e *Ingestor) returnWorker(w *worker) {+ select {+ case e.workerQueue <- w:+ case <-e.dispatchCtx.Done():}}@@ -333,104 +376,61 @@ func (e *Ingestor) compileLoop(id int32) {}}-// processMessage handles a single incoming MQ message: filter by type,-// activate the task (state transition), guard against duplicate execution-// (claim), and enqueue to the worker pool (or backpressure-reject). It-// settles (ack/nack) every message itself and never returns an error: a-// single message can never terminate the consume loop. Only ctx cancellation-// (graceful shutdown) stops the consumer - see Start.-func (e *Ingestor) processMessage(handle common.TaskHandle) {+// handleAndExecute owns one handle from its hand-off to final settlement. It+// runs only in the worker that received the handle through its private inbox.+func (e *Ingestor) handleAndExecute(handle common.TaskHandle) {+ startTime := time.Now()taskMessage := handle.GetMessage()- common.Info(fmt.Sprintf("Received task id: %s, type: %s", taskMessage.TaskID, taskMessage.TaskType))-- // Deferred claim release: if this function claims a task but the task- // is not successfully enqueued to the worker pool (e.g. backpressure,- // or a future error path added between claim and enqueue), the defer- // cleans up so the task can be reclaimed on MQ redelivery. When the- // task IS enqueued, claimedTaskID is cleared and executeTask's own- // defer takes ownership of the release.- var claimedTaskID string+ common.Info(fmt.Sprintf("Received task id: %s, type: %s", taskMessage.TaskID, taskMessage.TaskType),+ zap.String("task_id", taskMessage.TaskID),+ zap.String("task_type", taskMessage.TaskType),+ )defer func() {- if claimedTaskID != "" {- e.releaseTask(claimedTaskID)- }+ settlementDuration := time.Since(startTime)+ common.Debug(fmt.Sprintf("Task %s settlement finished in %v", taskMessage.TaskID, settlementDuration),+ zap.String("task_id", taskMessage.TaskID),+ zap.Duration("settlement_duration", settlementDuration),+ )}()+ hb := NewHeartbeat(taskMessage.TaskID, handle, e.heartbeatInterval).+ WithContext(context.Background())+ hb.Start()+ e.registerLease(hb)+ defer e.unregisterLease(hb)+// Memory-extraction tasks share the tasks.RAGFLOW consumer and the worker- // pool with ingestion tasks. They do NOT use the ingestion state machine- // (no ingestion_task row): the message body is dispatched straight to a- // worker via TaskContext.Kind==TaskKindMemory, which runs the memory- // extractor and acks/nacks on its own.+ // pool with ingestion tasks. They do NOT use the ingestion state machine.if taskMessage.TaskType == common.TaskTypeMemory {if e.memorySvc == nil {common.Warn(fmt.Sprintf("memory task %s received but memory extractor is disabled, ack", taskMessage.TaskID))- if err := handle.Ack(); err != nil {- common.Error(fmt.Sprintf("error ack memory task %s", taskMessage.TaskID), err)- }+ e.ackHandle(hb, handle, taskMessage.TaskID)return}- // A memory task without an envelope id has no valid identity: claiming- // an empty key would strand a no-op claim in currentTasks and block- // nothing useful, so Ack-skip it instead.if taskMessage.TaskID == "" {common.Warn("memory task with empty task id received, ack")- if err := handle.Ack(); err != nil {- common.Error("error ack memory task with empty task id", err)- }+ e.ackHandle(hb, handle, taskMessage.TaskID)return}var payload map[string]anyif len(taskMessage.Payload) == 0 || json.Unmarshal(taskMessage.Payload, &payload) != nil {common.Warn(fmt.Sprintf("memory task %s has no parseable payload, ack", taskMessage.TaskID))- if err := handle.Ack(); err != nil {- common.Error(fmt.Sprintf("error ack memory task %s", taskMessage.TaskID), err)- }+ e.ackHandle(hb, handle, taskMessage.TaskID)return}taskCtx := taskpkg.NewMemoryTaskContextForScheduling(e.ctx, taskMessage.TaskID, payload, handle)- // Claim the task before enqueueing so a redelivered copy of the same- // memory task (NATS AckWait/BackOff redelivery, or a restart replay) is- // not executed again while its owner is still running. Memory tasks have no- // ingestion_task row / status CAS to dedupe against, so this claim- // guards duplicate execution within this ingestor. The claim is released- // by executeMemoryTask when the worker finishes.if !e.claimTask(taskMessage.TaskID) {common.Warn(fmt.Sprintf("memory task %s redelivered while worker still processing, renew lease", taskMessage.TaskID))- if err := handle.InProgress(); err != nil {- common.Error(fmt.Sprintf("renew redelivered memory task %s", taskMessage.TaskID), err)- }- return- }- claimedTaskID = taskMessage.TaskID- // Begin renewing the broker lease before the task can wait in taskChan.- // Starting only in executeMemoryTask would leave queued work unprotected- // until a worker becomes available: a task waiting past AckWait would be- // redelivered before its owner can begin running it.- // The stop function rides on the task context so the worker can stop it- // before settlement (the lease is owned by the task, not by the map).- hb := NewHeartbeat(taskMessage.TaskID, handle, e.heartbeatInterval).WithContext(e.ctx)- hb.Start()- taskCtx.SetStopLease(hb.Stop)- select {- case e.taskChan <- taskCtx:- claimedTaskID = "" // executeMemoryTask owns the release now- common.Info(fmt.Sprintf("Memory task %s queued (channel: %d/%d)", taskMessage.TaskID, len(e.taskChan), cap(e.taskChan)))- case <-e.ctx.Done():- // Shutdown won the race: stop the lease, release the claim, and- // return without settling so the broker redelivers the memory task- // after restart.- taskCtx.StopLease()- common.Info(fmt.Sprintf("Ingestor shutting down; memory task %s not enqueued", taskMessage.TaskID))+ e.renewDuplicateHandle(hb, handle, taskMessage.TaskID)return}+ e.executeMemoryTaskWithHeartbeat(e.ctx, taskCtx, hb)return}if taskMessage.TaskType != common.TaskTypeIngestionTask {common.Info(fmt.Sprintf("task %s is not an ingestion task", taskMessage.TaskID))- if err := handle.Ack(); err != nil {- common.Error(fmt.Sprintf("error ack task %s", taskMessage.TaskID), err)- }+ e.ackHandle(hb, handle, taskMessage.TaskID)return}@@ -438,192 +438,171 @@ func (e *Ingestor) processMessage(handle common.TaskHandle) {if err != nil {if errors.Is(err, common.ErrTaskNotFound) {common.Warn(fmt.Sprintf("task %s not found, skipping", taskMessage.TaskID))- if ackErr := handle.Ack(); ackErr != nil {- common.Error(fmt.Sprintf("error ack task %s", taskMessage.TaskID), ackErr)- }+ e.ackHandle(hb, handle, taskMessage.TaskID)return}- // Recoverable activation failure (e.g. a DB blip): nack for- // redelivery instead of dropping the message or killing the- // consumer.common.Error(fmt.Sprintf("error setting task %s to running", taskMessage.TaskID), err)- if nackErr := handle.Nack(); nackErr != nil {- common.Error(fmt.Sprintf("error nack task %s", taskMessage.TaskID), nackErr)- }+ e.nackHandle(hb, handle, taskMessage.TaskID)return}if task == nil {common.Info(fmt.Sprintf("task %s is already removed", taskMessage.TaskID))- if ackErr := handle.Ack(); ackErr != nil {- common.Error(fmt.Sprintf("error ack task %s", taskMessage.TaskID), ackErr)- }+ e.ackHandle(hb, handle, taskMessage.TaskID)return}switch task.Status {case common.COMPLETED, common.STOPPED, common.FAILED:common.Info(fmt.Sprintf("task %s is already %s", taskMessage.TaskID, task.Status))- if ackErr := handle.Ack(); ackErr != nil {- common.Error(fmt.Sprintf("error ack task %s", taskMessage.TaskID), ackErr)- }+ e.ackHandle(hb, handle, taskMessage.TaskID)returncase common.RUNNING:- // Guard against MQ redelivery: if another worker in this- // process is already processing this task, renew the redelivered- // message's lease and skip instead of scheduling it again. The- // owning worker remains the only code path that may Ack/Nack.if !e.claimTask(task.ID) {common.Warn(fmt.Sprintf("task %s redelivered while worker still processing, renew lease (task_id=%s doc_id=%s kb_id=%s)",taskMessage.TaskID, task.ID, task.DocumentID, task.DatasetID))- if err := handle.InProgress(); err != nil {- common.Error(fmt.Sprintf("renew redelivered task %s", taskMessage.TaskID), err)- }+ e.renewDuplicateHandle(hb, handle, taskMessage.TaskID)return}- claimedTaskID = task.IDdefault:- // Unreachable given StartRunning normalizes every status to- // RUNNING/COMPLETED/STOPPED/FAILED, but defensive: ack-skip an- // unknown status instead of enqueuing it for execution.common.Warn(fmt.Sprintf("task %s in unexpected status %s, ack-skip", taskMessage.TaskID, task.Status))- if ackErr := handle.Ack(); ackErr != nil {- common.Error(fmt.Sprintf("error ack task %s", taskMessage.TaskID), ackErr)- }+ e.ackHandle(hb, handle, taskMessage.TaskID)return}- // Construct TaskContext and carry the MQ handle so the worker can- // Ack/Nack when the task reaches a terminal status.taskCtx := taskpkg.NewTaskContextForScheduling(e.ctx, task)taskCtx.Handle = handle+ e.executeTaskWithHeartbeat(e.ctx, taskCtx, hb)+}- // Push to the task channel. Use a blocking send so backpressure is- // applied at the consumer: the consume loop waits for a free worker slot- // instead of dropping the message. Dropping on backpressure is unsafe- // because StartRunning (above) has already flipped the task — and its- // document — to RUNNING in the DB, and there is no scan-and-re-enqueue- // path on completion. A Nack that exceeds the broker's MaxDeliver (16,- // with no dead-letter in nats.go) would permanently lose the task and- // leave the document stuck in RUNNING — the "files get stuck after the- // first few parse" defect.- //- // The in-flight claim (set just above) guards against a redelivery racing- // the blocked send: a duplicate delivery sees claimTask fail, renews its- // lease, and cannot double-execute a task.- select {- case e.taskChan <- taskCtx:- claimedTaskID = "" // executeTask owns the release now- common.Info(fmt.Sprintf("Task %s queued (channel: %d/%d)", task.ID, len(e.taskChan), cap(e.taskChan)))- case <-e.ctx.Done():- // Shutdown won the race after StartRunning already flipped the task- // to RUNNING: finalize it as STOPPED so the row cannot linger in- // non-terminal RUNNING with no in-flight worker (the user can- // retry). The message is left unsettled - on redelivery the terminal- // STOPPED status ack-skips it. Both paths- // run with a detached 2s timeout so shutdown is not slowed by a- // stalled DB.- stopCtx, cancel := context.WithTimeout(context.Background(), 2*time.Second)- defer cancel()- if e.markStopped(stopCtx, task.ID) {- common.Info(fmt.Sprintf("Ingestor shutting down; task %s finalized as STOPPED", task.ID))- } else {- common.Warn(fmt.Sprintf("Ingestor shutting down; task %s left RUNNING for NATS redelivery", task.ID))- }+func (e *Ingestor) ackHandle(hb *Heartbeat, handle common.TaskHandle, taskID string) {+ hb.Stop()+ if e.leaseAbandoned(hb) {+ return+ }+ if err := handle.Ack(); err != nil {+ common.Error(fmt.Sprintf("ack task %s", taskID), err)+ }+}++func (e *Ingestor) nackHandle(hb *Heartbeat, handle common.TaskHandle, taskID string) {+ hb.Stop()+ if e.leaseAbandoned(hb) {return}+ if err := handle.Nack(); err != nil {+ common.Error(fmt.Sprintf("nack task %s", taskID), err)+ }+}++func (e *Ingestor) renewDuplicateHandle(hb *Heartbeat, handle common.TaskHandle, taskID string) {+ hb.Stop()+ if e.leaseAbandoned(hb) {+ return+ }+ if err := handle.InProgress(); err != nil {+ common.Error(fmt.Sprintf("renew redelivered task %s", taskID), err)+ }+}++func (e *Ingestor) registerLease(hb *Heartbeat) {+ lease := &activeLease{}+ e.leasesMu.Lock()+ stop := e.stopLeases.Load()+ if stop {+ lease.abandoned.Store(true)+ }+ e.activeLeases[hb] = lease+ e.leasesMu.Unlock()+ if stop {+ hb.Stop()+ }+}++func (e *Ingestor) unregisterLease(hb *Heartbeat) {+ e.leasesMu.Lock()+ delete(e.activeLeases, hb)+ e.leasesMu.Unlock()+}++func (e *Ingestor) leaseAbandoned(hb *Heartbeat) bool {+ e.leasesMu.Lock()+ lease := e.activeLeases[hb]+ e.leasesMu.Unlock()+ return lease != nil && lease.abandoned.Load()+}++func (e *Ingestor) stopActiveLeases() {+ e.leasesMu.Lock()+ e.stopLeases.Store(true)+ leases := make([]*Heartbeat, 0, len(e.activeLeases))+ for heartbeat, lease := range e.activeLeases {+ lease.abandoned.Store(true)+ leases = append(leases, heartbeat)+ }+ e.leasesMu.Unlock()++ for _, lease := range leases {+ lease.Stop()+ }}func (e *Ingestor) startWorkerPool() {e.workerOnce.Do(func() {for i := int32(0); i < e.maxConcurrency; i++ {e.workerWg.Add(1)- go e.workerLoop(i)+ w := &worker{id: i, inbox: make(chan common.TaskHandle)}+ go e.workerLoop(w)}common.Info(fmt.Sprintf("Worker pool started with %d workers", e.maxConcurrency))})}-func (e *Ingestor) workerLoop(id int32) {+func (e *Ingestor) workerLoop(w *worker) {defer e.workerWg.Done()defer e.activeWorkers.Add(-1)e.activeWorkers.Add(1)- common.Info(fmt.Sprintf("Worker %d started", id))+ common.Info(fmt.Sprintf("Worker %d started", w.id))for {select {- case <-e.ctx.Done():+ case e.workerQueue <- w:+ case <-e.dispatchCtx.Done():+ return+ }++ select {+ case handle := <-w.inbox:+ e.handleAndExecute(handle)+ case <-e.dispatchCtx.Done():return- case taskCtx := <-e.taskChan:- if taskCtx.Kind == taskpkg.TaskKindMemory {- e.executeMemoryTask(e.ctx, taskCtx)- continue- }- common.Info("task context:" + taskCtx.IngestionTask.ID)- e.executeTask(e.ctx, taskCtx)}}}-// executeMemoryTask runs one async memory-extraction task (TaskKindMemory) on-// a worker of the shared pool. Unlike ingestion tasks, memory tasks have no-// ingestion_task row / state machine: the runner persists the extracted-// messages and settles task progress on the way out.-//-// The task id comes exclusively from taskCtx.ID() (the envelope TaskID set by-// the scheduler). It is never re-derived from MemoryPayload — the payload no-// longer carries identity.-//-// Settlement runs in one deferred closed loop on every exit path (success,-// terminal/transient failure, panic) in a fixed order: stop the admission-// heartbeat and wait for it to exit, then Ack/Nack exactly once, then release-// the in-process claim. Stopping the heartbeat before the Ack/Nack is required-// by Heartbeat's contract — no InProgress may be in flight on the same message-// while it is being settled. The claim is released after settlement so a-// redelivery that races the settlement stays owned by this worker; a redelivery-// after the function returns (e.g. a lost Ack) re-claims and re-runs the task-// as normal at-least-once delivery.-//-// Settlement is error-category aware:-// - Terminal failure (task row absent, already-failed, or progress=-1 already-// persisted) is Acked so an already-consumed message is never redelivered-// into an infinite nack loop.-// - A task row already at progress>=1.0 is treated as success (the completed-// short-circuit in HandleSaveToMemoryTask) and Acked.-// - Transient failure (a task-load DB error before any durable marker, or an-// LLM/network failure that did not reach progress=-1) is Nacked so the-// message is redelivered and retried instead of being silently dropped.-//-// A panic in the memory path is recovered so the worker goroutine survives:-// the ingestion path already does this via settleMessage's recover, but a-// panic here previously crashed the worker. With max_concurrent_workers as low-// as 1, one panicking memory task could permanently remove the only worker and-// stall every subsequent document parse. The recovered panic is treated as a-// transient failure and Nacked for redelivery.func (e *Ingestor) executeMemoryTask(ctx context.Context, taskCtx *taskpkg.TaskContext) {- taskID := taskCtx.ID()+ hb := NewHeartbeat(taskCtx.ID(), taskCtx.Handle, e.heartbeatInterval).WithContext(ctx)+ hb.Start()+ e.executeMemoryTaskWithHeartbeat(ctx, taskCtx, hb)+}- // The admission path (processMessage) starts the heartbeat before queueing- // and attaches its stop function via SetStopLease. Direct-execution callers- // (unit tests, non-queued paths) carry no lease, so start a local heartbeat- // as a fallback. The two are mutually exclusive — never both running.- if taskCtx.StopLeaseFn() == nil && taskCtx.Handle != nil && e.heartbeatInterval > 0 {- hb := NewHeartbeat(taskID, taskCtx.Handle, e.heartbeatInterval).WithContext(e.ctx)- hb.Start()- taskCtx.SetStopLease(hb.Stop)- }+// executeMemoryTaskWithHeartbeat settles a memory handle with the heartbeat+// that the receiving worker started before admission.+func (e *Ingestor) executeMemoryTaskWithHeartbeat(ctx context.Context, taskCtx *taskpkg.TaskContext, hb *Heartbeat) {+ taskID := taskCtx.ID()var (settleAck boolsettleNack bool)defer func() {- // Stop the lease (and wait for any in-flight InProgress) before settling.- taskCtx.StopLease()+ hb.Stop()if r := recover(); r != nil {- // A panic is treated as a transient failure: Nack so the broker- // redelivers and retries the message.common.Error(fmt.Sprintf("memory task %s panicked: %v", taskID, r), fmt.Errorf("%v", r))settleNack = true}+ if taskCtx.Handle == nil || e.leaseAbandoned(hb) {+ e.releaseTask(taskID)+ return+ }if settleAck {if err := taskCtx.Handle.Ack(); err != nil {common.Error(fmt.Sprintf("ack memory task %s", taskID), err)@@ -672,6 +651,14 @@ func (e *Ingestor) defaultRunMemoryTask(ctx context.Context, taskID string, payl}func (e *Ingestor) executeTask(ctx context.Context, taskCtx *taskpkg.TaskContext) {+ hb := NewHeartbeat(taskCtx.ID(), taskCtx.Handle, e.heartbeatInterval).WithContext(taskCtx.Ctx)+ hb.Start()+ e.executeTaskWithHeartbeat(ctx, taskCtx, hb)+}++// executeTaskWithHeartbeat executes one document task using the lease already+// started by the worker that owns its Handle.+func (e *Ingestor) executeTaskWithHeartbeat(ctx context.Context, taskCtx *taskpkg.TaskContext, hb *Heartbeat) {task := taskCtx.IngestionTaskcommon.Info(fmt.Sprintf("Starting task %s", task.ID))@@ -703,7 +690,7 @@ func (e *Ingestor) executeTask(ctx context.Context, taskCtx *taskpkg.TaskContextperTaskCancel()}- e.settleMessage(ctx, taskCtx, func(ctx context.Context) bool {+ e.settleMessage(ctx, taskCtx, hb, func(ctx context.Context) bool {return e.runTask(ctx, task)})}@@ -757,6 +744,10 @@ func (e *Ingestor) markFailed(ctx context.Context, taskID string) bool {func (e *Ingestor) runTask(ctx context.Context, task *entity.IngestionTask) bool {select {case <-ctx.Done():+ if e.ctx.Err() != nil || e.dispatchCtx.Err() != nil {+ common.Info(fmt.Sprintf("Task %s aborted due to ingestor shutdown, leaving for redelivery", task.ID))+ return false+ }common.Info(fmt.Sprintf("Task %s cancelled", task.ID))e.markCancelProgress(task)stopped := e.markStopped(context.Background(), task.ID)@@ -823,6 +814,10 @@ func (e *Ingestor) runTask(ctx context.Context, task *entity.IngestionTask) boolif err := e.runDocumentTask(ctx, task); err != nil {if errors.Is(err, context.Canceled) {+ if e.ctx.Err() != nil || e.dispatchCtx.Err() != nil {+ common.Info(fmt.Sprintf("Task %s pipeline interrupted by ingestor shutdown, leaving for redelivery", task.ID))+ return false+ }common.Info(fmt.Sprintf("Task %s cancelled during pipeline", task.ID))e.markCancelProgress(task)stopped := e.markStopped(ctx, task.ID)@@ -918,28 +913,29 @@ func (e *Ingestor) settleToTerminal(ctx context.Context, taskID string) error {}}-// settleMessage runs body under a heartbeat, then settles the MQ message. The-// heartbeat is stopped (and waited on) before ack/nack — see Heartbeat.Stop.+// settleMessage runs body under an injected heartbeat lease, then settles the MQ message.+// The heartbeat is stopped (and waited on) before ack/nack — see Heartbeat.Stop.// A panic in body is recovered: the task is marked FAILED and the message is// Nacked for redelivery, so a single task's panic never crashes the worker.// Settlement queries the DB for the task's actual status: a terminal state// (COMPLETED/STOPPED/FAILED) means Ack; anything else means Nack. The body's// return value is advisory only — DB truth is authoritative (BP1).-func (e *Ingestor) settleMessage(ctx context.Context, taskCtx *taskpkg.TaskContext, body func(context.Context) bool) (terminal bool) {- hb := NewHeartbeat(taskCtx.ID(), taskCtx.Handle, e.heartbeatInterval).WithContext(taskCtx.Ctx)- hb.Start()+func (e *Ingestor) settleMessage(ctx context.Context, taskCtx *taskpkg.TaskContext, hb *Heartbeat, body func(context.Context) bool) (terminal bool) {defer func() {hb.Stop() // stop heartbeat (and wait) before ack/nackif r := recover(); r != nil {// Recover the panic so the worker process survives. Mark the// task FAILED so a redelivery does not re-run a poison message- // (processMessage Ack-skips an already-FAILED task); Nack for+ // (handleAndExecute Ack-skips an already-FAILED task); Nack for// redelivery. The broker's redelivery limit handles deterministic// poison messages.common.Error(fmt.Sprintf("task %s panicked: %v", taskCtx.IngestionTask.ID, r), fmt.Errorf("%v", r))e.markFailed(ctx, taskCtx.IngestionTask.ID)terminal = false}+ if e.leaseAbandoned(hb) {+ return+ }// Settlement authority is the DB, not the in-memory bool (BP1).// Fall back to the in-memory bool only when the DB is unavailable.if dbTerminal, ok := e.safeGetTerminal(ctx, taskCtx.IngestionTask.ID); ok {@@ -1093,8 +1089,7 @@ func (e *Ingestor) markTimeoutProgress(task *entity.IngestionTask) {// worker has already claimed it (e.g. MQ redelivery), true on first claim.// The claim is released by releaseTask when the worker finishes, so a future// redelivery (after restart) can re-claim the task. Broker lease renewal is a-// separate concern handled by Heartbeat (doc: settleMessage; memory: admission-// in processMessage + stop via TaskContext.StopLease).+// separate concern handled by the worker-owned Heartbeat until settlement.func (e *Ingestor) claimTask(taskID string) bool {e.tasksMu.Lock()defer e.tasksMu.Unlock()@@ -1171,19 +1166,16 @@ func (e *Ingestor) recordTerminalPipelineLog(ctx context.Context, ingestionTask}}-// Stop gracefully shuts down the ingestor. It cancels the root context so-// idle workers exit immediately and in-flight pipelines abort at their next-// ctx.Err() check, then waits for workers to return. The wait is bounded by-// ctx: a stage that does not honor cancellation (e.g. a native CGO parse)-// would otherwise block workerWg.Wait() indefinitely; when ctx expires Stop-// returns and leaves the broker to redeliver any in-flight messages-// (at-least-once). Callers must pass a deadline-bearing context.+// Stop first stops worker registration and the dispatcher's Pull, then cancels+// execution. The caller's deadline bounds the wait for non-cooperative tasks.func (e *Ingestor) Stop(ctx context.Context) {common.Info(fmt.Sprintf("Stopping ingestor %s", e.id))- e.cancel()+ e.dispatchCancel()waitDone := make(chan struct{})go func() {+ e.dispatcherWg.Wait()+ e.cancel()e.workerWg.Wait()e.compileWg.Wait()close(waitDone)@@ -1193,6 +1185,8 @@ func (e *Ingestor) Stop(ctx context.Context) {case <-waitDone:common.Info("All tasks completed")case <-ctx.Done():+ e.stopActiveLeases()+ e.cancel()e.tasksMu.RLock()ids := make([]string, 0, len(e.currentTasks))for id := range e.currentTasks {diff --git a/internal/ingestion/service/ingestor_lifecycle_test.go b/internal/ingestion/service/ingestor_lifecycle_test.goindex 1594cae941..3bf1accb5c 100644--- a/internal/ingestion/service/ingestor_lifecycle_test.go+++ b/internal/ingestion/service/ingestor_lifecycle_test.go@@ -18,6 +18,7 @@ package serviceimport ("context"+ "encoding/json""errors""strings""testing"@@ -28,6 +29,7 @@ import ("ragflow/internal/entity"taskpkg "ragflow/internal/ingestion/task""ragflow/internal/ingestion/testutil"+ servicepkg "ragflow/internal/service")type startupTaskPublisher struct {@@ -69,8 +71,11 @@ func TestStartWorkerPool_StartOnceIdempotent(t *testing.T) {t.Fatalf("activeWorkers after second startWorkerPool = %d, want %d (sync.Once not idempotent)", got, concurrency)}- ingestor.cancel()+ ingestor.dispatchCancel()ingestor.workerWg.Wait()+ if got := ingestor.activeWorkers.Load(); got != 0 {+ t.Fatalf("activeWorkers after worker shutdown = %d, want 0", got)+ }}// TestStop_GracefulShutdown verifies that Stop cancels the context and waits@@ -120,7 +125,7 @@ func TestStop_TimesOutWhenWorkerStuck(t *testing.T) {db := testutil.SetupTestDB(t)cleanup := testutil.ReplaceDBForTest(t, db)defer cleanup()- _, _, docID, taskID := testutil.SeedTestData(t, db, testutil.WithPipelineID("flow-1"))+ _, _, _, taskID := testutil.SeedTestData(t, db, testutil.WithPipelineID("flow-1"))const concurrency int32 = 1ingestor := newUnitIngestor("test-stuck", concurrency, []string{"pdf"})@@ -142,10 +147,8 @@ func TestStop_TimesOutWhenWorkerStuck(t *testing.T) {t.Fatalf("set task RUNNING: %v", err)}- taskCtx := taskpkg.NewTaskContextForScheduling(ingestor.ctx, &entity.IngestionTask{- ID: taskID, DocumentID: docID, DatasetID: "kb-1", Status: common.RUNNING,- })- ingestor.taskChan <- taskCtx+ w := <-ingestor.workerQueue+ w.inbox <- &fakeTaskHandle{msg: common.TaskMessage{TaskID: taskID, TaskType: common.TaskTypeIngestionTask}}select {case <-started:@@ -174,6 +177,103 @@ func TestStop_TimesOutWhenWorkerStuck(t *testing.T) {ingestor.workerWg.Wait()}+// TestStopDeadlineStopsStuckWorkerHeartbeat prevents a task that ignores+// execution cancellation from renewing its broker lease after graceful+// shutdown has timed out. Once Stop returns at its deadline, the unfinished+// handle must be left for broker redelivery instead of being kept alive by a+// leaked heartbeat.+func TestStopDeadlineStopsStuckWorkerHeartbeat(t *testing.T) {+ db := testutil.SetupTestDB(t)+ cleanup := testutil.ReplaceDBForTest(t, db)+ defer cleanup()+ _, _, _, taskID := testutil.SeedTestData(t, db, testutil.WithPipelineID("flow-1"))++ ingestor := newUnitIngestor("test-stuck-heartbeat", 1, []string{"pdf"})+ ingestor.heartbeatInterval = 10 * time.Millisecond+ ingestor.startWorkerPool()++ release := make(chan struct{})+ started := make(chan struct{})+ ingestor.runDocumentTask = func(context.Context, *entity.IngestionTask) error {+ close(started)+ <-release+ return nil+ }++ handle := &fakeTaskHandle{msg: common.TaskMessage{TaskID: taskID, TaskType: common.TaskTypeIngestionTask}}+ w := <-ingestor.workerQueue+ w.inbox <- handle+ select {+ case <-started:+ case <-time.After(2 * time.Second):+ t.Fatal("worker did not enter runDocumentTask")+ }++ deadline := time.Now().Add(time.Second)+ for handle.inProgress.Load() == 0 && time.Now().Before(deadline) {+ time.Sleep(time.Millisecond)+ }+ if handle.inProgress.Load() == 0 {+ t.Fatal("heartbeat did not renew the running handle")+ }++ stopCtx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond)+ ingestor.Stop(stopCtx)+ cancel()++ pulsesAtStop := handle.inProgress.Load()+ time.Sleep(50 * time.Millisecond)+ if got := handle.inProgress.Load(); got != pulsesAtStop {+ t.Fatalf("heartbeat renewals after Stop deadline = %d, want none", got-pulsesAtStop)+ }++ close(release)+ ingestor.workerWg.Wait()+ if handle.acks.Load() != 0 || handle.nacks.Load() != 0 {+ t.Fatalf("timed-out handle settlement = %d Ack / %d Nack, want none", handle.acks.Load(), handle.nacks.Load())+ }+}++// TestStopDeadlineLeavesStuckMemoryHandleUnsettled applies the same broker+// redelivery rule to memory extraction: a non-cooperative memory runner must+// not settle its old handle after the ingestor's Stop deadline passes.+func TestStopDeadlineLeavesStuckMemoryHandleUnsettled(t *testing.T) {+ ingestor := newUnitIngestor("test-stuck-memory", 1, nil)+ ingestor.memorySvc = &servicepkg.MemoryMessageService{}+ ingestor.startWorkerPool()++ release := make(chan struct{})+ started := make(chan struct{})+ ingestor.runMemoryTask = func(context.Context, string, map[string]any) error {+ close(started)+ <-release+ return nil+ }++ handle := &fakeTaskHandle{msg: common.TaskMessage{+ TaskID: "memory-stop-timeout",+ TaskType: common.TaskTypeMemory,+ Payload: []byte(`{}`),+ }}+ w := <-ingestor.workerQueue+ w.inbox <- handle+ select {+ case <-started:+ case <-time.After(2 * time.Second):+ t.Fatal("worker did not enter memory runner")+ }++ stopCtx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond)+ ingestor.Stop(stopCtx)+ cancel()++ close(release)+ ingestor.workerWg.Wait()+ if handle.acks.Load() != 0 || handle.nacks.Load() != 0 {+ t.Fatalf("timed-out memory handle settlement = %d Ack / %d Nack, want none", handle.acks.Load(), handle.nacks.Load())+ }+}+// TestPollCancel_ExitsWhenDoneClosed verifies that closing the done channel// causes pollCancel to return even when cancelCheck is blocked (e.g. on a// long DB query). Without BP3, the initial cancelCheck call runs@@ -232,6 +332,27 @@ func TestStartNilEngine(t *testing.T) {}}+// TestStartRetainsStartupFailure prevents a second Start call from+// reporting success after initialization failed on the first attempt.+func TestStartRetainsStartupFailure(t *testing.T) {+ previousEngine := engine.GetMessageQueueEngine()+ engine.SetMessageQueueEngine(nil)+ t.Cleanup(func() { engine.SetMessageQueueEngine(previousEngine) })++ ingestor := newUnitIngestor("test-startup-failure", 1, nil)+ t.Cleanup(func() { ingestor.Stop(context.Background()) })++ if err := ingestor.Start(); err == nil {+ t.Fatal("first Start unexpectedly succeeded with nil engine")+ }+ if err := ingestor.Start(); err == nil {+ t.Fatal("second Start hid the prior startup failure")+ }+ if got := ingestor.activeWorkers.Load(); got != 0 {+ t.Fatalf("active workers after failed Start = %d, want 0", got)+ }+}+// TestExecuteTask_MarkFailedAfterCtxCancelAcks verifies that a generic task// failure is persisted and acknowledged even after the task context is// cancelled by the pipeline.@@ -352,3 +473,73 @@ func TestStartSchedulesCreatedTasks(t *testing.T) {t.Fatalf("task status = %q, want %q", task.Status, common.SCHEDULED)}}++// TestWorkerDispatcherDoesNotActivateTaskUntilWorkerReceivesTask prevents a busy+// worker from prefetching its next task. task-2 must remain SCHEDULED while+// task-1 still occupies the only worker.+func TestWorkerDispatcherDoesNotActivateTaskUntilWorkerReceivesTask(t *testing.T) {+ db := testutil.SetupTestDB(t)+ cleanup := testutil.ReplaceDBForTest(t, db)+ t.Cleanup(cleanup)++ taskIDs := seedBurstTasks(t, db, 2)+ for _, taskID := range taskIDs {+ if err := db.Model(&entity.IngestionTask{}).Where("id = ?", taskID).+ Update("status", common.SCHEDULED).Error; err != nil {+ t.Fatalf("schedule task %s: %v", taskID, err)+ }+ }++ queue := testutil.SetupNatsEngine(t)+ previousEngine := engine.GetMessageQueueEngine()+ engine.SetMessageQueueEngine(queue)+ t.Cleanup(func() { engine.SetMessageQueueEngine(previousEngine) })++ ingestor := newUnitIngestor("test-worker-dispatch", 1, []string{"pdf"})+ releaseFirst := make(chan struct{})+ firstStarted := make(chan struct{})+ ingestor.runDocumentTask = func(_ context.Context, task *entity.IngestionTask) error {+ if task.ID == taskIDs[0] {+ close(firstStarted)+ <-releaseFirst+ }+ return nil+ }+ t.Cleanup(func() {+ close(releaseFirst)+ ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)+ defer cancel()+ ingestor.Stop(ctx)+ })++ for _, taskID := range taskIDs {+ payload, err := json.Marshal(common.TaskMessage{+ TaskID: taskID,+ TaskType: common.TaskTypeIngestionTask,+ })+ if err != nil {+ t.Fatalf("marshal task %s: %v", taskID, err)+ }+ if err := queue.PublishTask(common.TaskSubject, payload); err != nil {+ t.Fatalf("publish task %s: %v", taskID, err)+ }+ }++ if err := ingestor.Start(); err != nil {+ t.Fatalf("Start: %v", err)+ }+ select {+ case <-firstStarted:+ case <-time.After(5 * time.Second):+ t.Fatal("first task did not start")+ }++ time.Sleep(150 * time.Millisecond)+ var second entity.IngestionTask+ if err := db.Where("id = ?", taskIDs[1]).First(&second).Error; err != nil {+ t.Fatalf("load second task: %v", err)+ }+ if second.Status != common.SCHEDULED {+ t.Fatalf("second task status = %q, want %q while the only worker is busy", second.Status, common.SCHEDULED)+ }+}diff --git a/internal/ingestion/service/process_message_test.go b/internal/ingestion/service/process_message_test.godeleted file mode 100644index 0d5e5383c9..0000000000--- a/internal/ingestion/service/process_message_test.go+++ /dev/null@@ -1,765 +0,0 @@-package service--import (- "context"- "encoding/json"- "errors"- "testing"- "time"-- "ragflow/internal/common"- "ragflow/internal/dao"- "ragflow/internal/entity"- taskpkg "ragflow/internal/ingestion/task"- "ragflow/internal/ingestion/testutil"- "ragflow/internal/service"-)--func newFakeHandle(taskID, taskType string) *fakeTaskHandle {- return &fakeTaskHandle{msg: common.TaskMessage{TaskID: taskID, TaskType: taskType}}-}--// TestProcessMessage_MemoryTaskDispatches verifies that a task_type="memory"-// message (with a NATS payload) is dispatched to the shared worker pool as a-// TaskKindMemory context rather than being acked-skipped as a non-ingestion-// task. It must NOT touch the ingestion state machine (no StartRunning).-func TestProcessMessage_MemoryTaskDispatches(t *testing.T) {- db := testutil.SetupTestDB(t)- cleanup := testutil.ReplaceDBForTest(t, db)- defer cleanup()-- ingestor := newUnitIngestor("test", 1, []string{"pdf"})- // Memory extractor must be enabled for the memory branch to enqueue.- ingestor.SetMemoryMessageService(service.NewMemoryMessageService(nil))-- payload, err := json.Marshal(map[string]any{- "memory_id": "mem-1",- "source_id": 42,- "message_dict": map[string]any{- "user_id": "u1",- "agent_id": "a1",- "session_id": "s1",- "user_input": "hi",- "agent_response": "hello",- },- })- if err != nil {- t.Fatalf("marshal payload: %v", err)- }- handle := &fakeTaskHandle{msg: common.TaskMessage{TaskID: "mem-task-1", TaskType: common.TaskTypeMemory, Payload: payload}}-- ingestor.processMessage(handle)- if handle.acks.Load() != 0 || handle.nacks.Load() != 0 {- t.Fatalf("memory dispatch: expected 0 Ack/0 Nack (settled by worker), got acks=%d nacks=%d", handle.acks.Load(), handle.nacks.Load())- }- if len(ingestor.taskChan) != 1 {- t.Fatalf("expected 1 memory task enqueued, got %d", len(ingestor.taskChan))- }- taskCtx := <-ingestor.taskChan- if taskCtx.Kind != taskpkg.TaskKindMemory {- t.Fatalf("task kind = %v, want TaskKindMemory", taskCtx.Kind)- }- if taskCtx.IngestionTask != nil {- t.Fatalf("memory task must not carry an IngestionTask (no ingestion state machine), got %+v", taskCtx.IngestionTask)- }- if taskCtx.MemoryPayload == nil || taskCtx.MemoryPayload["memory_id"] != "mem-1" {- t.Fatalf("memory payload not carried correctly: %+v", taskCtx.MemoryPayload)- }-}--// TestProcessMessage_MemoryTaskDisabledAcks verifies that when the memory-// extractor is not installed (nil), a memory task is acked and skipped so it-// does not loop forever on the worker pool.-func TestProcessMessage_MemoryTaskDisabledAcks(t *testing.T) {- db := testutil.SetupTestDB(t)- cleanup := testutil.ReplaceDBForTest(t, db)- defer cleanup()-- ingestor := newUnitIngestor("test", 1, []string{"pdf"}) // memorySvc nil by default- handle := newFakeHandle("mem-task-2", common.TaskTypeMemory)-- ingestor.processMessage(handle)- if handle.acks.Load() != 1 || handle.nacks.Load() != 0 {- t.Fatalf("memory disabled: expected 1 Ack/0 Nack, got acks=%d nacks=%d", handle.acks.Load(), handle.nacks.Load())- }- if len(ingestor.taskChan) != 0 {- t.Fatal("expected no memory task enqueued when extractor disabled")- }-}--// TestExecuteMemoryTaskAlreadyFailedAcks verifies the Ack-on-failure contract in-// the case that matters: the task row is already persisted with progress=-1 (the-// durable "failed" marker written by HandleSaveToMemoryTask). In that state-// HandleSaveToMemoryTask returns "already failed", and executeMemoryTask must-// Ack — a Nack would redeliver the message into an infinite retry loop against-// an already-failed row. Using a real taskDAO + MemoryService (not a nil-guard)-// so the test exercises the "task already failed" branch, not the nil-guard.-func TestExecuteMemoryTaskAlreadyFailedAcks(t *testing.T) {- db := testutil.SetupTestDB(t)- cleanup := testutil.ReplaceDBForTest(t, db)- defer cleanup()-- // Seed a task row already marked failed (progress=-1), mirroring what- // HandleSaveToMemoryTask persists before returning an error.- taskID := "mem-task-3"- if err := dao.DB.Create(&entity.Task{ID: taskID, DocID: "doc-mem-3", Progress: -1}).Error; err != nil {- t.Fatalf("insert already-failed task: %v", err)- }-- ingestor := newUnitIngestor("test", 1, []string{"pdf"})- // Real memory service (non-nil memories) so HandleSaveToMemoryTask gets past- // the nil-guard and reaches the progress==-1 "already failed" branch.- ingestor.SetMemoryMessageService(service.NewMemoryMessageService(service.NewMemoryService()))-- handle := &fakeTaskHandle{msg: common.TaskMessage{TaskID: taskID, TaskType: common.TaskTypeMemory}}- taskCtx := taskpkg.NewMemoryTaskContextForScheduling(context.Background(), taskID, map[string]any{- "memory_id": "mem-3", "source_id": 7,- "message_dict": map[string]any{"user_id": "u", "agent_id": "a", "session_id": "s"},- }, handle)-- // Prove the failure precondition: the seeded progress=-1 row must make- // HandleSaveToMemoryTask return the "already failed" error. Otherwise the- // Ack below would only reflect the success path and prove nothing about- // the Ack-on-failure contract.- if err := ingestor.memorySvc.HandleSaveToMemoryTask(context.Background(), taskID, taskCtx.MemoryPayload); err == nil {- t.Fatal("expected HandleSaveToMemoryTask to fail on an already-failed (progress=-1) task, got nil")- }-- ingestor.executeMemoryTask(context.Background(), taskCtx)- if handle.acks.Load() != 1 || handle.nacks.Load() != 0 {- t.Fatalf("already-failed memory task: expected 1 Ack/0 Nack, got acks=%d nacks=%d", handle.acks.Load(), handle.nacks.Load())- }-}--// TestExecuteMemoryTaskTransientFailureNacks verifies that a transient (non-terminal)-// failure from HandleSaveToMemoryTask — e.g. a task-load DB error before any durable-// progress=-1 marker is written — is Nacked so the message is redelivered, instead of-// being Acked and permanently dropped. The tasks table is intentionally NOT migrated-// so taskDAO.GetByID fails with a "no such table" error rather than gorm.ErrRecordNotFound.-func TestExecuteMemoryTaskTransientFailureNacks(t *testing.T) {- // Migrate only a table unrelated to tasks so GetByID returns a transient- // "no such table: tasks" error (not gorm.ErrRecordNotFound).- db := testutil.SetupTestDB(t, &entity.IngestionTask{})- cleanup := testutil.ReplaceDBForTest(t, db)- defer cleanup()-- ingestor := newUnitIngestor("test", 1, []string{"pdf"})- ingestor.SetMemoryMessageService(service.NewMemoryMessageService(service.NewMemoryService()))-- handle := &fakeTaskHandle{msg: common.TaskMessage{TaskID: "mem-task-x", TaskType: common.TaskTypeMemory}}- taskCtx := taskpkg.NewMemoryTaskContextForScheduling(context.Background(), "mem-task-x", map[string]any{- "memory_id": "mem-x", "source_id": 1,- "message_dict": map[string]any{"user_id": "u", "agent_id": "a", "session_id": "s"},- }, handle)-- // Precondition: with no tasks table, HandleSaveToMemoryTask must fail with a- // transient error that is NOT wrapped in ErrMemoryTaskTerminal.- err := ingestor.memorySvc.HandleSaveToMemoryTask(context.Background(), "mem-task-x", taskCtx.MemoryPayload)- if err == nil {- t.Fatal("expected HandleSaveToMemoryTask to fail with missing tasks table, got nil")- }- if errors.Is(err, service.ErrMemoryTaskTerminal) {- t.Fatalf("expected a transient error, got terminal: %v", err)- }-- ingestor.executeMemoryTask(context.Background(), taskCtx)- if handle.nacks.Load() != 1 || handle.acks.Load() != 0 {- t.Fatalf("transient memory task failure: expected 0 Ack/1 Nack, got acks=%d nacks=%d", handle.acks.Load(), handle.nacks.Load())- }-}--// TestProcessMessage_NonIngestionTaskAcks: a non-ingestion task is acked and-// skipped without touching the task DB or enqueuing.-func TestProcessMessage_NonIngestionTaskAcks(t *testing.T) {- db := testutil.SetupTestDB(t)- cleanup := testutil.ReplaceDBForTest(t, db)- defer cleanup()-- ingestor := newUnitIngestor("test", 1, []string{"pdf"})- handle := newFakeHandle("task-1", "not-ingestion")-- ingestor.processMessage(handle)- if handle.acks.Load() != 1 || handle.nacks.Load() != 0 {- t.Fatalf("non-ingestion: expected 1 Ack/0 Nack, got acks=%d nacks=%d", handle.acks.Load(), handle.nacks.Load())- }- if len(ingestor.taskChan) != 0 {- t.Fatal("expected no task enqueued")- }-}--// TestProcessMessage_TaskNotFoundAcks: when StartRunning returns-// ErrTaskNotFound the message is acked and skipped.-func TestProcessMessage_TaskNotFoundAcks(t *testing.T) {- db := testutil.SetupTestDB(t)- cleanup := testutil.ReplaceDBForTest(t, db)- defer cleanup()-- ingestor := newUnitIngestor("test", 1, []string{"pdf"})- // No task seeded in DB — StartRunning returns ErrTaskNotFound.- handle := newFakeHandle("no-such-task", common.TaskTypeIngestionTask)-- ingestor.processMessage(handle)- if handle.acks.Load() != 1 || handle.nacks.Load() != 0 {- t.Fatalf("not-found: expected 1 Ack/0 Nack, got acks=%d nacks=%d", handle.acks.Load(), handle.nacks.Load())- }-}--// TestProcessMessage_CreatedTaskStartsRunning verifies that a worker can claim-// a task after NATS accepts its message but before the API records SCHEDULED.-func TestProcessMessage_CreatedTaskStartsRunning(t *testing.T) {- db := testutil.SetupTestDB(t)- cleanup := testutil.ReplaceDBForTest(t, db)- defer cleanup()- _, _, _, taskID := testutil.SeedTestData(t, db, testutil.WithPipelineID("flow-1"))-- if err := db.Model(&entity.IngestionTask{}).Where("id = ?", taskID).- Update("status", common.CREATED).Error; err != nil {- t.Fatalf("reset task to CREATED: %v", err)- }-- ingestor := newUnitIngestor("test", 1, []string{"pdf"})- handle := newFakeHandle(taskID, common.TaskTypeIngestionTask)-- ingestor.processMessage(handle)- if handle.acks.Load() != 0 || handle.nacks.Load() != 0 {- t.Fatalf("CREATED task: expected 0 Ack/0 Nack (deferred), got acks=%d nacks=%d", handle.acks.Load(), handle.nacks.Load())- }- if len(ingestor.taskChan) != 1 {- t.Fatalf("expected 1 task enqueued, got %d", len(ingestor.taskChan))- }-- var task entity.IngestionTask- if err := db.Where("id = ?", taskID).First(&task).Error; err != nil {- t.Fatalf("reload task: %v", err)- }- if task.Status != common.RUNNING {- t.Fatalf("status = %q, want %q", task.Status, common.RUNNING)- }-}--// TestProcessMessage_AlreadyCompletedAcks: a task already in a terminal state-// (COMPLETED) is acked and skipped — no enqueue, and the document is NOT-// resurrected to RUNNING. A redelivered terminal task must not reset a-// finished document's run status or counters.-func TestProcessMessage_AlreadyCompletedAcks(t *testing.T) {- db := testutil.SetupTestDB(t)- cleanup := testutil.ReplaceDBForTest(t, db)- defer cleanup()- _, _, docID, taskID := testutil.SeedTestData(t, db, testutil.WithPipelineID("flow-1"))-- // Mark the document as a finished parse: a non-RUNNING run status and- // non-zero counters that StartRunning would clobber to "1"/0.- finishedRun := "3"- if err := db.Model(&entity.Document{}).Where("id = ?", docID).- Updates(map[string]interface{}{- "run": finishedRun,- "chunk_num": 42,- "token_num": 99,- }).Error; err != nil {- t.Fatalf("set document finished sentinel: %v", err)- }-- // Set the task COMPLETED so the status switch skips it.- if err := db.Model(&entity.IngestionTask{}).Where("id = ?", taskID).- Update("status", common.COMPLETED).Error; err != nil {- t.Fatalf("set COMPLETED: %v", err)- }-- ingestor := newUnitIngestor("test", 1, []string{"pdf"})- handle := newFakeHandle(taskID, common.TaskTypeIngestionTask)-- ingestor.processMessage(handle)- if handle.acks.Load() != 1 || handle.nacks.Load() != 0 {- t.Fatalf("completed: expected 1 Ack/0 Nack, got acks=%d nacks=%d", handle.acks.Load(), handle.nacks.Load())- }- if len(ingestor.taskChan) != 0 {- t.Fatal("expected no task enqueued for completed task")- }-- // The finished document must be untouched - no resurrection to RUNNING.- var doc entity.Document- if err := db.Where("id = ?", docID).First(&doc).Error; err != nil {- t.Fatalf("reload document: %v", err)- }- if doc.Run == nil || *doc.Run != finishedRun {- t.Fatalf("document.run = %v, want %q (must not be resurrected to RUNNING)", doc.Run, finishedRun)- }- if doc.ChunkNum != 42 || doc.TokenNum != 99 {- t.Fatalf("document counters changed: chunk_num=%d token_num=%d, want 42/99", doc.ChunkNum, doc.TokenNum)- }-}--// TestProcessMessage_ClaimFailsRenewsLease: a RUNNING task whose claim fails-// is already owned by another worker. The duplicate must not Ack the broker-// message before that owner reaches a durable outcome; it only renews the-// delivery lease and stays out of the worker queue.-func TestProcessMessage_ClaimFailsRenewsLease(t *testing.T) {- db := testutil.SetupTestDB(t)- cleanup := testutil.ReplaceDBForTest(t, db)- defer cleanup()- _, _, _, taskID := testutil.SeedTestData(t, db, testutil.WithPipelineID("flow-1"))-- ingestor := newUnitIngestor("test", 1, []string{"pdf"})- // Pre-claim the task so processMessage sees a claim conflict.- ingestor.claimTask(taskID)-- handle := newFakeHandle(taskID, common.TaskTypeIngestionTask)-- ingestor.processMessage(handle)- if handle.acks.Load() != 0 || handle.nacks.Load() != 0 {- t.Fatalf("claim-fail: expected 0 Ack/0 Nack while owner runs, got acks=%d nacks=%d", handle.acks.Load(), handle.nacks.Load())- }- if handle.inProgress.Load() != 1 {- t.Fatalf("claim-fail: expected duplicate lease renewal, got InProgress=%d", handle.inProgress.Load())- }- if len(ingestor.taskChan) != 0 {- t.Fatal("expected no task enqueued when claim fails")- }-}--// TestProcessMessage_ClaimSucceedsEnqueues: a RUNNING task with a successful-// claim is enqueued to the worker pool and the message is NOT settled yet-// (ack/nack is deferred to the worker).-func TestProcessMessage_ClaimSucceedsEnqueues(t *testing.T) {- db := testutil.SetupTestDB(t)- cleanup := testutil.ReplaceDBForTest(t, db)- defer cleanup()- _, _, _, taskID := testutil.SeedTestData(t, db, testutil.WithPipelineID("flow-1"))-- ingestor := newUnitIngestor("test", 1, []string{"pdf"})- handle := newFakeHandle(taskID, common.TaskTypeIngestionTask)-- ingestor.processMessage(handle)- // Ack/Nack must not be called — settlement is deferred to the worker.- if handle.acks.Load() != 0 || handle.nacks.Load() != 0 {- t.Fatalf("enqueued: expected 0 Ack/0 Nack (deferred), got acks=%d nacks=%d", handle.acks.Load(), handle.nacks.Load())- }- // Task must be in the channel.- if len(ingestor.taskChan) != 1 {- t.Fatalf("expected 1 task enqueued, got %d", len(ingestor.taskChan))- }- // Drain and verify.- taskCtx := <-ingestor.taskChan- if taskCtx.IngestionTask.ID != taskID {- t.Fatalf("enqueued task ID = %s, want %s", taskCtx.IngestionTask.ID, taskID)- }-}--// TestProcessMessage_ChannelFullBlocksUntilSlot: backpressure must NOT drop-// the task. When the task channel is at capacity, processMessage blocks on the-// send (consuming no slot, no Nack) until a worker frees one; the message is-// then enqueued and settled by the worker. Dropping on backpressure would-// permanently lose the task once the broker's MaxDeliver is exceeded.-func TestProcessMessage_ChannelFullBlocksUntilSlot(t *testing.T) {- db := testutil.SetupTestDB(t)- cleanup := testutil.ReplaceDBForTest(t, db)- defer cleanup()- _, _, _, taskID := testutil.SeedTestData(t, db, testutil.WithPipelineID("flow-1"))-- // maxConcurrency=2 → channel cap=4. Fill it completely.- ingestor := newUnitIngestor("test", 2, []string{"pdf"})- for i := 0; i < cap(ingestor.taskChan); i++ {- ingestor.taskChan <- taskpkg.NewTaskContextForScheduling(nil, &entity.IngestionTask{ID: "filler"})- }-- handle := newFakeHandle(taskID, common.TaskTypeIngestionTask)-- // processMessage must BLOCK on the full channel: no ack, no nack, no- // immediate enqueue.- done := make(chan struct{})- go func() {- ingestor.processMessage(handle)- close(done)- }()- select {- case <-done:- t.Fatal("processMessage returned while channel was full; it must block under backpressure")- case <-time.After(100 * time.Millisecond):- }- if handle.nacks.Load() != 0 || handle.acks.Load() != 0 {- t.Fatalf("expected 0 Ack/0 Nack while blocked, got acks=%d nacks=%d", handle.acks.Load(), handle.nacks.Load())- }- if len(ingestor.taskChan) != cap(ingestor.taskChan) {- t.Fatalf("expected channel still full, got %d/%d", len(ingestor.taskChan), cap(ingestor.taskChan))- }-- // Free a slot: the blocked processMessage must now enqueue the task.- <-ingestor.taskChan- select {- case <-done:- case <-time.After(2 * time.Second):- t.Fatal("processMessage did not enqueue after a slot freed")- }- if handle.nacks.Load() != 0 || handle.acks.Load() != 0 {- t.Fatalf("expected 0 Ack/0 Nack (settlement deferred to worker), got acks=%d nacks=%d", handle.acks.Load(), handle.nacks.Load())- }- // The real task must now be in the channel.- found := false- for i := 0; i < cap(ingestor.taskChan); i++ {- tc := <-ingestor.taskChan- if tc.IngestionTask != nil && tc.IngestionTask.ID == taskID {- found = true- }- }- if !found {- t.Fatal("real task was not enqueued after a slot freed")- }-}--// TestProcessMessage_ShutdownRaceMarksStopped: shutdown winning the race-// against the taskChan send must not leave the task in non-terminal RUNNING-// with no in-flight worker. StartRunning has already flipped the task to-// RUNNING, so the ctx.Done branch finalizes it as STOPPED (user-retryable)-// with a detached timeout, and leaves the message unsettled — the broker-// redelivers it and the terminal status ack-skips.-func TestProcessMessage_ShutdownRaceMarksStopped(t *testing.T) {- db := testutil.SetupTestDB(t)- cleanup := testutil.ReplaceDBForTest(t, db)- defer cleanup()- _, _, _, taskID := testutil.SeedTestData(t, db, testutil.WithPipelineID("flow-1"))- // SeedTestData creates the task RUNNING; reset to CREATED so the race- // below is the real one: processMessage's StartRunning flips it RUNNING- // and then blocks on the full channel.- if err := db.Model(&entity.IngestionTask{}).Where("id = ?", taskID).- Update("status", common.CREATED).Error; err != nil {- t.Fatalf("reset task to CREATED: %v", err)- }-- ingestor := newUnitIngestor("test", 1, []string{"pdf"}) // channel cap = 2- for i := 0; i < cap(ingestor.taskChan); i++ {- ingestor.taskChan <- taskpkg.NewTaskContextForScheduling(nil, &entity.IngestionTask{ID: "filler"})- }-- handle := newFakeHandle(taskID, common.TaskTypeIngestionTask)- done := make(chan struct{})- go func() {- ingestor.processMessage(handle)- close(done)- }()-- // Wait until StartRunning flipped the task RUNNING and processMessage- // is blocked on the full channel.- deadline := time.Now().Add(2 * time.Second)- for {- var task entity.IngestionTask- if err := db.Where("id = ?", taskID).First(&task).Error; err == nil && task.Status == common.RUNNING {- break- }- if time.Now().After(deadline) {- t.Fatal("task never reached RUNNING; processMessage did not reach the blocking send")- }- time.Sleep(2 * time.Millisecond)- }-- // Shutdown wins the race against the channel send.- ingestor.cancel()-- select {- case <-done:- case <-time.After(5 * time.Second):- t.Fatal("processMessage did not return after shutdown")- }-- var task entity.IngestionTask- if err := db.Where("id = ?", taskID).First(&task).Error; err != nil {- t.Fatalf("reload task: %v", err)- }- if task.Status != common.STOPPED {- t.Fatalf("task status = %q, want %q (shutdown race must not strand a RUNNING row)", task.Status, common.STOPPED)- }- // Message unsettled: on redelivery the STOPPED status ack-skips it.- if handle.acks.Load() != 0 || handle.nacks.Load() != 0 {- t.Fatalf("expected 0 Ack/0 Nack (unsettled for redelivery), got acks=%d nacks=%d", handle.acks.Load(), handle.nacks.Load())- }-}--// TestProcessMessage_StartRunningErrorNacks: when StartRunning returns a-// non-ErrTaskNotFound error (e.g. a DB blip), processMessage nacks the-// message for redelivery instead of killing the consumer (B2 fix). The-// consume loop's resilience relies on this never being a fatal return.-func TestProcessMessage_StartRunningErrorNacks(t *testing.T) {- db := testutil.SetupTestDB(t)- cleanup := testutil.ReplaceDBForTest(t, db)- defer cleanup()- _, _, _, taskID := testutil.SeedTestData(t, db, testutil.WithPipelineID("flow-1"))-- // Drop the tasks table so GetTask fails with a generic SQL error, not- // ErrRecordNotFound (which would map to ErrTaskNotFound and ack-skip).- if err := db.Migrator().DropTable(&entity.IngestionTask{}); err != nil {- t.Fatalf("drop table: %v", err)- }-- ingestor := newUnitIngestor("test", 1, []string{"pdf"})- handle := newFakeHandle(taskID, common.TaskTypeIngestionTask)-- ingestor.processMessage(handle)-- if handle.nacks.Load() != 1 || handle.acks.Load() != 0 {- t.Fatalf("start-running error: expected 1 Nack/0 Ack (redeliver), got acks=%d nacks=%d", handle.acks.Load(), handle.nacks.Load())- }- if len(ingestor.taskChan) != 0 {- t.Fatal("expected no task enqueued on activation failure")- }-}--// TestProcessMessage_MemoryTaskHeartbeatsWhileQueued verifies that a claimed-// memory message renews its broker lease before a worker starts it. Without-// this, a task waiting in taskChan can exceed AckWait and consume unnecessary-// broker deliveries before a worker starts it.-func TestProcessMessage_MemoryTaskHeartbeatsWhileQueued(t *testing.T) {- ingestor := newUnitIngestor("test", 1, []string{"pdf"})- ingestor.SetMemoryMessageService(service.NewMemoryMessageService(nil))- ingestor.heartbeatInterval = 5 * time.Millisecond-- payload, err := json.Marshal(map[string]any{- "memory_id": "mem-1", "source_id": 42,- "message_dict": map[string]any{"user_id": "u", "agent_id": "a", "session_id": "s"},- })- if err != nil {- t.Fatalf("marshal payload: %v", err)- }- handle := &fakeTaskHandle{msg: common.TaskMessage{TaskID: "mem-queued-hb-1", TaskType: common.TaskTypeMemory, Payload: payload}}-- ingestor.processMessage(handle)-- deadline := time.Now().Add(2 * time.Second)- for handle.inProgress.Load() == 0 && time.Now().Before(deadline) {- time.Sleep(5 * time.Millisecond)- }- if handle.inProgress.Load() == 0 {- t.Fatal("expected InProgress heartbeat while memory task was queued without a worker")- }-- // Drain and stop the lease so the heartbeat goroutine does not leak.- taskCtx := <-ingestor.taskChan- taskCtx.StopLease()- if handle.acks.Load() != 0 || handle.nacks.Load() != 0 {- t.Fatalf("queued memory task must not be settled by admission, got acks=%d nacks=%d", handle.acks.Load(), handle.nacks.Load())- }-}--// TestProcessMessage_MemoryTaskRedeliveryRenewsLease verifies the claim guard on-// the memory dispatch path: when a memory task is redelivered by the broker-// while the first copy is still queued/in-flight, the duplicate must renew its-// lease without Acking it or enqueuing a second worker. A different task id-// must still be accepted (claim is per-task-id).-func TestProcessMessage_MemoryTaskRedeliveryRenewsLease(t *testing.T) {- ingestor := newUnitIngestor("test", 1, []string{"pdf"})- ingestor.SetMemoryMessageService(service.NewMemoryMessageService(nil))-- payload, err := json.Marshal(map[string]any{- "memory_id": "mem-1", "source_id": 42,- "message_dict": map[string]any{"user_id": "u", "agent_id": "a", "session_id": "s"},- })- if err != nil {- t.Fatalf("marshal payload: %v", err)- }-- // First delivery: claim succeeds, task is enqueued, message unsettled.- first := &fakeTaskHandle{msg: common.TaskMessage{TaskID: "mem-redeliver-1", TaskType: common.TaskTypeMemory, Payload: payload}}- ingestor.processMessage(first)- if first.acks.Load() != 0 || first.nacks.Load() != 0 {- t.Fatalf("first delivery: expected 0 Ack/0 Nack (settled by worker), got acks=%d nacks=%d", first.acks.Load(), first.nacks.Load())- }- if len(ingestor.taskChan) != 1 {- t.Fatalf("first delivery: expected 1 memory task enqueued, got %d", len(ingestor.taskChan))- }-- // Redelivery while the first copy is still queued: claim fails, the- // duplicate must renew its lease and NOT be enqueued.- dup := &fakeTaskHandle{msg: common.TaskMessage{TaskID: "mem-redeliver-1", TaskType: common.TaskTypeMemory, Payload: payload}}- ingestor.processMessage(dup)- if dup.acks.Load() != 0 || dup.nacks.Load() != 0 {- t.Fatalf("redelivered copy: expected 0 Ack/0 Nack while owner runs, got acks=%d nacks=%d", dup.acks.Load(), dup.nacks.Load())- }- if dup.inProgress.Load() != 1 {- t.Fatalf("redelivered copy: expected 1 lease renewal, got InProgress=%d", dup.inProgress.Load())- }- if len(ingestor.taskChan) != 1 {- t.Fatalf("redelivered copy must not be enqueued, got %d tasks in channel", len(ingestor.taskChan))- }-- // A different memory task must still be accepted (claim is per-task-id).- otherPayload, err := json.Marshal(map[string]any{- "memory_id": "mem-2", "source_id": 43,- "message_dict": map[string]any{"user_id": "u", "agent_id": "a", "session_id": "s"},- })- if err != nil {- t.Fatalf("marshal other payload: %v", err)- }- other := &fakeTaskHandle{msg: common.TaskMessage{TaskID: "mem-redeliver-2", TaskType: common.TaskTypeMemory, Payload: otherPayload}}- ingestor.processMessage(other)- if len(ingestor.taskChan) != 2 {- t.Fatalf("different memory task should be enqueued, got %d tasks in channel", len(ingestor.taskChan))- }-- // Drain both and stop leases so heartbeat goroutines do not leak.- firstCtx := <-ingestor.taskChan- firstCtx.StopLease()- secondCtx := <-ingestor.taskChan- secondCtx.StopLease()-}--// TestProcessMessage_MemoryTaskEmptyIDAcks verifies that a memory task with an-// empty envelope task id is Acked and skipped before claiming — an empty claim-// key would otherwise strand a no-op entry in currentTasks.-func TestProcessMessage_MemoryTaskEmptyIDAcks(t *testing.T) {- ingestor := newUnitIngestor("test", 1, []string{"pdf"})- ingestor.SetMemoryMessageService(service.NewMemoryMessageService(nil))-- payload, err := json.Marshal(map[string]any{- "memory_id": "mem-1", "source_id": 42,- "message_dict": map[string]any{"user_id": "u", "agent_id": "a", "session_id": "s"},- })- if err != nil {- t.Fatalf("marshal payload: %v", err)- }- handle := &fakeTaskHandle{msg: common.TaskMessage{TaskID: "", TaskType: common.TaskTypeMemory, Payload: payload}}-- ingestor.processMessage(handle)- if handle.acks.Load() != 1 || handle.nacks.Load() != 0 {- t.Fatalf("empty-id memory task: expected 1 Ack/0 Nack, got acks=%d nacks=%d", handle.acks.Load(), handle.nacks.Load())- }- if len(ingestor.taskChan) != 0 {- t.Fatalf("empty-id memory task must not be enqueued, got %d tasks", len(ingestor.taskChan))- }- if _, claimed := ingestor.currentTasks[""]; claimed {- t.Fatal("empty-id memory task must not claim the empty key")- }-}--// TestExecuteMemoryTask_ReleasesClaim verifies that executeMemoryTask releases-// the claim taken by processMessage once the worker finishes, so a later-// redelivery (e.g. after restart) can re-claim and re-run the task instead of-// being permanently ack-skipped.-func TestExecuteMemoryTask_ReleasesClaim(t *testing.T) {- ingestor := newUnitIngestor("test", 1, []string{"pdf"})- ingestor.SetMemoryMessageService(service.NewMemoryMessageService(nil))- // Stub the runner so no DB / LLM is touched.- ingestor.runMemoryTask = func(_ context.Context, _ string, _ map[string]any) error {- return nil- }-- if !ingestor.claimTask("mem-release-1") {- t.Fatal("expected claim to succeed")- }-- handle := &fakeTaskHandle{msg: common.TaskMessage{TaskID: "mem-release-1", TaskType: common.TaskTypeMemory}}- taskCtx := taskpkg.NewMemoryTaskContextForScheduling(context.Background(), "mem-release-1", map[string]any{- "memory_id": "mem-r", "source_id": 1,- "message_dict": map[string]any{"user_id": "u", "agent_id": "a", "session_id": "s"},- }, handle)-- ingestor.executeMemoryTask(context.Background(), taskCtx)-- if handle.acks.Load() != 1 || handle.nacks.Load() != 0 {- t.Fatalf("expected 1 Ack/0 Nack on success, got acks=%d nacks=%d", handle.acks.Load(), handle.nacks.Load())- }- if _, stillClaimed := ingestor.currentTasks["mem-release-1"]; stillClaimed {- t.Fatal("expected claim released after executeMemoryTask finished")- }- if !ingestor.claimTask("mem-release-1") {- t.Fatal("expected re-claim to succeed after release (future redelivery can re-run)")- }- ingestor.releaseTask("mem-release-1")-}--// TestExecuteMemoryTask_HeartbeatsInProgressDuringLongTask drives the real-// admission path (processMessage claim + lease) then executes the task: a-// long-running memory task (LLM extraction can take 10-65s) must renew the-// broker lease via InProgress, and settlement must stop the heartbeat before-// Acking (no Ack while an InProgress is in flight).-func TestExecuteMemoryTask_HeartbeatsInProgressDuringLongTask(t *testing.T) {- ingestor := newUnitIngestor("test", 1, []string{"pdf"})- ingestor.SetMemoryMessageService(service.NewMemoryMessageService(nil))- ingestor.heartbeatInterval = 5 * time.Millisecond-- started := make(chan struct{})- proceed := make(chan struct{})- ingestor.runMemoryTask = func(_ context.Context, _ string, _ map[string]any) error {- close(started) // admission heartbeat is running by now- <-proceed // simulate a long LLM extraction- return nil- }-- payload, err := json.Marshal(map[string]any{- "memory_id": "mem-h", "source_id": 1,- "message_dict": map[string]any{"user_id": "u", "agent_id": "a", "session_id": "s"},- })- if err != nil {- t.Fatalf("marshal payload: %v", err)- }- handle := &fakeTaskHandle{msg: common.TaskMessage{TaskID: "mem-hb-1", TaskType: common.TaskTypeMemory, Payload: payload}}-- // Admit through processMessage so the claim + lease are real, then drain- // and execute like a worker would. executeMemoryTask's deferred settlement- // (stop lease → Ack → release claim) runs before the goroutine exits, so- // waiting on done before reading currentTasks is race-free.- ingestor.processMessage(handle)- taskCtx := <-ingestor.taskChan- done := make(chan struct{})- go func() {- defer close(done)- ingestor.executeMemoryTask(context.Background(), taskCtx)- }()- <-started-- // Poll for heartbeats with a generous deadline so the test is resilient- // to slow CI schedulers.- heartbeatDeadline := time.Now().Add(2 * time.Second)- for handle.inProgress.Load() == 0 && time.Now().Before(heartbeatDeadline) {- time.Sleep(5 * time.Millisecond)- }- if handle.inProgress.Load() == 0 {- t.Fatal("expected InProgress heartbeats while runMemoryTask was blocked, got 0")- }-- close(proceed) // release the long task — only after confirming heartbeats-- select {- case <-done:- case <-time.After(5 * time.Second):- t.Fatal("executeMemoryTask did not return after the long task was released")- }- if handle.acks.Load() != 1 || handle.nacks.Load() != 0 {- t.Fatalf("expected 1 Ack/0 Nack on completion after heartbeat, got acks=%d nacks=%d inProgress=%d",- handle.acks.Load(), handle.nacks.Load(), handle.inProgress.Load())- }- if handle.wasSettledWithInProgress() {- t.Fatal("Ack ran while an InProgress was still in flight — heartbeat must stop before settlement")- }- if _, stillClaimed := ingestor.currentTasks["mem-hb-1"]; stillClaimed {- t.Fatal("expected claim released after executeMemoryTask finished")- }-}--// TestExecuteMemoryTaskAlreadyCompletedAcks verifies the idempotent-// short-circuit for a task row already extracted to completion (progress=1.0):-// a redelivery after restart (or a duplicate copy) must NOT re-run the LLM-// extraction — which would insert duplicate memory entries — but must Ack the-// message. The production runner short-circuits on the progress=1.0 row.-func TestExecuteMemoryTaskAlreadyCompletedAcks(t *testing.T) {- db := testutil.SetupTestDB(t)- cleanup := testutil.ReplaceDBForTest(t, db)- defer cleanup()-- taskID := "mem-completed-1"- if err := dao.DB.Create(&entity.Task{ID: taskID, DocID: "doc-mem-c", Progress: 1.0}).Error; err != nil {- t.Fatalf("insert already-completed task: %v", err)- }-- ingestor := newUnitIngestor("test", 1, []string{"pdf"})- // Real memory service so the production runner path is exercised.- ingestor.SetMemoryMessageService(service.NewMemoryMessageService(service.NewMemoryService()))-- handle := &fakeTaskHandle{msg: common.TaskMessage{TaskID: taskID, TaskType: common.TaskTypeMemory}}- taskCtx := taskpkg.NewMemoryTaskContextForScheduling(context.Background(), taskID, map[string]any{- "memory_id": "mem-c", "source_id": 7,- "message_dict": map[string]any{"user_id": "u", "agent_id": "a", "session_id": "s"},- }, handle)-- ingestor.executeMemoryTask(context.Background(), taskCtx)- if handle.acks.Load() != 1 || handle.nacks.Load() != 0 {- t.Fatalf("already-completed memory task: expected 1 Ack/0 Nack, got acks=%d nacks=%d", handle.acks.Load(), handle.nacks.Load())- }-}diff --git a/internal/ingestion/service/real_consumer_pipeline_test.go b/internal/ingestion/service/real_consumer_pipeline_test.goindex beed984c24..a51edf73ce 100644--- a/internal/ingestion/service/real_consumer_pipeline_test.go+++ b/internal/ingestion/service/real_consumer_pipeline_test.go@@ -6,6 +6,7 @@ import ("context""encoding/json""testing"+ "time""ragflow/internal/common""ragflow/internal/dao"@@ -21,7 +22,12 @@ func TestRealConsumer_PipelineMessageRoutesToExecuteTask(t *testing.T) {}for {- handles, _ := natsEngine.GetMessages(1)+ pullCtx, cancel := context.WithTimeout(t.Context(), time.Second)+ handles, err := natsEngine.PullMessages(pullCtx, 1)+ cancel()+ if err != nil {+ t.Fatalf("drain queue: %v", err)+ }if len(handles) == 0 {break}@@ -62,9 +68,11 @@ func TestRealConsumer_PipelineMessageRoutesToExecuteTask(t *testing.T) {t.Fatalf("PublishTask: %v", err)}- handles, err := natsEngine.GetMessages(1)+ pullCtx, cancel := context.WithTimeout(t.Context(), time.Second)+ defer cancel()+ handles, err := natsEngine.PullMessages(pullCtx, 1)if err != nil {- t.Fatalf("GetMessages: %v", err)+ t.Fatalf("PullMessages: %v", err)}if len(handles) != 1 {t.Fatalf("expected 1 message, got %d", len(handles))diff --git a/internal/ingestion/service/worker_dispatcher_burst_test.go b/internal/ingestion/service/worker_dispatcher_burst_test.gonew file mode 100644index 0000000000..87ebae996d--- /dev/null+++ b/internal/ingestion/service/worker_dispatcher_burst_test.go@@ -0,0 +1,129 @@+//+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.+//++package service++import (+ "context"+ "encoding/json"+ "fmt"+ "sync"+ "testing"+ "time"++ "ragflow/internal/common"+ "ragflow/internal/engine"+ "ragflow/internal/entity"+ "ragflow/internal/ingestion/testutil"+)++// TestWorkerDispatcherBurstCompletesEveryTask verifies that a burst stays in+// JetStream until a concrete worker is ready, rather than being Nacked+// repeatedly until MaxDeliver drops it.+func TestWorkerDispatcherBurstCompletesEveryTask(t *testing.T) {+ const (+ taskCount = 20+ concurrency = 1+ )++ db := testutil.SetupTestDB(t)+ cleanupDB := testutil.ReplaceDBForTest(t, db)+ defer cleanupDB()+ taskIDs := seedBurstTasks(t, db, taskCount)++ queue := testutil.SetupNatsEngine(t)+ previousQueue := engine.GetMessageQueueEngine()+ engine.SetMessageQueueEngine(queue)+ t.Cleanup(func() { engine.SetMessageQueueEngine(previousQueue) })++ ingestor := newUnitIngestor("test-worker-burst", concurrency, []string{"pdf"})+ firstTaskStarted := make(chan struct{})+ releaseFirstTask := make(chan struct{})+ var firstTask sync.Once+ var release sync.Once+ releaseFirst := func() {+ release.Do(func() { close(releaseFirstTask) })+ }+ t.Cleanup(releaseFirst)+ t.Cleanup(func() {+ ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)+ defer cancel()+ ingestor.Stop(ctx)+ })+ ingestor.runDocumentTask = func(context.Context, *entity.IngestionTask) error {+ firstTask.Do(func() {+ close(firstTaskStarted)+ <-releaseFirstTask+ })+ return nil+ }++ for _, taskID := range taskIDs {+ if err := db.Model(&entity.IngestionTask{}).Where("id = ?", taskID).+ Update("status", common.SCHEDULED).Error; err != nil {+ t.Fatalf("schedule task %s: %v", taskID, err)+ }+ payload, err := json.Marshal(common.TaskMessage{+ TaskID: taskID,+ TaskType: common.TaskTypeIngestionTask,+ })+ if err != nil {+ t.Fatalf("marshal task %s: %v", taskID, err)+ }+ if err := queue.PublishTask(common.TaskSubject, payload); err != nil {+ t.Fatalf("publish task %s: %v", taskID, err)+ }+ }++ if err := ingestor.Start(); err != nil {+ t.Fatalf("Start: %v", err)+ }+ select {+ case <-firstTaskStarted:+ case <-time.After(5 * time.Second):+ t.Fatal("first task did not start")+ }++ // The sole worker is busy, so the remaining 19 tasks must remain broker+ // pending. A buffered dispatcher can Nack this burst until MaxDeliver.+ releaseFirst()++ deadline := time.Now().Add(10 * time.Second)+ for time.Now().Before(deadline) {+ var completed int64+ if err := db.Model(&entity.IngestionTask{}).+ Where("id IN ? AND status = ?", taskIDs, common.COMPLETED).+ Count(&completed).Error; err != nil {+ t.Fatalf("count completed burst tasks: %v", err)+ }+ if completed == taskCount {+ return+ }+ time.Sleep(10 * time.Millisecond)+ }++ incomplete := make([]string, 0, taskCount)+ for _, taskID := range taskIDs {+ var task entity.IngestionTask+ if err := db.Where("id = ?", taskID).First(&task).Error; err != nil {+ t.Fatalf("load burst task %s: %v", taskID, err)+ }+ if task.Status != common.COMPLETED {+ incomplete = append(incomplete, fmt.Sprintf("%s=%s", taskID, task.Status))+ }+ }+ t.Fatalf("burst left %d/%d task(s) incomplete: %v", len(incomplete), taskCount, incomplete)+}diff --git a/internal/ingestion/service/worker_dispatcher_helpers_test.go b/internal/ingestion/service/worker_dispatcher_helpers_test.gonew file mode 100644index 0000000000..65026fafdb--- /dev/null+++ b/internal/ingestion/service/worker_dispatcher_helpers_test.go@@ -0,0 +1,61 @@+//+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.+//++package service++import (+ "fmt"+ "testing"++ "ragflow/internal/common"+ "ragflow/internal/entity"+ "ragflow/internal/ingestion/testutil"++ "gorm.io/gorm"+)++func seedBurstTasks(t *testing.T, db *gorm.DB, n int) []string {+ t.Helper()+ const tenantID, kbID = "burst-tenant", "burst-kb"+ if err := db.Create(&entity.Tenant{ID: tenantID, LLMID: "gpt-4", Status: testutil.StrPtr("1")}).Error; err != nil {+ t.Fatalf("create tenant: %v", err)+ }+ if err := db.Create(&entity.Knowledgebase{+ ID: kbID, TenantID: tenantID, EmbdID: "embd-1", Status: testutil.StrPtr("1"), ParserConfig: entity.JSONMap{},+ }).Error; err != nil {+ t.Fatalf("create knowledgebase: %v", err)+ }++ ids := make([]string, 0, n)+ for i := 0; i < n; i++ {+ docID := fmt.Sprintf("burst-doc-%d", i)+ taskID := fmt.Sprintf("burst-task-%d", i)+ location := "doc_store/" + docID+ if err := db.Create(&entity.Document{+ ID: docID, KbID: kbID, Name: &docID, ParserID: "naive", ParserConfig: entity.JSONMap{},+ PipelineID: testutil.StrPtr("flow-1"), Status: testutil.StrPtr("1"), Type: "pdf", Location: &location,+ }).Error; err != nil {+ t.Fatalf("create document %s: %v", docID, err)+ }+ if err := db.Create(&entity.IngestionTask{+ ID: taskID, UserID: "u1", DocumentID: docID, DatasetID: kbID, Status: common.CREATED,+ }).Error; err != nil {+ t.Fatalf("create ingestion task %s: %v", taskID, err)+ }+ ids = append(ids, taskID)+ }+ return ids+}diff --git a/internal/ingestion/service/worker_dispatcher_integration_test.go b/internal/ingestion/service/worker_dispatcher_integration_test.gonew file mode 100644index 0000000000..7eb44a4af2--- /dev/null+++ b/internal/ingestion/service/worker_dispatcher_integration_test.go@@ -0,0 +1,427 @@+//go:build integration++//+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.+//++package service++import (+ "context"+ "encoding/json"+ "net"+ "sync"+ "sync/atomic"+ "testing"+ "time"++ "ragflow/internal/common"+ "ragflow/internal/engine"+ natsengine "ragflow/internal/engine/nats"+ "ragflow/internal/entity"+ "ragflow/internal/ingestion/testutil"++ "github.com/nats-io/nats-server/v2/server"+)++func setupRealNatsCluster(t *testing.T) (host string, port int) {+ t.Helper()+ opts := &server.Options{+ Port: -1,+ JetStream: true,+ StoreDir: t.TempDir(),+ NoLog: true,+ NoSigs: true,+ }+ ns, err := server.NewServer(opts)+ if err != nil {+ t.Fatalf("create embedded NATS server: %v", err)+ }+ ns.Start()+ if !ns.ReadyForConnections(10 * time.Second) {+ ns.Shutdown()+ t.Fatal("embedded NATS server did not become ready within 10s")+ }+ t.Cleanup(func() {+ ns.Shutdown()+ ns.WaitForShutdown()+ })+ addr := ns.Addr().(*net.TCPAddr)+ return "127.0.0.1", addr.Port+}++// TestIntegration_MultiInstanceSharedConsumerNoOverExecution verifies TaskRP.md §6.2:+// Multiple ingestor instances sharing the same JetStream durable consumer (RAGFLOW_CONSUMER)+// compete for tasks without duplicate or overlapping execution while the broker is healthy.+func TestIntegration_MultiInstanceSharedConsumerNoOverExecution(t *testing.T) {+ db := testutil.SetupTestDB(t)+ cleanup := testutil.ReplaceDBForTest(t, db)+ defer cleanup()++ host, port := setupRealNatsCluster(t)++ // Engine 1 initializes stream and consumer+ engine1 := natsengine.NewNatsEngine(host, port)+ if err := engine1.Init(); err != nil {+ t.Fatalf("engine1.Init: %v", err)+ }+ if err := engine1.InitConsumer(common.TaskSubject); err != nil {+ t.Fatalf("engine1.InitConsumer: %v", err)+ }++ // Engine 2 connects to the same stream and consumer+ engine2 := natsengine.NewNatsEngine(host, port)+ if err := engine2.Init(); err != nil {+ t.Fatalf("engine2.Init: %v", err)+ }+ if err := engine2.InitConsumer(common.TaskSubject); err != nil {+ t.Fatalf("engine2.InitConsumer: %v", err)+ }++ const taskCount = 6+ taskIDs := seedBurstTasks(t, db, taskCount)+ for _, id := range taskIDs {+ if err := db.Model(&entity.IngestionTask{}).Where("id = ?", id).+ Update("status", common.SCHEDULED).Error; err != nil {+ t.Fatalf("schedule task %s: %v", id, err)+ }+ }++ var mu sync.Mutex+ executionCounts := make(map[string]int)+ var activeConcurrentRuns atomic.Int32+ var maxConcurrentObserved atomic.Int32++ runner := func(ctx context.Context, task *entity.IngestionTask) error {+ cur := activeConcurrentRuns.Add(1)+ defer activeConcurrentRuns.Add(-1)+ for {+ oldMax := maxConcurrentObserved.Load()+ if cur <= oldMax || maxConcurrentObserved.CompareAndSwap(oldMax, cur) {+ break+ }+ }++ mu.Lock()+ executionCounts[task.ID]+++ mu.Unlock()++ time.Sleep(30 * time.Millisecond)+ return nil+ }++ ingestor1 := newUnitIngestor("instance-1", 2, []string{"pdf"})+ ingestor1.runDocumentTask = runner++ ingestor2 := newUnitIngestor("instance-2", 2, []string{"pdf"})+ ingestor2.runDocumentTask = runner++ // Point global MQ engine to engine1 then start ingestor1, then engine2 and start ingestor2+ previousEngine := engine.GetMessageQueueEngine()+ t.Cleanup(func() { engine.SetMessageQueueEngine(previousEngine) })++ engine.SetMessageQueueEngine(engine1)+ if err := ingestor1.Start(); err != nil {+ t.Fatalf("ingestor1.Start: %v", err)+ }+ t.Cleanup(func() { ingestor1.Stop(context.Background()) })++ engine.SetMessageQueueEngine(engine2)+ if err := ingestor2.Start(); err != nil {+ t.Fatalf("ingestor2.Start: %v", err)+ }+ t.Cleanup(func() { ingestor2.Stop(context.Background()) })++ // Publish tasks+ for _, id := range taskIDs {+ payload, err := json.Marshal(common.TaskMessage{+ TaskID: id,+ TaskType: common.TaskTypeIngestionTask,+ })+ if err != nil {+ t.Fatalf("marshal task %s: %v", id, err)+ }+ if err := engine1.PublishTask(common.TaskSubject, payload); err != nil {+ t.Fatalf("publish task %s: %v", id, err)+ }+ }++ // Wait for all tasks to be completed in DB+ deadline := time.Now().Add(10 * time.Second)+ for time.Now().Before(deadline) {+ var completedCount int64+ db.Model(&entity.IngestionTask{}).Where("id IN ? AND status = ?", taskIDs, common.COMPLETED).Count(&completedCount)+ if int(completedCount) == taskCount {+ break+ }+ time.Sleep(50 * time.Millisecond)+ }++ mu.Lock()+ defer mu.Unlock()+ for _, id := range taskIDs {+ if count := executionCounts[id]; count != 1 {+ t.Fatalf("task %s execution count = %d, want 1 during healthy shared-consumer delivery", id, count)+ }+ }+ t.Logf("Multi-instance executed %d tasks cleanly. Max concurrency across instances = %d", taskCount, maxConcurrentObserved.Load())+}++// TestIntegration_SlowTaskHeartbeatPreventsPrematureRedelivery verifies TaskRP.md §6.2:+// Under the NATS consumer's BackOff schedule (first retry window = 5s), a task whose execution+// takes longer than 5s is NOT redelivered mid-flight because the worker's heartbeat calls InProgress.+func TestIntegration_SlowTaskHeartbeatPreventsPrematureRedelivery(t *testing.T) {+ db := testutil.SetupTestDB(t)+ cleanup := testutil.ReplaceDBForTest(t, db)+ defer cleanup()++ host, port := setupRealNatsCluster(t)+ mq := natsengine.NewNatsEngine(host, port)+ if err := mq.Init(); err != nil {+ t.Fatalf("mq.Init: %v", err)+ }+ if err := mq.InitConsumer(common.TaskSubject); err != nil {+ t.Fatalf("mq.InitConsumer: %v", err)+ }++ _, _, _, taskID := testutil.SeedTestData(t, db)+ if err := db.Model(&entity.IngestionTask{}).Where("id = ?", taskID).+ Update("status", common.SCHEDULED).Error; err != nil {+ t.Fatalf("schedule task %s: %v", taskID, err)+ }++ ingestor := newUnitIngestor("slow-task-ingestor", 1, []string{"pdf"})+ ingestor.heartbeatInterval = 1 * time.Second // comfortably below BackOff[0] = 5s++ taskExecutionStarted := make(chan struct{})+ var executions atomic.Int32++ ingestor.runDocumentTask = func(ctx context.Context, task *entity.IngestionTask) error {+ executions.Add(1)+ close(taskExecutionStarted)+ // Simulate a slow task that runs for 6 seconds (> 5s BackOff[0])+ select {+ case <-time.After(6 * time.Second):+ return nil+ case <-ctx.Done():+ return ctx.Err()+ }+ }++ previousEngine := engine.GetMessageQueueEngine()+ engine.SetMessageQueueEngine(mq)+ t.Cleanup(func() { engine.SetMessageQueueEngine(previousEngine) })++ if err := ingestor.Start(); err != nil {+ t.Fatalf("ingestor.Start: %v", err)+ }+ t.Cleanup(func() { ingestor.Stop(context.Background()) })++ payload, err := json.Marshal(common.TaskMessage{+ TaskID: taskID,+ TaskType: common.TaskTypeIngestionTask,+ })+ if err != nil {+ t.Fatalf("marshal task: %v", err)+ }+ if err := mq.PublishTask(common.TaskSubject, payload); err != nil {+ t.Fatalf("publish task: %v", err)+ }++ select {+ case <-taskExecutionStarted:+ case <-time.After(5 * time.Second):+ t.Fatal("task execution did not start")+ }++ // While task is still running at t = 5.5s (after BackOff[0] elapsed):+ // A manual pull on the same consumer must find NO messages available,+ // because the heartbeat extended the lease.+ time.Sleep(5500 * time.Millisecond)++ pullCtx, cancel := context.WithTimeout(t.Context(), time.Second)+ defer cancel()+ handles, err := mq.PullMessages(pullCtx, 1)+ if err != nil {+ t.Fatalf("PullMessages: %v", err)+ }+ if len(handles) != 0 {+ t.Fatalf("expected 0 redelivered messages while task is running with heartbeat, got %d", len(handles))+ }++ // Wait for task completion+ deadline := time.Now().Add(5 * time.Second)+ for time.Now().Before(deadline) {+ var task entity.IngestionTask+ if err := db.Where("id = ?", taskID).First(&task).Error; err == nil && task.Status == common.COMPLETED {+ break+ }+ time.Sleep(100 * time.Millisecond)+ }++ if executions.Load() != 1 {+ t.Fatalf("task executions = %d, want 1 (heartbeat should prevent second delivery)", executions.Load())+ }+}++// TestIntegration_ShutdownRedeliveryRecovery verifies TaskRP.md §6.2 item 4:+// After an ingestor shutdown times out under full load (SIGTERM simulation):+// - Already finished tasks are settled (Acked) and never redelivered;+// - In-flight uncompleted tasks have their leases abandoned (stopActiveLeases),+// and are redelivered by the broker to a successor ingestor and completed.+func TestIntegration_ShutdownRedeliveryRecovery(t *testing.T) {+ db := testutil.SetupTestDB(t)+ cleanup := testutil.ReplaceDBForTest(t, db)+ defer cleanup()++ host, port := setupRealNatsCluster(t)+ mq := natsengine.NewNatsEngine(host, port)+ if err := mq.Init(); err != nil {+ t.Fatalf("mq.Init: %v", err)+ }+ if err := mq.InitConsumer(common.TaskSubject); err != nil {+ t.Fatalf("mq.InitConsumer: %v", err)+ }++ previousEngine := engine.GetMessageQueueEngine()+ engine.SetMessageQueueEngine(mq)+ t.Cleanup(func() { engine.SetMessageQueueEngine(previousEngine) })++ // Seed two tasks: taskA (fast) and taskB (slow, will be in-flight during SIGTERM)+ taskIDs := seedBurstTasks(t, db, 2)+ taskIDA := taskIDs[0]+ taskIDB := taskIDs[1]++ for _, id := range []string{taskIDA, taskIDB} {+ if err := db.Model(&entity.IngestionTask{}).Where("id = ?", id).+ Update("status", common.SCHEDULED).Error; err != nil {+ t.Fatalf("schedule task %s: %v", id, err)+ }+ }++ ingestor1 := newUnitIngestor("shutdown-ingestor-1", 2, []string{"pdf"})+ ingestor1.heartbeatInterval = 500 * time.Millisecond++ taskBStarted := make(chan struct{})+ var (+ taskAExecutions atomic.Int32+ taskBExecutions atomic.Int32+ )++ ingestor1.runDocumentTask = func(ctx context.Context, task *entity.IngestionTask) error {+ if task.ID == taskIDA {+ taskAExecutions.Add(1)+ return nil+ }+ if task.ID == taskIDB {+ taskBExecutions.Add(1)+ close(taskBStarted)+ // Simulate long-running task interrupted by shutdown+ select {+ case <-time.After(30 * time.Second):+ return nil+ case <-ctx.Done():+ return ctx.Err()+ }+ }+ return nil+ }++ if err := ingestor1.Start(); err != nil {+ t.Fatalf("ingestor1.Start: %v", err)+ }++ // Publish taskA and taskB+ for _, id := range []string{taskIDA, taskIDB} {+ payload, err := json.Marshal(common.TaskMessage{+ TaskID: id,+ TaskType: common.TaskTypeIngestionTask,+ })+ if err != nil {+ t.Fatalf("marshal %s: %v", id, err)+ }+ if err := mq.PublishTask(common.TaskSubject, payload); err != nil {+ t.Fatalf("publish %s: %v", id, err)+ }+ }++ // Wait for taskB to start execution and taskA to complete+ <-taskBStarted+ deadlineA := time.Now().Add(5 * time.Second)+ for time.Now().Before(deadlineA) {+ var taskA entity.IngestionTask+ if err := db.Where("id = ?", taskIDA).First(&taskA).Error; err == nil && taskA.Status == common.COMPLETED {+ break+ }+ time.Sleep(50 * time.Millisecond)+ }++ // Simulate SIGTERM with 200ms graceful shutdown timeout (which will time out for taskB)+ stopCtx, stopCancel := context.WithTimeout(context.Background(), 200*time.Millisecond)+ defer stopCancel()+ ingestor1.Stop(stopCtx)++ // Ingestor1 timed out and cancelled taskB, calling stopActiveLeases().+ // Now start ingestor2 to recover un-acked tasks from the shared consumer.+ ingestor2 := newUnitIngestor("recovery-ingestor-2", 2, []string{"pdf"})+ ingestor2.heartbeatInterval = 500 * time.Millisecond++ var (+ taskAExec2 atomic.Int32+ taskBExec2 atomic.Int32+ )+ ingestor2.runDocumentTask = func(ctx context.Context, task *entity.IngestionTask) error {+ if task.ID == taskIDA {+ taskAExec2.Add(1)+ return nil+ }+ if task.ID == taskIDB {+ taskBExec2.Add(1)+ return nil+ }+ return nil+ }++ if err := ingestor2.Start(); err != nil {+ t.Fatalf("ingestor2.Start: %v", err)+ }+ defer ingestor2.Stop(context.Background())++ // Wait for taskB to be redelivered and completed by ingestor2+ deadlineB := time.Now().Add(12 * time.Second)+ taskBCompleted := false+ for time.Now().Before(deadlineB) {+ var taskB entity.IngestionTask+ if err := db.Where("id = ?", taskIDB).First(&taskB).Error; err == nil && taskB.Status == common.COMPLETED {+ taskBCompleted = true+ break+ }+ time.Sleep(100 * time.Millisecond)+ }++ if !taskBCompleted {+ t.Fatalf("taskB was not redelivered and completed after ingestor1 shutdown")+ }++ // Assertions:+ // 1. taskA was settled by ingestor1 and NEVER executed by ingestor2 (no duplicate delivery)+ if taskAExec2.Load() != 0 {+ t.Fatalf("taskA was redelivered to ingestor2 (%d times), but it was already completed and Acked", taskAExec2.Load())+ }+ // 2. taskB was executed once on ingestor2 to completion+ if taskBExec2.Load() != 1 {+ t.Fatalf("taskB executions on ingestor2 = %d, want 1", taskBExec2.Load())+ }+}diff --git a/internal/ingestion/service/worker_dispatcher_test.go b/internal/ingestion/service/worker_dispatcher_test.gonew file mode 100644index 0000000000..62dac42949--- /dev/null+++ b/internal/ingestion/service/worker_dispatcher_test.go@@ -0,0 +1,200 @@+//+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.+//++package service++import (+ "context"+ "sync"+ "testing"+ "time"++ "ragflow/internal/common"+ "ragflow/internal/engine"+)++type dispatcherPullResult struct {+ handle common.TaskHandle+ err error+ ready <-chan struct{}+ returned chan<- struct{}+}++type dispatcherTestQueue struct {+ engine.MessageQueue++ mu sync.Mutex+ pulls []dispatcherPullResult+ next int+ calls chan struct{}+}++func (q *dispatcherTestQueue) PullMessage(ctx context.Context) (common.TaskHandle, error) {+ q.mu.Lock()+ if q.next == len(q.pulls) {+ q.mu.Unlock()+ q.calls <- struct{}{}+ <-ctx.Done()+ return nil, ctx.Err()+ }+ pull := q.pulls[q.next]+ q.next+++ q.mu.Unlock()++ q.calls <- struct{}{}+ select {+ case <-pull.ready:+ if pull.returned != nil {+ close(pull.returned)+ }+ return pull.handle, pull.err+ case <-ctx.Done():+ return nil, ctx.Err()+ }+}++func closedChannel() <-chan struct{} {+ ready := make(chan struct{})+ close(ready)+ return ready+}++// TestWorkerDispatcherLimitsPendingPullsToOne ensures an empty pull does not+// cause the dispatcher to create another outstanding broker request.+func TestWorkerDispatcherLimitsPendingPullsToOne(t *testing.T) {+ firstReady := make(chan struct{})+ queue := &dispatcherTestQueue{+ pulls: []dispatcherPullResult{{ready: firstReady}, {}},+ calls: make(chan struct{}, 2),+ }+ previousQueue := engine.GetMessageQueueEngine()+ engine.SetMessageQueueEngine(queue)+ t.Cleanup(func() { engine.SetMessageQueueEngine(previousQueue) })++ ingestor := newUnitIngestor("test-single-pull", 2, nil)+ ingestor.dispatcherWg.Add(1)+ go ingestor.consumeLoop()+ t.Cleanup(func() {+ ingestor.dispatchCancel()+ ingestor.dispatcherWg.Wait()+ })++ ingestor.workerQueue <- &worker{id: 1, inbox: make(chan common.TaskHandle)}+ select {+ case <-queue.calls:+ case <-time.After(250 * time.Millisecond):+ t.Fatal("first idle worker did not start PullMessage")+ }++ ingestor.workerQueue <- &worker{id: 2, inbox: make(chan common.TaskHandle)}+ select {+ case <-queue.calls:+ t.Fatal("dispatcher started a second PullMessage before the first completed")+ case <-time.After(100 * time.Millisecond):+ }++ close(firstReady)+ select {+ case <-queue.calls:+ case <-time.After(250 * time.Millisecond):+ t.Fatal("dispatcher did not start the next pull after the empty result")+ }+}++func TestWorkerDispatcherHandsOffPulledMessageImmediately(t *testing.T) {+ ready := make(chan struct{})+ handle := &fakeTaskHandle{msg: common.TaskMessage{TaskID: "single-pull-message"}}+ queue := &dispatcherTestQueue{+ pulls: []dispatcherPullResult{{handle: handle, ready: ready}},+ calls: make(chan struct{}, 1),+ }+ previousQueue := engine.GetMessageQueueEngine()+ engine.SetMessageQueueEngine(queue)+ t.Cleanup(func() { engine.SetMessageQueueEngine(previousQueue) })++ ingestor := newUnitIngestor("test-single-message-handoff", 1, nil)+ worker := &worker{id: 1, inbox: make(chan common.TaskHandle)}+ ingestor.workerQueue <- worker+ ingestor.dispatcherWg.Add(1)+ go ingestor.consumeLoop()+ t.Cleanup(func() {+ ingestor.dispatchCancel()+ ingestor.dispatcherWg.Wait()+ })++ select {+ case <-queue.calls:+ case <-time.After(250 * time.Millisecond):+ t.Fatal("worker did not start PullMessages(1)")+ }++ close(ready)+ select {+ case received := <-worker.inbox:+ if received != handle {+ t.Fatalf("handed-off handle = %v, want pulled handle", received)+ }+ case <-time.After(250 * time.Millisecond):+ t.Fatal("pulled message was not handed off immediately")+ }+}++func TestWorkerDispatcherNacksReservedHandleOnShutdown(t *testing.T) {+ returned := make(chan struct{})+ handle := &fakeTaskHandle{msg: common.TaskMessage{TaskID: "reserved-on-stop"}}+ queue := &dispatcherTestQueue{+ pulls: []dispatcherPullResult{{handle: handle, ready: closedChannel(), returned: returned}},+ calls: make(chan struct{}, 1),+ }+ previousQueue := engine.GetMessageQueueEngine()+ engine.SetMessageQueueEngine(queue)+ t.Cleanup(func() { engine.SetMessageQueueEngine(previousQueue) })+ ingestor := newUnitIngestor("test-nack-reserved-handle", 1, nil)+ worker := &worker{id: 1, inbox: make(chan common.TaskHandle)}+ t.Cleanup(func() {+ ingestor.dispatchCancel()+ ingestor.dispatcherWg.Wait()+ })+ ingestor.workerQueue <- worker+ ingestor.dispatcherWg.Add(1)+ go ingestor.consumeLoop()++ select {+ case <-queue.calls:+ case <-time.After(250 * time.Millisecond):+ t.Fatal("pull did not start")+ }+ select {+ case <-returned:+ case <-time.After(250 * time.Millisecond):+ t.Fatal("pull did not reserve the handle")+ }++ ingestor.dispatchCancel()+ waitDone := make(chan struct{})+ go func() {+ ingestor.dispatcherWg.Wait()+ close(waitDone)+ }()+ select {+ case <-waitDone:+ case <-time.After(250 * time.Millisecond):+ t.Fatal("cancelled hand-off blocked dispatcher shutdown")+ }+ if handle.acks.Load() != 0 || handle.nacks.Load() != 1 {+ t.Fatalf("reserved handle settlement = %d Ack / %d Nack, want 0 Ack / 1 Nack", handle.acks.Load(), handle.nacks.Load())+ }+}diff --git a/internal/ingestion/task/real_consumer_test.go b/internal/ingestion/task/real_consumer_test.goindex baed6516e4..804770040a 100644--- a/internal/ingestion/task/real_consumer_test.go+++ b/internal/ingestion/task/real_consumer_test.go@@ -21,6 +21,7 @@ import ("context""encoding/json""testing"+ "time""ragflow/internal/common""ragflow/internal/dao"@@ -41,7 +42,12 @@ func TestRealProducerConsumer(t *testing.T) {// Purge stale messagesfor {- h, _ := natsEngine.GetMessages(1)+ pullCtx, cancel := context.WithTimeout(t.Context(), time.Second)+ h, err := natsEngine.PullMessages(pullCtx, 1)+ cancel()+ if err != nil {+ t.Fatalf("drain queue: %v", err)+ }if len(h) == 0 {break}@@ -83,9 +89,11 @@ func TestRealProducerConsumer(t *testing.T) {t.Logf("Producer: Published %s", payload)// ── 4. Consumer: Mirrors Ingestor.Start():131-189 exactly ──- handles, err := natsEngine.GetMessages(1)+ pullCtx, cancel := context.WithTimeout(t.Context(), time.Second)+ defer cancel()+ handles, err := natsEngine.PullMessages(pullCtx, 1)if err != nil {- t.Fatalf("GetMessages: %v", err)+ t.Fatalf("PullMessages: %v", err)}if len(handles) != 1 {t.Fatalf("expected 1 message, got %d", len(handles))diff --git a/internal/ingestion/task/task_context.go b/internal/ingestion/task/task_context.goindex 15f88d290d..d41dfcb1ec 100644--- a/internal/ingestion/task/task_context.go+++ b/internal/ingestion/task/task_context.go@@ -71,13 +71,6 @@ type TaskContext struct {// lives only here, never re-derived from IngestionTask or MemoryPayload.taskID string- // stopLease stops the broker-lease heartbeat started at admission- // (processMessage) for a queued/running memory task. It is set by the- // memory dispatch path before enqueueing and invoked by the worker's- // settlement defer before Ack/Nack. Doc (ingestion) tasks leave it nil —- // their heartbeat is started and stopped entirely inside settleMessage.- stopLease func()-// Handle is the message-queue ack handle for the task message that scheduled// this context. The scheduler sets it before queueing; the worker decides// the terminal Ack/Nack:@@ -118,34 +111,6 @@ func NewTaskContextForScheduling(ctx context.Context, task *entity.IngestionTask}}-// SetStopLease attaches the admission-started broker-lease stop function to a-// memory task context so the worker can stop the heartbeat before settlement.-// A nil stop is a no-op. Doc (ingestion) contexts never call this.-func (c *TaskContext) SetStopLease(stop func()) {- if c != nil {- c.stopLease = stop- }-}--// StopLease invokes the attached stop function, if any, blocking until no-// InProgress is in flight (Heartbeat.Stop semantics). It is safe to call with-// no attached lease (doc path or direct-execution test path).-func (c *TaskContext) StopLease() {- if c != nil && c.stopLease != nil {- c.stopLease()- }-}--// StopLeaseFn returns the attached stop function (nil when no admission-// heartbeat was started), so callers can detect whether a lease exists without-// invoking it.-func (c *TaskContext) StopLeaseFn() func() {- if c == nil {- return nil- }- return c.stopLease-}-// ID returns the task identifier for claim/release and logging. It is the// envelope TaskMessage.TaskID captured at construction — there is deliberately// no fallback to IngestionTask or MemoryPayload, so identity is neverdiff --git a/internal/server/config/ingestor_config.go b/internal/server/config/ingestor_config.goindex 7b3d33c029..9b5a2b8646 100644--- a/internal/server/config/ingestor_config.go+++ b/internal/server/config/ingestor_config.go@@ -15,12 +15,14 @@package config-import "github.com/spf13/viper"+import (+ "github.com/spf13/viper"+)type IngestorConfig struct {// MaxConcurrentWorkers bounds how many ingestion tasks the ingestor runs in- // parallel (the task channel width and dataset-level compile worker count- // default to this value). 0/negative falls back to runtime.NumCPU().+ // parallel (and dataset-level compile worker count defaults to this value).+ // 0/negative falls back to runtime.NumCPU().MaxConcurrentWorkers int `mapstructure:"max_concurrent_workers"`// CompilerPoolSize bounds the process-wide knowledge-compilation worker// pool that drives the cross-doc KNN / LLM-merge / write stages. 0/negativediff --git a/internal/server/config/ingestor_config_test.go b/internal/server/config/ingestor_config_test.gonew file mode 100644index 0000000000..8dad538c93--- /dev/null+++ b/internal/server/config/ingestor_config_test.go@@ -0,0 +1,42 @@+//+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.+//++package config++import (+ "testing"++ "github.com/spf13/viper"+)++func TestParseIngestorConfigReadsWorkerAndCompilerSettings(t *testing.T) {+ v := viper.New()+ v.Set("ingestor", map[string]any{+ "max_concurrent_workers": 4,+ "compiler_pool_size": 2,+ })++ config := &Config{}+ if err := config.ParseIngestorConfig(v); err != nil {+ t.Fatalf("ParseIngestorConfig: %v", err)+ }+ if got := config.GetIngestorConfig().MaxConcurrentWorkers; got != 4 {+ t.Fatalf("max concurrent workers = %d, want 4", got)+ }+ if got := config.GetIngestorConfig().CompilerPoolSize; got != 2 {+ t.Fatalf("compiler pool size = %d, want 2", got)+ }+}diff --git a/internal/service/document/document_rerun_e2e_test.go b/internal/service/document/document_rerun_e2e_test.goindex ab5c7e51da..e0ea6b3367 100644--- a/internal/service/document/document_rerun_e2e_test.go+++ b/internal/service/document/document_rerun_e2e_test.go@@ -3,7 +3,9 @@package documentimport (+ "context""testing"+ "time""ragflow/internal/common""ragflow/internal/engine"@@ -86,9 +88,11 @@ func TestRerunDocument_E2E_EnqueuesThroughRealMessageQueue(t *testing.T) {}// The enqueue actually landed on tasks.RAGFLOW.- handles, err := mq.GetMessages(1)+ pullCtx, cancel := context.WithTimeout(t.Context(), time.Second)+ defer cancel()+ handles, err := mq.PullMessages(pullCtx, 1)if err != nil {- t.Fatalf("GetMessages: %v", err)+ t.Fatalf("PullMessages: %v", err)}if len(handles) != 1 {t.Fatalf("expected 1 message on tasks.RAGFLOW, got %d", len(handles))diff --git a/internal/service/ingestion_task_service.go b/internal/service/ingestion_task_service.goindex 87e7b58017..8395578ed0 100644--- a/internal/service/ingestion_task_service.go+++ b/internal/service/ingestion_task_service.go@@ -509,7 +509,7 @@ func (s *IngestionTaskService) enqueueTask(taskID string) error {TaskID: taskID,TaskType: common.TaskTypeIngestionTask,}- return s.taskPublisher.PublishTaskMessage("tasks.RAGFLOW", taskMessage)+ return s.taskPublisher.PublishTaskMessage(common.TaskSubject, taskMessage)}// UpdateComponentTotal records the number of components in the task's DSLdiff --git a/internal/service/memory_extractor.go b/internal/service/memory_extractor.goindex fc63fa879d..d6e5d8735f 100644--- a/internal/service/memory_extractor.go+++ b/internal/service/memory_extractor.go@@ -25,7 +25,7 @@// QueueSaveToMemoryTask persists the raw message and publishes a// task_type="memory" TaskMessage on the NATS tasks.RAGFLOW subject. The// Ingestor's shared consumer + worker pool dispatches it by TaskType to-// HandleSaveToMemoryTask (see internal/ingestion/service/processMessage and+// HandleSaveToMemoryTask (see internal/ingestion/service/handleAndExecute and// executeMemoryTask), which runs LLM extraction for the non-raw memory types// configured on the memory and persists the extracted messages with source_id// pointing at the raw message so listMemoryMessages can aggregate them underdiff --git a/internal/service/memory_message_service.go b/internal/service/memory_message_service.goindex 201c712b82..f77b688a1e 100644--- a/internal/service/memory_message_service.go+++ b/internal/service/memory_message_service.go@@ -362,7 +362,7 @@ func queueMemoryTask(ctx context.Context, taskID, memoryID, tenantID string, raw}// Publish the memory-extraction task to NATS (tasks.RAGFLOW) so it is// consumed by the Ingestor's shared consumer + worker pool, dispatched by- // TaskType=="memory" in processMessage. This keeps Go out of the Python+ // TaskType=="memory" in handleAndExecute. This keeps Go out of the Python// te.*.common Redis stream entirely, removing the cross-consumer// contention that previously stole Python dataflow tasks.mq := engine.GetMessageQueueEngine()