Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

refactor(ingestion): use worker-driven task pull dispatcher (#19314)

jay777211ed2fc26f019
diff --git a/cmd/ragflow_server.go b/cmd/ragflow_server.go
index b3f2071090..641ca7cbdf 100644
--- a/cmd/ragflow_server.go
+++ b/cmd/ragflow_server.go
@@ -593,7 +593,7 @@ func runIngestor(ctx context.Context, cancel context.CancelFunc, args *serverArg
// Search and UpsertDoc can embed queries/summaries automatically.
nav.SetNavService(nlp.NewNavService(service.NewNavEmbedder(service.NewModelProviderService(), "")))
// Memory extraction runs on the Ingestor's shared NATS consumer + worker
- // pool (task_type="memory" dispatched by processMessage -> executeMemoryTask),
+ // pool (task_type="memory" dispatched by handleAndExecute -> executeMemoryTask),
// so there is no longer a dedicated Redis memory consumer to start.
ingestor.SetMemoryMessageService(service.NewMemoryMessageService(service.NewMemoryService()))
diff --git a/internal/admin/handler.go b/internal/admin/handler.go
index abf47d18cc..3393c779e9 100644
--- a/internal/admin/handler.go
+++ b/internal/admin/handler.go
@@ -17,6 +17,7 @@
package admin
import (
+ "context"
"encoding/json"
"errors"
"fmt"
@@ -923,7 +924,7 @@ func (h *Handler) PublishMessageToQueue(c *gin.Context) {
}
msgQueueEngine := engine.GetMessageQueueEngine()
- err = msgQueueEngine.PublishTask("tasks.RAGFLOW", taskMessageStr)
+ err = msgQueueEngine.PublishTask(common.TaskSubject, taskMessageStr)
if err != nil {
common.ErrorWithCode(c, common.CodeBadRequest, err.Error())
return
@@ -933,7 +934,7 @@ func (h *Handler) PublishMessageToQueue(c *gin.Context) {
}
type PullMessageFromQueueRequest struct {
- MessageCount int `json:"message_count" binding:"required"`
+ MessageCount int `json:"message_count" binding:"required,gt=0"`
AckPolicy string `json:"ack_policy" binding:"required"`
}
@@ -943,14 +944,20 @@ func (h *Handler) PullMessageFromQueue(c *gin.Context) {
common.ErrorWithCode(c, common.CodeBadRequest, fmt.Sprintf("Message count error: %s", err.Error()))
return
}
+ if req.MessageCount > common.MaxManualPullMessages {
+ common.ErrorWithCode(c, common.CodeBadRequest,
+ fmt.Sprintf("message count must be between 1 and %d", common.MaxManualPullMessages))
+ return
+ }
msgQueueEngine := engine.GetMessageQueueEngine()
- err := msgQueueEngine.InitConsumer("tasks.RAGFLOW")
- if err != nil {
+ if err := msgQueueEngine.InitConsumer(common.TaskSubject); err != nil {
common.ErrorWithCode(c, common.CodeBadRequest, err.Error())
return
}
- messages, err := msgQueueEngine.GetMessages(req.MessageCount)
+ pullCtx, cancel := context.WithTimeout(c.Request.Context(), time.Second)
+ defer cancel()
+ messages, err := msgQueueEngine.PullMessages(pullCtx, req.MessageCount)
if err != nil {
common.ErrorWithCode(c, common.CodeBadRequest, err.Error())
return
@@ -978,6 +985,7 @@ func (h *Handler) PullMessageFromQueue(c *gin.Context) {
"id": taskMessage.TaskID,
"type": taskMessage.TaskType,
}
+ err = message.Nack()
if err == nil {
resultMessage["nack"] = "true"
} else {
diff --git a/internal/admin/handler_queue_test.go b/internal/admin/handler_queue_test.go
new file mode 100644
index 0000000000..80a3a15266
--- /dev/null
+++ b/internal/admin/handler_queue_test.go
@@ -0,0 +1,118 @@
+//
+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+package admin
+
+import (
+ "bytes"
+ "encoding/json"
+ "fmt"
+ "net/http"
+ "net/http/httptest"
+ "testing"
+
+ "ragflow/internal/common"
+ "ragflow/internal/engine"
+ "ragflow/internal/ingestion/testutil"
+
+ "github.com/gin-gonic/gin"
+)
+
+// TestPullMessageFromQueueInitializesSharedConsumer preserves the admin
+// endpoint's ability to pull messages before the ingestor starts. The endpoint
+// must initialize the existing durable consumer rather than creating a
+// dispatcher-specific consumer.
+func TestPullMessageFromQueueInitializesSharedConsumer(t *testing.T) {
+ gin.SetMode(gin.TestMode)
+ queue := testutil.SetupNatsEngine(t)
+ previousQueue := engine.GetMessageQueueEngine()
+ engine.SetMessageQueueEngine(queue)
+ t.Cleanup(func() { engine.SetMessageQueueEngine(previousQueue) })
+
+ payload, err := json.Marshal(common.TaskMessage{
+ TaskID: "admin-pull-before-ingestor",
+ TaskType: common.TaskTypeIngestionTask,
+ })
+ if err != nil {
+ t.Fatalf("marshal task: %v", err)
+ }
+ if err := queue.PublishTask(common.TaskSubject, payload); err != nil {
+ t.Fatalf("publish task: %v", err)
+ }
+
+ recorder := httptest.NewRecorder()
+ ctx, _ := gin.CreateTestContext(recorder)
+ ctx.Request = httptest.NewRequest(
+ http.MethodPost,
+ "/",
+ bytes.NewBufferString(`{"message_count":1,"ack_policy":"ACK"}`),
+ )
+ ctx.Request.Header.Set("Content-Type", "application/json")
+
+ (&Handler{}).PullMessageFromQueue(ctx)
+
+ var response struct {
+ Code int `json:"code"`
+ Data []struct {
+ ID string `json:"id"`
+ Ack string `json:"ack"`
+ } `json:"data"`
+ }
+ if err := json.Unmarshal(recorder.Body.Bytes(), &response); err != nil {
+ t.Fatalf("decode response: %v", err)
+ }
+ if response.Code != int(common.CodeSuccess) {
+ t.Fatalf("response code = %d, want %d; body = %s", response.Code, common.CodeSuccess, recorder.Body.String())
+ }
+ if len(response.Data) != 1 || response.Data[0].ID != "admin-pull-before-ingestor" || response.Data[0].Ack != "true" {
+ t.Fatalf("pull response = %+v, want acked admin-pull-before-ingestor", response.Data)
+ }
+}
+
+func TestPullMessageFromQueueRejectsOutOfRangeMessageCount(t *testing.T) {
+ gin.SetMode(gin.TestMode)
+ for _, testCase := range []struct {
+ name string
+ messageCount int
+ }{
+ {name: "negative", messageCount: -1},
+ {name: "zero", messageCount: 0},
+ {name: "above limit", messageCount: 101},
+ } {
+ t.Run(testCase.name, func(t *testing.T) {
+ recorder := httptest.NewRecorder()
+ ctx, _ := gin.CreateTestContext(recorder)
+ ctx.Request = httptest.NewRequest(
+ http.MethodPost,
+ "/",
+ bytes.NewBufferString(fmt.Sprintf(`{"message_count":%d,"ack_policy":"ACK"}`, testCase.messageCount)),
+ )
+ ctx.Request.Header.Set("Content-Type", "application/json")
+
+ (&Handler{}).PullMessageFromQueue(ctx)
+
+ var response struct {
+ Code int `json:"code"`
+ }
+ if err := json.Unmarshal(recorder.Body.Bytes(), &response); err != nil {
+ t.Fatalf("decode response: %v", err)
+ }
+ if response.Code != int(common.CodeBadRequest) {
+ t.Fatalf("response code = %d, want %d; body = %s", response.Code, common.CodeBadRequest, recorder.Body.String())
+ }
+ })
+ }
+}
diff --git a/internal/cli/admin_parser.go b/internal/cli/admin_parser.go
index f3a6d18054..ca93a8c409 100644
--- a/internal/cli/admin_parser.go
+++ b/internal/cli/admin_parser.go
@@ -2714,8 +2714,8 @@ func (p *Parser) parseMessageQueueCommand() (*Command, error) {
p.nextToken() // consume NUMBER
}
- if messageCount <= 0 || messageCount > 100 {
- return nil, fmt.Errorf("message count cannot be less than 0 or greater than 100")
+ if messageCount <= 0 || messageCount > common.MaxManualPullMessages {
+ return nil, fmt.Errorf("message count must be between 1 and %d", common.MaxManualPullMessages)
}
cmd = NewCommand("user_pull_message_command")
diff --git a/internal/common/task.go b/internal/common/task.go
index d6b1237e9e..c8c2447cde 100644
--- a/internal/common/task.go
+++ b/internal/common/task.go
@@ -24,6 +24,9 @@ const (
// symbol so the routing contract cannot diverge (mirrors the RAGFLOW_TASKS
// JetStream subject in internal/engine/nats).
TaskSubject = "tasks.RAGFLOW"
+ // MaxManualPullMessages is the largest task batch the administrative queue
+ // pull endpoint accepts.
+ MaxManualPullMessages = 100
TaskTypeIngestionTask = "ingestion_task"
TaskTypeIngestionTest = "ingestion_test"
@@ -31,7 +34,7 @@ const (
TaskTypeSyncer = "syncer"
// TaskTypeMemory is the async memory-extraction task type. Memory tasks
// share the tasks.RAGFLOW subject and the Ingestor's consumer + worker
- // pool with ingestion tasks; processMessage dispatches them by TaskType.
+ // pool with ingestion tasks; handleAndExecute dispatches them by TaskType.
// The memory-specific payload (message_dict/memory_id/source_id) is
// carried in TaskMessage.Payload.
TaskTypeMemory = "memory"
diff --git a/internal/engine/engine.go b/internal/engine/engine.go
index a23fa1dfb5..9b1af9b44c 100644
--- a/internal/engine/engine.go
+++ b/internal/engine/engine.go
@@ -113,7 +113,8 @@ type MessageQueue interface {
Type() string
InitConsumer(subject string) error
PublishTask(subject string, payload []byte) error
- GetMessages(messageCount int) ([]common.TaskHandle, error)
+ PullMessages(ctx context.Context, messageCount int) ([]common.TaskHandle, error)
+ PullMessage(ctx context.Context) (common.TaskHandle, error)
ListMessages(messageType string, pending bool) ([]map[string]string, error)
ShowMessageQueue() (map[string]string, error)
CheckStatus() string
diff --git a/internal/engine/global.go b/internal/engine/global.go
index 1058d7b085..d701d4457c 100644
--- a/internal/engine/global.go
+++ b/internal/engine/global.go
@@ -111,7 +111,10 @@ func InitMessageQueue() error {
switch messageQueueType {
case "nats":
natsConfig := globalConfig.GetNATSConfig()
- messageQueueEngine = nats.NewNatsEngine(natsConfig.Host, natsConfig.Port)
+ messageQueueEngine = nats.NewNatsEngine(
+ natsConfig.Host,
+ natsConfig.Port,
+ )
err := messageQueueEngine.Init()
if err != nil {
return err
diff --git a/internal/engine/nats/nats.go b/internal/engine/nats/nats.go
index 7fceec5727..2657e4cc48 100644
--- a/internal/engine/nats/nats.go
+++ b/internal/engine/nats/nats.go
@@ -121,7 +121,7 @@ func (n *NatsEngine) PublishTask(subject string, payload []byte) error {
// Duplicate delivery is instead made safe at the consumer level:
// StartRunning's CREATED/SCHEDULED→RUNNING CAS plus the in-process claim guard
// prevent a second copy from executing while the first owner is active (see
- // Ingestor.processMessage).
+ // Ingestor.handleAndExecute).
ack, err := n.jetStream.Publish(ctx, subject, payload)
if err != nil {
return err
@@ -256,22 +256,54 @@ func (n *NatsEngine) InitConsumer(subject string) error {
}
return nil
}
-func (n *NatsEngine) GetMessages(messageCount int) ([]common.TaskHandle, error) {
+
+// PullMessages fetches up to messageCount messages before ctx expires.
+func (n *NatsEngine) PullMessages(ctx context.Context, messageCount int) ([]common.TaskHandle, error) {
+ if messageCount < 1 || messageCount > common.MaxManualPullMessages {
+ return nil, fmt.Errorf("message count must be between 1 and %d", common.MaxManualPullMessages)
+ }
if n.consumer == nil {
return nil, errors.New("NATS consumer is nil, engine not properly initialized")
}
+ if _, ok := ctx.Deadline(); !ok {
+ return nil, errors.New("pull messages context must have a deadline")
+ }
- resultMessages := make([]common.TaskHandle, 0)
- messages, err := n.consumer.Fetch(messageCount, jetstream.FetchMaxWait(1*time.Second))
+ resultMessages := make([]common.TaskHandle, 0, messageCount)
+ messages, err := n.consumer.Fetch(messageCount, jetstream.FetchContext(ctx))
if err != nil {
return nil, fmt.Errorf("failed to fetch messages: %w", err)
}
- for msg := range messages.Messages() {
- resultMessages = append(resultMessages, NewNatsMessageHandle(msg))
+ for message := range messages.Messages() {
+ resultMessages = append(resultMessages, NewNatsMessageHandle(message))
+ }
+ if batchErr := messages.Error(); batchErr != nil {
+ if errors.Is(batchErr, context.DeadlineExceeded) {
+ return resultMessages, nil
+ }
+ for _, message := range resultMessages {
+ if nackErr := message.Nack(); nackErr != nil {
+ common.Error("nack message after failed pull", nackErr)
+ }
+ }
+ return nil, fmt.Errorf("failed to fetch messages: %w", batchErr)
}
return resultMessages, nil
}
+// PullMessage returns one task handle from PullMessages. A nil handle
+// with a nil error means the pull expired without an available task.
+func (n *NatsEngine) PullMessage(ctx context.Context) (common.TaskHandle, error) {
+ messages, err := n.PullMessages(ctx, 1)
+ if err != nil {
+ return nil, err
+ }
+ if len(messages) == 0 {
+ return nil, nil
+ }
+ return messages[0], nil
+}
+
func (n *NatsEngine) CheckStatus() string {
if n.nc == nil {
return "NATS connection is nil, engine not properly initialized"
diff --git a/internal/engine/nats/nats_uninitialized_test.go b/internal/engine/nats/nats_uninitialized_test.go
index 2aade2f5a3..5533fc64b2 100644
--- a/internal/engine/nats/nats_uninitialized_test.go
+++ b/internal/engine/nats/nats_uninitialized_test.go
@@ -19,8 +19,8 @@ func TestUninitializedEngineMethodsReturnErrors(t *testing.T) {
t.Fatalf("PublishTask on uninitialized engine: err = %v, want 'not properly initialized'", err)
}
- if _, err := e.GetMessages(1); err == nil || !strings.Contains(err.Error(), "not properly initialized") {
- t.Fatalf("GetMessages on uninitialized engine: err = %v, want 'not properly initialized'", err)
+ if _, err := e.PullMessages(t.Context(), 1); err == nil || !strings.Contains(err.Error(), "not properly initialized") {
+ t.Fatalf("PullMessages on uninitialized engine: err = %v, want 'not properly initialized'", err)
}
if _, err := e.ShowMessageQueue(); err == nil || !strings.Contains(err.Error(), "not properly initialized") {
diff --git a/internal/engine/nats/pull_messages_test.go b/internal/engine/nats/pull_messages_test.go
new file mode 100644
index 0000000000..56f41b74ad
--- /dev/null
+++ b/internal/engine/nats/pull_messages_test.go
@@ -0,0 +1,289 @@
+//
+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+package nats
+
+import (
+ "context"
+ "encoding/json"
+ "errors"
+ "strings"
+ "testing"
+ "time"
+
+ "ragflow/internal/common"
+
+ "github.com/nats-io/nats.go/jetstream"
+)
+
+func newTaskQueue(t *testing.T) *NatsEngine {
+ t.Helper()
+ host, port := newEmbeddedNatsServer(t)
+ queue := NewNatsEngine(host, port)
+ if err := queue.Init(); err != nil {
+ t.Fatalf("Init: %v", err)
+ }
+ if err := queue.InitConsumer(common.TaskSubject); err != nil {
+ t.Fatalf("InitConsumer: %v", err)
+ }
+ return queue
+}
+
+func taskPullContext(t *testing.T) (context.Context, context.CancelFunc) {
+ t.Helper()
+ return context.WithTimeout(t.Context(), time.Second)
+}
+
+func publishTask(t *testing.T, queue *NatsEngine, taskID string) {
+ t.Helper()
+ payload, err := json.Marshal(common.TaskMessage{
+ TaskID: taskID,
+ TaskType: common.TaskTypeIngestionTask,
+ })
+ if err != nil {
+ t.Fatalf("marshal task: %v", err)
+ }
+ if err := queue.PublishTask(common.TaskSubject, payload); err != nil {
+ t.Fatalf("publish task: %v", err)
+ }
+}
+
+func TestPullMessageFetchesOneMessage(t *testing.T) {
+ queue := newTaskQueue(t)
+ publishTask(t, queue, "stream-one")
+ publishTask(t, queue, "stream-two")
+
+ ctx, cancel := taskPullContext(t)
+ defer cancel()
+ handle, err := queue.PullMessage(ctx)
+ if err != nil {
+ t.Fatalf("PullMessage: %v", err)
+ }
+ if handle == nil || handle.GetMessage().TaskID != "stream-one" {
+ t.Fatalf("stream handle = %v, want stream-one", handle)
+ }
+
+ handles, err := queue.PullMessages(ctx, 1)
+ if err != nil {
+ t.Fatalf("PullMessages: %v", err)
+ }
+ if len(handles) != 1 || handles[0].GetMessage().TaskID != "stream-two" {
+ t.Fatalf("remaining messages = %+v, want stream-two", handles)
+ }
+}
+
+func TestPullMessageRequiresDeadline(t *testing.T) {
+ queue := newTaskQueue(t)
+ _, err := queue.PullMessage(t.Context())
+ if err == nil {
+ t.Fatal("PullMessage without a deadline succeeded")
+ }
+ if !strings.Contains(err.Error(), "deadline") {
+ t.Fatalf("PullMessage error = %v, want deadline error", err)
+ }
+}
+
+func TestPullMessageReturnsNilForEmptyQueue(t *testing.T) {
+ queue := newTaskQueue(t)
+ ctx, cancel := context.WithTimeout(t.Context(), 100*time.Millisecond)
+ defer cancel()
+
+ handle, err := queue.PullMessage(ctx)
+ if err != nil {
+ t.Fatalf("PullMessage: %v", err)
+ }
+ if handle != nil {
+ t.Fatalf("empty pull handle = %v, want nil", handle)
+ }
+}
+
+func TestPullMessageReturnsCancellation(t *testing.T) {
+ queue := newTaskQueue(t)
+ ctx, cancel := taskPullContext(t)
+ cancel()
+
+ _, err := queue.PullMessage(ctx)
+ if !errors.Is(err, context.Canceled) {
+ t.Fatalf("PullMessage error = %v, want cancellation", err)
+ }
+}
+
+func TestPullMessageReportsMaxWaiting(t *testing.T) {
+ queue := newTaskQueue(t)
+ ctx, cancel := taskPullContext(t)
+ defer cancel()
+ if err := queue.stream.DeleteConsumer(ctx, "RAGFLOW_CONSUMER"); err != nil {
+ t.Fatalf("delete default consumer: %v", err)
+ }
+ consumer, err := queue.stream.CreateOrUpdateConsumer(ctx, jetstream.ConsumerConfig{
+ Name: "RAGFLOW_CONSUMER",
+ AckPolicy: jetstream.AckExplicitPolicy,
+ FilterSubject: "tasks.>",
+ MaxWaiting: 1,
+ })
+ if err != nil {
+ t.Fatalf("create limited consumer: %v", err)
+ }
+ queue.consumer = consumer
+
+ firstCtx, cancelFirst := taskPullContext(t)
+ defer cancelFirst()
+ firstResult := make(chan error, 1)
+ go func() {
+ _, err := queue.PullMessage(firstCtx)
+ firstResult <- err
+ }()
+
+ deadline := time.Now().Add(time.Second)
+ for time.Now().Before(deadline) {
+ info, err := queue.consumer.Info(t.Context())
+ if err != nil {
+ t.Fatalf("consumer info: %v", err)
+ }
+ if info.NumWaiting == 1 {
+ break
+ }
+ time.Sleep(10 * time.Millisecond)
+ }
+ info, err := queue.consumer.Info(t.Context())
+ if err != nil {
+ t.Fatalf("consumer info: %v", err)
+ }
+ if info.NumWaiting != 1 {
+ t.Fatalf("waiting pulls = %d, want 1", info.NumWaiting)
+ }
+
+ secondCtx, cancelSecond := taskPullContext(t)
+ defer cancelSecond()
+ _, err = queue.PullMessage(secondCtx)
+ if err == nil || errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) {
+ t.Fatalf("MaxWaiting error = %v, want capacity rejection", err)
+ }
+
+ cancelFirst()
+ select {
+ case <-firstResult:
+ case <-time.After(250 * time.Millisecond):
+ t.Fatal("first pending pull did not exit after cancellation")
+ }
+}
+
+func TestPullMessagesFetchesMessages(t *testing.T) {
+ queue := newTaskQueue(t)
+ publishTask(t, queue, "admin-direct-pull-1")
+ publishTask(t, queue, "admin-direct-pull-2")
+
+ ctx, cancel := taskPullContext(t)
+ defer cancel()
+ handles, err := queue.PullMessages(ctx, 2)
+ if err != nil {
+ t.Fatalf("PullMessages: %v", err)
+ }
+ if len(handles) != 2 {
+ t.Fatalf("len(handles) = %d, want 2", len(handles))
+ }
+ for index, taskID := range []string{"admin-direct-pull-1", "admin-direct-pull-2"} {
+ if got := handles[index].GetMessage().TaskID; got != taskID {
+ t.Fatalf("task id = %s, want %s", got, taskID)
+ }
+ }
+}
+
+func TestPullMessagesReturnsPartialBatchOnDeadline(t *testing.T) {
+ queue := newTaskQueue(t)
+ publishTask(t, queue, "partial-batch")
+
+ ctx, cancel := context.WithTimeout(t.Context(), 100*time.Millisecond)
+ defer cancel()
+ handles, err := queue.PullMessages(ctx, 2)
+ if err != nil {
+ t.Fatalf("PullMessages: %v", err)
+ }
+ if len(handles) != 1 || handles[0].GetMessage().TaskID != "partial-batch" {
+ t.Fatalf("partial handles = %+v, want partial-batch", handles)
+ }
+}
+
+func TestPullMessagesRejectsOutOfRangeMessageCount(t *testing.T) {
+ queue := newTaskQueue(t)
+ ctx, cancel := taskPullContext(t)
+ defer cancel()
+
+ for _, testCase := range []struct {
+ name string
+ messageCount int
+ }{
+ {name: "zero", messageCount: 0},
+ {name: "negative", messageCount: -1},
+ {name: "above limit", messageCount: 101},
+ } {
+ t.Run(testCase.name, func(t *testing.T) {
+ _, err := queue.PullMessages(ctx, testCase.messageCount)
+ if err == nil {
+ t.Fatal("PullMessages succeeded for an out-of-range message count")
+ }
+ })
+ }
+}
+
+func TestPullMessagesReportsBatchError(t *testing.T) {
+ queue := newTaskQueue(t)
+ ctx, cancel := taskPullContext(t)
+ defer cancel()
+ if err := queue.stream.DeleteConsumer(ctx, "RAGFLOW_CONSUMER"); err != nil {
+ t.Fatalf("delete default consumer: %v", err)
+ }
+ consumer, err := queue.stream.CreateOrUpdateConsumer(ctx, jetstream.ConsumerConfig{
+ Name: "RAGFLOW_CONSUMER",
+ AckPolicy: jetstream.AckExplicitPolicy,
+ FilterSubject: "tasks.>",
+ MaxWaiting: 1,
+ })
+ if err != nil {
+ t.Fatalf("create limited consumer: %v", err)
+ }
+ queue.consumer = consumer
+
+ occupiedCtx, cancelOccupied := taskPullContext(t)
+ defer cancelOccupied()
+ occupied := make(chan error, 1)
+ go func() {
+ _, err := queue.PullMessage(occupiedCtx)
+ occupied <- err
+ }()
+
+ deadline := time.Now().Add(time.Second)
+ for time.Now().Before(deadline) {
+ info, err := queue.consumer.Info(t.Context())
+ if err != nil {
+ t.Fatalf("consumer info: %v", err)
+ }
+ if info.NumWaiting == 1 {
+ break
+ }
+ time.Sleep(10 * time.Millisecond)
+ }
+
+ if _, err := queue.PullMessages(ctx, 1); err == nil {
+ t.Fatal("PullMessages succeeded after the consumer rejected its pull")
+ }
+ cancelOccupied()
+ select {
+ case <-occupied:
+ case <-time.After(250 * time.Millisecond):
+ t.Fatal("occupied pull did not exit after cancellation")
+ }
+}
diff --git a/internal/ingestion/service/burst_backpressure_test.go b/internal/ingestion/service/burst_backpressure_test.go
deleted file mode 100644
index 3728e1fd77..0000000000
--- a/internal/ingestion/service/burst_backpressure_test.go
+++ /dev/null
@@ -1,258 +0,0 @@
-//
-// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-//
-//
-// This test reproduces the "20+ files selected, only some parse, the rest
-// get stuck forever" report against the Go ingestor (canvas / pipeline path).
-//
-// Root cause it guards against: processMessage applies backpressure by
-// NACKing the message when taskChan is full. The NATS consumer is configured
-// with MaxDeliver: 16 and NO dead-letter subject (internal/engine/nats/nats.go),
-// so a message NACKed 16 times is permanently dropped. By that point
-// StartRunning has already flipped the task (and its document) to RUNNING in
-// the DB, and the ingestor has no scan-and-re-enqueue path on completion — so
-// the dropped document is stuck in RUNNING forever.
-//
-// The test drives the REAL processMessage + REAL worker pool, but substitutes
-// a deterministic "broker" that mirrors NATS' contract: bounded redelivery
-// (MaxDeliver) with no dead-letter. A slow worker models a saturated pipeline
-// (e.g. LLM calls in a canvas). The invariant under test is that NO task is
-// ever lost: all N delivered tasks eventually reach COMPLETED.
-//
-// Under the buggy (NACK-on-backpressure) code, the burst overflow is dropped
-// after MaxDeliver and the test FAILS. After the fix (blocking send so the
-// consume loop applies backpressure instead of dropping), every task is
-// processed and the test PASSES.
-
-package service
-
-import (
- "context"
- "fmt"
- "sync"
- "testing"
- "time"
-
- "ragflow/internal/common"
- "ragflow/internal/entity"
- taskpkg "ragflow/internal/ingestion/task"
- "ragflow/internal/ingestion/testutil"
-
- "gorm.io/gorm"
-)
-
-const (
- burstTaskCount = 20
- burstMaxDeliver = 16 // mirrors NATS MaxDeliver in internal/engine/nats/nats.go
- burstWorkerMs = 50 // slow worker models a saturated pipeline
-)
-
-// burstMsg models one in-flight broker message: the handle processMessage
-// settles, plus how many times it has been delivered (1 = initial delivery).
-type burstMsg struct {
- handle *fakeTaskHandle
- deliveries int
-}
-
-// TestProcessMessage_BurstNoTaskLossUnderBackpressure drives a 20-file burst
-// through the real processMessage + worker pool with a simulated NATS broker
-// (bounded redelivery, no dead-letter). It asserts that every task completes
-// and none are dropped.
-func TestProcessMessage_BurstNoTaskLossUnderBackpressure(t *testing.T) {
- db := testutil.SetupTestDB(t)
- cleanup := testutil.ReplaceDBForTest(t, db)
- defer cleanup()
-
- taskIDs := seedBurstTasks(t, db, burstTaskCount)
-
- ingestor := newUnitIngestor("test", 1, []string{"pdf"})
- // Slow worker: model a saturated pipeline so the channel is full during
- // the burst, forcing the backpressure path.
- ingestor.runDocumentTask = func(ctx context.Context, _ *entity.IngestionTask) error {
- time.Sleep(burstWorkerMs * time.Millisecond)
- return nil
- }
- ingestor.startWorkerPool()
- defer ingestor.Stop(context.Background())
-
- // Simulated NATS broker: pending holds messages waiting to be delivered.
- var mu sync.Mutex
- var pending []*burstMsg
- dropped := 0
-
- seedBroker := func() {
- mu.Lock()
- defer mu.Unlock()
- for _, id := range taskIDs {
- pending = append(pending, &burstMsg{
- handle: newFakeHandle(id, common.TaskTypeIngestionTask),
- deliveries: 1,
- })
- }
- }
- seedBroker()
-
- // Simulated consume loop: mirrors consumeLoop -> GetMessages(4) ->
- // processMessage, with NATS-style bounded redelivery on Nack.
- stop := make(chan struct{})
- var wg sync.WaitGroup
- wg.Add(1)
- go func() {
- defer wg.Done()
- for {
- select {
- case <-stop:
- return
- default:
- }
- mu.Lock()
- if len(pending) == 0 {
- mu.Unlock()
- time.Sleep(5 * time.Millisecond) // idle backoff
- continue
- }
- batchSize := 4
- if len(pending) < batchSize {
- batchSize = len(pending)
- }
- batch := pending[:batchSize]
- pending = pending[batchSize:]
- mu.Unlock()
-
- for _, bm := range batch {
- ingestor.processMessage(bm.handle)
- if bm.handle.nacks.Load() > 0 {
- mu.Lock()
- bm.deliveries++
- if bm.deliveries < burstMaxDeliver {
- // Redeliver (no delay — worst case for loss).
- pending = append(pending, bm)
- } else {
- // NATS drops after MaxDeliver with no dead-letter:
- // the task is permanently lost.
- dropped++
- }
- mu.Unlock()
- }
- }
- }
- }()
-
- // Wait until all tasks reach COMPLETED, or time out.
- deadline := time.Now().Add(30 * time.Second)
- completed := 0
- for time.Now().Before(deadline) {
- completed = countTasksWithStatus(t, db, taskIDs, common.COMPLETED)
- if completed == burstTaskCount {
- break
- }
- time.Sleep(50 * time.Millisecond)
- }
- close(stop)
- wg.Wait()
-
- if dropped > 0 {
- t.Fatalf("BUG: %d/%d task(s) were permanently dropped (lost) under burst backpressure "+
- "with MaxDeliver=%d and no dead-letter; only %d/%d completed. "+
- "This is the 'files get stuck after the first few parse' defect.",
- dropped, burstTaskCount, burstMaxDeliver, completed, burstTaskCount)
- }
- if completed != burstTaskCount {
- t.Fatalf("expected all %d tasks to complete, got %d (stuck in RUNNING)",
- burstTaskCount, completed)
- }
-}
-
-// TestFetchBudget_NeverExceedsChannelCapacity: the consume loop must never
-// pull more messages than it can enqueue immediately. The fetch budget is
-// min(free channel slots, maxConcurrency) with a floor of 1 so a fully
-// saturated channel still polls (and lets blocked sends drain) instead of
-// stalling until AckWait expires on in-flight messages.
-func TestFetchBudget_NeverExceedsChannelCapacity(t *testing.T) {
- ingestor := newUnitIngestor("test", 2, []string{"pdf"}) // channel cap = 4
- defer ingestor.Stop(context.Background())
-
- if got := ingestor.fetchBudget(); got != 2 {
- t.Fatalf("empty channel: fetchBudget = %d, want 2 (capped by maxConcurrency)", got)
- }
-
- for i := 0; i < cap(ingestor.taskChan)-1; i++ {
- ingestor.taskChan <- taskpkg.NewTaskContextForScheduling(t.Context(), &entity.IngestionTask{ID: "filler"})
- }
- if got := ingestor.fetchBudget(); got != 1 {
- t.Fatalf("3/4-full channel: fetchBudget = %d, want 1 (one free slot)", got)
- }
-
- ingestor.taskChan <- taskpkg.NewTaskContextForScheduling(t.Context(), &entity.IngestionTask{ID: "filler"})
- if got := ingestor.fetchBudget(); got != 1 {
- t.Fatalf("full channel: fetchBudget = %d, want 1 (floor: must never fetch 0)", got)
- }
-
- // A wide pool is bounded by the channel, not the worker count.
- wide := newUnitIngestor("test-wide", 8, []string{"pdf"}) // channel cap = 16
- defer wide.Stop(context.Background())
- if got := wide.fetchBudget(); got != 8 {
- t.Fatalf("wide empty channel: fetchBudget = %d, want 8 (capped by maxConcurrency)", got)
- }
-}
-
-// seedBurstTasks creates one tenant/kb plus N (document, ingestion_task) pairs
-// in CREATED status, mirroring how CreateAndEnqueue publishes a message for a
-// freshly-created task. Returns the ingestion task IDs.
-func seedBurstTasks(t *testing.T, db *gorm.DB, n int) []string {
- t.Helper()
- const tenantID, kbID = "burst-tenant", "burst-kb"
- if err := db.Create(&entity.Tenant{ID: tenantID, LLMID: "gpt-4", Status: testutil.StrPtr("1")}).Error; err != nil {
- t.Fatalf("create tenant: %v", err)
- }
- if err := db.Create(&entity.Knowledgebase{
- ID: kbID, TenantID: tenantID, EmbdID: "embd-1", Status: testutil.StrPtr("1"), ParserConfig: entity.JSONMap{},
- }).Error; err != nil {
- t.Fatalf("create kb: %v", err)
- }
-
- ids := make([]string, 0, n)
- for i := 0; i < n; i++ {
- docID := fmt.Sprintf("burst-doc-%d", i)
- taskID := fmt.Sprintf("burst-task-%d", i)
- loc := "doc_store/" + docID
- if err := db.Create(&entity.Document{
- ID: docID, KbID: kbID, Name: &docID, ParserID: "naive",
- ParserConfig: entity.JSONMap{}, PipelineID: testutil.StrPtr("flow-1"),
- Status: testutil.StrPtr("1"), Type: "pdf", Location: &loc,
- }).Error; err != nil {
- t.Fatalf("create doc %s: %v", docID, err)
- }
- if err := db.Create(&entity.IngestionTask{
- ID: taskID, UserID: "u1", DocumentID: docID, DatasetID: kbID, Status: common.CREATED,
- }).Error; err != nil {
- t.Fatalf("create ingestion task %s: %v", taskID, err)
- }
- ids = append(ids, taskID)
- }
- return ids
-}
-
-// countTasksWithStatus returns how many of the given task IDs have the given status.
-func countTasksWithStatus(t *testing.T, db *gorm.DB, ids []string, status string) int {
- t.Helper()
- var count int64
- if err := db.Model(&entity.IngestionTask{}).
- Where("id IN ? AND status = ?", ids, status).
- Count(&count).Error; err != nil {
- t.Fatalf("count tasks: %v", err)
- }
- return int(count)
-}
diff --git a/internal/ingestion/service/execute_memory_task_panic_test.go b/internal/ingestion/service/execute_memory_task_panic_test.go
index ecb21e6806..45cb9ac82d 100644
--- a/internal/ingestion/service/execute_memory_task_panic_test.go
+++ b/internal/ingestion/service/execute_memory_task_panic_test.go
@@ -3,7 +3,6 @@ package service
import (
"context"
"testing"
- "time"
"ragflow/internal/common"
"ragflow/internal/entity"
@@ -62,23 +61,15 @@ func TestExecuteMemoryTask_PanicDoesNotPropagate(t *testing.T) {
}
}
-// TestWorkerLoop_SurvivesMemoryPanicAndKeepsServing proves the production
-// failure mode end-to-end: a panicking memory task must NOT kill the worker
-// goroutine, so a subsequent normal document parse is still processed. This is
-// the exact "parsing works at first, then breaks after some memory tasks"
-// symptom — at max_concurrent_workers=1 the only worker is lost on the first
-// panicking memory task, and every later PDF stalls in the queue.
-//
-// The test drives the real workerLoop (not executeMemoryTask directly): it
-// starts one worker, enqueues (1) a panicking memory task, then (2) a normal
-// ingestion task with a stubbed runDocumentTask, and asserts the worker both
-// Nacked the poison memory task and Acked the document task before exiting.
-func TestWorkerLoop_SurvivesMemoryPanicAndKeepsServing(t *testing.T) {
+// TestHandleAndExecute_SurvivesMemoryPanicAndKeepsServing proves a recovered
+// memory panic does not prevent the same worker from processing its next
+// hand-off.
+func TestHandleAndExecute_SurvivesMemoryPanicAndKeepsServing(t *testing.T) {
db := testutil.SetupTestDB(t)
cleanup := testutil.ReplaceDBForTest(t, db)
defer cleanup()
- _, _, docID, taskID := testutil.SeedTestData(t, db, testutil.WithPipelineID("flow-1"))
+ _, _, _, taskID := testutil.SeedTestData(t, db, testutil.WithPipelineID("flow-1"))
ingestor := NewIngestor("test", 1, []string{"pdf"})
ingestor.SetMemoryMessageService(servicepkg.NewMemoryMessageService(servicepkg.NewMemoryService()))
@@ -89,37 +80,16 @@ func TestWorkerLoop_SurvivesMemoryPanicAndKeepsServing(t *testing.T) {
return nil
}
- memHandle := &fakeTaskHandle{msg: common.TaskMessage{TaskID: "mem-panic-2", TaskType: common.TaskTypeMemory}}
- memCtx := taskpkg.NewMemoryTaskContextForScheduling(context.Background(), "mem-panic-2", map[string]any{
- "memory_id": "mem-p2", "source_id": 1,
- "message_dict": map[string]any{"user_id": "u", "agent_id": "a", "session_id": "s"},
- }, memHandle)
-
- docHandle := &fakeTaskHandle{}
- docCtx := newAckTaskCtx(context.Background(), taskID, docID, docHandle)
+ memHandle := &fakeTaskHandle{msg: common.TaskMessage{
+ TaskID: "mem-panic-2",
+ TaskType: common.TaskTypeMemory,
+ Payload: []byte(`{"memory_id":"mem-p2","source_id":1,"message_dict":{"user_id":"u","agent_id":"a","session_id":"s"}}`),
+ }}
- ingestor.startWorkerPool()
- // Feed the poison memory task first, then a normal document task.
- ingestor.taskChan <- memCtx
- ingestor.taskChan <- docCtx
-
- // Wait for the worker to drain both tasks (poison Nacked, document Acked)
- // before cancelling, so the loop does not exit on ctx.Done() first.
- deadline := time.After(5 * time.Second)
- for memHandle.nacks.Load() != 1 || docHandle.acks.Load() != 1 {
- select {
- case <-deadline:
- t.Fatalf("worker did not drain tasks in time: mem(acks=%d nacks=%d) doc(acks=%d nacks=%d)",
- memHandle.acks.Load(), memHandle.nacks.Load(), docHandle.acks.Load(), docHandle.nacks.Load())
- default:
- time.Sleep(5 * time.Millisecond)
- }
- }
+ ingestor.handleAndExecute(memHandle)
- // Cancel the ingestor context so the worker loop exits; then wait for the
- // worker goroutine to finish cleanly (proves it was not crashed by the panic).
- ingestor.cancel()
- ingestor.workerWg.Wait()
+ docHandle := &fakeTaskHandle{msg: common.TaskMessage{TaskID: taskID, TaskType: common.TaskTypeIngestionTask}}
+ ingestor.handleAndExecute(docHandle)
if memHandle.nacks.Load() != 1 || memHandle.acks.Load() != 0 {
t.Fatalf("poison memory task: expected 0 Ack/1 Nack, got acks=%d nacks=%d", memHandle.acks.Load(), memHandle.nacks.Load())
diff --git a/internal/ingestion/service/execute_task_ack_test.go b/internal/ingestion/service/execute_task_ack_test.go
index e658601591..aef6513249 100644
--- a/internal/ingestion/service/execute_task_ack_test.go
+++ b/internal/ingestion/service/execute_task_ack_test.go
@@ -315,8 +315,10 @@ func TestSettleMessage_AckOnTerminal(t *testing.T) {
handle := &fakeTaskHandle{}
ctx := t.Context()
taskCtx := newAckTaskCtx(t.Context(), "task-1", "doc-1", handle)
+ hb := NewHeartbeat(taskCtx.ID(), taskCtx.Handle, ingestor.heartbeatInterval).WithContext(taskCtx.Ctx)
+ hb.Start()
- ingestor.settleMessage(ctx, taskCtx, func(ctx context.Context) bool { return true })
+ ingestor.settleMessage(ctx, taskCtx, hb, func(ctx context.Context) bool { return true })
if handle.acks.Load() != 1 || handle.nacks.Load() != 0 {
t.Fatalf("body=true: expected 1 Ack/0 Nack, got acks=%d nacks=%d", handle.acks.Load(), handle.nacks.Load())
@@ -329,8 +331,10 @@ func TestSettleMessage_NackOnNonTerminal(t *testing.T) {
handle := &fakeTaskHandle{}
ctx := t.Context()
taskCtx := newAckTaskCtx(t.Context(), "task-1", "doc-1", handle)
+ hb := NewHeartbeat(taskCtx.ID(), taskCtx.Handle, ingestor.heartbeatInterval).WithContext(taskCtx.Ctx)
+ hb.Start()
- ingestor.settleMessage(ctx, taskCtx, func(ctx context.Context) bool { return false })
+ ingestor.settleMessage(ctx, taskCtx, hb, func(ctx context.Context) bool { return false })
if handle.nacks.Load() != 1 || handle.acks.Load() != 0 {
t.Fatalf("body=false: expected 1 Nack/0 Ack, got acks=%d nacks=%d", handle.acks.Load(), handle.nacks.Load())
@@ -355,6 +359,8 @@ func TestSettleMessage_RecoversPanicAndAcksWhenTaskTerminal(t *testing.T) {
handle := &fakeTaskHandle{}
ctx := t.Context()
taskCtx := newAckTaskCtx(t.Context(), taskID, docID, handle)
+ hb := NewHeartbeat(taskCtx.ID(), taskCtx.Handle, ingestor.heartbeatInterval).WithContext(taskCtx.Ctx)
+ hb.Start()
panicked := false
func() {
@@ -363,7 +369,7 @@ func TestSettleMessage_RecoversPanicAndAcksWhenTaskTerminal(t *testing.T) {
panicked = true
}
}()
- ingestor.settleMessage(ctx, taskCtx, func(ctx context.Context) bool {
+ ingestor.settleMessage(ctx, taskCtx, hb, func(ctx context.Context) bool {
panic("boom")
})
}()
@@ -451,7 +457,9 @@ func TestSettleMessage_DBTruthOverridesBodyReturn(t *testing.T) {
}
ctx := t.Context()
- ingestor.settleMessage(ctx, taskCtx, body)
+ hb := NewHeartbeat(taskCtx.ID(), taskCtx.Handle, ingestor.heartbeatInterval).WithContext(taskCtx.Ctx)
+ hb.Start()
+ ingestor.settleMessage(ctx, taskCtx, hb, body)
// DB shows FAILED → terminal → Ack, overriding body's false.
if handle.acks.Load() != 1 || handle.nacks.Load() != 0 {
diff --git a/internal/ingestion/service/handle_and_execute_test.go b/internal/ingestion/service/handle_and_execute_test.go
new file mode 100644
index 0000000000..4df6dd4944
--- /dev/null
+++ b/internal/ingestion/service/handle_and_execute_test.go
@@ -0,0 +1,366 @@
+//
+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+package service
+
+import (
+ "context"
+ "testing"
+ "time"
+
+ "ragflow/internal/common"
+ "ragflow/internal/entity"
+ "ragflow/internal/ingestion/testutil"
+ servicepkg "ragflow/internal/service"
+)
+
+// TestHandleAndExecute_MalformedMemoryPayloadAcks verifies that a memory task with
+// an unparseable payload is acked and skipped without executing the runner.
+func TestHandleAndExecute_MalformedMemoryPayloadAcks(t *testing.T) {
+ ingestor := newUnitIngestor("test-mem-malformed", 1, nil)
+ ingestor.SetMemoryMessageService(servicepkg.NewMemoryMessageService(servicepkg.NewMemoryService()))
+
+ runnerCalled := false
+ ingestor.runMemoryTask = func(ctx context.Context, taskID string, payload map[string]any) error {
+ runnerCalled = true
+ return nil
+ }
+
+ handle := &fakeTaskHandle{msg: common.TaskMessage{
+ TaskID: "mem-bad-payload",
+ TaskType: common.TaskTypeMemory,
+ Payload: []byte(`{invalid-json`),
+ }}
+
+ ingestor.handleAndExecute(handle)
+
+ if runnerCalled {
+ t.Fatal("expected runMemoryTask to not be called for malformed payload")
+ }
+ if handle.acks.Load() != 1 || handle.nacks.Load() != 0 {
+ t.Fatalf("expected 1 Ack/0 Nack, got acks=%d nacks=%d", handle.acks.Load(), handle.nacks.Load())
+ }
+}
+
+// TestHandleAndExecute_MemoryExtractorDisabledAcks verifies that a memory task
+// received when memory extractor is disabled is ack-skipped.
+func TestHandleAndExecute_MemoryExtractorDisabledAcks(t *testing.T) {
+ ingestor := newUnitIngestor("test-mem-disabled", 1, nil)
+ // memorySvc is nil
+
+ handle := &fakeTaskHandle{msg: common.TaskMessage{
+ TaskID: "mem-disabled",
+ TaskType: common.TaskTypeMemory,
+ Payload: []byte(`{"memory_id":"m1"}`),
+ }}
+
+ ingestor.handleAndExecute(handle)
+
+ if handle.acks.Load() != 1 || handle.nacks.Load() != 0 {
+ t.Fatalf("expected 1 Ack/0 Nack, got acks=%d nacks=%d", handle.acks.Load(), handle.nacks.Load())
+ }
+}
+
+// TestHandleAndExecute_MemoryEmptyTaskIDAcks verifies that a memory task with
+// an empty task ID is ack-skipped.
+func TestHandleAndExecute_MemoryEmptyTaskIDAcks(t *testing.T) {
+ ingestor := newUnitIngestor("test-mem-empty-id", 1, nil)
+ ingestor.SetMemoryMessageService(servicepkg.NewMemoryMessageService(servicepkg.NewMemoryService()))
+
+ handle := &fakeTaskHandle{msg: common.TaskMessage{
+ TaskID: "",
+ TaskType: common.TaskTypeMemory,
+ Payload: []byte(`{"memory_id":"m1"}`),
+ }}
+
+ ingestor.handleAndExecute(handle)
+
+ if handle.acks.Load() != 1 || handle.nacks.Load() != 0 {
+ t.Fatalf("expected 1 Ack/0 Nack, got acks=%d nacks=%d", handle.acks.Load(), handle.nacks.Load())
+ }
+}
+
+// TestHandleAndExecute_UnknownTaskTypeAcks verifies that a message with an
+// unknown task type is ack-skipped without touching the ingestion task service.
+func TestHandleAndExecute_UnknownTaskTypeAcks(t *testing.T) {
+ ingestor := newUnitIngestor("test-unknown-type", 1, nil)
+
+ handle := &fakeTaskHandle{msg: common.TaskMessage{
+ TaskID: "unknown-task-1",
+ TaskType: "completely_unknown_type",
+ }}
+
+ ingestor.handleAndExecute(handle)
+
+ if handle.acks.Load() != 1 || handle.nacks.Load() != 0 {
+ t.Fatalf("expected 1 Ack/0 Nack for unknown type, got acks=%d nacks=%d", handle.acks.Load(), handle.nacks.Load())
+ }
+}
+
+// TestHandleAndExecute_TaskNotFoundAcks verifies that when StartRunning returns
+// ErrTaskNotFound, the message is Acked to prevent indefinite redeliveries.
+func TestHandleAndExecute_TaskNotFoundAcks(t *testing.T) {
+ db := testutil.SetupTestDB(t)
+ cleanup := testutil.ReplaceDBForTest(t, db)
+ defer cleanup()
+
+ ingestor := newUnitIngestor("test-task-not-found", 1, []string{"pdf"})
+ handle := &fakeTaskHandle{msg: common.TaskMessage{
+ TaskID: "non-existent-task-id",
+ TaskType: common.TaskTypeIngestionTask,
+ }}
+
+ ingestor.handleAndExecute(handle)
+
+ if handle.acks.Load() != 1 || handle.nacks.Load() != 0 {
+ t.Fatalf("expected 1 Ack/0 Nack for not-found task, got acks=%d nacks=%d", handle.acks.Load(), handle.nacks.Load())
+ }
+}
+
+// TestHandleAndExecute_StartRunningTransientErrorNacks verifies that a transient
+// database error during StartRunning causes the message to be Nacked for redelivery.
+func TestHandleAndExecute_StartRunningTransientErrorNacks(t *testing.T) {
+ // Close/nil DB to simulate transient database failure
+ db := testutil.SetupTestDB(t)
+ cleanup := testutil.ReplaceDBForTest(t, db)
+ defer cleanup()
+
+ _, _, _, taskID := testutil.SeedTestData(t, db)
+
+ // Close underlying sql db to force query errors
+ sqlDB, err := db.DB()
+ if err == nil {
+ _ = sqlDB.Close()
+ }
+
+ ingestor := newUnitIngestor("test-start-running-err", 1, []string{"pdf"})
+ handle := &fakeTaskHandle{msg: common.TaskMessage{
+ TaskID: taskID,
+ TaskType: common.TaskTypeIngestionTask,
+ }}
+
+ ingestor.handleAndExecute(handle)
+
+ if handle.nacks.Load() != 1 || handle.acks.Load() != 0 {
+ t.Fatalf("expected 0 Ack/1 Nack on transient StartRunning error, got acks=%d nacks=%d", handle.acks.Load(), handle.nacks.Load())
+ }
+}
+
+// TestHandleAndExecute_AlreadyTerminalTaskAcks verifies that tasks already in a
+// terminal state (COMPLETED, STOPPED, FAILED) are Ack-skipped without re-execution.
+func TestHandleAndExecute_AlreadyTerminalTaskAcks(t *testing.T) {
+ statuses := []string{common.COMPLETED, common.STOPPED, common.FAILED}
+ for _, status := range statuses {
+ t.Run(status, func(t *testing.T) {
+ db := testutil.SetupTestDB(t)
+ cleanup := testutil.ReplaceDBForTest(t, db)
+ defer cleanup()
+
+ _, _, _, taskID := testutil.SeedTestData(t, db)
+ if err := db.Model(&entity.IngestionTask{}).Where("id = ?", taskID).
+ Update("status", status).Error; err != nil {
+ t.Fatalf("set status %s: %v", status, err)
+ }
+
+ ingestor := newUnitIngestor("test-terminal-"+status, 1, []string{"pdf"})
+ pipelineRan := false
+ ingestor.runDocumentTask = func(ctx context.Context, task *entity.IngestionTask) error {
+ pipelineRan = true
+ return nil
+ }
+
+ handle := &fakeTaskHandle{msg: common.TaskMessage{
+ TaskID: taskID,
+ TaskType: common.TaskTypeIngestionTask,
+ }}
+
+ ingestor.handleAndExecute(handle)
+
+ if pipelineRan {
+ t.Fatalf("pipeline should not run for already %s task", status)
+ }
+ if handle.acks.Load() != 1 || handle.nacks.Load() != 0 {
+ t.Fatalf("status %s: expected 1 Ack/0 Nack, got acks=%d nacks=%d", status, handle.acks.Load(), handle.nacks.Load())
+ }
+ })
+ }
+}
+
+// TestHandleAndExecute_StoppingTaskConvergedAndAcks verifies that a task in STOPPING
+// status is transitioned to STOPPED by StartRunning and then Acked.
+func TestHandleAndExecute_StoppingTaskConvergedAndAcks(t *testing.T) {
+ db := testutil.SetupTestDB(t)
+ cleanup := testutil.ReplaceDBForTest(t, db)
+ defer cleanup()
+
+ _, _, _, taskID := testutil.SeedTestData(t, db)
+ if err := db.Model(&entity.IngestionTask{}).Where("id = ?", taskID).
+ Update("status", common.STOPPING).Error; err != nil {
+ t.Fatalf("set status STOPPING: %v", err)
+ }
+
+ ingestor := newUnitIngestor("test-stopping-converge", 1, []string{"pdf"})
+ handle := &fakeTaskHandle{msg: common.TaskMessage{
+ TaskID: taskID,
+ TaskType: common.TaskTypeIngestionTask,
+ }}
+
+ ingestor.handleAndExecute(handle)
+
+ if handle.acks.Load() != 1 || handle.nacks.Load() != 0 {
+ t.Fatalf("expected 1 Ack/0 Nack for STOPPING task, got acks=%d nacks=%d", handle.acks.Load(), handle.nacks.Load())
+ }
+
+ var task entity.IngestionTask
+ if err := db.Where("id = ?", taskID).First(&task).Error; err != nil {
+ t.Fatalf("reload task: %v", err)
+ }
+ if task.Status != common.STOPPED {
+ t.Fatalf("task status = %s, want STOPPED", task.Status)
+ }
+}
+
+// TestHandleAndExecute_DocumentDuplicateClaimRenewsLease verifies that a redelivered
+// document task while the worker is still processing renews its lease via InProgress
+// and does not Ack or start a second execution.
+func TestHandleAndExecute_DocumentDuplicateClaimRenewsLease(t *testing.T) {
+ db := testutil.SetupTestDB(t)
+ cleanup := testutil.ReplaceDBForTest(t, db)
+ defer cleanup()
+
+ _, _, _, taskID := testutil.SeedTestData(t, db)
+ if err := db.Model(&entity.IngestionTask{}).Where("id = ?", taskID).
+ Update("status", common.RUNNING).Error; err != nil {
+ t.Fatalf("set status RUNNING: %v", err)
+ }
+
+ ingestor := newUnitIngestor("test-doc-dup", 1, []string{"pdf"})
+ // Simulate active claim by another worker
+ if !ingestor.claimTask(taskID) {
+ t.Fatal("first claim should succeed")
+ }
+
+ pipelineRan := false
+ ingestor.runDocumentTask = func(ctx context.Context, task *entity.IngestionTask) error {
+ pipelineRan = true
+ return nil
+ }
+
+ handle := &fakeTaskHandle{msg: common.TaskMessage{
+ TaskID: taskID,
+ TaskType: common.TaskTypeIngestionTask,
+ }}
+
+ ingestor.handleAndExecute(handle)
+
+ if pipelineRan {
+ t.Fatal("duplicate delivery should not execute pipeline")
+ }
+ if handle.inProgress.Load() != 1 {
+ t.Fatalf("expected InProgress = 1 for duplicate delivery, got %d", handle.inProgress.Load())
+ }
+ if handle.acks.Load() != 0 || handle.nacks.Load() != 0 {
+ t.Fatalf("expected 0 Ack/0 Nack for duplicate delivery, got acks=%d nacks=%d", handle.acks.Load(), handle.nacks.Load())
+ }
+}
+
+// TestHandleAndExecute_MemoryDuplicateClaimRenewsLease verifies duplicate delivery
+// protection for memory extraction tasks.
+func TestHandleAndExecute_MemoryDuplicateClaimRenewsLease(t *testing.T) {
+ ingestor := newUnitIngestor("test-mem-dup", 1, nil)
+ ingestor.SetMemoryMessageService(servicepkg.NewMemoryMessageService(servicepkg.NewMemoryService()))
+
+ taskID := "mem-dup-1"
+ if !ingestor.claimTask(taskID) {
+ t.Fatal("first claim should succeed")
+ }
+
+ handle := &fakeTaskHandle{msg: common.TaskMessage{
+ TaskID: taskID,
+ TaskType: common.TaskTypeMemory,
+ Payload: []byte(`{"memory_id":"m1","source_id":1,"message_dict":{"user_id":"u"}}`),
+ }}
+
+ ingestor.handleAndExecute(handle)
+
+ if handle.inProgress.Load() != 1 {
+ t.Fatalf("expected InProgress = 1 for duplicate memory delivery, got %d", handle.inProgress.Load())
+ }
+ if handle.acks.Load() != 0 || handle.nacks.Load() != 0 {
+ t.Fatalf("expected 0 Ack/0 Nack, got acks=%d nacks=%d", handle.acks.Load(), handle.nacks.Load())
+ }
+}
+
+// TestHandleAndExecute_SlowAdmissionHeartbeatsUnderLeaseProtection verifies that
+// when admission (StartRunning) is slow (e.g. database lag exceeding ack deadline),
+// the heartbeat started upon receiving the handle renews the broker lease before
+// StartRunning finishes.
+func TestHandleAndExecute_SlowAdmissionHeartbeatsUnderLeaseProtection(t *testing.T) {
+ db := testutil.SetupTestDB(t)
+ cleanup := testutil.ReplaceDBForTest(t, db)
+ defer cleanup()
+
+ _, _, _, taskID := testutil.SeedTestData(t, db)
+ if err := db.Model(&entity.IngestionTask{}).Where("id = ?", taskID).
+ Update("status", common.RUNNING).Error; err != nil {
+ t.Fatalf("set status RUNNING: %v", err)
+ }
+
+ ingestor := newUnitIngestor("test-slow-admission", 1, []string{"pdf"})
+ ingestor.heartbeatInterval = 10 * time.Millisecond
+
+ // Delay document pipeline execution briefly so heartbeats fire during run
+ ingestor.runDocumentTask = func(ctx context.Context, task *entity.IngestionTask) error {
+ time.Sleep(50 * time.Millisecond)
+ return nil
+ }
+
+ handle := &fakeTaskHandle{msg: common.TaskMessage{
+ TaskID: taskID,
+ TaskType: common.TaskTypeIngestionTask,
+ }}
+
+ ingestor.handleAndExecute(handle)
+
+ if handle.inProgress.Load() == 0 {
+ t.Fatal("expected InProgress heartbeats during task execution, got 0")
+ }
+ if handle.acks.Load() != 1 {
+ t.Fatalf("expected 1 Ack on completion, got %d", handle.acks.Load())
+ }
+}
+
+// TestWorkerDispatcher_WaitAfterPullErrorCancelsCleanly verifies that waitAfterPullError
+// unblocks promptly when dispatch context is cancelled, without hanging on the sleep.
+func TestWorkerDispatcher_WaitAfterPullErrorCancelsCleanly(t *testing.T) {
+ ingestor := newUnitIngestor("test-pull-backoff-cancel", 1, nil)
+
+ done := make(chan struct{})
+ go func() {
+ ingestor.waitAfterPullError()
+ close(done)
+ }()
+
+ // Cancel dispatch context
+ ingestor.dispatchCancel()
+
+ select {
+ case <-done:
+ // unblocked promptly
+ case <-time.After(200 * time.Millisecond):
+ t.Fatal("waitAfterPullError did not cancel promptly on dispatchCtx.Done")
+ }
+}
diff --git a/internal/ingestion/service/heartbeat.go b/internal/ingestion/service/heartbeat.go
index 5ef6ff6383..796716a2dd 100644
--- a/internal/ingestion/service/heartbeat.go
+++ b/internal/ingestion/service/heartbeat.go
@@ -40,6 +40,7 @@ type Heartbeat struct {
handle common.TaskHandle
interval time.Duration
ctx context.Context
+ onError func(error)
mu sync.Mutex
started bool
@@ -61,6 +62,14 @@ func NewHeartbeat(id string, handle common.TaskHandle, interval time.Duration) *
}
}
+// WithOnError sets an optional error callback invoked when handle.InProgress fails.
+func (h *Heartbeat) WithOnError(fn func(error)) *Heartbeat {
+ if h != nil {
+ h.onError = fn
+ }
+ return h
+}
+
// WithContext binds an external cancellation signal (e.g. ingestor shutdown) so
// the renewal goroutine also exits when ctx is done. It must be called before
// Start.
@@ -119,6 +128,9 @@ func (h *Heartbeat) loop() {
case <-ticker.C:
if err := h.handle.InProgress(); err != nil {
common.Error(fmt.Sprintf("heartbeat task %s", h.id), err)
+ if h.onError != nil {
+ h.onError(err)
+ }
}
case <-h.stopCh:
return
diff --git a/internal/ingestion/service/ingestion_service.go b/internal/ingestion/service/ingestion_service.go
index b0e1d10234..f82719fe94 100644
--- a/internal/ingestion/service/ingestion_service.go
+++ b/internal/ingestion/service/ingestion_service.go
@@ -41,6 +41,7 @@ import (
"ragflow/internal/utility"
"github.com/cenkalti/backoff/v5"
+ "go.uber.org/zap"
)
// defaultHeartbeatInterval paces the InProgress() working pulse that keeps an
@@ -56,9 +57,12 @@ const defaultHeartbeatInterval = 2 * time.Second
type Ingestor struct {
id string
name string
- ctx context.Context
+ ctx context.Context // execution context for handles already owned by workers
cancel context.CancelFunc
+ dispatchCtx context.Context // worker queue registration and Pull lifecycle
+ dispatchCancel context.CancelFunc
+
// Configuration
maxConcurrency int32
supportedDocTypes []string
@@ -69,16 +73,21 @@ type Ingestor struct {
currentTasks map[string]struct{} // set of task IDs currently claimed by a worker
tasksMu sync.RWMutex
activeWorkers atomic.Int32 // number of worker goroutines currently in workerLoop
+ activeLeases map[*Heartbeat]*activeLease
+ leasesMu sync.Mutex
+ stopLeases atomic.Bool
// Shutdown channel - receive on this to trigger graceful shutdown
ShutdownCh chan struct{}
// Worker pool
- taskChan chan *taskpkg.TaskContext
- workerWg sync.WaitGroup
- startOnce sync.Once
- workerOnce sync.Once // guards startWorkerPool; must NOT be startOnce (Start wraps start() in startOnce, and start() calls startWorkerPool -> re-entry deadlock)
- stopOnce sync.Once // guards close(ShutdownCh) against double-close on repeated Stop
+ workerQueue chan *worker
+ dispatcherWg sync.WaitGroup
+ workerWg sync.WaitGroup
+ startOnce sync.Once
+ startErr error
+ workerOnce sync.Once // guards startWorkerPool; must NOT be startOnce (Start wraps start() in startOnce, and start() calls startWorkerPool -> re-entry deadlock)
+ stopOnce sync.Once // guards close(ShutdownCh) against double-close on repeated Stop
ingestionTaskSvc *servicepkg.IngestionTaskService
docState *docStateUpdater
@@ -122,27 +131,38 @@ type Ingestor struct {
// override this to simulate cancel without Redis.
cancelCheck func(ctx context.Context, taskID string) bool
- // checkpointExists distinguishes a fresh run from a checkpoint resume.
- // Tests inject this dependency so unit tests do not require Redis.
checkpointExists func(ctx context.Context, taskID string) (bool, error)
}
+type worker struct {
+ id int32
+ inbox chan common.TaskHandle
+}
+
+type activeLease struct {
+ abandoned atomic.Bool
+}
+
func NewIngestor(name string, maxConcurrency int32, supportedTypes []string) *Ingestor {
if maxConcurrency <= 0 {
maxConcurrency = int32(runtime.NumCPU())
}
ctx, cancel := context.WithCancel(context.Background())
+ dispatchCtx, dispatchCancel := context.WithCancel(context.Background())
id := utility.GenerateUUID()
ingestor := &Ingestor{
id: id,
name: name,
ctx: ctx,
cancel: cancel,
+ dispatchCtx: dispatchCtx,
+ dispatchCancel: dispatchCancel,
maxConcurrency: maxConcurrency,
supportedDocTypes: supportedTypes,
version: "1.0.0",
currentTasks: make(map[string]struct{}),
- taskChan: make(chan *taskpkg.TaskContext, maxConcurrency*2),
+ activeLeases: make(map[*Heartbeat]*activeLease),
+ workerQueue: make(chan *worker, maxConcurrency),
ShutdownCh: make(chan struct{}, 1),
ingestionTaskSvc: servicepkg.NewIngestionTaskService(),
docState: newDocStateUpdater(),
@@ -160,18 +180,19 @@ func (e *Ingestor) ID() string {
return e.id
}
-// consumeErrorBackoff paces the consume loop when GetMessages returns an
-// error, so a persistent MQ failure does not pin a CPU. The backoff is
-// cancellable so a shutdown during backoff returns promptly.
+// consumeErrorBackoff paces failed Pull requests so a persistent MQ failure
+// does not pin a CPU. The backoff is cancellable and does not block another
+// idle worker from making its own Pull request.
const consumeErrorBackoff = 1 * time.Second
+const taskPullRequestTimeout = 1 * time.Second
+
func (e *Ingestor) Start() error {
common.Info(fmt.Sprintf("Ingestor %s initialized", e.id))
- var startErr error
e.startOnce.Do(func() {
- startErr = e.start()
+ e.startErr = e.start()
})
- return startErr
+ return e.startErr
}
// start runs the full startup sequence. It is invoked at most once (guarded by
@@ -199,66 +220,88 @@ func (e *Ingestor) start() error {
e.startWorkerPool()
e.startDatasetKnowledgeCompile()
- // Run the main tasks.RAGFLOW consume loop off the caller's goroutine so
- // Start returns promptly; it is joined by Stop via workerWg.
- e.workerWg.Add(1)
+ // Run the main tasks.RAGFLOW dispatcher off the caller's goroutine so Start
+ // returns promptly. Stop joins it before cancelling task execution.
+ e.dispatcherWg.Add(1)
go e.consumeLoop()
return nil
}
-// fetchBudget caps each Fetch batch by the worker channel's free capacity (and
-// the worker width) so the consumer never pulls more messages than it can
-// enqueue immediately: messages beyond the budget would sit claimed in
-// taskChan's senders' hands or force blocking sends that stall the consume
-// loop while their AckWait runs out.
-func (e *Ingestor) fetchBudget() int {
- budget := cap(e.taskChan) - len(e.taskChan)
- if budget > int(e.maxConcurrency) {
- budget = int(e.maxConcurrency)
- }
- if budget < 1 {
- budget = 1
- }
- return budget
-}
-
-// consumeLoop is the main tasks.RAGFLOW consume loop. It runs until e.ctx is
-// cancelled (graceful shutdown); per-message failures are settled by
-// processMessage and never terminate the consumer. Dataset-level compile work
-// is owned by the kcConcurrency compileLoop workers (each running the
-// Consumer's Run loop), so this loop stays focused on tasks.RAGFLOW and is
-// never held up by compile work.
+// consumeLoop is the worker dispatcher for tasks.RAGFLOW. It waits for an
+// available worker before synchronously pulling one task, preventing prefetch
+// and limiting this ingestor to one outstanding broker request.
func (e *Ingestor) consumeLoop() {
- defer e.workerWg.Done()
+ defer e.dispatcherWg.Done()
msgQueueEngine := engine.GetMessageQueueEngine()
for {
- // Graceful shutdown is the only condition under which the consume
- // loop exits. Per-message processing failures never terminate the
- // consumer: processMessage settles (ack/nack) each message itself.
- if err := e.ctx.Err(); err != nil {
+ var w *worker
+ select {
+ case <-e.dispatchCtx.Done():
+ return
+ case w = <-e.workerQueue:
+ }
+ if e.dispatchCtx.Err() != nil {
+ e.returnWorker(w)
return
}
- taskHandles, err := msgQueueEngine.GetMessages(e.fetchBudget())
+
+ pullStart := time.Now()
+ pullCtx, cancel := context.WithTimeout(e.dispatchCtx, taskPullRequestTimeout)
+ handle, err := msgQueueEngine.PullMessage(pullCtx)
+ cancel()
if err != nil {
- common.Error("error consuming message", err)
- select {
- case <-time.After(consumeErrorBackoff):
- case <-e.ctx.Done():
- return
+ e.returnWorker(w)
+ e.logPullError(err)
+ if !errors.Is(err, context.Canceled) && !errors.Is(err, context.DeadlineExceeded) {
+ e.waitAfterPullError()
}
continue
}
- if len(taskHandles) > 0 {
- for _, taskHandle := range taskHandles {
- e.processMessage(taskHandle)
- }
+ if handle == nil {
+ e.returnWorker(w)
continue
}
- // tasks.RAGFLOW idle: dataset-level compile work is owned by the
- // kcConcurrency compileLoop workers (started in
- // startDatasetKnowledgeCompile), so the task loop stays focused on
- // tasks.RAGFLOW and is never held up by compile work.
+
+ recvTime := time.Now()
+ latency := time.Since(pullStart)
+ common.Debug(fmt.Sprintf("Pull delivered handle in %v", latency),
+ zap.Duration("pull_first_handle_latency", latency),
+ )
+ select {
+ case w.inbox <- handle:
+ handoffDuration := time.Since(recvTime)
+ common.Debug(fmt.Sprintf("Handed off handle to worker %d in %v", w.id, handoffDuration),
+ zap.Int32("worker_id", w.id),
+ zap.Duration("handoff_duration", handoffDuration),
+ )
+ case <-e.dispatchCtx.Done():
+ if err := handle.Nack(); err != nil {
+ common.Error("nack task after shutdown interrupted handoff", err)
+ }
+ return
+ }
+ }
+}
+
+func (e *Ingestor) logPullError(err error) {
+ if err == nil || errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) {
+ return
+ }
+ common.Error("error pulling task message", err)
+}
+
+func (e *Ingestor) waitAfterPullError() {
+ select {
+ case <-time.After(consumeErrorBackoff):
+ case <-e.dispatchCtx.Done():
+ }
+}
+
+func (e *Ingestor) returnWorker(w *worker) {
+ select {
+ case e.workerQueue <- w:
+ case <-e.dispatchCtx.Done():
}
}
@@ -333,104 +376,61 @@ func (e *Ingestor) compileLoop(id int32) {
}
}
-// processMessage handles a single incoming MQ message: filter by type,
-// activate the task (state transition), guard against duplicate execution
-// (claim), and enqueue to the worker pool (or backpressure-reject). It
-// settles (ack/nack) every message itself and never returns an error: a
-// single message can never terminate the consume loop. Only ctx cancellation
-// (graceful shutdown) stops the consumer - see Start.
-func (e *Ingestor) processMessage(handle common.TaskHandle) {
+// handleAndExecute owns one handle from its hand-off to final settlement. It
+// runs only in the worker that received the handle through its private inbox.
+func (e *Ingestor) handleAndExecute(handle common.TaskHandle) {
+ startTime := time.Now()
taskMessage := handle.GetMessage()
- common.Info(fmt.Sprintf("Received task id: %s, type: %s", taskMessage.TaskID, taskMessage.TaskType))
-
- // Deferred claim release: if this function claims a task but the task
- // is not successfully enqueued to the worker pool (e.g. backpressure,
- // or a future error path added between claim and enqueue), the defer
- // cleans up so the task can be reclaimed on MQ redelivery. When the
- // task IS enqueued, claimedTaskID is cleared and executeTask's own
- // defer takes ownership of the release.
- var claimedTaskID string
+ common.Info(fmt.Sprintf("Received task id: %s, type: %s", taskMessage.TaskID, taskMessage.TaskType),
+ zap.String("task_id", taskMessage.TaskID),
+ zap.String("task_type", taskMessage.TaskType),
+ )
defer func() {
- if claimedTaskID != "" {
- e.releaseTask(claimedTaskID)
- }
+ settlementDuration := time.Since(startTime)
+ common.Debug(fmt.Sprintf("Task %s settlement finished in %v", taskMessage.TaskID, settlementDuration),
+ zap.String("task_id", taskMessage.TaskID),
+ zap.Duration("settlement_duration", settlementDuration),
+ )
}()
+ hb := NewHeartbeat(taskMessage.TaskID, handle, e.heartbeatInterval).
+ WithContext(context.Background())
+ hb.Start()
+ e.registerLease(hb)
+ defer e.unregisterLease(hb)
+
// Memory-extraction tasks share the tasks.RAGFLOW consumer and the worker
- // pool with ingestion tasks. They do NOT use the ingestion state machine
- // (no ingestion_task row): the message body is dispatched straight to a
- // worker via TaskContext.Kind==TaskKindMemory, which runs the memory
- // extractor and acks/nacks on its own.
+ // pool with ingestion tasks. They do NOT use the ingestion state machine.
if taskMessage.TaskType == common.TaskTypeMemory {
if e.memorySvc == nil {
common.Warn(fmt.Sprintf("memory task %s received but memory extractor is disabled, ack", taskMessage.TaskID))
- if err := handle.Ack(); err != nil {
- common.Error(fmt.Sprintf("error ack memory task %s", taskMessage.TaskID), err)
- }
+ e.ackHandle(hb, handle, taskMessage.TaskID)
return
}
- // A memory task without an envelope id has no valid identity: claiming
- // an empty key would strand a no-op claim in currentTasks and block
- // nothing useful, so Ack-skip it instead.
if taskMessage.TaskID == "" {
common.Warn("memory task with empty task id received, ack")
- if err := handle.Ack(); err != nil {
- common.Error("error ack memory task with empty task id", err)
- }
+ e.ackHandle(hb, handle, taskMessage.TaskID)
return
}
var payload map[string]any
if len(taskMessage.Payload) == 0 || json.Unmarshal(taskMessage.Payload, &payload) != nil {
common.Warn(fmt.Sprintf("memory task %s has no parseable payload, ack", taskMessage.TaskID))
- if err := handle.Ack(); err != nil {
- common.Error(fmt.Sprintf("error ack memory task %s", taskMessage.TaskID), err)
- }
+ e.ackHandle(hb, handle, taskMessage.TaskID)
return
}
taskCtx := taskpkg.NewMemoryTaskContextForScheduling(e.ctx, taskMessage.TaskID, payload, handle)
- // Claim the task before enqueueing so a redelivered copy of the same
- // memory task (NATS AckWait/BackOff redelivery, or a restart replay) is
- // not executed again while its owner is still running. Memory tasks have no
- // ingestion_task row / status CAS to dedupe against, so this claim
- // guards duplicate execution within this ingestor. The claim is released
- // by executeMemoryTask when the worker finishes.
if !e.claimTask(taskMessage.TaskID) {
common.Warn(fmt.Sprintf("memory task %s redelivered while worker still processing, renew lease", taskMessage.TaskID))
- if err := handle.InProgress(); err != nil {
- common.Error(fmt.Sprintf("renew redelivered memory task %s", taskMessage.TaskID), err)
- }
- return
- }
- claimedTaskID = taskMessage.TaskID
- // Begin renewing the broker lease before the task can wait in taskChan.
- // Starting only in executeMemoryTask would leave queued work unprotected
- // until a worker becomes available: a task waiting past AckWait would be
- // redelivered before its owner can begin running it.
- // The stop function rides on the task context so the worker can stop it
- // before settlement (the lease is owned by the task, not by the map).
- hb := NewHeartbeat(taskMessage.TaskID, handle, e.heartbeatInterval).WithContext(e.ctx)
- hb.Start()
- taskCtx.SetStopLease(hb.Stop)
- select {
- case e.taskChan <- taskCtx:
- claimedTaskID = "" // executeMemoryTask owns the release now
- common.Info(fmt.Sprintf("Memory task %s queued (channel: %d/%d)", taskMessage.TaskID, len(e.taskChan), cap(e.taskChan)))
- case <-e.ctx.Done():
- // Shutdown won the race: stop the lease, release the claim, and
- // return without settling so the broker redelivers the memory task
- // after restart.
- taskCtx.StopLease()
- common.Info(fmt.Sprintf("Ingestor shutting down; memory task %s not enqueued", taskMessage.TaskID))
+ e.renewDuplicateHandle(hb, handle, taskMessage.TaskID)
return
}
+ e.executeMemoryTaskWithHeartbeat(e.ctx, taskCtx, hb)
return
}
if taskMessage.TaskType != common.TaskTypeIngestionTask {
common.Info(fmt.Sprintf("task %s is not an ingestion task", taskMessage.TaskID))
- if err := handle.Ack(); err != nil {
- common.Error(fmt.Sprintf("error ack task %s", taskMessage.TaskID), err)
- }
+ e.ackHandle(hb, handle, taskMessage.TaskID)
return
}
@@ -438,192 +438,171 @@ func (e *Ingestor) processMessage(handle common.TaskHandle) {
if err != nil {
if errors.Is(err, common.ErrTaskNotFound) {
common.Warn(fmt.Sprintf("task %s not found, skipping", taskMessage.TaskID))
- if ackErr := handle.Ack(); ackErr != nil {
- common.Error(fmt.Sprintf("error ack task %s", taskMessage.TaskID), ackErr)
- }
+ e.ackHandle(hb, handle, taskMessage.TaskID)
return
}
- // Recoverable activation failure (e.g. a DB blip): nack for
- // redelivery instead of dropping the message or killing the
- // consumer.
common.Error(fmt.Sprintf("error setting task %s to running", taskMessage.TaskID), err)
- if nackErr := handle.Nack(); nackErr != nil {
- common.Error(fmt.Sprintf("error nack task %s", taskMessage.TaskID), nackErr)
- }
+ e.nackHandle(hb, handle, taskMessage.TaskID)
return
}
if task == nil {
common.Info(fmt.Sprintf("task %s is already removed", taskMessage.TaskID))
- if ackErr := handle.Ack(); ackErr != nil {
- common.Error(fmt.Sprintf("error ack task %s", taskMessage.TaskID), ackErr)
- }
+ e.ackHandle(hb, handle, taskMessage.TaskID)
return
}
switch task.Status {
case common.COMPLETED, common.STOPPED, common.FAILED:
common.Info(fmt.Sprintf("task %s is already %s", taskMessage.TaskID, task.Status))
- if ackErr := handle.Ack(); ackErr != nil {
- common.Error(fmt.Sprintf("error ack task %s", taskMessage.TaskID), ackErr)
- }
+ e.ackHandle(hb, handle, taskMessage.TaskID)
return
case common.RUNNING:
- // Guard against MQ redelivery: if another worker in this
- // process is already processing this task, renew the redelivered
- // message's lease and skip instead of scheduling it again. The
- // owning worker remains the only code path that may Ack/Nack.
if !e.claimTask(task.ID) {
common.Warn(fmt.Sprintf("task %s redelivered while worker still processing, renew lease (task_id=%s doc_id=%s kb_id=%s)",
taskMessage.TaskID, task.ID, task.DocumentID, task.DatasetID))
- if err := handle.InProgress(); err != nil {
- common.Error(fmt.Sprintf("renew redelivered task %s", taskMessage.TaskID), err)
- }
+ e.renewDuplicateHandle(hb, handle, taskMessage.TaskID)
return
}
- claimedTaskID = task.ID
default:
- // Unreachable given StartRunning normalizes every status to
- // RUNNING/COMPLETED/STOPPED/FAILED, but defensive: ack-skip an
- // unknown status instead of enqueuing it for execution.
common.Warn(fmt.Sprintf("task %s in unexpected status %s, ack-skip", taskMessage.TaskID, task.Status))
- if ackErr := handle.Ack(); ackErr != nil {
- common.Error(fmt.Sprintf("error ack task %s", taskMessage.TaskID), ackErr)
- }
+ e.ackHandle(hb, handle, taskMessage.TaskID)
return
}
- // Construct TaskContext and carry the MQ handle so the worker can
- // Ack/Nack when the task reaches a terminal status.
taskCtx := taskpkg.NewTaskContextForScheduling(e.ctx, task)
taskCtx.Handle = handle
+ e.executeTaskWithHeartbeat(e.ctx, taskCtx, hb)
+}
- // Push to the task channel. Use a blocking send so backpressure is
- // applied at the consumer: the consume loop waits for a free worker slot
- // instead of dropping the message. Dropping on backpressure is unsafe
- // because StartRunning (above) has already flipped the task — and its
- // document — to RUNNING in the DB, and there is no scan-and-re-enqueue
- // path on completion. A Nack that exceeds the broker's MaxDeliver (16,
- // with no dead-letter in nats.go) would permanently lose the task and
- // leave the document stuck in RUNNING — the "files get stuck after the
- // first few parse" defect.
- //
- // The in-flight claim (set just above) guards against a redelivery racing
- // the blocked send: a duplicate delivery sees claimTask fail, renews its
- // lease, and cannot double-execute a task.
- select {
- case e.taskChan <- taskCtx:
- claimedTaskID = "" // executeTask owns the release now
- common.Info(fmt.Sprintf("Task %s queued (channel: %d/%d)", task.ID, len(e.taskChan), cap(e.taskChan)))
- case <-e.ctx.Done():
- // Shutdown won the race after StartRunning already flipped the task
- // to RUNNING: finalize it as STOPPED so the row cannot linger in
- // non-terminal RUNNING with no in-flight worker (the user can
- // retry). The message is left unsettled - on redelivery the terminal
- // STOPPED status ack-skips it. Both paths
- // run with a detached 2s timeout so shutdown is not slowed by a
- // stalled DB.
- stopCtx, cancel := context.WithTimeout(context.Background(), 2*time.Second)
- defer cancel()
- if e.markStopped(stopCtx, task.ID) {
- common.Info(fmt.Sprintf("Ingestor shutting down; task %s finalized as STOPPED", task.ID))
- } else {
- common.Warn(fmt.Sprintf("Ingestor shutting down; task %s left RUNNING for NATS redelivery", task.ID))
- }
+func (e *Ingestor) ackHandle(hb *Heartbeat, handle common.TaskHandle, taskID string) {
+ hb.Stop()
+ if e.leaseAbandoned(hb) {
+ return
+ }
+ if err := handle.Ack(); err != nil {
+ common.Error(fmt.Sprintf("ack task %s", taskID), err)
+ }
+}
+
+func (e *Ingestor) nackHandle(hb *Heartbeat, handle common.TaskHandle, taskID string) {
+ hb.Stop()
+ if e.leaseAbandoned(hb) {
return
}
+ if err := handle.Nack(); err != nil {
+ common.Error(fmt.Sprintf("nack task %s", taskID), err)
+ }
+}
+
+func (e *Ingestor) renewDuplicateHandle(hb *Heartbeat, handle common.TaskHandle, taskID string) {
+ hb.Stop()
+ if e.leaseAbandoned(hb) {
+ return
+ }
+ if err := handle.InProgress(); err != nil {
+ common.Error(fmt.Sprintf("renew redelivered task %s", taskID), err)
+ }
+}
+
+func (e *Ingestor) registerLease(hb *Heartbeat) {
+ lease := &activeLease{}
+ e.leasesMu.Lock()
+ stop := e.stopLeases.Load()
+ if stop {
+ lease.abandoned.Store(true)
+ }
+ e.activeLeases[hb] = lease
+ e.leasesMu.Unlock()
+ if stop {
+ hb.Stop()
+ }
+}
+
+func (e *Ingestor) unregisterLease(hb *Heartbeat) {
+ e.leasesMu.Lock()
+ delete(e.activeLeases, hb)
+ e.leasesMu.Unlock()
+}
+
+func (e *Ingestor) leaseAbandoned(hb *Heartbeat) bool {
+ e.leasesMu.Lock()
+ lease := e.activeLeases[hb]
+ e.leasesMu.Unlock()
+ return lease != nil && lease.abandoned.Load()
+}
+
+func (e *Ingestor) stopActiveLeases() {
+ e.leasesMu.Lock()
+ e.stopLeases.Store(true)
+ leases := make([]*Heartbeat, 0, len(e.activeLeases))
+ for heartbeat, lease := range e.activeLeases {
+ lease.abandoned.Store(true)
+ leases = append(leases, heartbeat)
+ }
+ e.leasesMu.Unlock()
+
+ for _, lease := range leases {
+ lease.Stop()
+ }
}
func (e *Ingestor) startWorkerPool() {
e.workerOnce.Do(func() {
for i := int32(0); i < e.maxConcurrency; i++ {
e.workerWg.Add(1)
- go e.workerLoop(i)
+ w := &worker{id: i, inbox: make(chan common.TaskHandle)}
+ go e.workerLoop(w)
}
common.Info(fmt.Sprintf("Worker pool started with %d workers", e.maxConcurrency))
})
}
-func (e *Ingestor) workerLoop(id int32) {
+func (e *Ingestor) workerLoop(w *worker) {
defer e.workerWg.Done()
defer e.activeWorkers.Add(-1)
e.activeWorkers.Add(1)
- common.Info(fmt.Sprintf("Worker %d started", id))
+ common.Info(fmt.Sprintf("Worker %d started", w.id))
for {
select {
- case <-e.ctx.Done():
+ case e.workerQueue <- w:
+ case <-e.dispatchCtx.Done():
+ return
+ }
+
+ select {
+ case handle := <-w.inbox:
+ e.handleAndExecute(handle)
+ case <-e.dispatchCtx.Done():
return
- case taskCtx := <-e.taskChan:
- if taskCtx.Kind == taskpkg.TaskKindMemory {
- e.executeMemoryTask(e.ctx, taskCtx)
- continue
- }
- common.Info("task context:" + taskCtx.IngestionTask.ID)
- e.executeTask(e.ctx, taskCtx)
}
}
}
-// executeMemoryTask runs one async memory-extraction task (TaskKindMemory) on
-// a worker of the shared pool. Unlike ingestion tasks, memory tasks have no
-// ingestion_task row / state machine: the runner persists the extracted
-// messages and settles task progress on the way out.
-//
-// The task id comes exclusively from taskCtx.ID() (the envelope TaskID set by
-// the scheduler). It is never re-derived from MemoryPayload — the payload no
-// longer carries identity.
-//
-// Settlement runs in one deferred closed loop on every exit path (success,
-// terminal/transient failure, panic) in a fixed order: stop the admission
-// heartbeat and wait for it to exit, then Ack/Nack exactly once, then release
-// the in-process claim. Stopping the heartbeat before the Ack/Nack is required
-// by Heartbeat's contract — no InProgress may be in flight on the same message
-// while it is being settled. The claim is released after settlement so a
-// redelivery that races the settlement stays owned by this worker; a redelivery
-// after the function returns (e.g. a lost Ack) re-claims and re-runs the task
-// as normal at-least-once delivery.
-//
-// Settlement is error-category aware:
-// - Terminal failure (task row absent, already-failed, or progress=-1 already
-// persisted) is Acked so an already-consumed message is never redelivered
-// into an infinite nack loop.
-// - A task row already at progress>=1.0 is treated as success (the completed
-// short-circuit in HandleSaveToMemoryTask) and Acked.
-// - Transient failure (a task-load DB error before any durable marker, or an
-// LLM/network failure that did not reach progress=-1) is Nacked so the
-// message is redelivered and retried instead of being silently dropped.
-//
-// A panic in the memory path is recovered so the worker goroutine survives:
-// the ingestion path already does this via settleMessage's recover, but a
-// panic here previously crashed the worker. With max_concurrent_workers as low
-// as 1, one panicking memory task could permanently remove the only worker and
-// stall every subsequent document parse. The recovered panic is treated as a
-// transient failure and Nacked for redelivery.
func (e *Ingestor) executeMemoryTask(ctx context.Context, taskCtx *taskpkg.TaskContext) {
- taskID := taskCtx.ID()
+ hb := NewHeartbeat(taskCtx.ID(), taskCtx.Handle, e.heartbeatInterval).WithContext(ctx)
+ hb.Start()
+ e.executeMemoryTaskWithHeartbeat(ctx, taskCtx, hb)
+}
- // The admission path (processMessage) starts the heartbeat before queueing
- // and attaches its stop function via SetStopLease. Direct-execution callers
- // (unit tests, non-queued paths) carry no lease, so start a local heartbeat
- // as a fallback. The two are mutually exclusive — never both running.
- if taskCtx.StopLeaseFn() == nil && taskCtx.Handle != nil && e.heartbeatInterval > 0 {
- hb := NewHeartbeat(taskID, taskCtx.Handle, e.heartbeatInterval).WithContext(e.ctx)
- hb.Start()
- taskCtx.SetStopLease(hb.Stop)
- }
+// executeMemoryTaskWithHeartbeat settles a memory handle with the heartbeat
+// that the receiving worker started before admission.
+func (e *Ingestor) executeMemoryTaskWithHeartbeat(ctx context.Context, taskCtx *taskpkg.TaskContext, hb *Heartbeat) {
+ taskID := taskCtx.ID()
var (
settleAck bool
settleNack bool
)
defer func() {
- // Stop the lease (and wait for any in-flight InProgress) before settling.
- taskCtx.StopLease()
+ hb.Stop()
if r := recover(); r != nil {
- // A panic is treated as a transient failure: Nack so the broker
- // redelivers and retries the message.
common.Error(fmt.Sprintf("memory task %s panicked: %v", taskID, r), fmt.Errorf("%v", r))
settleNack = true
}
+ if taskCtx.Handle == nil || e.leaseAbandoned(hb) {
+ e.releaseTask(taskID)
+ return
+ }
if settleAck {
if err := taskCtx.Handle.Ack(); err != nil {
common.Error(fmt.Sprintf("ack memory task %s", taskID), err)
@@ -672,6 +651,14 @@ func (e *Ingestor) defaultRunMemoryTask(ctx context.Context, taskID string, payl
}
func (e *Ingestor) executeTask(ctx context.Context, taskCtx *taskpkg.TaskContext) {
+ hb := NewHeartbeat(taskCtx.ID(), taskCtx.Handle, e.heartbeatInterval).WithContext(taskCtx.Ctx)
+ hb.Start()
+ e.executeTaskWithHeartbeat(ctx, taskCtx, hb)
+}
+
+// executeTaskWithHeartbeat executes one document task using the lease already
+// started by the worker that owns its Handle.
+func (e *Ingestor) executeTaskWithHeartbeat(ctx context.Context, taskCtx *taskpkg.TaskContext, hb *Heartbeat) {
task := taskCtx.IngestionTask
common.Info(fmt.Sprintf("Starting task %s", task.ID))
@@ -703,7 +690,7 @@ func (e *Ingestor) executeTask(ctx context.Context, taskCtx *taskpkg.TaskContext
perTaskCancel()
}
- e.settleMessage(ctx, taskCtx, func(ctx context.Context) bool {
+ e.settleMessage(ctx, taskCtx, hb, func(ctx context.Context) bool {
return e.runTask(ctx, task)
})
}
@@ -757,6 +744,10 @@ func (e *Ingestor) markFailed(ctx context.Context, taskID string) bool {
func (e *Ingestor) runTask(ctx context.Context, task *entity.IngestionTask) bool {
select {
case <-ctx.Done():
+ if e.ctx.Err() != nil || e.dispatchCtx.Err() != nil {
+ common.Info(fmt.Sprintf("Task %s aborted due to ingestor shutdown, leaving for redelivery", task.ID))
+ return false
+ }
common.Info(fmt.Sprintf("Task %s cancelled", task.ID))
e.markCancelProgress(task)
stopped := e.markStopped(context.Background(), task.ID)
@@ -823,6 +814,10 @@ func (e *Ingestor) runTask(ctx context.Context, task *entity.IngestionTask) bool
if err := e.runDocumentTask(ctx, task); err != nil {
if errors.Is(err, context.Canceled) {
+ if e.ctx.Err() != nil || e.dispatchCtx.Err() != nil {
+ common.Info(fmt.Sprintf("Task %s pipeline interrupted by ingestor shutdown, leaving for redelivery", task.ID))
+ return false
+ }
common.Info(fmt.Sprintf("Task %s cancelled during pipeline", task.ID))
e.markCancelProgress(task)
stopped := e.markStopped(ctx, task.ID)
@@ -918,28 +913,29 @@ func (e *Ingestor) settleToTerminal(ctx context.Context, taskID string) error {
}
}
-// settleMessage runs body under a heartbeat, then settles the MQ message. The
-// heartbeat is stopped (and waited on) before ack/nack — see Heartbeat.Stop.
+// settleMessage runs body under an injected heartbeat lease, then settles the MQ message.
+// The heartbeat is stopped (and waited on) before ack/nack — see Heartbeat.Stop.
// A panic in body is recovered: the task is marked FAILED and the message is
// Nacked for redelivery, so a single task's panic never crashes the worker.
// Settlement queries the DB for the task's actual status: a terminal state
// (COMPLETED/STOPPED/FAILED) means Ack; anything else means Nack. The body's
// return value is advisory only — DB truth is authoritative (BP1).
-func (e *Ingestor) settleMessage(ctx context.Context, taskCtx *taskpkg.TaskContext, body func(context.Context) bool) (terminal bool) {
- hb := NewHeartbeat(taskCtx.ID(), taskCtx.Handle, e.heartbeatInterval).WithContext(taskCtx.Ctx)
- hb.Start()
+func (e *Ingestor) settleMessage(ctx context.Context, taskCtx *taskpkg.TaskContext, hb *Heartbeat, body func(context.Context) bool) (terminal bool) {
defer func() {
hb.Stop() // stop heartbeat (and wait) before ack/nack
if r := recover(); r != nil {
// Recover the panic so the worker process survives. Mark the
// task FAILED so a redelivery does not re-run a poison message
- // (processMessage Ack-skips an already-FAILED task); Nack for
+ // (handleAndExecute Ack-skips an already-FAILED task); Nack for
// redelivery. The broker's redelivery limit handles deterministic
// poison messages.
common.Error(fmt.Sprintf("task %s panicked: %v", taskCtx.IngestionTask.ID, r), fmt.Errorf("%v", r))
e.markFailed(ctx, taskCtx.IngestionTask.ID)
terminal = false
}
+ if e.leaseAbandoned(hb) {
+ return
+ }
// Settlement authority is the DB, not the in-memory bool (BP1).
// Fall back to the in-memory bool only when the DB is unavailable.
if dbTerminal, ok := e.safeGetTerminal(ctx, taskCtx.IngestionTask.ID); ok {
@@ -1093,8 +1089,7 @@ func (e *Ingestor) markTimeoutProgress(task *entity.IngestionTask) {
// worker has already claimed it (e.g. MQ redelivery), true on first claim.
// The claim is released by releaseTask when the worker finishes, so a future
// redelivery (after restart) can re-claim the task. Broker lease renewal is a
-// separate concern handled by Heartbeat (doc: settleMessage; memory: admission
-// in processMessage + stop via TaskContext.StopLease).
+// separate concern handled by the worker-owned Heartbeat until settlement.
func (e *Ingestor) claimTask(taskID string) bool {
e.tasksMu.Lock()
defer e.tasksMu.Unlock()
@@ -1171,19 +1166,16 @@ func (e *Ingestor) recordTerminalPipelineLog(ctx context.Context, ingestionTask
}
}
-// Stop gracefully shuts down the ingestor. It cancels the root context so
-// idle workers exit immediately and in-flight pipelines abort at their next
-// ctx.Err() check, then waits for workers to return. The wait is bounded by
-// ctx: a stage that does not honor cancellation (e.g. a native CGO parse)
-// would otherwise block workerWg.Wait() indefinitely; when ctx expires Stop
-// returns and leaves the broker to redeliver any in-flight messages
-// (at-least-once). Callers must pass a deadline-bearing context.
+// Stop first stops worker registration and the dispatcher's Pull, then cancels
+// execution. The caller's deadline bounds the wait for non-cooperative tasks.
func (e *Ingestor) Stop(ctx context.Context) {
common.Info(fmt.Sprintf("Stopping ingestor %s", e.id))
- e.cancel()
+ e.dispatchCancel()
waitDone := make(chan struct{})
go func() {
+ e.dispatcherWg.Wait()
+ e.cancel()
e.workerWg.Wait()
e.compileWg.Wait()
close(waitDone)
@@ -1193,6 +1185,8 @@ func (e *Ingestor) Stop(ctx context.Context) {
case <-waitDone:
common.Info("All tasks completed")
case <-ctx.Done():
+ e.stopActiveLeases()
+ e.cancel()
e.tasksMu.RLock()
ids := make([]string, 0, len(e.currentTasks))
for id := range e.currentTasks {
diff --git a/internal/ingestion/service/ingestor_lifecycle_test.go b/internal/ingestion/service/ingestor_lifecycle_test.go
index 1594cae941..3bf1accb5c 100644
--- a/internal/ingestion/service/ingestor_lifecycle_test.go
+++ b/internal/ingestion/service/ingestor_lifecycle_test.go
@@ -18,6 +18,7 @@ package service
import (
"context"
+ "encoding/json"
"errors"
"strings"
"testing"
@@ -28,6 +29,7 @@ import (
"ragflow/internal/entity"
taskpkg "ragflow/internal/ingestion/task"
"ragflow/internal/ingestion/testutil"
+ servicepkg "ragflow/internal/service"
)
type startupTaskPublisher struct {
@@ -69,8 +71,11 @@ func TestStartWorkerPool_StartOnceIdempotent(t *testing.T) {
t.Fatalf("activeWorkers after second startWorkerPool = %d, want %d (sync.Once not idempotent)", got, concurrency)
}
- ingestor.cancel()
+ ingestor.dispatchCancel()
ingestor.workerWg.Wait()
+ if got := ingestor.activeWorkers.Load(); got != 0 {
+ t.Fatalf("activeWorkers after worker shutdown = %d, want 0", got)
+ }
}
// TestStop_GracefulShutdown verifies that Stop cancels the context and waits
@@ -120,7 +125,7 @@ func TestStop_TimesOutWhenWorkerStuck(t *testing.T) {
db := testutil.SetupTestDB(t)
cleanup := testutil.ReplaceDBForTest(t, db)
defer cleanup()
- _, _, docID, taskID := testutil.SeedTestData(t, db, testutil.WithPipelineID("flow-1"))
+ _, _, _, taskID := testutil.SeedTestData(t, db, testutil.WithPipelineID("flow-1"))
const concurrency int32 = 1
ingestor := newUnitIngestor("test-stuck", concurrency, []string{"pdf"})
@@ -142,10 +147,8 @@ func TestStop_TimesOutWhenWorkerStuck(t *testing.T) {
t.Fatalf("set task RUNNING: %v", err)
}
- taskCtx := taskpkg.NewTaskContextForScheduling(ingestor.ctx, &entity.IngestionTask{
- ID: taskID, DocumentID: docID, DatasetID: "kb-1", Status: common.RUNNING,
- })
- ingestor.taskChan <- taskCtx
+ w := <-ingestor.workerQueue
+ w.inbox <- &fakeTaskHandle{msg: common.TaskMessage{TaskID: taskID, TaskType: common.TaskTypeIngestionTask}}
select {
case <-started:
@@ -174,6 +177,103 @@ func TestStop_TimesOutWhenWorkerStuck(t *testing.T) {
ingestor.workerWg.Wait()
}
+// TestStopDeadlineStopsStuckWorkerHeartbeat prevents a task that ignores
+// execution cancellation from renewing its broker lease after graceful
+// shutdown has timed out. Once Stop returns at its deadline, the unfinished
+// handle must be left for broker redelivery instead of being kept alive by a
+// leaked heartbeat.
+func TestStopDeadlineStopsStuckWorkerHeartbeat(t *testing.T) {
+ db := testutil.SetupTestDB(t)
+ cleanup := testutil.ReplaceDBForTest(t, db)
+ defer cleanup()
+ _, _, _, taskID := testutil.SeedTestData(t, db, testutil.WithPipelineID("flow-1"))
+
+ ingestor := newUnitIngestor("test-stuck-heartbeat", 1, []string{"pdf"})
+ ingestor.heartbeatInterval = 10 * time.Millisecond
+ ingestor.startWorkerPool()
+
+ release := make(chan struct{})
+ started := make(chan struct{})
+ ingestor.runDocumentTask = func(context.Context, *entity.IngestionTask) error {
+ close(started)
+ <-release
+ return nil
+ }
+
+ handle := &fakeTaskHandle{msg: common.TaskMessage{TaskID: taskID, TaskType: common.TaskTypeIngestionTask}}
+ w := <-ingestor.workerQueue
+ w.inbox <- handle
+ select {
+ case <-started:
+ case <-time.After(2 * time.Second):
+ t.Fatal("worker did not enter runDocumentTask")
+ }
+
+ deadline := time.Now().Add(time.Second)
+ for handle.inProgress.Load() == 0 && time.Now().Before(deadline) {
+ time.Sleep(time.Millisecond)
+ }
+ if handle.inProgress.Load() == 0 {
+ t.Fatal("heartbeat did not renew the running handle")
+ }
+
+ stopCtx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond)
+ ingestor.Stop(stopCtx)
+ cancel()
+
+ pulsesAtStop := handle.inProgress.Load()
+ time.Sleep(50 * time.Millisecond)
+ if got := handle.inProgress.Load(); got != pulsesAtStop {
+ t.Fatalf("heartbeat renewals after Stop deadline = %d, want none", got-pulsesAtStop)
+ }
+
+ close(release)
+ ingestor.workerWg.Wait()
+ if handle.acks.Load() != 0 || handle.nacks.Load() != 0 {
+ t.Fatalf("timed-out handle settlement = %d Ack / %d Nack, want none", handle.acks.Load(), handle.nacks.Load())
+ }
+}
+
+// TestStopDeadlineLeavesStuckMemoryHandleUnsettled applies the same broker
+// redelivery rule to memory extraction: a non-cooperative memory runner must
+// not settle its old handle after the ingestor's Stop deadline passes.
+func TestStopDeadlineLeavesStuckMemoryHandleUnsettled(t *testing.T) {
+ ingestor := newUnitIngestor("test-stuck-memory", 1, nil)
+ ingestor.memorySvc = &servicepkg.MemoryMessageService{}
+ ingestor.startWorkerPool()
+
+ release := make(chan struct{})
+ started := make(chan struct{})
+ ingestor.runMemoryTask = func(context.Context, string, map[string]any) error {
+ close(started)
+ <-release
+ return nil
+ }
+
+ handle := &fakeTaskHandle{msg: common.TaskMessage{
+ TaskID: "memory-stop-timeout",
+ TaskType: common.TaskTypeMemory,
+ Payload: []byte(`{}`),
+ }}
+ w := <-ingestor.workerQueue
+ w.inbox <- handle
+ select {
+ case <-started:
+ case <-time.After(2 * time.Second):
+ t.Fatal("worker did not enter memory runner")
+ }
+
+ stopCtx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond)
+ ingestor.Stop(stopCtx)
+ cancel()
+
+ close(release)
+ ingestor.workerWg.Wait()
+ if handle.acks.Load() != 0 || handle.nacks.Load() != 0 {
+ t.Fatalf("timed-out memory handle settlement = %d Ack / %d Nack, want none", handle.acks.Load(), handle.nacks.Load())
+ }
+}
+
// TestPollCancel_ExitsWhenDoneClosed verifies that closing the done channel
// causes pollCancel to return even when cancelCheck is blocked (e.g. on a
// long DB query). Without BP3, the initial cancelCheck call runs
@@ -232,6 +332,27 @@ func TestStartNilEngine(t *testing.T) {
}
}
+// TestStartRetainsStartupFailure prevents a second Start call from
+// reporting success after initialization failed on the first attempt.
+func TestStartRetainsStartupFailure(t *testing.T) {
+ previousEngine := engine.GetMessageQueueEngine()
+ engine.SetMessageQueueEngine(nil)
+ t.Cleanup(func() { engine.SetMessageQueueEngine(previousEngine) })
+
+ ingestor := newUnitIngestor("test-startup-failure", 1, nil)
+ t.Cleanup(func() { ingestor.Stop(context.Background()) })
+
+ if err := ingestor.Start(); err == nil {
+ t.Fatal("first Start unexpectedly succeeded with nil engine")
+ }
+ if err := ingestor.Start(); err == nil {
+ t.Fatal("second Start hid the prior startup failure")
+ }
+ if got := ingestor.activeWorkers.Load(); got != 0 {
+ t.Fatalf("active workers after failed Start = %d, want 0", got)
+ }
+}
+
// TestExecuteTask_MarkFailedAfterCtxCancelAcks verifies that a generic task
// failure is persisted and acknowledged even after the task context is
// cancelled by the pipeline.
@@ -352,3 +473,73 @@ func TestStartSchedulesCreatedTasks(t *testing.T) {
t.Fatalf("task status = %q, want %q", task.Status, common.SCHEDULED)
}
}
+
+// TestWorkerDispatcherDoesNotActivateTaskUntilWorkerReceivesTask prevents a busy
+// worker from prefetching its next task. task-2 must remain SCHEDULED while
+// task-1 still occupies the only worker.
+func TestWorkerDispatcherDoesNotActivateTaskUntilWorkerReceivesTask(t *testing.T) {
+ db := testutil.SetupTestDB(t)
+ cleanup := testutil.ReplaceDBForTest(t, db)
+ t.Cleanup(cleanup)
+
+ taskIDs := seedBurstTasks(t, db, 2)
+ for _, taskID := range taskIDs {
+ if err := db.Model(&entity.IngestionTask{}).Where("id = ?", taskID).
+ Update("status", common.SCHEDULED).Error; err != nil {
+ t.Fatalf("schedule task %s: %v", taskID, err)
+ }
+ }
+
+ queue := testutil.SetupNatsEngine(t)
+ previousEngine := engine.GetMessageQueueEngine()
+ engine.SetMessageQueueEngine(queue)
+ t.Cleanup(func() { engine.SetMessageQueueEngine(previousEngine) })
+
+ ingestor := newUnitIngestor("test-worker-dispatch", 1, []string{"pdf"})
+ releaseFirst := make(chan struct{})
+ firstStarted := make(chan struct{})
+ ingestor.runDocumentTask = func(_ context.Context, task *entity.IngestionTask) error {
+ if task.ID == taskIDs[0] {
+ close(firstStarted)
+ <-releaseFirst
+ }
+ return nil
+ }
+ t.Cleanup(func() {
+ close(releaseFirst)
+ ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+ defer cancel()
+ ingestor.Stop(ctx)
+ })
+
+ for _, taskID := range taskIDs {
+ payload, err := json.Marshal(common.TaskMessage{
+ TaskID: taskID,
+ TaskType: common.TaskTypeIngestionTask,
+ })
+ if err != nil {
+ t.Fatalf("marshal task %s: %v", taskID, err)
+ }
+ if err := queue.PublishTask(common.TaskSubject, payload); err != nil {
+ t.Fatalf("publish task %s: %v", taskID, err)
+ }
+ }
+
+ if err := ingestor.Start(); err != nil {
+ t.Fatalf("Start: %v", err)
+ }
+ select {
+ case <-firstStarted:
+ case <-time.After(5 * time.Second):
+ t.Fatal("first task did not start")
+ }
+
+ time.Sleep(150 * time.Millisecond)
+ var second entity.IngestionTask
+ if err := db.Where("id = ?", taskIDs[1]).First(&second).Error; err != nil {
+ t.Fatalf("load second task: %v", err)
+ }
+ if second.Status != common.SCHEDULED {
+ t.Fatalf("second task status = %q, want %q while the only worker is busy", second.Status, common.SCHEDULED)
+ }
+}
diff --git a/internal/ingestion/service/process_message_test.go b/internal/ingestion/service/process_message_test.go
deleted file mode 100644
index 0d5e5383c9..0000000000
--- a/internal/ingestion/service/process_message_test.go
+++ /dev/null
@@ -1,765 +0,0 @@
-package service
-
-import (
- "context"
- "encoding/json"
- "errors"
- "testing"
- "time"
-
- "ragflow/internal/common"
- "ragflow/internal/dao"
- "ragflow/internal/entity"
- taskpkg "ragflow/internal/ingestion/task"
- "ragflow/internal/ingestion/testutil"
- "ragflow/internal/service"
-)
-
-func newFakeHandle(taskID, taskType string) *fakeTaskHandle {
- return &fakeTaskHandle{msg: common.TaskMessage{TaskID: taskID, TaskType: taskType}}
-}
-
-// TestProcessMessage_MemoryTaskDispatches verifies that a task_type="memory"
-// message (with a NATS payload) is dispatched to the shared worker pool as a
-// TaskKindMemory context rather than being acked-skipped as a non-ingestion
-// task. It must NOT touch the ingestion state machine (no StartRunning).
-func TestProcessMessage_MemoryTaskDispatches(t *testing.T) {
- db := testutil.SetupTestDB(t)
- cleanup := testutil.ReplaceDBForTest(t, db)
- defer cleanup()
-
- ingestor := newUnitIngestor("test", 1, []string{"pdf"})
- // Memory extractor must be enabled for the memory branch to enqueue.
- ingestor.SetMemoryMessageService(service.NewMemoryMessageService(nil))
-
- payload, err := json.Marshal(map[string]any{
- "memory_id": "mem-1",
- "source_id": 42,
- "message_dict": map[string]any{
- "user_id": "u1",
- "agent_id": "a1",
- "session_id": "s1",
- "user_input": "hi",
- "agent_response": "hello",
- },
- })
- if err != nil {
- t.Fatalf("marshal payload: %v", err)
- }
- handle := &fakeTaskHandle{msg: common.TaskMessage{TaskID: "mem-task-1", TaskType: common.TaskTypeMemory, Payload: payload}}
-
- ingestor.processMessage(handle)
- if handle.acks.Load() != 0 || handle.nacks.Load() != 0 {
- t.Fatalf("memory dispatch: expected 0 Ack/0 Nack (settled by worker), got acks=%d nacks=%d", handle.acks.Load(), handle.nacks.Load())
- }
- if len(ingestor.taskChan) != 1 {
- t.Fatalf("expected 1 memory task enqueued, got %d", len(ingestor.taskChan))
- }
- taskCtx := <-ingestor.taskChan
- if taskCtx.Kind != taskpkg.TaskKindMemory {
- t.Fatalf("task kind = %v, want TaskKindMemory", taskCtx.Kind)
- }
- if taskCtx.IngestionTask != nil {
- t.Fatalf("memory task must not carry an IngestionTask (no ingestion state machine), got %+v", taskCtx.IngestionTask)
- }
- if taskCtx.MemoryPayload == nil || taskCtx.MemoryPayload["memory_id"] != "mem-1" {
- t.Fatalf("memory payload not carried correctly: %+v", taskCtx.MemoryPayload)
- }
-}
-
-// TestProcessMessage_MemoryTaskDisabledAcks verifies that when the memory
-// extractor is not installed (nil), a memory task is acked and skipped so it
-// does not loop forever on the worker pool.
-func TestProcessMessage_MemoryTaskDisabledAcks(t *testing.T) {
- db := testutil.SetupTestDB(t)
- cleanup := testutil.ReplaceDBForTest(t, db)
- defer cleanup()
-
- ingestor := newUnitIngestor("test", 1, []string{"pdf"}) // memorySvc nil by default
- handle := newFakeHandle("mem-task-2", common.TaskTypeMemory)
-
- ingestor.processMessage(handle)
- if handle.acks.Load() != 1 || handle.nacks.Load() != 0 {
- t.Fatalf("memory disabled: expected 1 Ack/0 Nack, got acks=%d nacks=%d", handle.acks.Load(), handle.nacks.Load())
- }
- if len(ingestor.taskChan) != 0 {
- t.Fatal("expected no memory task enqueued when extractor disabled")
- }
-}
-
-// TestExecuteMemoryTaskAlreadyFailedAcks verifies the Ack-on-failure contract in
-// the case that matters: the task row is already persisted with progress=-1 (the
-// durable "failed" marker written by HandleSaveToMemoryTask). In that state
-// HandleSaveToMemoryTask returns "already failed", and executeMemoryTask must
-// Ack — a Nack would redeliver the message into an infinite retry loop against
-// an already-failed row. Using a real taskDAO + MemoryService (not a nil-guard)
-// so the test exercises the "task already failed" branch, not the nil-guard.
-func TestExecuteMemoryTaskAlreadyFailedAcks(t *testing.T) {
- db := testutil.SetupTestDB(t)
- cleanup := testutil.ReplaceDBForTest(t, db)
- defer cleanup()
-
- // Seed a task row already marked failed (progress=-1), mirroring what
- // HandleSaveToMemoryTask persists before returning an error.
- taskID := "mem-task-3"
- if err := dao.DB.Create(&entity.Task{ID: taskID, DocID: "doc-mem-3", Progress: -1}).Error; err != nil {
- t.Fatalf("insert already-failed task: %v", err)
- }
-
- ingestor := newUnitIngestor("test", 1, []string{"pdf"})
- // Real memory service (non-nil memories) so HandleSaveToMemoryTask gets past
- // the nil-guard and reaches the progress==-1 "already failed" branch.
- ingestor.SetMemoryMessageService(service.NewMemoryMessageService(service.NewMemoryService()))
-
- handle := &fakeTaskHandle{msg: common.TaskMessage{TaskID: taskID, TaskType: common.TaskTypeMemory}}
- taskCtx := taskpkg.NewMemoryTaskContextForScheduling(context.Background(), taskID, map[string]any{
- "memory_id": "mem-3", "source_id": 7,
- "message_dict": map[string]any{"user_id": "u", "agent_id": "a", "session_id": "s"},
- }, handle)
-
- // Prove the failure precondition: the seeded progress=-1 row must make
- // HandleSaveToMemoryTask return the "already failed" error. Otherwise the
- // Ack below would only reflect the success path and prove nothing about
- // the Ack-on-failure contract.
- if err := ingestor.memorySvc.HandleSaveToMemoryTask(context.Background(), taskID, taskCtx.MemoryPayload); err == nil {
- t.Fatal("expected HandleSaveToMemoryTask to fail on an already-failed (progress=-1) task, got nil")
- }
-
- ingestor.executeMemoryTask(context.Background(), taskCtx)
- if handle.acks.Load() != 1 || handle.nacks.Load() != 0 {
- t.Fatalf("already-failed memory task: expected 1 Ack/0 Nack, got acks=%d nacks=%d", handle.acks.Load(), handle.nacks.Load())
- }
-}
-
-// TestExecuteMemoryTaskTransientFailureNacks verifies that a transient (non-terminal)
-// failure from HandleSaveToMemoryTask — e.g. a task-load DB error before any durable
-// progress=-1 marker is written — is Nacked so the message is redelivered, instead of
-// being Acked and permanently dropped. The tasks table is intentionally NOT migrated
-// so taskDAO.GetByID fails with a "no such table" error rather than gorm.ErrRecordNotFound.
-func TestExecuteMemoryTaskTransientFailureNacks(t *testing.T) {
- // Migrate only a table unrelated to tasks so GetByID returns a transient
- // "no such table: tasks" error (not gorm.ErrRecordNotFound).
- db := testutil.SetupTestDB(t, &entity.IngestionTask{})
- cleanup := testutil.ReplaceDBForTest(t, db)
- defer cleanup()
-
- ingestor := newUnitIngestor("test", 1, []string{"pdf"})
- ingestor.SetMemoryMessageService(service.NewMemoryMessageService(service.NewMemoryService()))
-
- handle := &fakeTaskHandle{msg: common.TaskMessage{TaskID: "mem-task-x", TaskType: common.TaskTypeMemory}}
- taskCtx := taskpkg.NewMemoryTaskContextForScheduling(context.Background(), "mem-task-x", map[string]any{
- "memory_id": "mem-x", "source_id": 1,
- "message_dict": map[string]any{"user_id": "u", "agent_id": "a", "session_id": "s"},
- }, handle)
-
- // Precondition: with no tasks table, HandleSaveToMemoryTask must fail with a
- // transient error that is NOT wrapped in ErrMemoryTaskTerminal.
- err := ingestor.memorySvc.HandleSaveToMemoryTask(context.Background(), "mem-task-x", taskCtx.MemoryPayload)
- if err == nil {
- t.Fatal("expected HandleSaveToMemoryTask to fail with missing tasks table, got nil")
- }
- if errors.Is(err, service.ErrMemoryTaskTerminal) {
- t.Fatalf("expected a transient error, got terminal: %v", err)
- }
-
- ingestor.executeMemoryTask(context.Background(), taskCtx)
- if handle.nacks.Load() != 1 || handle.acks.Load() != 0 {
- t.Fatalf("transient memory task failure: expected 0 Ack/1 Nack, got acks=%d nacks=%d", handle.acks.Load(), handle.nacks.Load())
- }
-}
-
-// TestProcessMessage_NonIngestionTaskAcks: a non-ingestion task is acked and
-// skipped without touching the task DB or enqueuing.
-func TestProcessMessage_NonIngestionTaskAcks(t *testing.T) {
- db := testutil.SetupTestDB(t)
- cleanup := testutil.ReplaceDBForTest(t, db)
- defer cleanup()
-
- ingestor := newUnitIngestor("test", 1, []string{"pdf"})
- handle := newFakeHandle("task-1", "not-ingestion")
-
- ingestor.processMessage(handle)
- if handle.acks.Load() != 1 || handle.nacks.Load() != 0 {
- t.Fatalf("non-ingestion: expected 1 Ack/0 Nack, got acks=%d nacks=%d", handle.acks.Load(), handle.nacks.Load())
- }
- if len(ingestor.taskChan) != 0 {
- t.Fatal("expected no task enqueued")
- }
-}
-
-// TestProcessMessage_TaskNotFoundAcks: when StartRunning returns
-// ErrTaskNotFound the message is acked and skipped.
-func TestProcessMessage_TaskNotFoundAcks(t *testing.T) {
- db := testutil.SetupTestDB(t)
- cleanup := testutil.ReplaceDBForTest(t, db)
- defer cleanup()
-
- ingestor := newUnitIngestor("test", 1, []string{"pdf"})
- // No task seeded in DB — StartRunning returns ErrTaskNotFound.
- handle := newFakeHandle("no-such-task", common.TaskTypeIngestionTask)
-
- ingestor.processMessage(handle)
- if handle.acks.Load() != 1 || handle.nacks.Load() != 0 {
- t.Fatalf("not-found: expected 1 Ack/0 Nack, got acks=%d nacks=%d", handle.acks.Load(), handle.nacks.Load())
- }
-}
-
-// TestProcessMessage_CreatedTaskStartsRunning verifies that a worker can claim
-// a task after NATS accepts its message but before the API records SCHEDULED.
-func TestProcessMessage_CreatedTaskStartsRunning(t *testing.T) {
- db := testutil.SetupTestDB(t)
- cleanup := testutil.ReplaceDBForTest(t, db)
- defer cleanup()
- _, _, _, taskID := testutil.SeedTestData(t, db, testutil.WithPipelineID("flow-1"))
-
- if err := db.Model(&entity.IngestionTask{}).Where("id = ?", taskID).
- Update("status", common.CREATED).Error; err != nil {
- t.Fatalf("reset task to CREATED: %v", err)
- }
-
- ingestor := newUnitIngestor("test", 1, []string{"pdf"})
- handle := newFakeHandle(taskID, common.TaskTypeIngestionTask)
-
- ingestor.processMessage(handle)
- if handle.acks.Load() != 0 || handle.nacks.Load() != 0 {
- t.Fatalf("CREATED task: expected 0 Ack/0 Nack (deferred), got acks=%d nacks=%d", handle.acks.Load(), handle.nacks.Load())
- }
- if len(ingestor.taskChan) != 1 {
- t.Fatalf("expected 1 task enqueued, got %d", len(ingestor.taskChan))
- }
-
- var task entity.IngestionTask
- if err := db.Where("id = ?", taskID).First(&task).Error; err != nil {
- t.Fatalf("reload task: %v", err)
- }
- if task.Status != common.RUNNING {
- t.Fatalf("status = %q, want %q", task.Status, common.RUNNING)
- }
-}
-
-// TestProcessMessage_AlreadyCompletedAcks: a task already in a terminal state
-// (COMPLETED) is acked and skipped — no enqueue, and the document is NOT
-// resurrected to RUNNING. A redelivered terminal task must not reset a
-// finished document's run status or counters.
-func TestProcessMessage_AlreadyCompletedAcks(t *testing.T) {
- db := testutil.SetupTestDB(t)
- cleanup := testutil.ReplaceDBForTest(t, db)
- defer cleanup()
- _, _, docID, taskID := testutil.SeedTestData(t, db, testutil.WithPipelineID("flow-1"))
-
- // Mark the document as a finished parse: a non-RUNNING run status and
- // non-zero counters that StartRunning would clobber to "1"/0.
- finishedRun := "3"
- if err := db.Model(&entity.Document{}).Where("id = ?", docID).
- Updates(map[string]interface{}{
- "run": finishedRun,
- "chunk_num": 42,
- "token_num": 99,
- }).Error; err != nil {
- t.Fatalf("set document finished sentinel: %v", err)
- }
-
- // Set the task COMPLETED so the status switch skips it.
- if err := db.Model(&entity.IngestionTask{}).Where("id = ?", taskID).
- Update("status", common.COMPLETED).Error; err != nil {
- t.Fatalf("set COMPLETED: %v", err)
- }
-
- ingestor := newUnitIngestor("test", 1, []string{"pdf"})
- handle := newFakeHandle(taskID, common.TaskTypeIngestionTask)
-
- ingestor.processMessage(handle)
- if handle.acks.Load() != 1 || handle.nacks.Load() != 0 {
- t.Fatalf("completed: expected 1 Ack/0 Nack, got acks=%d nacks=%d", handle.acks.Load(), handle.nacks.Load())
- }
- if len(ingestor.taskChan) != 0 {
- t.Fatal("expected no task enqueued for completed task")
- }
-
- // The finished document must be untouched - no resurrection to RUNNING.
- var doc entity.Document
- if err := db.Where("id = ?", docID).First(&doc).Error; err != nil {
- t.Fatalf("reload document: %v", err)
- }
- if doc.Run == nil || *doc.Run != finishedRun {
- t.Fatalf("document.run = %v, want %q (must not be resurrected to RUNNING)", doc.Run, finishedRun)
- }
- if doc.ChunkNum != 42 || doc.TokenNum != 99 {
- t.Fatalf("document counters changed: chunk_num=%d token_num=%d, want 42/99", doc.ChunkNum, doc.TokenNum)
- }
-}
-
-// TestProcessMessage_ClaimFailsRenewsLease: a RUNNING task whose claim fails
-// is already owned by another worker. The duplicate must not Ack the broker
-// message before that owner reaches a durable outcome; it only renews the
-// delivery lease and stays out of the worker queue.
-func TestProcessMessage_ClaimFailsRenewsLease(t *testing.T) {
- db := testutil.SetupTestDB(t)
- cleanup := testutil.ReplaceDBForTest(t, db)
- defer cleanup()
- _, _, _, taskID := testutil.SeedTestData(t, db, testutil.WithPipelineID("flow-1"))
-
- ingestor := newUnitIngestor("test", 1, []string{"pdf"})
- // Pre-claim the task so processMessage sees a claim conflict.
- ingestor.claimTask(taskID)
-
- handle := newFakeHandle(taskID, common.TaskTypeIngestionTask)
-
- ingestor.processMessage(handle)
- if handle.acks.Load() != 0 || handle.nacks.Load() != 0 {
- t.Fatalf("claim-fail: expected 0 Ack/0 Nack while owner runs, got acks=%d nacks=%d", handle.acks.Load(), handle.nacks.Load())
- }
- if handle.inProgress.Load() != 1 {
- t.Fatalf("claim-fail: expected duplicate lease renewal, got InProgress=%d", handle.inProgress.Load())
- }
- if len(ingestor.taskChan) != 0 {
- t.Fatal("expected no task enqueued when claim fails")
- }
-}
-
-// TestProcessMessage_ClaimSucceedsEnqueues: a RUNNING task with a successful
-// claim is enqueued to the worker pool and the message is NOT settled yet
-// (ack/nack is deferred to the worker).
-func TestProcessMessage_ClaimSucceedsEnqueues(t *testing.T) {
- db := testutil.SetupTestDB(t)
- cleanup := testutil.ReplaceDBForTest(t, db)
- defer cleanup()
- _, _, _, taskID := testutil.SeedTestData(t, db, testutil.WithPipelineID("flow-1"))
-
- ingestor := newUnitIngestor("test", 1, []string{"pdf"})
- handle := newFakeHandle(taskID, common.TaskTypeIngestionTask)
-
- ingestor.processMessage(handle)
- // Ack/Nack must not be called — settlement is deferred to the worker.
- if handle.acks.Load() != 0 || handle.nacks.Load() != 0 {
- t.Fatalf("enqueued: expected 0 Ack/0 Nack (deferred), got acks=%d nacks=%d", handle.acks.Load(), handle.nacks.Load())
- }
- // Task must be in the channel.
- if len(ingestor.taskChan) != 1 {
- t.Fatalf("expected 1 task enqueued, got %d", len(ingestor.taskChan))
- }
- // Drain and verify.
- taskCtx := <-ingestor.taskChan
- if taskCtx.IngestionTask.ID != taskID {
- t.Fatalf("enqueued task ID = %s, want %s", taskCtx.IngestionTask.ID, taskID)
- }
-}
-
-// TestProcessMessage_ChannelFullBlocksUntilSlot: backpressure must NOT drop
-// the task. When the task channel is at capacity, processMessage blocks on the
-// send (consuming no slot, no Nack) until a worker frees one; the message is
-// then enqueued and settled by the worker. Dropping on backpressure would
-// permanently lose the task once the broker's MaxDeliver is exceeded.
-func TestProcessMessage_ChannelFullBlocksUntilSlot(t *testing.T) {
- db := testutil.SetupTestDB(t)
- cleanup := testutil.ReplaceDBForTest(t, db)
- defer cleanup()
- _, _, _, taskID := testutil.SeedTestData(t, db, testutil.WithPipelineID("flow-1"))
-
- // maxConcurrency=2 → channel cap=4. Fill it completely.
- ingestor := newUnitIngestor("test", 2, []string{"pdf"})
- for i := 0; i < cap(ingestor.taskChan); i++ {
- ingestor.taskChan <- taskpkg.NewTaskContextForScheduling(nil, &entity.IngestionTask{ID: "filler"})
- }
-
- handle := newFakeHandle(taskID, common.TaskTypeIngestionTask)
-
- // processMessage must BLOCK on the full channel: no ack, no nack, no
- // immediate enqueue.
- done := make(chan struct{})
- go func() {
- ingestor.processMessage(handle)
- close(done)
- }()
- select {
- case <-done:
- t.Fatal("processMessage returned while channel was full; it must block under backpressure")
- case <-time.After(100 * time.Millisecond):
- }
- if handle.nacks.Load() != 0 || handle.acks.Load() != 0 {
- t.Fatalf("expected 0 Ack/0 Nack while blocked, got acks=%d nacks=%d", handle.acks.Load(), handle.nacks.Load())
- }
- if len(ingestor.taskChan) != cap(ingestor.taskChan) {
- t.Fatalf("expected channel still full, got %d/%d", len(ingestor.taskChan), cap(ingestor.taskChan))
- }
-
- // Free a slot: the blocked processMessage must now enqueue the task.
- <-ingestor.taskChan
- select {
- case <-done:
- case <-time.After(2 * time.Second):
- t.Fatal("processMessage did not enqueue after a slot freed")
- }
- if handle.nacks.Load() != 0 || handle.acks.Load() != 0 {
- t.Fatalf("expected 0 Ack/0 Nack (settlement deferred to worker), got acks=%d nacks=%d", handle.acks.Load(), handle.nacks.Load())
- }
- // The real task must now be in the channel.
- found := false
- for i := 0; i < cap(ingestor.taskChan); i++ {
- tc := <-ingestor.taskChan
- if tc.IngestionTask != nil && tc.IngestionTask.ID == taskID {
- found = true
- }
- }
- if !found {
- t.Fatal("real task was not enqueued after a slot freed")
- }
-}
-
-// TestProcessMessage_ShutdownRaceMarksStopped: shutdown winning the race
-// against the taskChan send must not leave the task in non-terminal RUNNING
-// with no in-flight worker. StartRunning has already flipped the task to
-// RUNNING, so the ctx.Done branch finalizes it as STOPPED (user-retryable)
-// with a detached timeout, and leaves the message unsettled — the broker
-// redelivers it and the terminal status ack-skips.
-func TestProcessMessage_ShutdownRaceMarksStopped(t *testing.T) {
- db := testutil.SetupTestDB(t)
- cleanup := testutil.ReplaceDBForTest(t, db)
- defer cleanup()
- _, _, _, taskID := testutil.SeedTestData(t, db, testutil.WithPipelineID("flow-1"))
- // SeedTestData creates the task RUNNING; reset to CREATED so the race
- // below is the real one: processMessage's StartRunning flips it RUNNING
- // and then blocks on the full channel.
- if err := db.Model(&entity.IngestionTask{}).Where("id = ?", taskID).
- Update("status", common.CREATED).Error; err != nil {
- t.Fatalf("reset task to CREATED: %v", err)
- }
-
- ingestor := newUnitIngestor("test", 1, []string{"pdf"}) // channel cap = 2
- for i := 0; i < cap(ingestor.taskChan); i++ {
- ingestor.taskChan <- taskpkg.NewTaskContextForScheduling(nil, &entity.IngestionTask{ID: "filler"})
- }
-
- handle := newFakeHandle(taskID, common.TaskTypeIngestionTask)
- done := make(chan struct{})
- go func() {
- ingestor.processMessage(handle)
- close(done)
- }()
-
- // Wait until StartRunning flipped the task RUNNING and processMessage
- // is blocked on the full channel.
- deadline := time.Now().Add(2 * time.Second)
- for {
- var task entity.IngestionTask
- if err := db.Where("id = ?", taskID).First(&task).Error; err == nil && task.Status == common.RUNNING {
- break
- }
- if time.Now().After(deadline) {
- t.Fatal("task never reached RUNNING; processMessage did not reach the blocking send")
- }
- time.Sleep(2 * time.Millisecond)
- }
-
- // Shutdown wins the race against the channel send.
- ingestor.cancel()
-
- select {
- case <-done:
- case <-time.After(5 * time.Second):
- t.Fatal("processMessage did not return after shutdown")
- }
-
- var task entity.IngestionTask
- if err := db.Where("id = ?", taskID).First(&task).Error; err != nil {
- t.Fatalf("reload task: %v", err)
- }
- if task.Status != common.STOPPED {
- t.Fatalf("task status = %q, want %q (shutdown race must not strand a RUNNING row)", task.Status, common.STOPPED)
- }
- // Message unsettled: on redelivery the STOPPED status ack-skips it.
- if handle.acks.Load() != 0 || handle.nacks.Load() != 0 {
- t.Fatalf("expected 0 Ack/0 Nack (unsettled for redelivery), got acks=%d nacks=%d", handle.acks.Load(), handle.nacks.Load())
- }
-}
-
-// TestProcessMessage_StartRunningErrorNacks: when StartRunning returns a
-// non-ErrTaskNotFound error (e.g. a DB blip), processMessage nacks the
-// message for redelivery instead of killing the consumer (B2 fix). The
-// consume loop's resilience relies on this never being a fatal return.
-func TestProcessMessage_StartRunningErrorNacks(t *testing.T) {
- db := testutil.SetupTestDB(t)
- cleanup := testutil.ReplaceDBForTest(t, db)
- defer cleanup()
- _, _, _, taskID := testutil.SeedTestData(t, db, testutil.WithPipelineID("flow-1"))
-
- // Drop the tasks table so GetTask fails with a generic SQL error, not
- // ErrRecordNotFound (which would map to ErrTaskNotFound and ack-skip).
- if err := db.Migrator().DropTable(&entity.IngestionTask{}); err != nil {
- t.Fatalf("drop table: %v", err)
- }
-
- ingestor := newUnitIngestor("test", 1, []string{"pdf"})
- handle := newFakeHandle(taskID, common.TaskTypeIngestionTask)
-
- ingestor.processMessage(handle)
-
- if handle.nacks.Load() != 1 || handle.acks.Load() != 0 {
- t.Fatalf("start-running error: expected 1 Nack/0 Ack (redeliver), got acks=%d nacks=%d", handle.acks.Load(), handle.nacks.Load())
- }
- if len(ingestor.taskChan) != 0 {
- t.Fatal("expected no task enqueued on activation failure")
- }
-}
-
-// TestProcessMessage_MemoryTaskHeartbeatsWhileQueued verifies that a claimed
-// memory message renews its broker lease before a worker starts it. Without
-// this, a task waiting in taskChan can exceed AckWait and consume unnecessary
-// broker deliveries before a worker starts it.
-func TestProcessMessage_MemoryTaskHeartbeatsWhileQueued(t *testing.T) {
- ingestor := newUnitIngestor("test", 1, []string{"pdf"})
- ingestor.SetMemoryMessageService(service.NewMemoryMessageService(nil))
- ingestor.heartbeatInterval = 5 * time.Millisecond
-
- payload, err := json.Marshal(map[string]any{
- "memory_id": "mem-1", "source_id": 42,
- "message_dict": map[string]any{"user_id": "u", "agent_id": "a", "session_id": "s"},
- })
- if err != nil {
- t.Fatalf("marshal payload: %v", err)
- }
- handle := &fakeTaskHandle{msg: common.TaskMessage{TaskID: "mem-queued-hb-1", TaskType: common.TaskTypeMemory, Payload: payload}}
-
- ingestor.processMessage(handle)
-
- deadline := time.Now().Add(2 * time.Second)
- for handle.inProgress.Load() == 0 && time.Now().Before(deadline) {
- time.Sleep(5 * time.Millisecond)
- }
- if handle.inProgress.Load() == 0 {
- t.Fatal("expected InProgress heartbeat while memory task was queued without a worker")
- }
-
- // Drain and stop the lease so the heartbeat goroutine does not leak.
- taskCtx := <-ingestor.taskChan
- taskCtx.StopLease()
- if handle.acks.Load() != 0 || handle.nacks.Load() != 0 {
- t.Fatalf("queued memory task must not be settled by admission, got acks=%d nacks=%d", handle.acks.Load(), handle.nacks.Load())
- }
-}
-
-// TestProcessMessage_MemoryTaskRedeliveryRenewsLease verifies the claim guard on
-// the memory dispatch path: when a memory task is redelivered by the broker
-// while the first copy is still queued/in-flight, the duplicate must renew its
-// lease without Acking it or enqueuing a second worker. A different task id
-// must still be accepted (claim is per-task-id).
-func TestProcessMessage_MemoryTaskRedeliveryRenewsLease(t *testing.T) {
- ingestor := newUnitIngestor("test", 1, []string{"pdf"})
- ingestor.SetMemoryMessageService(service.NewMemoryMessageService(nil))
-
- payload, err := json.Marshal(map[string]any{
- "memory_id": "mem-1", "source_id": 42,
- "message_dict": map[string]any{"user_id": "u", "agent_id": "a", "session_id": "s"},
- })
- if err != nil {
- t.Fatalf("marshal payload: %v", err)
- }
-
- // First delivery: claim succeeds, task is enqueued, message unsettled.
- first := &fakeTaskHandle{msg: common.TaskMessage{TaskID: "mem-redeliver-1", TaskType: common.TaskTypeMemory, Payload: payload}}
- ingestor.processMessage(first)
- if first.acks.Load() != 0 || first.nacks.Load() != 0 {
- t.Fatalf("first delivery: expected 0 Ack/0 Nack (settled by worker), got acks=%d nacks=%d", first.acks.Load(), first.nacks.Load())
- }
- if len(ingestor.taskChan) != 1 {
- t.Fatalf("first delivery: expected 1 memory task enqueued, got %d", len(ingestor.taskChan))
- }
-
- // Redelivery while the first copy is still queued: claim fails, the
- // duplicate must renew its lease and NOT be enqueued.
- dup := &fakeTaskHandle{msg: common.TaskMessage{TaskID: "mem-redeliver-1", TaskType: common.TaskTypeMemory, Payload: payload}}
- ingestor.processMessage(dup)
- if dup.acks.Load() != 0 || dup.nacks.Load() != 0 {
- t.Fatalf("redelivered copy: expected 0 Ack/0 Nack while owner runs, got acks=%d nacks=%d", dup.acks.Load(), dup.nacks.Load())
- }
- if dup.inProgress.Load() != 1 {
- t.Fatalf("redelivered copy: expected 1 lease renewal, got InProgress=%d", dup.inProgress.Load())
- }
- if len(ingestor.taskChan) != 1 {
- t.Fatalf("redelivered copy must not be enqueued, got %d tasks in channel", len(ingestor.taskChan))
- }
-
- // A different memory task must still be accepted (claim is per-task-id).
- otherPayload, err := json.Marshal(map[string]any{
- "memory_id": "mem-2", "source_id": 43,
- "message_dict": map[string]any{"user_id": "u", "agent_id": "a", "session_id": "s"},
- })
- if err != nil {
- t.Fatalf("marshal other payload: %v", err)
- }
- other := &fakeTaskHandle{msg: common.TaskMessage{TaskID: "mem-redeliver-2", TaskType: common.TaskTypeMemory, Payload: otherPayload}}
- ingestor.processMessage(other)
- if len(ingestor.taskChan) != 2 {
- t.Fatalf("different memory task should be enqueued, got %d tasks in channel", len(ingestor.taskChan))
- }
-
- // Drain both and stop leases so heartbeat goroutines do not leak.
- firstCtx := <-ingestor.taskChan
- firstCtx.StopLease()
- secondCtx := <-ingestor.taskChan
- secondCtx.StopLease()
-}
-
-// TestProcessMessage_MemoryTaskEmptyIDAcks verifies that a memory task with an
-// empty envelope task id is Acked and skipped before claiming — an empty claim
-// key would otherwise strand a no-op entry in currentTasks.
-func TestProcessMessage_MemoryTaskEmptyIDAcks(t *testing.T) {
- ingestor := newUnitIngestor("test", 1, []string{"pdf"})
- ingestor.SetMemoryMessageService(service.NewMemoryMessageService(nil))
-
- payload, err := json.Marshal(map[string]any{
- "memory_id": "mem-1", "source_id": 42,
- "message_dict": map[string]any{"user_id": "u", "agent_id": "a", "session_id": "s"},
- })
- if err != nil {
- t.Fatalf("marshal payload: %v", err)
- }
- handle := &fakeTaskHandle{msg: common.TaskMessage{TaskID: "", TaskType: common.TaskTypeMemory, Payload: payload}}
-
- ingestor.processMessage(handle)
- if handle.acks.Load() != 1 || handle.nacks.Load() != 0 {
- t.Fatalf("empty-id memory task: expected 1 Ack/0 Nack, got acks=%d nacks=%d", handle.acks.Load(), handle.nacks.Load())
- }
- if len(ingestor.taskChan) != 0 {
- t.Fatalf("empty-id memory task must not be enqueued, got %d tasks", len(ingestor.taskChan))
- }
- if _, claimed := ingestor.currentTasks[""]; claimed {
- t.Fatal("empty-id memory task must not claim the empty key")
- }
-}
-
-// TestExecuteMemoryTask_ReleasesClaim verifies that executeMemoryTask releases
-// the claim taken by processMessage once the worker finishes, so a later
-// redelivery (e.g. after restart) can re-claim and re-run the task instead of
-// being permanently ack-skipped.
-func TestExecuteMemoryTask_ReleasesClaim(t *testing.T) {
- ingestor := newUnitIngestor("test", 1, []string{"pdf"})
- ingestor.SetMemoryMessageService(service.NewMemoryMessageService(nil))
- // Stub the runner so no DB / LLM is touched.
- ingestor.runMemoryTask = func(_ context.Context, _ string, _ map[string]any) error {
- return nil
- }
-
- if !ingestor.claimTask("mem-release-1") {
- t.Fatal("expected claim to succeed")
- }
-
- handle := &fakeTaskHandle{msg: common.TaskMessage{TaskID: "mem-release-1", TaskType: common.TaskTypeMemory}}
- taskCtx := taskpkg.NewMemoryTaskContextForScheduling(context.Background(), "mem-release-1", map[string]any{
- "memory_id": "mem-r", "source_id": 1,
- "message_dict": map[string]any{"user_id": "u", "agent_id": "a", "session_id": "s"},
- }, handle)
-
- ingestor.executeMemoryTask(context.Background(), taskCtx)
-
- if handle.acks.Load() != 1 || handle.nacks.Load() != 0 {
- t.Fatalf("expected 1 Ack/0 Nack on success, got acks=%d nacks=%d", handle.acks.Load(), handle.nacks.Load())
- }
- if _, stillClaimed := ingestor.currentTasks["mem-release-1"]; stillClaimed {
- t.Fatal("expected claim released after executeMemoryTask finished")
- }
- if !ingestor.claimTask("mem-release-1") {
- t.Fatal("expected re-claim to succeed after release (future redelivery can re-run)")
- }
- ingestor.releaseTask("mem-release-1")
-}
-
-// TestExecuteMemoryTask_HeartbeatsInProgressDuringLongTask drives the real
-// admission path (processMessage claim + lease) then executes the task: a
-// long-running memory task (LLM extraction can take 10-65s) must renew the
-// broker lease via InProgress, and settlement must stop the heartbeat before
-// Acking (no Ack while an InProgress is in flight).
-func TestExecuteMemoryTask_HeartbeatsInProgressDuringLongTask(t *testing.T) {
- ingestor := newUnitIngestor("test", 1, []string{"pdf"})
- ingestor.SetMemoryMessageService(service.NewMemoryMessageService(nil))
- ingestor.heartbeatInterval = 5 * time.Millisecond
-
- started := make(chan struct{})
- proceed := make(chan struct{})
- ingestor.runMemoryTask = func(_ context.Context, _ string, _ map[string]any) error {
- close(started) // admission heartbeat is running by now
- <-proceed // simulate a long LLM extraction
- return nil
- }
-
- payload, err := json.Marshal(map[string]any{
- "memory_id": "mem-h", "source_id": 1,
- "message_dict": map[string]any{"user_id": "u", "agent_id": "a", "session_id": "s"},
- })
- if err != nil {
- t.Fatalf("marshal payload: %v", err)
- }
- handle := &fakeTaskHandle{msg: common.TaskMessage{TaskID: "mem-hb-1", TaskType: common.TaskTypeMemory, Payload: payload}}
-
- // Admit through processMessage so the claim + lease are real, then drain
- // and execute like a worker would. executeMemoryTask's deferred settlement
- // (stop lease → Ack → release claim) runs before the goroutine exits, so
- // waiting on done before reading currentTasks is race-free.
- ingestor.processMessage(handle)
- taskCtx := <-ingestor.taskChan
- done := make(chan struct{})
- go func() {
- defer close(done)
- ingestor.executeMemoryTask(context.Background(), taskCtx)
- }()
- <-started
-
- // Poll for heartbeats with a generous deadline so the test is resilient
- // to slow CI schedulers.
- heartbeatDeadline := time.Now().Add(2 * time.Second)
- for handle.inProgress.Load() == 0 && time.Now().Before(heartbeatDeadline) {
- time.Sleep(5 * time.Millisecond)
- }
- if handle.inProgress.Load() == 0 {
- t.Fatal("expected InProgress heartbeats while runMemoryTask was blocked, got 0")
- }
-
- close(proceed) // release the long task — only after confirming heartbeats
-
- select {
- case <-done:
- case <-time.After(5 * time.Second):
- t.Fatal("executeMemoryTask did not return after the long task was released")
- }
- if handle.acks.Load() != 1 || handle.nacks.Load() != 0 {
- t.Fatalf("expected 1 Ack/0 Nack on completion after heartbeat, got acks=%d nacks=%d inProgress=%d",
- handle.acks.Load(), handle.nacks.Load(), handle.inProgress.Load())
- }
- if handle.wasSettledWithInProgress() {
- t.Fatal("Ack ran while an InProgress was still in flight — heartbeat must stop before settlement")
- }
- if _, stillClaimed := ingestor.currentTasks["mem-hb-1"]; stillClaimed {
- t.Fatal("expected claim released after executeMemoryTask finished")
- }
-}
-
-// TestExecuteMemoryTaskAlreadyCompletedAcks verifies the idempotent
-// short-circuit for a task row already extracted to completion (progress=1.0):
-// a redelivery after restart (or a duplicate copy) must NOT re-run the LLM
-// extraction — which would insert duplicate memory entries — but must Ack the
-// message. The production runner short-circuits on the progress=1.0 row.
-func TestExecuteMemoryTaskAlreadyCompletedAcks(t *testing.T) {
- db := testutil.SetupTestDB(t)
- cleanup := testutil.ReplaceDBForTest(t, db)
- defer cleanup()
-
- taskID := "mem-completed-1"
- if err := dao.DB.Create(&entity.Task{ID: taskID, DocID: "doc-mem-c", Progress: 1.0}).Error; err != nil {
- t.Fatalf("insert already-completed task: %v", err)
- }
-
- ingestor := newUnitIngestor("test", 1, []string{"pdf"})
- // Real memory service so the production runner path is exercised.
- ingestor.SetMemoryMessageService(service.NewMemoryMessageService(service.NewMemoryService()))
-
- handle := &fakeTaskHandle{msg: common.TaskMessage{TaskID: taskID, TaskType: common.TaskTypeMemory}}
- taskCtx := taskpkg.NewMemoryTaskContextForScheduling(context.Background(), taskID, map[string]any{
- "memory_id": "mem-c", "source_id": 7,
- "message_dict": map[string]any{"user_id": "u", "agent_id": "a", "session_id": "s"},
- }, handle)
-
- ingestor.executeMemoryTask(context.Background(), taskCtx)
- if handle.acks.Load() != 1 || handle.nacks.Load() != 0 {
- t.Fatalf("already-completed memory task: expected 1 Ack/0 Nack, got acks=%d nacks=%d", handle.acks.Load(), handle.nacks.Load())
- }
-}
diff --git a/internal/ingestion/service/real_consumer_pipeline_test.go b/internal/ingestion/service/real_consumer_pipeline_test.go
index beed984c24..a51edf73ce 100644
--- a/internal/ingestion/service/real_consumer_pipeline_test.go
+++ b/internal/ingestion/service/real_consumer_pipeline_test.go
@@ -6,6 +6,7 @@ import (
"context"
"encoding/json"
"testing"
+ "time"
"ragflow/internal/common"
"ragflow/internal/dao"
@@ -21,7 +22,12 @@ func TestRealConsumer_PipelineMessageRoutesToExecuteTask(t *testing.T) {
}
for {
- handles, _ := natsEngine.GetMessages(1)
+ pullCtx, cancel := context.WithTimeout(t.Context(), time.Second)
+ handles, err := natsEngine.PullMessages(pullCtx, 1)
+ cancel()
+ if err != nil {
+ t.Fatalf("drain queue: %v", err)
+ }
if len(handles) == 0 {
break
}
@@ -62,9 +68,11 @@ func TestRealConsumer_PipelineMessageRoutesToExecuteTask(t *testing.T) {
t.Fatalf("PublishTask: %v", err)
}
- handles, err := natsEngine.GetMessages(1)
+ pullCtx, cancel := context.WithTimeout(t.Context(), time.Second)
+ defer cancel()
+ handles, err := natsEngine.PullMessages(pullCtx, 1)
if err != nil {
- t.Fatalf("GetMessages: %v", err)
+ t.Fatalf("PullMessages: %v", err)
}
if len(handles) != 1 {
t.Fatalf("expected 1 message, got %d", len(handles))
diff --git a/internal/ingestion/service/worker_dispatcher_burst_test.go b/internal/ingestion/service/worker_dispatcher_burst_test.go
new file mode 100644
index 0000000000..87ebae996d
--- /dev/null
+++ b/internal/ingestion/service/worker_dispatcher_burst_test.go
@@ -0,0 +1,129 @@
+//
+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+package service
+
+import (
+ "context"
+ "encoding/json"
+ "fmt"
+ "sync"
+ "testing"
+ "time"
+
+ "ragflow/internal/common"
+ "ragflow/internal/engine"
+ "ragflow/internal/entity"
+ "ragflow/internal/ingestion/testutil"
+)
+
+// TestWorkerDispatcherBurstCompletesEveryTask verifies that a burst stays in
+// JetStream until a concrete worker is ready, rather than being Nacked
+// repeatedly until MaxDeliver drops it.
+func TestWorkerDispatcherBurstCompletesEveryTask(t *testing.T) {
+ const (
+ taskCount = 20
+ concurrency = 1
+ )
+
+ db := testutil.SetupTestDB(t)
+ cleanupDB := testutil.ReplaceDBForTest(t, db)
+ defer cleanupDB()
+ taskIDs := seedBurstTasks(t, db, taskCount)
+
+ queue := testutil.SetupNatsEngine(t)
+ previousQueue := engine.GetMessageQueueEngine()
+ engine.SetMessageQueueEngine(queue)
+ t.Cleanup(func() { engine.SetMessageQueueEngine(previousQueue) })
+
+ ingestor := newUnitIngestor("test-worker-burst", concurrency, []string{"pdf"})
+ firstTaskStarted := make(chan struct{})
+ releaseFirstTask := make(chan struct{})
+ var firstTask sync.Once
+ var release sync.Once
+ releaseFirst := func() {
+ release.Do(func() { close(releaseFirstTask) })
+ }
+ t.Cleanup(releaseFirst)
+ t.Cleanup(func() {
+ ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+ defer cancel()
+ ingestor.Stop(ctx)
+ })
+ ingestor.runDocumentTask = func(context.Context, *entity.IngestionTask) error {
+ firstTask.Do(func() {
+ close(firstTaskStarted)
+ <-releaseFirstTask
+ })
+ return nil
+ }
+
+ for _, taskID := range taskIDs {
+ if err := db.Model(&entity.IngestionTask{}).Where("id = ?", taskID).
+ Update("status", common.SCHEDULED).Error; err != nil {
+ t.Fatalf("schedule task %s: %v", taskID, err)
+ }
+ payload, err := json.Marshal(common.TaskMessage{
+ TaskID: taskID,
+ TaskType: common.TaskTypeIngestionTask,
+ })
+ if err != nil {
+ t.Fatalf("marshal task %s: %v", taskID, err)
+ }
+ if err := queue.PublishTask(common.TaskSubject, payload); err != nil {
+ t.Fatalf("publish task %s: %v", taskID, err)
+ }
+ }
+
+ if err := ingestor.Start(); err != nil {
+ t.Fatalf("Start: %v", err)
+ }
+ select {
+ case <-firstTaskStarted:
+ case <-time.After(5 * time.Second):
+ t.Fatal("first task did not start")
+ }
+
+ // The sole worker is busy, so the remaining 19 tasks must remain broker
+ // pending. A buffered dispatcher can Nack this burst until MaxDeliver.
+ releaseFirst()
+
+ deadline := time.Now().Add(10 * time.Second)
+ for time.Now().Before(deadline) {
+ var completed int64
+ if err := db.Model(&entity.IngestionTask{}).
+ Where("id IN ? AND status = ?", taskIDs, common.COMPLETED).
+ Count(&completed).Error; err != nil {
+ t.Fatalf("count completed burst tasks: %v", err)
+ }
+ if completed == taskCount {
+ return
+ }
+ time.Sleep(10 * time.Millisecond)
+ }
+
+ incomplete := make([]string, 0, taskCount)
+ for _, taskID := range taskIDs {
+ var task entity.IngestionTask
+ if err := db.Where("id = ?", taskID).First(&task).Error; err != nil {
+ t.Fatalf("load burst task %s: %v", taskID, err)
+ }
+ if task.Status != common.COMPLETED {
+ incomplete = append(incomplete, fmt.Sprintf("%s=%s", taskID, task.Status))
+ }
+ }
+ t.Fatalf("burst left %d/%d task(s) incomplete: %v", len(incomplete), taskCount, incomplete)
+}
diff --git a/internal/ingestion/service/worker_dispatcher_helpers_test.go b/internal/ingestion/service/worker_dispatcher_helpers_test.go
new file mode 100644
index 0000000000..65026fafdb
--- /dev/null
+++ b/internal/ingestion/service/worker_dispatcher_helpers_test.go
@@ -0,0 +1,61 @@
+//
+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+package service
+
+import (
+ "fmt"
+ "testing"
+
+ "ragflow/internal/common"
+ "ragflow/internal/entity"
+ "ragflow/internal/ingestion/testutil"
+
+ "gorm.io/gorm"
+)
+
+func seedBurstTasks(t *testing.T, db *gorm.DB, n int) []string {
+ t.Helper()
+ const tenantID, kbID = "burst-tenant", "burst-kb"
+ if err := db.Create(&entity.Tenant{ID: tenantID, LLMID: "gpt-4", Status: testutil.StrPtr("1")}).Error; err != nil {
+ t.Fatalf("create tenant: %v", err)
+ }
+ if err := db.Create(&entity.Knowledgebase{
+ ID: kbID, TenantID: tenantID, EmbdID: "embd-1", Status: testutil.StrPtr("1"), ParserConfig: entity.JSONMap{},
+ }).Error; err != nil {
+ t.Fatalf("create knowledgebase: %v", err)
+ }
+
+ ids := make([]string, 0, n)
+ for i := 0; i < n; i++ {
+ docID := fmt.Sprintf("burst-doc-%d", i)
+ taskID := fmt.Sprintf("burst-task-%d", i)
+ location := "doc_store/" + docID
+ if err := db.Create(&entity.Document{
+ ID: docID, KbID: kbID, Name: &docID, ParserID: "naive", ParserConfig: entity.JSONMap{},
+ PipelineID: testutil.StrPtr("flow-1"), Status: testutil.StrPtr("1"), Type: "pdf", Location: &location,
+ }).Error; err != nil {
+ t.Fatalf("create document %s: %v", docID, err)
+ }
+ if err := db.Create(&entity.IngestionTask{
+ ID: taskID, UserID: "u1", DocumentID: docID, DatasetID: kbID, Status: common.CREATED,
+ }).Error; err != nil {
+ t.Fatalf("create ingestion task %s: %v", taskID, err)
+ }
+ ids = append(ids, taskID)
+ }
+ return ids
+}
diff --git a/internal/ingestion/service/worker_dispatcher_integration_test.go b/internal/ingestion/service/worker_dispatcher_integration_test.go
new file mode 100644
index 0000000000..7eb44a4af2
--- /dev/null
+++ b/internal/ingestion/service/worker_dispatcher_integration_test.go
@@ -0,0 +1,427 @@
+//go:build integration
+
+//
+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+package service
+
+import (
+ "context"
+ "encoding/json"
+ "net"
+ "sync"
+ "sync/atomic"
+ "testing"
+ "time"
+
+ "ragflow/internal/common"
+ "ragflow/internal/engine"
+ natsengine "ragflow/internal/engine/nats"
+ "ragflow/internal/entity"
+ "ragflow/internal/ingestion/testutil"
+
+ "github.com/nats-io/nats-server/v2/server"
+)
+
+func setupRealNatsCluster(t *testing.T) (host string, port int) {
+ t.Helper()
+ opts := &server.Options{
+ Port: -1,
+ JetStream: true,
+ StoreDir: t.TempDir(),
+ NoLog: true,
+ NoSigs: true,
+ }
+ ns, err := server.NewServer(opts)
+ if err != nil {
+ t.Fatalf("create embedded NATS server: %v", err)
+ }
+ ns.Start()
+ if !ns.ReadyForConnections(10 * time.Second) {
+ ns.Shutdown()
+ t.Fatal("embedded NATS server did not become ready within 10s")
+ }
+ t.Cleanup(func() {
+ ns.Shutdown()
+ ns.WaitForShutdown()
+ })
+ addr := ns.Addr().(*net.TCPAddr)
+ return "127.0.0.1", addr.Port
+}
+
+// TestIntegration_MultiInstanceSharedConsumerNoOverExecution verifies TaskRP.md §6.2:
+// Multiple ingestor instances sharing the same JetStream durable consumer (RAGFLOW_CONSUMER)
+// compete for tasks without duplicate or overlapping execution while the broker is healthy.
+func TestIntegration_MultiInstanceSharedConsumerNoOverExecution(t *testing.T) {
+ db := testutil.SetupTestDB(t)
+ cleanup := testutil.ReplaceDBForTest(t, db)
+ defer cleanup()
+
+ host, port := setupRealNatsCluster(t)
+
+ // Engine 1 initializes stream and consumer
+ engine1 := natsengine.NewNatsEngine(host, port)
+ if err := engine1.Init(); err != nil {
+ t.Fatalf("engine1.Init: %v", err)
+ }
+ if err := engine1.InitConsumer(common.TaskSubject); err != nil {
+ t.Fatalf("engine1.InitConsumer: %v", err)
+ }
+
+ // Engine 2 connects to the same stream and consumer
+ engine2 := natsengine.NewNatsEngine(host, port)
+ if err := engine2.Init(); err != nil {
+ t.Fatalf("engine2.Init: %v", err)
+ }
+ if err := engine2.InitConsumer(common.TaskSubject); err != nil {
+ t.Fatalf("engine2.InitConsumer: %v", err)
+ }
+
+ const taskCount = 6
+ taskIDs := seedBurstTasks(t, db, taskCount)
+ for _, id := range taskIDs {
+ if err := db.Model(&entity.IngestionTask{}).Where("id = ?", id).
+ Update("status", common.SCHEDULED).Error; err != nil {
+ t.Fatalf("schedule task %s: %v", id, err)
+ }
+ }
+
+ var mu sync.Mutex
+ executionCounts := make(map[string]int)
+ var activeConcurrentRuns atomic.Int32
+ var maxConcurrentObserved atomic.Int32
+
+ runner := func(ctx context.Context, task *entity.IngestionTask) error {
+ cur := activeConcurrentRuns.Add(1)
+ defer activeConcurrentRuns.Add(-1)
+ for {
+ oldMax := maxConcurrentObserved.Load()
+ if cur <= oldMax || maxConcurrentObserved.CompareAndSwap(oldMax, cur) {
+ break
+ }
+ }
+
+ mu.Lock()
+ executionCounts[task.ID]++
+ mu.Unlock()
+
+ time.Sleep(30 * time.Millisecond)
+ return nil
+ }
+
+ ingestor1 := newUnitIngestor("instance-1", 2, []string{"pdf"})
+ ingestor1.runDocumentTask = runner
+
+ ingestor2 := newUnitIngestor("instance-2", 2, []string{"pdf"})
+ ingestor2.runDocumentTask = runner
+
+ // Point global MQ engine to engine1 then start ingestor1, then engine2 and start ingestor2
+ previousEngine := engine.GetMessageQueueEngine()
+ t.Cleanup(func() { engine.SetMessageQueueEngine(previousEngine) })
+
+ engine.SetMessageQueueEngine(engine1)
+ if err := ingestor1.Start(); err != nil {
+ t.Fatalf("ingestor1.Start: %v", err)
+ }
+ t.Cleanup(func() { ingestor1.Stop(context.Background()) })
+
+ engine.SetMessageQueueEngine(engine2)
+ if err := ingestor2.Start(); err != nil {
+ t.Fatalf("ingestor2.Start: %v", err)
+ }
+ t.Cleanup(func() { ingestor2.Stop(context.Background()) })
+
+ // Publish tasks
+ for _, id := range taskIDs {
+ payload, err := json.Marshal(common.TaskMessage{
+ TaskID: id,
+ TaskType: common.TaskTypeIngestionTask,
+ })
+ if err != nil {
+ t.Fatalf("marshal task %s: %v", id, err)
+ }
+ if err := engine1.PublishTask(common.TaskSubject, payload); err != nil {
+ t.Fatalf("publish task %s: %v", id, err)
+ }
+ }
+
+ // Wait for all tasks to be completed in DB
+ deadline := time.Now().Add(10 * time.Second)
+ for time.Now().Before(deadline) {
+ var completedCount int64
+ db.Model(&entity.IngestionTask{}).Where("id IN ? AND status = ?", taskIDs, common.COMPLETED).Count(&completedCount)
+ if int(completedCount) == taskCount {
+ break
+ }
+ time.Sleep(50 * time.Millisecond)
+ }
+
+ mu.Lock()
+ defer mu.Unlock()
+ for _, id := range taskIDs {
+ if count := executionCounts[id]; count != 1 {
+ t.Fatalf("task %s execution count = %d, want 1 during healthy shared-consumer delivery", id, count)
+ }
+ }
+ t.Logf("Multi-instance executed %d tasks cleanly. Max concurrency across instances = %d", taskCount, maxConcurrentObserved.Load())
+}
+
+// TestIntegration_SlowTaskHeartbeatPreventsPrematureRedelivery verifies TaskRP.md §6.2:
+// Under the NATS consumer's BackOff schedule (first retry window = 5s), a task whose execution
+// takes longer than 5s is NOT redelivered mid-flight because the worker's heartbeat calls InProgress.
+func TestIntegration_SlowTaskHeartbeatPreventsPrematureRedelivery(t *testing.T) {
+ db := testutil.SetupTestDB(t)
+ cleanup := testutil.ReplaceDBForTest(t, db)
+ defer cleanup()
+
+ host, port := setupRealNatsCluster(t)
+ mq := natsengine.NewNatsEngine(host, port)
+ if err := mq.Init(); err != nil {
+ t.Fatalf("mq.Init: %v", err)
+ }
+ if err := mq.InitConsumer(common.TaskSubject); err != nil {
+ t.Fatalf("mq.InitConsumer: %v", err)
+ }
+
+ _, _, _, taskID := testutil.SeedTestData(t, db)
+ if err := db.Model(&entity.IngestionTask{}).Where("id = ?", taskID).
+ Update("status", common.SCHEDULED).Error; err != nil {
+ t.Fatalf("schedule task %s: %v", taskID, err)
+ }
+
+ ingestor := newUnitIngestor("slow-task-ingestor", 1, []string{"pdf"})
+ ingestor.heartbeatInterval = 1 * time.Second // comfortably below BackOff[0] = 5s
+
+ taskExecutionStarted := make(chan struct{})
+ var executions atomic.Int32
+
+ ingestor.runDocumentTask = func(ctx context.Context, task *entity.IngestionTask) error {
+ executions.Add(1)
+ close(taskExecutionStarted)
+ // Simulate a slow task that runs for 6 seconds (> 5s BackOff[0])
+ select {
+ case <-time.After(6 * time.Second):
+ return nil
+ case <-ctx.Done():
+ return ctx.Err()
+ }
+ }
+
+ previousEngine := engine.GetMessageQueueEngine()
+ engine.SetMessageQueueEngine(mq)
+ t.Cleanup(func() { engine.SetMessageQueueEngine(previousEngine) })
+
+ if err := ingestor.Start(); err != nil {
+ t.Fatalf("ingestor.Start: %v", err)
+ }
+ t.Cleanup(func() { ingestor.Stop(context.Background()) })
+
+ payload, err := json.Marshal(common.TaskMessage{
+ TaskID: taskID,
+ TaskType: common.TaskTypeIngestionTask,
+ })
+ if err != nil {
+ t.Fatalf("marshal task: %v", err)
+ }
+ if err := mq.PublishTask(common.TaskSubject, payload); err != nil {
+ t.Fatalf("publish task: %v", err)
+ }
+
+ select {
+ case <-taskExecutionStarted:
+ case <-time.After(5 * time.Second):
+ t.Fatal("task execution did not start")
+ }
+
+ // While task is still running at t = 5.5s (after BackOff[0] elapsed):
+ // A manual pull on the same consumer must find NO messages available,
+ // because the heartbeat extended the lease.
+ time.Sleep(5500 * time.Millisecond)
+
+ pullCtx, cancel := context.WithTimeout(t.Context(), time.Second)
+ defer cancel()
+ handles, err := mq.PullMessages(pullCtx, 1)
+ if err != nil {
+ t.Fatalf("PullMessages: %v", err)
+ }
+ if len(handles) != 0 {
+ t.Fatalf("expected 0 redelivered messages while task is running with heartbeat, got %d", len(handles))
+ }
+
+ // Wait for task completion
+ deadline := time.Now().Add(5 * time.Second)
+ for time.Now().Before(deadline) {
+ var task entity.IngestionTask
+ if err := db.Where("id = ?", taskID).First(&task).Error; err == nil && task.Status == common.COMPLETED {
+ break
+ }
+ time.Sleep(100 * time.Millisecond)
+ }
+
+ if executions.Load() != 1 {
+ t.Fatalf("task executions = %d, want 1 (heartbeat should prevent second delivery)", executions.Load())
+ }
+}
+
+// TestIntegration_ShutdownRedeliveryRecovery verifies TaskRP.md §6.2 item 4:
+// After an ingestor shutdown times out under full load (SIGTERM simulation):
+// - Already finished tasks are settled (Acked) and never redelivered;
+// - In-flight uncompleted tasks have their leases abandoned (stopActiveLeases),
+// and are redelivered by the broker to a successor ingestor and completed.
+func TestIntegration_ShutdownRedeliveryRecovery(t *testing.T) {
+ db := testutil.SetupTestDB(t)
+ cleanup := testutil.ReplaceDBForTest(t, db)
+ defer cleanup()
+
+ host, port := setupRealNatsCluster(t)
+ mq := natsengine.NewNatsEngine(host, port)
+ if err := mq.Init(); err != nil {
+ t.Fatalf("mq.Init: %v", err)
+ }
+ if err := mq.InitConsumer(common.TaskSubject); err != nil {
+ t.Fatalf("mq.InitConsumer: %v", err)
+ }
+
+ previousEngine := engine.GetMessageQueueEngine()
+ engine.SetMessageQueueEngine(mq)
+ t.Cleanup(func() { engine.SetMessageQueueEngine(previousEngine) })
+
+ // Seed two tasks: taskA (fast) and taskB (slow, will be in-flight during SIGTERM)
+ taskIDs := seedBurstTasks(t, db, 2)
+ taskIDA := taskIDs[0]
+ taskIDB := taskIDs[1]
+
+ for _, id := range []string{taskIDA, taskIDB} {
+ if err := db.Model(&entity.IngestionTask{}).Where("id = ?", id).
+ Update("status", common.SCHEDULED).Error; err != nil {
+ t.Fatalf("schedule task %s: %v", id, err)
+ }
+ }
+
+ ingestor1 := newUnitIngestor("shutdown-ingestor-1", 2, []string{"pdf"})
+ ingestor1.heartbeatInterval = 500 * time.Millisecond
+
+ taskBStarted := make(chan struct{})
+ var (
+ taskAExecutions atomic.Int32
+ taskBExecutions atomic.Int32
+ )
+
+ ingestor1.runDocumentTask = func(ctx context.Context, task *entity.IngestionTask) error {
+ if task.ID == taskIDA {
+ taskAExecutions.Add(1)
+ return nil
+ }
+ if task.ID == taskIDB {
+ taskBExecutions.Add(1)
+ close(taskBStarted)
+ // Simulate long-running task interrupted by shutdown
+ select {
+ case <-time.After(30 * time.Second):
+ return nil
+ case <-ctx.Done():
+ return ctx.Err()
+ }
+ }
+ return nil
+ }
+
+ if err := ingestor1.Start(); err != nil {
+ t.Fatalf("ingestor1.Start: %v", err)
+ }
+
+ // Publish taskA and taskB
+ for _, id := range []string{taskIDA, taskIDB} {
+ payload, err := json.Marshal(common.TaskMessage{
+ TaskID: id,
+ TaskType: common.TaskTypeIngestionTask,
+ })
+ if err != nil {
+ t.Fatalf("marshal %s: %v", id, err)
+ }
+ if err := mq.PublishTask(common.TaskSubject, payload); err != nil {
+ t.Fatalf("publish %s: %v", id, err)
+ }
+ }
+
+ // Wait for taskB to start execution and taskA to complete
+ <-taskBStarted
+ deadlineA := time.Now().Add(5 * time.Second)
+ for time.Now().Before(deadlineA) {
+ var taskA entity.IngestionTask
+ if err := db.Where("id = ?", taskIDA).First(&taskA).Error; err == nil && taskA.Status == common.COMPLETED {
+ break
+ }
+ time.Sleep(50 * time.Millisecond)
+ }
+
+ // Simulate SIGTERM with 200ms graceful shutdown timeout (which will time out for taskB)
+ stopCtx, stopCancel := context.WithTimeout(context.Background(), 200*time.Millisecond)
+ defer stopCancel()
+ ingestor1.Stop(stopCtx)
+
+ // Ingestor1 timed out and cancelled taskB, calling stopActiveLeases().
+ // Now start ingestor2 to recover un-acked tasks from the shared consumer.
+ ingestor2 := newUnitIngestor("recovery-ingestor-2", 2, []string{"pdf"})
+ ingestor2.heartbeatInterval = 500 * time.Millisecond
+
+ var (
+ taskAExec2 atomic.Int32
+ taskBExec2 atomic.Int32
+ )
+ ingestor2.runDocumentTask = func(ctx context.Context, task *entity.IngestionTask) error {
+ if task.ID == taskIDA {
+ taskAExec2.Add(1)
+ return nil
+ }
+ if task.ID == taskIDB {
+ taskBExec2.Add(1)
+ return nil
+ }
+ return nil
+ }
+
+ if err := ingestor2.Start(); err != nil {
+ t.Fatalf("ingestor2.Start: %v", err)
+ }
+ defer ingestor2.Stop(context.Background())
+
+ // Wait for taskB to be redelivered and completed by ingestor2
+ deadlineB := time.Now().Add(12 * time.Second)
+ taskBCompleted := false
+ for time.Now().Before(deadlineB) {
+ var taskB entity.IngestionTask
+ if err := db.Where("id = ?", taskIDB).First(&taskB).Error; err == nil && taskB.Status == common.COMPLETED {
+ taskBCompleted = true
+ break
+ }
+ time.Sleep(100 * time.Millisecond)
+ }
+
+ if !taskBCompleted {
+ t.Fatalf("taskB was not redelivered and completed after ingestor1 shutdown")
+ }
+
+ // Assertions:
+ // 1. taskA was settled by ingestor1 and NEVER executed by ingestor2 (no duplicate delivery)
+ if taskAExec2.Load() != 0 {
+ t.Fatalf("taskA was redelivered to ingestor2 (%d times), but it was already completed and Acked", taskAExec2.Load())
+ }
+ // 2. taskB was executed once on ingestor2 to completion
+ if taskBExec2.Load() != 1 {
+ t.Fatalf("taskB executions on ingestor2 = %d, want 1", taskBExec2.Load())
+ }
+}
diff --git a/internal/ingestion/service/worker_dispatcher_test.go b/internal/ingestion/service/worker_dispatcher_test.go
new file mode 100644
index 0000000000..62dac42949
--- /dev/null
+++ b/internal/ingestion/service/worker_dispatcher_test.go
@@ -0,0 +1,200 @@
+//
+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+package service
+
+import (
+ "context"
+ "sync"
+ "testing"
+ "time"
+
+ "ragflow/internal/common"
+ "ragflow/internal/engine"
+)
+
+type dispatcherPullResult struct {
+ handle common.TaskHandle
+ err error
+ ready <-chan struct{}
+ returned chan<- struct{}
+}
+
+type dispatcherTestQueue struct {
+ engine.MessageQueue
+
+ mu sync.Mutex
+ pulls []dispatcherPullResult
+ next int
+ calls chan struct{}
+}
+
+func (q *dispatcherTestQueue) PullMessage(ctx context.Context) (common.TaskHandle, error) {
+ q.mu.Lock()
+ if q.next == len(q.pulls) {
+ q.mu.Unlock()
+ q.calls <- struct{}{}
+ <-ctx.Done()
+ return nil, ctx.Err()
+ }
+ pull := q.pulls[q.next]
+ q.next++
+ q.mu.Unlock()
+
+ q.calls <- struct{}{}
+ select {
+ case <-pull.ready:
+ if pull.returned != nil {
+ close(pull.returned)
+ }
+ return pull.handle, pull.err
+ case <-ctx.Done():
+ return nil, ctx.Err()
+ }
+}
+
+func closedChannel() <-chan struct{} {
+ ready := make(chan struct{})
+ close(ready)
+ return ready
+}
+
+// TestWorkerDispatcherLimitsPendingPullsToOne ensures an empty pull does not
+// cause the dispatcher to create another outstanding broker request.
+func TestWorkerDispatcherLimitsPendingPullsToOne(t *testing.T) {
+ firstReady := make(chan struct{})
+ queue := &dispatcherTestQueue{
+ pulls: []dispatcherPullResult{{ready: firstReady}, {}},
+ calls: make(chan struct{}, 2),
+ }
+ previousQueue := engine.GetMessageQueueEngine()
+ engine.SetMessageQueueEngine(queue)
+ t.Cleanup(func() { engine.SetMessageQueueEngine(previousQueue) })
+
+ ingestor := newUnitIngestor("test-single-pull", 2, nil)
+ ingestor.dispatcherWg.Add(1)
+ go ingestor.consumeLoop()
+ t.Cleanup(func() {
+ ingestor.dispatchCancel()
+ ingestor.dispatcherWg.Wait()
+ })
+
+ ingestor.workerQueue <- &worker{id: 1, inbox: make(chan common.TaskHandle)}
+ select {
+ case <-queue.calls:
+ case <-time.After(250 * time.Millisecond):
+ t.Fatal("first idle worker did not start PullMessage")
+ }
+
+ ingestor.workerQueue <- &worker{id: 2, inbox: make(chan common.TaskHandle)}
+ select {
+ case <-queue.calls:
+ t.Fatal("dispatcher started a second PullMessage before the first completed")
+ case <-time.After(100 * time.Millisecond):
+ }
+
+ close(firstReady)
+ select {
+ case <-queue.calls:
+ case <-time.After(250 * time.Millisecond):
+ t.Fatal("dispatcher did not start the next pull after the empty result")
+ }
+}
+
+func TestWorkerDispatcherHandsOffPulledMessageImmediately(t *testing.T) {
+ ready := make(chan struct{})
+ handle := &fakeTaskHandle{msg: common.TaskMessage{TaskID: "single-pull-message"}}
+ queue := &dispatcherTestQueue{
+ pulls: []dispatcherPullResult{{handle: handle, ready: ready}},
+ calls: make(chan struct{}, 1),
+ }
+ previousQueue := engine.GetMessageQueueEngine()
+ engine.SetMessageQueueEngine(queue)
+ t.Cleanup(func() { engine.SetMessageQueueEngine(previousQueue) })
+
+ ingestor := newUnitIngestor("test-single-message-handoff", 1, nil)
+ worker := &worker{id: 1, inbox: make(chan common.TaskHandle)}
+ ingestor.workerQueue <- worker
+ ingestor.dispatcherWg.Add(1)
+ go ingestor.consumeLoop()
+ t.Cleanup(func() {
+ ingestor.dispatchCancel()
+ ingestor.dispatcherWg.Wait()
+ })
+
+ select {
+ case <-queue.calls:
+ case <-time.After(250 * time.Millisecond):
+ t.Fatal("worker did not start PullMessages(1)")
+ }
+
+ close(ready)
+ select {
+ case received := <-worker.inbox:
+ if received != handle {
+ t.Fatalf("handed-off handle = %v, want pulled handle", received)
+ }
+ case <-time.After(250 * time.Millisecond):
+ t.Fatal("pulled message was not handed off immediately")
+ }
+}
+
+func TestWorkerDispatcherNacksReservedHandleOnShutdown(t *testing.T) {
+ returned := make(chan struct{})
+ handle := &fakeTaskHandle{msg: common.TaskMessage{TaskID: "reserved-on-stop"}}
+ queue := &dispatcherTestQueue{
+ pulls: []dispatcherPullResult{{handle: handle, ready: closedChannel(), returned: returned}},
+ calls: make(chan struct{}, 1),
+ }
+ previousQueue := engine.GetMessageQueueEngine()
+ engine.SetMessageQueueEngine(queue)
+ t.Cleanup(func() { engine.SetMessageQueueEngine(previousQueue) })
+ ingestor := newUnitIngestor("test-nack-reserved-handle", 1, nil)
+ worker := &worker{id: 1, inbox: make(chan common.TaskHandle)}
+ t.Cleanup(func() {
+ ingestor.dispatchCancel()
+ ingestor.dispatcherWg.Wait()
+ })
+ ingestor.workerQueue <- worker
+ ingestor.dispatcherWg.Add(1)
+ go ingestor.consumeLoop()
+
+ select {
+ case <-queue.calls:
+ case <-time.After(250 * time.Millisecond):
+ t.Fatal("pull did not start")
+ }
+ select {
+ case <-returned:
+ case <-time.After(250 * time.Millisecond):
+ t.Fatal("pull did not reserve the handle")
+ }
+
+ ingestor.dispatchCancel()
+ waitDone := make(chan struct{})
+ go func() {
+ ingestor.dispatcherWg.Wait()
+ close(waitDone)
+ }()
+ select {
+ case <-waitDone:
+ case <-time.After(250 * time.Millisecond):
+ t.Fatal("cancelled hand-off blocked dispatcher shutdown")
+ }
+ if handle.acks.Load() != 0 || handle.nacks.Load() != 1 {
+ t.Fatalf("reserved handle settlement = %d Ack / %d Nack, want 0 Ack / 1 Nack", handle.acks.Load(), handle.nacks.Load())
+ }
+}
diff --git a/internal/ingestion/task/real_consumer_test.go b/internal/ingestion/task/real_consumer_test.go
index baed6516e4..804770040a 100644
--- a/internal/ingestion/task/real_consumer_test.go
+++ b/internal/ingestion/task/real_consumer_test.go
@@ -21,6 +21,7 @@ import (
"context"
"encoding/json"
"testing"
+ "time"
"ragflow/internal/common"
"ragflow/internal/dao"
@@ -41,7 +42,12 @@ func TestRealProducerConsumer(t *testing.T) {
// Purge stale messages
for {
- h, _ := natsEngine.GetMessages(1)
+ pullCtx, cancel := context.WithTimeout(t.Context(), time.Second)
+ h, err := natsEngine.PullMessages(pullCtx, 1)
+ cancel()
+ if err != nil {
+ t.Fatalf("drain queue: %v", err)
+ }
if len(h) == 0 {
break
}
@@ -83,9 +89,11 @@ func TestRealProducerConsumer(t *testing.T) {
t.Logf("Producer: Published %s", payload)
// ── 4. Consumer: Mirrors Ingestor.Start():131-189 exactly ──
- handles, err := natsEngine.GetMessages(1)
+ pullCtx, cancel := context.WithTimeout(t.Context(), time.Second)
+ defer cancel()
+ handles, err := natsEngine.PullMessages(pullCtx, 1)
if err != nil {
- t.Fatalf("GetMessages: %v", err)
+ t.Fatalf("PullMessages: %v", err)
}
if len(handles) != 1 {
t.Fatalf("expected 1 message, got %d", len(handles))
diff --git a/internal/ingestion/task/task_context.go b/internal/ingestion/task/task_context.go
index 15f88d290d..d41dfcb1ec 100644
--- a/internal/ingestion/task/task_context.go
+++ b/internal/ingestion/task/task_context.go
@@ -71,13 +71,6 @@ type TaskContext struct {
// lives only here, never re-derived from IngestionTask or MemoryPayload.
taskID string
- // stopLease stops the broker-lease heartbeat started at admission
- // (processMessage) for a queued/running memory task. It is set by the
- // memory dispatch path before enqueueing and invoked by the worker's
- // settlement defer before Ack/Nack. Doc (ingestion) tasks leave it nil —
- // their heartbeat is started and stopped entirely inside settleMessage.
- stopLease func()
-
// Handle is the message-queue ack handle for the task message that scheduled
// this context. The scheduler sets it before queueing; the worker decides
// the terminal Ack/Nack:
@@ -118,34 +111,6 @@ func NewTaskContextForScheduling(ctx context.Context, task *entity.IngestionTask
}
}
-// SetStopLease attaches the admission-started broker-lease stop function to a
-// memory task context so the worker can stop the heartbeat before settlement.
-// A nil stop is a no-op. Doc (ingestion) contexts never call this.
-func (c *TaskContext) SetStopLease(stop func()) {
- if c != nil {
- c.stopLease = stop
- }
-}
-
-// StopLease invokes the attached stop function, if any, blocking until no
-// InProgress is in flight (Heartbeat.Stop semantics). It is safe to call with
-// no attached lease (doc path or direct-execution test path).
-func (c *TaskContext) StopLease() {
- if c != nil && c.stopLease != nil {
- c.stopLease()
- }
-}
-
-// StopLeaseFn returns the attached stop function (nil when no admission
-// heartbeat was started), so callers can detect whether a lease exists without
-// invoking it.
-func (c *TaskContext) StopLeaseFn() func() {
- if c == nil {
- return nil
- }
- return c.stopLease
-}
-
// ID returns the task identifier for claim/release and logging. It is the
// envelope TaskMessage.TaskID captured at construction — there is deliberately
// no fallback to IngestionTask or MemoryPayload, so identity is never
diff --git a/internal/server/config/ingestor_config.go b/internal/server/config/ingestor_config.go
index 7b3d33c029..9b5a2b8646 100644
--- a/internal/server/config/ingestor_config.go
+++ b/internal/server/config/ingestor_config.go
@@ -15,12 +15,14 @@
package config
-import "github.com/spf13/viper"
+import (
+ "github.com/spf13/viper"
+)
type IngestorConfig struct {
// MaxConcurrentWorkers bounds how many ingestion tasks the ingestor runs in
- // parallel (the task channel width and dataset-level compile worker count
- // default to this value). 0/negative falls back to runtime.NumCPU().
+ // parallel (and dataset-level compile worker count defaults to this value).
+ // 0/negative falls back to runtime.NumCPU().
MaxConcurrentWorkers int `mapstructure:"max_concurrent_workers"`
// CompilerPoolSize bounds the process-wide knowledge-compilation worker
// pool that drives the cross-doc KNN / LLM-merge / write stages. 0/negative
diff --git a/internal/server/config/ingestor_config_test.go b/internal/server/config/ingestor_config_test.go
new file mode 100644
index 0000000000..8dad538c93
--- /dev/null
+++ b/internal/server/config/ingestor_config_test.go
@@ -0,0 +1,42 @@
+//
+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+package config
+
+import (
+ "testing"
+
+ "github.com/spf13/viper"
+)
+
+func TestParseIngestorConfigReadsWorkerAndCompilerSettings(t *testing.T) {
+ v := viper.New()
+ v.Set("ingestor", map[string]any{
+ "max_concurrent_workers": 4,
+ "compiler_pool_size": 2,
+ })
+
+ config := &Config{}
+ if err := config.ParseIngestorConfig(v); err != nil {
+ t.Fatalf("ParseIngestorConfig: %v", err)
+ }
+ if got := config.GetIngestorConfig().MaxConcurrentWorkers; got != 4 {
+ t.Fatalf("max concurrent workers = %d, want 4", got)
+ }
+ if got := config.GetIngestorConfig().CompilerPoolSize; got != 2 {
+ t.Fatalf("compiler pool size = %d, want 2", got)
+ }
+}
diff --git a/internal/service/document/document_rerun_e2e_test.go b/internal/service/document/document_rerun_e2e_test.go
index ab5c7e51da..e0ea6b3367 100644
--- a/internal/service/document/document_rerun_e2e_test.go
+++ b/internal/service/document/document_rerun_e2e_test.go
@@ -3,7 +3,9 @@
package document
import (
+ "context"
"testing"
+ "time"
"ragflow/internal/common"
"ragflow/internal/engine"
@@ -86,9 +88,11 @@ func TestRerunDocument_E2E_EnqueuesThroughRealMessageQueue(t *testing.T) {
}
// The enqueue actually landed on tasks.RAGFLOW.
- handles, err := mq.GetMessages(1)
+ pullCtx, cancel := context.WithTimeout(t.Context(), time.Second)
+ defer cancel()
+ handles, err := mq.PullMessages(pullCtx, 1)
if err != nil {
- t.Fatalf("GetMessages: %v", err)
+ t.Fatalf("PullMessages: %v", err)
}
if len(handles) != 1 {
t.Fatalf("expected 1 message on tasks.RAGFLOW, got %d", len(handles))
diff --git a/internal/service/ingestion_task_service.go b/internal/service/ingestion_task_service.go
index 87e7b58017..8395578ed0 100644
--- a/internal/service/ingestion_task_service.go
+++ b/internal/service/ingestion_task_service.go
@@ -509,7 +509,7 @@ func (s *IngestionTaskService) enqueueTask(taskID string) error {
TaskID: taskID,
TaskType: common.TaskTypeIngestionTask,
}
- return s.taskPublisher.PublishTaskMessage("tasks.RAGFLOW", taskMessage)
+ return s.taskPublisher.PublishTaskMessage(common.TaskSubject, taskMessage)
}
// UpdateComponentTotal records the number of components in the task's DSL
diff --git a/internal/service/memory_extractor.go b/internal/service/memory_extractor.go
index fc63fa879d..d6e5d8735f 100644
--- a/internal/service/memory_extractor.go
+++ b/internal/service/memory_extractor.go
@@ -25,7 +25,7 @@
// QueueSaveToMemoryTask persists the raw message and publishes a
// task_type="memory" TaskMessage on the NATS tasks.RAGFLOW subject. The
// Ingestor's shared consumer + worker pool dispatches it by TaskType to
-// HandleSaveToMemoryTask (see internal/ingestion/service/processMessage and
+// HandleSaveToMemoryTask (see internal/ingestion/service/handleAndExecute and
// executeMemoryTask), which runs LLM extraction for the non-raw memory types
// configured on the memory and persists the extracted messages with source_id
// pointing at the raw message so listMemoryMessages can aggregate them under
diff --git a/internal/service/memory_message_service.go b/internal/service/memory_message_service.go
index 201c712b82..f77b688a1e 100644
--- a/internal/service/memory_message_service.go
+++ b/internal/service/memory_message_service.go
@@ -362,7 +362,7 @@ func queueMemoryTask(ctx context.Context, taskID, memoryID, tenantID string, raw
}
// Publish the memory-extraction task to NATS (tasks.RAGFLOW) so it is
// consumed by the Ingestor's shared consumer + worker pool, dispatched by
- // TaskType=="memory" in processMessage. This keeps Go out of the Python
+ // TaskType=="memory" in handleAndExecute. This keeps Go out of the Python
// te.*.common Redis stream entirely, removing the cross-consumer
// contention that previously stole Python dataflow tasks.
mq := engine.GetMessageQueueEngine()