Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

fix(ingestor,nats): recover consumption, fail fast on NATS startup and harden reconciliation (P0 blockers) (#18942)

jay77721de6b12a2b912
diff --git a/cmd/ragflow_server.go b/cmd/ragflow_server.go
index 900913e5e3..aeb75bb0b1 100644
--- a/cmd/ragflow_server.go
+++ b/cmd/ragflow_server.go
@@ -376,7 +376,7 @@ func main() {
defer storage.CloseStorage()
if err = engine.InitMessageQueue(); err != nil {
- common.Error("Failed to initialize message queue engine", err)
+ common.Fatal("Failed to initialize message queue engine", zap.Error(err))
}
// Initialize server variables (runtime variables that can change during operation)
@@ -583,10 +583,12 @@ func runIngestor(ctx context.Context, cancel context.CancelFunc, args *serverArg
ingestor.SetMemoryMessageService(service.NewMemoryMessageService(service.NewMemoryService()))
// Start returns immediately (it launches the owned consume/compile
- // goroutines and joins them via Stop); a provisioning failure here is
- // logged and the server keeps running without the ingestor.
+ // goroutines and joins them via Stop); a provisioning failure here must
+ // fail the server (main's os.Exit(1) path) instead of reporting a
+ // healthy ingestor that can never consume.
if err := ingestor.Start(); err != nil {
common.Error("Failed to initialize ingestor", err)
+ return err
}
common.Info("\n ____ __ _\n" +
diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml
index 6321b72b32..2cf741a42b 100644
--- a/docker/docker-compose.yml
+++ b/docker/docker-compose.yml
@@ -23,6 +23,9 @@ services:
mysql:
condition: service_healthy
required: false
+ nats:
+ condition: service_healthy
+ required: false
profiles:
- cpu
image: ${RAGFLOW_IMAGE}
@@ -86,6 +89,9 @@ services:
mysql:
condition: service_healthy
required: false
+ nats:
+ condition: service_healthy
+ required: false
profiles:
- gpu
image: ${RAGFLOW_IMAGE}
diff --git a/internal/dao/ingestion.go b/internal/dao/ingestion.go
index 2afef863e4..dae1a10659 100644
--- a/internal/dao/ingestion.go
+++ b/internal/dao/ingestion.go
@@ -23,6 +23,7 @@ import (
"ragflow/internal/common"
"ragflow/internal/entity"
"ragflow/internal/utility"
+ "time"
"gorm.io/gorm"
)
@@ -195,6 +196,33 @@ func (dao *IngestionTaskDAO) GetByDocumentID(ctx context.Context, db *gorm.DB, d
return tasks[0], nil
}
+// ListStaleByStatus returns tasks in the given statuses whose timestamp column
+// is older than the threshold, for ingestor startup reconciliation. timeColumn
+// selects which staleness clock applies: "update_time" for RUNNING orphans
+// (frozen at the moment the previous process claimed them) or "create_time"
+// for CREATED orphans (never picked up). Results are ordered oldest first and
+// capped at limit when limit > 0.
+func (dao *IngestionTaskDAO) ListStaleByStatus(ctx context.Context, db *gorm.DB, statuses []string, timeColumn string, olderThan time.Time, limit int) ([]*entity.IngestionTask, error) {
+ switch timeColumn {
+ case "create_time", "update_time":
+ default:
+ return nil, fmt.Errorf("unsupported staleness column %q", timeColumn)
+ }
+ if len(statuses) == 0 {
+ return []*entity.IngestionTask{}, nil
+ }
+ query := db.WithContext(ctx).
+ Where("status IN ?", statuses).
+ Where(timeColumn+" < ?", olderThan.UnixMilli()).
+ Order(timeColumn + " ASC")
+ if limit > 0 {
+ query = query.Limit(limit)
+ }
+ var tasks []*entity.IngestionTask
+ err := query.Find(&tasks).Error
+ return tasks, err
+}
+
// DeleteIfTerminal deletes ingestion tasks for a document that are in a
// terminal state (COMPLETED, STOPPED, FAILED) or still queued (CREATED).
// RUNNING and STOPPING tasks are NOT deleted because an in-flight worker
diff --git a/internal/dao/ingestion_task_test.go b/internal/dao/ingestion_task_test.go
index 35d9931745..f047a8a2ba 100644
--- a/internal/dao/ingestion_task_test.go
+++ b/internal/dao/ingestion_task_test.go
@@ -3,6 +3,7 @@ package dao
import (
"errors"
"testing"
+ "time"
"fmt"
"ragflow/internal/common"
@@ -138,6 +139,96 @@ func TestIngestionTaskDAOUpdateStatusIfCurrentRejectsMismatchedStatus(t *testing
}
}
+// seedStaleTask inserts an ingestion task whose create/update timestamps are
+// aged back so ListStaleByStatus staleness windows can be asserted.
+func seedStaleTask(t *testing.T, db *gorm.DB, id, status string, age time.Duration) {
+ t.Helper()
+ ts := time.Now().Add(-age).UnixMilli()
+ task := &entity.IngestionTask{
+ ID: id,
+ UserID: "user-1",
+ DocumentID: "doc-" + id,
+ DatasetID: "kb-1",
+ Status: status,
+ BaseModel: entity.BaseModel{CreateTime: &ts, UpdateTime: &ts},
+ }
+ if err := db.Create(task).Error; err != nil {
+ t.Fatalf("create task %s: %v", id, err)
+ }
+}
+
+// TestIngestionTaskDAOListStaleByStatus covers the three contract axes of the
+// startup-reconciliation query: status filtering, the staleness time window
+// (per column), and the result limit.
+func TestIngestionTaskDAOListStaleByStatus(t *testing.T) {
+ db := setupTaskTestDB(t)
+ orig := DB
+ DB = db
+ t.Cleanup(func() { DB = orig })
+
+ seedStaleTask(t, db, "run-stale", common.RUNNING, 20*time.Minute)
+ seedStaleTask(t, db, "run-fresh", common.RUNNING, 1*time.Minute)
+ seedStaleTask(t, db, "created-stale", common.CREATED, 20*time.Minute)
+ seedStaleTask(t, db, "created-fresh", common.CREATED, 1*time.Minute)
+ seedStaleTask(t, db, "completed-stale", common.COMPLETED, 20*time.Minute)
+
+ ctx := t.Context()
+ d := NewIngestionTaskDAO()
+ threshold := time.Now().Add(-15 * time.Minute)
+
+ // Status filter + update_time window: only the stale RUNNING row matches.
+ running, err := d.ListStaleByStatus(ctx, db, []string{common.RUNNING}, "update_time", threshold, 0)
+ if err != nil {
+ t.Fatalf("ListStaleByStatus(RUNNING): %v", err)
+ }
+ if len(running) != 1 || running[0].ID != "run-stale" {
+ t.Fatalf("RUNNING stale rows = %v, want [run-stale]", idsOfTasks(running))
+ }
+
+ // CREATED is judged by create_time: fresh CREATED rows stay outside the
+ // window even though their status is in the filter.
+ created, err := d.ListStaleByStatus(ctx, db, []string{common.CREATED}, "create_time", threshold, 0)
+ if err != nil {
+ t.Fatalf("ListStaleByStatus(CREATED): %v", err)
+ }
+ if len(created) != 1 || created[0].ID != "created-stale" {
+ t.Fatalf("CREATED stale rows = %v, want [created-stale]", idsOfTasks(created))
+ }
+
+ // Rows in unlisted statuses (COMPLETED) are never returned.
+ all, err := d.ListStaleByStatus(ctx, db, []string{common.RUNNING, common.CREATED}, "create_time", threshold, 0)
+ if err != nil {
+ t.Fatalf("ListStaleByStatus(RUNNING+CREATED): %v", err)
+ }
+ for _, task := range all {
+ if task.ID == "completed-stale" {
+ t.Fatal("COMPLETED row must not be returned for RUNNING/CREATED filter")
+ }
+ }
+
+ // Limit caps the (oldest-first) result set.
+ limited, err := d.ListStaleByStatus(ctx, db, []string{common.RUNNING, common.CREATED}, "create_time", threshold, 1)
+ if err != nil {
+ t.Fatalf("ListStaleByStatus(limit): %v", err)
+ }
+ if len(limited) != 1 {
+ t.Fatalf("limited rows = %v, want exactly 1", idsOfTasks(limited))
+ }
+
+ // An unsupported staleness column is rejected, not silently ignored.
+ if _, err = d.ListStaleByStatus(ctx, db, []string{common.RUNNING}, "status; --", threshold, 0); err == nil {
+ t.Fatal("expected error for unsupported staleness column")
+ }
+}
+
+func idsOfTasks(tasks []*entity.IngestionTask) []string {
+ ids := make([]string, 0, len(tasks))
+ for _, task := range tasks {
+ ids = append(ids, task.ID)
+ }
+ return ids
+}
+
func TestIngestionTaskDAODeleteIfTerminal_RemovesOnlyTerminal(t *testing.T) {
db := setupTaskTestDB(t)
orig := DB
diff --git a/internal/engine/nats/knowledgecompile.go b/internal/engine/nats/knowledgecompile.go
index 8e1fbbbbfc..c2a7be41a4 100644
--- a/internal/engine/nats/knowledgecompile.go
+++ b/internal/engine/nats/knowledgecompile.go
@@ -53,17 +53,18 @@ func (n *NatsEngine) InitKnowledgeCompileStream() error {
}
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
defer cancel()
- cfg := jetstream.StreamConfig{
+ stream, err := ensureStreamConfig(ctx, n.jetStream, jetstream.StreamConfig{
Name: knowledgeCompileStreamName,
Subjects: []string{knowledgeCompileSubjectPrefix},
Retention: jetstream.WorkQueuePolicy,
Storage: jetstream.FileStorage,
MaxMsgs: 1024 * 1024,
MaxBytes: 1024 * 1024 * 64,
- }
- if _, err := n.jetStream.CreateStream(ctx, cfg); err != nil && !strings.Contains(err.Error(), "already exists") {
+ })
+ if err != nil {
return fmt.Errorf("knowledgecompile: create stream: %w", err)
}
+ n.knowledgeCompileStream = stream
return nil
}
diff --git a/internal/engine/nats/nats.go b/internal/engine/nats/nats.go
index cc09ebb42f..ae91d9faac 100644
--- a/internal/engine/nats/nats.go
+++ b/internal/engine/nats/nats.go
@@ -80,25 +80,20 @@ func (n *NatsEngine) Init() error {
Retention: jetstream.WorkQueuePolicy,
Storage: jetstream.FileStorage,
MaxMsgs: 1024 * 128,
- MaxBytes: 1024 * 1024,
+ MaxBytes: 1024 * 1024 * 64,
+ // Server-side dedup window. Inert for task publishes: PublishTask
+ // intentionally sends no MsgID (see below — dedup would swallow
+ // retry republishes of a reused task_id). It only takes effect for
+ // a publisher that opts into MsgIDs.
+ Duplicates: 10 * time.Minute,
}
- n.stream, err = n.jetStream.CreateStream(ctx, streamCfg)
+ n.stream, err = ensureStreamConfig(ctx, n.jetStream, streamCfg)
if err != nil {
- if !strings.Contains(err.Error(), "already exists") {
- n.nc.Close()
- return fmt.Errorf("fail to create stream at %s: %w", natsURL, err)
- }
-
- common.Info("NATS stream already exists, use existing stream")
- n.stream, err = n.jetStream.Stream(ctx, "RAGFLOW_TASKS")
- if err != nil {
- n.nc.Close()
- return fmt.Errorf("fail to get existing stream at %s: %w", natsURL, err)
- }
- } else {
- common.Info(fmt.Sprintf("NATS stream create successfully at %s", natsURL))
+ n.nc.Close()
+ return fmt.Errorf("fail to create stream at %s: %w", natsURL, err)
}
+ common.Info(fmt.Sprintf("NATS stream RAGFLOW_TASKS ready at %s", natsURL))
return nil
}
@@ -115,6 +110,16 @@ func (n *NatsEngine) PublishTask(subject string, payload []byte) error {
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
+ // Deliberately NO MsgID/dedup here. Ingestion tasks reuse the same
+ // task_id across publish attempts (the FAILED/STOPPED→CREATED retry
+ // path), and the server's Duplicates window suppresses a republished
+ // MsgID for its whole lifetime regardless of whether the original
+ // message was already consumed and acked. A deduped retry publish
+ // strands the task in CREATED with no message behind it — unreachable
+ // by any consumer and un-reparsable ("already exists, status: CREATED").
+ // Duplicate delivery is instead made safe at the consumer level:
+ // StartRunning's CREATED→RUNNING CAS plus the in-process claim guard
+ // ack-skip any second copy (see Ingestor.processMessage).
ack, err := n.jetStream.Publish(ctx, subject, payload)
if err != nil {
return err
@@ -212,15 +217,32 @@ func (n *NatsEngine) InitConsumer(subject string) error {
defer cancel()
var err error
+ // Explicit redelivery schedule: BackOff paces successive redeliveries
+ // (5s/15s/30s, then 60s repeated) so an unsettled message (crash, slow
+ // DB) is retried with breathing room instead of the broker default. The
+ // server normalizes AckWait to BackOff[0] when BackOff is present; the
+ // 60s AckWait is the effective schedule if BackOff is ever dropped.
+ // INVARIANT: the worker's InProgress heartbeat (Ingestor
+ // defaultHeartbeatInterval) must stay below BackOff[0] = 5s, or in-flight
+ // messages get redelivered mid-run and the claim-guard ack-skip acks the
+ // only live copy.
+ // Note: CreateOrUpdateConsumer is atomic. MaxWaiting is immutable after
+ // creation; if it mismatches the whole update fails (error "max waiting
+ // can not be updated") and NONE of AckWait/BackOff/MaxAckPending are
+ // applied. When MaxWaiting matches (the default, since this config omits
+ // it), the other three update in place. The fallback below handles the
+ // MaxWaiting mismatch by keeping the existing consumer.
n.consumer, err = n.stream.CreateOrUpdateConsumer(ctx, jetstream.ConsumerConfig{
Name: "RAGFLOW_CONSUMER",
AckPolicy: jetstream.AckExplicitPolicy,
MaxDeliver: 16,
MaxAckPending: 1024 * 128,
FilterSubject: "tasks.>",
+ AckWait: 60 * time.Second,
+ BackOff: []time.Duration{5 * time.Second, 15 * time.Second, 30 * time.Second, 60 * time.Second},
})
if err != nil {
- // MaxAckPending is immutable after consumer creation.
+ // MaxWaiting is immutable after consumer creation (AckWait/BackOff/MaxAckPending remain mutable when MaxWaiting matches; the update is atomic).
// If the consumer already exists, fall back to fetching it.
if strings.Contains(err.Error(), "max waiting can not be updated") {
n.consumer, err = n.stream.Consumer(ctx, "RAGFLOW_CONSUMER")
diff --git a/internal/engine/nats/stream_config_test.go b/internal/engine/nats/stream_config_test.go
new file mode 100644
index 0000000000..331fb44ce6
--- /dev/null
+++ b/internal/engine/nats/stream_config_test.go
@@ -0,0 +1,207 @@
+//
+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+package nats
+
+import (
+ "context"
+ "encoding/json"
+ "net"
+ "strconv"
+ "testing"
+ "time"
+
+ "ragflow/internal/common"
+
+ "github.com/nats-io/nats-server/v2/server"
+ "github.com/nats-io/nats.go"
+ "github.com/nats-io/nats.go/jetstream"
+)
+
+// newEmbeddedNatsServer starts an in-process JetStream-enabled NATS server on
+// a random port and returns its host/port.
+func newEmbeddedNatsServer(t *testing.T) (string, int) {
+ t.Helper()
+ opts := &server.Options{
+ Port: -1,
+ JetStream: true,
+ StoreDir: t.TempDir(),
+ NoLog: true,
+ NoSigs: true,
+ }
+ ns, err := server.NewServer(opts)
+ if err != nil {
+ t.Fatalf("create embedded NATS server: %v", err)
+ }
+ ns.Start()
+ if !ns.ReadyForConnections(10 * time.Second) {
+ ns.Shutdown()
+ t.Fatal("embedded NATS server did not become ready")
+ }
+ t.Cleanup(func() {
+ ns.Shutdown()
+ ns.WaitForShutdown()
+ })
+ addr := ns.Addr().(*net.TCPAddr)
+ return "127.0.0.1", addr.Port
+}
+
+// TestPublishTaskDeliversRepeatedTaskIDs: publishing the same task_id twice
+// MUST land two messages. Ingestion tasks reuse the task_id across publish
+// attempts (the FAILED/STOPPED→CREATED retry path), and a JetStream MsgID
+// dedup would suppress the retry republish within the Duplicates window even
+// though the original message is long gone — stranding the task in CREATED
+// with no message behind it ("already exists, status: CREATED" forever).
+// This test is the regression guard against reintroducing publish dedup.
+func TestPublishTaskDeliversRepeatedTaskIDs(t *testing.T) {
+ host, port := newEmbeddedNatsServer(t)
+ engine := NewNatsEngine(host, port)
+ if err := engine.Init(); err != nil {
+ t.Fatalf("Init: %v", err)
+ }
+ if err := engine.InitConsumer("tasks.>"); err != nil {
+ t.Fatalf("InitConsumer: %v", err)
+ }
+
+ payload, err := json.Marshal(common.TaskMessage{TaskID: "task-repeat-1", TaskType: common.TaskTypeIngestionTask})
+ if err != nil {
+ t.Fatalf("marshal: %v", err)
+ }
+ if err = engine.PublishTask("tasks.RAGFLOW", payload); err != nil {
+ t.Fatalf("first PublishTask: %v", err)
+ }
+ if err = engine.PublishTask("tasks.RAGFLOW", payload); err != nil {
+ t.Fatalf("repeated PublishTask: %v", err)
+ }
+
+ ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+ defer cancel()
+ info, err := engine.stream.Info(ctx)
+ if err != nil {
+ t.Fatalf("stream info: %v", err)
+ }
+ if info.State.Msgs != 2 {
+ t.Fatalf("stream holds %d messages after two publishes of the same task_id, want 2 (publish dedup strands retry republishes)", info.State.Msgs)
+ }
+
+ // A payload without a decodable TaskMessage shape must not fail either.
+ if err = engine.PublishTask("tasks.RAGFLOW", []byte("not-json")); err != nil {
+ t.Fatalf("non-TaskMessage PublishTask: %v", err)
+ }
+}
+
+// TestInitMigratesLegacyStreamConfig: a stream created by an older deployment
+// (no Duplicates, 1MB MaxBytes) must be migrated in place by Init instead of
+// being left stale behind an "already exists" error. The server-side config is
+// the merge base: fields the helper does not own (Subjects, Retention) must
+// survive the update.
+func TestInitMigratesLegacyStreamConfig(t *testing.T) {
+ host, port := newEmbeddedNatsServer(t)
+
+ // Pre-create the stream with the legacy (pre-migration) configuration.
+ nc, err := nats.Connect("nats://" + net.JoinHostPort(host, strconv.Itoa(port)))
+ if err != nil {
+ t.Fatalf("connect legacy stream creator: %v", err)
+ }
+ defer nc.Close()
+ js, err := jetstream.New(nc)
+ if err != nil {
+ t.Fatalf("jetstream context: %v", err)
+ }
+ ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+ defer cancel()
+ legacy, err := js.CreateStream(ctx, jetstream.StreamConfig{
+ Name: "RAGFLOW_TASKS",
+ Subjects: []string{"tasks.>"},
+ Retention: jetstream.WorkQueuePolicy,
+ Storage: jetstream.FileStorage,
+ MaxMsgs: 1024 * 128,
+ MaxBytes: 1024 * 1024,
+ })
+ if err != nil {
+ t.Fatalf("create legacy stream: %v", err)
+ }
+ legacyInfo, err := legacy.Info(ctx)
+ if err != nil {
+ t.Fatalf("legacy stream info: %v", err)
+ }
+ // Precondition: the legacy stream differs from the wanted config (server
+ // fills a 2m default Duplicates when unspecified).
+ if legacyInfo.Config.MaxBytes != int64(1024*1024) {
+ t.Fatalf("precondition: legacy MaxBytes = %d, want 1MB", legacyInfo.Config.MaxBytes)
+ }
+ if legacyInfo.Config.Duplicates == 10*time.Minute {
+ t.Fatalf("precondition: legacy Duplicates already migrated (%v)", legacyInfo.Config.Duplicates)
+ }
+
+ engine := NewNatsEngine(host, port)
+ if err := engine.Init(); err != nil {
+ t.Fatalf("Init over legacy stream: %v", err)
+ }
+
+ info, err := engine.stream.Info(ctx)
+ if err != nil {
+ t.Fatalf("stream info after migration: %v", err)
+ }
+ if got := info.Config.MaxBytes; got != int64(1024*1024*64) {
+ t.Fatalf("MaxBytes after migration = %d, want %d", got, int64(1024*1024*64))
+ }
+ if got := info.Config.Duplicates; got != 10*time.Minute {
+ t.Fatalf("Duplicates after migration = %v, want 10m", got)
+ }
+ // Non-owned fields must not be reset by the partial update.
+ if got := info.Config.Retention; got != jetstream.WorkQueuePolicy {
+ t.Fatalf("Retention after migration = %v, want WorkQueuePolicy (must not be reset)", got)
+ }
+ if len(info.Config.Subjects) != 1 || info.Config.Subjects[0] != "tasks.>" {
+ t.Fatalf("Subjects after migration = %v, want [tasks.>] (must not be reset)", info.Config.Subjects)
+ }
+}
+
+// TestInitConsumerSetsAckWaitAndBackOff: the consumer must carry an explicit
+// redelivery schedule (AckWait + BackOff) so unacked messages are retried on
+// a paced backoff instead of the broker default.
+func TestInitConsumerSetsAckWaitAndBackOff(t *testing.T) {
+ host, port := newEmbeddedNatsServer(t)
+ engine := NewNatsEngine(host, port)
+ if err := engine.Init(); err != nil {
+ t.Fatalf("Init: %v", err)
+ }
+ if err := engine.InitConsumer("tasks.>"); err != nil {
+ t.Fatalf("InitConsumer: %v", err)
+ }
+
+ ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+ defer cancel()
+ info, err := engine.consumer.Info(ctx)
+ if err != nil {
+ t.Fatalf("consumer info: %v", err)
+ }
+ wantBackOff := []time.Duration{5 * time.Second, 15 * time.Second, 30 * time.Second, 60 * time.Second}
+ if len(info.Config.BackOff) != len(wantBackOff) {
+ t.Fatalf("BackOff = %v, want %v", info.Config.BackOff, wantBackOff)
+ }
+ for i, d := range wantBackOff {
+ if info.Config.BackOff[i] != d {
+ t.Fatalf("BackOff[%d] = %v, want %v", i, info.Config.BackOff[i], d)
+ }
+ }
+ // The server normalizes AckWait to BackOff[0] when BackOff is present;
+ // anything else means the schedule was not persisted as configured.
+ if got := info.Config.AckWait; got != wantBackOff[0] {
+ t.Fatalf("AckWait = %v, want %v (server-normalized to BackOff[0])", got, wantBackOff[0])
+ }
+}
diff --git a/internal/engine/nats/stream_migrate.go b/internal/engine/nats/stream_migrate.go
new file mode 100644
index 0000000000..8c7d91ff73
--- /dev/null
+++ b/internal/engine/nats/stream_migrate.go
@@ -0,0 +1,67 @@
+//
+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+package nats
+
+import (
+ "context"
+ "errors"
+
+ "github.com/nats-io/nats.go/jetstream"
+)
+
+// ensureStreamConfig creates the stream, or - when it already exists with an
+// older configuration (e.g. created by a previous deployment) - migrates it in
+// place via UpdateStream instead of failing with "stream name already in
+// use". Only the capacity/dedup fields (MaxBytes, MaxMsgs, Duplicates) are
+// migrated; the server-side current config is the merge base so a partial
+// update can never reset fields this helper does not own (Subjects, Retention,
+// Storage, ...).
+func ensureStreamConfig(ctx context.Context, js jetstream.JetStream, want jetstream.StreamConfig) (jetstream.Stream, error) {
+ st, err := js.CreateStream(ctx, want)
+ if err == nil {
+ return st, nil
+ }
+ // Match the API error code, not a message substring: the server reports
+ // 10058 ("stream name already in use"), which a naive "already exists"
+ // contains-check never matches.
+ var jsErr jetstream.JetStreamError
+ if !errors.As(err, &jsErr) || jsErr.APIError() == nil || jsErr.APIError().ErrorCode != jetstream.JSErrCodeStreamNameInUse {
+ return nil, err
+ }
+ st, err = js.Stream(ctx, want.Name)
+ if err != nil {
+ return nil, err
+ }
+ cur := st.CachedInfo().Config
+ needUpdate := cur.MaxBytes != want.MaxBytes || cur.MaxMsgs != want.MaxMsgs
+ if want.Duplicates != 0 && cur.Duplicates != want.Duplicates {
+ needUpdate = true
+ }
+ if needUpdate {
+ merged := cur
+ merged.MaxBytes = want.MaxBytes
+ merged.MaxMsgs = want.MaxMsgs
+ if want.Duplicates != 0 {
+ merged.Duplicates = want.Duplicates
+ }
+ st, err = js.UpdateStream(ctx, merged)
+ if err != nil {
+ return nil, err
+ }
+ }
+ return st, nil
+}
diff --git a/internal/engine/nats/syncer.go b/internal/engine/nats/syncer.go
index 83cf0179df..13a93baae1 100644
--- a/internal/engine/nats/syncer.go
+++ b/internal/engine/nats/syncer.go
@@ -62,7 +62,7 @@ func (n *NatsEngine) initSyncerStreamLocked() error {
defer cancel()
// create jetStream
- stream, err := n.jetStream.CreateStream(ctx, jetstream.StreamConfig{
+ stream, err := ensureStreamConfig(ctx, n.jetStream, jetstream.StreamConfig{
Name: syncerStreamName,
Subjects: []string{syncerSubjectPattern},
Retention: jetstream.WorkQueuePolicy,
@@ -72,13 +72,7 @@ func (n *NatsEngine) initSyncerStreamLocked() error {
Duplicates: 10 * time.Minute,
})
if err != nil {
- if !strings.Contains(err.Error(), "already exists") {
- return fmt.Errorf("syncer: create stream: %w", err)
- }
- stream, err = n.jetStream.Stream(ctx, syncerStreamName)
- if err != nil {
- return fmt.Errorf("syncer: get existing stream: %w", err)
- }
+ return fmt.Errorf("syncer: create stream: %w", err)
}
n.syncerStream = stream
return nil
diff --git a/internal/ingestion/service/burst_backpressure_test.go b/internal/ingestion/service/burst_backpressure_test.go
index 2da6a308ab..18266f6cc7 100644
--- a/internal/ingestion/service/burst_backpressure_test.go
+++ b/internal/ingestion/service/burst_backpressure_test.go
@@ -47,6 +47,7 @@ import (
"ragflow/internal/common"
"ragflow/internal/entity"
+ taskpkg "ragflow/internal/ingestion/task"
"ragflow/internal/ingestion/testutil"
"gorm.io/gorm"
@@ -174,6 +175,39 @@ func TestProcessMessage_BurstNoTaskLossUnderBackpressure(t *testing.T) {
}
}
+// TestFetchBudget_NeverExceedsChannelCapacity: the consume loop must never
+// pull more messages than it can enqueue immediately. The fetch budget is
+// min(free channel slots, maxConcurrency) with a floor of 1 so a fully
+// saturated channel still polls (and lets blocked sends drain) instead of
+// stalling until AckWait expires on in-flight messages.
+func TestFetchBudget_NeverExceedsChannelCapacity(t *testing.T) {
+ ingestor := newUnitIngestor("test", 2, []string{"pdf"}) // channel cap = 4
+ defer ingestor.Stop(context.Background())
+
+ if got := ingestor.fetchBudget(); got != 2 {
+ t.Fatalf("empty channel: fetchBudget = %d, want 2 (capped by maxConcurrency)", got)
+ }
+
+ for i := 0; i < cap(ingestor.taskChan)-1; i++ {
+ ingestor.taskChan <- taskpkg.NewTaskContextForScheduling(context.Background(), &entity.IngestionTask{ID: "filler"})
+ }
+ if got := ingestor.fetchBudget(); got != 1 {
+ t.Fatalf("3/4-full channel: fetchBudget = %d, want 1 (one free slot)", got)
+ }
+
+ ingestor.taskChan <- taskpkg.NewTaskContextForScheduling(context.Background(), &entity.IngestionTask{ID: "filler"})
+ if got := ingestor.fetchBudget(); got != 1 {
+ t.Fatalf("full channel: fetchBudget = %d, want 1 (floor: must never fetch 0)", got)
+ }
+
+ // A wide pool is bounded by the channel, not the worker count.
+ wide := newUnitIngestor("test-wide", 8, []string{"pdf"}) // channel cap = 16
+ defer wide.Stop(context.Background())
+ if got := wide.fetchBudget(); got != 8 {
+ t.Fatalf("wide empty channel: fetchBudget = %d, want 8 (capped by maxConcurrency)", got)
+ }
+}
+
// seedBurstTasks creates one tenant/kb plus N (document, ingestion_task) pairs
// in CREATED status, mirroring how CreateAndEnqueue publishes a message for a
// freshly-created task. Returns the ingestion task IDs.
diff --git a/internal/ingestion/service/ingestion_service.go b/internal/ingestion/service/ingestion_service.go
index cd78abc129..acba81b7e0 100644
--- a/internal/ingestion/service/ingestion_service.go
+++ b/internal/ingestion/service/ingestion_service.go
@@ -43,7 +43,15 @@ import (
"github.com/cenkalti/backoff/v5"
)
-const defaultHeartbeatInterval = 10 * time.Second
+// defaultHeartbeatInterval paces the InProgress() working pulse that keeps an
+// in-flight message's ack deadline renewed while a worker parses. It MUST stay
+// comfortably below the consumer's ack deadline, which the server normalizes
+// to BackOff[0] = 5s (see NatsEngine.InitConsumer): a pulse slower than the
+// deadline lets the broker redeliver mid-run, and the redelivered copy is
+// ack-skipped by the claim guard — acking the only in-flight copy, so a worker
+// crash afterwards would lose the delivery (downgraded to the 15min
+// reconciliation backstop instead of automatic redelivery).
+const defaultHeartbeatInterval = 2 * time.Second
type Ingestor struct {
id string
@@ -172,6 +180,9 @@ func (e *Ingestor) Start() error {
// error is retained and returned to every later caller.
func (e *Ingestor) start() error {
msgQueueEngine := engine.GetMessageQueueEngine()
+ if msgQueueEngine == nil {
+ return fmt.Errorf("message queue engine not initialized; run engine.InitMessageQueue first")
+ }
if err := msgQueueEngine.InitConsumer(common.TaskSubject); err != nil {
return err
}
@@ -187,9 +198,41 @@ func (e *Ingestor) start() error {
// Start returns promptly; it is joined by Stop via workerWg.
e.workerWg.Add(1)
go e.consumeLoop()
+
+ // Startup reconciliation heals tasks orphaned by a previous process
+ // crash (RUNNING stuck mid-run, CREATED never delivered). It runs off
+ // the caller's goroutine so Start is not blocked by the DB scan; it is
+ // joined by Stop via workerWg. It runs concurrently with consumeLoop:
+ // the CAS in StartRunning (CREATED→RUNNING) and markFailed
+ // (RUNNING→FAILED) handles the race where a stale row is both
+ // reconciled and redelivered, and pagination handles >500 orphans
+ // without stranding tasks. Keeping it async avoids delaying consumer
+ // liveness; a synchronous scan before consumeLoop would add startup
+ // latency with no stronger correctness.
+ e.workerWg.Add(1)
+ go func() {
+ defer e.workerWg.Done()
+ e.reconcileStartupTasks()
+ }()
return nil
}
+// fetchBudget caps each Fetch batch by the worker channel's free capacity (and
+// the worker width) so the consumer never pulls more messages than it can
+// enqueue immediately: messages beyond the budget would sit claimed in
+// taskChan's senders' hands or force blocking sends that stall the consume
+// loop while their AckWait runs out.
+func (e *Ingestor) fetchBudget() int {
+ budget := cap(e.taskChan) - len(e.taskChan)
+ if budget > int(e.maxConcurrency) {
+ budget = int(e.maxConcurrency)
+ }
+ if budget < 1 {
+ budget = 1
+ }
+ return budget
+}
+
// consumeLoop is the main tasks.RAGFLOW consume loop. It runs until e.ctx is
// cancelled (graceful shutdown); per-message failures are settled by
// processMessage and never terminate the consumer. Dataset-level compile work
@@ -206,7 +249,7 @@ func (e *Ingestor) consumeLoop() {
if err := e.ctx.Err(); err != nil {
return
}
- taskHandles, err := msgQueueEngine.GetMessages(4)
+ taskHandles, err := msgQueueEngine.GetMessages(e.fetchBudget())
if err != nil {
common.Error("error consuming message", err)
select {
@@ -253,6 +296,84 @@ func (e *Ingestor) SetKnowledgeCompileConcurrency(n int32) {
e.kcConcurrency = n
}
+// Startup reconciliation thresholds: how long a CREATED task must sit
+// without being picked up before the startup scan treats it as an orphan
+// of a previous process (judged by create_time, never updated after creation).
+const (
+ reconcileCreatedStaleAfter = 5 * time.Minute
+ reconcileBatchLimit = 500
+ reconcileOverallTimeout = 5 * time.Minute
+)
+
+// reconcileStartupTasks heals CREATED tasks orphaned by a previous process
+// crash (never consumed for 5min) by re-publishing via EnqueueByID.
+// Republishes are safe without broker-side dedup: the CREATED→RUNNING
+// transition in StartRunning is a CAS, and the in-process claim guard
+// ack-skips a second copy while a worker holds the task. (JetStream MsgID
+// dedup is intentionally NOT used - see NatsEngine.PublishTask: it would
+// swallow the retry republish of a task_id reused across runs, stranding
+// the row in CREATED.)
+//
+// RUNNING orphans are intentionally NOT touched here; recovery relies on
+// NATS redelivery (BackOff/InProgress) and the knowledge-compile style
+// lease would be the proper fix for long tasks, not a startup 15m→FAILED.
+// Errors are logged and skipped per task; the next restart re-runs the scan.
+// reconcileStartupTasks scans and heals orphaned rows. It carries no
+// WaitGroup bookkeeping of its own: start() wraps it with workerWg Add/Done,
+// and tests invoke it synchronously.
+func (e *Ingestor) reconcileStartupTasks() {
+ // Bounded by the ingestor's own lifetime: Stop cancels e.ctx, which
+ // aborts an in-flight scan instead of letting workerWg.Wait() linger.
+ ctx, cancel := context.WithTimeout(e.ctx, reconcileOverallTimeout)
+ defer cancel()
+
+ if dao.DB == nil {
+ common.Warn("startup reconciliation skipped: DB not initialized")
+ return
+ }
+
+ // CREATED orphans are re-published without a status change, so a
+ // drain-style loop (as used for RUNNING) would revisit the same head
+ // batch forever. Use offset pagination to walk the stale snapshot.
+ // The CREATED→RUNNING CAS in StartRunning and the in-process claim
+ // guard make concurrent races with consumeLoop safe: duplicate
+ // deliveries are ack-skipped while a worker holds the claim.
+ createdThreshold := time.Now().Add(-reconcileCreatedStaleAfter)
+ for offset := 0; ; offset += reconcileBatchLimit {
+ if ctx.Err() != nil {
+ return
+ }
+ // ListStaleByStatus does not support offset; query with offset
+ // directly to paginate the immutable CREATED snapshot.
+ var created []*entity.IngestionTask
+ q := dao.DB.WithContext(ctx).
+ Where("status IN ?", []string{common.CREATED}).
+ Where("create_time < ?", createdThreshold.UnixMilli()).
+ Order("create_time ASC").
+ Offset(offset).Limit(reconcileBatchLimit)
+ if err := q.Find(&created).Error; err != nil {
+ common.Warn(fmt.Sprintf("startup reconciliation: list CREATED orphans: %v", err))
+ return
+ }
+ if len(created) == 0 {
+ break
+ }
+ for _, task := range created {
+ if ctx.Err() != nil {
+ return
+ }
+ if err := e.ingestionTaskSvc.EnqueueByID(task.ID); err != nil {
+ common.Warn(fmt.Sprintf("startup reconciliation: re-enqueue CREATED task %s: %v", task.ID, err))
+ continue
+ }
+ common.Info(fmt.Sprintf("startup reconciliation: orphaned CREATED task %s re-enqueued", task.ID))
+ }
+ if len(created) < reconcileBatchLimit {
+ break
+ }
+ }
+}
+
// startDatasetKnowledgeCompile provisions the dataset-level compile scheduling
// store (MySQL knowledge_compile_docs + NATS notify subject), builds the
// Consumer, and starts the owned compile-worker pool. The consumer is driven by
@@ -445,10 +566,21 @@ func (e *Ingestor) processMessage(handle common.TaskHandle) {
claimedTaskID = "" // executeTask owns the release now
common.Info(fmt.Sprintf("Task %s queued (channel: %d/%d)", task.ID, len(e.taskChan), cap(e.taskChan)))
case <-e.ctx.Done():
- // Shutdown won the race: release the claim and return without
- // settling so the broker redelivers the message after restart
- // rather than blocking the consume loop forever.
- common.Info(fmt.Sprintf("Ingestor shutting down; releasing slot for task %s without ack", task.ID))
+ // Shutdown won the race after StartRunning already flipped the task
+ // to RUNNING: finalize it as STOPPED so the row cannot linger in
+ // non-terminal RUNNING with no in-flight worker (the user can
+ // retry). If the DB write itself fails, the startup reconciliation
+ // scan is the backstop. The message is left unsettled - on
+ // redelivery the terminal STOPPED status ack-skips it. Both paths
+ // run with a detached 2s timeout so shutdown is not slowed by a
+ // stalled DB.
+ stopCtx, cancel := context.WithTimeout(context.Background(), 2*time.Second)
+ defer cancel()
+ if e.markStopped(stopCtx, task.ID) {
+ common.Info(fmt.Sprintf("Ingestor shutting down; task %s finalized as STOPPED", task.ID))
+ } else {
+ common.Warn(fmt.Sprintf("Ingestor shutting down; task %s left RUNNING (startup reconciliation will finalize it)", task.ID))
+ }
return
}
}
@@ -604,7 +736,13 @@ func (e *Ingestor) executeTask(ctx context.Context, taskCtx *taskpkg.TaskContext
// RequestStop to handle RUNNING → STOPPING, then MarkStopped for the final
// STOPPING → STOPPED transition. Finally it cleans up the Redis cancel flag
// so that a future retry of the same task does not immediately re-cancel.
+// The caller's context is detached before touching the DB: markStopped runs
+// on failure/cancel/shutdown paths whose context is typically already
+// cancelled, and a contaminated context would make the terminal write fail
+// exactly when it matters most.
func (e *Ingestor) markStopped(ctx context.Context, taskID string) bool {
+ ctx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 5*time.Second)
+ defer cancel()
if _, err := e.ingestionTaskSvc.RequestStop(ctx, taskID); err != nil {
common.Error(fmt.Sprintf("markStopped: RequestStop task %s: %v", taskID, err), err)
return false
@@ -624,7 +762,12 @@ func (e *Ingestor) markStopped(ctx context.Context, taskID string) bool {
// markFailed persists FAILED status for the task and reports whether the
// terminal status was durably written, so the caller can decide Ack vs Nack.
+// The caller's context is detached before touching the DB (see markStopped):
+// failure paths usually carry an already-cancelled context, and losing the
+// FAILED write would turn a settled task into a redelivery loop.
func (e *Ingestor) markFailed(ctx context.Context, taskID string) bool {
+ ctx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 5*time.Second)
+ defer cancel()
if uErr := e.ingestionTaskSvc.MarkFailed(ctx, taskID); uErr != nil {
common.Error(fmt.Sprintf("Failed to set task %s to FAILED", taskID), uErr)
return false
@@ -648,7 +791,13 @@ func (e *Ingestor) runTask(ctx context.Context, task *entity.IngestionTask) bool
default:
}
- if err := e.ingestionTaskSvc.IncrementRunCount(ctx, task.ID); err != nil {
+ // The three DB/Redis lookups below must survive a context cancelled
+ // between the pre-check and the call (cancel poll races the pipeline
+ // start): detach the caller's ctx with a short timeout so a cancelled
+ // run fails through markFailed with a real error, not a context error.
+ dbCtx, dbCancel := context.WithTimeout(context.WithoutCancel(ctx), 5*time.Second)
+ defer dbCancel()
+ if err := e.ingestionTaskSvc.IncrementRunCount(dbCtx, task.ID); err != nil {
common.Error(fmt.Sprintf("Failed to increment run count for task %s", task.ID), err)
ok := e.markFailed(ctx, task.ID)
if ok {
@@ -660,7 +809,7 @@ func (e *Ingestor) runTask(ctx context.Context, task *entity.IngestionTask) bool
if checkpointExists == nil {
checkpointExists = canvas.RedisCheckpointExists
}
- resumeCheckpoint, checkpointErr := checkpointExists(ctx, task.ID)
+ resumeCheckpoint, checkpointErr := checkpointExists(dbCtx, task.ID)
if checkpointErr != nil {
common.Error(fmt.Sprintf("Failed to check checkpoint for task %s", task.ID), checkpointErr)
ok := e.markFailed(ctx, task.ID)
@@ -670,7 +819,7 @@ func (e *Ingestor) runTask(ctx context.Context, task *entity.IngestionTask) bool
return ok
}
if !resumeCheckpoint {
- if err := e.ingestionTaskSvc.ClearComponentProgress(ctx, task.ID); err != nil {
+ if err := e.ingestionTaskSvc.ClearComponentProgress(dbCtx, task.ID); err != nil {
common.Error(fmt.Sprintf("Failed to clear previous component progress for task %s", task.ID), err)
ok := e.markFailed(ctx, task.ID)
if ok {
diff --git a/internal/ingestion/service/ingestor_reconcile_test.go b/internal/ingestion/service/ingestor_reconcile_test.go
new file mode 100644
index 0000000000..bb31adb87f
--- /dev/null
+++ b/internal/ingestion/service/ingestor_reconcile_test.go
@@ -0,0 +1,188 @@
+//
+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+package service
+
+import (
+ "context"
+ "errors"
+ "strings"
+ "sync"
+ "testing"
+ "time"
+
+ "ragflow/internal/common"
+ "ragflow/internal/engine"
+ "ragflow/internal/entity"
+ taskpkg "ragflow/internal/ingestion/task"
+ "ragflow/internal/ingestion/testutil"
+
+ "gorm.io/gorm"
+)
+
+// enqueueRecorder captures re-enqueue publishes so the reconciliation scan
+// can be asserted without a live broker.
+type enqueueRecorder struct {
+ mu sync.Mutex
+ taskIDs []string
+}
+
+func (p *enqueueRecorder) PublishTaskMessage(_ string, msg common.TaskMessage) error {
+ p.mu.Lock()
+ defer p.mu.Unlock()
+ p.taskIDs = append(p.taskIDs, msg.TaskID)
+ return nil
+}
+
+func (p *enqueueRecorder) published() []string {
+ p.mu.Lock()
+ defer p.mu.Unlock()
+ return append([]string(nil), p.taskIDs...)
+}
+
+// seedAgedTask inserts an ingestion task with explicit create/update
+// timestamps aged back by the given duration (startup reconciliation judges
+// staleness by these columns).
+func seedAgedTask(t *testing.T, db *gorm.DB, id, docID, status string, age time.Duration) {
+ t.Helper()
+ ts := time.Now().Add(-age).UnixMilli()
+ if err := db.Create(&entity.IngestionTask{
+ ID: id,
+ UserID: "u1",
+ DocumentID: docID,
+ DatasetID: "kb-1",
+ Status: status,
+ BaseModel: entity.BaseModel{CreateTime: &ts, UpdateTime: &ts},
+ }).Error; err != nil {
+ t.Fatalf("seed task %s: %v", id, err)
+ }
+}
+
+// TestStartupReconciliation drives the startup scan: stale CREATED orphans
+// are re-published for consumption, stale RUNNING orphans are intentionally
+// left untouched (recovery relies on NATS redelivery, not startup FAILED;
+// see ingestion_service.go reconcileStartupTasks), and fresh rows are left alone.
+func TestStartupReconciliation(t *testing.T) {
+ db := testutil.SetupTestDB(t)
+ cleanup := testutil.ReplaceDBForTest(t, db)
+ defer cleanup()
+
+ seedAgedTask(t, db, "task-run-stale", "doc-run-stale", common.RUNNING, 16*time.Minute)
+ seedAgedTask(t, db, "task-run-fresh", "doc-run-fresh", common.RUNNING, 1*time.Minute)
+ seedAgedTask(t, db, "task-created-stale", "doc-created-stale", common.CREATED, 6*time.Minute)
+ seedAgedTask(t, db, "task-created-fresh", "doc-created-fresh", common.CREATED, 1*time.Minute)
+ seedAgedTask(t, db, "task-completed-old", "doc-completed-old", common.COMPLETED, 1*time.Hour)
+
+ ingestor := newUnitIngestor("test", 1, []string{"pdf"})
+ recorder := &enqueueRecorder{}
+ ingestor.ingestionTaskSvc.SetTaskPublisher(recorder)
+
+ ingestor.reconcileStartupTasks()
+
+ statusOf := func(id string) string {
+ t.Helper()
+ var task entity.IngestionTask
+ if err := db.Where("id = ?", id).First(&task).Error; err != nil {
+ t.Fatalf("reload task %s: %v", id, err)
+ }
+ return task.Status
+ }
+
+ // RUNNING orphans are intentionally left untouched (NATS redelivery path).
+ if got := statusOf("task-run-stale"); got != common.RUNNING {
+ t.Fatalf("stale RUNNING task status = %q, want %q (should be left for NATS redelivery)", got, common.RUNNING)
+ }
+ // Fresh RUNNING row belongs to a live worker — untouched.
+ if got := statusOf("task-run-fresh"); got != common.RUNNING {
+ t.Fatalf("fresh RUNNING task status = %q, want %q", got, common.RUNNING)
+ }
+ // CREATED orphans are only re-published; their status stays CREATED so
+ // the normal consume path (StartRunning) owns the transition.
+ if got := statusOf("task-created-stale"); got != common.CREATED {
+ t.Fatalf("stale CREATED task status = %q, want %q", got, common.CREATED)
+ }
+ if got := statusOf("task-created-fresh"); got != common.CREATED {
+ t.Fatalf("fresh CREATED task status = %q, want %q", got, common.CREATED)
+ }
+
+ // Exactly the stale CREATED task was re-enqueued — fresh CREATED rows
+ // still have their original message in flight, and terminal rows are
+ // never resurrected by the scan.
+ published := recorder.published()
+ if len(published) != 1 || published[0] != "task-created-stale" {
+ t.Fatalf("re-enqueued tasks = %v, want [task-created-stale]", published)
+ }
+}
+
+// TestStartNilEngine verifies that Start on an ingestor whose message queue
+// engine was never initialized returns an error instead of panicking on the
+// nil engine (mirror of the nats package uninitialized-engine contract).
+func TestStartNilEngine(t *testing.T) {
+ previousEngine := engine.GetMessageQueueEngine()
+ engine.SetMessageQueueEngine(nil)
+ t.Cleanup(func() { engine.SetMessageQueueEngine(previousEngine) })
+
+ ingestor := newUnitIngestor("test-nil-engine", 1, []string{"pdf"})
+ defer ingestor.Stop(context.Background())
+
+ err := ingestor.Start()
+ if err == nil || !strings.Contains(err.Error(), "not initialized") {
+ t.Fatalf("Start() with nil engine: err = %v, want 'not initialized'", err)
+ }
+ // The failure must happen before the worker pool is provisioned.
+ if got := ingestor.activeWorkers.Load(); got != 0 {
+ t.Fatalf("activeWorkers after failed Start = %d, want 0", got)
+ }
+}
+
+// TestExecuteTask_MarkFailedAfterCtxCancelAcks verifies the ctx de-contamination
+// of the failure path: the pipeline cancels the task context and then fails with
+// a generic error. The FAILED write must still land (detached short timeout) and
+// the message must be Acked (terminal), not Nack-requeued against a dead run.
+func TestExecuteTask_MarkFailedAfterCtxCancelAcks(t *testing.T) {
+ db := testutil.SetupTestDB(t)
+ cleanup := testutil.ReplaceDBForTest(t, db)
+ defer cleanup()
+ _, _, docID, taskID := testutil.SeedTestData(t, db, testutil.WithPipelineID("flow-1"))
+
+ ingestor := newUnitIngestor("test", 1, []string{"pdf"})
+
+ parentCtx, parentCancel := context.WithCancel(context.Background())
+ defer parentCancel()
+ handle := &fakeTaskHandle{}
+ taskCtx := taskpkg.NewTaskContextForScheduling(parentCtx, &entity.IngestionTask{
+ ID: taskID, DocumentID: docID, DatasetID: "kb-1", Status: common.RUNNING,
+ })
+ taskCtx.Handle = handle
+
+ ingestor.runDocumentTask = func(_ context.Context, _ *entity.IngestionTask) error {
+ parentCancel() // cancel races the pipeline...
+ return errors.New("boom") // ...and a generic failure lands afterwards
+ }
+
+ ingestor.executeTask(context.Background(), taskCtx)
+
+ var task entity.IngestionTask
+ if err := db.Where("id = ?", taskID).First(&task).Error; err != nil {
+ t.Fatalf("reload task: %v", err)
+ }
+ if task.Status != common.FAILED {
+ t.Fatalf("task status = %q, want %q (FAILED write must survive a cancelled ctx)", task.Status, common.FAILED)
+ }
+ if handle.acks.Load() != 1 || handle.nacks.Load() != 0 {
+ t.Fatalf("expected 1 Ack/0 Nack (terminal), got acks=%d nacks=%d", handle.acks.Load(), handle.nacks.Load())
+ }
+}
diff --git a/internal/ingestion/service/process_message_test.go b/internal/ingestion/service/process_message_test.go
index 671db5d30c..3e9441461e 100644
--- a/internal/ingestion/service/process_message_test.go
+++ b/internal/ingestion/service/process_message_test.go
@@ -371,6 +371,73 @@ func TestProcessMessage_ChannelFullBlocksUntilSlot(t *testing.T) {
}
}
+// TestProcessMessage_ShutdownRaceMarksStopped: shutdown winning the race
+// against the taskChan send must not leave the task in non-terminal RUNNING
+// with no in-flight worker. StartRunning has already flipped the task to
+// RUNNING, so the ctx.Done branch finalizes it as STOPPED (user-retryable)
+// with a detached timeout, and leaves the message unsettled — the broker
+// redelivers it and the terminal status ack-skips.
+func TestProcessMessage_ShutdownRaceMarksStopped(t *testing.T) {
+ db := testutil.SetupTestDB(t)
+ cleanup := testutil.ReplaceDBForTest(t, db)
+ defer cleanup()
+ _, _, _, taskID := testutil.SeedTestData(t, db, testutil.WithPipelineID("flow-1"))
+ // SeedTestData creates the task RUNNING; reset to CREATED so the race
+ // below is the real one: processMessage's StartRunning flips it RUNNING
+ // and then blocks on the full channel.
+ if err := db.Model(&entity.IngestionTask{}).Where("id = ?", taskID).
+ Update("status", common.CREATED).Error; err != nil {
+ t.Fatalf("reset task to CREATED: %v", err)
+ }
+
+ ingestor := newUnitIngestor("test", 1, []string{"pdf"}) // channel cap = 2
+ for i := 0; i < cap(ingestor.taskChan); i++ {
+ ingestor.taskChan <- taskpkg.NewTaskContextForScheduling(nil, &entity.IngestionTask{ID: "filler"})
+ }
+
+ handle := newFakeHandle(taskID, common.TaskTypeIngestionTask)
+ done := make(chan struct{})
+ go func() {
+ ingestor.processMessage(handle)
+ close(done)
+ }()
+
+ // Wait until StartRunning flipped the task RUNNING and processMessage
+ // is blocked on the full channel.
+ deadline := time.Now().Add(2 * time.Second)
+ for {
+ var task entity.IngestionTask
+ if err := db.Where("id = ?", taskID).First(&task).Error; err == nil && task.Status == common.RUNNING {
+ break
+ }
+ if time.Now().After(deadline) {
+ t.Fatal("task never reached RUNNING; processMessage did not reach the blocking send")
+ }
+ time.Sleep(2 * time.Millisecond)
+ }
+
+ // Shutdown wins the race against the channel send.
+ ingestor.cancel()
+
+ select {
+ case <-done:
+ case <-time.After(5 * time.Second):
+ t.Fatal("processMessage did not return after shutdown")
+ }
+
+ var task entity.IngestionTask
+ if err := db.Where("id = ?", taskID).First(&task).Error; err != nil {
+ t.Fatalf("reload task: %v", err)
+ }
+ if task.Status != common.STOPPED {
+ t.Fatalf("task status = %q, want %q (shutdown race must not strand a RUNNING row)", task.Status, common.STOPPED)
+ }
+ // Message unsettled: on redelivery the STOPPED status ack-skips it.
+ if handle.acks.Load() != 0 || handle.nacks.Load() != 0 {
+ t.Fatalf("expected 0 Ack/0 Nack (unsettled for redelivery), got acks=%d nacks=%d", handle.acks.Load(), handle.nacks.Load())
+ }
+}
+
// TestProcessMessage_StartRunningErrorNacks: when StartRunning returns a
// non-ErrTaskNotFound error (e.g. a DB blip), processMessage nacks the
// message for redelivery instead of killing the consumer (B2 fix). The
diff --git a/internal/service/ingestion_task_service.go b/internal/service/ingestion_task_service.go
index 036791133a..c037d59ed6 100644
--- a/internal/service/ingestion_task_service.go
+++ b/internal/service/ingestion_task_service.go
@@ -458,6 +458,15 @@ func (s *IngestionTaskService) enqueueTask(taskID string) error {
return s.taskPublisher.PublishTaskMessage("tasks.RAGFLOW", taskMessage)
}
+// EnqueueByID re-publishes the wake-up message for an existing task without
+// touching its status. Used by ingestor startup reconciliation to redeliver
+// CREATED orphans; duplicate delivery is made safe at the consumer level
+// (StartRunning CAS + in-process claim guard), not by broker dedup — see
+// NatsEngine.PublishTask for why publish-time MsgID dedup is harmful here.
+func (s *IngestionTaskService) EnqueueByID(taskID string) error {
+ return s.enqueueTask(taskID)
+}
+
// UpdateComponentTotal records the number of components in the task's DSL
// graph - the authoritative denominator for progress percentage.
func (s *IngestionTaskService) UpdateComponentTotal(ctx context.Context, taskID string, total int) error {