RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
fix(ingestor,nats): recover consumption, fail fast on NATS startup and harden reconciliation (P0 blockers) (#18942)
jay77721de6b12a2b912
diff --git a/cmd/ragflow_server.go b/cmd/ragflow_server.goindex 900913e5e3..aeb75bb0b1 100644--- a/cmd/ragflow_server.go+++ b/cmd/ragflow_server.go@@ -376,7 +376,7 @@ func main() {defer storage.CloseStorage()if err = engine.InitMessageQueue(); err != nil {- common.Error("Failed to initialize message queue engine", err)+ common.Fatal("Failed to initialize message queue engine", zap.Error(err))}// Initialize server variables (runtime variables that can change during operation)@@ -583,10 +583,12 @@ func runIngestor(ctx context.Context, cancel context.CancelFunc, args *serverArgingestor.SetMemoryMessageService(service.NewMemoryMessageService(service.NewMemoryService()))// Start returns immediately (it launches the owned consume/compile- // goroutines and joins them via Stop); a provisioning failure here is- // logged and the server keeps running without the ingestor.+ // goroutines and joins them via Stop); a provisioning failure here must+ // fail the server (main's os.Exit(1) path) instead of reporting a+ // healthy ingestor that can never consume.if err := ingestor.Start(); err != nil {common.Error("Failed to initialize ingestor", err)+ return err}common.Info("\n ____ __ _\n" +diff --git a/docker/docker-compose.yml b/docker/docker-compose.ymlindex 6321b72b32..2cf741a42b 100644--- a/docker/docker-compose.yml+++ b/docker/docker-compose.yml@@ -23,6 +23,9 @@ services:mysql:condition: service_healthyrequired: false+ nats:+ condition: service_healthy+ required: falseprofiles:- cpuimage: ${RAGFLOW_IMAGE}@@ -86,6 +89,9 @@ services:mysql:condition: service_healthyrequired: false+ nats:+ condition: service_healthy+ required: falseprofiles:- gpuimage: ${RAGFLOW_IMAGE}diff --git a/internal/dao/ingestion.go b/internal/dao/ingestion.goindex 2afef863e4..dae1a10659 100644--- a/internal/dao/ingestion.go+++ b/internal/dao/ingestion.go@@ -23,6 +23,7 @@ import ("ragflow/internal/common""ragflow/internal/entity""ragflow/internal/utility"+ "time""gorm.io/gorm")@@ -195,6 +196,33 @@ func (dao *IngestionTaskDAO) GetByDocumentID(ctx context.Context, db *gorm.DB, dreturn tasks[0], nil}+// ListStaleByStatus returns tasks in the given statuses whose timestamp column+// is older than the threshold, for ingestor startup reconciliation. timeColumn+// selects which staleness clock applies: "update_time" for RUNNING orphans+// (frozen at the moment the previous process claimed them) or "create_time"+// for CREATED orphans (never picked up). Results are ordered oldest first and+// capped at limit when limit > 0.+func (dao *IngestionTaskDAO) ListStaleByStatus(ctx context.Context, db *gorm.DB, statuses []string, timeColumn string, olderThan time.Time, limit int) ([]*entity.IngestionTask, error) {+ switch timeColumn {+ case "create_time", "update_time":+ default:+ return nil, fmt.Errorf("unsupported staleness column %q", timeColumn)+ }+ if len(statuses) == 0 {+ return []*entity.IngestionTask{}, nil+ }+ query := db.WithContext(ctx).+ Where("status IN ?", statuses).+ Where(timeColumn+" < ?", olderThan.UnixMilli()).+ Order(timeColumn + " ASC")+ if limit > 0 {+ query = query.Limit(limit)+ }+ var tasks []*entity.IngestionTask+ err := query.Find(&tasks).Error+ return tasks, err+}+// DeleteIfTerminal deletes ingestion tasks for a document that are in a// terminal state (COMPLETED, STOPPED, FAILED) or still queued (CREATED).// RUNNING and STOPPING tasks are NOT deleted because an in-flight workerdiff --git a/internal/dao/ingestion_task_test.go b/internal/dao/ingestion_task_test.goindex 35d9931745..f047a8a2ba 100644--- a/internal/dao/ingestion_task_test.go+++ b/internal/dao/ingestion_task_test.go@@ -3,6 +3,7 @@ package daoimport ("errors""testing"+ "time""fmt""ragflow/internal/common"@@ -138,6 +139,96 @@ func TestIngestionTaskDAOUpdateStatusIfCurrentRejectsMismatchedStatus(t *testing}}+// seedStaleTask inserts an ingestion task whose create/update timestamps are+// aged back so ListStaleByStatus staleness windows can be asserted.+func seedStaleTask(t *testing.T, db *gorm.DB, id, status string, age time.Duration) {+ t.Helper()+ ts := time.Now().Add(-age).UnixMilli()+ task := &entity.IngestionTask{+ ID: id,+ UserID: "user-1",+ DocumentID: "doc-" + id,+ DatasetID: "kb-1",+ Status: status,+ BaseModel: entity.BaseModel{CreateTime: &ts, UpdateTime: &ts},+ }+ if err := db.Create(task).Error; err != nil {+ t.Fatalf("create task %s: %v", id, err)+ }+}++// TestIngestionTaskDAOListStaleByStatus covers the three contract axes of the+// startup-reconciliation query: status filtering, the staleness time window+// (per column), and the result limit.+func TestIngestionTaskDAOListStaleByStatus(t *testing.T) {+ db := setupTaskTestDB(t)+ orig := DB+ DB = db+ t.Cleanup(func() { DB = orig })++ seedStaleTask(t, db, "run-stale", common.RUNNING, 20*time.Minute)+ seedStaleTask(t, db, "run-fresh", common.RUNNING, 1*time.Minute)+ seedStaleTask(t, db, "created-stale", common.CREATED, 20*time.Minute)+ seedStaleTask(t, db, "created-fresh", common.CREATED, 1*time.Minute)+ seedStaleTask(t, db, "completed-stale", common.COMPLETED, 20*time.Minute)++ ctx := t.Context()+ d := NewIngestionTaskDAO()+ threshold := time.Now().Add(-15 * time.Minute)++ // Status filter + update_time window: only the stale RUNNING row matches.+ running, err := d.ListStaleByStatus(ctx, db, []string{common.RUNNING}, "update_time", threshold, 0)+ if err != nil {+ t.Fatalf("ListStaleByStatus(RUNNING): %v", err)+ }+ if len(running) != 1 || running[0].ID != "run-stale" {+ t.Fatalf("RUNNING stale rows = %v, want [run-stale]", idsOfTasks(running))+ }++ // CREATED is judged by create_time: fresh CREATED rows stay outside the+ // window even though their status is in the filter.+ created, err := d.ListStaleByStatus(ctx, db, []string{common.CREATED}, "create_time", threshold, 0)+ if err != nil {+ t.Fatalf("ListStaleByStatus(CREATED): %v", err)+ }+ if len(created) != 1 || created[0].ID != "created-stale" {+ t.Fatalf("CREATED stale rows = %v, want [created-stale]", idsOfTasks(created))+ }++ // Rows in unlisted statuses (COMPLETED) are never returned.+ all, err := d.ListStaleByStatus(ctx, db, []string{common.RUNNING, common.CREATED}, "create_time", threshold, 0)+ if err != nil {+ t.Fatalf("ListStaleByStatus(RUNNING+CREATED): %v", err)+ }+ for _, task := range all {+ if task.ID == "completed-stale" {+ t.Fatal("COMPLETED row must not be returned for RUNNING/CREATED filter")+ }+ }++ // Limit caps the (oldest-first) result set.+ limited, err := d.ListStaleByStatus(ctx, db, []string{common.RUNNING, common.CREATED}, "create_time", threshold, 1)+ if err != nil {+ t.Fatalf("ListStaleByStatus(limit): %v", err)+ }+ if len(limited) != 1 {+ t.Fatalf("limited rows = %v, want exactly 1", idsOfTasks(limited))+ }++ // An unsupported staleness column is rejected, not silently ignored.+ if _, err = d.ListStaleByStatus(ctx, db, []string{common.RUNNING}, "status; --", threshold, 0); err == nil {+ t.Fatal("expected error for unsupported staleness column")+ }+}++func idsOfTasks(tasks []*entity.IngestionTask) []string {+ ids := make([]string, 0, len(tasks))+ for _, task := range tasks {+ ids = append(ids, task.ID)+ }+ return ids+}+func TestIngestionTaskDAODeleteIfTerminal_RemovesOnlyTerminal(t *testing.T) {db := setupTaskTestDB(t)orig := DBdiff --git a/internal/engine/nats/knowledgecompile.go b/internal/engine/nats/knowledgecompile.goindex 8e1fbbbbfc..c2a7be41a4 100644--- a/internal/engine/nats/knowledgecompile.go+++ b/internal/engine/nats/knowledgecompile.go@@ -53,17 +53,18 @@ func (n *NatsEngine) InitKnowledgeCompileStream() error {}ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)defer cancel()- cfg := jetstream.StreamConfig{+ stream, err := ensureStreamConfig(ctx, n.jetStream, jetstream.StreamConfig{Name: knowledgeCompileStreamName,Subjects: []string{knowledgeCompileSubjectPrefix},Retention: jetstream.WorkQueuePolicy,Storage: jetstream.FileStorage,MaxMsgs: 1024 * 1024,MaxBytes: 1024 * 1024 * 64,- }- if _, err := n.jetStream.CreateStream(ctx, cfg); err != nil && !strings.Contains(err.Error(), "already exists") {+ })+ if err != nil {return fmt.Errorf("knowledgecompile: create stream: %w", err)}+ n.knowledgeCompileStream = streamreturn nil}diff --git a/internal/engine/nats/nats.go b/internal/engine/nats/nats.goindex cc09ebb42f..ae91d9faac 100644--- a/internal/engine/nats/nats.go+++ b/internal/engine/nats/nats.go@@ -80,25 +80,20 @@ func (n *NatsEngine) Init() error {Retention: jetstream.WorkQueuePolicy,Storage: jetstream.FileStorage,MaxMsgs: 1024 * 128,- MaxBytes: 1024 * 1024,+ MaxBytes: 1024 * 1024 * 64,+ // Server-side dedup window. Inert for task publishes: PublishTask+ // intentionally sends no MsgID (see below — dedup would swallow+ // retry republishes of a reused task_id). It only takes effect for+ // a publisher that opts into MsgIDs.+ Duplicates: 10 * time.Minute,}- n.stream, err = n.jetStream.CreateStream(ctx, streamCfg)+ n.stream, err = ensureStreamConfig(ctx, n.jetStream, streamCfg)if err != nil {- if !strings.Contains(err.Error(), "already exists") {- n.nc.Close()- return fmt.Errorf("fail to create stream at %s: %w", natsURL, err)- }-- common.Info("NATS stream already exists, use existing stream")- n.stream, err = n.jetStream.Stream(ctx, "RAGFLOW_TASKS")- if err != nil {- n.nc.Close()- return fmt.Errorf("fail to get existing stream at %s: %w", natsURL, err)- }- } else {- common.Info(fmt.Sprintf("NATS stream create successfully at %s", natsURL))+ n.nc.Close()+ return fmt.Errorf("fail to create stream at %s: %w", natsURL, err)}+ common.Info(fmt.Sprintf("NATS stream RAGFLOW_TASKS ready at %s", natsURL))return nil}@@ -115,6 +110,16 @@ func (n *NatsEngine) PublishTask(subject string, payload []byte) error {ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)defer cancel()+ // Deliberately NO MsgID/dedup here. Ingestion tasks reuse the same+ // task_id across publish attempts (the FAILED/STOPPED→CREATED retry+ // path), and the server's Duplicates window suppresses a republished+ // MsgID for its whole lifetime regardless of whether the original+ // message was already consumed and acked. A deduped retry publish+ // strands the task in CREATED with no message behind it — unreachable+ // by any consumer and un-reparsable ("already exists, status: CREATED").+ // Duplicate delivery is instead made safe at the consumer level:+ // StartRunning's CREATED→RUNNING CAS plus the in-process claim guard+ // ack-skip any second copy (see Ingestor.processMessage).ack, err := n.jetStream.Publish(ctx, subject, payload)if err != nil {return err@@ -212,15 +217,32 @@ func (n *NatsEngine) InitConsumer(subject string) error {defer cancel()var err error+ // Explicit redelivery schedule: BackOff paces successive redeliveries+ // (5s/15s/30s, then 60s repeated) so an unsettled message (crash, slow+ // DB) is retried with breathing room instead of the broker default. The+ // server normalizes AckWait to BackOff[0] when BackOff is present; the+ // 60s AckWait is the effective schedule if BackOff is ever dropped.+ // INVARIANT: the worker's InProgress heartbeat (Ingestor+ // defaultHeartbeatInterval) must stay below BackOff[0] = 5s, or in-flight+ // messages get redelivered mid-run and the claim-guard ack-skip acks the+ // only live copy.+ // Note: CreateOrUpdateConsumer is atomic. MaxWaiting is immutable after+ // creation; if it mismatches the whole update fails (error "max waiting+ // can not be updated") and NONE of AckWait/BackOff/MaxAckPending are+ // applied. When MaxWaiting matches (the default, since this config omits+ // it), the other three update in place. The fallback below handles the+ // MaxWaiting mismatch by keeping the existing consumer.n.consumer, err = n.stream.CreateOrUpdateConsumer(ctx, jetstream.ConsumerConfig{Name: "RAGFLOW_CONSUMER",AckPolicy: jetstream.AckExplicitPolicy,MaxDeliver: 16,MaxAckPending: 1024 * 128,FilterSubject: "tasks.>",+ AckWait: 60 * time.Second,+ BackOff: []time.Duration{5 * time.Second, 15 * time.Second, 30 * time.Second, 60 * time.Second},})if err != nil {- // MaxAckPending is immutable after consumer creation.+ // MaxWaiting is immutable after consumer creation (AckWait/BackOff/MaxAckPending remain mutable when MaxWaiting matches; the update is atomic).// If the consumer already exists, fall back to fetching it.if strings.Contains(err.Error(), "max waiting can not be updated") {n.consumer, err = n.stream.Consumer(ctx, "RAGFLOW_CONSUMER")diff --git a/internal/engine/nats/stream_config_test.go b/internal/engine/nats/stream_config_test.gonew file mode 100644index 0000000000..331fb44ce6--- /dev/null+++ b/internal/engine/nats/stream_config_test.go@@ -0,0 +1,207 @@+//+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.+//++package nats++import (+ "context"+ "encoding/json"+ "net"+ "strconv"+ "testing"+ "time"++ "ragflow/internal/common"++ "github.com/nats-io/nats-server/v2/server"+ "github.com/nats-io/nats.go"+ "github.com/nats-io/nats.go/jetstream"+)++// newEmbeddedNatsServer starts an in-process JetStream-enabled NATS server on+// a random port and returns its host/port.+func newEmbeddedNatsServer(t *testing.T) (string, int) {+ t.Helper()+ opts := &server.Options{+ Port: -1,+ JetStream: true,+ StoreDir: t.TempDir(),+ NoLog: true,+ NoSigs: true,+ }+ ns, err := server.NewServer(opts)+ if err != nil {+ t.Fatalf("create embedded NATS server: %v", err)+ }+ ns.Start()+ if !ns.ReadyForConnections(10 * time.Second) {+ ns.Shutdown()+ t.Fatal("embedded NATS server did not become ready")+ }+ t.Cleanup(func() {+ ns.Shutdown()+ ns.WaitForShutdown()+ })+ addr := ns.Addr().(*net.TCPAddr)+ return "127.0.0.1", addr.Port+}++// TestPublishTaskDeliversRepeatedTaskIDs: publishing the same task_id twice+// MUST land two messages. Ingestion tasks reuse the task_id across publish+// attempts (the FAILED/STOPPED→CREATED retry path), and a JetStream MsgID+// dedup would suppress the retry republish within the Duplicates window even+// though the original message is long gone — stranding the task in CREATED+// with no message behind it ("already exists, status: CREATED" forever).+// This test is the regression guard against reintroducing publish dedup.+func TestPublishTaskDeliversRepeatedTaskIDs(t *testing.T) {+ host, port := newEmbeddedNatsServer(t)+ engine := NewNatsEngine(host, port)+ if err := engine.Init(); err != nil {+ t.Fatalf("Init: %v", err)+ }+ if err := engine.InitConsumer("tasks.>"); err != nil {+ t.Fatalf("InitConsumer: %v", err)+ }++ payload, err := json.Marshal(common.TaskMessage{TaskID: "task-repeat-1", TaskType: common.TaskTypeIngestionTask})+ if err != nil {+ t.Fatalf("marshal: %v", err)+ }+ if err = engine.PublishTask("tasks.RAGFLOW", payload); err != nil {+ t.Fatalf("first PublishTask: %v", err)+ }+ if err = engine.PublishTask("tasks.RAGFLOW", payload); err != nil {+ t.Fatalf("repeated PublishTask: %v", err)+ }++ ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)+ defer cancel()+ info, err := engine.stream.Info(ctx)+ if err != nil {+ t.Fatalf("stream info: %v", err)+ }+ if info.State.Msgs != 2 {+ t.Fatalf("stream holds %d messages after two publishes of the same task_id, want 2 (publish dedup strands retry republishes)", info.State.Msgs)+ }++ // A payload without a decodable TaskMessage shape must not fail either.+ if err = engine.PublishTask("tasks.RAGFLOW", []byte("not-json")); err != nil {+ t.Fatalf("non-TaskMessage PublishTask: %v", err)+ }+}++// TestInitMigratesLegacyStreamConfig: a stream created by an older deployment+// (no Duplicates, 1MB MaxBytes) must be migrated in place by Init instead of+// being left stale behind an "already exists" error. The server-side config is+// the merge base: fields the helper does not own (Subjects, Retention) must+// survive the update.+func TestInitMigratesLegacyStreamConfig(t *testing.T) {+ host, port := newEmbeddedNatsServer(t)++ // Pre-create the stream with the legacy (pre-migration) configuration.+ nc, err := nats.Connect("nats://" + net.JoinHostPort(host, strconv.Itoa(port)))+ if err != nil {+ t.Fatalf("connect legacy stream creator: %v", err)+ }+ defer nc.Close()+ js, err := jetstream.New(nc)+ if err != nil {+ t.Fatalf("jetstream context: %v", err)+ }+ ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)+ defer cancel()+ legacy, err := js.CreateStream(ctx, jetstream.StreamConfig{+ Name: "RAGFLOW_TASKS",+ Subjects: []string{"tasks.>"},+ Retention: jetstream.WorkQueuePolicy,+ Storage: jetstream.FileStorage,+ MaxMsgs: 1024 * 128,+ MaxBytes: 1024 * 1024,+ })+ if err != nil {+ t.Fatalf("create legacy stream: %v", err)+ }+ legacyInfo, err := legacy.Info(ctx)+ if err != nil {+ t.Fatalf("legacy stream info: %v", err)+ }+ // Precondition: the legacy stream differs from the wanted config (server+ // fills a 2m default Duplicates when unspecified).+ if legacyInfo.Config.MaxBytes != int64(1024*1024) {+ t.Fatalf("precondition: legacy MaxBytes = %d, want 1MB", legacyInfo.Config.MaxBytes)+ }+ if legacyInfo.Config.Duplicates == 10*time.Minute {+ t.Fatalf("precondition: legacy Duplicates already migrated (%v)", legacyInfo.Config.Duplicates)+ }++ engine := NewNatsEngine(host, port)+ if err := engine.Init(); err != nil {+ t.Fatalf("Init over legacy stream: %v", err)+ }++ info, err := engine.stream.Info(ctx)+ if err != nil {+ t.Fatalf("stream info after migration: %v", err)+ }+ if got := info.Config.MaxBytes; got != int64(1024*1024*64) {+ t.Fatalf("MaxBytes after migration = %d, want %d", got, int64(1024*1024*64))+ }+ if got := info.Config.Duplicates; got != 10*time.Minute {+ t.Fatalf("Duplicates after migration = %v, want 10m", got)+ }+ // Non-owned fields must not be reset by the partial update.+ if got := info.Config.Retention; got != jetstream.WorkQueuePolicy {+ t.Fatalf("Retention after migration = %v, want WorkQueuePolicy (must not be reset)", got)+ }+ if len(info.Config.Subjects) != 1 || info.Config.Subjects[0] != "tasks.>" {+ t.Fatalf("Subjects after migration = %v, want [tasks.>] (must not be reset)", info.Config.Subjects)+ }+}++// TestInitConsumerSetsAckWaitAndBackOff: the consumer must carry an explicit+// redelivery schedule (AckWait + BackOff) so unacked messages are retried on+// a paced backoff instead of the broker default.+func TestInitConsumerSetsAckWaitAndBackOff(t *testing.T) {+ host, port := newEmbeddedNatsServer(t)+ engine := NewNatsEngine(host, port)+ if err := engine.Init(); err != nil {+ t.Fatalf("Init: %v", err)+ }+ if err := engine.InitConsumer("tasks.>"); err != nil {+ t.Fatalf("InitConsumer: %v", err)+ }++ ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)+ defer cancel()+ info, err := engine.consumer.Info(ctx)+ if err != nil {+ t.Fatalf("consumer info: %v", err)+ }+ wantBackOff := []time.Duration{5 * time.Second, 15 * time.Second, 30 * time.Second, 60 * time.Second}+ if len(info.Config.BackOff) != len(wantBackOff) {+ t.Fatalf("BackOff = %v, want %v", info.Config.BackOff, wantBackOff)+ }+ for i, d := range wantBackOff {+ if info.Config.BackOff[i] != d {+ t.Fatalf("BackOff[%d] = %v, want %v", i, info.Config.BackOff[i], d)+ }+ }+ // The server normalizes AckWait to BackOff[0] when BackOff is present;+ // anything else means the schedule was not persisted as configured.+ if got := info.Config.AckWait; got != wantBackOff[0] {+ t.Fatalf("AckWait = %v, want %v (server-normalized to BackOff[0])", got, wantBackOff[0])+ }+}diff --git a/internal/engine/nats/stream_migrate.go b/internal/engine/nats/stream_migrate.gonew file mode 100644index 0000000000..8c7d91ff73--- /dev/null+++ b/internal/engine/nats/stream_migrate.go@@ -0,0 +1,67 @@+//+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.+//++package nats++import (+ "context"+ "errors"++ "github.com/nats-io/nats.go/jetstream"+)++// ensureStreamConfig creates the stream, or - when it already exists with an+// older configuration (e.g. created by a previous deployment) - migrates it in+// place via UpdateStream instead of failing with "stream name already in+// use". Only the capacity/dedup fields (MaxBytes, MaxMsgs, Duplicates) are+// migrated; the server-side current config is the merge base so a partial+// update can never reset fields this helper does not own (Subjects, Retention,+// Storage, ...).+func ensureStreamConfig(ctx context.Context, js jetstream.JetStream, want jetstream.StreamConfig) (jetstream.Stream, error) {+ st, err := js.CreateStream(ctx, want)+ if err == nil {+ return st, nil+ }+ // Match the API error code, not a message substring: the server reports+ // 10058 ("stream name already in use"), which a naive "already exists"+ // contains-check never matches.+ var jsErr jetstream.JetStreamError+ if !errors.As(err, &jsErr) || jsErr.APIError() == nil || jsErr.APIError().ErrorCode != jetstream.JSErrCodeStreamNameInUse {+ return nil, err+ }+ st, err = js.Stream(ctx, want.Name)+ if err != nil {+ return nil, err+ }+ cur := st.CachedInfo().Config+ needUpdate := cur.MaxBytes != want.MaxBytes || cur.MaxMsgs != want.MaxMsgs+ if want.Duplicates != 0 && cur.Duplicates != want.Duplicates {+ needUpdate = true+ }+ if needUpdate {+ merged := cur+ merged.MaxBytes = want.MaxBytes+ merged.MaxMsgs = want.MaxMsgs+ if want.Duplicates != 0 {+ merged.Duplicates = want.Duplicates+ }+ st, err = js.UpdateStream(ctx, merged)+ if err != nil {+ return nil, err+ }+ }+ return st, nil+}diff --git a/internal/engine/nats/syncer.go b/internal/engine/nats/syncer.goindex 83cf0179df..13a93baae1 100644--- a/internal/engine/nats/syncer.go+++ b/internal/engine/nats/syncer.go@@ -62,7 +62,7 @@ func (n *NatsEngine) initSyncerStreamLocked() error {defer cancel()// create jetStream- stream, err := n.jetStream.CreateStream(ctx, jetstream.StreamConfig{+ stream, err := ensureStreamConfig(ctx, n.jetStream, jetstream.StreamConfig{Name: syncerStreamName,Subjects: []string{syncerSubjectPattern},Retention: jetstream.WorkQueuePolicy,@@ -72,13 +72,7 @@ func (n *NatsEngine) initSyncerStreamLocked() error {Duplicates: 10 * time.Minute,})if err != nil {- if !strings.Contains(err.Error(), "already exists") {- return fmt.Errorf("syncer: create stream: %w", err)- }- stream, err = n.jetStream.Stream(ctx, syncerStreamName)- if err != nil {- return fmt.Errorf("syncer: get existing stream: %w", err)- }+ return fmt.Errorf("syncer: create stream: %w", err)}n.syncerStream = streamreturn nildiff --git a/internal/ingestion/service/burst_backpressure_test.go b/internal/ingestion/service/burst_backpressure_test.goindex 2da6a308ab..18266f6cc7 100644--- a/internal/ingestion/service/burst_backpressure_test.go+++ b/internal/ingestion/service/burst_backpressure_test.go@@ -47,6 +47,7 @@ import ("ragflow/internal/common""ragflow/internal/entity"+ taskpkg "ragflow/internal/ingestion/task""ragflow/internal/ingestion/testutil""gorm.io/gorm"@@ -174,6 +175,39 @@ func TestProcessMessage_BurstNoTaskLossUnderBackpressure(t *testing.T) {}}+// TestFetchBudget_NeverExceedsChannelCapacity: the consume loop must never+// pull more messages than it can enqueue immediately. The fetch budget is+// min(free channel slots, maxConcurrency) with a floor of 1 so a fully+// saturated channel still polls (and lets blocked sends drain) instead of+// stalling until AckWait expires on in-flight messages.+func TestFetchBudget_NeverExceedsChannelCapacity(t *testing.T) {+ ingestor := newUnitIngestor("test", 2, []string{"pdf"}) // channel cap = 4+ defer ingestor.Stop(context.Background())++ if got := ingestor.fetchBudget(); got != 2 {+ t.Fatalf("empty channel: fetchBudget = %d, want 2 (capped by maxConcurrency)", got)+ }++ for i := 0; i < cap(ingestor.taskChan)-1; i++ {+ ingestor.taskChan <- taskpkg.NewTaskContextForScheduling(context.Background(), &entity.IngestionTask{ID: "filler"})+ }+ if got := ingestor.fetchBudget(); got != 1 {+ t.Fatalf("3/4-full channel: fetchBudget = %d, want 1 (one free slot)", got)+ }++ ingestor.taskChan <- taskpkg.NewTaskContextForScheduling(context.Background(), &entity.IngestionTask{ID: "filler"})+ if got := ingestor.fetchBudget(); got != 1 {+ t.Fatalf("full channel: fetchBudget = %d, want 1 (floor: must never fetch 0)", got)+ }++ // A wide pool is bounded by the channel, not the worker count.+ wide := newUnitIngestor("test-wide", 8, []string{"pdf"}) // channel cap = 16+ defer wide.Stop(context.Background())+ if got := wide.fetchBudget(); got != 8 {+ t.Fatalf("wide empty channel: fetchBudget = %d, want 8 (capped by maxConcurrency)", got)+ }+}+// seedBurstTasks creates one tenant/kb plus N (document, ingestion_task) pairs// in CREATED status, mirroring how CreateAndEnqueue publishes a message for a// freshly-created task. Returns the ingestion task IDs.diff --git a/internal/ingestion/service/ingestion_service.go b/internal/ingestion/service/ingestion_service.goindex cd78abc129..acba81b7e0 100644--- a/internal/ingestion/service/ingestion_service.go+++ b/internal/ingestion/service/ingestion_service.go@@ -43,7 +43,15 @@ import ("github.com/cenkalti/backoff/v5")-const defaultHeartbeatInterval = 10 * time.Second+// defaultHeartbeatInterval paces the InProgress() working pulse that keeps an+// in-flight message's ack deadline renewed while a worker parses. It MUST stay+// comfortably below the consumer's ack deadline, which the server normalizes+// to BackOff[0] = 5s (see NatsEngine.InitConsumer): a pulse slower than the+// deadline lets the broker redeliver mid-run, and the redelivered copy is+// ack-skipped by the claim guard — acking the only in-flight copy, so a worker+// crash afterwards would lose the delivery (downgraded to the 15min+// reconciliation backstop instead of automatic redelivery).+const defaultHeartbeatInterval = 2 * time.Secondtype Ingestor struct {id string@@ -172,6 +180,9 @@ func (e *Ingestor) Start() error {// error is retained and returned to every later caller.func (e *Ingestor) start() error {msgQueueEngine := engine.GetMessageQueueEngine()+ if msgQueueEngine == nil {+ return fmt.Errorf("message queue engine not initialized; run engine.InitMessageQueue first")+ }if err := msgQueueEngine.InitConsumer(common.TaskSubject); err != nil {return err}@@ -187,9 +198,41 @@ func (e *Ingestor) start() error {// Start returns promptly; it is joined by Stop via workerWg.e.workerWg.Add(1)go e.consumeLoop()++ // Startup reconciliation heals tasks orphaned by a previous process+ // crash (RUNNING stuck mid-run, CREATED never delivered). It runs off+ // the caller's goroutine so Start is not blocked by the DB scan; it is+ // joined by Stop via workerWg. It runs concurrently with consumeLoop:+ // the CAS in StartRunning (CREATED→RUNNING) and markFailed+ // (RUNNING→FAILED) handles the race where a stale row is both+ // reconciled and redelivered, and pagination handles >500 orphans+ // without stranding tasks. Keeping it async avoids delaying consumer+ // liveness; a synchronous scan before consumeLoop would add startup+ // latency with no stronger correctness.+ e.workerWg.Add(1)+ go func() {+ defer e.workerWg.Done()+ e.reconcileStartupTasks()+ }()return nil}+// fetchBudget caps each Fetch batch by the worker channel's free capacity (and+// the worker width) so the consumer never pulls more messages than it can+// enqueue immediately: messages beyond the budget would sit claimed in+// taskChan's senders' hands or force blocking sends that stall the consume+// loop while their AckWait runs out.+func (e *Ingestor) fetchBudget() int {+ budget := cap(e.taskChan) - len(e.taskChan)+ if budget > int(e.maxConcurrency) {+ budget = int(e.maxConcurrency)+ }+ if budget < 1 {+ budget = 1+ }+ return budget+}+// consumeLoop is the main tasks.RAGFLOW consume loop. It runs until e.ctx is// cancelled (graceful shutdown); per-message failures are settled by// processMessage and never terminate the consumer. Dataset-level compile work@@ -206,7 +249,7 @@ func (e *Ingestor) consumeLoop() {if err := e.ctx.Err(); err != nil {return}- taskHandles, err := msgQueueEngine.GetMessages(4)+ taskHandles, err := msgQueueEngine.GetMessages(e.fetchBudget())if err != nil {common.Error("error consuming message", err)select {@@ -253,6 +296,84 @@ func (e *Ingestor) SetKnowledgeCompileConcurrency(n int32) {e.kcConcurrency = n}+// Startup reconciliation thresholds: how long a CREATED task must sit+// without being picked up before the startup scan treats it as an orphan+// of a previous process (judged by create_time, never updated after creation).+const (+ reconcileCreatedStaleAfter = 5 * time.Minute+ reconcileBatchLimit = 500+ reconcileOverallTimeout = 5 * time.Minute+)++// reconcileStartupTasks heals CREATED tasks orphaned by a previous process+// crash (never consumed for 5min) by re-publishing via EnqueueByID.+// Republishes are safe without broker-side dedup: the CREATED→RUNNING+// transition in StartRunning is a CAS, and the in-process claim guard+// ack-skips a second copy while a worker holds the task. (JetStream MsgID+// dedup is intentionally NOT used - see NatsEngine.PublishTask: it would+// swallow the retry republish of a task_id reused across runs, stranding+// the row in CREATED.)+//+// RUNNING orphans are intentionally NOT touched here; recovery relies on+// NATS redelivery (BackOff/InProgress) and the knowledge-compile style+// lease would be the proper fix for long tasks, not a startup 15m→FAILED.+// Errors are logged and skipped per task; the next restart re-runs the scan.+// reconcileStartupTasks scans and heals orphaned rows. It carries no+// WaitGroup bookkeeping of its own: start() wraps it with workerWg Add/Done,+// and tests invoke it synchronously.+func (e *Ingestor) reconcileStartupTasks() {+ // Bounded by the ingestor's own lifetime: Stop cancels e.ctx, which+ // aborts an in-flight scan instead of letting workerWg.Wait() linger.+ ctx, cancel := context.WithTimeout(e.ctx, reconcileOverallTimeout)+ defer cancel()++ if dao.DB == nil {+ common.Warn("startup reconciliation skipped: DB not initialized")+ return+ }++ // CREATED orphans are re-published without a status change, so a+ // drain-style loop (as used for RUNNING) would revisit the same head+ // batch forever. Use offset pagination to walk the stale snapshot.+ // The CREATED→RUNNING CAS in StartRunning and the in-process claim+ // guard make concurrent races with consumeLoop safe: duplicate+ // deliveries are ack-skipped while a worker holds the claim.+ createdThreshold := time.Now().Add(-reconcileCreatedStaleAfter)+ for offset := 0; ; offset += reconcileBatchLimit {+ if ctx.Err() != nil {+ return+ }+ // ListStaleByStatus does not support offset; query with offset+ // directly to paginate the immutable CREATED snapshot.+ var created []*entity.IngestionTask+ q := dao.DB.WithContext(ctx).+ Where("status IN ?", []string{common.CREATED}).+ Where("create_time < ?", createdThreshold.UnixMilli()).+ Order("create_time ASC").+ Offset(offset).Limit(reconcileBatchLimit)+ if err := q.Find(&created).Error; err != nil {+ common.Warn(fmt.Sprintf("startup reconciliation: list CREATED orphans: %v", err))+ return+ }+ if len(created) == 0 {+ break+ }+ for _, task := range created {+ if ctx.Err() != nil {+ return+ }+ if err := e.ingestionTaskSvc.EnqueueByID(task.ID); err != nil {+ common.Warn(fmt.Sprintf("startup reconciliation: re-enqueue CREATED task %s: %v", task.ID, err))+ continue+ }+ common.Info(fmt.Sprintf("startup reconciliation: orphaned CREATED task %s re-enqueued", task.ID))+ }+ if len(created) < reconcileBatchLimit {+ break+ }+ }+}+// startDatasetKnowledgeCompile provisions the dataset-level compile scheduling// store (MySQL knowledge_compile_docs + NATS notify subject), builds the// Consumer, and starts the owned compile-worker pool. The consumer is driven by@@ -445,10 +566,21 @@ func (e *Ingestor) processMessage(handle common.TaskHandle) {claimedTaskID = "" // executeTask owns the release nowcommon.Info(fmt.Sprintf("Task %s queued (channel: %d/%d)", task.ID, len(e.taskChan), cap(e.taskChan)))case <-e.ctx.Done():- // Shutdown won the race: release the claim and return without- // settling so the broker redelivers the message after restart- // rather than blocking the consume loop forever.- common.Info(fmt.Sprintf("Ingestor shutting down; releasing slot for task %s without ack", task.ID))+ // Shutdown won the race after StartRunning already flipped the task+ // to RUNNING: finalize it as STOPPED so the row cannot linger in+ // non-terminal RUNNING with no in-flight worker (the user can+ // retry). If the DB write itself fails, the startup reconciliation+ // scan is the backstop. The message is left unsettled - on+ // redelivery the terminal STOPPED status ack-skips it. Both paths+ // run with a detached 2s timeout so shutdown is not slowed by a+ // stalled DB.+ stopCtx, cancel := context.WithTimeout(context.Background(), 2*time.Second)+ defer cancel()+ if e.markStopped(stopCtx, task.ID) {+ common.Info(fmt.Sprintf("Ingestor shutting down; task %s finalized as STOPPED", task.ID))+ } else {+ common.Warn(fmt.Sprintf("Ingestor shutting down; task %s left RUNNING (startup reconciliation will finalize it)", task.ID))+ }return}}@@ -604,7 +736,13 @@ func (e *Ingestor) executeTask(ctx context.Context, taskCtx *taskpkg.TaskContext// RequestStop to handle RUNNING → STOPPING, then MarkStopped for the final// STOPPING → STOPPED transition. Finally it cleans up the Redis cancel flag// so that a future retry of the same task does not immediately re-cancel.+// The caller's context is detached before touching the DB: markStopped runs+// on failure/cancel/shutdown paths whose context is typically already+// cancelled, and a contaminated context would make the terminal write fail+// exactly when it matters most.func (e *Ingestor) markStopped(ctx context.Context, taskID string) bool {+ ctx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 5*time.Second)+ defer cancel()if _, err := e.ingestionTaskSvc.RequestStop(ctx, taskID); err != nil {common.Error(fmt.Sprintf("markStopped: RequestStop task %s: %v", taskID, err), err)return false@@ -624,7 +762,12 @@ func (e *Ingestor) markStopped(ctx context.Context, taskID string) bool {// markFailed persists FAILED status for the task and reports whether the// terminal status was durably written, so the caller can decide Ack vs Nack.+// The caller's context is detached before touching the DB (see markStopped):+// failure paths usually carry an already-cancelled context, and losing the+// FAILED write would turn a settled task into a redelivery loop.func (e *Ingestor) markFailed(ctx context.Context, taskID string) bool {+ ctx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 5*time.Second)+ defer cancel()if uErr := e.ingestionTaskSvc.MarkFailed(ctx, taskID); uErr != nil {common.Error(fmt.Sprintf("Failed to set task %s to FAILED", taskID), uErr)return false@@ -648,7 +791,13 @@ func (e *Ingestor) runTask(ctx context.Context, task *entity.IngestionTask) booldefault:}- if err := e.ingestionTaskSvc.IncrementRunCount(ctx, task.ID); err != nil {+ // The three DB/Redis lookups below must survive a context cancelled+ // between the pre-check and the call (cancel poll races the pipeline+ // start): detach the caller's ctx with a short timeout so a cancelled+ // run fails through markFailed with a real error, not a context error.+ dbCtx, dbCancel := context.WithTimeout(context.WithoutCancel(ctx), 5*time.Second)+ defer dbCancel()+ if err := e.ingestionTaskSvc.IncrementRunCount(dbCtx, task.ID); err != nil {common.Error(fmt.Sprintf("Failed to increment run count for task %s", task.ID), err)ok := e.markFailed(ctx, task.ID)if ok {@@ -660,7 +809,7 @@ func (e *Ingestor) runTask(ctx context.Context, task *entity.IngestionTask) boolif checkpointExists == nil {checkpointExists = canvas.RedisCheckpointExists}- resumeCheckpoint, checkpointErr := checkpointExists(ctx, task.ID)+ resumeCheckpoint, checkpointErr := checkpointExists(dbCtx, task.ID)if checkpointErr != nil {common.Error(fmt.Sprintf("Failed to check checkpoint for task %s", task.ID), checkpointErr)ok := e.markFailed(ctx, task.ID)@@ -670,7 +819,7 @@ func (e *Ingestor) runTask(ctx context.Context, task *entity.IngestionTask) boolreturn ok}if !resumeCheckpoint {- if err := e.ingestionTaskSvc.ClearComponentProgress(ctx, task.ID); err != nil {+ if err := e.ingestionTaskSvc.ClearComponentProgress(dbCtx, task.ID); err != nil {common.Error(fmt.Sprintf("Failed to clear previous component progress for task %s", task.ID), err)ok := e.markFailed(ctx, task.ID)if ok {diff --git a/internal/ingestion/service/ingestor_reconcile_test.go b/internal/ingestion/service/ingestor_reconcile_test.gonew file mode 100644index 0000000000..bb31adb87f--- /dev/null+++ b/internal/ingestion/service/ingestor_reconcile_test.go@@ -0,0 +1,188 @@+//+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.+//++package service++import (+ "context"+ "errors"+ "strings"+ "sync"+ "testing"+ "time"++ "ragflow/internal/common"+ "ragflow/internal/engine"+ "ragflow/internal/entity"+ taskpkg "ragflow/internal/ingestion/task"+ "ragflow/internal/ingestion/testutil"++ "gorm.io/gorm"+)++// enqueueRecorder captures re-enqueue publishes so the reconciliation scan+// can be asserted without a live broker.+type enqueueRecorder struct {+ mu sync.Mutex+ taskIDs []string+}++func (p *enqueueRecorder) PublishTaskMessage(_ string, msg common.TaskMessage) error {+ p.mu.Lock()+ defer p.mu.Unlock()+ p.taskIDs = append(p.taskIDs, msg.TaskID)+ return nil+}++func (p *enqueueRecorder) published() []string {+ p.mu.Lock()+ defer p.mu.Unlock()+ return append([]string(nil), p.taskIDs...)+}++// seedAgedTask inserts an ingestion task with explicit create/update+// timestamps aged back by the given duration (startup reconciliation judges+// staleness by these columns).+func seedAgedTask(t *testing.T, db *gorm.DB, id, docID, status string, age time.Duration) {+ t.Helper()+ ts := time.Now().Add(-age).UnixMilli()+ if err := db.Create(&entity.IngestionTask{+ ID: id,+ UserID: "u1",+ DocumentID: docID,+ DatasetID: "kb-1",+ Status: status,+ BaseModel: entity.BaseModel{CreateTime: &ts, UpdateTime: &ts},+ }).Error; err != nil {+ t.Fatalf("seed task %s: %v", id, err)+ }+}++// TestStartupReconciliation drives the startup scan: stale CREATED orphans+// are re-published for consumption, stale RUNNING orphans are intentionally+// left untouched (recovery relies on NATS redelivery, not startup FAILED;+// see ingestion_service.go reconcileStartupTasks), and fresh rows are left alone.+func TestStartupReconciliation(t *testing.T) {+ db := testutil.SetupTestDB(t)+ cleanup := testutil.ReplaceDBForTest(t, db)+ defer cleanup()++ seedAgedTask(t, db, "task-run-stale", "doc-run-stale", common.RUNNING, 16*time.Minute)+ seedAgedTask(t, db, "task-run-fresh", "doc-run-fresh", common.RUNNING, 1*time.Minute)+ seedAgedTask(t, db, "task-created-stale", "doc-created-stale", common.CREATED, 6*time.Minute)+ seedAgedTask(t, db, "task-created-fresh", "doc-created-fresh", common.CREATED, 1*time.Minute)+ seedAgedTask(t, db, "task-completed-old", "doc-completed-old", common.COMPLETED, 1*time.Hour)++ ingestor := newUnitIngestor("test", 1, []string{"pdf"})+ recorder := &enqueueRecorder{}+ ingestor.ingestionTaskSvc.SetTaskPublisher(recorder)++ ingestor.reconcileStartupTasks()++ statusOf := func(id string) string {+ t.Helper()+ var task entity.IngestionTask+ if err := db.Where("id = ?", id).First(&task).Error; err != nil {+ t.Fatalf("reload task %s: %v", id, err)+ }+ return task.Status+ }++ // RUNNING orphans are intentionally left untouched (NATS redelivery path).+ if got := statusOf("task-run-stale"); got != common.RUNNING {+ t.Fatalf("stale RUNNING task status = %q, want %q (should be left for NATS redelivery)", got, common.RUNNING)+ }+ // Fresh RUNNING row belongs to a live worker — untouched.+ if got := statusOf("task-run-fresh"); got != common.RUNNING {+ t.Fatalf("fresh RUNNING task status = %q, want %q", got, common.RUNNING)+ }+ // CREATED orphans are only re-published; their status stays CREATED so+ // the normal consume path (StartRunning) owns the transition.+ if got := statusOf("task-created-stale"); got != common.CREATED {+ t.Fatalf("stale CREATED task status = %q, want %q", got, common.CREATED)+ }+ if got := statusOf("task-created-fresh"); got != common.CREATED {+ t.Fatalf("fresh CREATED task status = %q, want %q", got, common.CREATED)+ }++ // Exactly the stale CREATED task was re-enqueued — fresh CREATED rows+ // still have their original message in flight, and terminal rows are+ // never resurrected by the scan.+ published := recorder.published()+ if len(published) != 1 || published[0] != "task-created-stale" {+ t.Fatalf("re-enqueued tasks = %v, want [task-created-stale]", published)+ }+}++// TestStartNilEngine verifies that Start on an ingestor whose message queue+// engine was never initialized returns an error instead of panicking on the+// nil engine (mirror of the nats package uninitialized-engine contract).+func TestStartNilEngine(t *testing.T) {+ previousEngine := engine.GetMessageQueueEngine()+ engine.SetMessageQueueEngine(nil)+ t.Cleanup(func() { engine.SetMessageQueueEngine(previousEngine) })++ ingestor := newUnitIngestor("test-nil-engine", 1, []string{"pdf"})+ defer ingestor.Stop(context.Background())++ err := ingestor.Start()+ if err == nil || !strings.Contains(err.Error(), "not initialized") {+ t.Fatalf("Start() with nil engine: err = %v, want 'not initialized'", err)+ }+ // The failure must happen before the worker pool is provisioned.+ if got := ingestor.activeWorkers.Load(); got != 0 {+ t.Fatalf("activeWorkers after failed Start = %d, want 0", got)+ }+}++// TestExecuteTask_MarkFailedAfterCtxCancelAcks verifies the ctx de-contamination+// of the failure path: the pipeline cancels the task context and then fails with+// a generic error. The FAILED write must still land (detached short timeout) and+// the message must be Acked (terminal), not Nack-requeued against a dead run.+func TestExecuteTask_MarkFailedAfterCtxCancelAcks(t *testing.T) {+ db := testutil.SetupTestDB(t)+ cleanup := testutil.ReplaceDBForTest(t, db)+ defer cleanup()+ _, _, docID, taskID := testutil.SeedTestData(t, db, testutil.WithPipelineID("flow-1"))++ ingestor := newUnitIngestor("test", 1, []string{"pdf"})++ parentCtx, parentCancel := context.WithCancel(context.Background())+ defer parentCancel()+ handle := &fakeTaskHandle{}+ taskCtx := taskpkg.NewTaskContextForScheduling(parentCtx, &entity.IngestionTask{+ ID: taskID, DocumentID: docID, DatasetID: "kb-1", Status: common.RUNNING,+ })+ taskCtx.Handle = handle++ ingestor.runDocumentTask = func(_ context.Context, _ *entity.IngestionTask) error {+ parentCancel() // cancel races the pipeline...+ return errors.New("boom") // ...and a generic failure lands afterwards+ }++ ingestor.executeTask(context.Background(), taskCtx)++ var task entity.IngestionTask+ if err := db.Where("id = ?", taskID).First(&task).Error; err != nil {+ t.Fatalf("reload task: %v", err)+ }+ if task.Status != common.FAILED {+ t.Fatalf("task status = %q, want %q (FAILED write must survive a cancelled ctx)", task.Status, common.FAILED)+ }+ if handle.acks.Load() != 1 || handle.nacks.Load() != 0 {+ t.Fatalf("expected 1 Ack/0 Nack (terminal), got acks=%d nacks=%d", handle.acks.Load(), handle.nacks.Load())+ }+}diff --git a/internal/ingestion/service/process_message_test.go b/internal/ingestion/service/process_message_test.goindex 671db5d30c..3e9441461e 100644--- a/internal/ingestion/service/process_message_test.go+++ b/internal/ingestion/service/process_message_test.go@@ -371,6 +371,73 @@ func TestProcessMessage_ChannelFullBlocksUntilSlot(t *testing.T) {}}+// TestProcessMessage_ShutdownRaceMarksStopped: shutdown winning the race+// against the taskChan send must not leave the task in non-terminal RUNNING+// with no in-flight worker. StartRunning has already flipped the task to+// RUNNING, so the ctx.Done branch finalizes it as STOPPED (user-retryable)+// with a detached timeout, and leaves the message unsettled — the broker+// redelivers it and the terminal status ack-skips.+func TestProcessMessage_ShutdownRaceMarksStopped(t *testing.T) {+ db := testutil.SetupTestDB(t)+ cleanup := testutil.ReplaceDBForTest(t, db)+ defer cleanup()+ _, _, _, taskID := testutil.SeedTestData(t, db, testutil.WithPipelineID("flow-1"))+ // SeedTestData creates the task RUNNING; reset to CREATED so the race+ // below is the real one: processMessage's StartRunning flips it RUNNING+ // and then blocks on the full channel.+ if err := db.Model(&entity.IngestionTask{}).Where("id = ?", taskID).+ Update("status", common.CREATED).Error; err != nil {+ t.Fatalf("reset task to CREATED: %v", err)+ }++ ingestor := newUnitIngestor("test", 1, []string{"pdf"}) // channel cap = 2+ for i := 0; i < cap(ingestor.taskChan); i++ {+ ingestor.taskChan <- taskpkg.NewTaskContextForScheduling(nil, &entity.IngestionTask{ID: "filler"})+ }++ handle := newFakeHandle(taskID, common.TaskTypeIngestionTask)+ done := make(chan struct{})+ go func() {+ ingestor.processMessage(handle)+ close(done)+ }()++ // Wait until StartRunning flipped the task RUNNING and processMessage+ // is blocked on the full channel.+ deadline := time.Now().Add(2 * time.Second)+ for {+ var task entity.IngestionTask+ if err := db.Where("id = ?", taskID).First(&task).Error; err == nil && task.Status == common.RUNNING {+ break+ }+ if time.Now().After(deadline) {+ t.Fatal("task never reached RUNNING; processMessage did not reach the blocking send")+ }+ time.Sleep(2 * time.Millisecond)+ }++ // Shutdown wins the race against the channel send.+ ingestor.cancel()++ select {+ case <-done:+ case <-time.After(5 * time.Second):+ t.Fatal("processMessage did not return after shutdown")+ }++ var task entity.IngestionTask+ if err := db.Where("id = ?", taskID).First(&task).Error; err != nil {+ t.Fatalf("reload task: %v", err)+ }+ if task.Status != common.STOPPED {+ t.Fatalf("task status = %q, want %q (shutdown race must not strand a RUNNING row)", task.Status, common.STOPPED)+ }+ // Message unsettled: on redelivery the STOPPED status ack-skips it.+ if handle.acks.Load() != 0 || handle.nacks.Load() != 0 {+ t.Fatalf("expected 0 Ack/0 Nack (unsettled for redelivery), got acks=%d nacks=%d", handle.acks.Load(), handle.nacks.Load())+ }+}+// TestProcessMessage_StartRunningErrorNacks: when StartRunning returns a// non-ErrTaskNotFound error (e.g. a DB blip), processMessage nacks the// message for redelivery instead of killing the consumer (B2 fix). Thediff --git a/internal/service/ingestion_task_service.go b/internal/service/ingestion_task_service.goindex 036791133a..c037d59ed6 100644--- a/internal/service/ingestion_task_service.go+++ b/internal/service/ingestion_task_service.go@@ -458,6 +458,15 @@ func (s *IngestionTaskService) enqueueTask(taskID string) error {return s.taskPublisher.PublishTaskMessage("tasks.RAGFLOW", taskMessage)}+// EnqueueByID re-publishes the wake-up message for an existing task without+// touching its status. Used by ingestor startup reconciliation to redeliver+// CREATED orphans; duplicate delivery is made safe at the consumer level+// (StartRunning CAS + in-process claim guard), not by broker dedup — see+// NatsEngine.PublishTask for why publish-time MsgID dedup is harmful here.+func (s *IngestionTaskService) EnqueueByID(taskID string) error {+ return s.enqueueTask(taskID)+}+// UpdateComponentTotal records the number of components in the task's DSL// graph - the authoritative denominator for progress percentage.func (s *IngestionTaskService) UpdateComponentTotal(ctx context.Context, taskID string, total int) error {