Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

fix(ingestion): make NATS task scheduling crash-safe (#19074)

jay77721686663845eff
diff --git a/internal/common/constants.go b/internal/common/constants.go
index ed0e45ef94..96a301ba39 100644
--- a/internal/common/constants.go
+++ b/internal/common/constants.go
@@ -31,6 +31,7 @@ const (
// task status
const (
CREATED = "CREATED"
+ SCHEDULED = "SCHEDULED"
RUNNING = "RUNNING"
COMPLETED = "COMPLETED"
FAILED = "FAILED"
diff --git a/internal/dao/ingestion.go b/internal/dao/ingestion.go
index dae1a10659..46864a315a 100644
--- a/internal/dao/ingestion.go
+++ b/internal/dao/ingestion.go
@@ -23,7 +23,6 @@ import (
"ragflow/internal/common"
"ragflow/internal/entity"
"ragflow/internal/utility"
- "time"
"gorm.io/gorm"
)
@@ -121,15 +120,20 @@ func (dao *IngestionTaskDAO) Delete(ctx context.Context, db *gorm.DB, taskID str
taskStatus := tasks[0].Status
switch taskStatus {
- case common.CREATED, common.STOPPED, common.COMPLETED, common.FAILED:
+ case common.CREATED, common.SCHEDULED, common.STOPPED, common.COMPLETED, common.FAILED:
// ingestion_task_log no longer carries file references (the old
// checkpoint JSON column was dropped in favor of typed columns), so
// there are no task-level files to delete here.
var filesToDelete []string
- err = tx.Model(&entity.IngestionTask{}).Where("id = ?", taskID).Delete(&entity.IngestionTask{}).Error
- if err != nil {
- return nil, err
+ result := tx.Model(&entity.IngestionTask{}).
+ Where("id = ? AND status IN ?", taskID, []string{common.CREATED, common.SCHEDULED, common.STOPPED, common.COMPLETED, common.FAILED}).
+ Delete(&entity.IngestionTask{})
+ if result.Error != nil {
+ return nil, result.Error
+ }
+ if result.RowsAffected != 1 {
+ return nil, fmt.Errorf("task %s status changed, cannot be removed", taskID)
}
taskInfo := &TaskInfo{
@@ -145,22 +149,24 @@ func (dao *IngestionTaskDAO) Delete(ctx context.Context, db *gorm.DB, taskID str
func (dao *IngestionTaskDAO) GetAllTasks(ctx context.Context, db *gorm.DB, page, pageSize int) ([]*entity.IngestionTask, error) {
var tasks []*entity.IngestionTask
+ query := db.WithContext(ctx)
var err error
if pageSize == 0 {
- err = db.WithContext(ctx).Find(&tasks).Error
+ err = query.Find(&tasks).Error
} else {
- err = db.WithContext(ctx).Order("create_time DESC").Offset((page - 1) * pageSize).Limit(pageSize).Find(&tasks).Error
+ err = query.Order("create_time DESC").Offset((page - 1) * pageSize).Limit(pageSize).Find(&tasks).Error
}
return tasks, err
}
func (dao *IngestionTaskDAO) ListByUserID(ctx context.Context, db *gorm.DB, userID string, page, pageSize int) ([]*entity.IngestionTask, error) {
var tasks []*entity.IngestionTask
+ query := db.WithContext(ctx).Where("user_id = ?", userID)
var err error
if pageSize == 0 {
- err = db.WithContext(ctx).Where("user_id = ?", userID).Order("create_time DESC").Find(&tasks).Error
+ err = query.Order("create_time DESC").Find(&tasks).Error
} else {
- err = db.WithContext(ctx).Where("user_id = ?", userID).Order("create_time DESC").Offset((page - 1) * pageSize).Limit(pageSize).Find(&tasks).Error
+ err = query.Order("create_time DESC").Offset((page - 1) * pageSize).Limit(pageSize).Find(&tasks).Error
}
return tasks, err
@@ -168,16 +174,23 @@ func (dao *IngestionTaskDAO) ListByUserID(ctx context.Context, db *gorm.DB, user
func (dao *IngestionTaskDAO) ListByUserIDAndDatasetID(ctx context.Context, db *gorm.DB, userID, datasetID string, page, pageSize int) ([]*entity.IngestionTask, error) {
var tasks []*entity.IngestionTask
+ query := db.WithContext(ctx).Where("user_id = ? AND dataset_id = ?", userID, datasetID)
var err error
if pageSize == 0 {
- err = db.WithContext(ctx).Where("user_id = ? AND dataset_id = ?", userID, datasetID).Order("create_time DESC").Find(&tasks).Error
+ err = query.Order("create_time DESC").Find(&tasks).Error
} else {
- err = db.WithContext(ctx).Where("user_id = ? AND dataset_id = ?", userID, datasetID).Order("create_time DESC").Offset((page - 1) * pageSize).Limit(pageSize).Find(&tasks).Error
+ err = query.Order("create_time DESC").Offset((page - 1) * pageSize).Limit(pageSize).Find(&tasks).Error
}
return tasks, err
}
+func (dao *IngestionTaskDAO) ListByStatus(ctx context.Context, db *gorm.DB, status string) ([]*entity.IngestionTask, error) {
+ var tasks []*entity.IngestionTask
+ err := db.WithContext(ctx).Where("status = ?", status).Order("create_time ASC").Find(&tasks).Error
+ return tasks, err
+}
+
func (dao *IngestionTaskDAO) GetByID(ctx context.Context, db *gorm.DB, id string) (*entity.IngestionTask, error) {
var task *entity.IngestionTask
err := db.WithContext(ctx).Where("id = ?", id).First(&task).Error
@@ -196,35 +209,9 @@ func (dao *IngestionTaskDAO) GetByDocumentID(ctx context.Context, db *gorm.DB, d
return tasks[0], nil
}
-// ListStaleByStatus returns tasks in the given statuses whose timestamp column
-// is older than the threshold, for ingestor startup reconciliation. timeColumn
-// selects which staleness clock applies: "update_time" for RUNNING orphans
-// (frozen at the moment the previous process claimed them) or "create_time"
-// for CREATED orphans (never picked up). Results are ordered oldest first and
-// capped at limit when limit > 0.
-func (dao *IngestionTaskDAO) ListStaleByStatus(ctx context.Context, db *gorm.DB, statuses []string, timeColumn string, olderThan time.Time, limit int) ([]*entity.IngestionTask, error) {
- switch timeColumn {
- case "create_time", "update_time":
- default:
- return nil, fmt.Errorf("unsupported staleness column %q", timeColumn)
- }
- if len(statuses) == 0 {
- return []*entity.IngestionTask{}, nil
- }
- query := db.WithContext(ctx).
- Where("status IN ?", statuses).
- Where(timeColumn+" < ?", olderThan.UnixMilli()).
- Order(timeColumn + " ASC")
- if limit > 0 {
- query = query.Limit(limit)
- }
- var tasks []*entity.IngestionTask
- err := query.Find(&tasks).Error
- return tasks, err
-}
-
// DeleteIfTerminal deletes ingestion tasks for a document that are in a
-// terminal state (COMPLETED, STOPPED, FAILED) or still queued (CREATED).
+// terminal state (COMPLETED, STOPPED, FAILED), or not yet running
+// (CREATED, SCHEDULED).
// RUNNING and STOPPING tasks are NOT deleted because an in-flight worker
// would keep writing chunks and corrupt a new run's results.
// Returns the number of rows deleted.
diff --git a/internal/dao/ingestion_task_test.go b/internal/dao/ingestion_task_test.go
index f047a8a2ba..c0b4280dec 100644
--- a/internal/dao/ingestion_task_test.go
+++ b/internal/dao/ingestion_task_test.go
@@ -3,7 +3,6 @@ package dao
import (
"errors"
"testing"
- "time"
"fmt"
"ragflow/internal/common"
@@ -104,129 +103,169 @@ func TestIngestionTaskDAODocumentIDIsUniqueAtDBLevel(t *testing.T) {
}
}
-func TestIngestionTaskDAOUpdateStatusIfCurrentRejectsMismatchedStatus(t *testing.T) {
+func TestIngestionTaskDAODeleteAllowsScheduledTask(t *testing.T) {
db := setupTaskTestDB(t)
orig := DB
DB = db
t.Cleanup(func() { DB = orig })
task := &entity.IngestionTask{
- ID: "task-1",
+ ID: "task-scheduled",
UserID: "user-1",
- DocumentID: "doc-1",
+ DocumentID: "doc-scheduled",
DatasetID: "kb-1",
- Status: common.STOPPING,
+ Status: common.SCHEDULED,
}
if err := db.Create(task).Error; err != nil {
- t.Fatalf("create task: %v", err)
- }
-
- ctx := t.Context()
- updated, err := NewIngestionTaskDAO().UpdateStatusIfCurrent(ctx, db, "task-1", common.CREATED, common.RUNNING)
- if err != nil {
- t.Fatalf("UpdateStatusIfCurrent failed: %v", err)
- }
- if updated {
- t.Fatal("expected update to be rejected")
+ t.Fatalf("create scheduled task: %v", err)
}
- reloaded, err := NewIngestionTaskDAO().GetByID(ctx, db, "task-1")
+ info, err := NewIngestionTaskDAO().Delete(t.Context(), db, task.ID, nil)
if err != nil {
- t.Fatalf("reload task: %v", err)
+ t.Fatalf("delete scheduled task: %v", err)
}
- if reloaded.Status != common.STOPPING {
- t.Fatalf("status = %q, want %q", reloaded.Status, common.STOPPING)
+ if info == nil || info.TaskID != task.ID {
+ t.Fatalf("unexpected task info: %+v", info)
}
}
-// seedStaleTask inserts an ingestion task whose create/update timestamps are
-// aged back so ListStaleByStatus staleness windows can be asserted.
-func seedStaleTask(t *testing.T, db *gorm.DB, id, status string, age time.Duration) {
- t.Helper()
- ts := time.Now().Add(-age).UnixMilli()
+func TestIngestionTaskDAODeleteDoesNotRemoveTaskClaimedDuringDelete(t *testing.T) {
+ db := setupTaskTestDB(t)
task := &entity.IngestionTask{
- ID: id,
+ ID: "task-scheduled",
UserID: "user-1",
- DocumentID: "doc-" + id,
+ DocumentID: "doc-scheduled",
DatasetID: "kb-1",
- Status: status,
- BaseModel: entity.BaseModel{CreateTime: &ts, UpdateTime: &ts},
+ Status: common.SCHEDULED,
}
if err := db.Create(task).Error; err != nil {
- t.Fatalf("create task %s: %v", id, err)
+ t.Fatalf("create scheduled task: %v", err)
+ }
+
+ const callbackName = "test:claim-ingestion-task-before-delete"
+ if err := db.Callback().Delete().Before("gorm:delete").Register(callbackName, func(tx *gorm.DB) {
+ if tx.Statement.Table != task.TableName() {
+ return
+ }
+ if err := tx.Session(&gorm.Session{NewDB: true}).Model(&entity.IngestionTask{}).Where("id = ?", task.ID).
+ Update("status", common.RUNNING).Error; err != nil {
+ tx.AddError(err)
+ }
+ }); err != nil {
+ t.Fatalf("register delete callback: %v", err)
+ }
+
+ if _, err := NewIngestionTaskDAO().Delete(t.Context(), db, task.ID, nil); err == nil {
+ t.Fatal("expected delete to reject task claimed during delete")
+ }
+
+ reloaded, err := NewIngestionTaskDAO().GetByID(t.Context(), db, task.ID)
+ if err != nil {
+ t.Fatalf("reload task after rejected delete: %v", err)
+ }
+ if reloaded.Status != common.SCHEDULED {
+ t.Fatalf("status after rejected delete = %q, want %q", reloaded.Status, common.SCHEDULED)
}
}
-// TestIngestionTaskDAOListStaleByStatus covers the three contract axes of the
-// startup-reconciliation query: status filtering, the staleness time window
-// (per column), and the result limit.
-func TestIngestionTaskDAOListStaleByStatus(t *testing.T) {
+func TestIngestionTaskDAOListByStatus(t *testing.T) {
db := setupTaskTestDB(t)
orig := DB
DB = db
t.Cleanup(func() { DB = orig })
- seedStaleTask(t, db, "run-stale", common.RUNNING, 20*time.Minute)
- seedStaleTask(t, db, "run-fresh", common.RUNNING, 1*time.Minute)
- seedStaleTask(t, db, "created-stale", common.CREATED, 20*time.Minute)
- seedStaleTask(t, db, "created-fresh", common.CREATED, 1*time.Minute)
- seedStaleTask(t, db, "completed-stale", common.COMPLETED, 20*time.Minute)
-
- ctx := t.Context()
- d := NewIngestionTaskDAO()
- threshold := time.Now().Add(-15 * time.Minute)
+ for _, task := range []*entity.IngestionTask{
+ {ID: "task-created", UserID: "user-1", DocumentID: "doc-created", DatasetID: "kb-1", Status: common.CREATED},
+ {ID: "task-scheduled", UserID: "user-1", DocumentID: "doc-scheduled", DatasetID: "kb-1", Status: common.SCHEDULED},
+ } {
+ if err := db.Create(task).Error; err != nil {
+ t.Fatalf("create task %s: %v", task.ID, err)
+ }
+ }
- // Status filter + update_time window: only the stale RUNNING row matches.
- running, err := d.ListStaleByStatus(ctx, db, []string{common.RUNNING}, "update_time", threshold, 0)
+ tasks, err := NewIngestionTaskDAO().ListByStatus(t.Context(), db, common.CREATED)
if err != nil {
- t.Fatalf("ListStaleByStatus(RUNNING): %v", err)
+ t.Fatalf("list CREATED tasks: %v", err)
}
- if len(running) != 1 || running[0].ID != "run-stale" {
- t.Fatalf("RUNNING stale rows = %v, want [run-stale]", idsOfTasks(running))
+ if len(tasks) != 1 || tasks[0].ID != "task-created" {
+ t.Fatalf("CREATED tasks = %+v, want only task-created", tasks)
}
+}
- // CREATED is judged by create_time: fresh CREATED rows stay outside the
- // window even though their status is in the filter.
- created, err := d.ListStaleByStatus(ctx, db, []string{common.CREATED}, "create_time", threshold, 0)
+func TestIngestionTaskDAOListsTasks(t *testing.T) {
+ db := setupTaskTestDB(t)
+ orig := DB
+ DB = db
+ t.Cleanup(func() { DB = orig })
+
+ for _, task := range []*entity.IngestionTask{
+ {ID: "task-created", UserID: "user-1", DocumentID: "doc-created", DatasetID: "kb-1", Status: common.CREATED},
+ {ID: "task-scheduled", UserID: "user-1", DocumentID: "doc-scheduled", DatasetID: "kb-1", Status: common.SCHEDULED},
+ } {
+ if err := db.Create(task).Error; err != nil {
+ t.Fatalf("create task %s: %v", task.ID, err)
+ }
+ }
+
+ d := NewIngestionTaskDAO()
+ tasks, err := d.ListByUserID(t.Context(), db, "user-1", 0, 0)
if err != nil {
- t.Fatalf("ListStaleByStatus(CREATED): %v", err)
+ t.Fatalf("list tasks by user: %v", err)
}
- if len(created) != 1 || created[0].ID != "created-stale" {
- t.Fatalf("CREATED stale rows = %v, want [created-stale]", idsOfTasks(created))
+ if len(tasks) != 2 {
+ t.Fatalf("listed tasks = %+v, want 2 tasks", tasks)
}
- // Rows in unlisted statuses (COMPLETED) are never returned.
- all, err := d.ListStaleByStatus(ctx, db, []string{common.RUNNING, common.CREATED}, "create_time", threshold, 0)
+ tasks, err = d.ListByUserIDAndDatasetID(t.Context(), db, "user-1", "kb-1", 0, 0)
if err != nil {
- t.Fatalf("ListStaleByStatus(RUNNING+CREATED): %v", err)
+ t.Fatalf("list tasks by user and dataset: %v", err)
}
- for _, task := range all {
- if task.ID == "completed-stale" {
- t.Fatal("COMPLETED row must not be returned for RUNNING/CREATED filter")
- }
+ if len(tasks) != 2 {
+ t.Fatalf("listed tasks by dataset = %+v, want 2 tasks", tasks)
}
- // Limit caps the (oldest-first) result set.
- limited, err := d.ListStaleByStatus(ctx, db, []string{common.RUNNING, common.CREATED}, "create_time", threshold, 1)
+ tasks, err = d.GetAllTasks(t.Context(), db, 0, 0)
if err != nil {
- t.Fatalf("ListStaleByStatus(limit): %v", err)
+ t.Fatalf("list all tasks: %v", err)
}
- if len(limited) != 1 {
- t.Fatalf("limited rows = %v, want exactly 1", idsOfTasks(limited))
+ if len(tasks) != 2 {
+ t.Fatalf("all listed tasks = %+v, want 2 tasks", tasks)
}
+}
+
+func TestIngestionTaskDAOUpdateStatusIfCurrentRejectsMismatchedStatus(t *testing.T) {
+ db := setupTaskTestDB(t)
+ orig := DB
+ DB = db
+ t.Cleanup(func() { DB = orig })
- // An unsupported staleness column is rejected, not silently ignored.
- if _, err = d.ListStaleByStatus(ctx, db, []string{common.RUNNING}, "status; --", threshold, 0); err == nil {
- t.Fatal("expected error for unsupported staleness column")
+ task := &entity.IngestionTask{
+ ID: "task-1",
+ UserID: "user-1",
+ DocumentID: "doc-1",
+ DatasetID: "kb-1",
+ Status: common.STOPPING,
+ }
+ if err := db.Create(task).Error; err != nil {
+ t.Fatalf("create task: %v", err)
}
-}
-func idsOfTasks(tasks []*entity.IngestionTask) []string {
- ids := make([]string, 0, len(tasks))
- for _, task := range tasks {
- ids = append(ids, task.ID)
+ ctx := t.Context()
+ updated, err := NewIngestionTaskDAO().UpdateStatusIfCurrent(ctx, db, "task-1", common.CREATED, common.RUNNING)
+ if err != nil {
+ t.Fatalf("UpdateStatusIfCurrent failed: %v", err)
+ }
+ if updated {
+ t.Fatal("expected update to be rejected")
+ }
+
+ reloaded, err := NewIngestionTaskDAO().GetByID(ctx, db, "task-1")
+ if err != nil {
+ t.Fatalf("reload task: %v", err)
+ }
+ if reloaded.Status != common.STOPPING {
+ t.Fatalf("status = %q, want %q", reloaded.Status, common.STOPPING)
}
- return ids
}
func TestIngestionTaskDAODeleteIfTerminal_RemovesOnlyTerminal(t *testing.T) {
@@ -236,7 +275,7 @@ func TestIngestionTaskDAODeleteIfTerminal_RemovesOnlyTerminal(t *testing.T) {
t.Cleanup(func() { DB = orig })
// Create tasks in different statuses, each with a unique docID.
- statuses := []string{common.CREATED, common.RUNNING, common.STOPPING, common.COMPLETED, common.STOPPED, common.FAILED}
+ statuses := []string{common.CREATED, common.SCHEDULED, common.RUNNING, common.STOPPING, common.COMPLETED, common.STOPPED, common.FAILED}
for i, status := range statuses {
docID := fmt.Sprintf("doc-%d", i)
task := &entity.IngestionTask{
@@ -253,7 +292,7 @@ func TestIngestionTaskDAODeleteIfTerminal_RemovesOnlyTerminal(t *testing.T) {
ctx := t.Context()
// DeleteIfTerminal deletes everything except RUNNING and STOPPING.
- // CREATED is safe to delete (no worker has claimed it yet);
+ // CREATED and SCHEDULED are safe to delete (no worker has claimed them yet);
// COMPLETED/STOPPED/FAILED are terminal.
// Call it for every doc and verify the negative cases survived.
for i := 0; i < len(statuses); i++ {
@@ -265,7 +304,7 @@ func TestIngestionTaskDAODeleteIfTerminal_RemovesOnlyTerminal(t *testing.T) {
}
// RUNNING and STOPPING must survive.
- for _, i := range []int{1, 2} {
+ for _, i := range []int{2, 3} {
docID := fmt.Sprintf("doc-%d", i)
task, err := NewIngestionTaskDAO().GetByDocumentID(ctx, db, docID)
if err != nil {
@@ -275,8 +314,8 @@ func TestIngestionTaskDAODeleteIfTerminal_RemovesOnlyTerminal(t *testing.T) {
t.Fatalf("%s task (doc=%d) must not be deleted", statuses[i], i)
}
}
- // CREATED, COMPLETED, STOPPED, FAILED must be gone.
- for _, i := range []int{0, 3, 4, 5} {
+ // CREATED, SCHEDULED, COMPLETED, STOPPED, FAILED must be gone.
+ for _, i := range []int{0, 1, 4, 5, 6} {
docID := fmt.Sprintf("doc-%d", i)
task, err := NewIngestionTaskDAO().GetByDocumentID(ctx, db, docID)
if err != nil {
diff --git a/internal/engine/nats/nats.go b/internal/engine/nats/nats.go
index ae91d9faac..66700a67a2 100644
--- a/internal/engine/nats/nats.go
+++ b/internal/engine/nats/nats.go
@@ -111,14 +111,14 @@ func (n *NatsEngine) PublishTask(subject string, payload []byte) error {
defer cancel()
// Deliberately NO MsgID/dedup here. Ingestion tasks reuse the same
- // task_id across publish attempts (the FAILED/STOPPED→CREATED retry
+ // task_id across publish attempts (the FAILED/STOPPED→CREATED→SCHEDULED retry
// path), and the server's Duplicates window suppresses a republished
// MsgID for its whole lifetime regardless of whether the original
// message was already consumed and acked. A deduped retry publish
- // strands the task in CREATED with no message behind it — unreachable
- // by any consumer and un-reparsable ("already exists, status: CREATED").
+ // strands the task in SCHEDULED with no message behind it — unreachable
+ // by any consumer and un-reparsable ("already exists, status: SCHEDULED").
// Duplicate delivery is instead made safe at the consumer level:
- // StartRunning's CREATED→RUNNING CAS plus the in-process claim guard
+ // StartRunning's CREATED/SCHEDULED→RUNNING CAS plus the in-process claim guard
// ack-skip any second copy (see Ingestor.processMessage).
ack, err := n.jetStream.Publish(ctx, subject, payload)
if err != nil {
diff --git a/internal/ingestion/service/ingestion_service.go b/internal/ingestion/service/ingestion_service.go
index acba81b7e0..d82def8cfd 100644
--- a/internal/ingestion/service/ingestion_service.go
+++ b/internal/ingestion/service/ingestion_service.go
@@ -48,9 +48,8 @@ import (
// comfortably below the consumer's ack deadline, which the server normalizes
// to BackOff[0] = 5s (see NatsEngine.InitConsumer): a pulse slower than the
// deadline lets the broker redeliver mid-run, and the redelivered copy is
-// ack-skipped by the claim guard — acking the only in-flight copy, so a worker
-// crash afterwards would lose the delivery (downgraded to the 15min
-// reconciliation backstop instead of automatic redelivery).
+// ack-skipped by the claim guard. If the worker or process stops, the broker
+// can redeliver the unsettled message after the ack deadline.
const defaultHeartbeatInterval = 2 * time.Second
type Ingestor struct {
@@ -186,6 +185,11 @@ func (e *Ingestor) start() error {
if err := msgQueueEngine.InitConsumer(common.TaskSubject); err != nil {
return err
}
+ if dao.DB != nil {
+ if err := e.ingestionTaskSvc.ScheduleCreatedTasks(e.ctx); err != nil {
+ common.Warn(fmt.Sprintf("schedule CREATED ingestion tasks at startup: %v", err))
+ }
+ }
// Start the task worker pool and the dataset-level compile consumer as
// owned goroutines joined by Stop via workerWg/compileWg. Start follows
@@ -199,21 +203,6 @@ func (e *Ingestor) start() error {
e.workerWg.Add(1)
go e.consumeLoop()
- // Startup reconciliation heals tasks orphaned by a previous process
- // crash (RUNNING stuck mid-run, CREATED never delivered). It runs off
- // the caller's goroutine so Start is not blocked by the DB scan; it is
- // joined by Stop via workerWg. It runs concurrently with consumeLoop:
- // the CAS in StartRunning (CREATED→RUNNING) and markFailed
- // (RUNNING→FAILED) handles the race where a stale row is both
- // reconciled and redelivered, and pagination handles >500 orphans
- // without stranding tasks. Keeping it async avoids delaying consumer
- // liveness; a synchronous scan before consumeLoop would add startup
- // latency with no stronger correctness.
- e.workerWg.Add(1)
- go func() {
- defer e.workerWg.Done()
- e.reconcileStartupTasks()
- }()
return nil
}
@@ -296,84 +285,6 @@ func (e *Ingestor) SetKnowledgeCompileConcurrency(n int32) {
e.kcConcurrency = n
}
-// Startup reconciliation thresholds: how long a CREATED task must sit
-// without being picked up before the startup scan treats it as an orphan
-// of a previous process (judged by create_time, never updated after creation).
-const (
- reconcileCreatedStaleAfter = 5 * time.Minute
- reconcileBatchLimit = 500
- reconcileOverallTimeout = 5 * time.Minute
-)
-
-// reconcileStartupTasks heals CREATED tasks orphaned by a previous process
-// crash (never consumed for 5min) by re-publishing via EnqueueByID.
-// Republishes are safe without broker-side dedup: the CREATED→RUNNING
-// transition in StartRunning is a CAS, and the in-process claim guard
-// ack-skips a second copy while a worker holds the task. (JetStream MsgID
-// dedup is intentionally NOT used - see NatsEngine.PublishTask: it would
-// swallow the retry republish of a task_id reused across runs, stranding
-// the row in CREATED.)
-//
-// RUNNING orphans are intentionally NOT touched here; recovery relies on
-// NATS redelivery (BackOff/InProgress) and the knowledge-compile style
-// lease would be the proper fix for long tasks, not a startup 15m→FAILED.
-// Errors are logged and skipped per task; the next restart re-runs the scan.
-// reconcileStartupTasks scans and heals orphaned rows. It carries no
-// WaitGroup bookkeeping of its own: start() wraps it with workerWg Add/Done,
-// and tests invoke it synchronously.
-func (e *Ingestor) reconcileStartupTasks() {
- // Bounded by the ingestor's own lifetime: Stop cancels e.ctx, which
- // aborts an in-flight scan instead of letting workerWg.Wait() linger.
- ctx, cancel := context.WithTimeout(e.ctx, reconcileOverallTimeout)
- defer cancel()
-
- if dao.DB == nil {
- common.Warn("startup reconciliation skipped: DB not initialized")
- return
- }
-
- // CREATED orphans are re-published without a status change, so a
- // drain-style loop (as used for RUNNING) would revisit the same head
- // batch forever. Use offset pagination to walk the stale snapshot.
- // The CREATED→RUNNING CAS in StartRunning and the in-process claim
- // guard make concurrent races with consumeLoop safe: duplicate
- // deliveries are ack-skipped while a worker holds the claim.
- createdThreshold := time.Now().Add(-reconcileCreatedStaleAfter)
- for offset := 0; ; offset += reconcileBatchLimit {
- if ctx.Err() != nil {
- return
- }
- // ListStaleByStatus does not support offset; query with offset
- // directly to paginate the immutable CREATED snapshot.
- var created []*entity.IngestionTask
- q := dao.DB.WithContext(ctx).
- Where("status IN ?", []string{common.CREATED}).
- Where("create_time < ?", createdThreshold.UnixMilli()).
- Order("create_time ASC").
- Offset(offset).Limit(reconcileBatchLimit)
- if err := q.Find(&created).Error; err != nil {
- common.Warn(fmt.Sprintf("startup reconciliation: list CREATED orphans: %v", err))
- return
- }
- if len(created) == 0 {
- break
- }
- for _, task := range created {
- if ctx.Err() != nil {
- return
- }
- if err := e.ingestionTaskSvc.EnqueueByID(task.ID); err != nil {
- common.Warn(fmt.Sprintf("startup reconciliation: re-enqueue CREATED task %s: %v", task.ID, err))
- continue
- }
- common.Info(fmt.Sprintf("startup reconciliation: orphaned CREATED task %s re-enqueued", task.ID))
- }
- if len(created) < reconcileBatchLimit {
- break
- }
- }
-}
-
// startDatasetKnowledgeCompile provisions the dataset-level compile scheduling
// store (MySQL knowledge_compile_docs + NATS notify subject), builds the
// Consumer, and starts the owned compile-worker pool. The consumer is driven by
@@ -569,9 +480,8 @@ func (e *Ingestor) processMessage(handle common.TaskHandle) {
// Shutdown won the race after StartRunning already flipped the task
// to RUNNING: finalize it as STOPPED so the row cannot linger in
// non-terminal RUNNING with no in-flight worker (the user can
- // retry). If the DB write itself fails, the startup reconciliation
- // scan is the backstop. The message is left unsettled - on
- // redelivery the terminal STOPPED status ack-skips it. Both paths
+ // retry). The message is left unsettled - on redelivery the terminal
+ // STOPPED status ack-skips it. Both paths
// run with a detached 2s timeout so shutdown is not slowed by a
// stalled DB.
stopCtx, cancel := context.WithTimeout(context.Background(), 2*time.Second)
@@ -579,7 +489,7 @@ func (e *Ingestor) processMessage(handle common.TaskHandle) {
if e.markStopped(stopCtx, task.ID) {
common.Info(fmt.Sprintf("Ingestor shutting down; task %s finalized as STOPPED", task.ID))
} else {
- common.Warn(fmt.Sprintf("Ingestor shutting down; task %s left RUNNING (startup reconciliation will finalize it)", task.ID))
+ common.Warn(fmt.Sprintf("Ingestor shutting down; task %s left RUNNING for NATS redelivery", task.ID))
}
return
}
diff --git a/internal/ingestion/service/ingestor_lifecycle_test.go b/internal/ingestion/service/ingestor_lifecycle_test.go
index 01b65c1f3b..1594cae941 100644
--- a/internal/ingestion/service/ingestor_lifecycle_test.go
+++ b/internal/ingestion/service/ingestor_lifecycle_test.go
@@ -18,6 +18,8 @@ package service
import (
"context"
+ "errors"
+ "strings"
"testing"
"time"
@@ -28,6 +30,15 @@ import (
"ragflow/internal/ingestion/testutil"
)
+type startupTaskPublisher struct {
+ messages []common.TaskMessage
+}
+
+func (p *startupTaskPublisher) PublishTaskMessage(_ string, msg common.TaskMessage) error {
+ p.messages = append(p.messages, msg)
+ return nil
+}
+
// TestStartWorkerPool_StartOnceIdempotent verifies that calling startWorkerPool
// twice only starts maxConcurrency workers (sync.Once gate). It observes the
// active worker count directly: a broken sync.Once would double the worker
@@ -202,6 +213,61 @@ func TestPollCancel_ExitsWhenDoneClosed(t *testing.T) {
close(released) // cleanup
}
+// TestStartNilEngine verifies that Start returns an error instead of panicking
+// when the message queue engine has not been initialized.
+func TestStartNilEngine(t *testing.T) {
+ previousEngine := engine.GetMessageQueueEngine()
+ engine.SetMessageQueueEngine(nil)
+ t.Cleanup(func() { engine.SetMessageQueueEngine(previousEngine) })
+
+ ingestor := newUnitIngestor("test-nil-engine", 1, []string{"pdf"})
+ defer ingestor.Stop(context.Background())
+
+ err := ingestor.Start()
+ if err == nil || !strings.Contains(err.Error(), "not initialized") {
+ t.Fatalf("Start() with nil engine: err = %v, want 'not initialized'", err)
+ }
+ if got := ingestor.activeWorkers.Load(); got != 0 {
+ t.Fatalf("activeWorkers after failed Start = %d, want 0", got)
+ }
+}
+
+// TestExecuteTask_MarkFailedAfterCtxCancelAcks verifies that a generic task
+// failure is persisted and acknowledged even after the task context is
+// cancelled by the pipeline.
+func TestExecuteTask_MarkFailedAfterCtxCancelAcks(t *testing.T) {
+ db := testutil.SetupTestDB(t)
+ cleanup := testutil.ReplaceDBForTest(t, db)
+ defer cleanup()
+ _, _, docID, taskID := testutil.SeedTestData(t, db, testutil.WithPipelineID("flow-1"))
+
+ ingestor := newUnitIngestor("test", 1, []string{"pdf"})
+ parentCtx, parentCancel := context.WithCancel(context.Background())
+ defer parentCancel()
+ handle := &fakeTaskHandle{}
+ taskCtx := taskpkg.NewTaskContextForScheduling(parentCtx, &entity.IngestionTask{
+ ID: taskID, DocumentID: docID, DatasetID: "kb-1", Status: common.RUNNING,
+ })
+ taskCtx.Handle = handle
+ ingestor.runDocumentTask = func(_ context.Context, _ *entity.IngestionTask) error {
+ parentCancel()
+ return errors.New("boom")
+ }
+
+ ingestor.executeTask(context.Background(), taskCtx)
+
+ var task entity.IngestionTask
+ if err := db.Where("id = ?", taskID).First(&task).Error; err != nil {
+ t.Fatalf("reload task: %v", err)
+ }
+ if task.Status != common.FAILED {
+ t.Fatalf("task status = %q, want %q", task.Status, common.FAILED)
+ }
+ if handle.acks.Load() != 1 || handle.nacks.Load() != 0 {
+ t.Fatalf("expected 1 Ack/0 Nack, got acks=%d nacks=%d", handle.acks.Load(), handle.nacks.Load())
+ }
+}
+
// TestStart_FullPathReturnsAndStartsWorkers is a regression test for the
// sync.Once re-entrancy deadlock. Before the fix, Start() wrapped the whole
// startup (start()) in e.startOnce.Do, but start() also called startWorkerPool()
@@ -247,3 +313,42 @@ func TestStart_FullPathReturnsAndStartsWorkers(t *testing.T) {
t.Fatalf("expected activeWorkers > 0 after Start(), got %d", got)
}
}
+
+func TestStartSchedulesCreatedTasks(t *testing.T) {
+ db := testutil.SetupTestDB(t)
+ cleanup := testutil.ReplaceDBForTest(t, db)
+ defer cleanup()
+
+ _, _, _, taskID := testutil.SeedTestData(t, db)
+ if err := db.Model(&entity.IngestionTask{}).Where("id = ?", taskID).
+ Update("status", common.CREATED).Error; err != nil {
+ t.Fatalf("set task CREATED: %v", err)
+ }
+
+ previousEngine := engine.GetMessageQueueEngine()
+ engine.SetMessageQueueEngine(testutil.SetupNatsEngine(t))
+ t.Cleanup(func() { engine.SetMessageQueueEngine(previousEngine) })
+
+ ingestor := newUnitIngestor("test-schedule-created", 1, nil)
+ publisher := &startupTaskPublisher{}
+ ingestor.ingestionTaskSvc.SetTaskPublisher(publisher)
+ t.Cleanup(func() { ingestor.Stop(context.Background()) })
+
+ if err := ingestor.Start(); err != nil {
+ t.Fatalf("Start() returned error: %v", err)
+ }
+ if len(publisher.messages) != 1 {
+ t.Fatalf("published messages = %d, want 1", len(publisher.messages))
+ }
+ if publisher.messages[0].TaskID != taskID {
+ t.Fatalf("published task ID = %q, want %q", publisher.messages[0].TaskID, taskID)
+ }
+
+ var task entity.IngestionTask
+ if err := db.Where("id = ?", taskID).First(&task).Error; err != nil {
+ t.Fatalf("load scheduled task: %v", err)
+ }
+ if task.Status != common.SCHEDULED {
+ t.Fatalf("task status = %q, want %q", task.Status, common.SCHEDULED)
+ }
+}
diff --git a/internal/ingestion/service/ingestor_reconcile_test.go b/internal/ingestion/service/ingestor_reconcile_test.go
deleted file mode 100644
index bb31adb87f..0000000000
--- a/internal/ingestion/service/ingestor_reconcile_test.go
+++ /dev/null
@@ -1,188 +0,0 @@
-//
-// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-//
-
-package service
-
-import (
- "context"
- "errors"
- "strings"
- "sync"
- "testing"
- "time"
-
- "ragflow/internal/common"
- "ragflow/internal/engine"
- "ragflow/internal/entity"
- taskpkg "ragflow/internal/ingestion/task"
- "ragflow/internal/ingestion/testutil"
-
- "gorm.io/gorm"
-)
-
-// enqueueRecorder captures re-enqueue publishes so the reconciliation scan
-// can be asserted without a live broker.
-type enqueueRecorder struct {
- mu sync.Mutex
- taskIDs []string
-}
-
-func (p *enqueueRecorder) PublishTaskMessage(_ string, msg common.TaskMessage) error {
- p.mu.Lock()
- defer p.mu.Unlock()
- p.taskIDs = append(p.taskIDs, msg.TaskID)
- return nil
-}
-
-func (p *enqueueRecorder) published() []string {
- p.mu.Lock()
- defer p.mu.Unlock()
- return append([]string(nil), p.taskIDs...)
-}
-
-// seedAgedTask inserts an ingestion task with explicit create/update
-// timestamps aged back by the given duration (startup reconciliation judges
-// staleness by these columns).
-func seedAgedTask(t *testing.T, db *gorm.DB, id, docID, status string, age time.Duration) {
- t.Helper()
- ts := time.Now().Add(-age).UnixMilli()
- if err := db.Create(&entity.IngestionTask{
- ID: id,
- UserID: "u1",
- DocumentID: docID,
- DatasetID: "kb-1",
- Status: status,
- BaseModel: entity.BaseModel{CreateTime: &ts, UpdateTime: &ts},
- }).Error; err != nil {
- t.Fatalf("seed task %s: %v", id, err)
- }
-}
-
-// TestStartupReconciliation drives the startup scan: stale CREATED orphans
-// are re-published for consumption, stale RUNNING orphans are intentionally
-// left untouched (recovery relies on NATS redelivery, not startup FAILED;
-// see ingestion_service.go reconcileStartupTasks), and fresh rows are left alone.
-func TestStartupReconciliation(t *testing.T) {
- db := testutil.SetupTestDB(t)
- cleanup := testutil.ReplaceDBForTest(t, db)
- defer cleanup()
-
- seedAgedTask(t, db, "task-run-stale", "doc-run-stale", common.RUNNING, 16*time.Minute)
- seedAgedTask(t, db, "task-run-fresh", "doc-run-fresh", common.RUNNING, 1*time.Minute)
- seedAgedTask(t, db, "task-created-stale", "doc-created-stale", common.CREATED, 6*time.Minute)
- seedAgedTask(t, db, "task-created-fresh", "doc-created-fresh", common.CREATED, 1*time.Minute)
- seedAgedTask(t, db, "task-completed-old", "doc-completed-old", common.COMPLETED, 1*time.Hour)
-
- ingestor := newUnitIngestor("test", 1, []string{"pdf"})
- recorder := &enqueueRecorder{}
- ingestor.ingestionTaskSvc.SetTaskPublisher(recorder)
-
- ingestor.reconcileStartupTasks()
-
- statusOf := func(id string) string {
- t.Helper()
- var task entity.IngestionTask
- if err := db.Where("id = ?", id).First(&task).Error; err != nil {
- t.Fatalf("reload task %s: %v", id, err)
- }
- return task.Status
- }
-
- // RUNNING orphans are intentionally left untouched (NATS redelivery path).
- if got := statusOf("task-run-stale"); got != common.RUNNING {
- t.Fatalf("stale RUNNING task status = %q, want %q (should be left for NATS redelivery)", got, common.RUNNING)
- }
- // Fresh RUNNING row belongs to a live worker — untouched.
- if got := statusOf("task-run-fresh"); got != common.RUNNING {
- t.Fatalf("fresh RUNNING task status = %q, want %q", got, common.RUNNING)
- }
- // CREATED orphans are only re-published; their status stays CREATED so
- // the normal consume path (StartRunning) owns the transition.
- if got := statusOf("task-created-stale"); got != common.CREATED {
- t.Fatalf("stale CREATED task status = %q, want %q", got, common.CREATED)
- }
- if got := statusOf("task-created-fresh"); got != common.CREATED {
- t.Fatalf("fresh CREATED task status = %q, want %q", got, common.CREATED)
- }
-
- // Exactly the stale CREATED task was re-enqueued — fresh CREATED rows
- // still have their original message in flight, and terminal rows are
- // never resurrected by the scan.
- published := recorder.published()
- if len(published) != 1 || published[0] != "task-created-stale" {
- t.Fatalf("re-enqueued tasks = %v, want [task-created-stale]", published)
- }
-}
-
-// TestStartNilEngine verifies that Start on an ingestor whose message queue
-// engine was never initialized returns an error instead of panicking on the
-// nil engine (mirror of the nats package uninitialized-engine contract).
-func TestStartNilEngine(t *testing.T) {
- previousEngine := engine.GetMessageQueueEngine()
- engine.SetMessageQueueEngine(nil)
- t.Cleanup(func() { engine.SetMessageQueueEngine(previousEngine) })
-
- ingestor := newUnitIngestor("test-nil-engine", 1, []string{"pdf"})
- defer ingestor.Stop(context.Background())
-
- err := ingestor.Start()
- if err == nil || !strings.Contains(err.Error(), "not initialized") {
- t.Fatalf("Start() with nil engine: err = %v, want 'not initialized'", err)
- }
- // The failure must happen before the worker pool is provisioned.
- if got := ingestor.activeWorkers.Load(); got != 0 {
- t.Fatalf("activeWorkers after failed Start = %d, want 0", got)
- }
-}
-
-// TestExecuteTask_MarkFailedAfterCtxCancelAcks verifies the ctx de-contamination
-// of the failure path: the pipeline cancels the task context and then fails with
-// a generic error. The FAILED write must still land (detached short timeout) and
-// the message must be Acked (terminal), not Nack-requeued against a dead run.
-func TestExecuteTask_MarkFailedAfterCtxCancelAcks(t *testing.T) {
- db := testutil.SetupTestDB(t)
- cleanup := testutil.ReplaceDBForTest(t, db)
- defer cleanup()
- _, _, docID, taskID := testutil.SeedTestData(t, db, testutil.WithPipelineID("flow-1"))
-
- ingestor := newUnitIngestor("test", 1, []string{"pdf"})
-
- parentCtx, parentCancel := context.WithCancel(context.Background())
- defer parentCancel()
- handle := &fakeTaskHandle{}
- taskCtx := taskpkg.NewTaskContextForScheduling(parentCtx, &entity.IngestionTask{
- ID: taskID, DocumentID: docID, DatasetID: "kb-1", Status: common.RUNNING,
- })
- taskCtx.Handle = handle
-
- ingestor.runDocumentTask = func(_ context.Context, _ *entity.IngestionTask) error {
- parentCancel() // cancel races the pipeline...
- return errors.New("boom") // ...and a generic failure lands afterwards
- }
-
- ingestor.executeTask(context.Background(), taskCtx)
-
- var task entity.IngestionTask
- if err := db.Where("id = ?", taskID).First(&task).Error; err != nil {
- t.Fatalf("reload task: %v", err)
- }
- if task.Status != common.FAILED {
- t.Fatalf("task status = %q, want %q (FAILED write must survive a cancelled ctx)", task.Status, common.FAILED)
- }
- if handle.acks.Load() != 1 || handle.nacks.Load() != 0 {
- t.Fatalf("expected 1 Ack/0 Nack (terminal), got acks=%d nacks=%d", handle.acks.Load(), handle.nacks.Load())
- }
-}
diff --git a/internal/ingestion/service/process_message_test.go b/internal/ingestion/service/process_message_test.go
index 3e9441461e..e80f462c49 100644
--- a/internal/ingestion/service/process_message_test.go
+++ b/internal/ingestion/service/process_message_test.go
@@ -206,6 +206,39 @@ func TestProcessMessage_TaskNotFoundAcks(t *testing.T) {
}
}
+// TestProcessMessage_CreatedTaskStartsRunning verifies that a worker can claim
+// a task after NATS accepts its message but before the API records SCHEDULED.
+func TestProcessMessage_CreatedTaskStartsRunning(t *testing.T) {
+ db := testutil.SetupTestDB(t)
+ cleanup := testutil.ReplaceDBForTest(t, db)
+ defer cleanup()
+ _, _, _, taskID := testutil.SeedTestData(t, db, testutil.WithPipelineID("flow-1"))
+
+ if err := db.Model(&entity.IngestionTask{}).Where("id = ?", taskID).
+ Update("status", common.CREATED).Error; err != nil {
+ t.Fatalf("reset task to CREATED: %v", err)
+ }
+
+ ingestor := newUnitIngestor("test", 1, []string{"pdf"})
+ handle := newFakeHandle(taskID, common.TaskTypeIngestionTask)
+
+ ingestor.processMessage(handle)
+ if handle.acks.Load() != 0 || handle.nacks.Load() != 0 {
+ t.Fatalf("CREATED task: expected 0 Ack/0 Nack (deferred), got acks=%d nacks=%d", handle.acks.Load(), handle.nacks.Load())
+ }
+ if len(ingestor.taskChan) != 1 {
+ t.Fatalf("expected 1 task enqueued, got %d", len(ingestor.taskChan))
+ }
+
+ var task entity.IngestionTask
+ if err := db.Where("id = ?", taskID).First(&task).Error; err != nil {
+ t.Fatalf("reload task: %v", err)
+ }
+ if task.Status != common.RUNNING {
+ t.Fatalf("status = %q, want %q", task.Status, common.RUNNING)
+ }
+}
+
// TestProcessMessage_AlreadyCompletedAcks: a task already in a terminal state
// (COMPLETED) is acked and skipped — no enqueue, and the document is NOT
// resurrected to RUNNING. A redelivered terminal task must not reset a
diff --git a/internal/service/document/document_ingest.go b/internal/service/document/document_ingest.go
index acbdc24a35..5a969bc426 100644
--- a/internal/service/document/document_ingest.go
+++ b/internal/service/document/document_ingest.go
@@ -85,7 +85,8 @@ func (s *DocumentService) Ingest(ctx context.Context, userID string, req *Ingest
// Start parsing: delegates to the shared start-parse flow. The
// document run status is set by service.IngestionTaskService.StartRunning
- // when the task transitions from CREATED, not here.
+ // when the task transitions from CREATED or SCHEDULED,
+ // not here.
if run == string(entity.TaskStatusRunning) {
if err = s.StartParseDocuments(ctx, doc, kb, userID, StartParseOptions{
ApplyKB: req.ApplyKB,
diff --git a/internal/service/document/document_parse.go b/internal/service/document/document_parse.go
index 4c42b1d27b..ce1c7f8da7 100644
--- a/internal/service/document/document_parse.go
+++ b/internal/service/document/document_parse.go
@@ -7,6 +7,7 @@ import (
"ragflow/internal/service"
"strconv"
"strings"
+ "sync"
"ragflow/internal/common"
"ragflow/internal/dao"
@@ -18,12 +19,45 @@ import (
"gorm.io/gorm/clause"
)
+type documentParseLock struct {
+ mu sync.Mutex
+ refs int
+}
+
+var documentParseLocks = struct {
+ sync.Mutex
+ locks map[string]*documentParseLock
+}{locks: make(map[string]*documentParseLock)}
+
+func lockDocumentParse(docID string) func() {
+ documentParseLocks.Lock()
+ lock := documentParseLocks.locks[docID]
+ if lock == nil {
+ lock = &documentParseLock{}
+ documentParseLocks.locks[docID] = lock
+ }
+ lock.refs++
+ documentParseLocks.Unlock()
+
+ lock.mu.Lock()
+ return func() {
+ lock.mu.Unlock()
+ documentParseLocks.Lock()
+ lock.refs--
+ if lock.refs == 0 {
+ delete(documentParseLocks.locks, docID)
+ }
+ documentParseLocks.Unlock()
+ }
+}
+
// StartParseDocuments starts parsing a document via the DSL ingestion
// pipeline. It optionally clears prior results (RerunWithDelete), applies
// KB config (ApplyKB), validates storage, and enqueues an ingestion task.
// The document run status is NOT set here; service.IngestionTaskService.StartRunning
// sets it to RUNNING when the worker picks up the task and transitions it from
-// CREATED. Extracted from Ingest so other entry points (e.g. ChunkService.Parse)
+// CREATED or SCHEDULED. Extracted from Ingest so
+// other entry points (e.g. ChunkService.Parse)
// can reuse the same start-parse flow.
func (s *DocumentService) StartParseDocuments(ctx context.Context, doc *entity.Document, kb *entity.Knowledgebase, userID string, opts StartParseOptions) error {
// Validate storage first so we don't clear prior results and then fail
@@ -32,6 +66,8 @@ func (s *DocumentService) StartParseDocuments(ctx context.Context, doc *entity.D
if _, _, err := s.GetDocumentStorageAddress(ctx, doc); err != nil {
return err
}
+ unlock := lockDocumentParse(doc.ID)
+ defer unlock()
if opts.RerunWithDelete {
if err := s.clearDocumentParseResults(ctx, doc, kb.TenantID); err != nil {
@@ -80,20 +116,31 @@ func (s *DocumentService) clearDocumentParseResults(ctx context.Context, doc *en
// Refuse to clear a non-terminal ingestion task. An in-flight worker
// (RUNNING) or one mid-stop (STOPPING) would keep writing chunks and
// corrupt the new run's results. The caller must stop the task first
- // and wait for a terminal state (COMPLETED/STOPPED/FAILED) or CREATED.
- if task, _ := s.ingestionTaskDAO.GetByDocumentID(ctx, dao.DB, doc.ID); task != nil {
+ // and wait for a terminal state (COMPLETED/STOPPED/FAILED), CREATED, or
+ // SCHEDULED.
+ task, err := s.ingestionTaskDAO.GetByDocumentID(ctx, dao.DB, doc.ID)
+ if err != nil {
+ return fmt.Errorf("get ingestion task for document %s: %w", doc.ID, err)
+ }
+ taskExisted := task != nil
+ if task != nil {
if task.Status == common.RUNNING || task.Status == common.STOPPING {
return fmt.Errorf("document %s ingestion task is %s; stop it and wait for a terminal state before re-parsing", doc.ID, task.Status)
}
+ taskExisted = true
}
- // Delete terminal and CREATED ingestion tasks atomically, leaving
+ // Delete terminal, CREATED, and SCHEDULED ingestion tasks atomically, leaving
// RUNNING/STOPPING tasks untouched so the check-then-delete window
- // between GetByDocumentID and the delete above cannot delete a task
- // that just transitioned to RUNNING.
- if _, err := s.ingestionTaskDAO.DeleteIfTerminal(ctx, dao.DB, doc.ID); err != nil {
+ // between GetByDocumentID and the delete cannot delete a task that just
+ // transitioned to RUNNING. In that case, do not clear its parse results.
+ deleted, err := s.ingestionTaskDAO.DeleteIfTerminal(ctx, dao.DB, doc.ID)
+ if err != nil {
return err
}
+ if taskExisted && deleted == 0 {
+ return fmt.Errorf("document %s ingestion task started running; stop it before re-parsing", doc.ID)
+ }
if err := s.clearDocumentAndKBCountersForRerun(doc.ID, doc.KbID); err != nil {
return err
@@ -345,7 +392,7 @@ var errParseNotRunning = errors.New("parse task is not in running status")
// RequestStop (STOPPING), then marks the document run status as CANCEL.
// It mirrors the Python cancel precondition: only a document whose run status
// is RUNNING or CANCEL, or one with an in-flight ingestion task
-// (CREATED/RUNNING/STOPPING), can be canceled; otherwise errParseNotRunning
+// (CREATED/SCHEDULED/RUNNING/STOPPING), can be canceled; otherwise errParseNotRunning
// is returned. A missing ingestion task is not an error by itself — cancel is
// then a no-op on the task side, matching Python's cancel_all_task_of.
func (s *DocumentService) CancelDocParse(ctx context.Context, doc *entity.Document) error {
@@ -358,7 +405,7 @@ func (s *DocumentService) CancelDocParse(ctx context.Context, doc *entity.Docume
if doc.Run != nil {
docRun = *doc.Run
}
- inFlight := task != nil && (task.Status == common.CREATED || task.Status == common.RUNNING || task.Status == common.STOPPING)
+ inFlight := task != nil && (task.Status == common.CREATED || task.Status == common.SCHEDULED || task.Status == common.RUNNING || task.Status == common.STOPPING)
if docRun != string(entity.TaskStatusRunning) && docRun != string(entity.TaskStatusCancel) && !inFlight {
return errParseNotRunning
}
diff --git a/internal/service/document/document_test.go b/internal/service/document/document_test.go
index bc161f2b9a..d97a7dd1b6 100644
--- a/internal/service/document/document_test.go
+++ b/internal/service/document/document_test.go
@@ -28,6 +28,7 @@ import (
"path/filepath"
"reflect"
"strings"
+ "sync"
"testing"
"time"
@@ -51,6 +52,18 @@ type recordingTaskPublisher struct {
messages []common.TaskMessage
}
+type runningTaskPublisher struct {
+ db *gorm.DB
+ published chan struct{}
+ once sync.Once
+}
+
+func (p *runningTaskPublisher) PublishTaskMessage(_ string, msg common.TaskMessage) error {
+ p.once.Do(func() { close(p.published) })
+ return p.db.Model(&entity.IngestionTask{}).Where("id = ?", msg.TaskID).
+ Update("status", common.RUNNING).Error
+}
+
func (r *recordingTaskPublisher) PublishTaskMessage(subject string, msg common.TaskMessage) error {
r.subject = subject
r.messages = append(r.messages, msg)
@@ -422,8 +435,19 @@ func (f *failingDeleteMetadataEngine) UpdateMetadata(ctx context.Context, docID
// setupServiceTestDB initializes an in-memory SQLite database for service tests.
func setupServiceTestDB(t *testing.T) *gorm.DB {
t.Helper()
+ return setupServiceTestDBWithDSN(t, ":memory:")
+}
+
+func setupConcurrentServiceTestDB(t *testing.T) *gorm.DB {
+ t.Helper()
+ dsn := "file:" + strings.ReplaceAll(t.Name(), "/", "_") + "?mode=memory&cache=shared"
+ return setupServiceTestDBWithDSN(t, dsn)
+}
+
+func setupServiceTestDBWithDSN(t *testing.T, dsn string) *gorm.DB {
+ t.Helper()
- db, err := gorm.Open(sqlite.Open(":memory:"), &gorm.Config{
+ db, err := gorm.Open(sqlite.Open(dsn), &gorm.Config{
TranslateError: true,
})
if err != nil {
@@ -1266,8 +1290,8 @@ func TestStartParseDocuments_EnqueuesIngestionTask(t *testing.T) {
if ingestionTask.UserID != "user-1" {
t.Fatalf("ingestion task user id = %q, want %q", ingestionTask.UserID, "user-1")
}
- if ingestionTask.Status != common.CREATED {
- t.Fatalf("ingestion task status = %q, want %q", ingestionTask.Status, common.CREATED)
+ if ingestionTask.Status != common.SCHEDULED {
+ t.Fatalf("ingestion task status = %q, want %q", ingestionTask.Status, common.SCHEDULED)
}
tasks, err := svc.taskDAO.GetByDocID(ctx, db, "doc-1")
@@ -1279,6 +1303,76 @@ func TestStartParseDocuments_EnqueuesIngestionTask(t *testing.T) {
}
}
+func TestStartParseDocumentsSerializesConcurrentReruns(t *testing.T) {
+ db := setupConcurrentServiceTestDB(t)
+ pushServiceDB(t, db)
+ insertTestKB(t, "kb-1", "tenant-1", 0, 10, 5)
+ insertTestDocWithRun(t, "doc-1", "kb-1", string(entity.TaskStatusDone), 10, 5)
+ if err := db.Model(&entity.Document{}).Where("id = ?", "doc-1").Update("location", "loc-1").Error; err != nil {
+ t.Fatalf("set document location: %v", err)
+ }
+
+ firstTaskLookup := make(chan struct{})
+ releaseFirstTaskLookup := make(chan struct{})
+ var blockFirstLookup sync.Once
+ const callbackName = "test:block-first-ingestion-task-lookup"
+ if err := db.Callback().Query().After("gorm:query").Register(callbackName, func(tx *gorm.DB) {
+ if tx.Statement.Table != (&entity.IngestionTask{}).TableName() {
+ return
+ }
+ blockFirstLookup.Do(func() {
+ close(firstTaskLookup)
+ <-releaseFirstTaskLookup
+ })
+ }); err != nil {
+ t.Fatalf("register query callback: %v", err)
+ }
+
+ firstService := testDocumentService(t)
+ firstService.ingestionTaskSvc.SetTaskPublisher(&recordingTaskPublisher{})
+ secondPublisher := &runningTaskPublisher{db: db, published: make(chan struct{})}
+ secondService := testDocumentService(t)
+ secondService.ingestionTaskSvc.SetTaskPublisher(secondPublisher)
+ ctx := t.Context()
+ kb, err := firstService.kbDAO.GetByID(ctx, db, "kb-1")
+ if err != nil {
+ t.Fatalf("load knowledgebase: %v", err)
+ }
+ doc, err := firstService.documentDAO.GetByID(ctx, db, "doc-1")
+ if err != nil {
+ t.Fatalf("load document: %v", err)
+ }
+
+ firstDone := make(chan error, 1)
+ go func() {
+ firstDone <- firstService.StartParseDocuments(ctx, doc, kb, "user-1", StartParseOptions{RerunWithDelete: true})
+ }()
+ <-firstTaskLookup
+
+ secondDone := make(chan error, 1)
+ go func() {
+ secondDone <- secondService.StartParseDocuments(ctx, doc, kb, "user-1", StartParseOptions{RerunWithDelete: true})
+ }()
+
+ secondPublishedBeforeFirstCompletes := false
+ select {
+ case <-secondPublisher.published:
+ secondPublishedBeforeFirstCompletes = true
+ case <-time.After(200 * time.Millisecond):
+ }
+
+ close(releaseFirstTaskLookup)
+ if err := <-firstDone; err != nil {
+ t.Fatalf("first concurrent reparse: %v", err)
+ }
+ if err := <-secondDone; err != nil {
+ t.Fatalf("second concurrent reparse: %v", err)
+ }
+ if secondPublishedBeforeFirstCompletes {
+ t.Fatal("second reparse published a task before the first reparse completed cleanup")
+ }
+}
+
func TestStopParseDocuments_Success(t *testing.T) {
db := setupServiceTestDB(t)
pushServiceDB(t, db)
@@ -2060,6 +2154,58 @@ func TestClearDocumentParseResults_RejectsNonTerminalIngestionTask(t *testing.T)
}
}
+func TestClearDocumentParseResultsDoesNotClearResultsWhenTaskStartsRunning(t *testing.T) {
+ db := setupServiceTestDB(t)
+ pushServiceDB(t, db)
+ insertTestKB(t, "kb-1", "tenant-1", 1, 10, 5)
+ insertTestDocWithRun(t, "doc-1", "kb-1", string(entity.TaskStatusDone), 10, 5)
+ insertTestIngestionTaskWithStatus(t, "task-1", "user-1", "doc-1", "kb-1", common.SCHEDULED)
+
+ const callbackName = "test:start-ingestion-task-before-conditional-delete"
+ if err := db.Callback().Delete().Before("gorm:delete").Register(callbackName, func(tx *gorm.DB) {
+ if tx.Statement.Table != (&entity.IngestionTask{}).TableName() {
+ return
+ }
+ if err := tx.Session(&gorm.Session{NewDB: true}).Model(&entity.IngestionTask{}).
+ Where("id = ?", "task-1").Update("status", common.RUNNING).Error; err != nil {
+ _ = tx.AddError(err)
+ }
+ }); err != nil {
+ t.Fatalf("register delete callback: %v", err)
+ }
+
+ ctx := t.Context()
+ doc, err := dao.NewDocumentDAO().GetByID(ctx, db, "doc-1")
+ if err != nil {
+ t.Fatalf("get document: %v", err)
+ }
+ if err = testDocumentService(t).clearDocumentParseResults(ctx, doc, "tenant-1"); err == nil || !strings.Contains(err.Error(), "started running") {
+ t.Fatalf("expected reparse cleanup to reject a task that started running, got %v", err)
+ }
+ task, err := dao.NewIngestionTaskDAO().GetByID(ctx, db, "task-1")
+ if err != nil {
+ t.Fatalf("reload task: %v", err)
+ }
+ if task.Status != common.RUNNING {
+ t.Fatalf("task status = %q, want %q", task.Status, common.RUNNING)
+ }
+
+ updatedDoc, err := dao.NewDocumentDAO().GetByID(ctx, db, "doc-1")
+ if err != nil {
+ t.Fatalf("reload document: %v", err)
+ }
+ if updatedDoc.TokenNum != 10 || updatedDoc.ChunkNum != 5 {
+ t.Fatalf("document counters = token:%d chunk:%d, want unchanged 10/5", updatedDoc.TokenNum, updatedDoc.ChunkNum)
+ }
+ kb, err := dao.NewKnowledgebaseDAO().GetByID(ctx, db, "kb-1")
+ if err != nil {
+ t.Fatalf("reload knowledgebase: %v", err)
+ }
+ if kb.TokenNum != 10 || kb.ChunkNum != 5 {
+ t.Fatalf("knowledgebase counters = token:%d chunk:%d, want unchanged 10/5", kb.TokenNum, kb.ChunkNum)
+ }
+}
+
// TestClearDocumentParseResults_DeletesTerminalIngestionTask verifies
// that a terminal ingestion task (STOPPED/COMPLETED/FAILED) and a CREATED
// (queued) task are deleted so the new run can proceed.
diff --git a/internal/service/ingestion_task_service.go b/internal/service/ingestion_task_service.go
index c037d59ed6..87e7b58017 100644
--- a/internal/service/ingestion_task_service.go
+++ b/internal/service/ingestion_task_service.go
@@ -17,9 +17,7 @@ import (
// Run-count key for IngestionTaskLog.Checkpoint, consumed by
// ListAllForAdmin and IncrementRunCount to track how many times
// the task has been picked up by a worker.
-const (
- stepKeyRunCount = "run_count"
-)
+const stepKeyRunCount = "run_count"
type InvalidTaskTransitionError struct {
TaskID string
@@ -206,7 +204,7 @@ func (s *IngestionTaskService) StartRunning(ctx context.Context, taskID string)
return nil, err
}
switch task.Status {
- case common.CREATED:
+ case common.CREATED, common.SCHEDULED:
task, err = s.transition(ctx, taskID, common.RUNNING)
if err != nil {
return nil, err
@@ -252,7 +250,7 @@ func (s *IngestionTaskService) RequestStop(ctx context.Context, taskID string) (
return nil, err
}
switch task.Status {
- case common.CREATED:
+ case common.CREATED, common.SCHEDULED:
return s.transition(ctx, taskID, common.STOPPED)
case common.RUNNING:
task, err = s.transition(ctx, taskID, common.STOPPING)
@@ -328,6 +326,10 @@ func (s *IngestionTaskService) GetTask(ctx context.Context, taskID string) (*ent
func validateTransition(from, to string) error {
switch from {
case common.CREATED:
+ if to == common.SCHEDULED || to == common.RUNNING || to == common.STOPPED {
+ return nil
+ }
+ case common.SCHEDULED:
if to == common.RUNNING || to == common.STOPPED {
return nil
}
@@ -390,6 +392,11 @@ func (s *IngestionTaskService) CreateAndEnqueue(ctx context.Context, task *entit
}
if existing != nil {
switch existing.Status {
+ case common.CREATED:
+ if err = s.enqueueTask(existing.ID); err != nil {
+ return nil, err
+ }
+ return s.markScheduledAfterPublish(ctx, existing.ID)
case common.FAILED, common.STOPPED:
originalStatus := existing.Status
existing, err = s.transition(ctx, existing.ID, common.CREATED)
@@ -407,11 +414,12 @@ func (s *IngestionTaskService) CreateAndEnqueue(ctx context.Context, task *entit
}
return nil, err
}
- return existing, nil
+ return s.markScheduledAfterPublish(ctx, existing.ID)
default:
return nil, fmt.Errorf("document id %s already exists, status: %s, task id: %s", task.DocumentID, existing.Status, existing.ID)
}
}
+ task.Status = common.CREATED
created, err := s.ingestionTaskDAO.Create(ctx, dao.DB, task)
if err != nil {
return nil, err
@@ -422,7 +430,7 @@ func (s *IngestionTaskService) CreateAndEnqueue(ctx context.Context, task *entit
}
return nil, err
}
- return created, nil
+ return s.markScheduledAfterPublish(ctx, created.ID)
}
func (s *IngestionTaskService) rollbackRetriedTask(ctx context.Context, taskID, status string) error {
@@ -441,6 +449,52 @@ func (s *IngestionTaskService) rollbackCreatedTask(ctx context.Context, taskID s
return err
}
+// markScheduledAfterPublish records a successful NATS publish. A worker can
+// claim the CREATED task before this write and move it to RUNNING, which is
+// also a successful outcome.
+func (s *IngestionTaskService) markScheduledAfterPublish(ctx context.Context, taskID string) (*entity.IngestionTask, error) {
+ updated, err := s.ingestionTaskDAO.UpdateStatusIfCurrent(ctx, dao.DB, taskID, common.CREATED, common.SCHEDULED)
+ if err != nil {
+ return nil, err
+ }
+ if updated {
+ return s.GetTask(ctx, taskID)
+ }
+
+ task, err := s.GetTask(ctx, taskID)
+ if err != nil {
+ return nil, err
+ }
+ switch task.Status {
+ case common.SCHEDULED, common.RUNNING, common.STOPPING, common.COMPLETED, common.FAILED, common.STOPPED:
+ return task, nil
+ default:
+ return nil, s.newTaskStatusConflictError(ctx, taskID, common.CREATED, common.SCHEDULED)
+ }
+}
+
+// ScheduleCreatedTasks publishes the tasks that were persisted before a
+// process stopped but were not confirmed as scheduled. It is intended for the
+// single startup recovery pass; a publish error leaves the task CREATED for a
+// future startup or explicit parse request to retry.
+func (s *IngestionTaskService) ScheduleCreatedTasks(ctx context.Context) error {
+ tasks, err := s.ingestionTaskDAO.ListByStatus(ctx, dao.DB, common.CREATED)
+ if err != nil {
+ return err
+ }
+ var recoveryErr error
+ for _, task := range tasks {
+ if err := s.enqueueTask(task.ID); err != nil {
+ recoveryErr = errors.Join(recoveryErr, fmt.Errorf("schedule created task %s: %w", task.ID, err))
+ continue
+ }
+ if _, err := s.markScheduledAfterPublish(ctx, task.ID); err != nil {
+ recoveryErr = errors.Join(recoveryErr, fmt.Errorf("mark task %s scheduled: %w", task.ID, err))
+ }
+ }
+ return recoveryErr
+}
+
// clearCancelFlag removes the Redis cancel marker ({task_id}-cancel) that
// RequestStop sets for a RUNNING task. No-op when Redis is unavailable —
// the DB STOPPING status remains the fallback cancel signal.
@@ -458,15 +512,6 @@ func (s *IngestionTaskService) enqueueTask(taskID string) error {
return s.taskPublisher.PublishTaskMessage("tasks.RAGFLOW", taskMessage)
}
-// EnqueueByID re-publishes the wake-up message for an existing task without
-// touching its status. Used by ingestor startup reconciliation to redeliver
-// CREATED orphans; duplicate delivery is made safe at the consumer level
-// (StartRunning CAS + in-process claim guard), not by broker dedup — see
-// NatsEngine.PublishTask for why publish-time MsgID dedup is harmful here.
-func (s *IngestionTaskService) EnqueueByID(taskID string) error {
- return s.enqueueTask(taskID)
-}
-
// UpdateComponentTotal records the number of components in the task's DSL
// graph - the authoritative denominator for progress percentage.
func (s *IngestionTaskService) UpdateComponentTotal(ctx context.Context, taskID string, total int) error {
diff --git a/internal/service/ingestion_task_service_test.go b/internal/service/ingestion_task_service_test.go
index e8a05d4ec6..0820046c16 100644
--- a/internal/service/ingestion_task_service_test.go
+++ b/internal/service/ingestion_task_service_test.go
@@ -2,6 +2,8 @@ package service
import (
"errors"
+ "strings"
+ "sync"
"testing"
"ragflow/internal/common"
@@ -10,14 +12,56 @@ import (
)
type recordingTaskPublisher struct {
- subject string
+ subject string
+ messages []common.TaskMessage
+ err error
+ beforeReturn func(taskID string)
+}
+
+type failFirstTaskPublisher struct {
messages []common.TaskMessage
- err error
+}
+
+type duplicatePublishRecorder struct {
+ mu sync.Mutex
+ messages []common.TaskMessage
+ firstPublished chan struct{}
+ secondPublished chan struct{}
+ releaseFirst chan struct{}
+ releaseSecond chan struct{}
+}
+
+func (p *failFirstTaskPublisher) PublishTaskMessage(_ string, msg common.TaskMessage) error {
+ p.messages = append(p.messages, msg)
+ if len(p.messages) == 1 {
+ return errors.New("publish failed")
+ }
+ return nil
+}
+
+func (p *duplicatePublishRecorder) PublishTaskMessage(_ string, msg common.TaskMessage) error {
+ p.mu.Lock()
+ p.messages = append(p.messages, msg)
+ publishNumber := len(p.messages)
+ p.mu.Unlock()
+
+ switch publishNumber {
+ case 1:
+ close(p.firstPublished)
+ <-p.releaseFirst
+ case 2:
+ close(p.secondPublished)
+ <-p.releaseSecond
+ }
+ return nil
}
func (p *recordingTaskPublisher) PublishTaskMessage(subject string, msg common.TaskMessage) error {
p.subject = subject
p.messages = append(p.messages, msg)
+ if p.beforeReturn != nil {
+ p.beforeReturn(msg.TaskID)
+ }
return p.err
}
@@ -58,6 +102,95 @@ func TestIngestionTaskServiceCreateForDocumentsPublishesTaskMessages(t *testing.
}
}
+func TestIngestionTaskServiceMarksTaskScheduledOnlyAfterPublish(t *testing.T) {
+ db := setupServiceTestDB(t)
+ pushServiceDB(t, db)
+ insertTestKB(t, "kb-1", "tenant-1", 1, 0, 0)
+ insertTestDoc(t, "doc-1", "kb-1", 0, 0)
+
+ statusDuringPublish := ""
+ publisher := &recordingTaskPublisher{
+ beforeReturn: func(taskID string) {
+ task, err := dao.NewIngestionTaskDAO().GetByID(t.Context(), db, taskID)
+ if err != nil {
+ t.Fatalf("load task during publish: %v", err)
+ }
+ statusDuringPublish = task.Status
+ },
+ }
+ svc := NewIngestionTaskService()
+ svc.taskPublisher = publisher
+
+ responses, err := svc.CreateForDocuments(t.Context(), "kb-1", "user-1", []string{"doc-1"})
+ if err != nil {
+ t.Fatalf("CreateForDocuments failed: %v", err)
+ }
+ if statusDuringPublish != common.CREATED {
+ t.Fatalf("status during publish = %q, want %q", statusDuringPublish, common.CREATED)
+ }
+ if len(responses) != 1 || !strings.HasPrefix(responses[0].Result, "task_id: ") {
+ t.Fatalf("unexpected parse response: %+v", responses)
+ }
+
+ task, err := dao.NewIngestionTaskDAO().GetByDocumentID(t.Context(), db, "doc-1")
+ if err != nil {
+ t.Fatalf("reload task: %v", err)
+ }
+ if task.Status != common.SCHEDULED {
+ t.Fatalf("status after publish = %q, want %q", task.Status, common.SCHEDULED)
+ }
+}
+
+func TestIngestionTaskServiceAcceptsConsumerWinningPublishRace(t *testing.T) {
+ db := setupServiceTestDB(t)
+ pushServiceDB(t, db)
+ insertTestKB(t, "kb-1", "tenant-1", 1, 0, 0)
+ insertTestDoc(t, "doc-1", "kb-1", 0, 0)
+
+ publisher := &recordingTaskPublisher{
+ beforeReturn: func(taskID string) {
+ if err := db.Model(&entity.IngestionTask{}).Where("id = ?", taskID).
+ Update("status", common.RUNNING).Error; err != nil {
+ t.Fatalf("simulate consumer start: %v", err)
+ }
+ },
+ }
+ svc := NewIngestionTaskService()
+ svc.taskPublisher = publisher
+
+ responses, err := svc.CreateForDocuments(t.Context(), "kb-1", "user-1", []string{"doc-1"})
+ if err != nil {
+ t.Fatalf("CreateForDocuments failed: %v", err)
+ }
+ if len(responses) != 1 || !strings.HasPrefix(responses[0].Result, "task_id: ") {
+ t.Fatalf("unexpected parse response: %+v", responses)
+ }
+
+ task, err := dao.NewIngestionTaskDAO().GetByDocumentID(t.Context(), db, "doc-1")
+ if err != nil {
+ t.Fatalf("reload task: %v", err)
+ }
+ if task.Status != common.RUNNING {
+ t.Fatalf("status after consumer race = %q, want %q", task.Status, common.RUNNING)
+ }
+}
+
+func TestIngestionTaskServiceStartRunningTransitionsScheduledTask(t *testing.T) {
+ db := setupServiceTestDB(t)
+ pushServiceDB(t, db)
+ insertTestKB(t, "kb-1", "tenant-1", 1, 0, 0)
+ insertTestDoc(t, "doc-1", "kb-1", 0, 0)
+ insertTestIngestionTaskWithStatus(t, "task-1", "user-1", "doc-1", "kb-1", common.SCHEDULED)
+
+ task, err := NewIngestionTaskService().StartRunning(t.Context(), "task-1")
+ if err != nil {
+ t.Fatalf("StartRunning failed: %v", err)
+ }
+ if task.Status != common.RUNNING {
+ t.Fatalf("status = %q, want %q", task.Status, common.RUNNING)
+ }
+}
+
func TestIngestionTaskServiceListByUserFiltersDataset(t *testing.T) {
db := setupServiceTestDB(t)
pushServiceDB(t, db)
@@ -499,8 +632,8 @@ func TestIngestionTaskServiceCreateAndEnqueueRetriesTerminalTask(t *testing.T) {
if task.ID != "task-1" {
t.Fatalf("task ID = %q, want task-1", task.ID)
}
- if task.Status != common.CREATED {
- t.Fatalf("status = %q, want %q", task.Status, common.CREATED)
+ if task.Status != common.SCHEDULED {
+ t.Fatalf("status = %q, want %q", task.Status, common.SCHEDULED)
}
if len(publisher.messages) != 1 || publisher.messages[0].TaskID != "task-1" {
t.Fatalf("unexpected published messages: %+v", publisher.messages)
@@ -509,17 +642,178 @@ func TestIngestionTaskServiceCreateAndEnqueueRetriesTerminalTask(t *testing.T) {
if err != nil {
t.Fatalf("reload task: %v", err)
}
- if reloaded.Status != common.CREATED {
- t.Fatalf("reloaded status = %q, want %q", reloaded.Status, common.CREATED)
+ if reloaded.Status != common.SCHEDULED {
+ t.Fatalf("reloaded status = %q, want %q", reloaded.Status, common.SCHEDULED)
}
})
}
}
+func TestIngestionTaskServiceCreateAndEnqueueSchedulesExistingCreatedTask(t *testing.T) {
+ db := setupServiceTestDB(t)
+ pushServiceDB(t, db)
+ insertTestIngestionTaskWithStatus(t, "stale-task", "user-1", "doc-1", "kb-1", common.CREATED)
+
+ publisher := &recordingTaskPublisher{}
+ svc := NewIngestionTaskService()
+ svc.taskPublisher = publisher
+
+ task, err := svc.CreateAndEnqueue(t.Context(), &entity.IngestionTask{
+ DocumentID: "doc-1",
+ UserID: "user-1",
+ DatasetID: "kb-1",
+ Status: common.CREATED,
+ })
+ if err != nil {
+ t.Fatalf("CreateAndEnqueue failed: %v", err)
+ }
+ if task.ID != "stale-task" {
+ t.Fatalf("task ID = %q, want stale-task", task.ID)
+ }
+ if task.Status != common.SCHEDULED {
+ t.Fatalf("status = %q, want %q", task.Status, common.SCHEDULED)
+ }
+ if len(publisher.messages) != 1 || publisher.messages[0].TaskID != task.ID {
+ t.Fatalf("unexpected published messages: %+v", publisher.messages)
+ }
+
+ reloaded, err := dao.NewIngestionTaskDAO().GetByDocumentID(t.Context(), db, "doc-1")
+ if err != nil {
+ t.Fatalf("load task after scheduling created task: %v", err)
+ }
+ if reloaded == nil || reloaded.ID != "stale-task" || reloaded.Status != common.SCHEDULED {
+ t.Fatalf("expected created task to be scheduled in place, task=%+v", reloaded)
+ }
+}
+
+func TestIngestionTaskServiceConcurrentCreatedTaskPublicationsConverge(t *testing.T) {
+ db := setupServiceTestDB(t)
+ pushServiceDB(t, db)
+ insertTestIngestionTaskWithStatus(t, "task-1", "user-1", "doc-1", "kb-1", common.CREATED)
+
+ publisher := &duplicatePublishRecorder{
+ firstPublished: make(chan struct{}),
+ secondPublished: make(chan struct{}),
+ releaseFirst: make(chan struct{}),
+ releaseSecond: make(chan struct{}),
+ }
+ firstService := NewIngestionTaskService()
+ firstService.taskPublisher = publisher
+ secondService := NewIngestionTaskService()
+ secondService.taskPublisher = publisher
+
+ type result struct {
+ task *entity.IngestionTask
+ err error
+ }
+ results := make(chan result, 2)
+ enqueue := func(svc *IngestionTaskService) {
+ task, err := svc.CreateAndEnqueue(t.Context(), &entity.IngestionTask{
+ DocumentID: "doc-1",
+ UserID: "user-1",
+ DatasetID: "kb-1",
+ Status: common.CREATED,
+ })
+ results <- result{task: task, err: err}
+ }
+
+ go enqueue(firstService)
+ <-publisher.firstPublished
+ go enqueue(secondService)
+ <-publisher.secondPublished
+
+ close(publisher.releaseFirst)
+ first := <-results
+ if first.err != nil {
+ t.Fatalf("first CreateAndEnqueue: %v", first.err)
+ }
+ close(publisher.releaseSecond)
+ second := <-results
+ if second.err != nil {
+ t.Fatalf("second CreateAndEnqueue: %v", second.err)
+ }
+ if first.task.ID != "task-1" || second.task.ID != "task-1" {
+ t.Fatalf("concurrent requests returned task IDs %q and %q, want task-1", first.task.ID, second.task.ID)
+ }
+
+ publisher.mu.Lock()
+ messages := append([]common.TaskMessage(nil), publisher.messages...)
+ publisher.mu.Unlock()
+ if len(messages) != 2 || messages[0].TaskID != "task-1" || messages[1].TaskID != "task-1" {
+ t.Fatalf("published messages = %+v, want two messages for task-1", messages)
+ }
+
+ task, err := dao.NewIngestionTaskDAO().GetByID(t.Context(), db, "task-1")
+ if err != nil {
+ t.Fatalf("reload task: %v", err)
+ }
+ if task.Status != common.SCHEDULED {
+ t.Fatalf("task status after concurrent publication = %q, want %q", task.Status, common.SCHEDULED)
+ }
+}
+
+func TestIngestionTaskServiceScheduleCreatedTasksKeepsTaskCreatedOnPublishFailure(t *testing.T) {
+ db := setupServiceTestDB(t)
+ pushServiceDB(t, db)
+ insertTestIngestionTaskWithStatus(t, "task-1", "user-1", "doc-1", "kb-1", common.CREATED)
+
+ svc := NewIngestionTaskService()
+ svc.taskPublisher = &recordingTaskPublisher{err: errors.New("publish failed")}
+ if err := svc.ScheduleCreatedTasks(t.Context()); err == nil {
+ t.Fatal("expected ScheduleCreatedTasks to return publish error")
+ }
+
+ task, err := dao.NewIngestionTaskDAO().GetByID(t.Context(), db, "task-1")
+ if err != nil {
+ t.Fatalf("reload task: %v", err)
+ }
+ if task.Status != common.CREATED {
+ t.Fatalf("status after failed recovery publish = %q, want %q", task.Status, common.CREATED)
+ }
+}
+
+func TestIngestionTaskServiceScheduleCreatedTasksContinuesAfterPublishFailure(t *testing.T) {
+ db := setupServiceTestDB(t)
+ pushServiceDB(t, db)
+ insertTestIngestionTaskWithStatus(t, "task-1", "user-1", "doc-1", "kb-1", common.CREATED)
+ insertTestIngestionTaskWithStatus(t, "task-2", "user-1", "doc-2", "kb-1", common.CREATED)
+ if err := db.Model(&entity.IngestionTask{}).Where("id = ?", "task-1").Update("create_time", 1).Error; err != nil {
+ t.Fatalf("set first task create time: %v", err)
+ }
+ if err := db.Model(&entity.IngestionTask{}).Where("id = ?", "task-2").Update("create_time", 2).Error; err != nil {
+ t.Fatalf("set second task create time: %v", err)
+ }
+
+ publisher := &failFirstTaskPublisher{}
+ svc := NewIngestionTaskService()
+ svc.taskPublisher = publisher
+ if err := svc.ScheduleCreatedTasks(t.Context()); err == nil {
+ t.Fatal("expected ScheduleCreatedTasks to return publish error")
+ }
+ if len(publisher.messages) != 2 {
+ t.Fatalf("published messages = %d, want 2", len(publisher.messages))
+ }
+
+ first, err := dao.NewIngestionTaskDAO().GetByID(t.Context(), db, "task-1")
+ if err != nil {
+ t.Fatalf("reload first task: %v", err)
+ }
+ if first.Status != common.CREATED {
+ t.Fatalf("first task status = %q, want %q", first.Status, common.CREATED)
+ }
+ second, err := dao.NewIngestionTaskDAO().GetByID(t.Context(), db, "task-2")
+ if err != nil {
+ t.Fatalf("reload second task: %v", err)
+ }
+ if second.Status != common.SCHEDULED {
+ t.Fatalf("second task status = %q, want %q", second.Status, common.SCHEDULED)
+ }
+}
+
func TestIngestionTaskServiceCreateAndEnqueueRejectsActiveExistingTask(t *testing.T) {
db := setupServiceTestDB(t)
pushServiceDB(t, db)
- insertTestIngestionTask(t, "task-1", "user-1", "doc-1", "kb-1")
+ insertTestIngestionTaskWithStatus(t, "task-1", "user-1", "doc-1", "kb-1", common.SCHEDULED)
publisher := &recordingTaskPublisher{}
svc := NewIngestionTaskService()
svc.taskPublisher = publisher
diff --git a/internal/service/ingestion_task_transition_test.go b/internal/service/ingestion_task_transition_test.go
index 2b05d0fdeb..43ac780b65 100644
--- a/internal/service/ingestion_task_transition_test.go
+++ b/internal/service/ingestion_task_transition_test.go
@@ -12,8 +12,11 @@ func TestValidateTransitionAllowsExpectedEdges(t *testing.T) {
from string
to string
}{
+ {from: common.CREATED, to: common.SCHEDULED},
{from: common.CREATED, to: common.RUNNING},
{from: common.CREATED, to: common.STOPPED},
+ {from: common.SCHEDULED, to: common.RUNNING},
+ {from: common.SCHEDULED, to: common.STOPPED},
{from: common.RUNNING, to: common.STOPPING},
{from: common.RUNNING, to: common.COMPLETED},
{from: common.RUNNING, to: common.FAILED},