Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

fix(ingestion): make running transition atomic across created and scheduled (#19764)

jay77721b193b5741eb6
diff --git a/internal/dao/ingestion.go b/internal/dao/ingestion.go
index 7cdf9e223e..fd8b1e375e 100644
--- a/internal/dao/ingestion.go
+++ b/internal/dao/ingestion.go
@@ -60,9 +60,13 @@ func (dao *IngestionTaskDAO) Create(ctx context.Context, db *gorm.DB, ingestionT
return ingestionTask, nil
}
-func (dao *IngestionTaskDAO) UpdateStatusIfCurrent(ctx context.Context, db *gorm.DB, taskID, fromStatus, toStatus string) (bool, error) {
+// UpdateStatusIfCurrent updates the task status if its current status matches any of the given fromStatuses.
+func (dao *IngestionTaskDAO) UpdateStatusIfCurrent(ctx context.Context, db *gorm.DB, taskID string, fromStatuses []string, toStatus string) (bool, error) {
+ if len(fromStatuses) == 0 {
+ return false, nil
+ }
result := db.WithContext(ctx).Model(&entity.IngestionTask{}).
- Where("id = ? AND status = ?", taskID, fromStatus).
+ Where("id = ? AND status IN (?)", taskID, fromStatuses).
Update("status", toStatus)
if result.Error != nil {
return false, result.Error
diff --git a/internal/dao/ingestion_task_test.go b/internal/dao/ingestion_task_test.go
index 22ca81aa42..b914182335 100644
--- a/internal/dao/ingestion_task_test.go
+++ b/internal/dao/ingestion_task_test.go
@@ -29,7 +29,7 @@ func TestIngestionTaskDAOUpdateStatusIfCurrentSucceeds(t *testing.T) {
}
ctx := t.Context()
- updated, err := NewIngestionTaskDAO().UpdateStatusIfCurrent(ctx, db, "task-1", common.CREATED, common.RUNNING)
+ updated, err := NewIngestionTaskDAO().UpdateStatusIfCurrent(ctx, db, "task-1", []string{common.CREATED}, common.RUNNING)
if err != nil {
t.Fatalf("UpdateStatusIfCurrent failed: %v", err)
}
@@ -46,6 +46,76 @@ func TestIngestionTaskDAOUpdateStatusIfCurrentSucceeds(t *testing.T) {
}
}
+func TestIngestionTaskDAOUpdateStatusIfCurrentMatchesMultipleStatuses(t *testing.T) {
+ db := setupTaskTestDB(t)
+ orig := DB
+ DB = db
+ t.Cleanup(func() { DB = orig })
+
+ task := &entity.IngestionTask{
+ ID: "task-1",
+ UserID: "user-1",
+ DocumentID: "doc-1",
+ DatasetID: "kb-1",
+ Status: common.SCHEDULED,
+ }
+ if err := db.Create(task).Error; err != nil {
+ t.Fatalf("create task: %v", err)
+ }
+
+ ctx := t.Context()
+ updated, err := NewIngestionTaskDAO().UpdateStatusIfCurrent(ctx, db, "task-1", []string{common.CREATED, common.SCHEDULED}, common.RUNNING)
+ if err != nil {
+ t.Fatalf("UpdateStatusIfCurrent failed: %v", err)
+ }
+ if !updated {
+ t.Fatal("expected update to succeed")
+ }
+
+ reloaded, err := NewIngestionTaskDAO().GetByID(ctx, db, "task-1")
+ if err != nil {
+ t.Fatalf("reload task: %v", err)
+ }
+ if reloaded.Status != common.RUNNING {
+ t.Fatalf("status = %q, want %q", reloaded.Status, common.RUNNING)
+ }
+}
+
+func TestIngestionTaskDAOUpdateStatusIfCurrentEmptyStatusesReturnsFalse(t *testing.T) {
+ db := setupTaskTestDB(t)
+ orig := DB
+ DB = db
+ t.Cleanup(func() { DB = orig })
+
+ task := &entity.IngestionTask{
+ ID: "task-1",
+ UserID: "user-1",
+ DocumentID: "doc-1",
+ DatasetID: "kb-1",
+ Status: common.CREATED,
+ }
+ if err := db.Create(task).Error; err != nil {
+ t.Fatalf("create task: %v", err)
+ }
+
+ ctx := t.Context()
+ updated, err := NewIngestionTaskDAO().UpdateStatusIfCurrent(ctx, db, "task-1", nil, common.RUNNING)
+ if err != nil {
+ t.Fatalf("UpdateStatusIfCurrent failed: %v", err)
+ }
+ if updated {
+ t.Fatal("expected update with empty fromStatuses to be rejected")
+ }
+
+ reloaded, err := NewIngestionTaskDAO().GetByID(ctx, db, "task-1")
+ if err != nil {
+ t.Fatalf("reload task: %v", err)
+ }
+ if reloaded.Status != common.CREATED {
+ t.Fatalf("status = %q, want %q", reloaded.Status, common.CREATED)
+ }
+}
+
func TestIngestionTaskDAOCreateRejectsExistingTerminalTask(t *testing.T) {
db := setupTaskTestDB(t)
orig := DB
@@ -251,7 +321,7 @@ func TestIngestionTaskDAOUpdateStatusIfCurrentRejectsMismatchedStatus(t *testing
}
ctx := t.Context()
- updated, err := NewIngestionTaskDAO().UpdateStatusIfCurrent(ctx, db, "task-1", common.CREATED, common.RUNNING)
+ updated, err := NewIngestionTaskDAO().UpdateStatusIfCurrent(ctx, db, "task-1", []string{common.CREATED}, common.RUNNING)
if err != nil {
t.Fatalf("UpdateStatusIfCurrent failed: %v", err)
}
diff --git a/internal/ingestion/service/real_consumer_pipeline_test.go b/internal/ingestion/service/real_consumer_pipeline_test.go
index a51edf73ce..8735cbdcf8 100644
--- a/internal/ingestion/service/real_consumer_pipeline_test.go
+++ b/internal/ingestion/service/real_consumer_pipeline_test.go
@@ -87,7 +87,7 @@ func TestRealConsumer_PipelineMessageRoutesToExecuteTask(t *testing.T) {
}
ingestionTaskDAO := dao.NewIngestionTaskDAO()
- _, err = ingestionTaskDAO.UpdateStatusIfCurrent(context.Background(), db, taskMsg.TaskID, common.CREATED, common.RUNNING)
+ _, err = ingestionTaskDAO.UpdateStatusIfCurrent(context.Background(), db, taskMsg.TaskID, []string{common.CREATED}, common.RUNNING)
if err != nil {
t.Fatalf("UpdateStatusIfCurrent: %v", err)
}
diff --git a/internal/ingestion/task/real_consumer_test.go b/internal/ingestion/task/real_consumer_test.go
index 804770040a..b3b56a7810 100644
--- a/internal/ingestion/task/real_consumer_test.go
+++ b/internal/ingestion/task/real_consumer_test.go
@@ -110,7 +110,7 @@ func TestRealProducerConsumer(t *testing.T) {
// Mirrors Start():142-143 — UpdateStatusIfCurrent
ingestionTaskDAO := dao.NewIngestionTaskDAO()
- _, err = ingestionTaskDAO.UpdateStatusIfCurrent(context.Background(), db, taskMsg.TaskID, common.CREATED, common.RUNNING)
+ _, err = ingestionTaskDAO.UpdateStatusIfCurrent(context.Background(), db, taskMsg.TaskID, []string{common.CREATED}, common.RUNNING)
if err != nil {
t.Fatalf("UpdateStatusIfCurrent: %v", err)
}
@@ -168,7 +168,7 @@ func TestRealProducerConsumer(t *testing.T) {
t.Log("Consumer: PipelineExecutor.Execute() - OK")
// Mirrors executeTask — mark as completed
- if _, err := ingestionTaskDAO.UpdateStatusIfCurrent(context.Background(), db, task.ID, common.RUNNING, common.COMPLETED); err != nil {
+ if _, err := ingestionTaskDAO.UpdateStatusIfCurrent(context.Background(), db, task.ID, []string{common.RUNNING}, common.COMPLETED); err != nil {
t.Fatalf("UpdateStatus: %v", err)
}
diff --git a/internal/service/ingestion_task_service.go b/internal/service/ingestion_task_service.go
index bdf50e59ab..429950db7d 100644
--- a/internal/service/ingestion_task_service.go
+++ b/internal/service/ingestion_task_service.go
@@ -216,7 +216,7 @@ func (s *IngestionTaskService) StartRunning(ctx context.Context, taskID string)
}
switch task.Status {
case common.CREATED, common.SCHEDULED:
- task, err = s.transition(ctx, taskID, common.RUNNING)
+ task, err = s.transitionFrom(ctx, taskID, []string{common.CREATED, common.SCHEDULED}, common.RUNNING)
if err != nil {
return nil, err
}
@@ -263,7 +263,7 @@ func (s *IngestionTaskService) RequestStop(ctx context.Context, taskID string) (
}
switch task.Status {
case common.CREATED, common.SCHEDULED:
- stopped, err := s.transition(ctx, taskID, common.STOPPED)
+ stopped, err := s.transitionFrom(ctx, taskID, []string{common.CREATED, common.SCHEDULED}, common.STOPPED)
if err != nil {
return nil, err
}
@@ -394,6 +394,27 @@ func (s *IngestionTaskService) newTaskStatusConflictError(ctx context.Context, t
}
}
+func (s *IngestionTaskService) transitionFrom(ctx context.Context, taskID string, fromStatuses []string, to string) (*entity.IngestionTask, error) {
+ for _, from := range fromStatuses {
+ if err := validateTransition(from, to); err != nil {
+ var transitionErr *InvalidTaskTransitionError
+ if errors.As(err, &transitionErr) {
+ return nil, &InvalidTaskTransitionError{TaskID: taskID, From: transitionErr.From, To: transitionErr.To}
+ }
+ return nil, err
+ }
+ }
+ updated, err := s.ingestionTaskDAO.UpdateStatusIfCurrent(ctx, dao.DB, taskID, fromStatuses, to)
+ if err != nil {
+ return nil, err
+ }
+ if !updated {
+ expected := strings.Join(fromStatuses, "/")
+ return nil, s.newTaskStatusConflictError(ctx, taskID, expected, to)
+ }
+ return s.GetTask(ctx, taskID)
+}
+
func (s *IngestionTaskService) transition(ctx context.Context, taskID string, to string) (*entity.IngestionTask, error) {
task, err := s.GetTask(ctx, taskID)
if err != nil {
@@ -406,7 +427,7 @@ func (s *IngestionTaskService) transition(ctx context.Context, taskID string, to
}
return task, err
}
- updated, err := s.ingestionTaskDAO.UpdateStatusIfCurrent(ctx, dao.DB, taskID, task.Status, to)
+ updated, err := s.ingestionTaskDAO.UpdateStatusIfCurrent(ctx, dao.DB, taskID, []string{task.Status}, to)
if err != nil {
return nil, err
}
@@ -492,7 +513,7 @@ func (s *IngestionTaskService) createAndEnqueueWithKBCache(ctx context.Context,
}
func (s *IngestionTaskService) rollbackRetriedTask(ctx context.Context, taskID, status string) error {
- updated, err := s.ingestionTaskDAO.UpdateStatusIfCurrent(ctx, dao.DB, taskID, common.CREATED, status)
+ updated, err := s.ingestionTaskDAO.UpdateStatusIfCurrent(ctx, dao.DB, taskID, []string{common.CREATED}, status)
if err != nil {
return err
}
@@ -512,7 +533,7 @@ func (s *IngestionTaskService) rollbackCreatedTask(ctx context.Context, taskID s
// the unstarted from-state, so it never regresses a row the worker already
// advanced.
func (s *IngestionTaskService) markScheduledAfterPublish(ctx context.Context, taskID string) (*entity.IngestionTask, error) {
- updated, err := s.ingestionTaskDAO.UpdateStatusIfCurrent(ctx, dao.DB, taskID, common.CREATED, common.SCHEDULED)
+ updated, err := s.ingestionTaskDAO.UpdateStatusIfCurrent(ctx, dao.DB, taskID, []string{common.CREATED}, common.SCHEDULED)
if err != nil {
return nil, err
}
diff --git a/internal/service/ingestion_task_service_test.go b/internal/service/ingestion_task_service_test.go
index 20a27980ae..47cbaf414b 100644
--- a/internal/service/ingestion_task_service_test.go
+++ b/internal/service/ingestion_task_service_test.go
@@ -225,6 +225,47 @@ func TestIngestionTaskServiceStartRunningTransitionsScheduledTask(t *testing.T)
}
}
+func TestIngestionTaskServiceStartRunningFromCreatedTask(t *testing.T) {
+ db := setupServiceTestDB(t)
+ pushServiceDB(t, db)
+ insertTestKB(t, "kb-1", "tenant-1", 1, 0, 0)
+ insertTestDoc(t, "doc-1", "kb-1", 0, 0)
+ insertTestIngestionTaskWithStatus(t, "task-1", "user-1", "doc-1", "kb-1", common.CREATED)
+
+ task, err := NewIngestionTaskService().StartRunning(t.Context(), "task-1")
+ if err != nil {
+ t.Fatalf("StartRunning failed: %v", err)
+ }
+ if task.Status != common.RUNNING {
+ t.Fatalf("status = %q, want %q", task.Status, common.RUNNING)
+ }
+}
+
+func TestIngestionTaskServiceTransitionFromRejectsConflict(t *testing.T) {
+ db := setupServiceTestDB(t)
+ pushServiceDB(t, db)
+ insertTestKB(t, "kb-1", "tenant-1", 1, 0, 0)
+ insertTestDoc(t, "doc-1", "kb-1", 0, 0)
+ insertTestIngestionTaskWithStatus(t, "task-1", "user-1", "doc-1", "kb-1", common.STOPPED)
+
+ svc := NewIngestionTaskService()
+ ctx := t.Context()
+ _, err := svc.transitionFrom(ctx, "task-1", []string{common.CREATED, common.SCHEDULED}, common.RUNNING)
+ if err == nil {
+ t.Fatal("expected conflict error, got nil")
+ }
+ var conflictErr *TaskStatusConflictError
+ if !errors.As(err, &conflictErr) {
+ t.Fatalf("expected TaskStatusConflictError, got %T (%v)", err, err)
+ }
+ if conflictErr.ExpectedFrom != "CREATED/SCHEDULED" {
+ t.Fatalf("expected %q, got %q", "CREATED/SCHEDULED", conflictErr.ExpectedFrom)
+ }
+ if conflictErr.ActualCurrent != common.STOPPED {
+ t.Fatalf("actual current = %q, want %q", conflictErr.ActualCurrent, common.STOPPED)
+ }
+}
+
func TestIngestionTaskServiceListByUserFiltersDataset(t *testing.T) {
db := setupServiceTestDB(t)
pushServiceDB(t, db)