RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
fix(ingestion): make running transition atomic across created and scheduled (#19764)
jay77721b193b5741eb6
diff --git a/internal/dao/ingestion.go b/internal/dao/ingestion.goindex 7cdf9e223e..fd8b1e375e 100644--- a/internal/dao/ingestion.go+++ b/internal/dao/ingestion.go@@ -60,9 +60,13 @@ func (dao *IngestionTaskDAO) Create(ctx context.Context, db *gorm.DB, ingestionTreturn ingestionTask, nil}-func (dao *IngestionTaskDAO) UpdateStatusIfCurrent(ctx context.Context, db *gorm.DB, taskID, fromStatus, toStatus string) (bool, error) {+// UpdateStatusIfCurrent updates the task status if its current status matches any of the given fromStatuses.+func (dao *IngestionTaskDAO) UpdateStatusIfCurrent(ctx context.Context, db *gorm.DB, taskID string, fromStatuses []string, toStatus string) (bool, error) {+ if len(fromStatuses) == 0 {+ return false, nil+ }result := db.WithContext(ctx).Model(&entity.IngestionTask{}).- Where("id = ? AND status = ?", taskID, fromStatus).+ Where("id = ? AND status IN (?)", taskID, fromStatuses).Update("status", toStatus)if result.Error != nil {return false, result.Errordiff --git a/internal/dao/ingestion_task_test.go b/internal/dao/ingestion_task_test.goindex 22ca81aa42..b914182335 100644--- a/internal/dao/ingestion_task_test.go+++ b/internal/dao/ingestion_task_test.go@@ -29,7 +29,7 @@ func TestIngestionTaskDAOUpdateStatusIfCurrentSucceeds(t *testing.T) {}ctx := t.Context()- updated, err := NewIngestionTaskDAO().UpdateStatusIfCurrent(ctx, db, "task-1", common.CREATED, common.RUNNING)+ updated, err := NewIngestionTaskDAO().UpdateStatusIfCurrent(ctx, db, "task-1", []string{common.CREATED}, common.RUNNING)if err != nil {t.Fatalf("UpdateStatusIfCurrent failed: %v", err)}@@ -46,6 +46,76 @@ func TestIngestionTaskDAOUpdateStatusIfCurrentSucceeds(t *testing.T) {}}+func TestIngestionTaskDAOUpdateStatusIfCurrentMatchesMultipleStatuses(t *testing.T) {+ db := setupTaskTestDB(t)+ orig := DB+ DB = db+ t.Cleanup(func() { DB = orig })++ task := &entity.IngestionTask{+ ID: "task-1",+ UserID: "user-1",+ DocumentID: "doc-1",+ DatasetID: "kb-1",+ Status: common.SCHEDULED,+ }+ if err := db.Create(task).Error; err != nil {+ t.Fatalf("create task: %v", err)+ }++ ctx := t.Context()+ updated, err := NewIngestionTaskDAO().UpdateStatusIfCurrent(ctx, db, "task-1", []string{common.CREATED, common.SCHEDULED}, common.RUNNING)+ if err != nil {+ t.Fatalf("UpdateStatusIfCurrent failed: %v", err)+ }+ if !updated {+ t.Fatal("expected update to succeed")+ }++ reloaded, err := NewIngestionTaskDAO().GetByID(ctx, db, "task-1")+ if err != nil {+ t.Fatalf("reload task: %v", err)+ }+ if reloaded.Status != common.RUNNING {+ t.Fatalf("status = %q, want %q", reloaded.Status, common.RUNNING)+ }+}++func TestIngestionTaskDAOUpdateStatusIfCurrentEmptyStatusesReturnsFalse(t *testing.T) {+ db := setupTaskTestDB(t)+ orig := DB+ DB = db+ t.Cleanup(func() { DB = orig })++ task := &entity.IngestionTask{+ ID: "task-1",+ UserID: "user-1",+ DocumentID: "doc-1",+ DatasetID: "kb-1",+ Status: common.CREATED,+ }+ if err := db.Create(task).Error; err != nil {+ t.Fatalf("create task: %v", err)+ }++ ctx := t.Context()+ updated, err := NewIngestionTaskDAO().UpdateStatusIfCurrent(ctx, db, "task-1", nil, common.RUNNING)+ if err != nil {+ t.Fatalf("UpdateStatusIfCurrent failed: %v", err)+ }+ if updated {+ t.Fatal("expected update with empty fromStatuses to be rejected")+ }++ reloaded, err := NewIngestionTaskDAO().GetByID(ctx, db, "task-1")+ if err != nil {+ t.Fatalf("reload task: %v", err)+ }+ if reloaded.Status != common.CREATED {+ t.Fatalf("status = %q, want %q", reloaded.Status, common.CREATED)+ }+}+func TestIngestionTaskDAOCreateRejectsExistingTerminalTask(t *testing.T) {db := setupTaskTestDB(t)orig := DB@@ -251,7 +321,7 @@ func TestIngestionTaskDAOUpdateStatusIfCurrentRejectsMismatchedStatus(t *testing}ctx := t.Context()- updated, err := NewIngestionTaskDAO().UpdateStatusIfCurrent(ctx, db, "task-1", common.CREATED, common.RUNNING)+ updated, err := NewIngestionTaskDAO().UpdateStatusIfCurrent(ctx, db, "task-1", []string{common.CREATED}, common.RUNNING)if err != nil {t.Fatalf("UpdateStatusIfCurrent failed: %v", err)}diff --git a/internal/ingestion/service/real_consumer_pipeline_test.go b/internal/ingestion/service/real_consumer_pipeline_test.goindex a51edf73ce..8735cbdcf8 100644--- a/internal/ingestion/service/real_consumer_pipeline_test.go+++ b/internal/ingestion/service/real_consumer_pipeline_test.go@@ -87,7 +87,7 @@ func TestRealConsumer_PipelineMessageRoutesToExecuteTask(t *testing.T) {}ingestionTaskDAO := dao.NewIngestionTaskDAO()- _, err = ingestionTaskDAO.UpdateStatusIfCurrent(context.Background(), db, taskMsg.TaskID, common.CREATED, common.RUNNING)+ _, err = ingestionTaskDAO.UpdateStatusIfCurrent(context.Background(), db, taskMsg.TaskID, []string{common.CREATED}, common.RUNNING)if err != nil {t.Fatalf("UpdateStatusIfCurrent: %v", err)}diff --git a/internal/ingestion/task/real_consumer_test.go b/internal/ingestion/task/real_consumer_test.goindex 804770040a..b3b56a7810 100644--- a/internal/ingestion/task/real_consumer_test.go+++ b/internal/ingestion/task/real_consumer_test.go@@ -110,7 +110,7 @@ func TestRealProducerConsumer(t *testing.T) {// Mirrors Start():142-143 — UpdateStatusIfCurrentingestionTaskDAO := dao.NewIngestionTaskDAO()- _, err = ingestionTaskDAO.UpdateStatusIfCurrent(context.Background(), db, taskMsg.TaskID, common.CREATED, common.RUNNING)+ _, err = ingestionTaskDAO.UpdateStatusIfCurrent(context.Background(), db, taskMsg.TaskID, []string{common.CREATED}, common.RUNNING)if err != nil {t.Fatalf("UpdateStatusIfCurrent: %v", err)}@@ -168,7 +168,7 @@ func TestRealProducerConsumer(t *testing.T) {t.Log("Consumer: PipelineExecutor.Execute() - OK")// Mirrors executeTask — mark as completed- if _, err := ingestionTaskDAO.UpdateStatusIfCurrent(context.Background(), db, task.ID, common.RUNNING, common.COMPLETED); err != nil {+ if _, err := ingestionTaskDAO.UpdateStatusIfCurrent(context.Background(), db, task.ID, []string{common.RUNNING}, common.COMPLETED); err != nil {t.Fatalf("UpdateStatus: %v", err)}diff --git a/internal/service/ingestion_task_service.go b/internal/service/ingestion_task_service.goindex bdf50e59ab..429950db7d 100644--- a/internal/service/ingestion_task_service.go+++ b/internal/service/ingestion_task_service.go@@ -216,7 +216,7 @@ func (s *IngestionTaskService) StartRunning(ctx context.Context, taskID string)}switch task.Status {case common.CREATED, common.SCHEDULED:- task, err = s.transition(ctx, taskID, common.RUNNING)+ task, err = s.transitionFrom(ctx, taskID, []string{common.CREATED, common.SCHEDULED}, common.RUNNING)if err != nil {return nil, err}@@ -263,7 +263,7 @@ func (s *IngestionTaskService) RequestStop(ctx context.Context, taskID string) (}switch task.Status {case common.CREATED, common.SCHEDULED:- stopped, err := s.transition(ctx, taskID, common.STOPPED)+ stopped, err := s.transitionFrom(ctx, taskID, []string{common.CREATED, common.SCHEDULED}, common.STOPPED)if err != nil {return nil, err}@@ -394,6 +394,27 @@ func (s *IngestionTaskService) newTaskStatusConflictError(ctx context.Context, t}}+func (s *IngestionTaskService) transitionFrom(ctx context.Context, taskID string, fromStatuses []string, to string) (*entity.IngestionTask, error) {+ for _, from := range fromStatuses {+ if err := validateTransition(from, to); err != nil {+ var transitionErr *InvalidTaskTransitionError+ if errors.As(err, &transitionErr) {+ return nil, &InvalidTaskTransitionError{TaskID: taskID, From: transitionErr.From, To: transitionErr.To}+ }+ return nil, err+ }+ }+ updated, err := s.ingestionTaskDAO.UpdateStatusIfCurrent(ctx, dao.DB, taskID, fromStatuses, to)+ if err != nil {+ return nil, err+ }+ if !updated {+ expected := strings.Join(fromStatuses, "/")+ return nil, s.newTaskStatusConflictError(ctx, taskID, expected, to)+ }+ return s.GetTask(ctx, taskID)+}+func (s *IngestionTaskService) transition(ctx context.Context, taskID string, to string) (*entity.IngestionTask, error) {task, err := s.GetTask(ctx, taskID)if err != nil {@@ -406,7 +427,7 @@ func (s *IngestionTaskService) transition(ctx context.Context, taskID string, to}return task, err}- updated, err := s.ingestionTaskDAO.UpdateStatusIfCurrent(ctx, dao.DB, taskID, task.Status, to)+ updated, err := s.ingestionTaskDAO.UpdateStatusIfCurrent(ctx, dao.DB, taskID, []string{task.Status}, to)if err != nil {return nil, err}@@ -492,7 +513,7 @@ func (s *IngestionTaskService) createAndEnqueueWithKBCache(ctx context.Context,}func (s *IngestionTaskService) rollbackRetriedTask(ctx context.Context, taskID, status string) error {- updated, err := s.ingestionTaskDAO.UpdateStatusIfCurrent(ctx, dao.DB, taskID, common.CREATED, status)+ updated, err := s.ingestionTaskDAO.UpdateStatusIfCurrent(ctx, dao.DB, taskID, []string{common.CREATED}, status)if err != nil {return err}@@ -512,7 +533,7 @@ func (s *IngestionTaskService) rollbackCreatedTask(ctx context.Context, taskID s// the unstarted from-state, so it never regresses a row the worker already// advanced.func (s *IngestionTaskService) markScheduledAfterPublish(ctx context.Context, taskID string) (*entity.IngestionTask, error) {- updated, err := s.ingestionTaskDAO.UpdateStatusIfCurrent(ctx, dao.DB, taskID, common.CREATED, common.SCHEDULED)+ updated, err := s.ingestionTaskDAO.UpdateStatusIfCurrent(ctx, dao.DB, taskID, []string{common.CREATED}, common.SCHEDULED)if err != nil {return nil, err}diff --git a/internal/service/ingestion_task_service_test.go b/internal/service/ingestion_task_service_test.goindex 20a27980ae..47cbaf414b 100644--- a/internal/service/ingestion_task_service_test.go+++ b/internal/service/ingestion_task_service_test.go@@ -225,6 +225,47 @@ func TestIngestionTaskServiceStartRunningTransitionsScheduledTask(t *testing.T)}}+func TestIngestionTaskServiceStartRunningFromCreatedTask(t *testing.T) {+ db := setupServiceTestDB(t)+ pushServiceDB(t, db)+ insertTestKB(t, "kb-1", "tenant-1", 1, 0, 0)+ insertTestDoc(t, "doc-1", "kb-1", 0, 0)+ insertTestIngestionTaskWithStatus(t, "task-1", "user-1", "doc-1", "kb-1", common.CREATED)++ task, err := NewIngestionTaskService().StartRunning(t.Context(), "task-1")+ if err != nil {+ t.Fatalf("StartRunning failed: %v", err)+ }+ if task.Status != common.RUNNING {+ t.Fatalf("status = %q, want %q", task.Status, common.RUNNING)+ }+}++func TestIngestionTaskServiceTransitionFromRejectsConflict(t *testing.T) {+ db := setupServiceTestDB(t)+ pushServiceDB(t, db)+ insertTestKB(t, "kb-1", "tenant-1", 1, 0, 0)+ insertTestDoc(t, "doc-1", "kb-1", 0, 0)+ insertTestIngestionTaskWithStatus(t, "task-1", "user-1", "doc-1", "kb-1", common.STOPPED)++ svc := NewIngestionTaskService()+ ctx := t.Context()+ _, err := svc.transitionFrom(ctx, "task-1", []string{common.CREATED, common.SCHEDULED}, common.RUNNING)+ if err == nil {+ t.Fatal("expected conflict error, got nil")+ }+ var conflictErr *TaskStatusConflictError+ if !errors.As(err, &conflictErr) {+ t.Fatalf("expected TaskStatusConflictError, got %T (%v)", err, err)+ }+ if conflictErr.ExpectedFrom != "CREATED/SCHEDULED" {+ t.Fatalf("expected %q, got %q", "CREATED/SCHEDULED", conflictErr.ExpectedFrom)+ }+ if conflictErr.ActualCurrent != common.STOPPED {+ t.Fatalf("actual current = %q, want %q", conflictErr.ActualCurrent, common.STOPPED)+ }+}+func TestIngestionTaskServiceListByUserFiltersDataset(t *testing.T) {db := setupServiceTestDB(t)pushServiceDB(t, db)