RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
fix(ingestion): make NATS task scheduling crash-safe (#19074)
jay77721686663845eff
diff --git a/internal/common/constants.go b/internal/common/constants.goindex ed0e45ef94..96a301ba39 100644--- a/internal/common/constants.go+++ b/internal/common/constants.go@@ -31,6 +31,7 @@ const (// task statusconst (CREATED = "CREATED"+ SCHEDULED = "SCHEDULED"RUNNING = "RUNNING"COMPLETED = "COMPLETED"FAILED = "FAILED"diff --git a/internal/dao/ingestion.go b/internal/dao/ingestion.goindex dae1a10659..46864a315a 100644--- a/internal/dao/ingestion.go+++ b/internal/dao/ingestion.go@@ -23,7 +23,6 @@ import ("ragflow/internal/common""ragflow/internal/entity""ragflow/internal/utility"- "time""gorm.io/gorm")@@ -121,15 +120,20 @@ func (dao *IngestionTaskDAO) Delete(ctx context.Context, db *gorm.DB, taskID strtaskStatus := tasks[0].Statusswitch taskStatus {- case common.CREATED, common.STOPPED, common.COMPLETED, common.FAILED:+ case common.CREATED, common.SCHEDULED, common.STOPPED, common.COMPLETED, common.FAILED:// ingestion_task_log no longer carries file references (the old// checkpoint JSON column was dropped in favor of typed columns), so// there are no task-level files to delete here.var filesToDelete []string- err = tx.Model(&entity.IngestionTask{}).Where("id = ?", taskID).Delete(&entity.IngestionTask{}).Error- if err != nil {- return nil, err+ result := tx.Model(&entity.IngestionTask{}).+ Where("id = ? AND status IN ?", taskID, []string{common.CREATED, common.SCHEDULED, common.STOPPED, common.COMPLETED, common.FAILED}).+ Delete(&entity.IngestionTask{})+ if result.Error != nil {+ return nil, result.Error+ }+ if result.RowsAffected != 1 {+ return nil, fmt.Errorf("task %s status changed, cannot be removed", taskID)}taskInfo := &TaskInfo{@@ -145,22 +149,24 @@ func (dao *IngestionTaskDAO) Delete(ctx context.Context, db *gorm.DB, taskID strfunc (dao *IngestionTaskDAO) GetAllTasks(ctx context.Context, db *gorm.DB, page, pageSize int) ([]*entity.IngestionTask, error) {var tasks []*entity.IngestionTask+ query := db.WithContext(ctx)var err errorif pageSize == 0 {- err = db.WithContext(ctx).Find(&tasks).Error+ err = query.Find(&tasks).Error} else {- err = db.WithContext(ctx).Order("create_time DESC").Offset((page - 1) * pageSize).Limit(pageSize).Find(&tasks).Error+ err = query.Order("create_time DESC").Offset((page - 1) * pageSize).Limit(pageSize).Find(&tasks).Error}return tasks, err}func (dao *IngestionTaskDAO) ListByUserID(ctx context.Context, db *gorm.DB, userID string, page, pageSize int) ([]*entity.IngestionTask, error) {var tasks []*entity.IngestionTask+ query := db.WithContext(ctx).Where("user_id = ?", userID)var err errorif pageSize == 0 {- err = db.WithContext(ctx).Where("user_id = ?", userID).Order("create_time DESC").Find(&tasks).Error+ err = query.Order("create_time DESC").Find(&tasks).Error} else {- err = db.WithContext(ctx).Where("user_id = ?", userID).Order("create_time DESC").Offset((page - 1) * pageSize).Limit(pageSize).Find(&tasks).Error+ err = query.Order("create_time DESC").Offset((page - 1) * pageSize).Limit(pageSize).Find(&tasks).Error}return tasks, err@@ -168,16 +174,23 @@ func (dao *IngestionTaskDAO) ListByUserID(ctx context.Context, db *gorm.DB, userfunc (dao *IngestionTaskDAO) ListByUserIDAndDatasetID(ctx context.Context, db *gorm.DB, userID, datasetID string, page, pageSize int) ([]*entity.IngestionTask, error) {var tasks []*entity.IngestionTask+ query := db.WithContext(ctx).Where("user_id = ? AND dataset_id = ?", userID, datasetID)var err errorif pageSize == 0 {- err = db.WithContext(ctx).Where("user_id = ? AND dataset_id = ?", userID, datasetID).Order("create_time DESC").Find(&tasks).Error+ err = query.Order("create_time DESC").Find(&tasks).Error} else {- err = db.WithContext(ctx).Where("user_id = ? AND dataset_id = ?", userID, datasetID).Order("create_time DESC").Offset((page - 1) * pageSize).Limit(pageSize).Find(&tasks).Error+ err = query.Order("create_time DESC").Offset((page - 1) * pageSize).Limit(pageSize).Find(&tasks).Error}return tasks, err}+func (dao *IngestionTaskDAO) ListByStatus(ctx context.Context, db *gorm.DB, status string) ([]*entity.IngestionTask, error) {+ var tasks []*entity.IngestionTask+ err := db.WithContext(ctx).Where("status = ?", status).Order("create_time ASC").Find(&tasks).Error+ return tasks, err+}+func (dao *IngestionTaskDAO) GetByID(ctx context.Context, db *gorm.DB, id string) (*entity.IngestionTask, error) {var task *entity.IngestionTaskerr := db.WithContext(ctx).Where("id = ?", id).First(&task).Error@@ -196,35 +209,9 @@ func (dao *IngestionTaskDAO) GetByDocumentID(ctx context.Context, db *gorm.DB, dreturn tasks[0], nil}-// ListStaleByStatus returns tasks in the given statuses whose timestamp column-// is older than the threshold, for ingestor startup reconciliation. timeColumn-// selects which staleness clock applies: "update_time" for RUNNING orphans-// (frozen at the moment the previous process claimed them) or "create_time"-// for CREATED orphans (never picked up). Results are ordered oldest first and-// capped at limit when limit > 0.-func (dao *IngestionTaskDAO) ListStaleByStatus(ctx context.Context, db *gorm.DB, statuses []string, timeColumn string, olderThan time.Time, limit int) ([]*entity.IngestionTask, error) {- switch timeColumn {- case "create_time", "update_time":- default:- return nil, fmt.Errorf("unsupported staleness column %q", timeColumn)- }- if len(statuses) == 0 {- return []*entity.IngestionTask{}, nil- }- query := db.WithContext(ctx).- Where("status IN ?", statuses).- Where(timeColumn+" < ?", olderThan.UnixMilli()).- Order(timeColumn + " ASC")- if limit > 0 {- query = query.Limit(limit)- }- var tasks []*entity.IngestionTask- err := query.Find(&tasks).Error- return tasks, err-}-// DeleteIfTerminal deletes ingestion tasks for a document that are in a-// terminal state (COMPLETED, STOPPED, FAILED) or still queued (CREATED).+// terminal state (COMPLETED, STOPPED, FAILED), or not yet running+// (CREATED, SCHEDULED).// RUNNING and STOPPING tasks are NOT deleted because an in-flight worker// would keep writing chunks and corrupt a new run's results.// Returns the number of rows deleted.diff --git a/internal/dao/ingestion_task_test.go b/internal/dao/ingestion_task_test.goindex f047a8a2ba..c0b4280dec 100644--- a/internal/dao/ingestion_task_test.go+++ b/internal/dao/ingestion_task_test.go@@ -3,7 +3,6 @@ package daoimport ("errors""testing"- "time""fmt""ragflow/internal/common"@@ -104,129 +103,169 @@ func TestIngestionTaskDAODocumentIDIsUniqueAtDBLevel(t *testing.T) {}}-func TestIngestionTaskDAOUpdateStatusIfCurrentRejectsMismatchedStatus(t *testing.T) {+func TestIngestionTaskDAODeleteAllowsScheduledTask(t *testing.T) {db := setupTaskTestDB(t)orig := DBDB = dbt.Cleanup(func() { DB = orig })task := &entity.IngestionTask{- ID: "task-1",+ ID: "task-scheduled",UserID: "user-1",- DocumentID: "doc-1",+ DocumentID: "doc-scheduled",DatasetID: "kb-1",- Status: common.STOPPING,+ Status: common.SCHEDULED,}if err := db.Create(task).Error; err != nil {- t.Fatalf("create task: %v", err)- }-- ctx := t.Context()- updated, err := NewIngestionTaskDAO().UpdateStatusIfCurrent(ctx, db, "task-1", common.CREATED, common.RUNNING)- if err != nil {- t.Fatalf("UpdateStatusIfCurrent failed: %v", err)- }- if updated {- t.Fatal("expected update to be rejected")+ t.Fatalf("create scheduled task: %v", err)}- reloaded, err := NewIngestionTaskDAO().GetByID(ctx, db, "task-1")+ info, err := NewIngestionTaskDAO().Delete(t.Context(), db, task.ID, nil)if err != nil {- t.Fatalf("reload task: %v", err)+ t.Fatalf("delete scheduled task: %v", err)}- if reloaded.Status != common.STOPPING {- t.Fatalf("status = %q, want %q", reloaded.Status, common.STOPPING)+ if info == nil || info.TaskID != task.ID {+ t.Fatalf("unexpected task info: %+v", info)}}-// seedStaleTask inserts an ingestion task whose create/update timestamps are-// aged back so ListStaleByStatus staleness windows can be asserted.-func seedStaleTask(t *testing.T, db *gorm.DB, id, status string, age time.Duration) {- t.Helper()- ts := time.Now().Add(-age).UnixMilli()+func TestIngestionTaskDAODeleteDoesNotRemoveTaskClaimedDuringDelete(t *testing.T) {+ db := setupTaskTestDB(t)task := &entity.IngestionTask{- ID: id,+ ID: "task-scheduled",UserID: "user-1",- DocumentID: "doc-" + id,+ DocumentID: "doc-scheduled",DatasetID: "kb-1",- Status: status,- BaseModel: entity.BaseModel{CreateTime: &ts, UpdateTime: &ts},+ Status: common.SCHEDULED,}if err := db.Create(task).Error; err != nil {- t.Fatalf("create task %s: %v", id, err)+ t.Fatalf("create scheduled task: %v", err)+ }++ const callbackName = "test:claim-ingestion-task-before-delete"+ if err := db.Callback().Delete().Before("gorm:delete").Register(callbackName, func(tx *gorm.DB) {+ if tx.Statement.Table != task.TableName() {+ return+ }+ if err := tx.Session(&gorm.Session{NewDB: true}).Model(&entity.IngestionTask{}).Where("id = ?", task.ID).+ Update("status", common.RUNNING).Error; err != nil {+ tx.AddError(err)+ }+ }); err != nil {+ t.Fatalf("register delete callback: %v", err)+ }++ if _, err := NewIngestionTaskDAO().Delete(t.Context(), db, task.ID, nil); err == nil {+ t.Fatal("expected delete to reject task claimed during delete")+ }++ reloaded, err := NewIngestionTaskDAO().GetByID(t.Context(), db, task.ID)+ if err != nil {+ t.Fatalf("reload task after rejected delete: %v", err)+ }+ if reloaded.Status != common.SCHEDULED {+ t.Fatalf("status after rejected delete = %q, want %q", reloaded.Status, common.SCHEDULED)}}-// TestIngestionTaskDAOListStaleByStatus covers the three contract axes of the-// startup-reconciliation query: status filtering, the staleness time window-// (per column), and the result limit.-func TestIngestionTaskDAOListStaleByStatus(t *testing.T) {+func TestIngestionTaskDAOListByStatus(t *testing.T) {db := setupTaskTestDB(t)orig := DBDB = dbt.Cleanup(func() { DB = orig })- seedStaleTask(t, db, "run-stale", common.RUNNING, 20*time.Minute)- seedStaleTask(t, db, "run-fresh", common.RUNNING, 1*time.Minute)- seedStaleTask(t, db, "created-stale", common.CREATED, 20*time.Minute)- seedStaleTask(t, db, "created-fresh", common.CREATED, 1*time.Minute)- seedStaleTask(t, db, "completed-stale", common.COMPLETED, 20*time.Minute)-- ctx := t.Context()- d := NewIngestionTaskDAO()- threshold := time.Now().Add(-15 * time.Minute)+ for _, task := range []*entity.IngestionTask{+ {ID: "task-created", UserID: "user-1", DocumentID: "doc-created", DatasetID: "kb-1", Status: common.CREATED},+ {ID: "task-scheduled", UserID: "user-1", DocumentID: "doc-scheduled", DatasetID: "kb-1", Status: common.SCHEDULED},+ } {+ if err := db.Create(task).Error; err != nil {+ t.Fatalf("create task %s: %v", task.ID, err)+ }+ }- // Status filter + update_time window: only the stale RUNNING row matches.- running, err := d.ListStaleByStatus(ctx, db, []string{common.RUNNING}, "update_time", threshold, 0)+ tasks, err := NewIngestionTaskDAO().ListByStatus(t.Context(), db, common.CREATED)if err != nil {- t.Fatalf("ListStaleByStatus(RUNNING): %v", err)+ t.Fatalf("list CREATED tasks: %v", err)}- if len(running) != 1 || running[0].ID != "run-stale" {- t.Fatalf("RUNNING stale rows = %v, want [run-stale]", idsOfTasks(running))+ if len(tasks) != 1 || tasks[0].ID != "task-created" {+ t.Fatalf("CREATED tasks = %+v, want only task-created", tasks)}+}- // CREATED is judged by create_time: fresh CREATED rows stay outside the- // window even though their status is in the filter.- created, err := d.ListStaleByStatus(ctx, db, []string{common.CREATED}, "create_time", threshold, 0)+func TestIngestionTaskDAOListsTasks(t *testing.T) {+ db := setupTaskTestDB(t)+ orig := DB+ DB = db+ t.Cleanup(func() { DB = orig })++ for _, task := range []*entity.IngestionTask{+ {ID: "task-created", UserID: "user-1", DocumentID: "doc-created", DatasetID: "kb-1", Status: common.CREATED},+ {ID: "task-scheduled", UserID: "user-1", DocumentID: "doc-scheduled", DatasetID: "kb-1", Status: common.SCHEDULED},+ } {+ if err := db.Create(task).Error; err != nil {+ t.Fatalf("create task %s: %v", task.ID, err)+ }+ }++ d := NewIngestionTaskDAO()+ tasks, err := d.ListByUserID(t.Context(), db, "user-1", 0, 0)if err != nil {- t.Fatalf("ListStaleByStatus(CREATED): %v", err)+ t.Fatalf("list tasks by user: %v", err)}- if len(created) != 1 || created[0].ID != "created-stale" {- t.Fatalf("CREATED stale rows = %v, want [created-stale]", idsOfTasks(created))+ if len(tasks) != 2 {+ t.Fatalf("listed tasks = %+v, want 2 tasks", tasks)}- // Rows in unlisted statuses (COMPLETED) are never returned.- all, err := d.ListStaleByStatus(ctx, db, []string{common.RUNNING, common.CREATED}, "create_time", threshold, 0)+ tasks, err = d.ListByUserIDAndDatasetID(t.Context(), db, "user-1", "kb-1", 0, 0)if err != nil {- t.Fatalf("ListStaleByStatus(RUNNING+CREATED): %v", err)+ t.Fatalf("list tasks by user and dataset: %v", err)}- for _, task := range all {- if task.ID == "completed-stale" {- t.Fatal("COMPLETED row must not be returned for RUNNING/CREATED filter")- }+ if len(tasks) != 2 {+ t.Fatalf("listed tasks by dataset = %+v, want 2 tasks", tasks)}- // Limit caps the (oldest-first) result set.- limited, err := d.ListStaleByStatus(ctx, db, []string{common.RUNNING, common.CREATED}, "create_time", threshold, 1)+ tasks, err = d.GetAllTasks(t.Context(), db, 0, 0)if err != nil {- t.Fatalf("ListStaleByStatus(limit): %v", err)+ t.Fatalf("list all tasks: %v", err)}- if len(limited) != 1 {- t.Fatalf("limited rows = %v, want exactly 1", idsOfTasks(limited))+ if len(tasks) != 2 {+ t.Fatalf("all listed tasks = %+v, want 2 tasks", tasks)}+}++func TestIngestionTaskDAOUpdateStatusIfCurrentRejectsMismatchedStatus(t *testing.T) {+ db := setupTaskTestDB(t)+ orig := DB+ DB = db+ t.Cleanup(func() { DB = orig })- // An unsupported staleness column is rejected, not silently ignored.- if _, err = d.ListStaleByStatus(ctx, db, []string{common.RUNNING}, "status; --", threshold, 0); err == nil {- t.Fatal("expected error for unsupported staleness column")+ task := &entity.IngestionTask{+ ID: "task-1",+ UserID: "user-1",+ DocumentID: "doc-1",+ DatasetID: "kb-1",+ Status: common.STOPPING,+ }+ if err := db.Create(task).Error; err != nil {+ t.Fatalf("create task: %v", err)}-}-func idsOfTasks(tasks []*entity.IngestionTask) []string {- ids := make([]string, 0, len(tasks))- for _, task := range tasks {- ids = append(ids, task.ID)+ ctx := t.Context()+ updated, err := NewIngestionTaskDAO().UpdateStatusIfCurrent(ctx, db, "task-1", common.CREATED, common.RUNNING)+ if err != nil {+ t.Fatalf("UpdateStatusIfCurrent failed: %v", err)+ }+ if updated {+ t.Fatal("expected update to be rejected")+ }++ reloaded, err := NewIngestionTaskDAO().GetByID(ctx, db, "task-1")+ if err != nil {+ t.Fatalf("reload task: %v", err)+ }+ if reloaded.Status != common.STOPPING {+ t.Fatalf("status = %q, want %q", reloaded.Status, common.STOPPING)}- return ids}func TestIngestionTaskDAODeleteIfTerminal_RemovesOnlyTerminal(t *testing.T) {@@ -236,7 +275,7 @@ func TestIngestionTaskDAODeleteIfTerminal_RemovesOnlyTerminal(t *testing.T) {t.Cleanup(func() { DB = orig })// Create tasks in different statuses, each with a unique docID.- statuses := []string{common.CREATED, common.RUNNING, common.STOPPING, common.COMPLETED, common.STOPPED, common.FAILED}+ statuses := []string{common.CREATED, common.SCHEDULED, common.RUNNING, common.STOPPING, common.COMPLETED, common.STOPPED, common.FAILED}for i, status := range statuses {docID := fmt.Sprintf("doc-%d", i)task := &entity.IngestionTask{@@ -253,7 +292,7 @@ func TestIngestionTaskDAODeleteIfTerminal_RemovesOnlyTerminal(t *testing.T) {ctx := t.Context()// DeleteIfTerminal deletes everything except RUNNING and STOPPING.- // CREATED is safe to delete (no worker has claimed it yet);+ // CREATED and SCHEDULED are safe to delete (no worker has claimed them yet);// COMPLETED/STOPPED/FAILED are terminal.// Call it for every doc and verify the negative cases survived.for i := 0; i < len(statuses); i++ {@@ -265,7 +304,7 @@ func TestIngestionTaskDAODeleteIfTerminal_RemovesOnlyTerminal(t *testing.T) {}// RUNNING and STOPPING must survive.- for _, i := range []int{1, 2} {+ for _, i := range []int{2, 3} {docID := fmt.Sprintf("doc-%d", i)task, err := NewIngestionTaskDAO().GetByDocumentID(ctx, db, docID)if err != nil {@@ -275,8 +314,8 @@ func TestIngestionTaskDAODeleteIfTerminal_RemovesOnlyTerminal(t *testing.T) {t.Fatalf("%s task (doc=%d) must not be deleted", statuses[i], i)}}- // CREATED, COMPLETED, STOPPED, FAILED must be gone.- for _, i := range []int{0, 3, 4, 5} {+ // CREATED, SCHEDULED, COMPLETED, STOPPED, FAILED must be gone.+ for _, i := range []int{0, 1, 4, 5, 6} {docID := fmt.Sprintf("doc-%d", i)task, err := NewIngestionTaskDAO().GetByDocumentID(ctx, db, docID)if err != nil {diff --git a/internal/engine/nats/nats.go b/internal/engine/nats/nats.goindex ae91d9faac..66700a67a2 100644--- a/internal/engine/nats/nats.go+++ b/internal/engine/nats/nats.go@@ -111,14 +111,14 @@ func (n *NatsEngine) PublishTask(subject string, payload []byte) error {defer cancel()// Deliberately NO MsgID/dedup here. Ingestion tasks reuse the same- // task_id across publish attempts (the FAILED/STOPPED→CREATED retry+ // task_id across publish attempts (the FAILED/STOPPED→CREATED→SCHEDULED retry// path), and the server's Duplicates window suppresses a republished// MsgID for its whole lifetime regardless of whether the original// message was already consumed and acked. A deduped retry publish- // strands the task in CREATED with no message behind it — unreachable- // by any consumer and un-reparsable ("already exists, status: CREATED").+ // strands the task in SCHEDULED with no message behind it — unreachable+ // by any consumer and un-reparsable ("already exists, status: SCHEDULED").// Duplicate delivery is instead made safe at the consumer level:- // StartRunning's CREATED→RUNNING CAS plus the in-process claim guard+ // StartRunning's CREATED/SCHEDULED→RUNNING CAS plus the in-process claim guard// ack-skip any second copy (see Ingestor.processMessage).ack, err := n.jetStream.Publish(ctx, subject, payload)if err != nil {diff --git a/internal/ingestion/service/ingestion_service.go b/internal/ingestion/service/ingestion_service.goindex acba81b7e0..d82def8cfd 100644--- a/internal/ingestion/service/ingestion_service.go+++ b/internal/ingestion/service/ingestion_service.go@@ -48,9 +48,8 @@ import (// comfortably below the consumer's ack deadline, which the server normalizes// to BackOff[0] = 5s (see NatsEngine.InitConsumer): a pulse slower than the// deadline lets the broker redeliver mid-run, and the redelivered copy is-// ack-skipped by the claim guard — acking the only in-flight copy, so a worker-// crash afterwards would lose the delivery (downgraded to the 15min-// reconciliation backstop instead of automatic redelivery).+// ack-skipped by the claim guard. If the worker or process stops, the broker+// can redeliver the unsettled message after the ack deadline.const defaultHeartbeatInterval = 2 * time.Secondtype Ingestor struct {@@ -186,6 +185,11 @@ func (e *Ingestor) start() error {if err := msgQueueEngine.InitConsumer(common.TaskSubject); err != nil {return err}+ if dao.DB != nil {+ if err := e.ingestionTaskSvc.ScheduleCreatedTasks(e.ctx); err != nil {+ common.Warn(fmt.Sprintf("schedule CREATED ingestion tasks at startup: %v", err))+ }+ }// Start the task worker pool and the dataset-level compile consumer as// owned goroutines joined by Stop via workerWg/compileWg. Start follows@@ -199,21 +203,6 @@ func (e *Ingestor) start() error {e.workerWg.Add(1)go e.consumeLoop()- // Startup reconciliation heals tasks orphaned by a previous process- // crash (RUNNING stuck mid-run, CREATED never delivered). It runs off- // the caller's goroutine so Start is not blocked by the DB scan; it is- // joined by Stop via workerWg. It runs concurrently with consumeLoop:- // the CAS in StartRunning (CREATED→RUNNING) and markFailed- // (RUNNING→FAILED) handles the race where a stale row is both- // reconciled and redelivered, and pagination handles >500 orphans- // without stranding tasks. Keeping it async avoids delaying consumer- // liveness; a synchronous scan before consumeLoop would add startup- // latency with no stronger correctness.- e.workerWg.Add(1)- go func() {- defer e.workerWg.Done()- e.reconcileStartupTasks()- }()return nil}@@ -296,84 +285,6 @@ func (e *Ingestor) SetKnowledgeCompileConcurrency(n int32) {e.kcConcurrency = n}-// Startup reconciliation thresholds: how long a CREATED task must sit-// without being picked up before the startup scan treats it as an orphan-// of a previous process (judged by create_time, never updated after creation).-const (- reconcileCreatedStaleAfter = 5 * time.Minute- reconcileBatchLimit = 500- reconcileOverallTimeout = 5 * time.Minute-)--// reconcileStartupTasks heals CREATED tasks orphaned by a previous process-// crash (never consumed for 5min) by re-publishing via EnqueueByID.-// Republishes are safe without broker-side dedup: the CREATED→RUNNING-// transition in StartRunning is a CAS, and the in-process claim guard-// ack-skips a second copy while a worker holds the task. (JetStream MsgID-// dedup is intentionally NOT used - see NatsEngine.PublishTask: it would-// swallow the retry republish of a task_id reused across runs, stranding-// the row in CREATED.)-//-// RUNNING orphans are intentionally NOT touched here; recovery relies on-// NATS redelivery (BackOff/InProgress) and the knowledge-compile style-// lease would be the proper fix for long tasks, not a startup 15m→FAILED.-// Errors are logged and skipped per task; the next restart re-runs the scan.-// reconcileStartupTasks scans and heals orphaned rows. It carries no-// WaitGroup bookkeeping of its own: start() wraps it with workerWg Add/Done,-// and tests invoke it synchronously.-func (e *Ingestor) reconcileStartupTasks() {- // Bounded by the ingestor's own lifetime: Stop cancels e.ctx, which- // aborts an in-flight scan instead of letting workerWg.Wait() linger.- ctx, cancel := context.WithTimeout(e.ctx, reconcileOverallTimeout)- defer cancel()-- if dao.DB == nil {- common.Warn("startup reconciliation skipped: DB not initialized")- return- }-- // CREATED orphans are re-published without a status change, so a- // drain-style loop (as used for RUNNING) would revisit the same head- // batch forever. Use offset pagination to walk the stale snapshot.- // The CREATED→RUNNING CAS in StartRunning and the in-process claim- // guard make concurrent races with consumeLoop safe: duplicate- // deliveries are ack-skipped while a worker holds the claim.- createdThreshold := time.Now().Add(-reconcileCreatedStaleAfter)- for offset := 0; ; offset += reconcileBatchLimit {- if ctx.Err() != nil {- return- }- // ListStaleByStatus does not support offset; query with offset- // directly to paginate the immutable CREATED snapshot.- var created []*entity.IngestionTask- q := dao.DB.WithContext(ctx).- Where("status IN ?", []string{common.CREATED}).- Where("create_time < ?", createdThreshold.UnixMilli()).- Order("create_time ASC").- Offset(offset).Limit(reconcileBatchLimit)- if err := q.Find(&created).Error; err != nil {- common.Warn(fmt.Sprintf("startup reconciliation: list CREATED orphans: %v", err))- return- }- if len(created) == 0 {- break- }- for _, task := range created {- if ctx.Err() != nil {- return- }- if err := e.ingestionTaskSvc.EnqueueByID(task.ID); err != nil {- common.Warn(fmt.Sprintf("startup reconciliation: re-enqueue CREATED task %s: %v", task.ID, err))- continue- }- common.Info(fmt.Sprintf("startup reconciliation: orphaned CREATED task %s re-enqueued", task.ID))- }- if len(created) < reconcileBatchLimit {- break- }- }-}-// startDatasetKnowledgeCompile provisions the dataset-level compile scheduling// store (MySQL knowledge_compile_docs + NATS notify subject), builds the// Consumer, and starts the owned compile-worker pool. The consumer is driven by@@ -569,9 +480,8 @@ func (e *Ingestor) processMessage(handle common.TaskHandle) {// Shutdown won the race after StartRunning already flipped the task// to RUNNING: finalize it as STOPPED so the row cannot linger in// non-terminal RUNNING with no in-flight worker (the user can- // retry). If the DB write itself fails, the startup reconciliation- // scan is the backstop. The message is left unsettled - on- // redelivery the terminal STOPPED status ack-skips it. Both paths+ // retry). The message is left unsettled - on redelivery the terminal+ // STOPPED status ack-skips it. Both paths// run with a detached 2s timeout so shutdown is not slowed by a// stalled DB.stopCtx, cancel := context.WithTimeout(context.Background(), 2*time.Second)@@ -579,7 +489,7 @@ func (e *Ingestor) processMessage(handle common.TaskHandle) {if e.markStopped(stopCtx, task.ID) {common.Info(fmt.Sprintf("Ingestor shutting down; task %s finalized as STOPPED", task.ID))} else {- common.Warn(fmt.Sprintf("Ingestor shutting down; task %s left RUNNING (startup reconciliation will finalize it)", task.ID))+ common.Warn(fmt.Sprintf("Ingestor shutting down; task %s left RUNNING for NATS redelivery", task.ID))}return}diff --git a/internal/ingestion/service/ingestor_lifecycle_test.go b/internal/ingestion/service/ingestor_lifecycle_test.goindex 01b65c1f3b..1594cae941 100644--- a/internal/ingestion/service/ingestor_lifecycle_test.go+++ b/internal/ingestion/service/ingestor_lifecycle_test.go@@ -18,6 +18,8 @@ package serviceimport ("context"+ "errors"+ "strings""testing""time"@@ -28,6 +30,15 @@ import ("ragflow/internal/ingestion/testutil")+type startupTaskPublisher struct {+ messages []common.TaskMessage+}++func (p *startupTaskPublisher) PublishTaskMessage(_ string, msg common.TaskMessage) error {+ p.messages = append(p.messages, msg)+ return nil+}+// TestStartWorkerPool_StartOnceIdempotent verifies that calling startWorkerPool// twice only starts maxConcurrency workers (sync.Once gate). It observes the// active worker count directly: a broken sync.Once would double the worker@@ -202,6 +213,61 @@ func TestPollCancel_ExitsWhenDoneClosed(t *testing.T) {close(released) // cleanup}+// TestStartNilEngine verifies that Start returns an error instead of panicking+// when the message queue engine has not been initialized.+func TestStartNilEngine(t *testing.T) {+ previousEngine := engine.GetMessageQueueEngine()+ engine.SetMessageQueueEngine(nil)+ t.Cleanup(func() { engine.SetMessageQueueEngine(previousEngine) })++ ingestor := newUnitIngestor("test-nil-engine", 1, []string{"pdf"})+ defer ingestor.Stop(context.Background())++ err := ingestor.Start()+ if err == nil || !strings.Contains(err.Error(), "not initialized") {+ t.Fatalf("Start() with nil engine: err = %v, want 'not initialized'", err)+ }+ if got := ingestor.activeWorkers.Load(); got != 0 {+ t.Fatalf("activeWorkers after failed Start = %d, want 0", got)+ }+}++// TestExecuteTask_MarkFailedAfterCtxCancelAcks verifies that a generic task+// failure is persisted and acknowledged even after the task context is+// cancelled by the pipeline.+func TestExecuteTask_MarkFailedAfterCtxCancelAcks(t *testing.T) {+ db := testutil.SetupTestDB(t)+ cleanup := testutil.ReplaceDBForTest(t, db)+ defer cleanup()+ _, _, docID, taskID := testutil.SeedTestData(t, db, testutil.WithPipelineID("flow-1"))++ ingestor := newUnitIngestor("test", 1, []string{"pdf"})+ parentCtx, parentCancel := context.WithCancel(context.Background())+ defer parentCancel()+ handle := &fakeTaskHandle{}+ taskCtx := taskpkg.NewTaskContextForScheduling(parentCtx, &entity.IngestionTask{+ ID: taskID, DocumentID: docID, DatasetID: "kb-1", Status: common.RUNNING,+ })+ taskCtx.Handle = handle+ ingestor.runDocumentTask = func(_ context.Context, _ *entity.IngestionTask) error {+ parentCancel()+ return errors.New("boom")+ }++ ingestor.executeTask(context.Background(), taskCtx)++ var task entity.IngestionTask+ if err := db.Where("id = ?", taskID).First(&task).Error; err != nil {+ t.Fatalf("reload task: %v", err)+ }+ if task.Status != common.FAILED {+ t.Fatalf("task status = %q, want %q", task.Status, common.FAILED)+ }+ if handle.acks.Load() != 1 || handle.nacks.Load() != 0 {+ t.Fatalf("expected 1 Ack/0 Nack, got acks=%d nacks=%d", handle.acks.Load(), handle.nacks.Load())+ }+}+// TestStart_FullPathReturnsAndStartsWorkers is a regression test for the// sync.Once re-entrancy deadlock. Before the fix, Start() wrapped the whole// startup (start()) in e.startOnce.Do, but start() also called startWorkerPool()@@ -247,3 +313,42 @@ func TestStart_FullPathReturnsAndStartsWorkers(t *testing.T) {t.Fatalf("expected activeWorkers > 0 after Start(), got %d", got)}}++func TestStartSchedulesCreatedTasks(t *testing.T) {+ db := testutil.SetupTestDB(t)+ cleanup := testutil.ReplaceDBForTest(t, db)+ defer cleanup()++ _, _, _, taskID := testutil.SeedTestData(t, db)+ if err := db.Model(&entity.IngestionTask{}).Where("id = ?", taskID).+ Update("status", common.CREATED).Error; err != nil {+ t.Fatalf("set task CREATED: %v", err)+ }++ previousEngine := engine.GetMessageQueueEngine()+ engine.SetMessageQueueEngine(testutil.SetupNatsEngine(t))+ t.Cleanup(func() { engine.SetMessageQueueEngine(previousEngine) })++ ingestor := newUnitIngestor("test-schedule-created", 1, nil)+ publisher := &startupTaskPublisher{}+ ingestor.ingestionTaskSvc.SetTaskPublisher(publisher)+ t.Cleanup(func() { ingestor.Stop(context.Background()) })++ if err := ingestor.Start(); err != nil {+ t.Fatalf("Start() returned error: %v", err)+ }+ if len(publisher.messages) != 1 {+ t.Fatalf("published messages = %d, want 1", len(publisher.messages))+ }+ if publisher.messages[0].TaskID != taskID {+ t.Fatalf("published task ID = %q, want %q", publisher.messages[0].TaskID, taskID)+ }++ var task entity.IngestionTask+ if err := db.Where("id = ?", taskID).First(&task).Error; err != nil {+ t.Fatalf("load scheduled task: %v", err)+ }+ if task.Status != common.SCHEDULED {+ t.Fatalf("task status = %q, want %q", task.Status, common.SCHEDULED)+ }+}diff --git a/internal/ingestion/service/ingestor_reconcile_test.go b/internal/ingestion/service/ingestor_reconcile_test.godeleted file mode 100644index bb31adb87f..0000000000--- a/internal/ingestion/service/ingestor_reconcile_test.go+++ /dev/null@@ -1,188 +0,0 @@-//-// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.-//-// Licensed under the Apache License, Version 2.0 (the "License");-// you may not use this file except in compliance with the License.-// You may obtain a copy of the License at-//-// http://www.apache.org/licenses/LICENSE-2.0-//-// Unless required by applicable law or agreed to in writing, software-// distributed under the License is distributed on an "AS IS" BASIS,-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.-// See the License for the specific language governing permissions and-// limitations under the License.-//--package service--import (- "context"- "errors"- "strings"- "sync"- "testing"- "time"-- "ragflow/internal/common"- "ragflow/internal/engine"- "ragflow/internal/entity"- taskpkg "ragflow/internal/ingestion/task"- "ragflow/internal/ingestion/testutil"-- "gorm.io/gorm"-)--// enqueueRecorder captures re-enqueue publishes so the reconciliation scan-// can be asserted without a live broker.-type enqueueRecorder struct {- mu sync.Mutex- taskIDs []string-}--func (p *enqueueRecorder) PublishTaskMessage(_ string, msg common.TaskMessage) error {- p.mu.Lock()- defer p.mu.Unlock()- p.taskIDs = append(p.taskIDs, msg.TaskID)- return nil-}--func (p *enqueueRecorder) published() []string {- p.mu.Lock()- defer p.mu.Unlock()- return append([]string(nil), p.taskIDs...)-}--// seedAgedTask inserts an ingestion task with explicit create/update-// timestamps aged back by the given duration (startup reconciliation judges-// staleness by these columns).-func seedAgedTask(t *testing.T, db *gorm.DB, id, docID, status string, age time.Duration) {- t.Helper()- ts := time.Now().Add(-age).UnixMilli()- if err := db.Create(&entity.IngestionTask{- ID: id,- UserID: "u1",- DocumentID: docID,- DatasetID: "kb-1",- Status: status,- BaseModel: entity.BaseModel{CreateTime: &ts, UpdateTime: &ts},- }).Error; err != nil {- t.Fatalf("seed task %s: %v", id, err)- }-}--// TestStartupReconciliation drives the startup scan: stale CREATED orphans-// are re-published for consumption, stale RUNNING orphans are intentionally-// left untouched (recovery relies on NATS redelivery, not startup FAILED;-// see ingestion_service.go reconcileStartupTasks), and fresh rows are left alone.-func TestStartupReconciliation(t *testing.T) {- db := testutil.SetupTestDB(t)- cleanup := testutil.ReplaceDBForTest(t, db)- defer cleanup()-- seedAgedTask(t, db, "task-run-stale", "doc-run-stale", common.RUNNING, 16*time.Minute)- seedAgedTask(t, db, "task-run-fresh", "doc-run-fresh", common.RUNNING, 1*time.Minute)- seedAgedTask(t, db, "task-created-stale", "doc-created-stale", common.CREATED, 6*time.Minute)- seedAgedTask(t, db, "task-created-fresh", "doc-created-fresh", common.CREATED, 1*time.Minute)- seedAgedTask(t, db, "task-completed-old", "doc-completed-old", common.COMPLETED, 1*time.Hour)-- ingestor := newUnitIngestor("test", 1, []string{"pdf"})- recorder := &enqueueRecorder{}- ingestor.ingestionTaskSvc.SetTaskPublisher(recorder)-- ingestor.reconcileStartupTasks()-- statusOf := func(id string) string {- t.Helper()- var task entity.IngestionTask- if err := db.Where("id = ?", id).First(&task).Error; err != nil {- t.Fatalf("reload task %s: %v", id, err)- }- return task.Status- }-- // RUNNING orphans are intentionally left untouched (NATS redelivery path).- if got := statusOf("task-run-stale"); got != common.RUNNING {- t.Fatalf("stale RUNNING task status = %q, want %q (should be left for NATS redelivery)", got, common.RUNNING)- }- // Fresh RUNNING row belongs to a live worker — untouched.- if got := statusOf("task-run-fresh"); got != common.RUNNING {- t.Fatalf("fresh RUNNING task status = %q, want %q", got, common.RUNNING)- }- // CREATED orphans are only re-published; their status stays CREATED so- // the normal consume path (StartRunning) owns the transition.- if got := statusOf("task-created-stale"); got != common.CREATED {- t.Fatalf("stale CREATED task status = %q, want %q", got, common.CREATED)- }- if got := statusOf("task-created-fresh"); got != common.CREATED {- t.Fatalf("fresh CREATED task status = %q, want %q", got, common.CREATED)- }-- // Exactly the stale CREATED task was re-enqueued — fresh CREATED rows- // still have their original message in flight, and terminal rows are- // never resurrected by the scan.- published := recorder.published()- if len(published) != 1 || published[0] != "task-created-stale" {- t.Fatalf("re-enqueued tasks = %v, want [task-created-stale]", published)- }-}--// TestStartNilEngine verifies that Start on an ingestor whose message queue-// engine was never initialized returns an error instead of panicking on the-// nil engine (mirror of the nats package uninitialized-engine contract).-func TestStartNilEngine(t *testing.T) {- previousEngine := engine.GetMessageQueueEngine()- engine.SetMessageQueueEngine(nil)- t.Cleanup(func() { engine.SetMessageQueueEngine(previousEngine) })-- ingestor := newUnitIngestor("test-nil-engine", 1, []string{"pdf"})- defer ingestor.Stop(context.Background())-- err := ingestor.Start()- if err == nil || !strings.Contains(err.Error(), "not initialized") {- t.Fatalf("Start() with nil engine: err = %v, want 'not initialized'", err)- }- // The failure must happen before the worker pool is provisioned.- if got := ingestor.activeWorkers.Load(); got != 0 {- t.Fatalf("activeWorkers after failed Start = %d, want 0", got)- }-}--// TestExecuteTask_MarkFailedAfterCtxCancelAcks verifies the ctx de-contamination-// of the failure path: the pipeline cancels the task context and then fails with-// a generic error. The FAILED write must still land (detached short timeout) and-// the message must be Acked (terminal), not Nack-requeued against a dead run.-func TestExecuteTask_MarkFailedAfterCtxCancelAcks(t *testing.T) {- db := testutil.SetupTestDB(t)- cleanup := testutil.ReplaceDBForTest(t, db)- defer cleanup()- _, _, docID, taskID := testutil.SeedTestData(t, db, testutil.WithPipelineID("flow-1"))-- ingestor := newUnitIngestor("test", 1, []string{"pdf"})-- parentCtx, parentCancel := context.WithCancel(context.Background())- defer parentCancel()- handle := &fakeTaskHandle{}- taskCtx := taskpkg.NewTaskContextForScheduling(parentCtx, &entity.IngestionTask{- ID: taskID, DocumentID: docID, DatasetID: "kb-1", Status: common.RUNNING,- })- taskCtx.Handle = handle-- ingestor.runDocumentTask = func(_ context.Context, _ *entity.IngestionTask) error {- parentCancel() // cancel races the pipeline...- return errors.New("boom") // ...and a generic failure lands afterwards- }-- ingestor.executeTask(context.Background(), taskCtx)-- var task entity.IngestionTask- if err := db.Where("id = ?", taskID).First(&task).Error; err != nil {- t.Fatalf("reload task: %v", err)- }- if task.Status != common.FAILED {- t.Fatalf("task status = %q, want %q (FAILED write must survive a cancelled ctx)", task.Status, common.FAILED)- }- if handle.acks.Load() != 1 || handle.nacks.Load() != 0 {- t.Fatalf("expected 1 Ack/0 Nack (terminal), got acks=%d nacks=%d", handle.acks.Load(), handle.nacks.Load())- }-}diff --git a/internal/ingestion/service/process_message_test.go b/internal/ingestion/service/process_message_test.goindex 3e9441461e..e80f462c49 100644--- a/internal/ingestion/service/process_message_test.go+++ b/internal/ingestion/service/process_message_test.go@@ -206,6 +206,39 @@ func TestProcessMessage_TaskNotFoundAcks(t *testing.T) {}}+// TestProcessMessage_CreatedTaskStartsRunning verifies that a worker can claim+// a task after NATS accepts its message but before the API records SCHEDULED.+func TestProcessMessage_CreatedTaskStartsRunning(t *testing.T) {+ db := testutil.SetupTestDB(t)+ cleanup := testutil.ReplaceDBForTest(t, db)+ defer cleanup()+ _, _, _, taskID := testutil.SeedTestData(t, db, testutil.WithPipelineID("flow-1"))++ if err := db.Model(&entity.IngestionTask{}).Where("id = ?", taskID).+ Update("status", common.CREATED).Error; err != nil {+ t.Fatalf("reset task to CREATED: %v", err)+ }++ ingestor := newUnitIngestor("test", 1, []string{"pdf"})+ handle := newFakeHandle(taskID, common.TaskTypeIngestionTask)++ ingestor.processMessage(handle)+ if handle.acks.Load() != 0 || handle.nacks.Load() != 0 {+ t.Fatalf("CREATED task: expected 0 Ack/0 Nack (deferred), got acks=%d nacks=%d", handle.acks.Load(), handle.nacks.Load())+ }+ if len(ingestor.taskChan) != 1 {+ t.Fatalf("expected 1 task enqueued, got %d", len(ingestor.taskChan))+ }++ var task entity.IngestionTask+ if err := db.Where("id = ?", taskID).First(&task).Error; err != nil {+ t.Fatalf("reload task: %v", err)+ }+ if task.Status != common.RUNNING {+ t.Fatalf("status = %q, want %q", task.Status, common.RUNNING)+ }+}+// TestProcessMessage_AlreadyCompletedAcks: a task already in a terminal state// (COMPLETED) is acked and skipped — no enqueue, and the document is NOT// resurrected to RUNNING. A redelivered terminal task must not reset adiff --git a/internal/service/document/document_ingest.go b/internal/service/document/document_ingest.goindex acbdc24a35..5a969bc426 100644--- a/internal/service/document/document_ingest.go+++ b/internal/service/document/document_ingest.go@@ -85,7 +85,8 @@ func (s *DocumentService) Ingest(ctx context.Context, userID string, req *Ingest// Start parsing: delegates to the shared start-parse flow. The// document run status is set by service.IngestionTaskService.StartRunning- // when the task transitions from CREATED, not here.+ // when the task transitions from CREATED or SCHEDULED,+ // not here.if run == string(entity.TaskStatusRunning) {if err = s.StartParseDocuments(ctx, doc, kb, userID, StartParseOptions{ApplyKB: req.ApplyKB,diff --git a/internal/service/document/document_parse.go b/internal/service/document/document_parse.goindex 4c42b1d27b..ce1c7f8da7 100644--- a/internal/service/document/document_parse.go+++ b/internal/service/document/document_parse.go@@ -7,6 +7,7 @@ import ("ragflow/internal/service""strconv""strings"+ "sync""ragflow/internal/common""ragflow/internal/dao"@@ -18,12 +19,45 @@ import ("gorm.io/gorm/clause")+type documentParseLock struct {+ mu sync.Mutex+ refs int+}++var documentParseLocks = struct {+ sync.Mutex+ locks map[string]*documentParseLock+}{locks: make(map[string]*documentParseLock)}++func lockDocumentParse(docID string) func() {+ documentParseLocks.Lock()+ lock := documentParseLocks.locks[docID]+ if lock == nil {+ lock = &documentParseLock{}+ documentParseLocks.locks[docID] = lock+ }+ lock.refs+++ documentParseLocks.Unlock()++ lock.mu.Lock()+ return func() {+ lock.mu.Unlock()+ documentParseLocks.Lock()+ lock.refs--+ if lock.refs == 0 {+ delete(documentParseLocks.locks, docID)+ }+ documentParseLocks.Unlock()+ }+}+// StartParseDocuments starts parsing a document via the DSL ingestion// pipeline. It optionally clears prior results (RerunWithDelete), applies// KB config (ApplyKB), validates storage, and enqueues an ingestion task.// The document run status is NOT set here; service.IngestionTaskService.StartRunning// sets it to RUNNING when the worker picks up the task and transitions it from-// CREATED. Extracted from Ingest so other entry points (e.g. ChunkService.Parse)+// CREATED or SCHEDULED. Extracted from Ingest so+// other entry points (e.g. ChunkService.Parse)// can reuse the same start-parse flow.func (s *DocumentService) StartParseDocuments(ctx context.Context, doc *entity.Document, kb *entity.Knowledgebase, userID string, opts StartParseOptions) error {// Validate storage first so we don't clear prior results and then fail@@ -32,6 +66,8 @@ func (s *DocumentService) StartParseDocuments(ctx context.Context, doc *entity.Dif _, _, err := s.GetDocumentStorageAddress(ctx, doc); err != nil {return err}+ unlock := lockDocumentParse(doc.ID)+ defer unlock()if opts.RerunWithDelete {if err := s.clearDocumentParseResults(ctx, doc, kb.TenantID); err != nil {@@ -80,20 +116,31 @@ func (s *DocumentService) clearDocumentParseResults(ctx context.Context, doc *en// Refuse to clear a non-terminal ingestion task. An in-flight worker// (RUNNING) or one mid-stop (STOPPING) would keep writing chunks and// corrupt the new run's results. The caller must stop the task first- // and wait for a terminal state (COMPLETED/STOPPED/FAILED) or CREATED.- if task, _ := s.ingestionTaskDAO.GetByDocumentID(ctx, dao.DB, doc.ID); task != nil {+ // and wait for a terminal state (COMPLETED/STOPPED/FAILED), CREATED, or+ // SCHEDULED.+ task, err := s.ingestionTaskDAO.GetByDocumentID(ctx, dao.DB, doc.ID)+ if err != nil {+ return fmt.Errorf("get ingestion task for document %s: %w", doc.ID, err)+ }+ taskExisted := task != nil+ if task != nil {if task.Status == common.RUNNING || task.Status == common.STOPPING {return fmt.Errorf("document %s ingestion task is %s; stop it and wait for a terminal state before re-parsing", doc.ID, task.Status)}+ taskExisted = true}- // Delete terminal and CREATED ingestion tasks atomically, leaving+ // Delete terminal, CREATED, and SCHEDULED ingestion tasks atomically, leaving// RUNNING/STOPPING tasks untouched so the check-then-delete window- // between GetByDocumentID and the delete above cannot delete a task- // that just transitioned to RUNNING.- if _, err := s.ingestionTaskDAO.DeleteIfTerminal(ctx, dao.DB, doc.ID); err != nil {+ // between GetByDocumentID and the delete cannot delete a task that just+ // transitioned to RUNNING. In that case, do not clear its parse results.+ deleted, err := s.ingestionTaskDAO.DeleteIfTerminal(ctx, dao.DB, doc.ID)+ if err != nil {return err}+ if taskExisted && deleted == 0 {+ return fmt.Errorf("document %s ingestion task started running; stop it before re-parsing", doc.ID)+ }if err := s.clearDocumentAndKBCountersForRerun(doc.ID, doc.KbID); err != nil {return err@@ -345,7 +392,7 @@ var errParseNotRunning = errors.New("parse task is not in running status")// RequestStop (STOPPING), then marks the document run status as CANCEL.// It mirrors the Python cancel precondition: only a document whose run status// is RUNNING or CANCEL, or one with an in-flight ingestion task-// (CREATED/RUNNING/STOPPING), can be canceled; otherwise errParseNotRunning+// (CREATED/SCHEDULED/RUNNING/STOPPING), can be canceled; otherwise errParseNotRunning// is returned. A missing ingestion task is not an error by itself — cancel is// then a no-op on the task side, matching Python's cancel_all_task_of.func (s *DocumentService) CancelDocParse(ctx context.Context, doc *entity.Document) error {@@ -358,7 +405,7 @@ func (s *DocumentService) CancelDocParse(ctx context.Context, doc *entity.Documeif doc.Run != nil {docRun = *doc.Run}- inFlight := task != nil && (task.Status == common.CREATED || task.Status == common.RUNNING || task.Status == common.STOPPING)+ inFlight := task != nil && (task.Status == common.CREATED || task.Status == common.SCHEDULED || task.Status == common.RUNNING || task.Status == common.STOPPING)if docRun != string(entity.TaskStatusRunning) && docRun != string(entity.TaskStatusCancel) && !inFlight {return errParseNotRunning}diff --git a/internal/service/document/document_test.go b/internal/service/document/document_test.goindex bc161f2b9a..d97a7dd1b6 100644--- a/internal/service/document/document_test.go+++ b/internal/service/document/document_test.go@@ -28,6 +28,7 @@ import ("path/filepath""reflect""strings"+ "sync""testing""time"@@ -51,6 +52,18 @@ type recordingTaskPublisher struct {messages []common.TaskMessage}+type runningTaskPublisher struct {+ db *gorm.DB+ published chan struct{}+ once sync.Once+}++func (p *runningTaskPublisher) PublishTaskMessage(_ string, msg common.TaskMessage) error {+ p.once.Do(func() { close(p.published) })+ return p.db.Model(&entity.IngestionTask{}).Where("id = ?", msg.TaskID).+ Update("status", common.RUNNING).Error+}+func (r *recordingTaskPublisher) PublishTaskMessage(subject string, msg common.TaskMessage) error {r.subject = subjectr.messages = append(r.messages, msg)@@ -422,8 +435,19 @@ func (f *failingDeleteMetadataEngine) UpdateMetadata(ctx context.Context, docID// setupServiceTestDB initializes an in-memory SQLite database for service tests.func setupServiceTestDB(t *testing.T) *gorm.DB {t.Helper()+ return setupServiceTestDBWithDSN(t, ":memory:")+}++func setupConcurrentServiceTestDB(t *testing.T) *gorm.DB {+ t.Helper()+ dsn := "file:" + strings.ReplaceAll(t.Name(), "/", "_") + "?mode=memory&cache=shared"+ return setupServiceTestDBWithDSN(t, dsn)+}++func setupServiceTestDBWithDSN(t *testing.T, dsn string) *gorm.DB {+ t.Helper()- db, err := gorm.Open(sqlite.Open(":memory:"), &gorm.Config{+ db, err := gorm.Open(sqlite.Open(dsn), &gorm.Config{TranslateError: true,})if err != nil {@@ -1266,8 +1290,8 @@ func TestStartParseDocuments_EnqueuesIngestionTask(t *testing.T) {if ingestionTask.UserID != "user-1" {t.Fatalf("ingestion task user id = %q, want %q", ingestionTask.UserID, "user-1")}- if ingestionTask.Status != common.CREATED {- t.Fatalf("ingestion task status = %q, want %q", ingestionTask.Status, common.CREATED)+ if ingestionTask.Status != common.SCHEDULED {+ t.Fatalf("ingestion task status = %q, want %q", ingestionTask.Status, common.SCHEDULED)}tasks, err := svc.taskDAO.GetByDocID(ctx, db, "doc-1")@@ -1279,6 +1303,76 @@ func TestStartParseDocuments_EnqueuesIngestionTask(t *testing.T) {}}+func TestStartParseDocumentsSerializesConcurrentReruns(t *testing.T) {+ db := setupConcurrentServiceTestDB(t)+ pushServiceDB(t, db)+ insertTestKB(t, "kb-1", "tenant-1", 0, 10, 5)+ insertTestDocWithRun(t, "doc-1", "kb-1", string(entity.TaskStatusDone), 10, 5)+ if err := db.Model(&entity.Document{}).Where("id = ?", "doc-1").Update("location", "loc-1").Error; err != nil {+ t.Fatalf("set document location: %v", err)+ }++ firstTaskLookup := make(chan struct{})+ releaseFirstTaskLookup := make(chan struct{})+ var blockFirstLookup sync.Once+ const callbackName = "test:block-first-ingestion-task-lookup"+ if err := db.Callback().Query().After("gorm:query").Register(callbackName, func(tx *gorm.DB) {+ if tx.Statement.Table != (&entity.IngestionTask{}).TableName() {+ return+ }+ blockFirstLookup.Do(func() {+ close(firstTaskLookup)+ <-releaseFirstTaskLookup+ })+ }); err != nil {+ t.Fatalf("register query callback: %v", err)+ }++ firstService := testDocumentService(t)+ firstService.ingestionTaskSvc.SetTaskPublisher(&recordingTaskPublisher{})+ secondPublisher := &runningTaskPublisher{db: db, published: make(chan struct{})}+ secondService := testDocumentService(t)+ secondService.ingestionTaskSvc.SetTaskPublisher(secondPublisher)+ ctx := t.Context()+ kb, err := firstService.kbDAO.GetByID(ctx, db, "kb-1")+ if err != nil {+ t.Fatalf("load knowledgebase: %v", err)+ }+ doc, err := firstService.documentDAO.GetByID(ctx, db, "doc-1")+ if err != nil {+ t.Fatalf("load document: %v", err)+ }++ firstDone := make(chan error, 1)+ go func() {+ firstDone <- firstService.StartParseDocuments(ctx, doc, kb, "user-1", StartParseOptions{RerunWithDelete: true})+ }()+ <-firstTaskLookup++ secondDone := make(chan error, 1)+ go func() {+ secondDone <- secondService.StartParseDocuments(ctx, doc, kb, "user-1", StartParseOptions{RerunWithDelete: true})+ }()++ secondPublishedBeforeFirstCompletes := false+ select {+ case <-secondPublisher.published:+ secondPublishedBeforeFirstCompletes = true+ case <-time.After(200 * time.Millisecond):+ }++ close(releaseFirstTaskLookup)+ if err := <-firstDone; err != nil {+ t.Fatalf("first concurrent reparse: %v", err)+ }+ if err := <-secondDone; err != nil {+ t.Fatalf("second concurrent reparse: %v", err)+ }+ if secondPublishedBeforeFirstCompletes {+ t.Fatal("second reparse published a task before the first reparse completed cleanup")+ }+}+func TestStopParseDocuments_Success(t *testing.T) {db := setupServiceTestDB(t)pushServiceDB(t, db)@@ -2060,6 +2154,58 @@ func TestClearDocumentParseResults_RejectsNonTerminalIngestionTask(t *testing.T)}}+func TestClearDocumentParseResultsDoesNotClearResultsWhenTaskStartsRunning(t *testing.T) {+ db := setupServiceTestDB(t)+ pushServiceDB(t, db)+ insertTestKB(t, "kb-1", "tenant-1", 1, 10, 5)+ insertTestDocWithRun(t, "doc-1", "kb-1", string(entity.TaskStatusDone), 10, 5)+ insertTestIngestionTaskWithStatus(t, "task-1", "user-1", "doc-1", "kb-1", common.SCHEDULED)++ const callbackName = "test:start-ingestion-task-before-conditional-delete"+ if err := db.Callback().Delete().Before("gorm:delete").Register(callbackName, func(tx *gorm.DB) {+ if tx.Statement.Table != (&entity.IngestionTask{}).TableName() {+ return+ }+ if err := tx.Session(&gorm.Session{NewDB: true}).Model(&entity.IngestionTask{}).+ Where("id = ?", "task-1").Update("status", common.RUNNING).Error; err != nil {+ _ = tx.AddError(err)+ }+ }); err != nil {+ t.Fatalf("register delete callback: %v", err)+ }++ ctx := t.Context()+ doc, err := dao.NewDocumentDAO().GetByID(ctx, db, "doc-1")+ if err != nil {+ t.Fatalf("get document: %v", err)+ }+ if err = testDocumentService(t).clearDocumentParseResults(ctx, doc, "tenant-1"); err == nil || !strings.Contains(err.Error(), "started running") {+ t.Fatalf("expected reparse cleanup to reject a task that started running, got %v", err)+ }+ task, err := dao.NewIngestionTaskDAO().GetByID(ctx, db, "task-1")+ if err != nil {+ t.Fatalf("reload task: %v", err)+ }+ if task.Status != common.RUNNING {+ t.Fatalf("task status = %q, want %q", task.Status, common.RUNNING)+ }++ updatedDoc, err := dao.NewDocumentDAO().GetByID(ctx, db, "doc-1")+ if err != nil {+ t.Fatalf("reload document: %v", err)+ }+ if updatedDoc.TokenNum != 10 || updatedDoc.ChunkNum != 5 {+ t.Fatalf("document counters = token:%d chunk:%d, want unchanged 10/5", updatedDoc.TokenNum, updatedDoc.ChunkNum)+ }+ kb, err := dao.NewKnowledgebaseDAO().GetByID(ctx, db, "kb-1")+ if err != nil {+ t.Fatalf("reload knowledgebase: %v", err)+ }+ if kb.TokenNum != 10 || kb.ChunkNum != 5 {+ t.Fatalf("knowledgebase counters = token:%d chunk:%d, want unchanged 10/5", kb.TokenNum, kb.ChunkNum)+ }+}+// TestClearDocumentParseResults_DeletesTerminalIngestionTask verifies// that a terminal ingestion task (STOPPED/COMPLETED/FAILED) and a CREATED// (queued) task are deleted so the new run can proceed.diff --git a/internal/service/ingestion_task_service.go b/internal/service/ingestion_task_service.goindex c037d59ed6..87e7b58017 100644--- a/internal/service/ingestion_task_service.go+++ b/internal/service/ingestion_task_service.go@@ -17,9 +17,7 @@ import (// Run-count key for IngestionTaskLog.Checkpoint, consumed by// ListAllForAdmin and IncrementRunCount to track how many times// the task has been picked up by a worker.-const (- stepKeyRunCount = "run_count"-)+const stepKeyRunCount = "run_count"type InvalidTaskTransitionError struct {TaskID string@@ -206,7 +204,7 @@ func (s *IngestionTaskService) StartRunning(ctx context.Context, taskID string)return nil, err}switch task.Status {- case common.CREATED:+ case common.CREATED, common.SCHEDULED:task, err = s.transition(ctx, taskID, common.RUNNING)if err != nil {return nil, err@@ -252,7 +250,7 @@ func (s *IngestionTaskService) RequestStop(ctx context.Context, taskID string) (return nil, err}switch task.Status {- case common.CREATED:+ case common.CREATED, common.SCHEDULED:return s.transition(ctx, taskID, common.STOPPED)case common.RUNNING:task, err = s.transition(ctx, taskID, common.STOPPING)@@ -328,6 +326,10 @@ func (s *IngestionTaskService) GetTask(ctx context.Context, taskID string) (*entfunc validateTransition(from, to string) error {switch from {case common.CREATED:+ if to == common.SCHEDULED || to == common.RUNNING || to == common.STOPPED {+ return nil+ }+ case common.SCHEDULED:if to == common.RUNNING || to == common.STOPPED {return nil}@@ -390,6 +392,11 @@ func (s *IngestionTaskService) CreateAndEnqueue(ctx context.Context, task *entit}if existing != nil {switch existing.Status {+ case common.CREATED:+ if err = s.enqueueTask(existing.ID); err != nil {+ return nil, err+ }+ return s.markScheduledAfterPublish(ctx, existing.ID)case common.FAILED, common.STOPPED:originalStatus := existing.Statusexisting, err = s.transition(ctx, existing.ID, common.CREATED)@@ -407,11 +414,12 @@ func (s *IngestionTaskService) CreateAndEnqueue(ctx context.Context, task *entit}return nil, err}- return existing, nil+ return s.markScheduledAfterPublish(ctx, existing.ID)default:return nil, fmt.Errorf("document id %s already exists, status: %s, task id: %s", task.DocumentID, existing.Status, existing.ID)}}+ task.Status = common.CREATEDcreated, err := s.ingestionTaskDAO.Create(ctx, dao.DB, task)if err != nil {return nil, err@@ -422,7 +430,7 @@ func (s *IngestionTaskService) CreateAndEnqueue(ctx context.Context, task *entit}return nil, err}- return created, nil+ return s.markScheduledAfterPublish(ctx, created.ID)}func (s *IngestionTaskService) rollbackRetriedTask(ctx context.Context, taskID, status string) error {@@ -441,6 +449,52 @@ func (s *IngestionTaskService) rollbackCreatedTask(ctx context.Context, taskID sreturn err}+// markScheduledAfterPublish records a successful NATS publish. A worker can+// claim the CREATED task before this write and move it to RUNNING, which is+// also a successful outcome.+func (s *IngestionTaskService) markScheduledAfterPublish(ctx context.Context, taskID string) (*entity.IngestionTask, error) {+ updated, err := s.ingestionTaskDAO.UpdateStatusIfCurrent(ctx, dao.DB, taskID, common.CREATED, common.SCHEDULED)+ if err != nil {+ return nil, err+ }+ if updated {+ return s.GetTask(ctx, taskID)+ }++ task, err := s.GetTask(ctx, taskID)+ if err != nil {+ return nil, err+ }+ switch task.Status {+ case common.SCHEDULED, common.RUNNING, common.STOPPING, common.COMPLETED, common.FAILED, common.STOPPED:+ return task, nil+ default:+ return nil, s.newTaskStatusConflictError(ctx, taskID, common.CREATED, common.SCHEDULED)+ }+}++// ScheduleCreatedTasks publishes the tasks that were persisted before a+// process stopped but were not confirmed as scheduled. It is intended for the+// single startup recovery pass; a publish error leaves the task CREATED for a+// future startup or explicit parse request to retry.+func (s *IngestionTaskService) ScheduleCreatedTasks(ctx context.Context) error {+ tasks, err := s.ingestionTaskDAO.ListByStatus(ctx, dao.DB, common.CREATED)+ if err != nil {+ return err+ }+ var recoveryErr error+ for _, task := range tasks {+ if err := s.enqueueTask(task.ID); err != nil {+ recoveryErr = errors.Join(recoveryErr, fmt.Errorf("schedule created task %s: %w", task.ID, err))+ continue+ }+ if _, err := s.markScheduledAfterPublish(ctx, task.ID); err != nil {+ recoveryErr = errors.Join(recoveryErr, fmt.Errorf("mark task %s scheduled: %w", task.ID, err))+ }+ }+ return recoveryErr+}+// clearCancelFlag removes the Redis cancel marker ({task_id}-cancel) that// RequestStop sets for a RUNNING task. No-op when Redis is unavailable —// the DB STOPPING status remains the fallback cancel signal.@@ -458,15 +512,6 @@ func (s *IngestionTaskService) enqueueTask(taskID string) error {return s.taskPublisher.PublishTaskMessage("tasks.RAGFLOW", taskMessage)}-// EnqueueByID re-publishes the wake-up message for an existing task without-// touching its status. Used by ingestor startup reconciliation to redeliver-// CREATED orphans; duplicate delivery is made safe at the consumer level-// (StartRunning CAS + in-process claim guard), not by broker dedup — see-// NatsEngine.PublishTask for why publish-time MsgID dedup is harmful here.-func (s *IngestionTaskService) EnqueueByID(taskID string) error {- return s.enqueueTask(taskID)-}-// UpdateComponentTotal records the number of components in the task's DSL// graph - the authoritative denominator for progress percentage.func (s *IngestionTaskService) UpdateComponentTotal(ctx context.Context, taskID string, total int) error {diff --git a/internal/service/ingestion_task_service_test.go b/internal/service/ingestion_task_service_test.goindex e8a05d4ec6..0820046c16 100644--- a/internal/service/ingestion_task_service_test.go+++ b/internal/service/ingestion_task_service_test.go@@ -2,6 +2,8 @@ package serviceimport ("errors"+ "strings"+ "sync""testing""ragflow/internal/common"@@ -10,14 +12,56 @@ import ()type recordingTaskPublisher struct {- subject string+ subject string+ messages []common.TaskMessage+ err error+ beforeReturn func(taskID string)+}++type failFirstTaskPublisher struct {messages []common.TaskMessage- err error+}++type duplicatePublishRecorder struct {+ mu sync.Mutex+ messages []common.TaskMessage+ firstPublished chan struct{}+ secondPublished chan struct{}+ releaseFirst chan struct{}+ releaseSecond chan struct{}+}++func (p *failFirstTaskPublisher) PublishTaskMessage(_ string, msg common.TaskMessage) error {+ p.messages = append(p.messages, msg)+ if len(p.messages) == 1 {+ return errors.New("publish failed")+ }+ return nil+}++func (p *duplicatePublishRecorder) PublishTaskMessage(_ string, msg common.TaskMessage) error {+ p.mu.Lock()+ p.messages = append(p.messages, msg)+ publishNumber := len(p.messages)+ p.mu.Unlock()++ switch publishNumber {+ case 1:+ close(p.firstPublished)+ <-p.releaseFirst+ case 2:+ close(p.secondPublished)+ <-p.releaseSecond+ }+ return nil}func (p *recordingTaskPublisher) PublishTaskMessage(subject string, msg common.TaskMessage) error {p.subject = subjectp.messages = append(p.messages, msg)+ if p.beforeReturn != nil {+ p.beforeReturn(msg.TaskID)+ }return p.err}@@ -58,6 +102,95 @@ func TestIngestionTaskServiceCreateForDocumentsPublishesTaskMessages(t *testing.}}+func TestIngestionTaskServiceMarksTaskScheduledOnlyAfterPublish(t *testing.T) {+ db := setupServiceTestDB(t)+ pushServiceDB(t, db)+ insertTestKB(t, "kb-1", "tenant-1", 1, 0, 0)+ insertTestDoc(t, "doc-1", "kb-1", 0, 0)++ statusDuringPublish := ""+ publisher := &recordingTaskPublisher{+ beforeReturn: func(taskID string) {+ task, err := dao.NewIngestionTaskDAO().GetByID(t.Context(), db, taskID)+ if err != nil {+ t.Fatalf("load task during publish: %v", err)+ }+ statusDuringPublish = task.Status+ },+ }+ svc := NewIngestionTaskService()+ svc.taskPublisher = publisher++ responses, err := svc.CreateForDocuments(t.Context(), "kb-1", "user-1", []string{"doc-1"})+ if err != nil {+ t.Fatalf("CreateForDocuments failed: %v", err)+ }+ if statusDuringPublish != common.CREATED {+ t.Fatalf("status during publish = %q, want %q", statusDuringPublish, common.CREATED)+ }+ if len(responses) != 1 || !strings.HasPrefix(responses[0].Result, "task_id: ") {+ t.Fatalf("unexpected parse response: %+v", responses)+ }++ task, err := dao.NewIngestionTaskDAO().GetByDocumentID(t.Context(), db, "doc-1")+ if err != nil {+ t.Fatalf("reload task: %v", err)+ }+ if task.Status != common.SCHEDULED {+ t.Fatalf("status after publish = %q, want %q", task.Status, common.SCHEDULED)+ }+}++func TestIngestionTaskServiceAcceptsConsumerWinningPublishRace(t *testing.T) {+ db := setupServiceTestDB(t)+ pushServiceDB(t, db)+ insertTestKB(t, "kb-1", "tenant-1", 1, 0, 0)+ insertTestDoc(t, "doc-1", "kb-1", 0, 0)++ publisher := &recordingTaskPublisher{+ beforeReturn: func(taskID string) {+ if err := db.Model(&entity.IngestionTask{}).Where("id = ?", taskID).+ Update("status", common.RUNNING).Error; err != nil {+ t.Fatalf("simulate consumer start: %v", err)+ }+ },+ }+ svc := NewIngestionTaskService()+ svc.taskPublisher = publisher++ responses, err := svc.CreateForDocuments(t.Context(), "kb-1", "user-1", []string{"doc-1"})+ if err != nil {+ t.Fatalf("CreateForDocuments failed: %v", err)+ }+ if len(responses) != 1 || !strings.HasPrefix(responses[0].Result, "task_id: ") {+ t.Fatalf("unexpected parse response: %+v", responses)+ }++ task, err := dao.NewIngestionTaskDAO().GetByDocumentID(t.Context(), db, "doc-1")+ if err != nil {+ t.Fatalf("reload task: %v", err)+ }+ if task.Status != common.RUNNING {+ t.Fatalf("status after consumer race = %q, want %q", task.Status, common.RUNNING)+ }+}++func TestIngestionTaskServiceStartRunningTransitionsScheduledTask(t *testing.T) {+ db := setupServiceTestDB(t)+ pushServiceDB(t, db)+ insertTestKB(t, "kb-1", "tenant-1", 1, 0, 0)+ insertTestDoc(t, "doc-1", "kb-1", 0, 0)+ insertTestIngestionTaskWithStatus(t, "task-1", "user-1", "doc-1", "kb-1", common.SCHEDULED)++ task, err := NewIngestionTaskService().StartRunning(t.Context(), "task-1")+ if err != nil {+ t.Fatalf("StartRunning failed: %v", err)+ }+ if task.Status != common.RUNNING {+ t.Fatalf("status = %q, want %q", task.Status, common.RUNNING)+ }+}+func TestIngestionTaskServiceListByUserFiltersDataset(t *testing.T) {db := setupServiceTestDB(t)pushServiceDB(t, db)@@ -499,8 +632,8 @@ func TestIngestionTaskServiceCreateAndEnqueueRetriesTerminalTask(t *testing.T) {if task.ID != "task-1" {t.Fatalf("task ID = %q, want task-1", task.ID)}- if task.Status != common.CREATED {- t.Fatalf("status = %q, want %q", task.Status, common.CREATED)+ if task.Status != common.SCHEDULED {+ t.Fatalf("status = %q, want %q", task.Status, common.SCHEDULED)}if len(publisher.messages) != 1 || publisher.messages[0].TaskID != "task-1" {t.Fatalf("unexpected published messages: %+v", publisher.messages)@@ -509,17 +642,178 @@ func TestIngestionTaskServiceCreateAndEnqueueRetriesTerminalTask(t *testing.T) {if err != nil {t.Fatalf("reload task: %v", err)}- if reloaded.Status != common.CREATED {- t.Fatalf("reloaded status = %q, want %q", reloaded.Status, common.CREATED)+ if reloaded.Status != common.SCHEDULED {+ t.Fatalf("reloaded status = %q, want %q", reloaded.Status, common.SCHEDULED)}})}}+func TestIngestionTaskServiceCreateAndEnqueueSchedulesExistingCreatedTask(t *testing.T) {+ db := setupServiceTestDB(t)+ pushServiceDB(t, db)+ insertTestIngestionTaskWithStatus(t, "stale-task", "user-1", "doc-1", "kb-1", common.CREATED)++ publisher := &recordingTaskPublisher{}+ svc := NewIngestionTaskService()+ svc.taskPublisher = publisher++ task, err := svc.CreateAndEnqueue(t.Context(), &entity.IngestionTask{+ DocumentID: "doc-1",+ UserID: "user-1",+ DatasetID: "kb-1",+ Status: common.CREATED,+ })+ if err != nil {+ t.Fatalf("CreateAndEnqueue failed: %v", err)+ }+ if task.ID != "stale-task" {+ t.Fatalf("task ID = %q, want stale-task", task.ID)+ }+ if task.Status != common.SCHEDULED {+ t.Fatalf("status = %q, want %q", task.Status, common.SCHEDULED)+ }+ if len(publisher.messages) != 1 || publisher.messages[0].TaskID != task.ID {+ t.Fatalf("unexpected published messages: %+v", publisher.messages)+ }++ reloaded, err := dao.NewIngestionTaskDAO().GetByDocumentID(t.Context(), db, "doc-1")+ if err != nil {+ t.Fatalf("load task after scheduling created task: %v", err)+ }+ if reloaded == nil || reloaded.ID != "stale-task" || reloaded.Status != common.SCHEDULED {+ t.Fatalf("expected created task to be scheduled in place, task=%+v", reloaded)+ }+}++func TestIngestionTaskServiceConcurrentCreatedTaskPublicationsConverge(t *testing.T) {+ db := setupServiceTestDB(t)+ pushServiceDB(t, db)+ insertTestIngestionTaskWithStatus(t, "task-1", "user-1", "doc-1", "kb-1", common.CREATED)++ publisher := &duplicatePublishRecorder{+ firstPublished: make(chan struct{}),+ secondPublished: make(chan struct{}),+ releaseFirst: make(chan struct{}),+ releaseSecond: make(chan struct{}),+ }+ firstService := NewIngestionTaskService()+ firstService.taskPublisher = publisher+ secondService := NewIngestionTaskService()+ secondService.taskPublisher = publisher++ type result struct {+ task *entity.IngestionTask+ err error+ }+ results := make(chan result, 2)+ enqueue := func(svc *IngestionTaskService) {+ task, err := svc.CreateAndEnqueue(t.Context(), &entity.IngestionTask{+ DocumentID: "doc-1",+ UserID: "user-1",+ DatasetID: "kb-1",+ Status: common.CREATED,+ })+ results <- result{task: task, err: err}+ }++ go enqueue(firstService)+ <-publisher.firstPublished+ go enqueue(secondService)+ <-publisher.secondPublished++ close(publisher.releaseFirst)+ first := <-results+ if first.err != nil {+ t.Fatalf("first CreateAndEnqueue: %v", first.err)+ }+ close(publisher.releaseSecond)+ second := <-results+ if second.err != nil {+ t.Fatalf("second CreateAndEnqueue: %v", second.err)+ }+ if first.task.ID != "task-1" || second.task.ID != "task-1" {+ t.Fatalf("concurrent requests returned task IDs %q and %q, want task-1", first.task.ID, second.task.ID)+ }++ publisher.mu.Lock()+ messages := append([]common.TaskMessage(nil), publisher.messages...)+ publisher.mu.Unlock()+ if len(messages) != 2 || messages[0].TaskID != "task-1" || messages[1].TaskID != "task-1" {+ t.Fatalf("published messages = %+v, want two messages for task-1", messages)+ }++ task, err := dao.NewIngestionTaskDAO().GetByID(t.Context(), db, "task-1")+ if err != nil {+ t.Fatalf("reload task: %v", err)+ }+ if task.Status != common.SCHEDULED {+ t.Fatalf("task status after concurrent publication = %q, want %q", task.Status, common.SCHEDULED)+ }+}++func TestIngestionTaskServiceScheduleCreatedTasksKeepsTaskCreatedOnPublishFailure(t *testing.T) {+ db := setupServiceTestDB(t)+ pushServiceDB(t, db)+ insertTestIngestionTaskWithStatus(t, "task-1", "user-1", "doc-1", "kb-1", common.CREATED)++ svc := NewIngestionTaskService()+ svc.taskPublisher = &recordingTaskPublisher{err: errors.New("publish failed")}+ if err := svc.ScheduleCreatedTasks(t.Context()); err == nil {+ t.Fatal("expected ScheduleCreatedTasks to return publish error")+ }++ task, err := dao.NewIngestionTaskDAO().GetByID(t.Context(), db, "task-1")+ if err != nil {+ t.Fatalf("reload task: %v", err)+ }+ if task.Status != common.CREATED {+ t.Fatalf("status after failed recovery publish = %q, want %q", task.Status, common.CREATED)+ }+}++func TestIngestionTaskServiceScheduleCreatedTasksContinuesAfterPublishFailure(t *testing.T) {+ db := setupServiceTestDB(t)+ pushServiceDB(t, db)+ insertTestIngestionTaskWithStatus(t, "task-1", "user-1", "doc-1", "kb-1", common.CREATED)+ insertTestIngestionTaskWithStatus(t, "task-2", "user-1", "doc-2", "kb-1", common.CREATED)+ if err := db.Model(&entity.IngestionTask{}).Where("id = ?", "task-1").Update("create_time", 1).Error; err != nil {+ t.Fatalf("set first task create time: %v", err)+ }+ if err := db.Model(&entity.IngestionTask{}).Where("id = ?", "task-2").Update("create_time", 2).Error; err != nil {+ t.Fatalf("set second task create time: %v", err)+ }++ publisher := &failFirstTaskPublisher{}+ svc := NewIngestionTaskService()+ svc.taskPublisher = publisher+ if err := svc.ScheduleCreatedTasks(t.Context()); err == nil {+ t.Fatal("expected ScheduleCreatedTasks to return publish error")+ }+ if len(publisher.messages) != 2 {+ t.Fatalf("published messages = %d, want 2", len(publisher.messages))+ }++ first, err := dao.NewIngestionTaskDAO().GetByID(t.Context(), db, "task-1")+ if err != nil {+ t.Fatalf("reload first task: %v", err)+ }+ if first.Status != common.CREATED {+ t.Fatalf("first task status = %q, want %q", first.Status, common.CREATED)+ }+ second, err := dao.NewIngestionTaskDAO().GetByID(t.Context(), db, "task-2")+ if err != nil {+ t.Fatalf("reload second task: %v", err)+ }+ if second.Status != common.SCHEDULED {+ t.Fatalf("second task status = %q, want %q", second.Status, common.SCHEDULED)+ }+}+func TestIngestionTaskServiceCreateAndEnqueueRejectsActiveExistingTask(t *testing.T) {db := setupServiceTestDB(t)pushServiceDB(t, db)- insertTestIngestionTask(t, "task-1", "user-1", "doc-1", "kb-1")+ insertTestIngestionTaskWithStatus(t, "task-1", "user-1", "doc-1", "kb-1", common.SCHEDULED)publisher := &recordingTaskPublisher{}svc := NewIngestionTaskService()svc.taskPublisher = publisherdiff --git a/internal/service/ingestion_task_transition_test.go b/internal/service/ingestion_task_transition_test.goindex 2b05d0fdeb..43ac780b65 100644--- a/internal/service/ingestion_task_transition_test.go+++ b/internal/service/ingestion_task_transition_test.go@@ -12,8 +12,11 @@ func TestValidateTransitionAllowsExpectedEdges(t *testing.T) {from stringto string}{+ {from: common.CREATED, to: common.SCHEDULED},{from: common.CREATED, to: common.RUNNING},{from: common.CREATED, to: common.STOPPED},+ {from: common.SCHEDULED, to: common.RUNNING},+ {from: common.SCHEDULED, to: common.STOPPED},{from: common.RUNNING, to: common.STOPPING},{from: common.RUNNING, to: common.COMPLETED},{from: common.RUNNING, to: common.FAILED},