Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

fix(document): skip active and completed tasks on batch ingest (#19786)

jay77721132413d9a20e
diff --git a/internal/service/document/document_ingest.go b/internal/service/document/document_ingest.go
index 71e5b5c806..81da5d28b9 100644
--- a/internal/service/document/document_ingest.go
+++ b/internal/service/document/document_ingest.go
@@ -71,23 +71,44 @@ func (s *DocumentService) Ingest(ctx context.Context, userID string, req *Ingest
validatedIDs = append(validatedIDs, docID)
}
- // Batch pre-check for reparse with delete: use the validated doc IDs
- // so we don't silently skip non-existent or unauthorized documents.
- if run == string(entity.TaskStatusRunning) && req.Delete {
- if err = s.AssertIngestionTasksTerminal(ctx, validatedIDs); err != nil {
- return common.CodeDataError, err
+ // Start parsing: filter out in-flight documents (RUNNING, SCHEDULED,
+ // CREATED, STOPPING) so active parses continue undisturbed, and skip
+ // already COMPLETED documents when delete is false to avoid duplicate
+ // chunk errors. Only documents requiring a new parse run are started.
+ if run == string(entity.TaskStatusRunning) {
+ taskMap := make(map[string]*entity.IngestionTask, len(validatedIDs))
+ if s.ingestionTaskDAO != nil && len(validatedIDs) > 0 {
+ taskMap, err = s.ingestionTaskDAO.GetLatestByDocumentIDs(ctx, dao.DB, validatedIDs)
+ if err != nil {
+ return common.CodeExceptionError, fmt.Errorf("fail to get ingestion tasks: %w", err)
+ }
}
- }
- for _, vd := range validated {
- doc := vd.doc
- kb := vd.kb
+ toStart := make([]validatedDoc, 0, len(validated))
+ for _, vd := range validated {
+ task := taskMap[vd.doc.ID]
+ if task != nil && common.IsActiveTaskStatus(task.Status) {
+ common.Debug(fmt.Sprintf("skip document %s ingestion, active task status: %s", vd.doc.ID, task.Status))
+ continue
+ }
+ if !req.Delete && task != nil && task.Status == common.COMPLETED {
+ common.Debug(fmt.Sprintf("skip document %s ingestion, already completed and delete is false", vd.doc.ID))
+ continue
+ }
+ toStart = append(toStart, vd)
+ }
+
+ if skipped := len(validated) - len(toStart); skipped > 0 {
+ common.Info(fmt.Sprintf("batch ingest: requested %d, started %d, skipped %d", len(validated), len(toStart), skipped))
+ }
+
+ if len(toStart) == 0 {
+ return common.CodeSuccess, nil
+ }
- // Start parsing: delegates to the shared start-parse flow. The
- // document run status is set by service.IngestionTaskService.StartRunning
- // when the task transitions from CREATED or SCHEDULED,
- // not here.
- if run == string(entity.TaskStatusRunning) {
+ for _, vd := range toStart {
+ doc := vd.doc
+ kb := vd.kb
if err = s.StartParseDocuments(ctx, doc, kb, userID, StartParseOptions{
ApplyKB: req.ApplyKB,
RerunWithDelete: req.Delete,
@@ -95,8 +116,13 @@ func (s *DocumentService) Ingest(ctx context.Context, userID string, req *Ingest
common.Error(fmt.Sprintf("go side, doc %s, start parse", doc.ID), err)
return common.CodeExceptionError, err
}
- continue
}
+ return common.CodeSuccess, nil
+ }
+
+ for _, vd := range validated {
+ doc := vd.doc
+ kb := vd.kb
// Cancel: RequestStop (STOPPING) and update doc state. Do NOT
// delete the ingestion task or chunks here — deletion races with
diff --git a/internal/service/document/document_test.go b/internal/service/document/document_test.go
index c7000f8d96..f01b9b767c 100644
--- a/internal/service/document/document_test.go
+++ b/internal/service/document/document_test.go
@@ -2499,34 +2499,164 @@ func TestAssertIngestionTasksTerminal_AcceptsAllTerminal(t *testing.T) {
}
}
-// TestIngest_RerunWithDelete_RejectsBatchWithRunningTask verifies the batch
-// pre-check: when re-parsing with delete, if any document's ingestion task is
-// non-terminal, the whole request is rejected up front so no document is
-// partially cleaned.
-func TestIngest_RerunWithDelete_RejectsBatchWithRunningTask(t *testing.T) {
+// TestIngest_RunningTask_SkippedWhenBatchStartingParse verifies that when
+// starting parse on a batch of documents, in-flight active tasks (RUNNING,
+// CREATED, SCHEDULED, STOPPING) are skipped and leave existing tasks untouched,
+// while eligible documents are parsed.
+func TestIngest_RunningTask_SkippedWhenBatchStartingParse(t *testing.T) {
db := setupServiceTestDB(t)
pushServiceDB(t, db)
insertUserTenantForAccessCheck(t, "user-1", "tenant-1")
insertTestKB(t, "kb-1", "tenant-1", 0, 0, 0)
insertTestDoc(t, "doc-1", "kb-1", 0, 0)
insertTestDoc(t, "doc-2", "kb-1", 0, 0)
+ insertTestDoc(t, "doc-3", "kb-1", 0, 0)
+ if err := dao.DB.Model(&entity.Document{}).Where("id = ?", "doc-1").Update("location", "loc-1").Error; err != nil {
+ t.Fatalf("set location: %v", err)
+ }
+
insertTestIngestionTaskWithStatus(t, "task-1", "user-1", "doc-1", "kb-1", common.COMPLETED)
insertTestIngestionTaskWithStatus(t, "task-2", "user-1", "doc-2", "kb-1", common.RUNNING)
+ insertTestIngestionTaskWithStatus(t, "task-3", "user-1", "doc-3", "kb-1", common.CREATED)
+ publisher := &recordingTaskPublisher{}
+ svc := testDocumentService(t)
+ svc.ingestionTaskSvc.SetTaskPublisher(publisher)
ctx := t.Context()
+
+ code, err := svc.Ingest(ctx, "user-1", &IngestDocumentRequest{
+ DocIDs: []string{"doc-1", "doc-2", "doc-3"},
+ Run: string(entity.TaskStatusRunning),
+ Delete: true,
+ })
+ if err != nil {
+ t.Fatalf("Ingest failed: %v", err)
+ }
+ if code != common.CodeSuccess {
+ t.Fatalf("expected code %v, got %v", common.CodeSuccess, code)
+ }
+
+ // doc-2 (RUNNING) and doc-3 (CREATED) must be untouched
+ task2, _ := svc.ingestionTaskDAO.GetByDocumentID(ctx, db, "doc-2")
+ if task2 == nil || task2.Status != common.RUNNING {
+ t.Fatalf("doc-2 task status = %v, want %s", task2, common.RUNNING)
+ }
+ task3, _ := svc.ingestionTaskDAO.GetByDocumentID(ctx, db, "doc-3")
+ if task3 == nil || task3.Status != common.CREATED {
+ t.Fatalf("doc-3 task status = %v, want %s", task3, common.CREATED)
+ }
+
+ // Only doc-1 should have been re-enqueued
+ if len(publisher.messages) != 1 {
+ t.Fatalf("expected 1 published message for doc-1, got %d", len(publisher.messages))
+ }
+}
+
+func TestIngest_AllActiveTasks_ReturnsSuccessIdempotent(t *testing.T) {
+ db := setupServiceTestDB(t)
+ pushServiceDB(t, db)
+ insertUserTenantForAccessCheck(t, "user-1", "tenant-1")
+ insertTestKB(t, "kb-1", "tenant-1", 0, 0, 0)
+ insertTestDoc(t, "doc-1", "kb-1", 0, 0)
+ insertTestDoc(t, "doc-2", "kb-1", 0, 0)
+ insertTestIngestionTaskWithStatus(t, "task-1", "user-1", "doc-1", "kb-1", common.RUNNING)
+ insertTestIngestionTaskWithStatus(t, "task-2", "user-1", "doc-2", "kb-1", common.SCHEDULED)
+
+ publisher := &recordingTaskPublisher{}
svc := testDocumentService(t)
- _, err := svc.Ingest(ctx, "user-1", &IngestDocumentRequest{
+ svc.ingestionTaskSvc.SetTaskPublisher(publisher)
+ ctx := t.Context()
+
+ code, err := svc.Ingest(ctx, "user-1", &IngestDocumentRequest{
DocIDs: []string{"doc-1", "doc-2"},
Run: string(entity.TaskStatusRunning),
Delete: true,
})
- if err == nil {
- t.Fatal("expected error for batch with RUNNING task, got nil")
+ if err != nil {
+ t.Fatalf("Ingest failed: %v", err)
}
- // doc-1 (terminal) task must NOT be deleted - the whole batch was rejected.
+ if code != common.CodeSuccess {
+ t.Fatalf("expected code %v, got %v", common.CodeSuccess, code)
+ }
+ if len(publisher.messages) != 0 {
+ t.Fatalf("expected 0 published messages, got %d", len(publisher.messages))
+ }
+}
+
+func TestIngest_CompletedTask_SkippedWhenDeleteIsFalse(t *testing.T) {
+ db := setupServiceTestDB(t)
+ pushServiceDB(t, db)
+ insertUserTenantForAccessCheck(t, "user-1", "tenant-1")
+ insertTestKB(t, "kb-1", "tenant-1", 0, 0, 0)
+ insertTestDoc(t, "doc-1", "kb-1", 0, 0)
+ insertTestDoc(t, "doc-2", "kb-1", 0, 0)
+ if err := dao.DB.Model(&entity.Document{}).Where("id = ?", "doc-2").Update("location", "loc-2").Error; err != nil {
+ t.Fatalf("set location: %v", err)
+ }
+ insertTestIngestionTaskWithStatus(t, "task-1", "user-1", "doc-1", "kb-1", common.COMPLETED)
+
+ publisher := &recordingTaskPublisher{}
+ svc := testDocumentService(t)
+ svc.ingestionTaskSvc.SetTaskPublisher(publisher)
+ ctx := t.Context()
+
+ code, err := svc.Ingest(ctx, "user-1", &IngestDocumentRequest{
+ DocIDs: []string{"doc-1", "doc-2"},
+ Run: string(entity.TaskStatusRunning),
+ Delete: false,
+ })
+ if err != nil {
+ t.Fatalf("Ingest failed: %v", err)
+ }
+ if code != common.CodeSuccess {
+ t.Fatalf("expected code %v, got %v", common.CodeSuccess, code)
+ }
+
+ // doc-1 (COMPLETED with Delete=false) should remain COMPLETED and not fail
task1, _ := svc.ingestionTaskDAO.GetByDocumentID(ctx, db, "doc-1")
- if task1 == nil {
- t.Fatal("doc-1 terminal task should not be deleted when batch is rejected")
+ if task1 == nil || task1.Status != common.COMPLETED {
+ t.Fatalf("doc-1 task status = %v, want %s", task1, common.COMPLETED)
+ }
+
+ // doc-2 (unstarted) should have been started
+ if len(publisher.messages) != 1 {
+ t.Fatalf("expected 1 published message for doc-2, got %d", len(publisher.messages))
+ }
+}
+
+func TestIngest_FailedAndStoppedTasks_Restarted(t *testing.T) {
+ db := setupServiceTestDB(t)
+ pushServiceDB(t, db)
+ insertUserTenantForAccessCheck(t, "user-1", "tenant-1")
+ insertTestKB(t, "kb-1", "tenant-1", 0, 0, 0)
+ insertTestDoc(t, "doc-1", "kb-1", 0, 0)
+ insertTestDoc(t, "doc-2", "kb-1", 0, 0)
+ if err := dao.DB.Model(&entity.Document{}).Where("id IN ?", []string{"doc-1", "doc-2"}).Update("location", "loc").Error; err != nil {
+ t.Fatalf("set location: %v", err)
+ }
+ insertTestIngestionTaskWithStatus(t, "task-1", "user-1", "doc-1", "kb-1", common.FAILED)
+ insertTestIngestionTaskWithStatus(t, "task-2", "user-1", "doc-2", "kb-1", common.STOPPED)
+
+ publisher := &recordingTaskPublisher{}
+ svc := testDocumentService(t)
+ svc.ingestionTaskSvc.SetTaskPublisher(publisher)
+ ctx := t.Context()
+
+ code, err := svc.Ingest(ctx, "user-1", &IngestDocumentRequest{
+ DocIDs: []string{"doc-1", "doc-2"},
+ Run: string(entity.TaskStatusRunning),
+ Delete: false,
+ })
+ if err != nil {
+ t.Fatalf("Ingest failed: %v", err)
+ }
+ if code != common.CodeSuccess {
+ t.Fatalf("expected code %v, got %v", common.CodeSuccess, code)
+ }
+
+ // Both FAILED and STOPPED tasks should have been re-enqueued
+ if len(publisher.messages) != 2 {
+ t.Fatalf("expected 2 published messages, got %d", len(publisher.messages))
}
}