Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

fix(ingestion): allow re-parsing a document whose parse was canceled at progress 0 (#19258)

euvre347d0827a0fe
diff --git a/internal/ingestion/service/ingestion_service.go b/internal/ingestion/service/ingestion_service.go
index f82719fe94..921341f322 100644
--- a/internal/ingestion/service/ingestion_service.go
+++ b/internal/ingestion/service/ingestion_service.go
@@ -695,10 +695,31 @@ func (e *Ingestor) executeTaskWithHeartbeat(ctx context.Context, taskCtx *taskpk
})
}
-// markStopped transitions the task to STOPPED (terminal). It first calls
-// RequestStop to handle RUNNING → STOPPING, then MarkStopped for the final
-// STOPPING → STOPPED transition. Finally it cleans up the Redis cancel flag
-// so that a future retry of the same task does not immediately re-cancel.
+// ownsTaskRun reports whether the calling run still owns the task: a newer
+// run (created by a supersede/re-parse) bumps the run counter, so a mismatch
+// means this run's terminal write must be skipped. A run whose counter could
+// not be read (myRun <= 0) conservatively keeps ownership. Uses a detached
+// context: it is called from failure/cancel paths whose context is typically
+// already cancelled.
+func (e *Ingestor) ownsTaskRun(taskID string, myRun int) bool {
+ if myRun <= 0 {
+ return true
+ }
+ ctx, cancel := context.WithTimeout(context.WithoutCancel(e.ctx), 2*time.Second)
+ defer cancel()
+ current, ok := e.ingestionTaskSvc.CurrentRunCount(ctx, taskID)
+ if !ok {
+ return true
+ }
+ return current == myRun
+}
+
+// markStopped transitions the task to STOPPED (terminal). A RUNNING task
+// first goes through RequestStop (RUNNING → STOPPING); a STOPPING task is
+// settled directly; any other status (terminal, or CREATED/SCHEDULED because
+// a supersede re-armed the task for a newer run) is treated as settled with
+// nothing to finalize. Finally it cleans up the Redis cancel flag so that a
+// future retry of the same task does not immediately re-cancel.
// The caller's context is detached before touching the DB: markStopped runs
// on failure/cancel/shutdown paths whose context is typically already
// cancelled, and a contaminated context would make the terminal write fail
@@ -706,10 +727,29 @@ func (e *Ingestor) executeTaskWithHeartbeat(ctx context.Context, taskCtx *taskpk
func (e *Ingestor) markStopped(ctx context.Context, taskID string) bool {
ctx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 5*time.Second)
defer cancel()
- if _, err := e.ingestionTaskSvc.RequestStop(ctx, taskID); err != nil {
- common.Error(fmt.Sprintf("markStopped: RequestStop task %s: %v", taskID, err), err)
+ // Only a RUNNING/STOPPING task still belongs to this run. A task in
+ // CREATED/SCHEDULED here was re-armed for a newer run by a supersede
+ // (re-parse of a canceled document); finalizing it would kill the
+ // replacement run before it starts, so treat it as already settled.
+ // Terminal statuses need no work either.
+ task, err := e.ingestionTaskSvc.GetTask(ctx, taskID)
+ if err != nil {
+ common.Error(fmt.Sprintf("markStopped: load task %s: %v", taskID, err), err)
return false
}
+ switch task.Status {
+ case common.RUNNING:
+ if _, err := e.ingestionTaskSvc.RequestStop(ctx, taskID); err != nil {
+ common.Error(fmt.Sprintf("markStopped: RequestStop task %s: %v", taskID, err), err)
+ return false
+ }
+ case common.STOPPING:
+ default:
+ // COMPLETED/STOPPED/FAILED: already terminal. CREATED/SCHEDULED:
+ // re-armed by a newer run - nothing this run may finalize.
+ common.Info(fmt.Sprintf("markStopped: task %s is %s; nothing to finalize for this run", taskID, task.Status))
+ return true
+ }
if err := e.ingestionTaskSvc.MarkStopped(ctx, taskID); err != nil {
common.Error(fmt.Sprintf("markStopped: MarkStopped task %s: %v", taskID, err), err)
return false
@@ -772,6 +812,11 @@ func (e *Ingestor) runTask(ctx context.Context, task *entity.IngestionTask) bool
}
return ok
}
+ // Capture this run's counter: if a re-parse supersedes this run (the task
+ // is force-finalized and re-armed while this worker is still inside a
+ // non-cancellable section), a newer run bumps the counter and this run
+ // must not overwrite the newer run's terminal state or document progress.
+ myRun, _ := e.ingestionTaskSvc.CurrentRunCount(dbCtx, task.ID)
checkpointExists := e.checkpointExists
if checkpointExists == nil {
checkpointExists = canvas.RedisCheckpointExists
@@ -819,6 +864,10 @@ func (e *Ingestor) runTask(ctx context.Context, task *entity.IngestionTask) bool
return false
}
common.Info(fmt.Sprintf("Task %s cancelled during pipeline", task.ID))
+ if !e.ownsTaskRun(task.ID, myRun) {
+ common.Info(fmt.Sprintf("Task %s run %d superseded by a newer run; skip cancel finalize", task.ID, myRun))
+ return true
+ }
e.markCancelProgress(task)
stopped := e.markStopped(ctx, task.ID)
if stopped {
@@ -828,6 +877,10 @@ func (e *Ingestor) runTask(ctx context.Context, task *entity.IngestionTask) bool
}
if errors.Is(err, context.DeadlineExceeded) {
common.Info(fmt.Sprintf("Task %s timed out during pipeline", task.ID))
+ if !e.ownsTaskRun(task.ID, myRun) {
+ common.Info(fmt.Sprintf("Task %s run %d superseded by a newer run; skip timeout finalize", task.ID, myRun))
+ return true
+ }
e.markTimeoutProgress(task)
ok := e.markFailed(ctx, task.ID)
if ok {
@@ -836,6 +889,10 @@ func (e *Ingestor) runTask(ctx context.Context, task *entity.IngestionTask) bool
return ok
}
common.Error(fmt.Sprintf("Task %s failed", task.ID), err)
+ if !e.ownsTaskRun(task.ID, myRun) {
+ common.Info(fmt.Sprintf("Task %s run %d superseded by a newer run; skip failure finalize", task.ID, myRun))
+ return true
+ }
ok := e.markFailed(ctx, task.ID)
if ok {
e.recordTerminalPipelineLog(ctx, task, string(entity.TaskStatusFail))
@@ -843,6 +900,10 @@ func (e *Ingestor) runTask(ctx context.Context, task *entity.IngestionTask) bool
return ok
}
+ if !e.ownsTaskRun(task.ID, myRun) {
+ common.Info(fmt.Sprintf("Task %s run %d superseded by a newer run; skip completion", task.ID, myRun))
+ return true
+ }
if err := e.completeTask(ctx, task.ID); err != nil {
common.Error(fmt.Sprintf("Task %s update status failed", task.ID), err)
return false
diff --git a/internal/service/ingestion_task_service.go b/internal/service/ingestion_task_service.go
index 8395578ed0..3bd3f6babf 100644
--- a/internal/service/ingestion_task_service.go
+++ b/internal/service/ingestion_task_service.go
@@ -46,15 +46,20 @@ type IngestionTaskService struct {
ingestionTaskDAO *dao.IngestionTaskDAO
ingestionTaskLogDAO *dao.IngestionTaskLogDAO
taskPublisher TaskPublisher
+ // supersedeTerminalWait bounds how long supersedeAndRetryTask waits for a
+ // live worker to finalize a requested stop before force-finalizing the
+ // task. Overridable so tests do not pay the full production wait.
+ supersedeTerminalWait time.Duration
}
func NewIngestionTaskService() *IngestionTaskService {
return &IngestionTaskService{
- documentDAO: dao.NewDocumentDAO(),
- userDAO: dao.NewUserDAO(),
- ingestionTaskDAO: dao.NewIngestionTaskDAO(),
- ingestionTaskLogDAO: dao.NewIngestionTaskLogDAO(),
- taskPublisher: NewMessageQueueTaskPublisher(),
+ documentDAO: dao.NewDocumentDAO(),
+ userDAO: dao.NewUserDAO(),
+ ingestionTaskDAO: dao.NewIngestionTaskDAO(),
+ ingestionTaskLogDAO: dao.NewIngestionTaskLogDAO(),
+ taskPublisher: NewMessageQueueTaskPublisher(),
+ supersedeTerminalWait: defaultSupersedeTerminalWait,
}
}
@@ -341,7 +346,7 @@ func validateTransition(from, to string) error {
if to == common.STOPPED {
return nil
}
- case common.FAILED, common.STOPPED:
+ case common.FAILED, common.STOPPED, common.COMPLETED:
if to == common.CREATED {
return nil
}
@@ -397,7 +402,7 @@ func (s *IngestionTaskService) CreateAndEnqueue(ctx context.Context, task *entit
return nil, err
}
return s.markScheduledAfterPublish(ctx, existing.ID)
- case common.FAILED, common.STOPPED:
+ case common.FAILED, common.STOPPED, common.COMPLETED:
originalStatus := existing.Status
existing, err = s.transition(ctx, existing.ID, common.CREATED)
if err != nil {
@@ -415,6 +420,8 @@ func (s *IngestionTaskService) CreateAndEnqueue(ctx context.Context, task *entit
return nil, err
}
return s.markScheduledAfterPublish(ctx, existing.ID)
+ case common.SCHEDULED, common.RUNNING, common.STOPPING:
+ return s.supersedeAndRetryTask(ctx, existing)
default:
return nil, fmt.Errorf("document id %s already exists, status: %s, task id: %s", task.DocumentID, existing.Status, existing.ID)
}
@@ -433,6 +440,141 @@ func (s *IngestionTaskService) CreateAndEnqueue(ctx context.Context, task *entit
return s.markScheduledAfterPublish(ctx, created.ID)
}
+// defaultSupersedeTerminalWait bounds how long a re-parse request waits for a
+// live worker to finalize a requested stop before force-finalizing the task.
+// A live worker observes the stop (Redis cancel flag polled every 3s, plus the
+// DB STOPPING fallback) well within this window; only a worker stuck in a
+// non-cancellable section or a dead worker exceeds it.
+const defaultSupersedeTerminalWait = 8 * time.Second
+
+const supersedePollInterval = 250 * time.Millisecond
+
+// supersedeAndRetryTask finalizes an in-flight ingestion task
+// (SCHEDULED/RUNNING/STOPPING) and re-queues it for a fresh parse, mirroring
+// the Python /documents/ingest endpoint, which accepts a parse request
+// regardless of the previous task's in-flight state. Without this, a document
+// whose task lingers in a non-terminal state — for example a parse canceled at
+// progress 0 whose worker died before finalizing the stop (e.g. the service
+// restarted mid-parse), or a scheduled task whose queue message was lost —
+// could never be re-parsed: every attempt failed with "document id ... already
+// exists, status: ...".
+//
+// For a task a worker may still be executing (RUNNING/STOPPING) the stop is
+// requested first and the worker gets a bounded window to finalize on its own;
+// a live worker settles within its 3s cancel poll. Only when that window
+// expires is the task force-finalized. In every case the Redis cancel flag is
+// cleared before the re-arm: the replacement run's pre-start cancel check
+// (Ingestor.executeTask) reads the same flag and would abort the fresh parse
+// before it starts, silently reproducing the "cannot re-parse" symptom.
+func (s *IngestionTaskService) supersedeAndRetryTask(ctx context.Context, existing *entity.IngestionTask) (*entity.IngestionTask, error) {
+ inFlightStatus := existing.Status
+ if inFlightStatus == common.RUNNING {
+ // RequestStop moves RUNNING→STOPPING and sets the Redis cancel flag so
+ // a live worker aborts at its next cancellation checkpoint.
+ if _, err := s.RequestStop(ctx, existing.ID); err != nil {
+ // The worker may have finalized (stopped/completed/failed)
+ // concurrently; a terminal task is still supersedeable below.
+ refreshed, refreshErr := s.GetTask(ctx, existing.ID)
+ if refreshErr != nil || !isTerminalTaskStatus(refreshed.Status) {
+ return nil, fmt.Errorf("stop in-flight task %s before re-parse: %w", existing.ID, err)
+ }
+ existing = refreshed
+ }
+ }
+ if existing.Status == common.RUNNING || existing.Status == common.STOPPING {
+ final := s.awaitTerminalTask(ctx, existing.ID)
+ if final == nil {
+ var err error
+ if final, err = s.forceTerminalTask(ctx, existing.ID); err != nil {
+ return nil, fmt.Errorf("finalize in-flight task %s (%s) before re-parse: %w", existing.ID, inFlightStatus, err)
+ }
+ }
+ existing = final
+ }
+ if existing.Status == common.CREATED || existing.Status == common.SCHEDULED {
+ // Never claimed (or freshly re-armed by a concurrent request): no
+ // worker can be executing it, so finalize it immediately.
+ var err error
+ if existing, err = s.transition(ctx, existing.ID, common.STOPPED); err != nil {
+ return nil, fmt.Errorf("finalize in-flight task %s (%s) before re-parse: %w", existing.ID, inFlightStatus, err)
+ }
+ }
+ if !isTerminalTaskStatus(existing.Status) {
+ return nil, fmt.Errorf("task %s is in unexpected status %s before re-parse", existing.ID, existing.Status)
+ }
+ // The superseded run is final, so any leftover Redis cancel flag is stale
+ // for the replacement run (see the doc comment).
+ clearCancelFlag(ctx, existing.ID)
+ retried, err := s.transition(ctx, existing.ID, common.CREATED)
+ if err != nil {
+ return nil, fmt.Errorf("re-arm superseded task %s for re-parse: %w", existing.ID, err)
+ }
+ if err = s.enqueueTask(retried.ID); err != nil {
+ if rollbackErr := s.rollbackRetriedTask(ctx, retried.ID, existing.Status); rollbackErr != nil {
+ return nil, fmt.Errorf("enqueue task %s: %w (rollback failed: %w)", retried.ID, err, rollbackErr)
+ }
+ return nil, err
+ }
+ return s.markScheduledAfterPublish(ctx, retried.ID)
+}
+
+// awaitTerminalTask polls the task until it reaches a terminal status or the
+// supersede wait budget expires. Returns the terminal task, or nil on timeout.
+func (s *IngestionTaskService) awaitTerminalTask(ctx context.Context, taskID string) *entity.IngestionTask {
+ wait := s.supersedeTerminalWait
+ if wait <= 0 {
+ wait = defaultSupersedeTerminalWait
+ }
+ deadline := time.Now().Add(wait)
+ for {
+ task, err := s.GetTask(ctx, taskID)
+ if err == nil && isTerminalTaskStatus(task.Status) {
+ return task
+ }
+ if time.Now().After(deadline) {
+ return nil
+ }
+ select {
+ case <-ctx.Done():
+ return nil
+ case <-time.After(supersedePollInterval):
+ }
+ }
+}
+
+// forceTerminalTask drives a task that missed its bounded stop window to a
+// terminal state: STOPPING→STOPPED directly; RUNNING via RequestStop first (a
+// worker may still be inside a non-cancellable section — the flag that sets is
+// cleared by the caller right after the finalize, and the worker's late
+// terminal write is skipped by its run-count ownership check in the Ingestor);
+// CREATED/SCHEDULED→STOPPED because no worker is executing them.
+func (s *IngestionTaskService) forceTerminalTask(ctx context.Context, taskID string) (*entity.IngestionTask, error) {
+ task, err := s.GetTask(ctx, taskID)
+ if err != nil {
+ return nil, err
+ }
+ switch task.Status {
+ case common.STOPPING, common.CREATED, common.SCHEDULED:
+ return s.transition(ctx, taskID, common.STOPPED)
+ case common.RUNNING:
+ if _, err = s.RequestStop(ctx, taskID); err != nil {
+ if refreshed, refreshErr := s.GetTask(ctx, taskID); refreshErr == nil && isTerminalTaskStatus(refreshed.Status) {
+ return refreshed, nil
+ }
+ return nil, err
+ }
+ return s.transition(ctx, taskID, common.STOPPED)
+ case common.COMPLETED, common.STOPPED, common.FAILED:
+ return task, nil
+ default:
+ return nil, fmt.Errorf("task %s has unsupported status %s", taskID, task.Status)
+ }
+}
+
+func isTerminalTaskStatus(status string) bool {
+ return status == common.COMPLETED || status == common.STOPPED || status == common.FAILED
+}
+
func (s *IngestionTaskService) rollbackRetriedTask(ctx context.Context, taskID, status string) error {
updated, err := s.ingestionTaskDAO.UpdateStatusIfCurrent(ctx, dao.DB, taskID, common.CREATED, status)
if err != nil {
@@ -546,6 +688,14 @@ func (s *IngestionTaskService) AggregateTaskProgress(ctx context.Context, taskID
return s.ingestionTaskLogDAO.AggregateProgress(ctx, dao.DB, taskID, total)
}
+// CurrentRunCount returns the run counter of the most recent started run for
+// the task. Workers capture it right after IncrementRunCount and use it to
+// detect that their run was superseded (a newer run bumped the counter)
+// before writing terminal state.
+func (s *IngestionTaskService) CurrentRunCount(ctx context.Context, taskID string) (int, bool) {
+ return s.lastRunCount(ctx, taskID)
+}
+
// lastRunCount scans all task logs (newest first) for a run_count entry,
// skipping component-progress rows whose Checkpoint is empty. It returns
// the counter and whether one was found.
diff --git a/internal/service/ingestion_task_service_test.go b/internal/service/ingestion_task_service_test.go
index 0820046c16..d444e57faf 100644
--- a/internal/service/ingestion_task_service_test.go
+++ b/internal/service/ingestion_task_service_test.go
@@ -5,6 +5,7 @@ import (
"strings"
"sync"
"testing"
+ "time"
"ragflow/internal/common"
"ragflow/internal/dao"
@@ -810,21 +811,107 @@ func TestIngestionTaskServiceScheduleCreatedTasksContinuesAfterPublishFailure(t
}
}
-func TestIngestionTaskServiceCreateAndEnqueueRejectsActiveExistingTask(t *testing.T) {
+func TestIngestionTaskServiceCreateAndEnqueueSupersedesInFlightTask(t *testing.T) {
+ for _, tc := range []struct {
+ name string
+ existing string
+ wantStatus string
+ }{
+ {
+ // A parse canceled at progress 0 whose worker has not finalized the
+ // stop yet leaves the task in STOPPING; re-parsing waits a bounded
+ // window for the worker, then force-finalizes and re-queues.
+ name: "stopping",
+ existing: common.STOPPING,
+ wantStatus: common.SCHEDULED,
+ },
+ {
+ // A task queued but never claimed by a worker (e.g. its queue
+ // message was lost) must not block re-parsing either.
+ name: "scheduled",
+ existing: common.SCHEDULED,
+ wantStatus: common.SCHEDULED,
+ },
+ {
+ // A task whose worker died mid-run stays RUNNING with no owner;
+ // re-parsing must stop it (the only RUNNING→CREATED path in the
+ // transition table is RUNNING→STOPPING→STOPPED→CREATED) and queue
+ // a fresh run.
+ name: "running",
+ existing: common.RUNNING,
+ wantStatus: common.SCHEDULED,
+ },
+ } {
+ t.Run(tc.name, func(t *testing.T) {
+ db := setupServiceTestDB(t)
+ pushServiceDB(t, db)
+ insertTestIngestionTaskWithStatus(t, "task-1", "user-1", "doc-1", "kb-1", tc.existing)
+
+ publisher := &recordingTaskPublisher{}
+ svc := NewIngestionTaskService()
+ svc.taskPublisher = publisher
+ // No live worker in the test: do not pay the full production
+ // window before force-finalizing the in-flight task.
+ svc.supersedeTerminalWait = 50 * time.Millisecond
+
+ task, err := svc.CreateAndEnqueue(t.Context(), &entity.IngestionTask{
+ DocumentID: "doc-1",
+ UserID: "user-1",
+ DatasetID: "kb-1",
+ Status: common.CREATED,
+ })
+ if err != nil {
+ t.Fatalf("CreateAndEnqueue failed: %v", err)
+ }
+
+ if task.ID != "task-1" {
+ t.Fatalf("task ID = %q, want task-1 (superseded in place)", task.ID)
+ }
+ if task.Status != tc.wantStatus {
+ t.Fatalf("status = %q, want %q", task.Status, tc.wantStatus)
+ }
+ if len(publisher.messages) != 1 || publisher.messages[0].TaskID != task.ID {
+ t.Fatalf("unexpected published messages: %+v", publisher.messages)
+ }
+
+ reloaded, err := dao.NewIngestionTaskDAO().GetByDocumentID(t.Context(), db, "doc-1")
+ if err != nil {
+ t.Fatalf("reload task: %v", err)
+ }
+ if reloaded == nil || reloaded.Status != tc.wantStatus {
+ t.Fatalf("expected superseded task to be re-queued, task=%+v", reloaded)
+ }
+ })
+ }
+}
+
+// TestIngestionTaskServiceCreateAndEnqueueRearmsCompletedTask locks in that a
+// document whose previous parse COMPLETED can be re-parsed without first
+// deleting its ingestion task: the terminal task row is re-armed in place,
+// mirroring the FAILED/STOPPED handling.
+func TestIngestionTaskServiceCreateAndEnqueueRearmsCompletedTask(t *testing.T) {
db := setupServiceTestDB(t)
pushServiceDB(t, db)
- insertTestIngestionTaskWithStatus(t, "task-1", "user-1", "doc-1", "kb-1", common.SCHEDULED)
+ insertTestIngestionTaskWithStatus(t, "task-1", "user-1", "doc-1", "kb-1", common.COMPLETED)
+
publisher := &recordingTaskPublisher{}
svc := NewIngestionTaskService()
svc.taskPublisher = publisher
- ctx := t.Context()
- _, err := svc.CreateAndEnqueue(ctx, &entity.IngestionTask{DocumentID: "doc-1", UserID: "user-1", DatasetID: "kb-1", Status: common.CREATED})
- if err == nil {
- t.Fatal("expected CreateAndEnqueue to reject existing created task")
+ task, err := svc.CreateAndEnqueue(t.Context(), &entity.IngestionTask{
+ DocumentID: "doc-1",
+ UserID: "user-1",
+ DatasetID: "kb-1",
+ Status: common.CREATED,
+ })
+ if err != nil {
+ t.Fatalf("CreateAndEnqueue failed: %v", err)
+ }
+ if task.ID != "task-1" || task.Status != common.SCHEDULED {
+ t.Fatalf("task = %s/%s, want task-1/SCHEDULED", task.ID, task.Status)
}
- if len(publisher.messages) != 0 {
- t.Fatalf("expected no published messages, got %+v", publisher.messages)
+ if len(publisher.messages) != 1 || publisher.messages[0].TaskID != task.ID {
+ t.Fatalf("unexpected published messages: %+v", publisher.messages)
}
}