Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

Revert "fix(ingestion): allow re-parsing a document whose parse was canceled at progress 0" (#19503)

Wang Qi607f98666468
diff --git a/internal/ingestion/service/ingestion_service.go b/internal/ingestion/service/ingestion_service.go
index 921341f322..f82719fe94 100644
--- a/internal/ingestion/service/ingestion_service.go
+++ b/internal/ingestion/service/ingestion_service.go
@@ -695,31 +695,10 @@ func (e *Ingestor) executeTaskWithHeartbeat(ctx context.Context, taskCtx *taskpk
})
}
-// ownsTaskRun reports whether the calling run still owns the task: a newer
-// run (created by a supersede/re-parse) bumps the run counter, so a mismatch
-// means this run's terminal write must be skipped. A run whose counter could
-// not be read (myRun <= 0) conservatively keeps ownership. Uses a detached
-// context: it is called from failure/cancel paths whose context is typically
-// already cancelled.
-func (e *Ingestor) ownsTaskRun(taskID string, myRun int) bool {
- if myRun <= 0 {
- return true
- }
- ctx, cancel := context.WithTimeout(context.WithoutCancel(e.ctx), 2*time.Second)
- defer cancel()
- current, ok := e.ingestionTaskSvc.CurrentRunCount(ctx, taskID)
- if !ok {
- return true
- }
- return current == myRun
-}
-
-// markStopped transitions the task to STOPPED (terminal). A RUNNING task
-// first goes through RequestStop (RUNNING → STOPPING); a STOPPING task is
-// settled directly; any other status (terminal, or CREATED/SCHEDULED because
-// a supersede re-armed the task for a newer run) is treated as settled with
-// nothing to finalize. Finally it cleans up the Redis cancel flag so that a
-// future retry of the same task does not immediately re-cancel.
+// markStopped transitions the task to STOPPED (terminal). It first calls
+// RequestStop to handle RUNNING → STOPPING, then MarkStopped for the final
+// STOPPING → STOPPED transition. Finally it cleans up the Redis cancel flag
+// so that a future retry of the same task does not immediately re-cancel.
// The caller's context is detached before touching the DB: markStopped runs
// on failure/cancel/shutdown paths whose context is typically already
// cancelled, and a contaminated context would make the terminal write fail
@@ -727,29 +706,10 @@ func (e *Ingestor) ownsTaskRun(taskID string, myRun int) bool {
func (e *Ingestor) markStopped(ctx context.Context, taskID string) bool {
ctx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 5*time.Second)
defer cancel()
- // Only a RUNNING/STOPPING task still belongs to this run. A task in
- // CREATED/SCHEDULED here was re-armed for a newer run by a supersede
- // (re-parse of a canceled document); finalizing it would kill the
- // replacement run before it starts, so treat it as already settled.
- // Terminal statuses need no work either.
- task, err := e.ingestionTaskSvc.GetTask(ctx, taskID)
- if err != nil {
- common.Error(fmt.Sprintf("markStopped: load task %s: %v", taskID, err), err)
+ if _, err := e.ingestionTaskSvc.RequestStop(ctx, taskID); err != nil {
+ common.Error(fmt.Sprintf("markStopped: RequestStop task %s: %v", taskID, err), err)
return false
}
- switch task.Status {
- case common.RUNNING:
- if _, err := e.ingestionTaskSvc.RequestStop(ctx, taskID); err != nil {
- common.Error(fmt.Sprintf("markStopped: RequestStop task %s: %v", taskID, err), err)
- return false
- }
- case common.STOPPING:
- default:
- // COMPLETED/STOPPED/FAILED: already terminal. CREATED/SCHEDULED:
- // re-armed by a newer run - nothing this run may finalize.
- common.Info(fmt.Sprintf("markStopped: task %s is %s; nothing to finalize for this run", taskID, task.Status))
- return true
- }
if err := e.ingestionTaskSvc.MarkStopped(ctx, taskID); err != nil {
common.Error(fmt.Sprintf("markStopped: MarkStopped task %s: %v", taskID, err), err)
return false
@@ -812,11 +772,6 @@ func (e *Ingestor) runTask(ctx context.Context, task *entity.IngestionTask) bool
}
return ok
}
- // Capture this run's counter: if a re-parse supersedes this run (the task
- // is force-finalized and re-armed while this worker is still inside a
- // non-cancellable section), a newer run bumps the counter and this run
- // must not overwrite the newer run's terminal state or document progress.
- myRun, _ := e.ingestionTaskSvc.CurrentRunCount(dbCtx, task.ID)
checkpointExists := e.checkpointExists
if checkpointExists == nil {
checkpointExists = canvas.RedisCheckpointExists
@@ -864,10 +819,6 @@ func (e *Ingestor) runTask(ctx context.Context, task *entity.IngestionTask) bool
return false
}
common.Info(fmt.Sprintf("Task %s cancelled during pipeline", task.ID))
- if !e.ownsTaskRun(task.ID, myRun) {
- common.Info(fmt.Sprintf("Task %s run %d superseded by a newer run; skip cancel finalize", task.ID, myRun))
- return true
- }
e.markCancelProgress(task)
stopped := e.markStopped(ctx, task.ID)
if stopped {
@@ -877,10 +828,6 @@ func (e *Ingestor) runTask(ctx context.Context, task *entity.IngestionTask) bool
}
if errors.Is(err, context.DeadlineExceeded) {
common.Info(fmt.Sprintf("Task %s timed out during pipeline", task.ID))
- if !e.ownsTaskRun(task.ID, myRun) {
- common.Info(fmt.Sprintf("Task %s run %d superseded by a newer run; skip timeout finalize", task.ID, myRun))
- return true
- }
e.markTimeoutProgress(task)
ok := e.markFailed(ctx, task.ID)
if ok {
@@ -889,10 +836,6 @@ func (e *Ingestor) runTask(ctx context.Context, task *entity.IngestionTask) bool
return ok
}
common.Error(fmt.Sprintf("Task %s failed", task.ID), err)
- if !e.ownsTaskRun(task.ID, myRun) {
- common.Info(fmt.Sprintf("Task %s run %d superseded by a newer run; skip failure finalize", task.ID, myRun))
- return true
- }
ok := e.markFailed(ctx, task.ID)
if ok {
e.recordTerminalPipelineLog(ctx, task, string(entity.TaskStatusFail))
@@ -900,10 +843,6 @@ func (e *Ingestor) runTask(ctx context.Context, task *entity.IngestionTask) bool
return ok
}
- if !e.ownsTaskRun(task.ID, myRun) {
- common.Info(fmt.Sprintf("Task %s run %d superseded by a newer run; skip completion", task.ID, myRun))
- return true
- }
if err := e.completeTask(ctx, task.ID); err != nil {
common.Error(fmt.Sprintf("Task %s update status failed", task.ID), err)
return false
diff --git a/internal/service/ingestion_task_service.go b/internal/service/ingestion_task_service.go
index 3bd3f6babf..8395578ed0 100644
--- a/internal/service/ingestion_task_service.go
+++ b/internal/service/ingestion_task_service.go
@@ -46,20 +46,15 @@ type IngestionTaskService struct {
ingestionTaskDAO *dao.IngestionTaskDAO
ingestionTaskLogDAO *dao.IngestionTaskLogDAO
taskPublisher TaskPublisher
- // supersedeTerminalWait bounds how long supersedeAndRetryTask waits for a
- // live worker to finalize a requested stop before force-finalizing the
- // task. Overridable so tests do not pay the full production wait.
- supersedeTerminalWait time.Duration
}
func NewIngestionTaskService() *IngestionTaskService {
return &IngestionTaskService{
- documentDAO: dao.NewDocumentDAO(),
- userDAO: dao.NewUserDAO(),
- ingestionTaskDAO: dao.NewIngestionTaskDAO(),
- ingestionTaskLogDAO: dao.NewIngestionTaskLogDAO(),
- taskPublisher: NewMessageQueueTaskPublisher(),
- supersedeTerminalWait: defaultSupersedeTerminalWait,
+ documentDAO: dao.NewDocumentDAO(),
+ userDAO: dao.NewUserDAO(),
+ ingestionTaskDAO: dao.NewIngestionTaskDAO(),
+ ingestionTaskLogDAO: dao.NewIngestionTaskLogDAO(),
+ taskPublisher: NewMessageQueueTaskPublisher(),
}
}
@@ -346,7 +341,7 @@ func validateTransition(from, to string) error {
if to == common.STOPPED {
return nil
}
- case common.FAILED, common.STOPPED, common.COMPLETED:
+ case common.FAILED, common.STOPPED:
if to == common.CREATED {
return nil
}
@@ -402,7 +397,7 @@ func (s *IngestionTaskService) CreateAndEnqueue(ctx context.Context, task *entit
return nil, err
}
return s.markScheduledAfterPublish(ctx, existing.ID)
- case common.FAILED, common.STOPPED, common.COMPLETED:
+ case common.FAILED, common.STOPPED:
originalStatus := existing.Status
existing, err = s.transition(ctx, existing.ID, common.CREATED)
if err != nil {
@@ -420,8 +415,6 @@ func (s *IngestionTaskService) CreateAndEnqueue(ctx context.Context, task *entit
return nil, err
}
return s.markScheduledAfterPublish(ctx, existing.ID)
- case common.SCHEDULED, common.RUNNING, common.STOPPING:
- return s.supersedeAndRetryTask(ctx, existing)
default:
return nil, fmt.Errorf("document id %s already exists, status: %s, task id: %s", task.DocumentID, existing.Status, existing.ID)
}
@@ -440,141 +433,6 @@ func (s *IngestionTaskService) CreateAndEnqueue(ctx context.Context, task *entit
return s.markScheduledAfterPublish(ctx, created.ID)
}
-// defaultSupersedeTerminalWait bounds how long a re-parse request waits for a
-// live worker to finalize a requested stop before force-finalizing the task.
-// A live worker observes the stop (Redis cancel flag polled every 3s, plus the
-// DB STOPPING fallback) well within this window; only a worker stuck in a
-// non-cancellable section or a dead worker exceeds it.
-const defaultSupersedeTerminalWait = 8 * time.Second
-
-const supersedePollInterval = 250 * time.Millisecond
-
-// supersedeAndRetryTask finalizes an in-flight ingestion task
-// (SCHEDULED/RUNNING/STOPPING) and re-queues it for a fresh parse, mirroring
-// the Python /documents/ingest endpoint, which accepts a parse request
-// regardless of the previous task's in-flight state. Without this, a document
-// whose task lingers in a non-terminal state — for example a parse canceled at
-// progress 0 whose worker died before finalizing the stop (e.g. the service
-// restarted mid-parse), or a scheduled task whose queue message was lost —
-// could never be re-parsed: every attempt failed with "document id ... already
-// exists, status: ...".
-//
-// For a task a worker may still be executing (RUNNING/STOPPING) the stop is
-// requested first and the worker gets a bounded window to finalize on its own;
-// a live worker settles within its 3s cancel poll. Only when that window
-// expires is the task force-finalized. In every case the Redis cancel flag is
-// cleared before the re-arm: the replacement run's pre-start cancel check
-// (Ingestor.executeTask) reads the same flag and would abort the fresh parse
-// before it starts, silently reproducing the "cannot re-parse" symptom.
-func (s *IngestionTaskService) supersedeAndRetryTask(ctx context.Context, existing *entity.IngestionTask) (*entity.IngestionTask, error) {
- inFlightStatus := existing.Status
- if inFlightStatus == common.RUNNING {
- // RequestStop moves RUNNING→STOPPING and sets the Redis cancel flag so
- // a live worker aborts at its next cancellation checkpoint.
- if _, err := s.RequestStop(ctx, existing.ID); err != nil {
- // The worker may have finalized (stopped/completed/failed)
- // concurrently; a terminal task is still supersedeable below.
- refreshed, refreshErr := s.GetTask(ctx, existing.ID)
- if refreshErr != nil || !isTerminalTaskStatus(refreshed.Status) {
- return nil, fmt.Errorf("stop in-flight task %s before re-parse: %w", existing.ID, err)
- }
- existing = refreshed
- }
- }
- if existing.Status == common.RUNNING || existing.Status == common.STOPPING {
- final := s.awaitTerminalTask(ctx, existing.ID)
- if final == nil {
- var err error
- if final, err = s.forceTerminalTask(ctx, existing.ID); err != nil {
- return nil, fmt.Errorf("finalize in-flight task %s (%s) before re-parse: %w", existing.ID, inFlightStatus, err)
- }
- }
- existing = final
- }
- if existing.Status == common.CREATED || existing.Status == common.SCHEDULED {
- // Never claimed (or freshly re-armed by a concurrent request): no
- // worker can be executing it, so finalize it immediately.
- var err error
- if existing, err = s.transition(ctx, existing.ID, common.STOPPED); err != nil {
- return nil, fmt.Errorf("finalize in-flight task %s (%s) before re-parse: %w", existing.ID, inFlightStatus, err)
- }
- }
- if !isTerminalTaskStatus(existing.Status) {
- return nil, fmt.Errorf("task %s is in unexpected status %s before re-parse", existing.ID, existing.Status)
- }
- // The superseded run is final, so any leftover Redis cancel flag is stale
- // for the replacement run (see the doc comment).
- clearCancelFlag(ctx, existing.ID)
- retried, err := s.transition(ctx, existing.ID, common.CREATED)
- if err != nil {
- return nil, fmt.Errorf("re-arm superseded task %s for re-parse: %w", existing.ID, err)
- }
- if err = s.enqueueTask(retried.ID); err != nil {
- if rollbackErr := s.rollbackRetriedTask(ctx, retried.ID, existing.Status); rollbackErr != nil {
- return nil, fmt.Errorf("enqueue task %s: %w (rollback failed: %w)", retried.ID, err, rollbackErr)
- }
- return nil, err
- }
- return s.markScheduledAfterPublish(ctx, retried.ID)
-}
-
-// awaitTerminalTask polls the task until it reaches a terminal status or the
-// supersede wait budget expires. Returns the terminal task, or nil on timeout.
-func (s *IngestionTaskService) awaitTerminalTask(ctx context.Context, taskID string) *entity.IngestionTask {
- wait := s.supersedeTerminalWait
- if wait <= 0 {
- wait = defaultSupersedeTerminalWait
- }
- deadline := time.Now().Add(wait)
- for {
- task, err := s.GetTask(ctx, taskID)
- if err == nil && isTerminalTaskStatus(task.Status) {
- return task
- }
- if time.Now().After(deadline) {
- return nil
- }
- select {
- case <-ctx.Done():
- return nil
- case <-time.After(supersedePollInterval):
- }
- }
-}
-
-// forceTerminalTask drives a task that missed its bounded stop window to a
-// terminal state: STOPPING→STOPPED directly; RUNNING via RequestStop first (a
-// worker may still be inside a non-cancellable section — the flag that sets is
-// cleared by the caller right after the finalize, and the worker's late
-// terminal write is skipped by its run-count ownership check in the Ingestor);
-// CREATED/SCHEDULED→STOPPED because no worker is executing them.
-func (s *IngestionTaskService) forceTerminalTask(ctx context.Context, taskID string) (*entity.IngestionTask, error) {
- task, err := s.GetTask(ctx, taskID)
- if err != nil {
- return nil, err
- }
- switch task.Status {
- case common.STOPPING, common.CREATED, common.SCHEDULED:
- return s.transition(ctx, taskID, common.STOPPED)
- case common.RUNNING:
- if _, err = s.RequestStop(ctx, taskID); err != nil {
- if refreshed, refreshErr := s.GetTask(ctx, taskID); refreshErr == nil && isTerminalTaskStatus(refreshed.Status) {
- return refreshed, nil
- }
- return nil, err
- }
- return s.transition(ctx, taskID, common.STOPPED)
- case common.COMPLETED, common.STOPPED, common.FAILED:
- return task, nil
- default:
- return nil, fmt.Errorf("task %s has unsupported status %s", taskID, task.Status)
- }
-}
-
-func isTerminalTaskStatus(status string) bool {
- return status == common.COMPLETED || status == common.STOPPED || status == common.FAILED
-}
-
func (s *IngestionTaskService) rollbackRetriedTask(ctx context.Context, taskID, status string) error {
updated, err := s.ingestionTaskDAO.UpdateStatusIfCurrent(ctx, dao.DB, taskID, common.CREATED, status)
if err != nil {
@@ -688,14 +546,6 @@ func (s *IngestionTaskService) AggregateTaskProgress(ctx context.Context, taskID
return s.ingestionTaskLogDAO.AggregateProgress(ctx, dao.DB, taskID, total)
}
-// CurrentRunCount returns the run counter of the most recent started run for
-// the task. Workers capture it right after IncrementRunCount and use it to
-// detect that their run was superseded (a newer run bumped the counter)
-// before writing terminal state.
-func (s *IngestionTaskService) CurrentRunCount(ctx context.Context, taskID string) (int, bool) {
- return s.lastRunCount(ctx, taskID)
-}
-
// lastRunCount scans all task logs (newest first) for a run_count entry,
// skipping component-progress rows whose Checkpoint is empty. It returns
// the counter and whether one was found.
diff --git a/internal/service/ingestion_task_service_test.go b/internal/service/ingestion_task_service_test.go
index d444e57faf..0820046c16 100644
--- a/internal/service/ingestion_task_service_test.go
+++ b/internal/service/ingestion_task_service_test.go
@@ -5,7 +5,6 @@ import (
"strings"
"sync"
"testing"
- "time"
"ragflow/internal/common"
"ragflow/internal/dao"
@@ -811,107 +810,21 @@ func TestIngestionTaskServiceScheduleCreatedTasksContinuesAfterPublishFailure(t
}
}
-func TestIngestionTaskServiceCreateAndEnqueueSupersedesInFlightTask(t *testing.T) {
- for _, tc := range []struct {
- name string
- existing string
- wantStatus string
- }{
- {
- // A parse canceled at progress 0 whose worker has not finalized the
- // stop yet leaves the task in STOPPING; re-parsing waits a bounded
- // window for the worker, then force-finalizes and re-queues.
- name: "stopping",
- existing: common.STOPPING,
- wantStatus: common.SCHEDULED,
- },
- {
- // A task queued but never claimed by a worker (e.g. its queue
- // message was lost) must not block re-parsing either.
- name: "scheduled",
- existing: common.SCHEDULED,
- wantStatus: common.SCHEDULED,
- },
- {
- // A task whose worker died mid-run stays RUNNING with no owner;
- // re-parsing must stop it (the only RUNNING→CREATED path in the
- // transition table is RUNNING→STOPPING→STOPPED→CREATED) and queue
- // a fresh run.
- name: "running",
- existing: common.RUNNING,
- wantStatus: common.SCHEDULED,
- },
- } {
- t.Run(tc.name, func(t *testing.T) {
- db := setupServiceTestDB(t)
- pushServiceDB(t, db)
- insertTestIngestionTaskWithStatus(t, "task-1", "user-1", "doc-1", "kb-1", tc.existing)
-
- publisher := &recordingTaskPublisher{}
- svc := NewIngestionTaskService()
- svc.taskPublisher = publisher
- // No live worker in the test: do not pay the full production
- // window before force-finalizing the in-flight task.
- svc.supersedeTerminalWait = 50 * time.Millisecond
-
- task, err := svc.CreateAndEnqueue(t.Context(), &entity.IngestionTask{
- DocumentID: "doc-1",
- UserID: "user-1",
- DatasetID: "kb-1",
- Status: common.CREATED,
- })
- if err != nil {
- t.Fatalf("CreateAndEnqueue failed: %v", err)
- }
-
- if task.ID != "task-1" {
- t.Fatalf("task ID = %q, want task-1 (superseded in place)", task.ID)
- }
- if task.Status != tc.wantStatus {
- t.Fatalf("status = %q, want %q", task.Status, tc.wantStatus)
- }
- if len(publisher.messages) != 1 || publisher.messages[0].TaskID != task.ID {
- t.Fatalf("unexpected published messages: %+v", publisher.messages)
- }
-
- reloaded, err := dao.NewIngestionTaskDAO().GetByDocumentID(t.Context(), db, "doc-1")
- if err != nil {
- t.Fatalf("reload task: %v", err)
- }
- if reloaded == nil || reloaded.Status != tc.wantStatus {
- t.Fatalf("expected superseded task to be re-queued, task=%+v", reloaded)
- }
- })
- }
-}
-
-// TestIngestionTaskServiceCreateAndEnqueueRearmsCompletedTask locks in that a
-// document whose previous parse COMPLETED can be re-parsed without first
-// deleting its ingestion task: the terminal task row is re-armed in place,
-// mirroring the FAILED/STOPPED handling.
-func TestIngestionTaskServiceCreateAndEnqueueRearmsCompletedTask(t *testing.T) {
+func TestIngestionTaskServiceCreateAndEnqueueRejectsActiveExistingTask(t *testing.T) {
db := setupServiceTestDB(t)
pushServiceDB(t, db)
- insertTestIngestionTaskWithStatus(t, "task-1", "user-1", "doc-1", "kb-1", common.COMPLETED)
-
+ insertTestIngestionTaskWithStatus(t, "task-1", "user-1", "doc-1", "kb-1", common.SCHEDULED)
publisher := &recordingTaskPublisher{}
svc := NewIngestionTaskService()
svc.taskPublisher = publisher
- task, err := svc.CreateAndEnqueue(t.Context(), &entity.IngestionTask{
- DocumentID: "doc-1",
- UserID: "user-1",
- DatasetID: "kb-1",
- Status: common.CREATED,
- })
- if err != nil {
- t.Fatalf("CreateAndEnqueue failed: %v", err)
- }
- if task.ID != "task-1" || task.Status != common.SCHEDULED {
- t.Fatalf("task = %s/%s, want task-1/SCHEDULED", task.ID, task.Status)
+ ctx := t.Context()
+ _, err := svc.CreateAndEnqueue(ctx, &entity.IngestionTask{DocumentID: "doc-1", UserID: "user-1", DatasetID: "kb-1", Status: common.CREATED})
+ if err == nil {
+ t.Fatal("expected CreateAndEnqueue to reject existing created task")
}
- if len(publisher.messages) != 1 || publisher.messages[0].TaskID != task.ID {
- t.Fatalf("unexpected published messages: %+v", publisher.messages)
+ if len(publisher.messages) != 0 {
+ t.Fatalf("expected no published messages, got %+v", publisher.messages)
}
}