RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
fix(ingestion): allow re-parsing a document whose parse was canceled at progress 0 (#19258)
euvre347d0827a0fe
diff --git a/internal/ingestion/service/ingestion_service.go b/internal/ingestion/service/ingestion_service.goindex f82719fe94..921341f322 100644--- a/internal/ingestion/service/ingestion_service.go+++ b/internal/ingestion/service/ingestion_service.go@@ -695,10 +695,31 @@ func (e *Ingestor) executeTaskWithHeartbeat(ctx context.Context, taskCtx *taskpk})}-// markStopped transitions the task to STOPPED (terminal). It first calls-// RequestStop to handle RUNNING → STOPPING, then MarkStopped for the final-// STOPPING → STOPPED transition. Finally it cleans up the Redis cancel flag-// so that a future retry of the same task does not immediately re-cancel.+// ownsTaskRun reports whether the calling run still owns the task: a newer+// run (created by a supersede/re-parse) bumps the run counter, so a mismatch+// means this run's terminal write must be skipped. A run whose counter could+// not be read (myRun <= 0) conservatively keeps ownership. Uses a detached+// context: it is called from failure/cancel paths whose context is typically+// already cancelled.+func (e *Ingestor) ownsTaskRun(taskID string, myRun int) bool {+ if myRun <= 0 {+ return true+ }+ ctx, cancel := context.WithTimeout(context.WithoutCancel(e.ctx), 2*time.Second)+ defer cancel()+ current, ok := e.ingestionTaskSvc.CurrentRunCount(ctx, taskID)+ if !ok {+ return true+ }+ return current == myRun+}++// markStopped transitions the task to STOPPED (terminal). A RUNNING task+// first goes through RequestStop (RUNNING → STOPPING); a STOPPING task is+// settled directly; any other status (terminal, or CREATED/SCHEDULED because+// a supersede re-armed the task for a newer run) is treated as settled with+// nothing to finalize. Finally it cleans up the Redis cancel flag so that a+// future retry of the same task does not immediately re-cancel.// The caller's context is detached before touching the DB: markStopped runs// on failure/cancel/shutdown paths whose context is typically already// cancelled, and a contaminated context would make the terminal write fail@@ -706,10 +727,29 @@ func (e *Ingestor) executeTaskWithHeartbeat(ctx context.Context, taskCtx *taskpkfunc (e *Ingestor) markStopped(ctx context.Context, taskID string) bool {ctx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 5*time.Second)defer cancel()- if _, err := e.ingestionTaskSvc.RequestStop(ctx, taskID); err != nil {- common.Error(fmt.Sprintf("markStopped: RequestStop task %s: %v", taskID, err), err)+ // Only a RUNNING/STOPPING task still belongs to this run. A task in+ // CREATED/SCHEDULED here was re-armed for a newer run by a supersede+ // (re-parse of a canceled document); finalizing it would kill the+ // replacement run before it starts, so treat it as already settled.+ // Terminal statuses need no work either.+ task, err := e.ingestionTaskSvc.GetTask(ctx, taskID)+ if err != nil {+ common.Error(fmt.Sprintf("markStopped: load task %s: %v", taskID, err), err)return false}+ switch task.Status {+ case common.RUNNING:+ if _, err := e.ingestionTaskSvc.RequestStop(ctx, taskID); err != nil {+ common.Error(fmt.Sprintf("markStopped: RequestStop task %s: %v", taskID, err), err)+ return false+ }+ case common.STOPPING:+ default:+ // COMPLETED/STOPPED/FAILED: already terminal. CREATED/SCHEDULED:+ // re-armed by a newer run - nothing this run may finalize.+ common.Info(fmt.Sprintf("markStopped: task %s is %s; nothing to finalize for this run", taskID, task.Status))+ return true+ }if err := e.ingestionTaskSvc.MarkStopped(ctx, taskID); err != nil {common.Error(fmt.Sprintf("markStopped: MarkStopped task %s: %v", taskID, err), err)return false@@ -772,6 +812,11 @@ func (e *Ingestor) runTask(ctx context.Context, task *entity.IngestionTask) bool}return ok}+ // Capture this run's counter: if a re-parse supersedes this run (the task+ // is force-finalized and re-armed while this worker is still inside a+ // non-cancellable section), a newer run bumps the counter and this run+ // must not overwrite the newer run's terminal state or document progress.+ myRun, _ := e.ingestionTaskSvc.CurrentRunCount(dbCtx, task.ID)checkpointExists := e.checkpointExistsif checkpointExists == nil {checkpointExists = canvas.RedisCheckpointExists@@ -819,6 +864,10 @@ func (e *Ingestor) runTask(ctx context.Context, task *entity.IngestionTask) boolreturn false}common.Info(fmt.Sprintf("Task %s cancelled during pipeline", task.ID))+ if !e.ownsTaskRun(task.ID, myRun) {+ common.Info(fmt.Sprintf("Task %s run %d superseded by a newer run; skip cancel finalize", task.ID, myRun))+ return true+ }e.markCancelProgress(task)stopped := e.markStopped(ctx, task.ID)if stopped {@@ -828,6 +877,10 @@ func (e *Ingestor) runTask(ctx context.Context, task *entity.IngestionTask) bool}if errors.Is(err, context.DeadlineExceeded) {common.Info(fmt.Sprintf("Task %s timed out during pipeline", task.ID))+ if !e.ownsTaskRun(task.ID, myRun) {+ common.Info(fmt.Sprintf("Task %s run %d superseded by a newer run; skip timeout finalize", task.ID, myRun))+ return true+ }e.markTimeoutProgress(task)ok := e.markFailed(ctx, task.ID)if ok {@@ -836,6 +889,10 @@ func (e *Ingestor) runTask(ctx context.Context, task *entity.IngestionTask) boolreturn ok}common.Error(fmt.Sprintf("Task %s failed", task.ID), err)+ if !e.ownsTaskRun(task.ID, myRun) {+ common.Info(fmt.Sprintf("Task %s run %d superseded by a newer run; skip failure finalize", task.ID, myRun))+ return true+ }ok := e.markFailed(ctx, task.ID)if ok {e.recordTerminalPipelineLog(ctx, task, string(entity.TaskStatusFail))@@ -843,6 +900,10 @@ func (e *Ingestor) runTask(ctx context.Context, task *entity.IngestionTask) boolreturn ok}+ if !e.ownsTaskRun(task.ID, myRun) {+ common.Info(fmt.Sprintf("Task %s run %d superseded by a newer run; skip completion", task.ID, myRun))+ return true+ }if err := e.completeTask(ctx, task.ID); err != nil {common.Error(fmt.Sprintf("Task %s update status failed", task.ID), err)return falsediff --git a/internal/service/ingestion_task_service.go b/internal/service/ingestion_task_service.goindex 8395578ed0..3bd3f6babf 100644--- a/internal/service/ingestion_task_service.go+++ b/internal/service/ingestion_task_service.go@@ -46,15 +46,20 @@ type IngestionTaskService struct {ingestionTaskDAO *dao.IngestionTaskDAOingestionTaskLogDAO *dao.IngestionTaskLogDAOtaskPublisher TaskPublisher+ // supersedeTerminalWait bounds how long supersedeAndRetryTask waits for a+ // live worker to finalize a requested stop before force-finalizing the+ // task. Overridable so tests do not pay the full production wait.+ supersedeTerminalWait time.Duration}func NewIngestionTaskService() *IngestionTaskService {return &IngestionTaskService{- documentDAO: dao.NewDocumentDAO(),- userDAO: dao.NewUserDAO(),- ingestionTaskDAO: dao.NewIngestionTaskDAO(),- ingestionTaskLogDAO: dao.NewIngestionTaskLogDAO(),- taskPublisher: NewMessageQueueTaskPublisher(),+ documentDAO: dao.NewDocumentDAO(),+ userDAO: dao.NewUserDAO(),+ ingestionTaskDAO: dao.NewIngestionTaskDAO(),+ ingestionTaskLogDAO: dao.NewIngestionTaskLogDAO(),+ taskPublisher: NewMessageQueueTaskPublisher(),+ supersedeTerminalWait: defaultSupersedeTerminalWait,}}@@ -341,7 +346,7 @@ func validateTransition(from, to string) error {if to == common.STOPPED {return nil}- case common.FAILED, common.STOPPED:+ case common.FAILED, common.STOPPED, common.COMPLETED:if to == common.CREATED {return nil}@@ -397,7 +402,7 @@ func (s *IngestionTaskService) CreateAndEnqueue(ctx context.Context, task *entitreturn nil, err}return s.markScheduledAfterPublish(ctx, existing.ID)- case common.FAILED, common.STOPPED:+ case common.FAILED, common.STOPPED, common.COMPLETED:originalStatus := existing.Statusexisting, err = s.transition(ctx, existing.ID, common.CREATED)if err != nil {@@ -415,6 +420,8 @@ func (s *IngestionTaskService) CreateAndEnqueue(ctx context.Context, task *entitreturn nil, err}return s.markScheduledAfterPublish(ctx, existing.ID)+ case common.SCHEDULED, common.RUNNING, common.STOPPING:+ return s.supersedeAndRetryTask(ctx, existing)default:return nil, fmt.Errorf("document id %s already exists, status: %s, task id: %s", task.DocumentID, existing.Status, existing.ID)}@@ -433,6 +440,141 @@ func (s *IngestionTaskService) CreateAndEnqueue(ctx context.Context, task *entitreturn s.markScheduledAfterPublish(ctx, created.ID)}+// defaultSupersedeTerminalWait bounds how long a re-parse request waits for a+// live worker to finalize a requested stop before force-finalizing the task.+// A live worker observes the stop (Redis cancel flag polled every 3s, plus the+// DB STOPPING fallback) well within this window; only a worker stuck in a+// non-cancellable section or a dead worker exceeds it.+const defaultSupersedeTerminalWait = 8 * time.Second++const supersedePollInterval = 250 * time.Millisecond++// supersedeAndRetryTask finalizes an in-flight ingestion task+// (SCHEDULED/RUNNING/STOPPING) and re-queues it for a fresh parse, mirroring+// the Python /documents/ingest endpoint, which accepts a parse request+// regardless of the previous task's in-flight state. Without this, a document+// whose task lingers in a non-terminal state — for example a parse canceled at+// progress 0 whose worker died before finalizing the stop (e.g. the service+// restarted mid-parse), or a scheduled task whose queue message was lost —+// could never be re-parsed: every attempt failed with "document id ... already+// exists, status: ...".+//+// For a task a worker may still be executing (RUNNING/STOPPING) the stop is+// requested first and the worker gets a bounded window to finalize on its own;+// a live worker settles within its 3s cancel poll. Only when that window+// expires is the task force-finalized. In every case the Redis cancel flag is+// cleared before the re-arm: the replacement run's pre-start cancel check+// (Ingestor.executeTask) reads the same flag and would abort the fresh parse+// before it starts, silently reproducing the "cannot re-parse" symptom.+func (s *IngestionTaskService) supersedeAndRetryTask(ctx context.Context, existing *entity.IngestionTask) (*entity.IngestionTask, error) {+ inFlightStatus := existing.Status+ if inFlightStatus == common.RUNNING {+ // RequestStop moves RUNNING→STOPPING and sets the Redis cancel flag so+ // a live worker aborts at its next cancellation checkpoint.+ if _, err := s.RequestStop(ctx, existing.ID); err != nil {+ // The worker may have finalized (stopped/completed/failed)+ // concurrently; a terminal task is still supersedeable below.+ refreshed, refreshErr := s.GetTask(ctx, existing.ID)+ if refreshErr != nil || !isTerminalTaskStatus(refreshed.Status) {+ return nil, fmt.Errorf("stop in-flight task %s before re-parse: %w", existing.ID, err)+ }+ existing = refreshed+ }+ }+ if existing.Status == common.RUNNING || existing.Status == common.STOPPING {+ final := s.awaitTerminalTask(ctx, existing.ID)+ if final == nil {+ var err error+ if final, err = s.forceTerminalTask(ctx, existing.ID); err != nil {+ return nil, fmt.Errorf("finalize in-flight task %s (%s) before re-parse: %w", existing.ID, inFlightStatus, err)+ }+ }+ existing = final+ }+ if existing.Status == common.CREATED || existing.Status == common.SCHEDULED {+ // Never claimed (or freshly re-armed by a concurrent request): no+ // worker can be executing it, so finalize it immediately.+ var err error+ if existing, err = s.transition(ctx, existing.ID, common.STOPPED); err != nil {+ return nil, fmt.Errorf("finalize in-flight task %s (%s) before re-parse: %w", existing.ID, inFlightStatus, err)+ }+ }+ if !isTerminalTaskStatus(existing.Status) {+ return nil, fmt.Errorf("task %s is in unexpected status %s before re-parse", existing.ID, existing.Status)+ }+ // The superseded run is final, so any leftover Redis cancel flag is stale+ // for the replacement run (see the doc comment).+ clearCancelFlag(ctx, existing.ID)+ retried, err := s.transition(ctx, existing.ID, common.CREATED)+ if err != nil {+ return nil, fmt.Errorf("re-arm superseded task %s for re-parse: %w", existing.ID, err)+ }+ if err = s.enqueueTask(retried.ID); err != nil {+ if rollbackErr := s.rollbackRetriedTask(ctx, retried.ID, existing.Status); rollbackErr != nil {+ return nil, fmt.Errorf("enqueue task %s: %w (rollback failed: %w)", retried.ID, err, rollbackErr)+ }+ return nil, err+ }+ return s.markScheduledAfterPublish(ctx, retried.ID)+}++// awaitTerminalTask polls the task until it reaches a terminal status or the+// supersede wait budget expires. Returns the terminal task, or nil on timeout.+func (s *IngestionTaskService) awaitTerminalTask(ctx context.Context, taskID string) *entity.IngestionTask {+ wait := s.supersedeTerminalWait+ if wait <= 0 {+ wait = defaultSupersedeTerminalWait+ }+ deadline := time.Now().Add(wait)+ for {+ task, err := s.GetTask(ctx, taskID)+ if err == nil && isTerminalTaskStatus(task.Status) {+ return task+ }+ if time.Now().After(deadline) {+ return nil+ }+ select {+ case <-ctx.Done():+ return nil+ case <-time.After(supersedePollInterval):+ }+ }+}++// forceTerminalTask drives a task that missed its bounded stop window to a+// terminal state: STOPPING→STOPPED directly; RUNNING via RequestStop first (a+// worker may still be inside a non-cancellable section — the flag that sets is+// cleared by the caller right after the finalize, and the worker's late+// terminal write is skipped by its run-count ownership check in the Ingestor);+// CREATED/SCHEDULED→STOPPED because no worker is executing them.+func (s *IngestionTaskService) forceTerminalTask(ctx context.Context, taskID string) (*entity.IngestionTask, error) {+ task, err := s.GetTask(ctx, taskID)+ if err != nil {+ return nil, err+ }+ switch task.Status {+ case common.STOPPING, common.CREATED, common.SCHEDULED:+ return s.transition(ctx, taskID, common.STOPPED)+ case common.RUNNING:+ if _, err = s.RequestStop(ctx, taskID); err != nil {+ if refreshed, refreshErr := s.GetTask(ctx, taskID); refreshErr == nil && isTerminalTaskStatus(refreshed.Status) {+ return refreshed, nil+ }+ return nil, err+ }+ return s.transition(ctx, taskID, common.STOPPED)+ case common.COMPLETED, common.STOPPED, common.FAILED:+ return task, nil+ default:+ return nil, fmt.Errorf("task %s has unsupported status %s", taskID, task.Status)+ }+}++func isTerminalTaskStatus(status string) bool {+ return status == common.COMPLETED || status == common.STOPPED || status == common.FAILED+}+func (s *IngestionTaskService) rollbackRetriedTask(ctx context.Context, taskID, status string) error {updated, err := s.ingestionTaskDAO.UpdateStatusIfCurrent(ctx, dao.DB, taskID, common.CREATED, status)if err != nil {@@ -546,6 +688,14 @@ func (s *IngestionTaskService) AggregateTaskProgress(ctx context.Context, taskIDreturn s.ingestionTaskLogDAO.AggregateProgress(ctx, dao.DB, taskID, total)}+// CurrentRunCount returns the run counter of the most recent started run for+// the task. Workers capture it right after IncrementRunCount and use it to+// detect that their run was superseded (a newer run bumped the counter)+// before writing terminal state.+func (s *IngestionTaskService) CurrentRunCount(ctx context.Context, taskID string) (int, bool) {+ return s.lastRunCount(ctx, taskID)+}+// lastRunCount scans all task logs (newest first) for a run_count entry,// skipping component-progress rows whose Checkpoint is empty. It returns// the counter and whether one was found.diff --git a/internal/service/ingestion_task_service_test.go b/internal/service/ingestion_task_service_test.goindex 0820046c16..d444e57faf 100644--- a/internal/service/ingestion_task_service_test.go+++ b/internal/service/ingestion_task_service_test.go@@ -5,6 +5,7 @@ import ("strings""sync""testing"+ "time""ragflow/internal/common""ragflow/internal/dao"@@ -810,21 +811,107 @@ func TestIngestionTaskServiceScheduleCreatedTasksContinuesAfterPublishFailure(t}}-func TestIngestionTaskServiceCreateAndEnqueueRejectsActiveExistingTask(t *testing.T) {+func TestIngestionTaskServiceCreateAndEnqueueSupersedesInFlightTask(t *testing.T) {+ for _, tc := range []struct {+ name string+ existing string+ wantStatus string+ }{+ {+ // A parse canceled at progress 0 whose worker has not finalized the+ // stop yet leaves the task in STOPPING; re-parsing waits a bounded+ // window for the worker, then force-finalizes and re-queues.+ name: "stopping",+ existing: common.STOPPING,+ wantStatus: common.SCHEDULED,+ },+ {+ // A task queued but never claimed by a worker (e.g. its queue+ // message was lost) must not block re-parsing either.+ name: "scheduled",+ existing: common.SCHEDULED,+ wantStatus: common.SCHEDULED,+ },+ {+ // A task whose worker died mid-run stays RUNNING with no owner;+ // re-parsing must stop it (the only RUNNING→CREATED path in the+ // transition table is RUNNING→STOPPING→STOPPED→CREATED) and queue+ // a fresh run.+ name: "running",+ existing: common.RUNNING,+ wantStatus: common.SCHEDULED,+ },+ } {+ t.Run(tc.name, func(t *testing.T) {+ db := setupServiceTestDB(t)+ pushServiceDB(t, db)+ insertTestIngestionTaskWithStatus(t, "task-1", "user-1", "doc-1", "kb-1", tc.existing)++ publisher := &recordingTaskPublisher{}+ svc := NewIngestionTaskService()+ svc.taskPublisher = publisher+ // No live worker in the test: do not pay the full production+ // window before force-finalizing the in-flight task.+ svc.supersedeTerminalWait = 50 * time.Millisecond++ task, err := svc.CreateAndEnqueue(t.Context(), &entity.IngestionTask{+ DocumentID: "doc-1",+ UserID: "user-1",+ DatasetID: "kb-1",+ Status: common.CREATED,+ })+ if err != nil {+ t.Fatalf("CreateAndEnqueue failed: %v", err)+ }++ if task.ID != "task-1" {+ t.Fatalf("task ID = %q, want task-1 (superseded in place)", task.ID)+ }+ if task.Status != tc.wantStatus {+ t.Fatalf("status = %q, want %q", task.Status, tc.wantStatus)+ }+ if len(publisher.messages) != 1 || publisher.messages[0].TaskID != task.ID {+ t.Fatalf("unexpected published messages: %+v", publisher.messages)+ }++ reloaded, err := dao.NewIngestionTaskDAO().GetByDocumentID(t.Context(), db, "doc-1")+ if err != nil {+ t.Fatalf("reload task: %v", err)+ }+ if reloaded == nil || reloaded.Status != tc.wantStatus {+ t.Fatalf("expected superseded task to be re-queued, task=%+v", reloaded)+ }+ })+ }+}++// TestIngestionTaskServiceCreateAndEnqueueRearmsCompletedTask locks in that a+// document whose previous parse COMPLETED can be re-parsed without first+// deleting its ingestion task: the terminal task row is re-armed in place,+// mirroring the FAILED/STOPPED handling.+func TestIngestionTaskServiceCreateAndEnqueueRearmsCompletedTask(t *testing.T) {db := setupServiceTestDB(t)pushServiceDB(t, db)- insertTestIngestionTaskWithStatus(t, "task-1", "user-1", "doc-1", "kb-1", common.SCHEDULED)+ insertTestIngestionTaskWithStatus(t, "task-1", "user-1", "doc-1", "kb-1", common.COMPLETED)+publisher := &recordingTaskPublisher{}svc := NewIngestionTaskService()svc.taskPublisher = publisher- ctx := t.Context()- _, err := svc.CreateAndEnqueue(ctx, &entity.IngestionTask{DocumentID: "doc-1", UserID: "user-1", DatasetID: "kb-1", Status: common.CREATED})- if err == nil {- t.Fatal("expected CreateAndEnqueue to reject existing created task")+ task, err := svc.CreateAndEnqueue(t.Context(), &entity.IngestionTask{+ DocumentID: "doc-1",+ UserID: "user-1",+ DatasetID: "kb-1",+ Status: common.CREATED,+ })+ if err != nil {+ t.Fatalf("CreateAndEnqueue failed: %v", err)+ }+ if task.ID != "task-1" || task.Status != common.SCHEDULED {+ t.Fatalf("task = %s/%s, want task-1/SCHEDULED", task.ID, task.Status)}- if len(publisher.messages) != 0 {- t.Fatalf("expected no published messages, got %+v", publisher.messages)+ if len(publisher.messages) != 1 || publisher.messages[0].TaskID != task.ID {+ t.Fatalf("unexpected published messages: %+v", publisher.messages)}}