RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
Revert "fix(ingestion): allow re-parsing a document whose parse was canceled at progress 0" (#19503)
Wang Qi607f98666468
diff --git a/internal/ingestion/service/ingestion_service.go b/internal/ingestion/service/ingestion_service.goindex 921341f322..f82719fe94 100644--- a/internal/ingestion/service/ingestion_service.go+++ b/internal/ingestion/service/ingestion_service.go@@ -695,31 +695,10 @@ func (e *Ingestor) executeTaskWithHeartbeat(ctx context.Context, taskCtx *taskpk})}-// ownsTaskRun reports whether the calling run still owns the task: a newer-// run (created by a supersede/re-parse) bumps the run counter, so a mismatch-// means this run's terminal write must be skipped. A run whose counter could-// not be read (myRun <= 0) conservatively keeps ownership. Uses a detached-// context: it is called from failure/cancel paths whose context is typically-// already cancelled.-func (e *Ingestor) ownsTaskRun(taskID string, myRun int) bool {- if myRun <= 0 {- return true- }- ctx, cancel := context.WithTimeout(context.WithoutCancel(e.ctx), 2*time.Second)- defer cancel()- current, ok := e.ingestionTaskSvc.CurrentRunCount(ctx, taskID)- if !ok {- return true- }- return current == myRun-}--// markStopped transitions the task to STOPPED (terminal). A RUNNING task-// first goes through RequestStop (RUNNING → STOPPING); a STOPPING task is-// settled directly; any other status (terminal, or CREATED/SCHEDULED because-// a supersede re-armed the task for a newer run) is treated as settled with-// nothing to finalize. Finally it cleans up the Redis cancel flag so that a-// future retry of the same task does not immediately re-cancel.+// markStopped transitions the task to STOPPED (terminal). It first calls+// RequestStop to handle RUNNING → STOPPING, then MarkStopped for the final+// STOPPING → STOPPED transition. Finally it cleans up the Redis cancel flag+// so that a future retry of the same task does not immediately re-cancel.// The caller's context is detached before touching the DB: markStopped runs// on failure/cancel/shutdown paths whose context is typically already// cancelled, and a contaminated context would make the terminal write fail@@ -727,29 +706,10 @@ func (e *Ingestor) ownsTaskRun(taskID string, myRun int) bool {func (e *Ingestor) markStopped(ctx context.Context, taskID string) bool {ctx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 5*time.Second)defer cancel()- // Only a RUNNING/STOPPING task still belongs to this run. A task in- // CREATED/SCHEDULED here was re-armed for a newer run by a supersede- // (re-parse of a canceled document); finalizing it would kill the- // replacement run before it starts, so treat it as already settled.- // Terminal statuses need no work either.- task, err := e.ingestionTaskSvc.GetTask(ctx, taskID)- if err != nil {- common.Error(fmt.Sprintf("markStopped: load task %s: %v", taskID, err), err)+ if _, err := e.ingestionTaskSvc.RequestStop(ctx, taskID); err != nil {+ common.Error(fmt.Sprintf("markStopped: RequestStop task %s: %v", taskID, err), err)return false}- switch task.Status {- case common.RUNNING:- if _, err := e.ingestionTaskSvc.RequestStop(ctx, taskID); err != nil {- common.Error(fmt.Sprintf("markStopped: RequestStop task %s: %v", taskID, err), err)- return false- }- case common.STOPPING:- default:- // COMPLETED/STOPPED/FAILED: already terminal. CREATED/SCHEDULED:- // re-armed by a newer run - nothing this run may finalize.- common.Info(fmt.Sprintf("markStopped: task %s is %s; nothing to finalize for this run", taskID, task.Status))- return true- }if err := e.ingestionTaskSvc.MarkStopped(ctx, taskID); err != nil {common.Error(fmt.Sprintf("markStopped: MarkStopped task %s: %v", taskID, err), err)return false@@ -812,11 +772,6 @@ func (e *Ingestor) runTask(ctx context.Context, task *entity.IngestionTask) bool}return ok}- // Capture this run's counter: if a re-parse supersedes this run (the task- // is force-finalized and re-armed while this worker is still inside a- // non-cancellable section), a newer run bumps the counter and this run- // must not overwrite the newer run's terminal state or document progress.- myRun, _ := e.ingestionTaskSvc.CurrentRunCount(dbCtx, task.ID)checkpointExists := e.checkpointExistsif checkpointExists == nil {checkpointExists = canvas.RedisCheckpointExists@@ -864,10 +819,6 @@ func (e *Ingestor) runTask(ctx context.Context, task *entity.IngestionTask) boolreturn false}common.Info(fmt.Sprintf("Task %s cancelled during pipeline", task.ID))- if !e.ownsTaskRun(task.ID, myRun) {- common.Info(fmt.Sprintf("Task %s run %d superseded by a newer run; skip cancel finalize", task.ID, myRun))- return true- }e.markCancelProgress(task)stopped := e.markStopped(ctx, task.ID)if stopped {@@ -877,10 +828,6 @@ func (e *Ingestor) runTask(ctx context.Context, task *entity.IngestionTask) bool}if errors.Is(err, context.DeadlineExceeded) {common.Info(fmt.Sprintf("Task %s timed out during pipeline", task.ID))- if !e.ownsTaskRun(task.ID, myRun) {- common.Info(fmt.Sprintf("Task %s run %d superseded by a newer run; skip timeout finalize", task.ID, myRun))- return true- }e.markTimeoutProgress(task)ok := e.markFailed(ctx, task.ID)if ok {@@ -889,10 +836,6 @@ func (e *Ingestor) runTask(ctx context.Context, task *entity.IngestionTask) boolreturn ok}common.Error(fmt.Sprintf("Task %s failed", task.ID), err)- if !e.ownsTaskRun(task.ID, myRun) {- common.Info(fmt.Sprintf("Task %s run %d superseded by a newer run; skip failure finalize", task.ID, myRun))- return true- }ok := e.markFailed(ctx, task.ID)if ok {e.recordTerminalPipelineLog(ctx, task, string(entity.TaskStatusFail))@@ -900,10 +843,6 @@ func (e *Ingestor) runTask(ctx context.Context, task *entity.IngestionTask) boolreturn ok}- if !e.ownsTaskRun(task.ID, myRun) {- common.Info(fmt.Sprintf("Task %s run %d superseded by a newer run; skip completion", task.ID, myRun))- return true- }if err := e.completeTask(ctx, task.ID); err != nil {common.Error(fmt.Sprintf("Task %s update status failed", task.ID), err)return falsediff --git a/internal/service/ingestion_task_service.go b/internal/service/ingestion_task_service.goindex 3bd3f6babf..8395578ed0 100644--- a/internal/service/ingestion_task_service.go+++ b/internal/service/ingestion_task_service.go@@ -46,20 +46,15 @@ type IngestionTaskService struct {ingestionTaskDAO *dao.IngestionTaskDAOingestionTaskLogDAO *dao.IngestionTaskLogDAOtaskPublisher TaskPublisher- // supersedeTerminalWait bounds how long supersedeAndRetryTask waits for a- // live worker to finalize a requested stop before force-finalizing the- // task. Overridable so tests do not pay the full production wait.- supersedeTerminalWait time.Duration}func NewIngestionTaskService() *IngestionTaskService {return &IngestionTaskService{- documentDAO: dao.NewDocumentDAO(),- userDAO: dao.NewUserDAO(),- ingestionTaskDAO: dao.NewIngestionTaskDAO(),- ingestionTaskLogDAO: dao.NewIngestionTaskLogDAO(),- taskPublisher: NewMessageQueueTaskPublisher(),- supersedeTerminalWait: defaultSupersedeTerminalWait,+ documentDAO: dao.NewDocumentDAO(),+ userDAO: dao.NewUserDAO(),+ ingestionTaskDAO: dao.NewIngestionTaskDAO(),+ ingestionTaskLogDAO: dao.NewIngestionTaskLogDAO(),+ taskPublisher: NewMessageQueueTaskPublisher(),}}@@ -346,7 +341,7 @@ func validateTransition(from, to string) error {if to == common.STOPPED {return nil}- case common.FAILED, common.STOPPED, common.COMPLETED:+ case common.FAILED, common.STOPPED:if to == common.CREATED {return nil}@@ -402,7 +397,7 @@ func (s *IngestionTaskService) CreateAndEnqueue(ctx context.Context, task *entitreturn nil, err}return s.markScheduledAfterPublish(ctx, existing.ID)- case common.FAILED, common.STOPPED, common.COMPLETED:+ case common.FAILED, common.STOPPED:originalStatus := existing.Statusexisting, err = s.transition(ctx, existing.ID, common.CREATED)if err != nil {@@ -420,8 +415,6 @@ func (s *IngestionTaskService) CreateAndEnqueue(ctx context.Context, task *entitreturn nil, err}return s.markScheduledAfterPublish(ctx, existing.ID)- case common.SCHEDULED, common.RUNNING, common.STOPPING:- return s.supersedeAndRetryTask(ctx, existing)default:return nil, fmt.Errorf("document id %s already exists, status: %s, task id: %s", task.DocumentID, existing.Status, existing.ID)}@@ -440,141 +433,6 @@ func (s *IngestionTaskService) CreateAndEnqueue(ctx context.Context, task *entitreturn s.markScheduledAfterPublish(ctx, created.ID)}-// defaultSupersedeTerminalWait bounds how long a re-parse request waits for a-// live worker to finalize a requested stop before force-finalizing the task.-// A live worker observes the stop (Redis cancel flag polled every 3s, plus the-// DB STOPPING fallback) well within this window; only a worker stuck in a-// non-cancellable section or a dead worker exceeds it.-const defaultSupersedeTerminalWait = 8 * time.Second--const supersedePollInterval = 250 * time.Millisecond--// supersedeAndRetryTask finalizes an in-flight ingestion task-// (SCHEDULED/RUNNING/STOPPING) and re-queues it for a fresh parse, mirroring-// the Python /documents/ingest endpoint, which accepts a parse request-// regardless of the previous task's in-flight state. Without this, a document-// whose task lingers in a non-terminal state — for example a parse canceled at-// progress 0 whose worker died before finalizing the stop (e.g. the service-// restarted mid-parse), or a scheduled task whose queue message was lost —-// could never be re-parsed: every attempt failed with "document id ... already-// exists, status: ...".-//-// For a task a worker may still be executing (RUNNING/STOPPING) the stop is-// requested first and the worker gets a bounded window to finalize on its own;-// a live worker settles within its 3s cancel poll. Only when that window-// expires is the task force-finalized. In every case the Redis cancel flag is-// cleared before the re-arm: the replacement run's pre-start cancel check-// (Ingestor.executeTask) reads the same flag and would abort the fresh parse-// before it starts, silently reproducing the "cannot re-parse" symptom.-func (s *IngestionTaskService) supersedeAndRetryTask(ctx context.Context, existing *entity.IngestionTask) (*entity.IngestionTask, error) {- inFlightStatus := existing.Status- if inFlightStatus == common.RUNNING {- // RequestStop moves RUNNING→STOPPING and sets the Redis cancel flag so- // a live worker aborts at its next cancellation checkpoint.- if _, err := s.RequestStop(ctx, existing.ID); err != nil {- // The worker may have finalized (stopped/completed/failed)- // concurrently; a terminal task is still supersedeable below.- refreshed, refreshErr := s.GetTask(ctx, existing.ID)- if refreshErr != nil || !isTerminalTaskStatus(refreshed.Status) {- return nil, fmt.Errorf("stop in-flight task %s before re-parse: %w", existing.ID, err)- }- existing = refreshed- }- }- if existing.Status == common.RUNNING || existing.Status == common.STOPPING {- final := s.awaitTerminalTask(ctx, existing.ID)- if final == nil {- var err error- if final, err = s.forceTerminalTask(ctx, existing.ID); err != nil {- return nil, fmt.Errorf("finalize in-flight task %s (%s) before re-parse: %w", existing.ID, inFlightStatus, err)- }- }- existing = final- }- if existing.Status == common.CREATED || existing.Status == common.SCHEDULED {- // Never claimed (or freshly re-armed by a concurrent request): no- // worker can be executing it, so finalize it immediately.- var err error- if existing, err = s.transition(ctx, existing.ID, common.STOPPED); err != nil {- return nil, fmt.Errorf("finalize in-flight task %s (%s) before re-parse: %w", existing.ID, inFlightStatus, err)- }- }- if !isTerminalTaskStatus(existing.Status) {- return nil, fmt.Errorf("task %s is in unexpected status %s before re-parse", existing.ID, existing.Status)- }- // The superseded run is final, so any leftover Redis cancel flag is stale- // for the replacement run (see the doc comment).- clearCancelFlag(ctx, existing.ID)- retried, err := s.transition(ctx, existing.ID, common.CREATED)- if err != nil {- return nil, fmt.Errorf("re-arm superseded task %s for re-parse: %w", existing.ID, err)- }- if err = s.enqueueTask(retried.ID); err != nil {- if rollbackErr := s.rollbackRetriedTask(ctx, retried.ID, existing.Status); rollbackErr != nil {- return nil, fmt.Errorf("enqueue task %s: %w (rollback failed: %w)", retried.ID, err, rollbackErr)- }- return nil, err- }- return s.markScheduledAfterPublish(ctx, retried.ID)-}--// awaitTerminalTask polls the task until it reaches a terminal status or the-// supersede wait budget expires. Returns the terminal task, or nil on timeout.-func (s *IngestionTaskService) awaitTerminalTask(ctx context.Context, taskID string) *entity.IngestionTask {- wait := s.supersedeTerminalWait- if wait <= 0 {- wait = defaultSupersedeTerminalWait- }- deadline := time.Now().Add(wait)- for {- task, err := s.GetTask(ctx, taskID)- if err == nil && isTerminalTaskStatus(task.Status) {- return task- }- if time.Now().After(deadline) {- return nil- }- select {- case <-ctx.Done():- return nil- case <-time.After(supersedePollInterval):- }- }-}--// forceTerminalTask drives a task that missed its bounded stop window to a-// terminal state: STOPPING→STOPPED directly; RUNNING via RequestStop first (a-// worker may still be inside a non-cancellable section — the flag that sets is-// cleared by the caller right after the finalize, and the worker's late-// terminal write is skipped by its run-count ownership check in the Ingestor);-// CREATED/SCHEDULED→STOPPED because no worker is executing them.-func (s *IngestionTaskService) forceTerminalTask(ctx context.Context, taskID string) (*entity.IngestionTask, error) {- task, err := s.GetTask(ctx, taskID)- if err != nil {- return nil, err- }- switch task.Status {- case common.STOPPING, common.CREATED, common.SCHEDULED:- return s.transition(ctx, taskID, common.STOPPED)- case common.RUNNING:- if _, err = s.RequestStop(ctx, taskID); err != nil {- if refreshed, refreshErr := s.GetTask(ctx, taskID); refreshErr == nil && isTerminalTaskStatus(refreshed.Status) {- return refreshed, nil- }- return nil, err- }- return s.transition(ctx, taskID, common.STOPPED)- case common.COMPLETED, common.STOPPED, common.FAILED:- return task, nil- default:- return nil, fmt.Errorf("task %s has unsupported status %s", taskID, task.Status)- }-}--func isTerminalTaskStatus(status string) bool {- return status == common.COMPLETED || status == common.STOPPED || status == common.FAILED-}-func (s *IngestionTaskService) rollbackRetriedTask(ctx context.Context, taskID, status string) error {updated, err := s.ingestionTaskDAO.UpdateStatusIfCurrent(ctx, dao.DB, taskID, common.CREATED, status)if err != nil {@@ -688,14 +546,6 @@ func (s *IngestionTaskService) AggregateTaskProgress(ctx context.Context, taskIDreturn s.ingestionTaskLogDAO.AggregateProgress(ctx, dao.DB, taskID, total)}-// CurrentRunCount returns the run counter of the most recent started run for-// the task. Workers capture it right after IncrementRunCount and use it to-// detect that their run was superseded (a newer run bumped the counter)-// before writing terminal state.-func (s *IngestionTaskService) CurrentRunCount(ctx context.Context, taskID string) (int, bool) {- return s.lastRunCount(ctx, taskID)-}-// lastRunCount scans all task logs (newest first) for a run_count entry,// skipping component-progress rows whose Checkpoint is empty. It returns// the counter and whether one was found.diff --git a/internal/service/ingestion_task_service_test.go b/internal/service/ingestion_task_service_test.goindex d444e57faf..0820046c16 100644--- a/internal/service/ingestion_task_service_test.go+++ b/internal/service/ingestion_task_service_test.go@@ -5,7 +5,6 @@ import ("strings""sync""testing"- "time""ragflow/internal/common""ragflow/internal/dao"@@ -811,107 +810,21 @@ func TestIngestionTaskServiceScheduleCreatedTasksContinuesAfterPublishFailure(t}}-func TestIngestionTaskServiceCreateAndEnqueueSupersedesInFlightTask(t *testing.T) {- for _, tc := range []struct {- name string- existing string- wantStatus string- }{- {- // A parse canceled at progress 0 whose worker has not finalized the- // stop yet leaves the task in STOPPING; re-parsing waits a bounded- // window for the worker, then force-finalizes and re-queues.- name: "stopping",- existing: common.STOPPING,- wantStatus: common.SCHEDULED,- },- {- // A task queued but never claimed by a worker (e.g. its queue- // message was lost) must not block re-parsing either.- name: "scheduled",- existing: common.SCHEDULED,- wantStatus: common.SCHEDULED,- },- {- // A task whose worker died mid-run stays RUNNING with no owner;- // re-parsing must stop it (the only RUNNING→CREATED path in the- // transition table is RUNNING→STOPPING→STOPPED→CREATED) and queue- // a fresh run.- name: "running",- existing: common.RUNNING,- wantStatus: common.SCHEDULED,- },- } {- t.Run(tc.name, func(t *testing.T) {- db := setupServiceTestDB(t)- pushServiceDB(t, db)- insertTestIngestionTaskWithStatus(t, "task-1", "user-1", "doc-1", "kb-1", tc.existing)-- publisher := &recordingTaskPublisher{}- svc := NewIngestionTaskService()- svc.taskPublisher = publisher- // No live worker in the test: do not pay the full production- // window before force-finalizing the in-flight task.- svc.supersedeTerminalWait = 50 * time.Millisecond-- task, err := svc.CreateAndEnqueue(t.Context(), &entity.IngestionTask{- DocumentID: "doc-1",- UserID: "user-1",- DatasetID: "kb-1",- Status: common.CREATED,- })- if err != nil {- t.Fatalf("CreateAndEnqueue failed: %v", err)- }-- if task.ID != "task-1" {- t.Fatalf("task ID = %q, want task-1 (superseded in place)", task.ID)- }- if task.Status != tc.wantStatus {- t.Fatalf("status = %q, want %q", task.Status, tc.wantStatus)- }- if len(publisher.messages) != 1 || publisher.messages[0].TaskID != task.ID {- t.Fatalf("unexpected published messages: %+v", publisher.messages)- }-- reloaded, err := dao.NewIngestionTaskDAO().GetByDocumentID(t.Context(), db, "doc-1")- if err != nil {- t.Fatalf("reload task: %v", err)- }- if reloaded == nil || reloaded.Status != tc.wantStatus {- t.Fatalf("expected superseded task to be re-queued, task=%+v", reloaded)- }- })- }-}--// TestIngestionTaskServiceCreateAndEnqueueRearmsCompletedTask locks in that a-// document whose previous parse COMPLETED can be re-parsed without first-// deleting its ingestion task: the terminal task row is re-armed in place,-// mirroring the FAILED/STOPPED handling.-func TestIngestionTaskServiceCreateAndEnqueueRearmsCompletedTask(t *testing.T) {+func TestIngestionTaskServiceCreateAndEnqueueRejectsActiveExistingTask(t *testing.T) {db := setupServiceTestDB(t)pushServiceDB(t, db)- insertTestIngestionTaskWithStatus(t, "task-1", "user-1", "doc-1", "kb-1", common.COMPLETED)-+ insertTestIngestionTaskWithStatus(t, "task-1", "user-1", "doc-1", "kb-1", common.SCHEDULED)publisher := &recordingTaskPublisher{}svc := NewIngestionTaskService()svc.taskPublisher = publisher- task, err := svc.CreateAndEnqueue(t.Context(), &entity.IngestionTask{- DocumentID: "doc-1",- UserID: "user-1",- DatasetID: "kb-1",- Status: common.CREATED,- })- if err != nil {- t.Fatalf("CreateAndEnqueue failed: %v", err)- }- if task.ID != "task-1" || task.Status != common.SCHEDULED {- t.Fatalf("task = %s/%s, want task-1/SCHEDULED", task.ID, task.Status)+ ctx := t.Context()+ _, err := svc.CreateAndEnqueue(ctx, &entity.IngestionTask{DocumentID: "doc-1", UserID: "user-1", DatasetID: "kb-1", Status: common.CREATED})+ if err == nil {+ t.Fatal("expected CreateAndEnqueue to reject existing created task")}- if len(publisher.messages) != 1 || publisher.messages[0].TaskID != task.ID {- t.Fatalf("unexpected published messages: %+v", publisher.messages)+ if len(publisher.messages) != 0 {+ t.Fatalf("expected no published messages, got %+v", publisher.messages)}}