Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

fix: align Go ingestion progress and pipeline selection (#18366)

buua4361ef4ddddec5f
diff --git a/internal/dao/ingestion.go b/internal/dao/ingestion.go
index 94410964d7..2afef863e4 100644
--- a/internal/dao/ingestion.go
+++ b/internal/dao/ingestion.go
@@ -310,3 +310,23 @@ func (dao *IngestionTaskLogDAO) DeleteByTaskID(ctx context.Context, db *gorm.DB,
result := db.WithContext(ctx).Unscoped().Where("task_id = ?", taskID).Delete(&entity.IngestionTaskLog{})
return result.RowsAffected, result.Error
}
+
+// DeleteComponentLogsByTaskID removes component lifecycle rows from a new run
+// while preserving checkpoint rows such as run_count for task history.
+func (dao *IngestionTaskLogDAO) DeleteComponentLogsByTaskID(ctx context.Context, db *gorm.DB, taskID string) (int64, error) {
+ var logs []*entity.IngestionTaskLog
+ if err := db.WithContext(ctx).Where("task_id = ?", taskID).Find(&logs).Error; err != nil {
+ return 0, err
+ }
+ ids := make([]int, 0, len(logs))
+ for _, log := range logs {
+ if log != nil && len(log.Checkpoint) == 0 {
+ ids = append(ids, log.ID)
+ }
+ }
+ if len(ids) == 0 {
+ return 0, nil
+ }
+ result := db.WithContext(ctx).Unscoped().Where("id IN ?", ids).Delete(&entity.IngestionTaskLog{})
+ return result.RowsAffected, result.Error
+}
diff --git a/internal/handler/document.go b/internal/handler/document.go
index 3847b97f25..55b84e7398 100644
--- a/internal/handler/document.go
+++ b/internal/handler/document.go
@@ -1107,6 +1107,13 @@ func (h *DocumentHandler) DownloadDocument(c *gin.Context) {
}
func mapDocumentListItem(doc *entity.DocumentListItem, metaFields map[string]interface{}) map[string]interface{} {
+ processDuration := doc.ProcessDuration
+ if doc.Run != nil && strings.TrimSpace(*doc.Run) == "1" && doc.ProcessBeginAt != nil {
+ processDuration = time.Since(*doc.ProcessBeginAt).Seconds()
+ if processDuration < 0 {
+ processDuration = 0
+ }
+ }
item := map[string]interface{}{
"id": doc.ID,
"dataset_id": doc.KbID,
@@ -1121,7 +1128,7 @@ func mapDocumentListItem(doc *entity.DocumentListItem, metaFields map[string]int
"progress": doc.Progress,
"progress_msg": stringValue(doc.ProgressMsg),
"process_begin_at": formatTimePtr(doc.ProcessBeginAt),
- "process_duration": doc.ProcessDuration,
+ "process_duration": processDuration,
"suffix": doc.Suffix,
"run": mapRunStatus(doc.Run),
"status": stringValue(doc.Status),
diff --git a/internal/ingestion/service/ingestion_service.go b/internal/ingestion/service/ingestion_service.go
index 34d31c3f99..b221a48299 100644
--- a/internal/ingestion/service/ingestion_service.go
+++ b/internal/ingestion/service/ingestion_service.go
@@ -595,6 +595,10 @@ func (e *Ingestor) runTask(ctx context.Context, task *entity.IngestionTask) bool
common.Error(fmt.Sprintf("Failed to increment run count for task %s", task.ID), err)
return e.markFailed(ctx, task.ID)
}
+ if err := e.ingestionTaskSvc.ClearComponentProgress(ctx, task.ID); err != nil {
+ common.Error(fmt.Sprintf("Failed to clear previous component progress for task %s", task.ID), err)
+ return e.markFailed(ctx, task.ID)
+ }
// This is a new run (IncrementRunCount succeeded). Any Redis cancel flag
// that exists now is stale — a leftover from a previous run whose
diff --git a/internal/ingestion/service/progress_sink.go b/internal/ingestion/service/progress_sink.go
index c48b1f86b6..efb5469e3a 100644
--- a/internal/ingestion/service/progress_sink.go
+++ b/internal/ingestion/service/progress_sink.go
@@ -18,10 +18,12 @@ package service
import (
"context"
+ "errors"
"fmt"
"strings"
"sync"
"sync/atomic"
+ "time"
"ragflow/internal/common"
"ragflow/internal/dao"
@@ -32,8 +34,7 @@ import (
)
// progressLogMaxChars bounds the accumulated run log mirrored into
-// document.progress_msg. Mirrors Python's TASK_MAX_LOG_LENGTH
-// (api/db/services/task_service.py).
+// document.progress_msg.
const progressLogMaxChars = 3000
// progressSink implements pipeline.ProgressSink. It is the single writer of
@@ -54,9 +55,11 @@ type progressSink struct {
// logMu guards the accumulated run log below: OnComponentProgress fires
// from concurrent parallel-branch goroutines, so both the lazy seed and
// the append must hold the same mutex.
- logMu sync.Mutex
- seeded bool
- log strings.Builder
+ logMu sync.Mutex
+ seeded bool
+ log strings.Builder
+ pending []string
+ now func() time.Time
}
// docProgressSvc is the subset of *service.DocumentService the sink needs to
@@ -65,6 +68,7 @@ type progressSink struct {
// full DocumentService surface.
type docProgressSvc interface {
GetDocumentByID(ctx context.Context, docID string) (*documentpkg.DocumentResponse, error)
+ UpdateRunState(ctx context.Context, docID string, progress float64, run string) error
UpdateRunProgress(ctx context.Context, docID string, progress float64, run, progressMsg string) error
}
@@ -77,6 +81,7 @@ func newProgressSink(ctx context.Context, taskSvc *servicepkg.IngestionTaskServi
return &progressSink{
taskSvc: taskSvc,
docSvc: documentpkg.NewDocumentService(),
+ now: time.Now,
}
}
@@ -104,58 +109,94 @@ func (s *progressSink) OnComponentProgress(ctx context.Context, ev pipeline.Prog
return
}
progress, run := deriveDocumentProgress(agg, int(total))
- if err = s.docSvc.UpdateRunProgress(ctx, ev.DocumentID, progress, run, s.accumulateLog(ctx, ev.DocumentID, ev.Message)); err != nil {
+ if err = s.updateDocumentProgress(ctx, ev.DocumentID, progress, run, ev.Message); err != nil {
common.Error(fmt.Sprintf("progressSink: mirror progress to document %s for task %s failed: %v", ev.DocumentID, ev.TaskID, err), err)
}
}
-// accumulateLog appends one component lifecycle line to the run log and
-// returns the full accumulated log for mirroring into document.progress_msg.
-// The document-details dialog renders progress_msg verbatim as a multi-line
-// log (whitespace-pre-line), matching Python's append-and-trim semantics in
-// TaskService.update_progress; a single overwrite would leave the dialog
-// showing only the latest line. The buffer is lazily seeded from the
-// document row on the first event so a resumed run keeps the lines logged
-// before the crash/redelivery instead of restarting empty (StartRunning
-// resets the row to "" on a fresh run, which then seeds an empty log).
+// updateDocumentProgress serializes log accumulation with the corresponding
+// document write. Without holding the same lock across both operations, a
+// slower database write can store an older snapshot after a newer event has
+// already been persisted.
+func (s *progressSink) updateDocumentProgress(ctx context.Context, docID string, progress float64, run, msg string) error {
+ s.logMu.Lock()
+ defer s.logMu.Unlock()
+ log, err := s.accumulateLogLocked(ctx, docID, msg)
+ if err != nil {
+ // Keep the current run state durable even when the existing log cannot
+ // be seeded. The next event will retry the seed and persist the log.
+ stateErr := s.docSvc.UpdateRunState(ctx, docID, progress, run)
+ return errors.Join(err, stateErr)
+ }
+ return s.docSvc.UpdateRunProgress(ctx, docID, progress, run, log)
+}
+
+// accumulateLog appends one component lifecycle line to the run log. It is
+// retained as a small test seam; production writes use updateDocumentProgress
+// so the append and document update share one lock.
func (s *progressSink) accumulateLog(ctx context.Context, docID, msg string) string {
s.logMu.Lock()
defer s.logMu.Unlock()
+ log, _ := s.accumulateLogLocked(ctx, docID, msg)
+ return log
+}
+
+// accumulateLogLocked appends one component lifecycle line to the run log and
+// returns the full accumulated log, or an error when the existing log could
+// not be seeded. The caller must hold logMu.
+func (s *progressSink) accumulateLogLocked(ctx context.Context, docID, msg string) (string, error) {
if !s.seeded {
- s.seeded = true
doc, err := s.docSvc.GetDocumentByID(ctx, docID)
- switch {
- case err != nil:
+ if err != nil {
+ if msg != "" {
+ s.pending = append(s.pending, msg)
+ }
common.Warn(fmt.Sprintf("progressSink: seed run log from document %s: %v", docID, err))
- case doc != nil && doc.ProgressMsg != nil:
+ return "", fmt.Errorf("seed run log for document %s: %w", docID, err)
+ }
+ s.seeded = true
+ if doc != nil && doc.ProgressMsg != nil {
s.log.WriteString(*doc.ProgressMsg)
}
+ for _, pending := range s.pending {
+ s.appendLogLineLocked(pending)
+ }
+ s.pending = nil
}
if msg == "" {
- return s.log.String()
+ return s.log.String(), nil
}
+ s.appendLogLineLocked(msg)
+ return s.log.String(), nil
+}
+
+func (s *progressSink) appendLogLineLocked(msg string) {
if s.log.Len() > 0 {
s.log.WriteByte('\n')
}
+ now := time.Now
+ if s.now != nil {
+ now = s.now
+ }
+ s.log.WriteString(now().Format("15:04:05"))
+ s.log.WriteString(": ")
s.log.WriteString(msg)
log := trimLogHead(s.log.String(), progressLogMaxChars)
if len(log) != s.log.Len() {
s.log.Reset()
s.log.WriteString(log)
}
- return log
}
// trimLogHead drops whole lines from the head of text until the remainder
-// fits maxChars, keeping the newest lines. Mirrors Python's
-// trim_header_by_lines: text without a fitting newline boundary is returned
-// unchanged.
+// fits maxChars, keeping the newest lines. Text without a fitting newline
+// boundary is returned unchanged.
func trimLogHead(text string, maxChars int) string {
if len(text) <= maxChars {
return text
}
for i := 0; i < len(text); i++ {
- if text[i] == '\n' && len(text)-i <= maxChars {
+ if text[i] == '\n' && len(text)-(i+1) <= maxChars {
return text[i+1:]
}
}
@@ -176,5 +217,9 @@ func deriveDocumentProgress(agg *dao.TaskProgress, total int) (float64, string)
case agg.Done > 0 || agg.Running > 0:
run = string(entity.TaskStatusRunning)
}
- return agg.Percent / 100, run
+ progress := agg.Percent / 100
+ if progress > 1 {
+ progress = 1
+ }
+ return progress, run
}
diff --git a/internal/ingestion/service/progress_sink_test.go b/internal/ingestion/service/progress_sink_test.go
index 0c1cff2bc6..32833d7c18 100644
--- a/internal/ingestion/service/progress_sink_test.go
+++ b/internal/ingestion/service/progress_sink_test.go
@@ -18,11 +18,13 @@ package service
import (
"context"
+ "errors"
"fmt"
"runtime"
"strings"
"sync"
"testing"
+ "time"
"ragflow/internal/dao"
"ragflow/internal/entity"
@@ -150,10 +152,49 @@ type stubDocProgressSvc struct {
docErr error
}
+func setProgressSinkTestClock(sink *progressSink) {
+ sink.now = func() time.Time {
+ return time.Date(2026, 8, 17, 3, 4, 5, 0, time.UTC)
+ }
+}
+
+type blockingDocProgressSvc struct {
+ mu sync.Mutex
+ firstStarted chan struct{}
+ releaseFirst chan struct{}
+ messages []string
+ updates int
+}
+
+func (s *blockingDocProgressSvc) GetDocumentByID(context.Context, string) (*document.DocumentResponse, error) {
+ return nil, nil
+}
+
+func (s *blockingDocProgressSvc) UpdateRunState(context.Context, string, float64, string) error {
+ return nil
+}
+
+func (s *blockingDocProgressSvc) UpdateRunProgress(_ context.Context, _ string, _ float64, _ string, msg string) error {
+ s.mu.Lock()
+ s.updates++
+ update := s.updates
+ s.messages = append(s.messages, msg)
+ s.mu.Unlock()
+ if update == 1 {
+ close(s.firstStarted)
+ <-s.releaseFirst
+ }
+ return nil
+}
+
func (s *stubDocProgressSvc) GetDocumentByID(ctx context.Context, docID string) (*document.DocumentResponse, error) {
return s.doc, s.docErr
}
+func (s *stubDocProgressSvc) UpdateRunState(context.Context, string, float64, string) error {
+ return nil
+}
+
func (s *stubDocProgressSvc) UpdateRunProgress(ctx context.Context, docID string, progress float64, run, progressMsg string) error {
s.calls++
s.gotDocID = docID
@@ -174,6 +215,7 @@ func TestProgressSinkPersistsViaService(t *testing.T) {
ctx := t.Context()
sink := newProgressSink(ctx, servicepkg.NewIngestionTaskService())
+ setProgressSinkTestClock(sink)
stub := &stubDocProgressSvc{}
sink.docSvc = stub
@@ -218,8 +260,8 @@ func TestProgressSinkPersistsViaService(t *testing.T) {
if stub.gotRun != "1" {
t.Fatalf("run = %q, want 1 (RUNNING)", stub.gotRun)
}
- if stub.gotMsg != "Parser Done" {
- t.Fatalf("progress_msg = %q, want Parser Done", stub.gotMsg)
+ if stub.gotMsg != "03:04:05: Parser Done" {
+ t.Fatalf("progress_msg = %q, want timestamped Parser Done", stub.gotMsg)
}
}
@@ -234,6 +276,7 @@ func TestProgressSinkAccumulatesProgressLog(t *testing.T) {
ctx := t.Context()
sink := newProgressSink(ctx, servicepkg.NewIngestionTaskService())
+ setProgressSinkTestClock(sink)
stub := &stubDocProgressSvc{}
sink.docSvc = stub
sink.OnComponentTotal(ctx, taskID, 2)
@@ -245,15 +288,14 @@ func TestProgressSinkAccumulatesProgressLog(t *testing.T) {
TaskID: taskID, DocumentID: docID, Component: "Parser", Phase: 1, Message: "Parser Done",
})
- want := "File:naive Done\nParser Done"
+ want := "03:04:05: File:naive Done\n03:04:05: Parser Done"
if stub.gotMsg != want {
t.Fatalf("progress_msg = %q, want multi-line log %q", stub.gotMsg, want)
}
}
// TestProgressSinkSeedsLogFromDocument verifies the accumulated log keeps the
-// lines already stored on the document row, so a resumed run (post-crash
-// redelivery) appends to the pre-crash log instead of restarting empty.
+// lines already stored on the document row when a run resumes.
func TestProgressSinkSeedsLogFromDocument(t *testing.T) {
db := testutil.SetupTestDB(t)
cleanup := testutil.ReplaceDBForTest(t, db)
@@ -262,6 +304,7 @@ func TestProgressSinkSeedsLogFromDocument(t *testing.T) {
ctx := t.Context()
sink := newProgressSink(ctx, servicepkg.NewIngestionTaskService())
+ setProgressSinkTestClock(sink)
seed := "File:naive Done"
stub := &stubDocProgressSvc{doc: &document.DocumentResponse{ProgressMsg: &seed}}
sink.docSvc = stub
@@ -271,15 +314,74 @@ func TestProgressSinkSeedsLogFromDocument(t *testing.T) {
TaskID: taskID, DocumentID: docID, Component: "Parser", Phase: 1, Message: "Parser Done",
})
- want := "File:naive Done\nParser Done"
+ want := "File:naive Done\n03:04:05: Parser Done"
if stub.gotMsg != want {
t.Fatalf("progress_msg = %q, want seeded multi-line log %q", stub.gotMsg, want)
}
}
+func TestProgressSinkRetriesSeedAfterReadFailure(t *testing.T) {
+ seed := "existing"
+ stub := &stubDocProgressSvc{docErr: errors.New("temporary read failure")}
+ sink := &progressSink{docSvc: stub}
+
+ if got := sink.accumulateLog(context.Background(), "doc-1", "pending"); got != "" {
+ t.Fatalf("log after failed seed = %q, want empty mirror value", got)
+ }
+
+ stub.docErr = nil
+ stub.doc = &document.DocumentResponse{ProgressMsg: &seed}
+ setProgressSinkTestClock(sink)
+ got := sink.accumulateLog(context.Background(), "doc-1", "current")
+ if want := "existing\n03:04:05: pending\n03:04:05: current"; got != want {
+ t.Fatalf("log after seed retry = %q, want %q", got, want)
+ }
+}
+
+func TestProgressSinkSerializesLogAndDocumentWrite(t *testing.T) {
+ stub := &blockingDocProgressSvc{
+ firstStarted: make(chan struct{}),
+ releaseFirst: make(chan struct{}),
+ }
+ sink := &progressSink{docSvc: stub}
+ setProgressSinkTestClock(sink)
+
+ firstDone := make(chan struct{})
+ go func() {
+ defer close(firstDone)
+ if err := sink.updateDocumentProgress(context.Background(), "doc-1", 0.1, "1", "first"); err != nil {
+ t.Errorf("first update: %v", err)
+ }
+ }()
+ <-stub.firstStarted
+
+ secondDone := make(chan struct{})
+ go func() {
+ defer close(secondDone)
+ if err := sink.updateDocumentProgress(context.Background(), "doc-1", 0.2, "1", "second"); err != nil {
+ t.Errorf("second update: %v", err)
+ }
+ }()
+
+ select {
+ case <-secondDone:
+ t.Fatal("second update completed before the first document write was released")
+ default:
+ }
+ close(stub.releaseFirst)
+ <-firstDone
+ <-secondDone
+
+ stub.mu.Lock()
+ defer stub.mu.Unlock()
+ if len(stub.messages) != 2 || stub.messages[1] != "03:04:05: first\n03:04:05: second" {
+ t.Fatalf("document writes = %q, want accumulated second write", stub.messages)
+ }
+}
+
// TestProgressSinkTrimsLogHead verifies the accumulated log is head-trimmed at
// line boundaries once it exceeds progressLogMaxChars, keeping the newest lines
-// (Python trim_header_by_lines parity).
+// while keeping the newest complete lines.
func TestProgressSinkTrimsLogHead(t *testing.T) {
db := testutil.SetupTestDB(t)
cleanup := testutil.ReplaceDBForTest(t, db)
@@ -288,6 +390,7 @@ func TestProgressSinkTrimsLogHead(t *testing.T) {
ctx := t.Context()
sink := newProgressSink(ctx, servicepkg.NewIngestionTaskService())
+ setProgressSinkTestClock(sink)
stub := &stubDocProgressSvc{}
sink.docSvc = stub
sink.OnComponentTotal(ctx, taskID, 2)
@@ -304,7 +407,7 @@ func TestProgressSinkTrimsLogHead(t *testing.T) {
if len(stub.gotMsg) > progressLogMaxChars {
t.Fatalf("progress_msg len = %d, exceeds %d", len(stub.gotMsg), progressLogMaxChars)
}
- if stub.gotMsg != tail {
+ if stub.gotMsg != "03:04:05: "+tail {
t.Fatalf("progress_msg keeps the newest line: got prefix %q, want %q", stub.gotMsg[:20], tail[:20])
}
}
@@ -352,7 +455,7 @@ func TestTrimLogHead(t *testing.T) {
{name: "short text unchanged", text: "a\nb", max: 10, want: "a\nb"},
{name: "exactly max unchanged", text: "a\nb", max: 3, want: "a\nb"},
{name: "drops head line", text: "l1\nl2\nl3", max: 6, want: "l2\nl3"},
- {name: "drops lines until remainder fits", text: "l1\nl2\nl3", max: 5, want: "l3"},
+ {name: "keeps suffix at exact limit", text: "l1\nl2\nl3", max: 5, want: "l2\nl3"},
{name: "no fitting newline unchanged", text: "abcdef", max: 3, want: "abcdef"},
}
for _, tt := range tests {
diff --git a/internal/ingestion/task/pipeline_executor.go b/internal/ingestion/task/pipeline_executor.go
index fca13f455e..4b36b0f781 100644
--- a/internal/ingestion/task/pipeline_executor.go
+++ b/internal/ingestion/task/pipeline_executor.go
@@ -449,20 +449,70 @@ func (s *PipelineExecutor) recordPipelineLog(ctx context.Context, db *gorm.DB, d
if err := json.Unmarshal([]byte(dsl), &dslMap); err != nil {
dslMap = entity.JSONMap{"raw": dsl}
}
+
+ // The task context contains the document snapshot loaded when the task
+ // started. Reload it here so the operation log reflects the final progress
+ // state written by the progress sink, matching the Python operation-log
+ // creation path.
+ doc := s.taskCtx.Doc
+ if db != nil {
+ if persisted, err := dao.NewDocumentDAO().GetByID(ctx, db, docID); err == nil && persisted != nil {
+ doc = *persisted
+ } else if err != nil {
+ common.Warn(fmt.Sprintf("failed to reload document %s for pipeline log: %v", docID, err))
+ }
+ }
+
+ pipelineTitle := ""
+ var pipelineAvatar *string
+ if db != nil {
+ if canvas, err := dao.NewUserCanvasDAO().GetByID(ctx, db, s.canvasID); err == nil && canvas != nil {
+ if canvas.Title != nil {
+ pipelineTitle = *canvas.Title
+ }
+ pipelineAvatar = canvas.Avatar
+ } else if err != nil {
+ common.Warn(fmt.Sprintf("failed to reload pipeline %s for operation log: %v", s.canvasID, err))
+ }
+ }
+
+ operationStatus := status
+ if doc.Run != nil && *doc.Run != "" {
+ operationStatus = *doc.Run
+ }
+ statusValue := "1"
+ if doc.Status != nil && *doc.Status != "" {
+ statusValue = *doc.Status
+ }
+ sourceFrom := doc.SourceType
+ if parts := strings.SplitN(sourceFrom, "/", 2); len(parts) > 0 {
+ sourceFrom = parts[0]
+ }
+ documentName := ""
+ if doc.Name != nil {
+ documentName = *doc.Name
+ }
log := &entity.PipelineOperationLog{
ID: utility.GenerateUUID(),
TenantID: s.Tenant().ID,
KbID: s.KB().ID,
DocumentID: docID,
PipelineID: &s.canvasID,
+ PipelineTitle: &pipelineTitle,
TaskType: string(entity.PipelineTaskTypeParse),
DSL: dslMap,
- ParserID: s.taskCtx.Doc.ParserID,
- DocumentName: *s.Doc().Name,
- DocumentSuffix: s.taskCtx.Doc.Suffix,
- DocumentType: s.taskCtx.Doc.Type,
- SourceFrom: s.taskCtx.Doc.SourceType,
- OperationStatus: status,
+ ParserID: doc.ParserID,
+ DocumentName: documentName,
+ DocumentSuffix: doc.Suffix,
+ DocumentType: doc.Type,
+ SourceFrom: sourceFrom,
+ Progress: doc.Progress,
+ ProgressMsg: doc.ProgressMsg,
+ ProcessBeginAt: doc.ProcessBeginAt,
+ ProcessDuration: doc.ProcessDuration,
+ OperationStatus: operationStatus,
+ Avatar: pipelineAvatar,
+ Status: &statusValue,
}
if err := s.logCreateFunc(ctx, db, log); err != nil {
common.Warn(fmt.Sprintf("failed to record pipeline log: %v", err))
diff --git a/internal/ingestion/task/task_context.go b/internal/ingestion/task/task_context.go
index efd1af8391..9ca468a63e 100644
--- a/internal/ingestion/task/task_context.go
+++ b/internal/ingestion/task/task_context.go
@@ -119,7 +119,7 @@ func LoadFromIngestionTask(ctx context.Context, ingestionTask *entity.IngestionT
return nil, fmt.Errorf("error when load tenant %s: %w", kb.TenantID, err)
}
- pipelineID := resolvePipelineID(doc, kb)
+ pipelineID := resolvePipelineID(doc)
return &TaskContext{
Ctx: ctx,
@@ -131,16 +131,10 @@ func LoadFromIngestionTask(ctx context.Context, ingestionTask *entity.IngestionT
}, nil
}
-func resolvePipelineID(doc *entity.Document, kb *entity.Knowledgebase) string {
+// resolvePipelineID resolves the pipeline selected for a document.
+func resolvePipelineID(doc *entity.Document) string {
if doc != nil && doc.PipelineID != nil {
- if pipelineID := strings.TrimSpace(*doc.PipelineID); pipelineID != "" {
- return pipelineID
- }
- }
- if kb != nil && kb.PipelineID != nil {
- if pipelineID := strings.TrimSpace(*kb.PipelineID); pipelineID != "" {
- return pipelineID
- }
+ return strings.TrimSpace(*doc.PipelineID)
}
return ""
}
diff --git a/internal/ingestion/task/task_context_test.go b/internal/ingestion/task/task_context_test.go
index b8970445fe..1515bacba5 100644
--- a/internal/ingestion/task/task_context_test.go
+++ b/internal/ingestion/task/task_context_test.go
@@ -23,7 +23,7 @@ import (
"ragflow/internal/ingestion/testutil"
)
-func TestLoadFromIngestionTask_FallsBackToKnowledgebasePipelineID(t *testing.T) {
+func TestLoadFromIngestionTask_DoesNotFallbackToKnowledgebasePipelineID(t *testing.T) {
db := testutil.SetupTestDB(t)
cleanup := testutil.ReplaceDBForTest(t, db)
defer cleanup()
@@ -65,8 +65,8 @@ func TestLoadFromIngestionTask_FallsBackToKnowledgebasePipelineID(t *testing.T)
if err != nil {
t.Fatalf("LoadFromIngestionTask: %v", err)
}
- if taskCtx.PipelineID != kbPipelineID {
- t.Fatalf("PipelineID = %q, want %q", taskCtx.PipelineID, kbPipelineID)
+ if taskCtx.PipelineID != "" {
+ t.Fatalf("PipelineID = %q, want empty document pipeline selection", taskCtx.PipelineID)
}
}
diff --git a/internal/service/document/document.go b/internal/service/document/document.go
index 89296d0f51..28fa708295 100644
--- a/internal/service/document/document.go
+++ b/internal/service/document/document.go
@@ -77,26 +77,27 @@ type UpdateDocumentRequest struct {
// DocumentResponse document response
type DocumentResponse struct {
- ID string `json:"id"`
- Name *string `json:"name,omitempty"`
- KbID string `json:"kb_id"`
- ParserID string `json:"parser_id"`
- PipelineID *string `json:"pipeline_id,omitempty"`
- Type string `json:"type"`
- SourceType string `json:"source_type"`
- CreatedBy string `json:"created_by"`
- Location *string `json:"location,omitempty"`
- Size int64 `json:"size"`
- TokenNum int64 `json:"token_num"`
- ChunkNum int64 `json:"chunk_num"`
- Progress float64 `json:"progress"`
- ProgressMsg *string `json:"progress_msg,omitempty"`
- ProcessDuration float64 `json:"process_duration"`
- Suffix string `json:"suffix"`
- Run *string `json:"run,omitempty"`
- Status *string `json:"status,omitempty"`
- CreatedAt string `json:"created_at"`
- UpdatedAt string `json:"updated_at"`
+ ID string `json:"id"`
+ Name *string `json:"name,omitempty"`
+ KbID string `json:"kb_id"`
+ ParserID string `json:"parser_id"`
+ PipelineID *string `json:"pipeline_id,omitempty"`
+ Type string `json:"type"`
+ SourceType string `json:"source_type"`
+ CreatedBy string `json:"created_by"`
+ Location *string `json:"location,omitempty"`
+ Size int64 `json:"size"`
+ TokenNum int64 `json:"token_num"`
+ ChunkNum int64 `json:"chunk_num"`
+ Progress float64 `json:"progress"`
+ ProgressMsg *string `json:"progress_msg,omitempty"`
+ ProcessBeginAt *time.Time `json:"process_begin_at,omitempty"`
+ ProcessDuration float64 `json:"process_duration"`
+ Suffix string `json:"suffix"`
+ Run *string `json:"run,omitempty"`
+ Status *string `json:"status,omitempty"`
+ CreatedAt string `json:"created_at"`
+ UpdatedAt string `json:"updated_at"`
}
type ThumbnailResponse struct {
diff --git a/internal/service/document/document_crud.go b/internal/service/document/document_crud.go
index 2fc41953fd..23edc266f5 100644
--- a/internal/service/document/document_crud.go
+++ b/internal/service/document/document_crud.go
@@ -194,11 +194,41 @@ func (s *DocumentService) ApplyDocCounts(ctx context.Context, docID, kbID string
// progress_msg) reflects in-flight Go pipeline progress. Best-effort by
// design; callers log and continue on error.
func (s *DocumentService) UpdateRunProgress(ctx context.Context, docID string, progress float64, run, progressMsg string) error {
- return s.documentDAO.UpdateByID(ctx, dao.DB, docID, map[string]interface{}{
+ updates := map[string]interface{}{
"progress": progress,
"run": run,
"progress_msg": progressMsg,
- })
+ }
+ if doc, err := s.documentDAO.GetByID(ctx, dao.DB, docID); err != nil {
+ return err
+ } else if doc != nil && doc.ProcessBeginAt != nil {
+ duration := time.Since(*doc.ProcessBeginAt).Seconds()
+ if duration < 0 {
+ duration = 0
+ }
+ updates["process_duration"] = duration
+ }
+ return s.documentDAO.UpdateByID(ctx, dao.DB, docID, updates)
+}
+
+// UpdateRunState mirrors live progress and status into the document row when
+// the existing progress log cannot be read. It intentionally leaves the log
+// untouched so a later event can retry seeding and append it safely.
+func (s *DocumentService) UpdateRunState(ctx context.Context, docID string, progress float64, run string) error {
+ updates := map[string]interface{}{
+ "progress": progress,
+ "run": run,
+ }
+ if doc, err := s.documentDAO.GetByID(ctx, dao.DB, docID); err != nil {
+ return err
+ } else if doc != nil && doc.ProcessBeginAt != nil {
+ duration := time.Since(*doc.ProcessBeginAt).Seconds()
+ if duration < 0 {
+ duration = 0
+ }
+ updates["process_duration"] = duration
+ }
+ return s.documentDAO.UpdateByID(ctx, dao.DB, docID, updates)
}
// DeleteDocument delete document — delegates to full cleanup logic.
diff --git a/internal/service/document/document_list.go b/internal/service/document/document_list.go
index 58e8382bad..e01c9358f9 100644
--- a/internal/service/document/document_list.go
+++ b/internal/service/document/document_list.go
@@ -223,6 +223,7 @@ func (s *DocumentService) toResponse(doc *entity.Document) *DocumentResponse {
ChunkNum: doc.ChunkNum,
Progress: doc.Progress,
ProgressMsg: doc.ProgressMsg,
+ ProcessBeginAt: doc.ProcessBeginAt,
ProcessDuration: doc.ProcessDuration,
Suffix: doc.Suffix,
Run: doc.Run,
diff --git a/internal/service/document/document_test.go b/internal/service/document/document_test.go
index aede64b5f6..67e1c53e14 100644
--- a/internal/service/document/document_test.go
+++ b/internal/service/document/document_test.go
@@ -3095,6 +3095,10 @@ func TestUpdateRunProgressMirrorsFields(t *testing.T) {
db := setupServiceTestDB(t)
pushServiceDB(t, db)
insertTestDoc(t, "doc-1", "kb-1", 0, 0)
+ begin := time.Now().Add(-2 * time.Second)
+ if err := db.Model(&entity.Document{}).Where("id = ?", "doc-1").Update("process_begin_at", begin).Error; err != nil {
+ t.Fatalf("set process_begin_at: %v", err)
+ }
svc := testDocumentService(t)
ctx := t.Context()
@@ -3114,6 +3118,9 @@ func TestUpdateRunProgressMirrorsFields(t *testing.T) {
if doc.ProgressMsg == nil || *doc.ProgressMsg != "halfway" {
t.Fatalf("progress_msg = %v, want halfway", doc.ProgressMsg)
}
+ if doc.ProcessDuration <= 0 {
+ t.Fatalf("process_duration = %v, want positive live duration", doc.ProcessDuration)
+ }
}
func TestFileDeleteRemovesLinkedDocument(t *testing.T) {
diff --git a/internal/service/ingestion_task_service.go b/internal/service/ingestion_task_service.go
index 201957a5da..55674b9f78 100644
--- a/internal/service/ingestion_task_service.go
+++ b/internal/service/ingestion_task_service.go
@@ -479,6 +479,13 @@ func (s *IngestionTaskService) RecordComponentProgress(ctx context.Context, task
return s.ingestionTaskLogDAO.Create(ctx, dao.DB, entry)
}
+// ClearComponentProgress removes lifecycle rows left by a previous attempt of
+// the same reusable ingestion task. Run-count checkpoint rows are retained.
+func (s *IngestionTaskService) ClearComponentProgress(ctx context.Context, taskID string) error {
+ _, err := s.ingestionTaskLogDAO.DeleteComponentLogsByTaskID(ctx, dao.DB, taskID)
+ return err
+}
+
// AggregateTaskProgress returns the SQL-aggregated component progress for a
// task (done/failed/running/percent against the given total denominator).
func (s *IngestionTaskService) AggregateTaskProgress(ctx context.Context, taskID string, total int) (*dao.TaskProgress, error) {
diff --git a/web/src/hooks/use-document-request.ts b/web/src/hooks/use-document-request.ts
index 04ad407827..c28d478c59 100644
--- a/web/src/hooks/use-document-request.ts
+++ b/web/src/hooks/use-document-request.ts
@@ -52,6 +52,7 @@ import {
useQueryClient,
} from '@tanstack/react-query';
import { useDebounce } from 'ahooks';
+import dayjs from 'dayjs';
import { get } from 'lodash';
import { useCallback, useEffect, useMemo, useState } from 'react';
import { IHighlight } from 'react-pdf-highlighter';
@@ -91,6 +92,17 @@ export const enum DocumentStructureApiAction {
}
const DocumentKeys = {
+ list: (searchString?: string, pagination?: unknown, filter?: unknown) =>
+ [
+ DocumentApiAction.FetchDocumentList,
+ searchString,
+ pagination,
+ filter,
+ ] as const,
+ filter: (searchString?: string, knowledgeId?: string) =>
+ [DocumentApiAction.FetchDocumentFilter, searchString, knowledgeId] as const,
+ thumbnails: (ids: string[]) =>
+ [DocumentApiAction.FetchDocumentThumbnails, ids] as const,
byIds: (ids: string[]) =>
[DocumentApiAction.FetchDocumentList, 'byIds', ids] as const,
};
@@ -149,7 +161,7 @@ export const useUploadDocument = () => {
if (code === 0 || code === 500) {
queryClient.invalidateQueries({
- queryKey: [DocumentApiAction.FetchDocumentList],
+ queryKey: DocumentKeys.list(),
});
}
return ret;
@@ -191,12 +203,7 @@ export const useFetchDocumentList = (loop = true) => {
docs: IDocumentInfo[];
total: number;
}>({
- queryKey: [
- DocumentApiAction.FetchDocumentList,
- debouncedSearchString,
- pagination,
- filterValue,
- ],
+ queryKey: DocumentKeys.list(debouncedSearchString, pagination, filterValue),
initialData: { docs: [], total: 0 },
refetchInterval: isLoop ? 5000 : false,
enabled: !!knowledgeId || !!id,
@@ -231,7 +238,7 @@ export const useFetchDocumentList = (loop = true) => {
);
if (ret.data.code === 0) {
queryClient.invalidateQueries({
- queryKey: [DocumentApiAction.FetchDocumentFilter],
+ queryKey: DocumentKeys.filter(),
});
return ret.data.data;
}
@@ -320,11 +327,7 @@ export const useGetDocumentFilter = (): {
const [open, setOpen] = useState<number>(0);
const datasetId = knowledgeId || id;
const { data } = useQuery({
- queryKey: [
- DocumentApiAction.FetchDocumentFilter,
- debouncedSearchString,
- knowledgeId,
- ],
+ queryKey: DocumentKeys.filter(debouncedSearchString, knowledgeId),
queryFn: async () => {
if (!datasetId) {
return;
@@ -379,7 +382,7 @@ export const useSetDocumentStatus = () => {
if (data.code === 0) {
message.success(i18n.t('message.modified'));
queryClient.invalidateQueries({
- queryKey: [DocumentApiAction.FetchDocumentList],
+ queryKey: DocumentKeys.list(),
});
}
return data;
@@ -408,9 +411,37 @@ export const useRunDocument = () => {
run: number;
option?: { delete: boolean; apply_kb: boolean };
}) => {
- queryClient.invalidateQueries({
- queryKey: [DocumentApiAction.FetchDocumentList],
- });
+ if (run === 1) {
+ const documentIdSet = new Set(documentIds);
+ queryClient.setQueriesData<{
+ docs: IDocumentInfo[];
+ total: number;
+ }>({ queryKey: DocumentKeys.list() }, (current) => {
+ if (!current) {
+ return current;
+ }
+ return {
+ ...current,
+ docs: current.docs.map((doc) =>
+ documentIdSet.has(doc.id)
+ ? {
+ ...doc,
+ run: RunningStatus.RUNNING,
+ progress: 0,
+ process_duration: 0,
+ process_begin_at: dayjs().format('YYYY-MM-DD HH:mm:ss'),
+ progress_msg: '',
+ }
+ : doc,
+ ),
+ };
+ });
+ }
+ if (run !== 1) {
+ queryClient.invalidateQueries({
+ queryKey: DocumentKeys.list(),
+ });
+ }
const ret = await kbService.documentIngest({
doc_ids: documentIds,
run,
@@ -418,14 +449,29 @@ export const useRunDocument = () => {
});
const code = get(ret, 'data.code');
if (code === 0) {
+ // For a start request, keep the optimistic running state until the
+ // polling query observes the worker's state. Invalidating here can
+ // immediately fetch the pre-worker "not started" row and disable
+ // polling again.
+ if (run !== 1) {
+ queryClient.invalidateQueries({
+ queryKey: DocumentKeys.list(),
+ });
+ }
+ message.success(i18n.t('message.operated'));
+ } else {
queryClient.invalidateQueries({
- queryKey: [DocumentApiAction.FetchDocumentList],
+ queryKey: DocumentKeys.list(),
});
- message.success(i18n.t('message.operated'));
}
return code;
},
+ onError: () => {
+ queryClient.invalidateQueries({
+ queryKey: DocumentKeys.list(),
+ });
+ },
});
const runDocumentByIds = useCallback(
@@ -475,7 +521,7 @@ export const useRemoveDocument = () => {
if (data.code === 0) {
message.success(i18n.t('message.deleted'));
queryClient.invalidateQueries({
- queryKey: [DocumentApiAction.FetchDocumentList],
+ queryKey: DocumentKeys.list(),
});
}
return data.code;
@@ -509,7 +555,7 @@ export const useSaveDocumentName = () => {
if (data.code === 0) {
message.success(i18n.t('message.renamed'));
queryClient.invalidateQueries({
- queryKey: [DocumentApiAction.FetchDocumentList],
+ queryKey: DocumentKeys.list(),
});
}
return data.code;
@@ -561,7 +607,7 @@ export const useSetDocumentParser = () => {
);
if (data.code === 0) {
queryClient.invalidateQueries({
- queryKey: [DocumentApiAction.FetchDocumentList],
+ queryKey: DocumentKeys.list(),
});
message.success(i18n.t('message.modified'));
@@ -626,7 +672,7 @@ export const useSetDocumentPipelineParser = () => {
);
if (data.code === 0) {
queryClient.invalidateQueries({
- queryKey: [DocumentApiAction.FetchDocumentList],
+ queryKey: DocumentKeys.list(),
});
message.success(i18n.t('message.modified'));
@@ -656,7 +702,7 @@ export const useSetDocumentMeta = () => {
if (data?.code === 0) {
queryClient.invalidateQueries({
- queryKey: [DocumentApiAction.FetchDocumentList],
+ queryKey: DocumentKeys.list(),
});
message.success(i18n.t('message.modified'));
@@ -690,7 +736,7 @@ export const useCreateDocument = () => {
if (data.code === 0) {
if (page === 1) {
queryClient.invalidateQueries({
- queryKey: [DocumentApiAction.FetchDocumentList],
+ queryKey: DocumentKeys.list(),
});
} else {
setPaginationParams(); // fetch document list
@@ -740,7 +786,7 @@ export const useGetChunkHighlights = (
export const useFetchDocumentThumbnailsByIds = () => {
const [ids, setDocumentIds] = useState<string[]>([]);
const { data } = useQuery<Record<string, string>>({
- queryKey: [DocumentApiAction.FetchDocumentThumbnails, ids],
+ queryKey: DocumentKeys.thumbnails(ids),
enabled: ids.length > 0,
initialData: {},
queryFn: async () => {
diff --git a/web/src/pages/dataset/dataset/parsing-card.tsx b/web/src/pages/dataset/dataset/parsing-card.tsx
index 85572d5def..0a75b33ef5 100644
--- a/web/src/pages/dataset/dataset/parsing-card.tsx
+++ b/web/src/pages/dataset/dataset/parsing-card.tsx
@@ -51,12 +51,12 @@ export const PopoverContent = ({ record }: IProps) => {
{
key: 'knowledgeDetails.process_duration',
label: t('processDuration'),
- children: `${record.process_duration.toFixed(2)} s`,
+ children: `${(record.process_duration || 0).toFixed(2)} s`,
},
{
key: 'progress_msg',
label: t('knowledgeDetails.progressMsg'),
- children: replaceText(record.progress_msg.trim()),
+ children: replaceText((record.progress_msg || '').trim()),
},
];