RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
fix: align Go ingestion progress and pipeline selection (#18366)
buua4361ef4ddddec5f
diff --git a/internal/dao/ingestion.go b/internal/dao/ingestion.goindex 94410964d7..2afef863e4 100644--- a/internal/dao/ingestion.go+++ b/internal/dao/ingestion.go@@ -310,3 +310,23 @@ func (dao *IngestionTaskLogDAO) DeleteByTaskID(ctx context.Context, db *gorm.DB,result := db.WithContext(ctx).Unscoped().Where("task_id = ?", taskID).Delete(&entity.IngestionTaskLog{})return result.RowsAffected, result.Error}++// DeleteComponentLogsByTaskID removes component lifecycle rows from a new run+// while preserving checkpoint rows such as run_count for task history.+func (dao *IngestionTaskLogDAO) DeleteComponentLogsByTaskID(ctx context.Context, db *gorm.DB, taskID string) (int64, error) {+ var logs []*entity.IngestionTaskLog+ if err := db.WithContext(ctx).Where("task_id = ?", taskID).Find(&logs).Error; err != nil {+ return 0, err+ }+ ids := make([]int, 0, len(logs))+ for _, log := range logs {+ if log != nil && len(log.Checkpoint) == 0 {+ ids = append(ids, log.ID)+ }+ }+ if len(ids) == 0 {+ return 0, nil+ }+ result := db.WithContext(ctx).Unscoped().Where("id IN ?", ids).Delete(&entity.IngestionTaskLog{})+ return result.RowsAffected, result.Error+}diff --git a/internal/handler/document.go b/internal/handler/document.goindex 3847b97f25..55b84e7398 100644--- a/internal/handler/document.go+++ b/internal/handler/document.go@@ -1107,6 +1107,13 @@ func (h *DocumentHandler) DownloadDocument(c *gin.Context) {}func mapDocumentListItem(doc *entity.DocumentListItem, metaFields map[string]interface{}) map[string]interface{} {+ processDuration := doc.ProcessDuration+ if doc.Run != nil && strings.TrimSpace(*doc.Run) == "1" && doc.ProcessBeginAt != nil {+ processDuration = time.Since(*doc.ProcessBeginAt).Seconds()+ if processDuration < 0 {+ processDuration = 0+ }+ }item := map[string]interface{}{"id": doc.ID,"dataset_id": doc.KbID,@@ -1121,7 +1128,7 @@ func mapDocumentListItem(doc *entity.DocumentListItem, metaFields map[string]int"progress": doc.Progress,"progress_msg": stringValue(doc.ProgressMsg),"process_begin_at": formatTimePtr(doc.ProcessBeginAt),- "process_duration": doc.ProcessDuration,+ "process_duration": processDuration,"suffix": doc.Suffix,"run": mapRunStatus(doc.Run),"status": stringValue(doc.Status),diff --git a/internal/ingestion/service/ingestion_service.go b/internal/ingestion/service/ingestion_service.goindex 34d31c3f99..b221a48299 100644--- a/internal/ingestion/service/ingestion_service.go+++ b/internal/ingestion/service/ingestion_service.go@@ -595,6 +595,10 @@ func (e *Ingestor) runTask(ctx context.Context, task *entity.IngestionTask) boolcommon.Error(fmt.Sprintf("Failed to increment run count for task %s", task.ID), err)return e.markFailed(ctx, task.ID)}+ if err := e.ingestionTaskSvc.ClearComponentProgress(ctx, task.ID); err != nil {+ common.Error(fmt.Sprintf("Failed to clear previous component progress for task %s", task.ID), err)+ return e.markFailed(ctx, task.ID)+ }// This is a new run (IncrementRunCount succeeded). Any Redis cancel flag// that exists now is stale — a leftover from a previous run whosediff --git a/internal/ingestion/service/progress_sink.go b/internal/ingestion/service/progress_sink.goindex c48b1f86b6..efb5469e3a 100644--- a/internal/ingestion/service/progress_sink.go+++ b/internal/ingestion/service/progress_sink.go@@ -18,10 +18,12 @@ package serviceimport ("context"+ "errors""fmt""strings""sync""sync/atomic"+ "time""ragflow/internal/common""ragflow/internal/dao"@@ -32,8 +34,7 @@ import ()// progressLogMaxChars bounds the accumulated run log mirrored into-// document.progress_msg. Mirrors Python's TASK_MAX_LOG_LENGTH-// (api/db/services/task_service.py).+// document.progress_msg.const progressLogMaxChars = 3000// progressSink implements pipeline.ProgressSink. It is the single writer of@@ -54,9 +55,11 @@ type progressSink struct {// logMu guards the accumulated run log below: OnComponentProgress fires// from concurrent parallel-branch goroutines, so both the lazy seed and// the append must hold the same mutex.- logMu sync.Mutex- seeded bool- log strings.Builder+ logMu sync.Mutex+ seeded bool+ log strings.Builder+ pending []string+ now func() time.Time}// docProgressSvc is the subset of *service.DocumentService the sink needs to@@ -65,6 +68,7 @@ type progressSink struct {// full DocumentService surface.type docProgressSvc interface {GetDocumentByID(ctx context.Context, docID string) (*documentpkg.DocumentResponse, error)+ UpdateRunState(ctx context.Context, docID string, progress float64, run string) errorUpdateRunProgress(ctx context.Context, docID string, progress float64, run, progressMsg string) error}@@ -77,6 +81,7 @@ func newProgressSink(ctx context.Context, taskSvc *servicepkg.IngestionTaskServireturn &progressSink{taskSvc: taskSvc,docSvc: documentpkg.NewDocumentService(),+ now: time.Now,}}@@ -104,58 +109,94 @@ func (s *progressSink) OnComponentProgress(ctx context.Context, ev pipeline.Progreturn}progress, run := deriveDocumentProgress(agg, int(total))- if err = s.docSvc.UpdateRunProgress(ctx, ev.DocumentID, progress, run, s.accumulateLog(ctx, ev.DocumentID, ev.Message)); err != nil {+ if err = s.updateDocumentProgress(ctx, ev.DocumentID, progress, run, ev.Message); err != nil {common.Error(fmt.Sprintf("progressSink: mirror progress to document %s for task %s failed: %v", ev.DocumentID, ev.TaskID, err), err)}}-// accumulateLog appends one component lifecycle line to the run log and-// returns the full accumulated log for mirroring into document.progress_msg.-// The document-details dialog renders progress_msg verbatim as a multi-line-// log (whitespace-pre-line), matching Python's append-and-trim semantics in-// TaskService.update_progress; a single overwrite would leave the dialog-// showing only the latest line. The buffer is lazily seeded from the-// document row on the first event so a resumed run keeps the lines logged-// before the crash/redelivery instead of restarting empty (StartRunning-// resets the row to "" on a fresh run, which then seeds an empty log).+// updateDocumentProgress serializes log accumulation with the corresponding+// document write. Without holding the same lock across both operations, a+// slower database write can store an older snapshot after a newer event has+// already been persisted.+func (s *progressSink) updateDocumentProgress(ctx context.Context, docID string, progress float64, run, msg string) error {+ s.logMu.Lock()+ defer s.logMu.Unlock()+ log, err := s.accumulateLogLocked(ctx, docID, msg)+ if err != nil {+ // Keep the current run state durable even when the existing log cannot+ // be seeded. The next event will retry the seed and persist the log.+ stateErr := s.docSvc.UpdateRunState(ctx, docID, progress, run)+ return errors.Join(err, stateErr)+ }+ return s.docSvc.UpdateRunProgress(ctx, docID, progress, run, log)+}++// accumulateLog appends one component lifecycle line to the run log. It is+// retained as a small test seam; production writes use updateDocumentProgress+// so the append and document update share one lock.func (s *progressSink) accumulateLog(ctx context.Context, docID, msg string) string {s.logMu.Lock()defer s.logMu.Unlock()+ log, _ := s.accumulateLogLocked(ctx, docID, msg)+ return log+}++// accumulateLogLocked appends one component lifecycle line to the run log and+// returns the full accumulated log, or an error when the existing log could+// not be seeded. The caller must hold logMu.+func (s *progressSink) accumulateLogLocked(ctx context.Context, docID, msg string) (string, error) {if !s.seeded {- s.seeded = truedoc, err := s.docSvc.GetDocumentByID(ctx, docID)- switch {- case err != nil:+ if err != nil {+ if msg != "" {+ s.pending = append(s.pending, msg)+ }common.Warn(fmt.Sprintf("progressSink: seed run log from document %s: %v", docID, err))- case doc != nil && doc.ProgressMsg != nil:+ return "", fmt.Errorf("seed run log for document %s: %w", docID, err)+ }+ s.seeded = true+ if doc != nil && doc.ProgressMsg != nil {s.log.WriteString(*doc.ProgressMsg)}+ for _, pending := range s.pending {+ s.appendLogLineLocked(pending)+ }+ s.pending = nil}if msg == "" {- return s.log.String()+ return s.log.String(), nil}+ s.appendLogLineLocked(msg)+ return s.log.String(), nil+}++func (s *progressSink) appendLogLineLocked(msg string) {if s.log.Len() > 0 {s.log.WriteByte('\n')}+ now := time.Now+ if s.now != nil {+ now = s.now+ }+ s.log.WriteString(now().Format("15:04:05"))+ s.log.WriteString(": ")s.log.WriteString(msg)log := trimLogHead(s.log.String(), progressLogMaxChars)if len(log) != s.log.Len() {s.log.Reset()s.log.WriteString(log)}- return log}// trimLogHead drops whole lines from the head of text until the remainder-// fits maxChars, keeping the newest lines. Mirrors Python's-// trim_header_by_lines: text without a fitting newline boundary is returned-// unchanged.+// fits maxChars, keeping the newest lines. Text without a fitting newline+// boundary is returned unchanged.func trimLogHead(text string, maxChars int) string {if len(text) <= maxChars {return text}for i := 0; i < len(text); i++ {- if text[i] == '\n' && len(text)-i <= maxChars {+ if text[i] == '\n' && len(text)-(i+1) <= maxChars {return text[i+1:]}}@@ -176,5 +217,9 @@ func deriveDocumentProgress(agg *dao.TaskProgress, total int) (float64, string)case agg.Done > 0 || agg.Running > 0:run = string(entity.TaskStatusRunning)}- return agg.Percent / 100, run+ progress := agg.Percent / 100+ if progress > 1 {+ progress = 1+ }+ return progress, run}diff --git a/internal/ingestion/service/progress_sink_test.go b/internal/ingestion/service/progress_sink_test.goindex 0c1cff2bc6..32833d7c18 100644--- a/internal/ingestion/service/progress_sink_test.go+++ b/internal/ingestion/service/progress_sink_test.go@@ -18,11 +18,13 @@ package serviceimport ("context"+ "errors""fmt""runtime""strings""sync""testing"+ "time""ragflow/internal/dao""ragflow/internal/entity"@@ -150,10 +152,49 @@ type stubDocProgressSvc struct {docErr error}+func setProgressSinkTestClock(sink *progressSink) {+ sink.now = func() time.Time {+ return time.Date(2026, 8, 17, 3, 4, 5, 0, time.UTC)+ }+}++type blockingDocProgressSvc struct {+ mu sync.Mutex+ firstStarted chan struct{}+ releaseFirst chan struct{}+ messages []string+ updates int+}++func (s *blockingDocProgressSvc) GetDocumentByID(context.Context, string) (*document.DocumentResponse, error) {+ return nil, nil+}++func (s *blockingDocProgressSvc) UpdateRunState(context.Context, string, float64, string) error {+ return nil+}++func (s *blockingDocProgressSvc) UpdateRunProgress(_ context.Context, _ string, _ float64, _ string, msg string) error {+ s.mu.Lock()+ s.updates+++ update := s.updates+ s.messages = append(s.messages, msg)+ s.mu.Unlock()+ if update == 1 {+ close(s.firstStarted)+ <-s.releaseFirst+ }+ return nil+}+func (s *stubDocProgressSvc) GetDocumentByID(ctx context.Context, docID string) (*document.DocumentResponse, error) {return s.doc, s.docErr}+func (s *stubDocProgressSvc) UpdateRunState(context.Context, string, float64, string) error {+ return nil+}+func (s *stubDocProgressSvc) UpdateRunProgress(ctx context.Context, docID string, progress float64, run, progressMsg string) error {s.calls++s.gotDocID = docID@@ -174,6 +215,7 @@ func TestProgressSinkPersistsViaService(t *testing.T) {ctx := t.Context()sink := newProgressSink(ctx, servicepkg.NewIngestionTaskService())+ setProgressSinkTestClock(sink)stub := &stubDocProgressSvc{}sink.docSvc = stub@@ -218,8 +260,8 @@ func TestProgressSinkPersistsViaService(t *testing.T) {if stub.gotRun != "1" {t.Fatalf("run = %q, want 1 (RUNNING)", stub.gotRun)}- if stub.gotMsg != "Parser Done" {- t.Fatalf("progress_msg = %q, want Parser Done", stub.gotMsg)+ if stub.gotMsg != "03:04:05: Parser Done" {+ t.Fatalf("progress_msg = %q, want timestamped Parser Done", stub.gotMsg)}}@@ -234,6 +276,7 @@ func TestProgressSinkAccumulatesProgressLog(t *testing.T) {ctx := t.Context()sink := newProgressSink(ctx, servicepkg.NewIngestionTaskService())+ setProgressSinkTestClock(sink)stub := &stubDocProgressSvc{}sink.docSvc = stubsink.OnComponentTotal(ctx, taskID, 2)@@ -245,15 +288,14 @@ func TestProgressSinkAccumulatesProgressLog(t *testing.T) {TaskID: taskID, DocumentID: docID, Component: "Parser", Phase: 1, Message: "Parser Done",})- want := "File:naive Done\nParser Done"+ want := "03:04:05: File:naive Done\n03:04:05: Parser Done"if stub.gotMsg != want {t.Fatalf("progress_msg = %q, want multi-line log %q", stub.gotMsg, want)}}// TestProgressSinkSeedsLogFromDocument verifies the accumulated log keeps the-// lines already stored on the document row, so a resumed run (post-crash-// redelivery) appends to the pre-crash log instead of restarting empty.+// lines already stored on the document row when a run resumes.func TestProgressSinkSeedsLogFromDocument(t *testing.T) {db := testutil.SetupTestDB(t)cleanup := testutil.ReplaceDBForTest(t, db)@@ -262,6 +304,7 @@ func TestProgressSinkSeedsLogFromDocument(t *testing.T) {ctx := t.Context()sink := newProgressSink(ctx, servicepkg.NewIngestionTaskService())+ setProgressSinkTestClock(sink)seed := "File:naive Done"stub := &stubDocProgressSvc{doc: &document.DocumentResponse{ProgressMsg: &seed}}sink.docSvc = stub@@ -271,15 +314,74 @@ func TestProgressSinkSeedsLogFromDocument(t *testing.T) {TaskID: taskID, DocumentID: docID, Component: "Parser", Phase: 1, Message: "Parser Done",})- want := "File:naive Done\nParser Done"+ want := "File:naive Done\n03:04:05: Parser Done"if stub.gotMsg != want {t.Fatalf("progress_msg = %q, want seeded multi-line log %q", stub.gotMsg, want)}}+func TestProgressSinkRetriesSeedAfterReadFailure(t *testing.T) {+ seed := "existing"+ stub := &stubDocProgressSvc{docErr: errors.New("temporary read failure")}+ sink := &progressSink{docSvc: stub}++ if got := sink.accumulateLog(context.Background(), "doc-1", "pending"); got != "" {+ t.Fatalf("log after failed seed = %q, want empty mirror value", got)+ }++ stub.docErr = nil+ stub.doc = &document.DocumentResponse{ProgressMsg: &seed}+ setProgressSinkTestClock(sink)+ got := sink.accumulateLog(context.Background(), "doc-1", "current")+ if want := "existing\n03:04:05: pending\n03:04:05: current"; got != want {+ t.Fatalf("log after seed retry = %q, want %q", got, want)+ }+}++func TestProgressSinkSerializesLogAndDocumentWrite(t *testing.T) {+ stub := &blockingDocProgressSvc{+ firstStarted: make(chan struct{}),+ releaseFirst: make(chan struct{}),+ }+ sink := &progressSink{docSvc: stub}+ setProgressSinkTestClock(sink)++ firstDone := make(chan struct{})+ go func() {+ defer close(firstDone)+ if err := sink.updateDocumentProgress(context.Background(), "doc-1", 0.1, "1", "first"); err != nil {+ t.Errorf("first update: %v", err)+ }+ }()+ <-stub.firstStarted++ secondDone := make(chan struct{})+ go func() {+ defer close(secondDone)+ if err := sink.updateDocumentProgress(context.Background(), "doc-1", 0.2, "1", "second"); err != nil {+ t.Errorf("second update: %v", err)+ }+ }()++ select {+ case <-secondDone:+ t.Fatal("second update completed before the first document write was released")+ default:+ }+ close(stub.releaseFirst)+ <-firstDone+ <-secondDone++ stub.mu.Lock()+ defer stub.mu.Unlock()+ if len(stub.messages) != 2 || stub.messages[1] != "03:04:05: first\n03:04:05: second" {+ t.Fatalf("document writes = %q, want accumulated second write", stub.messages)+ }+}+// TestProgressSinkTrimsLogHead verifies the accumulated log is head-trimmed at// line boundaries once it exceeds progressLogMaxChars, keeping the newest lines-// (Python trim_header_by_lines parity).+// while keeping the newest complete lines.func TestProgressSinkTrimsLogHead(t *testing.T) {db := testutil.SetupTestDB(t)cleanup := testutil.ReplaceDBForTest(t, db)@@ -288,6 +390,7 @@ func TestProgressSinkTrimsLogHead(t *testing.T) {ctx := t.Context()sink := newProgressSink(ctx, servicepkg.NewIngestionTaskService())+ setProgressSinkTestClock(sink)stub := &stubDocProgressSvc{}sink.docSvc = stubsink.OnComponentTotal(ctx, taskID, 2)@@ -304,7 +407,7 @@ func TestProgressSinkTrimsLogHead(t *testing.T) {if len(stub.gotMsg) > progressLogMaxChars {t.Fatalf("progress_msg len = %d, exceeds %d", len(stub.gotMsg), progressLogMaxChars)}- if stub.gotMsg != tail {+ if stub.gotMsg != "03:04:05: "+tail {t.Fatalf("progress_msg keeps the newest line: got prefix %q, want %q", stub.gotMsg[:20], tail[:20])}}@@ -352,7 +455,7 @@ func TestTrimLogHead(t *testing.T) {{name: "short text unchanged", text: "a\nb", max: 10, want: "a\nb"},{name: "exactly max unchanged", text: "a\nb", max: 3, want: "a\nb"},{name: "drops head line", text: "l1\nl2\nl3", max: 6, want: "l2\nl3"},- {name: "drops lines until remainder fits", text: "l1\nl2\nl3", max: 5, want: "l3"},+ {name: "keeps suffix at exact limit", text: "l1\nl2\nl3", max: 5, want: "l2\nl3"},{name: "no fitting newline unchanged", text: "abcdef", max: 3, want: "abcdef"},}for _, tt := range tests {diff --git a/internal/ingestion/task/pipeline_executor.go b/internal/ingestion/task/pipeline_executor.goindex fca13f455e..4b36b0f781 100644--- a/internal/ingestion/task/pipeline_executor.go+++ b/internal/ingestion/task/pipeline_executor.go@@ -449,20 +449,70 @@ func (s *PipelineExecutor) recordPipelineLog(ctx context.Context, db *gorm.DB, dif err := json.Unmarshal([]byte(dsl), &dslMap); err != nil {dslMap = entity.JSONMap{"raw": dsl}}++ // The task context contains the document snapshot loaded when the task+ // started. Reload it here so the operation log reflects the final progress+ // state written by the progress sink, matching the Python operation-log+ // creation path.+ doc := s.taskCtx.Doc+ if db != nil {+ if persisted, err := dao.NewDocumentDAO().GetByID(ctx, db, docID); err == nil && persisted != nil {+ doc = *persisted+ } else if err != nil {+ common.Warn(fmt.Sprintf("failed to reload document %s for pipeline log: %v", docID, err))+ }+ }++ pipelineTitle := ""+ var pipelineAvatar *string+ if db != nil {+ if canvas, err := dao.NewUserCanvasDAO().GetByID(ctx, db, s.canvasID); err == nil && canvas != nil {+ if canvas.Title != nil {+ pipelineTitle = *canvas.Title+ }+ pipelineAvatar = canvas.Avatar+ } else if err != nil {+ common.Warn(fmt.Sprintf("failed to reload pipeline %s for operation log: %v", s.canvasID, err))+ }+ }++ operationStatus := status+ if doc.Run != nil && *doc.Run != "" {+ operationStatus = *doc.Run+ }+ statusValue := "1"+ if doc.Status != nil && *doc.Status != "" {+ statusValue = *doc.Status+ }+ sourceFrom := doc.SourceType+ if parts := strings.SplitN(sourceFrom, "/", 2); len(parts) > 0 {+ sourceFrom = parts[0]+ }+ documentName := ""+ if doc.Name != nil {+ documentName = *doc.Name+ }log := &entity.PipelineOperationLog{ID: utility.GenerateUUID(),TenantID: s.Tenant().ID,KbID: s.KB().ID,DocumentID: docID,PipelineID: &s.canvasID,+ PipelineTitle: &pipelineTitle,TaskType: string(entity.PipelineTaskTypeParse),DSL: dslMap,- ParserID: s.taskCtx.Doc.ParserID,- DocumentName: *s.Doc().Name,- DocumentSuffix: s.taskCtx.Doc.Suffix,- DocumentType: s.taskCtx.Doc.Type,- SourceFrom: s.taskCtx.Doc.SourceType,- OperationStatus: status,+ ParserID: doc.ParserID,+ DocumentName: documentName,+ DocumentSuffix: doc.Suffix,+ DocumentType: doc.Type,+ SourceFrom: sourceFrom,+ Progress: doc.Progress,+ ProgressMsg: doc.ProgressMsg,+ ProcessBeginAt: doc.ProcessBeginAt,+ ProcessDuration: doc.ProcessDuration,+ OperationStatus: operationStatus,+ Avatar: pipelineAvatar,+ Status: &statusValue,}if err := s.logCreateFunc(ctx, db, log); err != nil {common.Warn(fmt.Sprintf("failed to record pipeline log: %v", err))diff --git a/internal/ingestion/task/task_context.go b/internal/ingestion/task/task_context.goindex efd1af8391..9ca468a63e 100644--- a/internal/ingestion/task/task_context.go+++ b/internal/ingestion/task/task_context.go@@ -119,7 +119,7 @@ func LoadFromIngestionTask(ctx context.Context, ingestionTask *entity.IngestionTreturn nil, fmt.Errorf("error when load tenant %s: %w", kb.TenantID, err)}- pipelineID := resolvePipelineID(doc, kb)+ pipelineID := resolvePipelineID(doc)return &TaskContext{Ctx: ctx,@@ -131,16 +131,10 @@ func LoadFromIngestionTask(ctx context.Context, ingestionTask *entity.IngestionT}, nil}-func resolvePipelineID(doc *entity.Document, kb *entity.Knowledgebase) string {+// resolvePipelineID resolves the pipeline selected for a document.+func resolvePipelineID(doc *entity.Document) string {if doc != nil && doc.PipelineID != nil {- if pipelineID := strings.TrimSpace(*doc.PipelineID); pipelineID != "" {- return pipelineID- }- }- if kb != nil && kb.PipelineID != nil {- if pipelineID := strings.TrimSpace(*kb.PipelineID); pipelineID != "" {- return pipelineID- }+ return strings.TrimSpace(*doc.PipelineID)}return ""}diff --git a/internal/ingestion/task/task_context_test.go b/internal/ingestion/task/task_context_test.goindex b8970445fe..1515bacba5 100644--- a/internal/ingestion/task/task_context_test.go+++ b/internal/ingestion/task/task_context_test.go@@ -23,7 +23,7 @@ import ("ragflow/internal/ingestion/testutil")-func TestLoadFromIngestionTask_FallsBackToKnowledgebasePipelineID(t *testing.T) {+func TestLoadFromIngestionTask_DoesNotFallbackToKnowledgebasePipelineID(t *testing.T) {db := testutil.SetupTestDB(t)cleanup := testutil.ReplaceDBForTest(t, db)defer cleanup()@@ -65,8 +65,8 @@ func TestLoadFromIngestionTask_FallsBackToKnowledgebasePipelineID(t *testing.T)if err != nil {t.Fatalf("LoadFromIngestionTask: %v", err)}- if taskCtx.PipelineID != kbPipelineID {- t.Fatalf("PipelineID = %q, want %q", taskCtx.PipelineID, kbPipelineID)+ if taskCtx.PipelineID != "" {+ t.Fatalf("PipelineID = %q, want empty document pipeline selection", taskCtx.PipelineID)}}diff --git a/internal/service/document/document.go b/internal/service/document/document.goindex 89296d0f51..28fa708295 100644--- a/internal/service/document/document.go+++ b/internal/service/document/document.go@@ -77,26 +77,27 @@ type UpdateDocumentRequest struct {// DocumentResponse document responsetype DocumentResponse struct {- ID string `json:"id"`- Name *string `json:"name,omitempty"`- KbID string `json:"kb_id"`- ParserID string `json:"parser_id"`- PipelineID *string `json:"pipeline_id,omitempty"`- Type string `json:"type"`- SourceType string `json:"source_type"`- CreatedBy string `json:"created_by"`- Location *string `json:"location,omitempty"`- Size int64 `json:"size"`- TokenNum int64 `json:"token_num"`- ChunkNum int64 `json:"chunk_num"`- Progress float64 `json:"progress"`- ProgressMsg *string `json:"progress_msg,omitempty"`- ProcessDuration float64 `json:"process_duration"`- Suffix string `json:"suffix"`- Run *string `json:"run,omitempty"`- Status *string `json:"status,omitempty"`- CreatedAt string `json:"created_at"`- UpdatedAt string `json:"updated_at"`+ ID string `json:"id"`+ Name *string `json:"name,omitempty"`+ KbID string `json:"kb_id"`+ ParserID string `json:"parser_id"`+ PipelineID *string `json:"pipeline_id,omitempty"`+ Type string `json:"type"`+ SourceType string `json:"source_type"`+ CreatedBy string `json:"created_by"`+ Location *string `json:"location,omitempty"`+ Size int64 `json:"size"`+ TokenNum int64 `json:"token_num"`+ ChunkNum int64 `json:"chunk_num"`+ Progress float64 `json:"progress"`+ ProgressMsg *string `json:"progress_msg,omitempty"`+ ProcessBeginAt *time.Time `json:"process_begin_at,omitempty"`+ ProcessDuration float64 `json:"process_duration"`+ Suffix string `json:"suffix"`+ Run *string `json:"run,omitempty"`+ Status *string `json:"status,omitempty"`+ CreatedAt string `json:"created_at"`+ UpdatedAt string `json:"updated_at"`}type ThumbnailResponse struct {diff --git a/internal/service/document/document_crud.go b/internal/service/document/document_crud.goindex 2fc41953fd..23edc266f5 100644--- a/internal/service/document/document_crud.go+++ b/internal/service/document/document_crud.go@@ -194,11 +194,41 @@ func (s *DocumentService) ApplyDocCounts(ctx context.Context, docID, kbID string// progress_msg) reflects in-flight Go pipeline progress. Best-effort by// design; callers log and continue on error.func (s *DocumentService) UpdateRunProgress(ctx context.Context, docID string, progress float64, run, progressMsg string) error {- return s.documentDAO.UpdateByID(ctx, dao.DB, docID, map[string]interface{}{+ updates := map[string]interface{}{"progress": progress,"run": run,"progress_msg": progressMsg,- })+ }+ if doc, err := s.documentDAO.GetByID(ctx, dao.DB, docID); err != nil {+ return err+ } else if doc != nil && doc.ProcessBeginAt != nil {+ duration := time.Since(*doc.ProcessBeginAt).Seconds()+ if duration < 0 {+ duration = 0+ }+ updates["process_duration"] = duration+ }+ return s.documentDAO.UpdateByID(ctx, dao.DB, docID, updates)+}++// UpdateRunState mirrors live progress and status into the document row when+// the existing progress log cannot be read. It intentionally leaves the log+// untouched so a later event can retry seeding and append it safely.+func (s *DocumentService) UpdateRunState(ctx context.Context, docID string, progress float64, run string) error {+ updates := map[string]interface{}{+ "progress": progress,+ "run": run,+ }+ if doc, err := s.documentDAO.GetByID(ctx, dao.DB, docID); err != nil {+ return err+ } else if doc != nil && doc.ProcessBeginAt != nil {+ duration := time.Since(*doc.ProcessBeginAt).Seconds()+ if duration < 0 {+ duration = 0+ }+ updates["process_duration"] = duration+ }+ return s.documentDAO.UpdateByID(ctx, dao.DB, docID, updates)}// DeleteDocument delete document — delegates to full cleanup logic.diff --git a/internal/service/document/document_list.go b/internal/service/document/document_list.goindex 58e8382bad..e01c9358f9 100644--- a/internal/service/document/document_list.go+++ b/internal/service/document/document_list.go@@ -223,6 +223,7 @@ func (s *DocumentService) toResponse(doc *entity.Document) *DocumentResponse {ChunkNum: doc.ChunkNum,Progress: doc.Progress,ProgressMsg: doc.ProgressMsg,+ ProcessBeginAt: doc.ProcessBeginAt,ProcessDuration: doc.ProcessDuration,Suffix: doc.Suffix,Run: doc.Run,diff --git a/internal/service/document/document_test.go b/internal/service/document/document_test.goindex aede64b5f6..67e1c53e14 100644--- a/internal/service/document/document_test.go+++ b/internal/service/document/document_test.go@@ -3095,6 +3095,10 @@ func TestUpdateRunProgressMirrorsFields(t *testing.T) {db := setupServiceTestDB(t)pushServiceDB(t, db)insertTestDoc(t, "doc-1", "kb-1", 0, 0)+ begin := time.Now().Add(-2 * time.Second)+ if err := db.Model(&entity.Document{}).Where("id = ?", "doc-1").Update("process_begin_at", begin).Error; err != nil {+ t.Fatalf("set process_begin_at: %v", err)+ }svc := testDocumentService(t)ctx := t.Context()@@ -3114,6 +3118,9 @@ func TestUpdateRunProgressMirrorsFields(t *testing.T) {if doc.ProgressMsg == nil || *doc.ProgressMsg != "halfway" {t.Fatalf("progress_msg = %v, want halfway", doc.ProgressMsg)}+ if doc.ProcessDuration <= 0 {+ t.Fatalf("process_duration = %v, want positive live duration", doc.ProcessDuration)+ }}func TestFileDeleteRemovesLinkedDocument(t *testing.T) {diff --git a/internal/service/ingestion_task_service.go b/internal/service/ingestion_task_service.goindex 201957a5da..55674b9f78 100644--- a/internal/service/ingestion_task_service.go+++ b/internal/service/ingestion_task_service.go@@ -479,6 +479,13 @@ func (s *IngestionTaskService) RecordComponentProgress(ctx context.Context, taskreturn s.ingestionTaskLogDAO.Create(ctx, dao.DB, entry)}+// ClearComponentProgress removes lifecycle rows left by a previous attempt of+// the same reusable ingestion task. Run-count checkpoint rows are retained.+func (s *IngestionTaskService) ClearComponentProgress(ctx context.Context, taskID string) error {+ _, err := s.ingestionTaskLogDAO.DeleteComponentLogsByTaskID(ctx, dao.DB, taskID)+ return err+}+// AggregateTaskProgress returns the SQL-aggregated component progress for a// task (done/failed/running/percent against the given total denominator).func (s *IngestionTaskService) AggregateTaskProgress(ctx context.Context, taskID string, total int) (*dao.TaskProgress, error) {diff --git a/web/src/hooks/use-document-request.ts b/web/src/hooks/use-document-request.tsindex 04ad407827..c28d478c59 100644--- a/web/src/hooks/use-document-request.ts+++ b/web/src/hooks/use-document-request.ts@@ -52,6 +52,7 @@ import {useQueryClient,} from '@tanstack/react-query';import { useDebounce } from 'ahooks';+import dayjs from 'dayjs';import { get } from 'lodash';import { useCallback, useEffect, useMemo, useState } from 'react';import { IHighlight } from 'react-pdf-highlighter';@@ -91,6 +92,17 @@ export const enum DocumentStructureApiAction {}const DocumentKeys = {+ list: (searchString?: string, pagination?: unknown, filter?: unknown) =>+ [+ DocumentApiAction.FetchDocumentList,+ searchString,+ pagination,+ filter,+ ] as const,+ filter: (searchString?: string, knowledgeId?: string) =>+ [DocumentApiAction.FetchDocumentFilter, searchString, knowledgeId] as const,+ thumbnails: (ids: string[]) =>+ [DocumentApiAction.FetchDocumentThumbnails, ids] as const,byIds: (ids: string[]) =>[DocumentApiAction.FetchDocumentList, 'byIds', ids] as const,};@@ -149,7 +161,7 @@ export const useUploadDocument = () => {if (code === 0 || code === 500) {queryClient.invalidateQueries({- queryKey: [DocumentApiAction.FetchDocumentList],+ queryKey: DocumentKeys.list(),});}return ret;@@ -191,12 +203,7 @@ export const useFetchDocumentList = (loop = true) => {docs: IDocumentInfo[];total: number;}>({- queryKey: [- DocumentApiAction.FetchDocumentList,- debouncedSearchString,- pagination,- filterValue,- ],+ queryKey: DocumentKeys.list(debouncedSearchString, pagination, filterValue),initialData: { docs: [], total: 0 },refetchInterval: isLoop ? 5000 : false,enabled: !!knowledgeId || !!id,@@ -231,7 +238,7 @@ export const useFetchDocumentList = (loop = true) => {);if (ret.data.code === 0) {queryClient.invalidateQueries({- queryKey: [DocumentApiAction.FetchDocumentFilter],+ queryKey: DocumentKeys.filter(),});return ret.data.data;}@@ -320,11 +327,7 @@ export const useGetDocumentFilter = (): {const [open, setOpen] = useState<number>(0);const datasetId = knowledgeId || id;const { data } = useQuery({- queryKey: [- DocumentApiAction.FetchDocumentFilter,- debouncedSearchString,- knowledgeId,- ],+ queryKey: DocumentKeys.filter(debouncedSearchString, knowledgeId),queryFn: async () => {if (!datasetId) {return;@@ -379,7 +382,7 @@ export const useSetDocumentStatus = () => {if (data.code === 0) {message.success(i18n.t('message.modified'));queryClient.invalidateQueries({- queryKey: [DocumentApiAction.FetchDocumentList],+ queryKey: DocumentKeys.list(),});}return data;@@ -408,9 +411,37 @@ export const useRunDocument = () => {run: number;option?: { delete: boolean; apply_kb: boolean };}) => {- queryClient.invalidateQueries({- queryKey: [DocumentApiAction.FetchDocumentList],- });+ if (run === 1) {+ const documentIdSet = new Set(documentIds);+ queryClient.setQueriesData<{+ docs: IDocumentInfo[];+ total: number;+ }>({ queryKey: DocumentKeys.list() }, (current) => {+ if (!current) {+ return current;+ }+ return {+ ...current,+ docs: current.docs.map((doc) =>+ documentIdSet.has(doc.id)+ ? {+ ...doc,+ run: RunningStatus.RUNNING,+ progress: 0,+ process_duration: 0,+ process_begin_at: dayjs().format('YYYY-MM-DD HH:mm:ss'),+ progress_msg: '',+ }+ : doc,+ ),+ };+ });+ }+ if (run !== 1) {+ queryClient.invalidateQueries({+ queryKey: DocumentKeys.list(),+ });+ }const ret = await kbService.documentIngest({doc_ids: documentIds,run,@@ -418,14 +449,29 @@ export const useRunDocument = () => {});const code = get(ret, 'data.code');if (code === 0) {+ // For a start request, keep the optimistic running state until the+ // polling query observes the worker's state. Invalidating here can+ // immediately fetch the pre-worker "not started" row and disable+ // polling again.+ if (run !== 1) {+ queryClient.invalidateQueries({+ queryKey: DocumentKeys.list(),+ });+ }+ message.success(i18n.t('message.operated'));+ } else {queryClient.invalidateQueries({- queryKey: [DocumentApiAction.FetchDocumentList],+ queryKey: DocumentKeys.list(),});- message.success(i18n.t('message.operated'));}return code;},+ onError: () => {+ queryClient.invalidateQueries({+ queryKey: DocumentKeys.list(),+ });+ },});const runDocumentByIds = useCallback(@@ -475,7 +521,7 @@ export const useRemoveDocument = () => {if (data.code === 0) {message.success(i18n.t('message.deleted'));queryClient.invalidateQueries({- queryKey: [DocumentApiAction.FetchDocumentList],+ queryKey: DocumentKeys.list(),});}return data.code;@@ -509,7 +555,7 @@ export const useSaveDocumentName = () => {if (data.code === 0) {message.success(i18n.t('message.renamed'));queryClient.invalidateQueries({- queryKey: [DocumentApiAction.FetchDocumentList],+ queryKey: DocumentKeys.list(),});}return data.code;@@ -561,7 +607,7 @@ export const useSetDocumentParser = () => {);if (data.code === 0) {queryClient.invalidateQueries({- queryKey: [DocumentApiAction.FetchDocumentList],+ queryKey: DocumentKeys.list(),});message.success(i18n.t('message.modified'));@@ -626,7 +672,7 @@ export const useSetDocumentPipelineParser = () => {);if (data.code === 0) {queryClient.invalidateQueries({- queryKey: [DocumentApiAction.FetchDocumentList],+ queryKey: DocumentKeys.list(),});message.success(i18n.t('message.modified'));@@ -656,7 +702,7 @@ export const useSetDocumentMeta = () => {if (data?.code === 0) {queryClient.invalidateQueries({- queryKey: [DocumentApiAction.FetchDocumentList],+ queryKey: DocumentKeys.list(),});message.success(i18n.t('message.modified'));@@ -690,7 +736,7 @@ export const useCreateDocument = () => {if (data.code === 0) {if (page === 1) {queryClient.invalidateQueries({- queryKey: [DocumentApiAction.FetchDocumentList],+ queryKey: DocumentKeys.list(),});} else {setPaginationParams(); // fetch document list@@ -740,7 +786,7 @@ export const useGetChunkHighlights = (export const useFetchDocumentThumbnailsByIds = () => {const [ids, setDocumentIds] = useState<string[]>([]);const { data } = useQuery<Record<string, string>>({- queryKey: [DocumentApiAction.FetchDocumentThumbnails, ids],+ queryKey: DocumentKeys.thumbnails(ids),enabled: ids.length > 0,initialData: {},queryFn: async () => {diff --git a/web/src/pages/dataset/dataset/parsing-card.tsx b/web/src/pages/dataset/dataset/parsing-card.tsxindex 85572d5def..0a75b33ef5 100644--- a/web/src/pages/dataset/dataset/parsing-card.tsx+++ b/web/src/pages/dataset/dataset/parsing-card.tsx@@ -51,12 +51,12 @@ export const PopoverContent = ({ record }: IProps) => {{key: 'knowledgeDetails.process_duration',label: t('processDuration'),- children: `${record.process_duration.toFixed(2)} s`,+ children: `${(record.process_duration || 0).toFixed(2)} s`,},{key: 'progress_msg',label: t('knowledgeDetails.progressMsg'),- children: replaceText(record.progress_msg.trim()),+ children: replaceText((record.progress_msg || '').trim()),},];