RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
fix(ingestion): unify run duration and timestamp parsing log lines (#19849)
jay777218f8699cb6cd6
diff --git a/internal/ingestion/task/pipeline_executor.go b/internal/ingestion/task/pipeline_executor.goindex 310387520b..b45a7ac69a 100644--- a/internal/ingestion/task/pipeline_executor.go+++ b/internal/ingestion/task/pipeline_executor.go@@ -192,7 +192,18 @@ func (s *PipelineExecutor) Execute(ctx context.Context) (*PipelineResult, error)}if pipelineDSL != "" && s.taskCtx.IngestionTask != nil && s.taskCtx.IngestionTask.PipelineLogID != nil && *s.taskCtx.IngestionTask.PipelineLogID != "" {- s.recordPipelineLog(context.WithoutCancel(ctx), dao.DB, s.taskCtx.Doc.ID, pipelineDSL, string(entity.TaskStatusDone))+ var terminalDuration *float64+ if result != nil {+ terminalDuration = &result.Duration+ } else {+ // A successful run with no chunks never wrote a terminal duration+ // to the document, so recompute from the same process_begin_at+ // anchor here instead of letting the log copy the stale mid-run+ // value.+ d := s.terminalDuration(start)+ terminalDuration = &d+ }+ s.recordPipelineLog(context.WithoutCancel(ctx), dao.DB, s.taskCtx.Doc.ID, pipelineDSL, string(entity.TaskStatusDone), terminalDuration)}return result, nil@@ -335,13 +346,30 @@ func (s *PipelineExecutor) processOutput(ctx context.Context, pipelineOutput mapMetadata: metadata,ChunkCount: chunkCount,TokenConsumption: embeddingTokenConsumption,- Duration: time.Since(start).Seconds(),+ Duration: s.terminalDuration(start),DocName: docNameValue(s.taskCtx.Doc.Name),BuiltInMetadataConfig: builtInMetadata,AutoMetadataEnabled: autoMetaEnabled,}, nil}+// terminalDuration measures the run from the document's process_begin_at —+// the anchor PrepareValidatedRun stamped before the run and the one every+// mid-run progress-sink duration write uses — so the terminal value applied+// to document.process_duration and the value recorded in the pipeline+// operation log are the same number. Runs whose document carries no begin+// time fall back to the executor start.+func (s *PipelineExecutor) terminalDuration(start time.Time) float64 {+ if begin := s.taskCtx.Doc.ProcessBeginAt; begin != nil {+ duration := time.Since(*begin).Seconds()+ if duration < 0 {+ duration = 0+ }+ return duration+ }+ return time.Since(start).Seconds()+}+// builtInMetadataFromParserConfig extracts the built-in metadata config// (update_time / file_name) and whether auto-metadata is enabled from the// component-scoped Extractor node's modular metadata config. Legacy flat@@ -766,6 +794,14 @@ type PipelineLogInput struct {// exactly that row, never creating a second one — so a superseded run whose// row was deleted cannot adopt the replacement run's row.PipelineLogID string+ // TerminalDuration, when set, is this run's final duration in seconds,+ // measured from the document's process_begin_at. updateOpenLogRow records+ // it instead of the reloaded document's last mid-run value, so+ // pipeline_operation_log and document.process_duration hold one number.+ // Writers without it (failure/cancel, or a run that produced no chunks)+ // copy the document's stored value — still the last thing any writer put+ // there, so the tables stay aligned on those paths too.+ TerminalDuration *float64}// ErrMissingRunIdentity reports an attempt to persist an ingestion terminal@@ -898,6 +934,9 @@ func updateOpenLogRow(ctx context.Context, db *gorm.DB, input PipelineLogInput,if doc.Name != nil {updates["document_name"] = *doc.Name}+ if input.TerminalDuration != nil {+ updates["process_duration"] = *input.TerminalDuration+ }// The open row was created with a timestamp. Keep it when the reloaded// document carries none (a run that never reached the progress sink), so// the queued entry does not lose its start time.@@ -920,7 +959,7 @@ func updateOpenLogRow(ctx context.Context, db *gorm.DB, input PipelineLogInput,return nil}-func (s *PipelineExecutor) recordPipelineLog(ctx context.Context, db *gorm.DB, docID, dsl, status string) {+func (s *PipelineExecutor) recordPipelineLog(ctx context.Context, db *gorm.DB, docID, dsl, status string, terminalDuration *float64) {pipelineID := ""if s.taskCtx.PipelineID != "" {pipelineID = s.canvasID@@ -930,14 +969,15 @@ func (s *PipelineExecutor) recordPipelineLog(ctx context.Context, db *gorm.DB, dpipelineLogID = *s.taskCtx.IngestionTask.PipelineLogID}if err := recordPipelineLog(ctx, db, PipelineLogInput{- TenantID: s.Tenant().ID,- KbID: s.KB().ID,- DocumentID: docID,- PipelineID: pipelineID,- DSL: dsl,- Status: status,- Document: s.taskCtx.Doc,- PipelineLogID: pipelineLogID,+ TenantID: s.Tenant().ID,+ KbID: s.KB().ID,+ DocumentID: docID,+ PipelineID: pipelineID,+ DSL: dsl,+ Status: status,+ Document: s.taskCtx.Doc,+ PipelineLogID: pipelineLogID,+ TerminalDuration: terminalDuration,}); err != nil {common.Warn(fmt.Sprintf("failed to record pipeline log: %v", err))}diff --git a/internal/ingestion/task/pipeline_executor_test.go b/internal/ingestion/task/pipeline_executor_test.goindex a735d35e61..536fd2cc2a 100644--- a/internal/ingestion/task/pipeline_executor_test.go+++ b/internal/ingestion/task/pipeline_executor_test.go@@ -592,6 +592,88 @@ func TestRecordPipelineLog_KeepsEarlyTimestampWhenDocumentHasNone(t *testing.T)}}+// TestTerminalDurationAnchorsToProcessBeginAt pins the single duration anchor:+// a persisted run measures from the document's process_begin_at (the value+// PrepareValidatedRun stamped), not from executor start, so the document and+// the pipeline operation log record one number. A begin time in the future+// (DB/worker clock skew) clamps to zero like UpdateRunState does; a missing+// begin time falls back to the executor start.+func TestTerminalDurationAnchorsToProcessBeginAt(t *testing.T) {+ begin := time.Now().Add(-2 * time.Second)+ exec := &PipelineExecutor{taskCtx: &TaskContext{Doc: entity.Document{ProcessBeginAt: &begin}}}+ start := time.Now()+ if d := exec.terminalDuration(start); d < 2.0 || d > 3.0 {+ t.Fatalf("terminalDuration = %v, want ~2s measured from process_begin_at", d)+ }++ future := time.Now().Add(time.Minute)+ execFuture := &PipelineExecutor{taskCtx: &TaskContext{Doc: entity.Document{ProcessBeginAt: &future}}}+ if d := execFuture.terminalDuration(start); d != 0 {+ t.Fatalf("terminalDuration with future begin = %v, want 0", d)+ }++ execNoBegin := &PipelineExecutor{taskCtx: &TaskContext{}}+ if d := execNoBegin.terminalDuration(start); d < 0 || d > 1.0 {+ t.Fatalf("terminalDuration without begin = %v, want executor-start fallback", d)+ }+}++// TestRecordPipelineLog_TerminalDurationOverridesDocumentCopy locks the+// unification handover: when the run passes its measured terminal duration,+// the operation log records exactly that value — not the reloaded document's+// stale mid-run progress-sink value — so the final ApplyDocCounts write to+// document.process_duration and the log row agree.+func TestRecordPipelineLog_TerminalDurationOverridesDocumentCopy(t *testing.T) {+ cleanup := setupPipelineExecutorTestDB(t)+ defer cleanup()++ if err := dao.DB.Create(&entity.PipelineOperationLog{+ ID: "open-log",+ DocumentID: "doc-1",+ TenantID: "tenant-1",+ KbID: "kb-1",+ ParserID: "naive",+ TaskType: "Parse",+ OperationStatus: string(entity.TaskStatusRunning),+ }).Error; err != nil {+ t.Fatalf("seed open log: %v", err)+ }+ if err := dao.DB.Create(&entity.Document{+ ID: "doc-1",+ KbID: "kb-1",+ ParserID: "naive",+ ParserConfig: entity.JSONMap{},+ SourceType: "local",+ Type: "pdf",+ CreatedBy: "tenant-1",+ Suffix: ".pdf",+ ProcessDuration: 0.719,+ }).Error; err != nil {+ t.Fatalf("seed document: %v", err)+ }++ terminal := 0.627+ if err := RecordPipelineLog(t.Context(), dao.DB, PipelineLogInput{+ TenantID: "tenant-1",+ KbID: "kb-1",+ DocumentID: "doc-1",+ Status: "3",+ PipelineLogID: "open-log",+ TerminalDuration: &terminal,+ }); err != nil {+ t.Fatalf("RecordPipelineLog: %v", err)+ }++ var log entity.PipelineOperationLog+ if err := dao.DB.First(&log, "id = ?", "open-log").Error; err != nil {+ t.Fatalf("load open log: %v", err)+ }+ if log.ProcessDuration != terminal {+ t.Fatalf("ProcessDuration = %v, want the run's terminal value %v, not the document's mid-run 0.719",+ log.ProcessDuration, terminal)+ }+}+// TestRecordPipelineLog_DoesNotAdoptAnotherRunsRow locks the run-isolation// contract: a terminal write is bound to the row its own run opened, so a late// write from a superseded run (whose row was replaced) can neither finalize nor@@ -848,6 +930,73 @@ func TestPipelineExecutor_Execute_RecordsDoneOperationStatus(t *testing.T) {}}+// TestPipelineExecutor_Execute_EmptyOutputRecordsTerminalDuration pins the+// no-chunk success branch: such a run never writes a terminal duration to the+// document (processOutput returns before ApplyDocCounts), so Execute must+// recompute from the same process_begin_at anchor and hand that value to the+// operation log instead of letting the row keep the stale mid-run+// progress-sink duration.+func TestPipelineExecutor_Execute_EmptyOutputRecordsTerminalDuration(t *testing.T) {+ cleanup := setupPipelineExecutorTestDB(t)+ defer cleanup()++ begin := time.Now().Add(-2 * time.Second)+ if err := dao.DB.Create(&entity.PipelineOperationLog{+ ID: "open-log",+ DocumentID: "doc-1",+ TenantID: "tenant-1",+ KbID: "kb-1",+ ParserID: "naive",+ TaskType: "Parse",+ OperationStatus: string(entity.TaskStatusRunning),+ }).Error; err != nil {+ t.Fatalf("seed open log: %v", err)+ }+ if err := dao.DB.Create(&entity.Document{+ ID: "doc-1",+ KbID: "kb-1",+ ParserID: "naive",+ ParserConfig: entity.JSONMap{},+ SourceType: "local",+ Type: "pdf",+ CreatedBy: "tenant-1",+ Suffix: ".pdf",+ ProcessBeginAt: &begin,+ ProcessDuration: 0.719,+ }).Error; err != nil {+ t.Fatalf("seed document: %v", err)+ }++ logID := "open-log"+ taskCtx := makeTaskCtx()+ taskCtx.Ctx = t.Context()+ taskCtx.IngestionTask.PipelineLogID = &logID+ taskCtx.Doc.ProcessBeginAt = &begin++ svc := mustNewPipelineExecutor(t, taskCtx, "flow-1", 0).+ WithLoadDSLFunc(func(ctx context.Context, canvasID string) (string, string, error) {+ return `{"nodes":[{"id":"n1"}],"edges":[]}`, canvasID, nil+ }).+ WithRunPipelineFunc(func(runCtx context.Context, dsl string) (map[string]any, string, error) {+ return map[string]any{"chunks": []map[string]any{}}, dsl, nil+ })++ if _, err := svc.Execute(taskCtx.Ctx); err != nil {+ t.Fatalf("unexpected error: %v", err)+ }++ var log entity.PipelineOperationLog+ if err := dao.DB.First(&log, "id = ?", "open-log").Error; err != nil {+ t.Fatalf("load log row: %v", err)+ }+ if log.OperationStatus != string(entity.TaskStatusDone) {+ t.Fatalf("OperationStatus = %q, want DONE", log.OperationStatus)+ }+ if log.ProcessDuration < 2.0 || log.ProcessDuration > 3.0 {+ t.Fatalf("ProcessDuration = %v, want ~2s measured from process_begin_at, not the stale 0.719", log.ProcessDuration)+ }+}+// =============================================================================// Stub implementations for testing// =============================================================================diff --git a/internal/service/ingestion_task_service.go b/internal/service/ingestion_task_service.goindex 44374fc89b..7da99eeedf 100644--- a/internal/service/ingestion_task_service.go+++ b/internal/service/ingestion_task_service.go@@ -286,6 +286,7 @@ func (s *IngestionTaskService) PrepareValidatedRun(ctx context.Context, task *en"progress": float64(0),"chunk_num": int64(0),"token_num": int64(0),+ "process_duration": float64(0),"process_begin_at": time.Now(),}); err != nil {common.Warn(fmt.Sprintf("prepare validated run: mark document %s running for task %s: %v", task.DocumentID, task.ID, err))diff --git a/internal/service/ingestion_task_service_test.go b/internal/service/ingestion_task_service_test.goindex 77e9b71bc4..c9750269c2 100644--- a/internal/service/ingestion_task_service_test.go+++ b/internal/service/ingestion_task_service_test.go@@ -456,8 +456,9 @@ func TestPrepareValidatedRunResetsDocumentProgress(t *testing.T) {// Seed the document as a partially-processed state that the start transition must reset.if err := db.Model(&entity.Document{}).Where("id = ?", "doc-1").Updates(map[string]interface{}{- "progress": float64(0.5),- "progress_msg": "partial",+ "progress": float64(0.5),+ "progress_msg": "partial",+ "process_duration": 0.719,}).Error; err != nil {t.Fatalf("seed document: %v", err)}@@ -489,6 +490,9 @@ func TestPrepareValidatedRunResetsDocumentProgress(t *testing.T) {if doc.ProcessBeginAt == nil || doc.ProcessBeginAt.IsZero() {t.Fatal("process_begin_at not set")}+ if doc.ProcessDuration != 0 {+ t.Fatalf("process_duration = %f, want 0", doc.ProcessDuration)+ }}// TestStartRunningLeavesTerminalDocumentUntouched locks in the no-resurrectiondiff --git a/web/src/pages/dataset/dataset-overview/overview-table.tsx b/web/src/pages/dataset/dataset-overview/overview-table.tsxindex 4e7e089bf3..80fc2f9ae0 100644--- a/web/src/pages/dataset/dataset-overview/overview-table.tsx+++ b/web/src/pages/dataset/dataset-overview/overview-table.tsx@@ -385,8 +385,13 @@ const FileLogsTable: FC<FileLogsTableProps> = ({}return {...logInfo,- details: '',- events: messages?.items ?? [],+ // Keep the seeded progress_msg visible until events arrive: blanking+ // details unconditionally flashes an empty modal while the messages+ // query is still loading (mirrors the dataset page's useShowLog).+ details: messages?.items.length ? '' : logInfo.details,+ // An empty array is truthy, so passing it through would render a blank+ // events pane next to the seeded details; collapse it to undefined.+ events: messages?.items.length ? messages.items : undefined,loadPreviousEvents: hasPreviousPage? () => fetchPreviousPage(): undefined,diff --git a/web/src/pages/dataset/process-log-modal.test.tsx b/web/src/pages/dataset/process-log-modal.test.tsxnew file mode 100644index 0000000000..c517821836--- /dev/null+++ b/web/src/pages/dataset/process-log-modal.test.tsx@@ -0,0 +1,90 @@+import { render, screen } from '@testing-library/react';+import React from 'react';++jest.mock('react-i18next', () => ({+ useTranslation: () => ({ t: (key: string) => key }),+}));++jest.mock('@/hooks/common-hooks', () => ({+ useTranslate: () => ({ t: (key: string) => key }),+}));++import ProcessLogModal from './process-log-modal';+import { TooltipProvider } from '@/components/ui/tooltip';+import { formatTime } from '@/utils/date';++const eventTs = '2026-09-18T10:20:30+08:00';++type LogEvent = {+ id: number;+ ts: string;+ event_type: number;+ component: string;+ phase: number;+ message: string;+};++function renderModal(events: LogEvent[]) {+ return render(+ React.createElement(+ TooltipProvider,+ null,+ React.createElement(ProcessLogModal, {+ visible: true,+ onCancel: jest.fn(),+ title: 'log',+ logInfo: {+ fileName: 'doc.pdf',+ details: '',+ events,+ },+ }),+ ),+ );+}++describe('ProcessLogModal ingestion events', () => {+ it('renders each event line prefixed with its timestamp', () => {+ renderModal([+ {+ id: 1,+ ts: eventTs,+ event_type: 0,+ component: 'Parser',+ phase: 1,+ message: 'Parser Done',+ },+ {+ id: 2,+ ts: '2026-09-18T10:21:05+08:00',+ event_type: 1,+ component: '',+ phase: 0,+ message: 'embedding 40/120 chunks',+ },+ ]);++ const times = screen.getAllByTestId('ingestion-event-time');+ expect(times).toHaveLength(2);+ expect(times[0].textContent).toBe(formatTime(eventTs));+ expect(screen.getByText('embedding 40/120 chunks')).toBeInTheDocument();+ });++ it('omits the timestamp span when ts is absent', () => {+ renderModal([+ {+ id: 3,+ ts: '',+ event_type: 0,+ component: 'Tokenizer',+ phase: 0,+ message: 'Tokenizer Started',+ },+ ]);++ expect(+ screen.queryByTestId('ingestion-event-time'),+ ).not.toBeInTheDocument();+ expect(screen.getByText('Tokenizer Started')).toBeInTheDocument();+ });+});diff --git a/web/src/pages/dataset/process-log-modal.tsx b/web/src/pages/dataset/process-log-modal.tsxindex bebc354412..93216ac140 100644--- a/web/src/pages/dataset/process-log-modal.tsx+++ b/web/src/pages/dataset/process-log-modal.tsx@@ -13,6 +13,7 @@ import { useTranslation } from 'react-i18next';import reactStringReplace from 'react-string-replace';import { RunningStatus } from './dataset/constant';import { IngestionEventItem } from '@/interfaces/database/ingestion';+import { formatTime } from '@/utils/date';export interface ILogInfo {fileType?: string;uploadedBy?: string;@@ -166,18 +167,29 @@ const ProcessLogModal: React.FC<ProcessLogModalProps> = ({Loading earlier events…</div>)}- {logInfo.events?.map((event) => (- <div- className={- event.event_type === 3- ? 'text-text-secondary'- : undefined- }- key={event.id}- >- {replaceText(event.message)}- </div>- ))}+ {logInfo.events?.map((event) => {+ const time = formatTime(event.ts);+ return (+ <div+ className={+ event.event_type === 3+ ? 'text-text-secondary'+ : undefined+ }+ key={event.id}+ >+ {time && (+ <span+ className="text-text-secondary mr-1"+ data-testid="ingestion-event-time"+ >+ {time}+ </span>+ )}+ {replaceText(event.message)}+ </div>+ );+ })}</div>}/>