Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

fix(ingestion): unify run duration and timestamp parsing log lines (#19849)

jay777218f8699cb6cd6
diff --git a/internal/ingestion/task/pipeline_executor.go b/internal/ingestion/task/pipeline_executor.go
index 310387520b..b45a7ac69a 100644
--- a/internal/ingestion/task/pipeline_executor.go
+++ b/internal/ingestion/task/pipeline_executor.go
@@ -192,7 +192,18 @@ func (s *PipelineExecutor) Execute(ctx context.Context) (*PipelineResult, error)
}
if pipelineDSL != "" && s.taskCtx.IngestionTask != nil && s.taskCtx.IngestionTask.PipelineLogID != nil && *s.taskCtx.IngestionTask.PipelineLogID != "" {
- s.recordPipelineLog(context.WithoutCancel(ctx), dao.DB, s.taskCtx.Doc.ID, pipelineDSL, string(entity.TaskStatusDone))
+ var terminalDuration *float64
+ if result != nil {
+ terminalDuration = &result.Duration
+ } else {
+ // A successful run with no chunks never wrote a terminal duration
+ // to the document, so recompute from the same process_begin_at
+ // anchor here instead of letting the log copy the stale mid-run
+ // value.
+ d := s.terminalDuration(start)
+ terminalDuration = &d
+ }
+ s.recordPipelineLog(context.WithoutCancel(ctx), dao.DB, s.taskCtx.Doc.ID, pipelineDSL, string(entity.TaskStatusDone), terminalDuration)
}
return result, nil
@@ -335,13 +346,30 @@ func (s *PipelineExecutor) processOutput(ctx context.Context, pipelineOutput map
Metadata: metadata,
ChunkCount: chunkCount,
TokenConsumption: embeddingTokenConsumption,
- Duration: time.Since(start).Seconds(),
+ Duration: s.terminalDuration(start),
DocName: docNameValue(s.taskCtx.Doc.Name),
BuiltInMetadataConfig: builtInMetadata,
AutoMetadataEnabled: autoMetaEnabled,
}, nil
}
+// terminalDuration measures the run from the document's process_begin_at —
+// the anchor PrepareValidatedRun stamped before the run and the one every
+// mid-run progress-sink duration write uses — so the terminal value applied
+// to document.process_duration and the value recorded in the pipeline
+// operation log are the same number. Runs whose document carries no begin
+// time fall back to the executor start.
+func (s *PipelineExecutor) terminalDuration(start time.Time) float64 {
+ if begin := s.taskCtx.Doc.ProcessBeginAt; begin != nil {
+ duration := time.Since(*begin).Seconds()
+ if duration < 0 {
+ duration = 0
+ }
+ return duration
+ }
+ return time.Since(start).Seconds()
+}
+
// builtInMetadataFromParserConfig extracts the built-in metadata config
// (update_time / file_name) and whether auto-metadata is enabled from the
// component-scoped Extractor node's modular metadata config. Legacy flat
@@ -766,6 +794,14 @@ type PipelineLogInput struct {
// exactly that row, never creating a second one — so a superseded run whose
// row was deleted cannot adopt the replacement run's row.
PipelineLogID string
+ // TerminalDuration, when set, is this run's final duration in seconds,
+ // measured from the document's process_begin_at. updateOpenLogRow records
+ // it instead of the reloaded document's last mid-run value, so
+ // pipeline_operation_log and document.process_duration hold one number.
+ // Writers without it (failure/cancel, or a run that produced no chunks)
+ // copy the document's stored value — still the last thing any writer put
+ // there, so the tables stay aligned on those paths too.
+ TerminalDuration *float64
}
// ErrMissingRunIdentity reports an attempt to persist an ingestion terminal
@@ -898,6 +934,9 @@ func updateOpenLogRow(ctx context.Context, db *gorm.DB, input PipelineLogInput,
if doc.Name != nil {
updates["document_name"] = *doc.Name
}
+ if input.TerminalDuration != nil {
+ updates["process_duration"] = *input.TerminalDuration
+ }
// The open row was created with a timestamp. Keep it when the reloaded
// document carries none (a run that never reached the progress sink), so
// the queued entry does not lose its start time.
@@ -920,7 +959,7 @@ func updateOpenLogRow(ctx context.Context, db *gorm.DB, input PipelineLogInput,
return nil
}
-func (s *PipelineExecutor) recordPipelineLog(ctx context.Context, db *gorm.DB, docID, dsl, status string) {
+func (s *PipelineExecutor) recordPipelineLog(ctx context.Context, db *gorm.DB, docID, dsl, status string, terminalDuration *float64) {
pipelineID := ""
if s.taskCtx.PipelineID != "" {
pipelineID = s.canvasID
@@ -930,14 +969,15 @@ func (s *PipelineExecutor) recordPipelineLog(ctx context.Context, db *gorm.DB, d
pipelineLogID = *s.taskCtx.IngestionTask.PipelineLogID
}
if err := recordPipelineLog(ctx, db, PipelineLogInput{
- TenantID: s.Tenant().ID,
- KbID: s.KB().ID,
- DocumentID: docID,
- PipelineID: pipelineID,
- DSL: dsl,
- Status: status,
- Document: s.taskCtx.Doc,
- PipelineLogID: pipelineLogID,
+ TenantID: s.Tenant().ID,
+ KbID: s.KB().ID,
+ DocumentID: docID,
+ PipelineID: pipelineID,
+ DSL: dsl,
+ Status: status,
+ Document: s.taskCtx.Doc,
+ PipelineLogID: pipelineLogID,
+ TerminalDuration: terminalDuration,
}); err != nil {
common.Warn(fmt.Sprintf("failed to record pipeline log: %v", err))
}
diff --git a/internal/ingestion/task/pipeline_executor_test.go b/internal/ingestion/task/pipeline_executor_test.go
index a735d35e61..536fd2cc2a 100644
--- a/internal/ingestion/task/pipeline_executor_test.go
+++ b/internal/ingestion/task/pipeline_executor_test.go
@@ -592,6 +592,88 @@ func TestRecordPipelineLog_KeepsEarlyTimestampWhenDocumentHasNone(t *testing.T)
}
}
+// TestTerminalDurationAnchorsToProcessBeginAt pins the single duration anchor:
+// a persisted run measures from the document's process_begin_at (the value
+// PrepareValidatedRun stamped), not from executor start, so the document and
+// the pipeline operation log record one number. A begin time in the future
+// (DB/worker clock skew) clamps to zero like UpdateRunState does; a missing
+// begin time falls back to the executor start.
+func TestTerminalDurationAnchorsToProcessBeginAt(t *testing.T) {
+ begin := time.Now().Add(-2 * time.Second)
+ exec := &PipelineExecutor{taskCtx: &TaskContext{Doc: entity.Document{ProcessBeginAt: &begin}}}
+ start := time.Now()
+ if d := exec.terminalDuration(start); d < 2.0 || d > 3.0 {
+ t.Fatalf("terminalDuration = %v, want ~2s measured from process_begin_at", d)
+ }
+
+ future := time.Now().Add(time.Minute)
+ execFuture := &PipelineExecutor{taskCtx: &TaskContext{Doc: entity.Document{ProcessBeginAt: &future}}}
+ if d := execFuture.terminalDuration(start); d != 0 {
+ t.Fatalf("terminalDuration with future begin = %v, want 0", d)
+ }
+
+ execNoBegin := &PipelineExecutor{taskCtx: &TaskContext{}}
+ if d := execNoBegin.terminalDuration(start); d < 0 || d > 1.0 {
+ t.Fatalf("terminalDuration without begin = %v, want executor-start fallback", d)
+ }
+}
+
+// TestRecordPipelineLog_TerminalDurationOverridesDocumentCopy locks the
+// unification handover: when the run passes its measured terminal duration,
+// the operation log records exactly that value — not the reloaded document's
+// stale mid-run progress-sink value — so the final ApplyDocCounts write to
+// document.process_duration and the log row agree.
+func TestRecordPipelineLog_TerminalDurationOverridesDocumentCopy(t *testing.T) {
+ cleanup := setupPipelineExecutorTestDB(t)
+ defer cleanup()
+
+ if err := dao.DB.Create(&entity.PipelineOperationLog{
+ ID: "open-log",
+ DocumentID: "doc-1",
+ TenantID: "tenant-1",
+ KbID: "kb-1",
+ ParserID: "naive",
+ TaskType: "Parse",
+ OperationStatus: string(entity.TaskStatusRunning),
+ }).Error; err != nil {
+ t.Fatalf("seed open log: %v", err)
+ }
+ if err := dao.DB.Create(&entity.Document{
+ ID: "doc-1",
+ KbID: "kb-1",
+ ParserID: "naive",
+ ParserConfig: entity.JSONMap{},
+ SourceType: "local",
+ Type: "pdf",
+ CreatedBy: "tenant-1",
+ Suffix: ".pdf",
+ ProcessDuration: 0.719,
+ }).Error; err != nil {
+ t.Fatalf("seed document: %v", err)
+ }
+
+ terminal := 0.627
+ if err := RecordPipelineLog(t.Context(), dao.DB, PipelineLogInput{
+ TenantID: "tenant-1",
+ KbID: "kb-1",
+ DocumentID: "doc-1",
+ Status: "3",
+ PipelineLogID: "open-log",
+ TerminalDuration: &terminal,
+ }); err != nil {
+ t.Fatalf("RecordPipelineLog: %v", err)
+ }
+
+ var log entity.PipelineOperationLog
+ if err := dao.DB.First(&log, "id = ?", "open-log").Error; err != nil {
+ t.Fatalf("load open log: %v", err)
+ }
+ if log.ProcessDuration != terminal {
+ t.Fatalf("ProcessDuration = %v, want the run's terminal value %v, not the document's mid-run 0.719",
+ log.ProcessDuration, terminal)
+ }
+}
+
// TestRecordPipelineLog_DoesNotAdoptAnotherRunsRow locks the run-isolation
// contract: a terminal write is bound to the row its own run opened, so a late
// write from a superseded run (whose row was replaced) can neither finalize nor
@@ -848,6 +930,73 @@ func TestPipelineExecutor_Execute_RecordsDoneOperationStatus(t *testing.T) {
}
}
+// TestPipelineExecutor_Execute_EmptyOutputRecordsTerminalDuration pins the
+// no-chunk success branch: such a run never writes a terminal duration to the
+// document (processOutput returns before ApplyDocCounts), so Execute must
+// recompute from the same process_begin_at anchor and hand that value to the
+// operation log instead of letting the row keep the stale mid-run
+// progress-sink duration.
+func TestPipelineExecutor_Execute_EmptyOutputRecordsTerminalDuration(t *testing.T) {
+ cleanup := setupPipelineExecutorTestDB(t)
+ defer cleanup()
+
+ begin := time.Now().Add(-2 * time.Second)
+ if err := dao.DB.Create(&entity.PipelineOperationLog{
+ ID: "open-log",
+ DocumentID: "doc-1",
+ TenantID: "tenant-1",
+ KbID: "kb-1",
+ ParserID: "naive",
+ TaskType: "Parse",
+ OperationStatus: string(entity.TaskStatusRunning),
+ }).Error; err != nil {
+ t.Fatalf("seed open log: %v", err)
+ }
+ if err := dao.DB.Create(&entity.Document{
+ ID: "doc-1",
+ KbID: "kb-1",
+ ParserID: "naive",
+ ParserConfig: entity.JSONMap{},
+ SourceType: "local",
+ Type: "pdf",
+ CreatedBy: "tenant-1",
+ Suffix: ".pdf",
+ ProcessBeginAt: &begin,
+ ProcessDuration: 0.719,
+ }).Error; err != nil {
+ t.Fatalf("seed document: %v", err)
+ }
+
+ logID := "open-log"
+ taskCtx := makeTaskCtx()
+ taskCtx.Ctx = t.Context()
+ taskCtx.IngestionTask.PipelineLogID = &logID
+ taskCtx.Doc.ProcessBeginAt = &begin
+
+ svc := mustNewPipelineExecutor(t, taskCtx, "flow-1", 0).
+ WithLoadDSLFunc(func(ctx context.Context, canvasID string) (string, string, error) {
+ return `{"nodes":[{"id":"n1"}],"edges":[]}`, canvasID, nil
+ }).
+ WithRunPipelineFunc(func(runCtx context.Context, dsl string) (map[string]any, string, error) {
+ return map[string]any{"chunks": []map[string]any{}}, dsl, nil
+ })
+
+ if _, err := svc.Execute(taskCtx.Ctx); err != nil {
+ t.Fatalf("unexpected error: %v", err)
+ }
+
+ var log entity.PipelineOperationLog
+ if err := dao.DB.First(&log, "id = ?", "open-log").Error; err != nil {
+ t.Fatalf("load log row: %v", err)
+ }
+ if log.OperationStatus != string(entity.TaskStatusDone) {
+ t.Fatalf("OperationStatus = %q, want DONE", log.OperationStatus)
+ }
+ if log.ProcessDuration < 2.0 || log.ProcessDuration > 3.0 {
+ t.Fatalf("ProcessDuration = %v, want ~2s measured from process_begin_at, not the stale 0.719", log.ProcessDuration)
+ }
+}
+
// =============================================================================
// Stub implementations for testing
// =============================================================================
diff --git a/internal/service/ingestion_task_service.go b/internal/service/ingestion_task_service.go
index 44374fc89b..7da99eeedf 100644
--- a/internal/service/ingestion_task_service.go
+++ b/internal/service/ingestion_task_service.go
@@ -286,6 +286,7 @@ func (s *IngestionTaskService) PrepareValidatedRun(ctx context.Context, task *en
"progress": float64(0),
"chunk_num": int64(0),
"token_num": int64(0),
+ "process_duration": float64(0),
"process_begin_at": time.Now(),
}); err != nil {
common.Warn(fmt.Sprintf("prepare validated run: mark document %s running for task %s: %v", task.DocumentID, task.ID, err))
diff --git a/internal/service/ingestion_task_service_test.go b/internal/service/ingestion_task_service_test.go
index 77e9b71bc4..c9750269c2 100644
--- a/internal/service/ingestion_task_service_test.go
+++ b/internal/service/ingestion_task_service_test.go
@@ -456,8 +456,9 @@ func TestPrepareValidatedRunResetsDocumentProgress(t *testing.T) {
// Seed the document as a partially-processed state that the start transition must reset.
if err := db.Model(&entity.Document{}).Where("id = ?", "doc-1").
Updates(map[string]interface{}{
- "progress": float64(0.5),
- "progress_msg": "partial",
+ "progress": float64(0.5),
+ "progress_msg": "partial",
+ "process_duration": 0.719,
}).Error; err != nil {
t.Fatalf("seed document: %v", err)
}
@@ -489,6 +490,9 @@ func TestPrepareValidatedRunResetsDocumentProgress(t *testing.T) {
if doc.ProcessBeginAt == nil || doc.ProcessBeginAt.IsZero() {
t.Fatal("process_begin_at not set")
}
+ if doc.ProcessDuration != 0 {
+ t.Fatalf("process_duration = %f, want 0", doc.ProcessDuration)
+ }
}
// TestStartRunningLeavesTerminalDocumentUntouched locks in the no-resurrection
diff --git a/web/src/pages/dataset/dataset-overview/overview-table.tsx b/web/src/pages/dataset/dataset-overview/overview-table.tsx
index 4e7e089bf3..80fc2f9ae0 100644
--- a/web/src/pages/dataset/dataset-overview/overview-table.tsx
+++ b/web/src/pages/dataset/dataset-overview/overview-table.tsx
@@ -385,8 +385,13 @@ const FileLogsTable: FC<FileLogsTableProps> = ({
}
return {
...logInfo,
- details: '',
- events: messages?.items ?? [],
+ // Keep the seeded progress_msg visible until events arrive: blanking
+ // details unconditionally flashes an empty modal while the messages
+ // query is still loading (mirrors the dataset page's useShowLog).
+ details: messages?.items.length ? '' : logInfo.details,
+ // An empty array is truthy, so passing it through would render a blank
+ // events pane next to the seeded details; collapse it to undefined.
+ events: messages?.items.length ? messages.items : undefined,
loadPreviousEvents: hasPreviousPage
? () => fetchPreviousPage()
: undefined,
diff --git a/web/src/pages/dataset/process-log-modal.test.tsx b/web/src/pages/dataset/process-log-modal.test.tsx
new file mode 100644
index 0000000000..c517821836
--- /dev/null
+++ b/web/src/pages/dataset/process-log-modal.test.tsx
@@ -0,0 +1,90 @@
+import { render, screen } from '@testing-library/react';
+import React from 'react';
+
+jest.mock('react-i18next', () => ({
+ useTranslation: () => ({ t: (key: string) => key }),
+}));
+
+jest.mock('@/hooks/common-hooks', () => ({
+ useTranslate: () => ({ t: (key: string) => key }),
+}));
+
+import ProcessLogModal from './process-log-modal';
+import { TooltipProvider } from '@/components/ui/tooltip';
+import { formatTime } from '@/utils/date';
+
+const eventTs = '2026-09-18T10:20:30+08:00';
+
+type LogEvent = {
+ id: number;
+ ts: string;
+ event_type: number;
+ component: string;
+ phase: number;
+ message: string;
+};
+
+function renderModal(events: LogEvent[]) {
+ return render(
+ React.createElement(
+ TooltipProvider,
+ null,
+ React.createElement(ProcessLogModal, {
+ visible: true,
+ onCancel: jest.fn(),
+ title: 'log',
+ logInfo: {
+ fileName: 'doc.pdf',
+ details: '',
+ events,
+ },
+ }),
+ ),
+ );
+}
+
+describe('ProcessLogModal ingestion events', () => {
+ it('renders each event line prefixed with its timestamp', () => {
+ renderModal([
+ {
+ id: 1,
+ ts: eventTs,
+ event_type: 0,
+ component: 'Parser',
+ phase: 1,
+ message: 'Parser Done',
+ },
+ {
+ id: 2,
+ ts: '2026-09-18T10:21:05+08:00',
+ event_type: 1,
+ component: '',
+ phase: 0,
+ message: 'embedding 40/120 chunks',
+ },
+ ]);
+
+ const times = screen.getAllByTestId('ingestion-event-time');
+ expect(times).toHaveLength(2);
+ expect(times[0].textContent).toBe(formatTime(eventTs));
+ expect(screen.getByText('embedding 40/120 chunks')).toBeInTheDocument();
+ });
+
+ it('omits the timestamp span when ts is absent', () => {
+ renderModal([
+ {
+ id: 3,
+ ts: '',
+ event_type: 0,
+ component: 'Tokenizer',
+ phase: 0,
+ message: 'Tokenizer Started',
+ },
+ ]);
+
+ expect(
+ screen.queryByTestId('ingestion-event-time'),
+ ).not.toBeInTheDocument();
+ expect(screen.getByText('Tokenizer Started')).toBeInTheDocument();
+ });
+});
diff --git a/web/src/pages/dataset/process-log-modal.tsx b/web/src/pages/dataset/process-log-modal.tsx
index bebc354412..93216ac140 100644
--- a/web/src/pages/dataset/process-log-modal.tsx
+++ b/web/src/pages/dataset/process-log-modal.tsx
@@ -13,6 +13,7 @@ import { useTranslation } from 'react-i18next';
import reactStringReplace from 'react-string-replace';
import { RunningStatus } from './dataset/constant';
import { IngestionEventItem } from '@/interfaces/database/ingestion';
+import { formatTime } from '@/utils/date';
export interface ILogInfo {
fileType?: string;
uploadedBy?: string;
@@ -166,18 +167,29 @@ const ProcessLogModal: React.FC<ProcessLogModalProps> = ({
Loading earlier events…
</div>
)}
- {logInfo.events?.map((event) => (
- <div
- className={
- event.event_type === 3
- ? 'text-text-secondary'
- : undefined
- }
- key={event.id}
- >
- {replaceText(event.message)}
- </div>
- ))}
+ {logInfo.events?.map((event) => {
+ const time = formatTime(event.ts);
+ return (
+ <div
+ className={
+ event.event_type === 3
+ ? 'text-text-secondary'
+ : undefined
+ }
+ key={event.id}
+ >
+ {time && (
+ <span
+ className="text-text-secondary mr-1"
+ data-testid="ingestion-event-time"
+ >
+ {time}
+ </span>
+ )}
+ {replaceText(event.message)}
+ </div>
+ );
+ })}
</div>
}
/>