RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
Stop persisting process_end_at; recover end time from begin + duration (#21906)
xugangqiang58360341ab97
diff --git a/internal/entity/pipeline.go b/internal/entity/pipeline.goindex f6f29e9eac..409a82728d 100644--- a/internal/entity/pipeline.go+++ b/internal/entity/pipeline.go@@ -41,16 +41,15 @@ type PipelineOperationLog struct {Progress float64 `gorm:"column:progress;default:0;index" json:"progress"`ProgressMsg *string `gorm:"column:progress_msg;type:longtext" json:"progress_msg,omitempty"`ProcessBeginAt *time.Time `gorm:"column:process_begin_at;index" json:"process_begin_at,omitempty"`- // ProcessEndAt is set when the run reaches a terminal state (completed /- // failed / stopped). It is the authoritative end time used to compute parse- // duration for ingestion observability events.- ProcessEndAt *time.Time `gorm:"column:process_end_at;index" json:"process_end_at,omitempty"`- ProcessDuration float64 `gorm:"column:process_duration;default:0" json:"process_duration"`- DSL JSONMap `gorm:"column:dsl;type:longtext" json:"dsl,omitempty"`- TaskType string `gorm:"column:task_type;size:32;not null;default:''" json:"task_type"`- OperationStatus string `gorm:"column:operation_status;size:32;not null" json:"operation_status"`- Avatar *string `gorm:"column:avatar;type:longtext" json:"avatar,omitempty"`- Status *string `gorm:"column:status;size:1;index" json:"status,omitempty"`+ // ProcessDuration is the persisted terminal run duration in seconds. The+ // authoritative end time is recovered as ProcessBeginAt + ProcessDuration when+ // needed, so no separate process_end_at column is required.+ ProcessDuration float64 `gorm:"column:process_duration;default:0" json:"process_duration"`+ DSL JSONMap `gorm:"column:dsl;type:longtext" json:"dsl,omitempty"`+ TaskType string `gorm:"column:task_type;size:32;not null;default:''" json:"task_type"`+ OperationStatus string `gorm:"column:operation_status;size:32;not null" json:"operation_status"`+ Avatar *string `gorm:"column:avatar;type:longtext" json:"avatar,omitempty"`+ Status *string `gorm:"column:status;size:1;index" json:"status,omitempty"`BaseModel}diff --git a/internal/entity/pipeline_process_end_at_removed_test.go b/internal/entity/pipeline_process_end_at_removed_test.gonew file mode 100644index 0000000000..ab492558b1--- /dev/null+++ b/internal/entity/pipeline_process_end_at_removed_test.go@@ -0,0 +1,18 @@+package entity++import (+ "reflect"+ "testing"+)++// TestPipelineOperationLogHasNoProcessEndAtField guards that the process_end_at+// column/field was fully removed. The authoritative end time is recovered from+// ProcessBeginAt + ProcessDuration, so the struct must not carry a ProcessEndAt+// field at all (neither persisted nor in-memory). This prevents the field from+// being silently re-introduced.+func TestPipelineOperationLogHasNoProcessEndAtField(t *testing.T) {+ typ := reflect.TypeOf(PipelineOperationLog{})+ if _, ok := typ.FieldByName("ProcessEndAt"); ok {+ t.Fatalf("PipelineOperationLog must not have a ProcessEndAt field; remove it entirely")+ }+}diff --git a/internal/observability/ingestion_log.go b/internal/observability/ingestion_log.goindex 8d47120163..4c1648b0e8 100644--- a/internal/observability/ingestion_log.go+++ b/internal/observability/ingestion_log.go@@ -37,7 +37,10 @@ func ComputeDurations(enqueue, begin, end time.Time) (float64, float64) {wait = begin.Sub(enqueue).Seconds()}parse := 0.0- if end.After(begin) {+ // A zero begin means the worker-claim time is unknown (e.g. the document+ // lookup failed and claimTime returned nil). There is no parse window, so the+ // duration must clamp to 0 rather than blow up to end - year-0001.+ if !begin.IsZero() && end.After(begin) {parse = end.Sub(begin).Seconds()}return wait, parse@@ -118,17 +121,21 @@ func BuildIngestLog(task *entity.IngestionTask, doc *entity.Document, run *entitif run.CreateDate != nil {ev.EnqueueTime = run.CreateDate.UTC()}- if run.ProcessEndAt != nil {- ev.EndTime = run.ProcessEndAt.UTC()+ if run.ProcessBeginAt != nil && run.ProcessDuration > 0 {+ // The process_end_at column was removed; recover the authoritative end+ // time from the persisted begin time and duration so parse duration is+ // still accurate without a dedicated column.+ ev.EndTime = run.ProcessBeginAt.Add(time.Duration(run.ProcessDuration * float64(time.Second))).UTC()}// BeginTime only for running/terminal events; created/scheduled must not// leak a (possibly stale) claim time from a previous run.if ct := claimTime(doc, run); ct != nil && isBeginRelevant(eventType) {ev.BeginTime = ct.UTC()}- // ProcessEndAt is not always persisted on the run; for a terminal event- // whose run actually started, fall back to the transition time so parse- // duration is still reported (a still-queued stop gets no duration).+ // When the end time still could not be derived (the run had no+ // ProcessDuration, e.g. a stop on a still-queued task), fall back to the+ // transition time for a terminal event so a parse duration is still reported+ // if a begin time is present.if ev.EndTime.IsZero() && !ev.BeginTime.IsZero() &&(eventType == "completed" || eventType == "failed" || eventType == "stopped") {ev.EndTime = ev.EventTimediff --git a/internal/observability/ingestion_log_endtime_test.go b/internal/observability/ingestion_log_endtime_test.gonew file mode 100644index 0000000000..0986e09293--- /dev/null+++ b/internal/observability/ingestion_log_endtime_test.go@@ -0,0 +1,53 @@+package observability++import (+ "testing"+ "time"++ "ragflow/internal/entity"+)++// TestBuildIngestLogDerivesEndTimeFromBeginAndDuration covers the requirement+// that process_end_at need not be persisted: the authoritative end time is+// recovered from ProcessBeginAt + ProcessDuration.+func TestBuildIngestLogDerivesEndTimeFromBeginAndDuration(t *testing.T) {+ base := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC)+ begin := base.Add(10 * time.Second)+ duration := 30.0+ run := &entity.PipelineOperationLog{+ RunCount: ptrInt(1),+ CreateDate: ptrTime(base),+ ProcessBeginAt: ptrTime(begin),+ ProcessDuration: duration,+ }+ task := &entity.IngestionTask{ID: "task-1"}+ doc := &entity.Document{Name: ptrStr("d"), CreateDate: ptrTime(base)}++ ev := BuildIngestLog(task, doc, run, "completed", "COMPLETED", "")+ want := begin.Add(time.Duration(duration * float64(time.Second)))+ if !ev.EndTime.Equal(want) {+ t.Fatalf("EndTime = %v, want %v (ProcessBeginAt + ProcessDuration)", ev.EndTime, want)+ }+}++// TestBuildIngestLogZeroDurationFallsBackToEventTime guards against a naive+// "always derive end from begin" bug: when ProcessDuration is 0 (run not yet processed),+// EndTime must NOT collapse to ProcessBeginAt, and instead fall back to the transition+// EventTime for a terminal event.+func TestBuildIngestLogZeroDurationFallsBackToEventTime(t *testing.T) {+ base := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC)+ begin := base.Add(10 * time.Second)+ run := &entity.PipelineOperationLog{+ RunCount: ptrInt(1),+ CreateDate: ptrTime(base),+ ProcessBeginAt: ptrTime(begin),+ ProcessDuration: 0, // not yet processed+ }+ task := &entity.IngestionTask{ID: "task-1"}+ doc := &entity.Document{Name: ptrStr("d"), CreateDate: ptrTime(base), ProcessBeginAt: ptrTime(begin)}++ ev := BuildIngestLog(task, doc, run, "completed", "COMPLETED", "")+ if !ev.EndTime.Equal(ev.EventTime) {+ t.Fatalf("EndTime = %v, want EventTime %v (duration 0 fallback)", ev.EndTime, ev.EventTime)+ }+}diff --git a/internal/observability/ingestion_log_fix_test.go b/internal/observability/ingestion_log_fix_test.goindex 16610be8a0..073ffaaf3d 100644--- a/internal/observability/ingestion_log_fix_test.go+++ b/internal/observability/ingestion_log_fix_test.go@@ -12,10 +12,9 @@ func TestBuildIngestLogPrefersWorkerClaimTime(t *testing.T) {enqueue := baseopenTime := base.Add(1 * time.Minute)claimTime := base.Add(10 * time.Minute)- end := base.Add(20 * time.Minute)task := &entity.IngestionTask{ID: "t", DocumentID: "d", DatasetID: "kb", UserID: "u"}doc := &entity.Document{Name: ptrStr("r"), Suffix: ".pdf", Type: "pdf", Size: 10, ParserID: "pdf", ProcessBeginAt: ptrTime(claimTime)}- run := &entity.PipelineOperationLog{RunCount: ptrInt(1), CreateDate: ptrTime(enqueue), ProcessBeginAt: ptrTime(openTime), ProcessEndAt: ptrTime(end)}+ run := &entity.PipelineOperationLog{RunCount: ptrInt(1), CreateDate: ptrTime(enqueue), ProcessBeginAt: ptrTime(openTime)}log := BuildIngestLog(task, doc, run, "completed", "COMPLETED", "")if !log.BeginTime.Equal(claimTime) {@@ -30,7 +29,7 @@ func TestBuildIngestLogPopulatesDurationMS(t *testing.T) {base := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC)task := &entity.IngestionTask{ID: "t", DocumentID: "d", DatasetID: "kb", UserID: "u"}doc := &entity.Document{Name: ptrStr("r"), Suffix: ".pdf", Type: "pdf", Size: 10, ParserID: "pdf", ProcessBeginAt: ptrTime(base.Add(10 * time.Minute))}- run := &entity.PipelineOperationLog{RunCount: ptrInt(1), CreateDate: ptrTime(base), ProcessBeginAt: ptrTime(base.Add(1 * time.Minute)), ProcessEndAt: ptrTime(base.Add(20 * time.Minute))}+ run := &entity.PipelineOperationLog{RunCount: ptrInt(1), CreateDate: ptrTime(base), ProcessBeginAt: ptrTime(base.Add(1 * time.Minute)), ProcessDuration: 600}log := BuildIngestLog(task, doc, run, "completed", "COMPLETED", "")want := int64(log.ParseDurationS * 1000)diff --git a/internal/observability/ingestion_log_tdd_test.go b/internal/observability/ingestion_log_tdd_test.goindex bc6ce39045..016325c03d 100644--- a/internal/observability/ingestion_log_tdd_test.go+++ b/internal/observability/ingestion_log_tdd_test.go@@ -90,10 +90,9 @@ func TestBuildIngestLogRunningKeepsValidClaimTime(t *testing.T) {base := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC)runOpen := baseclaim := base.Add(10 * time.Minute)- end := base.Add(20 * time.Minute)task := &entity.IngestionTask{ID: "t", DocumentID: "d", DatasetID: "kb", UserID: "u"}doc := &entity.Document{Name: ptrStr("r"), Suffix: ".pdf", Type: "pdf", Size: 10, ParserID: "pdf", ProcessBeginAt: ptrTime(claim)}- run := &entity.PipelineOperationLog{RunCount: ptrInt(1), CreateDate: ptrTime(runOpen), ProcessEndAt: ptrTime(end)}+ run := &entity.PipelineOperationLog{RunCount: ptrInt(1), CreateDate: ptrTime(runOpen), ProcessBeginAt: ptrTime(runOpen), ProcessDuration: 1200}ev := BuildIngestLog(task, doc, run, "running", "RUNNING", "")if !ev.BeginTime.Equal(claim) {diff --git a/internal/observability/ingestion_log_test.go b/internal/observability/ingestion_log_test.goindex 1b6d605bd4..1d022e8ecc 100644--- a/internal/observability/ingestion_log_test.go+++ b/internal/observability/ingestion_log_test.go@@ -49,6 +49,24 @@ func TestComputeDurationsClampsEndBeforeBegin(t *testing.T) {}}+// TestComputeDurationsZeroBeginYieldsZeroParse guards against a real failure+// mode: when the worker-claim time is unknown (e.g. the document lookup failed+// and claimTime returned nil), BeginTime is the zero time while EndTime was+// still derived from ProcessBeginAt + ProcessDuration. Without a begin.IsZero()+// guard, parse would be end - year-0001 ≈ 9.2e9 seconds. A missing begin means+// no parse window, so parse (and wait) must clamp to 0.+func TestComputeDurationsZeroBeginYieldsZeroParse(t *testing.T) {+ enqueue := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC)+ end := enqueue.Add(10 * time.Minute)+ wait, parse := ComputeDurations(enqueue, time.Time{}, end)+ if wait != 0 {+ t.Errorf("wait_duration_s = %v, want 0 for zero begin", wait)+ }+ if parse != 0 {+ t.Errorf("parse_duration_s = %v, want 0 for zero begin (must not report a huge duration)", parse)+ }+}+func ptrStr(s string) *string { return &s }func ptrTime(t time.Time) *time.Time { return &t }func ptrInt(i int) *int { return &i }@@ -71,11 +89,12 @@ func TestBuildIngestLogHappy(t *testing.T) {ProcessBeginAt: ptrTime(now.Add(2 * time.Minute)),}run := &entity.PipelineOperationLog{- RunCount: ptrInt(1),- TenantID: "tenant-1",- Progress: 1,- CreateDate: ptrTime(now),- ProcessEndAt: ptrTime(now.Add(12 * time.Minute)),+ RunCount: ptrInt(1),+ TenantID: "tenant-1",+ Progress: 1,+ CreateDate: ptrTime(now),+ ProcessBeginAt: ptrTime(now.Add(2 * time.Minute)),+ ProcessDuration: 600,}log := BuildIngestLog(task, doc, run, "completed", "COMPLETED", "")diff --git a/internal/service/ingestion_task_service.go b/internal/service/ingestion_task_service.goindex 55714a336d..1f77c31472 100644--- a/internal/service/ingestion_task_service.go+++ b/internal/service/ingestion_task_service.go@@ -1182,35 +1182,11 @@ func (s *IngestionTaskService) RecordMessage(ctx context.Context, pipelineLogID,return s.insertEvent(ctx, ingestionEventMessage, pipelineLogID, taskID, "", 0, message)}-// persistProcessEndAt records the authoritative terminal end time for a run.-// It is idempotent: if a real process_end_at is already persisted, the existing-// value is left untouched, so a later terminal event cannot rewrite the recorded-// completion instant. Best-effort: errors are logged, never returned, because a-// failure here must not fail the terminal settlement.-func (s *IngestionTaskService) persistProcessEndAt(ctx context.Context, pipelineLogID string) {- if dao.DB == nil {- return- }- if updErr := dao.DB.WithContext(ctx).- Model(&entity.PipelineOperationLog{}).- Where("id = ?", pipelineLogID).- Where("process_end_at IS NULL").- Update("process_end_at", time.Now()).Error; updErr != nil {- common.Warn(fmt.Sprintf("persist process_end_at for run %s: %v", pipelineLogID, updErr))- }-}-// RecordTerminal writes the terminal explanation associated with a run.func (s *IngestionTaskService) RecordTerminal(ctx context.Context, pipelineLogID, taskID, message string) error {if err := s.ensureTerminalEvent(ctx, pipelineLogID, taskID, message); err != nil {return err}- // Persist the authoritative terminal end time so ingestion observability- // events and any downstream duration query read a real value instead of- // falling back to the transition time. Idempotent: a run whose end time- // was already recorded keeps it, so a repeated terminal settlement (e.g.- // overlapping failure/timeout events) cannot rewrite the completion instant.- s.persistProcessEndAt(context.WithoutCancel(ctx), pipelineLogID)foldContext := context.WithoutCancel(ctx)foldStarted := time.Now()foldResult, foldErr := s.foldIngestionRun(foldContext, pipelineLogID, s.logSettings.MaxRowsPerRun)diff --git a/internal/service/ingestion_task_service_test.go b/internal/service/ingestion_task_service_test.goindex c00ad0a19d..823da6512a 100644--- a/internal/service/ingestion_task_service_test.go+++ b/internal/service/ingestion_task_service_test.go@@ -2573,53 +2573,6 @@ func TestIngestionTaskServiceMarkFailedCarriesReason(t *testing.T) {}}-// TestIngestionTaskServicePersistProcessEndAtIdempotent verifies that-// persistProcessEndAt records the end time on first settlement and leaves an-// already-recorded value untouched on a later settlement (overlapping-// failure/timeout terminal events must not rewrite the completion instant).-func TestIngestionTaskServicePersistProcessEndAtIdempotent(t *testing.T) {- db := setupServiceTestDB(t)- pushServiceDB(t, db)- runID := "run-idempotent-1"- begin := time.Now().Add(-time.Minute)- run := &entity.PipelineOperationLog{- ID: runID,- DocumentID: "doc-idem-1",- TenantID: "tenant-1",- KbID: "kb-1",- ParserID: "parser-1",- DocumentName: "doc",- DocumentSuffix: ".pdf",- DocumentType: "pdf",- SourceFrom: "web",- OperationStatus: "RUNNING",- ProcessBeginAt: &begin,- }- if err := db.Create(run).Error; err != nil {- t.Fatalf("create run: %v", err)- }- ctx := t.Context()- svc := NewIngestionTaskService()- svc.persistProcessEndAt(ctx, runID)- var first entity.PipelineOperationLog- if err := db.Where("id = ?", runID).First(&first).Error; err != nil {- t.Fatalf("load run: %v", err)- }- if first.ProcessEndAt == nil {- t.Fatalf("process_end_at not persisted on first call")- }- firstEnd := *first.ProcessEndAt- // A second settlement must not overwrite the recorded end time.- svc.persistProcessEndAt(ctx, runID)- var second entity.PipelineOperationLog- if err := db.Where("id = ?", runID).First(&second).Error; err != nil {- t.Fatalf("load run again: %v", err)- }- if second.ProcessEndAt == nil || !second.ProcessEndAt.Equal(firstEnd) {- t.Fatalf("process_end_at changed on second call: first=%v second=%v", firstEnd, second.ProcessEndAt)- }-}-// TestPrepareValidatedRunEmitsRunningEventWithFreshClaimTime verifies that the// running lifecycle event is emitted AFTER PrepareValidatedRun stamps the// document's worker-claim time. Previously the running event was emitted in