RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
feat(ingestion): emit structured lifecycle events for document-parse tasks (#21711)
xugangqiang69fc4c9ec6e1
diff --git a/internal/common/ingest_log_path_test.go b/internal/common/ingest_log_path_test.gonew file mode 100644index 0000000000..d7ba0eed5c--- /dev/null+++ b/internal/common/ingest_log_path_test.go@@ -0,0 +1,68 @@+//go:build !ee++package common++import (+ "os"+ "path/filepath"+ "testing"+ "time"+)++// TestInitIngestLogPathTreatedAsFile is the regression test for P1b: the configured+// ingestor.log.path is a FILE name (per conf/service_conf.yaml comment), so InitIngestLog+// must write to that exact file and must NOT treat it as a directory.+func TestInitIngestLogPathTreatedAsFile(t *testing.T) {+ t.Cleanup(resetIngestLog)++ dir := t.TempDir()+ filePath := filepath.Join(dir, "ingest_log.log")++ if err := InitIngestLog("file", FileOutput{Path: filePath}, "api_server_9381"); err != nil {+ t.Fatalf("InitIngestLog: %v", err)+ }++ LogIngestEvent(IngestionLog{EventTime: time.Now().UTC(), TaskID: "t1", EventType: "created", Status: "CREATED"})+ SyncIngestLog()++ fi, err := os.Stat(filePath)+ if err != nil {+ t.Fatalf("expected log file at %s: %v", filePath, err)+ }+ if fi.IsDir() {+ t.Fatalf("%s was created as a DIRECTORY; path must be treated as a file", filePath)+ }++ if stray, err := os.Stat(filepath.Join(dir, "ingest_log.log")); err == nil && stray.IsDir() {+ t.Fatalf("stray directory %s/ created; path was misinterpreted as a directory", dir)+ }+}++// TestInitIngestLogEmptyPathNoFile asserts the documented default: an empty path emits to+// stdout only and opens no rotated file. This guards the off-by-default contract.+func TestInitIngestLogEmptyPathNoFile(t *testing.T) {+ t.Cleanup(resetIngestLog)++ dir := t.TempDir()+ if err := InitIngestLog("file", FileOutput{}, "api_server_9381"); err != nil {+ t.Fatalf("InitIngestLog: %v", err)+ }+ LogIngestEvent(IngestionLog{EventTime: time.Now().UTC(), TaskID: "t1"})+ SyncIngestLog()++ entries, _ := os.ReadDir(dir)+ for _, e := range entries {+ if nameContains(e.Name(), "ingest") {+ t.Fatalf("empty path opened a file under %s: %s", dir, e.Name())+ }+ }+}++func nameContains(s, sub string) bool {+ for i := 0; i+len(sub) <= len(s); i++ {+ if s[i:i+len(sub)] == sub {+ return true+ }+ }+ return false+}diff --git a/internal/common/ingest_log_test.go b/internal/common/ingest_log_test.gonew file mode 100644index 0000000000..9abaedcce0--- /dev/null+++ b/internal/common/ingest_log_test.go@@ -0,0 +1,186 @@+//go:build !ee++package common++import (+ "bytes"+ "encoding/json"+ "os"+ "path/filepath"+ "reflect"+ "strings"+ "testing"+ "time"++ "go.uber.org/zap"+)++func resetIngestLog() {+ ingestLogger.Store(zap.NewNop())+}++// TestIngestLogUnifiedStruct asserts the single unified type: renamed fields+// (DocID not DocumentID, EventType not TaskType) and the absence of the+// un-serializable Error error interface. Structural guard for 方案B unification.+func TestIngestLogUnifiedStruct(t *testing.T) {+ typ := reflect.TypeOf(IngestionLog{})+ for _, f := range []string{"DocID", "EventType", "ErrorInfo", "Code"} {+ if _, ok := typ.FieldByName(f); !ok {+ t.Errorf("IngestionLog missing expected unified field %q", f)+ }+ }+ for _, bad := range []string{"DocumentID", "TaskType", "Error"} {+ if _, ok := typ.FieldByName(bad); ok {+ t.Errorf("IngestionLog must not keep legacy field %q after unification", bad)+ }+ }+}++// TestIngestLogEmitsUnifiedJSON happy path: flat JSON line with error_info (string) and code.+func TestIngestLogEmitsUnifiedJSON(t *testing.T) {+ defer resetIngestLog()+ dir := t.TempDir()+ if err := InitIngestLog("json", FileOutput{Path: dir, Filename: "ingest.log"}, "svc"); err != nil {+ t.Fatalf("InitIngestLog: %v", err)+ }+ LogIngestEvent(IngestionLog{+ EventTime: time.Now().UTC(),+ TenantID: "t1",+ DatasetID: "kb1",+ DocID: "d1",+ UserID: "u1",+ TaskID: "task1",+ EventType: "completed",+ Status: "COMPLETED",+ ErrorInfo: "boom",+ Filename: "report.pdf",+ FileType: "pdf",+ FileSize: 12345,+ ParserID: "pdf",+ Code: 7,+ })+ SyncIngestLog()+ raw, err := os.ReadFile(filepath.Join(dir, "ingest.log"))+ if err != nil {+ t.Fatalf("read sink: %v", err)+ }+ var m map[string]any+ if err := json.Unmarshal(raw, &m); err != nil {+ t.Fatalf("emitted line not JSON: %v (raw=%q)", err, raw)+ }+ if m["error_info"] != "boom" {+ t.Errorf("error_info = %v, want boom (string, not error interface)", m["error_info"])+ }+ if c, ok := m["code"].(float64); !ok || c != 7 {+ t.Errorf("code = %v (%T), want 7", m["code"], m["code"])+ }+ if _, ok := m["error"]; ok {+ t.Errorf("legacy error key must not be present; only error_info is allowed")+ }+}++// TestIngestLogCodeRoundTrips adversarial: Code is a first-class struct field,+// not hard-coded 0 via a secondary mapping layer.+func TestIngestLogCodeRoundTrips(t *testing.T) {+ defer resetIngestLog()+ dir := t.TempDir()+ if err := InitIngestLog("json", FileOutput{Path: dir, Filename: "ingest.log"}, "svc"); err != nil {+ t.Fatalf("InitIngestLog: %v", err)+ }+ LogIngestEvent(IngestionLog{TaskID: "task1", EventType: "failed", Status: "FAILED", Code: 7})+ SyncIngestLog()+ raw, _ := os.ReadFile(filepath.Join(dir, "ingest.log"))+ var m map[string]any+ if err := json.Unmarshal(raw, &m); err != nil {+ t.Fatalf("emitted line not JSON: %v (raw=%q)", err, raw)+ }+ if c, ok := m["code"].(float64); !ok || c != 7 {+ t.Errorf("code = %v (%T), want 7 (first-class struct field, not hard-coded 0)", m["code"], m["code"])+ }+}++// TestIngestLogOmitsZeroTimes adversarial: zero-value times must not leak.+func TestIngestLogOmitsZeroTimes(t *testing.T) {+ defer resetIngestLog()+ dir := t.TempDir()+ if err := InitIngestLog("json", FileOutput{Path: dir, Filename: "ingest.log"}, "svc"); err != nil {+ t.Fatalf("InitIngestLog: %v", err)+ }+ LogIngestEvent(IngestionLog{EventTime: time.Now().UTC(), TaskID: "t1", EventType: "created", Status: "CREATED"})+ SyncIngestLog()+ raw, _ := os.ReadFile(filepath.Join(dir, "ingest.log"))+ if strings.Contains(string(raw), "0001-01-01") {+ t.Errorf("emitted zero-time sentinel 0001-01-01: %s", raw)+ }+ var m map[string]any+ json.Unmarshal(raw, &m)+ for _, k := range []string{"begin_time", "end_time", "upload_time", "enqueue_time"} {+ if _, ok := m[k]; ok {+ t.Errorf("zero-time field %q must be omitted, got %v", k, m[k])+ }+ }+ if _, ok := m["event_time"]; !ok {+ t.Errorf("event_time must always be present")+ }+}++// TestIngestLogOmitsEmptyErrorInfo adversarial: empty ErrorInfo omitted.+func TestIngestLogOmitsEmptyErrorInfo(t *testing.T) {+ defer resetIngestLog()+ dir := t.TempDir()+ if err := InitIngestLog("json", FileOutput{Path: dir, Filename: "ingest.log"}, "svc"); err != nil {+ t.Fatalf("InitIngestLog: %v", err)+ }+ LogIngestEvent(IngestionLog{TaskID: "t1", EventType: "created", Status: "CREATED"})+ SyncIngestLog()+ raw, _ := os.ReadFile(filepath.Join(dir, "ingest.log"))+ var m map[string]any+ json.Unmarshal(raw, &m)+ if _, ok := m["error_info"]; ok {+ t.Errorf("empty error_info must be omitted, got %v", m["error_info"])+ }+}++// TestIngestLogOmitsEmptyUserName corner.+func TestIngestLogOmitsEmptyUserName(t *testing.T) {+ defer resetIngestLog()+ dir := t.TempDir()+ if err := InitIngestLog("json", FileOutput{Path: dir, Filename: "ingest.log"}, "svc"); err != nil {+ t.Fatalf("InitIngestLog: %v", err)+ }+ LogIngestEvent(IngestionLog{TaskID: "task1", EventType: "created", Status: "CREATED"})+ SyncIngestLog()+ raw, _ := os.ReadFile(filepath.Join(dir, "ingest.log"))+ if strings.Contains(string(raw), "user_name") {+ t.Fatalf("emitted line contains user_name when UserName is empty: %s", raw)+ }+}++// TestIngestLogOneFlatLine adversarial: each event is exactly one JSON line.+func TestIngestLogOneFlatLine(t *testing.T) {+ defer resetIngestLog()+ dir := t.TempDir()+ if err := InitIngestLog("json", FileOutput{Path: dir, Filename: "ingest.log"}, "svc"); err != nil {+ t.Fatalf("InitIngestLog: %v", err)+ }+ LogIngestEvent(IngestionLog{TaskID: "task1", EventType: "completed", Status: "COMPLETED", ErrorInfo: "boom"})+ SyncIngestLog()+ raw, _ := os.ReadFile(filepath.Join(dir, "ingest.log"))+ if len(bytes.TrimSpace(raw)) == 0 {+ t.Fatal("LogIngest wrote no output")+ }+ lines := 0+ for _, l := range bytes.Split(raw, []byte("\n")) {+ if len(bytes.TrimSpace(l)) == 0 {+ continue+ }+ lines+++ var m map[string]any+ if err := json.Unmarshal(l, &m); err != nil {+ t.Fatalf("non-JSON line: %v (raw=%q)", err, l)+ }+ }+ if lines != 1 {+ t.Fatalf("expected 1 line, got %d (raw=%q)", lines, raw)+ }+}diff --git a/internal/common/stub.go b/internal/common/stub.goindex dcab51e087..0334a1184f 100644--- a/internal/common/stub.go+++ b/internal/common/stub.go@@ -19,9 +19,15 @@package commonimport (+ "fmt"+ "os"+ "path/filepath"+ "sync/atomic""time""go.uber.org/zap"+ "go.uber.org/zap/zapcore"+ "gopkg.in/natefinch/lumberjack.v2")var (@@ -55,23 +61,179 @@ type OperationLog struct {func Audit(operationLog *OperationLog, fields ...zap.Field) {}+// ingestLogger receives the serialized ingest events. It defaults to a no-op so a+// binary that never calls InitIngestLog emits nothing and never panics.+var ingestLogger atomic.Pointer[zap.Logger]++func init() {+ ingestLogger.Store(zap.NewNop())+}++// InitIngestLog configures the sink for ingestion lifecycle events. It mirrors+// InitLogger: output always goes to stdout, and an optional rotated file is added+// when both Path and Filename are set.func InitIngestLog(logType string, file FileOutput, serviceName string) error {+ encoderConfig := zapcore.EncoderConfig{+ TimeKey: "timestamp",+ LevelKey: "level",+ NameKey: "service",+ CallerKey: "caller",+ FunctionKey: "",+ MessageKey: "msg",+ StacktraceKey: "stacktrace",+ LineEnding: zapcore.DefaultLineEnding,+ EncodeLevel: zapcore.LowercaseLevelEncoder,+ EncodeTime: zapcore.TimeEncoderOfLayout("2006-01-02 15:04:05.000-07:00"),+ EncodeDuration: zapcore.SecondsDurationEncoder,+ EncodeCaller: zapcore.ShortCallerEncoder,+ EncodeName: zapcore.FullNameEncoder,+ }+ maxSize := file.MaxSize+ if maxSize <= 0 {+ maxSize = DefaultLogMaxSizeMB+ }+ maxBackups := file.MaxBackups+ if maxBackups <= 0 {+ maxBackups = DefaultLogMaxBackups+ }+ maxAge := file.MaxAge+ if maxAge <= 0 {+ maxAge = DefaultLogMaxAgeDays+ }+ compress := DefaultLogCompress+ if file.Compress != nil {+ compress = *file.Compress+ }+ stdoutSyncer := zapcore.AddSync(os.Stdout)+ syncers := []zapcore.WriteSyncer{stdoutSyncer}+ // File output is opt-in via a non-empty Path. Path is treated as the log FILE+ // (matching conf/service_conf.yaml's "path is the file" wording): a stray+ // directory named after the file is never created. Only when Path is itself an+ // existing directory do we synthesize a per-service file name inside it.+ if file.Path != "" {+ filename := file.Path+ if fi, statErr := os.Stat(file.Path); statErr == nil && fi.IsDir() {+ name := file.Filename+ if name == "" {+ name = fmt.Sprintf("%s_ingest.log", serviceName)+ }+ filename = filepath.Join(file.Path, name)+ }+ ljLogger := &lumberjack.Logger{+ Filename: filename,+ MaxSize: maxSize,+ MaxBackups: maxBackups,+ MaxAge: maxAge,+ Compress: compress,+ LocalTime: true,+ }+ syncers = append(syncers, zapcore.AddSync(ljLogger))+ }+ core := zapcore.NewCore(zapcore.NewJSONEncoder(encoderConfig), zapcore.NewMultiWriteSyncer(syncers...), zapcore.InfoLevel)+ oldLogger := ingestLogger.Swap(zap.New(core))+ if oldLogger != nil {+ _ = oldLogger.Sync()+ }return nil}type IngestionLog struct {- EventTime time.Time `json:"event_time"`- UserID string `json:"user_id"`- UserName string `json:"user_name"`- DocumentID string `json:"document_id"`- Worker string `json:"worker"`- TaskID string `json:"task_id"`- TaskType string `json:"task_type"`- Filename string `json:"filename"`- Code int `json:"code"`- Error error `json:"error"`- DurationMS int64 `json:"duration_ms"`- Operation map[string]any `json:"operation"`+ EventTime time.Time `json:"event_time"`+ TenantID string `json:"tenant_id"`+ DatasetID string `json:"dataset_id"`+ DocID string `json:"doc_id"`+ UserID string `json:"user_id"`+ UserName string `json:"user_name,omitempty"`+ Worker string `json:"worker"`+ TaskID string `json:"task_id"`+ EventType string `json:"event_type"` // created|scheduled|running|completed|failed|stopped+ Status string `json:"status"`+ // ErrorInfo is the terminal reason as a plain string (not an error+ // interface) so the event serializes cleanly and portably to external+ // dashboards / ClickHouse.+ ErrorInfo string `json:"error_info,omitempty"`+ Filename string `json:"filename"`+ FileType string `json:"file_type"`+ FileSize int64 `json:"file_size"`+ ParserID string `json:"parser_id"`+ UploadTime time.Time `json:"upload_time"`+ EnqueueTime time.Time `json:"enqueue_time"`+ BeginTime time.Time `json:"begin_time"`+ EndTime time.Time `json:"end_time"`+ DurationMS int64 `json:"duration_ms"`+ WaitDurationS float64 `json:"wait_duration_s"`+ ParseDurationS float64 `json:"parse_duration_s"`+ Progress float64 `json:"progress"`+ RunCount int `json:"run_count"`+ // Code is a machine-readable category for grouping in dashboards. It+ // defaults to 0 when the caller does not set it.+ Code int `json:"code"`+ Operation map[string]any `json:"operation,omitempty"`+}++// SyncIngestLog flushes any buffered ingest output. Call it before process exit+// so no events are lost.+func SyncIngestLog() {+ _ = ingestLogger.Load().Sync()+}++// LogIngestEvent transmits one structured ingestion lifecycle event as a single,+// flat JSON line. In the OSS build it writes to the local zap sink and also+// forwards to the shared LogIngest seam (a no-op in OSS, the real sink in EE).+func LogIngestEvent(log IngestionLog) {+ ingestLogger.Load().With(log.zapFields()...).Info("ingest_log")+ LogIngest(&log)+}++// zapFields renders the event as flat zap fields (top-level keys, UTC times) so+// the emitted JSON line is directly consumable by ClickHouse / Grafana.+func (l IngestionLog) zapFields() []zap.Field {+ fields := []zap.Field{+ zap.Time("event_time", l.EventTime.UTC()),+ zap.String("tenant_id", l.TenantID),+ zap.String("dataset_id", l.DatasetID),+ zap.String("doc_id", l.DocID),+ zap.String("user_id", l.UserID),+ zap.String("worker", l.Worker),+ zap.String("task_id", l.TaskID),+ zap.String("event_type", l.EventType),+ zap.String("status", l.Status),+ zap.String("filename", l.Filename),+ zap.String("file_type", l.FileType),+ zap.Int64("file_size", l.FileSize),+ zap.String("parser_id", l.ParserID),+ skipZeroTime("upload_time", l.UploadTime),+ skipZeroTime("enqueue_time", l.EnqueueTime),+ skipZeroTime("begin_time", l.BeginTime),+ skipZeroTime("end_time", l.EndTime),+ zap.Int64("duration_ms", l.DurationMS),+ zap.Float64("wait_duration_s", l.WaitDurationS),+ zap.Float64("parse_duration_s", l.ParseDurationS),+ zap.Float64("progress", l.Progress),+ zap.Int("run_count", l.RunCount),+ zap.Int("code", l.Code),+ }+ if l.UserName != "" {+ fields = append(fields, zap.String("user_name", l.UserName))+ }+ if l.ErrorInfo != "" {+ fields = append(fields, zap.String("error_info", l.ErrorInfo))+ }+ if len(l.Operation) > 0 {+ fields = append(fields, zap.Any("operation", l.Operation))+ }+ return fields+}++// skipZeroTime omits a time field from the emitted JSON when it is the zero+// value, keeping the line compact and ClickHouse-friendly.+func skipZeroTime(key string, t time.Time) zap.Field {+ if t.IsZero() {+ return zap.Skip()+ }+ return zap.Time(key, t.UTC())}+// LogIngest is the shared upstream seam. In the OSS build it is a no-op; the EE+// build provides the real sink. Our OSS LogIngestEvent forwards to it.func LogIngest(log *IngestionLog, fields ...zap.Field) {}diff --git a/internal/entity/pipeline.go b/internal/entity/pipeline.goindex 923a8835ed..f6f29e9eac 100644--- a/internal/entity/pipeline.go+++ b/internal/entity/pipeline.go@@ -28,19 +28,23 @@ type PipelineOperationLog struct {DocumentID string `gorm:"column:document_id;size:32;index;uniqueIndex:idx_pipeline_operation_log_document_run,priority:1" json:"document_id"`// RunCount is the monotonically allocated display number for a numbered// ingestion run.- RunCount *int `gorm:"column:run_count;uniqueIndex:idx_pipeline_operation_log_document_run,priority:2" json:"run_count,omitempty"`- TenantID string `gorm:"column:tenant_id;size:32;not null;index" json:"tenant_id"`- KbID string `gorm:"column:kb_id;size:32;not null;index" json:"kb_id"`- PipelineID *string `gorm:"column:pipeline_id;size:32;index" json:"pipeline_id,omitempty"`- PipelineTitle *string `gorm:"column:pipeline_title;size:32;index" json:"pipeline_title,omitempty"`- ParserID string `gorm:"column:parser_id;size:32;not null;index" json:"parser_id"`- DocumentName string `gorm:"column:document_name;size:255;not null" json:"document_name"`- DocumentSuffix string `gorm:"column:document_suffix;size:255;not null" json:"document_suffix"`- DocumentType string `gorm:"column:document_type;size:255;not null" json:"document_type"`- SourceFrom string `gorm:"column:source_from;size:255;not null" json:"source_from"`- Progress float64 `gorm:"column:progress;default:0;index" json:"progress"`- ProgressMsg *string `gorm:"column:progress_msg;type:longtext" json:"progress_msg,omitempty"`- ProcessBeginAt *time.Time `gorm:"column:process_begin_at;index" json:"process_begin_at,omitempty"`+ RunCount *int `gorm:"column:run_count;uniqueIndex:idx_pipeline_operation_log_document_run,priority:2" json:"run_count,omitempty"`+ TenantID string `gorm:"column:tenant_id;size:32;not null;index" json:"tenant_id"`+ KbID string `gorm:"column:kb_id;size:32;not null;index" json:"kb_id"`+ PipelineID *string `gorm:"column:pipeline_id;size:32;index" json:"pipeline_id,omitempty"`+ PipelineTitle *string `gorm:"column:pipeline_title;size:32;index" json:"pipeline_title,omitempty"`+ ParserID string `gorm:"column:parser_id;size:32;not null;index" json:"parser_id"`+ DocumentName string `gorm:"column:document_name;size:255;not null" json:"document_name"`+ DocumentSuffix string `gorm:"column:document_suffix;size:255;not null" json:"document_suffix"`+ DocumentType string `gorm:"column:document_type;size:255;not null" json:"document_type"`+ SourceFrom string `gorm:"column:source_from;size:255;not null" json:"source_from"`+ Progress float64 `gorm:"column:progress;default:0;index" json:"progress"`+ ProgressMsg *string `gorm:"column:progress_msg;type:longtext" json:"progress_msg,omitempty"`+ ProcessBeginAt *time.Time `gorm:"column:process_begin_at;index" json:"process_begin_at,omitempty"`+ // ProcessEndAt is set when the run reaches a terminal state (completed /+ // failed / stopped). It is the authoritative end time used to compute parse+ // duration for ingestion observability events.+ ProcessEndAt *time.Time `gorm:"column:process_end_at;index" json:"process_end_at,omitempty"`ProcessDuration float64 `gorm:"column:process_duration;default:0" json:"process_duration"`DSL JSONMap `gorm:"column:dsl;type:longtext" json:"dsl,omitempty"`TaskType string `gorm:"column:task_type;size:32;not null;default:''" json:"task_type"`diff --git a/internal/ingestion/service/execute_task_ack_test.go b/internal/ingestion/service/execute_task_ack_test.goindex 9e0372908d..cf2ea5b10d 100644--- a/internal/ingestion/service/execute_task_ack_test.go+++ b/internal/ingestion/service/execute_task_ack_test.go@@ -452,7 +452,7 @@ func TestSettleMessage_DBTruthOverridesBodyReturn(t *testing.T) {// recovery where markFailed succeeded: the task is terminal (FAILED)// but the body signals non-terminal.body := func(ctx context.Context) bool {- ingestor.markFailed(ctx, taskID)+ ingestor.markFailed(ctx, taskID, "")return false}diff --git a/internal/ingestion/service/ingestion_service.go b/internal/ingestion/service/ingestion_service.goindex fef61f75f6..339300d218 100644--- a/internal/ingestion/service/ingestion_service.go+++ b/internal/ingestion/service/ingestion_service.go@@ -547,7 +547,7 @@ func (e *Ingestor) handleAndExecute(handle common.TaskHandle) {var identityErr *servicepkg.InvalidRunIdentityErrorif errors.As(validateErr, &identityErr) {common.Error(fmt.Sprintf("task %s has permanent run identity error: %s", task.ID, identityErr.Reason), validateErr)- if e.markFailed(e.ctx, task.ID) {+ if e.markFailed(e.ctx, task.ID, fmt.Sprintf("Task rejected: invalid run identity (%s).", identityErr.Reason)) {e.recordTerminalPipelineLog(e.ctx, task, string(entity.TaskStatusFail), fmt.Sprintf("Task rejected: invalid run identity (%s).", identityErr.Reason))e.ackHandle(hb, handle, taskMessage.TaskID)} else {@@ -820,10 +820,10 @@ func (e *Ingestor) markStopped(ctx context.Context, taskID string) bool {// The caller's context is detached before touching the DB (see markStopped):// failure paths usually carry an already-cancelled context, and losing the// FAILED write would turn a settled task into a redelivery loop.-func (e *Ingestor) markFailed(ctx context.Context, taskID string) bool {+func (e *Ingestor) markFailed(ctx context.Context, taskID, reason string) bool {ctx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 5*time.Second)defer cancel()- if uErr := e.ingestionTaskSvc.MarkFailed(ctx, taskID); uErr != nil {+ if uErr := e.ingestionTaskSvc.MarkFailed(ctx, taskID, reason); uErr != nil {common.Error(fmt.Sprintf("Failed to set task %s to FAILED", taskID), uErr)return false}@@ -882,14 +882,14 @@ func (e *Ingestor) runTask(ctx context.Context, task *entity.IngestionTask) boolif errors.Is(err, context.DeadlineExceeded) {common.Info(fmt.Sprintf("Task %s timed out during pipeline", task.ID))e.markTerminalProgress(task)- ok := e.markFailed(ctx, task.ID)+ ok := e.markFailed(ctx, task.ID, "Task timed out.")if ok {e.recordTerminalPipelineLog(ctx, task, string(entity.TaskStatusFail), "Task timed out.")}return ok}common.Error(fmt.Sprintf("Task %s failed", task.ID), err)- ok := e.markFailed(ctx, task.ID)+ ok := e.markFailed(ctx, task.ID, fmt.Sprintf("Task failed: %v", err))if ok {e.recordTerminalPipelineLog(ctx, task, string(entity.TaskStatusFail), fmt.Sprintf("Task failed: %v", err))}@@ -1019,7 +1019,7 @@ func (e *Ingestor) settleMessage(ctx context.Context, taskCtx *taskpkg.TaskConte}}()common.Error(fmt.Sprintf("task %s panicked: %v", taskID, r), fmt.Errorf("%v", r))- e.markFailed(ctx, taskID)+ e.markFailed(ctx, taskID, fmt.Sprintf("Task panicked: %v", r))e.recordTerminalPipelineLog(ctx, task, string(entity.TaskStatusFail), fmt.Sprintf("Task panicked: %v", r))}()terminal = falsediff --git a/internal/ingestion/service/nil_database_guard_test.go b/internal/ingestion/service/nil_database_guard_test.goindex 50d6104e0f..08a053e493 100644--- a/internal/ingestion/service/nil_database_guard_test.go+++ b/internal/ingestion/service/nil_database_guard_test.go@@ -35,7 +35,7 @@ func TestMarkFailedToleratesNilDatabase(t *testing.T) {defer testutil.ReplaceDBForTest(t, nil)()ingestor := newUnitIngestor("test", 1, []string{"pdf"})- if terminal := ingestor.markFailed(context.Background(), "task-1"); terminal {+ if terminal := ingestor.markFailed(context.Background(), "task-1", ""); terminal {t.Fatal("markFailed reported a durable FAILED write without a database")}}diff --git a/internal/observability/ingestion_log.go b/internal/observability/ingestion_log.gonew file mode 100644index 0000000000..447bf9a56e--- /dev/null+++ b/internal/observability/ingestion_log.go@@ -0,0 +1,140 @@+// Package observability builds structured lifecycle events for ingestion+// (document-parse) tasks and forwards them to the common ingest-log sink+// (internal/common.LogIngestEvent). The default emitter routes straight to that+// sink; tests inject a recording emitter.+package observability++import (+ "context"+ "time"++ "ragflow/internal/common"+ "ragflow/internal/entity"+)++// IngestionEventEmitter receives lifecycle events. Implementations must be+// non-blocking and must never return an error that could fail the parse; the+// caller ignores errors by design.+type IngestionEventEmitter interface {+ EmitIngestLog(ctx context.Context, log common.IngestionLog)+}++// CommonLogEmitter is the default OSS emitter. It forwards the event to+// common.LogIngestEvent, which writes one JSON line to the configured sink.+type CommonLogEmitter struct{}++// EmitIngestLog forwards the event to the common ingest-log sink.+func (e *CommonLogEmitter) EmitIngestLog(_ context.Context, log common.IngestionLog) {+ common.LogIngestEvent(log)+}++// ComputeDurations derives wait (begin-enqueue) and parse (end-begin) seconds.+// Durations are clamped to >= 0 so clock skew never produces negative values+// downstream.+func ComputeDurations(enqueue, begin, end time.Time) (float64, float64) {+ wait := 0.0+ if begin.After(enqueue) {+ wait = begin.Sub(enqueue).Seconds()+ }+ parse := 0.0+ if end.After(begin) {+ parse = end.Sub(begin).Seconds()+ }+ return wait, parse+}++// isBeginRelevant reports whether the worker-claim time should be attached to+// this event. created/scheduled events happen before any worker has claimed the+// document, so they must not carry a (possibly stale, from a prior run) claim+// time.+func isBeginRelevant(eventType string) bool {+ switch eventType {+ case "running", "completed", "failed", "stopped":+ return true+ default:+ return false+ }+}++// claimTime returns the worker-claim time. It prefers the document's own+// ProcessBeginAt (set when the worker actually starts, in PrepareValidatedRun).+//+// doc.ProcessBeginAt is never cleared between runs, so on a run that never claimed+// the document (RequestStop/supersede on a queued task, or a publish failure) or a+// retry that reuses a stale document value, it can hold a PRIOR run's instant.+// We therefore only trust it when it falls at or after this run's open time+// (run.CreateDate); otherwise the run never started and must not report a+// claim/parse time. The run row's ProcessBeginAt is the open time (set in+// CreateOpenLog), not a claim time, so it is intentionally not used here.+func claimTime(doc *entity.Document, run *entity.PipelineOperationLog) *time.Time {+ if doc != nil && doc.ProcessBeginAt != nil && run != nil && run.CreateDate != nil {+ if !doc.ProcessBeginAt.Before(*run.CreateDate) {+ return doc.ProcessBeginAt+ }+ return nil+ }+ // No run to validate against: fall back to the document value as-is.+ if doc != nil && doc.ProcessBeginAt != nil {+ return doc.ProcessBeginAt+ }+ return nil+}++// BuildIngestLog assembles a common.IngestionLog from the task, its document, and+// the bound pipeline run. eventType is one of+// created|scheduled|running|completed|failed|stopped.+func BuildIngestLog(task *entity.IngestionTask, doc *entity.Document, run *entity.PipelineOperationLog, eventType, status, errorInfo string) common.IngestionLog {+ ev := common.IngestionLog{+ EventTime: time.Now().UTC(),+ TaskID: task.ID,+ DocID: task.DocumentID,+ DatasetID: task.DatasetID,+ UserID: task.UserID,+ EventType: eventType,+ Status: status,+ ErrorInfo: errorInfo,+ }+ if doc != nil {+ name := ""+ if doc.Name != nil {+ name = *doc.Name+ }+ ev.Filename = name + doc.Suffix+ ev.FileType = doc.Type+ ev.FileSize = doc.Size+ ev.ParserID = doc.ParserID+ if doc.CreateDate != nil {+ ev.UploadTime = doc.CreateDate.UTC()+ }+ }+ if run != nil {+ if run.RunCount != nil {+ ev.RunCount = *run.RunCount+ }+ if run.TenantID != "" {+ ev.TenantID = run.TenantID+ }+ ev.Progress = run.Progress+ if run.CreateDate != nil {+ ev.EnqueueTime = run.CreateDate.UTC()+ }+ if run.ProcessEndAt != nil {+ ev.EndTime = run.ProcessEndAt.UTC()+ }+ // BeginTime only for running/terminal events; created/scheduled must not+ // leak a (possibly stale) claim time from a previous run.+ if ct := claimTime(doc, run); ct != nil && isBeginRelevant(eventType) {+ ev.BeginTime = ct.UTC()+ }+ // ProcessEndAt is not always persisted on the run; for a terminal event+ // whose run actually started, fall back to the transition time so parse+ // duration is still reported (a still-queued stop gets no duration).+ if ev.EndTime.IsZero() && !ev.BeginTime.IsZero() &&+ (eventType == "completed" || eventType == "failed" || eventType == "stopped") {+ ev.EndTime = ev.EventTime+ }+ }+ ev.WaitDurationS, ev.ParseDurationS = ComputeDurations(ev.EnqueueTime, ev.BeginTime, ev.EndTime)+ ev.DurationMS = int64(ev.ParseDurationS * 1000)+ return ev+}diff --git a/internal/observability/ingestion_log_fix_test.go b/internal/observability/ingestion_log_fix_test.gonew file mode 100644index 0000000000..16610be8a0--- /dev/null+++ b/internal/observability/ingestion_log_fix_test.go@@ -0,0 +1,48 @@+package observability++import (+ "testing"+ "time"++ "ragflow/internal/entity"+)++func TestBuildIngestLogPrefersWorkerClaimTime(t *testing.T) {+ base := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC)+ enqueue := base+ openTime := base.Add(1 * time.Minute)+ claimTime := base.Add(10 * time.Minute)+ end := base.Add(20 * time.Minute)+ task := &entity.IngestionTask{ID: "t", DocumentID: "d", DatasetID: "kb", UserID: "u"}+ doc := &entity.Document{Name: ptrStr("r"), Suffix: ".pdf", Type: "pdf", Size: 10, ParserID: "pdf", ProcessBeginAt: ptrTime(claimTime)}+ run := &entity.PipelineOperationLog{RunCount: ptrInt(1), CreateDate: ptrTime(enqueue), ProcessBeginAt: ptrTime(openTime), ProcessEndAt: ptrTime(end)}++ log := BuildIngestLog(task, doc, run, "completed", "COMPLETED", "")+ if !log.BeginTime.Equal(claimTime) {+ t.Errorf("BeginTime = %v, want worker-claim time %v", log.BeginTime, claimTime)+ }+ if log.WaitDurationS <= 120 {+ t.Errorf("WaitDurationS = %v, want ~600 (enqueue -> worker claim), got near-zero", log.WaitDurationS)+ }+}++func TestBuildIngestLogPopulatesDurationMS(t *testing.T) {+ base := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC)+ task := &entity.IngestionTask{ID: "t", DocumentID: "d", DatasetID: "kb", UserID: "u"}+ doc := &entity.Document{Name: ptrStr("r"), Suffix: ".pdf", Type: "pdf", Size: 10, ParserID: "pdf", ProcessBeginAt: ptrTime(base.Add(10 * time.Minute))}+ run := &entity.PipelineOperationLog{RunCount: ptrInt(1), CreateDate: ptrTime(base), ProcessBeginAt: ptrTime(base.Add(1 * time.Minute)), ProcessEndAt: ptrTime(base.Add(20 * time.Minute))}++ log := BuildIngestLog(task, doc, run, "completed", "COMPLETED", "")+ want := int64(log.ParseDurationS * 1000)+ if want <= 0 {+ t.Fatalf("ParseDurationS = %v, want > 0 for setup", log.ParseDurationS)+ }+ if log.DurationMS != want {+ t.Errorf("DurationMS = %d, want %d", log.DurationMS, want)+ }++ created := BuildIngestLog(task, doc, &entity.PipelineOperationLog{RunCount: ptrInt(1)}, "created", "CREATED", "")+ if created.DurationMS != 0 {+ t.Errorf("created DurationMS = %d, want 0", created.DurationMS)+ }+}diff --git a/internal/observability/ingestion_log_tdd_test.go b/internal/observability/ingestion_log_tdd_test.gonew file mode 100644index 0000000000..bc6ce39045--- /dev/null+++ b/internal/observability/ingestion_log_tdd_test.go@@ -0,0 +1,105 @@+//go:build !ee++package observability++import (+ "testing"+ "time"++ "ragflow/internal/entity"+)++// TestClaimTimeRejectsStaleDocumentValue verifies that a document claim time left+// over from a PREVIOUS run (i.e. before this run even opened) is NOT treated as+// this run's claim time. This is the root cause of P1a: doc.ProcessBeginAt is+// stamped by PrepareValidatedRun and never cleared between runs, so a task that+// never reaches RUNNING in the current run must not inherit a stale value.+func TestClaimTimeRejectsStaleDocumentValue(t *testing.T) {+ base := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC)+ staleClaim := base.Add(-48 * time.Hour) // two days before this run opened+ runOpen := base.Add(-1 * time.Hour)+ doc := &entity.Document{ProcessBeginAt: ptrTime(staleClaim)}+ run := &entity.PipelineOperationLog{CreateDate: ptrTime(runOpen)}++ if got := claimTime(doc, run); got != nil {+ t.Fatalf("claimTime returned %v for a stale document value; want nil", got)+ }+}++// TestClaimTimeAcceptsValidClaimTime verifies a claim time that falls within this+// run's window is still honored (the happy path must keep working).+func TestClaimTimeAcceptsValidClaimTime(t *testing.T) {+ base := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC)+ runOpen := base+ claim := base.Add(10 * time.Minute)+ doc := &entity.Document{ProcessBeginAt: ptrTime(claim)}+ run := &entity.PipelineOperationLog{CreateDate: ptrTime(runOpen)}++ got := claimTime(doc, run)+ if got == nil || !got.Equal(claim) {+ t.Fatalf("claimTime = %v, want %v", got, claim)+ }+}++// TestBuildIngestLogPreClaimStoppedOmitsStaleClaimTime is the direct regression+// for P1a: a task stopped (by user / superseded) before any worker claimed it+// must not emit a BeginTime inherited from a prior run, and must not report a+// fabricated parse_duration_s of hours/days.+func TestBuildIngestLogPreClaimStoppedOmitsStaleClaimTime(t *testing.T) {+ base := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC)+ staleClaim := base.Add(-48 * time.Hour)+ runOpen := base.Add(-1 * time.Hour)+ task := &entity.IngestionTask{ID: "t", DocumentID: "d", DatasetID: "kb", UserID: "u"}+ doc := &entity.Document{Name: ptrStr("r"), Suffix: ".pdf", Type: "pdf", Size: 10, ParserID: "pdf", ProcessBeginAt: ptrTime(staleClaim)}+ run := &entity.PipelineOperationLog{RunCount: ptrInt(1), CreateDate: ptrTime(runOpen)}++ ev := BuildIngestLog(task, doc, run, "stopped", "STOPPED", "stopped by user")+ if !ev.BeginTime.IsZero() {+ t.Errorf("pre-claim stopped must omit stale BeginTime, got %v", ev.BeginTime)+ }+ if ev.ParseDurationS != 0 {+ t.Errorf("pre-claim stopped parse_duration_s must be 0, got %v", ev.ParseDurationS)+ }+ if ev.WaitDurationS != 0 {+ t.Errorf("pre-claim stopped wait_duration_s must be 0, got %v", ev.WaitDurationS)+ }+}++// TestBuildIngestLogPreClaimFailedOmitsStaleClaimTime covers the settlePublishFailure+// path: a run that fails before a worker claims it must not report a stale claim time.+func TestBuildIngestLogPreClaimFailedOmitsStaleClaimTime(t *testing.T) {+ base := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC)+ staleClaim := base.Add(-48 * time.Hour)+ runOpen := base.Add(-1 * time.Hour)+ task := &entity.IngestionTask{ID: "t", DocumentID: "d", DatasetID: "kb", UserID: "u"}+ doc := &entity.Document{Name: ptrStr("r"), Suffix: ".pdf", Type: "pdf", Size: 10, ParserID: "pdf", ProcessBeginAt: ptrTime(staleClaim)}+ run := &entity.PipelineOperationLog{RunCount: ptrInt(1), CreateDate: ptrTime(runOpen)}++ ev := BuildIngestLog(task, doc, run, "failed", "FAILED", "publish failed")+ if !ev.BeginTime.IsZero() {+ t.Errorf("pre-claim failed must omit stale BeginTime, got %v", ev.BeginTime)+ }+ if ev.ParseDurationS != 0 {+ t.Errorf("pre-claim failed parse_duration_s must be 0, got %v", ev.ParseDurationS)+ }+}++// TestBuildIngestLogRunningKeepsValidClaimTime guards the happy path: a task that+// actually started keeps its real claim time and a sane parse/wait duration.+func TestBuildIngestLogRunningKeepsValidClaimTime(t *testing.T) {+ base := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC)+ runOpen := base+ claim := base.Add(10 * time.Minute)+ end := base.Add(20 * time.Minute)+ task := &entity.IngestionTask{ID: "t", DocumentID: "d", DatasetID: "kb", UserID: "u"}+ doc := &entity.Document{Name: ptrStr("r"), Suffix: ".pdf", Type: "pdf", Size: 10, ParserID: "pdf", ProcessBeginAt: ptrTime(claim)}+ run := &entity.PipelineOperationLog{RunCount: ptrInt(1), CreateDate: ptrTime(runOpen), ProcessEndAt: ptrTime(end)}++ ev := BuildIngestLog(task, doc, run, "running", "RUNNING", "")+ if !ev.BeginTime.Equal(claim) {+ t.Errorf("running BeginTime = %v, want %v", ev.BeginTime, claim)+ }+ if ev.ParseDurationS <= 0 {+ t.Errorf("running parse_duration_s = %v, want > 0", ev.ParseDurationS)+ }+}diff --git a/internal/observability/ingestion_log_test.go b/internal/observability/ingestion_log_test.gonew file mode 100644index 0000000000..1b6d605bd4--- /dev/null+++ b/internal/observability/ingestion_log_test.go@@ -0,0 +1,150 @@+package observability++import (+ "testing"+ "time"++ "ragflow/internal/entity"+)++func TestComputeDurationsHappy(t *testing.T) {+ base := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC)+ enqueue := base+ begin := base.Add(5 * time.Minute)+ end := base.Add(15 * time.Minute)+ wait, parse := ComputeDurations(enqueue, begin, end)+ if wait != 300 {+ t.Errorf("wait_duration_s = %v, want 300", wait)+ }+ if parse != 600 {+ t.Errorf("parse_duration_s = %v, want 600", parse)+ }+}++func TestComputeDurationsClampsSkew(t *testing.T) {+ base := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC)+ enqueue := base.Add(10 * time.Minute)+ begin := base+ end := base.Add(20 * time.Minute)+ wait, parse := ComputeDurations(enqueue, begin, end)+ if wait < 0 {+ t.Errorf("wait_duration_s = %v, want >= 0 (clock skew must clamp)", wait)+ }+ if parse != 1200 {+ t.Errorf("parse_duration_s = %v, want 1200", parse)+ }+}++func TestComputeDurationsClampsEndBeforeBegin(t *testing.T) {+ base := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC)+ enqueue := base+ begin := base.Add(10 * time.Minute)+ end := base.Add(5 * time.Minute)+ wait, parse := ComputeDurations(enqueue, begin, end)+ if wait != 600 {+ t.Errorf("wait_duration_s = %v, want 600", wait)+ }+ if parse < 0 {+ t.Errorf("parse_duration_s = %v, want >= 0", parse)+ }+}++func ptrStr(s string) *string { return &s }+func ptrTime(t time.Time) *time.Time { return &t }+func ptrInt(i int) *int { return &i }++func TestBuildIngestLogHappy(t *testing.T) {+ now := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC)+ task := &entity.IngestionTask{+ ID: "task-1",+ DocumentID: "doc-1",+ DatasetID: "kb-1",+ UserID: "user-1",+ }+ doc := &entity.Document{+ Name: ptrStr("report"),+ Suffix: ".pdf",+ Type: "pdf",+ Size: 12345,+ ParserID: "pdf",+ CreateDate: ptrTime(now.Add(-time.Hour)),+ ProcessBeginAt: ptrTime(now.Add(2 * time.Minute)),+ }+ run := &entity.PipelineOperationLog{+ RunCount: ptrInt(1),+ TenantID: "tenant-1",+ Progress: 1,+ CreateDate: ptrTime(now),+ ProcessEndAt: ptrTime(now.Add(12 * time.Minute)),+ }+ log := BuildIngestLog(task, doc, run, "completed", "COMPLETED", "")++ if log.TaskID != "task-1" {+ t.Errorf("TaskID = %q, want task-1", log.TaskID)+ }+ if log.DocID != "doc-1" {+ t.Errorf("DocID = %q, want doc-1", log.DocID)+ }+ if log.DatasetID != "kb-1" {+ t.Errorf("DatasetID = %q, want kb-1", log.DatasetID)+ }+ if log.Filename != "report.pdf" {+ t.Errorf("Filename = %q, want report.pdf", log.Filename)+ }+ if log.FileType != "pdf" {+ t.Errorf("FileType = %q, want pdf", log.FileType)+ }+ if log.FileSize != 12345 {+ t.Errorf("FileSize = %d, want 12345", log.FileSize)+ }+ if log.ParserID != "pdf" {+ t.Errorf("ParserID = %q, want pdf", log.ParserID)+ }+ if log.EventType != "completed" {+ t.Errorf("EventType = %q, want completed", log.EventType)+ }+ if log.Status != "COMPLETED" {+ t.Errorf("Status = %q, want COMPLETED", log.Status)+ }+ if log.RunCount != 1 {+ t.Errorf("RunCount = %d, want 1", log.RunCount)+ }+ if log.TenantID != "tenant-1" {+ t.Errorf("TenantID = %q, want tenant-1", log.TenantID)+ }+ if log.WaitDurationS <= 0 {+ t.Errorf("WaitDurationS = %v, want > 0 (claim after enqueue)", log.WaitDurationS)+ }+ if log.ParseDurationS <= 0 {+ t.Errorf("ParseDurationS = %v, want > 0", log.ParseDurationS)+ }+}++func TestBuildIngestLogCreatedSkipsClaimTime(t *testing.T) {+ now := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC)+ task := &entity.IngestionTask{ID: "task-1", DocumentID: "doc-1", DatasetID: "kb-1", UserID: "user-1"}+ doc := &entity.Document{+ Name: ptrStr("report"),+ Suffix: ".pdf",+ Type: "pdf",+ Size: 10,+ ParserID: "pdf",+ ProcessBeginAt: ptrTime(now.Add(2 * time.Minute)),+ }+ run := &entity.PipelineOperationLog{RunCount: ptrInt(1), CreateDate: ptrTime(now)}+ log := BuildIngestLog(task, doc, run, "created", "CREATED", "")+ if !log.BeginTime.IsZero() {+ t.Errorf("created event BeginTime = %v, want zero", log.BeginTime)+ }+}++func TestBuildIngestLogFailedCarriesErrorInfo(t *testing.T) {+ now := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC)+ task := &entity.IngestionTask{ID: "task-1", DocumentID: "doc-1", DatasetID: "kb-1", UserID: "user-1"}+ doc := &entity.Document{Name: ptrStr("docname"), Suffix: ".pdf", Type: "pdf", Size: 10, ParserID: "pdf"}+ run := &entity.PipelineOperationLog{RunCount: ptrInt(1), CreateDate: ptrTime(now)}+ log := BuildIngestLog(task, doc, run, "failed", "FAILED", "boom")+ if log.ErrorInfo != "boom" {+ t.Errorf("ErrorInfo = %q, want boom", log.ErrorInfo)+ }+}diff --git a/internal/server/api/server.go b/internal/server/api/server.goindex 7c5fa85308..cbdf2fedf9 100644--- a/internal/server/api/server.go+++ b/internal/server/api/server.go@@ -458,6 +458,11 @@ func startServer(ctx context.Context, serverName string, arguments *bootstrap.Se// Set log levelbootstrap.SetLogger(serverName, arguments)+ // Initialize the ingestion lifecycle event sink (best-effort; stdout is+ // always on, ingestor.log.path adds an optional rotated file). Flush it on shutdown.+ bootstrap.SetIngestLog(serverName)+ defer common.SyncIngestLog()+// Start server in a goroutinego func() {serve(serverName, srv, apiListener)diff --git a/internal/server/bootstrap/runtime.go b/internal/server/bootstrap/runtime.goindex f3028e2537..a522f87ff9 100644--- a/internal/server/bootstrap/runtime.go+++ b/internal/server/bootstrap/runtime.go@@ -60,6 +60,22 @@ func SetLogger(serverName string, arguments *ServerArgs) {}}+// SetIngestLog (re)initializes the ingestion lifecycle event sink for the given+// server name from the configured ingest_log settings. It mirrors SetLogger so+// every server mode that can emit (the API server creating/enqueuing tasks, the+// ingestor running them, etc.) shares one sink. Emission is best-effort: a sink+// init failure is logged but never fatal. The sink always writes to stdout; an+// ingestor.log.path only adds an optional rotated file on top of it.+func SetIngestLog(serverName string) {+ globalConfig := server.GetConfig()+ ingestLogConfig := globalConfig.GetIngestLogConfig()+ // Flush any prior sink before swapping it out (e.g. on a reconfigure).+ common.SyncIngestLog()+ if err := common.InitIngestLog(ingestLogConfig.Type, ingestLogConfig.GetFileOutput(), serverName); err != nil {+ common.Error("Failed to initialize ingest log sink", err)+ }+}+// StartHeartbeat initializes and starts the heartbeat reporter to the admin// server. It is shared by the API, ingestor, and syncer server modes. The// caller must defer the returned *utility.ScheduledTask's Stop() method.diff --git a/internal/server/config/stub.go b/internal/server/config/stub.goindex 09f76af653..44f621407e 100644--- a/internal/server/config/stub.go+++ b/internal/server/config/stub.go@@ -107,8 +107,57 @@ func (c *AuditConfig) GetFileOutput() common.FileOutput {return common.FileOutput{}}-func (c *IngestLogConfig) ParseIngestLogConfig(v *viper.Viper) error { return nil }+// Default ingest-log rotation values mirror the documented IngestLogConfig+// defaults. File output is opt-in: leave ingestor.log.path empty and events emit+// to stdout only.+const (+ defaultIngestLogMaxSizeMB = 1000+ defaultIngestLogMaxBackups = 20+ defaultIngestLogMaxAgeDays = 60+)++// ParseIngestLogConfig reads the ingestor.log.* keys from the sub-viper passed by+// the ingestor config parser (already merged with the environment by+// server.Init), so env overrides such as RAGFLOW_INGESTOR_LOG_PATH win over the+// YAML file. A nil viper is a no-op so the struct keeps its zero (off) state.+func (c *IngestLogConfig) ParseIngestLogConfig(v *viper.Viper) error {+ if v == nil {+ return nil+ }+ c.Type = "file"+ c.Format = "json"+ c.MaxSize = defaultIngestLogMaxSizeMB+ c.MaxBackups = defaultIngestLogMaxBackups+ c.MaxAge = defaultIngestLogMaxAgeDays+ if v.IsSet("type") {+ c.Type = v.GetString("type")+ }+ if v.IsSet("format") {+ c.Format = v.GetString("format")+ }+ if v.IsSet("path") {+ c.Path = v.GetString("path")+ }+ if v.IsSet("max_size") {+ c.MaxSize = v.GetInt("max_size")+ }+ if v.IsSet("max_backups") {+ c.MaxBackups = v.GetInt("max_backups")+ }+ if v.IsSet("max_age") {+ c.MaxAge = v.GetInt("max_age")+ }+ return nil+}+// GetFileOutput projects the ingest log config onto the common.FileOutput sink+// descriptor. An empty Path means no file sink (stdout only), which keeps the+// feature off by default.func (c *IngestLogConfig) GetFileOutput() common.FileOutput {- return common.FileOutput{}+ return common.FileOutput{+ Path: c.Path,+ MaxSize: c.MaxSize,+ MaxBackups: c.MaxBackups,+ MaxAge: c.MaxAge,+ }}diff --git a/internal/server/ingestor/server.go b/internal/server/ingestor/server.goindex 3bd9d5f178..09ad677b11 100644--- a/internal/server/ingestor/server.go+++ b/internal/server/ingestor/server.go@@ -64,15 +64,14 @@ func logTokenizerCounters() {// admin, then gracefully stops the ingestor.func Run(ctx context.Context, cancel context.CancelFunc, serverName string, args *bootstrap.ServerArgs) error {globalConfig := server.GetConfig()- ingestLogConfig := globalConfig.GetIngestLogConfig()- err := common.InitIngestLog(ingestLogConfig.Type, ingestLogConfig.GetFileOutput(), serverName)- if err != nil {- common.Fatal("Failed to initialize ingest logger", zap.Error(err))- }+ // Initialize the ingestion lifecycle event sink (best-effort; stdout is+ // always on, ingestor.log.path adds an optional rotated file). Flush it on shutdown.+ bootstrap.SetIngestLog(serverName)+ defer common.SyncIngestLog()// Initialize tokenizer (rag_analyzer)// tokenizer.Init handles DictPath fallback: env var → /usr/share/infinity/resource- err = tokenizer.Init(&tokenizer.PoolConfig{})+ err := tokenizer.Init(&tokenizer.PoolConfig{})if err != nil {common.Fatal("Failed to initialize tokenizer", zap.Error(err))}diff --git a/internal/service/ingestion_task_event_test.go b/internal/service/ingestion_task_event_test.gonew file mode 100644index 0000000000..facdff867c--- /dev/null+++ b/internal/service/ingestion_task_event_test.go@@ -0,0 +1,88 @@+package service++import (+ "context"+ "sync"+ "testing"++ "ragflow/internal/common"+ "ragflow/internal/entity"+)++// recordingEmitter captures emitted ingest logs for assertions.+type recordingEmitter struct {+ mu sync.Mutex+ events []common.IngestionLog+}++func (r *recordingEmitter) EmitIngestLog(_ context.Context, log common.IngestionLog) {+ r.mu.Lock()+ defer r.mu.Unlock()+ r.events = append(r.events, log)+}++func (r *recordingEmitter) last() common.IngestionLog {+ r.mu.Lock()+ defer r.mu.Unlock()+ if len(r.events) == 0 {+ return common.IngestionLog{}+ }+ return r.events[len(r.events)-1]+}++// TestEmitLifecycleEventForwardsToEmitter verifies the service builds an+// observability event from the task and forwards it to the configured emitter.+func TestEmitLifecycleEventForwardsToEmitter(t *testing.T) {+ rec := &recordingEmitter{}+ svc := &IngestionTaskService{eventEmitter: rec}++ task := &entity.IngestionTask{+ ID: "task-1",+ DocumentID: "doc-1",+ DatasetID: "kb-1",+ UserID: "u-1",+ }+ svc.emitLifecycleEvent(context.Background(), task, "completed", "COMPLETED", "boom")++ got := rec.last()+ if got.EventType != "completed" {+ t.Errorf("EventType = %q, want completed", got.EventType)+ }+ if got.Status != "COMPLETED" {+ t.Errorf("Status = %q, want COMPLETED", got.Status)+ }+ if got.ErrorInfo != "boom" {+ t.Errorf("ErrorInfo = %q, want boom", got.ErrorInfo)+ }+ if got.TaskID != "task-1" || got.DocID != "doc-1" || got.DatasetID != "kb-1" || got.UserID != "u-1" {+ t.Errorf("identity mismatch: %+v", got)+ }+}++// TestEmitLifecycleEventNilEmitterIsNoop verifies a nil emitter never panics and+// a nil task is ignored (best-effort contract).+func TestEmitLifecycleEventNilEmitterIsNoop(t *testing.T) {+ svc := &IngestionTaskService{}+ task := &entity.IngestionTask{ID: "t", DocumentID: "d"}+ svc.emitLifecycleEvent(context.Background(), task, "created", "CREATED", "")+ svc.emitLifecycleEvent(context.Background(), nil, "created", "CREATED", "")+}++// TestSetEventEmitterSwapsSink verifies SetEventEmitter installs a new sink and+// that a nil argument is ignored (cannot disable emission by accident).+func TestSetEventEmitterSwapsSink(t *testing.T) {+ svc := &IngestionTaskService{}+ rec := &recordingEmitter{}+ svc.SetEventEmitter(nil) // ignored+ svc.SetEventEmitter(rec)++ task := &entity.IngestionTask{ID: "t2", DocumentID: "d2", DatasetID: "kb2", UserID: "u2"}+ svc.emitLifecycleEvent(context.Background(), task, "running", "RUNNING", "")++ if len(rec.events) != 1 {+ t.Fatalf("expected 1 emitted event, got %d", len(rec.events))+ }+ if rec.last().EventType != "running" {+ t.Errorf("EventType = %q, want running", rec.last().EventType)+ }+}diff --git a/internal/service/ingestion_task_service.go b/internal/service/ingestion_task_service.goindex 014deefd1a..b4405dde03 100644--- a/internal/service/ingestion_task_service.go+++ b/internal/service/ingestion_task_service.go@@ -13,6 +13,7 @@ import ("ragflow/internal/engine/kvrocks""ragflow/internal/entity"ingestionpipeline "ragflow/internal/ingestion/pipeline"+ "ragflow/internal/observability""go.uber.org/zap""gorm.io/gorm"@@ -125,6 +126,7 @@ type IngestionTaskService struct {userCanvasDAO *dao.UserCanvasDAOtaskPublisher TaskPublisherlogSettings IngestionLogSettings+ eventEmitter observability.IngestionEventEmitter// purgeTaskState removes the deleted task's run-scoped chunk and media// cache state. Nil resolves to the process-wide implementation.purgeTaskState func(context.Context, string) error@@ -139,6 +141,7 @@ func NewIngestionTaskService() *IngestionTaskService {userCanvasDAO: dao.NewUserCanvasDAO(),taskPublisher: NewMessageQueueTaskPublisher(),logSettings: DefaultIngestionLogSettings(),+ eventEmitter: &observability.CommonLogEmitter{},}}@@ -149,6 +152,42 @@ func (s *IngestionTaskService) SetTaskPublisher(taskPublisher TaskPublisher) {s.taskPublisher = taskPublisher}+// SetEventEmitter swaps the lifecycle event sink. A nil argument is ignored so+// callers cannot accidentally disable emission by passing nil.+func (s *IngestionTaskService) SetEventEmitter(emitter observability.IngestionEventEmitter) {+ if emitter == nil {+ return+ }+ s.eventEmitter = emitter+}++// emitLifecycleEvent builds and emits a structured observability event for a+// task lifecycle transition. It is best-effort: any lookup failure or a nil+// emitter is silently ignored so the parse pipeline is never disrupted by+// observability. Enrichment reads the document (file name/type/size/parser) and+// the bound pipeline run (run number, begin/end, progress) and forwards them to+// observability.BuildIngestLog, which owns the field mapping onto common.IngestionLog.+func (s *IngestionTaskService) emitLifecycleEvent(ctx context.Context, task *entity.IngestionTask, typ, status, errorInfo string) {+ if s.eventEmitter == nil || task == nil {+ return+ }+ var doc *entity.Document+ if s.documentDAO != nil {+ doc, _ = s.documentDAO.GetByID(ctx, dao.DB, task.DocumentID)+ }+ logID := task.PipelineLogID+ if (logID == nil || *logID == "") && s.documentDAO != nil {+ if fresh, err := s.GetTask(ctx, task.ID); err == nil {+ logID = fresh.PipelineLogID+ }+ }+ var run *entity.PipelineOperationLog+ if logID != nil && *logID != "" && s.pipelineLogDAO != nil {+ run, _ = s.pipelineLogDAO.GetByID(ctx, dao.DB, *logID)+ }+ s.eventEmitter.EmitIngestLog(ctx, observability.BuildIngestLog(task, doc, run, typ, status, errorInfo))+}+func (s *IngestionTaskService) ListByUser(ctx context.Context, userID string, datasetID *string, page, pageSize int) ([]*entity.IngestionTask, error) {if datasetID == nil {return s.ingestionTaskDAO.ListByUserID(ctx, dao.DB, userID, page, pageSize)@@ -265,6 +304,7 @@ func (s *IngestionTaskService) TransitionTaskToRunning(ctx context.Context, task// close the open row. Close it here or it stays RUNNING forever.s.advanceOpenLog(ctx, task, dao.OpenPipelineOperationStatuses(), string(entity.TaskStatusCancel))s.recordRunTerminal(ctx, task, "Task stopped by user.")+ s.emitLifecycleEvent(ctx, task, "stopped", common.STOPPED, "Task stopped by user.")return task, nilcase common.RUNNING, common.COMPLETED, common.STOPPED, common.FAILED:return task, nil@@ -290,6 +330,12 @@ func (s *IngestionTaskService) PrepareValidatedRun(ctx context.Context, task *en}s.advanceOpenLog(ctx, task, logFromUnstartOrScheduled, string(entity.TaskStatusRunning))s.recordRunMessage(ctx, task, "Task is running...")+ // Emit the running lifecycle event only after the document's worker-claim+ // time has been stamped, so the event's BeginTime (and wait_duration_s)+ // reflect this run rather than a zero or previous-run value. Emitting here+ // (guarded by a successful claim in the caller) keeps one running event per+ // actual run start.+ s.emitLifecycleEvent(ctx, task, "running", common.RUNNING, "")}func (s *IngestionTaskService) RequestStop(ctx context.Context, taskID string) (*entity.IngestionTask, error) {@@ -308,6 +354,7 @@ func (s *IngestionTaskService) RequestStop(ctx context.Context, taskID string) (// CANCEL here, otherwise the detail page keeps a queued entry.s.advanceOpenLog(ctx, stopped, logFromUnstartOrScheduled, string(entity.TaskStatusCancel))s.recordRunTerminal(ctx, stopped, "Task stopped by user.")+ s.emitLifecycleEvent(ctx, stopped, "stopped", common.STOPPED, "Task stopped by user.")return stopped, nilcase common.RUNNING:task, err = s.transition(ctx, taskID, common.STOPPING)@@ -348,6 +395,7 @@ func (s *IngestionTaskService) supersedeUnstartedTask(ctx context.Context, taskIif err := s.settleTaskRun(ctx, stopped, entity.TaskStatusCancel, terminalMessage); err != nil {return nil, err}+ s.emitLifecycleEvent(ctx, stopped, "stopped", common.STOPPED, terminalMessage)return stopped, nilcase common.RUNNING, common.STOPPING:return nil, fmt.Errorf("task %s is %s and cannot be superseded", taskID, task.Status)@@ -402,11 +450,27 @@ func (s *IngestionTaskService) MarkCompleted(ctx context.Context, taskID string)if task.Status == common.COMPLETED || task.Status == common.STOPPED || task.Status == common.FAILED {return nil // already terminal, idempotent — mirrors MarkStopped}- _, err = s.transition(ctx, taskID, common.COMPLETED)- return err+ updated, err := s.transition(ctx, taskID, common.COMPLETED)+ if err != nil {+ return err+ }+ s.emitLifecycleEvent(ctx, updated, "completed", common.COMPLETED, "")+ return nil}-func (s *IngestionTaskService) MarkFailed(ctx context.Context, taskID string) error {+// MarkFailed transitions a task to FAILED and emits a lifecycle event carrying+// the terminal reason. This is the public entry point used by the ingestor,+// which has the failure reason (timeout/panic/run-identity error) available at+// the call site; passing it here is what populates the event's ErrorInfo.+func (s *IngestionTaskService) MarkFailed(ctx context.Context, taskID, reason string) error {+ return s.markFailed(ctx, taskID, reason)+}++// markFailed transitions a task to FAILED and emits a lifecycle event carrying+// the terminal reason. The public MarkFailed forwards the reason supplied by+// the ingestor (timeout/panic/run-identity error); internal paths such as+// settlePublishFailure pass an explicit reason too.+func (s *IngestionTaskService) markFailed(ctx context.Context, taskID, reason string) error {task, err := s.GetTask(ctx, taskID)if err != nil {return err@@ -414,8 +478,12 @@ func (s *IngestionTaskService) MarkFailed(ctx context.Context, taskID string) erif task.Status == common.FAILED || task.Status == common.COMPLETED || task.Status == common.STOPPED {return nil // already terminal, idempotent — mirrors MarkStopped}- _, err = s.transition(ctx, taskID, common.FAILED)- return err+ updated, err := s.transition(ctx, taskID, common.FAILED)+ if err != nil {+ return err+ }+ s.emitLifecycleEvent(ctx, updated, "failed", common.FAILED, reason)+ return nil}// MarkStopped transitions the task from STOPPING to STOPPED (terminal).@@ -429,8 +497,12 @@ func (s *IngestionTaskService) MarkStopped(ctx context.Context, taskID string) eif task.Status == common.STOPPED || task.Status == common.COMPLETED || task.Status == common.FAILED {return nil}- _, err = s.transition(ctx, taskID, common.STOPPED)- return err+ updated, err := s.transition(ctx, taskID, common.STOPPED)+ if err != nil {+ return err+ }+ s.emitLifecycleEvent(ctx, updated, "stopped", common.STOPPED, "")+ return nil}func (s *IngestionTaskService) Remove(ctx context.Context, taskID string, userID *string) (*dao.TaskInfo, error) {@@ -672,6 +744,7 @@ func (s *IngestionTaskService) createAndEnqueueWithKBCache(ctx context.Context,if err = s.ensureRunIdentity(ctx, existing, kbCache); err != nil {return nil, fmt.Errorf("ensure run identity for task %s: %w", existing.ID, err)}+ s.emitLifecycleEvent(ctx, existing, "created", common.CREATED, "")if err = s.enqueueTask(existing.ID); err != nil {s.settlePublishFailure(ctx, existing)return nil, err@@ -698,6 +771,7 @@ func (s *IngestionTaskService) createAndEnqueueWithKBCache(ctx context.Context,}return nil, fmt.Errorf("ensure run identity for task %s: %w", existing.ID, err)}+ s.emitLifecycleEvent(ctx, existing, "created", common.CREATED, "")if err = s.enqueueTask(existing.ID); err != nil {s.settlePublishFailure(ctx, existing)return nil, err@@ -718,6 +792,7 @@ func (s *IngestionTaskService) createAndEnqueueWithKBCache(ctx context.Context,}return nil, fmt.Errorf("ensure run identity for task %s: %w", created.ID, err)}+ s.emitLifecycleEvent(ctx, created, "created", common.CREATED, "")if err = s.enqueueTask(created.ID); err != nil {s.settlePublishFailure(ctx, created)return nil, err@@ -757,6 +832,7 @@ func (s *IngestionTaskService) markScheduledAfterPublish(ctx context.Context, ta}s.advanceOpenLog(ctx, task, logFromUnstart, string(entity.TaskStatusSchedule))s.recordRunMessage(ctx, task, "Task is queued...")+ s.emitLifecycleEvent(ctx, task, "scheduled", common.SCHEDULED, "")return task, nil}@@ -810,7 +886,7 @@ func (s *IngestionTaskService) settlePublishFailure(ctx context.Context, task *eif task == nil {return}- if err := s.MarkFailed(ctx, task.ID); err != nil {+ if err := s.markFailed(ctx, task.ID, "Task publish failed."); err != nil {common.Error(fmt.Sprintf("mark task %s failed after publish failure", task.ID), err)return}@@ -1061,11 +1137,35 @@ func (s *IngestionTaskService) RecordMessage(ctx context.Context, pipelineLogID,return s.insertEvent(ctx, ingestionEventMessage, pipelineLogID, taskID, "", 0, message)}+// persistProcessEndAt records the authoritative terminal end time for a run.+// It is idempotent: if a real process_end_at is already persisted, the existing+// value is left untouched, so a later terminal event cannot rewrite the recorded+// completion instant. Best-effort: errors are logged, never returned, because a+// failure here must not fail the terminal settlement.+func (s *IngestionTaskService) persistProcessEndAt(ctx context.Context, pipelineLogID string) {+ if dao.DB == nil {+ return+ }+ if updErr := dao.DB.WithContext(ctx).+ Model(&entity.PipelineOperationLog{}).+ Where("id = ?", pipelineLogID).+ Where("process_end_at IS NULL").+ Update("process_end_at", time.Now()).Error; updErr != nil {+ common.Warn(fmt.Sprintf("persist process_end_at for run %s: %v", pipelineLogID, updErr))+ }+}+// RecordTerminal writes the terminal explanation associated with a run.func (s *IngestionTaskService) RecordTerminal(ctx context.Context, pipelineLogID, taskID, message string) error {if err := s.ensureTerminalEvent(ctx, pipelineLogID, taskID, message); err != nil {return err}+ // Persist the authoritative terminal end time so ingestion observability+ // events and any downstream duration query read a real value instead of+ // falling back to the transition time. Idempotent: a run whose end time+ // was already recorded keeps it, so a repeated terminal settlement (e.g.+ // overlapping failure/timeout events) cannot rewrite the completion instant.+ s.persistProcessEndAt(context.WithoutCancel(ctx), pipelineLogID)foldContext := context.WithoutCancel(ctx)foldStarted := time.Now()foldResult, foldErr := s.foldIngestionRun(foldContext, pipelineLogID, s.logSettings.MaxRowsPerRun)diff --git a/internal/service/ingestion_task_service_test.go b/internal/service/ingestion_task_service_test.goindex 349e9e7876..c00ad0a19d 100644--- a/internal/service/ingestion_task_service_test.go+++ b/internal/service/ingestion_task_service_test.go@@ -649,7 +649,7 @@ func TestIngestionTaskServiceMarkFailedUpdatesTaskStatus(t *testing.T) {ctx := t.Context()svc := NewIngestionTaskService()- if err := svc.MarkFailed(ctx, "task-1"); err != nil {+ if err := svc.MarkFailed(ctx, "task-1", ""); err != nil {t.Fatalf("MarkFailed failed: %v", err)}task, err := dao.NewIngestionTaskDAO().GetByID(ctx, db, "task-1")@@ -1883,7 +1883,7 @@ func TestIngestionTaskServiceMarkFailedIdempotentOnAlreadyTerminal(t *testing.T)svc := NewIngestionTaskService()ctx := t.Context()- if err := svc.MarkFailed(ctx, "task-1"); err != nil {+ if err := svc.MarkFailed(ctx, "task-1", ""); err != nil {t.Fatalf("MarkFailed on already COMPLETED task should be idempotent, got: %v", err)}}@@ -2523,3 +2523,168 @@ func TestIngestionTaskServiceCleanupMissingRun(t *testing.T) {})}}++// recordingIngestLogEmitter captures emitted lifecycle events for assertions.+type recordingIngestLogEmitter struct {+ mu sync.Mutex+ logs []common.IngestionLog+}++func (e *recordingIngestLogEmitter) EmitIngestLog(_ context.Context, log common.IngestionLog) {+ e.mu.Lock()+ defer e.mu.Unlock()+ e.logs = append(e.logs, log)+}++func (e *recordingIngestLogEmitter) last() common.IngestionLog {+ e.mu.Lock()+ defer e.mu.Unlock()+ if len(e.logs) == 0 {+ return common.IngestionLog{}+ }+ return e.logs[len(e.logs)-1]+}++// TestIngestionTaskServiceMarkFailedCarriesReason verifies that the failure+// reason supplied by the ingestor reaches the emitted lifecycle event's+// ErrorInfo, instead of being dropped (the previous gap: the run ledger got the+// message but the observability event carried an empty ErrorInfo).+func TestIngestionTaskServiceMarkFailedCarriesReason(t *testing.T) {+ db := setupServiceTestDB(t)+ pushServiceDB(t, db)+ insertTestIngestionTask(t, "task-1", "user-1", "doc-1", "kb-1")+ if err := dao.DB.Model(&entity.IngestionTask{}).Where("id = ?", "task-1").Update("status", common.RUNNING).Error; err != nil {+ t.Fatalf("set running status: %v", err)+ }+ ctx := t.Context()+ emitter := &recordingIngestLogEmitter{}+ svc := NewIngestionTaskService()+ svc.SetEventEmitter(emitter)+ const reason = "boom: nil deref"+ if err := svc.MarkFailed(ctx, "task-1", reason); err != nil {+ t.Fatalf("MarkFailed failed: %v", err)+ }+ got := emitter.last()+ if got.ErrorInfo != reason {+ t.Fatalf("emitted ErrorInfo = %q, want %q", got.ErrorInfo, reason)+ }+ if got.EventType != "failed" || got.Status != common.FAILED {+ t.Fatalf("emitted event type/status = %q/%q, want failed/%q", got.EventType, got.Status, common.FAILED)+ }+}++// TestIngestionTaskServicePersistProcessEndAtIdempotent verifies that+// persistProcessEndAt records the end time on first settlement and leaves an+// already-recorded value untouched on a later settlement (overlapping+// failure/timeout terminal events must not rewrite the completion instant).+func TestIngestionTaskServicePersistProcessEndAtIdempotent(t *testing.T) {+ db := setupServiceTestDB(t)+ pushServiceDB(t, db)+ runID := "run-idempotent-1"+ begin := time.Now().Add(-time.Minute)+ run := &entity.PipelineOperationLog{+ ID: runID,+ DocumentID: "doc-idem-1",+ TenantID: "tenant-1",+ KbID: "kb-1",+ ParserID: "parser-1",+ DocumentName: "doc",+ DocumentSuffix: ".pdf",+ DocumentType: "pdf",+ SourceFrom: "web",+ OperationStatus: "RUNNING",+ ProcessBeginAt: &begin,+ }+ if err := db.Create(run).Error; err != nil {+ t.Fatalf("create run: %v", err)+ }+ ctx := t.Context()+ svc := NewIngestionTaskService()+ svc.persistProcessEndAt(ctx, runID)+ var first entity.PipelineOperationLog+ if err := db.Where("id = ?", runID).First(&first).Error; err != nil {+ t.Fatalf("load run: %v", err)+ }+ if first.ProcessEndAt == nil {+ t.Fatalf("process_end_at not persisted on first call")+ }+ firstEnd := *first.ProcessEndAt+ // A second settlement must not overwrite the recorded end time.+ svc.persistProcessEndAt(ctx, runID)+ var second entity.PipelineOperationLog+ if err := db.Where("id = ?", runID).First(&second).Error; err != nil {+ t.Fatalf("load run again: %v", err)+ }+ if second.ProcessEndAt == nil || !second.ProcessEndAt.Equal(firstEnd) {+ t.Fatalf("process_end_at changed on second call: first=%v second=%v", firstEnd, second.ProcessEndAt)+ }+}++// TestPrepareValidatedRunEmitsRunningEventWithFreshClaimTime verifies that the+// running lifecycle event is emitted AFTER PrepareValidatedRun stamps the+// document's worker-claim time. Previously the running event was emitted in+// TransitionTaskToRunning, before the claim time was written, so its BeginTime+// (and therefore wait_duration_s) was either zero or the previous run's stale+// value.+func TestPrepareValidatedRunEmitsRunningEventWithFreshClaimTime(t *testing.T) {+ db := setupServiceTestDB(t)+ pushServiceDB(t, db)+ insertTestKB(t, "kb-1", "tenant-1", 1, 0, 0)+ insertTestDoc(t, "doc-1", "kb-1", 0, 0)+ insertTestIngestionTask(t, "task-1", "user-1", "doc-1", "kb-1")++ // Simulate a previous run that already stamped a claim time one hour ago.+ stale := time.Now().Add(-time.Hour)+ if err := db.Model(&entity.Document{}).Where("id = ?", "doc-1").Update("process_begin_at", stale).Error; err != nil {+ t.Fatalf("seed stale claim time: %v", err)+ }++ // Bind a pipeline-operation-log row so advanceOpenLog/recordRunMessage succeed.+ openLog := &entity.PipelineOperationLog{+ ID: "log-1",+ DocumentID: "doc-1",+ TenantID: "tenant-1",+ KbID: "kb-1",+ ParserID: "naive",+ TaskType: "Parse",+ OperationStatus: string(entity.TaskStatusUnstart),+ }+ if err := db.Create(openLog).Error; err != nil {+ t.Fatalf("seed open log: %v", err)+ }+ if err := db.Model(&entity.IngestionTask{}).Where("id = ?", "task-1").Update("pipeline_log_id", "log-1").Error; err != nil {+ t.Fatalf("bind task: %v", err)+ }++ ctx := t.Context()+ emitter := &recordingIngestLogEmitter{}+ svc := NewIngestionTaskService()+ svc.SetEventEmitter(emitter)++ task, err := dao.NewIngestionTaskDAO().GetByID(ctx, db, "task-1")+ if err != nil {+ t.Fatalf("load task: %v", err)+ }+ svc.PrepareValidatedRun(ctx, task)++ var running common.IngestionLog+ found := false+ for _, l := range emitter.logs {+ if l.EventType == "running" {+ running = l+ found = true+ }+ }+ if !found {+ t.Fatalf("no running event emitted by PrepareValidatedRun")+ }+ if running.BeginTime.IsZero() {+ t.Fatalf("running event BeginTime is zero - claim time was not stamped before emit")+ }+ if !running.BeginTime.After(stale) {+ t.Fatalf("running event BeginTime = %v, want after stale claim time %v (must reflect current run)", running.BeginTime, stale)+ }+ if time.Since(running.BeginTime) > 30*time.Second {+ t.Fatalf("running event BeginTime = %v, suspiciously old for a fresh claim", running.BeginTime)+ }+}