Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

feat(ingestion): emit structured lifecycle events for document-parse tasks (#21711)

xugangqiang69fc4c9ec6e1
diff --git a/internal/common/ingest_log_path_test.go b/internal/common/ingest_log_path_test.go
new file mode 100644
index 0000000000..d7ba0eed5c
--- /dev/null
+++ b/internal/common/ingest_log_path_test.go
@@ -0,0 +1,68 @@
+//go:build !ee
+
+package common
+
+import (
+ "os"
+ "path/filepath"
+ "testing"
+ "time"
+)
+
+// TestInitIngestLogPathTreatedAsFile is the regression test for P1b: the configured
+// ingestor.log.path is a FILE name (per conf/service_conf.yaml comment), so InitIngestLog
+// must write to that exact file and must NOT treat it as a directory.
+func TestInitIngestLogPathTreatedAsFile(t *testing.T) {
+ t.Cleanup(resetIngestLog)
+
+ dir := t.TempDir()
+ filePath := filepath.Join(dir, "ingest_log.log")
+
+ if err := InitIngestLog("file", FileOutput{Path: filePath}, "api_server_9381"); err != nil {
+ t.Fatalf("InitIngestLog: %v", err)
+ }
+
+ LogIngestEvent(IngestionLog{EventTime: time.Now().UTC(), TaskID: "t1", EventType: "created", Status: "CREATED"})
+ SyncIngestLog()
+
+ fi, err := os.Stat(filePath)
+ if err != nil {
+ t.Fatalf("expected log file at %s: %v", filePath, err)
+ }
+ if fi.IsDir() {
+ t.Fatalf("%s was created as a DIRECTORY; path must be treated as a file", filePath)
+ }
+
+ if stray, err := os.Stat(filepath.Join(dir, "ingest_log.log")); err == nil && stray.IsDir() {
+ t.Fatalf("stray directory %s/ created; path was misinterpreted as a directory", dir)
+ }
+}
+
+// TestInitIngestLogEmptyPathNoFile asserts the documented default: an empty path emits to
+// stdout only and opens no rotated file. This guards the off-by-default contract.
+func TestInitIngestLogEmptyPathNoFile(t *testing.T) {
+ t.Cleanup(resetIngestLog)
+
+ dir := t.TempDir()
+ if err := InitIngestLog("file", FileOutput{}, "api_server_9381"); err != nil {
+ t.Fatalf("InitIngestLog: %v", err)
+ }
+ LogIngestEvent(IngestionLog{EventTime: time.Now().UTC(), TaskID: "t1"})
+ SyncIngestLog()
+
+ entries, _ := os.ReadDir(dir)
+ for _, e := range entries {
+ if nameContains(e.Name(), "ingest") {
+ t.Fatalf("empty path opened a file under %s: %s", dir, e.Name())
+ }
+ }
+}
+
+func nameContains(s, sub string) bool {
+ for i := 0; i+len(sub) <= len(s); i++ {
+ if s[i:i+len(sub)] == sub {
+ return true
+ }
+ }
+ return false
+}
diff --git a/internal/common/ingest_log_test.go b/internal/common/ingest_log_test.go
new file mode 100644
index 0000000000..9abaedcce0
--- /dev/null
+++ b/internal/common/ingest_log_test.go
@@ -0,0 +1,186 @@
+//go:build !ee
+
+package common
+
+import (
+ "bytes"
+ "encoding/json"
+ "os"
+ "path/filepath"
+ "reflect"
+ "strings"
+ "testing"
+ "time"
+
+ "go.uber.org/zap"
+)
+
+func resetIngestLog() {
+ ingestLogger.Store(zap.NewNop())
+}
+
+// TestIngestLogUnifiedStruct asserts the single unified type: renamed fields
+// (DocID not DocumentID, EventType not TaskType) and the absence of the
+// un-serializable Error error interface. Structural guard for 方案B unification.
+func TestIngestLogUnifiedStruct(t *testing.T) {
+ typ := reflect.TypeOf(IngestionLog{})
+ for _, f := range []string{"DocID", "EventType", "ErrorInfo", "Code"} {
+ if _, ok := typ.FieldByName(f); !ok {
+ t.Errorf("IngestionLog missing expected unified field %q", f)
+ }
+ }
+ for _, bad := range []string{"DocumentID", "TaskType", "Error"} {
+ if _, ok := typ.FieldByName(bad); ok {
+ t.Errorf("IngestionLog must not keep legacy field %q after unification", bad)
+ }
+ }
+}
+
+// TestIngestLogEmitsUnifiedJSON happy path: flat JSON line with error_info (string) and code.
+func TestIngestLogEmitsUnifiedJSON(t *testing.T) {
+ defer resetIngestLog()
+ dir := t.TempDir()
+ if err := InitIngestLog("json", FileOutput{Path: dir, Filename: "ingest.log"}, "svc"); err != nil {
+ t.Fatalf("InitIngestLog: %v", err)
+ }
+ LogIngestEvent(IngestionLog{
+ EventTime: time.Now().UTC(),
+ TenantID: "t1",
+ DatasetID: "kb1",
+ DocID: "d1",
+ UserID: "u1",
+ TaskID: "task1",
+ EventType: "completed",
+ Status: "COMPLETED",
+ ErrorInfo: "boom",
+ Filename: "report.pdf",
+ FileType: "pdf",
+ FileSize: 12345,
+ ParserID: "pdf",
+ Code: 7,
+ })
+ SyncIngestLog()
+ raw, err := os.ReadFile(filepath.Join(dir, "ingest.log"))
+ if err != nil {
+ t.Fatalf("read sink: %v", err)
+ }
+ var m map[string]any
+ if err := json.Unmarshal(raw, &m); err != nil {
+ t.Fatalf("emitted line not JSON: %v (raw=%q)", err, raw)
+ }
+ if m["error_info"] != "boom" {
+ t.Errorf("error_info = %v, want boom (string, not error interface)", m["error_info"])
+ }
+ if c, ok := m["code"].(float64); !ok || c != 7 {
+ t.Errorf("code = %v (%T), want 7", m["code"], m["code"])
+ }
+ if _, ok := m["error"]; ok {
+ t.Errorf("legacy error key must not be present; only error_info is allowed")
+ }
+}
+
+// TestIngestLogCodeRoundTrips adversarial: Code is a first-class struct field,
+// not hard-coded 0 via a secondary mapping layer.
+func TestIngestLogCodeRoundTrips(t *testing.T) {
+ defer resetIngestLog()
+ dir := t.TempDir()
+ if err := InitIngestLog("json", FileOutput{Path: dir, Filename: "ingest.log"}, "svc"); err != nil {
+ t.Fatalf("InitIngestLog: %v", err)
+ }
+ LogIngestEvent(IngestionLog{TaskID: "task1", EventType: "failed", Status: "FAILED", Code: 7})
+ SyncIngestLog()
+ raw, _ := os.ReadFile(filepath.Join(dir, "ingest.log"))
+ var m map[string]any
+ if err := json.Unmarshal(raw, &m); err != nil {
+ t.Fatalf("emitted line not JSON: %v (raw=%q)", err, raw)
+ }
+ if c, ok := m["code"].(float64); !ok || c != 7 {
+ t.Errorf("code = %v (%T), want 7 (first-class struct field, not hard-coded 0)", m["code"], m["code"])
+ }
+}
+
+// TestIngestLogOmitsZeroTimes adversarial: zero-value times must not leak.
+func TestIngestLogOmitsZeroTimes(t *testing.T) {
+ defer resetIngestLog()
+ dir := t.TempDir()
+ if err := InitIngestLog("json", FileOutput{Path: dir, Filename: "ingest.log"}, "svc"); err != nil {
+ t.Fatalf("InitIngestLog: %v", err)
+ }
+ LogIngestEvent(IngestionLog{EventTime: time.Now().UTC(), TaskID: "t1", EventType: "created", Status: "CREATED"})
+ SyncIngestLog()
+ raw, _ := os.ReadFile(filepath.Join(dir, "ingest.log"))
+ if strings.Contains(string(raw), "0001-01-01") {
+ t.Errorf("emitted zero-time sentinel 0001-01-01: %s", raw)
+ }
+ var m map[string]any
+ json.Unmarshal(raw, &m)
+ for _, k := range []string{"begin_time", "end_time", "upload_time", "enqueue_time"} {
+ if _, ok := m[k]; ok {
+ t.Errorf("zero-time field %q must be omitted, got %v", k, m[k])
+ }
+ }
+ if _, ok := m["event_time"]; !ok {
+ t.Errorf("event_time must always be present")
+ }
+}
+
+// TestIngestLogOmitsEmptyErrorInfo adversarial: empty ErrorInfo omitted.
+func TestIngestLogOmitsEmptyErrorInfo(t *testing.T) {
+ defer resetIngestLog()
+ dir := t.TempDir()
+ if err := InitIngestLog("json", FileOutput{Path: dir, Filename: "ingest.log"}, "svc"); err != nil {
+ t.Fatalf("InitIngestLog: %v", err)
+ }
+ LogIngestEvent(IngestionLog{TaskID: "t1", EventType: "created", Status: "CREATED"})
+ SyncIngestLog()
+ raw, _ := os.ReadFile(filepath.Join(dir, "ingest.log"))
+ var m map[string]any
+ json.Unmarshal(raw, &m)
+ if _, ok := m["error_info"]; ok {
+ t.Errorf("empty error_info must be omitted, got %v", m["error_info"])
+ }
+}
+
+// TestIngestLogOmitsEmptyUserName corner.
+func TestIngestLogOmitsEmptyUserName(t *testing.T) {
+ defer resetIngestLog()
+ dir := t.TempDir()
+ if err := InitIngestLog("json", FileOutput{Path: dir, Filename: "ingest.log"}, "svc"); err != nil {
+ t.Fatalf("InitIngestLog: %v", err)
+ }
+ LogIngestEvent(IngestionLog{TaskID: "task1", EventType: "created", Status: "CREATED"})
+ SyncIngestLog()
+ raw, _ := os.ReadFile(filepath.Join(dir, "ingest.log"))
+ if strings.Contains(string(raw), "user_name") {
+ t.Fatalf("emitted line contains user_name when UserName is empty: %s", raw)
+ }
+}
+
+// TestIngestLogOneFlatLine adversarial: each event is exactly one JSON line.
+func TestIngestLogOneFlatLine(t *testing.T) {
+ defer resetIngestLog()
+ dir := t.TempDir()
+ if err := InitIngestLog("json", FileOutput{Path: dir, Filename: "ingest.log"}, "svc"); err != nil {
+ t.Fatalf("InitIngestLog: %v", err)
+ }
+ LogIngestEvent(IngestionLog{TaskID: "task1", EventType: "completed", Status: "COMPLETED", ErrorInfo: "boom"})
+ SyncIngestLog()
+ raw, _ := os.ReadFile(filepath.Join(dir, "ingest.log"))
+ if len(bytes.TrimSpace(raw)) == 0 {
+ t.Fatal("LogIngest wrote no output")
+ }
+ lines := 0
+ for _, l := range bytes.Split(raw, []byte("\n")) {
+ if len(bytes.TrimSpace(l)) == 0 {
+ continue
+ }
+ lines++
+ var m map[string]any
+ if err := json.Unmarshal(l, &m); err != nil {
+ t.Fatalf("non-JSON line: %v (raw=%q)", err, l)
+ }
+ }
+ if lines != 1 {
+ t.Fatalf("expected 1 line, got %d (raw=%q)", lines, raw)
+ }
+}
diff --git a/internal/common/stub.go b/internal/common/stub.go
index dcab51e087..0334a1184f 100644
--- a/internal/common/stub.go
+++ b/internal/common/stub.go
@@ -19,9 +19,15 @@
package common
import (
+ "fmt"
+ "os"
+ "path/filepath"
+ "sync/atomic"
"time"
"go.uber.org/zap"
+ "go.uber.org/zap/zapcore"
+ "gopkg.in/natefinch/lumberjack.v2"
)
var (
@@ -55,23 +61,179 @@ type OperationLog struct {
func Audit(operationLog *OperationLog, fields ...zap.Field) {
}
+// ingestLogger receives the serialized ingest events. It defaults to a no-op so a
+// binary that never calls InitIngestLog emits nothing and never panics.
+var ingestLogger atomic.Pointer[zap.Logger]
+
+func init() {
+ ingestLogger.Store(zap.NewNop())
+}
+
+// InitIngestLog configures the sink for ingestion lifecycle events. It mirrors
+// InitLogger: output always goes to stdout, and an optional rotated file is added
+// when both Path and Filename are set.
func InitIngestLog(logType string, file FileOutput, serviceName string) error {
+ encoderConfig := zapcore.EncoderConfig{
+ TimeKey: "timestamp",
+ LevelKey: "level",
+ NameKey: "service",
+ CallerKey: "caller",
+ FunctionKey: "",
+ MessageKey: "msg",
+ StacktraceKey: "stacktrace",
+ LineEnding: zapcore.DefaultLineEnding,
+ EncodeLevel: zapcore.LowercaseLevelEncoder,
+ EncodeTime: zapcore.TimeEncoderOfLayout("2006-01-02 15:04:05.000-07:00"),
+ EncodeDuration: zapcore.SecondsDurationEncoder,
+ EncodeCaller: zapcore.ShortCallerEncoder,
+ EncodeName: zapcore.FullNameEncoder,
+ }
+ maxSize := file.MaxSize
+ if maxSize <= 0 {
+ maxSize = DefaultLogMaxSizeMB
+ }
+ maxBackups := file.MaxBackups
+ if maxBackups <= 0 {
+ maxBackups = DefaultLogMaxBackups
+ }
+ maxAge := file.MaxAge
+ if maxAge <= 0 {
+ maxAge = DefaultLogMaxAgeDays
+ }
+ compress := DefaultLogCompress
+ if file.Compress != nil {
+ compress = *file.Compress
+ }
+ stdoutSyncer := zapcore.AddSync(os.Stdout)
+ syncers := []zapcore.WriteSyncer{stdoutSyncer}
+ // File output is opt-in via a non-empty Path. Path is treated as the log FILE
+ // (matching conf/service_conf.yaml's "path is the file" wording): a stray
+ // directory named after the file is never created. Only when Path is itself an
+ // existing directory do we synthesize a per-service file name inside it.
+ if file.Path != "" {
+ filename := file.Path
+ if fi, statErr := os.Stat(file.Path); statErr == nil && fi.IsDir() {
+ name := file.Filename
+ if name == "" {
+ name = fmt.Sprintf("%s_ingest.log", serviceName)
+ }
+ filename = filepath.Join(file.Path, name)
+ }
+ ljLogger := &lumberjack.Logger{
+ Filename: filename,
+ MaxSize: maxSize,
+ MaxBackups: maxBackups,
+ MaxAge: maxAge,
+ Compress: compress,
+ LocalTime: true,
+ }
+ syncers = append(syncers, zapcore.AddSync(ljLogger))
+ }
+ core := zapcore.NewCore(zapcore.NewJSONEncoder(encoderConfig), zapcore.NewMultiWriteSyncer(syncers...), zapcore.InfoLevel)
+ oldLogger := ingestLogger.Swap(zap.New(core))
+ if oldLogger != nil {
+ _ = oldLogger.Sync()
+ }
return nil
}
type IngestionLog struct {
- EventTime time.Time `json:"event_time"`
- UserID string `json:"user_id"`
- UserName string `json:"user_name"`
- DocumentID string `json:"document_id"`
- Worker string `json:"worker"`
- TaskID string `json:"task_id"`
- TaskType string `json:"task_type"`
- Filename string `json:"filename"`
- Code int `json:"code"`
- Error error `json:"error"`
- DurationMS int64 `json:"duration_ms"`
- Operation map[string]any `json:"operation"`
+ EventTime time.Time `json:"event_time"`
+ TenantID string `json:"tenant_id"`
+ DatasetID string `json:"dataset_id"`
+ DocID string `json:"doc_id"`
+ UserID string `json:"user_id"`
+ UserName string `json:"user_name,omitempty"`
+ Worker string `json:"worker"`
+ TaskID string `json:"task_id"`
+ EventType string `json:"event_type"` // created|scheduled|running|completed|failed|stopped
+ Status string `json:"status"`
+ // ErrorInfo is the terminal reason as a plain string (not an error
+ // interface) so the event serializes cleanly and portably to external
+ // dashboards / ClickHouse.
+ ErrorInfo string `json:"error_info,omitempty"`
+ Filename string `json:"filename"`
+ FileType string `json:"file_type"`
+ FileSize int64 `json:"file_size"`
+ ParserID string `json:"parser_id"`
+ UploadTime time.Time `json:"upload_time"`
+ EnqueueTime time.Time `json:"enqueue_time"`
+ BeginTime time.Time `json:"begin_time"`
+ EndTime time.Time `json:"end_time"`
+ DurationMS int64 `json:"duration_ms"`
+ WaitDurationS float64 `json:"wait_duration_s"`
+ ParseDurationS float64 `json:"parse_duration_s"`
+ Progress float64 `json:"progress"`
+ RunCount int `json:"run_count"`
+ // Code is a machine-readable category for grouping in dashboards. It
+ // defaults to 0 when the caller does not set it.
+ Code int `json:"code"`
+ Operation map[string]any `json:"operation,omitempty"`
+}
+
+// SyncIngestLog flushes any buffered ingest output. Call it before process exit
+// so no events are lost.
+func SyncIngestLog() {
+ _ = ingestLogger.Load().Sync()
+}
+
+// LogIngestEvent transmits one structured ingestion lifecycle event as a single,
+// flat JSON line. In the OSS build it writes to the local zap sink and also
+// forwards to the shared LogIngest seam (a no-op in OSS, the real sink in EE).
+func LogIngestEvent(log IngestionLog) {
+ ingestLogger.Load().With(log.zapFields()...).Info("ingest_log")
+ LogIngest(&log)
+}
+
+// zapFields renders the event as flat zap fields (top-level keys, UTC times) so
+// the emitted JSON line is directly consumable by ClickHouse / Grafana.
+func (l IngestionLog) zapFields() []zap.Field {
+ fields := []zap.Field{
+ zap.Time("event_time", l.EventTime.UTC()),
+ zap.String("tenant_id", l.TenantID),
+ zap.String("dataset_id", l.DatasetID),
+ zap.String("doc_id", l.DocID),
+ zap.String("user_id", l.UserID),
+ zap.String("worker", l.Worker),
+ zap.String("task_id", l.TaskID),
+ zap.String("event_type", l.EventType),
+ zap.String("status", l.Status),
+ zap.String("filename", l.Filename),
+ zap.String("file_type", l.FileType),
+ zap.Int64("file_size", l.FileSize),
+ zap.String("parser_id", l.ParserID),
+ skipZeroTime("upload_time", l.UploadTime),
+ skipZeroTime("enqueue_time", l.EnqueueTime),
+ skipZeroTime("begin_time", l.BeginTime),
+ skipZeroTime("end_time", l.EndTime),
+ zap.Int64("duration_ms", l.DurationMS),
+ zap.Float64("wait_duration_s", l.WaitDurationS),
+ zap.Float64("parse_duration_s", l.ParseDurationS),
+ zap.Float64("progress", l.Progress),
+ zap.Int("run_count", l.RunCount),
+ zap.Int("code", l.Code),
+ }
+ if l.UserName != "" {
+ fields = append(fields, zap.String("user_name", l.UserName))
+ }
+ if l.ErrorInfo != "" {
+ fields = append(fields, zap.String("error_info", l.ErrorInfo))
+ }
+ if len(l.Operation) > 0 {
+ fields = append(fields, zap.Any("operation", l.Operation))
+ }
+ return fields
+}
+
+// skipZeroTime omits a time field from the emitted JSON when it is the zero
+// value, keeping the line compact and ClickHouse-friendly.
+func skipZeroTime(key string, t time.Time) zap.Field {
+ if t.IsZero() {
+ return zap.Skip()
+ }
+ return zap.Time(key, t.UTC())
}
+// LogIngest is the shared upstream seam. In the OSS build it is a no-op; the EE
+// build provides the real sink. Our OSS LogIngestEvent forwards to it.
func LogIngest(log *IngestionLog, fields ...zap.Field) {}
diff --git a/internal/entity/pipeline.go b/internal/entity/pipeline.go
index 923a8835ed..f6f29e9eac 100644
--- a/internal/entity/pipeline.go
+++ b/internal/entity/pipeline.go
@@ -28,19 +28,23 @@ type PipelineOperationLog struct {
DocumentID string `gorm:"column:document_id;size:32;index;uniqueIndex:idx_pipeline_operation_log_document_run,priority:1" json:"document_id"`
// RunCount is the monotonically allocated display number for a numbered
// ingestion run.
- RunCount *int `gorm:"column:run_count;uniqueIndex:idx_pipeline_operation_log_document_run,priority:2" json:"run_count,omitempty"`
- TenantID string `gorm:"column:tenant_id;size:32;not null;index" json:"tenant_id"`
- KbID string `gorm:"column:kb_id;size:32;not null;index" json:"kb_id"`
- PipelineID *string `gorm:"column:pipeline_id;size:32;index" json:"pipeline_id,omitempty"`
- PipelineTitle *string `gorm:"column:pipeline_title;size:32;index" json:"pipeline_title,omitempty"`
- ParserID string `gorm:"column:parser_id;size:32;not null;index" json:"parser_id"`
- DocumentName string `gorm:"column:document_name;size:255;not null" json:"document_name"`
- DocumentSuffix string `gorm:"column:document_suffix;size:255;not null" json:"document_suffix"`
- DocumentType string `gorm:"column:document_type;size:255;not null" json:"document_type"`
- SourceFrom string `gorm:"column:source_from;size:255;not null" json:"source_from"`
- Progress float64 `gorm:"column:progress;default:0;index" json:"progress"`
- ProgressMsg *string `gorm:"column:progress_msg;type:longtext" json:"progress_msg,omitempty"`
- ProcessBeginAt *time.Time `gorm:"column:process_begin_at;index" json:"process_begin_at,omitempty"`
+ RunCount *int `gorm:"column:run_count;uniqueIndex:idx_pipeline_operation_log_document_run,priority:2" json:"run_count,omitempty"`
+ TenantID string `gorm:"column:tenant_id;size:32;not null;index" json:"tenant_id"`
+ KbID string `gorm:"column:kb_id;size:32;not null;index" json:"kb_id"`
+ PipelineID *string `gorm:"column:pipeline_id;size:32;index" json:"pipeline_id,omitempty"`
+ PipelineTitle *string `gorm:"column:pipeline_title;size:32;index" json:"pipeline_title,omitempty"`
+ ParserID string `gorm:"column:parser_id;size:32;not null;index" json:"parser_id"`
+ DocumentName string `gorm:"column:document_name;size:255;not null" json:"document_name"`
+ DocumentSuffix string `gorm:"column:document_suffix;size:255;not null" json:"document_suffix"`
+ DocumentType string `gorm:"column:document_type;size:255;not null" json:"document_type"`
+ SourceFrom string `gorm:"column:source_from;size:255;not null" json:"source_from"`
+ Progress float64 `gorm:"column:progress;default:0;index" json:"progress"`
+ ProgressMsg *string `gorm:"column:progress_msg;type:longtext" json:"progress_msg,omitempty"`
+ ProcessBeginAt *time.Time `gorm:"column:process_begin_at;index" json:"process_begin_at,omitempty"`
+ // ProcessEndAt is set when the run reaches a terminal state (completed /
+ // failed / stopped). It is the authoritative end time used to compute parse
+ // duration for ingestion observability events.
+ ProcessEndAt *time.Time `gorm:"column:process_end_at;index" json:"process_end_at,omitempty"`
ProcessDuration float64 `gorm:"column:process_duration;default:0" json:"process_duration"`
DSL JSONMap `gorm:"column:dsl;type:longtext" json:"dsl,omitempty"`
TaskType string `gorm:"column:task_type;size:32;not null;default:''" json:"task_type"`
diff --git a/internal/ingestion/service/execute_task_ack_test.go b/internal/ingestion/service/execute_task_ack_test.go
index 9e0372908d..cf2ea5b10d 100644
--- a/internal/ingestion/service/execute_task_ack_test.go
+++ b/internal/ingestion/service/execute_task_ack_test.go
@@ -452,7 +452,7 @@ func TestSettleMessage_DBTruthOverridesBodyReturn(t *testing.T) {
// recovery where markFailed succeeded: the task is terminal (FAILED)
// but the body signals non-terminal.
body := func(ctx context.Context) bool {
- ingestor.markFailed(ctx, taskID)
+ ingestor.markFailed(ctx, taskID, "")
return false
}
diff --git a/internal/ingestion/service/ingestion_service.go b/internal/ingestion/service/ingestion_service.go
index fef61f75f6..339300d218 100644
--- a/internal/ingestion/service/ingestion_service.go
+++ b/internal/ingestion/service/ingestion_service.go
@@ -547,7 +547,7 @@ func (e *Ingestor) handleAndExecute(handle common.TaskHandle) {
var identityErr *servicepkg.InvalidRunIdentityError
if errors.As(validateErr, &identityErr) {
common.Error(fmt.Sprintf("task %s has permanent run identity error: %s", task.ID, identityErr.Reason), validateErr)
- if e.markFailed(e.ctx, task.ID) {
+ if e.markFailed(e.ctx, task.ID, fmt.Sprintf("Task rejected: invalid run identity (%s).", identityErr.Reason)) {
e.recordTerminalPipelineLog(e.ctx, task, string(entity.TaskStatusFail), fmt.Sprintf("Task rejected: invalid run identity (%s).", identityErr.Reason))
e.ackHandle(hb, handle, taskMessage.TaskID)
} else {
@@ -820,10 +820,10 @@ func (e *Ingestor) markStopped(ctx context.Context, taskID string) bool {
// The caller's context is detached before touching the DB (see markStopped):
// failure paths usually carry an already-cancelled context, and losing the
// FAILED write would turn a settled task into a redelivery loop.
-func (e *Ingestor) markFailed(ctx context.Context, taskID string) bool {
+func (e *Ingestor) markFailed(ctx context.Context, taskID, reason string) bool {
ctx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 5*time.Second)
defer cancel()
- if uErr := e.ingestionTaskSvc.MarkFailed(ctx, taskID); uErr != nil {
+ if uErr := e.ingestionTaskSvc.MarkFailed(ctx, taskID, reason); uErr != nil {
common.Error(fmt.Sprintf("Failed to set task %s to FAILED", taskID), uErr)
return false
}
@@ -882,14 +882,14 @@ func (e *Ingestor) runTask(ctx context.Context, task *entity.IngestionTask) bool
if errors.Is(err, context.DeadlineExceeded) {
common.Info(fmt.Sprintf("Task %s timed out during pipeline", task.ID))
e.markTerminalProgress(task)
- ok := e.markFailed(ctx, task.ID)
+ ok := e.markFailed(ctx, task.ID, "Task timed out.")
if ok {
e.recordTerminalPipelineLog(ctx, task, string(entity.TaskStatusFail), "Task timed out.")
}
return ok
}
common.Error(fmt.Sprintf("Task %s failed", task.ID), err)
- ok := e.markFailed(ctx, task.ID)
+ ok := e.markFailed(ctx, task.ID, fmt.Sprintf("Task failed: %v", err))
if ok {
e.recordTerminalPipelineLog(ctx, task, string(entity.TaskStatusFail), fmt.Sprintf("Task failed: %v", err))
}
@@ -1019,7 +1019,7 @@ func (e *Ingestor) settleMessage(ctx context.Context, taskCtx *taskpkg.TaskConte
}
}()
common.Error(fmt.Sprintf("task %s panicked: %v", taskID, r), fmt.Errorf("%v", r))
- e.markFailed(ctx, taskID)
+ e.markFailed(ctx, taskID, fmt.Sprintf("Task panicked: %v", r))
e.recordTerminalPipelineLog(ctx, task, string(entity.TaskStatusFail), fmt.Sprintf("Task panicked: %v", r))
}()
terminal = false
diff --git a/internal/ingestion/service/nil_database_guard_test.go b/internal/ingestion/service/nil_database_guard_test.go
index 50d6104e0f..08a053e493 100644
--- a/internal/ingestion/service/nil_database_guard_test.go
+++ b/internal/ingestion/service/nil_database_guard_test.go
@@ -35,7 +35,7 @@ func TestMarkFailedToleratesNilDatabase(t *testing.T) {
defer testutil.ReplaceDBForTest(t, nil)()
ingestor := newUnitIngestor("test", 1, []string{"pdf"})
- if terminal := ingestor.markFailed(context.Background(), "task-1"); terminal {
+ if terminal := ingestor.markFailed(context.Background(), "task-1", ""); terminal {
t.Fatal("markFailed reported a durable FAILED write without a database")
}
}
diff --git a/internal/observability/ingestion_log.go b/internal/observability/ingestion_log.go
new file mode 100644
index 0000000000..447bf9a56e
--- /dev/null
+++ b/internal/observability/ingestion_log.go
@@ -0,0 +1,140 @@
+// Package observability builds structured lifecycle events for ingestion
+// (document-parse) tasks and forwards them to the common ingest-log sink
+// (internal/common.LogIngestEvent). The default emitter routes straight to that
+// sink; tests inject a recording emitter.
+package observability
+
+import (
+ "context"
+ "time"
+
+ "ragflow/internal/common"
+ "ragflow/internal/entity"
+)
+
+// IngestionEventEmitter receives lifecycle events. Implementations must be
+// non-blocking and must never return an error that could fail the parse; the
+// caller ignores errors by design.
+type IngestionEventEmitter interface {
+ EmitIngestLog(ctx context.Context, log common.IngestionLog)
+}
+
+// CommonLogEmitter is the default OSS emitter. It forwards the event to
+// common.LogIngestEvent, which writes one JSON line to the configured sink.
+type CommonLogEmitter struct{}
+
+// EmitIngestLog forwards the event to the common ingest-log sink.
+func (e *CommonLogEmitter) EmitIngestLog(_ context.Context, log common.IngestionLog) {
+ common.LogIngestEvent(log)
+}
+
+// ComputeDurations derives wait (begin-enqueue) and parse (end-begin) seconds.
+// Durations are clamped to >= 0 so clock skew never produces negative values
+// downstream.
+func ComputeDurations(enqueue, begin, end time.Time) (float64, float64) {
+ wait := 0.0
+ if begin.After(enqueue) {
+ wait = begin.Sub(enqueue).Seconds()
+ }
+ parse := 0.0
+ if end.After(begin) {
+ parse = end.Sub(begin).Seconds()
+ }
+ return wait, parse
+}
+
+// isBeginRelevant reports whether the worker-claim time should be attached to
+// this event. created/scheduled events happen before any worker has claimed the
+// document, so they must not carry a (possibly stale, from a prior run) claim
+// time.
+func isBeginRelevant(eventType string) bool {
+ switch eventType {
+ case "running", "completed", "failed", "stopped":
+ return true
+ default:
+ return false
+ }
+}
+
+// claimTime returns the worker-claim time. It prefers the document's own
+// ProcessBeginAt (set when the worker actually starts, in PrepareValidatedRun).
+//
+// doc.ProcessBeginAt is never cleared between runs, so on a run that never claimed
+// the document (RequestStop/supersede on a queued task, or a publish failure) or a
+// retry that reuses a stale document value, it can hold a PRIOR run's instant.
+// We therefore only trust it when it falls at or after this run's open time
+// (run.CreateDate); otherwise the run never started and must not report a
+// claim/parse time. The run row's ProcessBeginAt is the open time (set in
+// CreateOpenLog), not a claim time, so it is intentionally not used here.
+func claimTime(doc *entity.Document, run *entity.PipelineOperationLog) *time.Time {
+ if doc != nil && doc.ProcessBeginAt != nil && run != nil && run.CreateDate != nil {
+ if !doc.ProcessBeginAt.Before(*run.CreateDate) {
+ return doc.ProcessBeginAt
+ }
+ return nil
+ }
+ // No run to validate against: fall back to the document value as-is.
+ if doc != nil && doc.ProcessBeginAt != nil {
+ return doc.ProcessBeginAt
+ }
+ return nil
+}
+
+// BuildIngestLog assembles a common.IngestionLog from the task, its document, and
+// the bound pipeline run. eventType is one of
+// created|scheduled|running|completed|failed|stopped.
+func BuildIngestLog(task *entity.IngestionTask, doc *entity.Document, run *entity.PipelineOperationLog, eventType, status, errorInfo string) common.IngestionLog {
+ ev := common.IngestionLog{
+ EventTime: time.Now().UTC(),
+ TaskID: task.ID,
+ DocID: task.DocumentID,
+ DatasetID: task.DatasetID,
+ UserID: task.UserID,
+ EventType: eventType,
+ Status: status,
+ ErrorInfo: errorInfo,
+ }
+ if doc != nil {
+ name := ""
+ if doc.Name != nil {
+ name = *doc.Name
+ }
+ ev.Filename = name + doc.Suffix
+ ev.FileType = doc.Type
+ ev.FileSize = doc.Size
+ ev.ParserID = doc.ParserID
+ if doc.CreateDate != nil {
+ ev.UploadTime = doc.CreateDate.UTC()
+ }
+ }
+ if run != nil {
+ if run.RunCount != nil {
+ ev.RunCount = *run.RunCount
+ }
+ if run.TenantID != "" {
+ ev.TenantID = run.TenantID
+ }
+ ev.Progress = run.Progress
+ if run.CreateDate != nil {
+ ev.EnqueueTime = run.CreateDate.UTC()
+ }
+ if run.ProcessEndAt != nil {
+ ev.EndTime = run.ProcessEndAt.UTC()
+ }
+ // BeginTime only for running/terminal events; created/scheduled must not
+ // leak a (possibly stale) claim time from a previous run.
+ if ct := claimTime(doc, run); ct != nil && isBeginRelevant(eventType) {
+ ev.BeginTime = ct.UTC()
+ }
+ // ProcessEndAt is not always persisted on the run; for a terminal event
+ // whose run actually started, fall back to the transition time so parse
+ // duration is still reported (a still-queued stop gets no duration).
+ if ev.EndTime.IsZero() && !ev.BeginTime.IsZero() &&
+ (eventType == "completed" || eventType == "failed" || eventType == "stopped") {
+ ev.EndTime = ev.EventTime
+ }
+ }
+ ev.WaitDurationS, ev.ParseDurationS = ComputeDurations(ev.EnqueueTime, ev.BeginTime, ev.EndTime)
+ ev.DurationMS = int64(ev.ParseDurationS * 1000)
+ return ev
+}
diff --git a/internal/observability/ingestion_log_fix_test.go b/internal/observability/ingestion_log_fix_test.go
new file mode 100644
index 0000000000..16610be8a0
--- /dev/null
+++ b/internal/observability/ingestion_log_fix_test.go
@@ -0,0 +1,48 @@
+package observability
+
+import (
+ "testing"
+ "time"
+
+ "ragflow/internal/entity"
+)
+
+func TestBuildIngestLogPrefersWorkerClaimTime(t *testing.T) {
+ base := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC)
+ enqueue := base
+ openTime := base.Add(1 * time.Minute)
+ claimTime := base.Add(10 * time.Minute)
+ end := base.Add(20 * time.Minute)
+ task := &entity.IngestionTask{ID: "t", DocumentID: "d", DatasetID: "kb", UserID: "u"}
+ doc := &entity.Document{Name: ptrStr("r"), Suffix: ".pdf", Type: "pdf", Size: 10, ParserID: "pdf", ProcessBeginAt: ptrTime(claimTime)}
+ run := &entity.PipelineOperationLog{RunCount: ptrInt(1), CreateDate: ptrTime(enqueue), ProcessBeginAt: ptrTime(openTime), ProcessEndAt: ptrTime(end)}
+
+ log := BuildIngestLog(task, doc, run, "completed", "COMPLETED", "")
+ if !log.BeginTime.Equal(claimTime) {
+ t.Errorf("BeginTime = %v, want worker-claim time %v", log.BeginTime, claimTime)
+ }
+ if log.WaitDurationS <= 120 {
+ t.Errorf("WaitDurationS = %v, want ~600 (enqueue -> worker claim), got near-zero", log.WaitDurationS)
+ }
+}
+
+func TestBuildIngestLogPopulatesDurationMS(t *testing.T) {
+ base := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC)
+ task := &entity.IngestionTask{ID: "t", DocumentID: "d", DatasetID: "kb", UserID: "u"}
+ doc := &entity.Document{Name: ptrStr("r"), Suffix: ".pdf", Type: "pdf", Size: 10, ParserID: "pdf", ProcessBeginAt: ptrTime(base.Add(10 * time.Minute))}
+ run := &entity.PipelineOperationLog{RunCount: ptrInt(1), CreateDate: ptrTime(base), ProcessBeginAt: ptrTime(base.Add(1 * time.Minute)), ProcessEndAt: ptrTime(base.Add(20 * time.Minute))}
+
+ log := BuildIngestLog(task, doc, run, "completed", "COMPLETED", "")
+ want := int64(log.ParseDurationS * 1000)
+ if want <= 0 {
+ t.Fatalf("ParseDurationS = %v, want > 0 for setup", log.ParseDurationS)
+ }
+ if log.DurationMS != want {
+ t.Errorf("DurationMS = %d, want %d", log.DurationMS, want)
+ }
+
+ created := BuildIngestLog(task, doc, &entity.PipelineOperationLog{RunCount: ptrInt(1)}, "created", "CREATED", "")
+ if created.DurationMS != 0 {
+ t.Errorf("created DurationMS = %d, want 0", created.DurationMS)
+ }
+}
diff --git a/internal/observability/ingestion_log_tdd_test.go b/internal/observability/ingestion_log_tdd_test.go
new file mode 100644
index 0000000000..bc6ce39045
--- /dev/null
+++ b/internal/observability/ingestion_log_tdd_test.go
@@ -0,0 +1,105 @@
+//go:build !ee
+
+package observability
+
+import (
+ "testing"
+ "time"
+
+ "ragflow/internal/entity"
+)
+
+// TestClaimTimeRejectsStaleDocumentValue verifies that a document claim time left
+// over from a PREVIOUS run (i.e. before this run even opened) is NOT treated as
+// this run's claim time. This is the root cause of P1a: doc.ProcessBeginAt is
+// stamped by PrepareValidatedRun and never cleared between runs, so a task that
+// never reaches RUNNING in the current run must not inherit a stale value.
+func TestClaimTimeRejectsStaleDocumentValue(t *testing.T) {
+ base := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC)
+ staleClaim := base.Add(-48 * time.Hour) // two days before this run opened
+ runOpen := base.Add(-1 * time.Hour)
+ doc := &entity.Document{ProcessBeginAt: ptrTime(staleClaim)}
+ run := &entity.PipelineOperationLog{CreateDate: ptrTime(runOpen)}
+
+ if got := claimTime(doc, run); got != nil {
+ t.Fatalf("claimTime returned %v for a stale document value; want nil", got)
+ }
+}
+
+// TestClaimTimeAcceptsValidClaimTime verifies a claim time that falls within this
+// run's window is still honored (the happy path must keep working).
+func TestClaimTimeAcceptsValidClaimTime(t *testing.T) {
+ base := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC)
+ runOpen := base
+ claim := base.Add(10 * time.Minute)
+ doc := &entity.Document{ProcessBeginAt: ptrTime(claim)}
+ run := &entity.PipelineOperationLog{CreateDate: ptrTime(runOpen)}
+
+ got := claimTime(doc, run)
+ if got == nil || !got.Equal(claim) {
+ t.Fatalf("claimTime = %v, want %v", got, claim)
+ }
+}
+
+// TestBuildIngestLogPreClaimStoppedOmitsStaleClaimTime is the direct regression
+// for P1a: a task stopped (by user / superseded) before any worker claimed it
+// must not emit a BeginTime inherited from a prior run, and must not report a
+// fabricated parse_duration_s of hours/days.
+func TestBuildIngestLogPreClaimStoppedOmitsStaleClaimTime(t *testing.T) {
+ base := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC)
+ staleClaim := base.Add(-48 * time.Hour)
+ runOpen := base.Add(-1 * time.Hour)
+ task := &entity.IngestionTask{ID: "t", DocumentID: "d", DatasetID: "kb", UserID: "u"}
+ doc := &entity.Document{Name: ptrStr("r"), Suffix: ".pdf", Type: "pdf", Size: 10, ParserID: "pdf", ProcessBeginAt: ptrTime(staleClaim)}
+ run := &entity.PipelineOperationLog{RunCount: ptrInt(1), CreateDate: ptrTime(runOpen)}
+
+ ev := BuildIngestLog(task, doc, run, "stopped", "STOPPED", "stopped by user")
+ if !ev.BeginTime.IsZero() {
+ t.Errorf("pre-claim stopped must omit stale BeginTime, got %v", ev.BeginTime)
+ }
+ if ev.ParseDurationS != 0 {
+ t.Errorf("pre-claim stopped parse_duration_s must be 0, got %v", ev.ParseDurationS)
+ }
+ if ev.WaitDurationS != 0 {
+ t.Errorf("pre-claim stopped wait_duration_s must be 0, got %v", ev.WaitDurationS)
+ }
+}
+
+// TestBuildIngestLogPreClaimFailedOmitsStaleClaimTime covers the settlePublishFailure
+// path: a run that fails before a worker claims it must not report a stale claim time.
+func TestBuildIngestLogPreClaimFailedOmitsStaleClaimTime(t *testing.T) {
+ base := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC)
+ staleClaim := base.Add(-48 * time.Hour)
+ runOpen := base.Add(-1 * time.Hour)
+ task := &entity.IngestionTask{ID: "t", DocumentID: "d", DatasetID: "kb", UserID: "u"}
+ doc := &entity.Document{Name: ptrStr("r"), Suffix: ".pdf", Type: "pdf", Size: 10, ParserID: "pdf", ProcessBeginAt: ptrTime(staleClaim)}
+ run := &entity.PipelineOperationLog{RunCount: ptrInt(1), CreateDate: ptrTime(runOpen)}
+
+ ev := BuildIngestLog(task, doc, run, "failed", "FAILED", "publish failed")
+ if !ev.BeginTime.IsZero() {
+ t.Errorf("pre-claim failed must omit stale BeginTime, got %v", ev.BeginTime)
+ }
+ if ev.ParseDurationS != 0 {
+ t.Errorf("pre-claim failed parse_duration_s must be 0, got %v", ev.ParseDurationS)
+ }
+}
+
+// TestBuildIngestLogRunningKeepsValidClaimTime guards the happy path: a task that
+// actually started keeps its real claim time and a sane parse/wait duration.
+func TestBuildIngestLogRunningKeepsValidClaimTime(t *testing.T) {
+ base := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC)
+ runOpen := base
+ claim := base.Add(10 * time.Minute)
+ end := base.Add(20 * time.Minute)
+ task := &entity.IngestionTask{ID: "t", DocumentID: "d", DatasetID: "kb", UserID: "u"}
+ doc := &entity.Document{Name: ptrStr("r"), Suffix: ".pdf", Type: "pdf", Size: 10, ParserID: "pdf", ProcessBeginAt: ptrTime(claim)}
+ run := &entity.PipelineOperationLog{RunCount: ptrInt(1), CreateDate: ptrTime(runOpen), ProcessEndAt: ptrTime(end)}
+
+ ev := BuildIngestLog(task, doc, run, "running", "RUNNING", "")
+ if !ev.BeginTime.Equal(claim) {
+ t.Errorf("running BeginTime = %v, want %v", ev.BeginTime, claim)
+ }
+ if ev.ParseDurationS <= 0 {
+ t.Errorf("running parse_duration_s = %v, want > 0", ev.ParseDurationS)
+ }
+}
diff --git a/internal/observability/ingestion_log_test.go b/internal/observability/ingestion_log_test.go
new file mode 100644
index 0000000000..1b6d605bd4
--- /dev/null
+++ b/internal/observability/ingestion_log_test.go
@@ -0,0 +1,150 @@
+package observability
+
+import (
+ "testing"
+ "time"
+
+ "ragflow/internal/entity"
+)
+
+func TestComputeDurationsHappy(t *testing.T) {
+ base := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC)
+ enqueue := base
+ begin := base.Add(5 * time.Minute)
+ end := base.Add(15 * time.Minute)
+ wait, parse := ComputeDurations(enqueue, begin, end)
+ if wait != 300 {
+ t.Errorf("wait_duration_s = %v, want 300", wait)
+ }
+ if parse != 600 {
+ t.Errorf("parse_duration_s = %v, want 600", parse)
+ }
+}
+
+func TestComputeDurationsClampsSkew(t *testing.T) {
+ base := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC)
+ enqueue := base.Add(10 * time.Minute)
+ begin := base
+ end := base.Add(20 * time.Minute)
+ wait, parse := ComputeDurations(enqueue, begin, end)
+ if wait < 0 {
+ t.Errorf("wait_duration_s = %v, want >= 0 (clock skew must clamp)", wait)
+ }
+ if parse != 1200 {
+ t.Errorf("parse_duration_s = %v, want 1200", parse)
+ }
+}
+
+func TestComputeDurationsClampsEndBeforeBegin(t *testing.T) {
+ base := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC)
+ enqueue := base
+ begin := base.Add(10 * time.Minute)
+ end := base.Add(5 * time.Minute)
+ wait, parse := ComputeDurations(enqueue, begin, end)
+ if wait != 600 {
+ t.Errorf("wait_duration_s = %v, want 600", wait)
+ }
+ if parse < 0 {
+ t.Errorf("parse_duration_s = %v, want >= 0", parse)
+ }
+}
+
+func ptrStr(s string) *string { return &s }
+func ptrTime(t time.Time) *time.Time { return &t }
+func ptrInt(i int) *int { return &i }
+
+func TestBuildIngestLogHappy(t *testing.T) {
+ now := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC)
+ task := &entity.IngestionTask{
+ ID: "task-1",
+ DocumentID: "doc-1",
+ DatasetID: "kb-1",
+ UserID: "user-1",
+ }
+ doc := &entity.Document{
+ Name: ptrStr("report"),
+ Suffix: ".pdf",
+ Type: "pdf",
+ Size: 12345,
+ ParserID: "pdf",
+ CreateDate: ptrTime(now.Add(-time.Hour)),
+ ProcessBeginAt: ptrTime(now.Add(2 * time.Minute)),
+ }
+ run := &entity.PipelineOperationLog{
+ RunCount: ptrInt(1),
+ TenantID: "tenant-1",
+ Progress: 1,
+ CreateDate: ptrTime(now),
+ ProcessEndAt: ptrTime(now.Add(12 * time.Minute)),
+ }
+ log := BuildIngestLog(task, doc, run, "completed", "COMPLETED", "")
+
+ if log.TaskID != "task-1" {
+ t.Errorf("TaskID = %q, want task-1", log.TaskID)
+ }
+ if log.DocID != "doc-1" {
+ t.Errorf("DocID = %q, want doc-1", log.DocID)
+ }
+ if log.DatasetID != "kb-1" {
+ t.Errorf("DatasetID = %q, want kb-1", log.DatasetID)
+ }
+ if log.Filename != "report.pdf" {
+ t.Errorf("Filename = %q, want report.pdf", log.Filename)
+ }
+ if log.FileType != "pdf" {
+ t.Errorf("FileType = %q, want pdf", log.FileType)
+ }
+ if log.FileSize != 12345 {
+ t.Errorf("FileSize = %d, want 12345", log.FileSize)
+ }
+ if log.ParserID != "pdf" {
+ t.Errorf("ParserID = %q, want pdf", log.ParserID)
+ }
+ if log.EventType != "completed" {
+ t.Errorf("EventType = %q, want completed", log.EventType)
+ }
+ if log.Status != "COMPLETED" {
+ t.Errorf("Status = %q, want COMPLETED", log.Status)
+ }
+ if log.RunCount != 1 {
+ t.Errorf("RunCount = %d, want 1", log.RunCount)
+ }
+ if log.TenantID != "tenant-1" {
+ t.Errorf("TenantID = %q, want tenant-1", log.TenantID)
+ }
+ if log.WaitDurationS <= 0 {
+ t.Errorf("WaitDurationS = %v, want > 0 (claim after enqueue)", log.WaitDurationS)
+ }
+ if log.ParseDurationS <= 0 {
+ t.Errorf("ParseDurationS = %v, want > 0", log.ParseDurationS)
+ }
+}
+
+func TestBuildIngestLogCreatedSkipsClaimTime(t *testing.T) {
+ now := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC)
+ task := &entity.IngestionTask{ID: "task-1", DocumentID: "doc-1", DatasetID: "kb-1", UserID: "user-1"}
+ doc := &entity.Document{
+ Name: ptrStr("report"),
+ Suffix: ".pdf",
+ Type: "pdf",
+ Size: 10,
+ ParserID: "pdf",
+ ProcessBeginAt: ptrTime(now.Add(2 * time.Minute)),
+ }
+ run := &entity.PipelineOperationLog{RunCount: ptrInt(1), CreateDate: ptrTime(now)}
+ log := BuildIngestLog(task, doc, run, "created", "CREATED", "")
+ if !log.BeginTime.IsZero() {
+ t.Errorf("created event BeginTime = %v, want zero", log.BeginTime)
+ }
+}
+
+func TestBuildIngestLogFailedCarriesErrorInfo(t *testing.T) {
+ now := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC)
+ task := &entity.IngestionTask{ID: "task-1", DocumentID: "doc-1", DatasetID: "kb-1", UserID: "user-1"}
+ doc := &entity.Document{Name: ptrStr("docname"), Suffix: ".pdf", Type: "pdf", Size: 10, ParserID: "pdf"}
+ run := &entity.PipelineOperationLog{RunCount: ptrInt(1), CreateDate: ptrTime(now)}
+ log := BuildIngestLog(task, doc, run, "failed", "FAILED", "boom")
+ if log.ErrorInfo != "boom" {
+ t.Errorf("ErrorInfo = %q, want boom", log.ErrorInfo)
+ }
+}
diff --git a/internal/server/api/server.go b/internal/server/api/server.go
index 7c5fa85308..cbdf2fedf9 100644
--- a/internal/server/api/server.go
+++ b/internal/server/api/server.go
@@ -458,6 +458,11 @@ func startServer(ctx context.Context, serverName string, arguments *bootstrap.Se
// Set log level
bootstrap.SetLogger(serverName, arguments)
+ // Initialize the ingestion lifecycle event sink (best-effort; stdout is
+ // always on, ingestor.log.path adds an optional rotated file). Flush it on shutdown.
+ bootstrap.SetIngestLog(serverName)
+ defer common.SyncIngestLog()
+
// Start server in a goroutine
go func() {
serve(serverName, srv, apiListener)
diff --git a/internal/server/bootstrap/runtime.go b/internal/server/bootstrap/runtime.go
index f3028e2537..a522f87ff9 100644
--- a/internal/server/bootstrap/runtime.go
+++ b/internal/server/bootstrap/runtime.go
@@ -60,6 +60,22 @@ func SetLogger(serverName string, arguments *ServerArgs) {
}
}
+// SetIngestLog (re)initializes the ingestion lifecycle event sink for the given
+// server name from the configured ingest_log settings. It mirrors SetLogger so
+// every server mode that can emit (the API server creating/enqueuing tasks, the
+// ingestor running them, etc.) shares one sink. Emission is best-effort: a sink
+// init failure is logged but never fatal. The sink always writes to stdout; an
+// ingestor.log.path only adds an optional rotated file on top of it.
+func SetIngestLog(serverName string) {
+ globalConfig := server.GetConfig()
+ ingestLogConfig := globalConfig.GetIngestLogConfig()
+ // Flush any prior sink before swapping it out (e.g. on a reconfigure).
+ common.SyncIngestLog()
+ if err := common.InitIngestLog(ingestLogConfig.Type, ingestLogConfig.GetFileOutput(), serverName); err != nil {
+ common.Error("Failed to initialize ingest log sink", err)
+ }
+}
+
// StartHeartbeat initializes and starts the heartbeat reporter to the admin
// server. It is shared by the API, ingestor, and syncer server modes. The
// caller must defer the returned *utility.ScheduledTask's Stop() method.
diff --git a/internal/server/config/stub.go b/internal/server/config/stub.go
index 09f76af653..44f621407e 100644
--- a/internal/server/config/stub.go
+++ b/internal/server/config/stub.go
@@ -107,8 +107,57 @@ func (c *AuditConfig) GetFileOutput() common.FileOutput {
return common.FileOutput{}
}
-func (c *IngestLogConfig) ParseIngestLogConfig(v *viper.Viper) error { return nil }
+// Default ingest-log rotation values mirror the documented IngestLogConfig
+// defaults. File output is opt-in: leave ingestor.log.path empty and events emit
+// to stdout only.
+const (
+ defaultIngestLogMaxSizeMB = 1000
+ defaultIngestLogMaxBackups = 20
+ defaultIngestLogMaxAgeDays = 60
+)
+
+// ParseIngestLogConfig reads the ingestor.log.* keys from the sub-viper passed by
+// the ingestor config parser (already merged with the environment by
+// server.Init), so env overrides such as RAGFLOW_INGESTOR_LOG_PATH win over the
+// YAML file. A nil viper is a no-op so the struct keeps its zero (off) state.
+func (c *IngestLogConfig) ParseIngestLogConfig(v *viper.Viper) error {
+ if v == nil {
+ return nil
+ }
+ c.Type = "file"
+ c.Format = "json"
+ c.MaxSize = defaultIngestLogMaxSizeMB
+ c.MaxBackups = defaultIngestLogMaxBackups
+ c.MaxAge = defaultIngestLogMaxAgeDays
+ if v.IsSet("type") {
+ c.Type = v.GetString("type")
+ }
+ if v.IsSet("format") {
+ c.Format = v.GetString("format")
+ }
+ if v.IsSet("path") {
+ c.Path = v.GetString("path")
+ }
+ if v.IsSet("max_size") {
+ c.MaxSize = v.GetInt("max_size")
+ }
+ if v.IsSet("max_backups") {
+ c.MaxBackups = v.GetInt("max_backups")
+ }
+ if v.IsSet("max_age") {
+ c.MaxAge = v.GetInt("max_age")
+ }
+ return nil
+}
+// GetFileOutput projects the ingest log config onto the common.FileOutput sink
+// descriptor. An empty Path means no file sink (stdout only), which keeps the
+// feature off by default.
func (c *IngestLogConfig) GetFileOutput() common.FileOutput {
- return common.FileOutput{}
+ return common.FileOutput{
+ Path: c.Path,
+ MaxSize: c.MaxSize,
+ MaxBackups: c.MaxBackups,
+ MaxAge: c.MaxAge,
+ }
}
diff --git a/internal/server/ingestor/server.go b/internal/server/ingestor/server.go
index 3bd9d5f178..09ad677b11 100644
--- a/internal/server/ingestor/server.go
+++ b/internal/server/ingestor/server.go
@@ -64,15 +64,14 @@ func logTokenizerCounters() {
// admin, then gracefully stops the ingestor.
func Run(ctx context.Context, cancel context.CancelFunc, serverName string, args *bootstrap.ServerArgs) error {
globalConfig := server.GetConfig()
- ingestLogConfig := globalConfig.GetIngestLogConfig()
- err := common.InitIngestLog(ingestLogConfig.Type, ingestLogConfig.GetFileOutput(), serverName)
- if err != nil {
- common.Fatal("Failed to initialize ingest logger", zap.Error(err))
- }
+ // Initialize the ingestion lifecycle event sink (best-effort; stdout is
+ // always on, ingestor.log.path adds an optional rotated file). Flush it on shutdown.
+ bootstrap.SetIngestLog(serverName)
+ defer common.SyncIngestLog()
// Initialize tokenizer (rag_analyzer)
// tokenizer.Init handles DictPath fallback: env var → /usr/share/infinity/resource
- err = tokenizer.Init(&tokenizer.PoolConfig{})
+ err := tokenizer.Init(&tokenizer.PoolConfig{})
if err != nil {
common.Fatal("Failed to initialize tokenizer", zap.Error(err))
}
diff --git a/internal/service/ingestion_task_event_test.go b/internal/service/ingestion_task_event_test.go
new file mode 100644
index 0000000000..facdff867c
--- /dev/null
+++ b/internal/service/ingestion_task_event_test.go
@@ -0,0 +1,88 @@
+package service
+
+import (
+ "context"
+ "sync"
+ "testing"
+
+ "ragflow/internal/common"
+ "ragflow/internal/entity"
+)
+
+// recordingEmitter captures emitted ingest logs for assertions.
+type recordingEmitter struct {
+ mu sync.Mutex
+ events []common.IngestionLog
+}
+
+func (r *recordingEmitter) EmitIngestLog(_ context.Context, log common.IngestionLog) {
+ r.mu.Lock()
+ defer r.mu.Unlock()
+ r.events = append(r.events, log)
+}
+
+func (r *recordingEmitter) last() common.IngestionLog {
+ r.mu.Lock()
+ defer r.mu.Unlock()
+ if len(r.events) == 0 {
+ return common.IngestionLog{}
+ }
+ return r.events[len(r.events)-1]
+}
+
+// TestEmitLifecycleEventForwardsToEmitter verifies the service builds an
+// observability event from the task and forwards it to the configured emitter.
+func TestEmitLifecycleEventForwardsToEmitter(t *testing.T) {
+ rec := &recordingEmitter{}
+ svc := &IngestionTaskService{eventEmitter: rec}
+
+ task := &entity.IngestionTask{
+ ID: "task-1",
+ DocumentID: "doc-1",
+ DatasetID: "kb-1",
+ UserID: "u-1",
+ }
+ svc.emitLifecycleEvent(context.Background(), task, "completed", "COMPLETED", "boom")
+
+ got := rec.last()
+ if got.EventType != "completed" {
+ t.Errorf("EventType = %q, want completed", got.EventType)
+ }
+ if got.Status != "COMPLETED" {
+ t.Errorf("Status = %q, want COMPLETED", got.Status)
+ }
+ if got.ErrorInfo != "boom" {
+ t.Errorf("ErrorInfo = %q, want boom", got.ErrorInfo)
+ }
+ if got.TaskID != "task-1" || got.DocID != "doc-1" || got.DatasetID != "kb-1" || got.UserID != "u-1" {
+ t.Errorf("identity mismatch: %+v", got)
+ }
+}
+
+// TestEmitLifecycleEventNilEmitterIsNoop verifies a nil emitter never panics and
+// a nil task is ignored (best-effort contract).
+func TestEmitLifecycleEventNilEmitterIsNoop(t *testing.T) {
+ svc := &IngestionTaskService{}
+ task := &entity.IngestionTask{ID: "t", DocumentID: "d"}
+ svc.emitLifecycleEvent(context.Background(), task, "created", "CREATED", "")
+ svc.emitLifecycleEvent(context.Background(), nil, "created", "CREATED", "")
+}
+
+// TestSetEventEmitterSwapsSink verifies SetEventEmitter installs a new sink and
+// that a nil argument is ignored (cannot disable emission by accident).
+func TestSetEventEmitterSwapsSink(t *testing.T) {
+ svc := &IngestionTaskService{}
+ rec := &recordingEmitter{}
+ svc.SetEventEmitter(nil) // ignored
+ svc.SetEventEmitter(rec)
+
+ task := &entity.IngestionTask{ID: "t2", DocumentID: "d2", DatasetID: "kb2", UserID: "u2"}
+ svc.emitLifecycleEvent(context.Background(), task, "running", "RUNNING", "")
+
+ if len(rec.events) != 1 {
+ t.Fatalf("expected 1 emitted event, got %d", len(rec.events))
+ }
+ if rec.last().EventType != "running" {
+ t.Errorf("EventType = %q, want running", rec.last().EventType)
+ }
+}
diff --git a/internal/service/ingestion_task_service.go b/internal/service/ingestion_task_service.go
index 014deefd1a..b4405dde03 100644
--- a/internal/service/ingestion_task_service.go
+++ b/internal/service/ingestion_task_service.go
@@ -13,6 +13,7 @@ import (
"ragflow/internal/engine/kvrocks"
"ragflow/internal/entity"
ingestionpipeline "ragflow/internal/ingestion/pipeline"
+ "ragflow/internal/observability"
"go.uber.org/zap"
"gorm.io/gorm"
@@ -125,6 +126,7 @@ type IngestionTaskService struct {
userCanvasDAO *dao.UserCanvasDAO
taskPublisher TaskPublisher
logSettings IngestionLogSettings
+ eventEmitter observability.IngestionEventEmitter
// purgeTaskState removes the deleted task's run-scoped chunk and media
// cache state. Nil resolves to the process-wide implementation.
purgeTaskState func(context.Context, string) error
@@ -139,6 +141,7 @@ func NewIngestionTaskService() *IngestionTaskService {
userCanvasDAO: dao.NewUserCanvasDAO(),
taskPublisher: NewMessageQueueTaskPublisher(),
logSettings: DefaultIngestionLogSettings(),
+ eventEmitter: &observability.CommonLogEmitter{},
}
}
@@ -149,6 +152,42 @@ func (s *IngestionTaskService) SetTaskPublisher(taskPublisher TaskPublisher) {
s.taskPublisher = taskPublisher
}
+// SetEventEmitter swaps the lifecycle event sink. A nil argument is ignored so
+// callers cannot accidentally disable emission by passing nil.
+func (s *IngestionTaskService) SetEventEmitter(emitter observability.IngestionEventEmitter) {
+ if emitter == nil {
+ return
+ }
+ s.eventEmitter = emitter
+}
+
+// emitLifecycleEvent builds and emits a structured observability event for a
+// task lifecycle transition. It is best-effort: any lookup failure or a nil
+// emitter is silently ignored so the parse pipeline is never disrupted by
+// observability. Enrichment reads the document (file name/type/size/parser) and
+// the bound pipeline run (run number, begin/end, progress) and forwards them to
+// observability.BuildIngestLog, which owns the field mapping onto common.IngestionLog.
+func (s *IngestionTaskService) emitLifecycleEvent(ctx context.Context, task *entity.IngestionTask, typ, status, errorInfo string) {
+ if s.eventEmitter == nil || task == nil {
+ return
+ }
+ var doc *entity.Document
+ if s.documentDAO != nil {
+ doc, _ = s.documentDAO.GetByID(ctx, dao.DB, task.DocumentID)
+ }
+ logID := task.PipelineLogID
+ if (logID == nil || *logID == "") && s.documentDAO != nil {
+ if fresh, err := s.GetTask(ctx, task.ID); err == nil {
+ logID = fresh.PipelineLogID
+ }
+ }
+ var run *entity.PipelineOperationLog
+ if logID != nil && *logID != "" && s.pipelineLogDAO != nil {
+ run, _ = s.pipelineLogDAO.GetByID(ctx, dao.DB, *logID)
+ }
+ s.eventEmitter.EmitIngestLog(ctx, observability.BuildIngestLog(task, doc, run, typ, status, errorInfo))
+}
+
func (s *IngestionTaskService) ListByUser(ctx context.Context, userID string, datasetID *string, page, pageSize int) ([]*entity.IngestionTask, error) {
if datasetID == nil {
return s.ingestionTaskDAO.ListByUserID(ctx, dao.DB, userID, page, pageSize)
@@ -265,6 +304,7 @@ func (s *IngestionTaskService) TransitionTaskToRunning(ctx context.Context, task
// close the open row. Close it here or it stays RUNNING forever.
s.advanceOpenLog(ctx, task, dao.OpenPipelineOperationStatuses(), string(entity.TaskStatusCancel))
s.recordRunTerminal(ctx, task, "Task stopped by user.")
+ s.emitLifecycleEvent(ctx, task, "stopped", common.STOPPED, "Task stopped by user.")
return task, nil
case common.RUNNING, common.COMPLETED, common.STOPPED, common.FAILED:
return task, nil
@@ -290,6 +330,12 @@ func (s *IngestionTaskService) PrepareValidatedRun(ctx context.Context, task *en
}
s.advanceOpenLog(ctx, task, logFromUnstartOrScheduled, string(entity.TaskStatusRunning))
s.recordRunMessage(ctx, task, "Task is running...")
+ // Emit the running lifecycle event only after the document's worker-claim
+ // time has been stamped, so the event's BeginTime (and wait_duration_s)
+ // reflect this run rather than a zero or previous-run value. Emitting here
+ // (guarded by a successful claim in the caller) keeps one running event per
+ // actual run start.
+ s.emitLifecycleEvent(ctx, task, "running", common.RUNNING, "")
}
func (s *IngestionTaskService) RequestStop(ctx context.Context, taskID string) (*entity.IngestionTask, error) {
@@ -308,6 +354,7 @@ func (s *IngestionTaskService) RequestStop(ctx context.Context, taskID string) (
// CANCEL here, otherwise the detail page keeps a queued entry.
s.advanceOpenLog(ctx, stopped, logFromUnstartOrScheduled, string(entity.TaskStatusCancel))
s.recordRunTerminal(ctx, stopped, "Task stopped by user.")
+ s.emitLifecycleEvent(ctx, stopped, "stopped", common.STOPPED, "Task stopped by user.")
return stopped, nil
case common.RUNNING:
task, err = s.transition(ctx, taskID, common.STOPPING)
@@ -348,6 +395,7 @@ func (s *IngestionTaskService) supersedeUnstartedTask(ctx context.Context, taskI
if err := s.settleTaskRun(ctx, stopped, entity.TaskStatusCancel, terminalMessage); err != nil {
return nil, err
}
+ s.emitLifecycleEvent(ctx, stopped, "stopped", common.STOPPED, terminalMessage)
return stopped, nil
case common.RUNNING, common.STOPPING:
return nil, fmt.Errorf("task %s is %s and cannot be superseded", taskID, task.Status)
@@ -402,11 +450,27 @@ func (s *IngestionTaskService) MarkCompleted(ctx context.Context, taskID string)
if task.Status == common.COMPLETED || task.Status == common.STOPPED || task.Status == common.FAILED {
return nil // already terminal, idempotent — mirrors MarkStopped
}
- _, err = s.transition(ctx, taskID, common.COMPLETED)
- return err
+ updated, err := s.transition(ctx, taskID, common.COMPLETED)
+ if err != nil {
+ return err
+ }
+ s.emitLifecycleEvent(ctx, updated, "completed", common.COMPLETED, "")
+ return nil
}
-func (s *IngestionTaskService) MarkFailed(ctx context.Context, taskID string) error {
+// MarkFailed transitions a task to FAILED and emits a lifecycle event carrying
+// the terminal reason. This is the public entry point used by the ingestor,
+// which has the failure reason (timeout/panic/run-identity error) available at
+// the call site; passing it here is what populates the event's ErrorInfo.
+func (s *IngestionTaskService) MarkFailed(ctx context.Context, taskID, reason string) error {
+ return s.markFailed(ctx, taskID, reason)
+}
+
+// markFailed transitions a task to FAILED and emits a lifecycle event carrying
+// the terminal reason. The public MarkFailed forwards the reason supplied by
+// the ingestor (timeout/panic/run-identity error); internal paths such as
+// settlePublishFailure pass an explicit reason too.
+func (s *IngestionTaskService) markFailed(ctx context.Context, taskID, reason string) error {
task, err := s.GetTask(ctx, taskID)
if err != nil {
return err
@@ -414,8 +478,12 @@ func (s *IngestionTaskService) MarkFailed(ctx context.Context, taskID string) er
if task.Status == common.FAILED || task.Status == common.COMPLETED || task.Status == common.STOPPED {
return nil // already terminal, idempotent — mirrors MarkStopped
}
- _, err = s.transition(ctx, taskID, common.FAILED)
- return err
+ updated, err := s.transition(ctx, taskID, common.FAILED)
+ if err != nil {
+ return err
+ }
+ s.emitLifecycleEvent(ctx, updated, "failed", common.FAILED, reason)
+ return nil
}
// MarkStopped transitions the task from STOPPING to STOPPED (terminal).
@@ -429,8 +497,12 @@ func (s *IngestionTaskService) MarkStopped(ctx context.Context, taskID string) e
if task.Status == common.STOPPED || task.Status == common.COMPLETED || task.Status == common.FAILED {
return nil
}
- _, err = s.transition(ctx, taskID, common.STOPPED)
- return err
+ updated, err := s.transition(ctx, taskID, common.STOPPED)
+ if err != nil {
+ return err
+ }
+ s.emitLifecycleEvent(ctx, updated, "stopped", common.STOPPED, "")
+ return nil
}
func (s *IngestionTaskService) Remove(ctx context.Context, taskID string, userID *string) (*dao.TaskInfo, error) {
@@ -672,6 +744,7 @@ func (s *IngestionTaskService) createAndEnqueueWithKBCache(ctx context.Context,
if err = s.ensureRunIdentity(ctx, existing, kbCache); err != nil {
return nil, fmt.Errorf("ensure run identity for task %s: %w", existing.ID, err)
}
+ s.emitLifecycleEvent(ctx, existing, "created", common.CREATED, "")
if err = s.enqueueTask(existing.ID); err != nil {
s.settlePublishFailure(ctx, existing)
return nil, err
@@ -698,6 +771,7 @@ func (s *IngestionTaskService) createAndEnqueueWithKBCache(ctx context.Context,
}
return nil, fmt.Errorf("ensure run identity for task %s: %w", existing.ID, err)
}
+ s.emitLifecycleEvent(ctx, existing, "created", common.CREATED, "")
if err = s.enqueueTask(existing.ID); err != nil {
s.settlePublishFailure(ctx, existing)
return nil, err
@@ -718,6 +792,7 @@ func (s *IngestionTaskService) createAndEnqueueWithKBCache(ctx context.Context,
}
return nil, fmt.Errorf("ensure run identity for task %s: %w", created.ID, err)
}
+ s.emitLifecycleEvent(ctx, created, "created", common.CREATED, "")
if err = s.enqueueTask(created.ID); err != nil {
s.settlePublishFailure(ctx, created)
return nil, err
@@ -757,6 +832,7 @@ func (s *IngestionTaskService) markScheduledAfterPublish(ctx context.Context, ta
}
s.advanceOpenLog(ctx, task, logFromUnstart, string(entity.TaskStatusSchedule))
s.recordRunMessage(ctx, task, "Task is queued...")
+ s.emitLifecycleEvent(ctx, task, "scheduled", common.SCHEDULED, "")
return task, nil
}
@@ -810,7 +886,7 @@ func (s *IngestionTaskService) settlePublishFailure(ctx context.Context, task *e
if task == nil {
return
}
- if err := s.MarkFailed(ctx, task.ID); err != nil {
+ if err := s.markFailed(ctx, task.ID, "Task publish failed."); err != nil {
common.Error(fmt.Sprintf("mark task %s failed after publish failure", task.ID), err)
return
}
@@ -1061,11 +1137,35 @@ func (s *IngestionTaskService) RecordMessage(ctx context.Context, pipelineLogID,
return s.insertEvent(ctx, ingestionEventMessage, pipelineLogID, taskID, "", 0, message)
}
+// persistProcessEndAt records the authoritative terminal end time for a run.
+// It is idempotent: if a real process_end_at is already persisted, the existing
+// value is left untouched, so a later terminal event cannot rewrite the recorded
+// completion instant. Best-effort: errors are logged, never returned, because a
+// failure here must not fail the terminal settlement.
+func (s *IngestionTaskService) persistProcessEndAt(ctx context.Context, pipelineLogID string) {
+ if dao.DB == nil {
+ return
+ }
+ if updErr := dao.DB.WithContext(ctx).
+ Model(&entity.PipelineOperationLog{}).
+ Where("id = ?", pipelineLogID).
+ Where("process_end_at IS NULL").
+ Update("process_end_at", time.Now()).Error; updErr != nil {
+ common.Warn(fmt.Sprintf("persist process_end_at for run %s: %v", pipelineLogID, updErr))
+ }
+}
+
// RecordTerminal writes the terminal explanation associated with a run.
func (s *IngestionTaskService) RecordTerminal(ctx context.Context, pipelineLogID, taskID, message string) error {
if err := s.ensureTerminalEvent(ctx, pipelineLogID, taskID, message); err != nil {
return err
}
+ // Persist the authoritative terminal end time so ingestion observability
+ // events and any downstream duration query read a real value instead of
+ // falling back to the transition time. Idempotent: a run whose end time
+ // was already recorded keeps it, so a repeated terminal settlement (e.g.
+ // overlapping failure/timeout events) cannot rewrite the completion instant.
+ s.persistProcessEndAt(context.WithoutCancel(ctx), pipelineLogID)
foldContext := context.WithoutCancel(ctx)
foldStarted := time.Now()
foldResult, foldErr := s.foldIngestionRun(foldContext, pipelineLogID, s.logSettings.MaxRowsPerRun)
diff --git a/internal/service/ingestion_task_service_test.go b/internal/service/ingestion_task_service_test.go
index 349e9e7876..c00ad0a19d 100644
--- a/internal/service/ingestion_task_service_test.go
+++ b/internal/service/ingestion_task_service_test.go
@@ -649,7 +649,7 @@ func TestIngestionTaskServiceMarkFailedUpdatesTaskStatus(t *testing.T) {
ctx := t.Context()
svc := NewIngestionTaskService()
- if err := svc.MarkFailed(ctx, "task-1"); err != nil {
+ if err := svc.MarkFailed(ctx, "task-1", ""); err != nil {
t.Fatalf("MarkFailed failed: %v", err)
}
task, err := dao.NewIngestionTaskDAO().GetByID(ctx, db, "task-1")
@@ -1883,7 +1883,7 @@ func TestIngestionTaskServiceMarkFailedIdempotentOnAlreadyTerminal(t *testing.T)
svc := NewIngestionTaskService()
ctx := t.Context()
- if err := svc.MarkFailed(ctx, "task-1"); err != nil {
+ if err := svc.MarkFailed(ctx, "task-1", ""); err != nil {
t.Fatalf("MarkFailed on already COMPLETED task should be idempotent, got: %v", err)
}
}
@@ -2523,3 +2523,168 @@ func TestIngestionTaskServiceCleanupMissingRun(t *testing.T) {
})
}
}
+
+// recordingIngestLogEmitter captures emitted lifecycle events for assertions.
+type recordingIngestLogEmitter struct {
+ mu sync.Mutex
+ logs []common.IngestionLog
+}
+
+func (e *recordingIngestLogEmitter) EmitIngestLog(_ context.Context, log common.IngestionLog) {
+ e.mu.Lock()
+ defer e.mu.Unlock()
+ e.logs = append(e.logs, log)
+}
+
+func (e *recordingIngestLogEmitter) last() common.IngestionLog {
+ e.mu.Lock()
+ defer e.mu.Unlock()
+ if len(e.logs) == 0 {
+ return common.IngestionLog{}
+ }
+ return e.logs[len(e.logs)-1]
+}
+
+// TestIngestionTaskServiceMarkFailedCarriesReason verifies that the failure
+// reason supplied by the ingestor reaches the emitted lifecycle event's
+// ErrorInfo, instead of being dropped (the previous gap: the run ledger got the
+// message but the observability event carried an empty ErrorInfo).
+func TestIngestionTaskServiceMarkFailedCarriesReason(t *testing.T) {
+ db := setupServiceTestDB(t)
+ pushServiceDB(t, db)
+ insertTestIngestionTask(t, "task-1", "user-1", "doc-1", "kb-1")
+ if err := dao.DB.Model(&entity.IngestionTask{}).Where("id = ?", "task-1").Update("status", common.RUNNING).Error; err != nil {
+ t.Fatalf("set running status: %v", err)
+ }
+ ctx := t.Context()
+ emitter := &recordingIngestLogEmitter{}
+ svc := NewIngestionTaskService()
+ svc.SetEventEmitter(emitter)
+ const reason = "boom: nil deref"
+ if err := svc.MarkFailed(ctx, "task-1", reason); err != nil {
+ t.Fatalf("MarkFailed failed: %v", err)
+ }
+ got := emitter.last()
+ if got.ErrorInfo != reason {
+ t.Fatalf("emitted ErrorInfo = %q, want %q", got.ErrorInfo, reason)
+ }
+ if got.EventType != "failed" || got.Status != common.FAILED {
+ t.Fatalf("emitted event type/status = %q/%q, want failed/%q", got.EventType, got.Status, common.FAILED)
+ }
+}
+
+// TestIngestionTaskServicePersistProcessEndAtIdempotent verifies that
+// persistProcessEndAt records the end time on first settlement and leaves an
+// already-recorded value untouched on a later settlement (overlapping
+// failure/timeout terminal events must not rewrite the completion instant).
+func TestIngestionTaskServicePersistProcessEndAtIdempotent(t *testing.T) {
+ db := setupServiceTestDB(t)
+ pushServiceDB(t, db)
+ runID := "run-idempotent-1"
+ begin := time.Now().Add(-time.Minute)
+ run := &entity.PipelineOperationLog{
+ ID: runID,
+ DocumentID: "doc-idem-1",
+ TenantID: "tenant-1",
+ KbID: "kb-1",
+ ParserID: "parser-1",
+ DocumentName: "doc",
+ DocumentSuffix: ".pdf",
+ DocumentType: "pdf",
+ SourceFrom: "web",
+ OperationStatus: "RUNNING",
+ ProcessBeginAt: &begin,
+ }
+ if err := db.Create(run).Error; err != nil {
+ t.Fatalf("create run: %v", err)
+ }
+ ctx := t.Context()
+ svc := NewIngestionTaskService()
+ svc.persistProcessEndAt(ctx, runID)
+ var first entity.PipelineOperationLog
+ if err := db.Where("id = ?", runID).First(&first).Error; err != nil {
+ t.Fatalf("load run: %v", err)
+ }
+ if first.ProcessEndAt == nil {
+ t.Fatalf("process_end_at not persisted on first call")
+ }
+ firstEnd := *first.ProcessEndAt
+ // A second settlement must not overwrite the recorded end time.
+ svc.persistProcessEndAt(ctx, runID)
+ var second entity.PipelineOperationLog
+ if err := db.Where("id = ?", runID).First(&second).Error; err != nil {
+ t.Fatalf("load run again: %v", err)
+ }
+ if second.ProcessEndAt == nil || !second.ProcessEndAt.Equal(firstEnd) {
+ t.Fatalf("process_end_at changed on second call: first=%v second=%v", firstEnd, second.ProcessEndAt)
+ }
+}
+
+// TestPrepareValidatedRunEmitsRunningEventWithFreshClaimTime verifies that the
+// running lifecycle event is emitted AFTER PrepareValidatedRun stamps the
+// document's worker-claim time. Previously the running event was emitted in
+// TransitionTaskToRunning, before the claim time was written, so its BeginTime
+// (and therefore wait_duration_s) was either zero or the previous run's stale
+// value.
+func TestPrepareValidatedRunEmitsRunningEventWithFreshClaimTime(t *testing.T) {
+ db := setupServiceTestDB(t)
+ pushServiceDB(t, db)
+ insertTestKB(t, "kb-1", "tenant-1", 1, 0, 0)
+ insertTestDoc(t, "doc-1", "kb-1", 0, 0)
+ insertTestIngestionTask(t, "task-1", "user-1", "doc-1", "kb-1")
+
+ // Simulate a previous run that already stamped a claim time one hour ago.
+ stale := time.Now().Add(-time.Hour)
+ if err := db.Model(&entity.Document{}).Where("id = ?", "doc-1").Update("process_begin_at", stale).Error; err != nil {
+ t.Fatalf("seed stale claim time: %v", err)
+ }
+
+ // Bind a pipeline-operation-log row so advanceOpenLog/recordRunMessage succeed.
+ openLog := &entity.PipelineOperationLog{
+ ID: "log-1",
+ DocumentID: "doc-1",
+ TenantID: "tenant-1",
+ KbID: "kb-1",
+ ParserID: "naive",
+ TaskType: "Parse",
+ OperationStatus: string(entity.TaskStatusUnstart),
+ }
+ if err := db.Create(openLog).Error; err != nil {
+ t.Fatalf("seed open log: %v", err)
+ }
+ if err := db.Model(&entity.IngestionTask{}).Where("id = ?", "task-1").Update("pipeline_log_id", "log-1").Error; err != nil {
+ t.Fatalf("bind task: %v", err)
+ }
+
+ ctx := t.Context()
+ emitter := &recordingIngestLogEmitter{}
+ svc := NewIngestionTaskService()
+ svc.SetEventEmitter(emitter)
+
+ task, err := dao.NewIngestionTaskDAO().GetByID(ctx, db, "task-1")
+ if err != nil {
+ t.Fatalf("load task: %v", err)
+ }
+ svc.PrepareValidatedRun(ctx, task)
+
+ var running common.IngestionLog
+ found := false
+ for _, l := range emitter.logs {
+ if l.EventType == "running" {
+ running = l
+ found = true
+ }
+ }
+ if !found {
+ t.Fatalf("no running event emitted by PrepareValidatedRun")
+ }
+ if running.BeginTime.IsZero() {
+ t.Fatalf("running event BeginTime is zero - claim time was not stamped before emit")
+ }
+ if !running.BeginTime.After(stale) {
+ t.Fatalf("running event BeginTime = %v, want after stale claim time %v (must reflect current run)", running.BeginTime, stale)
+ }
+ if time.Since(running.BeginTime) > 30*time.Second {
+ t.Fatalf("running event BeginTime = %v, suspiciously old for a fresh claim", running.BeginTime)
+ }
+}