Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

Fix: to add dataset logs (#19973)

Wang Qi006eae3d2573
diff --git a/internal/dao/pipeline_operation_log.go b/internal/dao/pipeline_operation_log.go
index 990d1e09bc..95e4d127e5 100644
--- a/internal/dao/pipeline_operation_log.go
+++ b/internal/dao/pipeline_operation_log.go
@@ -52,11 +52,6 @@ func normalizePipelineOperationStatuses(statuses []string) []string {
return normalized
}
-// graphRaptorFakeDocID is the placeholder document_id used for dataset-level
-// (graph/raptor/mindmap) pipeline logs, mirroring GRAPH_RAPTOR_FAKE_DOC_ID in
-// api/db/services/task_service.py.
-const graphRaptorFakeDocID = "graph_raptor_x"
-
// PipelineOperationLogDAO data access object for pipeline_operation_log.
type PipelineOperationLogDAO struct{}
@@ -68,13 +63,15 @@ func NewPipelineOperationLogDAO() *PipelineOperationLogDAO {
// GetDatasetLogsByKBID lists dataset-level (graph/raptor/mindmap) ingestion
// logs for a knowledge base. Pagination is only applied when both page and
// pageSize are positive, matching peewee's paginate behavior.
+// Dataset-level writers leave run_count NULL; positive run counts identify
+// run-scoped ingestion rows and must not be returned from this list.
//
// documentID is honoured for the same reason as in GetFileLogsByKBID. Dataset
// logs belong to no single document, so a caller that names one gets an empty
// list rather than the whole dataset history.
func (dao *PipelineOperationLogDAO) GetDatasetLogsByKBID(ctx context.Context, db *gorm.DB, kbID string, page, pageSize int, terms []OrderTerm, operationStatus []string, createDateFrom, createDateTo, keywords, documentID string) ([]*entity.PipelineOperationLog, int64, error) {
query := db.WithContext(ctx).Model(&entity.PipelineOperationLog{}).
- Where("kb_id = ? AND document_id = ? AND run_count > 0", kbID, graphRaptorFakeDocID)
+ Where("kb_id = ? AND document_id = ? AND run_count IS NULL", kbID, entity.DatasetLogDocumentID)
if keywords != "" {
query = query.Where("LOWER(document_name) LIKE ?", "%"+strings.ToLower(keywords)+"%")
@@ -130,7 +127,7 @@ func (dao *PipelineOperationLogDAO) GetFileLogsByKBID(ctx context.Context, db *g
if documentID != "" {
query = query.Where("document_id = ?", documentID)
}
- query = query.Where("document_id <> ?", graphRaptorFakeDocID)
+ query = query.Where("document_id <> ?", entity.DatasetLogDocumentID)
if len(operationStatus) > 0 {
query = query.Where("operation_status IN ?", operationStatus)
diff --git a/internal/entity/pipeline.go b/internal/entity/pipeline.go
index a4b1bdf7a8..52b1998963 100644
--- a/internal/entity/pipeline.go
+++ b/internal/entity/pipeline.go
@@ -18,6 +18,10 @@ package entity
import "time"
+// DatasetLogDocumentID is the synthetic document identity shared by
+// dataset-level graph, RAPTOR, mind-map, and knowledge-compilation logs.
+const DatasetLogDocumentID = "graph_raptor_x"
+
// PipelineOperationLog pipeline operation log model
type PipelineOperationLog struct {
ID string `gorm:"column:id;primaryKey;size:32" json:"id"`
diff --git a/internal/ingestion/knowledge_compile/dataset_log.go b/internal/ingestion/knowledge_compile/dataset_log.go
index b79fab49c3..a40a1c7865 100644
--- a/internal/ingestion/knowledge_compile/dataset_log.go
+++ b/internal/ingestion/knowledge_compile/dataset_log.go
@@ -25,11 +25,12 @@ import (
"time"
"ragflow/internal/common"
+ "ragflow/internal/dao"
"ragflow/internal/entity"
kccommon "ragflow/internal/ingestion/component/knowledge_compiler/common"
-)
-const datasetLogDocumentID = "graph_raptor_x"
+ "gorm.io/gorm"
+)
var allDatasetTaskTypes = []string{
kccommon.TaskTypeWiki,
@@ -174,40 +175,48 @@ func startDatasetCompileLog(ctx context.Context, tenantID, datasetID, claimToken
groups[taskType] = entries
}
}
- for _, taskType := range sortedTaskTypesFromGroups(groups) {
- entryData := make([]any, 0, len(groups[taskType]))
- for _, entry := range groups[taskType] {
- entryData = append(entryData, map[string]any{
- "doc_id": entry.DocID,
- "event_type": entry.EventType,
- "variants": entry.Variants,
- "task_type": taskType,
- })
- }
- message := timestampProgressMessage(fmt.Sprintf("Created automatic %s dataset task for %d document event(s)", taskType, len(groups[taskType])))
- log := entity.PipelineOperationLog{
- ID: datasetCompileLogID(claimToken, taskType),
- DocumentID: datasetLogDocumentID,
- TenantID: tenantID,
- KbID: datasetID,
- ParserID: "knowledge_compile",
- DocumentName: taskType,
- DocumentSuffix: "",
- DocumentType: "dataset",
- SourceFrom: "knowledgebase",
- Progress: 0,
- ProgressMsg: &message,
- ProcessBeginAt: &now,
- DSL: entity.JSONMap{"entries": entryData, "task_type": taskType},
- TaskType: taskType,
- OperationStatus: common.RUNNING,
- Status: &status,
- }
- if err := kcDB.WithContext(ctx).Where("id = ?", log.ID).FirstOrCreate(&log).Error; err != nil {
- return err
+ return kcDB.WithContext(ctx).Transaction(func(tx *gorm.DB) error {
+ for _, taskType := range sortedTaskTypesFromGroups(groups) {
+ entryData := make([]any, 0, len(groups[taskType]))
+ for _, entry := range groups[taskType] {
+ entryData = append(entryData, map[string]any{
+ "doc_id": entry.DocID,
+ "event_type": entry.EventType,
+ "variants": entry.Variants,
+ "task_type": taskType,
+ })
+ }
+ log := entity.PipelineOperationLog{
+ ID: datasetCompileLogID(claimToken, taskType),
+ DocumentID: entity.DatasetLogDocumentID,
+ TenantID: tenantID,
+ KbID: datasetID,
+ ParserID: "knowledge_compile",
+ DocumentName: taskType,
+ DocumentSuffix: "",
+ DocumentType: "dataset",
+ SourceFrom: "knowledgebase",
+ Progress: 0,
+ ProcessBeginAt: &now,
+ DSL: entity.JSONMap{"entries": entryData, "task_type": taskType},
+ TaskType: taskType,
+ OperationStatus: common.RUNNING,
+ Status: &status,
+ }
+ result := tx.WithContext(ctx).Where("id = ?", log.ID).FirstOrCreate(&log)
+ if result.Error != nil {
+ return result.Error
+ }
+ if result.RowsAffected == 0 {
+ continue
+ }
+ message := fmt.Sprintf("Created automatic %s dataset task for %d document event(s)", taskType, len(groups[taskType]))
+ if err := createDatasetLogEvent(ctx, tx, log.ID, dao.EventTypeMessage, message); err != nil {
+ return err
+ }
}
- }
- return nil
+ return nil
+ })
}
func updateDatasetCompileLog(ctx context.Context, claimToken string, taskTypes []string, progress float64, message string) error {
@@ -223,25 +232,7 @@ func updateDatasetCompileLog(ctx context.Context, claimToken string, taskTypes [
}
func updateDatasetCompileLogForType(ctx context.Context, logID string, progress float64, message string) error {
- var log entity.PipelineOperationLog
- if err := kcDB.WithContext(ctx).Where("id = ?", logID).First(&log).Error; err != nil {
- return err
- }
- progressMessage := ""
- if log.ProgressMsg != nil {
- progressMessage = *log.ProgressMsg
- }
- progressMessage = appendProgressMessage(progressMessage, timestampProgressMessage(message))
- duration := log.ProcessDuration
- if log.ProcessBeginAt != nil {
- duration = max(0, time.Since(*log.ProcessBeginAt).Seconds())
- }
- return kcDB.WithContext(ctx).Model(&entity.PipelineOperationLog{}).Where("id = ?", logID).Updates(map[string]any{
- "progress": progress,
- "progress_msg": progressMessage,
- "process_duration": duration,
- "operation_status": common.RUNNING,
- }).Error
+ return recordDatasetCompileLogEvent(ctx, logID, progress, common.RUNNING, dao.EventTypeMessage, message)
}
func finishDatasetCompileLog(ctx context.Context, claimToken string, taskTypes []string, operationStatus, message string, progress float64) error {
@@ -250,29 +241,45 @@ func finishDatasetCompileLog(ctx context.Context, claimToken string, taskTypes [
}
for _, taskType := range taskTypes {
logID := datasetCompileLogID(claimToken, taskType)
- var log entity.PipelineOperationLog
- if err := kcDB.WithContext(ctx).Where("id = ?", logID).First(&log).Error; err != nil {
+ if err := recordDatasetCompileLogEvent(ctx, logID, progress, datasetLogOperationStatus(operationStatus), dao.EventTypeTerminal, message); err != nil {
return err
}
- progressMessage := ""
- if log.ProgressMsg != nil {
- progressMessage = *log.ProgressMsg
+ }
+ return nil
+}
+
+func recordDatasetCompileLogEvent(ctx context.Context, logID string, progress float64, operationStatus string, eventType int, message string) error {
+ return kcDB.WithContext(ctx).Transaction(func(tx *gorm.DB) error {
+ var log entity.PipelineOperationLog
+ if err := tx.WithContext(ctx).Where("id = ?", logID).First(&log).Error; err != nil {
+ return err
}
- progressMessage = appendProgressMessage(progressMessage, timestampProgressMessage(message))
duration := log.ProcessDuration
if log.ProcessBeginAt != nil {
duration = max(0, time.Since(*log.ProcessBeginAt).Seconds())
}
- if err := kcDB.WithContext(ctx).Model(&entity.PipelineOperationLog{}).Where("id = ?", logID).Updates(map[string]any{
+ if err := tx.WithContext(ctx).Model(&entity.PipelineOperationLog{}).Where("id = ?", logID).Updates(map[string]any{
"progress": progress,
- "progress_msg": progressMessage,
"process_duration": duration,
- "operation_status": datasetLogOperationStatus(operationStatus),
+ "operation_status": operationStatus,
}).Error; err != nil {
return err
}
- }
- return nil
+ return createDatasetLogEvent(ctx, tx, logID, eventType, message)
+ })
+}
+
+func createDatasetLogEvent(ctx context.Context, db *gorm.DB, logID string, eventType int, message string) error {
+ // Dataset compilation has no ingestion_task row, so its immutable pipeline
+ // log ID also serves as the event's stable task identity.
+ pipelineLogID := logID
+ return dao.NewIngestionTaskLogDAO().Create(ctx, db, &entity.IngestionTaskLog{
+ TaskID: logID,
+ PipelineLogID: &pipelineLogID,
+ Checkpoint: entity.JSONMap{},
+ EventType: eventType,
+ Message: message,
+ })
}
func sortedTaskTypesFromGroups(groups map[string][]BacklogEntry) []string {
diff --git a/internal/ingestion/knowledge_compile/dataset_log_test.go b/internal/ingestion/knowledge_compile/dataset_log_test.go
index a600fe619e..14114e7718 100644
--- a/internal/ingestion/knowledge_compile/dataset_log_test.go
+++ b/internal/ingestion/knowledge_compile/dataset_log_test.go
@@ -9,6 +9,7 @@ import (
"gorm.io/gorm"
"ragflow/internal/common"
+ "ragflow/internal/dao"
"ragflow/internal/entity"
kccommon "ragflow/internal/ingestion/component/knowledge_compiler/common"
)
@@ -18,8 +19,8 @@ func TestDatasetCompileLogLifecycle(t *testing.T) {
if err != nil {
t.Fatalf("open sqlite: %v", err)
}
- if err := db.AutoMigrate(&entity.PipelineOperationLog{}); err != nil {
- t.Fatalf("migrate pipeline log: %v", err)
+ if err := db.AutoMigrate(&entity.PipelineOperationLog{}, &entity.IngestionTaskLog{}); err != nil {
+ t.Fatalf("migrate ingestion logs: %v", err)
}
oldDB := kcDB
kcDB = db
@@ -32,6 +33,9 @@ func TestDatasetCompileLogLifecycle(t *testing.T) {
if err := startDatasetCompileLog(t.Context(), "tenant-1", "kb-1", "claim-1", entries); err != nil {
t.Fatalf("start dataset log: %v", err)
}
+ if err := startDatasetCompileLog(t.Context(), "tenant-1", "kb-1", "claim-1", entries); err != nil {
+ t.Fatalf("restart dataset log: %v", err)
+ }
if err := updateDatasetCompileLog(t.Context(), "claim-1", []string{kccommon.TaskTypeWiki, kccommon.TaskTypeGraph}, 0.5, "Comparing knowledge contributions: 2 affected page(s)"); err != nil {
t.Fatalf("update dataset log: %v", err)
}
@@ -47,14 +51,14 @@ func TestDatasetCompileLogLifecycle(t *testing.T) {
t.Fatalf("want one log per task type, got %d: %+v", len(logs), logs)
}
for _, log := range logs {
- if log.DocumentID != datasetLogDocumentID || log.OperationStatus != "DONE" || log.Progress != 1 {
+ if log.DocumentID != entity.DatasetLogDocumentID || log.RunCount != nil || log.OperationStatus != "DONE" || log.Progress != 1 {
t.Fatalf("unexpected dataset log state: %+v", log)
}
if log.ID != datasetCompileLogID("claim-1", log.TaskType) {
t.Fatalf("dataset log id is not task-type scoped: %+v", log)
}
- if log.ProgressMsg == nil || !strings.Contains(*log.ProgressMsg, "2 affected page(s)") || !strings.Contains(*log.ProgressMsg, "completed") {
- t.Fatalf("progress messages were not persisted: %v", log.ProgressMsg)
+ if log.ProgressMsg != nil {
+ t.Fatalf("dataset log retained progress_msg: %q", *log.ProgressMsg)
}
if log.DSL["task_type"] != log.TaskType {
t.Fatalf("dataset log DSL task type = %v, want %s", log.DSL["task_type"], log.TaskType)
@@ -63,6 +67,28 @@ func TestDatasetCompileLogLifecycle(t *testing.T) {
if !ok || len(entries) != 1 {
t.Fatalf("dataset log entries were not split by task type: %+v", log.DSL["entries"])
}
+
+ var events []entity.IngestionTaskLog
+ if err := db.Where("pipeline_log_id = ?", log.ID).Order("id ASC").Find(&events).Error; err != nil {
+ t.Fatalf("load dataset log events: %v", err)
+ }
+ if len(events) != 3 {
+ t.Fatalf("dataset log %s event count = %d, want 3: %+v", log.ID, len(events), events)
+ }
+ if events[0].EventType != dao.EventTypeMessage || !strings.Contains(events[0].Message, "Created automatic "+log.TaskType+" dataset task") {
+ t.Fatalf("unexpected initial event: %+v", events[0])
+ }
+ if events[1].EventType != dao.EventTypeMessage || !strings.Contains(events[1].Message, "2 affected page(s)") {
+ t.Fatalf("unexpected progress event: %+v", events[1])
+ }
+ if events[2].EventType != dao.EventTypeTerminal || !strings.Contains(events[2].Message, "completed") {
+ t.Fatalf("unexpected terminal event: %+v", events[2])
+ }
+ for _, event := range events {
+ if event.TaskID != log.ID || event.PipelineLogID == nil || *event.PipelineLogID != log.ID {
+ t.Fatalf("event is not scoped to dataset log %s: %+v", log.ID, event)
+ }
+ }
}
}
diff --git a/internal/service/dataset/helpers.go b/internal/service/dataset/helpers.go
index eff2b8392d..1ccd24f05d 100644
--- a/internal/service/dataset/helpers.go
+++ b/internal/service/dataset/helpers.go
@@ -56,7 +56,6 @@ var (
)
const (
- graphRaptorQueueDocID = "graph_raptor_x"
maximumTaskPageNumber = int64(100000000)
serverQueueNamePrefix = "te"
defaultEmbeddingCheckNum = 5
diff --git a/internal/service/dataset/ingestion.go b/internal/service/dataset/ingestion.go
index 20b65431ca..39b63952a6 100644
--- a/internal/service/dataset/ingestion.go
+++ b/internal/service/dataset/ingestion.go
@@ -93,11 +93,13 @@ func (d *DatasetService) ListIngestionMessages(ctx context.Context, datasetID, u
}
return nil, common.CodeServerError, fmt.Errorf("get ingestion log: %w", err)
}
- if run.RunCount == nil || *run.RunCount <= 0 {
+ if !isReadableIngestionLog(run) {
return nil, common.CodeDataError, errors.New("log not found")
}
response := &IngestionMessagesResponse{}
- response.RunCount = *run.RunCount
+ if run.RunCount != nil {
+ response.RunCount = *run.RunCount
+ }
response.Terminal = isTerminalIngestionLogStatus(run.OperationStatus)
page, err := dao.NewIngestionTaskLogDAO().ListEventsPageByPipelineLogID(ctx, dao.DB, logID, limit, afterID, beforeID)
@@ -126,6 +128,16 @@ func isTerminalIngestionLogStatus(status string) bool {
}
}
+func isReadableIngestionLog(log *entity.PipelineOperationLog) bool {
+ if log == nil {
+ return false
+ }
+ if log.DocumentID == entity.DatasetLogDocumentID {
+ return log.RunCount == nil
+ }
+ return log.RunCount != nil && *log.RunCount > 0
+}
+
func (d *DatasetService) ListIngestionLogs(ctx context.Context, datasetID, userID string, page, pageSize int, terms []dao.OrderTerm, operationStatus []string, createDateFrom, createDateTo, logType, keywords, documentID string) (map[string]interface{}, common.ErrorCode, error) {
if datasetID == "" {
return nil, common.CodeDataError, errors.New(`lack of "Dataset ID"`)
@@ -202,7 +214,7 @@ func (d *DatasetService) GetIngestionLog(ctx context.Context, datasetID, userID,
}
return nil, common.CodeServerError, fmt.Errorf("get ingestion log: %w", err)
}
- if log.RunCount == nil || *log.RunCount <= 0 {
+ if !isReadableIngestionLog(log) {
return nil, common.CodeDataError, errors.New("log not found")
}
diff --git a/internal/service/dataset/ingestion_messages_test.go b/internal/service/dataset/ingestion_messages_test.go
index 1472665fd2..5411ef8bfc 100644
--- a/internal/service/dataset/ingestion_messages_test.go
+++ b/internal/service/dataset/ingestion_messages_test.go
@@ -64,6 +64,55 @@ func TestListIngestionMessagesRejectsUnnumberedRun(t *testing.T) {
}
}
+func TestDatasetIngestionLogUsesEventStream(t *testing.T) {
+ db := setupServiceTestDB(t)
+ pushServiceDB(t, db)
+ if err := db.AutoMigrate(&entity.PipelineOperationLog{}); err != nil {
+ t.Fatalf("migrate pipeline operation log: %v", err)
+ }
+ insertCompilationOwnerKB(t, "kb-1", "user-1")
+ if err := db.Create(&entity.PipelineOperationLog{
+ ID: "dataset-run",
+ DocumentID: entity.DatasetLogDocumentID,
+ TenantID: "user-1",
+ KbID: "kb-1",
+ ParserID: "knowledge_compile",
+ DocumentName: "Wiki",
+ DocumentType: "dataset",
+ SourceFrom: "knowledgebase",
+ TaskType: "Wiki",
+ OperationStatus: "DONE",
+ }).Error; err != nil {
+ t.Fatalf("insert dataset run: %v", err)
+ }
+ insertMessageRun(t, "numbered-sentinel-run", "kb-1", entity.DatasetLogDocumentID, entity.TaskStatusDone, 1)
+ insertMessageEvent(t, db, "dataset-run", "dataset-run", dao.EventTypeTerminal, "Knowledge compilation completed")
+
+ result, code, err := NewDatasetService().ListIngestionLogs(t.Context(), "kb-1", "user-1", 1, 30, nil, nil, "", "", "dataset", "", "")
+ if err != nil || code != common.CodeSuccess {
+ t.Fatalf("ListIngestionLogs = (%+v, %v, %v), want success", result, code, err)
+ }
+ logs, ok := result["logs"].([]map[string]interface{})
+ if !ok || len(logs) != 1 || logs[0]["id"] != "dataset-run" {
+ t.Fatalf("dataset logs = %#v, want only unnumbered dataset run", result["logs"])
+ }
+ assertLatestEventMap(t, logs[0], 1, "Knowledge compilation completed")
+
+ messages, code, err := NewDatasetService().ListIngestionMessages(t.Context(), "kb-1", "user-1", "dataset-run", 200, nil, nil)
+ if err != nil || code != common.CodeSuccess {
+ t.Fatalf("ListIngestionMessages = (%+v, %v, %v), want success", messages, code, err)
+ }
+ if messages.RunCount != 0 || !messages.Terminal || len(messages.Items) != 1 || messages.Items[0].Message != "Knowledge compilation completed" {
+ t.Fatalf("dataset messages = %+v, want terminal event stream", messages)
+ }
+
+ log, code, err := NewDatasetService().GetIngestionLog(t.Context(), "kb-1", "user-1", "dataset-run")
+ if err != nil || code != common.CodeSuccess {
+ t.Fatalf("GetIngestionLog = (%+v, %v, %v), want success", log, code, err)
+ }
+ assertLatestEventMap(t, log, 1, "Knowledge compilation completed")
+}
+
func TestListIngestionLogsExcludesUnnumberedRuns(t *testing.T) {
db := setupServiceTestDB(t)
pushServiceDB(t, db)