RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
Fix: to add dataset logs (#19973)
Wang Qi006eae3d2573
diff --git a/internal/dao/pipeline_operation_log.go b/internal/dao/pipeline_operation_log.goindex 990d1e09bc..95e4d127e5 100644--- a/internal/dao/pipeline_operation_log.go+++ b/internal/dao/pipeline_operation_log.go@@ -52,11 +52,6 @@ func normalizePipelineOperationStatuses(statuses []string) []string {return normalized}-// graphRaptorFakeDocID is the placeholder document_id used for dataset-level-// (graph/raptor/mindmap) pipeline logs, mirroring GRAPH_RAPTOR_FAKE_DOC_ID in-// api/db/services/task_service.py.-const graphRaptorFakeDocID = "graph_raptor_x"-// PipelineOperationLogDAO data access object for pipeline_operation_log.type PipelineOperationLogDAO struct{}@@ -68,13 +63,15 @@ func NewPipelineOperationLogDAO() *PipelineOperationLogDAO {// GetDatasetLogsByKBID lists dataset-level (graph/raptor/mindmap) ingestion// logs for a knowledge base. Pagination is only applied when both page and// pageSize are positive, matching peewee's paginate behavior.+// Dataset-level writers leave run_count NULL; positive run counts identify+// run-scoped ingestion rows and must not be returned from this list.//// documentID is honoured for the same reason as in GetFileLogsByKBID. Dataset// logs belong to no single document, so a caller that names one gets an empty// list rather than the whole dataset history.func (dao *PipelineOperationLogDAO) GetDatasetLogsByKBID(ctx context.Context, db *gorm.DB, kbID string, page, pageSize int, terms []OrderTerm, operationStatus []string, createDateFrom, createDateTo, keywords, documentID string) ([]*entity.PipelineOperationLog, int64, error) {query := db.WithContext(ctx).Model(&entity.PipelineOperationLog{}).- Where("kb_id = ? AND document_id = ? AND run_count > 0", kbID, graphRaptorFakeDocID)+ Where("kb_id = ? AND document_id = ? AND run_count IS NULL", kbID, entity.DatasetLogDocumentID)if keywords != "" {query = query.Where("LOWER(document_name) LIKE ?", "%"+strings.ToLower(keywords)+"%")@@ -130,7 +127,7 @@ func (dao *PipelineOperationLogDAO) GetFileLogsByKBID(ctx context.Context, db *gif documentID != "" {query = query.Where("document_id = ?", documentID)}- query = query.Where("document_id <> ?", graphRaptorFakeDocID)+ query = query.Where("document_id <> ?", entity.DatasetLogDocumentID)if len(operationStatus) > 0 {query = query.Where("operation_status IN ?", operationStatus)diff --git a/internal/entity/pipeline.go b/internal/entity/pipeline.goindex a4b1bdf7a8..52b1998963 100644--- a/internal/entity/pipeline.go+++ b/internal/entity/pipeline.go@@ -18,6 +18,10 @@ package entityimport "time"+// DatasetLogDocumentID is the synthetic document identity shared by+// dataset-level graph, RAPTOR, mind-map, and knowledge-compilation logs.+const DatasetLogDocumentID = "graph_raptor_x"+// PipelineOperationLog pipeline operation log modeltype PipelineOperationLog struct {ID string `gorm:"column:id;primaryKey;size:32" json:"id"`diff --git a/internal/ingestion/knowledge_compile/dataset_log.go b/internal/ingestion/knowledge_compile/dataset_log.goindex b79fab49c3..a40a1c7865 100644--- a/internal/ingestion/knowledge_compile/dataset_log.go+++ b/internal/ingestion/knowledge_compile/dataset_log.go@@ -25,11 +25,12 @@ import ("time""ragflow/internal/common"+ "ragflow/internal/dao""ragflow/internal/entity"kccommon "ragflow/internal/ingestion/component/knowledge_compiler/common"-)-const datasetLogDocumentID = "graph_raptor_x"+ "gorm.io/gorm"+)var allDatasetTaskTypes = []string{kccommon.TaskTypeWiki,@@ -174,40 +175,48 @@ func startDatasetCompileLog(ctx context.Context, tenantID, datasetID, claimTokengroups[taskType] = entries}}- for _, taskType := range sortedTaskTypesFromGroups(groups) {- entryData := make([]any, 0, len(groups[taskType]))- for _, entry := range groups[taskType] {- entryData = append(entryData, map[string]any{- "doc_id": entry.DocID,- "event_type": entry.EventType,- "variants": entry.Variants,- "task_type": taskType,- })- }- message := timestampProgressMessage(fmt.Sprintf("Created automatic %s dataset task for %d document event(s)", taskType, len(groups[taskType])))- log := entity.PipelineOperationLog{- ID: datasetCompileLogID(claimToken, taskType),- DocumentID: datasetLogDocumentID,- TenantID: tenantID,- KbID: datasetID,- ParserID: "knowledge_compile",- DocumentName: taskType,- DocumentSuffix: "",- DocumentType: "dataset",- SourceFrom: "knowledgebase",- Progress: 0,- ProgressMsg: &message,- ProcessBeginAt: &now,- DSL: entity.JSONMap{"entries": entryData, "task_type": taskType},- TaskType: taskType,- OperationStatus: common.RUNNING,- Status: &status,- }- if err := kcDB.WithContext(ctx).Where("id = ?", log.ID).FirstOrCreate(&log).Error; err != nil {- return err+ return kcDB.WithContext(ctx).Transaction(func(tx *gorm.DB) error {+ for _, taskType := range sortedTaskTypesFromGroups(groups) {+ entryData := make([]any, 0, len(groups[taskType]))+ for _, entry := range groups[taskType] {+ entryData = append(entryData, map[string]any{+ "doc_id": entry.DocID,+ "event_type": entry.EventType,+ "variants": entry.Variants,+ "task_type": taskType,+ })+ }+ log := entity.PipelineOperationLog{+ ID: datasetCompileLogID(claimToken, taskType),+ DocumentID: entity.DatasetLogDocumentID,+ TenantID: tenantID,+ KbID: datasetID,+ ParserID: "knowledge_compile",+ DocumentName: taskType,+ DocumentSuffix: "",+ DocumentType: "dataset",+ SourceFrom: "knowledgebase",+ Progress: 0,+ ProcessBeginAt: &now,+ DSL: entity.JSONMap{"entries": entryData, "task_type": taskType},+ TaskType: taskType,+ OperationStatus: common.RUNNING,+ Status: &status,+ }+ result := tx.WithContext(ctx).Where("id = ?", log.ID).FirstOrCreate(&log)+ if result.Error != nil {+ return result.Error+ }+ if result.RowsAffected == 0 {+ continue+ }+ message := fmt.Sprintf("Created automatic %s dataset task for %d document event(s)", taskType, len(groups[taskType]))+ if err := createDatasetLogEvent(ctx, tx, log.ID, dao.EventTypeMessage, message); err != nil {+ return err+ }}- }- return nil+ return nil+ })}func updateDatasetCompileLog(ctx context.Context, claimToken string, taskTypes []string, progress float64, message string) error {@@ -223,25 +232,7 @@ func updateDatasetCompileLog(ctx context.Context, claimToken string, taskTypes [}func updateDatasetCompileLogForType(ctx context.Context, logID string, progress float64, message string) error {- var log entity.PipelineOperationLog- if err := kcDB.WithContext(ctx).Where("id = ?", logID).First(&log).Error; err != nil {- return err- }- progressMessage := ""- if log.ProgressMsg != nil {- progressMessage = *log.ProgressMsg- }- progressMessage = appendProgressMessage(progressMessage, timestampProgressMessage(message))- duration := log.ProcessDuration- if log.ProcessBeginAt != nil {- duration = max(0, time.Since(*log.ProcessBeginAt).Seconds())- }- return kcDB.WithContext(ctx).Model(&entity.PipelineOperationLog{}).Where("id = ?", logID).Updates(map[string]any{- "progress": progress,- "progress_msg": progressMessage,- "process_duration": duration,- "operation_status": common.RUNNING,- }).Error+ return recordDatasetCompileLogEvent(ctx, logID, progress, common.RUNNING, dao.EventTypeMessage, message)}func finishDatasetCompileLog(ctx context.Context, claimToken string, taskTypes []string, operationStatus, message string, progress float64) error {@@ -250,29 +241,45 @@ func finishDatasetCompileLog(ctx context.Context, claimToken string, taskTypes [}for _, taskType := range taskTypes {logID := datasetCompileLogID(claimToken, taskType)- var log entity.PipelineOperationLog- if err := kcDB.WithContext(ctx).Where("id = ?", logID).First(&log).Error; err != nil {+ if err := recordDatasetCompileLogEvent(ctx, logID, progress, datasetLogOperationStatus(operationStatus), dao.EventTypeTerminal, message); err != nil {return err}- progressMessage := ""- if log.ProgressMsg != nil {- progressMessage = *log.ProgressMsg+ }+ return nil+}++func recordDatasetCompileLogEvent(ctx context.Context, logID string, progress float64, operationStatus string, eventType int, message string) error {+ return kcDB.WithContext(ctx).Transaction(func(tx *gorm.DB) error {+ var log entity.PipelineOperationLog+ if err := tx.WithContext(ctx).Where("id = ?", logID).First(&log).Error; err != nil {+ return err}- progressMessage = appendProgressMessage(progressMessage, timestampProgressMessage(message))duration := log.ProcessDurationif log.ProcessBeginAt != nil {duration = max(0, time.Since(*log.ProcessBeginAt).Seconds())}- if err := kcDB.WithContext(ctx).Model(&entity.PipelineOperationLog{}).Where("id = ?", logID).Updates(map[string]any{+ if err := tx.WithContext(ctx).Model(&entity.PipelineOperationLog{}).Where("id = ?", logID).Updates(map[string]any{"progress": progress,- "progress_msg": progressMessage,"process_duration": duration,- "operation_status": datasetLogOperationStatus(operationStatus),+ "operation_status": operationStatus,}).Error; err != nil {return err}- }- return nil+ return createDatasetLogEvent(ctx, tx, logID, eventType, message)+ })+}++func createDatasetLogEvent(ctx context.Context, db *gorm.DB, logID string, eventType int, message string) error {+ // Dataset compilation has no ingestion_task row, so its immutable pipeline+ // log ID also serves as the event's stable task identity.+ pipelineLogID := logID+ return dao.NewIngestionTaskLogDAO().Create(ctx, db, &entity.IngestionTaskLog{+ TaskID: logID,+ PipelineLogID: &pipelineLogID,+ Checkpoint: entity.JSONMap{},+ EventType: eventType,+ Message: message,+ })}func sortedTaskTypesFromGroups(groups map[string][]BacklogEntry) []string {diff --git a/internal/ingestion/knowledge_compile/dataset_log_test.go b/internal/ingestion/knowledge_compile/dataset_log_test.goindex a600fe619e..14114e7718 100644--- a/internal/ingestion/knowledge_compile/dataset_log_test.go+++ b/internal/ingestion/knowledge_compile/dataset_log_test.go@@ -9,6 +9,7 @@ import ("gorm.io/gorm""ragflow/internal/common"+ "ragflow/internal/dao""ragflow/internal/entity"kccommon "ragflow/internal/ingestion/component/knowledge_compiler/common")@@ -18,8 +19,8 @@ func TestDatasetCompileLogLifecycle(t *testing.T) {if err != nil {t.Fatalf("open sqlite: %v", err)}- if err := db.AutoMigrate(&entity.PipelineOperationLog{}); err != nil {- t.Fatalf("migrate pipeline log: %v", err)+ if err := db.AutoMigrate(&entity.PipelineOperationLog{}, &entity.IngestionTaskLog{}); err != nil {+ t.Fatalf("migrate ingestion logs: %v", err)}oldDB := kcDBkcDB = db@@ -32,6 +33,9 @@ func TestDatasetCompileLogLifecycle(t *testing.T) {if err := startDatasetCompileLog(t.Context(), "tenant-1", "kb-1", "claim-1", entries); err != nil {t.Fatalf("start dataset log: %v", err)}+ if err := startDatasetCompileLog(t.Context(), "tenant-1", "kb-1", "claim-1", entries); err != nil {+ t.Fatalf("restart dataset log: %v", err)+ }if err := updateDatasetCompileLog(t.Context(), "claim-1", []string{kccommon.TaskTypeWiki, kccommon.TaskTypeGraph}, 0.5, "Comparing knowledge contributions: 2 affected page(s)"); err != nil {t.Fatalf("update dataset log: %v", err)}@@ -47,14 +51,14 @@ func TestDatasetCompileLogLifecycle(t *testing.T) {t.Fatalf("want one log per task type, got %d: %+v", len(logs), logs)}for _, log := range logs {- if log.DocumentID != datasetLogDocumentID || log.OperationStatus != "DONE" || log.Progress != 1 {+ if log.DocumentID != entity.DatasetLogDocumentID || log.RunCount != nil || log.OperationStatus != "DONE" || log.Progress != 1 {t.Fatalf("unexpected dataset log state: %+v", log)}if log.ID != datasetCompileLogID("claim-1", log.TaskType) {t.Fatalf("dataset log id is not task-type scoped: %+v", log)}- if log.ProgressMsg == nil || !strings.Contains(*log.ProgressMsg, "2 affected page(s)") || !strings.Contains(*log.ProgressMsg, "completed") {- t.Fatalf("progress messages were not persisted: %v", log.ProgressMsg)+ if log.ProgressMsg != nil {+ t.Fatalf("dataset log retained progress_msg: %q", *log.ProgressMsg)}if log.DSL["task_type"] != log.TaskType {t.Fatalf("dataset log DSL task type = %v, want %s", log.DSL["task_type"], log.TaskType)@@ -63,6 +67,28 @@ func TestDatasetCompileLogLifecycle(t *testing.T) {if !ok || len(entries) != 1 {t.Fatalf("dataset log entries were not split by task type: %+v", log.DSL["entries"])}++ var events []entity.IngestionTaskLog+ if err := db.Where("pipeline_log_id = ?", log.ID).Order("id ASC").Find(&events).Error; err != nil {+ t.Fatalf("load dataset log events: %v", err)+ }+ if len(events) != 3 {+ t.Fatalf("dataset log %s event count = %d, want 3: %+v", log.ID, len(events), events)+ }+ if events[0].EventType != dao.EventTypeMessage || !strings.Contains(events[0].Message, "Created automatic "+log.TaskType+" dataset task") {+ t.Fatalf("unexpected initial event: %+v", events[0])+ }+ if events[1].EventType != dao.EventTypeMessage || !strings.Contains(events[1].Message, "2 affected page(s)") {+ t.Fatalf("unexpected progress event: %+v", events[1])+ }+ if events[2].EventType != dao.EventTypeTerminal || !strings.Contains(events[2].Message, "completed") {+ t.Fatalf("unexpected terminal event: %+v", events[2])+ }+ for _, event := range events {+ if event.TaskID != log.ID || event.PipelineLogID == nil || *event.PipelineLogID != log.ID {+ t.Fatalf("event is not scoped to dataset log %s: %+v", log.ID, event)+ }+ }}}diff --git a/internal/service/dataset/helpers.go b/internal/service/dataset/helpers.goindex eff2b8392d..1ccd24f05d 100644--- a/internal/service/dataset/helpers.go+++ b/internal/service/dataset/helpers.go@@ -56,7 +56,6 @@ var ()const (- graphRaptorQueueDocID = "graph_raptor_x"maximumTaskPageNumber = int64(100000000)serverQueueNamePrefix = "te"defaultEmbeddingCheckNum = 5diff --git a/internal/service/dataset/ingestion.go b/internal/service/dataset/ingestion.goindex 20b65431ca..39b63952a6 100644--- a/internal/service/dataset/ingestion.go+++ b/internal/service/dataset/ingestion.go@@ -93,11 +93,13 @@ func (d *DatasetService) ListIngestionMessages(ctx context.Context, datasetID, u}return nil, common.CodeServerError, fmt.Errorf("get ingestion log: %w", err)}- if run.RunCount == nil || *run.RunCount <= 0 {+ if !isReadableIngestionLog(run) {return nil, common.CodeDataError, errors.New("log not found")}response := &IngestionMessagesResponse{}- response.RunCount = *run.RunCount+ if run.RunCount != nil {+ response.RunCount = *run.RunCount+ }response.Terminal = isTerminalIngestionLogStatus(run.OperationStatus)page, err := dao.NewIngestionTaskLogDAO().ListEventsPageByPipelineLogID(ctx, dao.DB, logID, limit, afterID, beforeID)@@ -126,6 +128,16 @@ func isTerminalIngestionLogStatus(status string) bool {}}+func isReadableIngestionLog(log *entity.PipelineOperationLog) bool {+ if log == nil {+ return false+ }+ if log.DocumentID == entity.DatasetLogDocumentID {+ return log.RunCount == nil+ }+ return log.RunCount != nil && *log.RunCount > 0+}+func (d *DatasetService) ListIngestionLogs(ctx context.Context, datasetID, userID string, page, pageSize int, terms []dao.OrderTerm, operationStatus []string, createDateFrom, createDateTo, logType, keywords, documentID string) (map[string]interface{}, common.ErrorCode, error) {if datasetID == "" {return nil, common.CodeDataError, errors.New(`lack of "Dataset ID"`)@@ -202,7 +214,7 @@ func (d *DatasetService) GetIngestionLog(ctx context.Context, datasetID, userID,}return nil, common.CodeServerError, fmt.Errorf("get ingestion log: %w", err)}- if log.RunCount == nil || *log.RunCount <= 0 {+ if !isReadableIngestionLog(log) {return nil, common.CodeDataError, errors.New("log not found")}diff --git a/internal/service/dataset/ingestion_messages_test.go b/internal/service/dataset/ingestion_messages_test.goindex 1472665fd2..5411ef8bfc 100644--- a/internal/service/dataset/ingestion_messages_test.go+++ b/internal/service/dataset/ingestion_messages_test.go@@ -64,6 +64,55 @@ func TestListIngestionMessagesRejectsUnnumberedRun(t *testing.T) {}}+func TestDatasetIngestionLogUsesEventStream(t *testing.T) {+ db := setupServiceTestDB(t)+ pushServiceDB(t, db)+ if err := db.AutoMigrate(&entity.PipelineOperationLog{}); err != nil {+ t.Fatalf("migrate pipeline operation log: %v", err)+ }+ insertCompilationOwnerKB(t, "kb-1", "user-1")+ if err := db.Create(&entity.PipelineOperationLog{+ ID: "dataset-run",+ DocumentID: entity.DatasetLogDocumentID,+ TenantID: "user-1",+ KbID: "kb-1",+ ParserID: "knowledge_compile",+ DocumentName: "Wiki",+ DocumentType: "dataset",+ SourceFrom: "knowledgebase",+ TaskType: "Wiki",+ OperationStatus: "DONE",+ }).Error; err != nil {+ t.Fatalf("insert dataset run: %v", err)+ }+ insertMessageRun(t, "numbered-sentinel-run", "kb-1", entity.DatasetLogDocumentID, entity.TaskStatusDone, 1)+ insertMessageEvent(t, db, "dataset-run", "dataset-run", dao.EventTypeTerminal, "Knowledge compilation completed")++ result, code, err := NewDatasetService().ListIngestionLogs(t.Context(), "kb-1", "user-1", 1, 30, nil, nil, "", "", "dataset", "", "")+ if err != nil || code != common.CodeSuccess {+ t.Fatalf("ListIngestionLogs = (%+v, %v, %v), want success", result, code, err)+ }+ logs, ok := result["logs"].([]map[string]interface{})+ if !ok || len(logs) != 1 || logs[0]["id"] != "dataset-run" {+ t.Fatalf("dataset logs = %#v, want only unnumbered dataset run", result["logs"])+ }+ assertLatestEventMap(t, logs[0], 1, "Knowledge compilation completed")++ messages, code, err := NewDatasetService().ListIngestionMessages(t.Context(), "kb-1", "user-1", "dataset-run", 200, nil, nil)+ if err != nil || code != common.CodeSuccess {+ t.Fatalf("ListIngestionMessages = (%+v, %v, %v), want success", messages, code, err)+ }+ if messages.RunCount != 0 || !messages.Terminal || len(messages.Items) != 1 || messages.Items[0].Message != "Knowledge compilation completed" {+ t.Fatalf("dataset messages = %+v, want terminal event stream", messages)+ }++ log, code, err := NewDatasetService().GetIngestionLog(t.Context(), "kb-1", "user-1", "dataset-run")+ if err != nil || code != common.CodeSuccess {+ t.Fatalf("GetIngestionLog = (%+v, %v, %v), want success", log, code, err)+ }+ assertLatestEventMap(t, log, 1, "Knowledge compilation completed")+}+func TestListIngestionLogsExcludesUnnumberedRuns(t *testing.T) {db := setupServiceTestDB(t)pushServiceDB(t, db)