RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
fix: dataset ingestion log has no other type instead of success (#18605)
Haruko386a33ee7d4a09a
diff --git a/internal/ingestion/service/ingestion_service.go b/internal/ingestion/service/ingestion_service.goindex 46d882b824..d0a5b24dec 100644--- a/internal/ingestion/service/ingestion_service.go+++ b/internal/ingestion/service/ingestion_service.go@@ -599,17 +599,29 @@ func (e *Ingestor) runTask(ctx context.Context, task *entity.IngestionTask) boolcase <-ctx.Done():common.Info(fmt.Sprintf("Task %s cancelled", task.ID))e.markCancelProgress(task)- return e.markStopped(context.Background(), task.ID)+ stopped := e.markStopped(context.Background(), task.ID)+ if stopped {+ e.recordTerminalPipelineLog(context.Background(), task, string(entity.TaskStatusCancel))+ }+ return stoppeddefault:}if err := e.ingestionTaskSvc.IncrementRunCount(ctx, task.ID); err != nil {common.Error(fmt.Sprintf("Failed to increment run count for task %s", task.ID), err)- return e.markFailed(ctx, task.ID)+ ok := e.markFailed(ctx, task.ID)+ if ok {+ e.recordTerminalPipelineLog(ctx, task, string(entity.TaskStatusFail))+ }+ return ok}if err := e.ingestionTaskSvc.ClearComponentProgress(ctx, task.ID); err != nil {common.Error(fmt.Sprintf("Failed to clear previous component progress for task %s", task.ID), err)- return e.markFailed(ctx, task.ID)+ ok := e.markFailed(ctx, task.ID)+ if ok {+ e.recordTerminalPipelineLog(ctx, task, string(entity.TaskStatusFail))+ }+ return ok}// This is a new run (IncrementRunCount succeeded). Any Redis cancel flag@@ -630,15 +642,27 @@ func (e *Ingestor) runTask(ctx context.Context, task *entity.IngestionTask) boolif errors.Is(err, context.Canceled) {common.Info(fmt.Sprintf("Task %s cancelled during pipeline", task.ID))e.markCancelProgress(task)- return e.markStopped(ctx, task.ID)+ stopped := e.markStopped(ctx, task.ID)+ if stopped {+ e.recordTerminalPipelineLog(ctx, task, string(entity.TaskStatusCancel))+ }+ return stopped}if errors.Is(err, context.DeadlineExceeded) {common.Info(fmt.Sprintf("Task %s timed out during pipeline", task.ID))e.markTimeoutProgress(task)- return e.markFailed(ctx, task.ID)+ ok := e.markFailed(ctx, task.ID)+ if ok {+ e.recordTerminalPipelineLog(ctx, task, string(entity.TaskStatusFail))+ }+ return ok}common.Error(fmt.Sprintf("Task %s failed", task.ID), err)- return e.markFailed(ctx, task.ID)+ ok := e.markFailed(ctx, task.ID)+ if ok {+ e.recordTerminalPipelineLog(ctx, task, string(entity.TaskStatusFail))+ }+ return ok}if err := e.completeTask(ctx, task.ID); err != nil {@@ -981,6 +1005,20 @@ func (e *Ingestor) defaultRunDocumentTask(ctx context.Context, ingestionTask *enreturn nil}+func (e *Ingestor) recordTerminalPipelineLog(ctx context.Context, ingestionTask *entity.IngestionTask, status string) {+ if ingestionTask == nil || status == "" {+ return+ }+ ctx = context.WithoutCancel(ctx)+ if err := taskpkg.RecordPipelineLog(ctx, dao.DB, taskpkg.PipelineLogInput{+ KbID: ingestionTask.DatasetID,+ DocumentID: ingestionTask.DocumentID,+ Status: status,+ }); err != nil {+ common.Warn(fmt.Sprintf("record terminal pipeline log for task %s document %s: %v", ingestionTask.ID, ingestionTask.DocumentID, err))+ }+}+// Stop gracefully shuts down the ingestor. It cancels the root context so// idle workers exit immediately and in-flight pipelines abort at their next// ctx.Err() check, then waits for workers to return. The wait is bounded bydiff --git a/internal/ingestion/task/pipeline_executor.go b/internal/ingestion/task/pipeline_executor.goindex dfab1be12e..496b989018 100644--- a/internal/ingestion/task/pipeline_executor.go+++ b/internal/ingestion/task/pipeline_executor.go@@ -198,7 +198,7 @@ func (s *PipelineExecutor) Execute(ctx context.Context) (*PipelineResult, error)}if pipelineDSL != "" {- s.recordPipelineLog(ctx, dao.DB, s.taskCtx.Doc.ID, pipelineDSL, "done")+ s.recordPipelineLog(context.WithoutCancel(ctx), dao.DB, s.taskCtx.Doc.ID, pipelineDSL, "")}return result, nil@@ -628,22 +628,65 @@ func wikiActiveStates(output map[string]any) ([]kccommon.WikiMapActiveState, err}}-func (s *PipelineExecutor) recordPipelineLog(ctx context.Context, db *gorm.DB, docID, dsl, status string) {+// PipelineLogInput contains the identifiers and optional snapshots needed to+// persist a pipeline operation log without constructing a PipelineExecutor.+type PipelineLogInput struct {+ TenantID string+ KbID string+ DocumentID string+ PipelineID string+ DSL string+ Status string+ Document entity.Document+}++// RecordPipelineLog persists a pipeline operation log without requiring+// executor setup. Callers that already know a terminal state should pass it in+// Status; otherwise the writer falls back to the latest document.run value.+func RecordPipelineLog(ctx context.Context, db *gorm.DB, input PipelineLogInput) error {+ return recordPipelineLog(ctx, db, input, dao.NewPipelineOperationLogDAO().Create)+}++func recordPipelineLog(+ ctx context.Context,+ db *gorm.DB,+ input PipelineLogInput,+ createFunc func(ctx context.Context, db *gorm.DB, log *entity.PipelineOperationLog) error,+) error {var dslMap entity.JSONMap- if err := json.Unmarshal([]byte(dsl), &dslMap); err != nil {- dslMap = entity.JSONMap{"raw": dsl}+ if strings.TrimSpace(input.DSL) == "" {+ dslMap = entity.JSONMap{}+ } else if err := json.Unmarshal([]byte(input.DSL), &dslMap); err != nil {+ dslMap = entity.JSONMap{"raw": input.DSL}}// The task context contains the document snapshot loaded when the task// started. Reload it here so the operation log reflects the final progress// state written by the progress sink, matching the Python operation-log// creation path.- doc := s.taskCtx.Doc+ doc := input.Document+ if doc.ID == "" {+ doc.ID = input.DocumentID+ doc.KbID = input.KbID+ }if db != nil {- if persisted, err := dao.NewDocumentDAO().GetByID(ctx, db, docID); err == nil && persisted != nil {+ if persisted, err := dao.NewDocumentDAO().GetByID(ctx, db, input.DocumentID); err == nil && persisted != nil {doc = *persisted} else if err != nil {- common.Warn(fmt.Sprintf("failed to reload document %s for pipeline log: %v", docID, err))+ common.Warn(fmt.Sprintf("failed to reload document %s for pipeline log: %v", input.DocumentID, err))+ }+ }+ if input.KbID == "" {+ input.KbID = doc.KbID+ }+ if input.PipelineID == "" && doc.PipelineID != nil {+ input.PipelineID = strings.TrimSpace(*doc.PipelineID)+ }+ if input.TenantID == "" && db != nil && input.KbID != "" {+ if kb, err := dao.NewKnowledgebaseDAO().GetByID(ctx, db, input.KbID); err == nil && kb != nil {+ input.TenantID = kb.TenantID+ } else if err != nil {+ return fmt.Errorf("load knowledgebase %s for pipeline log: %w", input.KbID, err)}}@@ -655,22 +698,22 @@ func (s *PipelineExecutor) recordPipelineLog(ctx context.Context, db *gorm.DB, dpipelineTitle := doc.ParserIDpipelineAvatar := doc.Thumbnailvar pipelineID *string- if s.taskCtx.PipelineID != "" {- pipelineID = &s.canvasID- if db != nil {- if canvas, err := dao.NewUserCanvasDAO().GetByID(ctx, db, s.canvasID); err == nil && canvas != nil {+ if input.PipelineID != "" {+ pipelineID = &input.PipelineID+ if db != nil && strings.TrimSpace(input.DSL) != "" {+ if canvas, err := dao.NewUserCanvasDAO().GetByID(ctx, db, input.PipelineID); err == nil && canvas != nil {if canvas.Title != nil {pipelineTitle = *canvas.Title}pipelineAvatar = canvas.Avatar} else if err != nil && !errors.Is(err, dao.ErrUserCanvasNotFound) {- common.Warn(fmt.Sprintf("failed to reload pipeline %s for operation log: %v", s.canvasID, err))+ common.Warn(fmt.Sprintf("failed to reload pipeline %s for operation log: %v", input.PipelineID, err))}}}- operationStatus := status- if doc.Run != nil && *doc.Run != "" {+ operationStatus := input.Status+ if operationStatus == "" && doc.Run != nil && *doc.Run != "" {operationStatus = *doc.Run}statusValue := "1"@@ -687,9 +730,9 @@ func (s *PipelineExecutor) recordPipelineLog(ctx context.Context, db *gorm.DB, d}log := &entity.PipelineOperationLog{ID: utility.GenerateUUID(),- TenantID: s.Tenant().ID,- KbID: s.KB().ID,- DocumentID: docID,+ TenantID: input.TenantID,+ KbID: input.KbID,+ DocumentID: input.DocumentID,PipelineID: pipelineID,PipelineTitle: &pipelineTitle,TaskType: string(entity.PipelineTaskTypeParse),@@ -707,7 +750,23 @@ func (s *PipelineExecutor) recordPipelineLog(ctx context.Context, db *gorm.DB, dAvatar: pipelineAvatar,Status: &statusValue,}- if err := s.logCreateFunc(ctx, db, log); err != nil {+ return createFunc(ctx, db, log)+}++func (s *PipelineExecutor) recordPipelineLog(ctx context.Context, db *gorm.DB, docID, dsl, status string) {+ pipelineID := ""+ if s.taskCtx.PipelineID != "" {+ pipelineID = s.canvasID+ }+ if err := recordPipelineLog(ctx, db, PipelineLogInput{+ TenantID: s.Tenant().ID,+ KbID: s.KB().ID,+ DocumentID: docID,+ PipelineID: pipelineID,+ DSL: dsl,+ Status: status,+ Document: s.taskCtx.Doc,+ }, s.logCreateFunc); err != nil {common.Warn(fmt.Sprintf("failed to record pipeline log: %v", err))}}diff --git a/internal/ingestion/task/pipeline_executor_test.go b/internal/ingestion/task/pipeline_executor_test.goindex 6c94417b0f..ab50c9a7e6 100644--- a/internal/ingestion/task/pipeline_executor_test.go+++ b/internal/ingestion/task/pipeline_executor_test.go@@ -2,6 +2,7 @@ package taskimport ("context"+ "errors""sync""testing""time"@@ -315,6 +316,44 @@ func TestRecordPipelineLog_ValidJSONParsed(t *testing.T) {}}+func TestRecordPipelineLog_SharedWriterTerminalWithoutDSL(t *testing.T) {+ cleanup := setupPipelineExecutorTestDB(t)+ defer cleanup()++ docName := "terminal.pdf"+ run := "1"+ if err := RecordPipelineLog(t.Context(), dao.DB, PipelineLogInput{+ TenantID: "tenant-1",+ KbID: "kb-1",+ DocumentID: "doc-1",+ Status: "3",+ Document: entity.Document{+ ID: "doc-1",+ KbID: "kb-1",+ ParserID: "naive",+ ParserConfig: entity.JSONMap{},+ SourceType: "local",+ Type: "pdf",+ Name: &docName,+ Suffix: ".pdf",+ Run: &run,+ },+ }); err != nil {+ t.Fatalf("RecordPipelineLog: %v", err)+ }++ var log entity.PipelineOperationLog+ if err := dao.DB.First(&log, "document_id = ?", "doc-1").Error; err != nil {+ t.Fatalf("load pipeline log: %v", err)+ }+ if log.OperationStatus != "3" {+ t.Fatalf("OperationStatus = %q, want explicit terminal status", log.OperationStatus)+ }+ if len(log.DSL) != 0 {+ t.Fatalf("DSL = %v, want empty object for terminal writer without DSL", log.DSL)+ }+}+func TestRecordPipelineLog_BuiltinUsesParserIDFallback(t *testing.T) {cleanup := setupPipelineExecutorTestDB(t)defer cleanup()@@ -609,6 +648,50 @@ func TestPipelineExecutor_Run_MainFlowWithStubs(t *testing.T) {}}+func TestPipelineExecutor_Execute_DoesNotLogFailedRun(t *testing.T) {+ logged := false+ svc := mustNewPipelineExecutor(t, makeTaskCtx(), "flow-1", 0).+ WithLoadDSLFunc(func(ctx context.Context, canvasID string) (string, string, error) {+ return `{"nodes":[{"id":"n1"}],"edges":[]}`, canvasID, nil+ }).+ WithRunPipelineFunc(func(ctx context.Context, dsl string) (map[string]any, string, error) {+ return nil, dsl, errors.New("pipeline failed")+ }).+ WithLogCreateFunc(func(ctx context.Context, db *gorm.DB, log *entity.PipelineOperationLog) error {+ logged = true+ return nil+ })++ if _, err := svc.Execute(context.Background()); err == nil {+ t.Fatal("Execute error = nil, want failure")+ }+ if logged {+ t.Fatal("executor must not log failed runs before ingestor writes final document status")+ }+}++func TestPipelineExecutor_Execute_DoesNotLogCanceledRun(t *testing.T) {+ logged := false+ svc := mustNewPipelineExecutor(t, makeTaskCtx(), "flow-1", 0).+ WithLoadDSLFunc(func(ctx context.Context, canvasID string) (string, string, error) {+ return `{"nodes":[{"id":"n1"}],"edges":[]}`, canvasID, nil+ }).+ WithRunPipelineFunc(func(ctx context.Context, dsl string) (map[string]any, string, error) {+ return nil, dsl, context.Canceled+ }).+ WithLogCreateFunc(func(ctx context.Context, db *gorm.DB, log *entity.PipelineOperationLog) error {+ logged = true+ return nil+ })++ if _, err := svc.Execute(context.Background()); !errors.Is(err, context.Canceled) {+ t.Fatalf("Execute error = %v, want context.Canceled", err)+ }+ if logged {+ t.Fatal("executor must not log canceled runs before ingestor writes final document status")+ }+}+// TestPipelineExecutor_Execute_PropagatesContext verifies the ctx passed to// Execute is the ctx received by runPipelineFunc - the task context must flow// through to the pipeline run.diff --git a/internal/service/ingestion_task_service.go b/internal/service/ingestion_task_service.goindex 55674b9f78..60a8ff321b 100644--- a/internal/service/ingestion_task_service.go+++ b/internal/service/ingestion_task_service.go@@ -515,9 +515,8 @@ func (s *IngestionTaskService) lastRunCount(ctx context.Context, taskID string)// failure. ListAllForAdmin reads run_count back to render the attempt number.//// A corrupted run_count value in an existing row is skipped (the row is-// ignored). A failure to persist the new row is best-effort (logged) and-// does not return an error — matching the legacy semantics that the run-// proceeds even if the counter write fails.+// ignored). A failure to persist the new row is returned so the caller can+// fail the task before running the pipeline.func (s *IngestionTaskService) IncrementRunCount(ctx context.Context, taskID string) error {prevCount, _ := s.lastRunCount(ctx, taskID)@@ -525,8 +524,5 @@ func (s *IngestionTaskService) IncrementRunCount(ctx context.Context, taskID strTaskID: taskID,Checkpoint: entity.JSONMap{stepKeyRunCount: prevCount + 1},}- if err := s.ingestionTaskLogDAO.Create(ctx, dao.DB, entry); err != nil {- common.Error(fmt.Sprintf("Failed to persist run_count for task %s", taskID), err)- }- return nil+ return s.ingestionTaskLogDAO.Create(ctx, dao.DB, entry)}