Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

feat(ingestion): track a run's pipeline log from queue to terminal (#19494)

jay77721aa4061471a85
diff --git a/internal/dao/database.go b/internal/dao/database.go
index 940ed7bfbf..2098f14f3c 100644
--- a/internal/dao/database.go
+++ b/internal/dao/database.go
@@ -187,6 +187,13 @@ func InitDB(ctx context.Context, migrateDB bool) error {
return fmt.Errorf("failed to auto-migrate runtime models: %w", err)
}
}
+ // ingestion_task.pipeline_log_id cannot be added by AutoMigrate (see the
+ // helper for why), and every ingestion_task query selects all columns, so a
+ // missing column fails the whole API with Error 1054. Ensure it on both
+ // startup paths rather than trusting AutoMigrate.
+ if err = migrateIngestionTaskPipelineLogID(ctx, DB); err != nil {
+ return err
+ }
// Seed built-in agent templates so the Go backend can serve the
// "create agent from template" catalogue without relying on Python-side
// initialization.
diff --git a/internal/dao/database_test.go b/internal/dao/database_test.go
index 6214b6a4cf..d3fa2d1b1e 100644
--- a/internal/dao/database_test.go
+++ b/internal/dao/database_test.go
@@ -101,6 +101,54 @@ func TestAutoMigrateRuntimeModelsCreatesGoRuntimeTables(t *testing.T) {
}
}
+// TestMigrateIngestionTaskPipelineLogID covers the upgrade path AutoMigrate
+// cannot handle on MySQL: an existing ingestion_task created before
+// pipeline_log_id existed must gain the column, and running the migration again
+// must be a no-op.
+func TestMigrateIngestionTaskPipelineLogID(t *testing.T) {
+ db, err := gorm.Open(sqlite.Open(":memory:"), &gorm.Config{})
+ if err != nil {
+ t.Fatalf("open sqlite: %v", err)
+ }
+ ctx := context.Background()
+
+ // An ingestion_task table without the column, as created by an older build.
+ if err := db.Exec(`CREATE TABLE ingestion_task (
+ id varchar(32) PRIMARY KEY,
+ user_id varchar(32) NOT NULL,
+ document_id varchar(32) NOT NULL,
+ dataset_id varchar(32) NOT NULL,
+ status varchar(32) NOT NULL
+ )`).Error; err != nil {
+ t.Fatalf("create legacy table: %v", err)
+ }
+ migrator := db.Migrator()
+ if migrator.HasColumn("ingestion_task", "pipeline_log_id") {
+ t.Fatal("precondition failed: column already present")
+ }
+
+ if err := migrateIngestionTaskPipelineLogID(ctx, db); err != nil {
+ t.Fatalf("migrateIngestionTaskPipelineLogID: %v", err)
+ }
+ if !migrator.HasColumn("ingestion_task", "pipeline_log_id") {
+ t.Fatal("expected pipeline_log_id to be added")
+ }
+
+ // Idempotent: a second run must not fail on the existing column.
+ if err := migrateIngestionTaskPipelineLogID(ctx, db); err != nil {
+ t.Fatalf("second migrateIngestionTaskPipelineLogID: %v", err)
+ }
+
+ // Missing table is a no-op, not an error.
+ fresh, err := gorm.Open(sqlite.Open(":memory:"), &gorm.Config{})
+ if err != nil {
+ t.Fatalf("open second sqlite: %v", err)
+ }
+ if err := migrateIngestionTaskPipelineLogID(ctx, fresh); err != nil {
+ t.Fatalf("migration on a table-less database should be a no-op: %v", err)
+ }
+}
+
func TestAutoMigrateRuntimeModelsNamesFailingTable(t *testing.T) {
db, err := gorm.Open(sqlite.Open(":memory:"), &gorm.Config{})
if err != nil {
diff --git a/internal/dao/ingestion.go b/internal/dao/ingestion.go
index 6ac2cef08b..7cdf9e223e 100644
--- a/internal/dao/ingestion.go
+++ b/internal/dao/ingestion.go
@@ -25,6 +25,7 @@ import (
"ragflow/internal/utility"
"gorm.io/gorm"
+ "gorm.io/gorm/clause"
)
type IngestionTaskDAO struct{}
@@ -75,6 +76,14 @@ func (dao *IngestionTaskDAO) UpdateComponentTotal(ctx context.Context, db *gorm.
return db.WithContext(ctx).Model(&entity.IngestionTask{}).Where("id = ?", taskID).Update("component_total", total).Error
}
+// UpdatePipelineLogID binds the task to the pipeline_operation_log row its
+// current run owns. The terminal writer updates exactly that row, so a
+// superseded run whose row was deleted or replaced cannot adopt the
+// replacement run's row.
+func (dao *IngestionTaskDAO) UpdatePipelineLogID(ctx context.Context, db *gorm.DB, taskID, logID string) error {
+ return db.WithContext(ctx).Model(&entity.IngestionTask{}).Where("id = ?", taskID).Update("pipeline_log_id", logID).Error
+}
+
type TaskInfo struct {
TaskID string `json:"task_id"`
FilesToDelete []string `json:"files_to_delete"`
@@ -197,6 +206,18 @@ func (dao *IngestionTaskDAO) GetByID(ctx context.Context, db *gorm.DB, id string
return task, err
}
+// GetByIDForUpdate fetches and locks a task for a short ownership-establishment
+// transaction. Callers must pass a transaction and keep metadata lookups and
+// message publishing outside the lock.
+func (dao *IngestionTaskDAO) GetByIDForUpdate(ctx context.Context, db *gorm.DB, id string) (*entity.IngestionTask, error) {
+ var task *entity.IngestionTask
+ err := db.WithContext(ctx).
+ Clauses(clause.Locking{Strength: "UPDATE"}).
+ Where("id = ?", id).
+ First(&task).Error
+ return task, err
+}
+
// GetByDocumentID returns the latest ingestion task for a document. Historical
// retries are ordered by create_time and then ID to match document-list state.
func (dao *IngestionTaskDAO) GetByDocumentID(ctx context.Context, db *gorm.DB, documentId string) (*entity.IngestionTask, error) {
diff --git a/internal/dao/migration.go b/internal/dao/migration.go
index cc5a890e4a..a99af13081 100644
--- a/internal/dao/migration.go
+++ b/internal/dao/migration.go
@@ -264,6 +264,44 @@ func migrateIngestionTaskDocumentIDUnique(ctx context.Context, db *gorm.DB) erro
return nil
}
+// migrateIngestionTaskPipelineLogID adds ingestion_task.pipeline_log_id when
+// the column is missing.
+//
+// It cannot be left to AutoMigrate. The model declares document_id's unique key
+// as a named unique index (uniqueIndex:idx_ingestion_task_document_id), while
+// the column is UNIQUE at the database level. GORM's MigrateColumnUnique reads
+// that pairing as a stray unique constraint: it issues
+// `ALTER TABLE ingestion_task DROP FOREIGN KEY uni_ingestion_task_document_id`
+// for the default constraint name, which does not exist, so MySQL fails with
+// 1091. AutoMigrate aborts on the first error, and because autoMigrateSafely
+// treats 1091 as a benign "already dropped" case, the failure is swallowed and
+// the migration reports success -- before ever reaching the missing column.
+// A column added to this table would therefore never be created, and every
+// ingestion_task query (the DAO selects all columns) would fail with
+// Error 1054.
+func migrateIngestionTaskPipelineLogID(ctx context.Context, db *gorm.DB) error {
+ // Use the model, not the bare table name: HasColumn resolves the field
+ // against the statement schema and dereferences it, which a plain string
+ // does not provide (the SQLite migrator tolerates it, the MySQL one panics).
+ migrator := db.WithContext(ctx).Migrator()
+ if !migrator.HasTable(&entity.IngestionTask{}) {
+ return nil
+ }
+ if migrator.HasColumn(&entity.IngestionTask{}, "pipeline_log_id") {
+ return nil
+ }
+ if err := db.WithContext(ctx).Exec(
+ "ALTER TABLE ingestion_task ADD COLUMN pipeline_log_id varchar(32) NULL",
+ ).Error; err != nil {
+ if strings.Contains(err.Error(), "Error 1060") && strings.Contains(err.Error(), "Duplicate column name") {
+ return nil
+ }
+ return fmt.Errorf("failed to add ingestion_task.pipeline_log_id: %w", err)
+ }
+ common.Info("Added ingestion_task.pipeline_log_id")
+ return nil
+}
+
// migrateKnowledgebaseNameUnique adds a case-insensitive unique constraint on
// (tenant_id, name) for valid knowledge bases. A VIRTUAL generated column
// (name_ci) computes LOWER(name) only for status='1' rows and is NULL otherwise,
diff --git a/internal/dao/pipeline_operation_log.go b/internal/dao/pipeline_operation_log.go
index bd3884b2ea..ec13661f1f 100644
--- a/internal/dao/pipeline_operation_log.go
+++ b/internal/dao/pipeline_operation_log.go
@@ -19,8 +19,11 @@ package dao
import (
"context"
"strings"
+ "time"
+ "ragflow/internal/common"
"ragflow/internal/entity"
+ "ragflow/internal/utility"
"gorm.io/gorm"
)
@@ -102,13 +105,20 @@ func NewPipelineOperationLogDAO() *PipelineOperationLogDAO {
// GetDatasetLogsByKBID lists dataset-level (graph/raptor/mindmap) ingestion
// logs for a knowledge base. Pagination is only applied when both page and
// pageSize are positive, matching peewee's paginate behavior.
-func (dao *PipelineOperationLogDAO) GetDatasetLogsByKBID(ctx context.Context, db *gorm.DB, kbID string, page, pageSize int, orderby string, desc bool, operationStatus []string, createDateFrom, createDateTo, keywords string) ([]*entity.PipelineOperationLog, int64, error) {
+//
+// documentID is honoured for the same reason as in GetFileLogsByKBID. Dataset
+// logs belong to no single document, so a caller that names one gets an empty
+// list rather than the whole dataset history.
+func (dao *PipelineOperationLogDAO) GetDatasetLogsByKBID(ctx context.Context, db *gorm.DB, kbID string, page, pageSize int, orderby string, desc bool, operationStatus []string, createDateFrom, createDateTo, keywords, documentID string) ([]*entity.PipelineOperationLog, int64, error) {
query := db.WithContext(ctx).Model(&entity.PipelineOperationLog{}).
Where("kb_id = ? AND document_id = ?", kbID, graphRaptorFakeDocID)
if keywords != "" {
query = query.Where("LOWER(document_name) LIKE ?", "%"+strings.ToLower(keywords)+"%")
}
+ if documentID != "" {
+ query = query.Where("document_id = ?", documentID)
+ }
if len(operationStatus) > 0 {
query = query.Where("operation_status IN ?", normalizePipelineOperationStatuses(operationStatus))
}
@@ -142,13 +152,21 @@ func (dao *PipelineOperationLogDAO) GetDatasetLogsByKBID(ctx context.Context, db
}
// GetFileLogsByKBID lists per-file ingestion logs for a knowledge base.
-func (dao *PipelineOperationLogDAO) GetFileLogsByKBID(ctx context.Context, db *gorm.DB, kbID string, page, pageSize int, orderby string, desc bool, keywords string, operationStatus []string, createDateFrom, createDateTo string) ([]*entity.PipelineOperationLog, int64, error) {
+//
+// documentID narrows the list to one document exactly. The frontend needs it to
+// resolve a queued document's early row: matching by document_name is a fuzzy
+// LIKE search that can push the row out of the first page when several
+// documents share a name.
+func (dao *PipelineOperationLogDAO) GetFileLogsByKBID(ctx context.Context, db *gorm.DB, kbID string, page, pageSize int, orderby string, desc bool, keywords, documentID string, operationStatus []string, createDateFrom, createDateTo string) ([]*entity.PipelineOperationLog, int64, error) {
query := db.WithContext(ctx).Model(&entity.PipelineOperationLog{}).
Where("kb_id = ?", kbID)
if keywords != "" {
query = query.Where("LOWER(document_name) LIKE ?", "%"+strings.ToLower(keywords)+"%")
}
+ if documentID != "" {
+ query = query.Where("document_id = ?", documentID)
+ }
query = query.Where("document_id <> ?", graphRaptorFakeDocID)
if len(operationStatus) > 0 {
@@ -183,6 +201,174 @@ func (dao *PipelineOperationLogDAO) GetFileLogsByKBID(ctx context.Context, db *g
return logs, count, nil
}
+// OpenPipelineOperationStatuses returns the operation_status values of a
+// pipeline operation log row that a run is still moving through. A run owns
+// exactly one such row from CREATED to its terminal write (DONE/FAIL/CANCEL),
+// so later stages advance the same row instead of inserting a new one. It is a
+// function, not a shared slice, so no caller can mutate the set.
+func OpenPipelineOperationStatuses() []string {
+ return []string{
+ string(entity.TaskStatusUnstart),
+ string(entity.TaskStatusSchedule),
+ string(entity.TaskStatusRunning),
+ }
+}
+
+// GetOpenLogByDocumentID returns the newest open pipeline operation log for a
+// document, or nil when the document has no in-flight run.
+//
+// It locates a row; it deliberately does not decide what may happen to it. The
+// create path drops whatever it finds (dropping a leftover from a run that no
+// longer exists), and the terminal writer adopts it only when the caller has no
+// bound row of its own. Whether a *terminal* write may touch a row is bound to
+// the run's own row id (ingestion_task.pipeline_log_id), so a superseded run
+// cannot reach the replacement run's row.
+func (dao *PipelineOperationLogDAO) GetOpenLogByDocumentID(ctx context.Context, db *gorm.DB, documentID string) (*entity.PipelineOperationLog, error) {
+ var log entity.PipelineOperationLog
+ err := db.WithContext(ctx).
+ Where("document_id = ? AND operation_status IN ?", documentID, OpenPipelineOperationStatuses()).
+ Order("create_time DESC").
+ Order("id DESC").
+ First(&log).Error
+ if err != nil {
+ if IsNotFoundErr(err) {
+ return nil, nil
+ }
+ return nil, err
+ }
+ return &log, nil
+}
+
+// OpenLogInput carries the bookkeeping needed to open or advance the
+// pre-terminal row for a queued run. It mirrors the PipelineOperationLog
+// columns that are known before the pipeline finishes; the DSL and the final
+// progress/counters stay empty until the terminal writer fills them in.
+type OpenLogInput struct {
+ DocumentID string
+ KbID string
+ TenantID string
+ PipelineID string
+ PipelineTitle string
+ ParserID string
+ DocumentName string
+ DocumentSuffix string
+ DocumentType string
+ SourceFrom string
+ Avatar *string
+ OperationStatus string
+ ProgressMsg string
+}
+
+// CreateOpenLog opens the pre-terminal row for a queued run. Callers must
+// have verified no open row exists (or accept a best-effort duplicate on a
+// lost race); the error is returned so tests can assert it, while production
+// callers log and continue.
+func (dao *PipelineOperationLogDAO) CreateOpenLog(ctx context.Context, db *gorm.DB, input OpenLogInput) (*entity.PipelineOperationLog, error) {
+ now := time.Now().Local()
+ msg := input.ProgressMsg
+ log := &entity.PipelineOperationLog{
+ ID: utility.GenerateUUID(),
+ DocumentID: input.DocumentID,
+ TenantID: input.TenantID,
+ KbID: input.KbID,
+ ParserID: input.ParserID,
+ DocumentName: input.DocumentName,
+ DocumentSuffix: input.DocumentSuffix,
+ DocumentType: input.DocumentType,
+ SourceFrom: input.SourceFrom,
+ Progress: 0,
+ ProgressMsg: &msg,
+ ProcessBeginAt: &now,
+ ProcessDuration: 0,
+ DSL: entity.JSONMap{},
+ TaskType: string(entity.PipelineTaskTypeParse),
+ OperationStatus: input.OperationStatus,
+ Avatar: input.Avatar,
+ }
+ statusValue := "1"
+ log.Status = &statusValue
+ if input.PipelineID != "" {
+ pipelineID := input.PipelineID
+ log.PipelineID = &pipelineID
+ }
+ if input.PipelineTitle != "" {
+ title := input.PipelineTitle
+ log.PipelineTitle = &title
+ }
+ if err := db.WithContext(ctx).Create(log).Error; err != nil {
+ return nil, err
+ }
+ return log, nil
+}
+
+// AdvanceOpenLog moves a run's own row to a later status, refreshing its
+// progress message. The row is targeted by id, so it can never touch another
+// run's row, and the update is guarded by the from-states the caller declares:
+// the transitions are monotonic (unstart -> schedule -> running), so a late
+// queued write cannot regress a row a concurrent writer already advanced.
+// A row that already left the declared from-states is left untouched.
+func (dao *PipelineOperationLogDAO) AdvanceOpenLog(ctx context.Context, db *gorm.DB, logID string, fromStatuses []string, operationStatus, progressMsg string) error {
+ if logID == "" || len(fromStatuses) == 0 {
+ return nil
+ }
+ return db.WithContext(ctx).Model(&entity.PipelineOperationLog{}).
+ Where("id = ? AND operation_status IN ?", logID, fromStatuses).
+ Updates(map[string]interface{}{
+ "operation_status": operationStatus,
+ "progress_msg": progressMsg,
+ }).Error
+}
+
+// DeleteOpenLogByID removes a run's own pre-terminal row. Used to clean up the
+// open row when a task is rolled back or removed, so the detail page is not
+// left with a permanently queued entry. Deleting by id (rather than by
+// document) keeps the cleanup away from a newer run's row, and the open-status
+// guard keeps it away from a terminal row, which is history.
+func (dao *PipelineOperationLogDAO) DeleteOpenLogByID(ctx context.Context, db *gorm.DB, logID string) error {
+ if logID == "" {
+ return nil
+ }
+ return db.WithContext(ctx).Model(&entity.PipelineOperationLog{}).
+ Where("id = ? AND operation_status IN ?", logID, OpenPipelineOperationStatuses()).
+ Delete(&entity.PipelineOperationLog{}).Error
+}
+
+// DeleteUnownedOpenLogByID removes an open row only when no live ingestion
+// task owns it. Existing databases may contain more than one task per document,
+// so document identity alone is not sufficient proof that a row is leftover.
+func (dao *PipelineOperationLogDAO) DeleteUnownedOpenLogByID(ctx context.Context, db *gorm.DB, logID string) (bool, error) {
+ if logID == "" {
+ return false, nil
+ }
+ result := db.WithContext(ctx).Model(&entity.PipelineOperationLog{}).
+ Where("id = ? AND operation_status IN ?", logID, OpenPipelineOperationStatuses()).
+ Where("NOT EXISTS (SELECT 1 FROM ingestion_task WHERE pipeline_log_id = ? AND status IN ?)", logID, common.ActiveTaskStatuses).
+ Delete(&entity.PipelineOperationLog{})
+ if result.Error != nil {
+ return false, result.Error
+ }
+ return result.RowsAffected == 1, nil
+}
+
+// HasLiveTaskOwner reports whether a live ingestion task is bound to the given
+// pipeline operation log row. Adoption of an open row is gated on this: the
+// create path never takes a row a live run owns, and the terminal fallback must
+// apply the same rule, or it would finalize the live run's entry with another
+// run's status and swallow the live run's own terminal write.
+func (dao *PipelineOperationLogDAO) HasLiveTaskOwner(ctx context.Context, db *gorm.DB, logID string) (bool, error) {
+ if logID == "" {
+ return false, nil
+ }
+ var count int64
+ err := db.WithContext(ctx).Model(&entity.IngestionTask{}).
+ Where("pipeline_log_id = ? AND status IN ?", logID, common.ActiveTaskStatuses).
+ Count(&count).Error
+ if err != nil {
+ return false, err
+ }
+ return count > 0, nil
+}
+
// GetByIDAndKBID fetches a single ingestion log scoped to its knowledge base.
func (dao *PipelineOperationLogDAO) GetByIDAndKBID(ctx context.Context, db *gorm.DB, logID, kbID string) (*entity.PipelineOperationLog, error) {
var log entity.PipelineOperationLog
diff --git a/internal/entity/ingestion_task.go b/internal/entity/ingestion_task.go
index 8676448466..003bff98c2 100644
--- a/internal/entity/ingestion_task.go
+++ b/internal/entity/ingestion_task.go
@@ -28,6 +28,14 @@ type IngestionTask struct {
// frontend does not have to count DSL nodes itself. Written once the
// pipeline compiles the canvas (see pipeline.Run).
ComponentTotal int `gorm:"column:component_total;default:0" json:"component_total"`
+ // PipelineLogID is the id of the pipeline_operation_log row the task's
+ // current run owns. The service opens that row when the run is queued and
+ // binds it here, so the running advance and the terminal write update
+ // exactly this row instead of adopting whichever row happens to be open for
+ // the document. A superseded run keeps the id of its (deleted) row, so its
+ // late terminal write cannot touch the replacement run's row. nil for runs
+ // that never opened one.
+ PipelineLogID *string `gorm:"column:pipeline_log_id;size:32" json:"-"`
BaseModel
}
diff --git a/internal/handler/dataset.go b/internal/handler/dataset.go
index d9d6b3cdcd..d0f341cdab 100644
--- a/internal/handler/dataset.go
+++ b/internal/handler/dataset.go
@@ -548,10 +548,13 @@ func (h *DatasetsHandler) ListIngestionLogs(c *gin.Context) {
createDateTo := c.Query("create_date_to")
logType := c.DefaultQuery("log_type", "dataset")
keywords := c.Query("keywords")
+ // Exact per-document filter for the file-log list. Python's endpoint has no
+ // equivalent; the frontend only sends it on the Go backend.
+ documentID := c.Query("document_id")
ctx := c.Request.Context()
- result, code, err := h.datasetsService.ListIngestionLogs(ctx, datasetID, user.ID, page, pageSize, orderby, desc, operationStatus, createDateFrom, createDateTo, logType, keywords)
+ result, code, err := h.datasetsService.ListIngestionLogs(ctx, datasetID, user.ID, page, pageSize, orderby, desc, operationStatus, createDateFrom, createDateTo, logType, keywords, documentID)
if err != nil {
common.ErrorWithCode(c, code, err.Error())
return
diff --git a/internal/ingestion/service/ingestion_service.go b/internal/ingestion/service/ingestion_service.go
index 53e18fdad8..ac3e2c6796 100644
--- a/internal/ingestion/service/ingestion_service.go
+++ b/internal/ingestion/service/ingestion_service.go
@@ -1192,11 +1192,18 @@ func (e *Ingestor) recordTerminalPipelineLog(ctx context.Context, ingestionTask
return
}
ctx = context.WithoutCancel(ctx)
- if err := taskpkg.RecordPipelineLog(ctx, dao.DB, taskpkg.PipelineLogInput{
+ input := taskpkg.PipelineLogInput{
KbID: ingestionTask.DatasetID,
DocumentID: ingestionTask.DocumentID,
Status: status,
- }); err != nil {
+ }
+ // Bind the write to this run's own row. A superseded run keeps the id of
+ // its (deleted or replaced) row, so its late terminal write can never reach
+ // into the replacement run's row.
+ if ingestionTask.PipelineLogID != nil {
+ input.PipelineLogID = *ingestionTask.PipelineLogID
+ }
+ if err := taskpkg.RecordPipelineLog(ctx, dao.DB, input); err != nil {
common.Warn(fmt.Sprintf("record terminal pipeline log for task %s document %s: %v", ingestionTask.ID, ingestionTask.DocumentID, err))
}
}
diff --git a/internal/ingestion/task/pipeline_executor.go b/internal/ingestion/task/pipeline_executor.go
index 5965060991..819afccaf2 100644
--- a/internal/ingestion/task/pipeline_executor.go
+++ b/internal/ingestion/task/pipeline_executor.go
@@ -769,10 +769,21 @@ type PipelineLogInput struct {
DSL string
Status string
Document entity.Document
+ // PipelineLogID is the id of the pipeline_operation_log row this run owns
+ // (ingestion_task.pipeline_log_id). When set, the terminal write updates
+ // exactly that row and never creates a second one — so a superseded run
+ // whose row was deleted cannot adopt the replacement run's row. Empty for
+ // legacy, debug-adjacent, or non-ingestion callers.
+ PipelineLogID string
}
// RecordPipelineLog persists a pipeline operation log without requiring
// executor setup. Callers should pass the status in Status.
+//
+// When the run created a queued row (CREATED/SCHEDULED), the terminal write
+// advances that same row so the dataset detail page shows one entry per run.
+// Without a bound row (legacy runs, debug-adjacent paths) it adopts the
+// document's open row, or creates a new row when none exists.
func RecordPipelineLog(ctx context.Context, db *gorm.DB, input PipelineLogInput) error {
return recordPipelineLog(ctx, db, input, dao.NewPipelineOperationLogDAO().Create)
}
@@ -858,6 +869,13 @@ func recordPipelineLog(
if doc.Name != nil {
documentName = *doc.Name
}
+ if db != nil {
+ if handled, err := updateOpenLogRow(ctx, db, input, operationStatus, statusValue, pipelineID, pipelineTitle, pipelineAvatar, dslMap, doc); err != nil {
+ common.Warn(fmt.Sprintf("failed to advance open pipeline log for document %s: %v", input.DocumentID, err))
+ } else if handled {
+ return nil
+ }
+ }
log := &entity.PipelineOperationLog{
ID: utility.GenerateUUID(),
TenantID: input.TenantID,
@@ -883,19 +901,98 @@ func recordPipelineLog(
return createFunc(ctx, db, log)
}
+// updateOpenLogRow advances the row this run already owns to its terminal
+// state, filling in the DSL, the pipeline identity, and the final progress
+// snapshot. It reports whether the caller must stop (true) or insert a new row
+// (false).
+//
+// The row is targeted by PipelineLogID when the caller has one, so a superseded run
+// whose row was deleted with the task can never reach into the replacement
+// run's row. Only when the caller has no bound row (legacy, debug-adjacent, or
+// non-ingestion callers) does it adopt the document's newest unowned open row,
+// so a stray open row is closed rather than left queued; a row a live run owns
+// is never adopted, because that run's own terminal write would then be lost.
+// RowsAffected is not used to decide whether to create: once a target row is
+// known, the write is final — a lost CAS means another writer already finalized
+// this run or the row was dropped with a superseded run, and neither may
+// produce a second entry.
+func updateOpenLogRow(ctx context.Context, db *gorm.DB, input PipelineLogInput, operationStatus, statusValue string, pipelineID *string, pipelineTitle string, pipelineAvatar *string, dslMap entity.JSONMap, doc entity.Document) (bool, error) {
+ targetID := input.PipelineLogID
+ if targetID == "" {
+ open, err := dao.NewPipelineOperationLogDAO().GetOpenLogByDocumentID(ctx, db, input.DocumentID)
+ if err != nil {
+ return false, err
+ }
+ if open == nil {
+ return false, nil
+ }
+ owned, err := dao.NewPipelineOperationLogDAO().HasLiveTaskOwner(ctx, db, open.ID)
+ if err != nil {
+ return false, err
+ }
+ if owned {
+ return false, nil
+ }
+ targetID = open.ID
+ }
+ updates := map[string]interface{}{
+ "operation_status": operationStatus,
+ "status": statusValue,
+ "dsl": dslMap,
+ "pipeline_id": pipelineID,
+ "pipeline_title": pipelineTitle,
+ "avatar": pipelineAvatar,
+ "progress": doc.Progress,
+ "progress_msg": doc.ProgressMsg,
+ "process_duration": doc.ProcessDuration,
+ "parser_id": doc.ParserID,
+ "source_from": strings.SplitN(doc.SourceType, "/", 2)[0],
+ "document_suffix": doc.Suffix,
+ "document_type": doc.Type,
+ }
+ if doc.Name != nil {
+ updates["document_name"] = *doc.Name
+ }
+ // The open row was created with a timestamp. Keep it when the reloaded
+ // document carries none (a run that never reached the progress sink), so
+ // the queued entry does not lose its start time.
+ if doc.ProcessBeginAt != nil {
+ updates["process_begin_at"] = doc.ProcessBeginAt
+ }
+ result := db.WithContext(ctx).Model(&entity.PipelineOperationLog{}).
+ Where("id = ? AND operation_status IN ?", targetID, dao.OpenPipelineOperationStatuses()).
+ Updates(updates)
+ if result.Error != nil {
+ return false, result.Error
+ }
+ // No second entry is created for a known target, so a row that is gone or
+ // already finalized means this run's outcome is not recorded anywhere. That
+ // is the intended outcome for a run whose row was dropped with a superseded
+ // task; log it so the missing entry is not silent.
+ if result.RowsAffected == 0 {
+ common.Warn(fmt.Sprintf("pipeline log %s for document %s is gone or already terminal; %s entry dropped", targetID, input.DocumentID, operationStatus))
+ }
+ return true, nil
+}
+
func (s *PipelineExecutor) recordPipelineLog(ctx context.Context, db *gorm.DB, docID, dsl, status string) {
pipelineID := ""
if s.taskCtx.PipelineID != "" {
pipelineID = s.canvasID
}
+ pipelineLogID := ""
+ if s.taskCtx.IngestionTask != nil && s.taskCtx.IngestionTask.PipelineLogID != nil {
+ pipelineLogID = *s.taskCtx.IngestionTask.PipelineLogID
+ }
if err := recordPipelineLog(ctx, db, PipelineLogInput{
- TenantID: s.Tenant().ID,
- KbID: s.KB().ID,
- DocumentID: docID,
- PipelineID: pipelineID,
- DSL: dsl,
- Status: status,
- Document: s.taskCtx.Doc,
+ TenantID: s.Tenant().ID,
+ KbID: s.KB().ID,
+ DocumentID: docID,
+ PipelineID: pipelineID,
+ DSL: dsl,
+ Status: status,
+ Document: s.taskCtx.Doc,
+ PipelineLogID: pipelineLogID,
}, s.logCreateFunc); err != nil {
common.Warn(fmt.Sprintf("failed to record pipeline log: %v", err))
}
diff --git a/internal/ingestion/task/pipeline_executor_test.go b/internal/ingestion/task/pipeline_executor_test.go
index a1006f5b3a..d109033ee4 100644
--- a/internal/ingestion/task/pipeline_executor_test.go
+++ b/internal/ingestion/task/pipeline_executor_test.go
@@ -148,7 +148,7 @@ func setupPipelineExecutorTestDB(t *testing.T) func() {
if err != nil {
t.Fatalf("open sqlite: %v", err)
}
- if err := db.AutoMigrate(&entity.UserCanvas{}, &entity.PipelineOperationLog{}, &entity.Document{}); err != nil {
+ if err := db.AutoMigrate(&entity.UserCanvas{}, &entity.PipelineOperationLog{}, &entity.Document{}, &entity.IngestionTask{}); err != nil {
t.Fatalf("auto-migrate sqlite: %v", err)
}
origDB := dao.DB
@@ -631,6 +631,386 @@ func TestRecordPipelineLog_SourceFromReloadedDoc(t *testing.T) {
}
}
+func TestRecordPipelineLog_ReusesOpenPreTerminalRow(t *testing.T) {
+ cleanup := setupPipelineExecutorTestDB(t)
+ defer cleanup()
+
+ queuedMsg := "Task is queued..."
+ openLog := &entity.PipelineOperationLog{
+ ID: "open-log",
+ DocumentID: "doc-1",
+ TenantID: "tenant-1",
+ KbID: "kb-1",
+ ParserID: "naive",
+ DocumentName: "test-doc.pdf",
+ DocumentSuffix: ".pdf",
+ DocumentType: "pdf",
+ SourceFrom: "local",
+ TaskType: "Parse",
+ OperationStatus: "5",
+ ProgressMsg: &queuedMsg,
+ }
+ if err := dao.DB.Create(openLog).Error; err != nil {
+ t.Fatalf("seed open log: %v", err)
+ }
+
+ progress := 1.0
+ finalMsg := "Parser Done"
+ if err := dao.DB.Create(&entity.Document{
+ ID: "doc-1",
+ KbID: "kb-1",
+ ParserID: "naive",
+ ParserConfig: entity.JSONMap{},
+ SourceType: "local",
+ Type: "pdf",
+ CreatedBy: "tenant-1",
+ Name: strPtr("test-doc.pdf"),
+ Suffix: ".pdf",
+ Progress: progress,
+ ProgressMsg: &finalMsg,
+ }).Error; err != nil {
+ t.Fatalf("seed document: %v", err)
+ }
+
+ if err := RecordPipelineLog(t.Context(), dao.DB, PipelineLogInput{
+ TenantID: "tenant-1",
+ KbID: "kb-1",
+ DocumentID: "doc-1",
+ Status: "3",
+ PipelineLogID: "open-log",
+ }); err != nil {
+ t.Fatalf("RecordPipelineLog: %v", err)
+ }
+
+ var count int64
+ if err := dao.DB.Model(&entity.PipelineOperationLog{}).Where("document_id = ?", "doc-1").Count(&count).Error; err != nil {
+ t.Fatalf("count pipeline logs: %v", err)
+ }
+ if count != 1 {
+ t.Fatalf("pipeline log rows = %d, want 1 (terminal must reuse the bound queued row)", count)
+ }
+ var log entity.PipelineOperationLog
+ if err := dao.DB.First(&log, "id = ?", "open-log").Error; err != nil {
+ t.Fatalf("load open log: %v", err)
+ }
+ if log.OperationStatus != "3" {
+ t.Fatalf("OperationStatus = %q, want terminal status", log.OperationStatus)
+ }
+ if log.Progress != progress {
+ t.Fatalf("Progress = %v, want %v", log.Progress, progress)
+ }
+ if log.ProgressMsg == nil || *log.ProgressMsg != finalMsg {
+ t.Fatalf("ProgressMsg = %v, want final message", log.ProgressMsg)
+ }
+ if len(log.DSL) != 0 {
+ t.Fatalf("DSL = %v, want empty object for terminal writer without DSL", log.DSL)
+ }
+}
+
+// TestRecordPipelineLog_KeepsEarlyTimestampWhenDocumentHasNone locks the
+// timestamp handover: the terminal snapshot must not blank the start time the
+// queued row was opened with when the document carries none (a run that never
+// reached the progress sink).
+func TestRecordPipelineLog_KeepsEarlyTimestampWhenDocumentHasNone(t *testing.T) {
+ cleanup := setupPipelineExecutorTestDB(t)
+ defer cleanup()
+
+ openedAt := time.Date(2026, 1, 2, 3, 4, 5, 0, time.Local)
+ queuedMsg := "Task is queued..."
+ if err := dao.DB.Create(&entity.PipelineOperationLog{
+ ID: "open-log",
+ DocumentID: "doc-1",
+ TenantID: "tenant-1",
+ KbID: "kb-1",
+ ParserID: "naive",
+ TaskType: "Parse",
+ OperationStatus: string(entity.TaskStatusRunning),
+ ProgressMsg: &queuedMsg,
+ ProcessBeginAt: &openedAt,
+ }).Error; err != nil {
+ t.Fatalf("seed open log: %v", err)
+ }
+
+ if err := dao.DB.Create(&entity.Document{
+ ID: "doc-1",
+ KbID: "kb-1",
+ ParserID: "naive",
+ ParserConfig: entity.JSONMap{},
+ SourceType: "local",
+ Type: "pdf",
+ CreatedBy: "tenant-1",
+ Suffix: ".pdf",
+ }).Error; err != nil {
+ t.Fatalf("seed document: %v", err)
+ }
+
+ if err := RecordPipelineLog(t.Context(), dao.DB, PipelineLogInput{
+ TenantID: "tenant-1",
+ KbID: "kb-1",
+ DocumentID: "doc-1",
+ Status: "4",
+ PipelineLogID: "open-log",
+ }); err != nil {
+ t.Fatalf("RecordPipelineLog: %v", err)
+ }
+
+ var log entity.PipelineOperationLog
+ if err := dao.DB.First(&log, "id = ?", "open-log").Error; err != nil {
+ t.Fatalf("load open log: %v", err)
+ }
+ if log.OperationStatus != "4" {
+ t.Fatalf("OperationStatus = %q, want terminal status", log.OperationStatus)
+ }
+ if log.ProcessBeginAt == nil || !log.ProcessBeginAt.Equal(openedAt) {
+ t.Fatalf("ProcessBeginAt = %v, want the queued row's %v", log.ProcessBeginAt, openedAt)
+ }
+ if log.SourceFrom != "local" || log.DocumentSuffix != ".pdf" || log.DocumentType != "pdf" {
+ t.Fatalf("terminal write did not refresh document metadata: source_from=%q suffix=%q type=%q",
+ log.SourceFrom, log.DocumentSuffix, log.DocumentType)
+ }
+}
+
+// TestRecordPipelineLog_DoesNotAdoptAnotherRunsRow locks the run-isolation
+// contract: a terminal write is bound to the row its own run opened, so a late
+// write from a superseded run (whose row was replaced) can neither finalize nor
+// even touch an open row belonging to another run of the same document.
+func TestRecordPipelineLog_DoesNotAdoptAnotherRunsRow(t *testing.T) {
+ cleanup := setupPipelineExecutorTestDB(t)
+ defer cleanup()
+
+ // An open row for the document that belongs to a different run: either the
+ // superseded run's row (cleanup failed) or the replacement run's fresh
+ // queued row. It must survive the write below untouched.
+ otherMsg := "Task is queued..."
+ other := &entity.PipelineOperationLog{
+ ID: "other-run-log",
+ DocumentID: "doc-1",
+ TenantID: "tenant-1",
+ KbID: "kb-1",
+ ParserID: "naive",
+ TaskType: "Parse",
+ OperationStatus: "5",
+ ProgressMsg: &otherMsg,
+ }
+ if err := dao.DB.Create(other).Error; err != nil {
+ t.Fatalf("seed other run's log: %v", err)
+ }
+
+ finalMsg := "Parser Done"
+ if err := dao.DB.Create(&entity.Document{
+ ID: "doc-1",
+ KbID: "kb-1",
+ ParserID: "naive",
+ ParserConfig: entity.JSONMap{},
+ SourceType: "local",
+ Type: "pdf",
+ CreatedBy: "tenant-1",
+ Suffix: ".pdf",
+ ProgressMsg: &finalMsg,
+ }).Error; err != nil {
+ t.Fatalf("seed document: %v", err)
+ }
+
+ // This run's own row was deleted along with the superseded task.
+ if err := RecordPipelineLog(t.Context(), dao.DB, PipelineLogInput{
+ TenantID: "tenant-1",
+ KbID: "kb-1",
+ DocumentID: "doc-1",
+ Status: "3",
+ PipelineLogID: "superseded-log",
+ }); err != nil {
+ t.Fatalf("RecordPipelineLog: %v", err)
+ }
+
+ var reloaded entity.PipelineOperationLog
+ if err := dao.DB.First(&reloaded, "id = ?", "other-run-log").Error; err != nil {
+ t.Fatalf("reload other run's log: %v", err)
+ }
+ if reloaded.OperationStatus != "5" {
+ t.Fatalf("other run's row OperationStatus = %q, want %q (must not be touched)", reloaded.OperationStatus, "5")
+ }
+ var count int64
+ if err := dao.DB.Model(&entity.PipelineOperationLog{}).Where("document_id = ?", "doc-1").Count(&count).Error; err != nil {
+ t.Fatalf("count pipeline logs: %v", err)
+ }
+ if count != 1 {
+ t.Fatalf("pipeline log rows = %d, want 1 (a bound missing row must not create a duplicate)", count)
+ }
+}
+
+// TestRecordPipelineLog_UnboundAdoptsOpenRow keeps the legacy contract for
+// callers that carry no bound row (debug-adjacent and non-ingestion paths): the
+// document's newest open row is adopted so a stray queued entry is closed
+// rather than left open.
+func TestRecordPipelineLog_UnboundAdoptsOpenRow(t *testing.T) {
+ cleanup := setupPipelineExecutorTestDB(t)
+ defer cleanup()
+
+ openMsg := "Task is queued..."
+ open := &entity.PipelineOperationLog{
+ ID: "open-log",
+ DocumentID: "doc-1",
+ TenantID: "tenant-1",
+ KbID: "kb-1",
+ ParserID: "naive",
+ TaskType: "Parse",
+ OperationStatus: "5",
+ ProgressMsg: &openMsg,
+ }
+ if err := dao.DB.Create(open).Error; err != nil {
+ t.Fatalf("seed open log: %v", err)
+ }
+
+ finalMsg := "Parser Done"
+ if err := dao.DB.Create(&entity.Document{
+ ID: "doc-1",
+ KbID: "kb-1",
+ ParserID: "naive",
+ ParserConfig: entity.JSONMap{},
+ SourceType: "local",
+ Type: "pdf",
+ CreatedBy: "tenant-1",
+ Suffix: ".pdf",
+ ProgressMsg: &finalMsg,
+ }).Error; err != nil {
+ t.Fatalf("seed document: %v", err)
+ }
+
+ if err := RecordPipelineLog(t.Context(), dao.DB, PipelineLogInput{
+ TenantID: "tenant-1",
+ KbID: "kb-1",
+ DocumentID: "doc-1",
+ Status: "3",
+ }); err != nil {
+ t.Fatalf("RecordPipelineLog: %v", err)
+ }
+
+ var reloaded entity.PipelineOperationLog
+ if err := dao.DB.First(&reloaded, "id = ?", "open-log").Error; err != nil {
+ t.Fatalf("reload open log: %v", err)
+ }
+ if reloaded.OperationStatus != "3" {
+ t.Fatalf("open row OperationStatus = %q, want %q (adopted by the unbound writer)", reloaded.OperationStatus, "3")
+ }
+ var count int64
+ if err := dao.DB.Model(&entity.PipelineOperationLog{}).Where("document_id = ?", "doc-1").Count(&count).Error; err != nil {
+ t.Fatalf("count pipeline logs: %v", err)
+ }
+ if count != 1 {
+ t.Fatalf("pipeline log rows = %d, want 1 (adopted, not duplicated)", count)
+ }
+}
+
+// TestRecordPipelineLog_DoesNotAdoptLiveRunsOpenRow locks the ownership rule on
+// the unbound fallback: the document's newest open row belongs to a live run, so
+// another run's terminal write must not finalize it. Adopting it would stamp the
+// live run's entry with a foreign status and swallow that run's own terminal
+// write (its CAS would find the row already closed).
+func TestRecordPipelineLog_DoesNotAdoptLiveRunsOpenRow(t *testing.T) {
+ cleanup := setupPipelineExecutorTestDB(t)
+ defer cleanup()
+
+ liveMsg := "Task is queued..."
+ if err := dao.DB.Create(&entity.PipelineOperationLog{
+ ID: "live-run-log",
+ DocumentID: "doc-1",
+ TenantID: "tenant-1",
+ KbID: "kb-1",
+ ParserID: "naive",
+ TaskType: "Parse",
+ OperationStatus: "5",
+ ProgressMsg: &liveMsg,
+ }).Error; err != nil {
+ t.Fatalf("seed live run's open log: %v", err)
+ }
+ if err := dao.DB.Create(&entity.IngestionTask{
+ ID: "live-task",
+ UserID: "user-1",
+ DocumentID: "doc-1",
+ DatasetID: "kb-1",
+ Status: "RUNNING",
+ }).Error; err != nil {
+ t.Fatalf("seed live task: %v", err)
+ }
+ if err := dao.DB.Model(&entity.IngestionTask{}).Where("id = ?", "live-task").
+ Update("pipeline_log_id", "live-run-log").Error; err != nil {
+ t.Fatalf("bind live task: %v", err)
+ }
+
+ finalMsg := "Parser Done"
+ if err := dao.DB.Create(&entity.Document{
+ ID: "doc-1",
+ KbID: "kb-1",
+ ParserID: "naive",
+ ParserConfig: entity.JSONMap{},
+ SourceType: "local",
+ Type: "pdf",
+ CreatedBy: "tenant-1",
+ Suffix: ".pdf",
+ ProgressMsg: &finalMsg,
+ }).Error; err != nil {
+ t.Fatalf("seed document: %v", err)
+ }
+
+ if err := RecordPipelineLog(t.Context(), dao.DB, PipelineLogInput{
+ TenantID: "tenant-1",
+ KbID: "kb-1",
+ DocumentID: "doc-1",
+ Status: "4",
+ }); err != nil {
+ t.Fatalf("RecordPipelineLog: %v", err)
+ }
+
+ var live entity.PipelineOperationLog
+ if err := dao.DB.First(&live, "id = ?", "live-run-log").Error; err != nil {
+ t.Fatalf("load live run's log: %v", err)
+ }
+ if live.OperationStatus != "5" {
+ t.Fatalf("live run's row was adopted: OperationStatus = %q, want it untouched at %q", live.OperationStatus, "5")
+ }
+ var count int64
+ if err := dao.DB.Model(&entity.PipelineOperationLog{}).Where("document_id = ?", "doc-1").Count(&count).Error; err != nil {
+ t.Fatalf("count pipeline logs: %v", err)
+ }
+ if count != 2 {
+ t.Fatalf("pipeline log rows = %d, want 2 (the unbound run records its own entry)", count)
+ }
+}
+
+func TestRecordPipelineLog_CreatesRowWithoutOpenPreTerminalRow(t *testing.T) {
+ cleanup := setupPipelineExecutorTestDB(t)
+ defer cleanup()
+
+ docName := "legacy.pdf"
+ if err := dao.DB.Create(&entity.Document{
+ ID: "doc-1",
+ KbID: "kb-1",
+ ParserID: "naive",
+ ParserConfig: entity.JSONMap{},
+ SourceType: "local",
+ Type: "pdf",
+ CreatedBy: "tenant-1",
+ Name: &docName,
+ Suffix: ".pdf",
+ }).Error; err != nil {
+ t.Fatalf("seed document: %v", err)
+ }
+
+ taskCtx := makeTaskCtx()
+ taskCtx.Doc.ParserID = "naive"
+ var captured *entity.PipelineOperationLog
+ svc := mustNewPipelineExecutor(t, taskCtx, "flow-1", 0).WithLogCreateFunc(
+ func(ctx context.Context, db *gorm.DB, log *entity.PipelineOperationLog) error {
+ captured = log
+ return nil
+ },
+ )
+ svc.recordPipelineLog(t.Context(), dao.DB, "doc-1", `{"components": {}}`, "done")
+ if captured == nil {
+ t.Fatal("logCreateFunc was not called: expected Create fallback without an open row")
+ }
+}
+
// =============================================================================
// updateDocumentMetadata
// =============================================================================
diff --git a/internal/service/dataset/ingestion.go b/internal/service/dataset/ingestion.go
index 75ee64fdaf..e7a68eac9c 100644
--- a/internal/service/dataset/ingestion.go
+++ b/internal/service/dataset/ingestion.go
@@ -39,7 +39,7 @@ func (d *DatasetService) GetIngestionSummary(ctx context.Context, datasetID, use
}, common.CodeSuccess, nil
}
-func (d *DatasetService) ListIngestionLogs(ctx context.Context, datasetID, userID string, page, pageSize int, orderby string, desc bool, operationStatus []string, createDateFrom, createDateTo, logType, keywords string) (map[string]interface{}, common.ErrorCode, error) {
+func (d *DatasetService) ListIngestionLogs(ctx context.Context, datasetID, userID string, page, pageSize int, orderby string, desc bool, operationStatus []string, createDateFrom, createDateTo, logType, keywords, documentID string) (map[string]interface{}, common.ErrorCode, error) {
if datasetID == "" {
return nil, common.CodeDataError, errors.New(`lack of "Dataset ID"`)
}
@@ -63,9 +63,9 @@ func (d *DatasetService) ListIngestionLogs(ctx context.Context, datasetID, userI
err error
)
if logType == "file" {
- logs, total, err = d.pipelineLogDAO.GetFileLogsByKBID(ctx, dao.DB, datasetID, page, pageSize, orderby, desc, keywords, operationStatus, createDateFrom, createDateTo)
+ logs, total, err = d.pipelineLogDAO.GetFileLogsByKBID(ctx, dao.DB, datasetID, page, pageSize, orderby, desc, keywords, documentID, operationStatus, createDateFrom, createDateTo)
} else {
- logs, total, err = d.pipelineLogDAO.GetDatasetLogsByKBID(ctx, dao.DB, datasetID, page, pageSize, orderby, desc, operationStatus, createDateFrom, createDateTo, keywords)
+ logs, total, err = d.pipelineLogDAO.GetDatasetLogsByKBID(ctx, dao.DB, datasetID, page, pageSize, orderby, desc, operationStatus, createDateFrom, createDateTo, keywords, documentID)
}
if err != nil {
return nil, common.CodeServerError, fmt.Errorf("list ingestion logs: %w", err)
diff --git a/internal/service/document/document_ingest.go b/internal/service/document/document_ingest.go
index 36cf515480..71e5b5c806 100644
--- a/internal/service/document/document_ingest.go
+++ b/internal/service/document/document_ingest.go
@@ -131,11 +131,22 @@ func (s *DocumentService) Ingest(ctx context.Context, userID string, req *Ingest
}
if req.Delete {
+ // Capture the run's own log row before its task is removed. The
+ // cleanup must drop only that row: a concurrent re-parse may
+ // already own a newer one for the same document.
+ var pipelineLogID string
+ if task, taskErr := s.ingestionTaskDAO.GetByDocumentID(ctx, dao.DB, doc.ID); taskErr != nil {
+ common.Error(fmt.Sprintf("go side, doc %s, load ingestion task for pipeline log cleanup failed", doc.ID), taskErr)
+ } else if task != nil && task.PipelineLogID != nil {
+ pipelineLogID = *task.PipelineLogID
+ }
if _, delErr := s.taskDAO.DeleteIngestionTasksByDocIDs(ctx, dao.DB, []string{doc.ID}); delErr != nil {
if errors.Is(delErr, context.Canceled) || errors.Is(delErr, context.DeadlineExceeded) {
return common.CodeExceptionError, fmt.Errorf("delete ingestion tasks: %w", delErr)
}
common.Error(fmt.Sprintf("go side, doc %s, DeleteIngestionTasksByDocIDs failed", doc.ID), delErr)
+ } else if err := s.pipelineLogDAO.DeleteOpenLogByID(ctx, dao.DB, pipelineLogID); err != nil {
+ common.Error(fmt.Sprintf("go side, doc %s, delete open pipeline log failed", doc.ID), err)
}
indexName := fmt.Sprintf("ragflow_%s", kb.TenantID)
if s.docEngine != nil {
diff --git a/internal/service/document/document_parse.go b/internal/service/document/document_parse.go
index cc1b29eaa3..f1ef27757c 100644
--- a/internal/service/document/document_parse.go
+++ b/internal/service/document/document_parse.go
@@ -138,6 +138,14 @@ func (s *DocumentService) clearDocumentParseResults(ctx context.Context, doc *en
if taskExisted && deleted == 0 {
return fmt.Errorf("document %s ingestion task started running; stop it before re-parsing", doc.ID)
}
+ // The deleted CREATED/SCHEDULED task (if any) owned an open
+ // pipeline-operation-log row that no worker will ever close. Drop that
+ // run's row only: a concurrent re-parse may already own a newer one.
+ if deleted > 0 && task != nil && task.PipelineLogID != nil {
+ if err := s.pipelineLogDAO.DeleteOpenLogByID(ctx, dao.DB, *task.PipelineLogID); err != nil {
+ common.Error(fmt.Sprintf("clear open pipeline log for document %s", doc.ID), err)
+ }
+ }
if err := s.clearDocumentAndKBCountersForRerun(doc.ID, doc.KbID); err != nil {
return err
diff --git a/internal/service/document/document_rerun_test.go b/internal/service/document/document_rerun_test.go
index 853ab11dc9..82bc344997 100644
--- a/internal/service/document/document_rerun_test.go
+++ b/internal/service/document/document_rerun_test.go
@@ -153,6 +153,102 @@ func TestRerunDocument_RerunsAndPersistsDSL(t *testing.T) {
}
}
+func TestRerunDocument_ClearsQueuedTaskPreTerminalLog(t *testing.T) {
+ db := setupServiceTestDB(t)
+ pushServiceDB(t, db)
+
+ insertTestKB(t, "kb-1", "tenant-1", 1, 0, 0)
+ insertTestDoc(t, "doc-1", "kb-1", 0, 0)
+ if err := dao.DB.Model(&entity.Document{}).Where("id = ?", "doc-1").Update("location", "loc-1").Error; err != nil {
+ t.Fatalf("set location: %v", err)
+ }
+ insertTestPipelineLog(t, "log-1", "doc-1", "kb-1", "tenant-1", entity.JSONMap{"components": map[string]interface{}{}})
+ insertTestIngestionTaskWithStatus(t, "task-1", "user-1", "doc-1", "kb-1", common.SCHEDULED)
+ queuedMsg := "Task is queued..."
+ if err := dao.DB.Create(&entity.PipelineOperationLog{
+ ID: "queued-log",
+ DocumentID: "doc-1",
+ TenantID: "tenant-1",
+ KbID: "kb-1",
+ ParserID: "naive",
+ TaskType: "Parse",
+ OperationStatus: string(entity.TaskStatusSchedule),
+ ProgressMsg: &queuedMsg,
+ }).Error; err != nil {
+ t.Fatalf("seed queued log: %v", err)
+ }
+ // The clearing run owns this row, exactly as createOpenLogBestEffort
+ // binds it in production.
+ if err := dao.DB.Model(&entity.IngestionTask{}).Where("id = ?", "task-1").
+ Update("pipeline_log_id", "queued-log").Error; err != nil {
+ t.Fatalf("bind queued log: %v", err)
+ }
+
+ svc, _ := rerunTestService(t)
+ if err := svc.RerunDocument(t.Context(), "tenant-1", "log-1", nil, "c1"); err != nil {
+ t.Fatalf("RerunDocument: %v", err)
+ }
+ var surviving entity.PipelineOperationLog
+ err := db.First(&surviving, "id = ?", "queued-log").Error
+ if err == nil {
+ t.Fatalf("stale queued row survived the rerun; the new run must start fresh")
+ }
+ if !dao.IsNotFoundErr(err) {
+ t.Fatalf("load queued log: %v", err)
+ }
+}
+
+// TestRerunDocument_KeepsAnotherRunsPreTerminalLog locks the ownership boundary: the
+// rerun cleanup drops only the row the cleared task was bound to, so an open
+// row belonging to another run is left alone.
+func TestRerunDocument_KeepsAnotherRunsPreTerminalLog(t *testing.T) {
+ db := setupServiceTestDB(t)
+ pushServiceDB(t, db)
+
+ insertTestKB(t, "kb-1", "tenant-1", 1, 0, 0)
+ insertTestDoc(t, "doc-1", "kb-1", 0, 0)
+ if err := dao.DB.Model(&entity.Document{}).Where("id = ?", "doc-1").Update("location", "loc-1").Error; err != nil {
+ t.Fatalf("set location: %v", err)
+ }
+ insertTestPipelineLog(t, "log-1", "doc-1", "kb-1", "tenant-1", entity.JSONMap{"components": map[string]interface{}{}})
+ insertTestIngestionTaskWithStatus(t, "task-1", "user-1", "doc-1", "kb-1", common.SCHEDULED)
+
+ queuedMsg := "Task is queued..."
+ seed := func(id, docID string) {
+ t.Helper()
+ if err := dao.DB.Create(&entity.PipelineOperationLog{
+ ID: id,
+ DocumentID: docID,
+ TenantID: "tenant-1",
+ KbID: "kb-1",
+ ParserID: "naive",
+ TaskType: "Parse",
+ OperationStatus: string(entity.TaskStatusSchedule),
+ ProgressMsg: &queuedMsg,
+ }).Error; err != nil {
+ t.Fatalf("seed %s: %v", id, err)
+ }
+ }
+ seed("own-log", "doc-1")
+ seed("other-run-log", "doc-2")
+ if err := dao.DB.Model(&entity.IngestionTask{}).Where("id = ?", "task-1").
+ Update("pipeline_log_id", "own-log").Error; err != nil {
+ t.Fatalf("bind own log: %v", err)
+ }
+
+ svc, _ := rerunTestService(t)
+ if err := svc.RerunDocument(t.Context(), "tenant-1", "log-1", nil, "c1"); err != nil {
+ t.Fatalf("RerunDocument: %v", err)
+ }
+
+ if err := db.First(&entity.PipelineOperationLog{}, "id = ?", "own-log").Error; !dao.IsNotFoundErr(err) {
+ t.Fatalf("cleared run's own row should be deleted, got err=%v", err)
+ }
+ if err := db.First(&entity.PipelineOperationLog{}, "id = ?", "other-run-log").Error; err != nil {
+ t.Fatalf("another run's open row was deleted by this run's cleanup: %v", err)
+ }
+}
+
func TestRerunDocument_EmptyDSLPersistsEntryPath(t *testing.T) {
db := setupServiceTestDB(t)
pushServiceDB(t, db)
diff --git a/internal/service/ingestion_task_service.go b/internal/service/ingestion_task_service.go
index 5109e1443a..bdf50e59ab 100644
--- a/internal/service/ingestion_task_service.go
+++ b/internal/service/ingestion_task_service.go
@@ -4,6 +4,7 @@ import (
"context"
"errors"
"fmt"
+ "strings"
"time"
"ragflow/internal/common"
@@ -45,6 +46,9 @@ type IngestionTaskService struct {
userDAO *dao.UserDAO
ingestionTaskDAO *dao.IngestionTaskDAO
ingestionTaskLogDAO *dao.IngestionTaskLogDAO
+ pipelineLogDAO *dao.PipelineOperationLogDAO
+ kbDAO *dao.KnowledgebaseDAO
+ userCanvasDAO *dao.UserCanvasDAO
taskPublisher TaskPublisher
}
@@ -54,6 +58,9 @@ func NewIngestionTaskService() *IngestionTaskService {
userDAO: dao.NewUserDAO(),
ingestionTaskDAO: dao.NewIngestionTaskDAO(),
ingestionTaskLogDAO: dao.NewIngestionTaskLogDAO(),
+ pipelineLogDAO: dao.NewPipelineOperationLogDAO(),
+ kbDAO: dao.NewKnowledgebaseDAO(),
+ userCanvasDAO: dao.NewUserCanvasDAO(),
taskPublisher: NewMessageQueueTaskPublisher(),
}
}
@@ -78,6 +85,10 @@ func (s *IngestionTaskService) CreateForDocuments(ctx context.Context, datasetID
return nil, fmt.Errorf("no documents to parse")
}
+ // Open-log metadata is best-effort. Populate this cache lazily so a missing
+ // or temporarily unavailable knowledge base cannot block task creation.
+ kbCache := make(map[string]*entity.Knowledgebase)
+
responses := make([]*ParseDocumentResponse, 0, len(uniqueDocIDs))
for _, docID := range uniqueDocIDs {
doc, err := s.documentDAO.GetByID(ctx, dao.DB, docID)
@@ -103,7 +114,7 @@ func (s *IngestionTaskService) CreateForDocuments(ctx context.Context, datasetID
Schema: nil,
Status: common.CREATED,
}
- task, err = s.CreateAndEnqueue(ctx, task)
+ task, err = s.createAndEnqueueWithKBCache(ctx, task, kbCache)
if err != nil {
responses = append(responses, &ParseDocumentResponse{
DocumentID: docID,
@@ -221,6 +232,7 @@ func (s *IngestionTaskService) StartRunning(ctx context.Context, taskID string)
}); err != nil {
common.Warn(fmt.Sprintf("StartRunning: mark document %s running for task %s: %v", task.DocumentID, taskID, err))
}
+ s.advanceOpenLog(ctx, task, logFromUnstartOrScheduled, string(entity.TaskStatusRunning), logMsgRunning, true)
return task, nil
case common.STOPPING:
task, err = s.transition(ctx, taskID, common.STOPPED)
@@ -232,6 +244,10 @@ func (s *IngestionTaskService) StartRunning(ctx context.Context, taskID string)
// that RequestStop set would otherwise leak until TTL and cancel the
// next run of this task at the worker's pre-start check.
clearCancelFlag(ctx, taskID)
+ // Same reason as RequestStop's CREATED/SCHEDULED branch: the stop
+ // finalizes without a worker, so no terminal pipeline-log writer will
+ // close the open row. Close it here or it stays RUNNING forever.
+ s.advanceOpenLog(ctx, task, dao.OpenPipelineOperationStatuses(), string(entity.TaskStatusCancel), logMsgStopped, false)
return task, nil
case common.RUNNING, common.COMPLETED, common.STOPPED, common.FAILED:
return task, nil
@@ -247,7 +263,15 @@ func (s *IngestionTaskService) RequestStop(ctx context.Context, taskID string) (
}
switch task.Status {
case common.CREATED, common.SCHEDULED:
- return s.transition(ctx, taskID, common.STOPPED)
+ stopped, err := s.transition(ctx, taskID, common.STOPPED)
+ if err != nil {
+ return nil, err
+ }
+ // The stop finalizes without a worker (no RUNNING phase, so no
+ // terminal pipeline-log writer will run). Advance the open row to
+ // CANCEL here, otherwise the detail page keeps a queued entry.
+ s.advanceOpenLog(ctx, stopped, logFromUnstartOrScheduled, string(entity.TaskStatusCancel), logMsgStopped, false)
+ return stopped, nil
case common.RUNNING:
task, err = s.transition(ctx, taskID, common.STOPPING)
if err != nil {
@@ -305,7 +329,19 @@ func (s *IngestionTaskService) MarkStopped(ctx context.Context, taskID string) e
}
func (s *IngestionTaskService) Remove(ctx context.Context, taskID string, userID *string) (*dao.TaskInfo, error) {
- return s.ingestionTaskDAO.Delete(ctx, dao.DB, taskID, userID)
+ task, err := s.GetTask(ctx, taskID)
+ if err != nil {
+ return nil, err
+ }
+ info, err := s.ingestionTaskDAO.Delete(ctx, dao.DB, taskID, userID)
+ if err != nil {
+ return nil, err
+ }
+ // The task row is gone, so no worker will ever reach the terminal writer
+ // for this run. Drop its open row, otherwise the detail page keeps a
+ // queued entry.
+ s.deleteOpenLogBestEffort(ctx, task)
+ return info, nil
}
func (s *IngestionTaskService) GetTask(ctx context.Context, taskID string) (*entity.IngestionTask, error) {
@@ -382,6 +418,13 @@ func (s *IngestionTaskService) transition(ctx context.Context, taskID string, to
}
func (s *IngestionTaskService) CreateAndEnqueue(ctx context.Context, task *entity.IngestionTask) (*entity.IngestionTask, error) {
+ return s.createAndEnqueueWithKBCache(ctx, task, nil)
+}
+
+// createAndEnqueueWithKBCache is CreateAndEnqueue with an optional
+// knowledge-base cache for batch callers (CreateForDocuments resolves one kb
+// for the whole batch). A nil cache behaves exactly like CreateAndEnqueue.
+func (s *IngestionTaskService) createAndEnqueueWithKBCache(ctx context.Context, task *entity.IngestionTask, kbCache map[string]*entity.Knowledgebase) (*entity.IngestionTask, error) {
existing, err := s.ingestionTaskDAO.GetByDocumentID(ctx, dao.DB, task.DocumentID)
if err != nil {
return nil, err
@@ -389,6 +432,11 @@ func (s *IngestionTaskService) CreateAndEnqueue(ctx context.Context, task *entit
if existing != nil {
switch existing.Status {
case common.CREATED:
+ // An existing CREATED task may already carry a queued row from
+ // before a crash; the helper adopts it (binding the task to it)
+ // instead of opening a second row, then the schedule transition
+ // below advances it.
+ s.createOpenLogBestEffort(ctx, existing, string(entity.TaskStatusUnstart), logMsgQueued, kbCache)
if err = s.enqueueTask(existing.ID); err != nil {
return nil, err
}
@@ -404,7 +452,12 @@ func (s *IngestionTaskService) CreateAndEnqueue(ctx context.Context, task *entit
// RequestStop once the task is RUNNING again. Clear it so the
// re-queued task is not cancelled at the worker's pre-start check.
clearCancelFlag(ctx, existing.ID)
+ // A retry starts a fresh pipeline-operation-log row: the prior
+ // run's row is terminal, so no open row matches and the helper
+ // opens (and binds) a new one.
+ s.createOpenLogBestEffort(ctx, existing, string(entity.TaskStatusUnstart), logMsgQueued, kbCache)
if err = s.enqueueTask(existing.ID); err != nil {
+ s.deleteOpenLogBestEffort(ctx, existing)
if rollbackErr := s.rollbackRetriedTask(ctx, existing.ID, originalStatus); rollbackErr != nil {
return nil, fmt.Errorf("enqueue task %s: %w (rollback failed: %w)", existing.ID, err, rollbackErr)
}
@@ -420,7 +473,16 @@ func (s *IngestionTaskService) CreateAndEnqueue(ctx context.Context, task *entit
if err != nil {
return nil, err
}
+ // Open the pre-terminal row the dataset detail page reads *before*
+ // publishing. Once the message is out a worker can claim the task and run
+ // it to a terminal state, and the terminal writer must always find the row
+ // it is bound to; a row created after publication could be inserted too
+ // late and stay queued forever. The prior run (if any) is terminal, so no
+ // open row exists and this never duplicates. Best-effort: on a DB blip the
+ // row is simply missing and markScheduledAfterPublish re-opens it at "5".
+ s.createOpenLogBestEffort(ctx, created, string(entity.TaskStatusUnstart), logMsgQueued, kbCache)
if err = s.enqueueTask(created.ID); err != nil {
+ s.deleteOpenLogBestEffort(ctx, created)
if rollbackErr := s.rollbackCreatedTask(ctx, created.ID); rollbackErr != nil {
return nil, fmt.Errorf("enqueue task %s: %w (rollback failed: %w)", created.ID, err, rollbackErr)
}
@@ -445,16 +507,22 @@ func (s *IngestionTaskService) rollbackCreatedTask(ctx context.Context, taskID s
return err
}
-// markScheduledAfterPublish records a successful NATS publish. A worker can
-// claim the CREATED task before this write and move it to RUNNING, which is
-// also a successful outcome.
+// markScheduledAfterPublish records a successful NATS publish. A worker may
+// claim the task first and move it to RUNNING; the log write below is scoped to
+// the unstarted from-state, so it never regresses a row the worker already
+// advanced.
func (s *IngestionTaskService) markScheduledAfterPublish(ctx context.Context, taskID string) (*entity.IngestionTask, error) {
updated, err := s.ingestionTaskDAO.UpdateStatusIfCurrent(ctx, dao.DB, taskID, common.CREATED, common.SCHEDULED)
if err != nil {
return nil, err
}
if updated {
- return s.GetTask(ctx, taskID)
+ task, err := s.GetTask(ctx, taskID)
+ if err != nil {
+ return nil, err
+ }
+ s.advanceOpenLog(ctx, task, logFromUnstart, string(entity.TaskStatusSchedule), logMsgQueued, true)
+ return task, nil
}
task, err := s.GetTask(ctx, taskID)
@@ -473,6 +541,10 @@ func (s *IngestionTaskService) markScheduledAfterPublish(ctx context.Context, ta
// process stopped but were not confirmed as scheduled. It is intended for the
// single startup recovery pass; a publish error leaves the task CREATED for a
// future startup or explicit parse request to retry.
+//
+// A recovered task may already carry a queued row written before the crash;
+// markScheduledAfterPublish advances it in place instead of opening a second
+// row, so startup recovery never duplicates the open row.
func (s *IngestionTaskService) ScheduleCreatedTasks(ctx context.Context) error {
tasks, err := s.ingestionTaskDAO.ListByStatus(ctx, dao.DB, common.CREATED)
if err != nil {
@@ -508,6 +580,208 @@ func (s *IngestionTaskService) enqueueTask(taskID string) error {
return s.taskPublisher.PublishTaskMessage(common.TaskSubject, taskMessage)
}
+// Messages the queued stages show on the dataset detail page.
+const (
+ logMsgQueued = "Task is queued..."
+ logMsgRunning = "Task is running..."
+ logMsgStopped = "Task stopped by user."
+)
+
+// logFromUnstart and logFromUnstartOrScheduled are the operation_status values
+// an open pipeline-log row may carry when a stage advances it. Scoping each
+// advance to its valid from-states keeps the transitions monotonic
+// (unstart -> schedule -> running), so a late queued write cannot regress a row
+// a worker already moved to running. Closing a row to a terminal status accepts
+// any open state, i.e. dao.OpenPipelineOperationStatuses itself.
+var (
+ logFromUnstart = []string{string(entity.TaskStatusUnstart)}
+ logFromUnstartOrScheduled = []string{string(entity.TaskStatusUnstart), string(entity.TaskStatusSchedule)}
+)
+
+// createOpenLogBestEffort opens the pre-terminal row the dataset detail page
+// reads and binds it to the task. Best-effort: never fails task creation. The
+// caller passes the status/message so a late-created row carries the current
+// status.
+//
+// Ownership establishment is serialized on the task row. A stale in-memory
+// task snapshot therefore cannot replace a binding another caller just wrote.
+// An unbound run never adopts a document-level row; it removes that row only
+// when no live task owns it, then creates and binds its own row atomically.
+func (s *IngestionTaskService) createOpenLogBestEffort(ctx context.Context, task *entity.IngestionTask, operationStatus, progressMsg string, kbCache map[string]*entity.Knowledgebase) {
+ if task == nil || s.pipelineLogDAO == nil {
+ return
+ }
+ input, err := s.buildOpenLogInput(ctx, task, operationStatus, progressMsg, kbCache)
+ if err != nil {
+ common.Warn(fmt.Sprintf("open pipeline log: build input for task %s document %s: %v", task.ID, task.DocumentID, err))
+ return
+ }
+ var boundLogID string
+ err = dao.DB.WithContext(ctx).Transaction(func(tx *gorm.DB) error {
+ lockedTask, err := s.ingestionTaskDAO.GetByIDForUpdate(ctx, tx, task.ID)
+ if err != nil {
+ return err
+ }
+ if isTerminalIngestionTask(lockedTask.Status) {
+ return nil
+ }
+ if lockedTask.PipelineLogID != nil && *lockedTask.PipelineLogID != "" {
+ bound, err := s.pipelineLogDAO.GetByID(ctx, tx, *lockedTask.PipelineLogID)
+ if err == nil && isOpenPipelineLog(bound.OperationStatus) {
+ boundLogID = bound.ID
+ return nil
+ }
+ if err != nil && !dao.IsNotFoundErr(err) {
+ return err
+ }
+ }
+
+ open, err := s.pipelineLogDAO.GetOpenLogByDocumentID(ctx, tx, lockedTask.DocumentID)
+ if err != nil {
+ return err
+ }
+ if open != nil {
+ deleted, err := s.pipelineLogDAO.DeleteUnownedOpenLogByID(ctx, tx, open.ID)
+ if err != nil {
+ return err
+ }
+ if !deleted {
+ return fmt.Errorf("open pipeline log %s is owned by another live task", open.ID)
+ }
+ common.Warn(fmt.Sprintf("dropped unowned open pipeline log %s for document %s (status %s)", open.ID, lockedTask.DocumentID, open.OperationStatus))
+ }
+
+ log, err := s.pipelineLogDAO.CreateOpenLog(ctx, tx, input)
+ if err != nil {
+ return err
+ }
+ if err := s.ingestionTaskDAO.UpdatePipelineLogID(ctx, tx, lockedTask.ID, log.ID); err != nil {
+ return err
+ }
+ boundLogID = log.ID
+ return nil
+ })
+ if err != nil {
+ common.Warn(fmt.Sprintf("open pipeline log: establish ownership for task %s document %s: %v", task.ID, task.DocumentID, err))
+ return
+ }
+ if boundLogID != "" {
+ task.PipelineLogID = &boundLogID
+ }
+}
+
+func isTerminalIngestionTask(status string) bool {
+ return status == common.COMPLETED || status == common.STOPPED || status == common.FAILED
+}
+
+func isOpenPipelineLog(status string) bool {
+ return status == string(entity.TaskStatusUnstart) ||
+ status == string(entity.TaskStatusSchedule) ||
+ status == string(entity.TaskStatusRunning)
+}
+
+// advanceOpenLog moves the task's own bound row to operationStatus from one of
+// fromStatuses. Best-effort: never fails task transitions.
+//
+// It never looks the row up by document: only the row this run is bound to is
+// advanced, so a concurrent run's row can be neither regressed nor adopted. A
+// task with no bound row (legacy task, or a lost best-effort create) opens one,
+// but only while the task is still live — a terminal task must never resurrect
+// a closed row as a permanently queued entry.
+func (s *IngestionTaskService) advanceOpenLog(ctx context.Context, task *entity.IngestionTask, fromStatuses []string, operationStatus, progressMsg string, reopen bool) {
+ if task == nil || task.DocumentID == "" || s.pipelineLogDAO == nil {
+ return
+ }
+ logID := ""
+ if task.PipelineLogID != nil {
+ logID = *task.PipelineLogID
+ }
+ if logID == "" {
+ terminal := task.Status == common.COMPLETED || task.Status == common.STOPPED || task.Status == common.FAILED
+ if !reopen || terminal {
+ return
+ }
+ s.createOpenLogBestEffort(ctx, task, operationStatus, progressMsg, nil)
+ return
+ }
+ if err := s.pipelineLogDAO.AdvanceOpenLog(ctx, dao.DB, logID, fromStatuses, operationStatus, progressMsg); err != nil {
+ common.Warn(fmt.Sprintf("advance open pipeline log for document %s to %s: %v", task.DocumentID, operationStatus, err))
+ }
+}
+
+// deleteOpenLogBestEffort drops the open row a run owns, so a rolled-back or
+// deleted run leaves no permanently queued entry on the detail page. Only the
+// run's own open row goes: a terminal row is history, and a newer run's row is
+// not this caller's to drop.
+func (s *IngestionTaskService) deleteOpenLogBestEffort(ctx context.Context, task *entity.IngestionTask) {
+ if task == nil || task.PipelineLogID == nil || s.pipelineLogDAO == nil {
+ return
+ }
+ if err := s.pipelineLogDAO.DeleteOpenLogByID(ctx, dao.DB, *task.PipelineLogID); err != nil {
+ common.Warn(fmt.Sprintf("CreateAndEnqueue: delete open pipeline log for document %s: %v", task.DocumentID, err))
+ }
+}
+
+// buildOpenLogInput resolves the pre-terminal row from the document and its
+// knowledge base. Status/message come from the caller so a late-created row
+// carries the current status. DSL and progress stay empty for the terminal write.
+func (s *IngestionTaskService) buildOpenLogInput(ctx context.Context, task *entity.IngestionTask, operationStatus, progressMsg string, kbCache map[string]*entity.Knowledgebase) (dao.OpenLogInput, error) {
+ var input dao.OpenLogInput
+ doc, err := s.documentDAO.GetByID(ctx, dao.DB, task.DocumentID)
+ if err != nil {
+ return input, err
+ }
+ kbID := task.DatasetID
+ if kbID == "" {
+ kbID = doc.KbID
+ }
+ kb := kbCache[kbID]
+ if kb == nil {
+ kb, err = s.kbDAO.GetByID(ctx, dao.DB, kbID)
+ if err != nil {
+ return input, err
+ }
+ if kb == nil {
+ return input, fmt.Errorf("knowledgebase %s not found", kbID)
+ }
+ if kbCache != nil {
+ kbCache[kbID] = kb
+ }
+ }
+ input = dao.OpenLogInput{
+ DocumentID: doc.ID,
+ KbID: kbID,
+ TenantID: kb.TenantID,
+ ParserID: doc.ParserID,
+ DocumentSuffix: doc.Suffix,
+ DocumentType: doc.Type,
+ OperationStatus: operationStatus,
+ ProgressMsg: progressMsg,
+ PipelineTitle: doc.ParserID,
+ Avatar: doc.Thumbnail,
+ }
+ if doc.Name != nil {
+ input.DocumentName = *doc.Name
+ }
+ // SplitN always returns at least one part, so the prefix before "/" is
+ // always available.
+ input.SourceFrom = strings.SplitN(doc.SourceType, "/", 2)[0]
+ if doc.PipelineID != nil {
+ if pipelineID := strings.TrimSpace(*doc.PipelineID); pipelineID != "" {
+ input.PipelineID = pipelineID
+ if canvas, err := s.userCanvasDAO.GetByID(ctx, dao.DB, pipelineID); err == nil && canvas != nil {
+ if canvas.Title != nil && *canvas.Title != "" {
+ input.PipelineTitle = *canvas.Title
+ }
+ input.Avatar = canvas.Avatar
+ } else if err != nil && !errors.Is(err, dao.ErrUserCanvasNotFound) {
+ common.Warn(fmt.Sprintf("CreateAndEnqueue: load pipeline %s for open log: %v", pipelineID, err))
+ }
+ }
+ }
+ return input, nil
+}
+
// UpdateComponentTotal records the number of components in the task's DSL
// graph - the authoritative denominator for progress percentage.
func (s *IngestionTaskService) UpdateComponentTotal(ctx context.Context, taskID string, total int) error {
diff --git a/internal/service/ingestion_task_service_test.go b/internal/service/ingestion_task_service_test.go
index 27aca6aa7c..20a27980ae 100644
--- a/internal/service/ingestion_task_service_test.go
+++ b/internal/service/ingestion_task_service_test.go
@@ -1,11 +1,14 @@
package service
import (
+ "context"
"errors"
"strings"
"sync"
"testing"
+ "gorm.io/gorm"
+
"ragflow/internal/common"
"ragflow/internal/dao"
"ragflow/internal/entity"
@@ -102,6 +105,37 @@ func TestIngestionTaskServiceCreateForDocumentsPublishesTaskMessages(t *testing.
}
}
+func TestIngestionTaskServiceCreateForDocumentsQueuesWithoutKnowledgebase(t *testing.T) {
+ db := setupServiceTestDB(t)
+ pushServiceDB(t, db)
+ insertTestDoc(t, "doc-1", "missing-kb", 0, 0)
+
+ publisher := &recordingTaskPublisher{}
+ svc := NewIngestionTaskService()
+ svc.taskPublisher = publisher
+
+ responses, err := svc.CreateForDocuments(t.Context(), "missing-kb", "user-1", []string{"doc-1"})
+ if err != nil {
+ t.Fatalf("CreateForDocuments should keep early-log lookup best-effort: %v", err)
+ }
+ if len(responses) != 1 || !strings.HasPrefix(responses[0].Result, "task_id:") {
+ t.Fatalf("unexpected responses: %+v", responses)
+ }
+ if len(publisher.messages) != 1 {
+ t.Fatalf("published messages = %d, want 1", len(publisher.messages))
+ }
+ task, err := dao.NewIngestionTaskDAO().GetByDocumentID(t.Context(), db, "doc-1")
+ if err != nil {
+ t.Fatalf("load queued task: %v", err)
+ }
+ if task == nil || task.Status != common.SCHEDULED {
+ t.Fatalf("queued task = %+v, want SCHEDULED", task)
+ }
+ if got := countPipelineLogs(t, db, "doc-1"); got != 0 {
+ t.Fatalf("pipeline log rows = %d, want 0 when metadata lookup fails", got)
+ }
+}
+
func TestIngestionTaskServiceMarksTaskScheduledOnlyAfterPublish(t *testing.T) {
db := setupServiceTestDB(t)
pushServiceDB(t, db)
@@ -844,6 +878,8 @@ func TestIngestionTaskServiceCreateAndEnqueueRejectsActiveExistingTask(t *testin
func TestIngestionTaskServiceCreateAndEnqueueRollsBackNewTaskOnPublishFailure(t *testing.T) {
db := setupServiceTestDB(t)
pushServiceDB(t, db)
+ insertTestKB(t, "kb-1", "tenant-1", 1, 0, 0)
+ insertTestDoc(t, "doc-1", "kb-1", 0, 0)
publisher := &recordingTaskPublisher{err: errors.New("publish failed")}
svc := NewIngestionTaskService()
svc.taskPublisher = publisher
@@ -865,11 +901,22 @@ func TestIngestionTaskServiceCreateAndEnqueueRollsBackNewTaskOnPublishFailure(t
if task != nil {
t.Fatalf("expected created task to be deleted after publish failure, got %+v", task)
}
+ open, openErr := dao.NewPipelineOperationLogDAO().GetOpenLogByDocumentID(ctx, db, "doc-1")
+ if openErr != nil {
+ t.Fatalf("load open pipeline log: %v", openErr)
+ }
+ if open != nil {
+ t.Fatalf("expected open pipeline log to be deleted after publish failure, got %+v", open)
+ }
}
func TestIngestionTaskServiceCreateAndEnqueueRollsBackRetriedTaskOnPublishFailure(t *testing.T) {
db := setupServiceTestDB(t)
pushServiceDB(t, db)
+ // Seed the document and KB so buildOpenLogInput succeeds: without them no
+ // open row is created and the rollback cleanup below is never exercised.
+ insertTestKB(t, "kb-1", "tenant-1", 1, 0, 0)
+ insertTestDoc(t, "doc-1", "kb-1", 0, 0)
insertTestIngestionTask(t, "task-1", "user-1", "doc-1", "kb-1")
if err := dao.DB.Model(&entity.IngestionTask{}).Where("id = ?", "task-1").Update("status", common.FAILED).Error; err != nil {
t.Fatalf("set failed status: %v", err)
@@ -896,16 +943,44 @@ func TestIngestionTaskServiceCreateAndEnqueueRollsBackRetriedTaskOnPublishFailur
if reloaded.Status != common.FAILED {
t.Fatalf("status = %q, want %q", reloaded.Status, common.FAILED)
}
+ // The retry opened an open row before publishing; the failed publish must
+ // take it back with the rolled-back task.
+ if got := countPipelineLogs(t, db, "doc-1"); got != 0 {
+ t.Fatalf("pipeline log rows = %d, want 0 after the retry rollback", got)
+ }
}
func TestIngestionTaskServiceRemoveDeletesOwnedTask(t *testing.T) {
db := setupServiceTestDB(t)
pushServiceDB(t, db)
+ insertTestKB(t, "kb-1", "tenant-1", 1, 0, 0)
+ insertTestDoc(t, "doc-1", "kb-1", 0, 0)
insertTestIngestionTask(t, "task-1", "user-1", "doc-1", "kb-1")
ctx := t.Context()
- userID := "user-1"
svc := NewIngestionTaskService()
+ queuedMsg := "Task is queued..."
+ openLog := &entity.PipelineOperationLog{
+ ID: "open-log",
+ DocumentID: "doc-1",
+ TenantID: "tenant-1",
+ KbID: "kb-1",
+ ParserID: "naive",
+ TaskType: "Parse",
+ OperationStatus: string(entity.TaskStatusUnstart),
+ ProgressMsg: &queuedMsg,
+ }
+ if err := dao.DB.Create(openLog).Error; err != nil {
+ t.Fatalf("seed open log: %v", err)
+ }
+ // Removal cleanup is scoped to the row the task owns, as bound in
+ // production by createOpenLogBestEffort.
+ if err := dao.DB.Model(&entity.IngestionTask{}).Where("id = ?", "task-1").
+ Update("pipeline_log_id", "open-log").Error; err != nil {
+ t.Fatalf("bind open log: %v", err)
+ }
+
+ userID := "user-1"
info, err := svc.Remove(ctx, "task-1", &userID)
if err != nil {
t.Fatalf("Remove failed: %v", err)
@@ -916,6 +991,13 @@ func TestIngestionTaskServiceRemoveDeletesOwnedTask(t *testing.T) {
if _, err = dao.NewIngestionTaskDAO().GetByID(ctx, db, "task-1"); err == nil {
t.Fatal("task should be removed")
}
+ open, err := dao.NewPipelineOperationLogDAO().GetOpenLogByDocumentID(ctx, db, "doc-1")
+ if err != nil {
+ t.Fatalf("load open pipeline log: %v", err)
+ }
+ if open != nil {
+ t.Fatalf("expected open row to be deleted with the task, got %+v", open)
+ }
}
func TestIngestionTaskServiceUpdateComponentTotalPersistsDenominator(t *testing.T) {
@@ -1252,3 +1334,576 @@ func TestIngestionTaskServiceMarkCompletedIdempotentOnAlreadyTerminal(t *testing
t.Fatalf("MarkCompleted on already FAILED task should be idempotent, got: %v", err)
}
}
+
+func loadOpenPipelineLog(t *testing.T, ctx context.Context, db *gorm.DB, documentID string) *entity.PipelineOperationLog {
+ t.Helper()
+ open, err := dao.NewPipelineOperationLogDAO().GetOpenLogByDocumentID(ctx, db, documentID)
+ if err != nil {
+ t.Fatalf("load open pipeline log: %v", err)
+ }
+ if open == nil {
+ t.Fatalf("expected open pipeline log for document %s", documentID)
+ }
+ return open
+}
+
+func countPipelineLogs(t *testing.T, db *gorm.DB, documentID string) int {
+ t.Helper()
+ var count int64
+ if err := db.Model(&entity.PipelineOperationLog{}).Where("document_id = ?", documentID).Count(&count).Error; err != nil {
+ t.Fatalf("count pipeline logs: %v", err)
+ }
+ return int(count)
+}
+
+func TestIngestionTaskServiceCreateForDocumentsOpensPreTerminalPipelineLog(t *testing.T) {
+ db := setupServiceTestDB(t)
+ pushServiceDB(t, db)
+ insertTestKB(t, "kb-1", "tenant-1", 1, 0, 0)
+ insertTestDoc(t, "doc-1", "kb-1", 0, 0)
+
+ svc := NewIngestionTaskService()
+ svc.taskPublisher = &recordingTaskPublisher{}
+ ctx := t.Context()
+ if _, err := svc.CreateForDocuments(ctx, "kb-1", "user-1", []string{"doc-1"}); err != nil {
+ t.Fatalf("CreateForDocuments failed: %v", err)
+ }
+
+ open := loadOpenPipelineLog(t, ctx, db, "doc-1")
+ if open.OperationStatus != string(entity.TaskStatusSchedule) {
+ t.Fatalf("OperationStatus = %q, want %q (scheduled write wins over created)", open.OperationStatus, string(entity.TaskStatusSchedule))
+ }
+ if open.KbID != "kb-1" || open.TenantID != "tenant-1" {
+ t.Fatalf("unexpected kb/tenant scope: %+v", open)
+ }
+ if open.ProgressMsg == nil || *open.ProgressMsg != "Task is queued..." {
+ t.Fatalf("ProgressMsg = %v, want queued message", open.ProgressMsg)
+ }
+ if countPipelineLogs(t, db, "doc-1") != 1 {
+ t.Fatalf("expected exactly 1 pipeline log row for one run")
+ }
+ // The task must be bound to the row so the terminal writer knows which row
+ // belongs to this run.
+ task, err := dao.NewIngestionTaskDAO().GetByDocumentID(ctx, db, "doc-1")
+ if err != nil {
+ t.Fatalf("reload task: %v", err)
+ }
+ if task.PipelineLogID == nil || *task.PipelineLogID != open.ID {
+ t.Fatalf("task PipelineLogID = %v, want %q (row bound to the owning run)", task.PipelineLogID, open.ID)
+ }
+}
+
+// loadTaskPipelineLogID returns the pipeline-operation-log id bound to a task.
+func loadTaskPipelineLogID(t *testing.T, ctx context.Context, db *gorm.DB, taskID string) string {
+ t.Helper()
+ task, err := dao.NewIngestionTaskDAO().GetByID(ctx, db, taskID)
+ if err != nil {
+ t.Fatalf("load task %s: %v", taskID, err)
+ }
+ if task.PipelineLogID == nil {
+ t.Fatalf("task %s has no bound pipeline log", taskID)
+ }
+ return *task.PipelineLogID
+}
+
+func TestIngestionTaskServiceStartRunningAdvancesPreTerminalPipelineLog(t *testing.T) {
+ db := setupServiceTestDB(t)
+ pushServiceDB(t, db)
+ insertTestKB(t, "kb-1", "tenant-1", 1, 0, 0)
+ insertTestDoc(t, "doc-1", "kb-1", 0, 0)
+
+ svc := NewIngestionTaskService()
+ svc.taskPublisher = &recordingTaskPublisher{}
+ ctx := t.Context()
+ resp, err := svc.CreateForDocuments(ctx, "kb-1", "user-1", []string{"doc-1"})
+ if err != nil {
+ t.Fatalf("CreateForDocuments failed: %v", err)
+ }
+ if len(resp) != 1 {
+ t.Fatalf("expected 1 response, got %d", len(resp))
+ }
+ open := loadOpenPipelineLog(t, ctx, db, "doc-1")
+ if open.OperationStatus != string(entity.TaskStatusSchedule) {
+ t.Fatalf("OperationStatus after schedule = %q, want %q", open.OperationStatus, string(entity.TaskStatusSchedule))
+ }
+ task, err := dao.NewIngestionTaskDAO().GetByDocumentID(ctx, db, "doc-1")
+ if err != nil {
+ t.Fatalf("reload task: %v", err)
+ }
+ if _, err := svc.StartRunning(ctx, task.ID); err != nil {
+ t.Fatalf("StartRunning failed: %v", err)
+ }
+ open = loadOpenPipelineLog(t, ctx, db, "doc-1")
+ if open.OperationStatus != string(entity.TaskStatusRunning) {
+ t.Fatalf("OperationStatus after start = %q, want %q", open.OperationStatus, string(entity.TaskStatusRunning))
+ }
+ if countPipelineLogs(t, db, "doc-1") != 1 {
+ t.Fatalf("expected running to reuse the queued row, not insert a second one")
+ }
+ if bound := loadTaskPipelineLogID(t, ctx, db, task.ID); bound != open.ID {
+ t.Fatalf("task bound to %q, want the same running row %q", bound, open.ID)
+ }
+}
+
+func TestIngestionTaskServiceRequestStopBeforeRunClosesPreTerminalPipelineLog(t *testing.T) {
+ db := setupServiceTestDB(t)
+ pushServiceDB(t, db)
+ insertTestKB(t, "kb-1", "tenant-1", 1, 0, 0)
+ insertTestDoc(t, "doc-1", "kb-1", 0, 0)
+
+ svc := NewIngestionTaskService()
+ svc.taskPublisher = &recordingTaskPublisher{}
+ ctx := t.Context()
+ if _, err := svc.CreateForDocuments(ctx, "kb-1", "user-1", []string{"doc-1"}); err != nil {
+ t.Fatalf("CreateForDocuments failed: %v", err)
+ }
+ loadOpenPipelineLog(t, ctx, db, "doc-1")
+ task, err := dao.NewIngestionTaskDAO().GetByDocumentID(ctx, db, "doc-1")
+ if err != nil {
+ t.Fatalf("reload task: %v", err)
+ }
+ stopped, err := svc.RequestStop(ctx, task.ID)
+ if err != nil {
+ t.Fatalf("RequestStop failed: %v", err)
+ }
+ if stopped.Status != common.STOPPED {
+ t.Fatalf("status = %q, want %q", stopped.Status, common.STOPPED)
+ }
+ var done entity.PipelineOperationLog
+ if err := db.Where("document_id = ?", "doc-1").First(&done).Error; err != nil {
+ t.Fatalf("load pipeline log: %v", err)
+ }
+ if done.OperationStatus != string(entity.TaskStatusCancel) {
+ t.Fatalf("OperationStatus = %q, want %q (stop without worker must close the row)", done.OperationStatus, string(entity.TaskStatusCancel))
+ }
+ if countPipelineLogs(t, db, "doc-1") != 1 {
+ t.Fatalf("expected stop to reuse the queued row, not insert a second one")
+ }
+}
+
+func TestIngestionTaskServiceRetryAfterTerminalOpensFreshPipelineLog(t *testing.T) {
+ db := setupServiceTestDB(t)
+ pushServiceDB(t, db)
+ insertTestKB(t, "kb-1", "tenant-1", 1, 0, 0)
+ insertTestDoc(t, "doc-1", "kb-1", 0, 0)
+ insertTestIngestionTask(t, "task-1", "user-1", "doc-1", "kb-1")
+ if err := dao.DB.Model(&entity.IngestionTask{}).Where("id = ?", "task-1").Update("status", common.FAILED).Error; err != nil {
+ t.Fatalf("set failed status: %v", err)
+ }
+
+ svc := NewIngestionTaskService()
+ svc.taskPublisher = &recordingTaskPublisher{}
+ ctx := t.Context()
+ finished := &entity.PipelineOperationLog{
+ ID: "finished-log",
+ DocumentID: "doc-1",
+ TenantID: "tenant-1",
+ KbID: "kb-1",
+ ParserID: "naive",
+ TaskType: "Parse",
+ OperationStatus: string(entity.TaskStatusFail),
+ }
+ if err := dao.DB.Create(finished).Error; err != nil {
+ t.Fatalf("seed finished log: %v", err)
+ }
+ if _, err := svc.CreateAndEnqueue(ctx, &entity.IngestionTask{
+ DocumentID: "doc-1",
+ UserID: "user-1",
+ DatasetID: "kb-1",
+ Status: common.CREATED,
+ }); err != nil {
+ t.Fatalf("CreateAndEnqueue retry failed: %v", err)
+ }
+ open := loadOpenPipelineLog(t, ctx, db, "doc-1")
+ if open.ID == "finished-log" {
+ t.Fatalf("retry reused the finished row; want a fresh queued row")
+ }
+ if countPipelineLogs(t, db, "doc-1") != 2 {
+ t.Fatalf("expected 2 pipeline log rows (finished + fresh queued)")
+ }
+}
+
+// TestIngestionTaskServiceOpensPreTerminalLogBeforePublish locks the ordering that
+// closes the orphan window: the run's row must exist before its message is
+// published, so a worker that claims and finishes the task immediately still
+// finds a row to terminalize.
+func TestIngestionTaskServiceOpensPreTerminalLogBeforePublish(t *testing.T) {
+ db := setupServiceTestDB(t)
+ pushServiceDB(t, db)
+ insertTestKB(t, "kb-1", "tenant-1", 1, 0, 0)
+ insertTestDoc(t, "doc-1", "kb-1", 0, 0)
+
+ var logIDAtPublish string
+ publisher := &recordingTaskPublisher{
+ beforeReturn: func(taskID string) {
+ task, err := dao.NewIngestionTaskDAO().GetByID(t.Context(), db, taskID)
+ if err != nil {
+ t.Fatalf("load task at publish: %v", err)
+ }
+ if task.PipelineLogID == nil {
+ t.Fatal("task was published before its open log row was bound")
+ }
+ logIDAtPublish = *task.PipelineLogID
+ },
+ }
+ svc := NewIngestionTaskService()
+ svc.taskPublisher = publisher
+
+ if _, err := svc.CreateForDocuments(t.Context(), "kb-1", "user-1", []string{"doc-1"}); err != nil {
+ t.Fatalf("CreateForDocuments failed: %v", err)
+ }
+ if logIDAtPublish == "" {
+ t.Fatal("publish did not observe a bound open log row")
+ }
+ open := loadOpenPipelineLog(t, t.Context(), db, "doc-1")
+ if open.ID != logIDAtPublish {
+ t.Fatalf("open row = %q, want the row bound before publish %q", open.ID, logIDAtPublish)
+ }
+}
+
+// TestIngestionTaskServiceStartRunningClosingStoppingTaskClosesPreTerminalLog locks
+// the workerless STOPPING finalize: MQ redelivery of a STOPPING task stops it
+// without ever running a terminal pipeline-log writer, so StartRunning must
+// close the row itself.
+func TestIngestionTaskServiceStartRunningClosingStoppingTaskClosesPreTerminalLog(t *testing.T) {
+ db := setupServiceTestDB(t)
+ pushServiceDB(t, db)
+ insertTestKB(t, "kb-1", "tenant-1", 1, 0, 0)
+ insertTestDoc(t, "doc-1", "kb-1", 0, 0)
+ insertTestIngestionTaskWithStatus(t, "task-1", "user-1", "doc-1", "kb-1", common.STOPPING)
+
+ runningMsg := "Task is running..."
+ if err := dao.DB.Create(&entity.PipelineOperationLog{
+ ID: "running-log",
+ DocumentID: "doc-1",
+ TenantID: "tenant-1",
+ KbID: "kb-1",
+ ParserID: "naive",
+ TaskType: "Parse",
+ OperationStatus: string(entity.TaskStatusRunning),
+ ProgressMsg: &runningMsg,
+ }).Error; err != nil {
+ t.Fatalf("seed running log: %v", err)
+ }
+ if err := dao.DB.Model(&entity.IngestionTask{}).Where("id = ?", "task-1").
+ Update("pipeline_log_id", "running-log").Error; err != nil {
+ t.Fatalf("bind running log: %v", err)
+ }
+
+ svc := NewIngestionTaskService()
+ task, err := svc.StartRunning(t.Context(), "task-1")
+ if err != nil {
+ t.Fatalf("StartRunning failed: %v", err)
+ }
+ if task.Status != common.STOPPED {
+ t.Fatalf("status = %q, want %q", task.Status, common.STOPPED)
+ }
+ var done entity.PipelineOperationLog
+ if err := db.First(&done, "id = ?", "running-log").Error; err != nil {
+ t.Fatalf("load running log: %v", err)
+ }
+ if done.OperationStatus != string(entity.TaskStatusCancel) {
+ t.Fatalf("OperationStatus = %q, want %q (workerless stop must close the row)", done.OperationStatus, string(entity.TaskStatusCancel))
+ }
+}
+
+// TestIngestionTaskServiceKeepsBoundRowOverUnrelatedOpenRow locks the ownership
+// rule: a run reuses the row it is bound to, never an unrelated open row that
+// happens to exist for the same document.
+func TestIngestionTaskServiceKeepsBoundRowOverUnrelatedOpenRow(t *testing.T) {
+ db := setupServiceTestDB(t)
+ pushServiceDB(t, db)
+ insertTestKB(t, "kb-1", "tenant-1", 1, 0, 0)
+ insertTestDoc(t, "doc-1", "kb-1", 0, 0)
+ insertTestIngestionTaskWithStatus(t, "task-1", "user-1", "doc-1", "kb-1", common.CREATED)
+
+ queuedMsg := "Task is queued..."
+ seed := func(id, status string) {
+ t.Helper()
+ if err := dao.DB.Create(&entity.PipelineOperationLog{
+ ID: id,
+ DocumentID: "doc-1",
+ TenantID: "tenant-1",
+ KbID: "kb-1",
+ ParserID: "naive",
+ TaskType: "Parse",
+ OperationStatus: status,
+ ProgressMsg: &queuedMsg,
+ }).Error; err != nil {
+ t.Fatalf("seed %s: %v", id, err)
+ }
+ }
+ seed("bound-log", string(entity.TaskStatusUnstart))
+ seed("unrelated-log", string(entity.TaskStatusSchedule))
+ if err := dao.DB.Model(&entity.IngestionTask{}).Where("id = ?", "task-1").
+ Update("pipeline_log_id", "bound-log").Error; err != nil {
+ t.Fatalf("bind: %v", err)
+ }
+
+ svc := NewIngestionTaskService()
+ svc.taskPublisher = &recordingTaskPublisher{}
+ if _, err := svc.CreateAndEnqueue(t.Context(), &entity.IngestionTask{
+ DocumentID: "doc-1",
+ UserID: "user-1",
+ DatasetID: "kb-1",
+ Status: common.CREATED,
+ }); err != nil {
+ t.Fatalf("CreateAndEnqueue failed: %v", err)
+ }
+
+ if got := countPipelineLogs(t, db, "doc-1"); got != 2 {
+ t.Fatalf("pipeline log rows = %d, want 2 (bound row advanced, no third row created)", got)
+ }
+ if bound := loadTaskPipelineLogID(t, t.Context(), db, "task-1"); bound != "bound-log" {
+ t.Fatalf("task bound to %q, want its own row bound-log", bound)
+ }
+ var bound entity.PipelineOperationLog
+ if err := db.First(&bound, "id = ?", "bound-log").Error; err != nil {
+ t.Fatalf("load bound log: %v", err)
+ }
+ if bound.OperationStatus != string(entity.TaskStatusSchedule) {
+ t.Fatalf("bound row status = %q, want %q", bound.OperationStatus, string(entity.TaskStatusSchedule))
+ }
+}
+
+func TestIngestionTaskServiceStaleSnapshotKeepsEstablishedOpenLogBinding(t *testing.T) {
+ db := setupServiceTestDB(t)
+ pushServiceDB(t, db)
+ insertTestKB(t, "kb-1", "tenant-1", 1, 0, 0)
+ insertTestDoc(t, "doc-1", "kb-1", 0, 0)
+ insertTestIngestionTaskWithStatus(t, "task-1", "user-1", "doc-1", "kb-1", common.CREATED)
+
+ taskDAO := dao.NewIngestionTaskDAO()
+ firstSnapshot, err := taskDAO.GetByID(t.Context(), db, "task-1")
+ if err != nil {
+ t.Fatalf("load first task snapshot: %v", err)
+ }
+ staleSnapshot, err := taskDAO.GetByID(t.Context(), db, "task-1")
+ if err != nil {
+ t.Fatalf("load stale task snapshot: %v", err)
+ }
+
+ svc := NewIngestionTaskService()
+ svc.createOpenLogBestEffort(t.Context(), firstSnapshot, string(entity.TaskStatusUnstart), logMsgQueued, nil)
+ firstLogID := loadTaskPipelineLogID(t, t.Context(), db, "task-1")
+ svc.createOpenLogBestEffort(t.Context(), staleSnapshot, string(entity.TaskStatusUnstart), logMsgQueued, nil)
+
+ if got := loadTaskPipelineLogID(t, t.Context(), db, "task-1"); got != firstLogID {
+ t.Fatalf("stale snapshot replaced binding %q with %q", firstLogID, got)
+ }
+ if got := countPipelineLogs(t, db, "doc-1"); got != 1 {
+ t.Fatalf("pipeline log rows = %d, want the original single row", got)
+ }
+ var firstLog entity.PipelineOperationLog
+ if err := db.First(&firstLog, "id = ?", firstLogID).Error; err != nil {
+ t.Fatalf("established open log was removed: %v", err)
+ }
+}
+
+func TestIngestionTaskServiceDoesNotDeleteOpenLogOwnedByActiveTask(t *testing.T) {
+ db := setupServiceTestDB(t)
+ pushServiceDB(t, db)
+ insertTestKB(t, "kb-1", "tenant-1", 1, 0, 0)
+ insertTestDoc(t, "doc-1", "kb-1", 0, 0)
+ if err := db.Migrator().DropIndex(&entity.IngestionTask{}, "idx_ingestion_task_document_id"); err != nil {
+ t.Fatalf("drop document uniqueness to model an existing database: %v", err)
+ }
+ insertTestIngestionTaskWithStatus(t, "owner-task", "user-1", "doc-1", "kb-1", common.RUNNING)
+ insertTestIngestionTaskWithStatus(t, "new-task", "user-1", "doc-1", "kb-1", common.CREATED)
+
+ queuedMsg := logMsgRunning
+ ownedLog := &entity.PipelineOperationLog{
+ ID: "owned-log",
+ DocumentID: "doc-1",
+ TenantID: "tenant-1",
+ KbID: "kb-1",
+ ParserID: "naive",
+ TaskType: string(entity.PipelineTaskTypeParse),
+ OperationStatus: string(entity.TaskStatusRunning),
+ ProgressMsg: &queuedMsg,
+ }
+ if err := db.Create(ownedLog).Error; err != nil {
+ t.Fatalf("seed owned open log: %v", err)
+ }
+ if err := db.Model(&entity.IngestionTask{}).Where("id = ?", "owner-task").Update("pipeline_log_id", ownedLog.ID).Error; err != nil {
+ t.Fatalf("bind owner task: %v", err)
+ }
+
+ newTask, err := dao.NewIngestionTaskDAO().GetByID(t.Context(), db, "new-task")
+ if err != nil {
+ t.Fatalf("load new task: %v", err)
+ }
+ svc := NewIngestionTaskService()
+ svc.createOpenLogBestEffort(t.Context(), newTask, string(entity.TaskStatusUnstart), logMsgQueued, nil)
+
+ var reloaded entity.PipelineOperationLog
+ if err := db.First(&reloaded, "id = ?", ownedLog.ID).Error; err != nil {
+ t.Fatalf("active task's open log was deleted: %v", err)
+ }
+ if got := countPipelineLogs(t, db, "doc-1"); got != 1 {
+ t.Fatalf("pipeline log rows = %d, want only the active task's row", got)
+ }
+ reloadedTask, err := dao.NewIngestionTaskDAO().GetByID(t.Context(), db, "new-task")
+ if err != nil {
+ t.Fatalf("reload new task: %v", err)
+ }
+ if reloadedTask.PipelineLogID != nil {
+ t.Fatalf("new task bound to %q while another active task owns the open row", *reloadedTask.PipelineLogID)
+ }
+}
+
+// TestIngestionTaskServiceNewRunReplacesStaleOpenRow locks the pre-binding
+// window: a brand-new task arrives with no bound row, and an open row left over
+// for the document (a run whose cleanup failed) must not be adopted. Since
+// ingestion_task.document_id is unique, this task is the document's only task,
+// so such a row cannot belong to a live run — the new run must open its own row
+// and the stale one must not linger as a permanently queued entry.
+func TestIngestionTaskServiceNewRunReplacesStaleOpenRow(t *testing.T) {
+ db := setupServiceTestDB(t)
+ pushServiceDB(t, db)
+ insertTestKB(t, "kb-1", "tenant-1", 1, 0, 0)
+ insertTestDoc(t, "doc-1", "kb-1", 0, 0)
+
+ // An orphaned row in a pre-terminal state, from a run that no longer owns a
+ // task, with stale content the new run must not inherit.
+ staleMsg := "Task is running..."
+ if err := dao.DB.Create(&entity.PipelineOperationLog{
+ ID: "stale-log",
+ DocumentID: "doc-1",
+ TenantID: "tenant-1",
+ KbID: "kb-1",
+ ParserID: "naive",
+ TaskType: "Parse",
+ OperationStatus: string(entity.TaskStatusRunning),
+ ProgressMsg: &staleMsg,
+ PipelineTitle: strPtr("stale-pipeline"),
+ }).Error; err != nil {
+ t.Fatalf("seed stale log: %v", err)
+ }
+
+ svc := NewIngestionTaskService()
+ svc.taskPublisher = &recordingTaskPublisher{}
+ ctx := t.Context()
+ if _, err := svc.CreateForDocuments(ctx, "kb-1", "user-1", []string{"doc-1"}); err != nil {
+ t.Fatalf("CreateForDocuments failed: %v", err)
+ }
+
+ task, err := dao.NewIngestionTaskDAO().GetByDocumentID(ctx, db, "doc-1")
+ if err != nil {
+ t.Fatalf("reload task: %v", err)
+ }
+ if task.PipelineLogID == nil {
+ t.Fatal("task has no bound open log row")
+ }
+ if *task.PipelineLogID == "stale-log" {
+ t.Fatalf("task adopted the stale row %q; want a fresh row", *task.PipelineLogID)
+ }
+ if err := db.First(&entity.PipelineOperationLog{}, "id = ?", "stale-log").Error; !dao.IsNotFoundErr(err) {
+ t.Fatalf("stale open row survived; want it dropped, got err=%v", err)
+ }
+ if got := countPipelineLogs(t, db, "doc-1"); got != 1 {
+ t.Fatalf("pipeline log rows = %d, want 1 (stale dropped, fresh queued)", got)
+ }
+ open := loadOpenPipelineLog(t, ctx, db, "doc-1")
+ if open.ID != *task.PipelineLogID {
+ t.Fatalf("open row %q is not the bound row %q", open.ID, *task.PipelineLogID)
+ }
+ if open.OperationStatus != string(entity.TaskStatusSchedule) {
+ t.Fatalf("OperationStatus = %q, want %q (fresh row, not stale RUNNING)", open.OperationStatus, string(entity.TaskStatusSchedule))
+ }
+ if open.PipelineTitle != nil && *open.PipelineTitle == "stale-pipeline" {
+ t.Fatalf("new run inherited the stale row's content: %+v", open)
+ }
+}
+
+// TestIngestionTaskServicePreTerminalLogAdvanceIsMonotonic locks the monotonic
+// transition contract: a late queued write must not regress a row a concurrent
+// worker already moved to running.
+func TestIngestionTaskServicePreTerminalLogAdvanceIsMonotonic(t *testing.T) {
+ db := setupServiceTestDB(t)
+ pushServiceDB(t, db)
+ insertTestKB(t, "kb-1", "tenant-1", 1, 0, 0)
+ insertTestDoc(t, "doc-1", "kb-1", 0, 0)
+ insertTestIngestionTaskWithStatus(t, "task-1", "user-1", "doc-1", "kb-1", common.CREATED)
+
+ queuedMsg := "Task is queued..."
+ openLog := &entity.PipelineOperationLog{
+ ID: "log-1",
+ DocumentID: "doc-1",
+ TenantID: "tenant-1",
+ KbID: "kb-1",
+ ParserID: "naive",
+ TaskType: "Parse",
+ OperationStatus: string(entity.TaskStatusUnstart),
+ ProgressMsg: &queuedMsg,
+ }
+ if err := dao.DB.Create(openLog).Error; err != nil {
+ t.Fatalf("seed open log: %v", err)
+ }
+ if err := dao.DB.Model(&entity.IngestionTask{}).Where("id = ?", "task-1").Update("pipeline_log_id", "log-1").Error; err != nil {
+ t.Fatalf("bind task: %v", err)
+ }
+
+ svc := NewIngestionTaskService()
+ ctx := t.Context()
+ task, err := dao.NewIngestionTaskDAO().GetByID(ctx, db, "task-1")
+ if err != nil {
+ t.Fatalf("load task: %v", err)
+ }
+
+ svc.advanceOpenLog(ctx, task, logFromUnstart, string(entity.TaskStatusSchedule), logMsgQueued, true)
+ if got := loadOpenPipelineLog(t, ctx, db, "doc-1").OperationStatus; got != string(entity.TaskStatusSchedule) {
+ t.Fatalf("after schedule = %q, want %q", got, string(entity.TaskStatusSchedule))
+ }
+ svc.advanceOpenLog(ctx, task, logFromUnstartOrScheduled, string(entity.TaskStatusRunning), logMsgRunning, true)
+ if got := loadOpenPipelineLog(t, ctx, db, "doc-1").OperationStatus; got != string(entity.TaskStatusRunning) {
+ t.Fatalf("after start = %q, want %q", got, string(entity.TaskStatusRunning))
+ }
+ // The enqueue-time scheduled write lands late; it must not regress the row.
+ svc.advanceOpenLog(ctx, task, logFromUnstart, string(entity.TaskStatusSchedule), logMsgQueued, true)
+ open := loadOpenPipelineLog(t, ctx, db, "doc-1")
+ if open.OperationStatus != string(entity.TaskStatusRunning) {
+ t.Fatalf("late scheduled write regressed the row to %q, want %q", open.OperationStatus, string(entity.TaskStatusRunning))
+ }
+ if open.ID != "log-1" || countPipelineLogs(t, db, "doc-1") != 1 {
+ t.Fatalf("expected the same single row, got id=%q count=%d", open.ID, countPipelineLogs(t, db, "doc-1"))
+ }
+}
+
+// TestIngestionTaskServiceAdvanceReopenGuard locks the reopen contract: a
+// missing open row is re-created only while the task is still live. A terminal
+// task must never resurrect its closed row as a permanently queued entry.
+func TestIngestionTaskServiceAdvanceReopenGuard(t *testing.T) {
+ db := setupServiceTestDB(t)
+ pushServiceDB(t, db)
+ insertTestKB(t, "kb-1", "tenant-1", 1, 0, 0)
+ insertTestDoc(t, "doc-terminal", "kb-1", 0, 0)
+ insertTestDoc(t, "doc-live", "kb-1", 0, 0)
+ insertTestIngestionTaskWithStatus(t, "task-terminal", "user-1", "doc-terminal", "kb-1", common.STOPPED)
+ insertTestIngestionTaskWithStatus(t, "task-live", "user-1", "doc-live", "kb-1", common.CREATED)
+
+ svc := NewIngestionTaskService()
+ ctx := t.Context()
+
+ terminal, err := dao.NewIngestionTaskDAO().GetByID(ctx, db, "task-terminal")
+ if err != nil {
+ t.Fatalf("load terminal task: %v", err)
+ }
+ svc.advanceOpenLog(ctx, terminal, logFromUnstart, string(entity.TaskStatusSchedule), logMsgQueued, true)
+ if got := countPipelineLogs(t, db, "doc-terminal"); got != 0 {
+ t.Fatalf("terminal task resurrected %d queued row(s); want none", got)
+ }
+
+ live, err := dao.NewIngestionTaskDAO().GetByID(ctx, db, "task-live")
+ if err != nil {
+ t.Fatalf("load live task: %v", err)
+ }
+ svc.advanceOpenLog(ctx, live, logFromUnstart, string(entity.TaskStatusSchedule), logMsgQueued, true)
+ open := loadOpenPipelineLog(t, ctx, db, "doc-live")
+ if open.OperationStatus != string(entity.TaskStatusSchedule) {
+ t.Fatalf("OperationStatus = %q, want %q (reopen for a live task)", open.OperationStatus, string(entity.TaskStatusSchedule))
+ }
+ if bound := loadTaskPipelineLogID(t, ctx, db, "task-live"); bound != open.ID {
+ t.Fatalf("task bound to %q, want reopened row %q", bound, open.ID)
+ }
+}
diff --git a/internal/service/test_helpers_test.go b/internal/service/test_helpers_test.go
index e35bb3c8e6..bafdadd48d 100644
--- a/internal/service/test_helpers_test.go
+++ b/internal/service/test_helpers_test.go
@@ -49,6 +49,8 @@ func setupServiceTestDB(t *testing.T) *gorm.DB {
&entity.Task{},
&entity.IngestionTask{},
&entity.IngestionTaskLog{},
+ &entity.PipelineOperationLog{},
+ &entity.UserCanvas{},
&entity.File2Document{},
&entity.File{},
&entity.User{},
diff --git a/web/src/pages/dataset/dataset/hooks.test.ts b/web/src/pages/dataset/dataset/hooks.test.ts
new file mode 100644
index 0000000000..2d2f4a19cb
--- /dev/null
+++ b/web/src/pages/dataset/dataset/hooks.test.ts
@@ -0,0 +1,185 @@
+import { QueryClient, QueryClientProvider } from '@tanstack/react-query';
+import { act, renderHook, waitFor } from '@testing-library/react';
+import React from 'react';
+import { IngestionTaskStatus, RunningStatus } from '@/constants/knowledge';
+
+// NOTE: this suite uses jest.mock, so it is transformed by esbuild-jest's
+// babel pipeline, which does not strip TypeScript. Imported bindings must not
+// appear in type annotations, so the document shape is declared locally and
+// the hook input is cast at the call site.
+type Doc = {
+ id: string;
+ name: string;
+ run: string;
+ progress_msg: string;
+ ingestion_status?: string;
+ [key: string]: unknown;
+};
+
+// The hook branches on the backend variant; drive it from a mutable flag so a
+// single suite can exercise both the Python and the Go path.
+let mockIsGo = false;
+
+jest.mock('react-router', () => ({
+ useParams: jest.fn(() => ({ id: 'kb-1' })),
+}));
+
+jest.mock('@/hooks/route-hook', () => ({
+ useGetKnowledgeSearchParams: jest.fn(() => ({ knowledgeId: 'kb-1' })),
+}));
+
+jest.mock('@/utils/backend-variant', () => ({
+ useIsGoBackend: () => mockIsGo,
+ pickByBackend: (variants: { go: unknown; python: unknown }) =>
+ mockIsGo ? variants.go : variants.python,
+}));
+
+jest.mock('@/hooks/use-document-request', () => ({
+ useFetchDocumentsByIds: () => ({ documents: [] }),
+}));
+
+jest.mock('@/services/knowledge-service', () => ({
+ listDataPipelineLogDocument: jest.fn(),
+}));
+
+import { listDataPipelineLogDocument } from '@/services/knowledge-service';
+import { useShowLog } from './hooks';
+
+const mockList = jest.mocked(listDataPipelineLogDocument);
+
+function makeDoc(overrides: Partial<Doc> = {}): Doc {
+ return {
+ id: 'doc-1',
+ name: 'a.pdf',
+ run: RunningStatus.UNSTART,
+ progress_msg: '',
+ ...overrides,
+ };
+}
+
+// eslint-disable-next-line @typescript-eslint/no-explicit-any
+function Wrapper(props: { children: any }) {
+ const client = new QueryClient({
+ defaultOptions: { queries: { retry: false } },
+ });
+ return React.createElement(QueryClientProvider, { client }, props.children);
+}
+
+// eslint-disable-next-line @typescript-eslint/no-explicit-any
+function renderLogs(documents: Doc[]): any {
+ // eslint-disable-next-line @typescript-eslint/no-explicit-any
+ return renderHook(() => useShowLog(documents as any), { wrapper: Wrapper });
+}
+
+beforeEach(() => {
+ jest.clearAllMocks();
+ mockIsGo = false;
+});
+
+describe('useShowLog — Python backend is unaffected by the early-log fallback', () => {
+ it('never queries pipeline logs and passes document.progress_msg through', () => {
+ mockIsGo = false;
+ const doc = makeDoc({
+ run: RunningStatus.RUNNING,
+ progress_msg: 'Parsing chunks',
+ });
+
+ const { result } = renderLogs([doc]);
+ act(() => result.current.showLog(doc));
+
+ expect(result.current.logInfo.details).toBe('Parsing chunks');
+ expect(result.current.logInfo.status).toBe(RunningStatus.RUNNING);
+ expect(mockList).not.toHaveBeenCalled();
+ });
+
+ it('leaves an empty progress_msg empty, exactly as before', () => {
+ mockIsGo = false;
+ const doc = makeDoc({ run: RunningStatus.UNSTART, progress_msg: '' });
+
+ const { result } = renderLogs([doc]);
+ act(() => result.current.showLog(doc));
+
+ // details is the document's own progress_msg, verbatim (the legacy "-"
+ // placeholder only applies when there is no source document at all).
+ expect(result.current.logInfo.details).toBe(doc.progress_msg);
+ expect(mockList).not.toHaveBeenCalled();
+ });
+
+ it('keeps the legacy "-" placeholder when there is no source document', () => {
+ mockIsGo = false;
+
+ const { result } = renderLogs([]);
+
+ expect(result.current.logInfo.details).toBe('-');
+ expect(mockList).not.toHaveBeenCalled();
+ });
+});
+
+describe('useShowLog — Go backend early-log fallback', () => {
+ it('shows the queued message from the early pipeline log row', async () => {
+ mockIsGo = true;
+ // eslint-disable-next-line @typescript-eslint/no-explicit-any
+ mockList.mockResolvedValue({
+ data: {
+ data: {
+ logs: [{ document_id: 'doc-1', progress_msg: 'Task is queued...' }],
+ total: 1,
+ },
+ },
+ } as any);
+
+ const doc = makeDoc({ ingestion_status: IngestionTaskStatus.CREATED });
+ const { result } = renderLogs([doc]);
+ act(() => result.current.showLog(doc));
+
+ await waitFor(() =>
+ expect(result.current.logInfo.details).toBe('Task is queued...'),
+ );
+ expect(result.current.logInfo.status).toBe(RunningStatus.QUEUED);
+ expect(mockList).toHaveBeenCalledWith(
+ 'kb-1',
+ expect.objectContaining({ log_type: 'file' }),
+ );
+ });
+
+ it('scopes the query to the document instead of a fuzzy name search', async () => {
+ mockIsGo = true;
+ // eslint-disable-next-line @typescript-eslint/no-explicit-any
+ mockList.mockResolvedValue({
+ data: { data: { logs: [], total: 0 } },
+ } as any);
+
+ const doc = makeDoc({
+ name: 'a.pdf',
+ ingestion_status: IngestionTaskStatus.SCHEDULED,
+ });
+ const { result } = renderLogs([doc]);
+ act(() => result.current.showLog(doc));
+
+ await waitFor(() => expect(mockList).toHaveBeenCalled());
+ expect(mockList).toHaveBeenCalledWith(
+ 'kb-1',
+ expect.objectContaining({ document_id: 'doc-1', log_type: 'file' }),
+ );
+ // The name search would return other documents' rows and could push this
+ // document's queued row past the first page.
+ expect(mockList.mock.calls[0][1]).not.toHaveProperty('keywords');
+ // No row came back, so the document's own (empty) progress_msg stands.
+ expect(result.current.logInfo.details).toBe('');
+ });
+
+ it('does not query once the document is running', () => {
+ mockIsGo = true;
+ const doc = makeDoc({
+ ingestion_status: IngestionTaskStatus.RUNNING,
+ run: RunningStatus.RUNNING,
+ progress_msg: 'Indexing done',
+ });
+
+ const { result } = renderLogs([doc]);
+ act(() => result.current.showLog(doc));
+
+ expect(result.current.logInfo.details).toBe('Indexing done');
+ expect(mockList).not.toHaveBeenCalled();
+ });
+});
diff --git a/web/src/pages/dataset/dataset/hooks.ts b/web/src/pages/dataset/dataset/hooks.ts
index d0b5866599..88ac71959c 100644
--- a/web/src/pages/dataset/dataset/hooks.ts
+++ b/web/src/pages/dataset/dataset/hooks.ts
@@ -1,17 +1,36 @@
import { useSetModalState } from '@/hooks/common-hooks';
import { useFetchDocumentsByIds } from '@/hooks/use-document-request';
import { IDocumentInfo } from '@/interfaces/database/document';
+import { useGetKnowledgeSearchParams } from '@/hooks/route-hook';
+import { useIsGoBackend } from '@/utils/backend-variant';
import { formatDate, formatSecondsToHumanReadable } from '@/utils/date';
import { formatBytes } from '@/utils/file-util';
+import { useQuery } from '@tanstack/react-query';
import { useCallback, useMemo, useState } from 'react';
+import { useParams } from 'react-router';
+import { listDataPipelineLogDocument } from '@/services/knowledge-service';
import { ILogInfo } from '../process-log-modal';
-import { getDocumentRunningStatus } from './utils';
+import { RunningStatus } from './constant';
+import {
+ getDocumentRunningStatus,
+ ingestionStatusToRunningStatus,
+} from './utils';
+import type { IFileLogList } from '../dataset-overview/interface';
const PollIntervalMs = 5000;
+export const DocumentLogKeys = {
+ queued: (datasetId: string | undefined, documentId: string | undefined) =>
+ ['queuedDocumentLog', datasetId, documentId] as const,
+};
+
export const useShowLog = (documents: IDocumentInfo[]) => {
const { showModal, hideModal, visible } = useSetModalState();
const [record, setRecord] = useState<IDocumentInfo>();
+ const { id: routeId } = useParams();
+ const { knowledgeId } = useGetKnowledgeSearchParams();
+ const datasetId = knowledgeId || routeId;
+ const isGoBackend = useIsGoBackend();
// When the modal is visible, poll the document directly by ID so progress_msg
// updates (e.g. "Indexing done") are captured even if the parent list no longer
@@ -22,16 +41,58 @@ export const useShowLog = (documents: IDocumentInfo[]) => {
{ enabled: visible, refetchInterval: PollIntervalMs },
);
const liveDoc = liveDocs?.[0];
+ const sourceDoc =
+ liveDoc ??
+ documents.find((item: IDocumentInfo) => item.id === record?.id) ??
+ record;
+ const queued =
+ isGoBackend &&
+ !!sourceDoc &&
+ ingestionStatusToRunningStatus(sourceDoc.ingestion_status) ===
+ RunningStatus.QUEUED;
+
+ // The Go backend reports a queued document via ingestion_status while the
+ // legacy document.progress_msg stays empty until the worker starts. Fall
+ // back to the early pipeline-operation-log row the Go API writes at task
+ // creation, so the queued modal shows "Task is queued..." instead of "-".
+ // Python never sets ingestion_status, so this query stays disabled there.
+ const { data: queuedLog } = useQuery<IFileLogList>({
+ queryKey: DocumentLogKeys.queued(datasetId, sourceDoc?.id),
+ enabled: visible && queued && !!datasetId && !!sourceDoc?.id,
+ refetchInterval: PollIntervalMs,
+ queryFn: async () => {
+ const { data: res = {} } = await listDataPipelineLogDocument(
+ datasetId || '',
+ {
+ page: 1,
+ page_size: 10,
+ // Exact match on the document: a name search is fuzzy and can push
+ // this document's row off the first page when several documents
+ // share a name, silently losing the queued message.
+ document_id: sourceDoc?.id,
+ log_type: 'file',
+ },
+ );
+ return (res.data || { logs: [], total: 0 }) as IFileLogList;
+ },
+ });
+ const queuedProgressMsg = useMemo(() => {
+ const logs = queuedLog?.logs ?? [];
+ // The endpoint filters by document_id, so the first row is this document's
+ // newest log. A miss falls back to the document progress_msg below.
+ return logs[0]?.progress_msg;
+ }, [queuedLog]);
+ // The queued fallback is only meaningful while the document is still
+ // queued. Once it starts running, the query cache may still hold the stale
+ // "Task is queued..." message and would shadow the live progress below.
+ const effectiveQueuedProgressMsg = queued ? queuedProgressMsg : undefined;
const logInfo = useMemo(() => {
- const source =
- liveDoc ??
- documents.find((item: IDocumentInfo) => item.id === record?.id) ??
- record;
+ const source = sourceDoc;
let log: ILogInfo = {
taskId: source?.id,
fileName: source?.name || '-',
- details: source?.progress_msg || '-',
+ details: effectiveQueuedProgressMsg || source?.progress_msg || '-',
};
if (source) {
log = {
@@ -46,11 +107,11 @@ export const useShowLog = (documents: IDocumentInfo[]) => {
// Go derives status from ingestion_status (queued included);
// Python reads the legacy run field.
status: getDocumentRunningStatus(source),
- details: source.progress_msg,
+ details: effectiveQueuedProgressMsg || source.progress_msg,
};
}
return log;
- }, [record, documents, liveDoc]);
+ }, [sourceDoc, effectiveQueuedProgressMsg]);
const showLog = useCallback(
(data: IDocumentInfo) => {
setRecord(data);