Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

feat(ingestion): surface queued document task status (#19137)

jay77721471dd91e43fc
diff --git a/internal/dao/database.go b/internal/dao/database.go
index 541731797c..af0fafc9be 100644
--- a/internal/dao/database.go
+++ b/internal/dao/database.go
@@ -180,6 +180,11 @@ func InitDB(ctx context.Context, migrateDB bool) error {
return fmt.Errorf("failed to run manual migrations: %w", err)
}
common.Info("Database schema migrated successfully")
+ } else {
+ // Ensure Go-exclusive runtime tables exist even if the server starts without --migrate
+ if err = autoMigrateRuntimeModels(ctx, DB); err != nil {
+ common.Warn("Failed to auto-migrate runtime models", zap.Error(err))
+ }
}
// Seed built-in agent templates so the Go backend can serve the
// "create agent from template" catalogue without relying on Python-side
@@ -289,3 +294,18 @@ func autoMigrateSafely(ctx context.Context, db *gorm.DB, model interface{}) erro
return err
}
+
+// autoMigrateRuntimeModels ensures Go-exclusive runtime tables exist even if
+// the server starts without --migrate.
+func autoMigrateRuntimeModels(ctx context.Context, db *gorm.DB) error {
+ goRuntimeModels := []interface{}{
+ &entity.IngestionTask{},
+ &entity.IngestionTaskLog{},
+ }
+ for _, m := range goRuntimeModels {
+ if err := autoMigrateSafely(ctx, db, m); err != nil {
+ return fmt.Errorf("failed to auto-migrate runtime model %T: %w", m, err)
+ }
+ }
+ return nil
+}
diff --git a/internal/dao/database_test.go b/internal/dao/database_test.go
new file mode 100644
index 0000000000..204f4bac86
--- /dev/null
+++ b/internal/dao/database_test.go
@@ -0,0 +1,60 @@
+//
+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+package dao
+
+import (
+ "context"
+ "testing"
+
+ "ragflow/internal/entity"
+
+ "github.com/glebarez/sqlite"
+ "gorm.io/gorm"
+)
+
+func TestAutoMigrateRuntimeModelsCreatesIngestionTaskTables(t *testing.T) {
+ db, err := gorm.Open(sqlite.Open(":memory:"), &gorm.Config{})
+ if err != nil {
+ t.Fatalf("open sqlite: %v", err)
+ }
+
+ // Verify tables do not exist initially
+ if db.Migrator().HasTable(&entity.IngestionTask{}) {
+ t.Fatal("expected ingestion_task to not exist initially")
+ }
+ if db.Migrator().HasTable(&entity.IngestionTaskLog{}) {
+ t.Fatal("expected ingestion_task_log to not exist initially")
+ }
+
+ ctx := context.Background()
+ if err = autoMigrateRuntimeModels(ctx, db); err != nil {
+ t.Fatalf("autoMigrateRuntimeModels failed: %v", err)
+ }
+
+ // Verify tables exist after auto migration
+ if !db.Migrator().HasTable(&entity.IngestionTask{}) {
+ t.Fatal("expected ingestion_task to exist after autoMigrateRuntimeModels")
+ }
+ if !db.Migrator().HasTable(&entity.IngestionTaskLog{}) {
+ t.Fatal("expected ingestion_task_log to exist after autoMigrateRuntimeModels")
+ }
+
+ // Verify idempotency
+ if err = autoMigrateRuntimeModels(ctx, db); err != nil {
+ t.Fatalf("second autoMigrateRuntimeModels failed: %v", err)
+ }
+}
diff --git a/internal/dao/document.go b/internal/dao/document.go
index 4c18747ac2..fe0f45e040 100644
--- a/internal/dao/document.go
+++ b/internal/dao/document.go
@@ -136,12 +136,28 @@ func (dao *DocumentDAO) ListByKBIDWithOptions(ctx context.Context, db *gorm.DB,
var documents []*entity.DocumentListItem
var total int64
+ // Historical retries can leave multiple ingestion tasks per document. Keep
+ // only the newest row, with ID as a deterministic tie-breaker for equal
+ // create times. This ordering must match IngestionTaskDAO's task lookups.
listQuery := db.WithContext(ctx).Table("document").
- Select(`document.*, user_canvas.title as pipeline_name, user.nickname`).
+ Select(`document.*, user_canvas.title as pipeline_name, user.nickname, ingestion_task.status as ingestion_status`).
Joins("JOIN file2document ON file2document.document_id = document.id").
Joins("JOIN file ON file.id = file2document.file_id").
Joins("LEFT JOIN user_canvas ON document.pipeline_id = user_canvas.id").
- Joins("LEFT JOIN user ON document.created_by = user.id")
+ Joins("LEFT JOIN user ON document.created_by = user.id").
+ Joins(`LEFT JOIN ingestion_task ON ingestion_task.document_id = document.id
+ AND NOT EXISTS (
+ SELECT 1
+ FROM ingestion_task newer_ingestion_task
+ WHERE newer_ingestion_task.document_id = ingestion_task.document_id
+ AND (
+ COALESCE(newer_ingestion_task.create_time, 0) > COALESCE(ingestion_task.create_time, 0)
+ OR (
+ COALESCE(newer_ingestion_task.create_time, 0) = COALESCE(ingestion_task.create_time, 0)
+ AND newer_ingestion_task.id > ingestion_task.id
+ )
+ )
+ )`)
listQuery = applyDocumentListFilters(listQuery, opts, true)
countQuery := applyDocumentListFilters(db.WithContext(ctx).Model(&entity.Document{}), opts, false)
diff --git a/internal/dao/document_test.go b/internal/dao/document_test.go
index 80dda70d2d..eec3f30a58 100644
--- a/internal/dao/document_test.go
+++ b/internal/dao/document_test.go
@@ -17,6 +17,7 @@
package dao
import (
+ "encoding/json"
"testing"
"github.com/glebarez/sqlite"
@@ -143,6 +144,173 @@ func TestDocumentGetByKBIDOrdersByCreateTime(t *testing.T) {
}
}
+func TestDocumentListIncludesScheduledIngestionStatus(t *testing.T) {
+ db := setupDocumentTestDB(t)
+ if err := db.AutoMigrate(
+ &entity.User{},
+ &entity.UserCanvas{},
+ &entity.File{},
+ &entity.File2Document{},
+ &entity.IngestionTask{},
+ ); err != nil {
+ t.Fatalf("migrate document-list dependencies: %v", err)
+ }
+ if err := db.Create(&entity.Document{
+ ID: "doc-scheduled",
+ KbID: "kb-1",
+ ParserID: "naive",
+ ParserConfig: entity.JSONMap{},
+ SourceType: "local",
+ Type: "document",
+ CreatedBy: "user-1",
+ Name: sp("scheduled.pdf"),
+ Suffix: "pdf",
+ }).Error; err != nil {
+ t.Fatalf("create document: %v", err)
+ }
+ if err := db.Create(&entity.File{
+ ID: "file-scheduled",
+ ParentID: "parent-1",
+ TenantID: "tenant-1",
+ CreatedBy: "user-1",
+ Name: "scheduled.pdf",
+ Type: "document",
+ }).Error; err != nil {
+ t.Fatalf("create file: %v", err)
+ }
+ if err := db.Create(&entity.File2Document{
+ ID: "link-scheduled",
+ FileID: sp("file-scheduled"),
+ DocumentID: sp("doc-scheduled"),
+ }).Error; err != nil {
+ t.Fatalf("link file to document: %v", err)
+ }
+ if err := db.Create(&entity.IngestionTask{
+ ID: "task-scheduled",
+ UserID: "user-1",
+ DocumentID: "doc-scheduled",
+ DatasetID: "kb-1",
+ Status: "SCHEDULED",
+ }).Error; err != nil {
+ t.Fatalf("create scheduled ingestion task: %v", err)
+ }
+
+ documents, total, err := NewDocumentDAO().ListByKBIDWithOptions(t.Context(), db, DocumentListOptions{
+ KbID: "kb-1",
+ OrderBy: "create_time",
+ Desc: true,
+ Offset: 0,
+ Limit: 10,
+ })
+ if err != nil {
+ t.Fatalf("list documents: %v", err)
+ }
+ if total != 1 || len(documents) != 1 {
+ t.Fatalf("listed %d documents (total %d), want 1", len(documents), total)
+ }
+
+ raw, err := json.Marshal(documents[0])
+ if err != nil {
+ t.Fatalf("marshal document list item: %v", err)
+ }
+ var listed map[string]interface{}
+ if err := json.Unmarshal(raw, &listed); err != nil {
+ t.Fatalf("unmarshal document list item: %v", err)
+ }
+ if got := listed["ingestion_status"]; got != "SCHEDULED" {
+ t.Fatalf("ingestion_status = %v, want %q", got, "SCHEDULED")
+ }
+}
+
+func TestDocumentListDeduplicatesHistoricalIngestionTasks(t *testing.T) {
+ db := setupDocumentTestDB(t)
+ if err := db.AutoMigrate(
+ &entity.User{},
+ &entity.UserCanvas{},
+ &entity.File{},
+ &entity.File2Document{},
+ &entity.IngestionTask{},
+ ); err != nil {
+ t.Fatalf("migrate document-list dependencies: %v", err)
+ }
+ if err := db.Exec("DROP INDEX idx_ingestion_task_document_id").Error; err != nil {
+ t.Fatalf("drop ingestion task unique index: %v", err)
+ }
+
+ if err := db.Create(&entity.Document{
+ ID: "doc-duplicate-tasks",
+ KbID: "kb-1",
+ ParserID: "naive",
+ ParserConfig: entity.JSONMap{},
+ SourceType: "local",
+ Type: "document",
+ CreatedBy: "user-1",
+ Name: sp("duplicate-tasks.pdf"),
+ Suffix: "pdf",
+ }).Error; err != nil {
+ t.Fatalf("create document: %v", err)
+ }
+ if err := db.Create(&entity.File{
+ ID: "file-duplicate-tasks",
+ ParentID: "parent-1",
+ TenantID: "tenant-1",
+ CreatedBy: "user-1",
+ Name: "duplicate-tasks.pdf",
+ Type: "document",
+ }).Error; err != nil {
+ t.Fatalf("create file: %v", err)
+ }
+ if err := db.Create(&entity.File2Document{
+ ID: "link-duplicate-tasks",
+ FileID: sp("file-duplicate-tasks"),
+ DocumentID: sp("doc-duplicate-tasks"),
+ }).Error; err != nil {
+ t.Fatalf("link file to document: %v", err)
+ }
+
+ oldTaskTime := int64(100)
+ newTaskTime := int64(200)
+ for _, task := range []*entity.IngestionTask{
+ {
+ ID: "task-old",
+ UserID: "user-1",
+ DocumentID: "doc-duplicate-tasks",
+ DatasetID: "kb-1",
+ Status: "FAILED",
+ BaseModel: entity.BaseModel{CreateTime: &oldTaskTime},
+ },
+ {
+ ID: "task-new",
+ UserID: "user-1",
+ DocumentID: "doc-duplicate-tasks",
+ DatasetID: "kb-1",
+ Status: "SCHEDULED",
+ BaseModel: entity.BaseModel{CreateTime: &newTaskTime},
+ },
+ } {
+ if err := db.Create(task).Error; err != nil {
+ t.Fatalf("create ingestion task %s: %v", task.ID, err)
+ }
+ }
+
+ documents, total, err := NewDocumentDAO().ListByKBIDWithOptions(t.Context(), db, DocumentListOptions{
+ KbID: "kb-1",
+ OrderBy: "create_time",
+ Desc: true,
+ Offset: 0,
+ Limit: 10,
+ })
+ if err != nil {
+ t.Fatalf("list documents: %v", err)
+ }
+ if total != 1 || len(documents) != 1 {
+ t.Fatalf("listed %d documents (total %d), want 1", len(documents), total)
+ }
+ if documents[0].IngestionStatus == nil || *documents[0].IngestionStatus != "SCHEDULED" {
+ t.Fatalf("ingestion status = %v, want %q", documents[0].IngestionStatus, "SCHEDULED")
+ }
+}
+
func TestDocumentGetByDocumentIDAndDatasetIDUsesKBID(t *testing.T) {
db := setupDocumentTestDB(t)
diff --git a/internal/dao/ingestion.go b/internal/dao/ingestion.go
index 46864a315a..fe6448036e 100644
--- a/internal/dao/ingestion.go
+++ b/internal/dao/ingestion.go
@@ -197,9 +197,16 @@ func (dao *IngestionTaskDAO) GetByID(ctx context.Context, db *gorm.DB, id string
return task, err
}
+// GetByDocumentID returns the latest ingestion task for a document. Historical
+// retries are ordered by create_time and then ID to match document-list state.
func (dao *IngestionTaskDAO) GetByDocumentID(ctx context.Context, db *gorm.DB, documentId string) (*entity.IngestionTask, error) {
var tasks []*entity.IngestionTask
- err := db.WithContext(ctx).Where("document_id = ?", documentId).Limit(1).Find(&tasks).Error
+ err := db.WithContext(ctx).
+ Where("document_id = ?", documentId).
+ Order("COALESCE(create_time, 0) DESC").
+ Order("id DESC").
+ Limit(1).
+ Find(&tasks).Error
if err != nil {
return nil, err
}
@@ -209,6 +216,31 @@ func (dao *IngestionTaskDAO) GetByDocumentID(ctx context.Context, db *gorm.DB, d
return tasks[0], nil
}
+// CountActiveByDatasetID returns the number of ingestion tasks for the
+// dataset whose latest task is non-terminal (CREATED/SCHEDULED/RUNNING/STOPPING).
+// It uses the same create_time/ID ordering as document-list state so historical
+// retries cannot keep polling alive after a newer task becomes terminal.
+func (dao *IngestionTaskDAO) CountActiveByDatasetID(ctx context.Context, db *gorm.DB, datasetID string) (int64, error) {
+ var count int64
+ err := db.WithContext(ctx).Model(&entity.IngestionTask{}).
+ Where(`ingestion_task.dataset_id = ?
+ AND ingestion_task.status IN ?
+ AND NOT EXISTS (
+ SELECT 1
+ FROM ingestion_task AS newer_ingestion_task
+ WHERE newer_ingestion_task.document_id = ingestion_task.document_id
+ AND (
+ COALESCE(newer_ingestion_task.create_time, 0) > COALESCE(ingestion_task.create_time, 0)
+ OR (
+ COALESCE(newer_ingestion_task.create_time, 0) = COALESCE(ingestion_task.create_time, 0)
+ AND newer_ingestion_task.id > ingestion_task.id
+ )
+ )
+ )`, datasetID, []string{common.CREATED, common.SCHEDULED, common.RUNNING, common.STOPPING}).
+ Count(&count).Error
+ return count, err
+}
+
// DeleteIfTerminal deletes ingestion tasks for a document that are in a
// terminal state (COMPLETED, STOPPED, FAILED), or not yet running
// (CREATED, SCHEDULED).
diff --git a/internal/dao/ingestion_task_test.go b/internal/dao/ingestion_task_test.go
index c0b4280dec..22ca81aa42 100644
--- a/internal/dao/ingestion_task_test.go
+++ b/internal/dao/ingestion_task_test.go
@@ -326,3 +326,175 @@ func TestIngestionTaskDAODeleteIfTerminal_RemovesOnlyTerminal(t *testing.T) {
}
}
}
+
+func TestIngestionTaskDAOCountActiveByDatasetID(t *testing.T) {
+ db := setupTaskTestDB(t)
+ for _, task := range []*entity.IngestionTask{
+ {ID: "task-active-1", UserID: "user-1", DocumentID: "doc-active-1", DatasetID: "kb-1", Status: common.SCHEDULED},
+ {ID: "task-active-2", UserID: "user-1", DocumentID: "doc-active-2", DatasetID: "kb-1", Status: common.RUNNING},
+ {ID: "task-active-3", UserID: "user-1", DocumentID: "doc-active-3", DatasetID: "kb-1", Status: common.CREATED},
+ {ID: "task-active-4", UserID: "user-1", DocumentID: "doc-active-4", DatasetID: "kb-1", Status: common.STOPPING},
+ {ID: "task-terminal", UserID: "user-1", DocumentID: "doc-terminal", DatasetID: "kb-1", Status: common.COMPLETED},
+ {ID: "task-other-kb", UserID: "user-1", DocumentID: "doc-other", DatasetID: "kb-2", Status: common.SCHEDULED},
+ } {
+ if err := db.Create(task).Error; err != nil {
+ t.Fatalf("create task %s: %v", task.ID, err)
+ }
+ }
+
+ ctx := t.Context()
+ count, err := NewIngestionTaskDAO().CountActiveByDatasetID(ctx, db, "kb-1")
+ if err != nil {
+ t.Fatalf("CountActiveByDatasetID: %v", err)
+ }
+ if count != 4 {
+ t.Fatalf("active count for kb-1 = %d, want 4", count)
+ }
+ count, err = NewIngestionTaskDAO().CountActiveByDatasetID(ctx, db, "kb-2")
+ if err != nil {
+ t.Fatalf("CountActiveByDatasetID kb-2: %v", err)
+ }
+ if count != 1 {
+ t.Fatalf("active count for kb-2 = %d, want 1", count)
+ }
+ count, err = NewIngestionTaskDAO().CountActiveByDatasetID(ctx, db, "kb-unknown")
+ if err != nil {
+ t.Fatalf("CountActiveByDatasetID unknown: %v", err)
+ }
+ if count != 0 {
+ t.Fatalf("active count for unknown = %d, want 0", count)
+ }
+}
+
+func TestIngestionTaskDAOCountActiveByDatasetIDUsesLatestTask(t *testing.T) {
+ db := setupTaskTestDB(t)
+ if err := db.Exec("DROP INDEX idx_ingestion_task_document_id").Error; err != nil {
+ t.Fatalf("drop ingestion task unique index: %v", err)
+ }
+
+ oldTaskTime := int64(100)
+ newTaskTime := int64(200)
+ for _, task := range []*entity.IngestionTask{
+ {
+ ID: "task-old-active",
+ UserID: "user-1",
+ DocumentID: "doc-retried",
+ DatasetID: "kb-1",
+ Status: common.RUNNING,
+ BaseModel: entity.BaseModel{CreateTime: &oldTaskTime},
+ },
+ {
+ ID: "task-new-terminal",
+ UserID: "user-1",
+ DocumentID: "doc-retried",
+ DatasetID: "kb-1",
+ Status: common.COMPLETED,
+ BaseModel: entity.BaseModel{CreateTime: &newTaskTime},
+ },
+ {
+ ID: "task-current-active",
+ UserID: "user-1",
+ DocumentID: "doc-current",
+ DatasetID: "kb-1",
+ Status: common.SCHEDULED,
+ },
+ } {
+ if err := db.Create(task).Error; err != nil {
+ t.Fatalf("create task %s: %v", task.ID, err)
+ }
+ }
+
+ count, err := NewIngestionTaskDAO().CountActiveByDatasetID(t.Context(), db, "kb-1")
+ if err != nil {
+ t.Fatalf("CountActiveByDatasetID: %v", err)
+ }
+ if count != 1 {
+ t.Fatalf("active count for kb-1 = %d, want 1 (latest task per document)", count)
+ }
+}
+
+func TestIngestionTaskDAOGetByDocumentIDUsesLatestTask(t *testing.T) {
+ db := setupTaskTestDB(t)
+ if err := db.Exec("DROP INDEX idx_ingestion_task_document_id").Error; err != nil {
+ t.Fatalf("drop ingestion task unique index: %v", err)
+ }
+
+ oldTaskTime := int64(100)
+ newTaskTime := int64(200)
+ for _, task := range []*entity.IngestionTask{
+ {
+ ID: "task-old",
+ UserID: "user-1",
+ DocumentID: "doc-retried",
+ DatasetID: "kb-1",
+ Status: common.RUNNING,
+ BaseModel: entity.BaseModel{CreateTime: &oldTaskTime},
+ },
+ {
+ ID: "task-new",
+ UserID: "user-1",
+ DocumentID: "doc-retried",
+ DatasetID: "kb-1",
+ Status: common.COMPLETED,
+ BaseModel: entity.BaseModel{CreateTime: &newTaskTime},
+ },
+ } {
+ if err := db.Create(task).Error; err != nil {
+ t.Fatalf("create task %s: %v", task.ID, err)
+ }
+ }
+
+ task, err := NewIngestionTaskDAO().GetByDocumentID(t.Context(), db, "doc-retried")
+ if err != nil {
+ t.Fatalf("GetByDocumentID: %v", err)
+ }
+ if task == nil || task.ID != "task-new" {
+ t.Fatalf("latest task = %+v, want task-new", task)
+ }
+}
+
+func TestIngestionTaskDAOGetByDocumentIDUsesIDAsTieBreaker(t *testing.T) {
+ db := setupTaskTestDB(t)
+ if err := db.Exec("DROP INDEX idx_ingestion_task_document_id").Error; err != nil {
+ t.Fatalf("drop ingestion task unique index: %v", err)
+ }
+
+ createTime := int64(100)
+ for _, task := range []*entity.IngestionTask{
+ {
+ ID: "task-z",
+ UserID: "user-1",
+ DocumentID: "doc-retried",
+ DatasetID: "kb-1",
+ Status: common.RUNNING,
+ BaseModel: entity.BaseModel{CreateTime: &createTime},
+ },
+ {
+ ID: "task-a",
+ UserID: "user-1",
+ DocumentID: "doc-retried",
+ DatasetID: "kb-1",
+ Status: common.COMPLETED,
+ BaseModel: entity.BaseModel{CreateTime: &createTime},
+ },
+ } {
+ if err := db.Create(task).Error; err != nil {
+ t.Fatalf("create task %s: %v", task.ID, err)
+ }
+ }
+
+ task, err := NewIngestionTaskDAO().GetByDocumentID(t.Context(), db, "doc-retried")
+ if err != nil {
+ t.Fatalf("GetByDocumentID: %v", err)
+ }
+ if task == nil || task.ID != "task-z" {
+ t.Fatalf("latest task = %+v, want task-z", task)
+ }
+}
+
+func TestIngestionTaskDAOHasDatasetStatusIndex(t *testing.T) {
+ db := setupTaskTestDB(t)
+ if !db.Migrator().HasIndex(&entity.IngestionTask{}, "idx_ingestion_task_dataset_status") {
+ t.Fatal("expected composite dataset/status index on ingestion_task")
+ }
+}
diff --git a/internal/entity/document.go b/internal/entity/document.go
index 168228c291..37b16d41b8 100644
--- a/internal/entity/document.go
+++ b/internal/entity/document.go
@@ -70,6 +70,7 @@ type DocumentListItem struct {
ContentHash *string `gorm:"column:content_hash" json:"content_hash,omitempty"`
Suffix string `gorm:"column:suffix" json:"suffix"`
Run *string `gorm:"column:run" json:"run,omitempty"`
+ IngestionStatus *string `gorm:"column:ingestion_status" json:"ingestion_status,omitempty"`
Status *string `gorm:"column:status" json:"status,omitempty"`
CreateTime *int64 `gorm:"column:create_time" json:"create_time,omitempty"`
CreateDate *time.Time `gorm:"column:create_date" json:"create_date,omitempty"`
diff --git a/internal/entity/ingestion_task.go b/internal/entity/ingestion_task.go
index 9746368d70..8676448466 100644
--- a/internal/entity/ingestion_task.go
+++ b/internal/entity/ingestion_task.go
@@ -20,9 +20,9 @@ type IngestionTask struct {
ID string `gorm:"column:id;primaryKey;size:32" json:"id"`
UserID string `gorm:"column:user_id;size:32;not null" json:"user_id"`
DocumentID string `gorm:"column:document_id;size:32;not null;uniqueIndex:idx_ingestion_task_document_id" json:"document_id"`
- DatasetID string `gorm:"column:dataset_id;size:32;not null" json:"dataset_id"`
+ DatasetID string `gorm:"column:dataset_id;size:32;not null;index:idx_ingestion_task_dataset_status,priority:1" json:"dataset_id"`
Schema JSONMap `gorm:"column:schema;type:longtext" json:"schema"`
- Status string `gorm:"column:status;size:32;not null;" json:"status"`
+ Status string `gorm:"column:status;size:32;not null;index:idx_ingestion_task_dataset_status,priority:2" json:"status"`
// ComponentTotal is the number of components in the task's DSL graph.
// It is the authoritative denominator for progress percentage so the
// frontend does not have to count DSL nodes itself. Written once the
diff --git a/internal/handler/document.go b/internal/handler/document.go
index 55b84e7398..bd2570a6f6 100644
--- a/internal/handler/document.go
+++ b/internal/handler/document.go
@@ -37,6 +37,7 @@ import (
"time"
"github.com/gin-gonic/gin"
+ "go.uber.org/zap"
"ragflow/internal/dao"
"ragflow/internal/service"
@@ -82,6 +83,7 @@ type documentServiceIface interface {
Ingest(ctx context.Context, userID string, req *document.IngestDocumentRequest) (common.ErrorCode, error)
RemoveIngestionTasks(ctx context.Context, tasks []string, userID string) ([]map[string]string, error)
BatchUpdateDocumentStatus(ctx context.Context, userID, datasetID, status string, DocumentIDs []string) (map[string]interface{}, common.ErrorCode, error)
+ HasActiveIngestionTasks(ctx context.Context, datasetID string) (bool, error)
}
// fileUploadIface defines the FileService upload methods used by DocumentHandler.
@@ -614,7 +616,14 @@ func (h *DocumentHandler) ListDocuments(c *gin.Context) {
docs = append(docs, mapDocumentListItem(doc, metaFields))
}
- common.SuccessWithData(c, gin.H{"total": total, "docs": docs}, "success")
+ hasActiveTasks, err := h.documentService.HasActiveIngestionTasks(ctx, datasetID)
+ if err != nil {
+ common.Warn("failed to check active ingestion tasks", zap.Error(err))
+ // Keep polling when the authoritative dataset-wide check is unavailable;
+ // otherwise an active task on another page could be missed.
+ hasActiveTasks = true
+ }
+ common.SuccessWithData(c, gin.H{"total": total, "docs": docs, "has_active_tasks": hasActiveTasks}, "success")
}
func parseDocumentListOptions(c *gin.Context, datasetID string) (dao.DocumentListOptions, string) {
@@ -1144,6 +1153,9 @@ func mapDocumentListItem(doc *entity.DocumentListItem, metaFields map[string]int
"update_time": int64(0),
"update_date": "",
}
+ if doc.IngestionStatus != nil {
+ item["ingestion_status"] = *doc.IngestionStatus
+ }
if doc.CreateTime != nil {
item["create_time"] = *doc.CreateTime
diff --git a/internal/handler/document_test.go b/internal/handler/document_test.go
index f6d06ca243..b795ea2e13 100644
--- a/internal/handler/document_test.go
+++ b/internal/handler/document_test.go
@@ -20,6 +20,7 @@ import (
"bytes"
"context"
"encoding/json"
+ "errors"
"fmt"
"mime/multipart"
"net/http"
@@ -80,7 +81,11 @@ type fakeDocumentService struct {
filterResult map[string]interface{}
filterTotal int64
listIDs []string
+ documentList []*entity.DocumentListItem
+ documentListTotal int64
metadataByKBs map[string]interface{}
+ hasActiveTasks bool
+ hasActiveTasksErr error
}
func (f *fakeDocumentService) Ingest(ctx context.Context, userID string, req *document.IngestDocumentRequest) (common.ErrorCode, error) {
@@ -171,7 +176,7 @@ func (f *fakeDocumentService) ListDocumentsByDatasetID(ctx context.Context, kbID
}
func (f *fakeDocumentService) ListDocumentsByDatasetIDWithOptions(ctx context.Context, opts dao.DocumentListOptions, page, pageSize int) ([]*entity.DocumentListItem, int64, error) {
f.listOpts = opts
- return nil, 0, nil
+ return f.documentList, f.documentListTotal, nil
}
func (f *fakeDocumentService) ListDocumentIDsByDatasetIDWithOptions(ctx context.Context, opts dao.DocumentListOptions) ([]string, error) {
f.listOpts = opts
@@ -209,6 +214,10 @@ func (f *fakeDocumentService) GetMetadataSummary(ctx context.Context, kbID strin
f.metadataDocIDs = docIDs
return f.metadataSummary, f.metadataErr
}
+func (f *fakeDocumentService) HasActiveIngestionTasks(ctx context.Context, datasetID string) (bool, error) {
+ return f.hasActiveTasks, f.hasActiveTasksErr
+}
+
func (f *fakeDocumentService) SetDocumentMetadata(ctx context.Context, docID string, meta map[string]interface{}) error {
f.setMetaCalled = true
f.setMetaDocID = docID
@@ -1108,6 +1117,243 @@ func TestListDocumentsHandler_FilterRequestUsesQueryFilters(t *testing.T) {
}
}
+func TestListDocumentsHandlerReturnsScheduledIngestionStatus(t *testing.T) {
+ db := setupHandlerAccessDB(t)
+ orig := dao.DB
+ dao.DB = db
+ t.Cleanup(func() { dao.DB = orig })
+
+ fake := &fakeDocumentService{
+ documentList: []*entity.DocumentListItem{{
+ ID: "doc-scheduled",
+ KbID: "ds-1",
+ ParserID: "naive",
+ ParserConfig: "{}",
+ SourceType: "local",
+ Type: "document",
+ CreatedBy: "user-1",
+ Name: sptr("scheduled.pdf"),
+ Suffix: "pdf",
+ IngestionStatus: sptr(common.SCHEDULED),
+ }},
+ documentListTotal: 1,
+ }
+ h := &DocumentHandler{
+ documentService: fake,
+ datasetService: dataset.NewDatasetService(),
+ }
+ c, w := setupGinContextWithUser("GET", "/api/v1/datasets/ds-1/documents", "")
+ c.Params = gin.Params{{Key: "dataset_id", Value: "ds-1"}}
+
+ h.ListDocuments(c)
+
+ if w.Code != http.StatusOK {
+ t.Fatalf("expected 200, got %d: %s", w.Code, w.Body.String())
+ }
+ var response struct {
+ Data struct {
+ Docs []map[string]interface{} `json:"docs"`
+ } `json:"data"`
+ }
+ if err := json.Unmarshal(w.Body.Bytes(), &response); err != nil {
+ t.Fatalf("unmarshal response: %v", err)
+ }
+ if len(response.Data.Docs) != 1 {
+ t.Fatalf("document count = %d, want 1", len(response.Data.Docs))
+ }
+ if got := response.Data.Docs[0]["ingestion_status"]; got != common.SCHEDULED {
+ t.Fatalf("ingestion_status = %v, want %q", got, common.SCHEDULED)
+ }
+}
+
+func TestListDocumentsHandlerOmitsEmptyIngestionStatus(t *testing.T) {
+ db := setupHandlerAccessDB(t)
+ orig := dao.DB
+ dao.DB = db
+ t.Cleanup(func() { dao.DB = orig })
+
+ fake := &fakeDocumentService{
+ documentList: []*entity.DocumentListItem{{
+ ID: "doc-without-task",
+ KbID: "ds-1",
+ ParserID: "naive",
+ ParserConfig: "{}",
+ SourceType: "local",
+ Type: "document",
+ CreatedBy: "user-1",
+ Name: sptr("without-task.pdf"),
+ Suffix: "pdf",
+ }},
+ documentListTotal: 1,
+ }
+ h := &DocumentHandler{
+ documentService: fake,
+ datasetService: dataset.NewDatasetService(),
+ }
+ c, w := setupGinContextWithUser("GET", "/api/v1/datasets/ds-1/documents", "")
+ c.Params = gin.Params{{Key: "dataset_id", Value: "ds-1"}}
+
+ h.ListDocuments(c)
+
+ if w.Code != http.StatusOK {
+ t.Fatalf("expected 200, got %d: %s", w.Code, w.Body.String())
+ }
+ var response struct {
+ Data struct {
+ Docs []map[string]interface{} `json:"docs"`
+ } `json:"data"`
+ }
+ if err := json.Unmarshal(w.Body.Bytes(), &response); err != nil {
+ t.Fatalf("unmarshal response: %v", err)
+ }
+ if len(response.Data.Docs) != 1 {
+ t.Fatalf("document count = %d, want 1", len(response.Data.Docs))
+ }
+ if _, exists := response.Data.Docs[0]["ingestion_status"]; exists {
+ t.Fatalf("ingestion_status should be omitted when no ingestion task exists")
+ }
+}
+
+func TestListDocumentsHandlerReturnsHasActiveTasks(t *testing.T) {
+ db := setupHandlerAccessDB(t)
+ orig := dao.DB
+ dao.DB = db
+ t.Cleanup(func() { dao.DB = orig })
+
+ fake := &fakeDocumentService{
+ documentList: []*entity.DocumentListItem{{
+ ID: "doc-1",
+ KbID: "ds-1",
+ ParserID: "naive",
+ ParserConfig: "{}",
+ SourceType: "local",
+ Type: "document",
+ CreatedBy: "user-1",
+ Name: sptr("doc.pdf"),
+ Suffix: "pdf",
+ }},
+ documentListTotal: 1,
+ hasActiveTasks: true,
+ }
+ h := &DocumentHandler{
+ documentService: fake,
+ datasetService: dataset.NewDatasetService(),
+ }
+ c, w := setupGinContextWithUser("GET", "/api/v1/datasets/ds-1/documents", "")
+ c.Params = gin.Params{{Key: "dataset_id", Value: "ds-1"}}
+
+ h.ListDocuments(c)
+
+ if w.Code != http.StatusOK {
+ t.Fatalf("expected 200, got %d: %s", w.Code, w.Body.String())
+ }
+ var response struct {
+ Data struct {
+ Docs []map[string]interface{} `json:"docs"`
+ HasActiveTasks bool `json:"has_active_tasks"`
+ } `json:"data"`
+ }
+ if err := json.Unmarshal(w.Body.Bytes(), &response); err != nil {
+ t.Fatalf("unmarshal response: %v", err)
+ }
+ if !response.Data.HasActiveTasks {
+ t.Fatalf("has_active_tasks = false, want true")
+ }
+}
+
+func TestListDocumentsHandlerReturnsHasActiveTasksFalse(t *testing.T) {
+ db := setupHandlerAccessDB(t)
+ orig := dao.DB
+ dao.DB = db
+ t.Cleanup(func() { dao.DB = orig })
+
+ fake := &fakeDocumentService{
+ documentList: []*entity.DocumentListItem{{
+ ID: "doc-1",
+ KbID: "ds-1",
+ ParserID: "naive",
+ ParserConfig: "{}",
+ SourceType: "local",
+ Type: "document",
+ CreatedBy: "user-1",
+ Name: sptr("doc.pdf"),
+ Suffix: "pdf",
+ }},
+ documentListTotal: 1,
+ hasActiveTasks: false,
+ }
+ h := &DocumentHandler{
+ documentService: fake,
+ datasetService: dataset.NewDatasetService(),
+ }
+ c, w := setupGinContextWithUser("GET", "/api/v1/datasets/ds-1/documents", "")
+ c.Params = gin.Params{{Key: "dataset_id", Value: "ds-1"}}
+
+ h.ListDocuments(c)
+
+ if w.Code != http.StatusOK {
+ t.Fatalf("expected 200, got %d: %s", w.Code, w.Body.String())
+ }
+ var response struct {
+ Data struct {
+ Docs []map[string]interface{} `json:"docs"`
+ HasActiveTasks bool `json:"has_active_tasks"`
+ } `json:"data"`
+ }
+ if err := json.Unmarshal(w.Body.Bytes(), &response); err != nil {
+ t.Fatalf("unmarshal response: %v", err)
+ }
+ if response.Data.HasActiveTasks {
+ t.Fatalf("has_active_tasks = true, want false")
+ }
+}
+
+func TestListDocumentsHandlerKeepsPollingWhenActiveTaskCheckFails(t *testing.T) {
+ db := setupHandlerAccessDB(t)
+ orig := dao.DB
+ dao.DB = db
+ t.Cleanup(func() { dao.DB = orig })
+
+ fake := &fakeDocumentService{
+ documentList: []*entity.DocumentListItem{{
+ ID: "doc-1",
+ KbID: "ds-1",
+ ParserID: "naive",
+ ParserConfig: "{}",
+ SourceType: "local",
+ Type: "document",
+ CreatedBy: "user-1",
+ Name: sptr("doc.pdf"),
+ Suffix: "pdf",
+ }},
+ documentListTotal: 1,
+ hasActiveTasksErr: errors.New("temporary task lookup failure"),
+ }
+ h := &DocumentHandler{
+ documentService: fake,
+ datasetService: dataset.NewDatasetService(),
+ }
+ c, w := setupGinContextWithUser("GET", "/api/v1/datasets/ds-1/documents", "")
+ c.Params = gin.Params{{Key: "dataset_id", Value: "ds-1"}}
+
+ h.ListDocuments(c)
+
+ if w.Code != http.StatusOK {
+ t.Fatalf("expected 200, got %d: %s", w.Code, w.Body.String())
+ }
+ var response struct {
+ Data struct {
+ HasActiveTasks bool `json:"has_active_tasks"`
+ } `json:"data"`
+ }
+ if err := json.Unmarshal(w.Body.Bytes(), &response); err != nil {
+ t.Fatalf("unmarshal response: %v", err)
+ }
+ if !response.Data.HasActiveTasks {
+ t.Fatal("has_active_tasks = false, want true when active task lookup fails")
+ }
+}
+
func TestListDocumentsHandler_MetadataFilterNarrowsDocumentIDs(t *testing.T) {
db := setupHandlerAccessDB(t)
orig := dao.DB
diff --git a/internal/service/document/document_active.go b/internal/service/document/document_active.go
new file mode 100644
index 0000000000..7c9780d744
--- /dev/null
+++ b/internal/service/document/document_active.go
@@ -0,0 +1,19 @@
+package document
+
+import (
+ "context"
+
+ "ragflow/internal/dao"
+)
+
+// HasActiveIngestionTasks reports whether the dataset has ingestion tasks
+// in a non-terminal state (CREATED/SCHEDULED/RUNNING/STOPPING).
+// It powers the list response's has_active_tasks field so pagination-
+// scoped polling does not stall when active docs are off the current page.
+func (s *DocumentService) HasActiveIngestionTasks(ctx context.Context, datasetID string) (bool, error) {
+ count, err := s.ingestionTaskDAO.CountActiveByDatasetID(ctx, dao.DB, datasetID)
+ if err != nil {
+ return false, err
+ }
+ return count > 0, nil
+}
diff --git a/web/src/constants/knowledge.ts b/web/src/constants/knowledge.ts
index 57568bc5d2..a32bcdfdb5 100644
--- a/web/src/constants/knowledge.ts
+++ b/web/src/constants/knowledge.ts
@@ -41,6 +41,16 @@ export enum RunningStatusOld {
SCHEDULE = '5',
}
+export enum IngestionTaskStatus {
+ CREATED = 'CREATED',
+ SCHEDULED = 'SCHEDULED',
+ RUNNING = 'RUNNING',
+ STOPPING = 'STOPPING',
+ COMPLETED = 'COMPLETED',
+ FAILED = 'FAILED',
+ STOPPED = 'STOPPED',
+}
+
export const RunningStatusMap = {
[RunningStatus.UNSTART]: 'Pending',
[RunningStatus.RUNNING]: 'Running',
diff --git a/web/src/hooks/use-document-request.ts b/web/src/hooks/use-document-request.ts
index 07f6bf36d8..cfcf80afd2 100644
--- a/web/src/hooks/use-document-request.ts
+++ b/web/src/hooks/use-document-request.ts
@@ -32,6 +32,7 @@ import {
} from '@/interfaces/request/document';
import i18n from '@/locales/config';
import { EMPTY_METADATA_FIELD } from '@/pages/dataset/dataset/use-select-filters';
+import { isDocumentProcessing } from '@/pages/dataset/dataset/utils';
import documentStructureService from '@/services/document-structure-service';
import kbService, {
changeDocumentParser,
@@ -173,12 +174,14 @@ export const useFetchDocumentList = (loop = true) => {
const { data, isFetching: loading } = useQuery<{
docs: IDocumentInfo[];
total: number;
+ has_active_tasks?: boolean;
}>({
queryKey: DocumentKeys.list(debouncedSearchString, pagination, filterValue),
- initialData: { docs: [], total: 0 },
+ initialData: { docs: [], total: 0, has_active_tasks: false },
refetchInterval: (query) =>
loop &&
- query.state.data?.docs.some((doc) => doc.run === RunningStatus.RUNNING)
+ (query.state.data?.has_active_tasks ||
+ !!query.state.data?.docs.some(isDocumentProcessing))
? 5000
: false,
enabled: !!knowledgeId || !!id,
@@ -221,6 +224,7 @@ export const useFetchDocumentList = (loop = true) => {
return {
docs: [],
total: 0,
+ has_active_tasks: false,
};
},
});
diff --git a/web/src/interfaces/database/document.ts b/web/src/interfaces/database/document.ts
index e4abbb4560..c1fc05f0ab 100644
--- a/web/src/interfaces/database/document.ts
+++ b/web/src/interfaces/database/document.ts
@@ -1,4 +1,4 @@
-import { RunningStatus } from '@/constants/knowledge';
+import { IngestionTaskStatus, RunningStatus } from '@/constants/knowledge';
export interface IDocumentInfo {
// chunk_num: number;
@@ -19,6 +19,7 @@ export interface IDocumentInfo {
progress: number;
progress_msg: string;
run: RunningStatus;
+ ingestion_status?: IngestionTaskStatus;
size: number;
source_type: string;
status: string;
diff --git a/web/src/locales/ar.ts b/web/src/locales/ar.ts
index c071d9c3a3..720c69a77a 100644
--- a/web/src/locales/ar.ts
+++ b/web/src/locales/ar.ts
@@ -359,6 +359,7 @@ export default {
runningStatus2: 'تم الإلغاء',
runningStatus3: 'نجاح',
runningStatus4: 'يفشل',
+ runningStatusQueued: 'في الانتظار',
pageRanges: 'نطاقات الصفحات',
pageRangesTip:
'نطاق الصفحات التي سيتم تحليلها؛ لن تتم معالجة الصفحات التي تقع خارج هذا النطاق.',
diff --git a/web/src/locales/bg.ts b/web/src/locales/bg.ts
index f61e322773..82bc69b542 100644
--- a/web/src/locales/bg.ts
+++ b/web/src/locales/bg.ts
@@ -352,6 +352,7 @@ export default {
runningStatus2: 'ОТМЕНЕНО',
runningStatus3: 'УСПЕХ',
runningStatus4: 'НЕУСПЕХ',
+ runningStatusQueued: 'На опашка',
pageRanges: 'Диапазон на страници',
pageRangesTip:
'Диапазон от страници за обработка; страниците извън този диапазон няма да бъдат обработени.',
diff --git a/web/src/locales/de.ts b/web/src/locales/de.ts
index 583a5796d7..a31eb79543 100644
--- a/web/src/locales/de.ts
+++ b/web/src/locales/de.ts
@@ -350,6 +350,7 @@ Beispiel: Eine 1 KB Nachricht mit 1024-dim Einbettung verwendet ~9 KB. Das Stand
runningStatus2: 'ABGEBROCHEN',
runningStatus3: 'ERFOLGREICH',
runningStatus4: 'FEHLGESCHLAGEN',
+ runningStatusQueued: 'In Warteschlange',
pageRanges: 'Seitenbereiche',
pageRangesTip:
'Bereich der zu analysierenden Seiten; Seiten außerhalb dieses Bereichs werden nicht verarbeitet.',
diff --git a/web/src/locales/en.ts b/web/src/locales/en.ts
index 3c19289cd6..e70655149b 100644
--- a/web/src/locales/en.ts
+++ b/web/src/locales/en.ts
@@ -560,6 +560,7 @@ Example: A 1 KB message with 1024-dim embedding uses ~9 KB. The 5 MB default lim
runningStatus2: 'Cancelled',
runningStatus3: 'Success',
runningStatus4: 'Fail',
+ runningStatusQueued: 'Queued',
pageRanges: 'Page ranges',
pageRangesTip:
'Range of pages to be parsed; pages outside this range will not be processed.',
diff --git a/web/src/locales/es.ts b/web/src/locales/es.ts
index ff10fa90ad..07eb154894 100644
--- a/web/src/locales/es.ts
+++ b/web/src/locales/es.ts
@@ -128,6 +128,7 @@ export default {
runningStatus2: 'CANCELADO',
runningStatus3: 'ÉXITO',
runningStatus4: 'FALLIDO',
+ runningStatusQueued: 'En cola',
pageRanges: 'Rangos de páginas',
pageRangesTip:
'rango de páginas: Define los rangos de páginas que necesitan ser analizados. Las páginas que no estén incluidas en estos rangos serán ignoradas.',
diff --git a/web/src/locales/fr.ts b/web/src/locales/fr.ts
index ee97af3582..714aecc4d2 100644
--- a/web/src/locales/fr.ts
+++ b/web/src/locales/fr.ts
@@ -416,6 +416,7 @@ export default {
runningStatus2: 'ANNULÉ',
runningStatus3: 'SUCCÈS',
runningStatus4: 'ÉCHEC',
+ runningStatusQueued: 'En attente',
pageRanges: 'Plages de pages',
pageRangesTip:
'Les pages en dehors de cette plage ne seront pas traitées.',
diff --git a/web/src/locales/id.ts b/web/src/locales/id.ts
index 0ae41036d4..9f3bad0a0a 100644
--- a/web/src/locales/id.ts
+++ b/web/src/locales/id.ts
@@ -133,6 +133,7 @@ export default {
runningStatus2: 'BATAL',
runningStatus3: 'SUKSES',
runningStatus4: 'GAGAL',
+ runningStatusQueued: 'Dalam antrean',
pageRanges: 'Rentang Halaman',
pageRangesTip:
'rentang halaman: Tentukan rentang halaman yang perlu diparsing. Halaman yang tidak termasuk dalam rentang ini akan diabaikan.',
diff --git a/web/src/locales/it.ts b/web/src/locales/it.ts
index 27c0c5f997..eb9fa6eab3 100644
--- a/web/src/locales/it.ts
+++ b/web/src/locales/it.ts
@@ -542,6 +542,7 @@ Esempio: un messaggio di 1 KB con embedding a 1024 dimensioni usa ~9 KB. Il limi
runningStatus2: 'Annullato',
runningStatus3: 'Successo',
runningStatus4: 'Fallito',
+ runningStatusQueued: 'In coda',
pageRanges: 'Intervallo pagine',
pageRangesTip:
'Intervallo di pagine da analizzare; le pagine al di fuori di questo intervallo non saranno elaborate.',
diff --git a/web/src/locales/ja.ts b/web/src/locales/ja.ts
index 88b62d28fe..294a2f67a4 100644
--- a/web/src/locales/ja.ts
+++ b/web/src/locales/ja.ts
@@ -207,6 +207,7 @@ export default {
runningStatus2: 'キャンセル',
runningStatus3: '成功',
runningStatus4: '失敗',
+ runningStatusQueued: '待機中',
pageRanges: 'ページ範囲',
pageRangesTip:
diff --git a/web/src/locales/ko.ts b/web/src/locales/ko.ts
index 4e27418532..1dc3d5d674 100644
--- a/web/src/locales/ko.ts
+++ b/web/src/locales/ko.ts
@@ -531,6 +531,7 @@ export default {
runningStatus2: '취소됨',
runningStatus3: '성공',
runningStatus4: '실패',
+ runningStatusQueued: '대기 중',
pageRanges: '페이지 범위',
pageRangesTip:
'파싱할 페이지 범위를 지정합니다. 범위 밖의 페이지는 처리되지 않습니다.',
diff --git a/web/src/locales/pt-br.ts b/web/src/locales/pt-br.ts
index 1f9df00837..0368b70de8 100644
--- a/web/src/locales/pt-br.ts
+++ b/web/src/locales/pt-br.ts
@@ -136,6 +136,7 @@ export default {
runningStatus2: 'Cancelado',
runningStatus3: 'Sucesso',
runningStatus4: 'Falha',
+ runningStatusQueued: 'Na fila',
pageRanges: 'Intervalos de páginas',
pageRangesTip:
'Intervalo de páginas a serem analisadas; páginas fora deste intervalo não serão processadas.',
diff --git a/web/src/locales/ru.ts b/web/src/locales/ru.ts
index 8f82f12295..0bc8c0a553 100644
--- a/web/src/locales/ru.ts
+++ b/web/src/locales/ru.ts
@@ -370,6 +370,7 @@ export default {
runningStatus2: 'ОТМЕНЕНО',
runningStatus3: 'УСПЕХ',
runningStatus4: 'НЕУДАЧА',
+ runningStatusQueued: 'В очереди',
pageRanges: 'Диапазон страниц',
pageRangesTip:
'Диапазон страниц для парсинга; страницы вне этого диапазона обрабатываться не будут.',
diff --git a/web/src/locales/tr.ts b/web/src/locales/tr.ts
index 8b435e6b6d..4fb302d2fa 100644
--- a/web/src/locales/tr.ts
+++ b/web/src/locales/tr.ts
@@ -562,6 +562,7 @@ Prosedürel Bellek: Öğrenilen beceriler, alışkanlıklar ve otomatik prosedü
runningStatus2: 'İptal edildi',
runningStatus3: 'Başarılı',
runningStatus4: 'Başarısız',
+ runningStatusQueued: 'Kuyrukta',
pageRanges: 'Sayfa aralıkları',
pageRangesTip:
'Ayrıştırılacak sayfa aralığı; bu aralığın dışındaki sayfalar işlenmeyecek.',
diff --git a/web/src/locales/vi.ts b/web/src/locales/vi.ts
index d9b3951545..fe222babf5 100644
--- a/web/src/locales/vi.ts
+++ b/web/src/locales/vi.ts
@@ -139,6 +139,7 @@ export default {
runningStatus2: 'CANCEL',
runningStatus3: 'SUCCESS',
runningStatus4: 'FAIL',
+ runningStatusQueued: 'Đang chờ',
pageRanges: 'Phạm vi trang',
pageRangesTip:
'Phạm vi trang: Xác định các phạm vi trang cần phân tích cú pháp. Các trang không nằm trong các phạm vi này sẽ bị bỏ qua.',
diff --git a/web/src/locales/zh-traditional.ts b/web/src/locales/zh-traditional.ts
index 6be6914589..b758f70809 100644
--- a/web/src/locales/zh-traditional.ts
+++ b/web/src/locales/zh-traditional.ts
@@ -140,6 +140,7 @@ export default {
runningStatus2: '取消',
runningStatus3: '成功',
runningStatus4: '失敗',
+ runningStatusQueued: '排隊中',
pageRanges: '頁碼範圍',
pageRangesTip:
'頁碼範圍:定義需要解析的頁面範圍。不包含在這些範圍內的頁面將被忽略。',
diff --git a/web/src/locales/zh.ts b/web/src/locales/zh.ts
index b9c2b74e42..be49eb3fca 100644
--- a/web/src/locales/zh.ts
+++ b/web/src/locales/zh.ts
@@ -504,6 +504,7 @@ export default {
runningStatus2: '取消',
runningStatus3: '成功',
runningStatus4: '失败',
+ runningStatusQueued: '排队中',
pageRanges: '页码范围',
pageRangesTip:
'页码范围:定义需要解析的页面范围。 不包含在这些范围内的页面将被忽略。',
diff --git a/web/src/pages/dataset/dataset/dataset-action-cell.tsx b/web/src/pages/dataset/dataset/dataset-action-cell.tsx
index 8761d7bce4..bb4aa5601b 100644
--- a/web/src/pages/dataset/dataset/dataset-action-cell.tsx
+++ b/web/src/pages/dataset/dataset/dataset-action-cell.tsx
@@ -17,7 +17,7 @@ import { Download, Eye, PenLine, Trash2 } from 'lucide-react';
import { omit } from 'lodash';
import { useCallback } from 'react';
import { UseRenameDocumentShowType } from './use-rename-document';
-import { isParserRunning } from './utils';
+import { isDocumentProcessing } from './utils';
const Fields = ['name', 'size', 'type', 'create_time', 'update_time'];
@@ -33,8 +33,8 @@ export function DatasetActionCell({
setRowSelection,
}: { record: IDocumentInfo } & UseRenameDocumentShowType &
Pick<UseRowSelectionType, 'setRowSelection'>) {
- const { id, run, type } = record;
- const isRunning = isParserRunning(run);
+ const { id, type } = record;
+ const isRunning = isDocumentProcessing(record);
const isVirtualDocument = type === DocumentType.Virtual;
const { removeDocument } = useRemoveDocument();
diff --git a/web/src/pages/dataset/dataset/parsing-status-cell.test.tsx b/web/src/pages/dataset/dataset/parsing-status-cell.test.tsx
new file mode 100644
index 0000000000..c9285811cb
--- /dev/null
+++ b/web/src/pages/dataset/dataset/parsing-status-cell.test.tsx
@@ -0,0 +1,111 @@
+import { render, screen } from '@testing-library/react';
+import React from 'react';
+
+jest.mock('react-i18next', () => ({
+ useTranslation: () => ({ t: (key: string) => key }),
+}));
+
+jest.mock('./reparse-dialog', () => ({
+ ReparseDialog: () => null,
+}));
+
+jest.mock('./use-run-document', () => ({
+ useHandleRunDocumentByIds: () => ({
+ handleRunDocumentByIds: jest.fn(),
+ visible: false,
+ showModal: jest.fn(),
+ hideModal: jest.fn(),
+ }),
+}));
+
+import { IngestionTaskStatus, RunningStatus } from './constant';
+import { ParsingStatusCell } from './parsing-status-cell';
+
+describe('ParsingStatusCell', () => {
+ it('shows a cancel icon while ingestion runs before the document run state updates', () => {
+ const { container } = render(
+ React.createElement(ParsingStatusCell, {
+ record: {
+ id: 'doc-1',
+ dataset_id: 'dataset-1',
+ name: 'doc.pdf',
+ type: 'document',
+ run: RunningStatus.UNSTART,
+ ingestion_status: IngestionTaskStatus.RUNNING,
+ progress: 0,
+ chunk_count: 0,
+ parser_config: {},
+ create_date: '',
+ create_time: 0,
+ created_by: 'user-1',
+ nickname: '',
+ location: '',
+ pipeline_id: '',
+ pipeline_name: '',
+ process_duration: 0,
+ progress_msg: '',
+ size: 0,
+ source_type: 'local',
+ status: '1',
+ suffix: 'pdf',
+ thumbnail: '',
+ token_num: 0,
+ update_date: '',
+ update_time: 0,
+ chunk_method: 'naive',
+ },
+ showLog: jest.fn(),
+ showChangeParserModal: jest.fn(),
+ }),
+ );
+
+ expect(container.querySelector('svg.lucide-circle-x')).toBeInTheDocument();
+ expect(
+ container.querySelector('use[href="#icon-play"]'),
+ ).not.toBeInTheDocument();
+ });
+
+ it('does not expose cancelling as a document status', () => {
+ const { container } = render(
+ React.createElement(ParsingStatusCell, {
+ record: {
+ id: 'doc-1',
+ dataset_id: 'dataset-1',
+ name: 'doc.pdf',
+ type: 'document',
+ run: RunningStatus.RUNNING,
+ ingestion_status: IngestionTaskStatus.STOPPING,
+ progress: 0,
+ chunk_count: 0,
+ parser_config: {},
+ create_date: '',
+ create_time: 0,
+ created_by: 'user-1',
+ nickname: '',
+ location: '',
+ pipeline_id: '',
+ pipeline_name: '',
+ process_duration: 0,
+ progress_msg: '',
+ size: 0,
+ source_type: 'local',
+ status: '1',
+ suffix: 'pdf',
+ thumbnail: '',
+ token_num: 0,
+ update_date: '',
+ update_time: 0,
+ chunk_method: 'naive',
+ },
+ showLog: jest.fn(),
+ showChangeParserModal: jest.fn(),
+ }),
+ );
+
+ expect(
+ screen.queryByText('knowledgeDetails.runningStatusStopping'),
+ ).not.toBeInTheDocument();
+ expect(container.querySelector('svg.lucide-circle-x')).toBeInTheDocument();
+ expect(container.querySelector('button[disabled]')).toBeInTheDocument();
+ });
+});
diff --git a/web/src/pages/dataset/dataset/parsing-status-cell.tsx b/web/src/pages/dataset/dataset/parsing-status-cell.tsx
index dcee61dea9..d3acbf0eda 100644
--- a/web/src/pages/dataset/dataset/parsing-status-cell.tsx
+++ b/web/src/pages/dataset/dataset/parsing-status-cell.tsx
@@ -14,15 +14,15 @@ import {
TooltipTrigger,
} from '@/components/ui/tooltip';
import { IDocumentInfo } from '@/interfaces/database/document';
-import { CircleQuestionMark, CircleX } from 'lucide-react';
+import { CircleQuestionMark, CircleX, Clock3 } from 'lucide-react';
import { useCallback, useMemo } from 'react';
import { useTranslation } from 'react-i18next';
-import { DocumentType, RunningStatus } from './constant';
+import { DocumentType, IngestionTaskStatus, RunningStatus } from './constant';
import { ParsingCard } from './parsing-card';
import { ReparseDialog } from './reparse-dialog';
import { UseChangeDocumentParserShowType } from './use-change-document-parser';
import { useHandleRunDocumentByIds } from './use-run-document';
-import { isParserRunning } from './utils';
+import { isDocumentProcessing } from './utils';
const IconMap = {
[RunningStatus.UNSTART]: (
<IconFontFill name="play" className="text-accent-primary size-[1em]" />
@@ -109,6 +109,7 @@ export function ParsingStatusCell({
record: IDocumentInfo;
showLog: (record: IDocumentInfo) => void;
} & UseChangeDocumentParserShowType) {
+ const { t } = useTranslation();
const { run, progress, chunk_count, id } = record;
const operationIcon = IconMap[run];
const p = Number((progress * 100).toFixed(2));
@@ -118,7 +119,11 @@ export function ParsingStatusCell({
showModal: showReparseDialogModal,
hideModal: hideReparseDialogModal,
} = useHandleRunDocumentByIds(id);
- const isRunning = isParserRunning(run);
+ const isRunning = isDocumentProcessing(record);
+ const isQueued =
+ record.ingestion_status === IngestionTaskStatus.CREATED ||
+ record.ingestion_status === IngestionTaskStatus.SCHEDULED;
+ const isStopping = record.ingestion_status === IngestionTaskStatus.STOPPING;
const isZeroChunk = chunk_count === 0;
const handleOperationIconClick = (option?: {
@@ -139,39 +144,49 @@ export function ParsingStatusCell({
<section
className="flex gap-8 items-center"
data-testid="document-parse-status"
- data-state={ParseStatusStateMap[run] ?? 'unknown'}
+ data-state={isQueued ? 'queued' : (ParseStatusStateMap[run] ?? 'unknown')}
>
{showParse && (
<div className="flex items-center gap-2">
<Separator orientation="vertical" className="h-[1em]" />
- {isParserRunning(run) ? (
+ {isRunning ? (
<>
- <Button
- size="auto"
- variant="static"
- onClick={() => handleShowLog(record)}
- >
- <Progress value={p} className="h-1 flex-1 min-w-10" />
- <div className="flex items-center gap-1">
- {p}%
- <span className="inline-flex items-center">
- <CircleQuestionMark className="size-[1em]" />
- </span>
- </div>
- </Button>
+ {isQueued ? (
+ <Button
+ size="auto"
+ variant="static"
+ onClick={() => handleShowLog(record)}
+ >
+ <Clock3 className="size-[1em]" />
+ {t('knowledgeDetails.runningStatusQueued')}
+ </Button>
+ ) : (
+ <Button
+ size="auto"
+ variant="static"
+ onClick={() => handleShowLog(record)}
+ >
+ <Progress value={p} className="h-1 flex-1 min-w-10" />
+ <div className="flex items-center gap-1">
+ {p}%
+ <span className="inline-flex items-center">
+ <CircleQuestionMark className="size-[1em]" />
+ </span>
+ </div>
+ </Button>
+ )}
<Button
variant="ghost"
size="icon-xs"
+ disabled={isStopping}
onClick={() => showReparseDialogModal()}
- // onClick={
- // isZeroChunk || isRunning
- // ? handleOperationIconClick(false)
- // : () => {}
- // }
>
- {operationIcon}
+ <CircleX
+ color="rgba(var(--state-error))"
+ className="size-[1em]"
+ />
</Button>
</>
) : (
@@ -197,7 +212,6 @@ export function ParsingStatusCell({
(isZeroChunk && !record?.parser_config?.enable_metadata) ||
isRunning
}
- // hidden={false}
enable_metadata={record?.parser_config?.enable_metadata}
handleOperationIconClick={handleOperationIconClick}
chunk_num={chunk_count}
diff --git a/web/src/pages/dataset/dataset/use-bulk-operate-dataset.tsx b/web/src/pages/dataset/dataset/use-bulk-operate-dataset.tsx
index 8858e81329..d83c5bb918 100644
--- a/web/src/pages/dataset/dataset/use-bulk-operate-dataset.tsx
+++ b/web/src/pages/dataset/dataset/use-bulk-operate-dataset.tsx
@@ -21,7 +21,8 @@ import { useCallback, useMemo } from 'react';
import { useTranslation } from 'react-i18next';
import { useParams } from 'react-router';
import { toast } from 'sonner';
-import { DocumentType, RunningStatus } from './constant';
+import { DocumentType } from './constant';
+import { isDocumentProcessing } from './utils';
export function useBulkOperateDataset({
rowSelection,
@@ -106,10 +107,7 @@ export function useBulkOperateDataset({
const handleDelete = useCallback(() => {
const deletedKeys = selectedRowKeys.filter(
- (x) =>
- !documents
- .filter((y) => y.run === RunningStatus.RUNNING)
- .some((y) => y.id === x),
+ (x) => !documents.filter(isDocumentProcessing).some((y) => y.id === x),
);
if (deletedKeys.length === 0) {
toast.error(
diff --git a/web/src/pages/dataset/dataset/utils.test.ts b/web/src/pages/dataset/dataset/utils.test.ts
new file mode 100644
index 0000000000..b1910a1321
--- /dev/null
+++ b/web/src/pages/dataset/dataset/utils.test.ts
@@ -0,0 +1,29 @@
+import { IngestionTaskStatus, RunningStatus } from './constant';
+import { isDocumentProcessing } from './utils';
+
+describe('isDocumentProcessing', () => {
+ it('treats a scheduled ingestion task as processing before parsing starts', () => {
+ expect(
+ isDocumentProcessing({
+ run: RunningStatus.UNSTART,
+ ingestion_status: IngestionTaskStatus.SCHEDULED,
+ }),
+ ).toBe(true);
+ });
+
+ it.each([
+ IngestionTaskStatus.CREATED,
+ IngestionTaskStatus.RUNNING,
+ IngestionTaskStatus.STOPPING,
+ ])(
+ 'treats an active %s ingestion task as processing before document state sync',
+ (ingestionStatus) => {
+ expect(
+ isDocumentProcessing({
+ run: RunningStatus.UNSTART,
+ ingestion_status: ingestionStatus,
+ }),
+ ).toBe(true);
+ },
+ );
+});
diff --git a/web/src/pages/dataset/dataset/utils.ts b/web/src/pages/dataset/dataset/utils.ts
index 0965ffdb9e..ee1d653b37 100644
--- a/web/src/pages/dataset/dataset/utils.ts
+++ b/web/src/pages/dataset/dataset/utils.ts
@@ -1,6 +1,18 @@
-import { RunningStatus } from './constant';
+import { IngestionTaskStatus, RunningStatus } from './constant';
+import type { IDocumentInfo } from '@/interfaces/database/document';
export const isParserRunning = (text: RunningStatus) => {
const isRunning = text === RunningStatus.RUNNING;
return isRunning;
};
+
+// Go ingestion status can advance before the legacy document run field. The
+// Python endpoint omits ingestion_status, so run remains the compatibility path.
+export const isDocumentProcessing = (
+ document: Pick<IDocumentInfo, 'run' | 'ingestion_status'>,
+) =>
+ isParserRunning(document.run) ||
+ document.ingestion_status === IngestionTaskStatus.CREATED ||
+ document.ingestion_status === IngestionTaskStatus.SCHEDULED ||
+ document.ingestion_status === IngestionTaskStatus.RUNNING ||
+ document.ingestion_status === IngestionTaskStatus.STOPPING;