Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

fix(go): preserve legacy document status in dataset lists (#20279)

adws6c04e4b98a06
diff --git a/internal/dao/document.go b/internal/dao/document.go
index cbb8a339e2..e0f114edba 100644
--- a/internal/dao/document.go
+++ b/internal/dao/document.go
@@ -160,16 +160,34 @@ const latestIngestionTaskJoin = `LEFT JOIN ingestion_task ON ingestion_task.docu
)
)`
+func buildDocumentListIngestionStatusExpression(db *gorm.DB) string {
+ const taskStatus = "COALESCE(NULLIF(ingestion_task.status, ''), 'UNSTART')"
+ if !db.Migrator().HasColumn(&entity.Document{}, "run") {
+ return taskStatus
+ }
+
+ return `CASE
+ WHEN ingestion_task.id IS NOT NULL THEN ` + taskStatus + `
+ WHEN document.run = '1' THEN 'RUNNING'
+ WHEN document.run = '2' THEN 'STOPPED'
+ WHEN document.run = '3' THEN 'COMPLETED'
+ WHEN document.run = '4' THEN 'FAILED'
+ WHEN document.run = '5' THEN 'SCHEDULED'
+ ELSE 'UNSTART'
+ END`
+}
+
// ListByKBIDWithOptions lists documents by knowledge base ID with filters.
func (dao *DocumentDAO) ListByKBIDWithOptions(ctx context.Context, db *gorm.DB, opts DocumentListOptions) ([]*entity.DocumentListItem, int64, error) {
var documents []*entity.DocumentListItem
var total int64
+ ingestionStatus := buildDocumentListIngestionStatusExpression(db)
// Historical retries can leave multiple ingestion tasks per document. Keep
// only the newest row, with ID as a deterministic tie-breaker for equal
// create times. This ordering must match IngestionTaskDAO's task lookups.
listQuery := db.WithContext(ctx).Table("document").
- Select(`document.*, user_canvas.title as pipeline_name, user.nickname, ingestion_task.status as ingestion_status, ingestion_task.pipeline_log_id as pipeline_log_id`).
+ Select(`document.*, user_canvas.title as pipeline_name, user.nickname, ` + ingestionStatus + ` as ingestion_status, ingestion_task.pipeline_log_id as pipeline_log_id`).
Joins("JOIN file2document ON file2document.document_id = document.id").
Joins("JOIN file ON file.id = file2document.file_id").
Joins("LEFT JOIN user_canvas ON document.pipeline_id = user_canvas.id").
@@ -183,14 +201,14 @@ func (dao *DocumentDAO) ListByKBIDWithOptions(ctx context.Context, db *gorm.DB,
countQuery = countQuery.Joins(latestIngestionTaskJoin)
}
- listQuery = applyDocumentListFilters(listQuery, opts, true)
- countQuery = applyDocumentListFilters(countQuery, opts, true)
+ listQuery = applyDocumentListFilters(listQuery, opts, true, ingestionStatus)
+ countQuery = applyDocumentListFilters(countQuery, opts, true, ingestionStatus)
if err := countQuery.Count(&total).Error; err != nil {
return nil, 0, err
}
- orderBy := documentListOrderColumn(opts.OrderBy)
+ orderBy := documentListOrderColumn(opts.OrderBy, ingestionStatus)
if opts.Desc {
orderBy += " DESC"
} else {
@@ -212,13 +230,14 @@ func (dao *DocumentDAO) GetFilterByKBID(ctx context.Context, db *gorm.DB, opts D
IngestionStatus *string `gorm:"column:ingestion_status"`
Suffix string `gorm:"column:suffix"`
}
+ ingestionStatus := buildDocumentListIngestionStatusExpression(db)
query := db.WithContext(ctx).Table("document").
- Select("document.id, ingestion_task.status as ingestion_status, document.suffix").
+ Select("document.id, " + ingestionStatus + " as ingestion_status, document.suffix").
Joins("JOIN file2document ON file2document.document_id = document.id").
Joins("JOIN file ON file.id = file2document.file_id").
Joins(latestIngestionTaskJoin)
- query = applyDocumentListFilters(query, opts, true)
+ query = applyDocumentListFilters(query, opts, true, ingestionStatus)
if err := query.Scan(&rows).Error; err != nil {
return nil, 0, err
@@ -247,21 +266,23 @@ func (dao *DocumentDAO) GetFilterByKBID(ctx context.Context, db *gorm.DB, opts D
// ListIDsByKBIDWithOptions lists matching document IDs without pagination.
func (dao *DocumentDAO) ListIDsByKBIDWithOptions(ctx context.Context, db *gorm.DB, opts DocumentListOptions) ([]string, error) {
var ids []string
+ var ingestionStatus string
query := db.WithContext(ctx).Table("document").
Select("document.id").
Joins("JOIN file2document ON file2document.document_id = document.id").
Joins("JOIN file ON file.id = file2document.file_id")
if len(opts.RunStatuses) > 0 {
+ ingestionStatus = buildDocumentListIngestionStatusExpression(db)
query = query.Joins(latestIngestionTaskJoin)
}
- query = applyDocumentListFilters(query, opts, true)
+ query = applyDocumentListFilters(query, opts, true, ingestionStatus)
if err := query.Scan(&ids).Error; err != nil {
return nil, err
}
return ids, nil
}
-func applyDocumentListFilters(query *gorm.DB, opts DocumentListOptions, qualified bool) *gorm.DB {
+func applyDocumentListFilters(query *gorm.DB, opts DocumentListOptions, qualified bool, ingestionStatus string) *gorm.DB {
column := func(name string) string {
if qualified {
return "document." + name
@@ -274,22 +295,7 @@ func applyDocumentListFilters(query *gorm.DB, opts DocumentListOptions, qualifie
query = query.Where("LOWER("+column("name")+") LIKE ?", "%"+strings.ToLower(strings.TrimSpace(opts.Keywords))+"%")
}
if len(opts.RunStatuses) > 0 {
- hasUnstart := false
- otherStatuses := make([]string, 0, len(opts.RunStatuses))
- for _, s := range opts.RunStatuses {
- if strings.EqualFold(s, "UNSTART") {
- hasUnstart = true
- } else {
- otherStatuses = append(otherStatuses, s)
- }
- }
- if hasUnstart && len(otherStatuses) > 0 {
- query = query.Where("(ingestion_task.status IN ? OR ingestion_task.id IS NULL OR ingestion_task.status = ?)", otherStatuses, "UNSTART")
- } else if hasUnstart {
- query = query.Where("(ingestion_task.id IS NULL OR ingestion_task.status = ?)", "UNSTART")
- } else {
- query = query.Where("ingestion_task.status IN ?", otherStatuses)
- }
+ query = query.Where("("+ingestionStatus+") IN ?", opts.RunStatuses)
}
if len(opts.Types) > 0 {
query = query.Where(column("type")+" IN ?", opts.Types)
@@ -313,7 +319,7 @@ func applyDocumentListFilters(query *gorm.DB, opts DocumentListOptions, qualifie
return query
}
-func documentListOrderColumn(orderBy string) string {
+func documentListOrderColumn(orderBy, ingestionStatus string) string {
switch orderBy {
case "update_time":
return "document.update_time"
@@ -324,7 +330,7 @@ func documentListOrderColumn(orderBy string) string {
case "type":
return "document.type"
case "run", "ingestion_status":
- return "COALESCE(ingestion_task.status, 'UNSTART')"
+ return ingestionStatus
default:
return "document.create_time"
}
diff --git a/internal/dao/document_test.go b/internal/dao/document_test.go
index 2bd47c4c6b..f127c33883 100644
--- a/internal/dao/document_test.go
+++ b/internal/dao/document_test.go
@@ -196,7 +196,8 @@ func TestDocumentListIncludesScheduledIngestionStatus(t *testing.T) {
t.Fatalf("create scheduled ingestion task: %v", err)
}
- documents, total, err := NewDocumentDAO().ListByKBIDWithOptions(t.Context(), db, DocumentListOptions{
+ dao := NewDocumentDAO()
+ documents, total, err := dao.ListByKBIDWithOptions(t.Context(), db, DocumentListOptions{
KbID: "kb-1",
OrderBy: "create_time",
Desc: true,
@@ -221,6 +222,176 @@ func TestDocumentListIncludesScheduledIngestionStatus(t *testing.T) {
if got := listed["ingestion_status"]; got != "SCHEDULED" {
t.Fatalf("ingestion_status = %v, want %q", got, "SCHEDULED")
}
+
+ opts := DocumentListOptions{KbID: "kb-1", RunStatuses: []string{common.SCHEDULED}, OrderBy: "ingestion_status", Limit: 10}
+ documents, total, err = dao.ListByKBIDWithOptions(t.Context(), db, opts)
+ if err != nil || total != 1 || len(documents) != 1 {
+ t.Fatalf("filter scheduled documents: got %d (total %d), err %v", len(documents), total, err)
+ }
+ ids, err := dao.ListIDsByKBIDWithOptions(t.Context(), db, opts)
+ if err != nil || len(ids) != 1 || ids[0] != "doc-scheduled" {
+ t.Fatalf("filter scheduled document IDs: got %v, err %v", ids, err)
+ }
+ filters, total, err := dao.GetFilterByKBID(t.Context(), db, DocumentListOptions{KbID: "kb-1"})
+ if err != nil || total != 1 {
+ t.Fatalf("get scheduled filter count: total %d, err %v", total, err)
+ }
+ if got := filters["ingestion_status"].(map[string]int64)[common.SCHEDULED]; got != 1 {
+ t.Fatalf("scheduled filter count = %d, want 1", got)
+ }
+ if db.Migrator().HasColumn(&entity.Document{}, "run") {
+ t.Fatal("document list query added the legacy run column")
+ }
+}
+
+func TestDocumentListFallsBackToReadOnlyLegacyRun(t *testing.T) {
+ db := setupDocumentTestDB(t)
+ if err := db.AutoMigrate(
+ &entity.User{},
+ &entity.UserCanvas{},
+ &entity.File{},
+ &entity.File2Document{},
+ &entity.IngestionTask{},
+ ); err != nil {
+ t.Fatalf("migrate document-list dependencies: %v", err)
+ }
+ if err := db.Exec("ALTER TABLE document ADD COLUMN run TEXT").Error; err != nil {
+ t.Fatalf("add legacy run fixture column: %v", err)
+ }
+
+ tests := []struct {
+ id, run, task, want string
+ hasTask bool
+ }{
+ {id: "unstart", run: "0", want: "UNSTART"},
+ {id: "empty", want: "UNSTART"},
+ {id: "running", run: "1", want: common.RUNNING},
+ {id: "stopped", run: "2", want: common.STOPPED},
+ {id: "completed", run: "3", want: common.COMPLETED},
+ {id: "failed", run: "4", want: common.FAILED},
+ {id: "scheduled", run: "5", want: common.SCHEDULED},
+ {id: "unknown", run: "9", want: "UNSTART"},
+ {id: "task-wins", run: "3", task: common.RUNNING, hasTask: true, want: common.RUNNING},
+ {id: "empty-task-wins", run: "3", hasTask: true, want: "UNSTART"},
+ }
+
+ create := func(value interface{}) {
+ t.Helper()
+ if err := db.Create(value).Error; err != nil {
+ t.Fatalf("create %T: %v", value, err)
+ }
+ }
+ fileID := "file-legacy"
+ create(&entity.File{ID: fileID, ParentID: "parent-1", TenantID: "tenant-1", CreatedBy: "user-1", Name: "legacy.pdf", Type: "document"})
+ for _, test := range tests {
+ documentID := "doc-" + test.id
+ create(&entity.Document{ID: documentID, KbID: "kb-legacy", ParserConfig: entity.JSONMap{}})
+ if err := db.Table("document").Where("id = ?", documentID).UpdateColumn("run", test.run).Error; err != nil {
+ t.Fatalf("set legacy run for %s: %v", documentID, err)
+ }
+ create(&entity.File2Document{ID: "link-" + test.id, FileID: sp(fileID), DocumentID: sp(documentID)})
+ if test.hasTask {
+ create(&entity.IngestionTask{ID: "task-" + test.id, DocumentID: documentID, Status: test.task})
+ }
+ }
+
+ dao := NewDocumentDAO()
+ documents, total, err := dao.ListByKBIDWithOptions(t.Context(), db, DocumentListOptions{
+ KbID: "kb-legacy", Limit: len(tests),
+ })
+ if err != nil {
+ t.Fatalf("list legacy documents: %v", err)
+ }
+ if total != int64(len(tests)) || len(documents) != len(tests) {
+ t.Fatalf("listed %d documents (total %d), want %d", len(documents), total, len(tests))
+ }
+
+ statusByID := make(map[string]string, len(documents))
+ for _, document := range documents {
+ if document.IngestionStatus != nil {
+ statusByID[document.ID] = *document.IngestionStatus
+ }
+ }
+ for _, test := range tests {
+ documentID := "doc-" + test.id
+ if got := statusByID[documentID]; got != test.want {
+ t.Errorf("status for %s = %q, want %q", documentID, got, test.want)
+ }
+ }
+
+ completedOpts := DocumentListOptions{KbID: "kb-legacy", RunStatuses: []string{common.COMPLETED}, Limit: len(tests)}
+ documents, total, err = dao.ListByKBIDWithOptions(t.Context(), db, completedOpts)
+ if err != nil || total != 1 || len(documents) != 1 || documents[0].ID != "doc-completed" {
+ t.Fatalf("filter completed documents: got %v (total %d), err %v", documents, total, err)
+ }
+ ids, err := dao.ListIDsByKBIDWithOptions(t.Context(), db, completedOpts)
+ if err != nil || len(ids) != 1 || ids[0] != "doc-completed" {
+ t.Fatalf("filter completed document IDs: got %v, err %v", ids, err)
+ }
+ filters, filterTotal, err := dao.GetFilterByKBID(t.Context(), db, completedOpts)
+ if err != nil || filterTotal != 1 || filters["ingestion_status"].(map[string]int64)[common.COMPLETED] != 1 {
+ t.Fatalf("filter completed count: got %v (total %d), err %v", filters, filterTotal, err)
+ }
+
+ mixedOpts := DocumentListOptions{KbID: "kb-legacy", RunStatuses: []string{"UNSTART", common.COMPLETED}, Limit: len(tests)}
+ documents, total, err = dao.ListByKBIDWithOptions(t.Context(), db, mixedOpts)
+ if err != nil || total != 5 || len(documents) != 5 {
+ t.Fatalf("filter mixed statuses: got %d (total %d), err %v", len(documents), total, err)
+ }
+ wantMixed := map[string]bool{
+ "doc-unstart": true, "doc-empty": true, "doc-completed": true, "doc-unknown": true, "doc-empty-task-wins": true,
+ }
+ for _, document := range documents {
+ delete(wantMixed, document.ID)
+ }
+ if len(wantMixed) != 0 {
+ t.Fatalf("mixed status filter missed documents: %v", wantMixed)
+ }
+ ids, err = dao.ListIDsByKBIDWithOptions(t.Context(), db, mixedOpts)
+ if err != nil || len(ids) != 5 {
+ t.Fatalf("filter mixed document IDs: got %v, err %v", ids, err)
+ }
+
+ filters, filterTotal, err = dao.GetFilterByKBID(t.Context(), db, DocumentListOptions{KbID: "kb-legacy"})
+ if err != nil || filterTotal != int64(len(tests)) {
+ t.Fatalf("get legacy filter counts: total %d, err %v", filterTotal, err)
+ }
+ counts := filters["ingestion_status"].(map[string]int64)
+ wantCounts := map[string]int64{"UNSTART": 4, common.RUNNING: 2, common.STOPPED: 1, common.COMPLETED: 1, common.FAILED: 1, common.SCHEDULED: 1}
+ for status, want := range wantCounts {
+ if got := counts[status]; got != want {
+ t.Errorf("filter count for %s = %d, want %d", status, got, want)
+ }
+ }
+
+ wantAscending := []string{common.COMPLETED, common.FAILED, common.RUNNING, common.RUNNING, common.SCHEDULED, common.STOPPED, "UNSTART", "UNSTART", "UNSTART", "UNSTART"}
+ for _, desc := range []bool{false, true} {
+ documents, total, err = dao.ListByKBIDWithOptions(t.Context(), db, DocumentListOptions{
+ KbID: "kb-legacy", OrderBy: "ingestion_status", Desc: desc, Limit: len(tests),
+ })
+ if err != nil || total != int64(len(tests)) || len(documents) != len(tests) {
+ t.Fatalf("sort legacy statuses (desc=%t): got %d (total %d), err %v", desc, len(documents), total, err)
+ }
+ for i, document := range documents {
+ want := wantAscending[i]
+ if desc {
+ want = wantAscending[len(wantAscending)-1-i]
+ }
+ if document.IngestionStatus == nil || *document.IngestionStatus != want {
+ t.Errorf("sorted status %d (desc=%t) = %v, want %s", i, desc, document.IngestionStatus, want)
+ }
+ }
+ }
+
+ for _, test := range tests {
+ var stored string
+ if err := db.Table("document").Select("run").Where("id = ?", "doc-"+test.id).Scan(&stored).Error; err != nil {
+ t.Fatalf("read legacy run for %s: %v", test.id, err)
+ }
+ if stored != test.run {
+ t.Fatalf("legacy run for %s changed from %q to %q", test.id, test.run, stored)
+ }
+ }
}
func TestDocumentListDeduplicatesHistoricalIngestionTasks(t *testing.T) {