RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
fix(go): preserve legacy document status in dataset lists (#20279)
adws6c04e4b98a06
diff --git a/internal/dao/document.go b/internal/dao/document.goindex cbb8a339e2..e0f114edba 100644--- a/internal/dao/document.go+++ b/internal/dao/document.go@@ -160,16 +160,34 @@ const latestIngestionTaskJoin = `LEFT JOIN ingestion_task ON ingestion_task.docu))`+func buildDocumentListIngestionStatusExpression(db *gorm.DB) string {+ const taskStatus = "COALESCE(NULLIF(ingestion_task.status, ''), 'UNSTART')"+ if !db.Migrator().HasColumn(&entity.Document{}, "run") {+ return taskStatus+ }++ return `CASE+ WHEN ingestion_task.id IS NOT NULL THEN ` + taskStatus + `+ WHEN document.run = '1' THEN 'RUNNING'+ WHEN document.run = '2' THEN 'STOPPED'+ WHEN document.run = '3' THEN 'COMPLETED'+ WHEN document.run = '4' THEN 'FAILED'+ WHEN document.run = '5' THEN 'SCHEDULED'+ ELSE 'UNSTART'+ END`+}+// ListByKBIDWithOptions lists documents by knowledge base ID with filters.func (dao *DocumentDAO) ListByKBIDWithOptions(ctx context.Context, db *gorm.DB, opts DocumentListOptions) ([]*entity.DocumentListItem, int64, error) {var documents []*entity.DocumentListItemvar total int64+ ingestionStatus := buildDocumentListIngestionStatusExpression(db)// Historical retries can leave multiple ingestion tasks per document. Keep// only the newest row, with ID as a deterministic tie-breaker for equal// create times. This ordering must match IngestionTaskDAO's task lookups.listQuery := db.WithContext(ctx).Table("document").- Select(`document.*, user_canvas.title as pipeline_name, user.nickname, ingestion_task.status as ingestion_status, ingestion_task.pipeline_log_id as pipeline_log_id`).+ Select(`document.*, user_canvas.title as pipeline_name, user.nickname, ` + ingestionStatus + ` as ingestion_status, ingestion_task.pipeline_log_id as pipeline_log_id`).Joins("JOIN file2document ON file2document.document_id = document.id").Joins("JOIN file ON file.id = file2document.file_id").Joins("LEFT JOIN user_canvas ON document.pipeline_id = user_canvas.id").@@ -183,14 +201,14 @@ func (dao *DocumentDAO) ListByKBIDWithOptions(ctx context.Context, db *gorm.DB,countQuery = countQuery.Joins(latestIngestionTaskJoin)}- listQuery = applyDocumentListFilters(listQuery, opts, true)- countQuery = applyDocumentListFilters(countQuery, opts, true)+ listQuery = applyDocumentListFilters(listQuery, opts, true, ingestionStatus)+ countQuery = applyDocumentListFilters(countQuery, opts, true, ingestionStatus)if err := countQuery.Count(&total).Error; err != nil {return nil, 0, err}- orderBy := documentListOrderColumn(opts.OrderBy)+ orderBy := documentListOrderColumn(opts.OrderBy, ingestionStatus)if opts.Desc {orderBy += " DESC"} else {@@ -212,13 +230,14 @@ func (dao *DocumentDAO) GetFilterByKBID(ctx context.Context, db *gorm.DB, opts DIngestionStatus *string `gorm:"column:ingestion_status"`Suffix string `gorm:"column:suffix"`}+ ingestionStatus := buildDocumentListIngestionStatusExpression(db)query := db.WithContext(ctx).Table("document").- Select("document.id, ingestion_task.status as ingestion_status, document.suffix").+ Select("document.id, " + ingestionStatus + " as ingestion_status, document.suffix").Joins("JOIN file2document ON file2document.document_id = document.id").Joins("JOIN file ON file.id = file2document.file_id").Joins(latestIngestionTaskJoin)- query = applyDocumentListFilters(query, opts, true)+ query = applyDocumentListFilters(query, opts, true, ingestionStatus)if err := query.Scan(&rows).Error; err != nil {return nil, 0, err@@ -247,21 +266,23 @@ func (dao *DocumentDAO) GetFilterByKBID(ctx context.Context, db *gorm.DB, opts D// ListIDsByKBIDWithOptions lists matching document IDs without pagination.func (dao *DocumentDAO) ListIDsByKBIDWithOptions(ctx context.Context, db *gorm.DB, opts DocumentListOptions) ([]string, error) {var ids []string+ var ingestionStatus stringquery := db.WithContext(ctx).Table("document").Select("document.id").Joins("JOIN file2document ON file2document.document_id = document.id").Joins("JOIN file ON file.id = file2document.file_id")if len(opts.RunStatuses) > 0 {+ ingestionStatus = buildDocumentListIngestionStatusExpression(db)query = query.Joins(latestIngestionTaskJoin)}- query = applyDocumentListFilters(query, opts, true)+ query = applyDocumentListFilters(query, opts, true, ingestionStatus)if err := query.Scan(&ids).Error; err != nil {return nil, err}return ids, nil}-func applyDocumentListFilters(query *gorm.DB, opts DocumentListOptions, qualified bool) *gorm.DB {+func applyDocumentListFilters(query *gorm.DB, opts DocumentListOptions, qualified bool, ingestionStatus string) *gorm.DB {column := func(name string) string {if qualified {return "document." + name@@ -274,22 +295,7 @@ func applyDocumentListFilters(query *gorm.DB, opts DocumentListOptions, qualifiequery = query.Where("LOWER("+column("name")+") LIKE ?", "%"+strings.ToLower(strings.TrimSpace(opts.Keywords))+"%")}if len(opts.RunStatuses) > 0 {- hasUnstart := false- otherStatuses := make([]string, 0, len(opts.RunStatuses))- for _, s := range opts.RunStatuses {- if strings.EqualFold(s, "UNSTART") {- hasUnstart = true- } else {- otherStatuses = append(otherStatuses, s)- }- }- if hasUnstart && len(otherStatuses) > 0 {- query = query.Where("(ingestion_task.status IN ? OR ingestion_task.id IS NULL OR ingestion_task.status = ?)", otherStatuses, "UNSTART")- } else if hasUnstart {- query = query.Where("(ingestion_task.id IS NULL OR ingestion_task.status = ?)", "UNSTART")- } else {- query = query.Where("ingestion_task.status IN ?", otherStatuses)- }+ query = query.Where("("+ingestionStatus+") IN ?", opts.RunStatuses)}if len(opts.Types) > 0 {query = query.Where(column("type")+" IN ?", opts.Types)@@ -313,7 +319,7 @@ func applyDocumentListFilters(query *gorm.DB, opts DocumentListOptions, qualifiereturn query}-func documentListOrderColumn(orderBy string) string {+func documentListOrderColumn(orderBy, ingestionStatus string) string {switch orderBy {case "update_time":return "document.update_time"@@ -324,7 +330,7 @@ func documentListOrderColumn(orderBy string) string {case "type":return "document.type"case "run", "ingestion_status":- return "COALESCE(ingestion_task.status, 'UNSTART')"+ return ingestionStatusdefault:return "document.create_time"}diff --git a/internal/dao/document_test.go b/internal/dao/document_test.goindex 2bd47c4c6b..f127c33883 100644--- a/internal/dao/document_test.go+++ b/internal/dao/document_test.go@@ -196,7 +196,8 @@ func TestDocumentListIncludesScheduledIngestionStatus(t *testing.T) {t.Fatalf("create scheduled ingestion task: %v", err)}- documents, total, err := NewDocumentDAO().ListByKBIDWithOptions(t.Context(), db, DocumentListOptions{+ dao := NewDocumentDAO()+ documents, total, err := dao.ListByKBIDWithOptions(t.Context(), db, DocumentListOptions{KbID: "kb-1",OrderBy: "create_time",Desc: true,@@ -221,6 +222,176 @@ func TestDocumentListIncludesScheduledIngestionStatus(t *testing.T) {if got := listed["ingestion_status"]; got != "SCHEDULED" {t.Fatalf("ingestion_status = %v, want %q", got, "SCHEDULED")}++ opts := DocumentListOptions{KbID: "kb-1", RunStatuses: []string{common.SCHEDULED}, OrderBy: "ingestion_status", Limit: 10}+ documents, total, err = dao.ListByKBIDWithOptions(t.Context(), db, opts)+ if err != nil || total != 1 || len(documents) != 1 {+ t.Fatalf("filter scheduled documents: got %d (total %d), err %v", len(documents), total, err)+ }+ ids, err := dao.ListIDsByKBIDWithOptions(t.Context(), db, opts)+ if err != nil || len(ids) != 1 || ids[0] != "doc-scheduled" {+ t.Fatalf("filter scheduled document IDs: got %v, err %v", ids, err)+ }+ filters, total, err := dao.GetFilterByKBID(t.Context(), db, DocumentListOptions{KbID: "kb-1"})+ if err != nil || total != 1 {+ t.Fatalf("get scheduled filter count: total %d, err %v", total, err)+ }+ if got := filters["ingestion_status"].(map[string]int64)[common.SCHEDULED]; got != 1 {+ t.Fatalf("scheduled filter count = %d, want 1", got)+ }+ if db.Migrator().HasColumn(&entity.Document{}, "run") {+ t.Fatal("document list query added the legacy run column")+ }+}++func TestDocumentListFallsBackToReadOnlyLegacyRun(t *testing.T) {+ db := setupDocumentTestDB(t)+ if err := db.AutoMigrate(+ &entity.User{},+ &entity.UserCanvas{},+ &entity.File{},+ &entity.File2Document{},+ &entity.IngestionTask{},+ ); err != nil {+ t.Fatalf("migrate document-list dependencies: %v", err)+ }+ if err := db.Exec("ALTER TABLE document ADD COLUMN run TEXT").Error; err != nil {+ t.Fatalf("add legacy run fixture column: %v", err)+ }++ tests := []struct {+ id, run, task, want string+ hasTask bool+ }{+ {id: "unstart", run: "0", want: "UNSTART"},+ {id: "empty", want: "UNSTART"},+ {id: "running", run: "1", want: common.RUNNING},+ {id: "stopped", run: "2", want: common.STOPPED},+ {id: "completed", run: "3", want: common.COMPLETED},+ {id: "failed", run: "4", want: common.FAILED},+ {id: "scheduled", run: "5", want: common.SCHEDULED},+ {id: "unknown", run: "9", want: "UNSTART"},+ {id: "task-wins", run: "3", task: common.RUNNING, hasTask: true, want: common.RUNNING},+ {id: "empty-task-wins", run: "3", hasTask: true, want: "UNSTART"},+ }++ create := func(value interface{}) {+ t.Helper()+ if err := db.Create(value).Error; err != nil {+ t.Fatalf("create %T: %v", value, err)+ }+ }+ fileID := "file-legacy"+ create(&entity.File{ID: fileID, ParentID: "parent-1", TenantID: "tenant-1", CreatedBy: "user-1", Name: "legacy.pdf", Type: "document"})+ for _, test := range tests {+ documentID := "doc-" + test.id+ create(&entity.Document{ID: documentID, KbID: "kb-legacy", ParserConfig: entity.JSONMap{}})+ if err := db.Table("document").Where("id = ?", documentID).UpdateColumn("run", test.run).Error; err != nil {+ t.Fatalf("set legacy run for %s: %v", documentID, err)+ }+ create(&entity.File2Document{ID: "link-" + test.id, FileID: sp(fileID), DocumentID: sp(documentID)})+ if test.hasTask {+ create(&entity.IngestionTask{ID: "task-" + test.id, DocumentID: documentID, Status: test.task})+ }+ }++ dao := NewDocumentDAO()+ documents, total, err := dao.ListByKBIDWithOptions(t.Context(), db, DocumentListOptions{+ KbID: "kb-legacy", Limit: len(tests),+ })+ if err != nil {+ t.Fatalf("list legacy documents: %v", err)+ }+ if total != int64(len(tests)) || len(documents) != len(tests) {+ t.Fatalf("listed %d documents (total %d), want %d", len(documents), total, len(tests))+ }++ statusByID := make(map[string]string, len(documents))+ for _, document := range documents {+ if document.IngestionStatus != nil {+ statusByID[document.ID] = *document.IngestionStatus+ }+ }+ for _, test := range tests {+ documentID := "doc-" + test.id+ if got := statusByID[documentID]; got != test.want {+ t.Errorf("status for %s = %q, want %q", documentID, got, test.want)+ }+ }++ completedOpts := DocumentListOptions{KbID: "kb-legacy", RunStatuses: []string{common.COMPLETED}, Limit: len(tests)}+ documents, total, err = dao.ListByKBIDWithOptions(t.Context(), db, completedOpts)+ if err != nil || total != 1 || len(documents) != 1 || documents[0].ID != "doc-completed" {+ t.Fatalf("filter completed documents: got %v (total %d), err %v", documents, total, err)+ }+ ids, err := dao.ListIDsByKBIDWithOptions(t.Context(), db, completedOpts)+ if err != nil || len(ids) != 1 || ids[0] != "doc-completed" {+ t.Fatalf("filter completed document IDs: got %v, err %v", ids, err)+ }+ filters, filterTotal, err := dao.GetFilterByKBID(t.Context(), db, completedOpts)+ if err != nil || filterTotal != 1 || filters["ingestion_status"].(map[string]int64)[common.COMPLETED] != 1 {+ t.Fatalf("filter completed count: got %v (total %d), err %v", filters, filterTotal, err)+ }++ mixedOpts := DocumentListOptions{KbID: "kb-legacy", RunStatuses: []string{"UNSTART", common.COMPLETED}, Limit: len(tests)}+ documents, total, err = dao.ListByKBIDWithOptions(t.Context(), db, mixedOpts)+ if err != nil || total != 5 || len(documents) != 5 {+ t.Fatalf("filter mixed statuses: got %d (total %d), err %v", len(documents), total, err)+ }+ wantMixed := map[string]bool{+ "doc-unstart": true, "doc-empty": true, "doc-completed": true, "doc-unknown": true, "doc-empty-task-wins": true,+ }+ for _, document := range documents {+ delete(wantMixed, document.ID)+ }+ if len(wantMixed) != 0 {+ t.Fatalf("mixed status filter missed documents: %v", wantMixed)+ }+ ids, err = dao.ListIDsByKBIDWithOptions(t.Context(), db, mixedOpts)+ if err != nil || len(ids) != 5 {+ t.Fatalf("filter mixed document IDs: got %v, err %v", ids, err)+ }++ filters, filterTotal, err = dao.GetFilterByKBID(t.Context(), db, DocumentListOptions{KbID: "kb-legacy"})+ if err != nil || filterTotal != int64(len(tests)) {+ t.Fatalf("get legacy filter counts: total %d, err %v", filterTotal, err)+ }+ counts := filters["ingestion_status"].(map[string]int64)+ wantCounts := map[string]int64{"UNSTART": 4, common.RUNNING: 2, common.STOPPED: 1, common.COMPLETED: 1, common.FAILED: 1, common.SCHEDULED: 1}+ for status, want := range wantCounts {+ if got := counts[status]; got != want {+ t.Errorf("filter count for %s = %d, want %d", status, got, want)+ }+ }++ wantAscending := []string{common.COMPLETED, common.FAILED, common.RUNNING, common.RUNNING, common.SCHEDULED, common.STOPPED, "UNSTART", "UNSTART", "UNSTART", "UNSTART"}+ for _, desc := range []bool{false, true} {+ documents, total, err = dao.ListByKBIDWithOptions(t.Context(), db, DocumentListOptions{+ KbID: "kb-legacy", OrderBy: "ingestion_status", Desc: desc, Limit: len(tests),+ })+ if err != nil || total != int64(len(tests)) || len(documents) != len(tests) {+ t.Fatalf("sort legacy statuses (desc=%t): got %d (total %d), err %v", desc, len(documents), total, err)+ }+ for i, document := range documents {+ want := wantAscending[i]+ if desc {+ want = wantAscending[len(wantAscending)-1-i]+ }+ if document.IngestionStatus == nil || *document.IngestionStatus != want {+ t.Errorf("sorted status %d (desc=%t) = %v, want %s", i, desc, document.IngestionStatus, want)+ }+ }+ }++ for _, test := range tests {+ var stored string+ if err := db.Table("document").Select("run").Where("id = ?", "doc-"+test.id).Scan(&stored).Error; err != nil {+ t.Fatalf("read legacy run for %s: %v", test.id, err)+ }+ if stored != test.run {+ t.Fatalf("legacy run for %s changed from %q to %q", test.id, test.run, stored)+ }+ }}func TestDocumentListDeduplicatesHistoricalIngestionTasks(t *testing.T) {