Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

refactor(ingestion): decouple document parsing state from document.run in favor of ingestion_task (#19533)

jay77721d45c8aa148fb
diff --git a/internal/common/constants.go b/internal/common/constants.go
index 96a301ba39..bddc6a87e2 100644
--- a/internal/common/constants.go
+++ b/internal/common/constants.go
@@ -39,6 +39,37 @@ const (
STOPPING = "STOPPING"
)
+// ActiveTaskStatuses contains all in-flight, non-terminal task statuses.
+var ActiveTaskStatuses = []string{CREATED, SCHEDULED, RUNNING, STOPPING}
+
+// IsActiveTaskStatus returns true if the status represents an active (non-terminal) task:
+// CREATED, SCHEDULED, RUNNING, or STOPPING.
+func IsActiveTaskStatus(status string) bool {
+ switch status {
+ case CREATED, SCHEDULED, RUNNING, STOPPING:
+ return true
+ default:
+ return false
+ }
+}
+
+// IsTerminalTaskStatus returns true if the status represents a terminal task state:
+// COMPLETED, STOPPED, or FAILED.
+func IsTerminalTaskStatus(status string) bool {
+ switch status {
+ case COMPLETED, STOPPED, FAILED:
+ return true
+ default:
+ return false
+ }
+}
+
+// IsRunningOrStopping returns true if a worker is actively executing or shutting down:
+// RUNNING or STOPPING.
+func IsRunningOrStopping(status string) bool {
+ return status == RUNNING || status == STOPPING
+}
+
// StatusDialogValid is the dialog.status value that gates public bot
// access. Mirrors Python's StatusEnum.VALID.value at
// api/common/constants.py (the string "1"). All chatbot/agentbot
diff --git a/internal/dao/document.go b/internal/dao/document.go
index fe0f45e040..f76d4858a1 100644
--- a/internal/dao/document.go
+++ b/internal/dao/document.go
@@ -18,6 +18,8 @@ package dao
import (
"context"
+ "fmt"
+ "ragflow/internal/common"
"ragflow/internal/entity"
"strings"
@@ -131,6 +133,20 @@ func (dao *DocumentDAO) ListByKBID(ctx context.Context, db *gorm.DB, kbID, keywo
})
}
+const latestIngestionTaskJoin = `LEFT JOIN ingestion_task ON ingestion_task.document_id = document.id
+ AND NOT EXISTS (
+ SELECT 1
+ FROM ingestion_task newer_ingestion_task
+ WHERE newer_ingestion_task.document_id = ingestion_task.document_id
+ AND (
+ COALESCE(newer_ingestion_task.create_time, 0) > COALESCE(ingestion_task.create_time, 0)
+ OR (
+ COALESCE(newer_ingestion_task.create_time, 0) = COALESCE(ingestion_task.create_time, 0)
+ AND newer_ingestion_task.id > ingestion_task.id
+ )
+ )
+ )`
+
// ListByKBIDWithOptions lists documents by knowledge base ID with filters.
func (dao *DocumentDAO) ListByKBIDWithOptions(ctx context.Context, db *gorm.DB, opts DocumentListOptions) ([]*entity.DocumentListItem, int64, error) {
var documents []*entity.DocumentListItem
@@ -145,22 +161,17 @@ func (dao *DocumentDAO) ListByKBIDWithOptions(ctx context.Context, db *gorm.DB,
Joins("JOIN file ON file.id = file2document.file_id").
Joins("LEFT JOIN user_canvas ON document.pipeline_id = user_canvas.id").
Joins("LEFT JOIN user ON document.created_by = user.id").
- Joins(`LEFT JOIN ingestion_task ON ingestion_task.document_id = document.id
- AND NOT EXISTS (
- SELECT 1
- FROM ingestion_task newer_ingestion_task
- WHERE newer_ingestion_task.document_id = ingestion_task.document_id
- AND (
- COALESCE(newer_ingestion_task.create_time, 0) > COALESCE(ingestion_task.create_time, 0)
- OR (
- COALESCE(newer_ingestion_task.create_time, 0) = COALESCE(ingestion_task.create_time, 0)
- AND newer_ingestion_task.id > ingestion_task.id
- )
- )
- )`)
+ Joins(latestIngestionTaskJoin)
+
+ countQuery := db.WithContext(ctx).Table("document").
+ Joins("JOIN file2document ON file2document.document_id = document.id").
+ Joins("JOIN file ON file.id = file2document.file_id")
+ if len(opts.RunStatuses) > 0 {
+ countQuery = countQuery.Joins(latestIngestionTaskJoin)
+ }
listQuery = applyDocumentListFilters(listQuery, opts, true)
- countQuery := applyDocumentListFilters(db.WithContext(ctx).Model(&entity.Document{}), opts, false)
+ countQuery = applyDocumentListFilters(countQuery, opts, true)
if err := countQuery.Count(&total).Error; err != nil {
return nil, 0, err
@@ -184,15 +195,16 @@ func (dao *DocumentDAO) ListByKBIDWithOptions(ctx context.Context, db *gorm.DB,
// GetFilterByKBID returns aggregate filter counts for documents in a dataset.
func (dao *DocumentDAO) GetFilterByKBID(ctx context.Context, db *gorm.DB, opts DocumentListOptions) (map[string]interface{}, int64, error) {
var rows []struct {
- ID string `gorm:"column:id"`
- Run *string `gorm:"column:run"`
- Suffix string `gorm:"column:suffix"`
+ ID string `gorm:"column:id"`
+ IngestionStatus *string `gorm:"column:ingestion_status"`
+ Suffix string `gorm:"column:suffix"`
}
query := db.WithContext(ctx).Table("document").
- Select("document.id, document.run, document.suffix").
+ Select("document.id, ingestion_task.status as ingestion_status, document.suffix").
Joins("JOIN file2document ON file2document.document_id = document.id").
- Joins("JOIN file ON file.id = file2document.file_id")
+ Joins("JOIN file ON file.id = file2document.file_id").
+ Joins(latestIngestionTaskJoin)
query = applyDocumentListFilters(query, opts, true)
if err := query.Scan(&rows).Error; err != nil {
@@ -200,20 +212,22 @@ func (dao *DocumentDAO) GetFilterByKBID(ctx context.Context, db *gorm.DB, opts D
}
suffixCounter := map[string]int64{}
- runStatusCounter := map[string]int64{}
+ statusCounter := map[string]int64{}
for _, row := range rows {
if row.Suffix != "" {
suffixCounter[row.Suffix]++
}
- if row.Run != nil {
- runStatusCounter[*row.Run]++
+ status := "UNSTART"
+ if row.IngestionStatus != nil && *row.IngestionStatus != "" {
+ status = *row.IngestionStatus
}
+ statusCounter[status]++
}
return map[string]interface{}{
- "suffix": suffixCounter,
- "run_status": runStatusCounter,
- "metadata": map[string]interface{}{},
+ "suffix": suffixCounter,
+ "ingestion_status": statusCounter,
+ "metadata": map[string]interface{}{},
}, int64(len(rows)), nil
}
@@ -224,6 +238,9 @@ func (dao *DocumentDAO) ListIDsByKBIDWithOptions(ctx context.Context, db *gorm.D
Select("document.id").
Joins("JOIN file2document ON file2document.document_id = document.id").
Joins("JOIN file ON file.id = file2document.file_id")
+ if len(opts.RunStatuses) > 0 {
+ query = query.Joins(latestIngestionTaskJoin)
+ }
query = applyDocumentListFilters(query, opts, true)
if err := query.Scan(&ids).Error; err != nil {
return nil, err
@@ -244,7 +261,22 @@ func applyDocumentListFilters(query *gorm.DB, opts DocumentListOptions, qualifie
query = query.Where("LOWER("+column("name")+") LIKE ?", "%"+strings.ToLower(strings.TrimSpace(opts.Keywords))+"%")
}
if len(opts.RunStatuses) > 0 {
- query = query.Where(column("run")+" IN ?", opts.RunStatuses)
+ hasUnstart := false
+ otherStatuses := make([]string, 0, len(opts.RunStatuses))
+ for _, s := range opts.RunStatuses {
+ if strings.EqualFold(s, "UNSTART") {
+ hasUnstart = true
+ } else {
+ otherStatuses = append(otherStatuses, s)
+ }
+ }
+ if hasUnstart && len(otherStatuses) > 0 {
+ query = query.Where("(ingestion_task.status IN ? OR ingestion_task.id IS NULL OR ingestion_task.status = ?)", otherStatuses, "UNSTART")
+ } else if hasUnstart {
+ query = query.Where("(ingestion_task.id IS NULL OR ingestion_task.status = ?)", "UNSTART")
+ } else {
+ query = query.Where("ingestion_task.status IN ?", otherStatuses)
+ }
}
if len(opts.Types) > 0 {
query = query.Where(column("type")+" IN ?", opts.Types)
@@ -278,8 +310,8 @@ func documentListOrderColumn(orderBy string) string {
return "document.size"
case "type":
return "document.type"
- case "run":
- return "document.run"
+ case "run", "ingestion_status":
+ return "COALESCE(ingestion_task.status, 'UNSTART')"
default:
return "document.create_time"
}
@@ -454,31 +486,37 @@ func (dao *DocumentDAO) GetParsingStatusByKBID(ctx context.Context, db *gorm.DB,
}
var rows []struct {
- Run *string `gorm:"column:run"`
- Cnt int64 `gorm:"column:cnt"`
+ Status *string `gorm:"column:status"`
+ Cnt int64 `gorm:"column:cnt"`
}
- err := db.WithContext(ctx).Model(&entity.Document{}).
- Select("run, COUNT(id) as cnt").
- Where("kb_id = ?", kbID).
- Group("run").
+ err := db.WithContext(ctx).Table("document").
+ Select("ingestion_task.status, COUNT(document.id) as cnt").
+ Joins(latestIngestionTaskJoin).
+ Where("document.kb_id = ?", kbID).
+ Group("ingestion_task.status").
Scan(&rows).Error
if err != nil {
return nil, err
}
- statusFieldMap := map[string]string{
- string(entity.TaskStatusUnstart): "unstart_count",
- string(entity.TaskStatusRunning): "running_count",
- string(entity.TaskStatusCancel): "cancel_count",
- string(entity.TaskStatusDone): "done_count",
- string(entity.TaskStatusFail): "fail_count",
- }
for _, row := range rows {
- if row.Run == nil {
+ if row.Status == nil || *row.Status == "" {
+ result["unstart_count"] += row.Cnt
continue
}
- if field, ok := statusFieldMap[*row.Run]; ok {
- result[field] = row.Cnt
+ switch *row.Status {
+ case common.CREATED, common.SCHEDULED:
+ result["unstart_count"] += row.Cnt
+ case common.RUNNING, common.STOPPING:
+ result["running_count"] += row.Cnt
+ case common.STOPPED:
+ result["cancel_count"] += row.Cnt
+ case common.COMPLETED:
+ result["done_count"] += row.Cnt
+ case common.FAILED:
+ result["fail_count"] += row.Cnt
+ default:
+ common.Warn(fmt.Sprintf("GetParsingStatusByKBID: unrecognized task status %q for dataset %s (count: %d)", *row.Status, kbID, row.Cnt))
}
}
return result, nil
diff --git a/internal/dao/document_test.go b/internal/dao/document_test.go
index eec3f30a58..141ab6c01f 100644
--- a/internal/dao/document_test.go
+++ b/internal/dao/document_test.go
@@ -23,6 +23,7 @@ import (
"github.com/glebarez/sqlite"
"gorm.io/gorm"
+ "ragflow/internal/common"
"ragflow/internal/entity"
)
@@ -389,4 +390,65 @@ func TestDocumentGetChunkingConfigScansParserConfig(t *testing.T) {
}
}
+func TestDocumentDAOGetParsingStatusByKBID(t *testing.T) {
+ db := setupDocumentTestDB(t)
+ if err := db.AutoMigrate(&entity.IngestionTask{}); err != nil {
+ t.Fatalf("migrate IngestionTask: %v", err)
+ }
+
+ // doc-unstart: no task
+ if err := db.Create(&entity.Document{ID: "doc-1", KbID: "kb-status", ParserConfig: entity.JSONMap{}}).Error; err != nil {
+ t.Fatalf("create doc-1: %v", err)
+ }
+ // doc-running: task RUNNING
+ if err := db.Create(&entity.Document{ID: "doc-2", KbID: "kb-status", ParserConfig: entity.JSONMap{}}).Error; err != nil {
+ t.Fatalf("create doc-2: %v", err)
+ }
+ if err := db.Create(&entity.IngestionTask{ID: "task-2", DocumentID: "doc-2", Status: common.RUNNING}).Error; err != nil {
+ t.Fatalf("create task-2: %v", err)
+ }
+ // doc-completed: task COMPLETED
+ if err := db.Create(&entity.Document{ID: "doc-3", KbID: "kb-status", ParserConfig: entity.JSONMap{}}).Error; err != nil {
+ t.Fatalf("create doc-3: %v", err)
+ }
+ if err := db.Create(&entity.IngestionTask{ID: "task-3", DocumentID: "doc-3", Status: common.COMPLETED}).Error; err != nil {
+ t.Fatalf("create task-3: %v", err)
+ }
+ // doc-failed: task FAILED
+ if err := db.Create(&entity.Document{ID: "doc-4", KbID: "kb-status", ParserConfig: entity.JSONMap{}}).Error; err != nil {
+ t.Fatalf("create doc-4: %v", err)
+ }
+ if err := db.Create(&entity.IngestionTask{ID: "task-4", DocumentID: "doc-4", Status: common.FAILED}).Error; err != nil {
+ t.Fatalf("create task-4: %v", err)
+ }
+ // doc-stopped: task STOPPED
+ if err := db.Create(&entity.Document{ID: "doc-5", KbID: "kb-status", ParserConfig: entity.JSONMap{}}).Error; err != nil {
+ t.Fatalf("create doc-5: %v", err)
+ }
+ if err := db.Create(&entity.IngestionTask{ID: "task-5", DocumentID: "doc-5", Status: common.STOPPED}).Error; err != nil {
+ t.Fatalf("create task-5: %v", err)
+ }
+
+ dao := NewDocumentDAO()
+ counts, err := dao.GetParsingStatusByKBID(t.Context(), db, "kb-status")
+ if err != nil {
+ t.Fatalf("GetParsingStatusByKBID failed: %v", err)
+ }
+ if counts["unstart_count"] != 1 {
+ t.Errorf("unstart_count = %d, want 1", counts["unstart_count"])
+ }
+ if counts["running_count"] != 1 {
+ t.Errorf("running_count = %d, want 1", counts["running_count"])
+ }
+ if counts["done_count"] != 1 {
+ t.Errorf("done_count = %d, want 1", counts["done_count"])
+ }
+ if counts["fail_count"] != 1 {
+ t.Errorf("fail_count = %d, want 1", counts["fail_count"])
+ }
+ if counts["cancel_count"] != 1 {
+ t.Errorf("cancel_count = %d, want 1", counts["cancel_count"])
+ }
+}
+
func sp(s string) *string { return &s }
diff --git a/internal/dao/ingestion.go b/internal/dao/ingestion.go
index fe6448036e..6ac2cef08b 100644
--- a/internal/dao/ingestion.go
+++ b/internal/dao/ingestion.go
@@ -216,6 +216,29 @@ func (dao *IngestionTaskDAO) GetByDocumentID(ctx context.Context, db *gorm.DB, d
return tasks[0], nil
}
+// GetLatestByDocumentIDs returns a map of documentID -> latest IngestionTask.
+func (dao *IngestionTaskDAO) GetLatestByDocumentIDs(ctx context.Context, db *gorm.DB, documentIDs []string) (map[string]*entity.IngestionTask, error) {
+ if len(documentIDs) == 0 {
+ return map[string]*entity.IngestionTask{}, nil
+ }
+ var tasks []*entity.IngestionTask
+ err := db.WithContext(ctx).
+ Where("document_id IN ?", documentIDs).
+ Order("COALESCE(create_time, 0) DESC").
+ Order("id DESC").
+ Find(&tasks).Error
+ if err != nil {
+ return nil, err
+ }
+ result := make(map[string]*entity.IngestionTask, len(documentIDs))
+ for _, task := range tasks {
+ if _, exists := result[task.DocumentID]; !exists {
+ result[task.DocumentID] = task
+ }
+ }
+ return result, nil
+}
+
// CountActiveByDatasetID returns the number of ingestion tasks for the
// dataset whose latest task is non-terminal (CREATED/SCHEDULED/RUNNING/STOPPING).
// It uses the same create_time/ID ordering as document-list state so historical
@@ -236,7 +259,7 @@ func (dao *IngestionTaskDAO) CountActiveByDatasetID(ctx context.Context, db *gor
AND newer_ingestion_task.id > ingestion_task.id
)
)
- )`, datasetID, []string{common.CREATED, common.SCHEDULED, common.RUNNING, common.STOPPING}).
+ )`, datasetID, common.ActiveTaskStatuses).
Count(&count).Error
return count, err
}
diff --git a/internal/entity/dataset.go b/internal/entity/dataset.go
index 37ed80226e..09d9a3d7f6 100644
--- a/internal/entity/dataset.go
+++ b/internal/entity/dataset.go
@@ -73,19 +73,6 @@ const (
TaskStatusSchedule TaskStatus = "5"
)
-// DocumentModifiableStatuses are the run statuses in which a document's
-// configuration (parser config, chunk method, pipeline id, name, enabled
-// flag, ...) may be edited via UpdateDatasetDocument. Documents that are
-// running ("1") or scheduled ("5") must not be edited — the in-flight parser
-// would race with the config change and the document can get stuck and become
-// undeletable.
-var DocumentModifiableStatuses = map[TaskStatus]bool{
- TaskStatusUnstart: true,
- TaskStatusCancel: true,
- TaskStatusDone: true,
- TaskStatusFail: true,
-}
-
// PipelineTaskType represents the type of pipeline task
type PipelineTaskType string
diff --git a/internal/entity/document.go b/internal/entity/document.go
index 37b16d41b8..6f202283fa 100644
--- a/internal/entity/document.go
+++ b/internal/entity/document.go
@@ -40,7 +40,6 @@ type Document struct {
ProcessDuration float64 `gorm:"column:process_duration;default:0" json:"process_duration"`
ContentHash *string `gorm:"column:content_hash;size:32;index;default:''" json:"content_hash,omitempty"`
Suffix string `gorm:"column:suffix;size:32;not null;index" json:"suffix"`
- Run *string `gorm:"column:run;size:1;index;default:'0'" json:"run,omitempty"`
Status *string `gorm:"column:status;size:1;index;default:'1'" json:"status,omitempty"`
BaseModel
}
@@ -69,7 +68,6 @@ type DocumentListItem struct {
ProcessDuration float64 `gorm:"column:process_duration" json:"process_duration"`
ContentHash *string `gorm:"column:content_hash" json:"content_hash,omitempty"`
Suffix string `gorm:"column:suffix" json:"suffix"`
- Run *string `gorm:"column:run" json:"run,omitempty"`
IngestionStatus *string `gorm:"column:ingestion_status" json:"ingestion_status,omitempty"`
Status *string `gorm:"column:status" json:"status,omitempty"`
CreateTime *int64 `gorm:"column:create_time" json:"create_time,omitempty"`
diff --git a/internal/handler/document.go b/internal/handler/document.go
index cc2e7bda17..a0afc4b833 100644
--- a/internal/handler/document.go
+++ b/internal/handler/document.go
@@ -658,9 +658,9 @@ func parseDocumentListOptions(c *gin.Context, datasetID string) (dao.DocumentLis
Types: queryValues(c, "types"),
}
- opts.RunStatuses = normalizeRunStatusFilter(queryValues(c, "run", "run_status"))
- if len(queryValues(c, "run", "run_status")) > 0 && len(opts.RunStatuses) == 0 {
- return opts, fmt.Sprintf("Invalid filter run status conditions: %s", strings.Join(invalidRunStatuses(queryValues(c, "run", "run_status")), ", "))
+ opts.RunStatuses = normalizeRunStatusFilter(queryValues(c, "run", "run_status", "ingestion_status"))
+ if len(queryValues(c, "run", "run_status", "ingestion_status")) > 0 && len(opts.RunStatuses) == 0 {
+ return opts, fmt.Sprintf("Invalid filter run status conditions: %s", strings.Join(invalidRunStatuses(queryValues(c, "run", "run_status", "ingestion_status")), ", "))
}
opts.Name = c.Query("name")
@@ -880,31 +880,25 @@ func queryValues(c *gin.Context, names ...string) []string {
return out
}
+var runStatusMap = map[string]string{
+ "UNSTART": "UNSTART",
+ "CREATED": common.CREATED,
+ "SCHEDULED": common.SCHEDULED,
+ "RUNNING": common.RUNNING,
+ "STOPPING": common.STOPPING,
+ "STOPPED": common.STOPPED,
+ "COMPLETED": common.COMPLETED,
+ "FAILED": common.FAILED,
+}
+
func normalizeRunStatusFilter(statuses []string) []string {
if len(statuses) == 0 {
return nil
}
- statusTextToNumeric := map[string]string{
- "UNSTART": string(entity.TaskStatusUnstart),
- "RUNNING": string(entity.TaskStatusRunning),
- "CANCEL": string(entity.TaskStatusCancel),
- "DONE": string(entity.TaskStatusDone),
- "FAIL": string(entity.TaskStatusFail),
- }
- validStatuses := map[string]bool{
- string(entity.TaskStatusUnstart): true,
- string(entity.TaskStatusRunning): true,
- string(entity.TaskStatusCancel): true,
- string(entity.TaskStatusDone): true,
- string(entity.TaskStatusFail): true,
- }
out := make([]string, 0, len(statuses))
for _, status := range statuses {
- normalized := statusTextToNumeric[strings.ToUpper(status)]
- if normalized == "" {
- normalized = status
- }
- if !validStatuses[normalized] {
+ normalized, ok := runStatusMap[strings.ToUpper(strings.TrimSpace(status))]
+ if !ok {
return nil
}
out = append(out, normalized)
@@ -915,10 +909,9 @@ func normalizeRunStatusFilter(statuses []string) []string {
// invalidRunStatuses returns the raw filter values that do not map to a valid
// run status, mirroring Python's "Invalid filter run status conditions: ...".
func invalidRunStatuses(statuses []string) []string {
- valid := map[string]bool{"UNSTART": true, "RUNNING": true, "CANCEL": true, "DONE": true, "FAIL": true}
invalid := make([]string, 0)
for _, status := range statuses {
- if !valid[strings.ToUpper(status)] {
+ if _, ok := runStatusMap[strings.ToUpper(strings.TrimSpace(status))]; !ok {
invalid = append(invalid, status)
}
}
@@ -1085,15 +1078,19 @@ func (h *DocumentHandler) uploadWebDocument(c *gin.Context, kb *entity.Knowledge
// mapDocKeysWithRunStatus renames a freshly-created document's raw keys to the
// public response shape (chunk_num→chunk_count, token_num→token_count,
-// kb_id→dataset_id) and reports run as a label.
+// kb_id→dataset_id) and sets ingestion_status to UNSTART.
// Mirrors Python map_doc_keys_with_run_status / map_doc_keys.
func mapDocKeysWithRunStatus(raw map[string]interface{}) map[string]interface{} {
+ ingestionStatus := "UNSTART"
+ if s, ok := raw["ingestion_status"].(string); ok && s != "" {
+ ingestionStatus = s
+ }
out := map[string]interface{}{
- "chunk_count": raw["chunk_num"],
- "token_count": raw["token_num"],
- "dataset_id": raw["kb_id"],
- "parser_id": raw["parser_id"],
- "run": "UNSTART",
+ "chunk_count": raw["chunk_num"],
+ "token_count": raw["token_num"],
+ "dataset_id": raw["kb_id"],
+ "parser_id": raw["parser_id"],
+ "ingestion_status": ingestionStatus,
}
for _, k := range []string{"id", "name", "type", "size", "suffix", "source_type", "created_by", "parser_config", "location", "pipeline_id", "content_hash"} {
if v, ok := raw[k]; ok {
@@ -1157,12 +1154,16 @@ func (h *DocumentHandler) DownloadDocument(c *gin.Context) {
func mapDocumentListItem(doc *entity.DocumentListItem, metaFields map[string]interface{}) map[string]interface{} {
processDuration := doc.ProcessDuration
- if doc.Run != nil && strings.TrimSpace(*doc.Run) == "1" && doc.ProcessBeginAt != nil {
+ if doc.IngestionStatus != nil && *doc.IngestionStatus == common.RUNNING && doc.ProcessBeginAt != nil {
processDuration = time.Since(*doc.ProcessBeginAt).Seconds()
if processDuration < 0 {
processDuration = 0
}
}
+ ingestionStatus := "UNSTART"
+ if doc.IngestionStatus != nil && *doc.IngestionStatus != "" {
+ ingestionStatus = *doc.IngestionStatus
+ }
item := map[string]interface{}{
"id": doc.ID,
"dataset_id": doc.KbID,
@@ -1179,7 +1180,7 @@ func mapDocumentListItem(doc *entity.DocumentListItem, metaFields map[string]int
"process_begin_at": formatTimePtr(doc.ProcessBeginAt),
"process_duration": processDuration,
"suffix": doc.Suffix,
- "run": mapRunStatus(doc.Run),
+ "ingestion_status": ingestionStatus,
"status": stringValue(doc.Status),
"parser_id": doc.ParserID,
"chunk_method": doc.ParserID,
@@ -1193,9 +1194,6 @@ func mapDocumentListItem(doc *entity.DocumentListItem, metaFields map[string]int
"update_time": int64(0),
"update_date": "",
}
- if doc.IngestionStatus != nil {
- item["ingestion_status"] = *doc.IngestionStatus
- }
if doc.CreateTime != nil {
item["create_time"] = *doc.CreateTime
@@ -1226,27 +1224,6 @@ func decodeJSONMap(raw string) map[string]interface{} {
return data
}
-func mapRunStatus(run *string) string {
- if run == nil {
- return "UNSTART"
- }
-
- switch strings.TrimSpace(*run) {
- case "0":
- return "UNSTART"
- case "1":
- return "RUNNING"
- case "2":
- return "CANCEL"
- case "3":
- return "DONE"
- case "4":
- return "FAIL"
- default:
- return strings.TrimSpace(*run)
- }
-}
-
func formatTimePtr(value *time.Time) string {
if value == nil {
return ""
diff --git a/internal/handler/document_test.go b/internal/handler/document_test.go
index 846814e92e..318395ee88 100644
--- a/internal/handler/document_test.go
+++ b/internal/handler/document_test.go
@@ -1041,6 +1041,40 @@ func TestDocumentHandlerIngestPropagatesServiceErrorCode(t *testing.T) {
}
}
+func TestDocumentHandlerIngest_CancelDispatchesToService(t *testing.T) {
+ gin.SetMode(gin.TestMode)
+
+ fake := &fakeDocumentService{}
+ h := &DocumentHandler{
+ documentService: fake,
+ datasetService: dataset.NewDatasetService(),
+ }
+
+ c, w := setupGinContextWithUser("POST", "/api/v1/documents/ingest", `{"doc_ids":["doc-1"],"run":2}`)
+ h.Ingest(c)
+
+ if w.Code != http.StatusOK {
+ t.Fatalf("expected 200, got %d: %s", w.Code, w.Body.String())
+ }
+
+ var resp map[string]interface{}
+ if err := json.Unmarshal(w.Body.Bytes(), &resp); err != nil {
+ t.Fatal(err)
+ }
+ if resp["code"] != float64(common.CodeSuccess) {
+ t.Fatalf("expected code 0, got %v", resp["code"])
+ }
+ if resp["data"] != true {
+ t.Fatalf("expected data true, got %#v", resp["data"])
+ }
+ if fake.ingestReq == nil || len(fake.ingestReq.DocIDs) != 1 || fake.ingestReq.DocIDs[0] != "doc-1" {
+ t.Fatalf("unexpected ingestReq: %#v", fake.ingestReq)
+ }
+ if fmt.Sprint(fake.ingestReq.Run) != "2" {
+ t.Fatalf("run = %v, want 2", fake.ingestReq.Run)
+ }
+}
+
func TestStopParseDocumentsHandler_EmptyDocIDs(t *testing.T) {
gin.SetMode(gin.TestMode)
@@ -1152,7 +1186,7 @@ func TestListDocumentsHandler_FilterRequestUsesQueryFilters(t *testing.T) {
datasetService: dataset.NewDatasetService(),
}
- c, w := setupGinContextWithUser("GET", "/api/v1/datasets/ds-1/documents?type=filter&keywords=report&suffix=pdf&run=DONE&types=doc&desc=false", "")
+ c, w := setupGinContextWithUser("GET", "/api/v1/datasets/ds-1/documents?type=filter&keywords=report&suffix=pdf&run=COMPLETED&types=doc&desc=false", "")
c.Params = gin.Params{{Key: "dataset_id", Value: "ds-1"}}
h.ListDocuments(c)
@@ -1169,8 +1203,8 @@ func TestListDocumentsHandler_FilterRequestUsesQueryFilters(t *testing.T) {
if len(fake.filterOpts.Suffixes) != 1 || fake.filterOpts.Suffixes[0] != "pdf" {
t.Fatalf("expected suffix pdf, got %#v", fake.filterOpts.Suffixes)
}
- if len(fake.filterOpts.RunStatuses) != 1 || fake.filterOpts.RunStatuses[0] != string(entity.TaskStatusDone) {
- t.Fatalf("expected run DONE to map to %q, got %#v", string(entity.TaskStatusDone), fake.filterOpts.RunStatuses)
+ if len(fake.filterOpts.RunStatuses) != 1 || fake.filterOpts.RunStatuses[0] != common.COMPLETED {
+ t.Fatalf("expected run COMPLETED to map to %q, got %#v", common.COMPLETED, fake.filterOpts.RunStatuses)
}
if len(fake.filterOpts.Types) != 1 || fake.filterOpts.Types[0] != "doc" {
t.Fatalf("expected type doc, got %#v", fake.filterOpts.Types)
@@ -1189,6 +1223,43 @@ func TestListDocumentsHandler_FilterRequestUsesQueryFilters(t *testing.T) {
}
}
+func TestListDocumentsRejectsInvalidRunFilter(t *testing.T) {
+ gin.SetMode(gin.TestMode)
+
+ db := setupHandlerAccessDB(t)
+ orig := dao.DB
+ dao.DB = db
+ t.Cleanup(func() { dao.DB = orig })
+
+ fake := &fakeDocumentService{}
+ h := &DocumentHandler{
+ documentService: fake,
+ datasetService: dataset.NewDatasetService(),
+ }
+
+ for _, invalid := range []string{"1", "DONE", "CANCEL", "FAIL", "SCHEDULE"} {
+ c, w := setupGinContextWithUser("GET", "/api/v1/datasets/ds-1/documents?run="+invalid, "user-1")
+ c.Params = gin.Params{{Key: "dataset_id", Value: "ds-1"}}
+
+ h.ListDocuments(c)
+
+ if w.Code != http.StatusOK {
+ t.Fatalf("expected 200 wrapper, got %d: %s", w.Code, w.Body.String())
+ }
+ var resp map[string]interface{}
+ if err := json.Unmarshal(w.Body.Bytes(), &resp); err != nil {
+ t.Fatalf("invalid json response: %v", err)
+ }
+ if int(resp["code"].(float64)) != int(common.CodeDataError) {
+ t.Fatalf("expected code %d for %s, got %v", common.CodeDataError, invalid, resp["code"])
+ }
+ msg, _ := resp["message"].(string)
+ if !strings.Contains(msg, "Invalid filter run status conditions: "+invalid) {
+ t.Fatalf("expected error message for %s to contain 'Invalid filter run status conditions: %s', got %q", invalid, invalid, msg)
+ }
+ }
+}
+
func TestListDocumentsHandlerReturnsScheduledIngestionStatus(t *testing.T) {
db := setupHandlerAccessDB(t)
orig := dao.DB
@@ -1238,7 +1309,7 @@ func TestListDocumentsHandlerReturnsScheduledIngestionStatus(t *testing.T) {
}
}
-func TestListDocumentsHandlerOmitsEmptyIngestionStatus(t *testing.T) {
+func TestListDocumentsHandlerReturnsUnstartWhenNoIngestionTask(t *testing.T) {
db := setupHandlerAccessDB(t)
orig := dao.DB
dao.DB = db
@@ -1281,8 +1352,11 @@ func TestListDocumentsHandlerOmitsEmptyIngestionStatus(t *testing.T) {
if len(response.Data.Docs) != 1 {
t.Fatalf("document count = %d, want 1", len(response.Data.Docs))
}
- if _, exists := response.Data.Docs[0]["ingestion_status"]; exists {
- t.Fatalf("ingestion_status should be omitted when no ingestion task exists")
+ if got := response.Data.Docs[0]["ingestion_status"]; got != "UNSTART" {
+ t.Fatalf("ingestion_status = %v, want UNSTART", got)
+ }
+ if _, exists := response.Data.Docs[0]["run"]; exists {
+ t.Fatalf("run field should not exist in response")
}
}
@@ -1438,7 +1512,7 @@ func TestListDocumentsHandler_MetadataFilterNarrowsDocumentIDs(t *testing.T) {
listIDs: []string{"doc-1", "doc-2", "doc-3"},
metadataByKBs: map[string]interface{}{
"author": map[string][]string{
- "Alice": []string{"doc-2", "doc-4"},
+ "Alice": {"doc-2", "doc-4"},
},
},
}
diff --git a/internal/ingestion/service/execute_task_ack_test.go b/internal/ingestion/service/execute_task_ack_test.go
index aef6513249..9e0372908d 100644
--- a/internal/ingestion/service/execute_task_ack_test.go
+++ b/internal/ingestion/service/execute_task_ack_test.go
@@ -156,7 +156,7 @@ func TestExecuteTask_AcksMessageOnFailure(t *testing.T) {
// TestExecuteTask_AcksMessageOnContextCancel: a task with a cancelled context
// (e.g. Redis cancel flag or Ingestor shutdown) is now terminal — the cancel
-// is durably recorded (progress=-1, run=CANCEL) and the message is Acked to
+// is durably recorded (progress=-1, STOPPED) and the message is Acked to
// prevent indefinite redeliveries of an already-cancelled task.
func TestExecuteTask_AcksMessageOnContextCancel(t *testing.T) {
db := testutil.SetupTestDB(t)
diff --git a/internal/ingestion/service/execute_task_test.go b/internal/ingestion/service/execute_task_test.go
index b283a02e76..23bf2e18fe 100644
--- a/internal/ingestion/service/execute_task_test.go
+++ b/internal/ingestion/service/execute_task_test.go
@@ -239,9 +239,6 @@ func TestExecuteTask_CancelBeforePipeline(t *testing.T) {
if doc.Progress != -1 {
t.Fatalf("document.progress = %v, want -1 (cancelled)", doc.Progress)
}
- if doc.Run == nil || *doc.Run != string(entity.TaskStatusCancel) {
- t.Fatalf("document.run = %v, want %s (CANCEL)", doc.Run, entity.TaskStatusCancel)
- }
if doc.ProgressMsg == nil || *doc.ProgressMsg == "" {
t.Fatal("document.progress_msg should contain cancel marker, got empty")
}
diff --git a/internal/ingestion/service/ingestion_service.go b/internal/ingestion/service/ingestion_service.go
index f82719fe94..4547454a4c 100644
--- a/internal/ingestion/service/ingestion_service.go
+++ b/internal/ingestion/service/ingestion_service.go
@@ -1050,8 +1050,7 @@ func (e *Ingestor) pollCancel(taskID string, cancel context.CancelFunc, done <-c
}
// markCancelProgress writes the cancelled-progress markers to the document
-// row. Mirrors Python's cancel_all_task_of: progress=-1, run=CANCEL, and an
-// appended timestamped cancel message (progress_msg += cancelMsg).
+// row: progress=-1 and an appended timestamped cancel message (progress_msg += cancelMsg).
func (e *Ingestor) markCancelProgress(task *entity.IngestionTask) {
svc := documentpkg.NewDocumentService()
doc, err := svc.GetDocumentByID(e.ctx, task.DocumentID)
@@ -1064,7 +1063,7 @@ func (e *Ingestor) markCancelProgress(task *entity.IngestionTask) {
if doc.ProgressMsg != nil {
existingMsg = *doc.ProgressMsg
}
- _ = svc.UpdateRunProgress(e.ctx, task.DocumentID, -1.0, string(entity.TaskStatusCancel), existingMsg+cancelMsg)
+ _ = svc.UpdateRunProgress(e.ctx, task.DocumentID, -1.0, existingMsg+cancelMsg)
}
// markTimeoutProgress writes the timeout-progress markers to the document
@@ -1082,7 +1081,7 @@ func (e *Ingestor) markTimeoutProgress(task *entity.IngestionTask) {
if doc.ProgressMsg != nil {
existingMsg = *doc.ProgressMsg
}
- _ = svc.UpdateRunProgress(e.ctx, task.DocumentID, -1.0, string(entity.TaskStatusFail), existingMsg+timeoutMsg)
+ _ = svc.UpdateRunProgress(e.ctx, task.DocumentID, -1.0, existingMsg+timeoutMsg)
}
// claimTask registers a worker claim on a task ID. Returns false if another
diff --git a/internal/ingestion/service/progress_sink.go b/internal/ingestion/service/progress_sink.go
index c7f5a826ec..41846787df 100644
--- a/internal/ingestion/service/progress_sink.go
+++ b/internal/ingestion/service/progress_sink.go
@@ -27,7 +27,6 @@ import (
"ragflow/internal/common"
"ragflow/internal/dao"
- "ragflow/internal/entity"
"ragflow/internal/ingestion/pipeline"
servicepkg "ragflow/internal/service"
documentpkg "ragflow/internal/service/document"
@@ -68,8 +67,8 @@ type progressSink struct {
// full DocumentService surface.
type docProgressSvc interface {
GetDocumentByID(ctx context.Context, docID string) (*documentpkg.DocumentResponse, error)
- UpdateRunState(ctx context.Context, docID string, progress float64, run string) error
- UpdateRunProgress(ctx context.Context, docID string, progress float64, run, progressMsg string) error
+ UpdateRunState(ctx context.Context, docID string, progress float64) error
+ UpdateRunProgress(ctx context.Context, docID string, progress float64, progressMsg string) error
}
func newProgressSink(ctx context.Context, taskSvc *servicepkg.IngestionTaskService) *progressSink {
@@ -108,8 +107,8 @@ func (s *progressSink) OnComponentProgress(ctx context.Context, ev pipeline.Prog
if agg == nil || total <= 0 {
return
}
- progress, run := deriveDocumentProgress(agg, int(total))
- if err = s.updateDocumentProgress(ctx, ev.DocumentID, progress, run, ev.Message); err != nil {
+ progress := deriveDocumentProgress(agg, int(total))
+ if err = s.updateDocumentProgress(ctx, ev.DocumentID, progress, ev.Message); err != nil {
common.Error(fmt.Sprintf("progressSink: mirror progress to document %s for task %s failed: %v", ev.DocumentID, ev.TaskID, err), err)
}
}
@@ -132,16 +131,12 @@ func (s *progressSink) OnComponentMessage(ctx context.Context, taskID, docID, co
if doc == nil {
return
}
- run := ""
- if doc.Run != nil {
- run = *doc.Run
- }
log, err := s.accumulateLogLocked(ctx, docID, fmt.Sprintf("%s: %s", component, message))
if err != nil {
common.Error(fmt.Sprintf("progressSink: append detail for document %s failed", docID), err)
return
}
- if err := s.docSvc.UpdateRunProgress(ctx, docID, doc.Progress, run, log); err != nil {
+ if err := s.docSvc.UpdateRunProgress(ctx, docID, doc.Progress, log); err != nil {
common.Error(fmt.Sprintf("progressSink: persist detail for document %s failed", docID), err)
}
}
@@ -150,17 +145,17 @@ func (s *progressSink) OnComponentMessage(ctx context.Context, taskID, docID, co
// document write. Without holding the same lock across both operations, a
// slower database write can store an older snapshot after a newer event has
// already been persisted.
-func (s *progressSink) updateDocumentProgress(ctx context.Context, docID string, progress float64, run, msg string) error {
+func (s *progressSink) updateDocumentProgress(ctx context.Context, docID string, progress float64, msg string) error {
s.logMu.Lock()
defer s.logMu.Unlock()
log, err := s.accumulateLogLocked(ctx, docID, msg)
if err != nil {
// Keep the current run state durable even when the existing log cannot
// be seeded. The next event will retry the seed and persist the log.
- stateErr := s.docSvc.UpdateRunState(ctx, docID, progress, run)
+ stateErr := s.docSvc.UpdateRunState(ctx, docID, progress)
return errors.Join(err, stateErr)
}
- return s.docSvc.UpdateRunProgress(ctx, docID, progress, run, log)
+ return s.docSvc.UpdateRunProgress(ctx, docID, progress, log)
}
// accumulateLog appends one component lifecycle line to the run log. It is
@@ -235,23 +230,12 @@ func trimLogHead(text string, maxChars int) string {
return text
}
-// deriveDocumentProgress computes the document-level progress (0..1) and run
-// label ("0".."4", matching Python's document.run enum) from the aggregated
-// ingestion_task_log. This logic is owned by the sink (the document-table
-// writer), not the pipeline.
-func deriveDocumentProgress(agg *dao.TaskProgress, total int) (float64, string) {
- run := string(entity.TaskStatusUnstart)
- switch {
- case agg.Failed > 0:
- run = string(entity.TaskStatusFail)
- case agg.Done == total:
- run = string(entity.TaskStatusDone)
- case agg.Done > 0 || agg.Running > 0:
- run = string(entity.TaskStatusRunning)
- }
+// deriveDocumentProgress computes the document-level progress (0..1) from the
+// aggregated ingestion_task_log.
+func deriveDocumentProgress(agg *dao.TaskProgress, total int) float64 {
progress := agg.Percent / 100
if progress > 1 {
progress = 1
}
- return progress, run
+ return progress
}
diff --git a/internal/ingestion/service/progress_sink_test.go b/internal/ingestion/service/progress_sink_test.go
index 32833d7c18..ba6bd3d780 100644
--- a/internal/ingestion/service/progress_sink_test.go
+++ b/internal/ingestion/service/progress_sink_test.go
@@ -27,7 +27,6 @@ import (
"time"
"ragflow/internal/dao"
- "ragflow/internal/entity"
"ragflow/internal/ingestion/pipeline"
"ragflow/internal/ingestion/testutil"
servicepkg "ragflow/internal/service"
@@ -145,7 +144,6 @@ func TestProgressSink_Total_NoDataRace(t *testing.T) {
type stubDocProgressSvc struct {
gotDocID string
gotProgress float64
- gotRun string
gotMsg string
calls int
doc *document.DocumentResponse
@@ -170,11 +168,11 @@ func (s *blockingDocProgressSvc) GetDocumentByID(context.Context, string) (*docu
return nil, nil
}
-func (s *blockingDocProgressSvc) UpdateRunState(context.Context, string, float64, string) error {
+func (s *blockingDocProgressSvc) UpdateRunState(context.Context, string, float64) error {
return nil
}
-func (s *blockingDocProgressSvc) UpdateRunProgress(_ context.Context, _ string, _ float64, _ string, msg string) error {
+func (s *blockingDocProgressSvc) UpdateRunProgress(_ context.Context, _ string, _ float64, msg string) error {
s.mu.Lock()
s.updates++
update := s.updates
@@ -191,15 +189,14 @@ func (s *stubDocProgressSvc) GetDocumentByID(ctx context.Context, docID string)
return s.doc, s.docErr
}
-func (s *stubDocProgressSvc) UpdateRunState(context.Context, string, float64, string) error {
+func (s *stubDocProgressSvc) UpdateRunState(context.Context, string, float64) error {
return nil
}
-func (s *stubDocProgressSvc) UpdateRunProgress(ctx context.Context, docID string, progress float64, run, progressMsg string) error {
+func (s *stubDocProgressSvc) UpdateRunProgress(ctx context.Context, docID string, progress float64, progressMsg string) error {
s.calls++
s.gotDocID = docID
s.gotProgress = progress
- s.gotRun = run
s.gotMsg = progressMsg
return nil
}
@@ -257,9 +254,6 @@ func TestProgressSinkPersistsViaService(t *testing.T) {
if stub.gotProgress != 0.5 {
t.Fatalf("progress = %v, want 0.5", stub.gotProgress)
}
- if stub.gotRun != "1" {
- t.Fatalf("run = %q, want 1 (RUNNING)", stub.gotRun)
- }
if stub.gotMsg != "03:04:05: Parser Done" {
t.Fatalf("progress_msg = %q, want timestamped Parser Done", stub.gotMsg)
}
@@ -349,7 +343,7 @@ func TestProgressSinkSerializesLogAndDocumentWrite(t *testing.T) {
firstDone := make(chan struct{})
go func() {
defer close(firstDone)
- if err := sink.updateDocumentProgress(context.Background(), "doc-1", 0.1, "1", "first"); err != nil {
+ if err := sink.updateDocumentProgress(context.Background(), "doc-1", 0.1, "first"); err != nil {
t.Errorf("first update: %v", err)
}
}()
@@ -358,7 +352,7 @@ func TestProgressSinkSerializesLogAndDocumentWrite(t *testing.T) {
secondDone := make(chan struct{})
go func() {
defer close(secondDone)
- if err := sink.updateDocumentProgress(context.Background(), "doc-1", 0.2, "1", "second"); err != nil {
+ if err := sink.updateDocumentProgress(context.Background(), "doc-1", 0.2, "second"); err != nil {
t.Errorf("second update: %v", err)
}
}()
@@ -507,69 +501,58 @@ func TestDeriveDocumentProgress(t *testing.T) {
name string
agg *dao.TaskProgress
total int
- wantRun string
wantProg float64
}{
{
- name: "failed component → fail",
+ name: "failed component progress",
agg: &dao.TaskProgress{Failed: 1, Done: 0, Running: 0, Percent: 0},
total: 5,
- wantRun: string(entity.TaskStatusFail),
wantProg: 0.0,
},
{
- name: "all done → done",
+ name: "all done",
agg: &dao.TaskProgress{Failed: 0, Done: 5, Running: 0, Percent: 100},
total: 5,
- wantRun: string(entity.TaskStatusDone),
wantProg: 1.0,
},
{
- name: "partial done → running",
+ name: "partial done",
agg: &dao.TaskProgress{Failed: 0, Done: 3, Running: 0, Percent: 60},
total: 5,
- wantRun: string(entity.TaskStatusRunning),
wantProg: 0.6,
},
{
- name: "running only → running",
+ name: "running only",
agg: &dao.TaskProgress{Failed: 0, Done: 0, Running: 2, Percent: 0},
total: 5,
- wantRun: string(entity.TaskStatusRunning),
wantProg: 0.0,
},
{
- name: "nothing started → unstart",
+ name: "nothing started",
agg: &dao.TaskProgress{Failed: 0, Done: 0, Running: 0, Percent: 0},
total: 5,
- wantRun: string(entity.TaskStatusUnstart),
wantProg: 0.0,
},
{
- name: "total zero, nothing done → done (0==0)",
+ name: "total zero, nothing done",
agg: &dao.TaskProgress{Failed: 0, Done: 0, Running: 0, Percent: 0},
total: 0,
- wantRun: string(entity.TaskStatusDone),
wantProg: 0.0,
},
{
- name: "failed overrides done=total",
+ name: "failed with 100 percent",
agg: &dao.TaskProgress{Failed: 1, Done: 5, Running: 0, Percent: 100},
total: 5,
- wantRun: string(entity.TaskStatusFail),
wantProg: 1.0,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
- prog, run := deriveDocumentProgress(tt.agg, tt.total)
+ prog := deriveDocumentProgress(tt.agg, tt.total)
if prog != tt.wantProg {
t.Errorf("progress = %v, want %v", prog, tt.wantProg)
}
- if run != tt.wantRun {
- t.Errorf("run = %q, want %q", run, tt.wantRun)
- }
})
}
}
diff --git a/internal/ingestion/task/pipeline_executor.go b/internal/ingestion/task/pipeline_executor.go
index 8afabef237..5965060991 100644
--- a/internal/ingestion/task/pipeline_executor.go
+++ b/internal/ingestion/task/pipeline_executor.go
@@ -200,7 +200,7 @@ func (s *PipelineExecutor) Execute(ctx context.Context) (*PipelineResult, error)
}
if pipelineDSL != "" {
- s.recordPipelineLog(context.WithoutCancel(ctx), dao.DB, s.taskCtx.Doc.ID, pipelineDSL, "")
+ s.recordPipelineLog(context.WithoutCancel(ctx), dao.DB, s.taskCtx.Doc.ID, pipelineDSL, string(entity.TaskStatusDone))
}
return result, nil
@@ -772,8 +772,7 @@ type PipelineLogInput struct {
}
// RecordPipelineLog persists a pipeline operation log without requiring
-// executor setup. Callers that already know a terminal state should pass it in
-// Status; otherwise the writer falls back to the latest document.run value.
+// executor setup. Callers should pass the status in Status.
func RecordPipelineLog(ctx context.Context, db *gorm.DB, input PipelineLogInput) error {
return recordPipelineLog(ctx, db, input, dao.NewPipelineOperationLogDAO().Create)
}
@@ -847,9 +846,6 @@ func recordPipelineLog(
}
operationStatus := input.Status
- if operationStatus == "" && doc.Run != nil && *doc.Run != "" {
- operationStatus = *doc.Run
- }
statusValue := "1"
if doc.Status != nil && *doc.Status != "" {
statusValue = *doc.Status
diff --git a/internal/ingestion/task/pipeline_executor_test.go b/internal/ingestion/task/pipeline_executor_test.go
index 80e9a2fcf5..a1006f5b3a 100644
--- a/internal/ingestion/task/pipeline_executor_test.go
+++ b/internal/ingestion/task/pipeline_executor_test.go
@@ -373,7 +373,6 @@ func TestRecordPipelineLog_SharedWriterTerminalWithoutDSL(t *testing.T) {
defer cleanup()
docName := "terminal.pdf"
- run := "1"
if err := RecordPipelineLog(t.Context(), dao.DB, PipelineLogInput{
TenantID: "tenant-1",
KbID: "kb-1",
@@ -388,7 +387,6 @@ func TestRecordPipelineLog_SharedWriterTerminalWithoutDSL(t *testing.T) {
Type: "pdf",
Name: &docName,
Suffix: ".pdf",
- Run: &run,
},
}); err != nil {
t.Fatalf("RecordPipelineLog: %v", err)
@@ -524,7 +522,6 @@ func TestRecordPipelineLog_TerminalWithoutDSLResolvesCanvasTitle(t *testing.T) {
t.Fatalf("seed knowledgebase: %v", err)
}
docName := "sample.avi"
- run := "1"
if err := dao.DB.Create(&entity.Document{
ID: "doc-1",
KbID: "kb-1",
@@ -532,7 +529,6 @@ func TestRecordPipelineLog_TerminalWithoutDSLResolvesCanvasTitle(t *testing.T) {
ParserID: "naive",
ParserConfig: entity.JSONMap{},
Name: &docName,
- Run: &run,
}).Error; err != nil {
t.Fatalf("seed document: %v", err)
}
@@ -837,6 +833,40 @@ func TestPipelineExecutor_Execute_PropagatesContext(t *testing.T) {
}
}
+func TestPipelineExecutor_Execute_RecordsDoneOperationStatus(t *testing.T) {
+ taskCtx := makeTaskCtx()
+ taskCtx.Ctx = t.Context()
+ var capturedLog *entity.PipelineOperationLog
+
+ svc := mustNewPipelineExecutor(t, taskCtx, "flow-1", 0).
+ WithLoadDSLFunc(func(ctx context.Context, canvasID string) (string, string, error) {
+ return `{"nodes":[{"id":"n1"}],"edges":[]}`, canvasID, nil
+ }).
+ WithRunPipelineFunc(func(runCtx context.Context, dsl string) (map[string]any, string, error) {
+ return map[string]any{"chunks": []map[string]any{{"text": "hello world"}}}, dsl, nil
+ }).
+ WithInsertFunc(func(ctx context.Context, chunks []map[string]any, baseName, datasetID string) ([]string, error) {
+ return nil, nil
+ }).
+ WithLogCreateFunc(func(ctx context.Context, db *gorm.DB, log *entity.PipelineOperationLog) error {
+ capturedLog = log
+ return nil
+ })
+
+ if _, err := svc.Execute(taskCtx.Ctx); err != nil {
+ t.Fatalf("unexpected error: %v", err)
+ }
+ if capturedLog == nil {
+ t.Fatalf("expected pipeline operation log to be recorded")
+ }
+ if capturedLog.OperationStatus == "" {
+ t.Fatalf("expected OperationStatus to be non-empty")
+ }
+ if capturedLog.OperationStatus != string(entity.TaskStatusDone) {
+ t.Fatalf("expected OperationStatus = %q, got %q", entity.TaskStatusDone, capturedLog.OperationStatus)
+ }
+}
+
// =============================================================================
// Stub implementations for testing
// =============================================================================
diff --git a/internal/service/chunk/chunk.go b/internal/service/chunk/chunk.go
index 103aee9dc7..6c06971728 100644
--- a/internal/service/chunk/chunk.go
+++ b/internal/service/chunk/chunk.go
@@ -74,13 +74,14 @@ func searchConfigMap(value interface{}) (map[string]interface{}, bool) {
// ChunkService chunk service
type ChunkService struct {
- docEngine engine.DocEngine
- embeddingCache *utility.EmbeddingLRU
- kbDAO *dao.KnowledgebaseDAO
- userTenantDAO *dao.UserTenantDAO
- documentDAO *dao.DocumentDAO
- taskDAO *dao.TaskDAO
- searchService *service.SearchService
+ docEngine engine.DocEngine
+ embeddingCache *utility.EmbeddingLRU
+ kbDAO *dao.KnowledgebaseDAO
+ userTenantDAO *dao.UserTenantDAO
+ documentDAO *dao.DocumentDAO
+ taskDAO *dao.TaskDAO
+ ingestionTaskDAO *dao.IngestionTaskDAO
+ searchService *service.SearchService
accessibleFunc func(string, string) bool
getKnowledgebaseByIDFunc func(string) (*entity.Knowledgebase, error)
@@ -105,13 +106,14 @@ type ChunkService struct {
// NewChunkService creates chunk service
func NewChunkService() *ChunkService {
return &ChunkService{
- docEngine: engine.Get(),
- embeddingCache: utility.NewEmbeddingLRU(1000), // default capacity
- kbDAO: dao.NewKnowledgebaseDAO(),
- userTenantDAO: dao.NewUserTenantDAO(),
- documentDAO: dao.NewDocumentDAO(),
- taskDAO: dao.NewTaskDAO(),
- searchService: service.NewSearchService(),
+ docEngine: engine.Get(),
+ embeddingCache: utility.NewEmbeddingLRU(1000), // default capacity
+ kbDAO: dao.NewKnowledgebaseDAO(),
+ userTenantDAO: dao.NewUserTenantDAO(),
+ documentDAO: dao.NewDocumentDAO(),
+ taskDAO: dao.NewTaskDAO(),
+ ingestionTaskDAO: dao.NewIngestionTaskDAO(),
+ searchService: service.NewSearchService(),
}
}
@@ -658,9 +660,8 @@ func (s *ChunkService) StopParsing(ctx context.Context, userID, datasetID string
return nil, common.CodeServerError, err
}
// CancelDocParse (inside cancelAllTasksOfDoc) already issues
- // RequestStop (STOPPING) and updates doc.run=CANCEL. Defer
- // destruction (chunk deletion, counter reset) until the worker
- // detects STOPPING and reaches a terminal state.
+ // RequestStop (STOPPING). Defer destruction (chunk deletion, counter
+ // reset) until the worker detects STOPPING and reaches a terminal state.
successCount++
}
@@ -752,12 +753,6 @@ func (s *ChunkService) Parse(ctx context.Context, userID, datasetID string, req
if len(notFound) > 0 {
return nil, common.CodeDataError, fmt.Errorf("documents not found: %v", notFound)
}
- for _, docID := range docIDs {
- doc := docByID[docID]
- if doc.Run != nil && *doc.Run == string(entity.TaskStatusRunning) {
- return nil, common.CodeDataError, fmt.Errorf("can't parse document that is currently being processed")
- }
- }
// Batch pre-check: refuse the whole request if any document's ingestion
// task is non-terminal (RUNNING/STOPPING), so we never partially clean.
@@ -982,9 +977,22 @@ func (s *ChunkService) List(ctx context.Context, req *service.ListChunksRequest,
chunks = append(chunks, result)
}
+ ingestionStatus := "UNSTART"
+ taskDAO := s.ingestionTaskDAO
+ if taskDAO == nil {
+ taskDAO = dao.NewIngestionTaskDAO()
+ }
+ task, err := taskDAO.GetByDocumentID(ctx, dao.DB, doc.ID)
+ if err != nil {
+ return nil, fmt.Errorf("failed to get ingestion task for document %s: %w", doc.ID, err)
+ }
+ if task != nil && task.Status != "" {
+ ingestionStatus = task.Status
+ }
+
// Build document info, mirroring Python's _map_doc key renames:
// kb_id→dataset_id, parser_id→chunk_method, token_num→token_count,
- // chunk_num→chunk_count, run→text status.
+ // chunk_num→chunk_count.
timeFormat := "2006-01-02T15:04:05"
docInfo := map[string]interface{}{
"id": doc.ID,
@@ -1007,7 +1015,7 @@ func (s *ChunkService) List(ctx context.Context, req *service.ListChunksRequest,
"process_duration": doc.ProcessDuration,
"content_hash": doc.ContentHash,
"suffix": doc.Suffix,
- "run": service.ChunkDocRunText(doc.Run),
+ "ingestion_status": ingestionStatus,
"status": doc.Status,
"create_time": doc.CreateTime,
"create_date": utility.FormatTimeToString(doc.CreateDate, timeFormat),
diff --git a/internal/service/chunk/chunk_test.go b/internal/service/chunk/chunk_test.go
index 880b5c25c9..79f85bdc13 100644
--- a/internal/service/chunk/chunk_test.go
+++ b/internal/service/chunk/chunk_test.go
@@ -199,12 +199,6 @@ func TestParsePrevalidatesDocumentsBeforeMutating(t *testing.T) {
if err != nil {
t.Fatalf("get doc: %v", err)
}
- if doc.Run == nil {
- t.Fatalf("expected doc run to remain nil, got %q", *doc.Run)
- }
- if *doc.Run != "0" {
- t.Fatalf("expected doc run status is '1', got %q", *doc.Run)
- }
if doc.ChunkNum != 7 {
t.Fatalf("expected chunk_num to remain 7, got %d", doc.ChunkNum)
}
@@ -289,10 +283,7 @@ func TestParseRejectsRunningDocument(t *testing.T) {
datasetID := "kb-1"
insertChunkTestKB(t, datasetID, userID)
insertChunkTestDoc(t, "doc-1", datasetID)
- running := string(entity.TaskStatusRunning)
- if err := db.Model(&entity.Document{}).Where("id = ?", "doc-1").Update("run", running).Error; err != nil {
- t.Fatalf("mark doc running: %v", err)
- }
+ insertChunkTestIngestionTask(t, "task-1", userID, "doc-1", datasetID, common.RUNNING)
svc := newParseTestService(t)
ctx := t.Context()
@@ -303,7 +294,7 @@ func TestParseRejectsRunningDocument(t *testing.T) {
if code != common.CodeDataError {
t.Fatalf("expected CodeDataError, got %v", code)
}
- if !strings.Contains(err.Error(), "currently being processed") {
+ if !strings.Contains(err.Error(), "ingestion task is RUNNING") {
t.Fatalf("unexpected error: %v", err)
}
}
@@ -1715,7 +1706,7 @@ func TestStopParsing_CallsCancelIngestionTask(t *testing.T) {
pushChunkTestDB(t, db)
insertChunkTestKB(t, "kb-1", "user-1") // owner = user-1, Accessible passes
insertChunkTestDoc(t, "doc-1", "kb-1")
- // StopParsing now checks the IngestionTask status (not doc.Run).
+ // StopParsing checks the IngestionTask status.
insertChunkTestIngestionTask(t, "task-1", "user-1", "doc-1", "kb-1", common.RUNNING)
svc := newParseTestService(t)
@@ -1773,9 +1764,9 @@ func TestStopParsing_DoesNotDeleteChunksOrResetCountersAfterCancel(t *testing.T)
svc := newParseTestService(t)
svc.cancelIngestionTaskFunc = func(ctx context.Context, doc *entity.Document) error {
- // Simulate CancelDocParse: set doc.run=CANCEL.
- return dao.DB.Model(&entity.Document{}).Where("id = ?", doc.ID).
- Update("run", string(entity.TaskStatusCancel)).Error
+ // Simulate CancelDocParse: stop task.
+ return dao.DB.Model(&entity.IngestionTask{}).Where("document_id = ?", doc.ID).
+ Update("status", common.STOPPED).Error
}
engine := &parseTestDocEngine{chunkStoreExists: true}
svc.docEngine = engine
diff --git a/internal/service/chunk_types.go b/internal/service/chunk_types.go
index 3dd01f2433..68cc1c92ce 100644
--- a/internal/service/chunk_types.go
+++ b/internal/service/chunk_types.go
@@ -17,47 +17,12 @@
package service
import (
- "context"
"encoding/json"
"fmt"
- "math"
- "strings"
- "time"
"ragflow/internal/common"
- "ragflow/internal/dao"
- "ragflow/internal/engine"
- "ragflow/internal/engine/redis"
- "ragflow/internal/engine/types"
- "ragflow/internal/entity"
- "ragflow/internal/tokenizer"
- "ragflow/internal/utility"
-
- "gorm.io/gorm"
-)
-
-var (
- UNSTART = "0"
- RUNNING = "1"
- CANCEL = "2"
- DONE = "3"
- FAIL = "4"
- SCHEDULE = "5"
)
-// ChunkService chunk service
-type ChunkService struct {
- docEngine engine.DocEngine
- embeddingCache *utility.EmbeddingLRU
- kbDAO *dao.KnowledgebaseDAO
- taskDAO *dao.TaskDAO
- userTenantDAO *dao.UserTenantDAO
- documentDAO *dao.DocumentDAO
- searchService *SearchService
-
- decrementChunkStatsFunc func(string, string, int64, int64, float64) error
-}
-
// RetrievalTestRequest retrieval test request
type RetrievalTestRequest struct {
Datasets common.StringSlice `json:"dataset_ids" binding:"required"` // string or []string
@@ -124,82 +89,6 @@ type ErrorCoder interface {
Code() common.ErrorCode
}
-// Get retrieves a chunk by ID
-func (s *ChunkService) Get(ctx context.Context, req *GetChunkRequest, userID string) (*GetChunkResponse, error) {
- if s.docEngine == nil {
- return nil, fmt.Errorf("doc engine not initialized")
- }
-
- if req.ChunkID == "" {
- return nil, fmt.Errorf("chunk_id is required")
- }
-
- // Get user's tenants
- tenants, err := s.userTenantDAO.GetByUserID(ctx, dao.DB, userID)
- if err != nil {
- return nil, fmt.Errorf("failed to get user tenants: %w", err)
- }
- if len(tenants) == 0 {
- return nil, fmt.Errorf("user has no accessible tenants")
- }
-
- // Try each tenant to find the chunk
- var chunk map[string]interface{}
- for _, tenant := range tenants {
- // Get kbIDs for this tenant
- kbIDs, err := s.kbDAO.GetKBIDsByTenantID(ctx, dao.DB, tenant.TenantID)
- if err != nil {
- continue
- }
-
- indexName := fmt.Sprintf("ragflow_%s", tenant.TenantID)
-
- doc, err := s.docEngine.GetChunk(ctx, indexName, req.ChunkID, kbIDs)
- if err != nil {
- continue
- }
-
- if doc != nil {
- chunk, ok := doc.(map[string]interface{})
- if ok {
- result := make(map[string]interface{})
- skipFields := map[string]bool{
- "id": true, "authors": true, "_score": true, "SCORE": true,
- }
- for k, v := range chunk {
- if skipFields[k] || isInternalField(k) {
- continue
- }
- if applyCommonChunkMapping(result, k, v) {
- continue
- }
- switch k {
- case "tag_feas":
- if utility.IsEmpty(v) {
- result[k] = map[string]interface{}{}
- } else {
- result[k] = v
- }
- case "create_timestamp_flt", "rank_flt", "weight_flt":
- if floatVal, ok := utility.ToFloat64(v); ok {
- result[k] = utility.JSONFloat64(floatVal)
- }
- default:
- result[k] = v
- }
- }
- return &GetChunkResponse{Chunk: result}, nil
- }
- }
- }
-
- if chunk == nil {
- return nil, fmt.Errorf("chunk not found")
- }
-
- return &GetChunkResponse{Chunk: chunk}, nil
-}
-
type StopParsingRequest struct {
DocumentIDs []string `json:"document_ids"`
}
@@ -209,28 +98,6 @@ type StopParsingResponse struct {
Message string
}
-func (s *ChunkService) cancelAllTasksOfDoc(ctx context.Context, docID string) error {
- tasks, err := s.taskDAO.GetByDocID(ctx, dao.DB, docID)
- if err != nil {
- return fmt.Errorf("failed to get tasks for document %s: %w", docID, err)
- }
-
- redisClient := redis.Get()
- if redisClient == nil {
- common.Logger.Warn(fmt.Sprintf("Redis unavailable; cannot cancel tasks for document %s", docID))
- return nil
- }
-
- for _, task := range tasks {
- if task == nil {
- continue
- }
- redisClient.Set(ctx, fmt.Sprintf("%s-cancel", task.ID), "x", time.Hour)
- }
-
- return nil
-}
-
func CheckDuplicateIDs(docList []string, idType string) ([]string, []string) {
uniqueDocIDs := make([]string, 0)
duplicateMessages := make([]string, 0)
@@ -253,80 +120,6 @@ func IndexName(uid string) string {
return fmt.Sprintf("ragflow_%s", uid)
}
-func (s *ChunkService) StopParsing(ctx context.Context, userID, datasetID string, req StopParsingRequest) (map[string]interface{}, common.ErrorCode, error) {
- if !s.kbDAO.Accessible(ctx, dao.DB, datasetID, userID) {
- return nil, common.CodeAuthenticationError, fmt.Errorf("You don't own the dataset %s", datasetID)
- }
-
- if req.DocumentIDs == nil || len(req.DocumentIDs) == 0 {
- return nil, common.CodeDataError, fmt.Errorf("document_ids is required")
- }
-
- docList, duplicateMessages := CheckDuplicateIDs(req.DocumentIDs, "document")
-
- kb, err := s.kbDAO.GetByID(ctx, dao.DB, datasetID)
- if err != nil {
- return nil, common.CodeDataError, fmt.Errorf("You don't own the dataset %s", datasetID)
- }
-
- successCount := 0
- for _, id := range docList {
- var doc *entity.Document
- doc, err = s.documentDAO.GetByDocumentIDAndDatasetID(ctx, dao.DB, id, datasetID)
- if err != nil {
- return nil, common.CodeDataError, fmt.Errorf("you don't own the document %s", id)
- }
- if doc == nil {
- return nil, common.CodeDataError, fmt.Errorf("you don't own the document %s", id)
- }
-
- if doc.Run == nil || *doc.Run != RUNNING {
- return nil, common.CodeDataError, fmt.Errorf("can't stop parsing document that has not started or already completed")
- }
-
- err = s.cancelAllTasksOfDoc(ctx, id)
- if err != nil {
- return nil, common.CodeServerError, err
- }
-
- info := map[string]interface{}{
- "run": "2",
- "progress": 0,
- "chunk_num": 0,
- }
- if err = s.documentDAO.UpdateByID(ctx, dao.DB, doc.ID, info); err != nil {
- return nil, common.CodeServerError, fmt.Errorf("failed to update document %s: %w", doc.ID, err)
- }
-
- indexName := IndexName(kb.TenantID)
-
- var exists bool
- exists, err = s.docEngine.ChunkStoreExists(ctx, indexName, datasetID)
- if err != nil {
- return nil, common.CodeServerError, fmt.Errorf("failed to check chunk store %s/%s: %w", indexName, datasetID, err)
- }
- if exists {
- if _, err = s.docEngine.DeleteChunks(ctx, map[string]interface{}{"doc_id": doc.ID}, indexName, datasetID); err != nil {
- return nil, common.CodeServerError, fmt.Errorf("failed to delete chunks for document %s: %w", doc.ID, err)
- }
- } else {
- common.Logger.Info(fmt.Sprintf("Skipping chunk delete during stop_parsing for doc %s: index %s/%s does not exist", doc.ID, indexName, datasetID))
- }
- successCount++
- }
- if len(duplicateMessages) > 0 {
- if successCount > 0 {
- return map[string]interface{}{
- "success_count": successCount,
- "errors": duplicateMessages,
- "message": fmt.Sprintf("Partially stopped %d documents with %d errors", successCount, len(duplicateMessages)),
- }, common.CodeSuccess, nil
- }
- return nil, common.CodeDataError, fmt.Errorf("%s", strings.Join(duplicateMessages, ";"))
- }
- return nil, common.CodeSuccess, nil
-}
-
// ListChunksRequest request for listing chunks
type ListChunksRequest struct {
DatasetID string `json:"dataset_id,omitempty"`
@@ -345,165 +138,6 @@ type ListChunksResponse struct {
Total int64 `json:"total"`
}
-// List retrieves chunks for a document
-func (s *ChunkService) List(ctx context.Context, req *ListChunksRequest, userID string) (*ListChunksResponse, error) {
- if s.docEngine == nil {
- return nil, fmt.Errorf("doc engine not initialized")
- }
-
- if req.DocID == "" {
- return nil, fmt.Errorf("doc_id is required")
- }
-
- // Get user's tenants
- tenants, err := s.userTenantDAO.GetByUserID(ctx, dao.DB, userID)
- if err != nil {
- return nil, fmt.Errorf("failed to get user tenants: %w", err)
- }
- if len(tenants) == 0 {
- return nil, fmt.Errorf("user has no accessible tenants")
- }
-
- // Get document to find its tenant
- docDAO := dao.NewDocumentDAO()
- doc, err := docDAO.GetByID(ctx, dao.DB, req.DocID)
- if err != nil || doc == nil {
- return nil, fmt.Errorf("document not found")
- }
- if req.DatasetID != "" && doc.KbID != req.DatasetID {
- return nil, fmt.Errorf("document not found")
- }
-
- // Get knowledge base to find tenant
- kb, err := s.kbDAO.GetByID(ctx, dao.DB, doc.KbID)
- if err != nil || kb == nil {
- return nil, fmt.Errorf("knowledge base not found")
- }
-
- // Find which tenant this document belongs to
- var targetTenantID string
- for _, tenant := range tenants {
- if tenant.TenantID == kb.TenantID {
- targetTenantID = tenant.TenantID
- break
- }
- }
- if targetTenantID == "" {
- return nil, fmt.Errorf("user does not have access to this document")
- }
-
- // Get kbIDs for this tenant
- kbIDs, err := s.kbDAO.GetKBIDsByTenantID(ctx, dao.DB, targetTenantID)
- if err != nil {
- return nil, fmt.Errorf("failed to get kb ids: %w", err)
- }
-
- indexName := fmt.Sprintf("ragflow_%s", targetTenantID)
-
- page := common.CoalesceInt(req.Page, 1)
- size := common.CoalesceInt(req.Size, 30)
- keywords := req.Keywords
-
- // Build search request - same as retrieval test but filtered by doc_id
- searchReq := &types.SearchRequest{
- IndexNames: []string{indexName},
- MatchExprs: []interface{}{keywords},
- KbIDs: kbIDs,
- Offset: (page - 1) * size,
- Limit: size,
- Filter: map[string]interface{}{
- "doc_id": req.DocID,
- },
- }
-
- // Add available_int filter if specified
- if req.AvailableInt != nil {
- searchReq.Filter["available_int"] = *req.AvailableInt
- }
-
- // Execute search through unified engine interface
- searchResp, err := s.docEngine.Search(ctx, searchReq)
- if err != nil {
- return nil, fmt.Errorf("search failed: %w", err)
- }
-
- chunks := make([]map[string]interface{}, 0, len(searchResp.Chunks))
- for _, chunk := range searchResp.Chunks {
- // Inline formatChunkForList
- result := make(map[string]interface{})
- skipFields := map[string]bool{
- "_id": true, "authors": true, "_score": true, "SCORE": true,
- "important_kwd_empty_count": true, "kb_id": true, "mom_id": true, "page_num_int": true,
- }
- for k, v := range chunk {
- if skipFields[k] || isInternalField(k) {
- continue
- }
- if applyCommonChunkMapping(result, k, v) {
- continue
- }
- switch k {
- case "img_id":
- if strVal, ok := v.(string); ok {
- result["image_id"] = strVal
- } else {
- result["image_id"] = ""
- }
- case "position_int":
- result["positions"] = v
- case "id":
- result["chunk_id"] = v
- default:
- if strings.HasSuffix(k, "_kwd") && k != "knowledge_graph_kwd" {
- result[k] = splitKwdHash(v)
- } else {
- result[k] = v
- }
- }
- }
- chunks = append(chunks, result)
- }
-
- // Build document info, mirroring Python's _map_doc key renames:
- // kb_id→dataset_id, parser_id→chunk_method, token_num→token_count,
- // chunk_num→chunk_count, run→text status.
- timeFormat := "2006-01-02T15:04:05"
- docInfo := map[string]interface{}{
- "id": doc.ID,
- "thumbnail": doc.Thumbnail,
- "dataset_id": doc.KbID,
- "chunk_method": doc.ParserID,
- "pipeline_id": doc.PipelineID,
- "parser_config": doc.ParserConfig,
- "source_type": doc.SourceType,
- "type": doc.Type,
- "created_by": doc.CreatedBy,
- "name": doc.Name,
- "location": doc.Location,
- "size": doc.Size,
- "token_count": doc.TokenNum,
- "chunk_count": doc.ChunkNum,
- "progress": utility.JSONFloat64(doc.Progress),
- "progress_msg": doc.ProgressMsg,
- "process_begin_at": utility.FormatTimeToString(doc.ProcessBeginAt, timeFormat),
- "process_duration": doc.ProcessDuration,
- "content_hash": doc.ContentHash,
- "suffix": doc.Suffix,
- "run": ChunkDocRunText(doc.Run),
- "status": doc.Status,
- "create_time": doc.CreateTime,
- "create_date": utility.FormatTimeToString(doc.CreateDate, timeFormat),
- "update_time": doc.UpdateTime,
- "update_date": utility.FormatTimeToString(doc.UpdateDate, timeFormat),
- }
-
- return &ListChunksResponse{
- Total: searchResp.Total,
- Chunks: chunks,
- Doc: docInfo,
- }, nil
-}
-
// UpdateChunkRequest request for updating a chunk
type UpdateChunkRequest struct {
DatasetID string `json:"dataset_id"`
@@ -518,198 +152,6 @@ type UpdateChunkRequest struct {
TagFeas interface{} `json:"tag_feas,omitempty"`
}
-// UpdateChunk updates a chunk fields
-func (s *ChunkService) UpdateChunk(ctx context.Context, req *UpdateChunkRequest, userID string) error {
- if s.docEngine == nil {
- return fmt.Errorf("doc engine not initialized")
- }
-
- if req.ChunkID == "" {
- return fmt.Errorf("chunk_id is required")
- }
-
- // Get user's tenants
- tenants, err := s.userTenantDAO.GetByUserID(ctx, dao.DB, userID)
- if err != nil {
- return fmt.Errorf("failed to get user tenants: %w", err)
- }
- if len(tenants) == 0 {
- return fmt.Errorf("user has no accessible tenants")
- }
-
- // Find the tenant that owns this dataset
- var targetTenantID string
- for _, tenant := range tenants {
- kb, err := s.kbDAO.GetByIDAndTenantID(ctx, dao.DB, req.DatasetID, tenant.TenantID)
- if err == nil && kb != nil {
- targetTenantID = tenant.TenantID
- break
- }
- }
- if targetTenantID == "" {
- return fmt.Errorf("user does not have access to this dataset")
- }
-
- // Verify document belongs to dataset
- docDAO := dao.NewDocumentDAO()
- doc, err := docDAO.GetByID(ctx, dao.DB, req.DocumentID)
- if err != nil || doc == nil {
- return fmt.Errorf("document not found")
- }
- if doc.KbID != req.DatasetID {
- return fmt.Errorf("document does not belong to this dataset")
- }
-
- // Fetch existing chunk first
- indexName := fmt.Sprintf("ragflow_%s", targetTenantID)
- existingChunk, err := s.docEngine.GetChunk(ctx, indexName, req.ChunkID, []string{req.DatasetID})
- if err != nil {
- return fmt.Errorf("failed to get existing chunk: %w", err)
- }
-
- existing, ok := existingChunk.(map[string]interface{})
- if !ok {
- return fmt.Errorf("invalid chunk format")
- }
-
- // Build update dict
- d := make(map[string]interface{})
-
- // Content - use new value or existing
- if req.Content != nil {
- d["content_with_weight"] = *req.Content
- } else {
- if v, ok := existing["content_with_weight"].(string); ok {
- d["content_with_weight"] = v
- } else if v, ok := existing["content"].(string); ok {
- d["content_with_weight"] = v
- } else {
- d["content_with_weight"] = ""
- }
- }
-
- // Tokenize content
- contentStr := d["content_with_weight"].(string)
- d["content_ltks"], _ = tokenizer.Tokenize(contentStr)
- d["content_sm_ltks"], _ = tokenizer.FineGrainedTokenize(d["content_ltks"].(string))
-
- // Important keywords - convert []string to []interface{} for transformChunkFields
- if req.ImportantKwd != nil {
- impKwd := make([]interface{}, len(req.ImportantKwd))
- for i, v := range req.ImportantKwd {
- impKwd[i] = v
- }
- d["important_kwd"] = impKwd
- }
-
- // Questions
- if req.Questions != nil {
- // Filter out empty questions and trim
- filteredQuestions := []string{}
- for _, q := range req.Questions {
- q = strings.TrimSpace(q)
- if q != "" {
- filteredQuestions = append(filteredQuestions, q)
- }
- }
- d["question_kwd"] = filteredQuestions
- }
-
- // Available
- if req.Available != nil {
- if *req.Available {
- d["available_int"] = 1
- } else {
- d["available_int"] = 0
- }
- }
-
- // Positions
- if req.Positions != nil {
- d["position_int"] = req.Positions
- }
-
- // Tag keywords
- if req.TagKwd != nil {
- d["tag_kwd"] = req.TagKwd
- }
-
- // Tag features
- if req.TagFeas != nil {
- tagFeas, err := validateUpdateTagFeatures(req.TagFeas)
- if err != nil {
- return updateChunkError{code: common.CodeArgumentError, message: "`tag_feas` " + err.Error()}
- }
- d["tag_feas"] = tagFeas
- }
-
- // Always include id
- d["id"] = req.ChunkID
-
- // Call update
- condition := map[string]interface{}{
- "id": req.ChunkID,
- }
-
- err = s.docEngine.UpdateChunks(ctx, condition, d, indexName, req.DatasetID)
- if err != nil {
- return fmt.Errorf("failed to update chunk: %w", err)
- }
-
- return nil
-}
-
-type updateChunkError struct {
- code common.ErrorCode
- message string
-}
-
-func (e updateChunkError) Error() string {
- return e.message
-}
-
-func (e updateChunkError) Code() common.ErrorCode {
- return e.code
-}
-
-func validateUpdateTagFeatures(raw interface{}) (map[string]float64, error) {
- parsed, ok := raw.(map[string]interface{})
- if !ok {
- return nil, fmt.Errorf("must be an object mapping string tags to finite numeric scores greater than 0")
- }
- cleaned := make(map[string]float64, len(parsed))
- for key, value := range parsed {
- key = strings.TrimSpace(key)
- if key == "" {
- return nil, fmt.Errorf("keys must be non-empty strings")
- }
- var numeric float64
- switch typed := value.(type) {
- case float64:
- numeric = typed
- case float32:
- numeric = float64(typed)
- case int:
- numeric = float64(typed)
- case int8:
- numeric = float64(typed)
- case int16:
- numeric = float64(typed)
- case int32:
- numeric = float64(typed)
- case int64:
- numeric = float64(typed)
- default:
- return nil, fmt.Errorf("values must be finite numbers greater than 0")
- }
- if math.IsNaN(numeric) || math.IsInf(numeric, 0) || numeric <= 0 {
- return nil, fmt.Errorf("values must be finite numbers greater than 0")
- }
- cleaned[key] = numeric
- }
- return cleaned, nil
-}
-
// RemoveChunksRequest request for removing chunks
type RemoveChunksRequest struct {
DocID string `json:"doc_id"`
@@ -717,118 +159,6 @@ type RemoveChunksRequest struct {
DeleteAll bool `json:"delete_all,omitempty"`
}
-// RemoveChunks removes chunks from the dataset table.
-// If ChunkIDs is empty and DeleteAll is true, removes all chunks for the document.
-// Otherwise, removes only the specified chunks.
-func (s *ChunkService) RemoveChunks(ctx context.Context, req *RemoveChunksRequest, userID string) (int64, error) {
- if s.docEngine == nil {
- return 0, fmt.Errorf("doc engine not initialized")
- }
-
- if req.DocID == "" {
- return 0, fmt.Errorf("doc_id is required")
- }
-
- // Get user's tenants
- tenants, err := s.userTenantDAO.GetByUserID(ctx, dao.DB, userID)
- if err != nil {
- return 0, fmt.Errorf("failed to get user tenants: %w", err)
- }
- if len(tenants) == 0 {
- return 0, fmt.Errorf("user has no accessible tenants")
- }
-
- // Verify document exists and belongs to a dataset (do this first to get doc.KbID)
- docDAO := dao.NewDocumentDAO()
- doc, err := docDAO.GetByID(ctx, dao.DB, req.DocID)
- if err != nil || doc == nil {
- return 0, fmt.Errorf("document not found")
- }
-
- // Find the tenant that owns this document
- var targetTenantID string
- for _, tenant := range tenants {
- kb, err := s.kbDAO.GetByIDAndTenantID(ctx, dao.DB, doc.KbID, tenant.TenantID)
- if err == nil && kb != nil {
- targetTenantID = tenant.TenantID
- break
- }
- }
- if targetTenantID == "" {
- return 0, fmt.Errorf("user does not have access to this document")
- }
-
- indexName := fmt.Sprintf("ragflow_%s", targetTenantID)
-
- // Build condition
- condition := make(map[string]interface{})
- switch {
- case len(req.ChunkIDs) > 0 && req.DeleteAll:
- return 0, fmt.Errorf("chunk_ids and delete_all are mutually exclusive")
- case len(req.ChunkIDs) > 0:
- // Delete specific chunks - convert []string to []interface{} for buildFilterFromCondition
- chunkIDsIf := make([]interface{}, len(req.ChunkIDs))
- for i, id := range req.ChunkIDs {
- chunkIDsIf[i] = id
- }
- condition["id"] = chunkIDsIf
- condition["doc_id"] = req.DocID
- case req.DeleteAll:
- // Delete all chunks for this document
- condition["doc_id"] = req.DocID
- default:
- return 0, fmt.Errorf("either chunk_ids or delete_all must be provided")
- }
-
- deletedCount, err := s.docEngine.DeleteChunks(ctx, condition, indexName, doc.KbID)
- if err != nil {
- return 0, fmt.Errorf("failed to delete chunks: %w", err)
- }
-
- if deletedCount > 0 {
- if err = s.decrementChunkStats(req.DocID, doc.KbID, 0, deletedCount, 0); err != nil {
- return deletedCount, fmt.Errorf("failed to update chunk stats: %w", err)
- }
- }
-
- return deletedCount, nil
-}
-
-func (s *ChunkService) decrementChunkStats(docID, kbID string, tokenNum, chunkNum int64, duration float64) error {
- if s.decrementChunkStatsFunc != nil {
- return s.decrementChunkStatsFunc(docID, kbID, tokenNum, chunkNum, duration)
- }
- return dao.DB.Transaction(func(tx *gorm.DB) error {
- result := tx.Model(&entity.Document{}).
- Where("id = ? AND kb_id = ?", docID, kbID).
- Updates(map[string]interface{}{
- "token_num": gorm.Expr("CASE WHEN token_num - ? >= 0 THEN token_num - ? ELSE 0 END", tokenNum, tokenNum),
- "chunk_num": gorm.Expr("CASE WHEN chunk_num - ? >= 0 THEN chunk_num - ? ELSE 0 END", chunkNum, chunkNum),
- "process_duration": gorm.Expr("CASE WHEN process_duration + ? >= 0 THEN process_duration + ? ELSE 0 END", duration, duration),
- })
- if result.Error != nil {
- return result.Error
- }
- if result.RowsAffected == 0 {
- return fmt.Errorf("document not found")
- }
-
- result = tx.Model(&entity.Knowledgebase{}).
- Where("id = ?", kbID).
- Updates(map[string]interface{}{
- "token_num": gorm.Expr("CASE WHEN token_num - ? >= 0 THEN token_num - ? ELSE 0 END", tokenNum, tokenNum),
- "chunk_num": gorm.Expr("CASE WHEN chunk_num - ? >= 0 THEN chunk_num - ? ELSE 0 END", chunkNum, chunkNum),
- })
- if result.Error != nil {
- return result.Error
- }
- if result.RowsAffected == 0 {
- return fmt.Errorf("knowledgebase not found")
- }
- return nil
- })
-}
-
// SourcedChunk is a typed, normalized view over a retrieval result chunk.
// It decouples the ask pipeline (KbPrompt, ChunksFormat) from the raw
// map[string]interface{} that flows through the retrieval engine.
@@ -925,78 +255,3 @@ func getMap(m map[string]interface{}, key string) map[string]interface{} {
}
return nil
}
-
-// isInternalField reports whether k is an internal/technical field that
-// should be excluded from API chunk responses.
-func isInternalField(k string) bool {
- return strings.HasSuffix(k, "_vec") ||
- strings.Contains(k, "_sm_") ||
- strings.HasSuffix(k, "_tks") ||
- strings.HasSuffix(k, "_ltks")
-}
-
-// applyCommonChunkMapping applies field mappings shared between GetChunk and
-// ListChunks. Returns true if the field was handled.
-// chunkDocRunText maps the document run code to its text form, mirroring
-// Python's _map_doc run_mapping.
-func ChunkDocRunText(run *string) interface{} {
- if run == nil {
- return nil
- }
- switch *run {
- case "0":
- return "UNSTART"
- case "1":
- return "RUNNING"
- case "2":
- return "CANCEL"
- case "3":
- return "DONE"
- case "4":
- return "FAIL"
- case "5":
- return "SCHEDULE"
- }
- return *run
-}
-
-func applyCommonChunkMapping(result map[string]interface{}, k string, v interface{}) bool {
- switch k {
- case "content":
- result["content_with_weight"] = v
- case "docnm":
- result["docnm_kwd"] = v
- case "important_keywords":
- utility.SetFieldArray(result, "important_kwd", v)
- case "questions":
- utility.SetFieldArray(result, "question_kwd", v)
- case "entities_kwd", "entity_kwd", "entity_type_kwd", "from_entity_kwd",
- "name_kwd", "raptor_kwd", "removed_kwd", "source_id", "tag_kwd",
- "to_entity_kwd", "toc_kwd", "doc_type_kwd":
- if utility.IsEmpty(v) {
- result[k] = []interface{}{}
- } else {
- result[k] = v
- }
- default:
- return false
- }
- return true
-}
-
-// splitKwdHash splits a "###"-separated _kwd string into a slice.
-// Non-string values or values without "###" are returned unchanged.
-func splitKwdHash(v interface{}) interface{} {
- strVal, ok := v.(string)
- if !ok || !strings.Contains(strVal, "###") {
- return v
- }
- parts := strings.Split(strVal, "###")
- filtered := make([]interface{}, 0, len(parts))
- for _, p := range parts {
- if p != "" {
- filtered = append(filtered, p)
- }
- }
- return filtered
-}
diff --git a/internal/service/chunk_types_test.go b/internal/service/chunk_types_test.go
index e8d56ce93d..356480f62f 100644
--- a/internal/service/chunk_types_test.go
+++ b/internal/service/chunk_types_test.go
@@ -18,12 +18,7 @@ package service
import (
"ragflow/internal/common"
- "ragflow/internal/dao"
- "ragflow/internal/entity"
"testing"
-
- "github.com/glebarez/sqlite"
- "gorm.io/gorm"
)
func TestNewSourcedChunks_Empty(t *testing.T) {
@@ -226,86 +221,6 @@ func TestSourcedChunk_ZeroValue(t *testing.T) {
}
}
-func TestChunkTypesDecrementChunkStats(t *testing.T) {
- db, err := gorm.Open(sqlite.Open(":memory:"), &gorm.Config{})
- if err != nil {
- t.Fatalf("open sqlite: %v", err)
- }
- if err = db.AutoMigrate(&entity.Knowledgebase{}, &entity.Document{}); err != nil {
- t.Fatalf("migrate sqlite: %v", err)
- }
- previousDB := dao.DB
- dao.DB = db
- t.Cleanup(func() { dao.DB = previousDB })
-
- status := string(entity.StatusValid)
- if err = dao.DB.Create(&entity.Knowledgebase{
- ID: "kb-1",
- TenantID: "tenant-1",
- Name: "kb-1",
- EmbdID: "embed",
- Permission: string(entity.TenantPermissionMe),
- CreatedBy: "user-1",
- ParserConfig: entity.JSONMap{},
- TokenNum: 20,
- ChunkNum: 5,
- Status: &status,
- }).Error; err != nil {
- t.Fatalf("create kb: %v", err)
- }
- if err = dao.DB.Create(&entity.Document{
- ID: "doc-1",
- KbID: "kb-1",
- ParserID: string(entity.ParserTypeNaive),
- ParserConfig: entity.JSONMap{},
- SourceType: string(entity.FileSourceLocal),
- Type: "txt",
- CreatedBy: "user-1",
- TokenNum: 10,
- ChunkNum: 3,
- Suffix: ".txt",
- Status: &status,
- }).Error; err != nil {
- t.Fatalf("create doc: %v", err)
- }
-
- svc := &ChunkService{}
- if err = svc.decrementChunkStats("doc-1", "kb-1", 0, 2, 0); err != nil {
- t.Fatalf("decrementChunkStats() error = %v", err)
- }
-
- var doc entity.Document
- if err = dao.DB.First(&doc, "id = ?", "doc-1").Error; err != nil {
- t.Fatalf("get doc: %v", err)
- }
- if doc.TokenNum != 10 || doc.ChunkNum != 1 {
- t.Fatalf("document stats token=%d chunk=%d, want token=10 chunk=1", doc.TokenNum, doc.ChunkNum)
- }
- var kb entity.Knowledgebase
- if err = dao.DB.First(&kb, "id = ?", "kb-1").Error; err != nil {
- t.Fatalf("get kb: %v", err)
- }
- if kb.TokenNum != 20 || kb.ChunkNum != 3 {
- t.Fatalf("knowledgebase stats token=%d chunk=%d, want token=20 chunk=3", kb.TokenNum, kb.ChunkNum)
- }
-
- if err = svc.decrementChunkStats("doc-1", "kb-1", 30, 30, -1); err != nil {
- t.Fatalf("decrementChunkStats() clamp error = %v", err)
- }
- if err = dao.DB.First(&doc, "id = ?", "doc-1").Error; err != nil {
- t.Fatalf("get doc after clamp: %v", err)
- }
- if doc.TokenNum != 0 || doc.ChunkNum != 0 || doc.ProcessDuration != 0 {
- t.Fatalf("document clamped stats token=%d chunk=%d duration=%v, want zeros", doc.TokenNum, doc.ChunkNum, doc.ProcessDuration)
- }
- if err = dao.DB.First(&kb, "id = ?", "kb-1").Error; err != nil {
- t.Fatalf("get kb after clamp: %v", err)
- }
- if kb.TokenNum != 0 || kb.ChunkNum != 0 {
- t.Fatalf("knowledgebase clamped stats token=%d chunk=%d, want zeros", kb.TokenNum, kb.ChunkNum)
- }
-}
-
func TestNewSourcedChunks_RoundTrip(t *testing.T) {
original := []SourcedChunk{{
ID: "id1",
@@ -385,123 +300,3 @@ func TestGetPageSize_Valid(t *testing.T) {
t.Errorf("CoalesceInt(&50, 20) = %d, want 50", got)
}
}
-
-func TestIsInternalField(t *testing.T) {
- tests := []struct {
- field string
- want bool
- }{
- {"vector_vec", true},
- {"content_sm_ltks", true},
- {"content_ltks", true},
- {"content", false},
- {"docnm_kwd", false},
- {"important_kwd", false},
- {"knowledge_graph_kwd", false},
- }
- for _, tt := range tests {
- if got := isInternalField(tt.field); got != tt.want {
- t.Errorf("isInternalField(%q) = %v, want %v", tt.field, got, tt.want)
- }
- }
-}
-
-func TestSplitKwdHash(t *testing.T) {
- tests := []struct {
- name string
- input interface{}
- }{
- {"non-string int", 42},
- {"no hash", "hello world"},
- {"simple hash", "a###b###c"},
- {"hash with empty", "a######b"},
- {"hash only", "###"},
- }
- for _, tt := range tests {
- _ = splitKwdHash(tt.input)
- }
-
- // Verify actual output
- result := splitKwdHash("a###b###c")
- slice, ok := result.([]interface{})
- if !ok {
- t.Errorf("expected []interface{}, got %T", result)
- return
- }
- if len(slice) != 3 {
- t.Errorf("expected 3 elements, got %d", len(slice))
- }
-
- // Non-string returns unchanged
- if splitKwdHash(42) != 42 {
- t.Error("non-string should return unchanged")
- }
-}
-
-func TestApplyCommonChunkMapping(t *testing.T) {
- result := make(map[string]interface{})
-
- // content -> content_with_weight
- if !applyCommonChunkMapping(result, "content", "hello") {
- t.Error("content should be handled")
- }
- if result["content_with_weight"] != "hello" {
- t.Errorf("content_with_weight = %v, want hello", result["content_with_weight"])
- }
-
- // docnm -> docnm_kwd
- result = make(map[string]interface{})
- applyCommonChunkMapping(result, "docnm", "mydoc")
- if result["docnm_kwd"] != "mydoc" {
- t.Errorf("docnm_kwd = %v, want mydoc", result["docnm_kwd"])
- }
-
- // important_keywords -> important_kwd
- result = make(map[string]interface{})
- applyCommonChunkMapping(result, "important_keywords", []interface{}{"kw1"})
- if _, ok := result["important_kwd"]; !ok {
- t.Error("important_kwd should be set")
- }
-
- // *_kwd empty handling
- result = make(map[string]interface{})
- applyCommonChunkMapping(result, "entity_kwd", "")
- if _, ok := result["entity_kwd"]; !ok {
- t.Error("entity_kwd should be set even for empty")
- }
-
- // Unknown field should not be handled
- result = make(map[string]interface{})
- if applyCommonChunkMapping(result, "unknown_field", "val") {
- t.Error("unknown_field should not be handled")
- }
- if len(result) != 0 {
- t.Error("result should be empty for unhandled field")
- }
-}
-
-func TestChunkDocRunText(t *testing.T) {
- strPtr := func(s string) *string { return &s }
- cases := []struct {
- name string
- run *string
- want interface{}
- }{
- {name: "nil returns nil", run: nil, want: nil},
- {name: "0 UNSTART", run: strPtr("0"), want: "UNSTART"},
- {name: "1 RUNNING", run: strPtr("1"), want: "RUNNING"},
- {name: "2 CANCEL", run: strPtr("2"), want: "CANCEL"},
- {name: "3 DONE", run: strPtr("3"), want: "DONE"},
- {name: "4 FAIL", run: strPtr("4"), want: "FAIL"},
- {name: "5 SCHEDULE", run: strPtr("5"), want: "SCHEDULE"},
- {name: "unknown returns raw", run: strPtr("999"), want: "999"},
- }
- for _, tc := range cases {
- t.Run(tc.name, func(t *testing.T) {
- got := ChunkDocRunText(tc.run)
- if got != tc.want {
- t.Fatalf("chunkDocRunText(%v) = %v, want %v", tc.run, got, tc.want)
- }
- })
- }
-}
diff --git a/internal/service/document/document.go b/internal/service/document/document.go
index cac341d018..f90b5d39b7 100644
--- a/internal/service/document/document.go
+++ b/internal/service/document/document.go
@@ -75,7 +75,6 @@ func NewDocumentService() *DocumentService {
// UpdateDocumentRequest update document request
type UpdateDocumentRequest struct {
Name *string `json:"name"`
- Run *string `json:"run"`
TokenNum *int64 `json:"token_num"`
ChunkNum *int64 `json:"chunk_num"`
Progress *float64 `json:"progress"`
@@ -101,7 +100,7 @@ type DocumentResponse struct {
ProcessBeginAt *time.Time `json:"process_begin_at,omitempty"`
ProcessDuration float64 `json:"process_duration"`
Suffix string `json:"suffix"`
- Run *string `json:"run,omitempty"`
+ IngestionStatus string `json:"ingestion_status"`
Status *string `json:"status,omitempty"`
CreatedAt string `json:"created_at"`
UpdatedAt string `json:"updated_at"`
@@ -161,7 +160,7 @@ type UpdateDatasetDocumentResponse struct {
ContentHash *string `json:"content_hash,omitempty"`
MetaFields map[string]interface{} `json:"meta_fields,omitempty"`
Suffix string `json:"suffix"`
- Run string `json:"run"`
+ IngestionStatus string `json:"ingestion_status"`
Status *string `json:"status,omitempty"`
CreateTime *int64 `json:"create_time,omitempty"`
CreateDate *time.Time `json:"create_date,omitempty"`
diff --git a/internal/service/document/document_crud.go b/internal/service/document/document_crud.go
index 502eb518c2..88b9a98094 100644
--- a/internal/service/document/document_crud.go
+++ b/internal/service/document/document_crud.go
@@ -111,7 +111,7 @@ func (s *DocumentService) GetDocumentByID(ctx context.Context, id string) (*Docu
return nil, err
}
- return s.toResponse(document), nil
+ return s.toResponse(ctx, document)
}
// UpdateDocument update document
@@ -134,7 +134,6 @@ func (s *DocumentService) UpdateDocument(ctx context.Context, id string, req *Up
tokenNum: req.TokenNum,
tokenNumRequestName: "token_num",
progress: req.Progress,
- run: req.Run,
progressMsg: req.ProgressMsg,
}); err != nil {
return common.CodeDataError, err
@@ -200,13 +199,12 @@ func (s *DocumentService) ApplyDocCounts(ctx context.Context, docID, kbID string
}
// UpdateRunProgress mirrors a pipeline run's live progress into the document
-// row so the document-list endpoint (which reads document.progress/run/
+// row so the document-list endpoint (which reads document.progress/
// progress_msg) reflects in-flight Go pipeline progress. Best-effort by
// design; callers log and continue on error.
-func (s *DocumentService) UpdateRunProgress(ctx context.Context, docID string, progress float64, run, progressMsg string) error {
+func (s *DocumentService) UpdateRunProgress(ctx context.Context, docID string, progress float64, progressMsg string) error {
updates := map[string]interface{}{
"progress": progress,
- "run": run,
"progress_msg": progressMsg,
}
if doc, err := s.documentDAO.GetByID(ctx, dao.DB, docID); err != nil {
@@ -221,13 +219,12 @@ func (s *DocumentService) UpdateRunProgress(ctx context.Context, docID string, p
return s.documentDAO.UpdateByID(ctx, dao.DB, docID, updates)
}
-// UpdateRunState mirrors live progress and status into the document row when
+// UpdateRunState mirrors live progress into the document row when
// the existing progress log cannot be read. It intentionally leaves the log
// untouched so a later event can retry seeding and append it safely.
-func (s *DocumentService) UpdateRunState(ctx context.Context, docID string, progress float64, run string) error {
+func (s *DocumentService) UpdateRunState(ctx context.Context, docID string, progress float64) error {
updates := map[string]interface{}{
"progress": progress,
- "run": run,
}
if doc, err := s.documentDAO.GetByID(ctx, dao.DB, docID); err != nil {
return err
diff --git a/internal/service/document/document_dataset_update.go b/internal/service/document/document_dataset_update.go
index ea2929dbe9..f578df14c1 100644
--- a/internal/service/document/document_dataset_update.go
+++ b/internal/service/document/document_dataset_update.go
@@ -235,21 +235,38 @@ func (s *DocumentService) UpdateDatasetDocument(ctx context.Context, userID, dat
metaFields, _ = s.GetDocumentMetadataByID(ctx, updatedDoc.ID)
}
- return s.toUpdateDatasetDocumentResponse(updatedDoc, metaFields), common.CodeSuccess, nil
+ resp, err := s.toUpdateDatasetDocumentResponse(ctx, updatedDoc, metaFields)
+ if err != nil {
+ return nil, common.CodeServerError, err
+ }
+ return resp, common.CodeSuccess, nil
}
// validateDocumentModifiable rejects configuration edits while the document is
-// actively parsing or scheduled. Editing a document that is mid-parse races
-// with the running worker and can leave the document stuck and undeletable.
-func (s *DocumentService) validateDocumentModifiable(doc *entity.Document) (common.ErrorCode, error) {
- if doc.Run != nil {
- run := entity.TaskStatus(*doc.Run)
- if !entity.DocumentModifiableStatuses[run] {
- return common.CodeDataError, fmt.Errorf(
- "document is currently %q and cannot be modified; stop parsing or wait for it to finish before updating its configuration", run)
- }
+// actively parsing or scheduled.
+func (s *DocumentService) validateDocumentModifiable(ctx context.Context, doc *entity.Document) (common.ErrorCode, error) {
+ if s.ingestionTaskDAO == nil {
+ return common.CodeServerError, errors.New("ingestion task DAO not initialized")
+ }
+ if doc == nil || doc.ID == "" {
+ return common.CodeDataError, errors.New("document is nil or has empty ID")
+ }
+ task, err := s.ingestionTaskDAO.GetByDocumentID(ctx, dao.DB, doc.ID)
+ if err != nil {
+ return common.CodeServerError, fmt.Errorf("failed to get ingestion task for document %s: %w", doc.ID, err)
+ }
+ if task == nil {
+ return common.CodeSuccess, nil
+ }
+ switch {
+ case common.IsActiveTaskStatus(task.Status):
+ return common.CodeDataError, fmt.Errorf(
+ "document is currently %q and cannot be modified; stop parsing or wait for it to finish before updating its configuration", task.Status)
+ case common.IsTerminalTaskStatus(task.Status):
+ return common.CodeSuccess, nil
+ default:
+ return common.CodeDataError, fmt.Errorf("document has unrecognized task status %q", task.Status)
}
- return common.CodeSuccess, nil
}
func (s *DocumentService) validateDatasetDocumentUpdate(ctx context.Context, datasetID, documentID, userID string, doc *entity.Document, req *UpdateDatasetDocumentRequest, present map[string]bool) (common.ErrorCode, error) {
@@ -262,7 +279,7 @@ func (s *DocumentService) validateDatasetDocumentUpdate(ctx context.Context, dat
// chunk_method, pipeline_id, enabled, meta_fields) is blocked so the
// in-flight parser never reads a config that changed underneath it.
if len(present) > 0 {
- if code, err := s.validateDocumentModifiable(doc); err != nil {
+ if code, err := s.validateDocumentModifiable(ctx, doc); err != nil {
return code, err
}
}
@@ -456,10 +473,20 @@ func (s *DocumentService) updateDocumentParserConfig(ctx context.Context, docume
})
}
-func (s *DocumentService) toUpdateDatasetDocumentResponse(doc *entity.Document, metaFields map[string]interface{}) *UpdateDatasetDocumentResponse {
+func (s *DocumentService) toUpdateDatasetDocumentResponse(ctx context.Context, doc *entity.Document, metaFields map[string]interface{}) (*UpdateDatasetDocumentResponse, error) {
if metaFields == nil {
metaFields = map[string]interface{}{}
}
+ ingestionStatus := "UNSTART"
+ if s.ingestionTaskDAO != nil && doc != nil && doc.ID != "" {
+ task, err := s.ingestionTaskDAO.GetByDocumentID(ctx, dao.DB, doc.ID)
+ if err != nil {
+ return nil, fmt.Errorf("failed to get ingestion task for document %s: %w", doc.ID, err)
+ }
+ if task != nil && task.Status != "" {
+ ingestionStatus = task.Status
+ }
+ }
return &UpdateDatasetDocumentResponse{
ID: doc.ID,
Thumbnail: doc.Thumbnail,
@@ -482,31 +509,13 @@ func (s *DocumentService) toUpdateDatasetDocumentResponse(doc *entity.Document,
ContentHash: doc.ContentHash,
MetaFields: metaFields,
Suffix: doc.Suffix,
- Run: mapDocumentRunStatus(doc.Run),
+ IngestionStatus: ingestionStatus,
Status: doc.Status,
CreateTime: doc.CreateTime,
CreateDate: doc.CreateDate,
UpdateTime: doc.UpdateTime,
UpdateDate: doc.UpdateDate,
- }
-}
-
-func mapDocumentRunStatus(run *string) string {
- if run == nil {
- return "UNSTART"
- }
- switch *run {
- case string(entity.TaskStatusRunning):
- return "RUNNING"
- case string(entity.TaskStatusCancel):
- return "CANCEL"
- case string(entity.TaskStatusDone):
- return "DONE"
- case string(entity.TaskStatusFail):
- return "FAIL"
- default:
- return "UNSTART"
- }
+ }, nil
}
// validDocumentChunkMethods mirrors Python's UpdateDocumentReq chunk_method set.
diff --git a/internal/service/document/document_ingest.go b/internal/service/document/document_ingest.go
index 5a969bc426..36cf515480 100644
--- a/internal/service/document/document_ingest.go
+++ b/internal/service/document/document_ingest.go
@@ -113,7 +113,6 @@ func (s *DocumentService) Ingest(ctx context.Context, userID string, req *Ingest
return common.CodeDataError, err
}
if err = s.documentDAO.UpdateByID(ctx, dao.DB, doc.ID, map[string]interface{}{
- "run": string(entity.TaskStatusCancel),
"progress": 0,
}); err != nil {
common.Error(fmt.Sprintf("go side, doc %s, UpdateByID failed", doc.ID), err)
@@ -125,7 +124,6 @@ func (s *DocumentService) Ingest(ctx context.Context, userID string, req *Ingest
// Delete-only: user asked to remove prior parse results without
// starting a new parse. RUNNING already continued above.
if err = s.documentDAO.UpdateByID(ctx, dao.DB, doc.ID, map[string]interface{}{
- "run": run,
"progress": 0,
}); err != nil {
common.Error(fmt.Sprintf("go side, doc %s, UpdateByID failed", doc.ID), err)
diff --git a/internal/service/document/document_list.go b/internal/service/document/document_list.go
index e01c9358f9..52b163b5c2 100644
--- a/internal/service/document/document_list.go
+++ b/internal/service/document/document_list.go
@@ -29,9 +29,28 @@ func (s *DocumentService) ListDocuments(ctx context.Context, page, pageSize int)
return nil, 0, err
}
+ docIDs := make([]string, 0, len(documents))
+ for _, doc := range documents {
+ if doc != nil && doc.ID != "" {
+ docIDs = append(docIDs, doc.ID)
+ }
+ }
+ var taskMap map[string]*entity.IngestionTask
+ if s.ingestionTaskDAO != nil && len(docIDs) > 0 {
+ var err error
+ taskMap, err = s.ingestionTaskDAO.GetLatestByDocumentIDs(ctx, dao.DB, docIDs)
+ if err != nil {
+ return nil, 0, fmt.Errorf("failed to get ingestion tasks for documents: %w", err)
+ }
+ }
+
responses := make([]*DocumentResponse, len(documents))
for i, doc := range documents {
- responses[i] = s.toResponse(doc)
+ var task *entity.IngestionTask
+ if taskMap != nil && doc != nil {
+ task = taskMap[doc.ID]
+ }
+ responses[i] = s.toResponseWithTask(doc, task)
}
return responses, total, nil
@@ -177,16 +196,49 @@ func (s *DocumentService) GetDocumentsByAuthorID(ctx context.Context, authorID,
return nil, 0, err
}
+ docIDs := make([]string, 0, len(documents))
+ for _, doc := range documents {
+ if doc != nil && doc.ID != "" {
+ docIDs = append(docIDs, doc.ID)
+ }
+ }
+ var taskMap map[string]*entity.IngestionTask
+ if s.ingestionTaskDAO != nil && len(docIDs) > 0 {
+ var err error
+ taskMap, err = s.ingestionTaskDAO.GetLatestByDocumentIDs(ctx, dao.DB, docIDs)
+ if err != nil {
+ return nil, 0, fmt.Errorf("failed to get ingestion tasks for documents: %w", err)
+ }
+ }
+
responses := make([]*DocumentResponse, len(documents))
for i, doc := range documents {
- responses[i] = s.toResponse(doc)
+ var task *entity.IngestionTask
+ if taskMap != nil && doc != nil {
+ task = taskMap[doc.ID]
+ }
+ responses[i] = s.toResponseWithTask(doc, task)
}
return responses, total, nil
}
// toResponse convert model.Document to DocumentResponse
-func (s *DocumentService) toResponse(doc *entity.Document) *DocumentResponse {
+func (s *DocumentService) toResponse(ctx context.Context, doc *entity.Document) (*DocumentResponse, error) {
+ if s.ingestionTaskDAO == nil || doc == nil || doc.ID == "" {
+ return s.toResponseWithTask(doc, nil), nil
+ }
+ task, err := s.ingestionTaskDAO.GetByDocumentID(ctx, dao.DB, doc.ID)
+ if err != nil {
+ return nil, fmt.Errorf("get ingestion task for document %s: %w", doc.ID, err)
+ }
+ return s.toResponseWithTask(doc, task), nil
+}
+
+func (s *DocumentService) toResponseWithTask(doc *entity.Document, task *entity.IngestionTask) *DocumentResponse {
+ if doc == nil {
+ return nil
+ }
createdAt := ""
if doc.CreateTime != nil {
// Check if timestamp is in milliseconds (13 digits) or seconds (10 digits)
@@ -208,6 +260,10 @@ func (s *DocumentService) toResponse(doc *entity.Document) *DocumentResponse {
}
updatedAt = time.Unix(ts, 0).Format("2006-01-02 15:04:05")
}
+ ingestionStatus := "UNSTART"
+ if task != nil && task.Status != "" {
+ ingestionStatus = task.Status
+ }
return &DocumentResponse{
ID: doc.ID,
Name: doc.Name,
@@ -226,7 +282,7 @@ func (s *DocumentService) toResponse(doc *entity.Document) *DocumentResponse {
ProcessBeginAt: doc.ProcessBeginAt,
ProcessDuration: doc.ProcessDuration,
Suffix: doc.Suffix,
- Run: doc.Run,
+ IngestionStatus: ingestionStatus,
Status: doc.Status,
CreatedAt: createdAt,
UpdatedAt: updatedAt,
diff --git a/internal/service/document/document_modify_guard_test.go b/internal/service/document/document_modify_guard_test.go
index 0073739420..915ce85e55 100644
--- a/internal/service/document/document_modify_guard_test.go
+++ b/internal/service/document/document_modify_guard_test.go
@@ -24,31 +24,40 @@ import (
"ragflow/internal/entity"
)
-// TestValidateDocumentModifiable_RunStatuses pins the allowed/disallowed run
-// statuses for editing a document via UpdateDatasetDocument. RUNNING ("1") and
-// SCHEDULE ("5") must be rejected; UNSTART/CANCEL/DONE/FAIL may be edited.
-// A nil Run (gorm default "0" UNSTART) is treated as editable.
-func TestValidateDocumentModifiable_RunStatuses(t *testing.T) {
+// TestValidateDocumentModifiable_TaskStatuses pins the allowed/disallowed task
+// statuses for editing a document via UpdateDatasetDocument.
+// CREATED, SCHEDULED, RUNNING, and STOPPING must be rejected;
+// no task, COMPLETED, STOPPED, and FAILED may be edited.
+func TestValidateDocumentModifiable_TaskStatuses(t *testing.T) {
cases := []struct {
- name string
- run *string
- wantCode common.ErrorCode
- wantErr bool
+ name string
+ taskStatus *string
+ wantCode common.ErrorCode
+ wantErr bool
}{
- {"nil run defaults to unstart", nil, common.CodeSuccess, false},
- {"unstart editable", sptr(string(entity.TaskStatusUnstart)), common.CodeSuccess, false},
- {"cancel editable", sptr(string(entity.TaskStatusCancel)), common.CodeSuccess, false},
- {"done editable", sptr(string(entity.TaskStatusDone)), common.CodeSuccess, false},
- {"fail editable", sptr(string(entity.TaskStatusFail)), common.CodeSuccess, false},
- {"running rejected", sptr(string(entity.TaskStatusRunning)), common.CodeDataError, true},
- {"schedule rejected", sptr(string(entity.TaskStatusSchedule)), common.CodeDataError, true},
+ {"no task defaults to unstart", nil, common.CodeSuccess, false},
+ {"created rejected", sptr(common.CREATED), common.CodeDataError, true},
+ {"scheduled rejected", sptr(common.SCHEDULED), common.CodeDataError, true},
+ {"running rejected", sptr(common.RUNNING), common.CodeDataError, true},
+ {"stopping rejected", sptr(common.STOPPING), common.CodeDataError, true},
+ {"completed editable", sptr(common.COMPLETED), common.CodeSuccess, false},
+ {"stopped editable", sptr(common.STOPPED), common.CodeSuccess, false},
+ {"failed editable", sptr(common.FAILED), common.CodeSuccess, false},
}
- svc := &DocumentService{}
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
- doc := &entity.Document{Run: tc.run}
- code, err := svc.validateDocumentModifiable(doc)
+ db := setupServiceTestDB(t)
+ pushServiceDB(t, db)
+ insertTestKB(t, "kb-1", "tenant-1", 1, 0, 0)
+ insertTestDoc(t, "doc-1", "kb-1", 0, 0)
+ if tc.taskStatus != nil {
+ insertTestIngestionTaskWithStatus(t, "task-1", "user-1", "doc-1", "kb-1", *tc.taskStatus)
+ }
+ svc := testDocumentService(t)
+ ctx := t.Context()
+ doc := &entity.Document{ID: "doc-1"}
+ code, err := svc.validateDocumentModifiable(ctx, doc)
if tc.wantErr {
if err == nil {
t.Fatalf("expected error, got nil")
@@ -71,20 +80,21 @@ func TestValidateDocumentModifiable_RunStatuses(t *testing.T) {
}
}
-// updateDatasetDocumentRejected asserts that editing a document in the given
-// run status with the supplied request is rejected by the run-state guard.
-func updateDatasetDocumentRejected(t *testing.T, run string, req *UpdateDatasetDocumentRequest, present map[string]bool) {
+// updateDatasetDocumentRejected asserts that editing a document with the given
+// task status is rejected by the modifiable guard.
+func updateDatasetDocumentRejected(t *testing.T, status string, req *UpdateDatasetDocumentRequest, present map[string]bool) {
t.Helper()
db := setupServiceTestDB(t)
pushServiceDB(t, db)
insertTestKB(t, "kb-1", "tenant-1", 1, 0, 0)
- insertTestDocWithRun(t, "doc-1", "kb-1", run, 0, 0)
+ insertTestDoc(t, "doc-1", "kb-1", 0, 0)
+ insertTestIngestionTaskWithStatus(t, "task-1", "user-1", "doc-1", "kb-1", status)
svc := testDocumentService(t)
ctx := t.Context()
_, code, err := svc.UpdateDatasetDocument(ctx, "tenant-1", "kb-1", "doc-1", req, present)
if err == nil {
- t.Fatalf("expected run-state rejection, got nil error (code=%v)", code)
+ t.Fatalf("expected task-state rejection, got nil error (code=%v)", code)
}
if code != common.CodeDataError {
t.Fatalf("code = %v, want %v", code, common.CodeDataError)
@@ -94,15 +104,17 @@ func updateDatasetDocumentRejected(t *testing.T, run string, req *UpdateDatasetD
}
}
-// updateDatasetDocumentAllowed asserts that editing a document in an editable
-// run status with the supplied request is NOT blocked by the run-state guard
-// (it may still fail later validation, but not with the run-state error).
-func updateDatasetDocumentAllowed(t *testing.T, run string, req *UpdateDatasetDocumentRequest, present map[string]bool) {
+// updateDatasetDocumentAllowed asserts that editing a document with an editable
+// task status is NOT blocked by the modifiable guard.
+func updateDatasetDocumentAllowed(t *testing.T, status string, req *UpdateDatasetDocumentRequest, present map[string]bool) {
t.Helper()
db := setupServiceTestDB(t)
pushServiceDB(t, db)
insertTestKB(t, "kb-1", "tenant-1", 1, 0, 0)
- insertTestDocWithRun(t, "doc-1", "kb-1", run, 0, 0)
+ insertTestDoc(t, "doc-1", "kb-1", 0, 0)
+ if status != "" {
+ insertTestIngestionTaskWithStatus(t, "task-1", "user-1", "doc-1", "kb-1", status)
+ }
svc := testDocumentService(t)
ctx := t.Context()
@@ -116,75 +128,76 @@ func updateDatasetDocumentAllowed(t *testing.T, run string, req *UpdateDatasetDo
}
func TestUpdateDatasetDocumentRejectsRunningParserConfig(t *testing.T) {
- updateDatasetDocumentRejected(t, string(entity.TaskStatusRunning),
+ updateDatasetDocumentRejected(t, common.RUNNING,
&UpdateDatasetDocumentRequest{ParserConfig: map[string]any{"chunk_token_num": float64(128)}},
map[string]bool{"parser_config": true})
}
func TestUpdateDatasetDocumentRejectsRunningChunkMethod(t *testing.T) {
cm := "naive"
- updateDatasetDocumentRejected(t, string(entity.TaskStatusRunning),
+ updateDatasetDocumentRejected(t, common.RUNNING,
&UpdateDatasetDocumentRequest{ChunkMethod: &cm},
map[string]bool{"chunk_method": true})
}
func TestUpdateDatasetDocumentRejectsRunningRename(t *testing.T) {
name := "renamed.txt"
- updateDatasetDocumentRejected(t, string(entity.TaskStatusRunning),
+ updateDatasetDocumentRejected(t, common.RUNNING,
&UpdateDatasetDocumentRequest{Name: &name},
map[string]bool{"name": true})
}
func TestUpdateDatasetDocumentRejectsRunningEnabled(t *testing.T) {
enabled := 0
- updateDatasetDocumentRejected(t, string(entity.TaskStatusRunning),
+ updateDatasetDocumentRejected(t, common.RUNNING,
&UpdateDatasetDocumentRequest{Enabled: &enabled},
map[string]bool{"enabled": true})
}
func TestUpdateDatasetDocumentRejectsScheduledRename(t *testing.T) {
name := "renamed.txt"
- updateDatasetDocumentRejected(t, string(entity.TaskStatusSchedule),
+ updateDatasetDocumentRejected(t, common.SCHEDULED,
&UpdateDatasetDocumentRequest{Name: &name},
map[string]bool{"name": true})
}
func TestUpdateDatasetDocumentAllowsDoneEnabled(t *testing.T) {
enabled := 0
- updateDatasetDocumentAllowed(t, string(entity.TaskStatusDone),
+ updateDatasetDocumentAllowed(t, common.COMPLETED,
&UpdateDatasetDocumentRequest{Enabled: &enabled},
map[string]bool{"enabled": true})
}
func TestUpdateDatasetDocumentAllowsCancelEnabled(t *testing.T) {
enabled := 0
- updateDatasetDocumentAllowed(t, string(entity.TaskStatusCancel),
+ updateDatasetDocumentAllowed(t, common.STOPPED,
&UpdateDatasetDocumentRequest{Enabled: &enabled},
map[string]bool{"enabled": true})
}
func TestUpdateDatasetDocumentAllowsFailEnabled(t *testing.T) {
enabled := 0
- updateDatasetDocumentAllowed(t, string(entity.TaskStatusFail),
+ updateDatasetDocumentAllowed(t, common.FAILED,
&UpdateDatasetDocumentRequest{Enabled: &enabled},
map[string]bool{"enabled": true})
}
func TestUpdateDatasetDocumentAllowsUnstartEnabled(t *testing.T) {
enabled := 0
- updateDatasetDocumentAllowed(t, string(entity.TaskStatusUnstart),
+ updateDatasetDocumentAllowed(t, "",
&UpdateDatasetDocumentRequest{Enabled: &enabled},
map[string]bool{"enabled": true})
}
// TestUpdateDatasetDocumentRunningEmptyPresentNotRejected locks the
// len(present) > 0 gate: an empty PATCH (no fields) on a RUNNING doc must not
-// be rejected by the run-state guard (it is a no-op, not an edit).
+// be rejected by the modifiable guard (it is a no-op, not an edit).
func TestUpdateDatasetDocumentRunningEmptyPresentNotRejected(t *testing.T) {
db := setupServiceTestDB(t)
pushServiceDB(t, db)
insertTestKB(t, "kb-1", "tenant-1", 1, 0, 0)
- insertTestDocWithRun(t, "doc-1", "kb-1", string(entity.TaskStatusRunning), 0, 0)
+ insertTestDoc(t, "doc-1", "kb-1", 0, 0)
+ insertTestIngestionTaskWithStatus(t, "task-1", "user-1", "doc-1", "kb-1", common.RUNNING)
svc := testDocumentService(t)
ctx := t.Context()
@@ -201,18 +214,19 @@ func TestUpdateDatasetDocumentRunningEmptyPresentNotRejected(t *testing.T) {
// TestUpdateDatasetDocumentRejectsRunningMetaFields pins that every editable
// field is blocked while RUNNING, including meta_fields.
func TestUpdateDatasetDocumentRejectsRunningMetaFields(t *testing.T) {
- updateDatasetDocumentRejected(t, string(entity.TaskStatusRunning),
+ updateDatasetDocumentRejected(t, common.RUNNING,
&UpdateDatasetDocumentRequest{MetaFields: map[string]any{"author": "x"}},
map[string]bool{"meta_fields": true})
}
// TestUpdateDatasetDocumentAllowsDoneRename proves an editable status is not
-// over-restricted: renaming a DONE document succeeds (no run-state error).
+// over-restricted: renaming a DONE document succeeds.
func TestUpdateDatasetDocumentAllowsDoneRename(t *testing.T) {
db := setupServiceTestDB(t)
pushServiceDB(t, db)
insertTestKB(t, "kb-1", "tenant-1", 1, 0, 0)
insertNamedTestDoc(t, "doc-1", "kb-1", "orig.txt", 0, 0)
+ insertTestIngestionTaskWithStatus(t, "task-1", "user-1", "doc-1", "kb-1", common.COMPLETED)
name := "renamed.txt"
svc := testDocumentService(t)
diff --git a/internal/service/document/document_parse.go b/internal/service/document/document_parse.go
index d4f4f826c8..cc1b29eaa3 100644
--- a/internal/service/document/document_parse.go
+++ b/internal/service/document/document_parse.go
@@ -54,10 +54,7 @@ func lockDocumentParse(docID string) func() {
// StartParseDocuments starts parsing a document via the DSL ingestion
// pipeline. It optionally clears prior results (RerunWithDelete), applies
// KB config (ApplyKB), validates storage, and enqueues an ingestion task.
-// The document run status is NOT set here; service.IngestionTaskService.StartRunning
-// sets it to RUNNING when the worker picks up the task and transitions it from
-// CREATED or SCHEDULED. Extracted from Ingest so
-// other entry points (e.g. ChunkService.Parse)
+// Extracted from Ingest so other entry points (e.g. ChunkService.Parse)
// can reuse the same start-parse flow.
func (s *DocumentService) StartParseDocuments(ctx context.Context, doc *entity.Document, kb *entity.Knowledgebase, userID string, opts StartParseOptions) error {
// Validate storage first so we don't clear prior results and then fail
@@ -101,7 +98,7 @@ func (s *DocumentService) AssertIngestionTasksTerminal(ctx context.Context, docI
if task == nil {
continue
}
- if task.Status == common.RUNNING || task.Status == common.STOPPING {
+ if common.IsRunningOrStopping(task.Status) {
return fmt.Errorf("document %s ingestion task is %s; stop it and wait for a terminal state before re-parsing", docID, task.Status)
}
}
@@ -207,14 +204,6 @@ func (s *DocumentService) clearDocumentAndKBCountersForRerun(docID, kbID string)
})
}
-func (s *DocumentService) countDoneDocuments(datasetID string) (int64, error) {
- var count int64
- err := dao.GetDB().Model(&entity.Document{}).
- Where("kb_id = ? AND run = ?", datasetID, string(entity.TaskStatusDone)).
- Count(&count).Error
- return count, err
-}
-
func (s *DocumentService) clearKBChunkNumWhenRerun(doc *entity.Document) error {
if doc == nil {
return fmt.Errorf("document is nil")
@@ -294,7 +283,7 @@ func (s *DocumentService) ParseDocuments(ctx context.Context, datasetID, userID
}
// StopParseDocuments stops parsing for the given documents in a dataset.
-// It sets Redis cancel signals for associated tasks and updates doc.run to CANCEL.
+// It requests stop for the associated ingestion tasks.
// Returns a map with success_count and optionally errors.
func (s *DocumentService) StopParseDocuments(ctx context.Context, datasetID string, docIDs []string) (map[string]interface{}, error) {
deduped := common.Deduplicate(docIDs)
@@ -382,53 +371,42 @@ func (s *DocumentService) validateDocsInDataset(ctx context.Context, docIDs []st
return docs, nil
}
-// errParseNotRunning is returned by CancelDocParse when the document is not in
-// a cancelable state: its run status is neither RUNNING nor CANCEL and it has
-// no in-flight ingestion task. Callers map it to their endpoint-specific
-// message (the Python /documents/ingest and /documents/stop messages differ).
+// errParseNotRunning is returned by CancelDocParse when the document has no
+// in-flight ingestion task and is not already stopped. Callers map it to their
+// endpoint-specific message (the Python /documents/ingest and /documents/stop messages differ).
var errParseNotRunning = errors.New("parse task is not in running status")
// CancelDocParse stops the ingestion task for the document by calling
-// RequestStop (STOPPING), then marks the document run status as CANCEL.
-// It mirrors the Python cancel precondition: only a document whose run status
-// is RUNNING or CANCEL, or one with an in-flight ingestion task
-// (CREATED/SCHEDULED/RUNNING/STOPPING), can be canceled; otherwise errParseNotRunning
-// is returned. A missing ingestion task is not an error by itself — cancel is
-// then a no-op on the task side, matching Python's cancel_all_task_of.
+// RequestStop (STOPPING).
+// It returns errParseNotRunning if the document has neither an in-flight
+// ingestion task (CREATED/SCHEDULED/RUNNING/STOPPING) nor an already stopped
+// task (STOPPED).
func (s *DocumentService) CancelDocParse(ctx context.Context, doc *entity.Document) error {
task, err := s.ingestionTaskDAO.GetByDocumentID(ctx, dao.DB, doc.ID)
if err != nil {
return fmt.Errorf("failed to get ingestion task for %s: %w", doc.ID, err)
}
- docRun := ""
- if doc.Run != nil {
- docRun = *doc.Run
- }
- inFlight := task != nil && (task.Status == common.CREATED || task.Status == common.SCHEDULED || task.Status == common.RUNNING || task.Status == common.STOPPING)
- if docRun != string(entity.TaskStatusRunning) && docRun != string(entity.TaskStatusCancel) && !inFlight {
+ inFlight := task != nil && common.IsActiveTaskStatus(task.Status)
+ isStopped := task != nil && task.Status == common.STOPPED
+ if !inFlight && !isStopped {
return errParseNotRunning
}
- if task != nil {
+ if inFlight {
if _, err = s.ingestionTaskSvc.RequestStop(ctx, task.ID); err != nil {
return fmt.Errorf("failed to stop ingestion task %s: %w", task.ID, err)
}
}
- if upErr := s.documentDAO.UpdateByID(ctx, dao.DB, doc.ID, map[string]interface{}{"run": string(entity.TaskStatusCancel)}); upErr != nil {
- return fmt.Errorf("failed to update document %s: %w", doc.ID, upErr)
- }
return nil
}
func (s *DocumentService) resetDocumentForReparse(ctx context.Context, doc *entity.Document, tenantID string, parserID *string, pipelineID *string) error {
progressMsg := ""
- run := string(entity.TaskStatusUnstart)
updates := map[string]interface{}{
"progress": 0,
"progress_msg": progressMsg,
- "run": run,
}
if parserID != nil {
updates["parser_id"] = *parserID
diff --git a/internal/service/document/document_test.go b/internal/service/document/document_test.go
index d5fc1525a5..f5be393bc0 100644
--- a/internal/service/document/document_test.go
+++ b/internal/service/document/document_test.go
@@ -1203,25 +1203,6 @@ func TestDeleteDocuments_Deduplicate(t *testing.T) {
}
}
-// insertTestDocWithRun inserts a document with the given Run status for StopParseDocuments tests.
-func insertTestDocWithRun(t *testing.T, id, kbID, run string, tokenNum, chunkNum int64) {
- t.Helper()
- doc := &entity.Document{
- ID: id,
- KbID: kbID,
- ParserID: "naive",
- ParserConfig: entity.JSONMap{},
- TokenNum: tokenNum,
- ChunkNum: chunkNum,
- Suffix: ".txt",
- Status: sptr("1"),
- Run: &run,
- }
- if err := dao.DB.Create(doc).Error; err != nil {
- t.Fatalf("insert test doc: %v", err)
- }
-}
-
// insertTestTaskWithProgress inserts a task with the given progress value.
func insertTestTaskWithProgress(t *testing.T, id, docID string, progress float64) {
t.Helper()
@@ -1309,7 +1290,7 @@ func TestStartParseDocumentsSerializesConcurrentReruns(t *testing.T) {
db := setupConcurrentServiceTestDB(t)
pushServiceDB(t, db)
insertTestKB(t, "kb-1", "tenant-1", 0, 10, 5)
- insertTestDocWithRun(t, "doc-1", "kb-1", string(entity.TaskStatusDone), 10, 5)
+ insertTestDoc(t, "doc-1", "kb-1", 10, 5)
if err := db.Model(&entity.Document{}).Where("id = ?", "doc-1").Update("location", "loc-1").Error; err != nil {
t.Fatalf("set document location: %v", err)
}
@@ -1380,7 +1361,7 @@ func TestStopParseDocuments_Success(t *testing.T) {
pushServiceDB(t, db)
insertTestKB(t, "kb-1", "tenant-1", 1, 10, 5)
- insertTestDocWithRun(t, "doc-1", "kb-1", string(entity.TaskStatusRunning), 10, 5)
+ insertTestDoc(t, "doc-1", "kb-1", 10, 5)
insertTestIngestionTask(t, "task-1", "user-1", "doc-1", "kb-1")
svc := testDocumentService(t)
@@ -1398,13 +1379,13 @@ func TestStopParseDocuments_Success(t *testing.T) {
t.Fatalf("expected success_count=1, got %d", sc)
}
- // Verify document run status updated to CANCEL
- doc, _ := dao.NewDocumentDAO().GetByID(ctx, db, "doc-1")
- if doc == nil || doc.Run == nil {
- t.Fatal("doc not found or run is nil")
+ // Verify ingestion task status updated to STOPPED
+ task, err := dao.NewIngestionTaskDAO().GetByID(ctx, db, "task-1")
+ if err != nil {
+ t.Fatalf("load task: %v", err)
}
- if *doc.Run != string(entity.TaskStatusCancel) {
- t.Fatalf("expected run=%q, got %q", string(entity.TaskStatusCancel), *doc.Run)
+ if task.Status != common.STOPPED {
+ t.Fatalf("expected task status=%s, got %s", common.STOPPED, task.Status)
}
}
@@ -1413,9 +1394,9 @@ func TestStopParseDocuments_CancelStatus(t *testing.T) {
pushServiceDB(t, db)
insertTestKB(t, "kb-1", "tenant-1", 1, 10, 5)
- // Doc is already in CANCEL state — should still be accepted
- insertTestDocWithRun(t, "doc-1", "kb-1", string(entity.TaskStatusCancel), 10, 5)
- insertTestIngestionTask(t, "task-1", "user-1", "doc-1", "kb-1")
+ // Task is already in STOPPED state — should still be accepted
+ insertTestDoc(t, "doc-1", "kb-1", 10, 5)
+ insertTestIngestionTaskWithStatus(t, "task-1", "user-1", "doc-1", "kb-1", common.STOPPED)
svc := testDocumentService(t)
ctx := t.Context()
@@ -1435,8 +1416,8 @@ func TestStopParseDocuments_NotRunningOrCancel(t *testing.T) {
pushServiceDB(t, db)
insertTestKB(t, "kb-1", "tenant-1", 1, 10, 5)
- // Doc with Run="0" (UNSTART) and no unfinished tasks → cannot cancel
- insertTestDocWithRun(t, "doc-1", "kb-1", string(entity.TaskStatusUnstart), 10, 5)
+ // Doc with no task → cannot cancel
+ insertTestDoc(t, "doc-1", "kb-1", 10, 5)
svc := testDocumentService(t)
ctx := t.Context()
@@ -1463,8 +1444,8 @@ func TestStopParseDocuments_UnfinishedTask(t *testing.T) {
pushServiceDB(t, db)
insertTestKB(t, "kb-1", "tenant-1", 1, 10, 5)
- // Doc with Run="0" → cancelDocParse calls RequestStop on the ingestion task.
- insertTestDocWithRun(t, "doc-1", "kb-1", string(entity.TaskStatusUnstart), 10, 5)
+ // Doc with active ingestion task → cancelDocParse calls RequestStop on the ingestion task.
+ insertTestDoc(t, "doc-1", "kb-1", 10, 5)
insertTestIngestionTask(t, "task-1", "user-1", "doc-1", "kb-1")
svc := testDocumentService(t)
@@ -1486,7 +1467,7 @@ func TestStopParseDocuments_WrongDataset(t *testing.T) {
insertTestKB(t, "kb-1", "tenant-1", 1, 10, 5)
insertTestKB(t, "kb-2", "tenant-1", 1, 10, 5)
- insertTestDocWithRun(t, "doc-1", "kb-2", string(entity.TaskStatusRunning), 10, 5)
+ insertTestDoc(t, "doc-1", "kb-2", 10, 5)
svc := testDocumentService(t)
ctx := t.Context()
@@ -1529,7 +1510,7 @@ func TestStopParseDocuments_Deduplicate(t *testing.T) {
pushServiceDB(t, db)
insertTestKB(t, "kb-1", "tenant-1", 1, 10, 5)
- insertTestDocWithRun(t, "doc-1", "kb-1", string(entity.TaskStatusRunning), 10, 5)
+ insertTestDoc(t, "doc-1", "kb-1", 10, 5)
insertTestIngestionTask(t, "task-1", "user-1", "doc-1", "kb-1")
svc := testDocumentService(t)
@@ -2075,7 +2056,6 @@ func TestUpdateDocumentRejectsImmutableFieldChanges(t *testing.T) {
wantErr string
}{
{name: "progress", request: UpdateDocumentRequest{Progress: float64Ptr(0)}, wantErr: "can't change `progress`"},
- {name: "run", request: UpdateDocumentRequest{Run: sptr("0")}, wantErr: "can't change `run`"},
{name: "progress message", request: UpdateDocumentRequest{ProgressMsg: sptr("reset")}, wantErr: "can't change `progress_msg`"},
{name: "chunk count", request: UpdateDocumentRequest{ChunkNum: int64Ptr(0)}, wantErr: "can't change `chunk_num`"},
{name: "token count", request: UpdateDocumentRequest{TokenNum: int64Ptr(0)}, wantErr: "can't change `token_num`"},
@@ -2086,11 +2066,9 @@ func TestUpdateDocumentRejectsImmutableFieldChanges(t *testing.T) {
db := setupServiceTestDB(t)
pushServiceDB(t, db)
insertTestDoc(t, "doc-1", "kb-1", 10, 5)
- run := "3"
progressMsg := "parsing"
if err := db.Model(&entity.Document{}).Where("id = ?", "doc-1").Updates(map[string]interface{}{
"progress": 0.5,
- "run": run,
"progress_msg": progressMsg,
}).Error; err != nil {
t.Fatalf("prepare document: %v", err)
@@ -2117,11 +2095,9 @@ func TestUpdateDocumentUsesSharedRenamePath(t *testing.T) {
insertNamedTestDoc(t, "doc-1", "kb-1", "old.pdf", 10, 5)
insertTestFile(t, "file-1", "folder-1", "old.pdf", sptr("old.pdf"))
insertTestFile2Document(t, "f2d-1", "file-1", "doc-1")
- run := "3"
progressMsg := "complete"
if err := db.Model(&entity.Document{}).Where("id = ?", "doc-1").Updates(map[string]interface{}{
"progress": 1.0,
- "run": run,
"progress_msg": progressMsg,
}).Error; err != nil {
t.Fatalf("prepare document: %v", err)
@@ -2133,7 +2109,6 @@ func TestUpdateDocumentUsesSharedRenamePath(t *testing.T) {
tokenNum := int64(10)
req := &UpdateDocumentRequest{
Name: &newName,
- Run: &run,
TokenNum: &tokenNum,
ChunkNum: &chunkNum,
Progress: &progress,
@@ -2158,7 +2133,7 @@ func TestUpdateDocumentUsesSharedRenamePath(t *testing.T) {
if file.Name != newName {
t.Fatalf("file name = %q, want %q", file.Name, newName)
}
- if doc.Progress != progress || doc.Run == nil || *doc.Run != run || doc.ProgressMsg == nil || *doc.ProgressMsg != progressMsg || doc.ChunkNum != chunkNum || doc.TokenNum != tokenNum {
+ if doc.Progress != progress || doc.ProgressMsg == nil || *doc.ProgressMsg != progressMsg || doc.ChunkNum != chunkNum || doc.TokenNum != tokenNum {
t.Fatalf("immutable fields changed: %#v", doc)
}
}
@@ -2243,8 +2218,8 @@ func TestUpdateDatasetDocumentParserIDResetsForReparse(t *testing.T) {
if err != nil {
t.Fatalf("UpdateDatasetDocument failed: code=%v err=%v", code, err)
}
- if resp.ParserID != chunkMethod || resp.Run != "UNSTART" || resp.TokenCount != 0 || resp.ChunkCount != 0 {
- t.Fatalf("response = %+v, want method=%s run=UNSTART counts=0", resp, chunkMethod)
+ if resp.ParserID != chunkMethod || resp.IngestionStatus != "UNSTART" || resp.TokenCount != 0 || resp.ChunkCount != 0 {
+ t.Fatalf("response = %+v, want method=%s ingestion_status=UNSTART counts=0", resp, chunkMethod)
}
doc, _ := dao.NewDocumentDAO().GetByID(ctx, db, "doc-1")
@@ -2295,7 +2270,7 @@ func TestClearDocumentParseResultsClearsCountersTasksAndChunks(t *testing.T) {
db := setupServiceTestDB(t)
pushServiceDB(t, db)
insertTestKB(t, "kb-1", "tenant-1", 1, 10, 5)
- insertTestDocWithRun(t, "doc-1", "kb-1", string(entity.TaskStatusDone), 10, 5)
+ insertTestDoc(t, "doc-1", "kb-1", 10, 5)
insertTestIngestionTaskWithStatus(t, "task-1", "user-1", "doc-1", "kb-1", common.COMPLETED)
ctx := t.Context()
@@ -2354,7 +2329,7 @@ func TestClearDocumentParseResultsIsIdempotentForStaleDocSnapshot(t *testing.T)
db := setupServiceTestDB(t)
pushServiceDB(t, db)
insertTestKB(t, "kb-1", "tenant-1", 1, 10, 5)
- insertTestDocWithRun(t, "doc-1", "kb-1", string(entity.TaskStatusDone), 10, 5)
+ insertTestDoc(t, "doc-1", "kb-1", 10, 5)
ctx := t.Context()
staleDoc, err := dao.NewDocumentDAO().GetByID(ctx, db, "doc-1")
@@ -2390,7 +2365,7 @@ func TestClearDocumentParseResults_RejectsNonTerminalIngestionTask(t *testing.T)
db := setupServiceTestDB(t)
pushServiceDB(t, db)
insertTestKB(t, "kb-1", "tenant-1", 1, 10, 5)
- insertTestDocWithRun(t, "doc-1", "kb-1", string(entity.TaskStatusRunning), 10, 5)
+ insertTestDoc(t, "doc-1", "kb-1", 10, 5)
insertTestIngestionTaskWithStatus(t, "task-1", "user-1", "doc-1", "kb-1", status)
ctx := t.Context()
doc, err := dao.NewDocumentDAO().GetByID(ctx, db, "doc-1")
@@ -2415,7 +2390,7 @@ func TestClearDocumentParseResultsDoesNotClearResultsWhenTaskStartsRunning(t *te
db := setupServiceTestDB(t)
pushServiceDB(t, db)
insertTestKB(t, "kb-1", "tenant-1", 1, 10, 5)
- insertTestDocWithRun(t, "doc-1", "kb-1", string(entity.TaskStatusDone), 10, 5)
+ insertTestDoc(t, "doc-1", "kb-1", 10, 5)
insertTestIngestionTaskWithStatus(t, "task-1", "user-1", "doc-1", "kb-1", common.SCHEDULED)
const callbackName = "test:start-ingestion-task-before-conditional-delete"
@@ -2472,7 +2447,7 @@ func TestClearDocumentParseResults_DeletesTerminalIngestionTask(t *testing.T) {
db := setupServiceTestDB(t)
pushServiceDB(t, db)
insertTestKB(t, "kb-1", "tenant-1", 1, 10, 5)
- insertTestDocWithRun(t, "doc-1", "kb-1", string(entity.TaskStatusDone), 10, 5)
+ insertTestDoc(t, "doc-1", "kb-1", 10, 5)
insertTestIngestionTaskWithStatus(t, "task-1", "user-1", "doc-1", "kb-1", status)
ctx := t.Context()
doc, err := dao.NewDocumentDAO().GetByID(ctx, db, "doc-1")
@@ -3107,7 +3082,6 @@ func insertNamedTestDoc(t *testing.T, id, kbID, name string, tokenNum, chunkNum
CreatedBy: "tenant-1",
Suffix: filepath.Ext(name),
Status: sptr("1"),
- Run: sptr(string(entity.TaskStatusDone)),
}
if err := dao.DB.Create(doc).Error; err != nil {
t.Fatalf("insert named test doc: %v", err)
@@ -3366,7 +3340,7 @@ func TestStartParseDocuments_FailsBeforeClearing(t *testing.T) {
db := setupServiceTestDB(t)
pushServiceDB(t, db)
insertTestKB(t, "kb-1", "tenant-1", 0, 10, 5)
- insertTestDocWithRun(t, "doc-1", "kb-1", string(entity.TaskStatusDone), 10, 5)
+ insertTestDoc(t, "doc-1", "kb-1", 10, 5)
insertTestIngestionTaskWithStatus(t, "task-1", "user-1", "doc-1", "kb-1", common.COMPLETED)
ctx := t.Context()
@@ -3428,7 +3402,7 @@ func TestIngest_CancelDoesNotDeleteIngestionTask(t *testing.T) {
}
// TestIngest_CancelUnstartedDocument mirrors the Python /documents/ingest
-// behavior: canceling a document whose parse never started (run=UNSTART, no
+// behavior: canceling a document whose parse never started (no
// ingestion task) must be rejected with code 102 and the Python message, not
// an internal "no ingestion task found" error.
func TestIngest_CancelUnstartedDocument(t *testing.T) {
@@ -3436,7 +3410,7 @@ func TestIngest_CancelUnstartedDocument(t *testing.T) {
pushServiceDB(t, db)
insertUserTenantForAccessCheck(t, "user-1", "tenant-1")
insertTestKB(t, "kb-1", "tenant-1", 0, 0, 0)
- insertTestDocWithRun(t, "doc-1", "kb-1", string(entity.TaskStatusUnstart), 10, 5)
+ insertTestDoc(t, "doc-1", "kb-1", 10, 5)
svc := testDocumentService(t)
ctx := t.Context()
@@ -3456,14 +3430,14 @@ func TestIngest_CancelUnstartedDocument(t *testing.T) {
}
// TestIngest_CancelCompletedDocument: canceling an already finished parse
-// (run=DONE, ingestion task COMPLETED) is rejected with the same Python
+// (ingestion task COMPLETED) is rejected with the same Python
// message as the un-started case.
func TestIngest_CancelCompletedDocument(t *testing.T) {
db := setupServiceTestDB(t)
pushServiceDB(t, db)
insertUserTenantForAccessCheck(t, "user-1", "tenant-1")
insertTestKB(t, "kb-1", "tenant-1", 1, 10, 5)
- insertTestDocWithRun(t, "doc-1", "kb-1", string(entity.TaskStatusDone), 10, 5)
+ insertTestDoc(t, "doc-1", "kb-1", 10, 5)
insertTestIngestionTaskWithStatus(t, "task-1", "user-1", "doc-1", "kb-1", common.COMPLETED)
svc := testDocumentService(t)
@@ -3483,7 +3457,7 @@ func TestIngest_CancelCompletedDocument(t *testing.T) {
}
}
-// TestIngest_CancelUnstartedWithInFlightTask: run=UNSTART but an ingestion
+// TestIngest_CancelUnstartedWithInFlightTask: an ingestion
// task was just enqueued (CREATED) — cancel is allowed, matching the Python
// has_unfinished_task branch.
func TestIngest_CancelUnstartedWithInFlightTask(t *testing.T) {
@@ -3491,7 +3465,7 @@ func TestIngest_CancelUnstartedWithInFlightTask(t *testing.T) {
pushServiceDB(t, db)
insertUserTenantForAccessCheck(t, "user-1", "tenant-1")
insertTestKB(t, "kb-1", "tenant-1", 0, 0, 0)
- insertTestDocWithRun(t, "doc-1", "kb-1", string(entity.TaskStatusUnstart), 10, 5)
+ insertTestDoc(t, "doc-1", "kb-1", 10, 5)
insertTestIngestionTask(t, "task-1", "user-1", "doc-1", "kb-1")
svc := testDocumentService(t)
@@ -3507,21 +3481,24 @@ func TestIngest_CancelUnstartedWithInFlightTask(t *testing.T) {
t.Fatalf("expected code %v, got %v", common.CodeSuccess, code)
}
- doc, _ := dao.NewDocumentDAO().GetByID(ctx, db, "doc-1")
- if doc == nil || doc.Run == nil || *doc.Run != string(entity.TaskStatusCancel) {
- t.Fatalf("expected doc run=CANCEL, got %v", doc.Run)
+ task, err := dao.NewIngestionTaskDAO().GetByID(ctx, db, "task-1")
+ if err != nil {
+ t.Fatalf("load task: %v", err)
+ }
+ if task.Status != common.STOPPED {
+ t.Fatalf("expected task status=%s, got %s", common.STOPPED, task.Status)
}
}
-// TestIngest_CancelAgainWithoutTask: re-canceling a document already in
-// CANCEL state is accepted even when its ingestion task is gone — Python
-// treats run=CANCEL as cancelable and cancel_all_task_of is a no-op.
-func TestIngest_CancelAgainWithoutTask(t *testing.T) {
+// TestIngest_CancelAgainWhenAlreadyStopped: re-canceling a document whose task is
+// already STOPPED is accepted as a no-op.
+func TestIngest_CancelAgainWhenAlreadyStopped(t *testing.T) {
db := setupServiceTestDB(t)
pushServiceDB(t, db)
insertUserTenantForAccessCheck(t, "user-1", "tenant-1")
insertTestKB(t, "kb-1", "tenant-1", 1, 10, 5)
- insertTestDocWithRun(t, "doc-1", "kb-1", string(entity.TaskStatusCancel), 10, 5)
+ insertTestDoc(t, "doc-1", "kb-1", 10, 5)
+ insertTestIngestionTaskWithStatus(t, "task-1", "user-1", "doc-1", "kb-1", common.STOPPED)
svc := testDocumentService(t)
ctx := t.Context()
@@ -3530,13 +3507,193 @@ func TestIngest_CancelAgainWithoutTask(t *testing.T) {
Run: string(entity.TaskStatusCancel),
})
if err != nil {
- t.Fatalf("Ingest(re-cancel) without task: %v", err)
+ t.Fatalf("Ingest(re-cancel) on stopped task: %v", err)
}
if code != common.CodeSuccess {
t.Fatalf("expected code %v, got %v", common.CodeSuccess, code)
}
}
+// TestIngest_CancelScheduledTask_TransitionsToStopped verifies that canceling a
+// SCHEDULED document task transitions the task immediately to STOPPED and resets
+// document progress to 0, ensuring the frontend never hangs in a transient state.
+func TestIngest_CancelScheduledTask_TransitionsToStopped(t *testing.T) {
+ db := setupServiceTestDB(t)
+ pushServiceDB(t, db)
+ insertUserTenantForAccessCheck(t, "user-1", "tenant-1")
+ insertTestKB(t, "kb-1", "tenant-1", 0, 0, 0)
+ insertTestDoc(t, "doc-1", "kb-1", 10, 5)
+ insertTestIngestionTaskWithStatus(t, "task-1", "user-1", "doc-1", "kb-1", common.SCHEDULED)
+
+ svc := testDocumentService(t)
+ ctx := t.Context()
+ code, err := svc.Ingest(ctx, "user-1", &IngestDocumentRequest{
+ DocIDs: []string{"doc-1"},
+ Run: string(entity.TaskStatusCancel),
+ })
+ if err != nil {
+ t.Fatalf("Ingest(cancel) on SCHEDULED task failed: %v", err)
+ }
+ if code != common.CodeSuccess {
+ t.Fatalf("expected code %v, got %v", common.CodeSuccess, code)
+ }
+
+ task, err := svc.ingestionTaskDAO.GetByDocumentID(ctx, db, "doc-1")
+ if err != nil {
+ t.Fatalf("load task: %v", err)
+ }
+ if task == nil || task.Status != common.STOPPED {
+ t.Fatalf("task status = %v, want %s", task, common.STOPPED)
+ }
+
+ doc, err := svc.documentDAO.GetByID(ctx, db, "doc-1")
+ if err != nil {
+ t.Fatalf("load doc: %v", err)
+ }
+ if doc.Progress != 0 {
+ t.Fatalf("doc.Progress = %v, want 0", doc.Progress)
+ }
+
+ // Verify that querying document reflects STOPPED status for frontend.
+ resp, err := svc.toResponse(ctx, doc)
+ if err != nil {
+ t.Fatalf("toResponse failed: %v", err)
+ }
+ if resp.IngestionStatus != common.STOPPED {
+ t.Fatalf("resp.IngestionStatus = %q, want %q", resp.IngestionStatus, common.STOPPED)
+ }
+}
+
+// TestIngest_CancelRunningTask_LifecycleToStopped verifies the complete cancel
+// lifecycle for a RUNNING task:
+// 1. Ingest(cancel) puts the task in STOPPING and resets progress to 0.
+// 2. A duplicate cancel call while STOPPING succeeds (idempotent guard).
+// 3. Once worker settles the cancellation via MarkStopped, the task reaches STOPPED.
+// 4. In STOPPED state, the document reflects ingestion_status=STOPPED and is
+// recognized as terminal, unblocking the frontend and allowing re-parse.
+func TestIngest_CancelRunningTask_LifecycleToStopped(t *testing.T) {
+ db := setupServiceTestDB(t)
+ pushServiceDB(t, db)
+ insertUserTenantForAccessCheck(t, "user-1", "tenant-1")
+ insertTestKB(t, "kb-1", "tenant-1", 0, 0, 0)
+ insertTestDoc(t, "doc-1", "kb-1", 10, 5)
+ if err := db.Model(&entity.Document{}).Where("id = ?", "doc-1").Update("progress", 0.6).Error; err != nil {
+ t.Fatalf("set initial progress: %v", err)
+ }
+ insertTestIngestionTaskWithStatus(t, "task-1", "user-1", "doc-1", "kb-1", common.RUNNING)
+
+ svc := testDocumentService(t)
+ ctx := t.Context()
+
+ // 1. First cancel call: transitions RUNNING -> STOPPING
+ code, err := svc.Ingest(ctx, "user-1", &IngestDocumentRequest{
+ DocIDs: []string{"doc-1"},
+ Run: string(entity.TaskStatusCancel),
+ })
+ if err != nil {
+ t.Fatalf("first Ingest(cancel): %v", err)
+ }
+ if code != common.CodeSuccess {
+ t.Fatalf("expected code %v, got %v", common.CodeSuccess, code)
+ }
+
+ task, err := svc.ingestionTaskDAO.GetByDocumentID(ctx, db, "doc-1")
+ if err != nil || task == nil {
+ t.Fatalf("load task after cancel: %v", err)
+ }
+ if task.Status != common.STOPPING {
+ t.Fatalf("task status = %q, want %q", task.Status, common.STOPPING)
+ }
+
+ doc, err := svc.documentDAO.GetByID(ctx, db, "doc-1")
+ if err != nil {
+ t.Fatalf("load doc: %v", err)
+ }
+ if doc.Progress != 0 {
+ t.Fatalf("doc.Progress = %v, want 0 after cancel", doc.Progress)
+ }
+
+ // 2. Duplicate cancel while in STOPPING must succeed as an idempotent no-op
+ code, err = svc.Ingest(ctx, "user-1", &IngestDocumentRequest{
+ DocIDs: []string{"doc-1"},
+ Run: string(entity.TaskStatusCancel),
+ })
+ if err != nil {
+ t.Fatalf("duplicate Ingest(cancel) while STOPPING: %v", err)
+ }
+ if code != common.CodeSuccess {
+ t.Fatalf("expected code %v, got %v", common.CodeSuccess, code)
+ }
+
+ // 3. Worker settlement: transitions STOPPING -> STOPPED
+ if err = svc.ingestionTaskSvc.MarkStopped(ctx, "task-1"); err != nil {
+ t.Fatalf("MarkStopped failed: %v", err)
+ }
+
+ task, err = svc.ingestionTaskDAO.GetByDocumentID(ctx, db, "doc-1")
+ if err != nil || task == nil {
+ t.Fatalf("load task after MarkStopped: %v", err)
+ }
+ if task.Status != common.STOPPED {
+ t.Fatalf("task status = %q, want %q", task.Status, common.STOPPED)
+ }
+
+ // 4. Verify document query returns STOPPED status for frontend consumption
+ resp, err := svc.toResponse(ctx, doc)
+ if err != nil {
+ t.Fatalf("toResponse failed: %v", err)
+ }
+ if resp.IngestionStatus != common.STOPPED {
+ t.Fatalf("resp.IngestionStatus = %q, want %q", resp.IngestionStatus, common.STOPPED)
+ }
+
+ // 5. AssertIngestionTasksTerminal succeeds on STOPPED (document unblocked)
+ if err = svc.AssertIngestionTasksTerminal(ctx, []string{"doc-1"}); err != nil {
+ t.Fatalf("AssertIngestionTasksTerminal should succeed on STOPPED: %v", err)
+ }
+}
+
+// TestIngest_DeleteOnlyCleansTasks verifies that when run is neither running nor
+// cancel, passing delete=true deletes prior tasks and resets progress to 0 without starting a parse.
+func TestIngest_DeleteOnlyCleansTasks(t *testing.T) {
+ db := setupServiceTestDB(t)
+ pushServiceDB(t, db)
+ insertUserTenantForAccessCheck(t, "user-1", "tenant-1")
+ insertTestKB(t, "kb-1", "tenant-1", 0, 0, 0)
+ insertTestDoc(t, "doc-1", "kb-1", 10, 5)
+ insertTestIngestionTaskWithStatus(t, "task-1", "user-1", "doc-1", "kb-1", common.COMPLETED)
+
+ svc := testDocumentService(t)
+ ctx := t.Context()
+ code, err := svc.Ingest(ctx, "user-1", &IngestDocumentRequest{
+ DocIDs: []string{"doc-1"},
+ Run: "0",
+ Delete: true,
+ })
+ if err != nil {
+ t.Fatalf("Ingest(delete-only): %v", err)
+ }
+ if code != common.CodeSuccess {
+ t.Fatalf("expected code %v, got %v", common.CodeSuccess, code)
+ }
+
+ task, err := svc.ingestionTaskDAO.GetByDocumentID(ctx, db, "doc-1")
+ if err != nil {
+ t.Fatalf("query task: %v", err)
+ }
+ if task != nil {
+ t.Fatal("expected task to be deleted in delete-only mode")
+ }
+
+ doc, err := dao.NewDocumentDAO().GetByID(ctx, db, "doc-1")
+ if err != nil {
+ t.Fatalf("query doc: %v", err)
+ }
+ if doc.Progress != 0 {
+ t.Fatalf("expected doc progress 0, got %v", doc.Progress)
+ }
+}
+
func TestUpdateRunProgressMirrorsFields(t *testing.T) {
db := setupServiceTestDB(t)
pushServiceDB(t, db)
@@ -3548,7 +3705,7 @@ func TestUpdateRunProgressMirrorsFields(t *testing.T) {
svc := testDocumentService(t)
ctx := t.Context()
- if err := svc.UpdateRunProgress(ctx, "doc-1", 0.5, "1", "halfway"); err != nil {
+ if err := svc.UpdateRunProgress(ctx, "doc-1", 0.5, "halfway"); err != nil {
t.Fatalf("UpdateRunProgress failed: %v", err)
}
doc, err := dao.NewDocumentDAO().GetByID(ctx, db, "doc-1")
@@ -3558,9 +3715,6 @@ func TestUpdateRunProgressMirrorsFields(t *testing.T) {
if doc.Progress != 0.5 {
t.Fatalf("progress = %v, want 0.5", doc.Progress)
}
- if doc.Run == nil || *doc.Run != "1" {
- t.Fatalf("run = %v, want 1", doc.Run)
- }
if doc.ProgressMsg == nil || *doc.ProgressMsg != "halfway" {
t.Fatalf("progress_msg = %v, want halfway", doc.ProgressMsg)
}
diff --git a/internal/service/document/document_upload.go b/internal/service/document/document_upload.go
index 341ee5c2fa..4d5a1495a4 100644
--- a/internal/service/document/document_upload.go
+++ b/internal/service/document/document_upload.go
@@ -322,7 +322,6 @@ func normalizeWebDocumentName(name, contentType string, blob []byte) string {
// suffix and content hash. blob may be nil for the empty/virtual document.
func (s *DocumentService) newDatasetDocument(kb *entity.Knowledgebase, tenantID, filename, location, filetype string, parserConfig entity.JSONMap, src string, size int64, blob []byte) *entity.Document {
docID := utility.GenerateToken()
- run := "0"
status := "1"
suffix := ""
if i := strings.LastIndex(filename, "."); i >= 0 {
@@ -346,7 +345,6 @@ func (s *DocumentService) newDatasetDocument(kb *entity.Knowledgebase, tenantID,
Location: &loc,
Size: size,
Suffix: suffix,
- Run: &run,
Status: &status,
}
if blob != nil {
@@ -360,18 +358,18 @@ func (s *DocumentService) newDatasetDocument(kb *entity.Knowledgebase, tenantID,
// handler remaps (chunk_num→chunk_count, kb_id→dataset_id).
func docToRawMap(doc *entity.Document) map[string]interface{} {
m := map[string]interface{}{
- "id": doc.ID,
- "kb_id": doc.KbID,
- "parser_id": doc.ParserID,
- "parser_config": map[string]interface{}(doc.ParserConfig),
- "created_by": doc.CreatedBy,
- "type": doc.Type,
- "source_type": doc.SourceType,
- "size": doc.Size,
- "chunk_num": doc.ChunkNum,
- "token_num": doc.TokenNum,
- "suffix": doc.Suffix,
- "run": "0",
+ "id": doc.ID,
+ "kb_id": doc.KbID,
+ "parser_id": doc.ParserID,
+ "parser_config": map[string]interface{}(doc.ParserConfig),
+ "created_by": doc.CreatedBy,
+ "type": doc.Type,
+ "source_type": doc.SourceType,
+ "size": doc.Size,
+ "chunk_num": doc.ChunkNum,
+ "token_num": doc.TokenNum,
+ "suffix": doc.Suffix,
+ "ingestion_status": "UNSTART",
}
if doc.Name != nil {
m["name"] = *doc.Name
diff --git a/internal/service/document/document_validation.go b/internal/service/document/document_validation.go
index 975437da67..3fccccb07a 100644
--- a/internal/service/document/document_validation.go
+++ b/internal/service/document/document_validation.go
@@ -30,7 +30,6 @@ type immutableDocumentFields struct {
tokenNum *int64
tokenNumRequestName string
progress *float64
- run *string
progressMsg *string
}
@@ -44,9 +43,6 @@ func validateImmutableDocumentFields(doc *entity.Document, fields immutableDocum
if fields.progress != nil && !documentProgressEqual(*fields.progress, doc.Progress) {
return errors.New("can't change `progress`")
}
- if fields.run != nil && !optionalStringEqual(fields.run, doc.Run) {
- return errors.New("can't change `run`")
- }
if fields.progressMsg != nil && !optionalStringEqual(fields.progressMsg, doc.ProgressMsg) {
return errors.New("can't change `progress_msg`")
}
diff --git a/internal/service/ingestion_task_service.go b/internal/service/ingestion_task_service.go
index 8395578ed0..5109e1443a 100644
--- a/internal/service/ingestion_task_service.go
+++ b/internal/service/ingestion_task_service.go
@@ -209,14 +209,10 @@ func (s *IngestionTaskService) StartRunning(ctx context.Context, taskID string)
if err != nil {
return nil, err
}
- // The task just started running: mirror it to the document so its
- // run status and progress counters reflect real processing, not
- // just API acceptance. Best-effort - a DB blip here must not fail
- // the task transition and trigger a redelivery loop. run uses the
- // document's numeric TaskStatus enum ("1"), not the task's string
- // status label.
+ // The task just started running: reset document progress counters to
+ // reflect real processing. Best-effort - a DB blip here must not fail
+ // the task transition and trigger a redelivery loop.
if err = s.documentDAO.UpdateByID(ctx, dao.DB, task.DocumentID, map[string]interface{}{
- "run": string(entity.TaskStatusRunning),
"progress": float64(0),
"chunk_num": int64(0),
"token_num": int64(0),
diff --git a/internal/service/ingestion_task_service_test.go b/internal/service/ingestion_task_service_test.go
index 0820046c16..27aca6aa7c 100644
--- a/internal/service/ingestion_task_service_test.go
+++ b/internal/service/ingestion_task_service_test.go
@@ -354,22 +354,20 @@ func TestIngestionTaskServiceStartRunningTransitionsCreatedTask(t *testing.T) {
}
}
-// TestStartRunningMarksDocumentRunning locks in that starting a CREATED task
-// mirrors the transition to its document: run=RUNNING and progress counters
-// reset, with a fresh process_begin_at. The document bookkeeping is owned by
-// the task-lifecycle transition, not the ingestion worker's execution path.
-func TestStartRunningMarksDocumentRunning(t *testing.T) {
+// TestStartRunningResetsDocumentProgress locks in that starting a CREATED task
+// resets its document progress counters, with a fresh process_begin_at. The
+// document bookkeeping is owned by the task-lifecycle transition, not the
+// ingestion worker's execution path.
+func TestStartRunningResetsDocumentProgress(t *testing.T) {
db := setupServiceTestDB(t)
pushServiceDB(t, db)
insertTestKB(t, "kb-1", "tenant-1", 1, 0, 0)
insertTestDoc(t, "doc-1", "kb-1", 100, 10)
insertTestIngestionTask(t, "task-1", "user-1", "doc-1", "kb-1")
- // Seed the document as a partially-processed, non-RUNNING state that the
- // start transition must clobber.
+ // Seed the document as a partially-processed state that the start transition must reset.
if err := db.Model(&entity.Document{}).Where("id = ?", "doc-1").
Updates(map[string]interface{}{
- "run": string(entity.TaskStatusDone),
"progress": float64(0.5),
"progress_msg": "partial",
}).Error; err != nil {
@@ -386,9 +384,6 @@ func TestStartRunningMarksDocumentRunning(t *testing.T) {
if err := db.Where("id = ?", "doc-1").First(&doc).Error; err != nil {
t.Fatalf("reload document: %v", err)
}
- if doc.Run == nil || *doc.Run != string(entity.TaskStatusRunning) {
- t.Fatalf("run = %v, want RUNNING(%q)", doc.Run, string(entity.TaskStatusRunning))
- }
if doc.Progress != 0 {
t.Fatalf("progress = %f, want 0", doc.Progress)
}
@@ -416,10 +411,8 @@ func TestStartRunningLeavesTerminalDocumentUntouched(t *testing.T) {
insertTestDoc(t, "doc-1", "kb-1", 100, 10)
insertTestIngestionTask(t, "task-1", "user-1", "doc-1", "kb-1")
- finishedRun := string(entity.TaskStatusDone)
if err := db.Model(&entity.Document{}).Where("id = ?", "doc-1").
Updates(map[string]interface{}{
- "run": finishedRun,
"progress": float64(1.0),
"progress_msg": "done",
}).Error; err != nil {
@@ -444,8 +437,8 @@ func TestStartRunningLeavesTerminalDocumentUntouched(t *testing.T) {
if err := db.Where("id = ?", "doc-1").First(&doc).Error; err != nil {
t.Fatalf("reload document: %v", err)
}
- if doc.Run == nil || *doc.Run != finishedRun {
- t.Fatalf("run = %v, want %q (terminal document must not be resurrected)", doc.Run, finishedRun)
+ if doc.Progress != 1.0 {
+ t.Fatalf("progress = %v, want 1.0", doc.Progress)
}
if doc.ChunkNum != 10 || doc.TokenNum != 100 {
t.Fatalf("counters changed: chunk_num=%d token_num=%d, want 10/100", doc.ChunkNum, doc.TokenNum)
@@ -492,6 +485,26 @@ func TestIngestionTaskServiceRequestStopTransitionsCreatedTaskToStopped(t *testi
}
}
+func TestIngestionTaskServiceRequestStopTransitionsScheduledTaskToStopped(t *testing.T) {
+ db := setupServiceTestDB(t)
+ pushServiceDB(t, db)
+ insertTestIngestionTask(t, "task-1", "user-1", "doc-1", "kb-1")
+ if err := db.Model(&entity.IngestionTask{}).Where("id = ?", "task-1").
+ Update("status", common.SCHEDULED).Error; err != nil {
+ t.Fatalf("set SCHEDULED: %v", err)
+ }
+ ctx := t.Context()
+
+ svc := NewIngestionTaskService()
+ task, err := svc.RequestStop(ctx, "task-1")
+ if err != nil {
+ t.Fatalf("RequestStop failed: %v", err)
+ }
+ if task.Status != common.STOPPED {
+ t.Fatalf("status = %q, want %q", task.Status, common.STOPPED)
+ }
+}
+
func TestIngestionTaskServiceMarkCompletedRejectsNonRunningTask(t *testing.T) {
db := setupServiceTestDB(t)
pushServiceDB(t, db)