RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
refactor(ingestion): decouple document parsing state from document.run in favor of ingestion_task (#19533)
jay77721d45c8aa148fb
diff --git a/internal/common/constants.go b/internal/common/constants.goindex 96a301ba39..bddc6a87e2 100644--- a/internal/common/constants.go+++ b/internal/common/constants.go@@ -39,6 +39,37 @@ const (STOPPING = "STOPPING")+// ActiveTaskStatuses contains all in-flight, non-terminal task statuses.+var ActiveTaskStatuses = []string{CREATED, SCHEDULED, RUNNING, STOPPING}++// IsActiveTaskStatus returns true if the status represents an active (non-terminal) task:+// CREATED, SCHEDULED, RUNNING, or STOPPING.+func IsActiveTaskStatus(status string) bool {+ switch status {+ case CREATED, SCHEDULED, RUNNING, STOPPING:+ return true+ default:+ return false+ }+}++// IsTerminalTaskStatus returns true if the status represents a terminal task state:+// COMPLETED, STOPPED, or FAILED.+func IsTerminalTaskStatus(status string) bool {+ switch status {+ case COMPLETED, STOPPED, FAILED:+ return true+ default:+ return false+ }+}++// IsRunningOrStopping returns true if a worker is actively executing or shutting down:+// RUNNING or STOPPING.+func IsRunningOrStopping(status string) bool {+ return status == RUNNING || status == STOPPING+}+// StatusDialogValid is the dialog.status value that gates public bot// access. Mirrors Python's StatusEnum.VALID.value at// api/common/constants.py (the string "1"). All chatbot/agentbotdiff --git a/internal/dao/document.go b/internal/dao/document.goindex fe0f45e040..f76d4858a1 100644--- a/internal/dao/document.go+++ b/internal/dao/document.go@@ -18,6 +18,8 @@ package daoimport ("context"+ "fmt"+ "ragflow/internal/common""ragflow/internal/entity""strings"@@ -131,6 +133,20 @@ func (dao *DocumentDAO) ListByKBID(ctx context.Context, db *gorm.DB, kbID, keywo})}+const latestIngestionTaskJoin = `LEFT JOIN ingestion_task ON ingestion_task.document_id = document.id+ AND NOT EXISTS (+ SELECT 1+ FROM ingestion_task newer_ingestion_task+ WHERE newer_ingestion_task.document_id = ingestion_task.document_id+ AND (+ COALESCE(newer_ingestion_task.create_time, 0) > COALESCE(ingestion_task.create_time, 0)+ OR (+ COALESCE(newer_ingestion_task.create_time, 0) = COALESCE(ingestion_task.create_time, 0)+ AND newer_ingestion_task.id > ingestion_task.id+ )+ )+ )`+// ListByKBIDWithOptions lists documents by knowledge base ID with filters.func (dao *DocumentDAO) ListByKBIDWithOptions(ctx context.Context, db *gorm.DB, opts DocumentListOptions) ([]*entity.DocumentListItem, int64, error) {var documents []*entity.DocumentListItem@@ -145,22 +161,17 @@ func (dao *DocumentDAO) ListByKBIDWithOptions(ctx context.Context, db *gorm.DB,Joins("JOIN file ON file.id = file2document.file_id").Joins("LEFT JOIN user_canvas ON document.pipeline_id = user_canvas.id").Joins("LEFT JOIN user ON document.created_by = user.id").- Joins(`LEFT JOIN ingestion_task ON ingestion_task.document_id = document.id- AND NOT EXISTS (- SELECT 1- FROM ingestion_task newer_ingestion_task- WHERE newer_ingestion_task.document_id = ingestion_task.document_id- AND (- COALESCE(newer_ingestion_task.create_time, 0) > COALESCE(ingestion_task.create_time, 0)- OR (- COALESCE(newer_ingestion_task.create_time, 0) = COALESCE(ingestion_task.create_time, 0)- AND newer_ingestion_task.id > ingestion_task.id- )- )- )`)+ Joins(latestIngestionTaskJoin)++ countQuery := db.WithContext(ctx).Table("document").+ Joins("JOIN file2document ON file2document.document_id = document.id").+ Joins("JOIN file ON file.id = file2document.file_id")+ if len(opts.RunStatuses) > 0 {+ countQuery = countQuery.Joins(latestIngestionTaskJoin)+ }listQuery = applyDocumentListFilters(listQuery, opts, true)- countQuery := applyDocumentListFilters(db.WithContext(ctx).Model(&entity.Document{}), opts, false)+ countQuery = applyDocumentListFilters(countQuery, opts, true)if err := countQuery.Count(&total).Error; err != nil {return nil, 0, err@@ -184,15 +195,16 @@ func (dao *DocumentDAO) ListByKBIDWithOptions(ctx context.Context, db *gorm.DB,// GetFilterByKBID returns aggregate filter counts for documents in a dataset.func (dao *DocumentDAO) GetFilterByKBID(ctx context.Context, db *gorm.DB, opts DocumentListOptions) (map[string]interface{}, int64, error) {var rows []struct {- ID string `gorm:"column:id"`- Run *string `gorm:"column:run"`- Suffix string `gorm:"column:suffix"`+ ID string `gorm:"column:id"`+ IngestionStatus *string `gorm:"column:ingestion_status"`+ Suffix string `gorm:"column:suffix"`}query := db.WithContext(ctx).Table("document").- Select("document.id, document.run, document.suffix").+ Select("document.id, ingestion_task.status as ingestion_status, document.suffix").Joins("JOIN file2document ON file2document.document_id = document.id").- Joins("JOIN file ON file.id = file2document.file_id")+ Joins("JOIN file ON file.id = file2document.file_id").+ Joins(latestIngestionTaskJoin)query = applyDocumentListFilters(query, opts, true)if err := query.Scan(&rows).Error; err != nil {@@ -200,20 +212,22 @@ func (dao *DocumentDAO) GetFilterByKBID(ctx context.Context, db *gorm.DB, opts D}suffixCounter := map[string]int64{}- runStatusCounter := map[string]int64{}+ statusCounter := map[string]int64{}for _, row := range rows {if row.Suffix != "" {suffixCounter[row.Suffix]++}- if row.Run != nil {- runStatusCounter[*row.Run]+++ status := "UNSTART"+ if row.IngestionStatus != nil && *row.IngestionStatus != "" {+ status = *row.IngestionStatus}+ statusCounter[status]++}return map[string]interface{}{- "suffix": suffixCounter,- "run_status": runStatusCounter,- "metadata": map[string]interface{}{},+ "suffix": suffixCounter,+ "ingestion_status": statusCounter,+ "metadata": map[string]interface{}{},}, int64(len(rows)), nil}@@ -224,6 +238,9 @@ func (dao *DocumentDAO) ListIDsByKBIDWithOptions(ctx context.Context, db *gorm.DSelect("document.id").Joins("JOIN file2document ON file2document.document_id = document.id").Joins("JOIN file ON file.id = file2document.file_id")+ if len(opts.RunStatuses) > 0 {+ query = query.Joins(latestIngestionTaskJoin)+ }query = applyDocumentListFilters(query, opts, true)if err := query.Scan(&ids).Error; err != nil {return nil, err@@ -244,7 +261,22 @@ func applyDocumentListFilters(query *gorm.DB, opts DocumentListOptions, qualifiequery = query.Where("LOWER("+column("name")+") LIKE ?", "%"+strings.ToLower(strings.TrimSpace(opts.Keywords))+"%")}if len(opts.RunStatuses) > 0 {- query = query.Where(column("run")+" IN ?", opts.RunStatuses)+ hasUnstart := false+ otherStatuses := make([]string, 0, len(opts.RunStatuses))+ for _, s := range opts.RunStatuses {+ if strings.EqualFold(s, "UNSTART") {+ hasUnstart = true+ } else {+ otherStatuses = append(otherStatuses, s)+ }+ }+ if hasUnstart && len(otherStatuses) > 0 {+ query = query.Where("(ingestion_task.status IN ? OR ingestion_task.id IS NULL OR ingestion_task.status = ?)", otherStatuses, "UNSTART")+ } else if hasUnstart {+ query = query.Where("(ingestion_task.id IS NULL OR ingestion_task.status = ?)", "UNSTART")+ } else {+ query = query.Where("ingestion_task.status IN ?", otherStatuses)+ }}if len(opts.Types) > 0 {query = query.Where(column("type")+" IN ?", opts.Types)@@ -278,8 +310,8 @@ func documentListOrderColumn(orderBy string) string {return "document.size"case "type":return "document.type"- case "run":- return "document.run"+ case "run", "ingestion_status":+ return "COALESCE(ingestion_task.status, 'UNSTART')"default:return "document.create_time"}@@ -454,31 +486,37 @@ func (dao *DocumentDAO) GetParsingStatusByKBID(ctx context.Context, db *gorm.DB,}var rows []struct {- Run *string `gorm:"column:run"`- Cnt int64 `gorm:"column:cnt"`+ Status *string `gorm:"column:status"`+ Cnt int64 `gorm:"column:cnt"`}- err := db.WithContext(ctx).Model(&entity.Document{}).- Select("run, COUNT(id) as cnt").- Where("kb_id = ?", kbID).- Group("run").+ err := db.WithContext(ctx).Table("document").+ Select("ingestion_task.status, COUNT(document.id) as cnt").+ Joins(latestIngestionTaskJoin).+ Where("document.kb_id = ?", kbID).+ Group("ingestion_task.status").Scan(&rows).Errorif err != nil {return nil, err}- statusFieldMap := map[string]string{- string(entity.TaskStatusUnstart): "unstart_count",- string(entity.TaskStatusRunning): "running_count",- string(entity.TaskStatusCancel): "cancel_count",- string(entity.TaskStatusDone): "done_count",- string(entity.TaskStatusFail): "fail_count",- }for _, row := range rows {- if row.Run == nil {+ if row.Status == nil || *row.Status == "" {+ result["unstart_count"] += row.Cntcontinue}- if field, ok := statusFieldMap[*row.Run]; ok {- result[field] = row.Cnt+ switch *row.Status {+ case common.CREATED, common.SCHEDULED:+ result["unstart_count"] += row.Cnt+ case common.RUNNING, common.STOPPING:+ result["running_count"] += row.Cnt+ case common.STOPPED:+ result["cancel_count"] += row.Cnt+ case common.COMPLETED:+ result["done_count"] += row.Cnt+ case common.FAILED:+ result["fail_count"] += row.Cnt+ default:+ common.Warn(fmt.Sprintf("GetParsingStatusByKBID: unrecognized task status %q for dataset %s (count: %d)", *row.Status, kbID, row.Cnt))}}return result, nildiff --git a/internal/dao/document_test.go b/internal/dao/document_test.goindex eec3f30a58..141ab6c01f 100644--- a/internal/dao/document_test.go+++ b/internal/dao/document_test.go@@ -23,6 +23,7 @@ import ("github.com/glebarez/sqlite""gorm.io/gorm"+ "ragflow/internal/common""ragflow/internal/entity")@@ -389,4 +390,65 @@ func TestDocumentGetChunkingConfigScansParserConfig(t *testing.T) {}}+func TestDocumentDAOGetParsingStatusByKBID(t *testing.T) {+ db := setupDocumentTestDB(t)+ if err := db.AutoMigrate(&entity.IngestionTask{}); err != nil {+ t.Fatalf("migrate IngestionTask: %v", err)+ }++ // doc-unstart: no task+ if err := db.Create(&entity.Document{ID: "doc-1", KbID: "kb-status", ParserConfig: entity.JSONMap{}}).Error; err != nil {+ t.Fatalf("create doc-1: %v", err)+ }+ // doc-running: task RUNNING+ if err := db.Create(&entity.Document{ID: "doc-2", KbID: "kb-status", ParserConfig: entity.JSONMap{}}).Error; err != nil {+ t.Fatalf("create doc-2: %v", err)+ }+ if err := db.Create(&entity.IngestionTask{ID: "task-2", DocumentID: "doc-2", Status: common.RUNNING}).Error; err != nil {+ t.Fatalf("create task-2: %v", err)+ }+ // doc-completed: task COMPLETED+ if err := db.Create(&entity.Document{ID: "doc-3", KbID: "kb-status", ParserConfig: entity.JSONMap{}}).Error; err != nil {+ t.Fatalf("create doc-3: %v", err)+ }+ if err := db.Create(&entity.IngestionTask{ID: "task-3", DocumentID: "doc-3", Status: common.COMPLETED}).Error; err != nil {+ t.Fatalf("create task-3: %v", err)+ }+ // doc-failed: task FAILED+ if err := db.Create(&entity.Document{ID: "doc-4", KbID: "kb-status", ParserConfig: entity.JSONMap{}}).Error; err != nil {+ t.Fatalf("create doc-4: %v", err)+ }+ if err := db.Create(&entity.IngestionTask{ID: "task-4", DocumentID: "doc-4", Status: common.FAILED}).Error; err != nil {+ t.Fatalf("create task-4: %v", err)+ }+ // doc-stopped: task STOPPED+ if err := db.Create(&entity.Document{ID: "doc-5", KbID: "kb-status", ParserConfig: entity.JSONMap{}}).Error; err != nil {+ t.Fatalf("create doc-5: %v", err)+ }+ if err := db.Create(&entity.IngestionTask{ID: "task-5", DocumentID: "doc-5", Status: common.STOPPED}).Error; err != nil {+ t.Fatalf("create task-5: %v", err)+ }++ dao := NewDocumentDAO()+ counts, err := dao.GetParsingStatusByKBID(t.Context(), db, "kb-status")+ if err != nil {+ t.Fatalf("GetParsingStatusByKBID failed: %v", err)+ }+ if counts["unstart_count"] != 1 {+ t.Errorf("unstart_count = %d, want 1", counts["unstart_count"])+ }+ if counts["running_count"] != 1 {+ t.Errorf("running_count = %d, want 1", counts["running_count"])+ }+ if counts["done_count"] != 1 {+ t.Errorf("done_count = %d, want 1", counts["done_count"])+ }+ if counts["fail_count"] != 1 {+ t.Errorf("fail_count = %d, want 1", counts["fail_count"])+ }+ if counts["cancel_count"] != 1 {+ t.Errorf("cancel_count = %d, want 1", counts["cancel_count"])+ }+}+func sp(s string) *string { return &s }diff --git a/internal/dao/ingestion.go b/internal/dao/ingestion.goindex fe6448036e..6ac2cef08b 100644--- a/internal/dao/ingestion.go+++ b/internal/dao/ingestion.go@@ -216,6 +216,29 @@ func (dao *IngestionTaskDAO) GetByDocumentID(ctx context.Context, db *gorm.DB, dreturn tasks[0], nil}+// GetLatestByDocumentIDs returns a map of documentID -> latest IngestionTask.+func (dao *IngestionTaskDAO) GetLatestByDocumentIDs(ctx context.Context, db *gorm.DB, documentIDs []string) (map[string]*entity.IngestionTask, error) {+ if len(documentIDs) == 0 {+ return map[string]*entity.IngestionTask{}, nil+ }+ var tasks []*entity.IngestionTask+ err := db.WithContext(ctx).+ Where("document_id IN ?", documentIDs).+ Order("COALESCE(create_time, 0) DESC").+ Order("id DESC").+ Find(&tasks).Error+ if err != nil {+ return nil, err+ }+ result := make(map[string]*entity.IngestionTask, len(documentIDs))+ for _, task := range tasks {+ if _, exists := result[task.DocumentID]; !exists {+ result[task.DocumentID] = task+ }+ }+ return result, nil+}+// CountActiveByDatasetID returns the number of ingestion tasks for the// dataset whose latest task is non-terminal (CREATED/SCHEDULED/RUNNING/STOPPING).// It uses the same create_time/ID ordering as document-list state so historical@@ -236,7 +259,7 @@ func (dao *IngestionTaskDAO) CountActiveByDatasetID(ctx context.Context, db *gorAND newer_ingestion_task.id > ingestion_task.id))- )`, datasetID, []string{common.CREATED, common.SCHEDULED, common.RUNNING, common.STOPPING}).+ )`, datasetID, common.ActiveTaskStatuses).Count(&count).Errorreturn count, err}diff --git a/internal/entity/dataset.go b/internal/entity/dataset.goindex 37ed80226e..09d9a3d7f6 100644--- a/internal/entity/dataset.go+++ b/internal/entity/dataset.go@@ -73,19 +73,6 @@ const (TaskStatusSchedule TaskStatus = "5")-// DocumentModifiableStatuses are the run statuses in which a document's-// configuration (parser config, chunk method, pipeline id, name, enabled-// flag, ...) may be edited via UpdateDatasetDocument. Documents that are-// running ("1") or scheduled ("5") must not be edited — the in-flight parser-// would race with the config change and the document can get stuck and become-// undeletable.-var DocumentModifiableStatuses = map[TaskStatus]bool{- TaskStatusUnstart: true,- TaskStatusCancel: true,- TaskStatusDone: true,- TaskStatusFail: true,-}-// PipelineTaskType represents the type of pipeline tasktype PipelineTaskType stringdiff --git a/internal/entity/document.go b/internal/entity/document.goindex 37b16d41b8..6f202283fa 100644--- a/internal/entity/document.go+++ b/internal/entity/document.go@@ -40,7 +40,6 @@ type Document struct {ProcessDuration float64 `gorm:"column:process_duration;default:0" json:"process_duration"`ContentHash *string `gorm:"column:content_hash;size:32;index;default:''" json:"content_hash,omitempty"`Suffix string `gorm:"column:suffix;size:32;not null;index" json:"suffix"`- Run *string `gorm:"column:run;size:1;index;default:'0'" json:"run,omitempty"`Status *string `gorm:"column:status;size:1;index;default:'1'" json:"status,omitempty"`BaseModel}@@ -69,7 +68,6 @@ type DocumentListItem struct {ProcessDuration float64 `gorm:"column:process_duration" json:"process_duration"`ContentHash *string `gorm:"column:content_hash" json:"content_hash,omitempty"`Suffix string `gorm:"column:suffix" json:"suffix"`- Run *string `gorm:"column:run" json:"run,omitempty"`IngestionStatus *string `gorm:"column:ingestion_status" json:"ingestion_status,omitempty"`Status *string `gorm:"column:status" json:"status,omitempty"`CreateTime *int64 `gorm:"column:create_time" json:"create_time,omitempty"`diff --git a/internal/handler/document.go b/internal/handler/document.goindex cc2e7bda17..a0afc4b833 100644--- a/internal/handler/document.go+++ b/internal/handler/document.go@@ -658,9 +658,9 @@ func parseDocumentListOptions(c *gin.Context, datasetID string) (dao.DocumentLisTypes: queryValues(c, "types"),}- opts.RunStatuses = normalizeRunStatusFilter(queryValues(c, "run", "run_status"))- if len(queryValues(c, "run", "run_status")) > 0 && len(opts.RunStatuses) == 0 {- return opts, fmt.Sprintf("Invalid filter run status conditions: %s", strings.Join(invalidRunStatuses(queryValues(c, "run", "run_status")), ", "))+ opts.RunStatuses = normalizeRunStatusFilter(queryValues(c, "run", "run_status", "ingestion_status"))+ if len(queryValues(c, "run", "run_status", "ingestion_status")) > 0 && len(opts.RunStatuses) == 0 {+ return opts, fmt.Sprintf("Invalid filter run status conditions: %s", strings.Join(invalidRunStatuses(queryValues(c, "run", "run_status", "ingestion_status")), ", "))}opts.Name = c.Query("name")@@ -880,31 +880,25 @@ func queryValues(c *gin.Context, names ...string) []string {return out}+var runStatusMap = map[string]string{+ "UNSTART": "UNSTART",+ "CREATED": common.CREATED,+ "SCHEDULED": common.SCHEDULED,+ "RUNNING": common.RUNNING,+ "STOPPING": common.STOPPING,+ "STOPPED": common.STOPPED,+ "COMPLETED": common.COMPLETED,+ "FAILED": common.FAILED,+}+func normalizeRunStatusFilter(statuses []string) []string {if len(statuses) == 0 {return nil}- statusTextToNumeric := map[string]string{- "UNSTART": string(entity.TaskStatusUnstart),- "RUNNING": string(entity.TaskStatusRunning),- "CANCEL": string(entity.TaskStatusCancel),- "DONE": string(entity.TaskStatusDone),- "FAIL": string(entity.TaskStatusFail),- }- validStatuses := map[string]bool{- string(entity.TaskStatusUnstart): true,- string(entity.TaskStatusRunning): true,- string(entity.TaskStatusCancel): true,- string(entity.TaskStatusDone): true,- string(entity.TaskStatusFail): true,- }out := make([]string, 0, len(statuses))for _, status := range statuses {- normalized := statusTextToNumeric[strings.ToUpper(status)]- if normalized == "" {- normalized = status- }- if !validStatuses[normalized] {+ normalized, ok := runStatusMap[strings.ToUpper(strings.TrimSpace(status))]+ if !ok {return nil}out = append(out, normalized)@@ -915,10 +909,9 @@ func normalizeRunStatusFilter(statuses []string) []string {// invalidRunStatuses returns the raw filter values that do not map to a valid// run status, mirroring Python's "Invalid filter run status conditions: ...".func invalidRunStatuses(statuses []string) []string {- valid := map[string]bool{"UNSTART": true, "RUNNING": true, "CANCEL": true, "DONE": true, "FAIL": true}invalid := make([]string, 0)for _, status := range statuses {- if !valid[strings.ToUpper(status)] {+ if _, ok := runStatusMap[strings.ToUpper(strings.TrimSpace(status))]; !ok {invalid = append(invalid, status)}}@@ -1085,15 +1078,19 @@ func (h *DocumentHandler) uploadWebDocument(c *gin.Context, kb *entity.Knowledge// mapDocKeysWithRunStatus renames a freshly-created document's raw keys to the// public response shape (chunk_num→chunk_count, token_num→token_count,-// kb_id→dataset_id) and reports run as a label.+// kb_id→dataset_id) and sets ingestion_status to UNSTART.// Mirrors Python map_doc_keys_with_run_status / map_doc_keys.func mapDocKeysWithRunStatus(raw map[string]interface{}) map[string]interface{} {+ ingestionStatus := "UNSTART"+ if s, ok := raw["ingestion_status"].(string); ok && s != "" {+ ingestionStatus = s+ }out := map[string]interface{}{- "chunk_count": raw["chunk_num"],- "token_count": raw["token_num"],- "dataset_id": raw["kb_id"],- "parser_id": raw["parser_id"],- "run": "UNSTART",+ "chunk_count": raw["chunk_num"],+ "token_count": raw["token_num"],+ "dataset_id": raw["kb_id"],+ "parser_id": raw["parser_id"],+ "ingestion_status": ingestionStatus,}for _, k := range []string{"id", "name", "type", "size", "suffix", "source_type", "created_by", "parser_config", "location", "pipeline_id", "content_hash"} {if v, ok := raw[k]; ok {@@ -1157,12 +1154,16 @@ func (h *DocumentHandler) DownloadDocument(c *gin.Context) {func mapDocumentListItem(doc *entity.DocumentListItem, metaFields map[string]interface{}) map[string]interface{} {processDuration := doc.ProcessDuration- if doc.Run != nil && strings.TrimSpace(*doc.Run) == "1" && doc.ProcessBeginAt != nil {+ if doc.IngestionStatus != nil && *doc.IngestionStatus == common.RUNNING && doc.ProcessBeginAt != nil {processDuration = time.Since(*doc.ProcessBeginAt).Seconds()if processDuration < 0 {processDuration = 0}}+ ingestionStatus := "UNSTART"+ if doc.IngestionStatus != nil && *doc.IngestionStatus != "" {+ ingestionStatus = *doc.IngestionStatus+ }item := map[string]interface{}{"id": doc.ID,"dataset_id": doc.KbID,@@ -1179,7 +1180,7 @@ func mapDocumentListItem(doc *entity.DocumentListItem, metaFields map[string]int"process_begin_at": formatTimePtr(doc.ProcessBeginAt),"process_duration": processDuration,"suffix": doc.Suffix,- "run": mapRunStatus(doc.Run),+ "ingestion_status": ingestionStatus,"status": stringValue(doc.Status),"parser_id": doc.ParserID,"chunk_method": doc.ParserID,@@ -1193,9 +1194,6 @@ func mapDocumentListItem(doc *entity.DocumentListItem, metaFields map[string]int"update_time": int64(0),"update_date": "",}- if doc.IngestionStatus != nil {- item["ingestion_status"] = *doc.IngestionStatus- }if doc.CreateTime != nil {item["create_time"] = *doc.CreateTime@@ -1226,27 +1224,6 @@ func decodeJSONMap(raw string) map[string]interface{} {return data}-func mapRunStatus(run *string) string {- if run == nil {- return "UNSTART"- }-- switch strings.TrimSpace(*run) {- case "0":- return "UNSTART"- case "1":- return "RUNNING"- case "2":- return "CANCEL"- case "3":- return "DONE"- case "4":- return "FAIL"- default:- return strings.TrimSpace(*run)- }-}-func formatTimePtr(value *time.Time) string {if value == nil {return ""diff --git a/internal/handler/document_test.go b/internal/handler/document_test.goindex 846814e92e..318395ee88 100644--- a/internal/handler/document_test.go+++ b/internal/handler/document_test.go@@ -1041,6 +1041,40 @@ func TestDocumentHandlerIngestPropagatesServiceErrorCode(t *testing.T) {}}+func TestDocumentHandlerIngest_CancelDispatchesToService(t *testing.T) {+ gin.SetMode(gin.TestMode)++ fake := &fakeDocumentService{}+ h := &DocumentHandler{+ documentService: fake,+ datasetService: dataset.NewDatasetService(),+ }++ c, w := setupGinContextWithUser("POST", "/api/v1/documents/ingest", `{"doc_ids":["doc-1"],"run":2}`)+ h.Ingest(c)++ if w.Code != http.StatusOK {+ t.Fatalf("expected 200, got %d: %s", w.Code, w.Body.String())+ }++ var resp map[string]interface{}+ if err := json.Unmarshal(w.Body.Bytes(), &resp); err != nil {+ t.Fatal(err)+ }+ if resp["code"] != float64(common.CodeSuccess) {+ t.Fatalf("expected code 0, got %v", resp["code"])+ }+ if resp["data"] != true {+ t.Fatalf("expected data true, got %#v", resp["data"])+ }+ if fake.ingestReq == nil || len(fake.ingestReq.DocIDs) != 1 || fake.ingestReq.DocIDs[0] != "doc-1" {+ t.Fatalf("unexpected ingestReq: %#v", fake.ingestReq)+ }+ if fmt.Sprint(fake.ingestReq.Run) != "2" {+ t.Fatalf("run = %v, want 2", fake.ingestReq.Run)+ }+}+func TestStopParseDocumentsHandler_EmptyDocIDs(t *testing.T) {gin.SetMode(gin.TestMode)@@ -1152,7 +1186,7 @@ func TestListDocumentsHandler_FilterRequestUsesQueryFilters(t *testing.T) {datasetService: dataset.NewDatasetService(),}- c, w := setupGinContextWithUser("GET", "/api/v1/datasets/ds-1/documents?type=filter&keywords=report&suffix=pdf&run=DONE&types=doc&desc=false", "")+ c, w := setupGinContextWithUser("GET", "/api/v1/datasets/ds-1/documents?type=filter&keywords=report&suffix=pdf&run=COMPLETED&types=doc&desc=false", "")c.Params = gin.Params{{Key: "dataset_id", Value: "ds-1"}}h.ListDocuments(c)@@ -1169,8 +1203,8 @@ func TestListDocumentsHandler_FilterRequestUsesQueryFilters(t *testing.T) {if len(fake.filterOpts.Suffixes) != 1 || fake.filterOpts.Suffixes[0] != "pdf" {t.Fatalf("expected suffix pdf, got %#v", fake.filterOpts.Suffixes)}- if len(fake.filterOpts.RunStatuses) != 1 || fake.filterOpts.RunStatuses[0] != string(entity.TaskStatusDone) {- t.Fatalf("expected run DONE to map to %q, got %#v", string(entity.TaskStatusDone), fake.filterOpts.RunStatuses)+ if len(fake.filterOpts.RunStatuses) != 1 || fake.filterOpts.RunStatuses[0] != common.COMPLETED {+ t.Fatalf("expected run COMPLETED to map to %q, got %#v", common.COMPLETED, fake.filterOpts.RunStatuses)}if len(fake.filterOpts.Types) != 1 || fake.filterOpts.Types[0] != "doc" {t.Fatalf("expected type doc, got %#v", fake.filterOpts.Types)@@ -1189,6 +1223,43 @@ func TestListDocumentsHandler_FilterRequestUsesQueryFilters(t *testing.T) {}}+func TestListDocumentsRejectsInvalidRunFilter(t *testing.T) {+ gin.SetMode(gin.TestMode)++ db := setupHandlerAccessDB(t)+ orig := dao.DB+ dao.DB = db+ t.Cleanup(func() { dao.DB = orig })++ fake := &fakeDocumentService{}+ h := &DocumentHandler{+ documentService: fake,+ datasetService: dataset.NewDatasetService(),+ }++ for _, invalid := range []string{"1", "DONE", "CANCEL", "FAIL", "SCHEDULE"} {+ c, w := setupGinContextWithUser("GET", "/api/v1/datasets/ds-1/documents?run="+invalid, "user-1")+ c.Params = gin.Params{{Key: "dataset_id", Value: "ds-1"}}++ h.ListDocuments(c)++ if w.Code != http.StatusOK {+ t.Fatalf("expected 200 wrapper, got %d: %s", w.Code, w.Body.String())+ }+ var resp map[string]interface{}+ if err := json.Unmarshal(w.Body.Bytes(), &resp); err != nil {+ t.Fatalf("invalid json response: %v", err)+ }+ if int(resp["code"].(float64)) != int(common.CodeDataError) {+ t.Fatalf("expected code %d for %s, got %v", common.CodeDataError, invalid, resp["code"])+ }+ msg, _ := resp["message"].(string)+ if !strings.Contains(msg, "Invalid filter run status conditions: "+invalid) {+ t.Fatalf("expected error message for %s to contain 'Invalid filter run status conditions: %s', got %q", invalid, invalid, msg)+ }+ }+}+func TestListDocumentsHandlerReturnsScheduledIngestionStatus(t *testing.T) {db := setupHandlerAccessDB(t)orig := dao.DB@@ -1238,7 +1309,7 @@ func TestListDocumentsHandlerReturnsScheduledIngestionStatus(t *testing.T) {}}-func TestListDocumentsHandlerOmitsEmptyIngestionStatus(t *testing.T) {+func TestListDocumentsHandlerReturnsUnstartWhenNoIngestionTask(t *testing.T) {db := setupHandlerAccessDB(t)orig := dao.DBdao.DB = db@@ -1281,8 +1352,11 @@ func TestListDocumentsHandlerOmitsEmptyIngestionStatus(t *testing.T) {if len(response.Data.Docs) != 1 {t.Fatalf("document count = %d, want 1", len(response.Data.Docs))}- if _, exists := response.Data.Docs[0]["ingestion_status"]; exists {- t.Fatalf("ingestion_status should be omitted when no ingestion task exists")+ if got := response.Data.Docs[0]["ingestion_status"]; got != "UNSTART" {+ t.Fatalf("ingestion_status = %v, want UNSTART", got)+ }+ if _, exists := response.Data.Docs[0]["run"]; exists {+ t.Fatalf("run field should not exist in response")}}@@ -1438,7 +1512,7 @@ func TestListDocumentsHandler_MetadataFilterNarrowsDocumentIDs(t *testing.T) {listIDs: []string{"doc-1", "doc-2", "doc-3"},metadataByKBs: map[string]interface{}{"author": map[string][]string{- "Alice": []string{"doc-2", "doc-4"},+ "Alice": {"doc-2", "doc-4"},},},}diff --git a/internal/ingestion/service/execute_task_ack_test.go b/internal/ingestion/service/execute_task_ack_test.goindex aef6513249..9e0372908d 100644--- a/internal/ingestion/service/execute_task_ack_test.go+++ b/internal/ingestion/service/execute_task_ack_test.go@@ -156,7 +156,7 @@ func TestExecuteTask_AcksMessageOnFailure(t *testing.T) {// TestExecuteTask_AcksMessageOnContextCancel: a task with a cancelled context// (e.g. Redis cancel flag or Ingestor shutdown) is now terminal — the cancel-// is durably recorded (progress=-1, run=CANCEL) and the message is Acked to+// is durably recorded (progress=-1, STOPPED) and the message is Acked to// prevent indefinite redeliveries of an already-cancelled task.func TestExecuteTask_AcksMessageOnContextCancel(t *testing.T) {db := testutil.SetupTestDB(t)diff --git a/internal/ingestion/service/execute_task_test.go b/internal/ingestion/service/execute_task_test.goindex b283a02e76..23bf2e18fe 100644--- a/internal/ingestion/service/execute_task_test.go+++ b/internal/ingestion/service/execute_task_test.go@@ -239,9 +239,6 @@ func TestExecuteTask_CancelBeforePipeline(t *testing.T) {if doc.Progress != -1 {t.Fatalf("document.progress = %v, want -1 (cancelled)", doc.Progress)}- if doc.Run == nil || *doc.Run != string(entity.TaskStatusCancel) {- t.Fatalf("document.run = %v, want %s (CANCEL)", doc.Run, entity.TaskStatusCancel)- }if doc.ProgressMsg == nil || *doc.ProgressMsg == "" {t.Fatal("document.progress_msg should contain cancel marker, got empty")}diff --git a/internal/ingestion/service/ingestion_service.go b/internal/ingestion/service/ingestion_service.goindex f82719fe94..4547454a4c 100644--- a/internal/ingestion/service/ingestion_service.go+++ b/internal/ingestion/service/ingestion_service.go@@ -1050,8 +1050,7 @@ func (e *Ingestor) pollCancel(taskID string, cancel context.CancelFunc, done <-c}// markCancelProgress writes the cancelled-progress markers to the document-// row. Mirrors Python's cancel_all_task_of: progress=-1, run=CANCEL, and an-// appended timestamped cancel message (progress_msg += cancelMsg).+// row: progress=-1 and an appended timestamped cancel message (progress_msg += cancelMsg).func (e *Ingestor) markCancelProgress(task *entity.IngestionTask) {svc := documentpkg.NewDocumentService()doc, err := svc.GetDocumentByID(e.ctx, task.DocumentID)@@ -1064,7 +1063,7 @@ func (e *Ingestor) markCancelProgress(task *entity.IngestionTask) {if doc.ProgressMsg != nil {existingMsg = *doc.ProgressMsg}- _ = svc.UpdateRunProgress(e.ctx, task.DocumentID, -1.0, string(entity.TaskStatusCancel), existingMsg+cancelMsg)+ _ = svc.UpdateRunProgress(e.ctx, task.DocumentID, -1.0, existingMsg+cancelMsg)}// markTimeoutProgress writes the timeout-progress markers to the document@@ -1082,7 +1081,7 @@ func (e *Ingestor) markTimeoutProgress(task *entity.IngestionTask) {if doc.ProgressMsg != nil {existingMsg = *doc.ProgressMsg}- _ = svc.UpdateRunProgress(e.ctx, task.DocumentID, -1.0, string(entity.TaskStatusFail), existingMsg+timeoutMsg)+ _ = svc.UpdateRunProgress(e.ctx, task.DocumentID, -1.0, existingMsg+timeoutMsg)}// claimTask registers a worker claim on a task ID. Returns false if anotherdiff --git a/internal/ingestion/service/progress_sink.go b/internal/ingestion/service/progress_sink.goindex c7f5a826ec..41846787df 100644--- a/internal/ingestion/service/progress_sink.go+++ b/internal/ingestion/service/progress_sink.go@@ -27,7 +27,6 @@ import ("ragflow/internal/common""ragflow/internal/dao"- "ragflow/internal/entity""ragflow/internal/ingestion/pipeline"servicepkg "ragflow/internal/service"documentpkg "ragflow/internal/service/document"@@ -68,8 +67,8 @@ type progressSink struct {// full DocumentService surface.type docProgressSvc interface {GetDocumentByID(ctx context.Context, docID string) (*documentpkg.DocumentResponse, error)- UpdateRunState(ctx context.Context, docID string, progress float64, run string) error- UpdateRunProgress(ctx context.Context, docID string, progress float64, run, progressMsg string) error+ UpdateRunState(ctx context.Context, docID string, progress float64) error+ UpdateRunProgress(ctx context.Context, docID string, progress float64, progressMsg string) error}func newProgressSink(ctx context.Context, taskSvc *servicepkg.IngestionTaskService) *progressSink {@@ -108,8 +107,8 @@ func (s *progressSink) OnComponentProgress(ctx context.Context, ev pipeline.Progif agg == nil || total <= 0 {return}- progress, run := deriveDocumentProgress(agg, int(total))- if err = s.updateDocumentProgress(ctx, ev.DocumentID, progress, run, ev.Message); err != nil {+ progress := deriveDocumentProgress(agg, int(total))+ if err = s.updateDocumentProgress(ctx, ev.DocumentID, progress, ev.Message); err != nil {common.Error(fmt.Sprintf("progressSink: mirror progress to document %s for task %s failed: %v", ev.DocumentID, ev.TaskID, err), err)}}@@ -132,16 +131,12 @@ func (s *progressSink) OnComponentMessage(ctx context.Context, taskID, docID, coif doc == nil {return}- run := ""- if doc.Run != nil {- run = *doc.Run- }log, err := s.accumulateLogLocked(ctx, docID, fmt.Sprintf("%s: %s", component, message))if err != nil {common.Error(fmt.Sprintf("progressSink: append detail for document %s failed", docID), err)return}- if err := s.docSvc.UpdateRunProgress(ctx, docID, doc.Progress, run, log); err != nil {+ if err := s.docSvc.UpdateRunProgress(ctx, docID, doc.Progress, log); err != nil {common.Error(fmt.Sprintf("progressSink: persist detail for document %s failed", docID), err)}}@@ -150,17 +145,17 @@ func (s *progressSink) OnComponentMessage(ctx context.Context, taskID, docID, co// document write. Without holding the same lock across both operations, a// slower database write can store an older snapshot after a newer event has// already been persisted.-func (s *progressSink) updateDocumentProgress(ctx context.Context, docID string, progress float64, run, msg string) error {+func (s *progressSink) updateDocumentProgress(ctx context.Context, docID string, progress float64, msg string) error {s.logMu.Lock()defer s.logMu.Unlock()log, err := s.accumulateLogLocked(ctx, docID, msg)if err != nil {// Keep the current run state durable even when the existing log cannot// be seeded. The next event will retry the seed and persist the log.- stateErr := s.docSvc.UpdateRunState(ctx, docID, progress, run)+ stateErr := s.docSvc.UpdateRunState(ctx, docID, progress)return errors.Join(err, stateErr)}- return s.docSvc.UpdateRunProgress(ctx, docID, progress, run, log)+ return s.docSvc.UpdateRunProgress(ctx, docID, progress, log)}// accumulateLog appends one component lifecycle line to the run log. It is@@ -235,23 +230,12 @@ func trimLogHead(text string, maxChars int) string {return text}-// deriveDocumentProgress computes the document-level progress (0..1) and run-// label ("0".."4", matching Python's document.run enum) from the aggregated-// ingestion_task_log. This logic is owned by the sink (the document-table-// writer), not the pipeline.-func deriveDocumentProgress(agg *dao.TaskProgress, total int) (float64, string) {- run := string(entity.TaskStatusUnstart)- switch {- case agg.Failed > 0:- run = string(entity.TaskStatusFail)- case agg.Done == total:- run = string(entity.TaskStatusDone)- case agg.Done > 0 || agg.Running > 0:- run = string(entity.TaskStatusRunning)- }+// deriveDocumentProgress computes the document-level progress (0..1) from the+// aggregated ingestion_task_log.+func deriveDocumentProgress(agg *dao.TaskProgress, total int) float64 {progress := agg.Percent / 100if progress > 1 {progress = 1}- return progress, run+ return progress}diff --git a/internal/ingestion/service/progress_sink_test.go b/internal/ingestion/service/progress_sink_test.goindex 32833d7c18..ba6bd3d780 100644--- a/internal/ingestion/service/progress_sink_test.go+++ b/internal/ingestion/service/progress_sink_test.go@@ -27,7 +27,6 @@ import ("time""ragflow/internal/dao"- "ragflow/internal/entity""ragflow/internal/ingestion/pipeline""ragflow/internal/ingestion/testutil"servicepkg "ragflow/internal/service"@@ -145,7 +144,6 @@ func TestProgressSink_Total_NoDataRace(t *testing.T) {type stubDocProgressSvc struct {gotDocID stringgotProgress float64- gotRun stringgotMsg stringcalls intdoc *document.DocumentResponse@@ -170,11 +168,11 @@ func (s *blockingDocProgressSvc) GetDocumentByID(context.Context, string) (*docureturn nil, nil}-func (s *blockingDocProgressSvc) UpdateRunState(context.Context, string, float64, string) error {+func (s *blockingDocProgressSvc) UpdateRunState(context.Context, string, float64) error {return nil}-func (s *blockingDocProgressSvc) UpdateRunProgress(_ context.Context, _ string, _ float64, _ string, msg string) error {+func (s *blockingDocProgressSvc) UpdateRunProgress(_ context.Context, _ string, _ float64, msg string) error {s.mu.Lock()s.updates++update := s.updates@@ -191,15 +189,14 @@ func (s *stubDocProgressSvc) GetDocumentByID(ctx context.Context, docID string)return s.doc, s.docErr}-func (s *stubDocProgressSvc) UpdateRunState(context.Context, string, float64, string) error {+func (s *stubDocProgressSvc) UpdateRunState(context.Context, string, float64) error {return nil}-func (s *stubDocProgressSvc) UpdateRunProgress(ctx context.Context, docID string, progress float64, run, progressMsg string) error {+func (s *stubDocProgressSvc) UpdateRunProgress(ctx context.Context, docID string, progress float64, progressMsg string) error {s.calls++s.gotDocID = docIDs.gotProgress = progress- s.gotRun = runs.gotMsg = progressMsgreturn nil}@@ -257,9 +254,6 @@ func TestProgressSinkPersistsViaService(t *testing.T) {if stub.gotProgress != 0.5 {t.Fatalf("progress = %v, want 0.5", stub.gotProgress)}- if stub.gotRun != "1" {- t.Fatalf("run = %q, want 1 (RUNNING)", stub.gotRun)- }if stub.gotMsg != "03:04:05: Parser Done" {t.Fatalf("progress_msg = %q, want timestamped Parser Done", stub.gotMsg)}@@ -349,7 +343,7 @@ func TestProgressSinkSerializesLogAndDocumentWrite(t *testing.T) {firstDone := make(chan struct{})go func() {defer close(firstDone)- if err := sink.updateDocumentProgress(context.Background(), "doc-1", 0.1, "1", "first"); err != nil {+ if err := sink.updateDocumentProgress(context.Background(), "doc-1", 0.1, "first"); err != nil {t.Errorf("first update: %v", err)}}()@@ -358,7 +352,7 @@ func TestProgressSinkSerializesLogAndDocumentWrite(t *testing.T) {secondDone := make(chan struct{})go func() {defer close(secondDone)- if err := sink.updateDocumentProgress(context.Background(), "doc-1", 0.2, "1", "second"); err != nil {+ if err := sink.updateDocumentProgress(context.Background(), "doc-1", 0.2, "second"); err != nil {t.Errorf("second update: %v", err)}}()@@ -507,69 +501,58 @@ func TestDeriveDocumentProgress(t *testing.T) {name stringagg *dao.TaskProgresstotal int- wantRun stringwantProg float64}{{- name: "failed component → fail",+ name: "failed component progress",agg: &dao.TaskProgress{Failed: 1, Done: 0, Running: 0, Percent: 0},total: 5,- wantRun: string(entity.TaskStatusFail),wantProg: 0.0,},{- name: "all done → done",+ name: "all done",agg: &dao.TaskProgress{Failed: 0, Done: 5, Running: 0, Percent: 100},total: 5,- wantRun: string(entity.TaskStatusDone),wantProg: 1.0,},{- name: "partial done → running",+ name: "partial done",agg: &dao.TaskProgress{Failed: 0, Done: 3, Running: 0, Percent: 60},total: 5,- wantRun: string(entity.TaskStatusRunning),wantProg: 0.6,},{- name: "running only → running",+ name: "running only",agg: &dao.TaskProgress{Failed: 0, Done: 0, Running: 2, Percent: 0},total: 5,- wantRun: string(entity.TaskStatusRunning),wantProg: 0.0,},{- name: "nothing started → unstart",+ name: "nothing started",agg: &dao.TaskProgress{Failed: 0, Done: 0, Running: 0, Percent: 0},total: 5,- wantRun: string(entity.TaskStatusUnstart),wantProg: 0.0,},{- name: "total zero, nothing done → done (0==0)",+ name: "total zero, nothing done",agg: &dao.TaskProgress{Failed: 0, Done: 0, Running: 0, Percent: 0},total: 0,- wantRun: string(entity.TaskStatusDone),wantProg: 0.0,},{- name: "failed overrides done=total",+ name: "failed with 100 percent",agg: &dao.TaskProgress{Failed: 1, Done: 5, Running: 0, Percent: 100},total: 5,- wantRun: string(entity.TaskStatusFail),wantProg: 1.0,},}for _, tt := range tests {t.Run(tt.name, func(t *testing.T) {- prog, run := deriveDocumentProgress(tt.agg, tt.total)+ prog := deriveDocumentProgress(tt.agg, tt.total)if prog != tt.wantProg {t.Errorf("progress = %v, want %v", prog, tt.wantProg)}- if run != tt.wantRun {- t.Errorf("run = %q, want %q", run, tt.wantRun)- }})}}diff --git a/internal/ingestion/task/pipeline_executor.go b/internal/ingestion/task/pipeline_executor.goindex 8afabef237..5965060991 100644--- a/internal/ingestion/task/pipeline_executor.go+++ b/internal/ingestion/task/pipeline_executor.go@@ -200,7 +200,7 @@ func (s *PipelineExecutor) Execute(ctx context.Context) (*PipelineResult, error)}if pipelineDSL != "" {- s.recordPipelineLog(context.WithoutCancel(ctx), dao.DB, s.taskCtx.Doc.ID, pipelineDSL, "")+ s.recordPipelineLog(context.WithoutCancel(ctx), dao.DB, s.taskCtx.Doc.ID, pipelineDSL, string(entity.TaskStatusDone))}return result, nil@@ -772,8 +772,7 @@ type PipelineLogInput struct {}// RecordPipelineLog persists a pipeline operation log without requiring-// executor setup. Callers that already know a terminal state should pass it in-// Status; otherwise the writer falls back to the latest document.run value.+// executor setup. Callers should pass the status in Status.func RecordPipelineLog(ctx context.Context, db *gorm.DB, input PipelineLogInput) error {return recordPipelineLog(ctx, db, input, dao.NewPipelineOperationLogDAO().Create)}@@ -847,9 +846,6 @@ func recordPipelineLog(}operationStatus := input.Status- if operationStatus == "" && doc.Run != nil && *doc.Run != "" {- operationStatus = *doc.Run- }statusValue := "1"if doc.Status != nil && *doc.Status != "" {statusValue = *doc.Statusdiff --git a/internal/ingestion/task/pipeline_executor_test.go b/internal/ingestion/task/pipeline_executor_test.goindex 80e9a2fcf5..a1006f5b3a 100644--- a/internal/ingestion/task/pipeline_executor_test.go+++ b/internal/ingestion/task/pipeline_executor_test.go@@ -373,7 +373,6 @@ func TestRecordPipelineLog_SharedWriterTerminalWithoutDSL(t *testing.T) {defer cleanup()docName := "terminal.pdf"- run := "1"if err := RecordPipelineLog(t.Context(), dao.DB, PipelineLogInput{TenantID: "tenant-1",KbID: "kb-1",@@ -388,7 +387,6 @@ func TestRecordPipelineLog_SharedWriterTerminalWithoutDSL(t *testing.T) {Type: "pdf",Name: &docName,Suffix: ".pdf",- Run: &run,},}); err != nil {t.Fatalf("RecordPipelineLog: %v", err)@@ -524,7 +522,6 @@ func TestRecordPipelineLog_TerminalWithoutDSLResolvesCanvasTitle(t *testing.T) {t.Fatalf("seed knowledgebase: %v", err)}docName := "sample.avi"- run := "1"if err := dao.DB.Create(&entity.Document{ID: "doc-1",KbID: "kb-1",@@ -532,7 +529,6 @@ func TestRecordPipelineLog_TerminalWithoutDSLResolvesCanvasTitle(t *testing.T) {ParserID: "naive",ParserConfig: entity.JSONMap{},Name: &docName,- Run: &run,}).Error; err != nil {t.Fatalf("seed document: %v", err)}@@ -837,6 +833,40 @@ func TestPipelineExecutor_Execute_PropagatesContext(t *testing.T) {}}+func TestPipelineExecutor_Execute_RecordsDoneOperationStatus(t *testing.T) {+ taskCtx := makeTaskCtx()+ taskCtx.Ctx = t.Context()+ var capturedLog *entity.PipelineOperationLog++ svc := mustNewPipelineExecutor(t, taskCtx, "flow-1", 0).+ WithLoadDSLFunc(func(ctx context.Context, canvasID string) (string, string, error) {+ return `{"nodes":[{"id":"n1"}],"edges":[]}`, canvasID, nil+ }).+ WithRunPipelineFunc(func(runCtx context.Context, dsl string) (map[string]any, string, error) {+ return map[string]any{"chunks": []map[string]any{{"text": "hello world"}}}, dsl, nil+ }).+ WithInsertFunc(func(ctx context.Context, chunks []map[string]any, baseName, datasetID string) ([]string, error) {+ return nil, nil+ }).+ WithLogCreateFunc(func(ctx context.Context, db *gorm.DB, log *entity.PipelineOperationLog) error {+ capturedLog = log+ return nil+ })++ if _, err := svc.Execute(taskCtx.Ctx); err != nil {+ t.Fatalf("unexpected error: %v", err)+ }+ if capturedLog == nil {+ t.Fatalf("expected pipeline operation log to be recorded")+ }+ if capturedLog.OperationStatus == "" {+ t.Fatalf("expected OperationStatus to be non-empty")+ }+ if capturedLog.OperationStatus != string(entity.TaskStatusDone) {+ t.Fatalf("expected OperationStatus = %q, got %q", entity.TaskStatusDone, capturedLog.OperationStatus)+ }+}+// =============================================================================// Stub implementations for testing// =============================================================================diff --git a/internal/service/chunk/chunk.go b/internal/service/chunk/chunk.goindex 103aee9dc7..6c06971728 100644--- a/internal/service/chunk/chunk.go+++ b/internal/service/chunk/chunk.go@@ -74,13 +74,14 @@ func searchConfigMap(value interface{}) (map[string]interface{}, bool) {// ChunkService chunk servicetype ChunkService struct {- docEngine engine.DocEngine- embeddingCache *utility.EmbeddingLRU- kbDAO *dao.KnowledgebaseDAO- userTenantDAO *dao.UserTenantDAO- documentDAO *dao.DocumentDAO- taskDAO *dao.TaskDAO- searchService *service.SearchService+ docEngine engine.DocEngine+ embeddingCache *utility.EmbeddingLRU+ kbDAO *dao.KnowledgebaseDAO+ userTenantDAO *dao.UserTenantDAO+ documentDAO *dao.DocumentDAO+ taskDAO *dao.TaskDAO+ ingestionTaskDAO *dao.IngestionTaskDAO+ searchService *service.SearchServiceaccessibleFunc func(string, string) boolgetKnowledgebaseByIDFunc func(string) (*entity.Knowledgebase, error)@@ -105,13 +106,14 @@ type ChunkService struct {// NewChunkService creates chunk servicefunc NewChunkService() *ChunkService {return &ChunkService{- docEngine: engine.Get(),- embeddingCache: utility.NewEmbeddingLRU(1000), // default capacity- kbDAO: dao.NewKnowledgebaseDAO(),- userTenantDAO: dao.NewUserTenantDAO(),- documentDAO: dao.NewDocumentDAO(),- taskDAO: dao.NewTaskDAO(),- searchService: service.NewSearchService(),+ docEngine: engine.Get(),+ embeddingCache: utility.NewEmbeddingLRU(1000), // default capacity+ kbDAO: dao.NewKnowledgebaseDAO(),+ userTenantDAO: dao.NewUserTenantDAO(),+ documentDAO: dao.NewDocumentDAO(),+ taskDAO: dao.NewTaskDAO(),+ ingestionTaskDAO: dao.NewIngestionTaskDAO(),+ searchService: service.NewSearchService(),}}@@ -658,9 +660,8 @@ func (s *ChunkService) StopParsing(ctx context.Context, userID, datasetID stringreturn nil, common.CodeServerError, err}// CancelDocParse (inside cancelAllTasksOfDoc) already issues- // RequestStop (STOPPING) and updates doc.run=CANCEL. Defer- // destruction (chunk deletion, counter reset) until the worker- // detects STOPPING and reaches a terminal state.+ // RequestStop (STOPPING). Defer destruction (chunk deletion, counter+ // reset) until the worker detects STOPPING and reaches a terminal state.successCount++}@@ -752,12 +753,6 @@ func (s *ChunkService) Parse(ctx context.Context, userID, datasetID string, reqif len(notFound) > 0 {return nil, common.CodeDataError, fmt.Errorf("documents not found: %v", notFound)}- for _, docID := range docIDs {- doc := docByID[docID]- if doc.Run != nil && *doc.Run == string(entity.TaskStatusRunning) {- return nil, common.CodeDataError, fmt.Errorf("can't parse document that is currently being processed")- }- }// Batch pre-check: refuse the whole request if any document's ingestion// task is non-terminal (RUNNING/STOPPING), so we never partially clean.@@ -982,9 +977,22 @@ func (s *ChunkService) List(ctx context.Context, req *service.ListChunksRequest,chunks = append(chunks, result)}+ ingestionStatus := "UNSTART"+ taskDAO := s.ingestionTaskDAO+ if taskDAO == nil {+ taskDAO = dao.NewIngestionTaskDAO()+ }+ task, err := taskDAO.GetByDocumentID(ctx, dao.DB, doc.ID)+ if err != nil {+ return nil, fmt.Errorf("failed to get ingestion task for document %s: %w", doc.ID, err)+ }+ if task != nil && task.Status != "" {+ ingestionStatus = task.Status+ }+// Build document info, mirroring Python's _map_doc key renames:// kb_id→dataset_id, parser_id→chunk_method, token_num→token_count,- // chunk_num→chunk_count, run→text status.+ // chunk_num→chunk_count.timeFormat := "2006-01-02T15:04:05"docInfo := map[string]interface{}{"id": doc.ID,@@ -1007,7 +1015,7 @@ func (s *ChunkService) List(ctx context.Context, req *service.ListChunksRequest,"process_duration": doc.ProcessDuration,"content_hash": doc.ContentHash,"suffix": doc.Suffix,- "run": service.ChunkDocRunText(doc.Run),+ "ingestion_status": ingestionStatus,"status": doc.Status,"create_time": doc.CreateTime,"create_date": utility.FormatTimeToString(doc.CreateDate, timeFormat),diff --git a/internal/service/chunk/chunk_test.go b/internal/service/chunk/chunk_test.goindex 880b5c25c9..79f85bdc13 100644--- a/internal/service/chunk/chunk_test.go+++ b/internal/service/chunk/chunk_test.go@@ -199,12 +199,6 @@ func TestParsePrevalidatesDocumentsBeforeMutating(t *testing.T) {if err != nil {t.Fatalf("get doc: %v", err)}- if doc.Run == nil {- t.Fatalf("expected doc run to remain nil, got %q", *doc.Run)- }- if *doc.Run != "0" {- t.Fatalf("expected doc run status is '1', got %q", *doc.Run)- }if doc.ChunkNum != 7 {t.Fatalf("expected chunk_num to remain 7, got %d", doc.ChunkNum)}@@ -289,10 +283,7 @@ func TestParseRejectsRunningDocument(t *testing.T) {datasetID := "kb-1"insertChunkTestKB(t, datasetID, userID)insertChunkTestDoc(t, "doc-1", datasetID)- running := string(entity.TaskStatusRunning)- if err := db.Model(&entity.Document{}).Where("id = ?", "doc-1").Update("run", running).Error; err != nil {- t.Fatalf("mark doc running: %v", err)- }+ insertChunkTestIngestionTask(t, "task-1", userID, "doc-1", datasetID, common.RUNNING)svc := newParseTestService(t)ctx := t.Context()@@ -303,7 +294,7 @@ func TestParseRejectsRunningDocument(t *testing.T) {if code != common.CodeDataError {t.Fatalf("expected CodeDataError, got %v", code)}- if !strings.Contains(err.Error(), "currently being processed") {+ if !strings.Contains(err.Error(), "ingestion task is RUNNING") {t.Fatalf("unexpected error: %v", err)}}@@ -1715,7 +1706,7 @@ func TestStopParsing_CallsCancelIngestionTask(t *testing.T) {pushChunkTestDB(t, db)insertChunkTestKB(t, "kb-1", "user-1") // owner = user-1, Accessible passesinsertChunkTestDoc(t, "doc-1", "kb-1")- // StopParsing now checks the IngestionTask status (not doc.Run).+ // StopParsing checks the IngestionTask status.insertChunkTestIngestionTask(t, "task-1", "user-1", "doc-1", "kb-1", common.RUNNING)svc := newParseTestService(t)@@ -1773,9 +1764,9 @@ func TestStopParsing_DoesNotDeleteChunksOrResetCountersAfterCancel(t *testing.T)svc := newParseTestService(t)svc.cancelIngestionTaskFunc = func(ctx context.Context, doc *entity.Document) error {- // Simulate CancelDocParse: set doc.run=CANCEL.- return dao.DB.Model(&entity.Document{}).Where("id = ?", doc.ID).- Update("run", string(entity.TaskStatusCancel)).Error+ // Simulate CancelDocParse: stop task.+ return dao.DB.Model(&entity.IngestionTask{}).Where("document_id = ?", doc.ID).+ Update("status", common.STOPPED).Error}engine := &parseTestDocEngine{chunkStoreExists: true}svc.docEngine = enginediff --git a/internal/service/chunk_types.go b/internal/service/chunk_types.goindex 3dd01f2433..68cc1c92ce 100644--- a/internal/service/chunk_types.go+++ b/internal/service/chunk_types.go@@ -17,47 +17,12 @@package serviceimport (- "context""encoding/json""fmt"- "math"- "strings"- "time""ragflow/internal/common"- "ragflow/internal/dao"- "ragflow/internal/engine"- "ragflow/internal/engine/redis"- "ragflow/internal/engine/types"- "ragflow/internal/entity"- "ragflow/internal/tokenizer"- "ragflow/internal/utility"-- "gorm.io/gorm"-)--var (- UNSTART = "0"- RUNNING = "1"- CANCEL = "2"- DONE = "3"- FAIL = "4"- SCHEDULE = "5")-// ChunkService chunk service-type ChunkService struct {- docEngine engine.DocEngine- embeddingCache *utility.EmbeddingLRU- kbDAO *dao.KnowledgebaseDAO- taskDAO *dao.TaskDAO- userTenantDAO *dao.UserTenantDAO- documentDAO *dao.DocumentDAO- searchService *SearchService-- decrementChunkStatsFunc func(string, string, int64, int64, float64) error-}-// RetrievalTestRequest retrieval test requesttype RetrievalTestRequest struct {Datasets common.StringSlice `json:"dataset_ids" binding:"required"` // string or []string@@ -124,82 +89,6 @@ type ErrorCoder interface {Code() common.ErrorCode}-// Get retrieves a chunk by ID-func (s *ChunkService) Get(ctx context.Context, req *GetChunkRequest, userID string) (*GetChunkResponse, error) {- if s.docEngine == nil {- return nil, fmt.Errorf("doc engine not initialized")- }-- if req.ChunkID == "" {- return nil, fmt.Errorf("chunk_id is required")- }-- // Get user's tenants- tenants, err := s.userTenantDAO.GetByUserID(ctx, dao.DB, userID)- if err != nil {- return nil, fmt.Errorf("failed to get user tenants: %w", err)- }- if len(tenants) == 0 {- return nil, fmt.Errorf("user has no accessible tenants")- }-- // Try each tenant to find the chunk- var chunk map[string]interface{}- for _, tenant := range tenants {- // Get kbIDs for this tenant- kbIDs, err := s.kbDAO.GetKBIDsByTenantID(ctx, dao.DB, tenant.TenantID)- if err != nil {- continue- }-- indexName := fmt.Sprintf("ragflow_%s", tenant.TenantID)-- doc, err := s.docEngine.GetChunk(ctx, indexName, req.ChunkID, kbIDs)- if err != nil {- continue- }-- if doc != nil {- chunk, ok := doc.(map[string]interface{})- if ok {- result := make(map[string]interface{})- skipFields := map[string]bool{- "id": true, "authors": true, "_score": true, "SCORE": true,- }- for k, v := range chunk {- if skipFields[k] || isInternalField(k) {- continue- }- if applyCommonChunkMapping(result, k, v) {- continue- }- switch k {- case "tag_feas":- if utility.IsEmpty(v) {- result[k] = map[string]interface{}{}- } else {- result[k] = v- }- case "create_timestamp_flt", "rank_flt", "weight_flt":- if floatVal, ok := utility.ToFloat64(v); ok {- result[k] = utility.JSONFloat64(floatVal)- }- default:- result[k] = v- }- }- return &GetChunkResponse{Chunk: result}, nil- }- }- }-- if chunk == nil {- return nil, fmt.Errorf("chunk not found")- }-- return &GetChunkResponse{Chunk: chunk}, nil-}-type StopParsingRequest struct {DocumentIDs []string `json:"document_ids"`}@@ -209,28 +98,6 @@ type StopParsingResponse struct {Message string}-func (s *ChunkService) cancelAllTasksOfDoc(ctx context.Context, docID string) error {- tasks, err := s.taskDAO.GetByDocID(ctx, dao.DB, docID)- if err != nil {- return fmt.Errorf("failed to get tasks for document %s: %w", docID, err)- }-- redisClient := redis.Get()- if redisClient == nil {- common.Logger.Warn(fmt.Sprintf("Redis unavailable; cannot cancel tasks for document %s", docID))- return nil- }-- for _, task := range tasks {- if task == nil {- continue- }- redisClient.Set(ctx, fmt.Sprintf("%s-cancel", task.ID), "x", time.Hour)- }-- return nil-}-func CheckDuplicateIDs(docList []string, idType string) ([]string, []string) {uniqueDocIDs := make([]string, 0)duplicateMessages := make([]string, 0)@@ -253,80 +120,6 @@ func IndexName(uid string) string {return fmt.Sprintf("ragflow_%s", uid)}-func (s *ChunkService) StopParsing(ctx context.Context, userID, datasetID string, req StopParsingRequest) (map[string]interface{}, common.ErrorCode, error) {- if !s.kbDAO.Accessible(ctx, dao.DB, datasetID, userID) {- return nil, common.CodeAuthenticationError, fmt.Errorf("You don't own the dataset %s", datasetID)- }-- if req.DocumentIDs == nil || len(req.DocumentIDs) == 0 {- return nil, common.CodeDataError, fmt.Errorf("document_ids is required")- }-- docList, duplicateMessages := CheckDuplicateIDs(req.DocumentIDs, "document")-- kb, err := s.kbDAO.GetByID(ctx, dao.DB, datasetID)- if err != nil {- return nil, common.CodeDataError, fmt.Errorf("You don't own the dataset %s", datasetID)- }-- successCount := 0- for _, id := range docList {- var doc *entity.Document- doc, err = s.documentDAO.GetByDocumentIDAndDatasetID(ctx, dao.DB, id, datasetID)- if err != nil {- return nil, common.CodeDataError, fmt.Errorf("you don't own the document %s", id)- }- if doc == nil {- return nil, common.CodeDataError, fmt.Errorf("you don't own the document %s", id)- }-- if doc.Run == nil || *doc.Run != RUNNING {- return nil, common.CodeDataError, fmt.Errorf("can't stop parsing document that has not started or already completed")- }-- err = s.cancelAllTasksOfDoc(ctx, id)- if err != nil {- return nil, common.CodeServerError, err- }-- info := map[string]interface{}{- "run": "2",- "progress": 0,- "chunk_num": 0,- }- if err = s.documentDAO.UpdateByID(ctx, dao.DB, doc.ID, info); err != nil {- return nil, common.CodeServerError, fmt.Errorf("failed to update document %s: %w", doc.ID, err)- }-- indexName := IndexName(kb.TenantID)-- var exists bool- exists, err = s.docEngine.ChunkStoreExists(ctx, indexName, datasetID)- if err != nil {- return nil, common.CodeServerError, fmt.Errorf("failed to check chunk store %s/%s: %w", indexName, datasetID, err)- }- if exists {- if _, err = s.docEngine.DeleteChunks(ctx, map[string]interface{}{"doc_id": doc.ID}, indexName, datasetID); err != nil {- return nil, common.CodeServerError, fmt.Errorf("failed to delete chunks for document %s: %w", doc.ID, err)- }- } else {- common.Logger.Info(fmt.Sprintf("Skipping chunk delete during stop_parsing for doc %s: index %s/%s does not exist", doc.ID, indexName, datasetID))- }- successCount++- }- if len(duplicateMessages) > 0 {- if successCount > 0 {- return map[string]interface{}{- "success_count": successCount,- "errors": duplicateMessages,- "message": fmt.Sprintf("Partially stopped %d documents with %d errors", successCount, len(duplicateMessages)),- }, common.CodeSuccess, nil- }- return nil, common.CodeDataError, fmt.Errorf("%s", strings.Join(duplicateMessages, ";"))- }- return nil, common.CodeSuccess, nil-}-// ListChunksRequest request for listing chunkstype ListChunksRequest struct {DatasetID string `json:"dataset_id,omitempty"`@@ -345,165 +138,6 @@ type ListChunksResponse struct {Total int64 `json:"total"`}-// List retrieves chunks for a document-func (s *ChunkService) List(ctx context.Context, req *ListChunksRequest, userID string) (*ListChunksResponse, error) {- if s.docEngine == nil {- return nil, fmt.Errorf("doc engine not initialized")- }-- if req.DocID == "" {- return nil, fmt.Errorf("doc_id is required")- }-- // Get user's tenants- tenants, err := s.userTenantDAO.GetByUserID(ctx, dao.DB, userID)- if err != nil {- return nil, fmt.Errorf("failed to get user tenants: %w", err)- }- if len(tenants) == 0 {- return nil, fmt.Errorf("user has no accessible tenants")- }-- // Get document to find its tenant- docDAO := dao.NewDocumentDAO()- doc, err := docDAO.GetByID(ctx, dao.DB, req.DocID)- if err != nil || doc == nil {- return nil, fmt.Errorf("document not found")- }- if req.DatasetID != "" && doc.KbID != req.DatasetID {- return nil, fmt.Errorf("document not found")- }-- // Get knowledge base to find tenant- kb, err := s.kbDAO.GetByID(ctx, dao.DB, doc.KbID)- if err != nil || kb == nil {- return nil, fmt.Errorf("knowledge base not found")- }-- // Find which tenant this document belongs to- var targetTenantID string- for _, tenant := range tenants {- if tenant.TenantID == kb.TenantID {- targetTenantID = tenant.TenantID- break- }- }- if targetTenantID == "" {- return nil, fmt.Errorf("user does not have access to this document")- }-- // Get kbIDs for this tenant- kbIDs, err := s.kbDAO.GetKBIDsByTenantID(ctx, dao.DB, targetTenantID)- if err != nil {- return nil, fmt.Errorf("failed to get kb ids: %w", err)- }-- indexName := fmt.Sprintf("ragflow_%s", targetTenantID)-- page := common.CoalesceInt(req.Page, 1)- size := common.CoalesceInt(req.Size, 30)- keywords := req.Keywords-- // Build search request - same as retrieval test but filtered by doc_id- searchReq := &types.SearchRequest{- IndexNames: []string{indexName},- MatchExprs: []interface{}{keywords},- KbIDs: kbIDs,- Offset: (page - 1) * size,- Limit: size,- Filter: map[string]interface{}{- "doc_id": req.DocID,- },- }-- // Add available_int filter if specified- if req.AvailableInt != nil {- searchReq.Filter["available_int"] = *req.AvailableInt- }-- // Execute search through unified engine interface- searchResp, err := s.docEngine.Search(ctx, searchReq)- if err != nil {- return nil, fmt.Errorf("search failed: %w", err)- }-- chunks := make([]map[string]interface{}, 0, len(searchResp.Chunks))- for _, chunk := range searchResp.Chunks {- // Inline formatChunkForList- result := make(map[string]interface{})- skipFields := map[string]bool{- "_id": true, "authors": true, "_score": true, "SCORE": true,- "important_kwd_empty_count": true, "kb_id": true, "mom_id": true, "page_num_int": true,- }- for k, v := range chunk {- if skipFields[k] || isInternalField(k) {- continue- }- if applyCommonChunkMapping(result, k, v) {- continue- }- switch k {- case "img_id":- if strVal, ok := v.(string); ok {- result["image_id"] = strVal- } else {- result["image_id"] = ""- }- case "position_int":- result["positions"] = v- case "id":- result["chunk_id"] = v- default:- if strings.HasSuffix(k, "_kwd") && k != "knowledge_graph_kwd" {- result[k] = splitKwdHash(v)- } else {- result[k] = v- }- }- }- chunks = append(chunks, result)- }-- // Build document info, mirroring Python's _map_doc key renames:- // kb_id→dataset_id, parser_id→chunk_method, token_num→token_count,- // chunk_num→chunk_count, run→text status.- timeFormat := "2006-01-02T15:04:05"- docInfo := map[string]interface{}{- "id": doc.ID,- "thumbnail": doc.Thumbnail,- "dataset_id": doc.KbID,- "chunk_method": doc.ParserID,- "pipeline_id": doc.PipelineID,- "parser_config": doc.ParserConfig,- "source_type": doc.SourceType,- "type": doc.Type,- "created_by": doc.CreatedBy,- "name": doc.Name,- "location": doc.Location,- "size": doc.Size,- "token_count": doc.TokenNum,- "chunk_count": doc.ChunkNum,- "progress": utility.JSONFloat64(doc.Progress),- "progress_msg": doc.ProgressMsg,- "process_begin_at": utility.FormatTimeToString(doc.ProcessBeginAt, timeFormat),- "process_duration": doc.ProcessDuration,- "content_hash": doc.ContentHash,- "suffix": doc.Suffix,- "run": ChunkDocRunText(doc.Run),- "status": doc.Status,- "create_time": doc.CreateTime,- "create_date": utility.FormatTimeToString(doc.CreateDate, timeFormat),- "update_time": doc.UpdateTime,- "update_date": utility.FormatTimeToString(doc.UpdateDate, timeFormat),- }-- return &ListChunksResponse{- Total: searchResp.Total,- Chunks: chunks,- Doc: docInfo,- }, nil-}-// UpdateChunkRequest request for updating a chunktype UpdateChunkRequest struct {DatasetID string `json:"dataset_id"`@@ -518,198 +152,6 @@ type UpdateChunkRequest struct {TagFeas interface{} `json:"tag_feas,omitempty"`}-// UpdateChunk updates a chunk fields-func (s *ChunkService) UpdateChunk(ctx context.Context, req *UpdateChunkRequest, userID string) error {- if s.docEngine == nil {- return fmt.Errorf("doc engine not initialized")- }-- if req.ChunkID == "" {- return fmt.Errorf("chunk_id is required")- }-- // Get user's tenants- tenants, err := s.userTenantDAO.GetByUserID(ctx, dao.DB, userID)- if err != nil {- return fmt.Errorf("failed to get user tenants: %w", err)- }- if len(tenants) == 0 {- return fmt.Errorf("user has no accessible tenants")- }-- // Find the tenant that owns this dataset- var targetTenantID string- for _, tenant := range tenants {- kb, err := s.kbDAO.GetByIDAndTenantID(ctx, dao.DB, req.DatasetID, tenant.TenantID)- if err == nil && kb != nil {- targetTenantID = tenant.TenantID- break- }- }- if targetTenantID == "" {- return fmt.Errorf("user does not have access to this dataset")- }-- // Verify document belongs to dataset- docDAO := dao.NewDocumentDAO()- doc, err := docDAO.GetByID(ctx, dao.DB, req.DocumentID)- if err != nil || doc == nil {- return fmt.Errorf("document not found")- }- if doc.KbID != req.DatasetID {- return fmt.Errorf("document does not belong to this dataset")- }-- // Fetch existing chunk first- indexName := fmt.Sprintf("ragflow_%s", targetTenantID)- existingChunk, err := s.docEngine.GetChunk(ctx, indexName, req.ChunkID, []string{req.DatasetID})- if err != nil {- return fmt.Errorf("failed to get existing chunk: %w", err)- }-- existing, ok := existingChunk.(map[string]interface{})- if !ok {- return fmt.Errorf("invalid chunk format")- }-- // Build update dict- d := make(map[string]interface{})-- // Content - use new value or existing- if req.Content != nil {- d["content_with_weight"] = *req.Content- } else {- if v, ok := existing["content_with_weight"].(string); ok {- d["content_with_weight"] = v- } else if v, ok := existing["content"].(string); ok {- d["content_with_weight"] = v- } else {- d["content_with_weight"] = ""- }- }-- // Tokenize content- contentStr := d["content_with_weight"].(string)- d["content_ltks"], _ = tokenizer.Tokenize(contentStr)- d["content_sm_ltks"], _ = tokenizer.FineGrainedTokenize(d["content_ltks"].(string))-- // Important keywords - convert []string to []interface{} for transformChunkFields- if req.ImportantKwd != nil {- impKwd := make([]interface{}, len(req.ImportantKwd))- for i, v := range req.ImportantKwd {- impKwd[i] = v- }- d["important_kwd"] = impKwd- }-- // Questions- if req.Questions != nil {- // Filter out empty questions and trim- filteredQuestions := []string{}- for _, q := range req.Questions {- q = strings.TrimSpace(q)- if q != "" {- filteredQuestions = append(filteredQuestions, q)- }- }- d["question_kwd"] = filteredQuestions- }-- // Available- if req.Available != nil {- if *req.Available {- d["available_int"] = 1- } else {- d["available_int"] = 0- }- }-- // Positions- if req.Positions != nil {- d["position_int"] = req.Positions- }-- // Tag keywords- if req.TagKwd != nil {- d["tag_kwd"] = req.TagKwd- }-- // Tag features- if req.TagFeas != nil {- tagFeas, err := validateUpdateTagFeatures(req.TagFeas)- if err != nil {- return updateChunkError{code: common.CodeArgumentError, message: "`tag_feas` " + err.Error()}- }- d["tag_feas"] = tagFeas- }-- // Always include id- d["id"] = req.ChunkID-- // Call update- condition := map[string]interface{}{- "id": req.ChunkID,- }-- err = s.docEngine.UpdateChunks(ctx, condition, d, indexName, req.DatasetID)- if err != nil {- return fmt.Errorf("failed to update chunk: %w", err)- }-- return nil-}--type updateChunkError struct {- code common.ErrorCode- message string-}--func (e updateChunkError) Error() string {- return e.message-}--func (e updateChunkError) Code() common.ErrorCode {- return e.code-}--func validateUpdateTagFeatures(raw interface{}) (map[string]float64, error) {- parsed, ok := raw.(map[string]interface{})- if !ok {- return nil, fmt.Errorf("must be an object mapping string tags to finite numeric scores greater than 0")- }- cleaned := make(map[string]float64, len(parsed))- for key, value := range parsed {- key = strings.TrimSpace(key)- if key == "" {- return nil, fmt.Errorf("keys must be non-empty strings")- }- var numeric float64- switch typed := value.(type) {- case float64:- numeric = typed- case float32:- numeric = float64(typed)- case int:- numeric = float64(typed)- case int8:- numeric = float64(typed)- case int16:- numeric = float64(typed)- case int32:- numeric = float64(typed)- case int64:- numeric = float64(typed)- default:- return nil, fmt.Errorf("values must be finite numbers greater than 0")- }- if math.IsNaN(numeric) || math.IsInf(numeric, 0) || numeric <= 0 {- return nil, fmt.Errorf("values must be finite numbers greater than 0")- }- cleaned[key] = numeric- }- return cleaned, nil-}-// RemoveChunksRequest request for removing chunkstype RemoveChunksRequest struct {DocID string `json:"doc_id"`@@ -717,118 +159,6 @@ type RemoveChunksRequest struct {DeleteAll bool `json:"delete_all,omitempty"`}-// RemoveChunks removes chunks from the dataset table.-// If ChunkIDs is empty and DeleteAll is true, removes all chunks for the document.-// Otherwise, removes only the specified chunks.-func (s *ChunkService) RemoveChunks(ctx context.Context, req *RemoveChunksRequest, userID string) (int64, error) {- if s.docEngine == nil {- return 0, fmt.Errorf("doc engine not initialized")- }-- if req.DocID == "" {- return 0, fmt.Errorf("doc_id is required")- }-- // Get user's tenants- tenants, err := s.userTenantDAO.GetByUserID(ctx, dao.DB, userID)- if err != nil {- return 0, fmt.Errorf("failed to get user tenants: %w", err)- }- if len(tenants) == 0 {- return 0, fmt.Errorf("user has no accessible tenants")- }-- // Verify document exists and belongs to a dataset (do this first to get doc.KbID)- docDAO := dao.NewDocumentDAO()- doc, err := docDAO.GetByID(ctx, dao.DB, req.DocID)- if err != nil || doc == nil {- return 0, fmt.Errorf("document not found")- }-- // Find the tenant that owns this document- var targetTenantID string- for _, tenant := range tenants {- kb, err := s.kbDAO.GetByIDAndTenantID(ctx, dao.DB, doc.KbID, tenant.TenantID)- if err == nil && kb != nil {- targetTenantID = tenant.TenantID- break- }- }- if targetTenantID == "" {- return 0, fmt.Errorf("user does not have access to this document")- }-- indexName := fmt.Sprintf("ragflow_%s", targetTenantID)-- // Build condition- condition := make(map[string]interface{})- switch {- case len(req.ChunkIDs) > 0 && req.DeleteAll:- return 0, fmt.Errorf("chunk_ids and delete_all are mutually exclusive")- case len(req.ChunkIDs) > 0:- // Delete specific chunks - convert []string to []interface{} for buildFilterFromCondition- chunkIDsIf := make([]interface{}, len(req.ChunkIDs))- for i, id := range req.ChunkIDs {- chunkIDsIf[i] = id- }- condition["id"] = chunkIDsIf- condition["doc_id"] = req.DocID- case req.DeleteAll:- // Delete all chunks for this document- condition["doc_id"] = req.DocID- default:- return 0, fmt.Errorf("either chunk_ids or delete_all must be provided")- }-- deletedCount, err := s.docEngine.DeleteChunks(ctx, condition, indexName, doc.KbID)- if err != nil {- return 0, fmt.Errorf("failed to delete chunks: %w", err)- }-- if deletedCount > 0 {- if err = s.decrementChunkStats(req.DocID, doc.KbID, 0, deletedCount, 0); err != nil {- return deletedCount, fmt.Errorf("failed to update chunk stats: %w", err)- }- }-- return deletedCount, nil-}--func (s *ChunkService) decrementChunkStats(docID, kbID string, tokenNum, chunkNum int64, duration float64) error {- if s.decrementChunkStatsFunc != nil {- return s.decrementChunkStatsFunc(docID, kbID, tokenNum, chunkNum, duration)- }- return dao.DB.Transaction(func(tx *gorm.DB) error {- result := tx.Model(&entity.Document{}).- Where("id = ? AND kb_id = ?", docID, kbID).- Updates(map[string]interface{}{- "token_num": gorm.Expr("CASE WHEN token_num - ? >= 0 THEN token_num - ? ELSE 0 END", tokenNum, tokenNum),- "chunk_num": gorm.Expr("CASE WHEN chunk_num - ? >= 0 THEN chunk_num - ? ELSE 0 END", chunkNum, chunkNum),- "process_duration": gorm.Expr("CASE WHEN process_duration + ? >= 0 THEN process_duration + ? ELSE 0 END", duration, duration),- })- if result.Error != nil {- return result.Error- }- if result.RowsAffected == 0 {- return fmt.Errorf("document not found")- }-- result = tx.Model(&entity.Knowledgebase{}).- Where("id = ?", kbID).- Updates(map[string]interface{}{- "token_num": gorm.Expr("CASE WHEN token_num - ? >= 0 THEN token_num - ? ELSE 0 END", tokenNum, tokenNum),- "chunk_num": gorm.Expr("CASE WHEN chunk_num - ? >= 0 THEN chunk_num - ? ELSE 0 END", chunkNum, chunkNum),- })- if result.Error != nil {- return result.Error- }- if result.RowsAffected == 0 {- return fmt.Errorf("knowledgebase not found")- }- return nil- })-}-// SourcedChunk is a typed, normalized view over a retrieval result chunk.// It decouples the ask pipeline (KbPrompt, ChunksFormat) from the raw// map[string]interface{} that flows through the retrieval engine.@@ -925,78 +255,3 @@ func getMap(m map[string]interface{}, key string) map[string]interface{} {}return nil}--// isInternalField reports whether k is an internal/technical field that-// should be excluded from API chunk responses.-func isInternalField(k string) bool {- return strings.HasSuffix(k, "_vec") ||- strings.Contains(k, "_sm_") ||- strings.HasSuffix(k, "_tks") ||- strings.HasSuffix(k, "_ltks")-}--// applyCommonChunkMapping applies field mappings shared between GetChunk and-// ListChunks. Returns true if the field was handled.-// chunkDocRunText maps the document run code to its text form, mirroring-// Python's _map_doc run_mapping.-func ChunkDocRunText(run *string) interface{} {- if run == nil {- return nil- }- switch *run {- case "0":- return "UNSTART"- case "1":- return "RUNNING"- case "2":- return "CANCEL"- case "3":- return "DONE"- case "4":- return "FAIL"- case "5":- return "SCHEDULE"- }- return *run-}--func applyCommonChunkMapping(result map[string]interface{}, k string, v interface{}) bool {- switch k {- case "content":- result["content_with_weight"] = v- case "docnm":- result["docnm_kwd"] = v- case "important_keywords":- utility.SetFieldArray(result, "important_kwd", v)- case "questions":- utility.SetFieldArray(result, "question_kwd", v)- case "entities_kwd", "entity_kwd", "entity_type_kwd", "from_entity_kwd",- "name_kwd", "raptor_kwd", "removed_kwd", "source_id", "tag_kwd",- "to_entity_kwd", "toc_kwd", "doc_type_kwd":- if utility.IsEmpty(v) {- result[k] = []interface{}{}- } else {- result[k] = v- }- default:- return false- }- return true-}--// splitKwdHash splits a "###"-separated _kwd string into a slice.-// Non-string values or values without "###" are returned unchanged.-func splitKwdHash(v interface{}) interface{} {- strVal, ok := v.(string)- if !ok || !strings.Contains(strVal, "###") {- return v- }- parts := strings.Split(strVal, "###")- filtered := make([]interface{}, 0, len(parts))- for _, p := range parts {- if p != "" {- filtered = append(filtered, p)- }- }- return filtered-}diff --git a/internal/service/chunk_types_test.go b/internal/service/chunk_types_test.goindex e8d56ce93d..356480f62f 100644--- a/internal/service/chunk_types_test.go+++ b/internal/service/chunk_types_test.go@@ -18,12 +18,7 @@ package serviceimport ("ragflow/internal/common"- "ragflow/internal/dao"- "ragflow/internal/entity""testing"-- "github.com/glebarez/sqlite"- "gorm.io/gorm")func TestNewSourcedChunks_Empty(t *testing.T) {@@ -226,86 +221,6 @@ func TestSourcedChunk_ZeroValue(t *testing.T) {}}-func TestChunkTypesDecrementChunkStats(t *testing.T) {- db, err := gorm.Open(sqlite.Open(":memory:"), &gorm.Config{})- if err != nil {- t.Fatalf("open sqlite: %v", err)- }- if err = db.AutoMigrate(&entity.Knowledgebase{}, &entity.Document{}); err != nil {- t.Fatalf("migrate sqlite: %v", err)- }- previousDB := dao.DB- dao.DB = db- t.Cleanup(func() { dao.DB = previousDB })-- status := string(entity.StatusValid)- if err = dao.DB.Create(&entity.Knowledgebase{- ID: "kb-1",- TenantID: "tenant-1",- Name: "kb-1",- EmbdID: "embed",- Permission: string(entity.TenantPermissionMe),- CreatedBy: "user-1",- ParserConfig: entity.JSONMap{},- TokenNum: 20,- ChunkNum: 5,- Status: &status,- }).Error; err != nil {- t.Fatalf("create kb: %v", err)- }- if err = dao.DB.Create(&entity.Document{- ID: "doc-1",- KbID: "kb-1",- ParserID: string(entity.ParserTypeNaive),- ParserConfig: entity.JSONMap{},- SourceType: string(entity.FileSourceLocal),- Type: "txt",- CreatedBy: "user-1",- TokenNum: 10,- ChunkNum: 3,- Suffix: ".txt",- Status: &status,- }).Error; err != nil {- t.Fatalf("create doc: %v", err)- }-- svc := &ChunkService{}- if err = svc.decrementChunkStats("doc-1", "kb-1", 0, 2, 0); err != nil {- t.Fatalf("decrementChunkStats() error = %v", err)- }-- var doc entity.Document- if err = dao.DB.First(&doc, "id = ?", "doc-1").Error; err != nil {- t.Fatalf("get doc: %v", err)- }- if doc.TokenNum != 10 || doc.ChunkNum != 1 {- t.Fatalf("document stats token=%d chunk=%d, want token=10 chunk=1", doc.TokenNum, doc.ChunkNum)- }- var kb entity.Knowledgebase- if err = dao.DB.First(&kb, "id = ?", "kb-1").Error; err != nil {- t.Fatalf("get kb: %v", err)- }- if kb.TokenNum != 20 || kb.ChunkNum != 3 {- t.Fatalf("knowledgebase stats token=%d chunk=%d, want token=20 chunk=3", kb.TokenNum, kb.ChunkNum)- }-- if err = svc.decrementChunkStats("doc-1", "kb-1", 30, 30, -1); err != nil {- t.Fatalf("decrementChunkStats() clamp error = %v", err)- }- if err = dao.DB.First(&doc, "id = ?", "doc-1").Error; err != nil {- t.Fatalf("get doc after clamp: %v", err)- }- if doc.TokenNum != 0 || doc.ChunkNum != 0 || doc.ProcessDuration != 0 {- t.Fatalf("document clamped stats token=%d chunk=%d duration=%v, want zeros", doc.TokenNum, doc.ChunkNum, doc.ProcessDuration)- }- if err = dao.DB.First(&kb, "id = ?", "kb-1").Error; err != nil {- t.Fatalf("get kb after clamp: %v", err)- }- if kb.TokenNum != 0 || kb.ChunkNum != 0 {- t.Fatalf("knowledgebase clamped stats token=%d chunk=%d, want zeros", kb.TokenNum, kb.ChunkNum)- }-}-func TestNewSourcedChunks_RoundTrip(t *testing.T) {original := []SourcedChunk{{ID: "id1",@@ -385,123 +300,3 @@ func TestGetPageSize_Valid(t *testing.T) {t.Errorf("CoalesceInt(&50, 20) = %d, want 50", got)}}--func TestIsInternalField(t *testing.T) {- tests := []struct {- field string- want bool- }{- {"vector_vec", true},- {"content_sm_ltks", true},- {"content_ltks", true},- {"content", false},- {"docnm_kwd", false},- {"important_kwd", false},- {"knowledge_graph_kwd", false},- }- for _, tt := range tests {- if got := isInternalField(tt.field); got != tt.want {- t.Errorf("isInternalField(%q) = %v, want %v", tt.field, got, tt.want)- }- }-}--func TestSplitKwdHash(t *testing.T) {- tests := []struct {- name string- input interface{}- }{- {"non-string int", 42},- {"no hash", "hello world"},- {"simple hash", "a###b###c"},- {"hash with empty", "a######b"},- {"hash only", "###"},- }- for _, tt := range tests {- _ = splitKwdHash(tt.input)- }-- // Verify actual output- result := splitKwdHash("a###b###c")- slice, ok := result.([]interface{})- if !ok {- t.Errorf("expected []interface{}, got %T", result)- return- }- if len(slice) != 3 {- t.Errorf("expected 3 elements, got %d", len(slice))- }-- // Non-string returns unchanged- if splitKwdHash(42) != 42 {- t.Error("non-string should return unchanged")- }-}--func TestApplyCommonChunkMapping(t *testing.T) {- result := make(map[string]interface{})-- // content -> content_with_weight- if !applyCommonChunkMapping(result, "content", "hello") {- t.Error("content should be handled")- }- if result["content_with_weight"] != "hello" {- t.Errorf("content_with_weight = %v, want hello", result["content_with_weight"])- }-- // docnm -> docnm_kwd- result = make(map[string]interface{})- applyCommonChunkMapping(result, "docnm", "mydoc")- if result["docnm_kwd"] != "mydoc" {- t.Errorf("docnm_kwd = %v, want mydoc", result["docnm_kwd"])- }-- // important_keywords -> important_kwd- result = make(map[string]interface{})- applyCommonChunkMapping(result, "important_keywords", []interface{}{"kw1"})- if _, ok := result["important_kwd"]; !ok {- t.Error("important_kwd should be set")- }-- // *_kwd empty handling- result = make(map[string]interface{})- applyCommonChunkMapping(result, "entity_kwd", "")- if _, ok := result["entity_kwd"]; !ok {- t.Error("entity_kwd should be set even for empty")- }-- // Unknown field should not be handled- result = make(map[string]interface{})- if applyCommonChunkMapping(result, "unknown_field", "val") {- t.Error("unknown_field should not be handled")- }- if len(result) != 0 {- t.Error("result should be empty for unhandled field")- }-}--func TestChunkDocRunText(t *testing.T) {- strPtr := func(s string) *string { return &s }- cases := []struct {- name string- run *string- want interface{}- }{- {name: "nil returns nil", run: nil, want: nil},- {name: "0 UNSTART", run: strPtr("0"), want: "UNSTART"},- {name: "1 RUNNING", run: strPtr("1"), want: "RUNNING"},- {name: "2 CANCEL", run: strPtr("2"), want: "CANCEL"},- {name: "3 DONE", run: strPtr("3"), want: "DONE"},- {name: "4 FAIL", run: strPtr("4"), want: "FAIL"},- {name: "5 SCHEDULE", run: strPtr("5"), want: "SCHEDULE"},- {name: "unknown returns raw", run: strPtr("999"), want: "999"},- }- for _, tc := range cases {- t.Run(tc.name, func(t *testing.T) {- got := ChunkDocRunText(tc.run)- if got != tc.want {- t.Fatalf("chunkDocRunText(%v) = %v, want %v", tc.run, got, tc.want)- }- })- }-}diff --git a/internal/service/document/document.go b/internal/service/document/document.goindex cac341d018..f90b5d39b7 100644--- a/internal/service/document/document.go+++ b/internal/service/document/document.go@@ -75,7 +75,6 @@ func NewDocumentService() *DocumentService {// UpdateDocumentRequest update document requesttype UpdateDocumentRequest struct {Name *string `json:"name"`- Run *string `json:"run"`TokenNum *int64 `json:"token_num"`ChunkNum *int64 `json:"chunk_num"`Progress *float64 `json:"progress"`@@ -101,7 +100,7 @@ type DocumentResponse struct {ProcessBeginAt *time.Time `json:"process_begin_at,omitempty"`ProcessDuration float64 `json:"process_duration"`Suffix string `json:"suffix"`- Run *string `json:"run,omitempty"`+ IngestionStatus string `json:"ingestion_status"`Status *string `json:"status,omitempty"`CreatedAt string `json:"created_at"`UpdatedAt string `json:"updated_at"`@@ -161,7 +160,7 @@ type UpdateDatasetDocumentResponse struct {ContentHash *string `json:"content_hash,omitempty"`MetaFields map[string]interface{} `json:"meta_fields,omitempty"`Suffix string `json:"suffix"`- Run string `json:"run"`+ IngestionStatus string `json:"ingestion_status"`Status *string `json:"status,omitempty"`CreateTime *int64 `json:"create_time,omitempty"`CreateDate *time.Time `json:"create_date,omitempty"`diff --git a/internal/service/document/document_crud.go b/internal/service/document/document_crud.goindex 502eb518c2..88b9a98094 100644--- a/internal/service/document/document_crud.go+++ b/internal/service/document/document_crud.go@@ -111,7 +111,7 @@ func (s *DocumentService) GetDocumentByID(ctx context.Context, id string) (*Docureturn nil, err}- return s.toResponse(document), nil+ return s.toResponse(ctx, document)}// UpdateDocument update document@@ -134,7 +134,6 @@ func (s *DocumentService) UpdateDocument(ctx context.Context, id string, req *UptokenNum: req.TokenNum,tokenNumRequestName: "token_num",progress: req.Progress,- run: req.Run,progressMsg: req.ProgressMsg,}); err != nil {return common.CodeDataError, err@@ -200,13 +199,12 @@ func (s *DocumentService) ApplyDocCounts(ctx context.Context, docID, kbID string}// UpdateRunProgress mirrors a pipeline run's live progress into the document-// row so the document-list endpoint (which reads document.progress/run/+// row so the document-list endpoint (which reads document.progress/// progress_msg) reflects in-flight Go pipeline progress. Best-effort by// design; callers log and continue on error.-func (s *DocumentService) UpdateRunProgress(ctx context.Context, docID string, progress float64, run, progressMsg string) error {+func (s *DocumentService) UpdateRunProgress(ctx context.Context, docID string, progress float64, progressMsg string) error {updates := map[string]interface{}{"progress": progress,- "run": run,"progress_msg": progressMsg,}if doc, err := s.documentDAO.GetByID(ctx, dao.DB, docID); err != nil {@@ -221,13 +219,12 @@ func (s *DocumentService) UpdateRunProgress(ctx context.Context, docID string, preturn s.documentDAO.UpdateByID(ctx, dao.DB, docID, updates)}-// UpdateRunState mirrors live progress and status into the document row when+// UpdateRunState mirrors live progress into the document row when// the existing progress log cannot be read. It intentionally leaves the log// untouched so a later event can retry seeding and append it safely.-func (s *DocumentService) UpdateRunState(ctx context.Context, docID string, progress float64, run string) error {+func (s *DocumentService) UpdateRunState(ctx context.Context, docID string, progress float64) error {updates := map[string]interface{}{"progress": progress,- "run": run,}if doc, err := s.documentDAO.GetByID(ctx, dao.DB, docID); err != nil {return errdiff --git a/internal/service/document/document_dataset_update.go b/internal/service/document/document_dataset_update.goindex ea2929dbe9..f578df14c1 100644--- a/internal/service/document/document_dataset_update.go+++ b/internal/service/document/document_dataset_update.go@@ -235,21 +235,38 @@ func (s *DocumentService) UpdateDatasetDocument(ctx context.Context, userID, datmetaFields, _ = s.GetDocumentMetadataByID(ctx, updatedDoc.ID)}- return s.toUpdateDatasetDocumentResponse(updatedDoc, metaFields), common.CodeSuccess, nil+ resp, err := s.toUpdateDatasetDocumentResponse(ctx, updatedDoc, metaFields)+ if err != nil {+ return nil, common.CodeServerError, err+ }+ return resp, common.CodeSuccess, nil}// validateDocumentModifiable rejects configuration edits while the document is-// actively parsing or scheduled. Editing a document that is mid-parse races-// with the running worker and can leave the document stuck and undeletable.-func (s *DocumentService) validateDocumentModifiable(doc *entity.Document) (common.ErrorCode, error) {- if doc.Run != nil {- run := entity.TaskStatus(*doc.Run)- if !entity.DocumentModifiableStatuses[run] {- return common.CodeDataError, fmt.Errorf(- "document is currently %q and cannot be modified; stop parsing or wait for it to finish before updating its configuration", run)- }+// actively parsing or scheduled.+func (s *DocumentService) validateDocumentModifiable(ctx context.Context, doc *entity.Document) (common.ErrorCode, error) {+ if s.ingestionTaskDAO == nil {+ return common.CodeServerError, errors.New("ingestion task DAO not initialized")+ }+ if doc == nil || doc.ID == "" {+ return common.CodeDataError, errors.New("document is nil or has empty ID")+ }+ task, err := s.ingestionTaskDAO.GetByDocumentID(ctx, dao.DB, doc.ID)+ if err != nil {+ return common.CodeServerError, fmt.Errorf("failed to get ingestion task for document %s: %w", doc.ID, err)+ }+ if task == nil {+ return common.CodeSuccess, nil+ }+ switch {+ case common.IsActiveTaskStatus(task.Status):+ return common.CodeDataError, fmt.Errorf(+ "document is currently %q and cannot be modified; stop parsing or wait for it to finish before updating its configuration", task.Status)+ case common.IsTerminalTaskStatus(task.Status):+ return common.CodeSuccess, nil+ default:+ return common.CodeDataError, fmt.Errorf("document has unrecognized task status %q", task.Status)}- return common.CodeSuccess, nil}func (s *DocumentService) validateDatasetDocumentUpdate(ctx context.Context, datasetID, documentID, userID string, doc *entity.Document, req *UpdateDatasetDocumentRequest, present map[string]bool) (common.ErrorCode, error) {@@ -262,7 +279,7 @@ func (s *DocumentService) validateDatasetDocumentUpdate(ctx context.Context, dat// chunk_method, pipeline_id, enabled, meta_fields) is blocked so the// in-flight parser never reads a config that changed underneath it.if len(present) > 0 {- if code, err := s.validateDocumentModifiable(doc); err != nil {+ if code, err := s.validateDocumentModifiable(ctx, doc); err != nil {return code, err}}@@ -456,10 +473,20 @@ func (s *DocumentService) updateDocumentParserConfig(ctx context.Context, docume})}-func (s *DocumentService) toUpdateDatasetDocumentResponse(doc *entity.Document, metaFields map[string]interface{}) *UpdateDatasetDocumentResponse {+func (s *DocumentService) toUpdateDatasetDocumentResponse(ctx context.Context, doc *entity.Document, metaFields map[string]interface{}) (*UpdateDatasetDocumentResponse, error) {if metaFields == nil {metaFields = map[string]interface{}{}}+ ingestionStatus := "UNSTART"+ if s.ingestionTaskDAO != nil && doc != nil && doc.ID != "" {+ task, err := s.ingestionTaskDAO.GetByDocumentID(ctx, dao.DB, doc.ID)+ if err != nil {+ return nil, fmt.Errorf("failed to get ingestion task for document %s: %w", doc.ID, err)+ }+ if task != nil && task.Status != "" {+ ingestionStatus = task.Status+ }+ }return &UpdateDatasetDocumentResponse{ID: doc.ID,Thumbnail: doc.Thumbnail,@@ -482,31 +509,13 @@ func (s *DocumentService) toUpdateDatasetDocumentResponse(doc *entity.Document,ContentHash: doc.ContentHash,MetaFields: metaFields,Suffix: doc.Suffix,- Run: mapDocumentRunStatus(doc.Run),+ IngestionStatus: ingestionStatus,Status: doc.Status,CreateTime: doc.CreateTime,CreateDate: doc.CreateDate,UpdateTime: doc.UpdateTime,UpdateDate: doc.UpdateDate,- }-}--func mapDocumentRunStatus(run *string) string {- if run == nil {- return "UNSTART"- }- switch *run {- case string(entity.TaskStatusRunning):- return "RUNNING"- case string(entity.TaskStatusCancel):- return "CANCEL"- case string(entity.TaskStatusDone):- return "DONE"- case string(entity.TaskStatusFail):- return "FAIL"- default:- return "UNSTART"- }+ }, nil}// validDocumentChunkMethods mirrors Python's UpdateDocumentReq chunk_method set.diff --git a/internal/service/document/document_ingest.go b/internal/service/document/document_ingest.goindex 5a969bc426..36cf515480 100644--- a/internal/service/document/document_ingest.go+++ b/internal/service/document/document_ingest.go@@ -113,7 +113,6 @@ func (s *DocumentService) Ingest(ctx context.Context, userID string, req *Ingestreturn common.CodeDataError, err}if err = s.documentDAO.UpdateByID(ctx, dao.DB, doc.ID, map[string]interface{}{- "run": string(entity.TaskStatusCancel),"progress": 0,}); err != nil {common.Error(fmt.Sprintf("go side, doc %s, UpdateByID failed", doc.ID), err)@@ -125,7 +124,6 @@ func (s *DocumentService) Ingest(ctx context.Context, userID string, req *Ingest// Delete-only: user asked to remove prior parse results without// starting a new parse. RUNNING already continued above.if err = s.documentDAO.UpdateByID(ctx, dao.DB, doc.ID, map[string]interface{}{- "run": run,"progress": 0,}); err != nil {common.Error(fmt.Sprintf("go side, doc %s, UpdateByID failed", doc.ID), err)diff --git a/internal/service/document/document_list.go b/internal/service/document/document_list.goindex e01c9358f9..52b163b5c2 100644--- a/internal/service/document/document_list.go+++ b/internal/service/document/document_list.go@@ -29,9 +29,28 @@ func (s *DocumentService) ListDocuments(ctx context.Context, page, pageSize int)return nil, 0, err}+ docIDs := make([]string, 0, len(documents))+ for _, doc := range documents {+ if doc != nil && doc.ID != "" {+ docIDs = append(docIDs, doc.ID)+ }+ }+ var taskMap map[string]*entity.IngestionTask+ if s.ingestionTaskDAO != nil && len(docIDs) > 0 {+ var err error+ taskMap, err = s.ingestionTaskDAO.GetLatestByDocumentIDs(ctx, dao.DB, docIDs)+ if err != nil {+ return nil, 0, fmt.Errorf("failed to get ingestion tasks for documents: %w", err)+ }+ }+responses := make([]*DocumentResponse, len(documents))for i, doc := range documents {- responses[i] = s.toResponse(doc)+ var task *entity.IngestionTask+ if taskMap != nil && doc != nil {+ task = taskMap[doc.ID]+ }+ responses[i] = s.toResponseWithTask(doc, task)}return responses, total, nil@@ -177,16 +196,49 @@ func (s *DocumentService) GetDocumentsByAuthorID(ctx context.Context, authorID,return nil, 0, err}+ docIDs := make([]string, 0, len(documents))+ for _, doc := range documents {+ if doc != nil && doc.ID != "" {+ docIDs = append(docIDs, doc.ID)+ }+ }+ var taskMap map[string]*entity.IngestionTask+ if s.ingestionTaskDAO != nil && len(docIDs) > 0 {+ var err error+ taskMap, err = s.ingestionTaskDAO.GetLatestByDocumentIDs(ctx, dao.DB, docIDs)+ if err != nil {+ return nil, 0, fmt.Errorf("failed to get ingestion tasks for documents: %w", err)+ }+ }+responses := make([]*DocumentResponse, len(documents))for i, doc := range documents {- responses[i] = s.toResponse(doc)+ var task *entity.IngestionTask+ if taskMap != nil && doc != nil {+ task = taskMap[doc.ID]+ }+ responses[i] = s.toResponseWithTask(doc, task)}return responses, total, nil}// toResponse convert model.Document to DocumentResponse-func (s *DocumentService) toResponse(doc *entity.Document) *DocumentResponse {+func (s *DocumentService) toResponse(ctx context.Context, doc *entity.Document) (*DocumentResponse, error) {+ if s.ingestionTaskDAO == nil || doc == nil || doc.ID == "" {+ return s.toResponseWithTask(doc, nil), nil+ }+ task, err := s.ingestionTaskDAO.GetByDocumentID(ctx, dao.DB, doc.ID)+ if err != nil {+ return nil, fmt.Errorf("get ingestion task for document %s: %w", doc.ID, err)+ }+ return s.toResponseWithTask(doc, task), nil+}++func (s *DocumentService) toResponseWithTask(doc *entity.Document, task *entity.IngestionTask) *DocumentResponse {+ if doc == nil {+ return nil+ }createdAt := ""if doc.CreateTime != nil {// Check if timestamp is in milliseconds (13 digits) or seconds (10 digits)@@ -208,6 +260,10 @@ func (s *DocumentService) toResponse(doc *entity.Document) *DocumentResponse {}updatedAt = time.Unix(ts, 0).Format("2006-01-02 15:04:05")}+ ingestionStatus := "UNSTART"+ if task != nil && task.Status != "" {+ ingestionStatus = task.Status+ }return &DocumentResponse{ID: doc.ID,Name: doc.Name,@@ -226,7 +282,7 @@ func (s *DocumentService) toResponse(doc *entity.Document) *DocumentResponse {ProcessBeginAt: doc.ProcessBeginAt,ProcessDuration: doc.ProcessDuration,Suffix: doc.Suffix,- Run: doc.Run,+ IngestionStatus: ingestionStatus,Status: doc.Status,CreatedAt: createdAt,UpdatedAt: updatedAt,diff --git a/internal/service/document/document_modify_guard_test.go b/internal/service/document/document_modify_guard_test.goindex 0073739420..915ce85e55 100644--- a/internal/service/document/document_modify_guard_test.go+++ b/internal/service/document/document_modify_guard_test.go@@ -24,31 +24,40 @@ import ("ragflow/internal/entity")-// TestValidateDocumentModifiable_RunStatuses pins the allowed/disallowed run-// statuses for editing a document via UpdateDatasetDocument. RUNNING ("1") and-// SCHEDULE ("5") must be rejected; UNSTART/CANCEL/DONE/FAIL may be edited.-// A nil Run (gorm default "0" UNSTART) is treated as editable.-func TestValidateDocumentModifiable_RunStatuses(t *testing.T) {+// TestValidateDocumentModifiable_TaskStatuses pins the allowed/disallowed task+// statuses for editing a document via UpdateDatasetDocument.+// CREATED, SCHEDULED, RUNNING, and STOPPING must be rejected;+// no task, COMPLETED, STOPPED, and FAILED may be edited.+func TestValidateDocumentModifiable_TaskStatuses(t *testing.T) {cases := []struct {- name string- run *string- wantCode common.ErrorCode- wantErr bool+ name string+ taskStatus *string+ wantCode common.ErrorCode+ wantErr bool}{- {"nil run defaults to unstart", nil, common.CodeSuccess, false},- {"unstart editable", sptr(string(entity.TaskStatusUnstart)), common.CodeSuccess, false},- {"cancel editable", sptr(string(entity.TaskStatusCancel)), common.CodeSuccess, false},- {"done editable", sptr(string(entity.TaskStatusDone)), common.CodeSuccess, false},- {"fail editable", sptr(string(entity.TaskStatusFail)), common.CodeSuccess, false},- {"running rejected", sptr(string(entity.TaskStatusRunning)), common.CodeDataError, true},- {"schedule rejected", sptr(string(entity.TaskStatusSchedule)), common.CodeDataError, true},+ {"no task defaults to unstart", nil, common.CodeSuccess, false},+ {"created rejected", sptr(common.CREATED), common.CodeDataError, true},+ {"scheduled rejected", sptr(common.SCHEDULED), common.CodeDataError, true},+ {"running rejected", sptr(common.RUNNING), common.CodeDataError, true},+ {"stopping rejected", sptr(common.STOPPING), common.CodeDataError, true},+ {"completed editable", sptr(common.COMPLETED), common.CodeSuccess, false},+ {"stopped editable", sptr(common.STOPPED), common.CodeSuccess, false},+ {"failed editable", sptr(common.FAILED), common.CodeSuccess, false},}- svc := &DocumentService{}for _, tc := range cases {t.Run(tc.name, func(t *testing.T) {- doc := &entity.Document{Run: tc.run}- code, err := svc.validateDocumentModifiable(doc)+ db := setupServiceTestDB(t)+ pushServiceDB(t, db)+ insertTestKB(t, "kb-1", "tenant-1", 1, 0, 0)+ insertTestDoc(t, "doc-1", "kb-1", 0, 0)+ if tc.taskStatus != nil {+ insertTestIngestionTaskWithStatus(t, "task-1", "user-1", "doc-1", "kb-1", *tc.taskStatus)+ }+ svc := testDocumentService(t)+ ctx := t.Context()+ doc := &entity.Document{ID: "doc-1"}+ code, err := svc.validateDocumentModifiable(ctx, doc)if tc.wantErr {if err == nil {t.Fatalf("expected error, got nil")@@ -71,20 +80,21 @@ func TestValidateDocumentModifiable_RunStatuses(t *testing.T) {}}-// updateDatasetDocumentRejected asserts that editing a document in the given-// run status with the supplied request is rejected by the run-state guard.-func updateDatasetDocumentRejected(t *testing.T, run string, req *UpdateDatasetDocumentRequest, present map[string]bool) {+// updateDatasetDocumentRejected asserts that editing a document with the given+// task status is rejected by the modifiable guard.+func updateDatasetDocumentRejected(t *testing.T, status string, req *UpdateDatasetDocumentRequest, present map[string]bool) {t.Helper()db := setupServiceTestDB(t)pushServiceDB(t, db)insertTestKB(t, "kb-1", "tenant-1", 1, 0, 0)- insertTestDocWithRun(t, "doc-1", "kb-1", run, 0, 0)+ insertTestDoc(t, "doc-1", "kb-1", 0, 0)+ insertTestIngestionTaskWithStatus(t, "task-1", "user-1", "doc-1", "kb-1", status)svc := testDocumentService(t)ctx := t.Context()_, code, err := svc.UpdateDatasetDocument(ctx, "tenant-1", "kb-1", "doc-1", req, present)if err == nil {- t.Fatalf("expected run-state rejection, got nil error (code=%v)", code)+ t.Fatalf("expected task-state rejection, got nil error (code=%v)", code)}if code != common.CodeDataError {t.Fatalf("code = %v, want %v", code, common.CodeDataError)@@ -94,15 +104,17 @@ func updateDatasetDocumentRejected(t *testing.T, run string, req *UpdateDatasetD}}-// updateDatasetDocumentAllowed asserts that editing a document in an editable-// run status with the supplied request is NOT blocked by the run-state guard-// (it may still fail later validation, but not with the run-state error).-func updateDatasetDocumentAllowed(t *testing.T, run string, req *UpdateDatasetDocumentRequest, present map[string]bool) {+// updateDatasetDocumentAllowed asserts that editing a document with an editable+// task status is NOT blocked by the modifiable guard.+func updateDatasetDocumentAllowed(t *testing.T, status string, req *UpdateDatasetDocumentRequest, present map[string]bool) {t.Helper()db := setupServiceTestDB(t)pushServiceDB(t, db)insertTestKB(t, "kb-1", "tenant-1", 1, 0, 0)- insertTestDocWithRun(t, "doc-1", "kb-1", run, 0, 0)+ insertTestDoc(t, "doc-1", "kb-1", 0, 0)+ if status != "" {+ insertTestIngestionTaskWithStatus(t, "task-1", "user-1", "doc-1", "kb-1", status)+ }svc := testDocumentService(t)ctx := t.Context()@@ -116,75 +128,76 @@ func updateDatasetDocumentAllowed(t *testing.T, run string, req *UpdateDatasetDo}func TestUpdateDatasetDocumentRejectsRunningParserConfig(t *testing.T) {- updateDatasetDocumentRejected(t, string(entity.TaskStatusRunning),+ updateDatasetDocumentRejected(t, common.RUNNING,&UpdateDatasetDocumentRequest{ParserConfig: map[string]any{"chunk_token_num": float64(128)}},map[string]bool{"parser_config": true})}func TestUpdateDatasetDocumentRejectsRunningChunkMethod(t *testing.T) {cm := "naive"- updateDatasetDocumentRejected(t, string(entity.TaskStatusRunning),+ updateDatasetDocumentRejected(t, common.RUNNING,&UpdateDatasetDocumentRequest{ChunkMethod: &cm},map[string]bool{"chunk_method": true})}func TestUpdateDatasetDocumentRejectsRunningRename(t *testing.T) {name := "renamed.txt"- updateDatasetDocumentRejected(t, string(entity.TaskStatusRunning),+ updateDatasetDocumentRejected(t, common.RUNNING,&UpdateDatasetDocumentRequest{Name: &name},map[string]bool{"name": true})}func TestUpdateDatasetDocumentRejectsRunningEnabled(t *testing.T) {enabled := 0- updateDatasetDocumentRejected(t, string(entity.TaskStatusRunning),+ updateDatasetDocumentRejected(t, common.RUNNING,&UpdateDatasetDocumentRequest{Enabled: &enabled},map[string]bool{"enabled": true})}func TestUpdateDatasetDocumentRejectsScheduledRename(t *testing.T) {name := "renamed.txt"- updateDatasetDocumentRejected(t, string(entity.TaskStatusSchedule),+ updateDatasetDocumentRejected(t, common.SCHEDULED,&UpdateDatasetDocumentRequest{Name: &name},map[string]bool{"name": true})}func TestUpdateDatasetDocumentAllowsDoneEnabled(t *testing.T) {enabled := 0- updateDatasetDocumentAllowed(t, string(entity.TaskStatusDone),+ updateDatasetDocumentAllowed(t, common.COMPLETED,&UpdateDatasetDocumentRequest{Enabled: &enabled},map[string]bool{"enabled": true})}func TestUpdateDatasetDocumentAllowsCancelEnabled(t *testing.T) {enabled := 0- updateDatasetDocumentAllowed(t, string(entity.TaskStatusCancel),+ updateDatasetDocumentAllowed(t, common.STOPPED,&UpdateDatasetDocumentRequest{Enabled: &enabled},map[string]bool{"enabled": true})}func TestUpdateDatasetDocumentAllowsFailEnabled(t *testing.T) {enabled := 0- updateDatasetDocumentAllowed(t, string(entity.TaskStatusFail),+ updateDatasetDocumentAllowed(t, common.FAILED,&UpdateDatasetDocumentRequest{Enabled: &enabled},map[string]bool{"enabled": true})}func TestUpdateDatasetDocumentAllowsUnstartEnabled(t *testing.T) {enabled := 0- updateDatasetDocumentAllowed(t, string(entity.TaskStatusUnstart),+ updateDatasetDocumentAllowed(t, "",&UpdateDatasetDocumentRequest{Enabled: &enabled},map[string]bool{"enabled": true})}// TestUpdateDatasetDocumentRunningEmptyPresentNotRejected locks the// len(present) > 0 gate: an empty PATCH (no fields) on a RUNNING doc must not-// be rejected by the run-state guard (it is a no-op, not an edit).+// be rejected by the modifiable guard (it is a no-op, not an edit).func TestUpdateDatasetDocumentRunningEmptyPresentNotRejected(t *testing.T) {db := setupServiceTestDB(t)pushServiceDB(t, db)insertTestKB(t, "kb-1", "tenant-1", 1, 0, 0)- insertTestDocWithRun(t, "doc-1", "kb-1", string(entity.TaskStatusRunning), 0, 0)+ insertTestDoc(t, "doc-1", "kb-1", 0, 0)+ insertTestIngestionTaskWithStatus(t, "task-1", "user-1", "doc-1", "kb-1", common.RUNNING)svc := testDocumentService(t)ctx := t.Context()@@ -201,18 +214,19 @@ func TestUpdateDatasetDocumentRunningEmptyPresentNotRejected(t *testing.T) {// TestUpdateDatasetDocumentRejectsRunningMetaFields pins that every editable// field is blocked while RUNNING, including meta_fields.func TestUpdateDatasetDocumentRejectsRunningMetaFields(t *testing.T) {- updateDatasetDocumentRejected(t, string(entity.TaskStatusRunning),+ updateDatasetDocumentRejected(t, common.RUNNING,&UpdateDatasetDocumentRequest{MetaFields: map[string]any{"author": "x"}},map[string]bool{"meta_fields": true})}// TestUpdateDatasetDocumentAllowsDoneRename proves an editable status is not-// over-restricted: renaming a DONE document succeeds (no run-state error).+// over-restricted: renaming a DONE document succeeds.func TestUpdateDatasetDocumentAllowsDoneRename(t *testing.T) {db := setupServiceTestDB(t)pushServiceDB(t, db)insertTestKB(t, "kb-1", "tenant-1", 1, 0, 0)insertNamedTestDoc(t, "doc-1", "kb-1", "orig.txt", 0, 0)+ insertTestIngestionTaskWithStatus(t, "task-1", "user-1", "doc-1", "kb-1", common.COMPLETED)name := "renamed.txt"svc := testDocumentService(t)diff --git a/internal/service/document/document_parse.go b/internal/service/document/document_parse.goindex d4f4f826c8..cc1b29eaa3 100644--- a/internal/service/document/document_parse.go+++ b/internal/service/document/document_parse.go@@ -54,10 +54,7 @@ func lockDocumentParse(docID string) func() {// StartParseDocuments starts parsing a document via the DSL ingestion// pipeline. It optionally clears prior results (RerunWithDelete), applies// KB config (ApplyKB), validates storage, and enqueues an ingestion task.-// The document run status is NOT set here; service.IngestionTaskService.StartRunning-// sets it to RUNNING when the worker picks up the task and transitions it from-// CREATED or SCHEDULED. Extracted from Ingest so-// other entry points (e.g. ChunkService.Parse)+// Extracted from Ingest so other entry points (e.g. ChunkService.Parse)// can reuse the same start-parse flow.func (s *DocumentService) StartParseDocuments(ctx context.Context, doc *entity.Document, kb *entity.Knowledgebase, userID string, opts StartParseOptions) error {// Validate storage first so we don't clear prior results and then fail@@ -101,7 +98,7 @@ func (s *DocumentService) AssertIngestionTasksTerminal(ctx context.Context, docIif task == nil {continue}- if task.Status == common.RUNNING || task.Status == common.STOPPING {+ if common.IsRunningOrStopping(task.Status) {return fmt.Errorf("document %s ingestion task is %s; stop it and wait for a terminal state before re-parsing", docID, task.Status)}}@@ -207,14 +204,6 @@ func (s *DocumentService) clearDocumentAndKBCountersForRerun(docID, kbID string)})}-func (s *DocumentService) countDoneDocuments(datasetID string) (int64, error) {- var count int64- err := dao.GetDB().Model(&entity.Document{}).- Where("kb_id = ? AND run = ?", datasetID, string(entity.TaskStatusDone)).- Count(&count).Error- return count, err-}-func (s *DocumentService) clearKBChunkNumWhenRerun(doc *entity.Document) error {if doc == nil {return fmt.Errorf("document is nil")@@ -294,7 +283,7 @@ func (s *DocumentService) ParseDocuments(ctx context.Context, datasetID, userID}// StopParseDocuments stops parsing for the given documents in a dataset.-// It sets Redis cancel signals for associated tasks and updates doc.run to CANCEL.+// It requests stop for the associated ingestion tasks.// Returns a map with success_count and optionally errors.func (s *DocumentService) StopParseDocuments(ctx context.Context, datasetID string, docIDs []string) (map[string]interface{}, error) {deduped := common.Deduplicate(docIDs)@@ -382,53 +371,42 @@ func (s *DocumentService) validateDocsInDataset(ctx context.Context, docIDs []streturn docs, nil}-// errParseNotRunning is returned by CancelDocParse when the document is not in-// a cancelable state: its run status is neither RUNNING nor CANCEL and it has-// no in-flight ingestion task. Callers map it to their endpoint-specific-// message (the Python /documents/ingest and /documents/stop messages differ).+// errParseNotRunning is returned by CancelDocParse when the document has no+// in-flight ingestion task and is not already stopped. Callers map it to their+// endpoint-specific message (the Python /documents/ingest and /documents/stop messages differ).var errParseNotRunning = errors.New("parse task is not in running status")// CancelDocParse stops the ingestion task for the document by calling-// RequestStop (STOPPING), then marks the document run status as CANCEL.-// It mirrors the Python cancel precondition: only a document whose run status-// is RUNNING or CANCEL, or one with an in-flight ingestion task-// (CREATED/SCHEDULED/RUNNING/STOPPING), can be canceled; otherwise errParseNotRunning-// is returned. A missing ingestion task is not an error by itself — cancel is-// then a no-op on the task side, matching Python's cancel_all_task_of.+// RequestStop (STOPPING).+// It returns errParseNotRunning if the document has neither an in-flight+// ingestion task (CREATED/SCHEDULED/RUNNING/STOPPING) nor an already stopped+// task (STOPPED).func (s *DocumentService) CancelDocParse(ctx context.Context, doc *entity.Document) error {task, err := s.ingestionTaskDAO.GetByDocumentID(ctx, dao.DB, doc.ID)if err != nil {return fmt.Errorf("failed to get ingestion task for %s: %w", doc.ID, err)}- docRun := ""- if doc.Run != nil {- docRun = *doc.Run- }- inFlight := task != nil && (task.Status == common.CREATED || task.Status == common.SCHEDULED || task.Status == common.RUNNING || task.Status == common.STOPPING)- if docRun != string(entity.TaskStatusRunning) && docRun != string(entity.TaskStatusCancel) && !inFlight {+ inFlight := task != nil && common.IsActiveTaskStatus(task.Status)+ isStopped := task != nil && task.Status == common.STOPPED+ if !inFlight && !isStopped {return errParseNotRunning}- if task != nil {+ if inFlight {if _, err = s.ingestionTaskSvc.RequestStop(ctx, task.ID); err != nil {return fmt.Errorf("failed to stop ingestion task %s: %w", task.ID, err)}}- if upErr := s.documentDAO.UpdateByID(ctx, dao.DB, doc.ID, map[string]interface{}{"run": string(entity.TaskStatusCancel)}); upErr != nil {- return fmt.Errorf("failed to update document %s: %w", doc.ID, upErr)- }return nil}func (s *DocumentService) resetDocumentForReparse(ctx context.Context, doc *entity.Document, tenantID string, parserID *string, pipelineID *string) error {progressMsg := ""- run := string(entity.TaskStatusUnstart)updates := map[string]interface{}{"progress": 0,"progress_msg": progressMsg,- "run": run,}if parserID != nil {updates["parser_id"] = *parserIDdiff --git a/internal/service/document/document_test.go b/internal/service/document/document_test.goindex d5fc1525a5..f5be393bc0 100644--- a/internal/service/document/document_test.go+++ b/internal/service/document/document_test.go@@ -1203,25 +1203,6 @@ func TestDeleteDocuments_Deduplicate(t *testing.T) {}}-// insertTestDocWithRun inserts a document with the given Run status for StopParseDocuments tests.-func insertTestDocWithRun(t *testing.T, id, kbID, run string, tokenNum, chunkNum int64) {- t.Helper()- doc := &entity.Document{- ID: id,- KbID: kbID,- ParserID: "naive",- ParserConfig: entity.JSONMap{},- TokenNum: tokenNum,- ChunkNum: chunkNum,- Suffix: ".txt",- Status: sptr("1"),- Run: &run,- }- if err := dao.DB.Create(doc).Error; err != nil {- t.Fatalf("insert test doc: %v", err)- }-}-// insertTestTaskWithProgress inserts a task with the given progress value.func insertTestTaskWithProgress(t *testing.T, id, docID string, progress float64) {t.Helper()@@ -1309,7 +1290,7 @@ func TestStartParseDocumentsSerializesConcurrentReruns(t *testing.T) {db := setupConcurrentServiceTestDB(t)pushServiceDB(t, db)insertTestKB(t, "kb-1", "tenant-1", 0, 10, 5)- insertTestDocWithRun(t, "doc-1", "kb-1", string(entity.TaskStatusDone), 10, 5)+ insertTestDoc(t, "doc-1", "kb-1", 10, 5)if err := db.Model(&entity.Document{}).Where("id = ?", "doc-1").Update("location", "loc-1").Error; err != nil {t.Fatalf("set document location: %v", err)}@@ -1380,7 +1361,7 @@ func TestStopParseDocuments_Success(t *testing.T) {pushServiceDB(t, db)insertTestKB(t, "kb-1", "tenant-1", 1, 10, 5)- insertTestDocWithRun(t, "doc-1", "kb-1", string(entity.TaskStatusRunning), 10, 5)+ insertTestDoc(t, "doc-1", "kb-1", 10, 5)insertTestIngestionTask(t, "task-1", "user-1", "doc-1", "kb-1")svc := testDocumentService(t)@@ -1398,13 +1379,13 @@ func TestStopParseDocuments_Success(t *testing.T) {t.Fatalf("expected success_count=1, got %d", sc)}- // Verify document run status updated to CANCEL- doc, _ := dao.NewDocumentDAO().GetByID(ctx, db, "doc-1")- if doc == nil || doc.Run == nil {- t.Fatal("doc not found or run is nil")+ // Verify ingestion task status updated to STOPPED+ task, err := dao.NewIngestionTaskDAO().GetByID(ctx, db, "task-1")+ if err != nil {+ t.Fatalf("load task: %v", err)}- if *doc.Run != string(entity.TaskStatusCancel) {- t.Fatalf("expected run=%q, got %q", string(entity.TaskStatusCancel), *doc.Run)+ if task.Status != common.STOPPED {+ t.Fatalf("expected task status=%s, got %s", common.STOPPED, task.Status)}}@@ -1413,9 +1394,9 @@ func TestStopParseDocuments_CancelStatus(t *testing.T) {pushServiceDB(t, db)insertTestKB(t, "kb-1", "tenant-1", 1, 10, 5)- // Doc is already in CANCEL state — should still be accepted- insertTestDocWithRun(t, "doc-1", "kb-1", string(entity.TaskStatusCancel), 10, 5)- insertTestIngestionTask(t, "task-1", "user-1", "doc-1", "kb-1")+ // Task is already in STOPPED state — should still be accepted+ insertTestDoc(t, "doc-1", "kb-1", 10, 5)+ insertTestIngestionTaskWithStatus(t, "task-1", "user-1", "doc-1", "kb-1", common.STOPPED)svc := testDocumentService(t)ctx := t.Context()@@ -1435,8 +1416,8 @@ func TestStopParseDocuments_NotRunningOrCancel(t *testing.T) {pushServiceDB(t, db)insertTestKB(t, "kb-1", "tenant-1", 1, 10, 5)- // Doc with Run="0" (UNSTART) and no unfinished tasks → cannot cancel- insertTestDocWithRun(t, "doc-1", "kb-1", string(entity.TaskStatusUnstart), 10, 5)+ // Doc with no task → cannot cancel+ insertTestDoc(t, "doc-1", "kb-1", 10, 5)svc := testDocumentService(t)ctx := t.Context()@@ -1463,8 +1444,8 @@ func TestStopParseDocuments_UnfinishedTask(t *testing.T) {pushServiceDB(t, db)insertTestKB(t, "kb-1", "tenant-1", 1, 10, 5)- // Doc with Run="0" → cancelDocParse calls RequestStop on the ingestion task.- insertTestDocWithRun(t, "doc-1", "kb-1", string(entity.TaskStatusUnstart), 10, 5)+ // Doc with active ingestion task → cancelDocParse calls RequestStop on the ingestion task.+ insertTestDoc(t, "doc-1", "kb-1", 10, 5)insertTestIngestionTask(t, "task-1", "user-1", "doc-1", "kb-1")svc := testDocumentService(t)@@ -1486,7 +1467,7 @@ func TestStopParseDocuments_WrongDataset(t *testing.T) {insertTestKB(t, "kb-1", "tenant-1", 1, 10, 5)insertTestKB(t, "kb-2", "tenant-1", 1, 10, 5)- insertTestDocWithRun(t, "doc-1", "kb-2", string(entity.TaskStatusRunning), 10, 5)+ insertTestDoc(t, "doc-1", "kb-2", 10, 5)svc := testDocumentService(t)ctx := t.Context()@@ -1529,7 +1510,7 @@ func TestStopParseDocuments_Deduplicate(t *testing.T) {pushServiceDB(t, db)insertTestKB(t, "kb-1", "tenant-1", 1, 10, 5)- insertTestDocWithRun(t, "doc-1", "kb-1", string(entity.TaskStatusRunning), 10, 5)+ insertTestDoc(t, "doc-1", "kb-1", 10, 5)insertTestIngestionTask(t, "task-1", "user-1", "doc-1", "kb-1")svc := testDocumentService(t)@@ -2075,7 +2056,6 @@ func TestUpdateDocumentRejectsImmutableFieldChanges(t *testing.T) {wantErr string}{{name: "progress", request: UpdateDocumentRequest{Progress: float64Ptr(0)}, wantErr: "can't change `progress`"},- {name: "run", request: UpdateDocumentRequest{Run: sptr("0")}, wantErr: "can't change `run`"},{name: "progress message", request: UpdateDocumentRequest{ProgressMsg: sptr("reset")}, wantErr: "can't change `progress_msg`"},{name: "chunk count", request: UpdateDocumentRequest{ChunkNum: int64Ptr(0)}, wantErr: "can't change `chunk_num`"},{name: "token count", request: UpdateDocumentRequest{TokenNum: int64Ptr(0)}, wantErr: "can't change `token_num`"},@@ -2086,11 +2066,9 @@ func TestUpdateDocumentRejectsImmutableFieldChanges(t *testing.T) {db := setupServiceTestDB(t)pushServiceDB(t, db)insertTestDoc(t, "doc-1", "kb-1", 10, 5)- run := "3"progressMsg := "parsing"if err := db.Model(&entity.Document{}).Where("id = ?", "doc-1").Updates(map[string]interface{}{"progress": 0.5,- "run": run,"progress_msg": progressMsg,}).Error; err != nil {t.Fatalf("prepare document: %v", err)@@ -2117,11 +2095,9 @@ func TestUpdateDocumentUsesSharedRenamePath(t *testing.T) {insertNamedTestDoc(t, "doc-1", "kb-1", "old.pdf", 10, 5)insertTestFile(t, "file-1", "folder-1", "old.pdf", sptr("old.pdf"))insertTestFile2Document(t, "f2d-1", "file-1", "doc-1")- run := "3"progressMsg := "complete"if err := db.Model(&entity.Document{}).Where("id = ?", "doc-1").Updates(map[string]interface{}{"progress": 1.0,- "run": run,"progress_msg": progressMsg,}).Error; err != nil {t.Fatalf("prepare document: %v", err)@@ -2133,7 +2109,6 @@ func TestUpdateDocumentUsesSharedRenamePath(t *testing.T) {tokenNum := int64(10)req := &UpdateDocumentRequest{Name: &newName,- Run: &run,TokenNum: &tokenNum,ChunkNum: &chunkNum,Progress: &progress,@@ -2158,7 +2133,7 @@ func TestUpdateDocumentUsesSharedRenamePath(t *testing.T) {if file.Name != newName {t.Fatalf("file name = %q, want %q", file.Name, newName)}- if doc.Progress != progress || doc.Run == nil || *doc.Run != run || doc.ProgressMsg == nil || *doc.ProgressMsg != progressMsg || doc.ChunkNum != chunkNum || doc.TokenNum != tokenNum {+ if doc.Progress != progress || doc.ProgressMsg == nil || *doc.ProgressMsg != progressMsg || doc.ChunkNum != chunkNum || doc.TokenNum != tokenNum {t.Fatalf("immutable fields changed: %#v", doc)}}@@ -2243,8 +2218,8 @@ func TestUpdateDatasetDocumentParserIDResetsForReparse(t *testing.T) {if err != nil {t.Fatalf("UpdateDatasetDocument failed: code=%v err=%v", code, err)}- if resp.ParserID != chunkMethod || resp.Run != "UNSTART" || resp.TokenCount != 0 || resp.ChunkCount != 0 {- t.Fatalf("response = %+v, want method=%s run=UNSTART counts=0", resp, chunkMethod)+ if resp.ParserID != chunkMethod || resp.IngestionStatus != "UNSTART" || resp.TokenCount != 0 || resp.ChunkCount != 0 {+ t.Fatalf("response = %+v, want method=%s ingestion_status=UNSTART counts=0", resp, chunkMethod)}doc, _ := dao.NewDocumentDAO().GetByID(ctx, db, "doc-1")@@ -2295,7 +2270,7 @@ func TestClearDocumentParseResultsClearsCountersTasksAndChunks(t *testing.T) {db := setupServiceTestDB(t)pushServiceDB(t, db)insertTestKB(t, "kb-1", "tenant-1", 1, 10, 5)- insertTestDocWithRun(t, "doc-1", "kb-1", string(entity.TaskStatusDone), 10, 5)+ insertTestDoc(t, "doc-1", "kb-1", 10, 5)insertTestIngestionTaskWithStatus(t, "task-1", "user-1", "doc-1", "kb-1", common.COMPLETED)ctx := t.Context()@@ -2354,7 +2329,7 @@ func TestClearDocumentParseResultsIsIdempotentForStaleDocSnapshot(t *testing.T)db := setupServiceTestDB(t)pushServiceDB(t, db)insertTestKB(t, "kb-1", "tenant-1", 1, 10, 5)- insertTestDocWithRun(t, "doc-1", "kb-1", string(entity.TaskStatusDone), 10, 5)+ insertTestDoc(t, "doc-1", "kb-1", 10, 5)ctx := t.Context()staleDoc, err := dao.NewDocumentDAO().GetByID(ctx, db, "doc-1")@@ -2390,7 +2365,7 @@ func TestClearDocumentParseResults_RejectsNonTerminalIngestionTask(t *testing.T)db := setupServiceTestDB(t)pushServiceDB(t, db)insertTestKB(t, "kb-1", "tenant-1", 1, 10, 5)- insertTestDocWithRun(t, "doc-1", "kb-1", string(entity.TaskStatusRunning), 10, 5)+ insertTestDoc(t, "doc-1", "kb-1", 10, 5)insertTestIngestionTaskWithStatus(t, "task-1", "user-1", "doc-1", "kb-1", status)ctx := t.Context()doc, err := dao.NewDocumentDAO().GetByID(ctx, db, "doc-1")@@ -2415,7 +2390,7 @@ func TestClearDocumentParseResultsDoesNotClearResultsWhenTaskStartsRunning(t *tedb := setupServiceTestDB(t)pushServiceDB(t, db)insertTestKB(t, "kb-1", "tenant-1", 1, 10, 5)- insertTestDocWithRun(t, "doc-1", "kb-1", string(entity.TaskStatusDone), 10, 5)+ insertTestDoc(t, "doc-1", "kb-1", 10, 5)insertTestIngestionTaskWithStatus(t, "task-1", "user-1", "doc-1", "kb-1", common.SCHEDULED)const callbackName = "test:start-ingestion-task-before-conditional-delete"@@ -2472,7 +2447,7 @@ func TestClearDocumentParseResults_DeletesTerminalIngestionTask(t *testing.T) {db := setupServiceTestDB(t)pushServiceDB(t, db)insertTestKB(t, "kb-1", "tenant-1", 1, 10, 5)- insertTestDocWithRun(t, "doc-1", "kb-1", string(entity.TaskStatusDone), 10, 5)+ insertTestDoc(t, "doc-1", "kb-1", 10, 5)insertTestIngestionTaskWithStatus(t, "task-1", "user-1", "doc-1", "kb-1", status)ctx := t.Context()doc, err := dao.NewDocumentDAO().GetByID(ctx, db, "doc-1")@@ -3107,7 +3082,6 @@ func insertNamedTestDoc(t *testing.T, id, kbID, name string, tokenNum, chunkNumCreatedBy: "tenant-1",Suffix: filepath.Ext(name),Status: sptr("1"),- Run: sptr(string(entity.TaskStatusDone)),}if err := dao.DB.Create(doc).Error; err != nil {t.Fatalf("insert named test doc: %v", err)@@ -3366,7 +3340,7 @@ func TestStartParseDocuments_FailsBeforeClearing(t *testing.T) {db := setupServiceTestDB(t)pushServiceDB(t, db)insertTestKB(t, "kb-1", "tenant-1", 0, 10, 5)- insertTestDocWithRun(t, "doc-1", "kb-1", string(entity.TaskStatusDone), 10, 5)+ insertTestDoc(t, "doc-1", "kb-1", 10, 5)insertTestIngestionTaskWithStatus(t, "task-1", "user-1", "doc-1", "kb-1", common.COMPLETED)ctx := t.Context()@@ -3428,7 +3402,7 @@ func TestIngest_CancelDoesNotDeleteIngestionTask(t *testing.T) {}// TestIngest_CancelUnstartedDocument mirrors the Python /documents/ingest-// behavior: canceling a document whose parse never started (run=UNSTART, no+// behavior: canceling a document whose parse never started (no// ingestion task) must be rejected with code 102 and the Python message, not// an internal "no ingestion task found" error.func TestIngest_CancelUnstartedDocument(t *testing.T) {@@ -3436,7 +3410,7 @@ func TestIngest_CancelUnstartedDocument(t *testing.T) {pushServiceDB(t, db)insertUserTenantForAccessCheck(t, "user-1", "tenant-1")insertTestKB(t, "kb-1", "tenant-1", 0, 0, 0)- insertTestDocWithRun(t, "doc-1", "kb-1", string(entity.TaskStatusUnstart), 10, 5)+ insertTestDoc(t, "doc-1", "kb-1", 10, 5)svc := testDocumentService(t)ctx := t.Context()@@ -3456,14 +3430,14 @@ func TestIngest_CancelUnstartedDocument(t *testing.T) {}// TestIngest_CancelCompletedDocument: canceling an already finished parse-// (run=DONE, ingestion task COMPLETED) is rejected with the same Python+// (ingestion task COMPLETED) is rejected with the same Python// message as the un-started case.func TestIngest_CancelCompletedDocument(t *testing.T) {db := setupServiceTestDB(t)pushServiceDB(t, db)insertUserTenantForAccessCheck(t, "user-1", "tenant-1")insertTestKB(t, "kb-1", "tenant-1", 1, 10, 5)- insertTestDocWithRun(t, "doc-1", "kb-1", string(entity.TaskStatusDone), 10, 5)+ insertTestDoc(t, "doc-1", "kb-1", 10, 5)insertTestIngestionTaskWithStatus(t, "task-1", "user-1", "doc-1", "kb-1", common.COMPLETED)svc := testDocumentService(t)@@ -3483,7 +3457,7 @@ func TestIngest_CancelCompletedDocument(t *testing.T) {}}-// TestIngest_CancelUnstartedWithInFlightTask: run=UNSTART but an ingestion+// TestIngest_CancelUnstartedWithInFlightTask: an ingestion// task was just enqueued (CREATED) — cancel is allowed, matching the Python// has_unfinished_task branch.func TestIngest_CancelUnstartedWithInFlightTask(t *testing.T) {@@ -3491,7 +3465,7 @@ func TestIngest_CancelUnstartedWithInFlightTask(t *testing.T) {pushServiceDB(t, db)insertUserTenantForAccessCheck(t, "user-1", "tenant-1")insertTestKB(t, "kb-1", "tenant-1", 0, 0, 0)- insertTestDocWithRun(t, "doc-1", "kb-1", string(entity.TaskStatusUnstart), 10, 5)+ insertTestDoc(t, "doc-1", "kb-1", 10, 5)insertTestIngestionTask(t, "task-1", "user-1", "doc-1", "kb-1")svc := testDocumentService(t)@@ -3507,21 +3481,24 @@ func TestIngest_CancelUnstartedWithInFlightTask(t *testing.T) {t.Fatalf("expected code %v, got %v", common.CodeSuccess, code)}- doc, _ := dao.NewDocumentDAO().GetByID(ctx, db, "doc-1")- if doc == nil || doc.Run == nil || *doc.Run != string(entity.TaskStatusCancel) {- t.Fatalf("expected doc run=CANCEL, got %v", doc.Run)+ task, err := dao.NewIngestionTaskDAO().GetByID(ctx, db, "task-1")+ if err != nil {+ t.Fatalf("load task: %v", err)+ }+ if task.Status != common.STOPPED {+ t.Fatalf("expected task status=%s, got %s", common.STOPPED, task.Status)}}-// TestIngest_CancelAgainWithoutTask: re-canceling a document already in-// CANCEL state is accepted even when its ingestion task is gone — Python-// treats run=CANCEL as cancelable and cancel_all_task_of is a no-op.-func TestIngest_CancelAgainWithoutTask(t *testing.T) {+// TestIngest_CancelAgainWhenAlreadyStopped: re-canceling a document whose task is+// already STOPPED is accepted as a no-op.+func TestIngest_CancelAgainWhenAlreadyStopped(t *testing.T) {db := setupServiceTestDB(t)pushServiceDB(t, db)insertUserTenantForAccessCheck(t, "user-1", "tenant-1")insertTestKB(t, "kb-1", "tenant-1", 1, 10, 5)- insertTestDocWithRun(t, "doc-1", "kb-1", string(entity.TaskStatusCancel), 10, 5)+ insertTestDoc(t, "doc-1", "kb-1", 10, 5)+ insertTestIngestionTaskWithStatus(t, "task-1", "user-1", "doc-1", "kb-1", common.STOPPED)svc := testDocumentService(t)ctx := t.Context()@@ -3530,13 +3507,193 @@ func TestIngest_CancelAgainWithoutTask(t *testing.T) {Run: string(entity.TaskStatusCancel),})if err != nil {- t.Fatalf("Ingest(re-cancel) without task: %v", err)+ t.Fatalf("Ingest(re-cancel) on stopped task: %v", err)}if code != common.CodeSuccess {t.Fatalf("expected code %v, got %v", common.CodeSuccess, code)}}+// TestIngest_CancelScheduledTask_TransitionsToStopped verifies that canceling a+// SCHEDULED document task transitions the task immediately to STOPPED and resets+// document progress to 0, ensuring the frontend never hangs in a transient state.+func TestIngest_CancelScheduledTask_TransitionsToStopped(t *testing.T) {+ db := setupServiceTestDB(t)+ pushServiceDB(t, db)+ insertUserTenantForAccessCheck(t, "user-1", "tenant-1")+ insertTestKB(t, "kb-1", "tenant-1", 0, 0, 0)+ insertTestDoc(t, "doc-1", "kb-1", 10, 5)+ insertTestIngestionTaskWithStatus(t, "task-1", "user-1", "doc-1", "kb-1", common.SCHEDULED)++ svc := testDocumentService(t)+ ctx := t.Context()+ code, err := svc.Ingest(ctx, "user-1", &IngestDocumentRequest{+ DocIDs: []string{"doc-1"},+ Run: string(entity.TaskStatusCancel),+ })+ if err != nil {+ t.Fatalf("Ingest(cancel) on SCHEDULED task failed: %v", err)+ }+ if code != common.CodeSuccess {+ t.Fatalf("expected code %v, got %v", common.CodeSuccess, code)+ }++ task, err := svc.ingestionTaskDAO.GetByDocumentID(ctx, db, "doc-1")+ if err != nil {+ t.Fatalf("load task: %v", err)+ }+ if task == nil || task.Status != common.STOPPED {+ t.Fatalf("task status = %v, want %s", task, common.STOPPED)+ }++ doc, err := svc.documentDAO.GetByID(ctx, db, "doc-1")+ if err != nil {+ t.Fatalf("load doc: %v", err)+ }+ if doc.Progress != 0 {+ t.Fatalf("doc.Progress = %v, want 0", doc.Progress)+ }++ // Verify that querying document reflects STOPPED status for frontend.+ resp, err := svc.toResponse(ctx, doc)+ if err != nil {+ t.Fatalf("toResponse failed: %v", err)+ }+ if resp.IngestionStatus != common.STOPPED {+ t.Fatalf("resp.IngestionStatus = %q, want %q", resp.IngestionStatus, common.STOPPED)+ }+}++// TestIngest_CancelRunningTask_LifecycleToStopped verifies the complete cancel+// lifecycle for a RUNNING task:+// 1. Ingest(cancel) puts the task in STOPPING and resets progress to 0.+// 2. A duplicate cancel call while STOPPING succeeds (idempotent guard).+// 3. Once worker settles the cancellation via MarkStopped, the task reaches STOPPED.+// 4. In STOPPED state, the document reflects ingestion_status=STOPPED and is+// recognized as terminal, unblocking the frontend and allowing re-parse.+func TestIngest_CancelRunningTask_LifecycleToStopped(t *testing.T) {+ db := setupServiceTestDB(t)+ pushServiceDB(t, db)+ insertUserTenantForAccessCheck(t, "user-1", "tenant-1")+ insertTestKB(t, "kb-1", "tenant-1", 0, 0, 0)+ insertTestDoc(t, "doc-1", "kb-1", 10, 5)+ if err := db.Model(&entity.Document{}).Where("id = ?", "doc-1").Update("progress", 0.6).Error; err != nil {+ t.Fatalf("set initial progress: %v", err)+ }+ insertTestIngestionTaskWithStatus(t, "task-1", "user-1", "doc-1", "kb-1", common.RUNNING)++ svc := testDocumentService(t)+ ctx := t.Context()++ // 1. First cancel call: transitions RUNNING -> STOPPING+ code, err := svc.Ingest(ctx, "user-1", &IngestDocumentRequest{+ DocIDs: []string{"doc-1"},+ Run: string(entity.TaskStatusCancel),+ })+ if err != nil {+ t.Fatalf("first Ingest(cancel): %v", err)+ }+ if code != common.CodeSuccess {+ t.Fatalf("expected code %v, got %v", common.CodeSuccess, code)+ }++ task, err := svc.ingestionTaskDAO.GetByDocumentID(ctx, db, "doc-1")+ if err != nil || task == nil {+ t.Fatalf("load task after cancel: %v", err)+ }+ if task.Status != common.STOPPING {+ t.Fatalf("task status = %q, want %q", task.Status, common.STOPPING)+ }++ doc, err := svc.documentDAO.GetByID(ctx, db, "doc-1")+ if err != nil {+ t.Fatalf("load doc: %v", err)+ }+ if doc.Progress != 0 {+ t.Fatalf("doc.Progress = %v, want 0 after cancel", doc.Progress)+ }++ // 2. Duplicate cancel while in STOPPING must succeed as an idempotent no-op+ code, err = svc.Ingest(ctx, "user-1", &IngestDocumentRequest{+ DocIDs: []string{"doc-1"},+ Run: string(entity.TaskStatusCancel),+ })+ if err != nil {+ t.Fatalf("duplicate Ingest(cancel) while STOPPING: %v", err)+ }+ if code != common.CodeSuccess {+ t.Fatalf("expected code %v, got %v", common.CodeSuccess, code)+ }++ // 3. Worker settlement: transitions STOPPING -> STOPPED+ if err = svc.ingestionTaskSvc.MarkStopped(ctx, "task-1"); err != nil {+ t.Fatalf("MarkStopped failed: %v", err)+ }++ task, err = svc.ingestionTaskDAO.GetByDocumentID(ctx, db, "doc-1")+ if err != nil || task == nil {+ t.Fatalf("load task after MarkStopped: %v", err)+ }+ if task.Status != common.STOPPED {+ t.Fatalf("task status = %q, want %q", task.Status, common.STOPPED)+ }++ // 4. Verify document query returns STOPPED status for frontend consumption+ resp, err := svc.toResponse(ctx, doc)+ if err != nil {+ t.Fatalf("toResponse failed: %v", err)+ }+ if resp.IngestionStatus != common.STOPPED {+ t.Fatalf("resp.IngestionStatus = %q, want %q", resp.IngestionStatus, common.STOPPED)+ }++ // 5. AssertIngestionTasksTerminal succeeds on STOPPED (document unblocked)+ if err = svc.AssertIngestionTasksTerminal(ctx, []string{"doc-1"}); err != nil {+ t.Fatalf("AssertIngestionTasksTerminal should succeed on STOPPED: %v", err)+ }+}++// TestIngest_DeleteOnlyCleansTasks verifies that when run is neither running nor+// cancel, passing delete=true deletes prior tasks and resets progress to 0 without starting a parse.+func TestIngest_DeleteOnlyCleansTasks(t *testing.T) {+ db := setupServiceTestDB(t)+ pushServiceDB(t, db)+ insertUserTenantForAccessCheck(t, "user-1", "tenant-1")+ insertTestKB(t, "kb-1", "tenant-1", 0, 0, 0)+ insertTestDoc(t, "doc-1", "kb-1", 10, 5)+ insertTestIngestionTaskWithStatus(t, "task-1", "user-1", "doc-1", "kb-1", common.COMPLETED)++ svc := testDocumentService(t)+ ctx := t.Context()+ code, err := svc.Ingest(ctx, "user-1", &IngestDocumentRequest{+ DocIDs: []string{"doc-1"},+ Run: "0",+ Delete: true,+ })+ if err != nil {+ t.Fatalf("Ingest(delete-only): %v", err)+ }+ if code != common.CodeSuccess {+ t.Fatalf("expected code %v, got %v", common.CodeSuccess, code)+ }++ task, err := svc.ingestionTaskDAO.GetByDocumentID(ctx, db, "doc-1")+ if err != nil {+ t.Fatalf("query task: %v", err)+ }+ if task != nil {+ t.Fatal("expected task to be deleted in delete-only mode")+ }++ doc, err := dao.NewDocumentDAO().GetByID(ctx, db, "doc-1")+ if err != nil {+ t.Fatalf("query doc: %v", err)+ }+ if doc.Progress != 0 {+ t.Fatalf("expected doc progress 0, got %v", doc.Progress)+ }+}+func TestUpdateRunProgressMirrorsFields(t *testing.T) {db := setupServiceTestDB(t)pushServiceDB(t, db)@@ -3548,7 +3705,7 @@ func TestUpdateRunProgressMirrorsFields(t *testing.T) {svc := testDocumentService(t)ctx := t.Context()- if err := svc.UpdateRunProgress(ctx, "doc-1", 0.5, "1", "halfway"); err != nil {+ if err := svc.UpdateRunProgress(ctx, "doc-1", 0.5, "halfway"); err != nil {t.Fatalf("UpdateRunProgress failed: %v", err)}doc, err := dao.NewDocumentDAO().GetByID(ctx, db, "doc-1")@@ -3558,9 +3715,6 @@ func TestUpdateRunProgressMirrorsFields(t *testing.T) {if doc.Progress != 0.5 {t.Fatalf("progress = %v, want 0.5", doc.Progress)}- if doc.Run == nil || *doc.Run != "1" {- t.Fatalf("run = %v, want 1", doc.Run)- }if doc.ProgressMsg == nil || *doc.ProgressMsg != "halfway" {t.Fatalf("progress_msg = %v, want halfway", doc.ProgressMsg)}diff --git a/internal/service/document/document_upload.go b/internal/service/document/document_upload.goindex 341ee5c2fa..4d5a1495a4 100644--- a/internal/service/document/document_upload.go+++ b/internal/service/document/document_upload.go@@ -322,7 +322,6 @@ func normalizeWebDocumentName(name, contentType string, blob []byte) string {// suffix and content hash. blob may be nil for the empty/virtual document.func (s *DocumentService) newDatasetDocument(kb *entity.Knowledgebase, tenantID, filename, location, filetype string, parserConfig entity.JSONMap, src string, size int64, blob []byte) *entity.Document {docID := utility.GenerateToken()- run := "0"status := "1"suffix := ""if i := strings.LastIndex(filename, "."); i >= 0 {@@ -346,7 +345,6 @@ func (s *DocumentService) newDatasetDocument(kb *entity.Knowledgebase, tenantID,Location: &loc,Size: size,Suffix: suffix,- Run: &run,Status: &status,}if blob != nil {@@ -360,18 +358,18 @@ func (s *DocumentService) newDatasetDocument(kb *entity.Knowledgebase, tenantID,// handler remaps (chunk_num→chunk_count, kb_id→dataset_id).func docToRawMap(doc *entity.Document) map[string]interface{} {m := map[string]interface{}{- "id": doc.ID,- "kb_id": doc.KbID,- "parser_id": doc.ParserID,- "parser_config": map[string]interface{}(doc.ParserConfig),- "created_by": doc.CreatedBy,- "type": doc.Type,- "source_type": doc.SourceType,- "size": doc.Size,- "chunk_num": doc.ChunkNum,- "token_num": doc.TokenNum,- "suffix": doc.Suffix,- "run": "0",+ "id": doc.ID,+ "kb_id": doc.KbID,+ "parser_id": doc.ParserID,+ "parser_config": map[string]interface{}(doc.ParserConfig),+ "created_by": doc.CreatedBy,+ "type": doc.Type,+ "source_type": doc.SourceType,+ "size": doc.Size,+ "chunk_num": doc.ChunkNum,+ "token_num": doc.TokenNum,+ "suffix": doc.Suffix,+ "ingestion_status": "UNSTART",}if doc.Name != nil {m["name"] = *doc.Namediff --git a/internal/service/document/document_validation.go b/internal/service/document/document_validation.goindex 975437da67..3fccccb07a 100644--- a/internal/service/document/document_validation.go+++ b/internal/service/document/document_validation.go@@ -30,7 +30,6 @@ type immutableDocumentFields struct {tokenNum *int64tokenNumRequestName stringprogress *float64- run *stringprogressMsg *string}@@ -44,9 +43,6 @@ func validateImmutableDocumentFields(doc *entity.Document, fields immutableDocumif fields.progress != nil && !documentProgressEqual(*fields.progress, doc.Progress) {return errors.New("can't change `progress`")}- if fields.run != nil && !optionalStringEqual(fields.run, doc.Run) {- return errors.New("can't change `run`")- }if fields.progressMsg != nil && !optionalStringEqual(fields.progressMsg, doc.ProgressMsg) {return errors.New("can't change `progress_msg`")}diff --git a/internal/service/ingestion_task_service.go b/internal/service/ingestion_task_service.goindex 8395578ed0..5109e1443a 100644--- a/internal/service/ingestion_task_service.go+++ b/internal/service/ingestion_task_service.go@@ -209,14 +209,10 @@ func (s *IngestionTaskService) StartRunning(ctx context.Context, taskID string)if err != nil {return nil, err}- // The task just started running: mirror it to the document so its- // run status and progress counters reflect real processing, not- // just API acceptance. Best-effort - a DB blip here must not fail- // the task transition and trigger a redelivery loop. run uses the- // document's numeric TaskStatus enum ("1"), not the task's string- // status label.+ // The task just started running: reset document progress counters to+ // reflect real processing. Best-effort - a DB blip here must not fail+ // the task transition and trigger a redelivery loop.if err = s.documentDAO.UpdateByID(ctx, dao.DB, task.DocumentID, map[string]interface{}{- "run": string(entity.TaskStatusRunning),"progress": float64(0),"chunk_num": int64(0),"token_num": int64(0),diff --git a/internal/service/ingestion_task_service_test.go b/internal/service/ingestion_task_service_test.goindex 0820046c16..27aca6aa7c 100644--- a/internal/service/ingestion_task_service_test.go+++ b/internal/service/ingestion_task_service_test.go@@ -354,22 +354,20 @@ func TestIngestionTaskServiceStartRunningTransitionsCreatedTask(t *testing.T) {}}-// TestStartRunningMarksDocumentRunning locks in that starting a CREATED task-// mirrors the transition to its document: run=RUNNING and progress counters-// reset, with a fresh process_begin_at. The document bookkeeping is owned by-// the task-lifecycle transition, not the ingestion worker's execution path.-func TestStartRunningMarksDocumentRunning(t *testing.T) {+// TestStartRunningResetsDocumentProgress locks in that starting a CREATED task+// resets its document progress counters, with a fresh process_begin_at. The+// document bookkeeping is owned by the task-lifecycle transition, not the+// ingestion worker's execution path.+func TestStartRunningResetsDocumentProgress(t *testing.T) {db := setupServiceTestDB(t)pushServiceDB(t, db)insertTestKB(t, "kb-1", "tenant-1", 1, 0, 0)insertTestDoc(t, "doc-1", "kb-1", 100, 10)insertTestIngestionTask(t, "task-1", "user-1", "doc-1", "kb-1")- // Seed the document as a partially-processed, non-RUNNING state that the- // start transition must clobber.+ // Seed the document as a partially-processed state that the start transition must reset.if err := db.Model(&entity.Document{}).Where("id = ?", "doc-1").Updates(map[string]interface{}{- "run": string(entity.TaskStatusDone),"progress": float64(0.5),"progress_msg": "partial",}).Error; err != nil {@@ -386,9 +384,6 @@ func TestStartRunningMarksDocumentRunning(t *testing.T) {if err := db.Where("id = ?", "doc-1").First(&doc).Error; err != nil {t.Fatalf("reload document: %v", err)}- if doc.Run == nil || *doc.Run != string(entity.TaskStatusRunning) {- t.Fatalf("run = %v, want RUNNING(%q)", doc.Run, string(entity.TaskStatusRunning))- }if doc.Progress != 0 {t.Fatalf("progress = %f, want 0", doc.Progress)}@@ -416,10 +411,8 @@ func TestStartRunningLeavesTerminalDocumentUntouched(t *testing.T) {insertTestDoc(t, "doc-1", "kb-1", 100, 10)insertTestIngestionTask(t, "task-1", "user-1", "doc-1", "kb-1")- finishedRun := string(entity.TaskStatusDone)if err := db.Model(&entity.Document{}).Where("id = ?", "doc-1").Updates(map[string]interface{}{- "run": finishedRun,"progress": float64(1.0),"progress_msg": "done",}).Error; err != nil {@@ -444,8 +437,8 @@ func TestStartRunningLeavesTerminalDocumentUntouched(t *testing.T) {if err := db.Where("id = ?", "doc-1").First(&doc).Error; err != nil {t.Fatalf("reload document: %v", err)}- if doc.Run == nil || *doc.Run != finishedRun {- t.Fatalf("run = %v, want %q (terminal document must not be resurrected)", doc.Run, finishedRun)+ if doc.Progress != 1.0 {+ t.Fatalf("progress = %v, want 1.0", doc.Progress)}if doc.ChunkNum != 10 || doc.TokenNum != 100 {t.Fatalf("counters changed: chunk_num=%d token_num=%d, want 10/100", doc.ChunkNum, doc.TokenNum)@@ -492,6 +485,26 @@ func TestIngestionTaskServiceRequestStopTransitionsCreatedTaskToStopped(t *testi}}+func TestIngestionTaskServiceRequestStopTransitionsScheduledTaskToStopped(t *testing.T) {+ db := setupServiceTestDB(t)+ pushServiceDB(t, db)+ insertTestIngestionTask(t, "task-1", "user-1", "doc-1", "kb-1")+ if err := db.Model(&entity.IngestionTask{}).Where("id = ?", "task-1").+ Update("status", common.SCHEDULED).Error; err != nil {+ t.Fatalf("set SCHEDULED: %v", err)+ }+ ctx := t.Context()++ svc := NewIngestionTaskService()+ task, err := svc.RequestStop(ctx, "task-1")+ if err != nil {+ t.Fatalf("RequestStop failed: %v", err)+ }+ if task.Status != common.STOPPED {+ t.Fatalf("status = %q, want %q", task.Status, common.STOPPED)+ }+}+func TestIngestionTaskServiceMarkCompletedRejectsNonRunningTask(t *testing.T) {db := setupServiceTestDB(t)pushServiceDB(t, db)