RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
feat(media): harden run media cache lifecycle and limits (#21694)
jay777219ba49c523eb2
diff --git a/docker/.env b/docker/.envindex d33ff98f99..348769a5a2 100644--- a/docker/.env+++ b/docker/.env@@ -549,6 +549,13 @@ LLM_POOL_RATE_LIMIT_RETRY_MAX_DELAY=30.0# -----------------------------------------------------------------------------# RAGFLOW_INGESTOR_CONCURRENT_WORKERS=1+# -----------------------------------------------------------------------------+# Media cache limits (Go ingestion)+# -----------------------------------------------------------------------------+# RAGFLOW_MEDIA_IMAGE_MAX_BYTES=33554432+# RAGFLOW_MEDIA_MAX_IMAGES_PER_RUN=1024+# RAGFLOW_MEDIA_MAX_BYTES_PER_RUN=1073741824+# -----------------------------------------------------------------------------# DeepDoc (in-process)# -----------------------------------------------------------------------------diff --git a/internal/common/environments.go b/internal/common/environments.goindex 82aebe81c2..ee6006eb5d 100644--- a/internal/common/environments.go+++ b/internal/common/environments.go@@ -322,6 +322,13 @@ const (// --deepdoc-inference-cpu-cores CLI flag.EnvDeepDocInferenceCPUCores = "RAGFLOW_DEEPDOC_INFERENCE_CPU_CORES"+ // Media extraction limits. The parser media budget and the run media cache+ // read these per call: the per-image payload limit, the number of distinct+ // images one run may cache, and the total decoded bytes one run may cache.+ EnvMediaImageMaxBytes = "RAGFLOW_MEDIA_IMAGE_MAX_BYTES"+ EnvMediaMaxImagesPerRun = "RAGFLOW_MEDIA_MAX_IMAGES_PER_RUN"+ EnvMediaMaxBytesPerRun = "RAGFLOW_MEDIA_MAX_BYTES_PER_RUN"+// Typed-decision model. Unset RAGFLOW_DECISION_DIR leaves the feature off:// every caller degrades to unfiltered behaviour, so a deployment without the// model directory is indistinguishable from one that never had the feature.diff --git a/internal/dao/database_test.go b/internal/dao/database_test.goindex 6cf67663f7..b3cdae4748 100644--- a/internal/dao/database_test.go+++ b/internal/dao/database_test.go@@ -23,6 +23,7 @@ import ("strings""testing"+ "ragflow/internal/common""ragflow/internal/entity""github.com/glebarez/sqlite"@@ -123,6 +124,181 @@ func TestAutoMigrateRuntimeModelsAddsIngestionTaskFilterIndexes(t *testing.T) {}}+func prepareDuplicateIngestionTaskDB(t *testing.T) *gorm.DB {+ t.Helper()++ db := setupTaskTestDB(t)+ if err := db.Migrator().DropIndex(&entity.IngestionTask{}, "idx_ingestion_task_document_id"); err != nil {+ t.Fatalf("drop ingestion task unique index: %v", err)+ }+ if err := db.Exec("CREATE INDEX idx_ingestion_task_document_id ON ingestion_task(document_id)").Error; err != nil {+ t.Fatalf("create legacy non-unique document index: %v", err)+ }+ return db+}++func seedDuplicateIngestionTasks(t *testing.T, db *gorm.DB, staleStatus string, stalePipelineLogID *string) {+ t.Helper()++ for _, task := range []*entity.IngestionTask{+ {+ ID: "task-old",+ UserID: "user-1",+ DocumentID: "doc-duplicate",+ DatasetID: "kb-1",+ Status: staleStatus,+ PipelineLogID: stalePipelineLogID,+ BaseModel: entity.BaseModel{+ CreateTime: int64Ptr(100),+ },+ },+ {+ ID: "task-new",+ UserID: "user-1",+ DocumentID: "doc-duplicate",+ DatasetID: "kb-1",+ Status: common.COMPLETED,+ BaseModel: entity.BaseModel{+ CreateTime: int64Ptr(200),+ },+ },+ } {+ if err := db.Create(task).Error; err != nil {+ t.Fatalf("create task %s: %v", task.ID, err)+ }+ }+}++func TestRepairDuplicateIngestionTasksDeletesOnlyTerminalStaleRows(t *testing.T) {+ db := prepareDuplicateIngestionTaskDB(t)+ doneLogID := "run-done"+ if err := db.Exec(`CREATE TABLE pipeline_operation_log (+ id varchar(32) PRIMARY KEY,+ document_id varchar(32) NOT NULL,+ operation_status varchar(32) NOT NULL+ )`).Error; err != nil {+ t.Fatalf("create pipeline operation log table: %v", err)+ }+ if err := db.Exec(+ "INSERT INTO pipeline_operation_log (id, document_id, operation_status) VALUES (?, ?, ?)",+ doneLogID, "doc-duplicate", string(entity.TaskStatusDone),+ ).Error; err != nil {+ t.Fatalf("create completed pipeline operation log: %v", err)+ }+ seedDuplicateIngestionTasks(t, db, common.COMPLETED, &doneLogID)++ if err := repairDuplicateIngestionTasks(t.Context(), db); err != nil {+ t.Fatalf("repairDuplicateIngestionTasks: %v", err)+ }+ var ids []string+ if err := db.Model(&entity.IngestionTask{}).+ Where("document_id = ?", "doc-duplicate").+ Pluck("id", &ids).Error; err != nil {+ t.Fatalf("list remaining ingestion tasks: %v", err)+ }+ if !reflect.DeepEqual(ids, []string{"task-new"}) {+ t.Fatalf("remaining task ids = %v, want [task-new]", ids)+ }+ if err := repairDuplicateIngestionTasks(t.Context(), db); err != nil {+ t.Fatalf("second repairDuplicateIngestionTasks: %v", err)+ }+}++func TestRepairDuplicateIngestionTasksRejectsNonTerminalStaleRows(t *testing.T) {+ for _, status := range common.ActiveTaskStatuses {+ t.Run(status, func(t *testing.T) {+ db := prepareDuplicateIngestionTaskDB(t)+ seedDuplicateIngestionTasks(t, db, status, nil)++ err := repairDuplicateIngestionTasks(t.Context(), db)+ for _, want := range []string{"doc-duplicate", "task-old", status} {+ if err == nil || !strings.Contains(err.Error(), want) {+ t.Fatalf("repair error = %v, want it to contain %q", err, want)+ }+ }+ var count int64+ if err := db.Model(&entity.IngestionTask{}).+ Where("document_id = ?", "doc-duplicate").+ Count(&count).Error; err != nil {+ t.Fatalf("count ingestion tasks: %v", err)+ }+ if count != 2 {+ t.Fatalf("ingestion task count = %d, want 2 after failed repair", count)+ }+ })+ }+}++func TestRepairDuplicateIngestionTasksRejectsOpenBoundPipelineLog(t *testing.T) {+ db := prepareDuplicateIngestionTaskDB(t)+ openLogID := "run-open"+ if err := db.Exec(`CREATE TABLE pipeline_operation_log (+ id varchar(32) PRIMARY KEY,+ document_id varchar(32) NOT NULL,+ operation_status varchar(32) NOT NULL+ )`).Error; err != nil {+ t.Fatalf("create pipeline operation log table: %v", err)+ }+ if err := db.Exec(+ "INSERT INTO pipeline_operation_log (id, document_id, operation_status) VALUES (?, ?, ?)",+ openLogID, "doc-duplicate", string(entity.TaskStatusRunning),+ ).Error; err != nil {+ t.Fatalf("create open pipeline operation log: %v", err)+ }+ seedDuplicateIngestionTasks(t, db, common.FAILED, &openLogID)++ err := repairDuplicateIngestionTasks(t.Context(), db)+ for _, want := range []string{"doc-duplicate", "task-old", openLogID} {+ if err == nil || !strings.Contains(err.Error(), want) {+ t.Fatalf("repair error = %v, want it to contain %q", err, want)+ }+ }+ var count int64+ if err := db.Model(&entity.IngestionTask{}).+ Where("document_id = ?", "doc-duplicate").+ Count(&count).Error; err != nil {+ t.Fatalf("count ingestion tasks: %v", err)+ }+ if count != 2 {+ t.Fatalf("ingestion task count = %d, want 2 after failed repair", count)+ }+}++func TestRepairDuplicateIngestionTasksSupportsLegacySchemaWithoutPipelineLogID(t *testing.T) {+ db, err := gorm.Open(sqlite.Open(":memory:"), &gorm.Config{})+ if err != nil {+ t.Fatalf("open sqlite: %v", err)+ }+ if err := db.Exec(`CREATE TABLE ingestion_task (+ id varchar(32) PRIMARY KEY,+ user_id varchar(32) NOT NULL,+ document_id varchar(32) NOT NULL,+ dataset_id varchar(32) NOT NULL,+ status varchar(32) NOT NULL,+ create_time bigint NULL+ )`).Error; err != nil {+ t.Fatalf("create legacy ingestion_task table: %v", err)+ }+ if err := db.Exec(`INSERT INTO ingestion_task+ (id, user_id, document_id, dataset_id, status, create_time)+ VALUES+ ('task-old', 'user-1', 'doc-duplicate', 'kb-1', 'COMPLETED', 100),+ ('task-new', 'user-1', 'doc-duplicate', 'kb-1', 'FAILED', 200)`).Error; err != nil {+ t.Fatalf("seed legacy ingestion tasks: %v", err)+ }++ if err := repairDuplicateIngestionTasks(t.Context(), db); err != nil {+ t.Fatalf("repairDuplicateIngestionTasks: %v", err)+ }+ var ids []string+ if err := db.Table("ingestion_task").Pluck("id", &ids).Error; err != nil {+ t.Fatalf("list remaining ingestion tasks: %v", err)+ }+ if !reflect.DeepEqual(ids, []string{"task-new"}) {+ t.Fatalf("remaining task ids = %v, want [task-new]", ids)+ }+}+// TestMigrateIngestionTaskPipelineLogID covers the upgrade path AutoMigrate// cannot handle on MySQL: an existing ingestion_task created before// pipeline_log_id existed must gain the column, and running the migration againdiff --git a/internal/dao/ingestion_task.go b/internal/dao/ingestion_task.goindex 8488e085f8..0770aff767 100644--- a/internal/dao/ingestion_task.go+++ b/internal/dao/ingestion_task.go@@ -294,15 +294,13 @@ func (dao *IngestionTaskDAO) CountActiveByDatasetID(ctx context.Context, db *gorreturn count, err}-// DeleteIfTerminal deletes ingestion tasks for a document that are in a-// terminal state (COMPLETED, STOPPED, FAILED), or not yet running-// (CREATED, SCHEDULED).-// RUNNING and STOPPING tasks are NOT deleted because an in-flight worker-// would keep writing chunks and corrupt a new run's results.-// Returns the number of rows deleted.-func (dao *IngestionTaskDAO) DeleteIfTerminal(ctx context.Context, db *gorm.DB, documentID string) (int64, error) {- result := db.WithContext(ctx).Where("document_id = ? AND status NOT IN (?, ?)",- documentID, common.RUNNING, common.STOPPING).+// DeleteIfTerminal deletes one terminal ingestion task by ID. Cleanup freezes+// CREATED and SCHEDULED tasks before calling this method; if another writer+// moves the task back to a non-terminal state, the conditional delete leaves it+// in place.+func (dao *IngestionTaskDAO) DeleteIfTerminal(ctx context.Context, db *gorm.DB, taskID string) (int64, error) {+ result := db.WithContext(ctx).+ Where("id = ? AND status IN ?", taskID, []string{common.COMPLETED, common.STOPPED, common.FAILED}).Delete(&entity.IngestionTask{})if result.Error != nil {return 0, result.Errordiff --git a/internal/dao/ingestion_task_test.go b/internal/dao/ingestion_task_test.goindex 3ead63809e..14ee9cc888 100644--- a/internal/dao/ingestion_task_test.go+++ b/internal/dao/ingestion_task_test.go@@ -4,7 +4,6 @@ import ("errors""testing"- "fmt""ragflow/internal/common""ragflow/internal/entity"@@ -338,65 +337,6 @@ func TestIngestionTaskDAOUpdateStatusIfCurrentRejectsMismatchedStatus(t *testing}}-func TestIngestionTaskDAODeleteIfTerminal_RemovesOnlyTerminal(t *testing.T) {- db := setupTaskTestDB(t)- orig := DB- DB = db- t.Cleanup(func() { DB = orig })-- // Create tasks in different statuses, each with a unique docID.- statuses := []string{common.CREATED, common.SCHEDULED, common.RUNNING, common.STOPPING, common.COMPLETED, common.STOPPED, common.FAILED}- for i, status := range statuses {- docID := fmt.Sprintf("doc-%d", i)- task := &entity.IngestionTask{- ID: fmt.Sprintf("task-%d", i),- UserID: "user-1",- DocumentID: docID,- DatasetID: "kb-1",- Status: status,- }- if err := db.Create(task).Error; err != nil {- t.Fatalf("create task %s: %v", status, err)- }- }-- ctx := t.Context()- // DeleteIfTerminal deletes everything except RUNNING and STOPPING.- // CREATED and SCHEDULED are safe to delete (no worker has claimed them yet);- // COMPLETED/STOPPED/FAILED are terminal.- // Call it for every doc and verify the negative cases survived.- for i := 0; i < len(statuses); i++ {- docID := fmt.Sprintf("doc-%d", i)- _, err := NewIngestionTaskDAO().DeleteIfTerminal(ctx, db, docID)- if err != nil {- t.Fatalf("DeleteIfTerminal(doc-%d): %v", i, err)- }- }-- // RUNNING and STOPPING must survive.- for _, i := range []int{2, 3} {- docID := fmt.Sprintf("doc-%d", i)- task, err := NewIngestionTaskDAO().GetByDocumentID(ctx, db, docID)- if err != nil {- t.Fatalf("GetByDocumentID %s: %v", docID, err)- }- if task == nil {- t.Fatalf("%s task (doc=%d) must not be deleted", statuses[i], i)- }- }- // CREATED, SCHEDULED, COMPLETED, STOPPED, FAILED must be gone.- for _, i := range []int{0, 1, 4, 5, 6} {- docID := fmt.Sprintf("doc-%d", i)- task, err := NewIngestionTaskDAO().GetByDocumentID(ctx, db, docID)- if err != nil {- t.Fatalf("GetByDocumentID %s: %v", docID, err)- }- if task != nil {- t.Fatalf("%s task (doc=%d) should be deleted, still present", statuses[i], i)- }- }-}-func TestIngestionTaskDAOCountActiveByDatasetID(t *testing.T) {db := setupTaskTestDB(t)for _, task := range []*entity.IngestionTask{@@ -436,6 +376,40 @@ func TestIngestionTaskDAOCountActiveByDatasetID(t *testing.T) {}}+func TestIngestionTaskDAODeleteIfTerminalRequiresMatchingTerminalTask(t *testing.T) {+ db := setupTaskTestDB(t)+ for _, task := range []*entity.IngestionTask{+ {ID: "task-terminal", UserID: "user-1", DocumentID: "doc-terminal", DatasetID: "kb-1", Status: common.COMPLETED},+ {ID: "task-running", UserID: "user-1", DocumentID: "doc-running", DatasetID: "kb-1", Status: common.RUNNING},+ {ID: "task-queued", UserID: "user-1", DocumentID: "doc-queued", DatasetID: "kb-1", Status: common.SCHEDULED},+ } {+ if err := db.Create(task).Error; err != nil {+ t.Fatalf("create task %s: %v", task.ID, err)+ }+ }++ taskDAO := NewIngestionTaskDAO()+ deleted, err := taskDAO.DeleteIfTerminal(t.Context(), db, "task-terminal")+ if err != nil {+ t.Fatalf("delete terminal task: %v", err)+ }+ if deleted != 1 {+ t.Fatalf("deleted terminal rows = %d, want 1", deleted)+ }+ for _, taskID := range []string{"task-running", "task-queued"} {+ deleted, err := taskDAO.DeleteIfTerminal(t.Context(), db, taskID)+ if err != nil {+ t.Fatalf("delete %s: %v", taskID, err)+ }+ if deleted != 0 {+ t.Fatalf("deleted %s rows = %d, want 0", taskID, deleted)+ }+ if _, err := taskDAO.GetByID(t.Context(), db, taskID); err != nil {+ t.Fatalf("%s was removed despite non-terminal status: %v", taskID, err)+ }+ }+}+func TestIngestionTaskDAOCountActiveByDatasetIDUsesLatestTask(t *testing.T) {db := setupTaskTestDB(t)if err := db.Exec("DROP INDEX idx_ingestion_task_document_id").Error; err != nil {diff --git a/internal/dao/migration.go b/internal/dao/migration.goindex 569014fe25..59c274273b 100644--- a/internal/dao/migration.go+++ b/internal/dao/migration.go@@ -19,10 +19,12 @@ package daoimport ("context""encoding/json"+ "errors""fmt""ragflow/internal/common""ragflow/internal/entity""regexp"+ "slices""sort""strings"@@ -262,18 +264,9 @@ func migrateIngestionTaskDocumentIDUnique(ctx context.Context, db *gorm.DB) erroreturn nil}- var duplicateCount int64- if err = db.WithContext(ctx).Raw(`- SELECT COUNT(*) FROM (- SELECT document_id FROM ingestion_task GROUP BY document_id HAVING COUNT(*) > 1- ) AS duplicates- `).Scan(&duplicateCount).Error; err != nil {+ if err = repairDuplicateIngestionTasks(ctx, db); err != nil {return err}- if duplicateCount > 0 {- common.Warn("Found duplicate document_id values in ingestion_task, cannot add unique index", zap.Int64("count", duplicateCount))- return nil- }// A non-unique index under the same name would block the unique one.if err = dropIndexIfExists(ctx, db, "ingestion_task", indexName); err != nil {@@ -283,6 +276,132 @@ func migrateIngestionTaskDocumentIDUnique(ctx context.Context, db *gorm.DB) erroreturn addUniqueIndex(ctx, db, "ingestion_task", indexName, columns)}+// repairDuplicateIngestionTasks keeps the task selected by GetByDocumentID and+// removes older terminal rows before the unique document index is created.+// Active duplicates and terminal rows whose bound pipeline log is still open+// require operator intervention because deleting them could orphan a worker or+// run ledger.+func repairDuplicateIngestionTasks(ctx context.Context, db *gorm.DB) error {+ return db.WithContext(ctx).Transaction(func(tx *gorm.DB) error {+ var documentIDs []string+ if err := tx.Model(&entity.IngestionTask{}).+ Select("document_id").+ Group("document_id").+ Having("COUNT(*) > 1").+ Order("document_id").+ Pluck("document_id", &documentIDs).Error; err != nil {+ return fmt.Errorf("find duplicate ingestion_task.document_id values: %w", err)+ }+ if len(documentIDs) == 0 {+ return nil+ }++ hasPipelineLogID := tx.Migrator().HasColumn(&entity.IngestionTask{}, "pipeline_log_id")+ columns := "id, document_id, status, create_time"+ if hasPipelineLogID {+ columns += ", pipeline_log_id"+ }++ var staleTasks []entity.IngestionTask+ for _, documentID := range documentIDs {+ var tasks []entity.IngestionTask+ if err := tx.Model(&entity.IngestionTask{}).+ Select(columns).+ Where("document_id = ?", documentID).+ Order("COALESCE(create_time, 0) DESC").+ Order("id DESC").+ Find(&tasks).Error; err != nil {+ return fmt.Errorf("load duplicate ingestion tasks for document %s: %w", documentID, err)+ }+ if len(tasks) < 2 {+ continue+ }+ for _, task := range tasks[1:] {+ if !common.IsTerminalTaskStatus(task.Status) {+ return fmt.Errorf(+ "cannot add unique index on ingestion_task.document_id: duplicate task %s for document %s is %s; stop ingestors and resolve the duplicate task before upgrading",+ task.ID, task.DocumentID, task.Status,+ )+ }+ if hasPipelineLogID && task.PipelineLogID != nil && *task.PipelineLogID != "" {+ if err := requireSettledDuplicateTaskRun(tx, task); err != nil {+ return err+ }+ }+ staleTasks = append(staleTasks, task)+ }+ }++ for _, task := range staleTasks {+ result := tx.Where("id = ? AND status = ?", task.ID, task.Status).+ Delete(&entity.IngestionTask{})+ if result.Error != nil {+ return fmt.Errorf("delete duplicate ingestion task %s: %w", task.ID, result.Error)+ }+ if result.RowsAffected != 1 {+ return fmt.Errorf(+ "cannot delete duplicate ingestion task %s for document %s: task changed during migration",+ task.ID, task.DocumentID,+ )+ }+ }+ common.Warn(+ "Removed duplicate terminal ingestion tasks before adding the unique document index",+ zap.Int("rows", len(staleTasks)),+ )+ return nil+ })+}++func requireSettledDuplicateTaskRun(tx *gorm.DB, task entity.IngestionTask) error {+ if !tx.Migrator().HasTable("pipeline_operation_log") {+ return fmt.Errorf(+ "cannot add unique index on ingestion_task.document_id: duplicate task %s for document %s references pipeline log %s, but pipeline_operation_log is missing",+ task.ID, task.DocumentID, *task.PipelineLogID,+ )+ }++ var operationStatus string+ err := tx.Table("pipeline_operation_log").+ Select("operation_status").+ Where("id = ? AND document_id = ?", *task.PipelineLogID, task.DocumentID).+ Take(&operationStatus).Error+ if errors.Is(err, gorm.ErrRecordNotFound) {+ return fmt.Errorf(+ "cannot add unique index on ingestion_task.document_id: duplicate task %s for document %s references missing pipeline log %s",+ task.ID, task.DocumentID, *task.PipelineLogID,+ )+ }+ if err != nil {+ return fmt.Errorf("load pipeline log %s for duplicate task %s: %w", *task.PipelineLogID, task.ID, err)+ }+ if isOpenPipelineOperationStatus(operationStatus) {+ return fmt.Errorf(+ "cannot add unique index on ingestion_task.document_id: duplicate task %s for document %s is bound to open pipeline log %s; settle the log before upgrading",+ task.ID, task.DocumentID, *task.PipelineLogID,+ )+ }+ if !isTerminalPipelineOperationStatus(operationStatus) {+ return fmt.Errorf(+ "cannot add unique index on ingestion_task.document_id: duplicate task %s for document %s is bound to pipeline log %s with unknown status %q",+ task.ID, task.DocumentID, *task.PipelineLogID, operationStatus,+ )+ }+ return nil+}++func isOpenPipelineOperationStatus(status string) bool {+ return slices.Contains(expandPipelineOperationStatuses(OpenPipelineOperationStatuses()), status)+}++func isTerminalPipelineOperationStatus(status string) bool {+ return slices.Contains(expandPipelineOperationStatuses([]string{+ string(entity.TaskStatusCancel),+ string(entity.TaskStatusDone),+ string(entity.TaskStatusFail),+ }), status)+}+// migrateIngestionTaskPipelineLogID adds ingestion_task.pipeline_log_id when// the column is missing.//diff --git a/internal/dao/migration_integration_test.go b/internal/dao/migration_integration_test.goindex a7d9d0efaa..d9daa936eb 100644--- a/internal/dao/migration_integration_test.go+++ b/internal/dao/migration_integration_test.go@@ -7,9 +7,11 @@ import ("context""fmt""os"+ "reflect""strings""testing"+ "ragflow/internal/common""ragflow/internal/entity""gorm.io/gorm"@@ -224,3 +226,146 @@ func TestMigrateKnowledgeCompileQueues_Integration(t *testing.T) {}}}++func openIngestionTaskMigrationDB(t *testing.T) (*gorm.DB, context.Context) {+ t.Helper()++ dsn := os.Getenv("RAGFLOW_TEST_MYSQL_DSN")+ if dsn == "" {+ t.Skip("RAGFLOW_TEST_MYSQL_DSN unset: this test needs a throwaway MySQL database")+ }++ db, err := gorm.Open(newMigrationAwareDialector(dsn), &gorm.Config{TranslateError: true})+ if err != nil {+ t.Fatalf("failed to connect: %v", err)+ }+ sqlDB, err := db.DB()+ if err != nil {+ t.Fatalf("failed to get database connection pool: %v", err)+ }+ t.Cleanup(func() {+ if err := sqlDB.Close(); err != nil {+ t.Errorf("failed to close database connection pool: %v", err)+ }+ })+ return db, context.Background()+}++func createDuplicateIngestionTaskSchema(t *testing.T, db *gorm.DB, ctx context.Context, withPipelineLog bool) {+ t.Helper()++ tables := []string{"ingestion_task"}+ if withPipelineLog {+ tables = append(tables, "pipeline_operation_log")+ }+ for _, table := range tables {+ if db.Migrator().HasTable(table) {+ t.Fatalf("%s already exists; this test requires an empty database", table)+ }+ }+ t.Cleanup(func() {+ for _, table := range tables {+ if err := db.WithContext(context.Background()).Migrator().DropTable(table); err != nil {+ t.Errorf("failed to drop %s test table: %v", table, err)+ }+ }+ })+ if err := db.WithContext(ctx).Exec(`CREATE TABLE ingestion_task (+ id varchar(32) NOT NULL PRIMARY KEY,+ document_id varchar(32) NOT NULL,+ status varchar(32) NOT NULL,+ create_time bigint NULL,+ pipeline_log_id varchar(32) NULL+ )`).Error; err != nil {+ t.Fatalf("create ingestion_task: %v", err)+ }+ if err := db.WithContext(ctx).Exec(+ "CREATE INDEX idx_ingestion_task_document_id ON ingestion_task(document_id)",+ ).Error; err != nil {+ t.Fatalf("create legacy document index: %v", err)+ }+ if !withPipelineLog {+ return+ }+ if err := db.WithContext(ctx).Exec(`CREATE TABLE pipeline_operation_log (+ id varchar(32) NOT NULL PRIMARY KEY,+ document_id varchar(32) NOT NULL,+ operation_status varchar(32) NOT NULL+ )`).Error; err != nil {+ t.Fatalf("create pipeline_operation_log: %v", err)+ }+}++func TestMigrateIngestionTaskDocumentIDUnique_Integration(t *testing.T) {+ db, ctx := openIngestionTaskMigrationDB(t)++ t.Run("repairs terminal duplicates", func(t *testing.T) {+ createDuplicateIngestionTaskSchema(t, db, ctx, true)+ if err := db.WithContext(ctx).Exec(+ "INSERT INTO pipeline_operation_log (id, document_id, operation_status) VALUES (?, ?, ?)",+ "run-done", "doc-duplicate", string(entity.TaskStatusDone),+ ).Error; err != nil {+ t.Fatalf("seed completed pipeline log: %v", err)+ }+ if err := db.WithContext(ctx).Exec(`INSERT INTO ingestion_task+ (id, document_id, status, create_time, pipeline_log_id)+ VALUES+ ('task-old', 'doc-duplicate', 'COMPLETED', 100, 'run-done'),+ ('task-new', 'doc-duplicate', 'FAILED', 200, NULL)`).Error; err != nil {+ t.Fatalf("seed duplicate ingestion tasks: %v", err)+ }++ if err := migrateIngestionTaskDocumentIDUnique(ctx, db); err != nil {+ t.Fatalf("migrateIngestionTaskDocumentIDUnique: %v", err)+ }+ var ids []string+ if err := db.WithContext(ctx).Table("ingestion_task").Pluck("id", &ids).Error; err != nil {+ t.Fatalf("list remaining ingestion tasks: %v", err)+ }+ if !reflect.DeepEqual(ids, []string{"task-new"}) {+ t.Fatalf("remaining task ids = %v, want [task-new]", ids)+ }+ hasUnique, err := hasUniqueIndex(ctx, db, "ingestion_task", []string{"document_id"})+ if err != nil {+ t.Fatalf("check unique document index: %v", err)+ }+ if !hasUnique {+ t.Fatal("ingestion_task.document_id is not unique after migration")+ }+ if err := migrateIngestionTaskDocumentIDUnique(ctx, db); err != nil {+ t.Fatalf("second migrateIngestionTaskDocumentIDUnique: %v", err)+ }+ })++ t.Run("rejects active duplicate", func(t *testing.T) {+ createDuplicateIngestionTaskSchema(t, db, ctx, false)+ if err := db.WithContext(ctx).Exec(`INSERT INTO ingestion_task+ (id, document_id, status, create_time, pipeline_log_id)+ VALUES+ ('task-active', 'doc-duplicate', 'RUNNING', 100, NULL),+ ('task-new', 'doc-duplicate', 'COMPLETED', 200, NULL)`).Error; err != nil {+ t.Fatalf("seed duplicate ingestion tasks: %v", err)+ }++ err := migrateIngestionTaskDocumentIDUnique(ctx, db)+ for _, want := range []string{"doc-duplicate", "task-active", common.RUNNING} {+ if err == nil || !strings.Contains(err.Error(), want) {+ t.Fatalf("migration error = %v, want it to contain %q", err, want)+ }+ }+ var count int64+ if err := db.WithContext(ctx).Table("ingestion_task").Count(&count).Error; err != nil {+ t.Fatalf("count ingestion tasks: %v", err)+ }+ if count != 2 {+ t.Fatalf("ingestion task count = %d, want 2 after failed migration", count)+ }+ hasUnique, err := hasUniqueIndex(ctx, db, "ingestion_task", []string{"document_id"})+ if err != nil {+ t.Fatalf("check unique document index: %v", err)+ }+ if hasUnique {+ t.Fatal("ingestion_task.document_id unexpectedly became unique")+ }+ })+}diff --git a/internal/dao/pipeline_operation_log.go b/internal/dao/pipeline_operation_log.goindex a541bfd641..8d85b155e1 100644--- a/internal/dao/pipeline_operation_log.go+++ b/internal/dao/pipeline_operation_log.go@@ -272,16 +272,20 @@ func (dao *PipelineOperationLogDAO) NextRunCount(ctx context.Context, db *gorm.D// the from-states the caller declares:// the transitions are monotonic (unstart -> schedule -> running), so a late// queued write cannot regress a row a concurrent writer already advanced.-// A row that already left the declared from-states is left untouched. The-// legacy progress_msg column is intentionally not updated; run text belongs-// to ingestion_task_log events.-func (dao *PipelineOperationLogDAO) AdvanceOpenLog(ctx context.Context, db *gorm.DB, logID string, fromStatuses []string, operationStatus string) error {+// A row that already left the declared from-states is left untouched and the+// returned bool is false. The legacy progress_msg column is intentionally not+// updated; run text belongs to ingestion_task_log events.+func (dao *PipelineOperationLogDAO) AdvanceOpenLog(ctx context.Context, db *gorm.DB, logID string, fromStatuses []string, operationStatus string) (bool, error) {if logID == "" || len(fromStatuses) == 0 {- return nil+ return false, nil}- return db.WithContext(ctx).Model(&entity.PipelineOperationLog{}).+ result := db.WithContext(ctx).Model(&entity.PipelineOperationLog{}).Where("id = ? AND operation_status IN ?", logID, fromStatuses).- Update("operation_status", operationStatus).Error+ Update("operation_status", operationStatus)+ if result.Error != nil {+ return false, result.Error+ }+ return result.RowsAffected == 1, nil}// GetByIDAndKBID fetches a single ingestion log scoped to its knowledge base.diff --git a/internal/development.md b/internal/development.mdindex 3b784556e9..8f857b44a8 100644--- a/internal/development.md+++ b/internal/development.md@@ -307,10 +307,18 @@ export KVROCKS_PORT=6379In docker the Go services override these to `kvrocks:6379` automatically(see `docker-compose-go.yml`). If Kvrocks is unreachable at startup theprocess fails fast with `failed to connect to Kvrocks`.- HTML ingestion uses Kvrocks to hand inline images from the parser to the- chunker; the ingestor also validates the injected Kvrocks client at startup.- HTML image payloads default to a 32 MiB per-image limit; set- `RAGFLOW_HTML_IMAGE_MAX_BYTES` to a positive byte count to change it.+ HTML and EPUB ingestion use Kvrocks to hand inline images from the parser to+ the chunker; the ingestor also validates the injected Kvrocks client at startup.+ Inline image payloads default to a 32 MiB per-image limit; set+ `RAGFLOW_MEDIA_IMAGE_MAX_BYTES` to a positive byte count to change it.+ The parser budget and media cache apply the same numeric run caps: 1024 items+ and 1 GiB of decoded image bytes per run. The parser counts distinct source+ references; the cache counts distinct SHA-256 payloads. Set+ `RAGFLOW_MEDIA_MAX_IMAGES_PER_RUN` and `RAGFLOW_MEDIA_MAX_BYTES_PER_RUN` to+ positive values to change them. Quota state is keyed by ingestion task and+ persists across retries of that task until cleanup or the 24-hour media TTL.+ Debug previews use a temporary `pipeline_<document-id>` run scope and purge+ their cache state when the preview finishes.- Start RAGFlowdiff --git a/internal/engine/kvrocks/redis.go b/internal/engine/kvrocks/redis.goindex 4c15a25503..8cef086819 100644--- a/internal/engine/kvrocks/redis.go+++ b/internal/engine/kvrocks/redis.go@@ -46,6 +46,7 @@ type Client struct {client *redis.ClientluaDeleteIfEqual *redis.ScriptluaTokenBucket *redis.Script+ luaMediaPut *redis.ScriptluaAutoIncrement *redis.Scriptconfig config.KvrocksConfig}@@ -104,6 +105,53 @@ const (return {1, tokens}`++ luaMediaPutScript = `+ local media_key = KEYS[1]+ local digest_key = KEYS[2]+ local state_key = KEYS[3]+ local digest = ARGV[1]+ local value = ARGV[2]+ local max_images = tonumber(ARGV[3])+ local max_bytes = tonumber(ARGV[4])+ local ttl = tonumber(ARGV[5])+ local manifest_key = ARGV[6]+ local manifest_member = ARGV[7]++ local function track_manifest()+ if manifest_key ~= "" then+ redis.call("SADD", manifest_key, manifest_member)+ redis.call("EXPIRE", manifest_key, ttl)+ end+ end++ if redis.call("SISMEMBER", digest_key, digest) == 1 then+ if redis.call("EXISTS", media_key) == 0 then+ redis.call("SET", media_key, value, "EX", ttl)+ redis.call("EXPIRE", digest_key, ttl)+ redis.call("EXPIRE", state_key, ttl)+ track_manifest()+ end+ return {1, 0}+ end++ local images = redis.call("SCARD", digest_key)+ local bytes = tonumber(redis.call("HGET", state_key, "bytes") or "0")+ if max_images > 0 and images >= max_images then+ return {0, 0}+ end+ if max_bytes > 0 and #value > max_bytes - bytes then+ return {0, 0}+ end++ redis.call("SET", media_key, value, "EX", ttl)+ track_manifest()+ redis.call("SADD", digest_key, digest)+ redis.call("HINCRBY", state_key, "bytes", #value)+ redis.call("EXPIRE", digest_key, ttl)+ redis.call("EXPIRE", state_key, ttl)+ return {2, 1}+ `)// Init initializes the Kvrocks client. Kvrocks is the only supported cache/queue@@ -139,6 +187,7 @@ func Init(ctx context.Context) error {config: kvrocksConfig,luaDeleteIfEqual: redis.NewScript(luaDeleteIfEqualScript),luaTokenBucket: redis.NewScript(luaTokenBucketScript),+ luaMediaPut: redis.NewScript(luaMediaPutScript),}common.Info("Kvrocks client initialized",@@ -368,6 +417,65 @@ func (r *Client) Set(ctx context.Context, key string, value string, exp time.Durreturn true}+// MediaPut atomically writes one media payload, records it on the manifest,+// and charges the run quota. accepted is false when a configured count or byte+// cap rejects the payload. inserted is false for a digest already charged by+// this scope; in that case a missing media value is restored without charging+// the quota again.+func (r *Client) MediaPut(+ ctx context.Context,+ mediaKey, digestKey, stateKey, digest, value string,+ manifestKey, manifestMember string,+ maxImages int,+ maxBytes int64,+ ttl time.Duration,+) (accepted, inserted bool, err error) {+ if r == nil || r.client == nil || r.luaMediaPut == nil {+ return false, false, fmt.Errorf("redis: media cache is not initialised")+ }+ ttlSeconds := int64(math.Ceil(ttl.Seconds()))+ if ttlSeconds <= 0 {+ return false, false, fmt.Errorf("redis: media cache ttl must be positive")+ }+ result, err := r.luaMediaPut.Run(+ ctx,+ r.client,+ []string{mediaKey, digestKey, stateKey},+ digest,+ value,+ maxImages,+ maxBytes,+ ttlSeconds,+ manifestKey,+ manifestMember,+ ).Result()+ if err != nil {+ return false, false, fmt.Errorf("media put: %w", err)+ }+ values, ok := result.([]interface{})+ if !ok || len(values) != 2 {+ return false, false, fmt.Errorf("media put: malformed reply")+ }+ status, ok := values[0].(int64)+ if !ok {+ return false, false, fmt.Errorf("media put: malformed status")+ }+ insertedValue, ok := values[1].(int64)+ if !ok {+ return false, false, fmt.Errorf("media put: malformed insert result")+ }+ switch status {+ case 0:+ return false, false, nil+ case 1:+ return true, insertedValue == 1, nil+ case 2:+ return true, true, nil+ default:+ return false, false, fmt.Errorf("media put: unknown status %d", status)+ }+}+// SetNX sets value only if key does not exist.// It returns false with a nil error when the key already exists.func (r *Client) SetNX(ctx context.Context, key string, value string, exp time.Duration) (bool, error) {diff --git a/internal/engine/kvrocks/redis_test.go b/internal/engine/kvrocks/redis_test.goindex fa1026324d..5395025b5e 100644--- a/internal/engine/kvrocks/redis_test.go+++ b/internal/engine/kvrocks/redis_test.go@@ -43,6 +43,7 @@ func newStrictTestClient(t *testing.T) (*Client, *miniredis.Miniredis) {client: rdb,luaDeleteIfEqual: redis.NewScript(luaDeleteIfEqualScript),luaTokenBucket: redis.NewScript(luaTokenBucketScript),+ luaMediaPut: redis.NewScript(luaMediaPutScript),// luaAutoIncrement intentionally not loaded; not used here.}, mr}@@ -145,6 +146,167 @@ func TestEvalTokenBucketStrict_NilClient(t *testing.T) {}}+func TestMediaPutEnforcesQuotaAndRestoresDuplicate(t *testing.T) {+ r, _ := newStrictTestClient(t)+ ctx := t.Context()+ mediaKey := "mc:media:test:a"+ digestKey := "mc:quota:test:digests"+ stateKey := "mc:quota:test:state"+ manifestKey := "mc:manifest:test"++ accepted, inserted, err := r.MediaPut(+ ctx,+ mediaKey,+ digestKey,+ stateKey,+ "digest-a",+ "abc",+ manifestKey,+ mediaKey,+ 1,+ 3,+ time.Hour,+ )+ if err != nil || !accepted || !inserted {+ t.Fatalf("first MediaPut = (%v, %v, %v), want (true, true, nil)", accepted, inserted, err)+ }+ accepted, inserted, err = r.MediaPut(+ ctx,+ "mc:media:test:b",+ digestKey,+ stateKey,+ "digest-b",+ "b",+ manifestKey,+ "mc:media:test:b",+ 1,+ 3,+ time.Hour,+ )+ if err != nil || accepted || inserted {+ t.Fatalf("limit MediaPut = (%v, %v, %v), want (false, false, nil)", accepted, inserted, err)+ }++ if !r.Delete(ctx, mediaKey) {+ t.Fatal("delete media value failed")+ }+ for _, key := range []string{digestKey, stateKey} {+ if err := r.GetClient().Expire(ctx, key, time.Second).Err(); err != nil {+ t.Fatalf("shorten %s TTL: %v", key, err)+ }+ }+ accepted, inserted, err = r.MediaPut(+ ctx,+ mediaKey,+ digestKey,+ stateKey,+ "digest-a",+ "abc",+ manifestKey,+ mediaKey,+ 1,+ 3,+ time.Hour,+ )+ if err != nil || !accepted || inserted {+ t.Fatalf("restore MediaPut = (%v, %v, %v), want (true, false, nil)", accepted, inserted, err)+ }+ value, err := r.Get(ctx, mediaKey)+ if err != nil || value != "abc" {+ t.Fatalf("restored value = %q, %v; want abc, nil", value, err)+ }+ for _, key := range []string{mediaKey, digestKey, stateKey} {+ ttl, err := r.GetClient().TTL(ctx, key).Result()+ if err != nil || ttl < 59*time.Minute {+ t.Fatalf("restored %s TTL = %v, %v; want the refreshed one-hour lifetime", key, ttl, err)+ }+ }+}++func TestMediaPutCountsDigestSetInsteadOfMirroredHash(t *testing.T) {+ r, _ := newStrictTestClient(t)+ ctx := t.Context()+ digestKey := "mc:quota:scard:digests"+ stateKey := "mc:quota:scard:state"++ accepted, inserted, err := r.MediaPut(+ ctx,+ "mc:media:scard:a",+ digestKey,+ stateKey,+ "digest-a",+ "a",+ "",+ "",+ 1,+ 100,+ time.Hour,+ )+ if err != nil || !accepted || !inserted {+ t.Fatalf("first MediaPut = (%v, %v, %v), want (true, true, nil)", accepted, inserted, err)+ }+ if err := r.GetClient().HSet(ctx, stateKey, "images", 0).Err(); err != nil {+ t.Fatalf("make mirrored image count diverge: %v", err)+ }++ accepted, inserted, err = r.MediaPut(+ ctx,+ "mc:media:scard:b",+ digestKey,+ stateKey,+ "digest-b",+ "b",+ "",+ "",+ 1,+ 100,+ time.Hour,+ )+ if err != nil || accepted || inserted {+ t.Fatalf("second MediaPut = (%v, %v, %v), want (false, false, nil)", accepted, inserted, err)+ }+}++func TestMediaPutRejectsByteOverflow(t *testing.T) {+ r, _ := newStrictTestClient(t)+ ctx := t.Context()+ digestKey := "mc:quota:bytes:digests"+ stateKey := "mc:quota:bytes:state"++ accepted, inserted, err := r.MediaPut(+ ctx,+ "mc:media:bytes:a",+ digestKey,+ stateKey,+ "digest-a",+ "abcde",+ "",+ "",+ 10,+ 5,+ time.Hour,+ )+ if err != nil || !accepted || !inserted {+ t.Fatalf("first MediaPut = (%v, %v, %v), want (true, true, nil)", accepted, inserted, err)+ }+ accepted, inserted, err = r.MediaPut(+ ctx,+ "mc:media:bytes:b",+ digestKey,+ stateKey,+ "digest-b",+ "b",+ "",+ "",+ 10,+ 5,+ time.Hour,+ )+ if err != nil || accepted || inserted {+ t.Fatalf("overflow MediaPut = (%v, %v, %v), want (false, false, nil)", accepted, inserted, err)+ }+}+func TestClientSetNXDistinguishesContention(t *testing.T) {r, _ := newStrictTestClient(t)ctx := t.Context()diff --git a/internal/ingestion/component/media_dispatch.go b/internal/ingestion/component/media_dispatch.goindex f0b9334c75..ad8073e320 100644--- a/internal/ingestion/component/media_dispatch.go+++ b/internal/ingestion/component/media_dispatch.go@@ -109,7 +109,7 @@ func maybeDispatchImage(return parser.ParseResult{}, true, err}defer release()- if len(binary) == 0 || len(binary) > media.MaxImagePayloadBytes {+ if len(binary) == 0 || len(binary) > media.ImageMaxBytes() {return parser.ParseResult{}, true, fmt.Errorf("parser: image payload exceeds size limits")}// SVG carries its text in the document, so it is read rather thandiff --git a/internal/ingestion/component/svg_text.go b/internal/ingestion/component/svg_text.goindex 6fde5604b1..93dea01bac 100644--- a/internal/ingestion/component/svg_text.go+++ b/internal/ingestion/component/svg_text.go@@ -34,14 +34,17 @@ import ("unicode/utf8"htmlcharset "golang.org/x/net/html/charset"-- "ragflow/internal/parser/media")// maxSVGDepth bounds element nesting: the XML decoder keeps one stack entry// per open element, so depth is the allocation an SVG payload could inflate.const maxSVGDepth = 1024+// maxSVGEntityExpansionBytes bounds the total expanded size of DOCTYPE+// entities. It is a fixed parser-safety bound, not the configurable image+// limit.+const maxSVGEntityExpansionBytes = 32 << 20+// maxSVGEntities bounds the entity declarations read from a DOCTYPE.const maxSVGEntities = 64@@ -117,7 +120,7 @@ func extractSVGText(ctx context.Context, data []byte) (string, error) {value := string(match[2]) + string(match[3])references := bytes.Count(data, []byte("&"+string(match[1])+";"))expansion += int64(references) * int64(len(value))- if expansion > media.MaxImagePayloadBytes {+ if expansion > maxSVGEntityExpansionBytes {return "", errors.New("parser: decode SVG: entity expansion exceeds size limits")}entities[string(match[1])] = valuediff --git a/internal/ingestion/component/vision_image.go b/internal/ingestion/component/vision_image.goindex f304dd6372..73d383ae26 100644--- a/internal/ingestion/component/vision_image.go+++ b/internal/ingestion/component/vision_image.go@@ -35,9 +35,12 @@ import (parserlib "ragflow/internal/parser/parser")-const (- maxVisionImageBytes = media.MaxImagePayloadBytes+var (+ maxVisionImageBytes = media.ImageMaxBytes()maxVLMEncodedBytes = (maxVisionImageBytes+2)/3*4 + 256+)++const (// maxVLMCropPixels and maxVLMCropEdge bound the raster of PDF page crops// rendered for the vision model (see vision_enhancement_cgo.go). They are a// pixel budget for the VLM crop path and are intentionally separate from@@ -100,7 +103,7 @@ func inspectVisionImagePayload(payload string) (string, error) {}for _, encoding := range []*base64.Encoding{base64.StdEncoding, base64.RawStdEncoding} {decoder := base64.NewDecoder(encoding, &visionPayloadReader{source: payload})- config, format, err := image.DecodeConfig(io.LimitReader(decoder, maxVisionImageBytes+1))+ config, format, err := image.DecodeConfig(io.LimitReader(decoder, int64(maxVisionImageBytes)+1))if err != nil {continue}@@ -130,7 +133,7 @@ func decodeVisionPayload(raw string) ([]byte, error) {}for _, encoding := range []*base64.Encoding{base64.StdEncoding, base64.RawStdEncoding} {decoder := base64.NewDecoder(encoding, &visionPayloadReader{source: encoded})- data, err := io.ReadAll(io.LimitReader(decoder, maxVisionImageBytes+1))+ data, err := io.ReadAll(io.LimitReader(decoder, int64(maxVisionImageBytes)+1))if err != nil {continue}diff --git a/internal/ingestion/mediacache/store.go b/internal/ingestion/mediacache/store.goindex 69ff9a38c5..c9c477a240 100644--- a/internal/ingestion/mediacache/store.go+++ b/internal/ingestion/mediacache/store.go@@ -25,7 +25,6 @@ import ("errors""fmt""strings"- "sync""time""github.com/cespare/xxhash/v2"@@ -41,26 +40,48 @@ const (TTL = 24 * time.Hour)-// Store is a run-scoped media cache backed by a chunkcache.Store.+// Store is a run-scoped media cache backed by a Backend. It dedupes writes by+// digest and bounds how many distinct images, and how many decoded bytes, one+// run may keep in the backend.type Store struct {- backend chunkcache.Store+ backend Backendscope string- mu sync.Mutex+ maxImages int+ maxBytes int64+}++// Backend provides the cache lifecycle operations and atomic quota admission.+// The quota, media value, and manifest update happen in one MediaPut operation+// so separate Store instances and processes cannot each admit a full run+// allowance.+type Backend interface {+ chunkcache.Store++ MediaPut(+ ctx context.Context,+ mediaKey, digestKey, stateKey, digest, value string,+ manifestKey, manifestMember string,+ maxImages int,+ maxBytes int64,+ ttl time.Duration,+ ) (accepted, inserted bool, err error)}// New returns a cache bound to one ingestion run. A nil backend is a valid// value so callers can surface configured-cache failures only when media is// actually encountered.-func New(backend chunkcache.Store, scope string) *Store {+func New(backend Backend, scope string) *Store {return &Store{- backend: backend,- scope: strings.TrimSpace(scope),+ backend: backend,+ scope: strings.TrimSpace(scope),+ maxImages: media.MediaMaxImagesPerRun(),+ maxBytes: media.MediaMaxBytesPerRun(),}}-// Put validates and stores one decoded image, then records the key on the-// current task manifest. Any write or lifecycle failure is returned.+// Put validates and atomically stores one decoded image with its manifest and+// quota state. Any write or lifecycle failure is returned.func (s *Store) Put(ctx context.Context, format, mime string, data []byte) (entity.MediaRef, error) {if s == nil || s.backend == nil {return entity.MediaRef{}, errors.New("mediacache: cache is not configured")@@ -77,8 +98,8 @@ func (s *Store) Put(ctx context.Context, format, mime string, data []byte) (entiif strings.TrimSpace(mime) == "" {return entity.MediaRef{}, errors.New("mediacache: media type is empty")}- if len(data) == 0 || len(data) > media.HTMLImageMaxBytes() {- return entity.MediaRef{}, fmt.Errorf("mediacache: image size %d is outside the 1..%d-byte limit", len(data), media.HTMLImageMaxBytes())+ if len(data) == 0 || len(data) > media.ImageMaxBytes() {+ return entity.MediaRef{}, fmt.Errorf("mediacache: image size %d is outside the 1..%d-byte limit", len(data), media.ImageMaxBytes())}sum := sha256.Sum256(data)@@ -92,18 +113,35 @@ func (s *Store) Put(ctx context.Context, format, mime string, data []byte) (entiSize: int64(len(data)),}- s.mu.Lock()- defer s.mu.Unlock()-key := s.key(digest)- if !s.backend.Set(ctx, key, string(data), TTL) {- return entity.MediaRef{}, fmt.Errorf("mediacache: write image %s", mediaShortDigest(digest))+ mk := ""+ if globals.TaskID(ctx) != "" {+ mk = manifestKey(s.scope)+ }+ digestKey, stateKey := quotaKeys(s.scope)+ accepted, _, err := s.backend.MediaPut(+ ctx,+ key,+ digestKey,+ stateKey,+ digest,+ string(data),+ mk,+ key,+ s.maxImages,+ s.maxBytes,+ TTL,+ )+ if err != nil {+ return entity.MediaRef{}, fmt.Errorf("mediacache: write image %s: %w", mediaShortDigest(digest), err)}- if err := s.track(ctx, key); err != nil {- if !s.backend.Delete(ctx, key) {- return entity.MediaRef{}, fmt.Errorf("mediacache: track image %s: %w (rollback failed)", mediaShortDigest(digest), err)- }- return entity.MediaRef{}, fmt.Errorf("mediacache: track image %s: %w", mediaShortDigest(digest), err)+ if !accepted {+ return entity.MediaRef{}, fmt.Errorf(+ "mediacache: run media limit exceeded (%d images / %d bytes): %w",+ s.maxImages,+ s.maxBytes,+ media.ErrMediaLimitExceeded,+ )}return ref, nil}@@ -130,8 +168,8 @@ func (s *Store) Get(ctx context.Context, ref entity.MediaRef) ([]byte, error) {if ref.Scope != s.scope {return nil, fmt.Errorf("mediacache: reference scope %q does not match run scope %q", ref.Scope, s.scope)}- if ref.Size > int64(media.HTMLImageMaxBytes()) {- return nil, fmt.Errorf("mediacache: image %s exceeds the %d-byte limit", mediaShortDigest(ref.SHA256), media.HTMLImageMaxBytes())+ if ref.Size > int64(media.ImageMaxBytes()) {+ return nil, fmt.Errorf("mediacache: image %s exceeds the %d-byte limit", mediaShortDigest(ref.SHA256), media.ImageMaxBytes())}value, err := s.backend.Get(ctx, s.key(ref.SHA256))@@ -163,20 +201,10 @@ func manifestKey(scope string) string {return fmt.Sprintf("%s:manifest:%x", keyNamespace, xxhash.Sum64String(scope))}-func (s *Store) track(ctx context.Context, key string) error {- // Taskless preview runs have no persist-time cleanup and rely on TTL.- if globals.TaskID(ctx) == "" {- return nil- }- mk := manifestKey(s.scope)- if !s.backend.SAdd(ctx, mk, key) {- return fmt.Errorf("mediacache: record manifest %s", mk)- }- if !s.backend.Expire(ctx, mk, TTL) {- s.backend.Delete(ctx, mk)- return fmt.Errorf("mediacache: expire manifest %s", mk)- }- return nil+func quotaKeys(scope string) (digestKey, stateKey string) {+ hashedScope := xxhash.Sum64String(scope)+ return fmt.Sprintf("%s:quota:%x:digests", keyNamespace, hashedScope),+ fmt.Sprintf("%s:quota:%x:state", keyNamespace, hashedScope)}// PurgeTask removes media after durable persistence. Failed runs rely on TTL.@@ -197,5 +225,11 @@ func PurgeTask(ctx context.Context, backend chunkcache.Store, taskID string) errif !backend.Delete(ctx, mk) {return fmt.Errorf("mediacache: delete manifest %s", mk)}+ digestKey, stateKey := quotaKeys(taskID)+ for _, key := range []string{digestKey, stateKey} {+ if !backend.Delete(ctx, key) {+ return fmt.Errorf("mediacache: delete quota state %s", key)+ }+ }return nil}diff --git a/internal/ingestion/mediacache/store_test.go b/internal/ingestion/mediacache/store_test.goindex 0a1b34166e..6aacd586ac 100644--- a/internal/ingestion/mediacache/store_test.go+++ b/internal/ingestion/mediacache/store_test.go@@ -38,14 +38,17 @@ type fakeStore struct {expireFails boolsetCalls intaddCalls int+ mediaPuts intdeleteFails map[string]bool+ quotaBytes map[string]int64}func newFakeStore() *fakeStore {return &fakeStore{- kv: map[string]string{},- sets: map[string]map[string]bool{},- ttl: map[string]time.Duration{},+ kv: map[string]string{},+ sets: map[string]map[string]bool{},+ ttl: map[string]time.Duration{},+ quotaBytes: map[string]int64{},}}@@ -66,6 +69,69 @@ func (f *fakeStore) Set(_ context.Context, key, value string, exp time.Duration)return true}+func (f *fakeStore) MediaPut(+ _ context.Context,+ mediaKey, digestKey, stateKey, digest, value string,+ manifestKey, manifestMember string,+ maxImages int,+ maxBytes int64,+ ttl time.Duration,+) (bool, bool, error) {+ f.mediaPuts+++ if f.failSet {+ return false, false, errors.New("fake media write failure")+ }+ if f.expireFails {+ return false, false, errors.New("fake manifest expire failure")+ }+ if f.sets[digestKey][digest] {+ if f.kv[mediaKey] == "" {+ f.setCalls+++ f.kv[mediaKey] = value+ f.ttl[mediaKey] = ttl+ f.ttl[digestKey] = ttl+ f.ttl[stateKey] = ttl+ if manifestKey != "" {+ f.addCalls+++ if f.sets[manifestKey] == nil {+ f.sets[manifestKey] = map[string]bool{}+ }+ f.sets[manifestKey][manifestMember] = true+ f.ttl[manifestKey] = ttl+ }+ }+ return true, false, nil+ }+ if maxImages > 0 && len(f.sets[digestKey]) >= maxImages {+ return false, false, nil+ }+ if maxBytes > 0 && int64(len(value)) > maxBytes-f.quotaBytes[stateKey] {+ return false, false, nil+ }++ f.setCalls+++ f.kv[mediaKey] = value+ f.ttl[mediaKey] = ttl+ if manifestKey != "" {+ f.addCalls+++ if f.sets[manifestKey] == nil {+ f.sets[manifestKey] = map[string]bool{}+ }+ f.sets[manifestKey][manifestMember] = true+ }+ if f.sets[digestKey] == nil {+ f.sets[digestKey] = map[string]bool{}+ }+ f.sets[digestKey][digest] = true+ f.quotaBytes[stateKey] += int64(len(value))+ f.ttl[digestKey] = ttl+ f.ttl[stateKey] = ttl+ if manifestKey != "" {+ f.ttl[manifestKey] = ttl+ }+ return true, true, nil+}+func (f *fakeStore) SAdd(_ context.Context, key, member string) bool {f.addCalls++if f.sets[key] == nil {@@ -90,6 +156,7 @@ func (f *fakeStore) Delete(_ context.Context, key string) bool {}delete(f.kv, key)delete(f.sets, key)+ delete(f.quotaBytes, key)return true}@@ -236,7 +303,7 @@ func TestPut_RejectsMissingStoreScopeAndOversizedImage(t *testing.T) {if _, err := New(nil, "task-1").Put(ctx, "html", "image/png", []byte("hello")); err == nil {t.Fatal("Put accepted an unavailable cache")}- t.Setenv("RAGFLOW_HTML_IMAGE_MAX_BYTES", "5")+ t.Setenv("RAGFLOW_MEDIA_IMAGE_MAX_BYTES", "5")data := make([]byte, 6)if _, err := New(backend, "task-1").Put(ctx, "html", "image/png", data); err == nil {t.Fatal("Put accepted an oversized image")@@ -349,7 +416,7 @@ func TestRetryRepopulatesMediaAfterCacheMiss(t *testing.T) {}}-func TestPutRestoresMissingMediaInSameStore(t *testing.T) {+func TestPutDuplicateWritesOnceAndReturnsCallerRef(t *testing.T) {backend := newFakeStore()ctx := taskContext("task-1")store := New(backend, "task-1")@@ -358,26 +425,127 @@ func TestPutRestoresMissingMediaInSameStore(t *testing.T) {if err != nil {t.Fatal(err)}- backend.Delete(ctx, store.key(ref.SHA256))- backend.Delete(ctx, manifestKey("task-1"))+ key := store.key(ref.SHA256)+ backend.ttl[key] = time.Second+ backend.ttl[manifestKey("task-1")] = time.Second- restored, err := store.Put(ctx, "html", "image/png", data)+ duplicate, err := store.Put(ctx, "epub", "image/jpeg", data)if err != nil {- t.Fatal(err)+ t.Fatalf("duplicate Put: %v", err)}- got, err := store.Get(ctx, restored)- if err != nil || string(got) != string(data) {- t.Fatalf("restored media = %q, %v", got, err)+ if duplicate.Format != "epub" || duplicate.MIME != "image/jpeg" || duplicate.SHA256 != ref.SHA256 || duplicate.Size != ref.Size {+ t.Fatalf("duplicate ref = %+v, want the current caller's format/mime on the same digest", duplicate)}- if err := PurgeTask(ctx, backend, "task-1"); err != nil {- t.Fatal(err)+ if backend.mediaPuts != 2 || backend.setCalls != 1 || backend.addCalls != 1 {+ t.Fatalf("backend calls = %d atomic Put, %d Set, %d SAdd, want 2, 1, 1", backend.mediaPuts, backend.setCalls, backend.addCalls)}- if len(backend.kv) != 0 || len(backend.sets) != 0 {- t.Fatal("restored media was not purged")+ if backend.ttl[key] != time.Second || backend.ttl[manifestKey("task-1")] != time.Second {+ t.Fatal("duplicate Put refreshed TTLs; dedupe must not touch the backend")+ }+}++func TestPutDedupeDoesNotConsumeRunLimits(t *testing.T) {+ t.Setenv("RAGFLOW_MEDIA_MAX_IMAGES_PER_RUN", "2")+ t.Setenv("RAGFLOW_MEDIA_MAX_BYTES_PER_RUN", "5")+ backend := newFakeStore()+ ctx := taskContext("task-1")+ store := New(backend, "task-1")+ for attempt := 0; attempt < 3; attempt++ {+ if _, err := store.Put(ctx, "html", "image/png", []byte("abc")); err != nil {+ t.Fatalf("duplicate Put #%d: %v", attempt+1, err)+ }+ }+ if _, err := store.Put(ctx, "html", "image/png", []byte("de")); err != nil {+ t.Fatalf("second distinct image: %v", err)+ }+ if _, err := store.Put(ctx, "html", "image/png", []byte("f")); !errors.Is(err, media.ErrMediaLimitExceeded) {+ t.Fatalf("limit error = %v, want ErrMediaLimitExceeded", err)+ }+}++func TestPutEnforcesRunImageCountLimit(t *testing.T) {+ t.Setenv("RAGFLOW_MEDIA_MAX_IMAGES_PER_RUN", "2")+ backend := newFakeStore()+ ctx := taskContext("task-1")+ store := New(backend, "task-1")+ for _, image := range []string{"a", "b"} {+ if _, err := store.Put(ctx, "html", "image/png", []byte(image)); err != nil {+ t.Fatalf("image %q: %v", image, err)+ }+ }+ if _, err := store.Put(ctx, "html", "image/png", []byte("c")); !errors.Is(err, media.ErrMediaLimitExceeded) {+ t.Fatalf("limit error = %v, want ErrMediaLimitExceeded", err)+ }+ if _, err := store.Put(ctx, "html", "image/png", []byte("c")); !errors.Is(err, media.ErrMediaLimitExceeded) {+ t.Fatalf("repeated rejected error = %v, want ErrMediaLimitExceeded", err)+ }+ if len(backend.kv) != 2 {+ t.Fatalf("cached entries = %d, want the 2 admitted images", len(backend.kv))+ }+ members, _ := backend.SMembers(ctx, manifestKey("task-1"))+ if len(members) != 2 {+ t.Fatalf("manifest = %v, want 2 entries", members)+ }+}++func TestPutEnforcesRunLimitsAcrossStoreInstances(t *testing.T) {+ t.Setenv("RAGFLOW_MEDIA_MAX_IMAGES_PER_RUN", "2")+ backend := newFakeStore()+ ctx := taskContext("task-1")+ first := New(backend, "task-1")+ second := New(backend, "task-1")++ if _, err := first.Put(ctx, "html", "image/png", []byte("a")); err != nil {+ t.Fatalf("first image: %v", err)+ }+ if _, err := second.Put(ctx, "html", "image/png", []byte("b")); err != nil {+ t.Fatalf("second image from another Store: %v", err)+ }+ if _, err := first.Put(ctx, "html", "image/png", []byte("c")); !errors.Is(err, media.ErrMediaLimitExceeded) {+ t.Fatalf("third image error = %v, want ErrMediaLimitExceeded", err)+ }+ if len(backend.kv) != 2 {+ t.Fatalf("cached entries = %d, want 2", len(backend.kv))}}-func TestPutDuplicateRefreshesTTLAndReportsWriteFailure(t *testing.T) {+func TestPutEnforcesRunBytesAcrossStoreInstances(t *testing.T) {+ t.Setenv("RAGFLOW_MEDIA_MAX_BYTES_PER_RUN", "5")+ backend := newFakeStore()+ ctx := taskContext("task-1")+ first := New(backend, "task-1")+ second := New(backend, "task-1")++ if _, err := first.Put(ctx, "html", "image/png", []byte("abc")); err != nil {+ t.Fatalf("first image: %v", err)+ }+ if _, err := second.Put(ctx, "html", "image/png", []byte("de")); err != nil {+ t.Fatalf("second image from another Store: %v", err)+ }+ if _, err := first.Put(ctx, "html", "image/png", []byte("f")); !errors.Is(err, media.ErrMediaLimitExceeded) {+ t.Fatalf("third image error = %v, want ErrMediaLimitExceeded", err)+ }+}++func TestPutEnforcesRunByteLimitAtExactBoundary(t *testing.T) {+ t.Setenv("RAGFLOW_MEDIA_MAX_BYTES_PER_RUN", "5")+ backend := newFakeStore()+ ctx := taskContext("task-1")+ store := New(backend, "task-1")+ for _, image := range []string{"abc", "de"} {+ if _, err := store.Put(ctx, "html", "image/png", []byte(image)); err != nil {+ t.Fatalf("image %q: %v", image, err)+ }+ }+ if _, err := store.Put(ctx, "html", "image/png", []byte("f")); !errors.Is(err, media.ErrMediaLimitExceeded) {+ t.Fatalf("limit error = %v, want ErrMediaLimitExceeded", err)+ }+ if len(backend.kv) != 2 {+ t.Fatalf("cached entries = %d, want the 2 admitted images", len(backend.kv))+ }+}++func TestPutRestoresLocallySeenDigestAfterBackendEviction(t *testing.T) {backend := newFakeStore()ctx := taskContext("task-1")store := New(backend, "task-1")@@ -387,16 +555,54 @@ func TestPutDuplicateRefreshesTTLAndReportsWriteFailure(t *testing.T) {t.Fatal(err)}key := store.key(ref.SHA256)- backend.ttl[key] = time.Second- backend.ttl[manifestKey("task-1")] = time.Second- if _, err := store.Put(ctx, "html", "image/png", data); err != nil {- t.Fatal(err)- }- if backend.ttl[key] != TTL || backend.ttl[manifestKey("task-1")] != TTL {- t.Fatal("duplicate put did not refresh media and manifest TTLs")+ delete(backend.kv, key)++ restored, err := store.Put(ctx, "html", "image/png", data)+ if err != nil {+ t.Fatalf("restore Put: %v", err)}- backend.failSet = true- if _, err := store.Put(ctx, "html", "image/png", data); err == nil {- t.Fatal("duplicate put succeeded when the backend rejected the write")+ got, err := store.Get(ctx, restored)+ if err != nil || string(got) != string(data) {+ t.Fatalf("restored read = %q, %v; want %q, nil", got, err, data)}}++func TestPutFailedWriteDoesNotConsumeRunLimits(t *testing.T) {+ t.Setenv("RAGFLOW_MEDIA_MAX_IMAGES_PER_RUN", "2")+ t.Setenv("RAGFLOW_MEDIA_MAX_BYTES_PER_RUN", "2")++ t.Run("value write", func(t *testing.T) {+ backend := newFakeStore()+ ctx := taskContext("task-1")+ store := New(backend, "task-1")+ backend.failSet = true+ if _, err := store.Put(ctx, "html", "image/png", []byte("a")); err == nil {+ t.Fatal("Put succeeded when the value write failed")+ }+ backend.failSet = false+ for _, image := range []string{"a", "b"} {+ if _, err := store.Put(ctx, "html", "image/png", []byte(image)); err != nil {+ t.Fatalf("image %q after recovered write: %v", image, err)+ }+ }+ })++ t.Run("manifest track", func(t *testing.T) {+ backend := newFakeStore()+ ctx := taskContext("task-1")+ store := New(backend, "task-1")+ backend.expireFails = true+ if _, err := store.Put(ctx, "html", "image/png", []byte("a")); err == nil {+ t.Fatal("Put succeeded when the manifest TTL failed")+ }+ if len(backend.kv) != 0 {+ t.Fatalf("untracked media entries = %v, want rollback", backend.kv)+ }+ backend.expireFails = false+ for _, image := range []string{"a", "b"} {+ if _, err := store.Put(ctx, "html", "image/png", []byte(image)); err != nil {+ t.Fatalf("image %q after recovered track: %v", image, err)+ }+ }+ })+}diff --git a/internal/ingestion/pipeline/cleanup.go b/internal/ingestion/pipeline/cleanup.goindex 0bb0b5b974..860ca3f5da 100644--- a/internal/ingestion/pipeline/cleanup.go+++ b/internal/ingestion/pipeline/cleanup.go@@ -11,8 +11,18 @@ import (// PurgeTaskState removes the run-scoped cache state for a task using the// process-wide Kvrocks client. Model results and media use separate manifests.func PurgeTaskState(ctx context.Context, taskID string) error {+ return purgeTaskStateWithBackend(ctx, chunkcache.Client(), taskID)+}++// purgeTaskStateWithBackend purges the chunk and media manifests on one+// backend. The task-id guard stays in both entry points so an empty id can+// never purge a shared or empty-keyed manifest.+func purgeTaskStateWithBackend(ctx context.Context, backend chunkcache.Store, taskID string) error {if taskID == "" {return errors.New("task state cleanup requires a task id")}- return errors.Join(chunkcache.PurgeTask(ctx, chunkcache.Client(), taskID), mediacache.PurgeTask(ctx, chunkcache.Client(), taskID))+ return errors.Join(+ chunkcache.PurgeTask(ctx, backend, taskID),+ mediacache.PurgeTask(ctx, backend, taskID),+ )}diff --git a/internal/ingestion/pipeline/cleanup_test.go b/internal/ingestion/pipeline/cleanup_test.gonew file mode 100644index 0000000000..b9bff1f8c9--- /dev/null+++ b/internal/ingestion/pipeline/cleanup_test.go@@ -0,0 +1,206 @@+//+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.+//++package pipeline++import (+ "context"+ "errors"+ "sort"+ "strings"+ "testing"+ "time"++ "ragflow/internal/agent/runtime"+ "ragflow/internal/ingestion/chunkcache"+ "ragflow/internal/ingestion/component/globals"+ "ragflow/internal/ingestion/mediacache"+)++type cleanupFakeStore struct {+ kv map[string]string+ sets map[string]map[string]bool+ ttl map[string]time.Duration+ failRead map[string]bool+ bytes map[string]int64+}++func newCleanupFakeStore() *cleanupFakeStore {+ return &cleanupFakeStore{+ kv: map[string]string{},+ sets: map[string]map[string]bool{},+ ttl: map[string]time.Duration{},+ failRead: map[string]bool{},+ bytes: map[string]int64{},+ }+}++func (f *cleanupFakeStore) Get(_ context.Context, key string) (string, error) {+ return f.kv[key], nil+}++func (f *cleanupFakeStore) Set(_ context.Context, key, value string, exp time.Duration) bool {+ f.kv[key] = value+ f.ttl[key] = exp+ return true+}++func (f *cleanupFakeStore) MediaPut(+ _ context.Context,+ mediaKey, digestKey, stateKey, digest, value string,+ manifestKey, manifestMember string,+ maxImages int,+ maxBytes int64,+ ttl time.Duration,+) (bool, bool, error) {+ if f.sets[digestKey][digest] {+ if f.kv[mediaKey] == "" {+ f.kv[mediaKey] = value+ f.ttl[mediaKey] = ttl+ f.ttl[digestKey] = ttl+ f.ttl[stateKey] = ttl+ if manifestKey != "" {+ if f.sets[manifestKey] == nil {+ f.sets[manifestKey] = map[string]bool{}+ }+ f.sets[manifestKey][manifestMember] = true+ f.ttl[manifestKey] = ttl+ }+ }+ return true, false, nil+ }+ if maxImages > 0 && len(f.sets[digestKey]) >= maxImages {+ return false, false, nil+ }+ if maxBytes > 0 && int64(len(value)) > maxBytes-f.bytes[stateKey] {+ return false, false, nil+ }+ f.kv[mediaKey] = value+ f.ttl[mediaKey] = ttl+ if manifestKey != "" {+ if f.sets[manifestKey] == nil {+ f.sets[manifestKey] = map[string]bool{}+ }+ f.sets[manifestKey][manifestMember] = true+ f.ttl[manifestKey] = ttl+ }+ if f.sets[digestKey] == nil {+ f.sets[digestKey] = map[string]bool{}+ }+ f.sets[digestKey][digest] = true+ f.bytes[stateKey] += int64(len(value))+ f.ttl[digestKey] = ttl+ f.ttl[stateKey] = ttl+ return true, true, nil+}++func (f *cleanupFakeStore) SAdd(_ context.Context, key, member string) bool {+ if f.sets[key] == nil {+ f.sets[key] = map[string]bool{}+ }+ f.sets[key][member] = true+ return true+}++func (f *cleanupFakeStore) SMembers(_ context.Context, key string) ([]string, error) {+ if f.failRead[key] {+ return nil, errors.New("kvrocks unavailable")+ }+ out := make([]string, 0, len(f.sets[key]))+ for member := range f.sets[key] {+ out = append(out, member)+ }+ sort.Strings(out)+ return out, nil+}++func (f *cleanupFakeStore) Delete(_ context.Context, key string) bool {+ delete(f.kv, key)+ delete(f.sets, key)+ delete(f.bytes, key)+ return true+}++func (f *cleanupFakeStore) Expire(_ context.Context, key string, exp time.Duration) bool {+ f.ttl[key] = exp+ return true+}++func cleanupTaskContext(taskID string) context.Context {+ ctx := runtime.WithState(context.Background(), runtime.NewCanvasState("", ""))+ globals.SetTaskID(ctx, taskID)+ return ctx+}++func seedTaskManifests(t *testing.T, backend *cleanupFakeStore, taskID string) {+ t.Helper()+ ctx := cleanupTaskContext(taskID)+ chunkcache.Set(ctx, backend, "kc:emb:chunk-1", "model result")+ if _, err := mediacache.New(backend, taskID).Put(ctx, "html", "image/png", []byte("hello")); err != nil {+ t.Fatalf("seed media cache: %v", err)+ }+ if len(backend.kv) != 2 || len(backend.sets) != 3 {+ t.Fatalf("seeded state = %d values, %d sets, want 2 and 3", len(backend.kv), len(backend.sets))+ }+}++func TestPurgeTaskStateWithBackendRemovesChunkAndMediaManifests(t *testing.T) {+ backend := newCleanupFakeStore()+ ctx := cleanupTaskContext("task-1")+ seedTaskManifests(t, backend, "task-1")++ if err := purgeTaskStateWithBackend(ctx, backend, "task-1"); err != nil {+ t.Fatalf("purgeTaskStateWithBackend: %v", err)+ }+ if len(backend.kv) != 0 || len(backend.sets) != 0 {+ t.Fatalf("state after purge = %v values, %v manifests, want empty", backend.kv, backend.sets)+ }+}++func TestPurgeTaskStateWithBackendJoinsErrorsAndStillPurgesMedia(t *testing.T) {+ backend := newCleanupFakeStore()+ ctx := cleanupTaskContext("task-1")+ seedTaskManifests(t, backend, "task-1")+ backend.failRead["kc:manifest:task-1"] = true++ err := purgeTaskStateWithBackend(ctx, backend, "task-1")+ if err == nil || !strings.Contains(err.Error(), "chunk cache manifest") {+ t.Fatalf("error = %v, want the chunk manifest failure", err)+ }+ for key := range backend.kv {+ if strings.HasPrefix(key, "mc:") {+ t.Fatalf("media value %s survived a failed chunk purge", key)+ }+ }+ for key := range backend.sets {+ if strings.HasPrefix(key, "mc:") {+ t.Fatalf("media manifest %s survived a failed chunk purge", key)+ }+ }+ if !backend.sets["kc:manifest:task-1"]["kc:emb:chunk-1"] {+ t.Fatal("failed chunk purge lost the chunk manifest it must retry")+ }+}++func TestPurgeTaskStateWithBackendRejectsEmptyTaskID(t *testing.T) {+ backend := newCleanupFakeStore()+ if err := purgeTaskStateWithBackend(context.Background(), backend, ""); err == nil {+ t.Fatal("purge accepted an empty task id")+ }+ if len(backend.kv) != 0 || len(backend.sets) != 0 {+ t.Fatal("empty task id touched backend state")+ }+}diff --git a/internal/ingestion/pipeline/html_media_real_integration_test.go b/internal/ingestion/pipeline/html_media_real_integration_test.goindex 44dc6f64a8..cfab629bdf 100644--- a/internal/ingestion/pipeline/html_media_real_integration_test.go+++ b/internal/ingestion/pipeline/html_media_real_integration_test.go@@ -204,12 +204,35 @@ func TestHTMLMediaRealServices(t *testing.T) {if got, err := retry.Get(t.Context(), retryRef); err != nil || !bytes.Equal(got, png) {t.Fatalf("retry cache bytes: %v", err)}+ scopeHash := xxhash.Sum64String(taskID)+ digestKey := fmt.Sprintf("mc:quota:%x:digests", scopeHash)+ stateKey := fmt.Sprintf("mc:quota:%x:state", scopeHash)+ manifestKey := fmt.Sprintf("mc:manifest:%x", scopeHash)+ if member, err := kv.GetClient().SIsMember(t.Context(), digestKey, retryRef.SHA256).Result(); err != nil || !member {+ t.Fatalf("quota digest membership = %v, %v; want true", member, err)+ }+ images, err := kv.GetClient().SCard(t.Context(), digestKey).Result()+ if err != nil || images != 1 {+ t.Fatalf("quota images = %d, %v; want 1", images, err)+ }+ bytesUsed, err := kv.GetClient().HGet(t.Context(), stateKey, "bytes").Result()+ if err != nil || bytesUsed != fmt.Sprint(len(png)) {+ t.Fatalf("quota bytes = %q, %v; want %d", bytesUsed, err, len(png))+ }+ if member, err := kv.GetClient().SIsMember(t.Context(), manifestKey, key).Result(); err != nil || !member {+ t.Fatalf("manifest membership = %v, %v; want true", member, err)+ }if err := mediacache.PurgeTask(t.Context(), kv, taskID); err != nil {t.Fatal(err)}if _, err := retry.Get(t.Context(), retryRef); !errors.Is(err, media.ErrMediaNotFound) {t.Fatalf("purged cache read: %v", err)}+ for _, purgedKey := range []string{key, digestKey, stateKey, manifestKey} {+ if exists, err := kv.GetClient().Exists(t.Context(), purgedKey).Result(); err != nil || exists != 0 {+ t.Fatalf("purged key %s exists=%d, err=%v", purgedKey, exists, err)+ }+ }}// loadHTMLMediaIntegrationConfig uses the CI service addresses when set.diff --git a/internal/ingestion/pipeline/pipeline.go b/internal/ingestion/pipeline/pipeline.goindex d227476282..12af493f2e 100644--- a/internal/ingestion/pipeline/pipeline.go+++ b/internal/ingestion/pipeline/pipeline.go@@ -37,6 +37,8 @@ import ("go.uber.org/zap")+var _ mediacache.Backend = (*kvrocks.Client)(nil)+// Pipeline is a compiled ingestion canvas plus task-scoped metadata.type Pipeline struct {taskID string@@ -111,6 +113,10 @@ func WithProgressSink(s ProgressSink) PipelineOption {// It accepts either the inner canvas DSL or the template wrapper whose// top-level `dsl` field carries that canvas.func NewPipelineFromDSL(dsl []byte, taskID string, opts ...PipelineOption) (*Pipeline, error) {+ taskID = strings.TrimSpace(taskID)+ if taskID == "" {+ return nil, errors.New("pipeline: task id is required")+ }// UnwrapCanvasDSL is the single source of truth for stripping the// optional {"dsl": {...}} canvas envelope; it also reports a nil/unparseable// DSL.@@ -270,11 +276,7 @@ func (p *Pipeline) Run(ctx context.Context, inputs map[string]any, overrideParamglobals.SetTaskID(runCtx, p.taskID)mediaCache := p.mediaif mediaCache == nil && p.kv != nil {- scope := p.taskID- if scope == "" {- scope = p.documentID- }- mediaCache = mediacache.New(p.kv, scope)+ mediaCache = mediacache.New(p.kv, p.taskID)}if mediaCache != nil {runCtx = media.WithCache(runCtx, mediaCache)diff --git a/internal/ingestion/pipeline/pipeline_test.go b/internal/ingestion/pipeline/pipeline_test.goindex 0ed1d0912d..eca521745f 100644--- a/internal/ingestion/pipeline/pipeline_test.go+++ b/internal/ingestion/pipeline/pipeline_test.go@@ -270,6 +270,22 @@ func TestPipelineRunAttachesInjectedMediaCacheWithTaskScope(t *testing.T) {}}+func TestNewPipelineFromDSLRejectsEmptyTaskID(t *testing.T) {+ _, err := NewPipelineFromDSL([]byte(`{+ "dsl": {+ "components": {+ "begin": {"obj": {"component_name": "Begin", "params": {}}, "downstream": ["stage"]},+ "stage": {"obj": {"component_name": "Begin", "params": {}}, "upstream": ["begin"]}+ },+ "path": ["begin", "stage"],+ "graph": {"nodes": []}+ }+ }`), " ")+ if err == nil {+ t.Fatal("NewPipelineFromDSL accepted an empty task id")+ }+}+func TestPipelineRun_TaskScopedFactoriesDoNotLeakAcrossConcurrentPipelines(t *testing.T) {origFactory := runtime.DefaultFactory()runtime.SetDefaultFactory(func(_ string, _ map[string]any) (runtime.Component, error) {diff --git a/internal/ingestion/task/debug_test.go b/internal/ingestion/task/debug_test.goindex b250b4e93e..93a6ae630c 100644--- a/internal/ingestion/task/debug_test.go+++ b/internal/ingestion/task/debug_test.go@@ -113,6 +113,40 @@ func TestExecute_DebugViaEntry(t *testing.T) {}}+func TestExecute_DebugViaEntryPurgesRunState(t *testing.T) {+ taskCtx := NewDebugTaskContext("t1", "canvas-1", "doc.pdf", []byte("page one"))+ runCtx, cancel := context.WithCancel(context.Background())++ exec, err := NewPipelineExecutor(taskCtx, "canvas-1", 0, testPipelineDeps())+ if err != nil {+ t.Fatalf("NewPipelineExecutor: %v", err)+ }+ var purgedTaskID string+ exec.purgeTaskState = func(ctx context.Context, taskID string) error {+ if err := ctx.Err(); err != nil {+ t.Errorf("purge context was cancelled with the debug request: %v", err)+ }+ purgedTaskID = taskID+ return nil+ }+ exec.+ WithLoadDSLFunc(func(ctx context.Context, canvasID string) (string, string, error) {+ return "dsl", "canvas-1", nil+ }).+ WithRunPipelineFunc(func(ctx context.Context, dsl string) (map[string]any, string, error) {+ cancel()+ return map[string]any{"chunks": []map[string]any{{"text": "c1"}}}, dsl, nil+ })++ if _, err := exec.Execute(runCtx); err != nil {+ t.Fatalf("Execute: %v", err)+ }+ want := "pipeline_" + taskCtx.Doc.ID+ if purgedTaskID != want {+ t.Fatalf("purged task id = %q, want %q", purgedTaskID, want)+ }+}+// TestInjectDebugPageCap verifies the canvas-debug page cap is delivered// through Run's override_params channel (the existing ParserConfig shape),// NOT through pipeline inputs. The cap must land atdiff --git a/internal/ingestion/task/pipeline_executor.go b/internal/ingestion/task/pipeline_executor.goindex dde2b6fb61..769b879e74 100644--- a/internal/ingestion/task/pipeline_executor.go+++ b/internal/ingestion/task/pipeline_executor.go@@ -82,6 +82,7 @@ type PipelineExecutor struct {deleteChunksFunc DeleteChunksFuncloadDSLFunc func(ctx context.Context, canvasID string) (string, string, error)runPipelineFunc func(ctx context.Context, dsl string) (map[string]any, string, error)+ purgeTaskState func(ctx context.Context, taskID string) errorprogressSink pipelinepkg.ProgressSink}@@ -176,6 +177,15 @@ func NewPipelineExecutor(}svc.loadDSLFunc = svc.loadDSLFromCanvassvc.runPipelineFunc = svc.runPipelineWithDSL+ svc.purgeTaskState = func(ctx context.Context, taskID string) error {+ if deps.KV == nil {+ return nil+ }+ return errors.Join(+ chunkcache.PurgeTask(ctx, deps.KV, taskID),+ mediacache.PurgeTask(ctx, deps.KV, taskID),+ )+ }return svc, nil}@@ -267,8 +277,10 @@ func (s *PipelineExecutor) Execute(ctx context.Context) (*PipelineResult, error)// collectDebugOutput builds a PipelineResult for a debug (dry-run) run.// It surfaces the pipeline's chunks so a debug endpoint can render them, but// performs no DB/index writes — the embedding vectors already computed by the-// pipeline run are left on the chunks. This keeps debug runs side-effect free.+// pipeline run are left on the chunks. Run-scoped cache state is purged before+// the preview is returned.func (s *PipelineExecutor) collectDebugOutput(ctx context.Context, pipelineOutput map[string]any, start time.Time) (*PipelineResult, error) {+ s.purgeRunState(ctx)chunks := indexdoc.NormalizeChunks(pipelineOutput)return &PipelineResult{DocID: s.taskCtx.Doc.ID,@@ -282,6 +294,7 @@ func (s *PipelineExecutor) collectDebugOutput(ctx context.Context, pipelineOutpufunc (s *PipelineExecutor) processOutput(ctx context.Context, pipelineOutput map[string]any, start time.Time) (*PipelineResult, error) {if pipelineOutput == nil {+ s.purgeRunState(ctx)return nil, nil}if err := ctx.Err(); err != nil {@@ -290,6 +303,7 @@ func (s *PipelineExecutor) processOutput(ctx context.Context, pipelineOutput mapchunks := indexdoc.NormalizeChunks(pipelineOutput)if len(chunks) == 0 {+ s.purgeRunState(ctx)return nil, nil}@@ -378,16 +392,8 @@ func (s *PipelineExecutor) processOutput(ctx context.Context, pipelineOutput map// (index write, compiled-product reconcile, Wiki active-MAP state) has// succeeded. Only now drop the run-scoped cache entries this task produced,// so a failure in any of those steps leaves chunk results and parser media- // intact for retry. Cleanup is best-effort; failed media cleanup is logged- // and the entries expire independently.- if s.taskCtx.IngestionTask != nil && s.taskCtx.IngestionTask.ID != "" {- chunkcache.PurgeTask(ctx, chunkcache.Client(), s.taskCtx.IngestionTask.ID)- if s.deps.KV != nil {- if err := mediacache.PurgeTask(ctx, s.deps.KV, s.taskCtx.IngestionTask.ID); err != nil {- common.Warn("failed to purge task media cache", zap.String("task_id", s.taskCtx.IngestionTask.ID), zap.Error(err))- }- }- }+ // intact for retry.+ s.purgeRunState(ctx)return &PipelineResult{DocID: s.taskCtx.Doc.ID,@@ -404,6 +410,30 @@ func (s *PipelineExecutor) processOutput(ctx context.Context, pipelineOutput map}, nil}+func (s *PipelineExecutor) purgeRunState(ctx context.Context) {+ if s == nil || s.taskCtx == nil || s.purgeTaskState == nil {+ return+ }+ taskID := ""+ if s.taskCtx.IngestionTask != nil {+ taskID = s.taskCtx.IngestionTask.ID+ }+ if taskID == "" && s.taskCtx.IsDebug() {+ taskID = "pipeline_" + s.taskCtx.Doc.ID+ }+ if taskID == "" {+ return+ }+ cleanupCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), runStateCleanupTimeout)+ defer cancel()+ if err := s.purgeTaskState(cleanupCtx, taskID); err != nil {+ common.Warn("failed to purge task run state",+ zap.String("task_id", taskID),+ zap.Error(err),+ )+ }+}+func (s *PipelineExecutor) compensateFailedIndexWrite(ctx context.Context, chunks []map[string]any) error {if s.deleteChunksFunc == nil {return nil@@ -1296,6 +1326,8 @@ func (s *PipelineExecutor) buildLogDSL(dsl string, output map[string]any) string// ParserConfig[cpnID][filetype]["pages"] shape (see NormalizeParserConfigPages).const debugPageCapPages = 2+const runStateCleanupTimeout = 5 * time.Second+// injectDebugChunkCap sets the canvas-debug chunk cap on the run inputs when// not already present. The cap is read by the chunker decorator (via// CanvasState.Globals / globals.DebugChunkCap) and limits a debug (dry-run)diff --git a/internal/ingestion/task/pipeline_executor_test.go b/internal/ingestion/task/pipeline_executor_test.goindex 1d93fab08b..1e1e6478ba 100644--- a/internal/ingestion/task/pipeline_executor_test.go+++ b/internal/ingestion/task/pipeline_executor_test.go@@ -27,6 +27,37 @@ import (func strPtr(s string) *string { return &s }+func TestProcessOutputPurgesRunStateOnSuccessfulNoOutput(t *testing.T) {+ for _, tc := range []struct {+ name string+ output map[string]any+ }{+ {name: "nil output", output: nil},+ {name: "empty output", output: map[string]any{}},+ } {+ t.Run(tc.name, func(t *testing.T) {+ taskCtx := &TaskContext{+ Doc: entity.Document{ID: "doc-1"},+ IngestionTask: &entity.IngestionTask{ID: "task-1"},+ }+ var purgedTaskID string+ exec := &PipelineExecutor{+ taskCtx: taskCtx,+ purgeTaskState: func(ctx context.Context, taskID string) error {+ purgedTaskID = taskID+ return nil+ },+ }+ if _, err := exec.processOutput(context.Background(), tc.output, time.Now()); err != nil {+ t.Fatalf("processOutput: %v", err)+ }+ if purgedTaskID != "task-1" {+ t.Fatalf("purged task id = %q, want task-1", purgedTaskID)+ }+ })+ }+}+// TestMarkCompiledProductsHidden verifies the pipeline caller stages ONLY the// wiki variant's per-document rows as available_int=0 (they are intermediate// state for the consumer's merged pages). Tree / structure (page_index) /diff --git a/internal/parser/media/budget.go b/internal/parser/media/budget.gonew file mode 100644index 0000000000..042c6eb57d--- /dev/null+++ b/internal/parser/media/budget.go@@ -0,0 +1,192 @@+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.++package media++import "fmt"++// Size is a declared or actual media payload size in bytes. UnknownSize lets a+// caller reserve an occurrence before it knows how much data it will read.+type Size int64++const (+ // UnknownSize defers the byte check until Reservation.Commit.+ UnknownSize Size = -1+)++// Admission explains why a source or occurrence was accepted or rejected.+type Admission uint8++const (+ // Admitted means the caller owns a reservation and may read the payload.+ Admitted Admission = iota+ // RejectedByCount means the run's distinct-image limit is exhausted.+ RejectedByCount+ // RejectedByBytes means the per-image or total-byte limit rejected it.+ RejectedByBytes+ // RejectedInvalid means the supplied size is invalid.+ RejectedInvalid+)++// Budget enforces one parser run's image count and byte limits. Reserve checks+// declared bytes before reserving a stable source; ReserveOccurrence preserves+// sequential extraction semantics by consuming the count slot first.+type Budget struct {+ limits Limits++ itemsUsed int+ totalBytes int64+ oversizedImages int+ totalOverflows int+ countExhausted bool+}++type budgetItem struct {+ included bool+ charged bool+}++// Reservation is one admitted item. Commit applies the actual payload size+// after the caller reads or decodes it.+type Reservation struct {+ budget *Budget+ item *budgetItem+}++// NewBudget returns a run budget using the current configured limits.+func NewBudget() *Budget {+ return NewBudgetWithLimits(LoadLimits())+}++// NewBudgetWithLimits returns a run budget with an explicit limits snapshot.+func NewBudgetWithLimits(limits Limits) *Budget {+ return &Budget{limits: limits}+}++// Reserve admits a stable source before its bytes are read. A declared size+// rejected by a byte limit does not consume a count slot. Callers memoize the+// source so a successful reservation is reused without another Budget lookup.+func (b *Budget) Reserve(declared Size) (*Reservation, Admission) {+ if declared != UnknownSize && declared < 0 {+ return nil, RejectedInvalid+ }+ if declared != UnknownSize && !b.canAdmitBytes(declared) {+ b.noteByteRejection(declared)+ return nil, RejectedByBytes+ }+ if b.countExhausted || b.itemsUsed >= b.limits.MaxImagesPerRun {+ b.countExhausted = true+ return nil, RejectedByCount+ }+ item := &budgetItem{}+ b.itemsUsed+++ return &Reservation{budget: b, item: item}, Admitted+}++// ReserveOccurrence admits a source-less payload. The occurrence consumes one+// item before its actual size is known, matching sequential Office and+// Markdown extraction where byte-limit omissions must not stop later text.+func (b *Budget) ReserveOccurrence(declared Size) (*Reservation, Admission) {+ if declared != UnknownSize && declared < 0 {+ return nil, RejectedInvalid+ }+ if b.countExhausted || b.itemsUsed >= b.limits.MaxImagesPerRun {+ b.countExhausted = true+ return nil, RejectedByCount+ }+ item := &budgetItem{}+ b.itemsUsed+++ if declared != UnknownSize && !b.admitBytes(item, declared) {+ return nil, RejectedByBytes+ }+ return &Reservation{budget: b, item: item}, Admitted+}++// Commit applies the actual payload size to a reservation. It is idempotent+// for an included item and keeps byte-rejected reservations rejected.+func (r *Reservation) Commit(actual Size) bool {+ if r == nil || r.budget == nil || r.item == nil {+ return false+ }+ if r.item.included {+ return true+ }+ if r.item.charged || actual < 0 {+ return false+ }+ if !r.budget.admitBytes(r.item, actual) {+ return false+ }+ r.item.included = true+ r.budget.totalBytes += int64(actual)+ return true+}++// Count reports how many items this run has reserved.+func (b *Budget) Count() int {+ return b.itemsUsed+}++func (b *Budget) canAdmitBytes(size Size) bool {+ if int64(size) > b.limits.ImageMaxBytes {+ return false+ }+ return int64(size) <= b.limits.MaxBytesPerRun-b.totalBytes+}++func (b *Budget) admitBytes(item *budgetItem, size Size) bool {+ if item.charged {+ return false+ }+ if b.canAdmitBytes(size) {+ return true+ }+ item.charged = true+ b.noteByteRejection(size)+ return false+}++func (b *Budget) noteByteRejection(size Size) {+ if int64(size) > b.limits.ImageMaxBytes {+ b.oversizedImages+++ return+ }+ b.totalOverflows+++}++// Warnings reports the budget omissions accumulated by this run.+func (b *Budget) Warnings() []string {+ var warnings []string+ if b.oversizedImages > 0 {+ warnings = append(warnings, fmt.Sprintf(+ "omitted %d embedded image payload(s) larger than the %d-byte per-image limit",+ b.oversizedImages,+ b.limits.ImageMaxBytes,+ ))+ }+ if b.totalOverflows > 0 {+ warnings = append(warnings, fmt.Sprintf(+ "omitted %d embedded image payload(s) after reaching the %d-byte run image budget",+ b.totalOverflows,+ b.limits.MaxBytesPerRun,+ ))+ }+ if b.countExhausted {+ warnings = append(warnings, fmt.Sprintf(+ "stopped extracting embedded images after the %d-item run limit",+ b.limits.MaxImagesPerRun,+ ))+ }+ return warnings+}diff --git a/internal/parser/media/budget_test.go b/internal/parser/media/budget_test.gonew file mode 100644index 0000000000..e3f482ac34--- /dev/null+++ b/internal/parser/media/budget_test.go@@ -0,0 +1,200 @@+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.++package media++import (+ "reflect"+ "strings"+ "testing"+)++func testBudget(maxItems int, maxImageBytes, maxTotalBytes int64) *Budget {+ return NewBudgetWithLimits(Limits{+ ImageMaxBytes: maxImageBytes,+ MaxImagesPerRun: maxItems,+ MaxBytesPerRun: maxTotalBytes,+ })+}++func reserveAndCommit(t *testing.T, budget *Budget, size Size) bool {+ t.Helper()+ reservation, admission := budget.Reserve(size)+ if admission != Admitted {+ return false+ }+ return reservation.Commit(size)+}++func TestNewBudgetUsesRunCaps(t *testing.T) {+ t.Setenv("RAGFLOW_MEDIA_IMAGE_MAX_BYTES", "2048")+ t.Setenv("RAGFLOW_MEDIA_MAX_IMAGES_PER_RUN", "7")+ t.Setenv("RAGFLOW_MEDIA_MAX_BYTES_PER_RUN", "1234")+ budget := NewBudget()+ want := Limits{ImageMaxBytes: 2048, MaxImagesPerRun: 7, MaxBytesPerRun: 1234}+ if budget.limits != want {+ t.Fatalf("budget limits = %+v, want %+v", budget.limits, want)+ }+}++func TestBudgetItemLimitExhaustion(t *testing.T) {+ budget := testBudget(2, 16, 16)+ for attempt := 0; attempt < 2; attempt++ {+ reservation, admission := budget.Reserve(UnknownSize)+ if admission != Admitted || !reservation.Commit(1) {+ t.Fatalf("item #%d was not admitted", attempt+1)+ }+ }+ if _, admission := budget.Reserve(UnknownSize); admission != RejectedByCount {+ t.Fatalf("third item admission = %v, want count rejection", admission)+ }+ if !strings.Contains(strings.Join(budget.Warnings(), "\n"), "2-item run limit") {+ t.Fatalf("warnings = %v, want the item-count warning", budget.Warnings())+ }+}++func TestBudgetDeclaredByteRejectionsDoNotConsumeItemSlots(t *testing.T) {+ budget := testBudget(1, 8, 5)+ if _, admission := budget.Reserve(9); admission != RejectedByBytes {+ t.Fatalf("per-image rejection = %v, want byte rejection", admission)+ }+ if _, admission := budget.Reserve(6); admission != RejectedByBytes {+ t.Fatalf("total-byte rejection = %v, want byte rejection", admission)+ }+ if !reserveAndCommit(t, budget, 1) {+ t.Fatal("byte-rejected items exhausted the item budget")+ }+ if budget.Count() != 1 || budget.totalBytes != 1 {+ t.Fatalf("items=%d bytes=%d, want 1 item and 1 byte", budget.Count(), budget.totalBytes)+ }+ if budget.oversizedImages != 1 || budget.totalOverflows != 1 {+ t.Fatalf("oversized=%d overflows=%d, want 1 and 1", budget.oversizedImages, budget.totalOverflows)+ }+}++func TestBudgetCommitRejectionsAreStable(t *testing.T) {+ budget := testBudget(4, 4, 8)+ reservation, admission := budget.Reserve(UnknownSize)+ if admission != Admitted {+ t.Fatalf("Reserve = %v, want admitted", admission)+ }+ for attempt := 0; attempt < 3; attempt++ {+ if reservation.Commit(5) {+ t.Fatal("Commit admitted an oversized payload")+ }+ }+ if budget.oversizedImages != 1 || budget.totalBytes != 0 {+ t.Fatalf("oversized=%d bytes=%d, want one oversized skip and no bytes", budget.oversizedImages, budget.totalBytes)+ }+}++func TestBudgetTotalByteLimitDoesNotRecount(t *testing.T) {+ budget := testBudget(4, 8, 5)+ if !reserveAndCommit(t, budget, 3) {+ t.Fatal("first payload should fit")+ }+ reservation, admission := budget.Reserve(UnknownSize)+ if admission != Admitted {+ t.Fatalf("second item admission = %v, want admitted", admission)+ }+ if reservation.Commit(3) {+ t.Fatal("Commit admitted a payload past the total byte limit")+ }+ if reservation.Commit(3) {+ t.Fatal("Commit recounted the rejected item")+ }+ if budget.totalOverflows != 1 || budget.totalBytes != 3 {+ t.Fatalf("overflows=%d bytes=%d, want one overflow and 3 bytes", budget.totalOverflows, budget.totalBytes)+ }+}++func TestBudgetOccurrenceConsumesCountBeforeByteAdmission(t *testing.T) {+ budget := testBudget(1, 8, 5)+ if _, admission := budget.ReserveOccurrence(6); admission != RejectedByBytes {+ t.Fatalf("byte rejection = %v, want rejected by bytes", admission)+ }+ if budget.Count() != 1 {+ t.Fatalf("count = %d, want the rejected occurrence to consume one slot", budget.Count())+ }+ if _, admission := budget.ReserveOccurrence(UnknownSize); admission != RejectedByCount {+ t.Fatalf("next occurrence = %v, want rejected by count", admission)+ }+}++func TestBudgetOccurrenceKeepsWalkingAfterByteRejection(t *testing.T) {+ budget := testBudget(3, 4, 5)+ reservation, admission := budget.ReserveOccurrence(UnknownSize)+ if admission != Admitted || !reservation.Commit(3) {+ t.Fatal("first occurrence should fit")+ }+ reservation, admission = budget.ReserveOccurrence(UnknownSize)+ if admission != Admitted {+ t.Fatalf("second occurrence admission = %v, want admitted", admission)+ }+ if reservation.Commit(3) {+ t.Fatal("second occurrence exceeded the byte limit")+ }+ reservation, admission = budget.ReserveOccurrence(UnknownSize)+ if admission != Admitted {+ t.Fatalf("third occurrence admission = %v, want admitted", admission)+ }+ if !reservation.Commit(2) {+ t.Fatal("third occurrence should fit after the byte rejection")+ }+ if budget.Count() != 3 || budget.totalBytes != 5 {+ t.Fatalf("count=%d bytes=%d, want 3 and 5", budget.Count(), budget.totalBytes)+ }+}++func TestBudgetExactByteLimits(t *testing.T) {+ budget := testBudget(2, 4, 4)+ if !reserveAndCommit(t, budget, 4) {+ t.Fatal("exact-limit image should be admitted")+ }+ reservation, admission := budget.ReserveOccurrence(UnknownSize)+ if admission != Admitted || reservation.Commit(1) {+ t.Fatal("image after exact run limit should be omitted")+ }+}++func TestBudgetInvalidSizes(t *testing.T) {+ budget := testBudget(2, 8, 8)+ if _, admission := budget.Reserve(-2); admission != RejectedInvalid {+ t.Fatalf("invalid declared size = %v, want invalid rejection", admission)+ }+ reservation, admission := budget.Reserve(UnknownSize)+ if admission != Admitted {+ t.Fatalf("valid reservation = %v, want admitted", admission)+ }+ if reservation.Commit(-2) {+ t.Fatal("Commit admitted an invalid actual size")+ }+}++func TestBudgetWarningsAreStable(t *testing.T) {+ budget := testBudget(3, 4, 5)+ _, _ = budget.Reserve(5)+ _, _ = budget.ReserveOccurrence(6)+ _, _ = budget.Reserve(UnknownSize)+ _, _ = budget.Reserve(UnknownSize)+ _, _ = budget.Reserve(UnknownSize)+ _, _ = budget.Reserve(UnknownSize)+ want := []string{+ "omitted 2 embedded image payload(s) larger than the 4-byte per-image limit",+ "stopped extracting embedded images after the 3-item run limit",+ }+ if got := budget.Warnings(); !reflect.DeepEqual(got, want) {+ t.Fatalf("warnings = %q, want %q", got, want)+ }+}diff --git a/internal/parser/media/cache.go b/internal/parser/media/cache.goindex d310fb2996..022f112ef9 100644--- a/internal/parser/media/cache.go+++ b/internal/parser/media/cache.go@@ -29,6 +29,11 @@ import (// be returned to the task rather than treated as a missing image.var ErrMediaNotFound = errors.New("media: reference not found")+// ErrMediaLimitExceeded reports that a run-scoped media limit rejected a+// write. Parser paths may skip the image and warn; every other cache error is+// fatal to the task.+var ErrMediaLimitExceeded = errors.New("media: run media limit exceeded")+// Cache stores decoded image bytes for materialization by later ingestion// components. Implementations own key scoping, lifecycle bookkeeping and// validation of the payload size and digest before returning bytes from Get.@@ -54,8 +59,8 @@ func CacheFromContext(ctx context.Context) Cache {// ValidatePayload performs the shared cache integrity and size checks.func ValidatePayload(data []byte, ref entity.MediaRef) ([]byte, error) {- if len(data) > HTMLImageMaxBytes() {- return nil, fmt.Errorf("media: image %s exceeds the %d-byte limit", shortMediaDigest(ref.SHA256), HTMLImageMaxBytes())+ if len(data) > ImageMaxBytes() {+ return nil, fmt.Errorf("media: image %s exceeds the %d-byte limit", shortMediaDigest(ref.SHA256), ImageMaxBytes())}sum := sha256.Sum256(data)if got := hex.EncodeToString(sum[:]); got != ref.SHA256 {diff --git a/internal/parser/media/html.go b/internal/parser/media/datauri.gosimilarity index 62%rename from internal/parser/media/html.gorename to internal/parser/media/datauri.goindex 44650b5cd1..a0431688a9 100644--- a/internal/parser/media/html.go+++ b/internal/parser/media/datauri.go@@ -16,41 +16,30 @@ package mediaimport ("crypto/sha256"- "os"- "strconv""strings")-// DecodeHTMLImageSource validates and decodes one inline HTML image. The HTML+// DecodeDataImageSource validates and decodes one inline image data URI. The// parser calls this once, then writes the bytes to the run media cache; later// ingestion components materialize only the emitted MediaRef.-func DecodeHTMLImageSource(src string) (string, []byte, bool) {- prefix, payload, ok := splitHTMLImageSource(src)+func DecodeDataImageSource(src string) (string, []byte, bool) {+ prefix, payload, ok := splitDataImageSource(src)if !ok {return "", nil, false}- data, ok := decodeBase64ImagePayload(payload, HTMLImageMaxBytes())+ data, ok := decodeBase64ImagePayload(payload, ImageMaxBytes())if !ok {return "", nil, false}return DataURIMediaType(prefix), data, true}-// HTMLImageMaxBytes is the per-image limit for HTML media.-func HTMLImageMaxBytes() int {- const defaultLimit = 32 << 20- if n, err := strconv.Atoi(os.Getenv("RAGFLOW_HTML_IMAGE_MAX_BYTES")); err == nil && n > 0 && n <= int(^uint(0)>>1)/4*3 {- return n- }- return defaultLimit-}--// HTMLImageDecodedSize validates an image data URI and returns the decoded+// DataImageDecodedSize validates an image data URI and returns the decoded// payload size without allocating the decoded bytes. Malformed payloads are-// rejected; a syntactically valid payload above HTMLImageMaxBytes is still+// rejected; a syntactically valid payload above ImageMaxBytes is still// reported by its size so callers can apply their own limit.-func HTMLImageDecodedSize(src string) (int, bool) {- _, payload, ok := splitHTMLImageSource(src)+func DataImageDecodedSize(src string) (int, bool) {+ _, payload, ok := splitDataImageSource(src)if !ok {return 0, false}@@ -62,11 +51,11 @@ func HTMLImageDecodedSize(src string) (int, bool) {return padded/4*3 - (padded - encoded) - padding, true}-// splitHTMLImageSource validates the data-URI header and returns the normalized+// splitDataImageSource validates the data-URI header and returns the normalized// prefix and base64 payload.-func splitHTMLImageSource(src string) (prefix, payload string, ok bool) {+func splitDataImageSource(src string) (prefix, payload string, ok bool) {src = strings.TrimSpace(src)- if !IsHTMLDataImageSource(src) {+ if !IsDataImageSource(src) {return "", "", false}separator := strings.IndexByte(src, ',')@@ -76,16 +65,16 @@ func splitHTMLImageSource(src string) (prefix, payload string, ok bool) {return strings.ToLower(src[:separator+1]), src[separator+1:], true}-// NormalizeHTMLImageSource validates an inline image while avoiding a decoded-// byte buffer. It remains the path for HTML layouts that intentionally keep-// the normalized data URI inline rather than storing it in the run cache.-func NormalizeHTMLImageSource(src string) (string, int64, [sha256.Size]byte, bool) {+// NormalizeDataImageSource validates an inline image while avoiding a decoded+// byte buffer. It remains the path for layouts that intentionally keep the+// normalized data URI inline rather than storing it in the run cache.+func NormalizeDataImageSource(src string) (string, int64, [sha256.Size]byte, bool) {var sum [sha256.Size]byte- prefix, payload, ok := splitHTMLImageSource(src)+ prefix, payload, ok := splitDataImageSource(src)if !ok {return "", 0, sum, false}- normalized, size, sum, ok := NormalizeBase64ImagePayload(payload, MaxImagePayloadBytes)+ normalized, size, sum, ok := NormalizeBase64ImagePayload(payload, ImageMaxBytes())if !ok {return "", 0, sum, false}@@ -103,8 +92,8 @@ func DataURIMediaType(uri string) string {return strings.ToLower(uri[colon+1 : semi])}-// IsHTMLDataImageSource reports whether src is an image data URI.-func IsHTMLDataImageSource(src string) bool {+// IsDataImageSource reports whether src is an image data URI.+func IsDataImageSource(src string) bool {const dataPrefix = "data:image/"src = strings.TrimSpace(src)return len(src) >= len(dataPrefix) && strings.EqualFold(src[:len(dataPrefix)], dataPrefix)diff --git a/internal/parser/media/html_test.go b/internal/parser/media/datauri_test.gosimilarity index 64%rename from internal/parser/media/html_test.gorename to internal/parser/media/datauri_test.goindex 902def6482..363e6ca880 100644--- a/internal/parser/media/html_test.go+++ b/internal/parser/media/datauri_test.go@@ -20,10 +20,10 @@ import ("testing")-func TestDecodeHTMLImageSourceCaseAndPadding(t *testing.T) {- mime, data, ok := DecodeHTMLImageSource("DATA:IMAGE/PNG;BASE64,aGVsbG8")+func TestDecodeDataImageSourceCaseAndPadding(t *testing.T) {+ mime, data, ok := DecodeDataImageSource("DATA:IMAGE/PNG;BASE64,aGVsbG8")if !ok {- t.Fatal("DecodeHTMLImageSource rejected a valid case-insensitive data URI")+ t.Fatal("DecodeDataImageSource rejected a valid case-insensitive data URI")}if mime != "image/png" {t.Errorf("mime = %q, want image/png", mime)@@ -33,7 +33,7 @@ func TestDecodeHTMLImageSourceCaseAndPadding(t *testing.T) {}}-func TestDecodeHTMLImageSourceRejectsInvalidAndOversizedHeaders(t *testing.T) {+func TestDecodeDataImageSourceRejectsInvalidAndOversizedHeaders(t *testing.T) {for _, src := range []string{"data:image/png;base64,","data:image/png;base64,YQ=",@@ -41,21 +41,21 @@ func TestDecodeHTMLImageSourceRejectsInvalidAndOversizedHeaders(t *testing.T) {"data:image/png;base64,G0==JbjQ","data:image/png;" + strings.Repeat("x", 200) + ",aGVsbG8=",} {- if _, _, ok := DecodeHTMLImageSource(src); ok {- t.Errorf("DecodeHTMLImageSource(%q) succeeded", src)+ if _, _, ok := DecodeDataImageSource(src); ok {+ t.Errorf("DecodeDataImageSource(%q) succeeded", src)}}}-func TestNormalizeHTMLImageSourceMatchesDecode(t *testing.T) {+func TestNormalizeDataImageSourceMatchesDecode(t *testing.T) {src := "data:image/png;base64,aGVs\nbG8"- normalized, size, normalizedSum, ok := NormalizeHTMLImageSource(src)+ normalized, size, normalizedSum, ok := NormalizeDataImageSource(src)if !ok {- t.Fatal("NormalizeHTMLImageSource rejected a valid payload")+ t.Fatal("NormalizeDataImageSource rejected a valid payload")}- mime, data, ok := DecodeHTMLImageSource(src)+ mime, data, ok := DecodeDataImageSource(src)if !ok {- t.Fatal("DecodeHTMLImageSource rejected a valid payload")+ t.Fatal("DecodeDataImageSource rejected a valid payload")}if normalized != "data:image/png;base64,aGVsbG8=" {t.Fatalf("normalized = %q", normalized)@@ -67,10 +67,10 @@ func TestNormalizeHTMLImageSourceMatchesDecode(t *testing.T) {}}-// TestHTMLImageDecodedSizeMatchesDecode checks that the allocation-free-// preflight accepts exactly what DecodeHTMLImageSource accepts and reports the+// TestDataImageDecodedSizeMatchesDecode checks that the allocation-free+// preflight accepts exactly what DecodeDataImageSource accepts and reports the// same decoded size, so budget admission never diverges from the actual write.-func TestHTMLImageDecodedSizeMatchesDecode(t *testing.T) {+func TestDataImageDecodedSizeMatchesDecode(t *testing.T) {cases := []struct {src stringwant int // -1 means the source is malformed@@ -90,10 +90,10 @@ func TestHTMLImageDecodedSizeMatchesDecode(t *testing.T) {{"data:image/png;" + strings.Repeat("x", 200) + ",aGVsbG8=", -1},}for _, tc := range cases {- size, ok := HTMLImageDecodedSize(tc.src)- _, data, decodeOK := DecodeHTMLImageSource(tc.src)+ size, ok := DataImageDecodedSize(tc.src)+ _, data, decodeOK := DecodeDataImageSource(tc.src)if ok != (tc.want >= 0) {- t.Errorf("%q: HTMLImageDecodedSize ok = %v, want %v", tc.src, ok, tc.want >= 0)+ t.Errorf("%q: DataImageDecodedSize ok = %v, want %v", tc.src, ok, tc.want >= 0)continue}if ok != decodeOK {@@ -106,27 +106,16 @@ func TestHTMLImageDecodedSizeMatchesDecode(t *testing.T) {}}-// TestHTMLImageDecodedSizeIgnoresPerImageLimit checks that the preflight+// TestDataImageDecodedSizeIgnoresPerImageLimit checks that the preflight// reports the size of a syntactically valid payload even when it exceeds the// configured per-image limit, leaving the limit decision to the caller.-func TestHTMLImageDecodedSizeIgnoresPerImageLimit(t *testing.T) {- t.Setenv("RAGFLOW_HTML_IMAGE_MAX_BYTES", "2")- size, ok := HTMLImageDecodedSize("data:image/png;base64,aGVsbG8=")+func TestDataImageDecodedSizeIgnoresPerImageLimit(t *testing.T) {+ t.Setenv("RAGFLOW_MEDIA_IMAGE_MAX_BYTES", "2")+ size, ok := DataImageDecodedSize("data:image/png;base64,aGVsbG8=")if !ok || size != 5 {t.Fatalf("size, ok = %d, %v, want 5, true", size, ok)}- if size <= HTMLImageMaxBytes() {- t.Fatalf("size %d does not exceed the configured limit %d", size, HTMLImageMaxBytes())- }-}--func TestHTMLImageLimitDefaultsAndConfiguration(t *testing.T) {- t.Setenv("RAGFLOW_HTML_IMAGE_MAX_BYTES", "")- if got := HTMLImageMaxBytes(); got != 32<<20 {- t.Fatalf("default = %d", got)- }- t.Setenv("RAGFLOW_HTML_IMAGE_MAX_BYTES", "67108864")- if got := HTMLImageMaxBytes(); got != 64<<20 {- t.Fatalf("configured = %d", got)+ if size <= ImageMaxBytes() {+ t.Fatalf("size %d does not exceed the configured limit %d", size, ImageMaxBytes())}}diff --git a/internal/parser/media/limits.go b/internal/parser/media/limits.gonew file mode 100644index 0000000000..2b5eb28695--- /dev/null+++ b/internal/parser/media/limits.go@@ -0,0 +1,75 @@+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.++package media++import (+ "os"+ "strconv"++ "ragflow/internal/common"+)++const (+ defaultImageMaxBytes = 32 << 20+ defaultMediaMaxImagesPerRun = 1024+ defaultMediaMaxBytesPerRun = int64(1 << 30)+)++// Limits is the run media quota snapshot shared by parser budgets.+type Limits struct {+ ImageMaxBytes int64+ MaxImagesPerRun int+ MaxBytesPerRun int64+}++// LoadLimits reads the configured media limits once for a parser run.+func LoadLimits() Limits {+ return Limits{+ ImageMaxBytes: int64(ImageMaxBytes()),+ MaxImagesPerRun: MediaMaxImagesPerRun(),+ MaxBytesPerRun: MediaMaxBytesPerRun(),+ }+}++// ImageMaxBytes is the single per-image limit shared by every media path:+// data URI decoding, ZIP entry extraction, picture parsing, vision images and+// the run media cache. Set RAGFLOW_MEDIA_IMAGE_MAX_BYTES to a positive byte+// count to change it.+func ImageMaxBytes() int {+ if n, err := strconv.Atoi(os.Getenv(common.EnvMediaImageMaxBytes)); err == nil && n > 0 && n <= int(^uint(0)>>1)/4*3 {+ return n+ }+ return defaultImageMaxBytes+}++// MediaMaxImagesPerRun bounds the number of distinct images one run may keep+// in the media cache. Set RAGFLOW_MEDIA_MAX_IMAGES_PER_RUN to a positive count+// to change it.+func MediaMaxImagesPerRun() int {+ if n, err := strconv.Atoi(os.Getenv(common.EnvMediaMaxImagesPerRun)); err == nil && n > 0 {+ return n+ }+ return defaultMediaMaxImagesPerRun+}++// MediaMaxBytesPerRun bounds the total decoded image bytes one run may keep in+// the media cache. Set RAGFLOW_MEDIA_MAX_BYTES_PER_RUN to a positive byte count+// to change it.+func MediaMaxBytesPerRun() int64 {+ if n, err := strconv.ParseInt(os.Getenv(common.EnvMediaMaxBytesPerRun), 10, 64); err == nil && n > 0 {+ return n+ }+ return defaultMediaMaxBytesPerRun+}diff --git a/internal/parser/media/limits_test.go b/internal/parser/media/limits_test.gonew file mode 100644index 0000000000..9ef172ab65--- /dev/null+++ b/internal/parser/media/limits_test.go@@ -0,0 +1,65 @@+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.++package media++import "testing"++func TestImageMaxBytesDefaultsAndConfiguration(t *testing.T) {+ t.Setenv("RAGFLOW_MEDIA_IMAGE_MAX_BYTES", "")+ if got := ImageMaxBytes(); got != 32<<20 {+ t.Fatalf("default = %d, want %d", got, 32<<20)+ }+ t.Setenv("RAGFLOW_MEDIA_IMAGE_MAX_BYTES", "67108864")+ if got := ImageMaxBytes(); got != 64<<20 {+ t.Fatalf("configured = %d, want %d", got, 64<<20)+ }+ for _, invalid := range []string{"0", "-1", "not-a-number"} {+ t.Setenv("RAGFLOW_MEDIA_IMAGE_MAX_BYTES", invalid)+ if got := ImageMaxBytes(); got != 32<<20 {+ t.Fatalf("invalid %q = %d, want the default", invalid, got)+ }+ }+}++func TestMediaRunLimitsDefaultsAndConfiguration(t *testing.T) {+ t.Setenv("RAGFLOW_MEDIA_MAX_IMAGES_PER_RUN", "")+ t.Setenv("RAGFLOW_MEDIA_MAX_BYTES_PER_RUN", "")+ if got := MediaMaxImagesPerRun(); got != 1024 {+ t.Fatalf("default images = %d, want 1024", got)+ }+ if got := MediaMaxBytesPerRun(); got != 1<<30 {+ t.Fatalf("default bytes = %d, want %d", got, 1<<30)+ }++ t.Setenv("RAGFLOW_MEDIA_MAX_IMAGES_PER_RUN", "7")+ t.Setenv("RAGFLOW_MEDIA_MAX_BYTES_PER_RUN", "2048")+ if got := MediaMaxImagesPerRun(); got != 7 {+ t.Fatalf("configured images = %d, want 7", got)+ }+ if got := MediaMaxBytesPerRun(); got != 2048 {+ t.Fatalf("configured bytes = %d, want 2048", got)+ }++ for _, invalid := range []string{"0", "-1", "not-a-number"} {+ t.Setenv("RAGFLOW_MEDIA_MAX_IMAGES_PER_RUN", invalid)+ t.Setenv("RAGFLOW_MEDIA_MAX_BYTES_PER_RUN", invalid)+ if got := MediaMaxImagesPerRun(); got != 1024 {+ t.Fatalf("invalid images %q = %d, want 1024", invalid, got)+ }+ if got := MediaMaxBytesPerRun(); got != 1<<30 {+ t.Fatalf("invalid bytes %q = %d, want %d", invalid, got, int64(1<<30))+ }+ }+}diff --git a/internal/parser/media/payload.go b/internal/parser/media/payload.goindex c0aec5ef90..e626585523 100644--- a/internal/parser/media/payload.go+++ b/internal/parser/media/payload.go@@ -23,12 +23,8 @@ import ("strings")-const (- // MaxImagePayloadBytes bounds compressed image data accepted by media paths.- MaxImagePayloadBytes = 32 << 20- // maxStreamBuffer caps the copy buffer used while validating a payload.- maxStreamBuffer = 32 << 10-)+// maxStreamBuffer caps the copy buffer used while validating a payload.+const maxStreamBuffer = 32 << 10// NormalizeBase64ImagePayload validates an external image payload within its// decoded byte limit and returns standard base64 with padding, no CR/LF, thediff --git a/internal/parser/parser/doc_parser.go b/internal/parser/parser/doc_parser.goindex 1daeb0394b..766ca90694 100644--- a/internal/parser/parser/doc_parser.go+++ b/internal/parser/parser/doc_parser.go@@ -22,6 +22,7 @@ import ("strings"deepdocoffice "ragflow/internal/deepdoc/parser/office"+ "ragflow/internal/parser/media")// docExtract is the native office_oxide extraction seam for the DOC family.@@ -60,12 +61,12 @@ func (p *DOCParser) ParseWithResult(ctx context.Context, filename string, data [// usable sections, so a document whose IR is empty/unusable still keeps its// content (PlainText / ToMarkdown) rather than silently producing 0 chunks.if strings.TrimSpace(irJSON) != "" {- budget := newEmbeddedMediaBudget()+ budget := media.NewBudget()if sections := buildDOCXJSONSections(irJSON, budget); len(sections) > 0 {return ParseResult{File: map[string]any{"name": filename, "format": "doc"},JSON: MapsToItems(sections),- Warnings: budget.warnings(),+ Warnings: budget.Warnings(),}}}diff --git a/internal/parser/parser/docx_caption_test.go b/internal/parser/parser/docx_caption_test.goindex 362dc1c2c5..84ec46f4bd 100644--- a/internal/parser/parser/docx_caption_test.go+++ b/internal/parser/parser/docx_caption_test.go@@ -39,7 +39,7 @@ func TestBuildDOCXJSONSections_TableCaption_ByHeading(t *testing.T) {`{"type":"heading","level":1,"content":[{"type":"text","text":"设备台账"}]}`,docxCaptionTableBody,)- sections := buildDOCXJSONSections(ir, newEmbeddedMediaBudget())+ sections := buildDOCXJSONSections(ir, newMediaBudgetForTest())if got := docxCaptionFindTableCaption(t, sections); got != "Table Location: 设备台账" {t.Errorf("caption = %q, want %q", got, "Table Location: 设备台账")}@@ -55,7 +55,7 @@ func TestBuildDOCXJSONSections_TableCaption_NestedHeading(t *testing.T) {`{"type":"heading","level":3,"content":[{"type":"text","text":"H3"}]}`,docxCaptionTableBody,)- sections := buildDOCXJSONSections(ir, newEmbeddedMediaBudget())+ sections := buildDOCXJSONSections(ir, newMediaBudgetForTest())if got := docxCaptionFindTableCaption(t, sections); got != "Table Location: H1 > H2 > H3" {t.Errorf("caption = %q, want %q", got, "Table Location: H1 > H2 > H3")}@@ -67,7 +67,7 @@ func TestBuildDOCXJSONSections_TableCaption_HeadingAfterTable(t *testing.T) {docxCaptionTableBody,`{"type":"heading","level":1,"content":[{"type":"text","text":"后续标题"}]}`,)- sections := buildDOCXJSONSections(ir, newEmbeddedMediaBudget())+ sections := buildDOCXJSONSections(ir, newMediaBudgetForTest())if got := docxCaptionFindTableCaption(t, sections); got != "" {t.Errorf("table should have no caption when heading is after it, got %q", got)}@@ -83,7 +83,7 @@ func TestBuildDOCXJSONSections_TableCaption_SiblingHeadingSupersedes(t *testing.`{"type":"heading","level":2,"content":[{"type":"text","text":"H2b"}]}`,docxCaptionTableBody,)- sections := buildDOCXJSONSections(ir, newEmbeddedMediaBudget())+ sections := buildDOCXJSONSections(ir, newMediaBudgetForTest())if got := docxCaptionFindTableCaption(t, sections); got != "Table Location: H1 > H2b" {t.Errorf("caption = %q, want %q", got, "Table Location: H1 > H2b")}@@ -99,7 +99,7 @@ func TestBuildDOCXJSONSections_TableCaption_DeeperHeadingAfterSibling(t *testing`{"type":"heading","level":3,"content":[{"type":"text","text":"H3b"}]}`,docxCaptionTableBody,)- sections := buildDOCXJSONSections(ir, newEmbeddedMediaBudget())+ sections := buildDOCXJSONSections(ir, newMediaBudgetForTest())if got := docxCaptionFindTableCaption(t, sections); got != "Table Location: H2b > H3b" {t.Errorf("caption = %q, want %q", got, "Table Location: H2b > H3b")}@@ -113,7 +113,7 @@ func TestBuildDOCXJSONSections_TableCaption_EmptyHeadingIgnored(t *testing.T) {`{"type":"heading","level":2,"content":[{"type":"text","text":" "}]}`,docxCaptionTableBody,)- sections := buildDOCXJSONSections(ir, newEmbeddedMediaBudget())+ sections := buildDOCXJSONSections(ir, newMediaBudgetForTest())if got := docxCaptionFindTableCaption(t, sections); got != "Table Location: H1" {t.Errorf("caption = %q, want %q", got, "Table Location: H1")}@@ -126,7 +126,7 @@ func TestBuildDOCXJSONSections_TableCaption_OutlineSpansSections(t *testing.T) {{"title":"","elements":[{"type":"heading","level":1,"content":[{"type":"text","text":"第一章"}]}]},{"title":"","elements":[` + docxCaptionTableBody + `]}]}`- sections := buildDOCXJSONSections(ir, newEmbeddedMediaBudget())+ sections := buildDOCXJSONSections(ir, newMediaBudgetForTest())if got := docxCaptionFindTableCaption(t, sections); got != "Table Location: 第一章" {t.Errorf("caption = %q, want %q", got, "Table Location: 第一章")}@@ -139,7 +139,7 @@ func TestBuildDOCXJSONSections_TableCaption_PlainParagraphNotCaption(t *testing.`{"type":"paragraph","content":[{"type":"text","text":"设备流转明细"}]}`,docxCaptionTableBody,)- sections := buildDOCXJSONSections(ir, newEmbeddedMediaBudget())+ sections := buildDOCXJSONSections(ir, newMediaBudgetForTest())if got := docxCaptionFindTableCaption(t, sections); got != "" {t.Errorf("plain paragraph must not become a caption, got %q", got)}@@ -147,7 +147,7 @@ func TestBuildDOCXJSONSections_TableCaption_PlainParagraphNotCaption(t *testing.// Corner: empty IR yields no sections.func TestBuildDOCXJSONSections_TableCaption_Empty(t *testing.T) {- sections := buildDOCXJSONSections(`{"sections":[]}`, newEmbeddedMediaBudget())+ sections := buildDOCXJSONSections(`{"sections":[]}`, newMediaBudgetForTest())if len(sections) != 0 {t.Errorf("empty IR should yield no sections, got %d", len(sections))}@@ -161,7 +161,7 @@ func TestBuildDOCXJSONSections_TableCaption_ExplicitPreferredOverLocation(t *tes`{"type":"heading","level":1,"content":[{"type":"text","text":"设备台账"}]}`,docxCaptionTableBody,)- sections := buildDOCXJSONSectionsWithCaptions(ir, newEmbeddedMediaBudget(), []string{"设备流转明细"})+ sections := buildDOCXJSONSectionsWithCaptions(ir, newMediaBudgetForTest(), []string{"设备流转明细"})if got := docxCaptionFindTableCaption(t, sections); got != "设备流转明细" {t.Errorf("explicit caption = %q, want %q", got, "设备流转明细")}@@ -170,7 +170,7 @@ func TestBuildDOCXJSONSections_TableCaption_ExplicitPreferredOverLocation(t *tes// An explicit caption is attached even when the table sits under no heading.func TestBuildDOCXJSONSections_TableCaption_ExplicitWithNoHeading(t *testing.T) {ir := docxCaptionTableIR(docxCaptionTableBody)- sections := buildDOCXJSONSectionsWithCaptions(ir, newEmbeddedMediaBudget(), []string{"设备流转明细"})+ sections := buildDOCXJSONSectionsWithCaptions(ir, newMediaBudgetForTest(), []string{"设备流转明细"})if got := docxCaptionFindTableCaption(t, sections); got != "设备流转明细" {t.Errorf("explicit caption = %q, want %q", got, "设备流转明细")}@@ -182,7 +182,7 @@ func TestBuildDOCXJSONSections_TableCaption_BlankDoesNotOverrideLocation(t *test`{"type":"heading","level":1,"content":[{"type":"text","text":"设备台账"}]}`,docxCaptionTableBody,)- sections := buildDOCXJSONSectionsWithCaptions(ir, newEmbeddedMediaBudget(), []string{""})+ sections := buildDOCXJSONSectionsWithCaptions(ir, newMediaBudgetForTest(), []string{""})if got := docxCaptionFindTableCaption(t, sections); got != "Table Location: 设备台账" {t.Errorf("caption = %q, want %q", got, "Table Location: 设备台账")}diff --git a/internal/parser/parser/docx_ir.go b/internal/parser/parser/docx_ir.goindex dfe3707ad1..3edcbe39a5 100644--- a/internal/parser/parser/docx_ir.go+++ b/internal/parser/parser/docx_ir.go@@ -31,6 +31,7 @@ import ("strings""ragflow/internal/entity"+ "ragflow/internal/parser/media""ragflow/internal/parser/tableutil")@@ -382,7 +383,7 @@ func docxHeadingLocationCaption(headings []docxHeading) string {// This is the caption-only (heading hierarchy) entry point. Use// buildDOCXJSONSectionsWithCaptions to also attach Caption-styled paragraph// text (a temporary stopgap; see docx_caption_scan.go).-func buildDOCXJSONSections(irJSON string, budget *embeddedMediaBudget) []map[string]any {+func buildDOCXJSONSections(irJSON string, budget *media.Budget) []map[string]any {return buildDOCXJSONSectionsCore(irJSON, budget, nil)}@@ -392,7 +393,7 @@ func buildDOCXJSONSections(irJSON string, budget *embeddedMediaBudget) []map[str//// TEMPORARY: remove once office_oxide emits paragraph styles// (yfedoseev/office_oxide#455).-func buildDOCXJSONSectionsWithCaptions(irJSON string, budget *embeddedMediaBudget, tableCaptions []string) []map[string]any {+func buildDOCXJSONSectionsWithCaptions(irJSON string, budget *media.Budget, tableCaptions []string) []map[string]any {return buildDOCXJSONSectionsCore(irJSON, budget, tableCaptions)}@@ -403,13 +404,13 @@ func buildDOCXJSONSectionsWithCaptions(irJSON string, budget *embeddedMediaBudge// between tableCaptions and the IR tables is approximate (document-body order// vs office_oxide IR order) and is a known limitation of the temporary// Caption-style stopgap in docx_caption_scan.go.-func buildDOCXJSONSectionsCore(irJSON string, budget *embeddedMediaBudget, tableCaptions []string) []map[string]any {+func buildDOCXJSONSectionsCore(irJSON string, budget *media.Budget, tableCaptions []string) []map[string]any {var ir docxIRDocumentif err := json.Unmarshal([]byte(irJSON), &ir); err != nil {return nil}if budget == nil {- budget = newEmbeddedMediaBudget()+ budget = media.NewBudget()}var sections []map[string]anytableSequence := 0@@ -539,12 +540,14 @@ func buildDOCXJSONSectionsCore(irJSON string, budget *embeddedMediaBudget, table}// appendOfficeTableSections emits a table and its embedded images together.-func appendOfficeTableSections(sections []map[string]any, el docxIRElement, caption, tableID string, budget *embeddedMediaBudget) []map[string]any {+func appendOfficeTableSections(sections []map[string]any, el docxIRElement, caption, tableID string, budget *media.Budget) []map[string]any {table := NewTableDataJSONItem(officeIRTableData(el, caption), "", nil).ToMap()var tableImages []docxTableImageforEachDOCXTableImage(el, func(tableImage docxTableImage) bool {- included, keepWalking := budget.include(tableImage.data)+ reservation, admission := budget.ReserveOccurrence(media.UnknownSize)+ keepWalking := admission != media.RejectedByCount+ included := admission == media.Admitted && reservation.Commit(media.Size(len(tableImage.data)))if included || keepWalking {tableImage.included = includedtableImages = append(tableImages, tableImage)@@ -572,7 +575,7 @@ func appendOfficeTableSections(sections []map[string]any, el docxIRElement, capt// order. A paragraph is not necessarily a single text unit: office_oxide can// place an image between two text runs, and dropping that run would make the// downstream DOCX chunker lose both media order and adjacency context.-func appendDOCXParagraphSections(sections []map[string]any, el docxIRElement, heading bool, budget *embeddedMediaBudget) []map[string]any {+func appendDOCXParagraphSections(sections []map[string]any, el docxIRElement, heading bool, budget *media.Budget) []map[string]any {var text strings.BuilderflushText := func() {value := strings.TrimSpace(text.String())@@ -611,11 +614,13 @@ func appendDOCXParagraphSections(sections []map[string]any, el docxIRElement, hereturn sections}-func appendDOCXImageSection(sections []map[string]any, data []byte, budget *embeddedMediaBudget, metadata map[string]any) ([]map[string]any, bool) {+func appendDOCXImageSection(sections []map[string]any, data []byte, budget *media.Budget, metadata map[string]any) ([]map[string]any, bool) {if len(data) == 0 {return sections, true}- included, keepWalking := budget.include(data)+ reservation, admission := budget.ReserveOccurrence(media.UnknownSize)+ keepWalking := admission != media.RejectedByCount+ included := admission == media.Admitted && reservation.Commit(media.Size(len(data)))if !included && !keepWalking {return sections, false}@@ -649,13 +654,13 @@ func appendDOCXImagePayload(sections []map[string]any, data []byte, included boo//// Reuses the IR already obtained from the doc handle in// ParseWithResult so the binary is not opened twice.-func extractDOCXFiguresFromIR(irJSON string, budget *embeddedMediaBudget) []DOCXFigure {+func extractDOCXFiguresFromIR(irJSON string, budget *media.Budget) []DOCXFigure {var ir docxIRDocumentif err := json.Unmarshal([]byte(irJSON), &ir); err != nil {return nil}if budget == nil {- budget = newEmbeddedMediaBudget()+ budget = media.NewBudget()}var flat []flatBlock@@ -668,7 +673,9 @@ func extractDOCXFiguresFromIR(irJSON string, budget *embeddedMediaBudget) []DOCXif len(block.imageData) == 0 {continue}- included, keepWalking := budget.include(block.imageData)+ reservation, admission := budget.ReserveOccurrence(media.UnknownSize)+ keepWalking := admission != media.RejectedByCount+ included := admission == media.Admitted && reservation.Commit(media.Size(len(block.imageData)))if !included && !keepWalking {break}diff --git a/internal/parser/parser/docx_ir_test.go b/internal/parser/parser/docx_ir_test.goindex 866ee36097..efe23b41c4 100644--- a/internal/parser/parser/docx_ir_test.go+++ b/internal/parser/parser/docx_ir_test.go@@ -27,7 +27,7 @@ func TestBuildDOCXJSONSections_FromJSON(t *testing.T) {{"type":"list","items":[{"content":[{"type":"paragraph","content":[{"type":"text","text":"item1"}]}]}]}]}]}`- got := buildDOCXJSONSections(irJSON, newEmbeddedMediaBudget())+ got := buildDOCXJSONSections(irJSON, newMediaBudgetForTest())want := []map[string]any{{"text": "Title", "image": nil, "doc_type_kwd": "text", "layout": "heading"},{"text": "Hello", "image": nil, "doc_type_kwd": "text"},@@ -49,7 +49,7 @@ func TestBuildDOCXJSONSections_EmptyTableSkipped(t *testing.T) {{"type":"paragraph","content":[{"type":"text","text":"keep"}]},{"type":"table","rows":[]}]}]}`- got := buildDOCXJSONSections(irJSON, newEmbeddedMediaBudget())+ got := buildDOCXJSONSections(irJSON, newMediaBudgetForTest())if len(got) != 1 || got[0]["text"] != "keep" {t.Fatalf("expected only the paragraph to survive, got %+v", got)}@@ -71,7 +71,7 @@ func TestBuildDOCXJSONSections_MergesSplitDocTable(t *testing.T) {{"type":"table","rows":[{"cells":[%s,%s,%s]},{"cells":[%s,%s,%s]}]}]}]}`, cell, cell, cell, cell, cell, cell, cell, cell, cell)- got := buildDOCXJSONSections(irJSON, newEmbeddedMediaBudget())+ got := buildDOCXJSONSections(irJSON, newMediaBudgetForTest())var tables []map[string]anyfor _, it := range got {if dt, _ := it["doc_type_kwd"].(string); dt == "table" {@@ -103,7 +103,7 @@ func TestBuildDOCXJSONSections_DifferentColumnTablesStaySeparate(t *testing.T) {{"type":"table","rows":[{"cells":[%s,%s,%s]}]},{"type":"table","rows":[{"cells":[%s,%s]},{"cells":[%s,%s]}]}]}]}`, cell, cell, cell, cell, cell, cell, cell)- got := buildDOCXJSONSections(irJSON, newEmbeddedMediaBudget())+ got := buildDOCXJSONSections(irJSON, newMediaBudgetForTest())var n intfor _, it := range got {if dt, _ := it["doc_type_kwd"].(string); dt == "table" {@@ -128,7 +128,7 @@ func TestBuildDOCXJSONSections_AdjacentMultiRowTablesStaySeparate(t *testing.T){"type":"table","rows":[{"cells":[%s,%s,%s]},{"cells":[%s,%s,%s]}]},{"type":"table","rows":[{"cells":[%s,%s,%s]},{"cells":[%s,%s,%s]}]}]}]}`, cell, cell, cell, cell, cell, cell, cell, cell, cell, cell, cell, cell)- got := buildDOCXJSONSections(irJSON, newEmbeddedMediaBudget())+ got := buildDOCXJSONSections(irJSON, newMediaBudgetForTest())var tables []map[string]anyfor _, it := range got {if dt, _ := it["doc_type_kwd"].(string); dt == "table" {@@ -160,7 +160,7 @@ func TestBuildDOCXJSONSections_HeaderFragmentWithParagraphBetweenStaysSplit(t *t{"type":"paragraph","content":[{"type":"text","text":"note"}]},{"type":"table","rows":[{"cells":[%s,%s,%s]},{"cells":[%s,%s,%s]}]}]}]}`, cell, cell, cell, cell, cell, cell, cell, cell, cell)- got := buildDOCXJSONSections(irJSON, newEmbeddedMediaBudget())+ got := buildDOCXJSONSections(irJSON, newMediaBudgetForTest())var n intfor _, it := range got {if dt, _ := it["doc_type_kwd"].(string); dt == "table" {@@ -180,7 +180,7 @@ func TestBuildDOCXJSONSections_PreservesInlineImageOrder(t *testing.T) {{"type":"text","text":"after"}]}]}]}`- got := buildDOCXJSONSections(irJSON, newEmbeddedMediaBudget())+ got := buildDOCXJSONSections(irJSON, newMediaBudgetForTest())if len(got) != 3 {t.Fatalf("sections = %+v, want text/image/text sequence", got)}@@ -207,7 +207,7 @@ func TestBuildDOCXJSONSections_ExtractsTableCellImages(t *testing.T) {]}]}]}]}`- got := buildDOCXJSONSections(irJSON, newEmbeddedMediaBudget())+ got := buildDOCXJSONSections(irJSON, newMediaBudgetForTest())if len(got) != 3 {t.Fatalf("sections = %+v, want table plus two cell image items", got)}@@ -234,7 +234,7 @@ func TestBuildDOCXJSONSections_BudgetKeepsInlineTextAroundOmittedImage(t *testin{"type":"image","data":"YWJjZA=="},{"type":"text","text":"after"}]}]}]}`- budget := &embeddedMediaBudget{maxImageBytes: 3, maxTotalBytes: 5, maxItems: 10}+ budget := newLimitedMediaBudgetForTest(t, 3, 5, 10)items := buildDOCXJSONSections(irJSON, budget)if len(items) != 3 {@@ -246,7 +246,7 @@ func TestBuildDOCXJSONSections_BudgetKeepsInlineTextAroundOmittedImage(t *testinif items[1]["doc_type_kwd"] != "image" || items[1]["image"] != nil || items[1]["media_omitted"] != true {t.Fatalf("omitted inline image item = %+v, want metadata without payload", items[1])}- if len(budget.warnings()) == 0 {+ if len(budget.Warnings()) == 0 {t.Fatal("omitted image should produce a parser warning")}}@@ -256,7 +256,7 @@ func TestBuildDOCXJSONSections_BudgetKeepsTableImageMetadata(t *testing.T) {{"content":[{"type":"image","data":"YWJj"}]},{"content":[{"type":"image","data":"ZGVm"}]}]}]}]}]}`- budget := &embeddedMediaBudget{maxImageBytes: 4, maxTotalBytes: 5, maxItems: 10}+ budget := newLimitedMediaBudgetForTest(t, 4, 5, 10)items := buildDOCXJSONSections(irJSON, budget)if len(items) != 3 {@@ -275,9 +275,11 @@ func TestBuildDOCXJSONSections_BudgetKeepsTableImageMetadata(t *testing.T) {}func TestBuildDOCXJSONSections_BudgetCapsMediaCount(t *testing.T) {+ const countLimit = 8+ budget := newLimitedMediaBudgetForTest(t, 8, 1024, countLimit)var ir strings.Builderir.WriteString(`{"sections":[{"elements":[`)- for i := 0; i < 257; i++ {+ for i := 0; i <= countLimit; i++ {if i > 0 {ir.WriteByte(',')}@@ -285,7 +287,6 @@ func TestBuildDOCXJSONSections_BudgetCapsMediaCount(t *testing.T) {}ir.WriteString(`]}]}`)- budget := newEmbeddedMediaBudget()items := buildDOCXJSONSections(ir.String(), budget)imageCount := 0for _, item := range items {@@ -293,10 +294,10 @@ func TestBuildDOCXJSONSections_BudgetCapsMediaCount(t *testing.T) {imageCount++}}- if imageCount != budget.maxItems {- t.Fatalf("image items = %d, want count limit %d", imageCount, budget.maxItems)+ if imageCount != countLimit {+ t.Fatalf("image items = %d, want count limit %d", imageCount, countLimit)}- if len(budget.warnings()) == 0 {+ if len(budget.Warnings()) == 0 {t.Fatal("truncated media count should produce a parser warning")}}@@ -306,7 +307,7 @@ func TestBuildPPTXJSONSections_BudgetKeepsSlideMetadata(t *testing.T) {{"type":"paragraph","content":[{"type":"text","text":"slide text"}]},{"type":"image","data":"YWJjZA=="}]}]}`- budget := &embeddedMediaBudget{maxImageBytes: 3, maxTotalBytes: 5, maxItems: 10}+ budget := newLimitedMediaBudgetForTest(t, 3, 5, 10)items, err := buildPPTXJSONSections(irJSON, budget)if err != nil {@@ -325,9 +326,11 @@ func TestBuildPPTXJSONSections_BudgetKeepsSlideMetadata(t *testing.T) {}func TestBuildPPTXJSONSections_BudgetCapsMediaCount(t *testing.T) {+ const countLimit = 8+ budget := newLimitedMediaBudgetForTest(t, 8, 1024, countLimit)var ir strings.Builderir.WriteString(`{"sections":[{"elements":[`)- for i := 0; i < 257; i++ {+ for i := 0; i <= countLimit; i++ {if i > 0 {ir.WriteByte(',')}@@ -335,7 +338,6 @@ func TestBuildPPTXJSONSections_BudgetCapsMediaCount(t *testing.T) {}ir.WriteString(`]}]}`)- budget := newEmbeddedMediaBudget()items, err := buildPPTXJSONSections(ir.String(), budget)if err != nil {t.Fatalf("buildPPTXJSONSections: %v", err)@@ -346,26 +348,26 @@ func TestBuildPPTXJSONSections_BudgetCapsMediaCount(t *testing.T) {imageCount++}}- if imageCount != budget.maxItems {- t.Fatalf("image items = %d, want count limit %d", imageCount, budget.maxItems)+ if imageCount != countLimit {+ t.Fatalf("image items = %d, want count limit %d", imageCount, countLimit)}- if len(budget.warnings()) == 0 {+ if len(budget.Warnings()) == 0 {t.Fatal("truncated media count should produce a parser warning")}}-func TestEmbeddedMediaBudgetRejectsPerImageAndDocumentByteLimits(t *testing.T) {- budget := &embeddedMediaBudget{maxImageBytes: 4, maxTotalBytes: 5, maxItems: 4}- if included, keepGoing := budget.include([]byte("abc")); !included || !keepGoing {+func TestMediaBudgetRejectsPerImageAndDocumentByteLimits(t *testing.T) {+ budget := newLimitedMediaBudgetForTest(t, 4, 5, 4)+ if included, keepGoing := includeMediaForTest(budget, []byte("abc")); !included || !keepGoing {t.Fatalf("first image admission = (%v, %v), want (true, true)", included, keepGoing)}- if included, keepGoing := budget.include([]byte("def")); included || !keepGoing {+ if included, keepGoing := includeMediaForTest(budget, []byte("def")); included || !keepGoing {t.Fatalf("aggregate overflow admission = (%v, %v), want (false, true)", included, keepGoing)}- if included, keepGoing := budget.include([]byte("12345")); included || !keepGoing {+ if included, keepGoing := includeMediaForTest(budget, []byte("12345")); included || !keepGoing {t.Fatalf("per-image overflow admission = (%v, %v), want (false, true)", included, keepGoing)}- if len(budget.warnings()) == 0 {+ if len(budget.Warnings()) == 0 {t.Fatal("omitted byte payloads should produce a parser warning")}}@@ -374,7 +376,7 @@ func TestBuildDOCXJSONSectionsEncodesOnlyAcceptedImageData(t *testing.T) {data := []byte("text is not a valid image, but the parser preserves it")encoded := base64.StdEncoding.EncodeToString(data)irJSON := fmt.Sprintf(`{"sections":[{"elements":[{"type":"image","data":%q}]}]}`, encoded)- budget := &embeddedMediaBudget{maxImageBytes: 8, maxTotalBytes: 8, maxItems: 4}+ budget := newLimitedMediaBudgetForTest(t, 8, 8, 4)items := buildDOCXJSONSections(irJSON, budget)if len(items) != 1 || items[0]["image"] != nil || items[0]["media_omitted"] != true {t.Fatalf("oversized image items = %+v, want metadata without payload", items)@@ -420,7 +422,7 @@ func TestExtractDOCXFiguresFromIR(t *testing.T) {{"type":"paragraph","content":[{"type":"text","text":"after"}]}]}]}`- figs := extractDOCXFiguresFromIR(irJSON, newEmbeddedMediaBudget())+ figs := extractDOCXFiguresFromIR(irJSON, newMediaBudgetForTest())if len(figs) != 1 {t.Fatalf("expected 1 figure, got %d", len(figs))}@@ -444,7 +446,7 @@ func TestExtractDOCXFiguresFromIR_InlineImage(t *testing.T) {{"type":"text","text":"after"}]}]}]}`- figs := extractDOCXFiguresFromIR(irJSON, newEmbeddedMediaBudget())+ figs := extractDOCXFiguresFromIR(irJSON, newMediaBudgetForTest())if len(figs) != 1 {t.Fatalf("expected 1 inline figure, got %d", len(figs))}@@ -459,7 +461,7 @@ func TestExtractDOCXFiguresFromIR_BudgetKeepsContextWithoutPayload(t *testing.T){"type":"image","data":"YWJjZA=="},{"type":"paragraph","content":[{"type":"text","text":"after"}]}]}]}`- budget := &embeddedMediaBudget{maxImageBytes: 3, maxTotalBytes: 5, maxItems: 10}+ budget := newLimitedMediaBudgetForTest(t, 3, 5, 10)figures := extractDOCXFiguresFromIR(irJSON, budget)if len(figures) != 1 {@@ -476,7 +478,7 @@ func TestExtractDOCXFiguresFromIR_NoImage(t *testing.T) {irJSON := `{"sections":[{"elements":[{"type":"paragraph","content":[{"type":"text","text":"only text"}]}]}]}`- if figs := extractDOCXFiguresFromIR(irJSON, newEmbeddedMediaBudget()); figs != nil {+ if figs := extractDOCXFiguresFromIR(irJSON, newMediaBudgetForTest()); figs != nil {t.Fatalf("expected nil, got %+v", figs)}}@@ -484,7 +486,7 @@ func TestExtractDOCXFiguresFromIR_NoImage(t *testing.T) {// TestExtractDOCXFiguresFromIR_BadJSON returns nil on unparseable IR,// mirroring the json.Unmarshal error guard.func TestExtractDOCXFiguresFromIR_BadJSON(t *testing.T) {- if figs := extractDOCXFiguresFromIR("{not json", newEmbeddedMediaBudget()); figs != nil {+ if figs := extractDOCXFiguresFromIR("{not json", newMediaBudgetForTest()); figs != nil {t.Fatalf("expected nil for bad JSON, got %+v", figs)}}@@ -502,7 +504,7 @@ func TestExtractDOCXFiguresFromIR_NonParagraphContext(t *testing.T) {{"type":"list","items":[{"content":[{"type":"paragraph","content":[{"type":"text","text":"list item"}]}]}]},{"type":"text_box","content":[{"type":"paragraph","content":[{"type":"text","text":"box text"}]}]}]}]}`- figs := extractDOCXFiguresFromIR(irJSON, newEmbeddedMediaBudget())+ figs := extractDOCXFiguresFromIR(irJSON, newMediaBudgetForTest())if len(figs) != 1 {t.Fatalf("expected 1 figure, got %d", len(figs))}@@ -526,7 +528,7 @@ func TestExtractDOCXFiguresFromIR_NestedImages(t *testing.T) {{"type":"table","rows":[{"cells":[{"content":[{"type":"paragraph","content":[{"type":"text","text":"cell"},{"type":"image","data":"aGVsbG8="}]}]}]}]},{"type":"text_box","content":[{"type":"paragraph","content":[{"type":"image","data":"d29ybGQ="}]}]}]}]}`- figs := extractDOCXFiguresFromIR(irJSON, newEmbeddedMediaBudget())+ figs := extractDOCXFiguresFromIR(irJSON, newMediaBudgetForTest())if len(figs) != 2 {t.Fatalf("expected 2 nested figures, got %d: %+v", len(figs), figs)}@@ -626,7 +628,7 @@ func TestExtractTextFromListItem_NestedNull(t *testing.T) {{"content":[{"type":"paragraph","content":[{"type":"text","text":"only item"}]}],"nested":null}]}]}]}`- sections := buildDOCXJSONSections(irJSON, newEmbeddedMediaBudget())+ sections := buildDOCXJSONSections(irJSON, newMediaBudgetForTest())if len(sections) != 1 || sections[0]["text"] != "only item" {t.Fatalf("expected single item \"only item\", got %+v", sections)}@@ -648,7 +650,7 @@ func TestExtractTextFromListItem_NestedTable(t *testing.T) {]}]}]}]}`- sections := buildDOCXJSONSections(irJSON, newEmbeddedMediaBudget())+ sections := buildDOCXJSONSections(irJSON, newMediaBudgetForTest())if len(sections) != 2 || sections[0]["text"] != "before" {t.Fatalf("sections=%#v", sections)}@@ -697,7 +699,7 @@ func TestBuildDOCXJSONSections_TextBoxTable(t *testing.T) {]}]}]}]}`- sections := buildDOCXJSONSections(irJSON, newEmbeddedMediaBudget())+ sections := buildDOCXJSONSections(irJSON, newMediaBudgetForTest())if len(sections) != 1 {t.Fatalf("expected 1 section, got %d: %+v", len(sections), sections)}@@ -716,7 +718,7 @@ func TestBuildDOCXJSONSections_NestedList(t *testing.T) {]}}]}]}]}`- sections := buildDOCXJSONSections(irJSON, newEmbeddedMediaBudget())+ sections := buildDOCXJSONSections(irJSON, newMediaBudgetForTest())if len(sections) != 3 {t.Fatalf("sections=%#v", sections)}@@ -744,7 +746,7 @@ func TestBuildDOCXJSONSections_EmitsListAndTextBoxImages(t *testing.T) {{"type":"image","data":"cGljdHVyZQ=="}]}]}]}]}`- sections := buildDOCXJSONSections(irJSON, newEmbeddedMediaBudget())+ sections := buildDOCXJSONSections(irJSON, newMediaBudgetForTest())want := []struct {docType stringtext stringdiff --git a/internal/parser/parser/docx_parser.go b/internal/parser/parser/docx_parser.goindex 0642327787..805bf6e6e9 100644--- a/internal/parser/parser/docx_parser.go+++ b/internal/parser/parser/docx_parser.go@@ -21,6 +21,7 @@ import ("fmt"deepdocoffice "ragflow/internal/deepdoc/parser/office"+ "ragflow/internal/parser/media")// docxExtract is a test seam for the native office_oxide extraction. It@@ -98,7 +99,7 @@ func (p *DOCXParser) ParseWithResult(ctx context.Context, filename string, data// Extract IR JSON for section building and embedded-image extraction.// The budget bounds image payloads so the downstream VLM enhancement// stays within limits.- budget := newEmbeddedMediaBudget()+ budget := media.NewBudget()if irJSON != "" {figures := extractDOCXFiguresFromIR(irJSON, budget)if len(figures) > 0 {@@ -145,7 +146,7 @@ func (p *DOCXParser) ParseWithResult(ctx context.Context, filename string, datareturn ParseResult{File: fileMeta,JSON: MapsToItems(sections),- Warnings: budget.warnings(),+ Warnings: budget.Warnings(),}}diff --git a/internal/parser/parser/docx_parser_test.go b/internal/parser/parser/docx_parser_test.goindex a4398a6b1b..62d9875b0c 100644--- a/internal/parser/parser/docx_parser_test.go+++ b/internal/parser/parser/docx_parser_test.go@@ -30,7 +30,7 @@ func TestBuildDOCXJSONSections_Paragraphs(t *testing.T) {ir := `{"sections":[{"title":"","elements":[{"type":"paragraph","content":[{"type":"text","text":"Hello world"}],"style":"Normal"}]}]}`- sections := buildDOCXJSONSections(ir, newEmbeddedMediaBudget())+ sections := buildDOCXJSONSections(ir, newMediaBudgetForTest())if len(sections) != 1 {t.Fatalf("got %d sections, want 1", len(sections))}@@ -47,7 +47,7 @@ func TestBuildDOCXJSONSections_Headings(t *testing.T) {ir := `{"sections":[{"title":"","elements":[{"type":"heading","level":1,"content":[{"type":"text","text":"Chapter 1"}]}]}]}`- sections := buildDOCXJSONSections(ir, newEmbeddedMediaBudget())+ sections := buildDOCXJSONSections(ir, newMediaBudgetForTest())if len(sections) != 1 {t.Fatalf("got %d sections, want 1", len(sections))}@@ -68,7 +68,7 @@ func TestBuildDOCXJSONSections_Images(t *testing.T) {ir := `{"sections":[{"title":"","elements":[{"type":"image","data":"` + b64 + `"}]}]}`- sections := buildDOCXJSONSections(ir, newEmbeddedMediaBudget())+ sections := buildDOCXJSONSections(ir, newMediaBudgetForTest())if len(sections) != 1 {t.Fatalf("got %d sections, want 1", len(sections))}@@ -91,7 +91,7 @@ func TestBuildDOCXJSONSections_Tables(t *testing.T) {{"cells":[{"content":[{"type":"paragraph","content":[{"type":"text","text":"A2"}]}]},{"content":[{"type":"paragraph","content":[{"type":"text","text":"B2"}]}]}]}]}]}]}`- sections := buildDOCXJSONSections(ir, newEmbeddedMediaBudget())+ sections := buildDOCXJSONSections(ir, newMediaBudgetForTest())if len(sections) != 1 {t.Fatalf("got %d sections, want 1", len(sections))}@@ -126,7 +126,7 @@ func TestBuildDOCXJSONSections_MixedContent(t *testing.T) {{"type":"table","rows":[{"cells":[{"content":[{"type":"paragraph","content":[{"type":"text","text":"cell"}]}]}]}]},{"type":"heading","level":2,"content":[{"type":"text","text":"Sub title"}]}]}]}`- sections := buildDOCXJSONSections(ir, newEmbeddedMediaBudget())+ sections := buildDOCXJSONSections(ir, newMediaBudgetForTest())if len(sections) != 4 {t.Fatalf("got %d sections, want 4", len(sections))}@@ -159,7 +159,7 @@ func TestBuildDOCXJSONSections_List(t *testing.T) {{"content":[{"type":"paragraph","content":[{"type":"text","text":"Item 3"}]}]}],"level":0}]}]}`- sections := buildDOCXJSONSections(ir, newEmbeddedMediaBudget())+ sections := buildDOCXJSONSections(ir, newMediaBudgetForTest())if len(sections) != 3 {t.Fatalf("got %d sections, want 3 (each list item should be a section)", len(sections))}@@ -177,7 +177,7 @@ func TestBuildDOCXJSONSections_TextBox(t *testing.T) {ir := `{"sections":[{"title":"","elements":[{"type":"text_box","content":[{"type":"paragraph","content":[{"type":"text","text":"Boxed paragraph"}]}],"width_emu":null}]}]}`- sections := buildDOCXJSONSections(ir, newEmbeddedMediaBudget())+ sections := buildDOCXJSONSections(ir, newMediaBudgetForTest())if len(sections) != 1 {t.Fatalf("got %d sections, want 1 (text_box content should become a section)", len(sections))}@@ -198,7 +198,7 @@ func TestBuildDOCXJSONSections_MixedWithList(t *testing.T) {],"level":0},{"type":"paragraph","content":[{"type":"text","text":"Trailer"}],"style":"Normal"}]}]}`- sections := buildDOCXJSONSections(ir, newEmbeddedMediaBudget())+ sections := buildDOCXJSONSections(ir, newMediaBudgetForTest())if len(sections) != 4 {t.Fatalf("got %d sections, want 4 (para + 2 list items + para)", len(sections))}@@ -216,7 +216,7 @@ func TestBuildDOCXJSONSections_EmptySkipped(t *testing.T) {{"type":"paragraph","content":[{"type":"text","text":"Only this matters"}]},{"type":"table","rows":[]}]}]}`- sections := buildDOCXJSONSections(ir, newEmbeddedMediaBudget())+ sections := buildDOCXJSONSections(ir, newMediaBudgetForTest())if len(sections) != 1 {t.Fatalf("got %d sections, want 1 (only non-empty paragraph)", len(sections))}diff --git a/internal/parser/parser/epub_parser.go b/internal/parser/parser/epub_parser.goindex 8bbda6785c..af1f08e431 100644--- a/internal/parser/parser/epub_parser.go+++ b/internal/parser/parser/epub_parser.go@@ -36,6 +36,7 @@ import ("bytes""context""encoding/xml"+ "errors""fmt""io""net/url"@@ -287,15 +288,11 @@ func extractEPUBTextItems(ctx context.Context, r *zip.Reader, opf *opfManifest,var items []map[string]anyvar failures []stringread := 0- budget := newEmbeddedMediaBudget()- // mediacache.Put enforces the same per-image limit, so keep the parser- // budget aligned with it instead of the legacy fixed constant.- budget.maxImageBytes = media.HTMLImageMaxBytes()state := &htmlWalkState{refOnly: true,ctx: ctx,mediaCache: media.CacheFromContext(ctx),- mediaBudget: budget,+ budget: media.NewBudget(),mediaFormat: "epub",skipImages: textOnly,}@@ -357,15 +354,16 @@ func extractEPUBTextItems(ctx context.Context, r *zip.Reader, opf *opfManifest,// the run media cache and the caller receives only the reference.//// A source that cannot be mapped to a readable manifest image is skipped and-// counted in the shared warning. Cache failures fail the task.+// counted in the shared warning. Cache failures fail the task; a run media+// limit skips the image with a warning instead.func epubMediaResolver(entries map[string]*zip.File, state *htmlWalkState, chapterEntry string, images map[string]string) func(string) (*entity.MediaRef, bool) {chapterDir := path.Dir(chapterEntry)return func(src string) (*entity.MediaRef, bool) {if state.err != nil {return nil, false}- // Inline data URIs keep the shared HTML data-URI path.- if media.IsHTMLDataImageSource(src) {+ // Inline data URIs keep the shared data-URI path.+ if media.IsDataImageSource(src) {return nil, false}if state.mediaCache == nil {@@ -377,49 +375,78 @@ func epubMediaResolver(entries map[string]*zip.File, state *htmlWalkState, chaptstate.unresolvedImages++return nil, false}+ // The memo is shared by every chapter: one ZIP entry is read, hashed,+ // and cached once per parse, even when different chapters reference it.+ // A nil value records a source that was already rejected, so repeated+ // references do not re-read it or inflate the warning counters.+ if ref, seen := state.epubMediaMemo[name]; seen {+ if ref == nil {+ return nil, false+ }+ return ref, true+ }mime, ok := images[name]if !ok {state.unresolvedImages+++ state.memoizeEPUBMedia(name, nil)return nil, false}entry, ok := entries[name]if !ok {state.unresolvedImages+++ state.memoizeEPUBMedia(name, nil)return nil, false}- maxBytes := state.mediaBudget.maxImageBytes- if entry.UncompressedSize64 > uint64(maxBytes) {- // Charge the per-image limit so the skip is reported through the- // shared budget warning instead of being silent.- state.mediaBudget.includeReservedSize(maxBytes + 1)- return nil, false- }- // Reserve only for an actual ZIP read: entries skipped above need no- // bytes and must not exhaust the attempt budget.- if !state.mediaBudget.reserveItem() {- return nil, false- }- f, err := entry.Open()- if err != nil {- state.unresolvedImages++- return nil, false- }- data, err := io.ReadAll(io.LimitReader(f, int64(maxBytes)+1))- f.Close()- if err != nil || len(data) == 0 {- state.unresolvedImages++- return nil, false- }- if !state.mediaBudget.includeReservedSize(len(data)) {- return nil, false- }- ref, err := state.mediaCache.Put(state.ctx, state.cacheFormat(), mime, data)- if err != nil {- state.err = fmt.Errorf("%s media cache write: %w", state.cacheFormat(), err)+ return state.loadEPUBMedia(name, mime, entry)+ }+}++// loadEPUBMedia applies size admission, reads one ZIP entry, and writes it to+// the run media cache. It memoizes both success and failure so a source shared+// by multiple chapters performs the work and records warnings once.+func (s *htmlWalkState) loadEPUBMedia(name, mime string, entry *zip.File) (*entity.MediaRef, bool) {+ maxBytes := media.ImageMaxBytes()+ declared := media.Size(maxBytes + 1)+ if entry.UncompressedSize64 <= uint64(maxBytes) {+ declared = media.Size(entry.UncompressedSize64)+ }+ // Check the declared uncompressed size against both byte limits before+ // reserving an item slot or decompressing the entry.+ reservation, admission := s.budget.Reserve(declared)+ if admission != media.Admitted {+ s.memoizeEPUBMedia(name, nil)+ return nil, false+ }+ f, err := entry.Open()+ if err != nil {+ s.unresolvedImages+++ s.memoizeEPUBMedia(name, nil)+ return nil, false+ }+ data, err := io.ReadAll(io.LimitReader(f, int64(maxBytes)+1))+ f.Close()+ if err != nil || len(data) == 0 {+ s.unresolvedImages+++ s.memoizeEPUBMedia(name, nil)+ return nil, false+ }+ if !reservation.Commit(media.Size(len(data))) {+ s.memoizeEPUBMedia(name, nil)+ return nil, false+ }+ ref, err := s.mediaCache.Put(s.ctx, s.cacheFormat(), mime, data)+ if err != nil {+ if errors.Is(err, media.ErrMediaLimitExceeded) {+ s.mediaLimitSkips+++ s.memoizeEPUBMedia(name, nil)return nil, false}- return &ref, true+ s.err = fmt.Errorf("%s media cache write: %w", s.cacheFormat(), err)+ s.memoizeEPUBMedia(name, nil)+ return nil, false}+ s.memoizeEPUBMedia(name, &ref)+ return &ref, true}// appendEPUBChapter keeps continuous chapter prose together, splitting onlydiff --git a/internal/parser/parser/epub_parser_test.go b/internal/parser/parser/epub_parser_test.goindex 0d484f0b74..1a5cc69281 100644--- a/internal/parser/parser/epub_parser_test.go+++ b/internal/parser/parser/epub_parser_test.go@@ -21,6 +21,7 @@ import ("bytes""context""crypto/sha256"+ "encoding/base64""encoding/hex""errors""fmt"@@ -143,32 +144,50 @@ func parseTestEPUBContext(t *testing.T, ctx context.Context, chapters []epubTestreturn NewEPUBParser().ParseWithResult(ctx, "book.epub", buildTestEPUBWithImages(t, chapters, images))}-// The parser-side embedded budget temporarily bounds how many ZIP images one-// EPUB extracts, until the shared media budget replaces it.+// The run media budget bounds how many distinct ZIP images one EPUB extracts.+// Distinct sources are required to reach the cap; repeated references share+// one source (see TestEPUBParserRepeatedZipImageReadsAndWritesOnce).func TestEPUBParserCapsZipImageExtractionAcrossChapters(t *testing.T) {- image := `<img src="Images/plate.png">`+ t.Setenv("RAGFLOW_MEDIA_MAX_IMAGES_PER_RUN", "256")+ const limit = 256+ var first, second strings.Builder+ images := make([]epubTestImage, 0, limit)+ for i := 0; i < limit; i++ {+ name := fmt.Sprintf("Images/plate-%d.png", i)+ if i < limit/2 {+ first.WriteString(`<img src="` + name + `">`)+ } else {+ second.WriteString(`<img src="` + name + `">`)+ }+ images = append(images, epubTestImage{name: name, content: fmt.Sprintf("plate-%d", i)})+ }+ // One distinct source past the cap, which must be omitted.+ const overflowName = "Images/overflow.png"+ second.WriteString(`<img src="` + overflowName + `">`)+ images = append(images, epubTestImage{name: overflowName, content: "overflow"})+ctx, cache := recordingMediaContext()res := parseTestEPUBContext(t, ctx, []epubTestChapter{- {name: "one.xhtml", content: epubTestHTML(strings.Repeat(image, 128))},- {name: "two.xhtml", content: epubTestHTML(strings.Repeat(image, 129) + "<p>After images.</p>")},- }, []epubTestImage{{name: "Images/plate.png", content: "plate-bytes"}})+ {name: "one.xhtml", content: epubTestHTML(first.String())},+ {name: "two.xhtml", content: epubTestHTML(second.String() + "<p>After images.</p>")},+ }, images)if res.Err != nil {t.Fatal(res.Err)}- images := 0+ items := 0for _, item := range ItemsToMaps(res.JSON) {if item["doc_type_kwd"] == DocTypeImage {- images+++ items++}}- if images != maxEmbeddedMediaItems || !strings.Contains(epubTestText(res), "After images.") {- t.Fatalf("images=%d text=%q", images, epubTestText(res))+ if items != limit || !strings.Contains(epubTestText(res), "After images.") {+ t.Fatalf("images=%d text=%q", items, epubTestText(res))}- if cache.putCalls != maxEmbeddedMediaItems {- t.Fatalf("cache writes = %d, want %d", cache.putCalls, maxEmbeddedMediaItems)+ if cache.putCalls != limit {+ t.Fatalf("cache writes = %d, want %d", cache.putCalls, limit)}- if !strings.Contains(strings.Join(res.Warnings, "\n"), "256-item document limit") {- t.Fatalf("missing document image limit warning: %v", res.Warnings)+ if !strings.Contains(strings.Join(res.Warnings, "\n"), "256-item run limit") {+ t.Fatalf("missing run image limit warning: %v", res.Warnings)}}@@ -432,6 +451,7 @@ func TestHTMLWalkStateResolveMediaPrecedesResolveImageAndDataURI(t *testing.T) {refOnly: true,ctx: ctx,mediaCache: cache,+ budget: media.NewBudget(),mediaFormat: "epub",resolveMedia: func(src string) (*entity.MediaRef, bool) {if src == "zip.png" || src == htmlMediaDataURI {@@ -467,7 +487,7 @@ func TestHTMLWalkStateDataURIWriteUsesMediaFormat(t *testing.T) {} {t.Run(tc.name, func(t *testing.T) {ctx, cache := recordingMediaContext()- state := &htmlWalkState{refOnly: true, ctx: ctx, mediaCache: cache, mediaFormat: tc.format}+ state := &htmlWalkState{refOnly: true, ctx: ctx, mediaCache: cache, budget: media.NewBudget(), mediaFormat: tc.format}_, ref, ok := state.imageSource(htmlMediaDataURI)if !ok || ref == nil {t.Fatalf("data URI image was not cached: ok=%v ref=%+v", ok, ref)@@ -621,27 +641,35 @@ func TestEPUBParserSkipsUnreadableImageEntries(t *testing.T) {}// A manifest image missing from the ZIP is skipped without reading bytes, so-// it must not consume an image attempt and hide a later readable image.+// it must not consume an image attempt and hide a later readable image. The+// missing sources are distinct so the count cap would be reachable if the+// resolver charged them.func TestEPUBParserMissingZipEntryDoesNotConsumeImageBudget(t *testing.T) {+ const missing = 256+ var content strings.Builder+ images := make([]epubTestImage, 0, missing+1)+ for i := 0; i < missing; i++ {+ name := fmt.Sprintf("Images/gone-%d.png", i)+ content.WriteString(`<img src="` + name + `">`)+ images = append(images, epubTestImage{name: name, missing: true})+ }+ content.WriteString(`<img src="Images/keep.png" alt="keep">`)+ images = append(images, epubTestImage{name: "Images/keep.png", content: "keep-bytes"})+ctx, cache := recordingMediaContext()- content := epubTestHTML(strings.Repeat(`<img src="Images/gone.png">`, maxEmbeddedMediaItems) +- `<img src="Images/keep.png" alt="keep">`)res := parseTestEPUBContext(t, ctx, []epubTestChapter{{name: "ch1.xhtml",- content: content,- }}, []epubTestImage{- {name: "Images/gone.png", missing: true},- {name: "Images/keep.png", content: "keep-bytes"},- })+ content: epubTestHTML(content.String()),+ }}, images)if res.Err != nil {t.Fatalf("ParseWithResult: %v", res.Err)}if cache.putCalls != 1 || len(cache.puts) != 1 || string(cache.puts[0].data) != "keep-bytes" {t.Fatalf("cache puts = %+v, want only the readable image", cache.puts)}- images := epubTestImageItems(t, res)- if len(images) != 1 || images[0]["text"] != "keep" {- t.Fatalf("image items = %#v, want the readable image", images)+ items := epubTestImageItems(t, res)+ if len(items) != 1 || items[0]["text"] != "keep" {+ t.Fatalf("image items = %#v, want the readable image", items)}if !strings.Contains(epubTestWarningText(res), "unresolved image reference") {t.Fatalf("missing unresolved-image warning: %v", res.Warnings)@@ -651,9 +679,10 @@ func TestEPUBParserMissingZipEntryDoesNotConsumeImageBudget(t *testing.T) {// An image above the per-image limit is reported through the budget warning// without reading bytes, so it must not consume an image attempt either.func TestEPUBParserOversizedZipImageDoesNotConsumeImageBudget(t *testing.T) {- t.Setenv("RAGFLOW_HTML_IMAGE_MAX_BYTES", "8")+ t.Setenv("RAGFLOW_MEDIA_IMAGE_MAX_BYTES", "8")+ t.Setenv("RAGFLOW_MEDIA_MAX_IMAGES_PER_RUN", "256")ctx, cache := recordingMediaContext()- content := epubTestHTML(strings.Repeat(`<img src="Images/huge.png">`, maxEmbeddedMediaItems) ++ content := epubTestHTML(strings.Repeat(`<img src="Images/huge.png">`, 256) +`<img src="Images/ok.png" alt="ok">`)res := parseTestEPUBContext(t, ctx, []epubTestChapter{{name: "ch1.xhtml",@@ -673,27 +702,60 @@ func TestEPUBParserOversizedZipImageDoesNotConsumeImageBudget(t *testing.T) {}}+func TestEPUBParserByteRejectedEntriesDoNotConsumeImageSlots(t *testing.T) {+ t.Setenv("RAGFLOW_MEDIA_IMAGE_MAX_BYTES", "8")+ t.Setenv("RAGFLOW_MEDIA_MAX_IMAGES_PER_RUN", "2")+ t.Setenv("RAGFLOW_MEDIA_MAX_BYTES_PER_RUN", "5")+ ctx, cache := recordingMediaContext()+ res := parseTestEPUBContext(t, ctx, []epubTestChapter{{+ name: "ch1.xhtml",+ content: epubTestHTML(+ `<img src="Images/large-1.png"><img src="Images/large-2.png"><img src="Images/small.png" alt="small">`),+ }}, []epubTestImage{+ {name: "Images/large-1.png", content: strings.Repeat("a", 6)},+ {name: "Images/large-2.png", content: strings.Repeat("b", 6)},+ {name: "Images/small.png", content: "c"},+ })+ if res.Err != nil {+ t.Fatalf("ParseWithResult: %v", res.Err)+ }+ if cache.putCalls != 1 || len(cache.puts) != 1 || string(cache.puts[0].data) != "c" {+ t.Fatalf("cache puts = %+v, want only the small image", cache.puts)+ }+ if !strings.Contains(epubTestWarningText(res), "5-byte run image budget") {+ t.Fatalf("missing total-byte warning: %v", res.Warnings)+ }+}+// A read failure is still a fetch attempt, so the item budget keeps bounding-// the reads a document full of unreadable entries can force.+// the reads a document full of unreadable entries can force. Distinct sources+// are required: repeating one damaged entry shares one admission.func TestEPUBParserUnreadableImageAttemptsCountAgainstBudget(t *testing.T) {+ t.Setenv("RAGFLOW_MEDIA_MAX_IMAGES_PER_RUN", "256")+ const attempts = 256+ var content strings.Builder+ images := make([]epubTestImage, 0, attempts+1)+ for i := 0; i < attempts; i++ {+ name := fmt.Sprintf("Images/bad-%d.png", i)+ content.WriteString(`<img src="` + name + `">`)+ images = append(images, epubTestImage{name: name, damaged: true})+ }+ content.WriteString(`<img src="Images/keep.png" alt="keep">`)+ images = append(images, epubTestImage{name: "Images/keep.png", content: "keep-bytes"})+ctx, cache := recordingMediaContext()- content := epubTestHTML(strings.Repeat(`<img src="Images/bad.png">`, maxEmbeddedMediaItems) +- `<img src="Images/keep.png" alt="keep">`)res := parseTestEPUBContext(t, ctx, []epubTestChapter{{name: "ch1.xhtml",- content: content,- }}, []epubTestImage{- {name: "Images/bad.png", damaged: true},- {name: "Images/keep.png", content: "keep-bytes"},- })+ content: epubTestHTML(content.String()),+ }}, images)if res.Err != nil {t.Fatalf("ParseWithResult: %v", res.Err)}if cache.putCalls != 0 {t.Fatalf("cache puts = %+v, want the readable image dropped after the attempt cap", cache.puts)}- if !strings.Contains(epubTestWarningText(res), "256-item document limit") {- t.Fatalf("missing document image limit warning: %v", res.Warnings)+ if !strings.Contains(epubTestWarningText(res), "256-item run limit") {+ t.Fatalf("missing run image limit warning: %v", res.Warnings)}}@@ -750,17 +812,28 @@ func TestEPUBTextParserSkipsImagesWithoutCache(t *testing.T) {}}-func TestEPUBParserDataURIImagesShareEmbeddedMediaBudget(t *testing.T) {+// Data URIs share the run budget with ZIP entries. Distinct sources reach the+// count cap; repeated data URIs share one admission.+func TestEPUBParserDataURIImagesShareRunBudget(t *testing.T) {+ t.Setenv("RAGFLOW_MEDIA_MAX_IMAGES_PER_RUN", "256")+ const limit = 256+ var content strings.Builder+ for i := 0; i <= limit; i++ {+ content.WriteString(`<img src="data:image/png;base64,`)+ content.WriteString(base64.StdEncoding.EncodeToString([]byte(fmt.Sprintf("image-%d", i))))+ content.WriteString(`">`)+ }+ content.WriteString("<p>After images.</p>")ctx, cache := recordingMediaContext()res := parseTestEPUBContext(t, ctx, []epubTestChapter{{name: "ch1.xhtml",- content: epubTestHTML(strings.Repeat(`<img src="`+htmlMediaDataURI+`">`, maxEmbeddedMediaItems+1) + "<p>After images.</p>"),+ content: epubTestHTML(content.String()),}}, nil)if res.Err != nil {t.Fatalf("ParseWithResult: %v", res.Err)}- if cache.putCalls != maxEmbeddedMediaItems {- t.Fatalf("cache writes = %d, want %d", cache.putCalls, maxEmbeddedMediaItems)+ if cache.putCalls != limit {+ t.Fatalf("cache writes = %d, want %d", cache.putCalls, limit)}images := 0for _, item := range ItemsToMaps(res.JSON) {@@ -768,11 +841,147 @@ func TestEPUBParserDataURIImagesShareEmbeddedMediaBudget(t *testing.T) {images++}}- if images != maxEmbeddedMediaItems || !strings.Contains(epubTestText(res), "After images.") {+ if images != limit || !strings.Contains(epubTestText(res), "After images.") {t.Fatalf("images=%d text=%q", images, epubTestText(res))}- if !strings.Contains(strings.Join(res.Warnings, "\n"), "256-item document limit") {- t.Fatalf("missing document image limit warning: %v", res.Warnings)+ if !strings.Contains(strings.Join(res.Warnings, "\n"), "256-item run limit") {+ t.Fatalf("missing run image limit warning: %v", res.Warnings)+ }+}++// TestEPUBParserRepeatedZipImageReadsAndWritesOnce pins source dedupe on the+// resolver: one ZIP entry referenced repeatedly is read and cached once, while+// every reference and its alt text stay in the result.+func TestEPUBParserRepeatedZipImageReadsAndWritesOnce(t *testing.T) {+ ctx, cache := recordingMediaContext()+ res := parseTestEPUBContext(t, ctx, []epubTestChapter{{+ name: "ch1.xhtml",+ content: epubTestHTML(+ `<p>one</p><img src="Images/icon.png" alt="first"><p>two</p>` ++ `<img src="Images/icon.png" alt="second"><img src="Images/icon.png" alt="third">`),+ }}, []epubTestImage{{name: "Images/icon.png", content: "icon-bytes"}})+ if res.Err != nil {+ t.Fatalf("ParseWithResult: %v", res.Err)+ }+ if cache.putCalls != 1 || len(cache.puts) != 1 || string(cache.puts[0].data) != "icon-bytes" {+ t.Fatalf("cache puts = %+v, want one write for the repeated entry", cache.puts)+ }+ items := epubTestImageItems(t, res)+ if len(items) != 3 {+ t.Fatalf("image items = %#v, want all three references", items)+ }+ first := entity.MediaRefFromAny(items[0]["media_ref"])+ for i, wantAlt := range []string{"first", "second", "third"} {+ if items[i]["text"] != wantAlt {+ t.Fatalf("image %d alt = %v, want %q", i, items[i]["text"], wantAlt)+ }+ ref := entity.MediaRefFromAny(items[i]["media_ref"])+ if first == nil || ref == nil || ref.SHA256 != first.SHA256 {+ t.Fatalf("image %d ref = %+v, want the shared reference %+v", i, ref, first)+ }+ }+ if len(res.Warnings) != 0 {+ t.Fatalf("warnings = %v, want none", res.Warnings)+ }+}++// TestEPUBParserRepeatedZipImageAcrossChaptersReadsAndWritesOnce pins that the+// source memo is shared by the whole parse, not recreated for each chapter.+func TestEPUBParserRepeatedZipImageAcrossChaptersReadsAndWritesOnce(t *testing.T) {+ ctx, cache := recordingMediaContext()+ res := parseTestEPUBContext(t, ctx, []epubTestChapter{+ {name: "ch1.xhtml", content: epubTestHTML(`<img src="Images/icon.png" alt="first">`)},+ {name: "ch2.xhtml", content: epubTestHTML(`<img src="Images/icon.png" alt="second">`)},+ }, []epubTestImage{{name: "Images/icon.png", content: "icon-bytes"}})+ if res.Err != nil {+ t.Fatalf("ParseWithResult: %v", res.Err)+ }+ if cache.putCalls != 1 || len(cache.puts) != 1 || string(cache.puts[0].data) != "icon-bytes" {+ t.Fatalf("cache puts = %+v, want one write across chapters", cache.puts)+ }+ items := epubTestImageItems(t, res)+ if len(items) != 2 {+ t.Fatalf("image items = %#v, want one per chapter", items)+ }+ first := entity.MediaRefFromAny(items[0]["media_ref"])+ second := entity.MediaRefFromAny(items[1]["media_ref"])+ if first == nil || second == nil || first.SHA256 != second.SHA256 {+ t.Fatalf("refs = %+v / %+v, want the shared reference", first, second)+ }+ if len(res.Warnings) != 0 {+ t.Fatalf("warnings = %v, want none", res.Warnings)+ }+}++func TestEPUBParserRepeatedRejectedZipImageAcrossChaptersWarnsOnce(t *testing.T) {+ tests := []struct {+ name string+ maxImage string+ maxTotal string+ payload string+ wantWarn string+ }{+ {+ name: "oversized",+ maxImage: "8",+ maxTotal: "8",+ payload: strings.Repeat("x", 9),+ wantWarn: "larger than the 8-byte per-image limit",+ },+ {+ name: "total bytes",+ maxImage: "8",+ maxTotal: "5",+ payload: strings.Repeat("x", 6),+ wantWarn: "5-byte run image budget",+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ t.Setenv("RAGFLOW_MEDIA_IMAGE_MAX_BYTES", tt.maxImage)+ t.Setenv("RAGFLOW_MEDIA_MAX_BYTES_PER_RUN", tt.maxTotal)+ ctx, cache := recordingMediaContext()+ res := parseTestEPUBContext(t, ctx, []epubTestChapter{+ {name: "ch1.xhtml", content: epubTestHTML(`<img src="Images/rejected.png">`)},+ {name: "ch2.xhtml", content: epubTestHTML(`<img src="Images/rejected.png">`)},+ }, []epubTestImage{{name: "Images/rejected.png", content: tt.payload}})+ if res.Err != nil {+ t.Fatalf("ParseWithResult: %v", res.Err)+ }+ if cache.putCalls != 0 {+ t.Fatalf("cache writes = %+v, want none", cache.puts)+ }+ warnings := epubTestWarningText(res)+ if strings.Count(warnings, tt.wantWarn) != 1 {+ t.Fatalf("warnings = %q, want one %q entry", warnings, tt.wantWarn)+ }+ })+ }+}++// TestEPUBParserSkipsImagesAfterRunCacheLimit pins that a cache limit error is+// a skip with a warning, not a task failure.+func TestEPUBParserSkipsImagesAfterRunCacheLimit(t *testing.T) {+ cache := &recordingMediaCache{putErr: fmt.Errorf("mediacache: run image count limit 1 exceeded: %w", media.ErrMediaLimitExceeded)}+ ctx := media.WithCache(context.Background(), cache)+ res := parseTestEPUBContext(t, ctx, []epubTestChapter{{+ name: "ch1.xhtml",+ content: epubTestHTML(`<p>before</p><img src="Images/a.png"><img src="Images/b.png"><p>after</p>`),+ }}, []epubTestImage{{name: "Images/a.png", content: "a"}, {name: "Images/b.png", content: "b"}})+ if res.Err != nil {+ t.Fatalf("ParseWithResult: %v", res.Err)+ }+ if cache.putCalls != 2 {+ t.Fatalf("cache writes = %d, want both references attempted", cache.putCalls)+ }+ if images := epubTestImageItems(t, res); len(images) != 0 {+ t.Fatalf("image items = %#v, want none", images)+ }+ if !strings.Contains(epubTestWarningText(res), "run media cache limit") {+ t.Fatalf("missing run-limit warning: %v", res.Warnings)+ }+ if !strings.Contains(epubTestText(res), "after") {+ t.Fatalf("text after skipped images lost: %q", epubTestText(res))}}diff --git a/internal/parser/parser/html_media_test.go b/internal/parser/parser/html_media_test.goindex c728c0a082..9ec4d6eae7 100644--- a/internal/parser/parser/html_media_test.go+++ b/internal/parser/parser/html_media_test.go@@ -7,6 +7,7 @@ import ("encoding/hex""encoding/json""errors"+ "fmt""strings""testing"@@ -175,7 +176,7 @@ func TestHTMLParser_EmitsTableCellImagesWithParentAndCellOrder(t *testing.T) {}func TestHTMLParser_SkipsOversizedInlineImagePayload(t *testing.T) {- t.Setenv("RAGFLOW_HTML_IMAGE_MAX_BYTES", "4")+ t.Setenv("RAGFLOW_MEDIA_IMAGE_MAX_BYTES", "4")for name, oversized := range map[string]string{"inline": `<p><img src="` + htmlMediaDataURI + `"></p>`,"table": `<table><tr><td><img src="` + htmlMediaDataURI + `"></td></tr></table>`,@@ -193,17 +194,42 @@ func TestHTMLParser_SkipsOversizedInlineImagePayload(t *testing.T) {if !strings.Contains(renderedText(result.JSON[len(result.JSON)-2].ToMap()), "after") {t.Fatalf("text after oversized image lost: %+v", result.JSON)}- if !strings.Contains(strings.Join(result.Warnings, " "), "invalid or oversized inline image") {- t.Fatalf("missing skipped-image warning: %v", result.Warnings)+ if !strings.Contains(strings.Join(result.Warnings, " "), "larger than the 4-byte per-image limit") {+ t.Fatalf("missing oversized-image warning: %v", result.Warnings)}})}}-func TestHTMLParser_HasNoParserSideImageCountLimit(t *testing.T) {- const count = maxEmbeddedMediaItems + 44+func TestHTMLParserByteRejectedDataURIsDoNotConsumeSourceSlots(t *testing.T) {+ t.Setenv("RAGFLOW_MEDIA_IMAGE_MAX_BYTES", "8")+ t.Setenv("RAGFLOW_MEDIA_MAX_IMAGES_PER_RUN", "2")+ t.Setenv("RAGFLOW_MEDIA_MAX_BYTES_PER_RUN", "5")+ large1 := "data:image/png;base64," + base64.StdEncoding.EncodeToString([]byte("aaaaaa"))+ large2 := "data:image/png;base64," + base64.StdEncoding.EncodeToString([]byte("bbbbbb"))+ small := "data:image/png;base64," + base64.StdEncoding.EncodeToString([]byte("c"))+ input := `<img src="` + large1 + `"><img src="` + large2 + `"><img src="` + small + `" alt="small">`++ ctx, cache := recordingMediaContext()+ result := NewHTMLParser().ParseWithResult(ctx, "byte-limit.html", []byte(input))+ if result.Err != nil {+ t.Fatalf("ParseWithResult: %v", result.Err)+ }+ if cache.putCalls != 1 || len(cache.puts) != 1 || string(cache.puts[0].data) != "c" {+ t.Fatalf("cache puts = %+v, want only the small image", cache.puts)+ }+ if !strings.Contains(strings.Join(result.Warnings, "\n"), "5-byte run image budget") {+ t.Fatalf("missing total-byte warning: %v", result.Warnings)+ }+}++// TestHTMLParser_RepeatedDataURIsShareOneSource pins that an icon repeated+// across a document is decoded and written once, while every reference is+// still emitted with the same MediaRef.+func TestHTMLParser_RepeatedDataURIsShareOneSource(t *testing.T) {+ const count = 300input := "<p>" + strings.Repeat(`<img src="`+htmlMediaDataURI+`">`, count) + "</p>"- ctx, _ := recordingMediaContext()+ ctx, cache := recordingMediaContext()result := NewHTMLParser().ParseWithResult(ctx, "many.html", []byte(input))if result.Err != nil {t.Fatalf("ParseWithResult: %v", result.Err)@@ -215,10 +241,75 @@ func TestHTMLParser_HasNoParserSideImageCountLimit(t *testing.T) {}}if images != count {- t.Fatalf("images = %d, want %d (no parser-side cap, matching PDF)", images, count)+ t.Fatalf("images = %d, want %d (one repeated source stays unbounded)", images, count)+ }+ if len(result.Warnings) != 0 {+ t.Fatalf("warnings = %v, want none", result.Warnings)+ }+ if cache.putCalls != 1 {+ t.Fatalf("cache writes = %d, want one for the repeated source", cache.putCalls)+ }+}++// TestHTMLParser_DistinctDataURIsHitSourceLimit pins that distinct sources+// remain bounded by the run budget and report the omission.+func TestHTMLParser_DistinctDataURIsHitSourceLimit(t *testing.T) {+ t.Setenv("RAGFLOW_MEDIA_MAX_IMAGES_PER_RUN", "256")+ const limit = 256+ var body strings.Builder+ body.WriteString("<p>")+ for i := 0; i <= limit; i++ {+ body.WriteString(`<img src="data:image/png;base64,`)+ body.WriteString(base64.StdEncoding.EncodeToString([]byte(fmt.Sprintf("image-%d", i))))+ body.WriteString(`">`)+ }+ body.WriteString("</p>")+ ctx, cache := recordingMediaContext()+ result := NewHTMLParser().ParseWithResult(ctx, "distinct.html", []byte(body.String()))+ if result.Err != nil {+ t.Fatalf("ParseWithResult: %v", result.Err)+ }+ images := 0+ for _, item := range result.JSON {+ if item.ToMap()["doc_type_kwd"] == "image" {+ images+++ }+ }+ if images != limit {+ t.Fatalf("images = %d, want %d", images, limit)+ }+ if cache.putCalls != limit {+ t.Fatalf("cache writes = %d, want %d", cache.putCalls, limit)+ }+ if !strings.Contains(strings.Join(result.Warnings, "\n"), fmt.Sprintf("%d-item run limit", limit)) {+ t.Fatalf("missing source-count warning: %v", result.Warnings)+ }+}++// TestHTMLParser_SkipsImagesAfterRunCacheLimit pins the parser-side behavior+// for a cache that has exhausted its run caps: the image is skipped with a+// warning and the rest of the document still parses.+func TestHTMLParser_SkipsImagesAfterRunCacheLimit(t *testing.T) {+ cache := &recordingMediaCache{putErr: fmt.Errorf("mediacache: run image count limit 1 exceeded: %w", media.ErrMediaLimitExceeded)}+ ctx := media.WithCache(context.Background(), cache)+ input := `<p>before</p><img src="` + htmlMediaDataURI + `"><img src="data:image/png;base64,YQ=="><p>after</p>`+ result := NewHTMLParser().ParseWithResult(ctx, "limit.html", []byte(input))+ if result.Err != nil {+ t.Fatalf("ParseWithResult: %v", result.Err)+ }+ if cache.putCalls != 2 {+ t.Fatalf("cache writes = %d, want both references attempted", cache.putCalls)+ }+ for _, item := range result.JSON {+ if item.ToMap()["doc_type_kwd"] == "image" {+ t.Fatalf("image item emitted after a cache-limit skip: %#v", item.ToMap())+ }+ }+ if text := renderedText(result.JSON[len(result.JSON)-1].ToMap()); text != "after" {+ t.Fatalf("text after skipped images = %q, want after", text)}- if strings.Contains(strings.Join(result.Warnings, "\n"), "256-item") {- t.Fatalf("unexpected parser-side image cap warning: %v", result.Warnings)+ if !strings.Contains(strings.Join(result.Warnings, "\n"), "run media cache limit") {+ t.Fatalf("missing run-limit warning: %v", result.Warnings)}}diff --git a/internal/parser/parser/html_parser.go b/internal/parser/parser/html_parser.goindex d319cd371e..4cfadf6c81 100644--- a/internal/parser/parser/html_parser.go+++ b/internal/parser/parser/html_parser.go@@ -19,6 +19,7 @@ package parserimport ("bytes""context"+ "errors""fmt""strings"@@ -91,6 +92,7 @@ func (p *HTMLParser) ParseWithResult(ctx context.Context, filename string, datarefOnly: true,ctx: ctx,mediaCache: media.CacheFromContext(ctx),+ budget: media.NewBudget(),}if p.textOnly {state.resolveImage = func(string) (string, bool) { return "", false }@@ -131,7 +133,13 @@ type htmlWalkState struct {mediaOrder intskippedInlineImages intunresolvedImages int- mediaBudget *embeddedMediaBudget+ mediaLimitSkips int+ budget *media.Budget+ // dataURIMemo and epubMediaMemo dedupe sources across one parse. A nil+ // value means the source was already processed and skipped; a non-nil+ // value is the reference every later occurrence reuses.+ dataURIMemo map[string]*entity.MediaRef+ epubMediaMemo map[string]*entity.MediaRef// refOnly defers image bytes to content-addressed references instead of// carrying the normalized data URI on the item. HTML and EPUB set it;// formats that still inline image bytes leave it false.@@ -150,7 +158,11 @@ type htmlWalkState struct {}func (s *htmlWalkState) warnings() []string {- warnings := s.mediaBudget.warnings()+ var warnings []string+ warnings = append(warnings, s.budget.Warnings()...)+ if s.mediaLimitSkips > 0 {+ warnings = append(warnings, fmt.Sprintf("document parser skipped %d image(s) after the run media cache limit", s.mediaLimitSkips))+ }if s.skippedInlineImages > 0 {warnings = append(warnings, fmt.Sprintf("document parser skipped %d invalid or oversized inline image(s)", s.skippedInlineImages))}@@ -184,56 +196,90 @@ func (s *htmlWalkState) imageSource(src string) (string, *entity.MediaRef, bool)normalized, ok := s.resolveImage(src)return normalized, nil, ok}- if !media.IsHTMLDataImageSource(src) {+ if !media.IsDataImageSource(src) {return "", nil, false}if s.refOnly {+ // A repeated source is decoded and written once; every reference still+ // receives the same MediaRef. A nil memo value means it was already+ // processed and skipped, so the warning counters are not inflated.+ if ref, seen := s.dataURIMemo[src]; seen {+ if ref == nil {+ return "", nil, false+ }+ return "", ref, true+ }// Validate and size the payload without allocating so budget admission- // happens before the decode. Malformed sources are never charged- // against the item budget.- size, ok := media.HTMLImageDecodedSize(src)- if !ok || size > media.HTMLImageMaxBytes() {+ // happens before the decode. Malformed sources never spend budget.+ size, ok := media.DataImageDecodedSize(src)+ if !ok {+ s.skippedInlineImages+++ s.memoizeDataURI(src, nil)return "", nil, false}if s.mediaCache == nil {s.err = fmt.Errorf("%s media cache is not configured", s.cacheFormat())return "", nil, false}- if s.mediaBudget != nil {- if !s.mediaBudget.reserveItem() {- return "", nil, false- }- if !s.mediaBudget.includeReservedSize(size) {- return "", nil, false- }+ // Repeated sources share one reservation, so a document made of one+ // repeated data URI stays bounded without capping its references.+ reservation, admission := s.budget.Reserve(media.Size(size))+ if admission != media.Admitted {+ s.memoizeDataURI(src, nil)+ return "", nil, false}- mime, data, ok := media.DecodeHTMLImageSource(src)+ mime, data, ok := media.DecodeDataImageSource(src)if !ok {+ s.skippedInlineImages+++ s.memoizeDataURI(src, nil)+ return "", nil, false+ }+ if !reservation.Commit(media.Size(len(data))) {+ s.memoizeDataURI(src, nil)return "", nil, false}ref, err := s.mediaCache.Put(s.ctx, s.cacheFormat(), mime, data)if err != nil {+ if errors.Is(err, media.ErrMediaLimitExceeded) {+ s.mediaLimitSkips+++ s.memoizeDataURI(src, nil)+ return "", nil, false+ }s.err = fmt.Errorf("%s media cache write: %w", s.cacheFormat(), err)return "", nil, false}+ s.memoizeDataURI(src, &ref)return "", &ref, true}- if s.mediaBudget == nil {- s.mediaBudget = newEmbeddedMediaBudget()- }- if !s.mediaBudget.reserveItem() {+ reservation, admission := s.budget.ReserveOccurrence(media.UnknownSize)+ if admission != media.Admitted {return "", nil, false}- normalized, size, _, ok := media.NormalizeHTMLImageSource(src)+ normalized, size, _, ok := media.NormalizeDataImageSource(src)if !ok {+ s.skippedInlineImages++return "", nil, false}- if !s.mediaBudget.includeReservedSize(int(size)) {+ if !reservation.Commit(media.Size(size)) {return "", nil, false}return normalized, nil, true}+func (s *htmlWalkState) memoizeDataURI(src string, ref *entity.MediaRef) {+ if s.dataURIMemo == nil {+ s.dataURIMemo = make(map[string]*entity.MediaRef)+ }+ s.dataURIMemo[src] = ref+}++func (s *htmlWalkState) memoizeEPUBMedia(name string, ref *entity.MediaRef) {+ if s.epubMediaMemo == nil {+ s.epubMediaMemo = make(map[string]*entity.MediaRef)+ }+ s.epubMediaMemo[name] = ref+}+// walkHTMLBlocksWithState emits normalized text and image items in document// order. Inline formatting is collapsed into its parent text; script, style,// and noscript subtrees are skipped.@@ -497,8 +543,6 @@ func walkHTMLLeaf(n *html.Node, w *leafWriter, out *[]map[string]any, layout strif !ok {if state.resolveImage != nil {w.writeText(htmlAttribute(n, "alt"))- } else if media.IsHTMLDataImageSource(src) {- state.skippedInlineImages++}return}@@ -562,8 +606,7 @@ func emitHTMLTable(n *html.Node, out *[]map[string]any, state *htmlWalkState) {if state.err != nil {return}- images, skippedInlineImages := htmlTableImages(n, state)- state.skippedInlineImages += skippedInlineImages+ images := htmlTableImages(n, state)markup := renderTableHTML(n)if strings.TrimSpace(markup) == "" {return@@ -591,9 +634,8 @@ func emitHTMLTable(n *html.Node, out *[]map[string]any, state *htmlWalkState) {}}-func htmlTableImages(table *html.Node, state *htmlWalkState) ([]htmlTableImage, int) {+func htmlTableImages(table *html.Node, state *htmlWalkState) []htmlTableImage {var images []htmlTableImage- skippedInlineImages := 0rowCount := 0columnCounts := make(map[int]int)var walk func(node *html.Node, row, column int)@@ -626,8 +668,7 @@ func htmlTableImages(table *html.Node, state *htmlWalkState) ([]htmlTableImage,replacement = ""}node.Type, node.Data, node.Attr = html.TextNode, replacement, nil- } else if media.IsHTMLDataImageSource(src) {- skippedInlineImages+++ } else if media.IsDataImageSource(src) {removeHTMLImageSource(node)}} else {@@ -653,7 +694,7 @@ func htmlTableImages(table *html.Node, state *htmlWalkState) ([]htmlTableImage,}}walk(table, 0, 0)- return images, skippedInlineImages+ return images}func removeHTMLImageSource(n *html.Node) {diff --git a/internal/parser/parser/markdown_parser.go b/internal/parser/parser/markdown_parser.goindex 0112047a0b..891154e237 100644--- a/internal/parser/parser/markdown_parser.go+++ b/internal/parser/parser/markdown_parser.go@@ -96,7 +96,7 @@ func (p *MarkdownParser) ConfigureFromSetup(setup map[string]any) {// the base64-encoded image payload. The legacy debug-print path has// been removed; callers consume ParseResult directly.func (p *MarkdownParser) ParseWithResult(ctx context.Context, filename string, data []byte) ParseResult {- state := &htmlWalkState{}+ state := &htmlWalkState{budget: media.NewBudget()}result := p.parseWithState(ctx, filename, data, state)result.Warnings = append(state.warnings(), result.Warnings...)return result@@ -156,10 +156,7 @@ func (p *MarkdownParser) parseWithState(ctx context.Context, filename string, dadoc := blockParser.Parse(nil)var items []map[string]any- if state.mediaBudget == nil {- state.mediaBudget = newEmbeddedMediaBudget()- }- mediaBudget := state.mediaBudget+ mediaBudget := state.budgetimageCtx, cancel := context.WithTimeout(ctx, 60*time.Second)defer cancel()unresolvedImages := walkMarkdownBlocksWithImages(imageCtx, doc, &items, p.FlattenMediaToText, p.FetchRemoteImages, state)@@ -170,7 +167,7 @@ func (p *MarkdownParser) parseWithState(ctx context.Context, filename string, daitems = []map[string]any{{"text": "", "doc_type_kwd": "text"}}}var warnings []string- if mediaBudget.items > 0 && imageCtx.Err() == context.DeadlineExceeded && ctx.Err() == nil {+ if mediaBudget.Count() > 0 && imageCtx.Err() == context.DeadlineExceeded && ctx.Err() == nil {warnings = append(warnings, "Markdown image processing reached the document time budget")}if unresolvedImages > 0 {@@ -447,10 +444,7 @@ func markdownTableCells(line string) []string {// Tables are emitted through the HTML walker as structured items in source// order, with embedded images linked to their parent table.func walkMarkdownBlocksWithImages(ctx context.Context, doc ast.Node, out *[]map[string]any, flatten, fetchRemoteImages bool, htmlState *htmlWalkState) int {- if htmlState.mediaBudget == nil {- htmlState.mediaBudget = newEmbeddedMediaBudget()- }- budget := htmlState.mediaBudget+ budget := htmlState.budgetstart := len(*out)defer func() {if flatten {@@ -635,8 +629,12 @@ func findBlockImage(n ast.Node) (string, bool) {// resolveImageURL resolves one bounded Markdown image attempt. It returns// either normalized base64 or downloaded bytes, and distinguishes a budget// or timeout omission from an unresolved source. Local paths are never fetched.-func resolveImageURL(ctx context.Context, imageURL string, budget *embeddedMediaBudget) (encoded string, raw []byte, omitted, resolved bool) {- if ctx.Err() != nil || !budget.reserveItem() {+func resolveImageURL(ctx context.Context, imageURL string, budget *media.Budget) (encoded string, raw []byte, omitted, resolved bool) {+ if ctx.Err() != nil {+ return "", nil, true, false+ }+ reservation, admission := budget.ReserveOccurrence(media.UnknownSize)+ if admission != media.Admitted {return "", nil, true, false}var size int@@ -646,7 +644,7 @@ func resolveImageURL(ctx context.Context, imageURL string, budget *embeddedMediareturn "", nil, false, false}var decodedSize int64- encoded, decodedSize, _, resolved = media.NormalizeBase64ImagePayload(imageURL[idx+len("base64,"):], media.MaxImagePayloadBytes)+ encoded, decodedSize, _, resolved = media.NormalizeBase64ImagePayload(imageURL[idx+len("base64,"):], media.ImageMaxBytes())if !resolved {return "", nil, false, false}@@ -661,7 +659,7 @@ func resolveImageURL(ctx context.Context, imageURL string, budget *embeddedMedia} else {return "", nil, false, false}- if !budget.includeReservedSize(size) {+ if !reservation.Commit(media.Size(size)) {return "", nil, true, false}return encoded, raw, false, true@@ -746,12 +744,12 @@ func fetchImage(ctx context.Context, rawURL string) ([]byte, error) {if resp.StatusCode < 200 || resp.StatusCode >= 300 {return nil, fmt.Errorf("markdown: fetch image %s: HTTP %d", rawURL, resp.StatusCode)}- body, err := io.ReadAll(io.LimitReader(resp.Body, media.MaxImagePayloadBytes+1))+ body, err := io.ReadAll(io.LimitReader(resp.Body, int64(media.ImageMaxBytes())+1))if err != nil {return nil, fmt.Errorf("markdown: read image %s: %w", rawURL, err)}- if len(body) > media.MaxImagePayloadBytes {- return nil, fmt.Errorf("markdown: image %s exceeds %d-byte limit", rawURL, media.MaxImagePayloadBytes)+ if len(body) > media.ImageMaxBytes() {+ return nil, fmt.Errorf("markdown: image %s exceeds %d-byte limit", rawURL, media.ImageMaxBytes())}if len(body) == 0 {return nil, fmt.Errorf("markdown: image %s is empty", rawURL)diff --git a/internal/parser/parser/markdown_parser_test.go b/internal/parser/parser/markdown_parser_test.goindex 3752aadd90..6a7e25e7b1 100644--- a/internal/parser/parser/markdown_parser_test.go+++ b/internal/parser/parser/markdown_parser_test.go@@ -552,7 +552,7 @@ func TestMarkdownParser_FlattenMediaToText(t *testing.T) {func TestResolveImageURL_DataURI(t *testing.T) {b64 := base64.StdEncoding.EncodeToString([]byte("fakeimage"))- budget := newEmbeddedMediaBudget()+ budget := newLimitedMediaBudgetForTest(t, len("fakeimage"), int64(len("fakeimage")), 2)result, raw, omitted, found := resolveImageURL(context.Background(), "data:image/png;base64,"+b64, budget)if !found {t.Fatal("expected image found for data URI")@@ -560,14 +560,17 @@ func TestResolveImageURL_DataURI(t *testing.T) {if result != b64 || raw != nil {t.Fatalf("got %q, want %q", result, b64)}- if omitted || budget.totalBytes != len("fakeimage") {- t.Errorf("charged bytes = %d, want %d", budget.totalBytes, len("fakeimage"))+ if omitted || len(budget.Warnings()) != 0 {+ t.Fatalf("first image omitted=%v warnings=%v", omitted, budget.Warnings())+ }+ if included, keepWalking := includeMediaForTest(budget, []byte("x")); included || !keepWalking {+ t.Fatalf("payload after the exact byte cap = (%v, %v), want (false, true)", included, keepWalking)}}func TestResolveImageURL_LocalPathNotFetched(t *testing.T) {// Local / relative paths are not fetched (security); resolution fails.- if _, _, _, found := resolveImageURL(context.Background(), "./local/image.png", newEmbeddedMediaBudget()); found {+ if _, _, _, found := resolveImageURL(context.Background(), "./local/image.png", newMediaBudgetForTest()); found {t.Fatal("expected no image resolved for a local path")}}@@ -586,7 +589,7 @@ func TestResolveImageURL_HTTPImage(t *testing.T) {}))defer ts.Close()- budget := newEmbeddedMediaBudget()+ budget := newLimitedMediaBudgetForTest(t, len("fake-png-bytes"), int64(len("fake-png-bytes")), 2)encoded, result, omitted, found := resolveImageURL(context.Background(), ts.URL+"/image.png", budget)if !found {t.Fatal("expected image found for HTTP URL")@@ -594,8 +597,11 @@ func TestResolveImageURL_HTTPImage(t *testing.T) {if encoded != "" || string(result) != "fake-png-bytes" {t.Fatalf("got encoded=%q raw=%q, want raw image bytes", encoded, result)}- if omitted || budget.totalBytes != len("fake-png-bytes") {- t.Errorf("charged bytes = %d, want %d", budget.totalBytes, len("fake-png-bytes"))+ if omitted || len(budget.Warnings()) != 0 {+ t.Fatalf("first image omitted=%v warnings=%v", omitted, budget.Warnings())+ }+ if included, keepWalking := includeMediaForTest(budget, []byte("x")); included || !keepWalking {+ t.Fatalf("payload after the exact byte cap = (%v, %v), want (false, true)", included, keepWalking)}}@@ -616,9 +622,9 @@ func TestMarkdownImageBudgetSkipsFetchAfterCountLimit(t *testing.T) {"\n\n" +"\n"doc := markdownNew().Parse([]byte(markdown))- budget := &embeddedMediaBudget{maxImageBytes: 4, maxTotalBytes: 4, maxItems: 2}+ budget := newLimitedMediaBudgetForTest(t, 4, 4, 2)var items []map[string]any- if unresolved := walkMarkdownBlocksWithImages(t.Context(), doc, &items, false, true, &htmlWalkState{mediaBudget: budget}); unresolved != 0 {+ if unresolved := walkMarkdownBlocksWithImages(t.Context(), doc, &items, false, true, &htmlWalkState{budget: budget}); unresolved != 0 {t.Fatalf("unresolved images = %d, want 0", unresolved)}if got := fetches.Load(); got != 2 {@@ -632,10 +638,10 @@ func TestMarkdownImageBudgetSkipsFetchAfterCountLimit(t *testing.T) {t.Fatalf("image %d = %+v, want omitted payload", index, items[index])}}- warnings := strings.Join(budget.warnings(), "\n")- if !strings.Contains(warnings, "omitted 1 embedded image payload(s) after reaching the 4-byte document image budget") ||- !strings.Contains(warnings, "stopped extracting embedded images after the 2-item document limit") {- t.Fatalf("warnings = %q, want document byte and item limit warnings", warnings)+ warnings := strings.Join(budget.Warnings(), "\n")+ if !strings.Contains(warnings, "omitted 1 embedded image payload(s) after reaching the 4-byte run image budget") ||+ !strings.Contains(warnings, "stopped extracting embedded images after the 2-item run limit") {+ t.Fatalf("warnings = %q, want run byte and item limit warnings", warnings)}}diff --git a/internal/parser/parser/media_budget.go b/internal/parser/parser/media_budget.godeleted file mode 100644index a3966bd3c5..0000000000--- a/internal/parser/parser/media_budget.go+++ /dev/null@@ -1,117 +0,0 @@-//-// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.-//-// Licensed under the Apache License, Version 2.0 (the "License");-// you may not use this file except in compliance with the License.-// You may obtain a copy of the License at-//-// http://www.apache.org/licenses/LICENSE-2.0-//-// Unless required by applicable law or agreed to in writing, software-// distributed under the License is distributed on an "AS IS" BASIS,-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.-// See the License for the specific language governing permissions and-// limitations under the License.-//--package parser--import (- "fmt"-- "ragflow/internal/parser/media"-)--const (- maxEmbeddedImageBytes = media.MaxImagePayloadBytes- maxEmbeddedDocumentImageBytes = 64 << 20- maxEmbeddedMediaItems = 256-)--type embeddedMediaBudget struct {- maxImageBytes int- maxTotalBytes int- maxItems int-- items int- totalBytes int- oversizedImages int- documentOverflows int- countExhausted bool-}--func newEmbeddedMediaBudget() *embeddedMediaBudget {- return &embeddedMediaBudget{- maxImageBytes: maxEmbeddedImageBytes,- maxTotalBytes: maxEmbeddedDocumentImageBytes,- maxItems: maxEmbeddedMediaItems,- }-}--// include reserves budget for one image payload. The second result is false-// when callers should stop walking images because the document item limit has-// been reached. Byte-limit omissions can keep walking so later text and media-// metadata remain available.-func (b *embeddedMediaBudget) include(data []byte) (bool, bool) {- if !b.reserveItem() {- return false, false- }- return b.includeReservedSize(len(data)), true-}--// reserveItem caps the image fetch attempts for one document. Callers reserve-// immediately before the I/O; skips that read no bytes are not charged.-func (b *embeddedMediaBudget) reserveItem() bool {- if b.countExhausted {- return false- }- if b.items >= b.maxItems {- b.countExhausted = true- return false- }- b.items++- return true-}--// includeReservedSize charges a payload against the per-image and document-// byte limits without retaining or decoding it just to measure its size.-func (b *embeddedMediaBudget) includeReservedSize(size int) bool {- if size > b.maxImageBytes {- b.oversizedImages++- return false- }- if size > b.maxTotalBytes-b.totalBytes {- b.documentOverflows++- return false- }- b.totalBytes += size- return true-}--func (b *embeddedMediaBudget) warnings() []string {- if b == nil {- return nil- }- var warnings []string- if b.oversizedImages > 0 {- warnings = append(warnings, fmt.Sprintf(- "omitted %d embedded image payload(s) larger than the %d-byte per-image limit",- b.oversizedImages,- b.maxImageBytes,- ))- }- if b.documentOverflows > 0 {- warnings = append(warnings, fmt.Sprintf(- "omitted %d embedded image payload(s) after reaching the %d-byte document image budget",- b.documentOverflows,- b.maxTotalBytes,- ))- }- if b.countExhausted {- warnings = append(warnings, fmt.Sprintf(- "stopped extracting embedded images after the %d-item document limit",- b.maxItems,- ))- }- return warnings-}diff --git a/internal/parser/parser/media_budget_test.go b/internal/parser/parser/media_budget_test.goindex df8f13ff6a..a47a9c5b0c 100644--- a/internal/parser/parser/media_budget_test.go+++ b/internal/parser/parser/media_budget_test.go@@ -1,50 +1,45 @@+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.+package parserimport (- "reflect""testing"++ "ragflow/internal/parser/media")-func TestEmbeddedMediaBudgetReservedItemsAndWarnings(t *testing.T) {- budget := &embeddedMediaBudget{maxImageBytes: 4, maxTotalBytes: 5, maxItems: 3}+func newMediaBudgetForTest() *media.Budget {+ return media.NewBudget()+}- if !budget.reserveItem() || !budget.includeReservedSize(3) {- t.Fatal("first image should fit")- }- if !budget.reserveItem() || budget.includeReservedSize(3) {- t.Fatal("second image should exceed the document byte limit")- }- if !budget.reserveItem() || budget.includeReservedSize(5) {- t.Fatal("third image should exceed the per-image byte limit")- }- if budget.reserveItem() {- t.Fatal("fourth image should be rejected before its source is read")- }- if included, keepWalking := budget.include([]byte("x")); included || keepWalking {- t.Fatalf("include after count exhaustion = (%v, %v), want (false, false)", included, keepWalking)- }- if budget.items != 3 || budget.totalBytes != 3 {- t.Fatalf("budget counters = (%d items, %d bytes), want (3, 3)", budget.items, budget.totalBytes)- }- want := []string{- "omitted 1 embedded image payload(s) larger than the 4-byte per-image limit",- "omitted 1 embedded image payload(s) after reaching the 5-byte document image budget",- "stopped extracting embedded images after the 3-item document limit",- }- if got := budget.warnings(); !reflect.DeepEqual(got, want) {- t.Fatalf("warnings = %q, want %q", got, want)- }+func newLimitedMediaBudgetForTest(t *testing.T, maxImageBytes int, maxTotalBytes int64, maxItems int) *media.Budget {+ t.Helper()+ return media.NewBudgetWithLimits(media.Limits{+ ImageMaxBytes: int64(maxImageBytes),+ MaxImagesPerRun: maxItems,+ MaxBytesPerRun: maxTotalBytes,+ })}-func TestEmbeddedMediaBudgetIncludesExactByteLimits(t *testing.T) {- budget := &embeddedMediaBudget{maxImageBytes: 4, maxTotalBytes: 4, maxItems: 2}- if included, keepWalking := budget.include([]byte("abcd")); !included || !keepWalking {- t.Fatalf("exact-limit image = (%v, %v), want (true, true)", included, keepWalking)- }- if included, keepWalking := budget.include([]byte("e")); included || !keepWalking {- t.Fatalf("image after exact document limit = (%v, %v), want (false, true)", included, keepWalking)+func includeMediaForTest(budget *media.Budget, data []byte) (included, keepWalking bool) {+ reservation, admission := budget.ReserveOccurrence(media.UnknownSize)+ if admission == media.RejectedByCount {+ return false, false}- if budget.totalBytes != 4 {- t.Fatalf("totalBytes = %d, want 4", budget.totalBytes)+ if admission != media.Admitted {+ return false, true}+ return reservation.Commit(media.Size(len(data))), true}diff --git a/internal/parser/parser/office_table_render.go b/internal/parser/parser/office_table_render.goindex 3013f7b76e..35312221d6 100644--- a/internal/parser/parser/office_table_render.go+++ b/internal/parser/parser/office_table_render.go@@ -29,6 +29,7 @@ import ("ragflow/internal/common""ragflow/internal/entity"+ "ragflow/internal/parser/media""ragflow/internal/parser/tableutil")@@ -598,7 +599,7 @@ func deprecatedHTML4Excel(setup map[string]any, parserName string) {// worksheet as structured parser items. Excelize exposes both kinds through// GetPictureCells/GetPictures; walking the reported anchor cells avoids// scanning the worksheet's entire coordinate space.-func extractXLSXImages(f *excelize.File, sheet string, budget *embeddedMediaBudget) ([]map[string]any, []string) {+func extractXLSXImages(f *excelize.File, sheet string, budget *media.Budget) ([]map[string]any, []string) {cells, err := f.GetPictureCells(sheet)if err != nil {return nil, []string{fmt.Sprintf("XLSX image discovery failed for sheet %q: %v", sheet, err)}@@ -627,7 +628,9 @@ cellLoop:}included, keepWalking := true, trueif budget != nil {- included, keepWalking = budget.include(picture.File)+ reservation, admission := budget.ReserveOccurrence(media.UnknownSize)+ keepWalking = admission != media.RejectedByCount+ included = admission == media.Admitted && reservation.Commit(media.Size(len(picture.File)))}if !keepWalking {break cellLoopdiff --git a/internal/parser/parser/parse_with_result_test.go b/internal/parser/parser/parse_with_result_test.goindex 4dcdea6760..8baed39852 100644--- a/internal/parser/parser/parse_with_result_test.go+++ b/internal/parser/parser/parse_with_result_test.go@@ -246,7 +246,7 @@ func TestWalkHTMLBlocksWithState_SkipsHeadLooseText(t *testing.T) {head.AppendChild(&html.Node{Type: html.TextNode, Data: "Head metadata"})var items []map[string]any- state := &htmlWalkState{}+ state := &htmlWalkState{budget: newMediaBudgetForTest()}walkHTMLBlocksWithState(head, &items, state)if len(items) != 0 {t.Fatalf("JSON len = %d, want 0: %#v", len(items), items)diff --git a/internal/parser/parser/pdf_parser_somark.go b/internal/parser/parser/pdf_parser_somark.goindex cd9b5a9ee0..7c09323685 100644--- a/internal/parser/parser/pdf_parser_somark.go+++ b/internal/parser/parser/pdf_parser_somark.go@@ -10,10 +10,12 @@ import ("mime/multipart""net/http""net/url"- "ragflow/internal/common""strings"pdfreader "github.com/ledongthuc/pdf"++ "ragflow/internal/common"+ "ragflow/internal/parser/media")// parsePDFWithSoMark submits a PDF and converts the task result into parser@@ -168,7 +170,7 @@ func soMarkItems(ctx context.Context, result map[string]any, keepHeaderFooter bojsonPayload, _ := outputs["json"].(map[string]any)pages, _ := jsonPayload["pages"].([]any)items := make([]map[string]any, 0)- state := &htmlWalkState{}+ state := &htmlWalkState{budget: media.NewBudget()}var warnings []string// Positions reference real document pages: prefer the source PDF's pagediff --git a/internal/parser/parser/pdf_parser_somark_test.go b/internal/parser/parser/pdf_parser_somark_test.goindex f9b898c5f3..7588e88f45 100644--- a/internal/parser/parser/pdf_parser_somark_test.go+++ b/internal/parser/parser/pdf_parser_somark_test.go@@ -98,7 +98,7 @@ func TestPDFParser_ParseWithResult_SoMarkRequiresBaseURL(t *testing.T) {}func TestSoMarkBlockToItem_DropsHeaderByDefault(t *testing.T) {- if item, _ := soMarkBlockToItems(t.Context(), map[string]any{"type": "header", "content": "x"}, false, &htmlWalkState{}); item != nil {+ if item, _ := soMarkBlockToItems(t.Context(), map[string]any{"type": "header", "content": "x"}, false, &htmlWalkState{budget: newMediaBudgetForTest()}); item != nil {t.Fatalf("item = %#v, want nil", item)}}diff --git a/internal/parser/parser/pdf_parser_table_contract_test.go b/internal/parser/parser/pdf_parser_table_contract_test.goindex b8501c8d4f..856b0335c6 100644--- a/internal/parser/parser/pdf_parser_table_contract_test.go+++ b/internal/parser/parser/pdf_parser_table_contract_test.go@@ -77,11 +77,11 @@ func TestPDFRemoteParsers_TableItemsCarryTableMarkup(t *testing.T) {return tcadpAnyToItems(map[string]any{"type": "table", "content": pipeText, "table_data": blankRows})}, "text"},{"somark html", func() []map[string]any {- items, _ := soMarkBlockToItems(t.Context(), map[string]any{"type": "table", "content": tableHTML}, false, &htmlWalkState{})+ items, _ := soMarkBlockToItems(t.Context(), map[string]any{"type": "table", "content": tableHTML}, false, &htmlWalkState{budget: newMediaBudgetForTest()})return items}, "table"},{"somark markdown", func() []map[string]any {- items, _ := soMarkBlockToItems(t.Context(), map[string]any{"type": "table", "content": markdownTable}, false, &htmlWalkState{})+ items, _ := soMarkBlockToItems(t.Context(), map[string]any{"type": "table", "content": markdownTable}, false, &htmlWalkState{budget: newMediaBudgetForTest()})return items}, "table"},}diff --git a/internal/parser/parser/pdf_table_no_html_text_test.go b/internal/parser/parser/pdf_table_no_html_text_test.goindex ee4c88f270..99bb7ee219 100644--- a/internal/parser/parser/pdf_table_no_html_text_test.go+++ b/internal/parser/parser/pdf_table_no_html_text_test.go@@ -34,11 +34,11 @@ func TestPDFTableItemsOmitHTMLText(t *testing.T) {return tcadpAnyToItems(map[string]any{"type": "table", "content": tableHTML})}},{"somark html", func() []map[string]any {- items, _ := soMarkBlockToItems(t.Context(), map[string]any{"type": "table", "content": tableHTML}, false, &htmlWalkState{})+ items, _ := soMarkBlockToItems(t.Context(), map[string]any{"type": "table", "content": tableHTML}, false, &htmlWalkState{budget: newMediaBudgetForTest()})return items}},{"docx table", func() []map[string]any {- return buildDOCXJSONSections(docxIR, newEmbeddedMediaBudget())+ return buildDOCXJSONSections(docxIR, newMediaBudgetForTest())}},}for _, tc := range cases {diff --git a/internal/parser/parser/picture_parser.go b/internal/parser/parser/picture_parser.goindex 5481c2bd70..80fc30b31d 100644--- a/internal/parser/parser/picture_parser.go+++ b/internal/parser/parser/picture_parser.go@@ -71,8 +71,8 @@ func (p *PictureParser) ParseWithResult(ctx context.Context, filename string, daErr: fmt.Errorf("picture: unsupported extension %q (filename: %s); accepted: .jpg/.jpeg/.png/.gif/.bmp/.tiff/.tif/.webp/.svg/.ico/.avif/.heic/...", ext, filename),}}- if len(data) == 0 || len(data) > media.MaxImagePayloadBytes {- return ParseResult{Err: fmt.Errorf("picture: image payload size %d is outside the %d-byte limit", len(data), media.MaxImagePayloadBytes)}+ if len(data) == 0 || len(data) > media.ImageMaxBytes() {+ return ParseResult{Err: fmt.Errorf("picture: image payload size %d is outside the %d-byte limit", len(data), media.ImageMaxBytes())}}release, err := AcquireImageMedia(ctx)diff --git a/internal/parser/parser/pptx_ir.go b/internal/parser/parser/pptx_ir.goindex 677686932c..b2e6d80434 100644--- a/internal/parser/parser/pptx_ir.go+++ b/internal/parser/parser/pptx_ir.go@@ -22,18 +22,19 @@ import ("strings""ragflow/internal/entity"+ "ragflow/internal/parser/media")// buildPPTXJSONSections emits text runs, tables and images in slide order.// Every item retains its slide number; a blank slide emits an empty text item.// Container elements are traversed recursively so grouped tables remain structured.-func buildPPTXJSONSections(irJSON string, budget *embeddedMediaBudget) ([]map[string]any, error) {+func buildPPTXJSONSections(irJSON string, budget *media.Budget) ([]map[string]any, error) {var ir docxIRDocumentif err := json.Unmarshal([]byte(irJSON), &ir); err != nil {return nil, fmt.Errorf("presentation ir-json decode: %w", err)}if budget == nil {- budget = newEmbeddedMediaBudget()+ budget = media.NewBudget()}items := make([]map[string]any, 0, len(ir.Sections))for i, sec := range ir.Sections {diff --git a/internal/parser/parser/pptx_ir_test.go b/internal/parser/parser/pptx_ir_test.goindex 7c6240024f..c1d77599e9 100644--- a/internal/parser/parser/pptx_ir_test.go+++ b/internal/parser/parser/pptx_ir_test.go@@ -172,7 +172,7 @@ func TestBuildPPTXJSONSections(t *testing.T) {}for _, tt := range tests {t.Run(tt.name, func(t *testing.T) {- items, err := buildPPTXJSONSections(tt.irJSON, newEmbeddedMediaBudget())+ items, err := buildPPTXJSONSections(tt.irJSON, newMediaBudgetForTest())if tt.wantErr {if err == nil {t.Fatalf("buildPPTXJSONSections: want error, got items %+v", items)@@ -217,7 +217,7 @@ func TestBuildPPTXJSONSections_ExtractsSlideImagesInOrder(t *testing.T) {{"type":"table","rows":[{"cells":[{"content":[{"type":"image","data":"aW1hZ2U="}]}]}]}]},{"elements":[{"type":"image","data":"c2Vjb25k"}]}]}`- items, err := buildPPTXJSONSections(irJSON, newEmbeddedMediaBudget())+ items, err := buildPPTXJSONSections(irJSON, newMediaBudgetForTest())if err != nil {t.Fatalf("buildPPTXJSONSections: %v", err)}diff --git a/internal/parser/parser/pptx_parser.go b/internal/parser/parser/pptx_parser.goindex b050d87e36..a54d9d24fd 100644--- a/internal/parser/parser/pptx_parser.go+++ b/internal/parser/parser/pptx_parser.go@@ -22,6 +22,7 @@ import ("strings"deepdocoffice "ragflow/internal/deepdoc/parser/office"+ "ragflow/internal/parser/media")// pptxExtract is a test seam for the native office_oxide extraction. It@@ -151,7 +152,7 @@ func (p *PPTXParser) ParseWithResult(ctx context.Context, filename string, dataJSON: MapsToItems(itemsFromPlainText(plainText)),}}- budget := newEmbeddedMediaBudget()+ budget := media.NewBudget()items, err := buildPPTXJSONSections(irJSON, budget)if err != nil {return ParseResult{Err: err}@@ -174,6 +175,6 @@ func (p *PPTXParser) ParseWithResult(ctx context.Context, filename string, datareturn ParseResult{File: map[string]any{"name": filename, "format": effFormat},JSON: MapsToItems(items),- Warnings: budget.warnings(),+ Warnings: budget.Warnings(),}}diff --git a/internal/parser/parser/table_regression_test.go b/internal/parser/parser/table_regression_test.goindex 6320ea815a..2d7099c7e0 100644--- a/internal/parser/parser/table_regression_test.go+++ b/internal/parser/parser/table_regression_test.go@@ -105,8 +105,10 @@ func TestTableRegressionSoMarkTableImageIDs(t *testing.T) {}func TestTableRegressionSoMarkDocumentImageBudget(t *testing.T) {+ const limit = 256+ t.Setenv("RAGFLOW_MEDIA_MAX_IMAGES_PER_RUN", "256")markup := "| Field | Value |\n|---|---|\n| Logo |  |"- blocks := make([]any, maxEmbeddedMediaItems+1)+ blocks := make([]any, limit+1)for i := range blocks {blocks[i] = map[string]any{"type": "table", "content": markup}}@@ -117,8 +119,8 @@ func TestTableRegressionSoMarkDocumentImageBudget(t *testing.T) {images++}}- t.Logf("retained images=%d document item limit=%d", images, maxEmbeddedMediaItems)- if images != maxEmbeddedMediaItems || !strings.Contains(strings.Join(warnings, "\n"), "256-item") {+ t.Logf("retained images=%d run item limit=%d", images, limit)+ if images != limit || !strings.Contains(strings.Join(warnings, "\n"), "256-item") {t.Errorf("retained=%d warnings=%v, want document-wide item limit", images, warnings)}}@@ -135,19 +137,21 @@ func TestTableRegressionHTMLTheadWithTDRetainsDefaultHeader(t *testing.T) {}func TestTableRegressionMarkdownRawHTMLImageBudget(t *testing.T) {- source := "<table><tr><td>" + strings.Repeat("<img src=\"data:image/png;base64,aGVsbG8=\">", maxEmbeddedMediaItems+1) + "</td></tr></table>\n\n"+ const limit = 256+ t.Setenv("RAGFLOW_MEDIA_MAX_IMAGES_PER_RUN", "256")+ source := "<table><tr><td>" + strings.Repeat("<img src=\"data:image/png;base64,aGVsbG8=\">", limit+1) + "</td></tr></table>\n\n"doc := markdownNew().Parse([]byte(source))- budget := newEmbeddedMediaBudget()+ budget := newMediaBudgetForTest()var items []map[string]any- walkMarkdownBlocksWithImages(t.Context(), doc, &items, false, true, &htmlWalkState{mediaBudget: budget})+ walkMarkdownBlocksWithImages(t.Context(), doc, &items, false, true, &htmlWalkState{budget: budget})images := 0for _, item := range items {if item["doc_type_kwd"] == "image" && item["image"] != nil {images++}}- t.Logf("retained images=%d budget attempts=%d document item limit=%d", images, budget.items, maxEmbeddedMediaItems)- if images != maxEmbeddedMediaItems || budget.items != maxEmbeddedMediaItems || !budget.countExhausted {+ t.Logf("retained images=%d budget attempts=%d run item limit=%d", images, budget.Count(), limit)+ if images != limit || budget.Count() != limit || !strings.Contains(strings.Join(budget.Warnings(), "\n"), "256-item run limit") {t.Errorf("raw HTML budget not enforced: retained=%d budget=%+v", images, budget)}}diff --git a/internal/parser/parser/table_structure_test.go b/internal/parser/parser/table_structure_test.goindex b15e880430..35aee1af44 100644--- a/internal/parser/parser/table_structure_test.go+++ b/internal/parser/parser/table_structure_test.go@@ -96,11 +96,11 @@ func TestMarkdownTableImageTimeoutDoesNotConsumeBudget(t *testing.T) {ctx, cancel := context.WithCancel(t.Context())cancel()doc := markdownNew().Parse([]byte("| A | B |\n|---|---|\n| Logo |  |\n"))- budget := newEmbeddedMediaBudget()+ budget := newMediaBudgetForTest()var items []map[string]any- unresolved := walkMarkdownBlocksWithImages(ctx, doc, &items, false, true, &htmlWalkState{mediaBudget: budget})- if budget.items != 0 || unresolved != 0 {- t.Fatalf("timed out image consumed budget: attempts=%d unresolved=%d", budget.items, unresolved)+ unresolved := walkMarkdownBlocksWithImages(ctx, doc, &items, false, true, &htmlWalkState{budget: budget})+ if budget.Count() != 0 || unresolved != 0 {+ t.Fatalf("timed out image consumed budget: attempts=%d unresolved=%d", budget.Count(), unresolved)}}@@ -268,7 +268,7 @@ func TestOfficeTableRetainsDefaultHeaderCount(t *testing.T) {func TestOfficeTablePreservesLiteralCellText(t *testing.T) {el := docxIRElement{Type: "table", Rows: []docxIRRow{{Cells: []docxIRCell{{Content: []docxIRElement{{Type: "paragraph", Content: json.RawMessage(`[{"type":"text","text":" <b>A & B</b>\nsecond line "}]`)}}}}}}}- items := appendOfficeTableSections(nil, el, " A & <B> ", "table-1", newEmbeddedMediaBudget())+ items := appendOfficeTableSections(nil, el, " A & <B> ", "table-1", newMediaBudgetForTest())if len(items) != 1 {t.Fatalf("items=%#v", items)}diff --git a/internal/parser/parser/xls_parser.go b/internal/parser/parser/xls_parser.goindex 76c5dfcd8a..5820b3bce9 100644--- a/internal/parser/parser/xls_parser.go+++ b/internal/parser/parser/xls_parser.go@@ -24,6 +24,8 @@ import ("time"excel "github.com/jloor/go-excel-reader"++ "ragflow/internal/parser/media")// CompoundDocumentSignature is the OLE compound-document magic@@ -100,9 +102,9 @@ func (p *XLSParser) ParseWithResult(ctx context.Context, filename string, data [if bytes.HasPrefix(data, CompoundDocumentSignature) {items, sheetsCount, err = parseBIFF8Bytes(ctx, data)} else {- mediaBudget := newEmbeddedMediaBudget()+ mediaBudget := media.NewBudget()items, warnings, sheetsCount, err = parseXLSXBytes(data, mediaBudget)- warnings = append(warnings, mediaBudget.warnings()...)+ warnings = append(warnings, mediaBudget.Warnings()...)}if err := ctx.Err(); err != nil {return ParseResult{Err: err}diff --git a/internal/parser/parser/xls_parser_test.go b/internal/parser/parser/xls_parser_test.goindex a4c952f47e..4d8f017cdf 100644--- a/internal/parser/parser/xls_parser_test.go+++ b/internal/parser/parser/xls_parser_test.go@@ -322,13 +322,15 @@ func TestXLSParser_ImagesDoNotRunOCR(t *testing.T) {}func TestXLSParser_CapsEmbeddedImageCount(t *testing.T) {+ const limit = 256+ t.Setenv("RAGFLOW_MEDIA_MAX_IMAGES_PER_RUN", "256")var imageData bytes.Bufferif err := png.Encode(&imageData, image.NewRGBA(image.Rect(0, 0, 2, 2))); err != nil {t.Fatalf("encode fixture: %v", err)}f := excelize.NewFile()defer f.Close()- for i := 0; i <= maxEmbeddedMediaItems; i++ {+ for i := 0; i <= limit; i++ {cell, err := excelize.CoordinatesToCellName(3, i+1)if err != nil {t.Fatalf("cell %d: %v", i, err)@@ -361,10 +363,10 @@ func TestXLSParser_CapsEmbeddedImageCount(t *testing.T) {imageCount++}}- if imageCount != maxEmbeddedMediaItems {- t.Fatalf("image items = %d, want %d", imageCount, maxEmbeddedMediaItems)+ if imageCount != limit {+ t.Fatalf("image items = %d, want %d", imageCount, limit)}- if warnings := strings.Join(result.Warnings, "\n"); !strings.Contains(warnings, "stopped extracting embedded images after the 256-item document limit") {+ if warnings := strings.Join(result.Warnings, "\n"); !strings.Contains(warnings, "stopped extracting embedded images after the 256-item run limit") {t.Fatalf("warnings = %q, want image-count limit warning", warnings)}}diff --git a/internal/parser/parser/xlsx_parser.go b/internal/parser/parser/xlsx_parser.goindex c5fde577e1..ad8061654d 100644--- a/internal/parser/parser/xlsx_parser.go+++ b/internal/parser/parser/xlsx_parser.go@@ -23,6 +23,8 @@ import ("strings""github.com/xuri/excelize/v2"++ "ragflow/internal/parser/media")type XLSXParser struct {@@ -102,13 +104,13 @@ func (p *XLSXParser) ParseWithResult(ctx context.Context, filename string, data// for spreadsheet processing.}- mediaBudget := newEmbeddedMediaBudget()+ mediaBudget := media.NewBudget()items, warnings, sheets, err := parseXLSXBytes(data, mediaBudget)if err == nil {if ctx.Err() != nil {return ParseResult{Err: ctx.Err()}}- warnings = append(warnings, mediaBudget.warnings()...)+ warnings = append(warnings, mediaBudget.Warnings()...)return xlsxParseResult(filename, items, warnings, sheets)}if ctx.Err() != nil {@@ -124,7 +126,7 @@ func (p *XLSXParser) ParseWithResult(ctx context.Context, filename string, data}// The first parse attempt is discarded, so its media reservations must not// affect the normalized retry's output.- mediaBudget = newEmbeddedMediaBudget()+ mediaBudget = media.NewBudget()items, warnings, sheets, retryErr := parseXLSXBytes(normalized, mediaBudget)if retryErr != nil {return ParseResult{Err: fmt.Errorf("xlsx parse: %w; retry after normalization: %v", err, retryErr)}@@ -133,11 +135,11 @@ func (p *XLSXParser) ParseWithResult(ctx context.Context, filename string, datareturn ParseResult{Err: ctx.Err()}}warnings = append(normalizeWarnings, warnings...)- warnings = append(warnings, mediaBudget.warnings()...)+ warnings = append(warnings, mediaBudget.Warnings()...)return xlsxParseResult(filename, items, warnings, sheets)}-func parseXLSXBytes(data []byte, mediaBudget *embeddedMediaBudget) ([]map[string]any, []string, int, error) {+func parseXLSXBytes(data []byte, mediaBudget *media.Budget) ([]map[string]any, []string, int, error) {f, err := excelize.OpenReader(bytes.NewReader(data))if err != nil {return nil, nil, 0, fmt.Errorf("open XLSX: %w", err)diff --git a/internal/service/chunk/chunk.go b/internal/service/chunk/chunk.goindex b6025c5b93..4ad9518d21 100644--- a/internal/service/chunk/chunk.go+++ b/internal/service/chunk/chunk.go@@ -20,6 +20,7 @@ import ("bytes""context""encoding/base64"+ "errors""fmt""image""image/color"@@ -657,18 +658,12 @@ func (s *ChunkService) StopParsing(ctx context.Context, userID, datasetID stringreturn nil, common.CodeDataError, fmt.Errorf("you don't own the document %s", docID)}- task, err := dao.NewIngestionTaskDAO().GetByDocumentID(ctx, dao.DB, docID)- if err != nil {- return nil, common.CodeServerError, fmt.Errorf("get ingestion task for %s: %w", docID, err)- }- if task == nil || task.Status == common.COMPLETED ||- task.Status == common.STOPPED || task.Status == common.FAILED {- return &service.StopParsingResponse{- Data: map[string]interface{}{"error_code": docStopParsingInvalidStateErrorCode},- }, common.CodeDataError, fmt.Errorf("%s", docStopParsingInvalidStateMessage)- }-if err = s.cancelAllTasksOfDoc(ctx, doc); err != nil {+ if errors.Is(err, document.ErrParseNotRunning) {+ return &service.StopParsingResponse{+ Data: map[string]interface{}{"error_code": docStopParsingInvalidStateErrorCode},+ }, common.CodeDataError, fmt.Errorf("%s", docStopParsingInvalidStateMessage)+ }return nil, common.CodeServerError, err}// CancelDocParse (inside cancelAllTasksOfDoc) already issuesdiff --git a/internal/service/chunk/chunk_test.go b/internal/service/chunk/chunk_test.goindex 75d62fbe22..b59791a7bd 100644--- a/internal/service/chunk/chunk_test.go+++ b/internal/service/chunk/chunk_test.go@@ -301,7 +301,7 @@ func TestParseRejectsRunningDocument(t *testing.T) {if code != common.CodeDataError {t.Fatalf("expected CodeDataError, got %v", code)}- if !strings.Contains(err.Error(), "ingestion task is RUNNING") {+ if !strings.Contains(err.Error(), "ingestion task task-1 is RUNNING") {t.Fatalf("unexpected error: %v", err)}}@@ -2755,8 +2755,6 @@ func TestStopParsing_CallsCancelIngestionTask(t *testing.T) {pushChunkTestDB(t, db)insertChunkTestKB(t, "kb-1", "user-1") // owner = user-1, Accessible passesinsertChunkTestDoc(t, "doc-1", "kb-1")- // StopParsing checks the IngestionTask status.- insertChunkTestIngestionTask(t, "task-1", "user-1", "doc-1", "kb-1", common.RUNNING)svc := newParseTestService(t)var calledDocID string@@ -2783,7 +2781,10 @@ func TestStopParsing_RejectsDocumentWithoutIngestionTask(t *testing.T) {svc := newParseTestService(t)var called bool- svc.cancelIngestionTaskFunc = func(ctx context.Context, doc *entity.Document) error { called = true; return nil }+ svc.cancelIngestionTaskFunc = func(ctx context.Context, doc *entity.Document) error {+ called = true+ return document.ErrParseNotRunning+ }ctx := chunkTestContext(t, "user-1", "user-1")_, code, err := svc.StopParsing(ctx, "user-1", "kb-1", service.StopParsingRequest{DocumentIDs: []string{"doc-1"}})if err == nil {@@ -2792,8 +2793,8 @@ func TestStopParsing_RejectsDocumentWithoutIngestionTask(t *testing.T) {if code != common.CodeDataError {t.Fatalf("expected CodeDataError, got %v", code)}- if called {- t.Fatal("cancel should not be called for doc without ingestion task")+ if !called {+ t.Fatal("StopParsing must delegate task-state validation to CancelDocParse")}}diff --git a/internal/service/document/document_crud.go b/internal/service/document/document_crud.goindex ddd2546a91..3550fbbeaf 100644--- a/internal/service/document/document_crud.go+++ b/internal/service/document/document_crud.go@@ -280,6 +280,14 @@ func (s *DocumentService) deleteDocumentFull(ctx context.Context, docID string)if err != nil {return err}+ unlock := lockDocumentParse(docID)+ defer unlock()++ // Settle and delete the task before purging its cache state. A conditional+ // delete failure leaves the document and the newly active run intact.+ if err := s.cleanupDocumentTask(ctx, docID, taskTerminalMessageDocumentDeleted); err != nil {+ return err+ }// Release an uncommitted parse hold before the document disappears.if billingService := billing.Get(); billingService != nil && billingService.Enabled() {@@ -288,21 +296,6 @@ func (s *DocumentService) deleteDocumentFull(ctx context.Context, docID string)}}- // Delete tasks from DB- ingestionTask, err := s.ingestionTaskDAO.GetByDocumentID(ctx, dao.DB, docID)- if err != nil {- common.Error(fmt.Sprintf("failed to get ingestion task by doc:%s", doc.ID), err)- return err- }- if ingestionTask != nil {- if err := s.purgeTaskStateForCleanup(ctx, ingestionTask.ID); err != nil {- return err- }- if _, err := s.ingestionTaskSvc.Remove(ctx, ingestionTask.ID, &ingestionTask.UserID); err != nil {- return err- }- }-if err := s.deleteDocEngineData(ctx, docID, kb.TenantID, doc.KbID); err != nil {return err}@@ -328,21 +321,18 @@ func (s *DocumentService) RemoveDocumentKeepFile(ctx context.Context, docID striif err != nil {return err}+ unlock := lockDocumentParse(docID)+ defer unlock()+variants, taskTypes, typeErr := s.documentKnowledgeCompileTypes(ctx, kb.TenantID, kb.ID, docID)if typeErr != nil {return fmt.Errorf("resolve knowledge compile types for document %s: %w", docID, typeErr)}- ingestionTask, err := s.ingestionTaskDAO.GetByDocumentID(ctx, dao.DB, docID)- if err != nil {- return fmt.Errorf("failed to get ingestion task for %s: %w", docID, err)- }- if ingestionTask != nil {- if err := s.purgeTaskStateForCleanup(ctx, ingestionTask.ID); err != nil {- return err- }- if _, err := s.ingestionTaskSvc.Remove(ctx, ingestionTask.ID, nil); err != nil {- return fmt.Errorf("remove ingestion task for document %s: %w", docID, err)- }++ // Settle and delete the task before purging its cache state. Active workers+ // must be stopped before this path can run.+ if err := s.cleanupDocumentTask(ctx, docID, taskTerminalMessageDocumentRemoved); err != nil {+ return err}if _, delErr := s.taskDAO.DeleteByDocIDs(ctx, dao.DB, []string{docID}); delErr != nil {if errors.Is(delErr, context.Canceled) || errors.Is(delErr, context.DeadlineExceeded) {diff --git a/internal/service/document/document_ingest.go b/internal/service/document/document_ingest.goindex 02835fedf1..15d839c582 100644--- a/internal/service/document/document_ingest.go+++ b/internal/service/document/document_ingest.go@@ -135,7 +135,7 @@ func (s *DocumentService) Ingest(ctx context.Context, userID string, req *Ingestif run == string(entity.TaskStatusCancel) {if err = s.CancelDocParse(ctx, doc); err != nil {common.Error(fmt.Sprintf("go side, start to process %s, run is cancel", doc.ID), err)- if errors.Is(err, errParseNotRunning) {+ if errors.Is(err, ErrParseNotRunning) {// Mirror the Python /documents/ingest endpoint's message.return common.CodeDataError, errors.New("Cannot cancel a task that is not in RUNNING status")}@@ -150,20 +150,33 @@ func (s *DocumentService) Ingest(ctx context.Context, userID string, req *Ingestcontinue}- // Delete-only: user asked to remove prior parse results without- // starting a new parse. RUNNING already continued above.+ if req.Delete {+ // Delete-only: user asked to remove prior parse results without+ // starting a new parse. Serialize the counter reset and cleanup with+ // StartParseDocuments so a concurrent parse cannot reuse the task+ // while its old cache state is being removed.+ unlock := lockDocumentParse(doc.ID)+ if err = s.documentDAO.UpdateByID(ctx, dao.DB, doc.ID, map[string]interface{}{+ "progress": 0,+ }); err != nil {+ unlock()+ common.Error(fmt.Sprintf("go side, doc %s, UpdateByID failed", doc.ID), err)+ return common.CodeExceptionError, err+ }+ if err := s.clearDocumentParseResults(ctx, doc, kb.TenantID, taskTerminalMessageParseResultsDeleted); err != nil {+ unlock()+ return common.CodeExceptionError, err+ }+ unlock()+ continue+ }+if err = s.documentDAO.UpdateByID(ctx, dao.DB, doc.ID, map[string]interface{}{"progress": 0,}); err != nil {common.Error(fmt.Sprintf("go side, doc %s, UpdateByID failed", doc.ID), err)return common.CodeExceptionError, err}-- if req.Delete {- if err := s.clearDocumentParseResults(ctx, doc, kb.TenantID); err != nil {- return common.CodeExceptionError, err- }- }}return common.CodeSuccess, nildiff --git a/internal/service/document/document_parse.go b/internal/service/document/document_parse.goindex 3992f8b4a3..98d248a042 100644--- a/internal/service/document/document_parse.go+++ b/internal/service/document/document_parse.go@@ -34,7 +34,11 @@ var documentParseLocks = struct {}{locks: make(map[string]*documentParseLock)}const (- cleanupBatchTimeout = 30 * time.Second+ cleanupBatchTimeout = 30 * time.Second+ taskTerminalMessageSuperseded = "Task superseded by a new parse request."+ taskTerminalMessageDocumentDeleted = "Task cancelled by document deletion."+ taskTerminalMessageDocumentRemoved = "Task cancelled by document removal."+ taskTerminalMessageParseResultsDeleted = "Task cancelled by parse result deletion.")func lockDocumentParse(docID string) func() {@@ -59,18 +63,48 @@ func lockDocumentParse(docID string) func() {}}-// purgeTaskStateForCleanup removes the per-chunk cache state for a task before-// its row is deleted. The bounded context makes Redis outages and hung clients-// observable to the caller instead of allowing a destructive task/document-// delete to proceed with stale cache state.-func (s *DocumentService) purgeTaskStateForCleanup(ctx context.Context, taskID string) error {+// cleanupDocumentTask settles and conditionally deletes the document's task+// before purging its run-scoped cache state. If the task starts running between+// settlement and deletion, the conditional delete fails and no cache state is+// removed for the new run.+func (s *DocumentService) cleanupDocumentTask(ctx context.Context, docID, terminalMessage string) error {+ task, err := s.ingestionTaskSvc.PrepareDocumentTaskForCleanup(ctx, docID, terminalMessage)+ if err != nil {+ return fmt.Errorf("prepare ingestion task for document %s cleanup: %w", docID, err)+ }+ if err := s.deletePreparedDocumentTask(ctx, docID, task); err != nil {+ return err+ }+ if task == nil {+ return nil+ }+purgeTaskState := s.purgeTaskStateif purgeTaskState == nil {purgeTaskState = ingestionpipeline.PurgeTaskState}- batchCtx, cancel := context.WithTimeout(ctx, cleanupBatchTimeout)+ // The task row is gone, so a request cancellation must not skip cleanup that+ // is now retryable only through the 24-hour TTL.+ batchCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), cleanupBatchTimeout)defer cancel()- return purgeTaskState(batchCtx, taskID)+ if err := purgeTaskState(batchCtx, task.ID); err != nil {+ return fmt.Errorf("purge task state for document %s task %s: %w", docID, task.ID, err)+ }+ return nil+}++func (s *DocumentService) deletePreparedDocumentTask(ctx context.Context, docID string, task *entity.IngestionTask) error {+ if task == nil {+ return nil+ }+ deleted, err := s.ingestionTaskDAO.DeleteIfTerminal(ctx, dao.DB, task.ID)+ if err != nil {+ return fmt.Errorf("delete ingestion task %s for document %s: %w", task.ID, docID, err)+ }+ if deleted != 1 {+ return fmt.Errorf("document %s ingestion task %s started running; stop it before cleanup", docID, task.ID)+ }+ return nil}// StartParseDocuments starts parsing a document via the DSL ingestion@@ -95,7 +129,7 @@ func (s *DocumentService) StartParseDocuments(ctx context.Context, doc *entity.D}if opts.RerunWithDelete {- if err := s.clearDocumentParseResults(ctx, doc, kb.TenantID); err != nil {+ if err := s.clearDocumentParseResults(ctx, doc, kb.TenantID, taskTerminalMessageSuperseded); err != nil {return err}}@@ -285,60 +319,24 @@ func (s *DocumentService) AssertIngestionTasksTerminal(ctx context.Context, docIif err != nil {return fmt.Errorf("check ingestion task for %s: %w", docID, err)}- if task == nil {- continue- }- if common.IsRunningOrStopping(task.Status) {- return fmt.Errorf("document %s ingestion task is %s; stop it and wait for a terminal state before re-parsing", docID, task.Status)+ if task != nil && common.IsRunningOrStopping(task.Status) {+ return fmt.Errorf("document %s ingestion task %s is %s; stop it and wait for a terminal state before re-parsing", docID, task.ID, task.Status)}}return nil}-func (s *DocumentService) clearDocumentParseResults(ctx context.Context, doc *entity.Document, tenantID string) error {+func (s *DocumentService) clearDocumentParseResults(ctx context.Context, doc *entity.Document, tenantID, terminalMessage string) error {if doc == nil {return fmt.Errorf("document is nil")}- // Refuse to clear a non-terminal ingestion task. An in-flight worker- // (RUNNING) or one mid-stop (STOPPING) would keep writing chunks and- // corrupt the new run's results. The caller must stop the task first- // and wait for a terminal state (COMPLETED/STOPPED/FAILED), CREATED, or- // SCHEDULED.- task, err := s.ingestionTaskDAO.GetByDocumentID(ctx, dao.DB, doc.ID)- if err != nil {- return fmt.Errorf("get ingestion task for document %s: %w", doc.ID, err)- }- taskExisted := task != nil- if task != nil {- if task.Status == common.RUNNING || task.Status == common.STOPPING {- return fmt.Errorf("document %s ingestion task is %s; stop it and wait for a terminal state before re-parsing", doc.ID, task.Status)- }- if task.Status == common.CREATED || task.Status == common.SCHEDULED {- if err := s.ingestionTaskSvc.SupersedeUnstartedTask(ctx, task.ID); err != nil {- return fmt.Errorf("supersede queued ingestion task for document %s: %w", doc.ID, err)- }- task.Status = common.STOPPED- }- taskExisted = true- }- if task != nil {- if err := s.purgeTaskStateForCleanup(ctx, task.ID); err != nil {- return fmt.Errorf("purge task state for document %s: %w", doc.ID, err)- }- }-- // Delete terminal, CREATED, and SCHEDULED ingestion tasks atomically, leaving- // RUNNING/STOPPING tasks untouched so the check-then-delete window- // between GetByDocumentID and the delete cannot delete a task that just- // transitioned to RUNNING. In that case, do not clear its parse results.- deleted, err := s.ingestionTaskDAO.DeleteIfTerminal(ctx, dao.DB, doc.ID)- if err != nil {+ // Settle and conditionally delete the task before touching cache state or+ // durable results. A task that starts running after preparation blocks the+ // delete and leaves both the document and its results intact.+ if err := s.cleanupDocumentTask(ctx, doc.ID, terminalMessage); err != nil {return err}- if taskExisted && deleted == 0 {- return fmt.Errorf("document %s ingestion task started running; stop it before re-parsing", doc.ID)- }if err := s.clearDocumentAndKBCountersForRerun(doc.ID, doc.KbID); err != nil {return err}@@ -504,7 +502,7 @@ func (s *DocumentService) StopParseDocuments(ctx context.Context, datasetID strisuccessCount := 0for _, doc := range docs {if cancelErr := s.CancelDocParse(ctx, doc); cancelErr != nil {- if errors.Is(cancelErr, errParseNotRunning) {+ if errors.Is(cancelErr, ErrParseNotRunning) {// Mirror the Python /documents/stop endpoint's message.errs = append(errs, "Can't stop parsing document that has not started or already completed")} else {@@ -566,35 +564,34 @@ func (s *DocumentService) validateDocsInDataset(ctx context.Context, docIDs []streturn docs, nil}-// errParseNotRunning is returned by CancelDocParse when the document has no+// ErrParseNotRunning is returned by CancelDocParse when the document has no// in-flight ingestion task and is not already stopped. Callers map it to their// endpoint-specific message (the Python /documents/ingest and /documents/stop messages differ).-var errParseNotRunning = errors.New("parse task is not in running status")+var ErrParseNotRunning = errors.New("parse task is not in running status")-// CancelDocParse stops the ingestion task for the document by calling-// RequestStop (STOPPING).-// It returns errParseNotRunning if the document has neither an in-flight-// ingestion task (CREATED/SCHEDULED/RUNNING/STOPPING) nor an already stopped-// task (STOPPED).+// CancelDocParse stops the document's active ingestion task. It returns+// ErrParseNotRunning when the task is absent or already completed/failed.func (s *DocumentService) CancelDocParse(ctx context.Context, doc *entity.Document) error {task, err := s.ingestionTaskDAO.GetByDocumentID(ctx, dao.DB, doc.ID)if err != nil {return fmt.Errorf("failed to get ingestion task for %s: %w", doc.ID, err)}-- inFlight := task != nil && common.IsActiveTaskStatus(task.Status)- isStopped := task != nil && task.Status == common.STOPPED- if !inFlight && !isStopped {- return errParseNotRunning+ if task == nil {+ return ErrParseNotRunning}-- if inFlight {- if _, err = s.ingestionTaskSvc.RequestStop(ctx, task.ID); err != nil {+ switch {+ case common.IsActiveTaskStatus(task.Status):+ if _, err := s.ingestionTaskSvc.RequestStop(ctx, task.ID); err != nil {return fmt.Errorf("failed to stop ingestion task %s: %w", task.ID, err)}+ return nil+ case task.Status == common.STOPPED:+ return nil+ case task.Status == common.COMPLETED || task.Status == common.FAILED:+ return ErrParseNotRunning+ default:+ return fmt.Errorf("ingestion task %s has unrecognized status %q", task.ID, task.Status)}-- return nil}// deleteDocumentChunkImages removes source chunk image objects in boundeddiff --git a/internal/service/document/document_test.go b/internal/service/document/document_test.goindex a6de761893..3d361f6b04 100644--- a/internal/service/document/document_test.go+++ b/internal/service/document/document_test.go@@ -789,6 +789,29 @@ func insertTestIngestionTaskWithStatus(t *testing.T, id, userID, docID, datasetI}}+func insertTestOpenTaskRun(t *testing.T, taskID, logID, userID, docID, datasetID string) {+ t.Helper()+ insertTestIngestionTaskWithStatus(t, taskID, userID, docID, datasetID, common.CREATED)++ runCount := 1+ if err := dao.DB.Create(&entity.PipelineOperationLog{+ ID: logID,+ DocumentID: docID,+ TenantID: "tenant-1",+ KbID: datasetID,+ ParserID: "naive",+ TaskType: string(entity.PipelineTaskTypeParse),+ OperationStatus: string(entity.TaskStatusUnstart),+ RunCount: &runCount,+ }).Error; err != nil {+ t.Fatalf("insert open pipeline run: %v", err)+ }+ if err := dao.DB.Model(&entity.IngestionTask{}).Where("id = ?", taskID).+ Update("pipeline_log_id", logID).Error; err != nil {+ t.Fatalf("bind open pipeline run: %v", err)+ }+}+func insertTestFile2Document(t *testing.T, id, fileID, docID string) {t.Helper()f2d := &entity.File2Document{@@ -863,7 +886,7 @@ func TestDeleteDocumentFull_Basic(t *testing.T) {}}-func TestDeleteDocumentFullPurgesTaskStateBeforeDeletingTask(t *testing.T) {+func TestDeleteDocumentFullDeletesTaskBeforePurgingState(t *testing.T) {db := setupServiceTestDB(t)pushServiceDB(t, db)insertTestKB(t, "kb-1", "tenant-1", 1, 0, 0)@@ -879,11 +902,11 @@ func TestDeleteDocumentFullPurgesTaskStateBeforeDeletingTask(t *testing.T) {}if task, err := svc.ingestionTaskDAO.GetByDocumentID(t.Context(), db, "doc-1"); err != nil {t.Fatalf("reload ingestion task: %v", err)- } else if task == nil {- t.Fatal("ingestion task was deleted despite resumable state cleanup failure")+ } else if task != nil {+ t.Fatal("ingestion task was not deleted before state cleanup")}if _, err := svc.documentDAO.GetByID(t.Context(), db, "doc-1"); err != nil {- t.Fatalf("document was deleted despite resumable state cleanup failure: %v", err)+ t.Fatalf("document was deleted before state cleanup succeeded: %v", err)}}@@ -905,7 +928,7 @@ func TestDeleteDocumentFullKeepsDocumentWhenChunkDeletionFails(t *testing.T) {}}-func TestRemoveDocumentKeepFilePurgesTaskStateBeforeDeletingDocument(t *testing.T) {+func TestRemoveDocumentKeepFileDeletesTaskBeforePurgingState(t *testing.T) {db := setupServiceTestDB(t)pushServiceDB(t, db)insertTestKB(t, "kb-1", "tenant-1", 1, 0, 0)@@ -921,11 +944,133 @@ func TestRemoveDocumentKeepFilePurgesTaskStateBeforeDeletingDocument(t *testing.}if task, err := svc.ingestionTaskDAO.GetByDocumentID(t.Context(), db, "doc-1"); err != nil {t.Fatalf("reload ingestion task: %v", err)- } else if task == nil {- t.Fatal("ingestion task was deleted despite resumable state cleanup failure")+ } else if task != nil {+ t.Fatal("ingestion task was not deleted before state cleanup")}if _, err := svc.documentDAO.GetByID(t.Context(), db, "doc-1"); err != nil {- t.Fatalf("document was deleted despite resumable state cleanup failure: %v", err)+ t.Fatalf("document was deleted before state cleanup succeeded: %v", err)+ }+}++func TestDocumentCleanupRecordsCallerSpecificTaskCancellationReason(t *testing.T) {+ tests := []struct {+ name string+ run func(*DocumentService, *entity.Document) error+ want string+ }{+ {+ name: "delete document",+ run: func(svc *DocumentService, _ *entity.Document) error {+ return svc.deleteDocumentFull(t.Context(), "doc-1")+ },+ want: taskTerminalMessageDocumentDeleted,+ },+ {+ name: "remove document keep file",+ run: func(svc *DocumentService, _ *entity.Document) error {+ return svc.RemoveDocumentKeepFile(t.Context(), "doc-1")+ },+ want: taskTerminalMessageDocumentRemoved,+ },+ {+ name: "clear parse results",+ run: func(svc *DocumentService, doc *entity.Document) error {+ return svc.clearDocumentParseResults(t.Context(), doc, "tenant-1", taskTerminalMessageSuperseded)+ },+ want: taskTerminalMessageSuperseded,+ },+ }++ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ db := setupServiceTestDB(t)+ pushServiceDB(t, db)+ insertTestKB(t, "kb-1", "tenant-1", 1, 0, 0)+ insertTestDoc(t, "doc-1", "kb-1", 0, 0)+ insertTestOpenTaskRun(t, "task-1", "parse-run-1", "user-1", "doc-1", "kb-1")++ svc := testDocumentService(t)+ svc.purgeTaskState = func(context.Context, string) error { return nil }+ doc, err := svc.documentDAO.GetByID(t.Context(), db, "doc-1")+ if err != nil {+ t.Fatalf("load document: %v", err)+ }+ if err := tt.run(svc, doc); err != nil {+ t.Fatalf("%s: %v", tt.name, err)+ }++ var terminal entity.IngestionTaskLog+ if err := db.Where("pipeline_log_id = ? AND event_type = ?", "parse-run-1", dao.EventTypeTerminal).+ First(&terminal).Error; err != nil {+ t.Fatalf("load terminal event: %v", err)+ }+ if terminal.Message != tt.want {+ t.Fatalf("terminal message = %q, want %q", terminal.Message, tt.want)+ }+ })+ }+}++func TestDocumentCleanupRejectsActiveTask(t *testing.T) {+ for _, status := range []string{common.RUNNING, common.STOPPING} {+ t.Run(status, func(t *testing.T) {+ db := setupServiceTestDB(t)+ pushServiceDB(t, db)+ insertTestKB(t, "kb-1", "tenant-1", 1, 10, 5)+ insertTestDoc(t, "doc-1", "kb-1", 10, 5)+ insertTestIngestionTaskWithStatus(t, "task-active", "user-1", "doc-1", "kb-1", status)++ svc := testDocumentService(t)+ doc, err := svc.documentDAO.GetByID(t.Context(), db, "doc-1")+ if err != nil {+ t.Fatalf("load document: %v", err)+ }++ tests := []struct {+ name string+ run func() error+ }{+ {+ name: "delete document",+ run: func() error { return svc.deleteDocumentFull(t.Context(), "doc-1") },+ },+ {+ name: "remove document keep file",+ run: func() error { return svc.RemoveDocumentKeepFile(t.Context(), "doc-1") },+ },+ {+ name: "clear parse results",+ run: func() error {+ return svc.clearDocumentParseResults(t.Context(), doc, "tenant-1", taskTerminalMessageSuperseded)+ },+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ purged := map[string]bool{}+ svc.purgeTaskState = func(_ context.Context, taskID string) error {+ purged[taskID] = true+ return nil+ }+ if err := tt.run(); err == nil {+ t.Fatalf("cleanup succeeded with an active %s task", status)+ }+ if len(purged) != 0 {+ t.Fatalf("purged task state %v before rejecting an active task", purged)+ }+ if _, err := svc.ingestionTaskDAO.GetByID(t.Context(), db, "task-active"); err != nil {+ t.Fatalf("task was removed after rejected cleanup: %v", err)+ }+ stored, err := svc.documentDAO.GetByID(t.Context(), db, "doc-1")+ if err != nil {+ t.Fatalf("document was removed after rejected cleanup: %v", err)+ }+ if stored.TokenNum != 10 || stored.ChunkNum != 5 {+ t.Fatalf("document counters changed after rejected cleanup: token=%d chunk=%d", stored.TokenNum, stored.ChunkNum)+ }+ })+ }+ })}}@@ -3311,7 +3456,7 @@ func TestClearDocumentParseResultsClearsCountersTasksAndChunks(t *testing.T) {svc := testDocumentService(t)svc.docEngine = engine- if err = svc.clearDocumentParseResults(ctx, doc, "tenant-1"); err != nil {+ if err = svc.clearDocumentParseResults(ctx, doc, "tenant-1", taskTerminalMessageSuperseded); err != nil {t.Fatalf("clearDocumentParseResults failed: %v", err)}@@ -3353,7 +3498,7 @@ func TestClearDocumentParseResultsDeletesDocumentGeneratedChunks(t *testing.T) {t.Fatalf("load document: %v", err)}- if err := svc.clearDocumentParseResults(t.Context(), doc, "tenant-1"); err != nil {+ if err := svc.clearDocumentParseResults(t.Context(), doc, "tenant-1", taskTerminalMessageSuperseded); err != nil {t.Fatalf("clearDocumentParseResults failed: %v", err)}if !engine.generatedDeleted {@@ -3386,7 +3531,7 @@ func TestClearDocumentParseResultsDeletesSourceChunkImages(t *testing.T) {t.Fatalf("load document: %v", err)}- if err := svc.clearDocumentParseResults(t.Context(), doc, "tenant-1"); err != nil {+ if err := svc.clearDocumentParseResults(t.Context(), doc, "tenant-1", taskTerminalMessageSuperseded); err != nil {t.Fatalf("clearDocumentParseResults failed: %v", err)}if imageStore.ObjExist(t.Context(), "kb-1", "image-001") {@@ -3394,7 +3539,7 @@ func TestClearDocumentParseResultsDeletesSourceChunkImages(t *testing.T) {}}-func TestClearDocumentParseResultsPurgesTaskStateBeforeDeletingTask(t *testing.T) {+func TestClearDocumentParseResultsDeletesTaskBeforePurgingState(t *testing.T) {db := setupServiceTestDB(t)pushServiceDB(t, db)insertTestKB(t, "kb-1", "tenant-1", 0, 0, 0)@@ -3409,15 +3554,15 @@ func TestClearDocumentParseResultsPurgesTaskStateBeforeDeletingTask(t *testing.Tt.Fatalf("load document: %v", err)}- if err := svc.clearDocumentParseResults(t.Context(), doc, "tenant-1"); !errors.Is(err, purgeErr) {+ if err := svc.clearDocumentParseResults(t.Context(), doc, "tenant-1", taskTerminalMessageSuperseded); !errors.Is(err, purgeErr) {t.Fatalf("clearDocumentParseResults error = %v, want purge error", err)}remaining, err := svc.ingestionTaskDAO.GetByDocumentID(t.Context(), db, "doc-1")if err != nil {t.Fatalf("reload ingestion task: %v", err)}- if remaining == nil {- t.Fatal("task was deleted despite resumable state cleanup failure")+ if remaining != nil {+ t.Fatal("task was not deleted before state cleanup")}}@@ -3487,10 +3632,10 @@ func TestClearDocumentParseResultsIsIdempotentForStaleDocSnapshot(t *testing.T)}svc := testDocumentService(t)- if err = svc.clearDocumentParseResults(ctx, staleDoc, "tenant-1"); err != nil {+ if err = svc.clearDocumentParseResults(ctx, staleDoc, "tenant-1", taskTerminalMessageSuperseded); err != nil {t.Fatalf("first clearDocumentParseResults failed: %v", err)}- if err = svc.clearDocumentParseResults(ctx, staleDoc, "tenant-1"); err != nil {+ if err = svc.clearDocumentParseResults(ctx, staleDoc, "tenant-1", taskTerminalMessageSuperseded); err != nil {t.Fatalf("second clearDocumentParseResults failed: %v", err)}@@ -3523,7 +3668,7 @@ func TestClearDocumentParseResults_RejectsNonTerminalIngestionTask(t *testing.T)}svc := testDocumentService(t)- if err = svc.clearDocumentParseResults(ctx, doc, "tenant-1"); err == nil {+ if err = svc.clearDocumentParseResults(ctx, doc, "tenant-1", taskTerminalMessageSuperseded); err == nil {t.Fatalf("expected error for %s ingestion task, got nil", status)}// The non-terminal task must NOT be deleted.@@ -3560,13 +3705,24 @@ func TestClearDocumentParseResultsDoesNotClearResultsWhenTaskStartsRunning(t *teif err != nil {t.Fatalf("get document: %v", err)}- if err = testDocumentService(t).clearDocumentParseResults(ctx, doc, "tenant-1"); err == nil || !strings.Contains(err.Error(), "started running") {+ purgeCalls := 0+ svc := testDocumentService(t)+ svc.purgeTaskState = func(context.Context, string) error {+ purgeCalls+++ return nil+ }+ if err = svc.clearDocumentParseResults(ctx, doc, "tenant-1", taskTerminalMessageSuperseded); err == nil || !strings.Contains(err.Error(), "started running") {t.Fatalf("expected reparse cleanup to reject a task that started running, got %v", err)}+ if purgeCalls != 0 {+ t.Fatalf("purged task state %d time(s) after the conditional task delete failed", purgeCalls)+ }task, err := dao.NewIngestionTaskDAO().GetByID(ctx, db, "task-1")if err != nil {t.Fatalf("reload task: %v", err)}+ // The task won the race and remains active, so the conditional delete did+ // not remove it.if task.Status != common.RUNNING {t.Fatalf("task status = %q, want %q", task.Status, common.RUNNING)}@@ -3605,7 +3761,7 @@ func TestClearDocumentParseResults_DeletesTerminalIngestionTask(t *testing.T) {}svc := testDocumentService(t)- if err = svc.clearDocumentParseResults(ctx, doc, "tenant-1"); err != nil {+ if err = svc.clearDocumentParseResults(ctx, doc, "tenant-1", taskTerminalMessageSuperseded); err != nil {t.Fatalf("clearDocumentParseResults for %s task: %v", status, err)}task, _ := svc.ingestionTaskDAO.GetByDocumentID(ctx, db, "doc-1")@@ -5115,9 +5271,10 @@ func TestIngest_DeleteOnlyCleansTasks(t *testing.T) {insertUserTenantForAccessCheck(t, "user-1", "tenant-1")insertTestKB(t, "kb-1", "tenant-1", 0, 0, 0)insertTestDoc(t, "doc-1", "kb-1", 10, 5)- insertTestIngestionTaskWithStatus(t, "task-1", "user-1", "doc-1", "kb-1", common.COMPLETED)+ insertTestOpenTaskRun(t, "task-1", "parse-run-1", "user-1", "doc-1", "kb-1")svc := testDocumentService(t)+ svc.purgeTaskState = func(context.Context, string) error { return nil }ctx := documentTestContext(t.Context())code, err := svc.Ingest(ctx, "user-1", &IngestDocumentRequest{DocIDs: []string{"doc-1"},@@ -5139,6 +5296,15 @@ func TestIngest_DeleteOnlyCleansTasks(t *testing.T) {t.Fatal("expected task to be deleted in delete-only mode")}+ var terminal entity.IngestionTaskLog+ if err := db.Where("pipeline_log_id = ? AND event_type = ?", "parse-run-1", dao.EventTypeTerminal).+ First(&terminal).Error; err != nil {+ t.Fatalf("load terminal event: %v", err)+ }+ if terminal.Message != taskTerminalMessageParseResultsDeleted {+ t.Fatalf("terminal message = %q, want delete-only reason", terminal.Message)+ }+doc, err := dao.NewDocumentDAO().GetByID(ctx, db, "doc-1")if err != nil {t.Fatalf("query doc: %v", err)diff --git a/internal/service/ingestion_task_service.go b/internal/service/ingestion_task_service.goindex 8f87c600f3..c427daba90 100644--- a/internal/service/ingestion_task_service.go+++ b/internal/service/ingestion_task_service.go@@ -5,15 +5,18 @@ import ("errors""fmt""strings"+ "sync""time""ragflow/internal/common""ragflow/internal/dao""ragflow/internal/engine/kvrocks""ragflow/internal/entity"+ ingestionpipeline "ragflow/internal/ingestion/pipeline""go.uber.org/zap""gorm.io/gorm"+ "gorm.io/gorm/clause")// ingestionEventKind intentionally has an invalid zero value. The persisted@@ -29,6 +32,58 @@ const (ingestionEventSystem)+const (+ taskStateCleanupTimeout = 5 * time.Second+ taskStateCleanupCapacity = 1024+ taskTerminalMessageComplete = "Task completed."+ taskTerminalMessageFailed = "Task failed."+)++type taskStatePurgeRequest struct {+ ctx context.Context+ taskID string+ purgeTask func(context.Context, string) error+}++var taskStateCleanupQueue = struct {+ once sync.Once+ queue chan taskStatePurgeRequest+}{+ queue: make(chan taskStatePurgeRequest, taskStateCleanupCapacity),+}++func scheduleTaskStatePurge(req taskStatePurgeRequest) bool {+ taskStateCleanupQueue.once.Do(func() {+ go func() {+ for req := range taskStateCleanupQueue.queue {+ started := time.Now()+ ctx, cancel := context.WithTimeout(context.WithoutCancel(req.ctx), taskStateCleanupTimeout)+ err := req.purgeTask(ctx, req.taskID)+ cancel()+ elapsed := time.Since(started)+ if err != nil {+ common.Warn("purge task state after delete",+ zap.String("task_id", req.taskID),+ zap.Duration("elapsed", elapsed),+ zap.Error(err),+ )+ continue+ }+ common.Debug("purged task state after delete",+ zap.String("task_id", req.taskID),+ zap.Duration("elapsed", elapsed),+ )+ }+ }()+ })+ select {+ case taskStateCleanupQueue.queue <- req:+ return true+ default:+ return false+ }+}+type InvalidTaskTransitionError struct {TaskID stringFrom string@@ -70,6 +125,9 @@ type IngestionTaskService struct {userCanvasDAO *dao.UserCanvasDAOtaskPublisher TaskPublisherlogSettings IngestionLogSettings+ // purgeTaskState removes the deleted task's run-scoped chunk and media+ // cache state. Nil resolves to the process-wide implementation.+ purgeTaskState func(context.Context, string) error}func NewIngestionTaskService() *IngestionTaskService {@@ -272,23 +330,65 @@ func (s *IngestionTaskService) RequestStop(ctx context.Context, taskID string) (// created. A numbered pipeline log is an immutable run ledger, so this must// record its cancellation rather than deleting the row with the task.func (s *IngestionTaskService) SupersedeUnstartedTask(ctx context.Context, taskID string) error {+ _, err := s.supersedeUnstartedTask(ctx, taskID, "Task superseded by a new parse request.")+ return err+}++func (s *IngestionTaskService) supersedeUnstartedTask(ctx context.Context, taskID, terminalMessage string) (*entity.IngestionTask, error) {task, err := s.GetTask(ctx, taskID)if err != nil {- return err+ return nil, err}switch task.Status {case common.CREATED, common.SCHEDULED:stopped, err := s.transition(ctx, taskID, common.STOPPED)if err != nil {- return err+ return nil, err}- s.advanceOpenLog(ctx, stopped, dao.OpenPipelineOperationStatuses(), string(entity.TaskStatusCancel))- s.recordRunTerminal(ctx, stopped, "Task superseded by a new parse request.")- return nil+ if err := s.settleTaskRun(ctx, stopped, entity.TaskStatusCancel, terminalMessage); err != nil {+ return nil, err+ }+ return stopped, nilcase common.RUNNING, common.STOPPING:- return fmt.Errorf("task %s is %s and cannot be superseded", taskID, task.Status)+ return nil, fmt.Errorf("task %s is %s and cannot be superseded", taskID, task.Status)default:- return nil+ return task, nil+ }+}++// PrepareDocumentTaskForCleanup rejects an active task and freezes a queued+// task before destructive document cleanup. The caller deletes the returned+// terminal task after purging its run-scoped cache state.+func (s *IngestionTaskService) PrepareDocumentTaskForCleanup(ctx context.Context, documentID, terminalMessage string) (*entity.IngestionTask, error) {+ task, err := s.ingestionTaskDAO.GetByDocumentID(ctx, dao.DB, documentID)+ if err != nil {+ return nil, err+ }+ if task == nil {+ return nil, nil+ }+ switch task.Status {+ case common.CREATED, common.SCHEDULED:+ return s.supersedeUnstartedTask(ctx, task.ID, terminalMessage)+ case common.RUNNING, common.STOPPING:+ return nil, fmt.Errorf("document %s ingestion task %s is %s; stop it and wait for a terminal state before cleanup", documentID, task.ID, task.Status)+ case common.STOPPED:+ if err := s.settleTaskRun(ctx, task, entity.TaskStatusCancel, terminalMessage); err != nil {+ return nil, err+ }+ return task, nil+ case common.COMPLETED:+ if err := s.settleTaskRun(ctx, task, entity.TaskStatusDone, taskTerminalMessageComplete); err != nil {+ return nil, err+ }+ return task, nil+ case common.FAILED:+ if err := s.settleTaskRun(ctx, task, entity.TaskStatusFail, taskTerminalMessageFailed); err != nil {+ return nil, err+ }+ return task, nil+ default:+ return nil, fmt.Errorf("document %s ingestion task %s has unrecognized status %q", documentID, task.ID, task.Status)}}@@ -355,10 +455,40 @@ func (s *IngestionTaskService) Remove(ctx context.Context, taskID string, userID}}}+ // A queued task becomes STOPPED before its run is settled. If settlement+ // failed, a retry must finish it before the task row is deleted.+ if task.Status == common.STOPPED && task.PipelineLogID != nil && *task.PipelineLogID != "" {+ run, runErr := s.pipelineLogDAO.GetByID(ctx, dao.DB, *task.PipelineLogID)+ if runErr != nil {+ return nil, runErr+ }+ if run.RunCount != nil && *run.RunCount > 0 {+ if err := s.settleTaskRun(ctx, task, entity.TaskStatusCancel, "Task superseded by a new parse request."); err != nil {+ return nil, err+ }+ }+ }info, err := s.ingestionTaskDAO.Delete(ctx, dao.DB, taskID, userID)if err != nil {return nil, err}+ // The row is gone, so purge failures cannot be retried by the caller and+ // TTL remains the backstop. Keep backend I/O off the request path; RemoveMany+ // may otherwise accumulate one slow Kvrocks round-trip per deleted task.+ purgeTaskState := s.purgeTaskState+ if purgeTaskState == nil {+ purgeTaskState = ingestionpipeline.PurgeTaskState+ }+ if !scheduleTaskStatePurge(taskStatePurgeRequest{+ ctx: ctx,+ taskID: taskID,+ purgeTask: purgeTaskState,+ }) {+ common.Warn("task-state cleanup queue is full",+ zap.String("task_id", taskID),+ zap.Int("capacity", taskStateCleanupCapacity),+ )+ }return info, nil}@@ -768,12 +898,17 @@ func isTerminalIngestionTask(status string) bool {return status == common.COMPLETED || status == common.STOPPED || status == common.FAILED}-// advanceOpenLog moves a task's already-bound row to operationStatus. It never-// creates or adopts a row, so a worker cannot repair a missing identity or-// affect a newer run.+// advanceOpenLog performs a best-effort ledger advance for non-cleanup flows.func (s *IngestionTaskService) advanceOpenLog(ctx context.Context, task *entity.IngestionTask, fromStatuses []string, operationStatus string) {+ _, _ = s.advanceOpenLogIfOpen(ctx, task, fromStatuses, operationStatus)+}++// advanceOpenLogIfOpen moves a task's already-bound row to operationStatus and+// reports whether this call changed it. It never creates or adopts a row, so a+// worker cannot repair a missing identity or affect a newer run.+func (s *IngestionTaskService) advanceOpenLogIfOpen(ctx context.Context, task *entity.IngestionTask, fromStatuses []string, operationStatus string) (bool, error) {if task == nil || task.DocumentID == "" || s.pipelineLogDAO == nil {- return+ return false, nil}logID := ""if task.PipelineLogID != nil {@@ -781,11 +916,29 @@ func (s *IngestionTaskService) advanceOpenLog(ctx context.Context, task *entity.}if logID == "" {common.Warn(fmt.Sprintf("advance open pipeline log for task %s: missing run identity", task.ID))- return+ return false, nil}- if err := s.pipelineLogDAO.AdvanceOpenLog(ctx, dao.DB, logID, fromStatuses, operationStatus); err != nil {+ advanced, err := s.pipelineLogDAO.AdvanceOpenLog(ctx, dao.DB, logID, fromStatuses, operationStatus)+ if err != nil {common.Warn(fmt.Sprintf("advance open pipeline log for document %s to %s: %v", task.DocumentID, operationStatus, err))}+ return advanced, err+}++// settleTaskRun closes a terminal task's run ledger and ensures its terminal+// event exists. The event write is retried even when a previous attempt already+// advanced the operation row, so a partial settlement never becomes permanent.+func (s *IngestionTaskService) settleTaskRun(ctx context.Context, task *entity.IngestionTask, operationStatus entity.TaskStatus, terminalMessage string) error {+ if task == nil || task.PipelineLogID == nil || *task.PipelineLogID == "" {+ return nil+ }+ if _, err := s.advanceOpenLogIfOpen(ctx, task, dao.OpenPipelineOperationStatuses(), string(operationStatus)); err != nil {+ return err+ }+ if err := s.RecordTerminal(ctx, *task.PipelineLogID, task.ID, terminalMessage); err != nil {+ return fmt.Errorf("record terminal event for task %s run %s: %w", task.ID, *task.PipelineLogID, err)+ }+ return nil}func (s *IngestionTaskService) recordRunMessage(ctx context.Context, task *entity.IngestionTask, message string) {@@ -891,7 +1044,7 @@ func (s *IngestionTaskService) RecordMessage(ctx context.Context, pipelineLogID,// RecordTerminal writes the terminal explanation associated with a run.func (s *IngestionTaskService) RecordTerminal(ctx context.Context, pipelineLogID, taskID, message string) error {- if err := s.insertEvent(ctx, ingestionEventTerminal, pipelineLogID, taskID, "", 0, message); err != nil {+ if err := s.ensureTerminalEvent(ctx, pipelineLogID, taskID, message); err != nil {return err}foldContext := context.WithoutCancel(ctx)@@ -916,6 +1069,32 @@ func (s *IngestionTaskService) RecordTerminal(ctx context.Context, pipelineLogIDreturn nil}+// ensureTerminalEvent serializes terminal writers on the run row and inserts+// the terminal event only when one is not already present. This makes retrying+// settlement safe after the operation status has already advanced.+func (s *IngestionTaskService) ensureTerminalEvent(ctx context.Context, pipelineLogID, taskID, message string) error {+ if dao.DB == nil {+ return errors.New("ingestion task: nil database")+ }+ return dao.DB.WithContext(ctx).Transaction(func(tx *gorm.DB) error {+ var run entity.PipelineOperationLog+ if err := tx.Clauses(clause.Locking{Strength: "UPDATE"}).+ Take(&run, "id = ?", pipelineLogID).Error; err != nil {+ return err+ }+ var count int64+ if err := tx.Model(&entity.IngestionTaskLog{}).+ Where("pipeline_log_id = ? AND event_type = ?", pipelineLogID, dao.EventTypeTerminal).+ Count(&count).Error; err != nil {+ return err+ }+ if count > 0 {+ return nil+ }+ return s.insertEventOnDB(ctx, tx, ingestionEventTerminal, pipelineLogID, taskID, "", 0, message)+ })+}+func rejectIngestionEvent(reason, pipelineLogID, taskID string, err error) error {common.Warn("ingestion_log_event_rejected",zap.String("event", "ingestion_log_event_rejected"),@@ -927,12 +1106,19 @@ func rejectIngestionEvent(reason, pipelineLogID, taskID string, err error) error}func (s *IngestionTaskService) insertEvent(ctx context.Context, kind ingestionEventKind, pipelineLogID, taskID, component string, phase int, message string) error {+ return s.insertEventOnDB(ctx, dao.DB, kind, pipelineLogID, taskID, component, phase, message)+}++func (s *IngestionTaskService) insertEventOnDB(ctx context.Context, db *gorm.DB, kind ingestionEventKind, pipelineLogID, taskID, component string, phase int, message string) error {if pipelineLogID == "" {return rejectIngestionEvent("missing_pipeline_log_id", pipelineLogID, taskID, errors.New("ingestion event requires a pipeline log id"))}if taskID == "" {return rejectIngestionEvent("missing_task_id", pipelineLogID, taskID, errors.New("ingestion event requires a task id"))}+ if db == nil {+ return errors.New("ingestion task: nil database")+ }eventType := 0switch kind {case ingestionEventLifecycle:@@ -955,7 +1141,7 @@ func (s *IngestionTaskService) insertEvent(ctx context.Context, kind ingestionEv// EventTypeLifecycle is zero while the database default is the defensive// legacy value. A map keeps the explicitly mapped protocol value intact;// GORM otherwise substitutes a default-tag value for a zero struct field.- return dao.DB.WithContext(ctx).Model(&entity.IngestionTaskLog{}).Create(map[string]interface{}{+ return db.WithContext(ctx).Model(&entity.IngestionTaskLog{}).Create(map[string]interface{}{"task_id": taskID,"pipeline_log_id": pipelineLogID,"checkpoint": entity.JSONMap{},diff --git a/internal/service/ingestion_task_service_test.go b/internal/service/ingestion_task_service_test.goindex 2fee206a6d..0f07d18771 100644--- a/internal/service/ingestion_task_service_test.go+++ b/internal/service/ingestion_task_service_test.go@@ -7,6 +7,7 @@ import ("strings""sync""testing"+ "time""unicode/utf8""gorm.io/gorm"@@ -1072,6 +1073,485 @@ func TestIngestionTaskServiceRemoveSettlesNumberedQueuedRun(t *testing.T) {}}+func TestIngestionTaskServiceRemoveRetriesStoppedRunSettlement(t *testing.T) {+ db := setupServiceTestDB(t)+ pushServiceDB(t, db)+ insertTestKB(t, "kb-1", "tenant-1", 1, 0, 0)+ insertTestDoc(t, "doc-1", "kb-1", 0, 0)+ insertTestIngestionTaskWithStatus(t, "task-1", "user-1", "doc-1", "kb-1", common.CREATED)++ runCount := 1+ if err := db.Create(&entity.PipelineOperationLog{+ ID: "remove-open-log",+ DocumentID: "doc-1",+ TenantID: "tenant-1",+ KbID: "kb-1",+ ParserID: "naive",+ TaskType: string(entity.PipelineTaskTypeParse),+ OperationStatus: string(entity.TaskStatusUnstart),+ RunCount: &runCount,+ }).Error; err != nil {+ t.Fatalf("seed numbered pipeline log: %v", err)+ }+ if err := db.Model(&entity.IngestionTask{}).Where("id = ?", "task-1").+ Update("pipeline_log_id", "remove-open-log").Error; err != nil {+ t.Fatalf("bind numbered pipeline log: %v", err)+ }++ settlementErr := errors.New("pipeline ledger unavailable")+ const callbackName = "test:fail-remove-ledger-settlement"+ if err := db.Callback().Update().Before("gorm:update").Register(callbackName, func(tx *gorm.DB) {+ if tx.Statement.Table == (&entity.PipelineOperationLog{}).TableName() {+ _ = tx.AddError(settlementErr)+ }+ }); err != nil {+ t.Fatalf("register update failure: %v", err)+ }++ svc := NewIngestionTaskService()+ if _, err := svc.Remove(t.Context(), "task-1", sptr("user-1")); !errors.Is(err, settlementErr) {+ t.Fatalf("first Remove error = %v, want settlement failure", err)+ }+ task, err := svc.ingestionTaskDAO.GetByID(t.Context(), db, "task-1")+ if err != nil {+ t.Fatalf("reload task after failed settlement: %v", err)+ }+ if task.Status != common.STOPPED {+ t.Fatalf("task status = %q, want %q", task.Status, common.STOPPED)+ }+ var run entity.PipelineOperationLog+ if err := db.First(&run, "id = ?", "remove-open-log").Error; err != nil {+ t.Fatalf("load pipeline log after failed settlement: %v", err)+ }+ if run.OperationStatus != string(entity.TaskStatusUnstart) {+ t.Fatalf("pipeline run status = %q, want %q", run.OperationStatus, entity.TaskStatusUnstart)+ }++ if err := db.Callback().Update().Remove(callbackName); err != nil {+ t.Fatalf("remove update failure: %v", err)+ }+ if _, err := svc.Remove(t.Context(), "task-1", sptr("user-1")); err != nil {+ t.Fatalf("retry Remove: %v", err)+ }+ if _, err := svc.ingestionTaskDAO.GetByID(t.Context(), db, "task-1"); !errors.Is(err, gorm.ErrRecordNotFound) {+ t.Fatalf("deleted task reload error = %v, want record not found", err)+ }+ if err := db.First(&run, "id = ?", "remove-open-log").Error; err != nil {+ t.Fatalf("load pipeline log after retry: %v", err)+ }+ if run.OperationStatus != string(entity.TaskStatusCancel) {+ t.Fatalf("pipeline run status after retry = %q, want %q", run.OperationStatus, entity.TaskStatusCancel)+ }+ var terminal entity.IngestionTaskLog+ if err := db.Where("pipeline_log_id = ? AND event_type = ?", "remove-open-log", dao.EventTypeTerminal).+ First(&terminal).Error; err != nil {+ t.Fatalf("load terminal event after retry: %v", err)+ }+ if terminal.Message != "Task superseded by a new parse request." {+ t.Fatalf("terminal message after retry = %q, want supersede reason", terminal.Message)+ }+}++func TestIngestionTaskServicePrepareDocumentTaskForCleanupRecordsCleanupReason(t *testing.T) {+ db := setupServiceTestDB(t)+ pushServiceDB(t, db)+ insertTestKB(t, "kb-1", "tenant-1", 1, 0, 0)+ insertTestDoc(t, "doc-1", "kb-1", 0, 0)+ insertTestIngestionTaskWithStatus(t, "task-1", "user-1", "doc-1", "kb-1", common.CREATED)++ runCount := 1+ if err := db.Create(&entity.PipelineOperationLog{+ ID: "cleanup-open-log",+ DocumentID: "doc-1",+ TenantID: "tenant-1",+ KbID: "kb-1",+ ParserID: "naive",+ TaskType: string(entity.PipelineTaskTypeParse),+ OperationStatus: string(entity.TaskStatusUnstart),+ RunCount: &runCount,+ }).Error; err != nil {+ t.Fatalf("seed numbered pipeline log: %v", err)+ }+ if err := db.Model(&entity.IngestionTask{}).Where("id = ?", "task-1").+ Update("pipeline_log_id", "cleanup-open-log").Error; err != nil {+ t.Fatalf("bind numbered pipeline log: %v", err)+ }++ task, err := NewIngestionTaskService().PrepareDocumentTaskForCleanup(+ t.Context(),+ "doc-1",+ "Task cancelled by document cleanup.",+ )+ if err != nil {+ t.Fatalf("PrepareDocumentTaskForCleanup failed: %v", err)+ }+ if task == nil || task.Status != common.STOPPED {+ t.Fatalf("cleanup task = %+v, want one STOPPED task", task)+ }++ var run entity.PipelineOperationLog+ if err := db.First(&run, "id = ?", "cleanup-open-log").Error; err != nil {+ t.Fatalf("load pipeline log: %v", err)+ }+ if run.OperationStatus != string(entity.TaskStatusCancel) {+ t.Fatalf("pipeline run status = %q, want %q", run.OperationStatus, entity.TaskStatusCancel)+ }++ var terminal entity.IngestionTaskLog+ if err := db.Where("pipeline_log_id = ? AND event_type = ?", "cleanup-open-log", dao.EventTypeTerminal).+ First(&terminal).Error; err != nil {+ t.Fatalf("load terminal event: %v", err)+ }+ if terminal.Message != "Task cancelled by document cleanup." {+ t.Fatalf("terminal message = %q, want cleanup reason", terminal.Message)+ }+}++func TestIngestionTaskServicePrepareDocumentTaskForCleanupRetriesOpenLedgerSettlement(t *testing.T) {+ db := setupServiceTestDB(t)+ pushServiceDB(t, db)+ insertTestKB(t, "kb-1", "tenant-1", 1, 0, 0)+ insertTestDoc(t, "doc-1", "kb-1", 0, 0)+ insertTestIngestionTaskWithStatus(t, "task-1", "user-1", "doc-1", "kb-1", common.CREATED)++ runCount := 1+ if err := db.Create(&entity.PipelineOperationLog{+ ID: "cleanup-open-log",+ DocumentID: "doc-1",+ TenantID: "tenant-1",+ KbID: "kb-1",+ ParserID: "naive",+ TaskType: string(entity.PipelineTaskTypeParse),+ OperationStatus: string(entity.TaskStatusUnstart),+ RunCount: &runCount,+ }).Error; err != nil {+ t.Fatalf("seed numbered pipeline log: %v", err)+ }+ if err := db.Model(&entity.IngestionTask{}).Where("id = ?", "task-1").+ Update("pipeline_log_id", "cleanup-open-log").Error; err != nil {+ t.Fatalf("bind numbered pipeline log: %v", err)+ }++ settlementErr := errors.New("pipeline ledger unavailable")+ const callbackName = "test:fail-cleanup-ledger-settlement"+ if err := db.Callback().Update().Before("gorm:update").Register(callbackName, func(tx *gorm.DB) {+ if tx.Statement.Table == (&entity.PipelineOperationLog{}).TableName() {+ _ = tx.AddError(settlementErr)+ }+ }); err != nil {+ t.Fatalf("register update failure: %v", err)+ }++ svc := NewIngestionTaskService()+ if _, err := svc.PrepareDocumentTaskForCleanup(+ t.Context(),+ "doc-1",+ "Task cancelled by document cleanup.",+ ); !errors.Is(err, settlementErr) {+ t.Fatalf("first cleanup error = %v, want settlement failure", err)+ }++ task, err := svc.ingestionTaskDAO.GetByID(t.Context(), db, "task-1")+ if err != nil {+ t.Fatalf("reload task after failed settlement: %v", err)+ }+ if task.Status != common.STOPPED {+ t.Fatalf("task status = %q, want %q", task.Status, common.STOPPED)+ }+ var run entity.PipelineOperationLog+ if err := db.First(&run, "id = ?", "cleanup-open-log").Error; err != nil {+ t.Fatalf("load pipeline log after failed settlement: %v", err)+ }+ if run.OperationStatus != string(entity.TaskStatusUnstart) {+ t.Fatalf("pipeline run status = %q, want %q", run.OperationStatus, entity.TaskStatusUnstart)+ }+ var terminalCount int64+ if err := db.Model(&entity.IngestionTaskLog{}).+ Where("pipeline_log_id = ? AND event_type = ?", "cleanup-open-log", dao.EventTypeTerminal).+ Count(&terminalCount).Error; err != nil {+ t.Fatalf("count terminal events after failed settlement: %v", err)+ }+ if terminalCount != 0 {+ t.Fatalf("terminal events after failed settlement = %d, want 0", terminalCount)+ }++ if err := db.Callback().Update().Remove(callbackName); err != nil {+ t.Fatalf("remove update failure: %v", err)+ }+ task, err = svc.PrepareDocumentTaskForCleanup(+ t.Context(),+ "doc-1",+ "Task cancelled by document cleanup.",+ )+ if err != nil {+ t.Fatalf("retry PrepareDocumentTaskForCleanup: %v", err)+ }+ if task == nil || task.Status != common.STOPPED {+ t.Fatalf("retried cleanup task = %+v, want one STOPPED task", task)+ }+ if err := db.First(&run, "id = ?", "cleanup-open-log").Error; err != nil {+ t.Fatalf("load pipeline log after retry: %v", err)+ }+ if run.OperationStatus != string(entity.TaskStatusCancel) {+ t.Fatalf("pipeline run status after retry = %q, want %q", run.OperationStatus, entity.TaskStatusCancel)+ }+ var terminal entity.IngestionTaskLog+ if err := db.Where("pipeline_log_id = ? AND event_type = ?", "cleanup-open-log", dao.EventTypeTerminal).+ First(&terminal).Error; err != nil {+ t.Fatalf("load terminal event after retry: %v", err)+ }+ if terminal.Message != "Task cancelled by document cleanup." {+ t.Fatalf("terminal message after retry = %q, want cleanup reason", terminal.Message)+ }+ if err := db.Model(&entity.IngestionTaskLog{}).+ Where("pipeline_log_id = ? AND event_type = ?", "cleanup-open-log", dao.EventTypeTerminal).+ Count(&terminalCount).Error; err != nil {+ t.Fatalf("count terminal events after retry: %v", err)+ }+ if terminalCount != 1 {+ t.Fatalf("terminal events after retry = %d, want 1", terminalCount)+ }+}++func TestIngestionTaskServicePrepareDocumentTaskForCleanupSettlesTerminalTaskLedger(t *testing.T) {+ tests := []struct {+ name string+ taskStatus string+ operationStatus entity.TaskStatus+ wantMessage string+ }{+ {+ name: "completed",+ taskStatus: common.COMPLETED,+ operationStatus: entity.TaskStatusDone,+ wantMessage: "Task completed.",+ },+ {+ name: "failed",+ taskStatus: common.FAILED,+ operationStatus: entity.TaskStatusFail,+ wantMessage: "Task failed.",+ },+ }+ for _, tc := range tests {+ t.Run(tc.name, func(t *testing.T) {+ db := setupServiceTestDB(t)+ pushServiceDB(t, db)+ insertTestKB(t, "kb-1", "tenant-1", 1, 0, 0)+ insertTestDoc(t, "doc-1", "kb-1", 0, 0)+ insertTestIngestionTaskWithStatus(t, "task-1", "user-1", "doc-1", "kb-1", tc.taskStatus)++ runCount := 1+ run := &entity.PipelineOperationLog{+ ID: "cleanup-terminal-log",+ DocumentID: "doc-1",+ TenantID: "tenant-1",+ KbID: "kb-1",+ ParserID: "naive",+ TaskType: string(entity.PipelineTaskTypeParse),+ OperationStatus: string(entity.TaskStatusRunning),+ RunCount: &runCount,+ }+ if err := db.Create(run).Error; err != nil {+ t.Fatalf("seed numbered pipeline log: %v", err)+ }+ if err := db.Model(&entity.IngestionTask{}).Where("id = ?", "task-1").+ Update("pipeline_log_id", run.ID).Error; err != nil {+ t.Fatalf("bind numbered pipeline log: %v", err)+ }++ task, err := NewIngestionTaskService().PrepareDocumentTaskForCleanup(+ t.Context(),+ "doc-1",+ "Task cancelled by document cleanup.",+ )+ if err != nil {+ t.Fatalf("PrepareDocumentTaskForCleanup failed: %v", err)+ }+ if task == nil || task.Status != tc.taskStatus {+ t.Fatalf("cleanup task = %+v, want status %q", task, tc.taskStatus)+ }+ if err := db.First(run, "id = ?", run.ID).Error; err != nil {+ t.Fatalf("load pipeline log: %v", err)+ }+ if run.OperationStatus != string(tc.operationStatus) {+ t.Fatalf("pipeline run status = %q, want %q", run.OperationStatus, tc.operationStatus)+ }+ var terminal entity.IngestionTaskLog+ if err := db.Where("pipeline_log_id = ? AND event_type = ?", run.ID, dao.EventTypeTerminal).+ First(&terminal).Error; err != nil {+ t.Fatalf("load terminal event: %v", err)+ }+ if terminal.Message != tc.wantMessage {+ t.Fatalf("terminal message = %q, want %q", terminal.Message, tc.wantMessage)+ }+ })+ }+}++func TestIngestionTaskServicePrepareDocumentTaskForCleanupRetriesMissingTerminalEvent(t *testing.T) {+ db := setupServiceTestDB(t)+ pushServiceDB(t, db)+ insertTestKB(t, "kb-1", "tenant-1", 1, 0, 0)+ insertTestDoc(t, "doc-1", "kb-1", 0, 0)+ insertTestIngestionTaskWithStatus(t, "task-1", "user-1", "doc-1", "kb-1", common.COMPLETED)++ runCount := 1+ run := &entity.PipelineOperationLog{+ ID: "cleanup-event-log",+ DocumentID: "doc-1",+ TenantID: "tenant-1",+ KbID: "kb-1",+ ParserID: "naive",+ TaskType: string(entity.PipelineTaskTypeParse),+ OperationStatus: string(entity.TaskStatusDone),+ RunCount: &runCount,+ }+ if err := db.Create(run).Error; err != nil {+ t.Fatalf("seed numbered pipeline log: %v", err)+ }+ if err := db.Model(&entity.IngestionTask{}).Where("id = ?", "task-1").+ Update("pipeline_log_id", run.ID).Error; err != nil {+ t.Fatalf("bind numbered pipeline log: %v", err)+ }++ insertErr := errors.New("terminal event unavailable")+ failInsert := true+ const callbackName = "test:fail-cleanup-terminal-event"+ if err := db.Callback().Create().Before("gorm:create").Register(callbackName, func(tx *gorm.DB) {+ if !failInsert || tx.Statement.Table != (&entity.IngestionTaskLog{}).TableName() {+ return+ }+ eventType, ok := tx.Statement.Dest.(map[string]interface{})["event_type"].(int)+ if ok && eventType == dao.EventTypeTerminal {+ _ = tx.AddError(insertErr)+ }+ }); err != nil {+ t.Fatalf("register terminal insert failure: %v", err)+ }++ svc := NewIngestionTaskService()+ if _, err := svc.PrepareDocumentTaskForCleanup(+ t.Context(),+ "doc-1",+ "Task cancelled by document cleanup.",+ ); !errors.Is(err, insertErr) {+ t.Fatalf("first cleanup error = %v, want terminal insert failure", err)+ }+ var terminalCount int64+ if err := db.Model(&entity.IngestionTaskLog{}).+ Where("pipeline_log_id = ? AND event_type = ?", run.ID, dao.EventTypeTerminal).+ Count(&terminalCount).Error; err != nil {+ t.Fatalf("count terminal events after failed insert: %v", err)+ }+ if terminalCount != 0 {+ t.Fatalf("terminal events after failed insert = %d, want 0", terminalCount)+ }++ failInsert = false+ if _, err := svc.PrepareDocumentTaskForCleanup(+ t.Context(),+ "doc-1",+ "Task cancelled by document cleanup.",+ ); err != nil {+ t.Fatalf("retry PrepareDocumentTaskForCleanup: %v", err)+ }+ if err := db.Model(&entity.IngestionTaskLog{}).+ Where("pipeline_log_id = ? AND event_type = ?", run.ID, dao.EventTypeTerminal).+ Count(&terminalCount).Error; err != nil {+ t.Fatalf("count terminal events after retry: %v", err)+ }+ if terminalCount != 1 {+ t.Fatalf("terminal events after retry = %d, want 1", terminalCount)+ }+}++func TestIngestionTaskServiceRemovePurgesTaskState(t *testing.T) {+ db := setupServiceTestDB(t)+ pushServiceDB(t, db)+ insertTestKB(t, "kb-1", "tenant-1", 1, 0, 0)+ insertTestDoc(t, "doc-1", "kb-1", 0, 0)+ insertTestIngestionTaskWithStatus(t, "task-1", "user-1", "doc-1", "kb-1", common.COMPLETED)++ purged := make(chan string, 1)+ svc := NewIngestionTaskService()+ svc.purgeTaskState = func(_ context.Context, taskID string) error {+ purged <- taskID+ return nil+ }+ if _, err := svc.Remove(t.Context(), "task-1", sptr("user-1")); err != nil {+ t.Fatalf("Remove failed: %v", err)+ }+ select {+ case taskID := <-purged:+ if taskID != "task-1" {+ t.Fatalf("purged task = %q, want task-1", taskID)+ }+ case <-time.After(2 * time.Second):+ t.Fatal("Remove did not schedule task-state purge")+ }+}++func TestIngestionTaskServiceRemoveDoesNotWaitForTaskStatePurge(t *testing.T) {+ db := setupServiceTestDB(t)+ pushServiceDB(t, db)+ insertTestKB(t, "kb-1", "tenant-1", 1, 0, 0)+ insertTestDoc(t, "doc-1", "kb-1", 0, 0)+ insertTestIngestionTaskWithStatus(t, "task-remove-async", "user-1", "doc-1", "kb-1", common.COMPLETED)++ started := make(chan struct{})+ release := make(chan struct{})+ svc := NewIngestionTaskService()+ svc.purgeTaskState = func(ctx context.Context, _ string) error {+ close(started)+ select {+ case <-release:+ return nil+ case <-ctx.Done():+ return ctx.Err()+ }+ }++ done := make(chan error, 1)+ go func() {+ _, err := svc.Remove(t.Context(), "task-remove-async", sptr("user-1"))+ done <- err+ }()+ select {+ case err := <-done:+ if err != nil {+ t.Fatalf("Remove failed: %v", err)+ }+ case <-time.After(2 * time.Second):+ t.Fatal("Remove blocked on task-state purge")+ }+ select {+ case <-started:+ case <-time.After(2 * time.Second):+ t.Fatal("background task-state purge did not start")+ }+ close(release)+}++func TestIngestionTaskServiceRemoveKeepsDeleteWhenPurgeFails(t *testing.T) {+ db := setupServiceTestDB(t)+ pushServiceDB(t, db)+ insertTestKB(t, "kb-1", "tenant-1", 1, 0, 0)+ insertTestDoc(t, "doc-1", "kb-1", 0, 0)+ insertTestIngestionTaskWithStatus(t, "task-1", "user-1", "doc-1", "kb-1", common.COMPLETED)++ svc := NewIngestionTaskService()+ svc.purgeTaskState = func(_ context.Context, _ string) error {+ return errors.New("kvrocks unavailable")+ }+ if _, err := svc.Remove(t.Context(), "task-1", sptr("user-1")); err != nil {+ t.Fatalf("Remove failed after a purge error: %v", err)+ }+ if _, err := dao.NewIngestionTaskDAO().GetByID(t.Context(), db, "task-1"); !errors.Is(err, gorm.ErrRecordNotFound) {+ t.Fatalf("deleted task reload error = %v, want record not found", err)+ }+}+func TestIngestionTaskServiceUpdateComponentTotalPersistsDenominator(t *testing.T) {db := setupServiceTestDB(t)pushServiceDB(t, db)