Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

feat(media): harden run media cache lifecycle and limits (#21694)

jay777219ba49c523eb2
diff --git a/docker/.env b/docker/.env
index d33ff98f99..348769a5a2 100644
--- a/docker/.env
+++ b/docker/.env
@@ -549,6 +549,13 @@ LLM_POOL_RATE_LIMIT_RETRY_MAX_DELAY=30.0
# -----------------------------------------------------------------------------
# RAGFLOW_INGESTOR_CONCURRENT_WORKERS=1
+# -----------------------------------------------------------------------------
+# Media cache limits (Go ingestion)
+# -----------------------------------------------------------------------------
+# RAGFLOW_MEDIA_IMAGE_MAX_BYTES=33554432
+# RAGFLOW_MEDIA_MAX_IMAGES_PER_RUN=1024
+# RAGFLOW_MEDIA_MAX_BYTES_PER_RUN=1073741824
+
# -----------------------------------------------------------------------------
# DeepDoc (in-process)
# -----------------------------------------------------------------------------
diff --git a/internal/common/environments.go b/internal/common/environments.go
index 82aebe81c2..ee6006eb5d 100644
--- a/internal/common/environments.go
+++ b/internal/common/environments.go
@@ -322,6 +322,13 @@ const (
// --deepdoc-inference-cpu-cores CLI flag.
EnvDeepDocInferenceCPUCores = "RAGFLOW_DEEPDOC_INFERENCE_CPU_CORES"
+ // Media extraction limits. The parser media budget and the run media cache
+ // read these per call: the per-image payload limit, the number of distinct
+ // images one run may cache, and the total decoded bytes one run may cache.
+ EnvMediaImageMaxBytes = "RAGFLOW_MEDIA_IMAGE_MAX_BYTES"
+ EnvMediaMaxImagesPerRun = "RAGFLOW_MEDIA_MAX_IMAGES_PER_RUN"
+ EnvMediaMaxBytesPerRun = "RAGFLOW_MEDIA_MAX_BYTES_PER_RUN"
+
// Typed-decision model. Unset RAGFLOW_DECISION_DIR leaves the feature off:
// every caller degrades to unfiltered behaviour, so a deployment without the
// model directory is indistinguishable from one that never had the feature.
diff --git a/internal/dao/database_test.go b/internal/dao/database_test.go
index 6cf67663f7..b3cdae4748 100644
--- a/internal/dao/database_test.go
+++ b/internal/dao/database_test.go
@@ -23,6 +23,7 @@ import (
"strings"
"testing"
+ "ragflow/internal/common"
"ragflow/internal/entity"
"github.com/glebarez/sqlite"
@@ -123,6 +124,181 @@ func TestAutoMigrateRuntimeModelsAddsIngestionTaskFilterIndexes(t *testing.T) {
}
}
+func prepareDuplicateIngestionTaskDB(t *testing.T) *gorm.DB {
+ t.Helper()
+
+ db := setupTaskTestDB(t)
+ if err := db.Migrator().DropIndex(&entity.IngestionTask{}, "idx_ingestion_task_document_id"); err != nil {
+ t.Fatalf("drop ingestion task unique index: %v", err)
+ }
+ if err := db.Exec("CREATE INDEX idx_ingestion_task_document_id ON ingestion_task(document_id)").Error; err != nil {
+ t.Fatalf("create legacy non-unique document index: %v", err)
+ }
+ return db
+}
+
+func seedDuplicateIngestionTasks(t *testing.T, db *gorm.DB, staleStatus string, stalePipelineLogID *string) {
+ t.Helper()
+
+ for _, task := range []*entity.IngestionTask{
+ {
+ ID: "task-old",
+ UserID: "user-1",
+ DocumentID: "doc-duplicate",
+ DatasetID: "kb-1",
+ Status: staleStatus,
+ PipelineLogID: stalePipelineLogID,
+ BaseModel: entity.BaseModel{
+ CreateTime: int64Ptr(100),
+ },
+ },
+ {
+ ID: "task-new",
+ UserID: "user-1",
+ DocumentID: "doc-duplicate",
+ DatasetID: "kb-1",
+ Status: common.COMPLETED,
+ BaseModel: entity.BaseModel{
+ CreateTime: int64Ptr(200),
+ },
+ },
+ } {
+ if err := db.Create(task).Error; err != nil {
+ t.Fatalf("create task %s: %v", task.ID, err)
+ }
+ }
+}
+
+func TestRepairDuplicateIngestionTasksDeletesOnlyTerminalStaleRows(t *testing.T) {
+ db := prepareDuplicateIngestionTaskDB(t)
+ doneLogID := "run-done"
+ if err := db.Exec(`CREATE TABLE pipeline_operation_log (
+ id varchar(32) PRIMARY KEY,
+ document_id varchar(32) NOT NULL,
+ operation_status varchar(32) NOT NULL
+ )`).Error; err != nil {
+ t.Fatalf("create pipeline operation log table: %v", err)
+ }
+ if err := db.Exec(
+ "INSERT INTO pipeline_operation_log (id, document_id, operation_status) VALUES (?, ?, ?)",
+ doneLogID, "doc-duplicate", string(entity.TaskStatusDone),
+ ).Error; err != nil {
+ t.Fatalf("create completed pipeline operation log: %v", err)
+ }
+ seedDuplicateIngestionTasks(t, db, common.COMPLETED, &doneLogID)
+
+ if err := repairDuplicateIngestionTasks(t.Context(), db); err != nil {
+ t.Fatalf("repairDuplicateIngestionTasks: %v", err)
+ }
+ var ids []string
+ if err := db.Model(&entity.IngestionTask{}).
+ Where("document_id = ?", "doc-duplicate").
+ Pluck("id", &ids).Error; err != nil {
+ t.Fatalf("list remaining ingestion tasks: %v", err)
+ }
+ if !reflect.DeepEqual(ids, []string{"task-new"}) {
+ t.Fatalf("remaining task ids = %v, want [task-new]", ids)
+ }
+ if err := repairDuplicateIngestionTasks(t.Context(), db); err != nil {
+ t.Fatalf("second repairDuplicateIngestionTasks: %v", err)
+ }
+}
+
+func TestRepairDuplicateIngestionTasksRejectsNonTerminalStaleRows(t *testing.T) {
+ for _, status := range common.ActiveTaskStatuses {
+ t.Run(status, func(t *testing.T) {
+ db := prepareDuplicateIngestionTaskDB(t)
+ seedDuplicateIngestionTasks(t, db, status, nil)
+
+ err := repairDuplicateIngestionTasks(t.Context(), db)
+ for _, want := range []string{"doc-duplicate", "task-old", status} {
+ if err == nil || !strings.Contains(err.Error(), want) {
+ t.Fatalf("repair error = %v, want it to contain %q", err, want)
+ }
+ }
+ var count int64
+ if err := db.Model(&entity.IngestionTask{}).
+ Where("document_id = ?", "doc-duplicate").
+ Count(&count).Error; err != nil {
+ t.Fatalf("count ingestion tasks: %v", err)
+ }
+ if count != 2 {
+ t.Fatalf("ingestion task count = %d, want 2 after failed repair", count)
+ }
+ })
+ }
+}
+
+func TestRepairDuplicateIngestionTasksRejectsOpenBoundPipelineLog(t *testing.T) {
+ db := prepareDuplicateIngestionTaskDB(t)
+ openLogID := "run-open"
+ if err := db.Exec(`CREATE TABLE pipeline_operation_log (
+ id varchar(32) PRIMARY KEY,
+ document_id varchar(32) NOT NULL,
+ operation_status varchar(32) NOT NULL
+ )`).Error; err != nil {
+ t.Fatalf("create pipeline operation log table: %v", err)
+ }
+ if err := db.Exec(
+ "INSERT INTO pipeline_operation_log (id, document_id, operation_status) VALUES (?, ?, ?)",
+ openLogID, "doc-duplicate", string(entity.TaskStatusRunning),
+ ).Error; err != nil {
+ t.Fatalf("create open pipeline operation log: %v", err)
+ }
+ seedDuplicateIngestionTasks(t, db, common.FAILED, &openLogID)
+
+ err := repairDuplicateIngestionTasks(t.Context(), db)
+ for _, want := range []string{"doc-duplicate", "task-old", openLogID} {
+ if err == nil || !strings.Contains(err.Error(), want) {
+ t.Fatalf("repair error = %v, want it to contain %q", err, want)
+ }
+ }
+ var count int64
+ if err := db.Model(&entity.IngestionTask{}).
+ Where("document_id = ?", "doc-duplicate").
+ Count(&count).Error; err != nil {
+ t.Fatalf("count ingestion tasks: %v", err)
+ }
+ if count != 2 {
+ t.Fatalf("ingestion task count = %d, want 2 after failed repair", count)
+ }
+}
+
+func TestRepairDuplicateIngestionTasksSupportsLegacySchemaWithoutPipelineLogID(t *testing.T) {
+ db, err := gorm.Open(sqlite.Open(":memory:"), &gorm.Config{})
+ if err != nil {
+ t.Fatalf("open sqlite: %v", err)
+ }
+ if err := db.Exec(`CREATE TABLE ingestion_task (
+ id varchar(32) PRIMARY KEY,
+ user_id varchar(32) NOT NULL,
+ document_id varchar(32) NOT NULL,
+ dataset_id varchar(32) NOT NULL,
+ status varchar(32) NOT NULL,
+ create_time bigint NULL
+ )`).Error; err != nil {
+ t.Fatalf("create legacy ingestion_task table: %v", err)
+ }
+ if err := db.Exec(`INSERT INTO ingestion_task
+ (id, user_id, document_id, dataset_id, status, create_time)
+ VALUES
+ ('task-old', 'user-1', 'doc-duplicate', 'kb-1', 'COMPLETED', 100),
+ ('task-new', 'user-1', 'doc-duplicate', 'kb-1', 'FAILED', 200)`).Error; err != nil {
+ t.Fatalf("seed legacy ingestion tasks: %v", err)
+ }
+
+ if err := repairDuplicateIngestionTasks(t.Context(), db); err != nil {
+ t.Fatalf("repairDuplicateIngestionTasks: %v", err)
+ }
+ var ids []string
+ if err := db.Table("ingestion_task").Pluck("id", &ids).Error; err != nil {
+ t.Fatalf("list remaining ingestion tasks: %v", err)
+ }
+ if !reflect.DeepEqual(ids, []string{"task-new"}) {
+ t.Fatalf("remaining task ids = %v, want [task-new]", ids)
+ }
+}
+
// TestMigrateIngestionTaskPipelineLogID covers the upgrade path AutoMigrate
// cannot handle on MySQL: an existing ingestion_task created before
// pipeline_log_id existed must gain the column, and running the migration again
diff --git a/internal/dao/ingestion_task.go b/internal/dao/ingestion_task.go
index 8488e085f8..0770aff767 100644
--- a/internal/dao/ingestion_task.go
+++ b/internal/dao/ingestion_task.go
@@ -294,15 +294,13 @@ func (dao *IngestionTaskDAO) CountActiveByDatasetID(ctx context.Context, db *gor
return count, err
}
-// DeleteIfTerminal deletes ingestion tasks for a document that are in a
-// terminal state (COMPLETED, STOPPED, FAILED), or not yet running
-// (CREATED, SCHEDULED).
-// RUNNING and STOPPING tasks are NOT deleted because an in-flight worker
-// would keep writing chunks and corrupt a new run's results.
-// Returns the number of rows deleted.
-func (dao *IngestionTaskDAO) DeleteIfTerminal(ctx context.Context, db *gorm.DB, documentID string) (int64, error) {
- result := db.WithContext(ctx).Where("document_id = ? AND status NOT IN (?, ?)",
- documentID, common.RUNNING, common.STOPPING).
+// DeleteIfTerminal deletes one terminal ingestion task by ID. Cleanup freezes
+// CREATED and SCHEDULED tasks before calling this method; if another writer
+// moves the task back to a non-terminal state, the conditional delete leaves it
+// in place.
+func (dao *IngestionTaskDAO) DeleteIfTerminal(ctx context.Context, db *gorm.DB, taskID string) (int64, error) {
+ result := db.WithContext(ctx).
+ Where("id = ? AND status IN ?", taskID, []string{common.COMPLETED, common.STOPPED, common.FAILED}).
Delete(&entity.IngestionTask{})
if result.Error != nil {
return 0, result.Error
diff --git a/internal/dao/ingestion_task_test.go b/internal/dao/ingestion_task_test.go
index 3ead63809e..14ee9cc888 100644
--- a/internal/dao/ingestion_task_test.go
+++ b/internal/dao/ingestion_task_test.go
@@ -4,7 +4,6 @@ import (
"errors"
"testing"
- "fmt"
"ragflow/internal/common"
"ragflow/internal/entity"
@@ -338,65 +337,6 @@ func TestIngestionTaskDAOUpdateStatusIfCurrentRejectsMismatchedStatus(t *testing
}
}
-func TestIngestionTaskDAODeleteIfTerminal_RemovesOnlyTerminal(t *testing.T) {
- db := setupTaskTestDB(t)
- orig := DB
- DB = db
- t.Cleanup(func() { DB = orig })
-
- // Create tasks in different statuses, each with a unique docID.
- statuses := []string{common.CREATED, common.SCHEDULED, common.RUNNING, common.STOPPING, common.COMPLETED, common.STOPPED, common.FAILED}
- for i, status := range statuses {
- docID := fmt.Sprintf("doc-%d", i)
- task := &entity.IngestionTask{
- ID: fmt.Sprintf("task-%d", i),
- UserID: "user-1",
- DocumentID: docID,
- DatasetID: "kb-1",
- Status: status,
- }
- if err := db.Create(task).Error; err != nil {
- t.Fatalf("create task %s: %v", status, err)
- }
- }
-
- ctx := t.Context()
- // DeleteIfTerminal deletes everything except RUNNING and STOPPING.
- // CREATED and SCHEDULED are safe to delete (no worker has claimed them yet);
- // COMPLETED/STOPPED/FAILED are terminal.
- // Call it for every doc and verify the negative cases survived.
- for i := 0; i < len(statuses); i++ {
- docID := fmt.Sprintf("doc-%d", i)
- _, err := NewIngestionTaskDAO().DeleteIfTerminal(ctx, db, docID)
- if err != nil {
- t.Fatalf("DeleteIfTerminal(doc-%d): %v", i, err)
- }
- }
-
- // RUNNING and STOPPING must survive.
- for _, i := range []int{2, 3} {
- docID := fmt.Sprintf("doc-%d", i)
- task, err := NewIngestionTaskDAO().GetByDocumentID(ctx, db, docID)
- if err != nil {
- t.Fatalf("GetByDocumentID %s: %v", docID, err)
- }
- if task == nil {
- t.Fatalf("%s task (doc=%d) must not be deleted", statuses[i], i)
- }
- }
- // CREATED, SCHEDULED, COMPLETED, STOPPED, FAILED must be gone.
- for _, i := range []int{0, 1, 4, 5, 6} {
- docID := fmt.Sprintf("doc-%d", i)
- task, err := NewIngestionTaskDAO().GetByDocumentID(ctx, db, docID)
- if err != nil {
- t.Fatalf("GetByDocumentID %s: %v", docID, err)
- }
- if task != nil {
- t.Fatalf("%s task (doc=%d) should be deleted, still present", statuses[i], i)
- }
- }
-}
-
func TestIngestionTaskDAOCountActiveByDatasetID(t *testing.T) {
db := setupTaskTestDB(t)
for _, task := range []*entity.IngestionTask{
@@ -436,6 +376,40 @@ func TestIngestionTaskDAOCountActiveByDatasetID(t *testing.T) {
}
}
+func TestIngestionTaskDAODeleteIfTerminalRequiresMatchingTerminalTask(t *testing.T) {
+ db := setupTaskTestDB(t)
+ for _, task := range []*entity.IngestionTask{
+ {ID: "task-terminal", UserID: "user-1", DocumentID: "doc-terminal", DatasetID: "kb-1", Status: common.COMPLETED},
+ {ID: "task-running", UserID: "user-1", DocumentID: "doc-running", DatasetID: "kb-1", Status: common.RUNNING},
+ {ID: "task-queued", UserID: "user-1", DocumentID: "doc-queued", DatasetID: "kb-1", Status: common.SCHEDULED},
+ } {
+ if err := db.Create(task).Error; err != nil {
+ t.Fatalf("create task %s: %v", task.ID, err)
+ }
+ }
+
+ taskDAO := NewIngestionTaskDAO()
+ deleted, err := taskDAO.DeleteIfTerminal(t.Context(), db, "task-terminal")
+ if err != nil {
+ t.Fatalf("delete terminal task: %v", err)
+ }
+ if deleted != 1 {
+ t.Fatalf("deleted terminal rows = %d, want 1", deleted)
+ }
+ for _, taskID := range []string{"task-running", "task-queued"} {
+ deleted, err := taskDAO.DeleteIfTerminal(t.Context(), db, taskID)
+ if err != nil {
+ t.Fatalf("delete %s: %v", taskID, err)
+ }
+ if deleted != 0 {
+ t.Fatalf("deleted %s rows = %d, want 0", taskID, deleted)
+ }
+ if _, err := taskDAO.GetByID(t.Context(), db, taskID); err != nil {
+ t.Fatalf("%s was removed despite non-terminal status: %v", taskID, err)
+ }
+ }
+}
+
func TestIngestionTaskDAOCountActiveByDatasetIDUsesLatestTask(t *testing.T) {
db := setupTaskTestDB(t)
if err := db.Exec("DROP INDEX idx_ingestion_task_document_id").Error; err != nil {
diff --git a/internal/dao/migration.go b/internal/dao/migration.go
index 569014fe25..59c274273b 100644
--- a/internal/dao/migration.go
+++ b/internal/dao/migration.go
@@ -19,10 +19,12 @@ package dao
import (
"context"
"encoding/json"
+ "errors"
"fmt"
"ragflow/internal/common"
"ragflow/internal/entity"
"regexp"
+ "slices"
"sort"
"strings"
@@ -262,18 +264,9 @@ func migrateIngestionTaskDocumentIDUnique(ctx context.Context, db *gorm.DB) erro
return nil
}
- var duplicateCount int64
- if err = db.WithContext(ctx).Raw(`
- SELECT COUNT(*) FROM (
- SELECT document_id FROM ingestion_task GROUP BY document_id HAVING COUNT(*) > 1
- ) AS duplicates
- `).Scan(&duplicateCount).Error; err != nil {
+ if err = repairDuplicateIngestionTasks(ctx, db); err != nil {
return err
}
- if duplicateCount > 0 {
- common.Warn("Found duplicate document_id values in ingestion_task, cannot add unique index", zap.Int64("count", duplicateCount))
- return nil
- }
// A non-unique index under the same name would block the unique one.
if err = dropIndexIfExists(ctx, db, "ingestion_task", indexName); err != nil {
@@ -283,6 +276,132 @@ func migrateIngestionTaskDocumentIDUnique(ctx context.Context, db *gorm.DB) erro
return addUniqueIndex(ctx, db, "ingestion_task", indexName, columns)
}
+// repairDuplicateIngestionTasks keeps the task selected by GetByDocumentID and
+// removes older terminal rows before the unique document index is created.
+// Active duplicates and terminal rows whose bound pipeline log is still open
+// require operator intervention because deleting them could orphan a worker or
+// run ledger.
+func repairDuplicateIngestionTasks(ctx context.Context, db *gorm.DB) error {
+ return db.WithContext(ctx).Transaction(func(tx *gorm.DB) error {
+ var documentIDs []string
+ if err := tx.Model(&entity.IngestionTask{}).
+ Select("document_id").
+ Group("document_id").
+ Having("COUNT(*) > 1").
+ Order("document_id").
+ Pluck("document_id", &documentIDs).Error; err != nil {
+ return fmt.Errorf("find duplicate ingestion_task.document_id values: %w", err)
+ }
+ if len(documentIDs) == 0 {
+ return nil
+ }
+
+ hasPipelineLogID := tx.Migrator().HasColumn(&entity.IngestionTask{}, "pipeline_log_id")
+ columns := "id, document_id, status, create_time"
+ if hasPipelineLogID {
+ columns += ", pipeline_log_id"
+ }
+
+ var staleTasks []entity.IngestionTask
+ for _, documentID := range documentIDs {
+ var tasks []entity.IngestionTask
+ if err := tx.Model(&entity.IngestionTask{}).
+ Select(columns).
+ Where("document_id = ?", documentID).
+ Order("COALESCE(create_time, 0) DESC").
+ Order("id DESC").
+ Find(&tasks).Error; err != nil {
+ return fmt.Errorf("load duplicate ingestion tasks for document %s: %w", documentID, err)
+ }
+ if len(tasks) < 2 {
+ continue
+ }
+ for _, task := range tasks[1:] {
+ if !common.IsTerminalTaskStatus(task.Status) {
+ return fmt.Errorf(
+ "cannot add unique index on ingestion_task.document_id: duplicate task %s for document %s is %s; stop ingestors and resolve the duplicate task before upgrading",
+ task.ID, task.DocumentID, task.Status,
+ )
+ }
+ if hasPipelineLogID && task.PipelineLogID != nil && *task.PipelineLogID != "" {
+ if err := requireSettledDuplicateTaskRun(tx, task); err != nil {
+ return err
+ }
+ }
+ staleTasks = append(staleTasks, task)
+ }
+ }
+
+ for _, task := range staleTasks {
+ result := tx.Where("id = ? AND status = ?", task.ID, task.Status).
+ Delete(&entity.IngestionTask{})
+ if result.Error != nil {
+ return fmt.Errorf("delete duplicate ingestion task %s: %w", task.ID, result.Error)
+ }
+ if result.RowsAffected != 1 {
+ return fmt.Errorf(
+ "cannot delete duplicate ingestion task %s for document %s: task changed during migration",
+ task.ID, task.DocumentID,
+ )
+ }
+ }
+ common.Warn(
+ "Removed duplicate terminal ingestion tasks before adding the unique document index",
+ zap.Int("rows", len(staleTasks)),
+ )
+ return nil
+ })
+}
+
+func requireSettledDuplicateTaskRun(tx *gorm.DB, task entity.IngestionTask) error {
+ if !tx.Migrator().HasTable("pipeline_operation_log") {
+ return fmt.Errorf(
+ "cannot add unique index on ingestion_task.document_id: duplicate task %s for document %s references pipeline log %s, but pipeline_operation_log is missing",
+ task.ID, task.DocumentID, *task.PipelineLogID,
+ )
+ }
+
+ var operationStatus string
+ err := tx.Table("pipeline_operation_log").
+ Select("operation_status").
+ Where("id = ? AND document_id = ?", *task.PipelineLogID, task.DocumentID).
+ Take(&operationStatus).Error
+ if errors.Is(err, gorm.ErrRecordNotFound) {
+ return fmt.Errorf(
+ "cannot add unique index on ingestion_task.document_id: duplicate task %s for document %s references missing pipeline log %s",
+ task.ID, task.DocumentID, *task.PipelineLogID,
+ )
+ }
+ if err != nil {
+ return fmt.Errorf("load pipeline log %s for duplicate task %s: %w", *task.PipelineLogID, task.ID, err)
+ }
+ if isOpenPipelineOperationStatus(operationStatus) {
+ return fmt.Errorf(
+ "cannot add unique index on ingestion_task.document_id: duplicate task %s for document %s is bound to open pipeline log %s; settle the log before upgrading",
+ task.ID, task.DocumentID, *task.PipelineLogID,
+ )
+ }
+ if !isTerminalPipelineOperationStatus(operationStatus) {
+ return fmt.Errorf(
+ "cannot add unique index on ingestion_task.document_id: duplicate task %s for document %s is bound to pipeline log %s with unknown status %q",
+ task.ID, task.DocumentID, *task.PipelineLogID, operationStatus,
+ )
+ }
+ return nil
+}
+
+func isOpenPipelineOperationStatus(status string) bool {
+ return slices.Contains(expandPipelineOperationStatuses(OpenPipelineOperationStatuses()), status)
+}
+
+func isTerminalPipelineOperationStatus(status string) bool {
+ return slices.Contains(expandPipelineOperationStatuses([]string{
+ string(entity.TaskStatusCancel),
+ string(entity.TaskStatusDone),
+ string(entity.TaskStatusFail),
+ }), status)
+}
+
// migrateIngestionTaskPipelineLogID adds ingestion_task.pipeline_log_id when
// the column is missing.
//
diff --git a/internal/dao/migration_integration_test.go b/internal/dao/migration_integration_test.go
index a7d9d0efaa..d9daa936eb 100644
--- a/internal/dao/migration_integration_test.go
+++ b/internal/dao/migration_integration_test.go
@@ -7,9 +7,11 @@ import (
"context"
"fmt"
"os"
+ "reflect"
"strings"
"testing"
+ "ragflow/internal/common"
"ragflow/internal/entity"
"gorm.io/gorm"
@@ -224,3 +226,146 @@ func TestMigrateKnowledgeCompileQueues_Integration(t *testing.T) {
}
}
}
+
+func openIngestionTaskMigrationDB(t *testing.T) (*gorm.DB, context.Context) {
+ t.Helper()
+
+ dsn := os.Getenv("RAGFLOW_TEST_MYSQL_DSN")
+ if dsn == "" {
+ t.Skip("RAGFLOW_TEST_MYSQL_DSN unset: this test needs a throwaway MySQL database")
+ }
+
+ db, err := gorm.Open(newMigrationAwareDialector(dsn), &gorm.Config{TranslateError: true})
+ if err != nil {
+ t.Fatalf("failed to connect: %v", err)
+ }
+ sqlDB, err := db.DB()
+ if err != nil {
+ t.Fatalf("failed to get database connection pool: %v", err)
+ }
+ t.Cleanup(func() {
+ if err := sqlDB.Close(); err != nil {
+ t.Errorf("failed to close database connection pool: %v", err)
+ }
+ })
+ return db, context.Background()
+}
+
+func createDuplicateIngestionTaskSchema(t *testing.T, db *gorm.DB, ctx context.Context, withPipelineLog bool) {
+ t.Helper()
+
+ tables := []string{"ingestion_task"}
+ if withPipelineLog {
+ tables = append(tables, "pipeline_operation_log")
+ }
+ for _, table := range tables {
+ if db.Migrator().HasTable(table) {
+ t.Fatalf("%s already exists; this test requires an empty database", table)
+ }
+ }
+ t.Cleanup(func() {
+ for _, table := range tables {
+ if err := db.WithContext(context.Background()).Migrator().DropTable(table); err != nil {
+ t.Errorf("failed to drop %s test table: %v", table, err)
+ }
+ }
+ })
+ if err := db.WithContext(ctx).Exec(`CREATE TABLE ingestion_task (
+ id varchar(32) NOT NULL PRIMARY KEY,
+ document_id varchar(32) NOT NULL,
+ status varchar(32) NOT NULL,
+ create_time bigint NULL,
+ pipeline_log_id varchar(32) NULL
+ )`).Error; err != nil {
+ t.Fatalf("create ingestion_task: %v", err)
+ }
+ if err := db.WithContext(ctx).Exec(
+ "CREATE INDEX idx_ingestion_task_document_id ON ingestion_task(document_id)",
+ ).Error; err != nil {
+ t.Fatalf("create legacy document index: %v", err)
+ }
+ if !withPipelineLog {
+ return
+ }
+ if err := db.WithContext(ctx).Exec(`CREATE TABLE pipeline_operation_log (
+ id varchar(32) NOT NULL PRIMARY KEY,
+ document_id varchar(32) NOT NULL,
+ operation_status varchar(32) NOT NULL
+ )`).Error; err != nil {
+ t.Fatalf("create pipeline_operation_log: %v", err)
+ }
+}
+
+func TestMigrateIngestionTaskDocumentIDUnique_Integration(t *testing.T) {
+ db, ctx := openIngestionTaskMigrationDB(t)
+
+ t.Run("repairs terminal duplicates", func(t *testing.T) {
+ createDuplicateIngestionTaskSchema(t, db, ctx, true)
+ if err := db.WithContext(ctx).Exec(
+ "INSERT INTO pipeline_operation_log (id, document_id, operation_status) VALUES (?, ?, ?)",
+ "run-done", "doc-duplicate", string(entity.TaskStatusDone),
+ ).Error; err != nil {
+ t.Fatalf("seed completed pipeline log: %v", err)
+ }
+ if err := db.WithContext(ctx).Exec(`INSERT INTO ingestion_task
+ (id, document_id, status, create_time, pipeline_log_id)
+ VALUES
+ ('task-old', 'doc-duplicate', 'COMPLETED', 100, 'run-done'),
+ ('task-new', 'doc-duplicate', 'FAILED', 200, NULL)`).Error; err != nil {
+ t.Fatalf("seed duplicate ingestion tasks: %v", err)
+ }
+
+ if err := migrateIngestionTaskDocumentIDUnique(ctx, db); err != nil {
+ t.Fatalf("migrateIngestionTaskDocumentIDUnique: %v", err)
+ }
+ var ids []string
+ if err := db.WithContext(ctx).Table("ingestion_task").Pluck("id", &ids).Error; err != nil {
+ t.Fatalf("list remaining ingestion tasks: %v", err)
+ }
+ if !reflect.DeepEqual(ids, []string{"task-new"}) {
+ t.Fatalf("remaining task ids = %v, want [task-new]", ids)
+ }
+ hasUnique, err := hasUniqueIndex(ctx, db, "ingestion_task", []string{"document_id"})
+ if err != nil {
+ t.Fatalf("check unique document index: %v", err)
+ }
+ if !hasUnique {
+ t.Fatal("ingestion_task.document_id is not unique after migration")
+ }
+ if err := migrateIngestionTaskDocumentIDUnique(ctx, db); err != nil {
+ t.Fatalf("second migrateIngestionTaskDocumentIDUnique: %v", err)
+ }
+ })
+
+ t.Run("rejects active duplicate", func(t *testing.T) {
+ createDuplicateIngestionTaskSchema(t, db, ctx, false)
+ if err := db.WithContext(ctx).Exec(`INSERT INTO ingestion_task
+ (id, document_id, status, create_time, pipeline_log_id)
+ VALUES
+ ('task-active', 'doc-duplicate', 'RUNNING', 100, NULL),
+ ('task-new', 'doc-duplicate', 'COMPLETED', 200, NULL)`).Error; err != nil {
+ t.Fatalf("seed duplicate ingestion tasks: %v", err)
+ }
+
+ err := migrateIngestionTaskDocumentIDUnique(ctx, db)
+ for _, want := range []string{"doc-duplicate", "task-active", common.RUNNING} {
+ if err == nil || !strings.Contains(err.Error(), want) {
+ t.Fatalf("migration error = %v, want it to contain %q", err, want)
+ }
+ }
+ var count int64
+ if err := db.WithContext(ctx).Table("ingestion_task").Count(&count).Error; err != nil {
+ t.Fatalf("count ingestion tasks: %v", err)
+ }
+ if count != 2 {
+ t.Fatalf("ingestion task count = %d, want 2 after failed migration", count)
+ }
+ hasUnique, err := hasUniqueIndex(ctx, db, "ingestion_task", []string{"document_id"})
+ if err != nil {
+ t.Fatalf("check unique document index: %v", err)
+ }
+ if hasUnique {
+ t.Fatal("ingestion_task.document_id unexpectedly became unique")
+ }
+ })
+}
diff --git a/internal/dao/pipeline_operation_log.go b/internal/dao/pipeline_operation_log.go
index a541bfd641..8d85b155e1 100644
--- a/internal/dao/pipeline_operation_log.go
+++ b/internal/dao/pipeline_operation_log.go
@@ -272,16 +272,20 @@ func (dao *PipelineOperationLogDAO) NextRunCount(ctx context.Context, db *gorm.D
// the from-states the caller declares:
// the transitions are monotonic (unstart -> schedule -> running), so a late
// queued write cannot regress a row a concurrent writer already advanced.
-// A row that already left the declared from-states is left untouched. The
-// legacy progress_msg column is intentionally not updated; run text belongs
-// to ingestion_task_log events.
-func (dao *PipelineOperationLogDAO) AdvanceOpenLog(ctx context.Context, db *gorm.DB, logID string, fromStatuses []string, operationStatus string) error {
+// A row that already left the declared from-states is left untouched and the
+// returned bool is false. The legacy progress_msg column is intentionally not
+// updated; run text belongs to ingestion_task_log events.
+func (dao *PipelineOperationLogDAO) AdvanceOpenLog(ctx context.Context, db *gorm.DB, logID string, fromStatuses []string, operationStatus string) (bool, error) {
if logID == "" || len(fromStatuses) == 0 {
- return nil
+ return false, nil
}
- return db.WithContext(ctx).Model(&entity.PipelineOperationLog{}).
+ result := db.WithContext(ctx).Model(&entity.PipelineOperationLog{}).
Where("id = ? AND operation_status IN ?", logID, fromStatuses).
- Update("operation_status", operationStatus).Error
+ Update("operation_status", operationStatus)
+ if result.Error != nil {
+ return false, result.Error
+ }
+ return result.RowsAffected == 1, nil
}
// GetByIDAndKBID fetches a single ingestion log scoped to its knowledge base.
diff --git a/internal/development.md b/internal/development.md
index 3b784556e9..8f857b44a8 100644
--- a/internal/development.md
+++ b/internal/development.md
@@ -307,10 +307,18 @@ export KVROCKS_PORT=6379
In docker the Go services override these to `kvrocks:6379` automatically
(see `docker-compose-go.yml`). If Kvrocks is unreachable at startup the
process fails fast with `failed to connect to Kvrocks`.
- HTML ingestion uses Kvrocks to hand inline images from the parser to the
- chunker; the ingestor also validates the injected Kvrocks client at startup.
- HTML image payloads default to a 32 MiB per-image limit; set
- `RAGFLOW_HTML_IMAGE_MAX_BYTES` to a positive byte count to change it.
+ HTML and EPUB ingestion use Kvrocks to hand inline images from the parser to
+ the chunker; the ingestor also validates the injected Kvrocks client at startup.
+ Inline image payloads default to a 32 MiB per-image limit; set
+ `RAGFLOW_MEDIA_IMAGE_MAX_BYTES` to a positive byte count to change it.
+ The parser budget and media cache apply the same numeric run caps: 1024 items
+ and 1 GiB of decoded image bytes per run. The parser counts distinct source
+ references; the cache counts distinct SHA-256 payloads. Set
+ `RAGFLOW_MEDIA_MAX_IMAGES_PER_RUN` and `RAGFLOW_MEDIA_MAX_BYTES_PER_RUN` to
+ positive values to change them. Quota state is keyed by ingestion task and
+ persists across retries of that task until cleanup or the 24-hour media TTL.
+ Debug previews use a temporary `pipeline_<document-id>` run scope and purge
+ their cache state when the preview finishes.
- Start RAGFlow
diff --git a/internal/engine/kvrocks/redis.go b/internal/engine/kvrocks/redis.go
index 4c15a25503..8cef086819 100644
--- a/internal/engine/kvrocks/redis.go
+++ b/internal/engine/kvrocks/redis.go
@@ -46,6 +46,7 @@ type Client struct {
client *redis.Client
luaDeleteIfEqual *redis.Script
luaTokenBucket *redis.Script
+ luaMediaPut *redis.Script
luaAutoIncrement *redis.Script
config config.KvrocksConfig
}
@@ -104,6 +105,53 @@ const (
return {1, tokens}
`
+
+ luaMediaPutScript = `
+ local media_key = KEYS[1]
+ local digest_key = KEYS[2]
+ local state_key = KEYS[3]
+ local digest = ARGV[1]
+ local value = ARGV[2]
+ local max_images = tonumber(ARGV[3])
+ local max_bytes = tonumber(ARGV[4])
+ local ttl = tonumber(ARGV[5])
+ local manifest_key = ARGV[6]
+ local manifest_member = ARGV[7]
+
+ local function track_manifest()
+ if manifest_key ~= "" then
+ redis.call("SADD", manifest_key, manifest_member)
+ redis.call("EXPIRE", manifest_key, ttl)
+ end
+ end
+
+ if redis.call("SISMEMBER", digest_key, digest) == 1 then
+ if redis.call("EXISTS", media_key) == 0 then
+ redis.call("SET", media_key, value, "EX", ttl)
+ redis.call("EXPIRE", digest_key, ttl)
+ redis.call("EXPIRE", state_key, ttl)
+ track_manifest()
+ end
+ return {1, 0}
+ end
+
+ local images = redis.call("SCARD", digest_key)
+ local bytes = tonumber(redis.call("HGET", state_key, "bytes") or "0")
+ if max_images > 0 and images >= max_images then
+ return {0, 0}
+ end
+ if max_bytes > 0 and #value > max_bytes - bytes then
+ return {0, 0}
+ end
+
+ redis.call("SET", media_key, value, "EX", ttl)
+ track_manifest()
+ redis.call("SADD", digest_key, digest)
+ redis.call("HINCRBY", state_key, "bytes", #value)
+ redis.call("EXPIRE", digest_key, ttl)
+ redis.call("EXPIRE", state_key, ttl)
+ return {2, 1}
+ `
)
// Init initializes the Kvrocks client. Kvrocks is the only supported cache/queue
@@ -139,6 +187,7 @@ func Init(ctx context.Context) error {
config: kvrocksConfig,
luaDeleteIfEqual: redis.NewScript(luaDeleteIfEqualScript),
luaTokenBucket: redis.NewScript(luaTokenBucketScript),
+ luaMediaPut: redis.NewScript(luaMediaPutScript),
}
common.Info("Kvrocks client initialized",
@@ -368,6 +417,65 @@ func (r *Client) Set(ctx context.Context, key string, value string, exp time.Dur
return true
}
+// MediaPut atomically writes one media payload, records it on the manifest,
+// and charges the run quota. accepted is false when a configured count or byte
+// cap rejects the payload. inserted is false for a digest already charged by
+// this scope; in that case a missing media value is restored without charging
+// the quota again.
+func (r *Client) MediaPut(
+ ctx context.Context,
+ mediaKey, digestKey, stateKey, digest, value string,
+ manifestKey, manifestMember string,
+ maxImages int,
+ maxBytes int64,
+ ttl time.Duration,
+) (accepted, inserted bool, err error) {
+ if r == nil || r.client == nil || r.luaMediaPut == nil {
+ return false, false, fmt.Errorf("redis: media cache is not initialised")
+ }
+ ttlSeconds := int64(math.Ceil(ttl.Seconds()))
+ if ttlSeconds <= 0 {
+ return false, false, fmt.Errorf("redis: media cache ttl must be positive")
+ }
+ result, err := r.luaMediaPut.Run(
+ ctx,
+ r.client,
+ []string{mediaKey, digestKey, stateKey},
+ digest,
+ value,
+ maxImages,
+ maxBytes,
+ ttlSeconds,
+ manifestKey,
+ manifestMember,
+ ).Result()
+ if err != nil {
+ return false, false, fmt.Errorf("media put: %w", err)
+ }
+ values, ok := result.([]interface{})
+ if !ok || len(values) != 2 {
+ return false, false, fmt.Errorf("media put: malformed reply")
+ }
+ status, ok := values[0].(int64)
+ if !ok {
+ return false, false, fmt.Errorf("media put: malformed status")
+ }
+ insertedValue, ok := values[1].(int64)
+ if !ok {
+ return false, false, fmt.Errorf("media put: malformed insert result")
+ }
+ switch status {
+ case 0:
+ return false, false, nil
+ case 1:
+ return true, insertedValue == 1, nil
+ case 2:
+ return true, true, nil
+ default:
+ return false, false, fmt.Errorf("media put: unknown status %d", status)
+ }
+}
+
// SetNX sets value only if key does not exist.
// It returns false with a nil error when the key already exists.
func (r *Client) SetNX(ctx context.Context, key string, value string, exp time.Duration) (bool, error) {
diff --git a/internal/engine/kvrocks/redis_test.go b/internal/engine/kvrocks/redis_test.go
index fa1026324d..5395025b5e 100644
--- a/internal/engine/kvrocks/redis_test.go
+++ b/internal/engine/kvrocks/redis_test.go
@@ -43,6 +43,7 @@ func newStrictTestClient(t *testing.T) (*Client, *miniredis.Miniredis) {
client: rdb,
luaDeleteIfEqual: redis.NewScript(luaDeleteIfEqualScript),
luaTokenBucket: redis.NewScript(luaTokenBucketScript),
+ luaMediaPut: redis.NewScript(luaMediaPutScript),
// luaAutoIncrement intentionally not loaded; not used here.
}, mr
}
@@ -145,6 +146,167 @@ func TestEvalTokenBucketStrict_NilClient(t *testing.T) {
}
}
+func TestMediaPutEnforcesQuotaAndRestoresDuplicate(t *testing.T) {
+ r, _ := newStrictTestClient(t)
+ ctx := t.Context()
+ mediaKey := "mc:media:test:a"
+ digestKey := "mc:quota:test:digests"
+ stateKey := "mc:quota:test:state"
+ manifestKey := "mc:manifest:test"
+
+ accepted, inserted, err := r.MediaPut(
+ ctx,
+ mediaKey,
+ digestKey,
+ stateKey,
+ "digest-a",
+ "abc",
+ manifestKey,
+ mediaKey,
+ 1,
+ 3,
+ time.Hour,
+ )
+ if err != nil || !accepted || !inserted {
+ t.Fatalf("first MediaPut = (%v, %v, %v), want (true, true, nil)", accepted, inserted, err)
+ }
+ accepted, inserted, err = r.MediaPut(
+ ctx,
+ "mc:media:test:b",
+ digestKey,
+ stateKey,
+ "digest-b",
+ "b",
+ manifestKey,
+ "mc:media:test:b",
+ 1,
+ 3,
+ time.Hour,
+ )
+ if err != nil || accepted || inserted {
+ t.Fatalf("limit MediaPut = (%v, %v, %v), want (false, false, nil)", accepted, inserted, err)
+ }
+
+ if !r.Delete(ctx, mediaKey) {
+ t.Fatal("delete media value failed")
+ }
+ for _, key := range []string{digestKey, stateKey} {
+ if err := r.GetClient().Expire(ctx, key, time.Second).Err(); err != nil {
+ t.Fatalf("shorten %s TTL: %v", key, err)
+ }
+ }
+ accepted, inserted, err = r.MediaPut(
+ ctx,
+ mediaKey,
+ digestKey,
+ stateKey,
+ "digest-a",
+ "abc",
+ manifestKey,
+ mediaKey,
+ 1,
+ 3,
+ time.Hour,
+ )
+ if err != nil || !accepted || inserted {
+ t.Fatalf("restore MediaPut = (%v, %v, %v), want (true, false, nil)", accepted, inserted, err)
+ }
+ value, err := r.Get(ctx, mediaKey)
+ if err != nil || value != "abc" {
+ t.Fatalf("restored value = %q, %v; want abc, nil", value, err)
+ }
+ for _, key := range []string{mediaKey, digestKey, stateKey} {
+ ttl, err := r.GetClient().TTL(ctx, key).Result()
+ if err != nil || ttl < 59*time.Minute {
+ t.Fatalf("restored %s TTL = %v, %v; want the refreshed one-hour lifetime", key, ttl, err)
+ }
+ }
+}
+
+func TestMediaPutCountsDigestSetInsteadOfMirroredHash(t *testing.T) {
+ r, _ := newStrictTestClient(t)
+ ctx := t.Context()
+ digestKey := "mc:quota:scard:digests"
+ stateKey := "mc:quota:scard:state"
+
+ accepted, inserted, err := r.MediaPut(
+ ctx,
+ "mc:media:scard:a",
+ digestKey,
+ stateKey,
+ "digest-a",
+ "a",
+ "",
+ "",
+ 1,
+ 100,
+ time.Hour,
+ )
+ if err != nil || !accepted || !inserted {
+ t.Fatalf("first MediaPut = (%v, %v, %v), want (true, true, nil)", accepted, inserted, err)
+ }
+ if err := r.GetClient().HSet(ctx, stateKey, "images", 0).Err(); err != nil {
+ t.Fatalf("make mirrored image count diverge: %v", err)
+ }
+
+ accepted, inserted, err = r.MediaPut(
+ ctx,
+ "mc:media:scard:b",
+ digestKey,
+ stateKey,
+ "digest-b",
+ "b",
+ "",
+ "",
+ 1,
+ 100,
+ time.Hour,
+ )
+ if err != nil || accepted || inserted {
+ t.Fatalf("second MediaPut = (%v, %v, %v), want (false, false, nil)", accepted, inserted, err)
+ }
+}
+
+func TestMediaPutRejectsByteOverflow(t *testing.T) {
+ r, _ := newStrictTestClient(t)
+ ctx := t.Context()
+ digestKey := "mc:quota:bytes:digests"
+ stateKey := "mc:quota:bytes:state"
+
+ accepted, inserted, err := r.MediaPut(
+ ctx,
+ "mc:media:bytes:a",
+ digestKey,
+ stateKey,
+ "digest-a",
+ "abcde",
+ "",
+ "",
+ 10,
+ 5,
+ time.Hour,
+ )
+ if err != nil || !accepted || !inserted {
+ t.Fatalf("first MediaPut = (%v, %v, %v), want (true, true, nil)", accepted, inserted, err)
+ }
+ accepted, inserted, err = r.MediaPut(
+ ctx,
+ "mc:media:bytes:b",
+ digestKey,
+ stateKey,
+ "digest-b",
+ "b",
+ "",
+ "",
+ 10,
+ 5,
+ time.Hour,
+ )
+ if err != nil || accepted || inserted {
+ t.Fatalf("overflow MediaPut = (%v, %v, %v), want (false, false, nil)", accepted, inserted, err)
+ }
+}
+
func TestClientSetNXDistinguishesContention(t *testing.T) {
r, _ := newStrictTestClient(t)
ctx := t.Context()
diff --git a/internal/ingestion/component/media_dispatch.go b/internal/ingestion/component/media_dispatch.go
index f0b9334c75..ad8073e320 100644
--- a/internal/ingestion/component/media_dispatch.go
+++ b/internal/ingestion/component/media_dispatch.go
@@ -109,7 +109,7 @@ func maybeDispatchImage(
return parser.ParseResult{}, true, err
}
defer release()
- if len(binary) == 0 || len(binary) > media.MaxImagePayloadBytes {
+ if len(binary) == 0 || len(binary) > media.ImageMaxBytes() {
return parser.ParseResult{}, true, fmt.Errorf("parser: image payload exceeds size limits")
}
// SVG carries its text in the document, so it is read rather than
diff --git a/internal/ingestion/component/svg_text.go b/internal/ingestion/component/svg_text.go
index 6fde5604b1..93dea01bac 100644
--- a/internal/ingestion/component/svg_text.go
+++ b/internal/ingestion/component/svg_text.go
@@ -34,14 +34,17 @@ import (
"unicode/utf8"
htmlcharset "golang.org/x/net/html/charset"
-
- "ragflow/internal/parser/media"
)
// maxSVGDepth bounds element nesting: the XML decoder keeps one stack entry
// per open element, so depth is the allocation an SVG payload could inflate.
const maxSVGDepth = 1024
+// maxSVGEntityExpansionBytes bounds the total expanded size of DOCTYPE
+// entities. It is a fixed parser-safety bound, not the configurable image
+// limit.
+const maxSVGEntityExpansionBytes = 32 << 20
+
// maxSVGEntities bounds the entity declarations read from a DOCTYPE.
const maxSVGEntities = 64
@@ -117,7 +120,7 @@ func extractSVGText(ctx context.Context, data []byte) (string, error) {
value := string(match[2]) + string(match[3])
references := bytes.Count(data, []byte("&"+string(match[1])+";"))
expansion += int64(references) * int64(len(value))
- if expansion > media.MaxImagePayloadBytes {
+ if expansion > maxSVGEntityExpansionBytes {
return "", errors.New("parser: decode SVG: entity expansion exceeds size limits")
}
entities[string(match[1])] = value
diff --git a/internal/ingestion/component/vision_image.go b/internal/ingestion/component/vision_image.go
index f304dd6372..73d383ae26 100644
--- a/internal/ingestion/component/vision_image.go
+++ b/internal/ingestion/component/vision_image.go
@@ -35,9 +35,12 @@ import (
parserlib "ragflow/internal/parser/parser"
)
-const (
- maxVisionImageBytes = media.MaxImagePayloadBytes
+var (
+ maxVisionImageBytes = media.ImageMaxBytes()
maxVLMEncodedBytes = (maxVisionImageBytes+2)/3*4 + 256
+)
+
+const (
// maxVLMCropPixels and maxVLMCropEdge bound the raster of PDF page crops
// rendered for the vision model (see vision_enhancement_cgo.go). They are a
// pixel budget for the VLM crop path and are intentionally separate from
@@ -100,7 +103,7 @@ func inspectVisionImagePayload(payload string) (string, error) {
}
for _, encoding := range []*base64.Encoding{base64.StdEncoding, base64.RawStdEncoding} {
decoder := base64.NewDecoder(encoding, &visionPayloadReader{source: payload})
- config, format, err := image.DecodeConfig(io.LimitReader(decoder, maxVisionImageBytes+1))
+ config, format, err := image.DecodeConfig(io.LimitReader(decoder, int64(maxVisionImageBytes)+1))
if err != nil {
continue
}
@@ -130,7 +133,7 @@ func decodeVisionPayload(raw string) ([]byte, error) {
}
for _, encoding := range []*base64.Encoding{base64.StdEncoding, base64.RawStdEncoding} {
decoder := base64.NewDecoder(encoding, &visionPayloadReader{source: encoded})
- data, err := io.ReadAll(io.LimitReader(decoder, maxVisionImageBytes+1))
+ data, err := io.ReadAll(io.LimitReader(decoder, int64(maxVisionImageBytes)+1))
if err != nil {
continue
}
diff --git a/internal/ingestion/mediacache/store.go b/internal/ingestion/mediacache/store.go
index 69ff9a38c5..c9c477a240 100644
--- a/internal/ingestion/mediacache/store.go
+++ b/internal/ingestion/mediacache/store.go
@@ -25,7 +25,6 @@ import (
"errors"
"fmt"
"strings"
- "sync"
"time"
"github.com/cespare/xxhash/v2"
@@ -41,26 +40,48 @@ const (
TTL = 24 * time.Hour
)
-// Store is a run-scoped media cache backed by a chunkcache.Store.
+// Store is a run-scoped media cache backed by a Backend. It dedupes writes by
+// digest and bounds how many distinct images, and how many decoded bytes, one
+// run may keep in the backend.
type Store struct {
- backend chunkcache.Store
+ backend Backend
scope string
- mu sync.Mutex
+ maxImages int
+ maxBytes int64
+}
+
+// Backend provides the cache lifecycle operations and atomic quota admission.
+// The quota, media value, and manifest update happen in one MediaPut operation
+// so separate Store instances and processes cannot each admit a full run
+// allowance.
+type Backend interface {
+ chunkcache.Store
+
+ MediaPut(
+ ctx context.Context,
+ mediaKey, digestKey, stateKey, digest, value string,
+ manifestKey, manifestMember string,
+ maxImages int,
+ maxBytes int64,
+ ttl time.Duration,
+ ) (accepted, inserted bool, err error)
}
// New returns a cache bound to one ingestion run. A nil backend is a valid
// value so callers can surface configured-cache failures only when media is
// actually encountered.
-func New(backend chunkcache.Store, scope string) *Store {
+func New(backend Backend, scope string) *Store {
return &Store{
- backend: backend,
- scope: strings.TrimSpace(scope),
+ backend: backend,
+ scope: strings.TrimSpace(scope),
+ maxImages: media.MediaMaxImagesPerRun(),
+ maxBytes: media.MediaMaxBytesPerRun(),
}
}
-// Put validates and stores one decoded image, then records the key on the
-// current task manifest. Any write or lifecycle failure is returned.
+// Put validates and atomically stores one decoded image with its manifest and
+// quota state. Any write or lifecycle failure is returned.
func (s *Store) Put(ctx context.Context, format, mime string, data []byte) (entity.MediaRef, error) {
if s == nil || s.backend == nil {
return entity.MediaRef{}, errors.New("mediacache: cache is not configured")
@@ -77,8 +98,8 @@ func (s *Store) Put(ctx context.Context, format, mime string, data []byte) (enti
if strings.TrimSpace(mime) == "" {
return entity.MediaRef{}, errors.New("mediacache: media type is empty")
}
- if len(data) == 0 || len(data) > media.HTMLImageMaxBytes() {
- return entity.MediaRef{}, fmt.Errorf("mediacache: image size %d is outside the 1..%d-byte limit", len(data), media.HTMLImageMaxBytes())
+ if len(data) == 0 || len(data) > media.ImageMaxBytes() {
+ return entity.MediaRef{}, fmt.Errorf("mediacache: image size %d is outside the 1..%d-byte limit", len(data), media.ImageMaxBytes())
}
sum := sha256.Sum256(data)
@@ -92,18 +113,35 @@ func (s *Store) Put(ctx context.Context, format, mime string, data []byte) (enti
Size: int64(len(data)),
}
- s.mu.Lock()
- defer s.mu.Unlock()
-
key := s.key(digest)
- if !s.backend.Set(ctx, key, string(data), TTL) {
- return entity.MediaRef{}, fmt.Errorf("mediacache: write image %s", mediaShortDigest(digest))
+ mk := ""
+ if globals.TaskID(ctx) != "" {
+ mk = manifestKey(s.scope)
+ }
+ digestKey, stateKey := quotaKeys(s.scope)
+ accepted, _, err := s.backend.MediaPut(
+ ctx,
+ key,
+ digestKey,
+ stateKey,
+ digest,
+ string(data),
+ mk,
+ key,
+ s.maxImages,
+ s.maxBytes,
+ TTL,
+ )
+ if err != nil {
+ return entity.MediaRef{}, fmt.Errorf("mediacache: write image %s: %w", mediaShortDigest(digest), err)
}
- if err := s.track(ctx, key); err != nil {
- if !s.backend.Delete(ctx, key) {
- return entity.MediaRef{}, fmt.Errorf("mediacache: track image %s: %w (rollback failed)", mediaShortDigest(digest), err)
- }
- return entity.MediaRef{}, fmt.Errorf("mediacache: track image %s: %w", mediaShortDigest(digest), err)
+ if !accepted {
+ return entity.MediaRef{}, fmt.Errorf(
+ "mediacache: run media limit exceeded (%d images / %d bytes): %w",
+ s.maxImages,
+ s.maxBytes,
+ media.ErrMediaLimitExceeded,
+ )
}
return ref, nil
}
@@ -130,8 +168,8 @@ func (s *Store) Get(ctx context.Context, ref entity.MediaRef) ([]byte, error) {
if ref.Scope != s.scope {
return nil, fmt.Errorf("mediacache: reference scope %q does not match run scope %q", ref.Scope, s.scope)
}
- if ref.Size > int64(media.HTMLImageMaxBytes()) {
- return nil, fmt.Errorf("mediacache: image %s exceeds the %d-byte limit", mediaShortDigest(ref.SHA256), media.HTMLImageMaxBytes())
+ if ref.Size > int64(media.ImageMaxBytes()) {
+ return nil, fmt.Errorf("mediacache: image %s exceeds the %d-byte limit", mediaShortDigest(ref.SHA256), media.ImageMaxBytes())
}
value, err := s.backend.Get(ctx, s.key(ref.SHA256))
@@ -163,20 +201,10 @@ func manifestKey(scope string) string {
return fmt.Sprintf("%s:manifest:%x", keyNamespace, xxhash.Sum64String(scope))
}
-func (s *Store) track(ctx context.Context, key string) error {
- // Taskless preview runs have no persist-time cleanup and rely on TTL.
- if globals.TaskID(ctx) == "" {
- return nil
- }
- mk := manifestKey(s.scope)
- if !s.backend.SAdd(ctx, mk, key) {
- return fmt.Errorf("mediacache: record manifest %s", mk)
- }
- if !s.backend.Expire(ctx, mk, TTL) {
- s.backend.Delete(ctx, mk)
- return fmt.Errorf("mediacache: expire manifest %s", mk)
- }
- return nil
+func quotaKeys(scope string) (digestKey, stateKey string) {
+ hashedScope := xxhash.Sum64String(scope)
+ return fmt.Sprintf("%s:quota:%x:digests", keyNamespace, hashedScope),
+ fmt.Sprintf("%s:quota:%x:state", keyNamespace, hashedScope)
}
// PurgeTask removes media after durable persistence. Failed runs rely on TTL.
@@ -197,5 +225,11 @@ func PurgeTask(ctx context.Context, backend chunkcache.Store, taskID string) err
if !backend.Delete(ctx, mk) {
return fmt.Errorf("mediacache: delete manifest %s", mk)
}
+ digestKey, stateKey := quotaKeys(taskID)
+ for _, key := range []string{digestKey, stateKey} {
+ if !backend.Delete(ctx, key) {
+ return fmt.Errorf("mediacache: delete quota state %s", key)
+ }
+ }
return nil
}
diff --git a/internal/ingestion/mediacache/store_test.go b/internal/ingestion/mediacache/store_test.go
index 0a1b34166e..6aacd586ac 100644
--- a/internal/ingestion/mediacache/store_test.go
+++ b/internal/ingestion/mediacache/store_test.go
@@ -38,14 +38,17 @@ type fakeStore struct {
expireFails bool
setCalls int
addCalls int
+ mediaPuts int
deleteFails map[string]bool
+ quotaBytes map[string]int64
}
func newFakeStore() *fakeStore {
return &fakeStore{
- kv: map[string]string{},
- sets: map[string]map[string]bool{},
- ttl: map[string]time.Duration{},
+ kv: map[string]string{},
+ sets: map[string]map[string]bool{},
+ ttl: map[string]time.Duration{},
+ quotaBytes: map[string]int64{},
}
}
@@ -66,6 +69,69 @@ func (f *fakeStore) Set(_ context.Context, key, value string, exp time.Duration)
return true
}
+func (f *fakeStore) MediaPut(
+ _ context.Context,
+ mediaKey, digestKey, stateKey, digest, value string,
+ manifestKey, manifestMember string,
+ maxImages int,
+ maxBytes int64,
+ ttl time.Duration,
+) (bool, bool, error) {
+ f.mediaPuts++
+ if f.failSet {
+ return false, false, errors.New("fake media write failure")
+ }
+ if f.expireFails {
+ return false, false, errors.New("fake manifest expire failure")
+ }
+ if f.sets[digestKey][digest] {
+ if f.kv[mediaKey] == "" {
+ f.setCalls++
+ f.kv[mediaKey] = value
+ f.ttl[mediaKey] = ttl
+ f.ttl[digestKey] = ttl
+ f.ttl[stateKey] = ttl
+ if manifestKey != "" {
+ f.addCalls++
+ if f.sets[manifestKey] == nil {
+ f.sets[manifestKey] = map[string]bool{}
+ }
+ f.sets[manifestKey][manifestMember] = true
+ f.ttl[manifestKey] = ttl
+ }
+ }
+ return true, false, nil
+ }
+ if maxImages > 0 && len(f.sets[digestKey]) >= maxImages {
+ return false, false, nil
+ }
+ if maxBytes > 0 && int64(len(value)) > maxBytes-f.quotaBytes[stateKey] {
+ return false, false, nil
+ }
+
+ f.setCalls++
+ f.kv[mediaKey] = value
+ f.ttl[mediaKey] = ttl
+ if manifestKey != "" {
+ f.addCalls++
+ if f.sets[manifestKey] == nil {
+ f.sets[manifestKey] = map[string]bool{}
+ }
+ f.sets[manifestKey][manifestMember] = true
+ }
+ if f.sets[digestKey] == nil {
+ f.sets[digestKey] = map[string]bool{}
+ }
+ f.sets[digestKey][digest] = true
+ f.quotaBytes[stateKey] += int64(len(value))
+ f.ttl[digestKey] = ttl
+ f.ttl[stateKey] = ttl
+ if manifestKey != "" {
+ f.ttl[manifestKey] = ttl
+ }
+ return true, true, nil
+}
+
func (f *fakeStore) SAdd(_ context.Context, key, member string) bool {
f.addCalls++
if f.sets[key] == nil {
@@ -90,6 +156,7 @@ func (f *fakeStore) Delete(_ context.Context, key string) bool {
}
delete(f.kv, key)
delete(f.sets, key)
+ delete(f.quotaBytes, key)
return true
}
@@ -236,7 +303,7 @@ func TestPut_RejectsMissingStoreScopeAndOversizedImage(t *testing.T) {
if _, err := New(nil, "task-1").Put(ctx, "html", "image/png", []byte("hello")); err == nil {
t.Fatal("Put accepted an unavailable cache")
}
- t.Setenv("RAGFLOW_HTML_IMAGE_MAX_BYTES", "5")
+ t.Setenv("RAGFLOW_MEDIA_IMAGE_MAX_BYTES", "5")
data := make([]byte, 6)
if _, err := New(backend, "task-1").Put(ctx, "html", "image/png", data); err == nil {
t.Fatal("Put accepted an oversized image")
@@ -349,7 +416,7 @@ func TestRetryRepopulatesMediaAfterCacheMiss(t *testing.T) {
}
}
-func TestPutRestoresMissingMediaInSameStore(t *testing.T) {
+func TestPutDuplicateWritesOnceAndReturnsCallerRef(t *testing.T) {
backend := newFakeStore()
ctx := taskContext("task-1")
store := New(backend, "task-1")
@@ -358,26 +425,127 @@ func TestPutRestoresMissingMediaInSameStore(t *testing.T) {
if err != nil {
t.Fatal(err)
}
- backend.Delete(ctx, store.key(ref.SHA256))
- backend.Delete(ctx, manifestKey("task-1"))
+ key := store.key(ref.SHA256)
+ backend.ttl[key] = time.Second
+ backend.ttl[manifestKey("task-1")] = time.Second
- restored, err := store.Put(ctx, "html", "image/png", data)
+ duplicate, err := store.Put(ctx, "epub", "image/jpeg", data)
if err != nil {
- t.Fatal(err)
+ t.Fatalf("duplicate Put: %v", err)
}
- got, err := store.Get(ctx, restored)
- if err != nil || string(got) != string(data) {
- t.Fatalf("restored media = %q, %v", got, err)
+ if duplicate.Format != "epub" || duplicate.MIME != "image/jpeg" || duplicate.SHA256 != ref.SHA256 || duplicate.Size != ref.Size {
+ t.Fatalf("duplicate ref = %+v, want the current caller's format/mime on the same digest", duplicate)
}
- if err := PurgeTask(ctx, backend, "task-1"); err != nil {
- t.Fatal(err)
+ if backend.mediaPuts != 2 || backend.setCalls != 1 || backend.addCalls != 1 {
+ t.Fatalf("backend calls = %d atomic Put, %d Set, %d SAdd, want 2, 1, 1", backend.mediaPuts, backend.setCalls, backend.addCalls)
}
- if len(backend.kv) != 0 || len(backend.sets) != 0 {
- t.Fatal("restored media was not purged")
+ if backend.ttl[key] != time.Second || backend.ttl[manifestKey("task-1")] != time.Second {
+ t.Fatal("duplicate Put refreshed TTLs; dedupe must not touch the backend")
+ }
+}
+
+func TestPutDedupeDoesNotConsumeRunLimits(t *testing.T) {
+ t.Setenv("RAGFLOW_MEDIA_MAX_IMAGES_PER_RUN", "2")
+ t.Setenv("RAGFLOW_MEDIA_MAX_BYTES_PER_RUN", "5")
+ backend := newFakeStore()
+ ctx := taskContext("task-1")
+ store := New(backend, "task-1")
+ for attempt := 0; attempt < 3; attempt++ {
+ if _, err := store.Put(ctx, "html", "image/png", []byte("abc")); err != nil {
+ t.Fatalf("duplicate Put #%d: %v", attempt+1, err)
+ }
+ }
+ if _, err := store.Put(ctx, "html", "image/png", []byte("de")); err != nil {
+ t.Fatalf("second distinct image: %v", err)
+ }
+ if _, err := store.Put(ctx, "html", "image/png", []byte("f")); !errors.Is(err, media.ErrMediaLimitExceeded) {
+ t.Fatalf("limit error = %v, want ErrMediaLimitExceeded", err)
+ }
+}
+
+func TestPutEnforcesRunImageCountLimit(t *testing.T) {
+ t.Setenv("RAGFLOW_MEDIA_MAX_IMAGES_PER_RUN", "2")
+ backend := newFakeStore()
+ ctx := taskContext("task-1")
+ store := New(backend, "task-1")
+ for _, image := range []string{"a", "b"} {
+ if _, err := store.Put(ctx, "html", "image/png", []byte(image)); err != nil {
+ t.Fatalf("image %q: %v", image, err)
+ }
+ }
+ if _, err := store.Put(ctx, "html", "image/png", []byte("c")); !errors.Is(err, media.ErrMediaLimitExceeded) {
+ t.Fatalf("limit error = %v, want ErrMediaLimitExceeded", err)
+ }
+ if _, err := store.Put(ctx, "html", "image/png", []byte("c")); !errors.Is(err, media.ErrMediaLimitExceeded) {
+ t.Fatalf("repeated rejected error = %v, want ErrMediaLimitExceeded", err)
+ }
+ if len(backend.kv) != 2 {
+ t.Fatalf("cached entries = %d, want the 2 admitted images", len(backend.kv))
+ }
+ members, _ := backend.SMembers(ctx, manifestKey("task-1"))
+ if len(members) != 2 {
+ t.Fatalf("manifest = %v, want 2 entries", members)
+ }
+}
+
+func TestPutEnforcesRunLimitsAcrossStoreInstances(t *testing.T) {
+ t.Setenv("RAGFLOW_MEDIA_MAX_IMAGES_PER_RUN", "2")
+ backend := newFakeStore()
+ ctx := taskContext("task-1")
+ first := New(backend, "task-1")
+ second := New(backend, "task-1")
+
+ if _, err := first.Put(ctx, "html", "image/png", []byte("a")); err != nil {
+ t.Fatalf("first image: %v", err)
+ }
+ if _, err := second.Put(ctx, "html", "image/png", []byte("b")); err != nil {
+ t.Fatalf("second image from another Store: %v", err)
+ }
+ if _, err := first.Put(ctx, "html", "image/png", []byte("c")); !errors.Is(err, media.ErrMediaLimitExceeded) {
+ t.Fatalf("third image error = %v, want ErrMediaLimitExceeded", err)
+ }
+ if len(backend.kv) != 2 {
+ t.Fatalf("cached entries = %d, want 2", len(backend.kv))
}
}
-func TestPutDuplicateRefreshesTTLAndReportsWriteFailure(t *testing.T) {
+func TestPutEnforcesRunBytesAcrossStoreInstances(t *testing.T) {
+ t.Setenv("RAGFLOW_MEDIA_MAX_BYTES_PER_RUN", "5")
+ backend := newFakeStore()
+ ctx := taskContext("task-1")
+ first := New(backend, "task-1")
+ second := New(backend, "task-1")
+
+ if _, err := first.Put(ctx, "html", "image/png", []byte("abc")); err != nil {
+ t.Fatalf("first image: %v", err)
+ }
+ if _, err := second.Put(ctx, "html", "image/png", []byte("de")); err != nil {
+ t.Fatalf("second image from another Store: %v", err)
+ }
+ if _, err := first.Put(ctx, "html", "image/png", []byte("f")); !errors.Is(err, media.ErrMediaLimitExceeded) {
+ t.Fatalf("third image error = %v, want ErrMediaLimitExceeded", err)
+ }
+}
+
+func TestPutEnforcesRunByteLimitAtExactBoundary(t *testing.T) {
+ t.Setenv("RAGFLOW_MEDIA_MAX_BYTES_PER_RUN", "5")
+ backend := newFakeStore()
+ ctx := taskContext("task-1")
+ store := New(backend, "task-1")
+ for _, image := range []string{"abc", "de"} {
+ if _, err := store.Put(ctx, "html", "image/png", []byte(image)); err != nil {
+ t.Fatalf("image %q: %v", image, err)
+ }
+ }
+ if _, err := store.Put(ctx, "html", "image/png", []byte("f")); !errors.Is(err, media.ErrMediaLimitExceeded) {
+ t.Fatalf("limit error = %v, want ErrMediaLimitExceeded", err)
+ }
+ if len(backend.kv) != 2 {
+ t.Fatalf("cached entries = %d, want the 2 admitted images", len(backend.kv))
+ }
+}
+
+func TestPutRestoresLocallySeenDigestAfterBackendEviction(t *testing.T) {
backend := newFakeStore()
ctx := taskContext("task-1")
store := New(backend, "task-1")
@@ -387,16 +555,54 @@ func TestPutDuplicateRefreshesTTLAndReportsWriteFailure(t *testing.T) {
t.Fatal(err)
}
key := store.key(ref.SHA256)
- backend.ttl[key] = time.Second
- backend.ttl[manifestKey("task-1")] = time.Second
- if _, err := store.Put(ctx, "html", "image/png", data); err != nil {
- t.Fatal(err)
- }
- if backend.ttl[key] != TTL || backend.ttl[manifestKey("task-1")] != TTL {
- t.Fatal("duplicate put did not refresh media and manifest TTLs")
+ delete(backend.kv, key)
+
+ restored, err := store.Put(ctx, "html", "image/png", data)
+ if err != nil {
+ t.Fatalf("restore Put: %v", err)
}
- backend.failSet = true
- if _, err := store.Put(ctx, "html", "image/png", data); err == nil {
- t.Fatal("duplicate put succeeded when the backend rejected the write")
+ got, err := store.Get(ctx, restored)
+ if err != nil || string(got) != string(data) {
+ t.Fatalf("restored read = %q, %v; want %q, nil", got, err, data)
}
}
+
+func TestPutFailedWriteDoesNotConsumeRunLimits(t *testing.T) {
+ t.Setenv("RAGFLOW_MEDIA_MAX_IMAGES_PER_RUN", "2")
+ t.Setenv("RAGFLOW_MEDIA_MAX_BYTES_PER_RUN", "2")
+
+ t.Run("value write", func(t *testing.T) {
+ backend := newFakeStore()
+ ctx := taskContext("task-1")
+ store := New(backend, "task-1")
+ backend.failSet = true
+ if _, err := store.Put(ctx, "html", "image/png", []byte("a")); err == nil {
+ t.Fatal("Put succeeded when the value write failed")
+ }
+ backend.failSet = false
+ for _, image := range []string{"a", "b"} {
+ if _, err := store.Put(ctx, "html", "image/png", []byte(image)); err != nil {
+ t.Fatalf("image %q after recovered write: %v", image, err)
+ }
+ }
+ })
+
+ t.Run("manifest track", func(t *testing.T) {
+ backend := newFakeStore()
+ ctx := taskContext("task-1")
+ store := New(backend, "task-1")
+ backend.expireFails = true
+ if _, err := store.Put(ctx, "html", "image/png", []byte("a")); err == nil {
+ t.Fatal("Put succeeded when the manifest TTL failed")
+ }
+ if len(backend.kv) != 0 {
+ t.Fatalf("untracked media entries = %v, want rollback", backend.kv)
+ }
+ backend.expireFails = false
+ for _, image := range []string{"a", "b"} {
+ if _, err := store.Put(ctx, "html", "image/png", []byte(image)); err != nil {
+ t.Fatalf("image %q after recovered track: %v", image, err)
+ }
+ }
+ })
+}
diff --git a/internal/ingestion/pipeline/cleanup.go b/internal/ingestion/pipeline/cleanup.go
index 0bb0b5b974..860ca3f5da 100644
--- a/internal/ingestion/pipeline/cleanup.go
+++ b/internal/ingestion/pipeline/cleanup.go
@@ -11,8 +11,18 @@ import (
// PurgeTaskState removes the run-scoped cache state for a task using the
// process-wide Kvrocks client. Model results and media use separate manifests.
func PurgeTaskState(ctx context.Context, taskID string) error {
+ return purgeTaskStateWithBackend(ctx, chunkcache.Client(), taskID)
+}
+
+// purgeTaskStateWithBackend purges the chunk and media manifests on one
+// backend. The task-id guard stays in both entry points so an empty id can
+// never purge a shared or empty-keyed manifest.
+func purgeTaskStateWithBackend(ctx context.Context, backend chunkcache.Store, taskID string) error {
if taskID == "" {
return errors.New("task state cleanup requires a task id")
}
- return errors.Join(chunkcache.PurgeTask(ctx, chunkcache.Client(), taskID), mediacache.PurgeTask(ctx, chunkcache.Client(), taskID))
+ return errors.Join(
+ chunkcache.PurgeTask(ctx, backend, taskID),
+ mediacache.PurgeTask(ctx, backend, taskID),
+ )
}
diff --git a/internal/ingestion/pipeline/cleanup_test.go b/internal/ingestion/pipeline/cleanup_test.go
new file mode 100644
index 0000000000..b9bff1f8c9
--- /dev/null
+++ b/internal/ingestion/pipeline/cleanup_test.go
@@ -0,0 +1,206 @@
+//
+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+package pipeline
+
+import (
+ "context"
+ "errors"
+ "sort"
+ "strings"
+ "testing"
+ "time"
+
+ "ragflow/internal/agent/runtime"
+ "ragflow/internal/ingestion/chunkcache"
+ "ragflow/internal/ingestion/component/globals"
+ "ragflow/internal/ingestion/mediacache"
+)
+
+type cleanupFakeStore struct {
+ kv map[string]string
+ sets map[string]map[string]bool
+ ttl map[string]time.Duration
+ failRead map[string]bool
+ bytes map[string]int64
+}
+
+func newCleanupFakeStore() *cleanupFakeStore {
+ return &cleanupFakeStore{
+ kv: map[string]string{},
+ sets: map[string]map[string]bool{},
+ ttl: map[string]time.Duration{},
+ failRead: map[string]bool{},
+ bytes: map[string]int64{},
+ }
+}
+
+func (f *cleanupFakeStore) Get(_ context.Context, key string) (string, error) {
+ return f.kv[key], nil
+}
+
+func (f *cleanupFakeStore) Set(_ context.Context, key, value string, exp time.Duration) bool {
+ f.kv[key] = value
+ f.ttl[key] = exp
+ return true
+}
+
+func (f *cleanupFakeStore) MediaPut(
+ _ context.Context,
+ mediaKey, digestKey, stateKey, digest, value string,
+ manifestKey, manifestMember string,
+ maxImages int,
+ maxBytes int64,
+ ttl time.Duration,
+) (bool, bool, error) {
+ if f.sets[digestKey][digest] {
+ if f.kv[mediaKey] == "" {
+ f.kv[mediaKey] = value
+ f.ttl[mediaKey] = ttl
+ f.ttl[digestKey] = ttl
+ f.ttl[stateKey] = ttl
+ if manifestKey != "" {
+ if f.sets[manifestKey] == nil {
+ f.sets[manifestKey] = map[string]bool{}
+ }
+ f.sets[manifestKey][manifestMember] = true
+ f.ttl[manifestKey] = ttl
+ }
+ }
+ return true, false, nil
+ }
+ if maxImages > 0 && len(f.sets[digestKey]) >= maxImages {
+ return false, false, nil
+ }
+ if maxBytes > 0 && int64(len(value)) > maxBytes-f.bytes[stateKey] {
+ return false, false, nil
+ }
+ f.kv[mediaKey] = value
+ f.ttl[mediaKey] = ttl
+ if manifestKey != "" {
+ if f.sets[manifestKey] == nil {
+ f.sets[manifestKey] = map[string]bool{}
+ }
+ f.sets[manifestKey][manifestMember] = true
+ f.ttl[manifestKey] = ttl
+ }
+ if f.sets[digestKey] == nil {
+ f.sets[digestKey] = map[string]bool{}
+ }
+ f.sets[digestKey][digest] = true
+ f.bytes[stateKey] += int64(len(value))
+ f.ttl[digestKey] = ttl
+ f.ttl[stateKey] = ttl
+ return true, true, nil
+}
+
+func (f *cleanupFakeStore) SAdd(_ context.Context, key, member string) bool {
+ if f.sets[key] == nil {
+ f.sets[key] = map[string]bool{}
+ }
+ f.sets[key][member] = true
+ return true
+}
+
+func (f *cleanupFakeStore) SMembers(_ context.Context, key string) ([]string, error) {
+ if f.failRead[key] {
+ return nil, errors.New("kvrocks unavailable")
+ }
+ out := make([]string, 0, len(f.sets[key]))
+ for member := range f.sets[key] {
+ out = append(out, member)
+ }
+ sort.Strings(out)
+ return out, nil
+}
+
+func (f *cleanupFakeStore) Delete(_ context.Context, key string) bool {
+ delete(f.kv, key)
+ delete(f.sets, key)
+ delete(f.bytes, key)
+ return true
+}
+
+func (f *cleanupFakeStore) Expire(_ context.Context, key string, exp time.Duration) bool {
+ f.ttl[key] = exp
+ return true
+}
+
+func cleanupTaskContext(taskID string) context.Context {
+ ctx := runtime.WithState(context.Background(), runtime.NewCanvasState("", ""))
+ globals.SetTaskID(ctx, taskID)
+ return ctx
+}
+
+func seedTaskManifests(t *testing.T, backend *cleanupFakeStore, taskID string) {
+ t.Helper()
+ ctx := cleanupTaskContext(taskID)
+ chunkcache.Set(ctx, backend, "kc:emb:chunk-1", "model result")
+ if _, err := mediacache.New(backend, taskID).Put(ctx, "html", "image/png", []byte("hello")); err != nil {
+ t.Fatalf("seed media cache: %v", err)
+ }
+ if len(backend.kv) != 2 || len(backend.sets) != 3 {
+ t.Fatalf("seeded state = %d values, %d sets, want 2 and 3", len(backend.kv), len(backend.sets))
+ }
+}
+
+func TestPurgeTaskStateWithBackendRemovesChunkAndMediaManifests(t *testing.T) {
+ backend := newCleanupFakeStore()
+ ctx := cleanupTaskContext("task-1")
+ seedTaskManifests(t, backend, "task-1")
+
+ if err := purgeTaskStateWithBackend(ctx, backend, "task-1"); err != nil {
+ t.Fatalf("purgeTaskStateWithBackend: %v", err)
+ }
+ if len(backend.kv) != 0 || len(backend.sets) != 0 {
+ t.Fatalf("state after purge = %v values, %v manifests, want empty", backend.kv, backend.sets)
+ }
+}
+
+func TestPurgeTaskStateWithBackendJoinsErrorsAndStillPurgesMedia(t *testing.T) {
+ backend := newCleanupFakeStore()
+ ctx := cleanupTaskContext("task-1")
+ seedTaskManifests(t, backend, "task-1")
+ backend.failRead["kc:manifest:task-1"] = true
+
+ err := purgeTaskStateWithBackend(ctx, backend, "task-1")
+ if err == nil || !strings.Contains(err.Error(), "chunk cache manifest") {
+ t.Fatalf("error = %v, want the chunk manifest failure", err)
+ }
+ for key := range backend.kv {
+ if strings.HasPrefix(key, "mc:") {
+ t.Fatalf("media value %s survived a failed chunk purge", key)
+ }
+ }
+ for key := range backend.sets {
+ if strings.HasPrefix(key, "mc:") {
+ t.Fatalf("media manifest %s survived a failed chunk purge", key)
+ }
+ }
+ if !backend.sets["kc:manifest:task-1"]["kc:emb:chunk-1"] {
+ t.Fatal("failed chunk purge lost the chunk manifest it must retry")
+ }
+}
+
+func TestPurgeTaskStateWithBackendRejectsEmptyTaskID(t *testing.T) {
+ backend := newCleanupFakeStore()
+ if err := purgeTaskStateWithBackend(context.Background(), backend, ""); err == nil {
+ t.Fatal("purge accepted an empty task id")
+ }
+ if len(backend.kv) != 0 || len(backend.sets) != 0 {
+ t.Fatal("empty task id touched backend state")
+ }
+}
diff --git a/internal/ingestion/pipeline/html_media_real_integration_test.go b/internal/ingestion/pipeline/html_media_real_integration_test.go
index 44dc6f64a8..cfab629bdf 100644
--- a/internal/ingestion/pipeline/html_media_real_integration_test.go
+++ b/internal/ingestion/pipeline/html_media_real_integration_test.go
@@ -204,12 +204,35 @@ func TestHTMLMediaRealServices(t *testing.T) {
if got, err := retry.Get(t.Context(), retryRef); err != nil || !bytes.Equal(got, png) {
t.Fatalf("retry cache bytes: %v", err)
}
+ scopeHash := xxhash.Sum64String(taskID)
+ digestKey := fmt.Sprintf("mc:quota:%x:digests", scopeHash)
+ stateKey := fmt.Sprintf("mc:quota:%x:state", scopeHash)
+ manifestKey := fmt.Sprintf("mc:manifest:%x", scopeHash)
+ if member, err := kv.GetClient().SIsMember(t.Context(), digestKey, retryRef.SHA256).Result(); err != nil || !member {
+ t.Fatalf("quota digest membership = %v, %v; want true", member, err)
+ }
+ images, err := kv.GetClient().SCard(t.Context(), digestKey).Result()
+ if err != nil || images != 1 {
+ t.Fatalf("quota images = %d, %v; want 1", images, err)
+ }
+ bytesUsed, err := kv.GetClient().HGet(t.Context(), stateKey, "bytes").Result()
+ if err != nil || bytesUsed != fmt.Sprint(len(png)) {
+ t.Fatalf("quota bytes = %q, %v; want %d", bytesUsed, err, len(png))
+ }
+ if member, err := kv.GetClient().SIsMember(t.Context(), manifestKey, key).Result(); err != nil || !member {
+ t.Fatalf("manifest membership = %v, %v; want true", member, err)
+ }
if err := mediacache.PurgeTask(t.Context(), kv, taskID); err != nil {
t.Fatal(err)
}
if _, err := retry.Get(t.Context(), retryRef); !errors.Is(err, media.ErrMediaNotFound) {
t.Fatalf("purged cache read: %v", err)
}
+ for _, purgedKey := range []string{key, digestKey, stateKey, manifestKey} {
+ if exists, err := kv.GetClient().Exists(t.Context(), purgedKey).Result(); err != nil || exists != 0 {
+ t.Fatalf("purged key %s exists=%d, err=%v", purgedKey, exists, err)
+ }
+ }
}
// loadHTMLMediaIntegrationConfig uses the CI service addresses when set.
diff --git a/internal/ingestion/pipeline/pipeline.go b/internal/ingestion/pipeline/pipeline.go
index d227476282..12af493f2e 100644
--- a/internal/ingestion/pipeline/pipeline.go
+++ b/internal/ingestion/pipeline/pipeline.go
@@ -37,6 +37,8 @@ import (
"go.uber.org/zap"
)
+var _ mediacache.Backend = (*kvrocks.Client)(nil)
+
// Pipeline is a compiled ingestion canvas plus task-scoped metadata.
type Pipeline struct {
taskID string
@@ -111,6 +113,10 @@ func WithProgressSink(s ProgressSink) PipelineOption {
// It accepts either the inner canvas DSL or the template wrapper whose
// top-level `dsl` field carries that canvas.
func NewPipelineFromDSL(dsl []byte, taskID string, opts ...PipelineOption) (*Pipeline, error) {
+ taskID = strings.TrimSpace(taskID)
+ if taskID == "" {
+ return nil, errors.New("pipeline: task id is required")
+ }
// UnwrapCanvasDSL is the single source of truth for stripping the
// optional {"dsl": {...}} canvas envelope; it also reports a nil/unparseable
// DSL.
@@ -270,11 +276,7 @@ func (p *Pipeline) Run(ctx context.Context, inputs map[string]any, overrideParam
globals.SetTaskID(runCtx, p.taskID)
mediaCache := p.media
if mediaCache == nil && p.kv != nil {
- scope := p.taskID
- if scope == "" {
- scope = p.documentID
- }
- mediaCache = mediacache.New(p.kv, scope)
+ mediaCache = mediacache.New(p.kv, p.taskID)
}
if mediaCache != nil {
runCtx = media.WithCache(runCtx, mediaCache)
diff --git a/internal/ingestion/pipeline/pipeline_test.go b/internal/ingestion/pipeline/pipeline_test.go
index 0ed1d0912d..eca521745f 100644
--- a/internal/ingestion/pipeline/pipeline_test.go
+++ b/internal/ingestion/pipeline/pipeline_test.go
@@ -270,6 +270,22 @@ func TestPipelineRunAttachesInjectedMediaCacheWithTaskScope(t *testing.T) {
}
}
+func TestNewPipelineFromDSLRejectsEmptyTaskID(t *testing.T) {
+ _, err := NewPipelineFromDSL([]byte(`{
+ "dsl": {
+ "components": {
+ "begin": {"obj": {"component_name": "Begin", "params": {}}, "downstream": ["stage"]},
+ "stage": {"obj": {"component_name": "Begin", "params": {}}, "upstream": ["begin"]}
+ },
+ "path": ["begin", "stage"],
+ "graph": {"nodes": []}
+ }
+ }`), " ")
+ if err == nil {
+ t.Fatal("NewPipelineFromDSL accepted an empty task id")
+ }
+}
+
func TestPipelineRun_TaskScopedFactoriesDoNotLeakAcrossConcurrentPipelines(t *testing.T) {
origFactory := runtime.DefaultFactory()
runtime.SetDefaultFactory(func(_ string, _ map[string]any) (runtime.Component, error) {
diff --git a/internal/ingestion/task/debug_test.go b/internal/ingestion/task/debug_test.go
index b250b4e93e..93a6ae630c 100644
--- a/internal/ingestion/task/debug_test.go
+++ b/internal/ingestion/task/debug_test.go
@@ -113,6 +113,40 @@ func TestExecute_DebugViaEntry(t *testing.T) {
}
}
+func TestExecute_DebugViaEntryPurgesRunState(t *testing.T) {
+ taskCtx := NewDebugTaskContext("t1", "canvas-1", "doc.pdf", []byte("page one"))
+ runCtx, cancel := context.WithCancel(context.Background())
+
+ exec, err := NewPipelineExecutor(taskCtx, "canvas-1", 0, testPipelineDeps())
+ if err != nil {
+ t.Fatalf("NewPipelineExecutor: %v", err)
+ }
+ var purgedTaskID string
+ exec.purgeTaskState = func(ctx context.Context, taskID string) error {
+ if err := ctx.Err(); err != nil {
+ t.Errorf("purge context was cancelled with the debug request: %v", err)
+ }
+ purgedTaskID = taskID
+ return nil
+ }
+ exec.
+ WithLoadDSLFunc(func(ctx context.Context, canvasID string) (string, string, error) {
+ return "dsl", "canvas-1", nil
+ }).
+ WithRunPipelineFunc(func(ctx context.Context, dsl string) (map[string]any, string, error) {
+ cancel()
+ return map[string]any{"chunks": []map[string]any{{"text": "c1"}}}, dsl, nil
+ })
+
+ if _, err := exec.Execute(runCtx); err != nil {
+ t.Fatalf("Execute: %v", err)
+ }
+ want := "pipeline_" + taskCtx.Doc.ID
+ if purgedTaskID != want {
+ t.Fatalf("purged task id = %q, want %q", purgedTaskID, want)
+ }
+}
+
// TestInjectDebugPageCap verifies the canvas-debug page cap is delivered
// through Run's override_params channel (the existing ParserConfig shape),
// NOT through pipeline inputs. The cap must land at
diff --git a/internal/ingestion/task/pipeline_executor.go b/internal/ingestion/task/pipeline_executor.go
index dde2b6fb61..769b879e74 100644
--- a/internal/ingestion/task/pipeline_executor.go
+++ b/internal/ingestion/task/pipeline_executor.go
@@ -82,6 +82,7 @@ type PipelineExecutor struct {
deleteChunksFunc DeleteChunksFunc
loadDSLFunc func(ctx context.Context, canvasID string) (string, string, error)
runPipelineFunc func(ctx context.Context, dsl string) (map[string]any, string, error)
+ purgeTaskState func(ctx context.Context, taskID string) error
progressSink pipelinepkg.ProgressSink
}
@@ -176,6 +177,15 @@ func NewPipelineExecutor(
}
svc.loadDSLFunc = svc.loadDSLFromCanvas
svc.runPipelineFunc = svc.runPipelineWithDSL
+ svc.purgeTaskState = func(ctx context.Context, taskID string) error {
+ if deps.KV == nil {
+ return nil
+ }
+ return errors.Join(
+ chunkcache.PurgeTask(ctx, deps.KV, taskID),
+ mediacache.PurgeTask(ctx, deps.KV, taskID),
+ )
+ }
return svc, nil
}
@@ -267,8 +277,10 @@ func (s *PipelineExecutor) Execute(ctx context.Context) (*PipelineResult, error)
// collectDebugOutput builds a PipelineResult for a debug (dry-run) run.
// It surfaces the pipeline's chunks so a debug endpoint can render them, but
// performs no DB/index writes — the embedding vectors already computed by the
-// pipeline run are left on the chunks. This keeps debug runs side-effect free.
+// pipeline run are left on the chunks. Run-scoped cache state is purged before
+// the preview is returned.
func (s *PipelineExecutor) collectDebugOutput(ctx context.Context, pipelineOutput map[string]any, start time.Time) (*PipelineResult, error) {
+ s.purgeRunState(ctx)
chunks := indexdoc.NormalizeChunks(pipelineOutput)
return &PipelineResult{
DocID: s.taskCtx.Doc.ID,
@@ -282,6 +294,7 @@ func (s *PipelineExecutor) collectDebugOutput(ctx context.Context, pipelineOutpu
func (s *PipelineExecutor) processOutput(ctx context.Context, pipelineOutput map[string]any, start time.Time) (*PipelineResult, error) {
if pipelineOutput == nil {
+ s.purgeRunState(ctx)
return nil, nil
}
if err := ctx.Err(); err != nil {
@@ -290,6 +303,7 @@ func (s *PipelineExecutor) processOutput(ctx context.Context, pipelineOutput map
chunks := indexdoc.NormalizeChunks(pipelineOutput)
if len(chunks) == 0 {
+ s.purgeRunState(ctx)
return nil, nil
}
@@ -378,16 +392,8 @@ func (s *PipelineExecutor) processOutput(ctx context.Context, pipelineOutput map
// (index write, compiled-product reconcile, Wiki active-MAP state) has
// succeeded. Only now drop the run-scoped cache entries this task produced,
// so a failure in any of those steps leaves chunk results and parser media
- // intact for retry. Cleanup is best-effort; failed media cleanup is logged
- // and the entries expire independently.
- if s.taskCtx.IngestionTask != nil && s.taskCtx.IngestionTask.ID != "" {
- chunkcache.PurgeTask(ctx, chunkcache.Client(), s.taskCtx.IngestionTask.ID)
- if s.deps.KV != nil {
- if err := mediacache.PurgeTask(ctx, s.deps.KV, s.taskCtx.IngestionTask.ID); err != nil {
- common.Warn("failed to purge task media cache", zap.String("task_id", s.taskCtx.IngestionTask.ID), zap.Error(err))
- }
- }
- }
+ // intact for retry.
+ s.purgeRunState(ctx)
return &PipelineResult{
DocID: s.taskCtx.Doc.ID,
@@ -404,6 +410,30 @@ func (s *PipelineExecutor) processOutput(ctx context.Context, pipelineOutput map
}, nil
}
+func (s *PipelineExecutor) purgeRunState(ctx context.Context) {
+ if s == nil || s.taskCtx == nil || s.purgeTaskState == nil {
+ return
+ }
+ taskID := ""
+ if s.taskCtx.IngestionTask != nil {
+ taskID = s.taskCtx.IngestionTask.ID
+ }
+ if taskID == "" && s.taskCtx.IsDebug() {
+ taskID = "pipeline_" + s.taskCtx.Doc.ID
+ }
+ if taskID == "" {
+ return
+ }
+ cleanupCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), runStateCleanupTimeout)
+ defer cancel()
+ if err := s.purgeTaskState(cleanupCtx, taskID); err != nil {
+ common.Warn("failed to purge task run state",
+ zap.String("task_id", taskID),
+ zap.Error(err),
+ )
+ }
+}
+
func (s *PipelineExecutor) compensateFailedIndexWrite(ctx context.Context, chunks []map[string]any) error {
if s.deleteChunksFunc == nil {
return nil
@@ -1296,6 +1326,8 @@ func (s *PipelineExecutor) buildLogDSL(dsl string, output map[string]any) string
// ParserConfig[cpnID][filetype]["pages"] shape (see NormalizeParserConfigPages).
const debugPageCapPages = 2
+const runStateCleanupTimeout = 5 * time.Second
+
// injectDebugChunkCap sets the canvas-debug chunk cap on the run inputs when
// not already present. The cap is read by the chunker decorator (via
// CanvasState.Globals / globals.DebugChunkCap) and limits a debug (dry-run)
diff --git a/internal/ingestion/task/pipeline_executor_test.go b/internal/ingestion/task/pipeline_executor_test.go
index 1d93fab08b..1e1e6478ba 100644
--- a/internal/ingestion/task/pipeline_executor_test.go
+++ b/internal/ingestion/task/pipeline_executor_test.go
@@ -27,6 +27,37 @@ import (
func strPtr(s string) *string { return &s }
+func TestProcessOutputPurgesRunStateOnSuccessfulNoOutput(t *testing.T) {
+ for _, tc := range []struct {
+ name string
+ output map[string]any
+ }{
+ {name: "nil output", output: nil},
+ {name: "empty output", output: map[string]any{}},
+ } {
+ t.Run(tc.name, func(t *testing.T) {
+ taskCtx := &TaskContext{
+ Doc: entity.Document{ID: "doc-1"},
+ IngestionTask: &entity.IngestionTask{ID: "task-1"},
+ }
+ var purgedTaskID string
+ exec := &PipelineExecutor{
+ taskCtx: taskCtx,
+ purgeTaskState: func(ctx context.Context, taskID string) error {
+ purgedTaskID = taskID
+ return nil
+ },
+ }
+ if _, err := exec.processOutput(context.Background(), tc.output, time.Now()); err != nil {
+ t.Fatalf("processOutput: %v", err)
+ }
+ if purgedTaskID != "task-1" {
+ t.Fatalf("purged task id = %q, want task-1", purgedTaskID)
+ }
+ })
+ }
+}
+
// TestMarkCompiledProductsHidden verifies the pipeline caller stages ONLY the
// wiki variant's per-document rows as available_int=0 (they are intermediate
// state for the consumer's merged pages). Tree / structure (page_index) /
diff --git a/internal/parser/media/budget.go b/internal/parser/media/budget.go
new file mode 100644
index 0000000000..042c6eb57d
--- /dev/null
+++ b/internal/parser/media/budget.go
@@ -0,0 +1,192 @@
+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package media
+
+import "fmt"
+
+// Size is a declared or actual media payload size in bytes. UnknownSize lets a
+// caller reserve an occurrence before it knows how much data it will read.
+type Size int64
+
+const (
+ // UnknownSize defers the byte check until Reservation.Commit.
+ UnknownSize Size = -1
+)
+
+// Admission explains why a source or occurrence was accepted or rejected.
+type Admission uint8
+
+const (
+ // Admitted means the caller owns a reservation and may read the payload.
+ Admitted Admission = iota
+ // RejectedByCount means the run's distinct-image limit is exhausted.
+ RejectedByCount
+ // RejectedByBytes means the per-image or total-byte limit rejected it.
+ RejectedByBytes
+ // RejectedInvalid means the supplied size is invalid.
+ RejectedInvalid
+)
+
+// Budget enforces one parser run's image count and byte limits. Reserve checks
+// declared bytes before reserving a stable source; ReserveOccurrence preserves
+// sequential extraction semantics by consuming the count slot first.
+type Budget struct {
+ limits Limits
+
+ itemsUsed int
+ totalBytes int64
+ oversizedImages int
+ totalOverflows int
+ countExhausted bool
+}
+
+type budgetItem struct {
+ included bool
+ charged bool
+}
+
+// Reservation is one admitted item. Commit applies the actual payload size
+// after the caller reads or decodes it.
+type Reservation struct {
+ budget *Budget
+ item *budgetItem
+}
+
+// NewBudget returns a run budget using the current configured limits.
+func NewBudget() *Budget {
+ return NewBudgetWithLimits(LoadLimits())
+}
+
+// NewBudgetWithLimits returns a run budget with an explicit limits snapshot.
+func NewBudgetWithLimits(limits Limits) *Budget {
+ return &Budget{limits: limits}
+}
+
+// Reserve admits a stable source before its bytes are read. A declared size
+// rejected by a byte limit does not consume a count slot. Callers memoize the
+// source so a successful reservation is reused without another Budget lookup.
+func (b *Budget) Reserve(declared Size) (*Reservation, Admission) {
+ if declared != UnknownSize && declared < 0 {
+ return nil, RejectedInvalid
+ }
+ if declared != UnknownSize && !b.canAdmitBytes(declared) {
+ b.noteByteRejection(declared)
+ return nil, RejectedByBytes
+ }
+ if b.countExhausted || b.itemsUsed >= b.limits.MaxImagesPerRun {
+ b.countExhausted = true
+ return nil, RejectedByCount
+ }
+ item := &budgetItem{}
+ b.itemsUsed++
+ return &Reservation{budget: b, item: item}, Admitted
+}
+
+// ReserveOccurrence admits a source-less payload. The occurrence consumes one
+// item before its actual size is known, matching sequential Office and
+// Markdown extraction where byte-limit omissions must not stop later text.
+func (b *Budget) ReserveOccurrence(declared Size) (*Reservation, Admission) {
+ if declared != UnknownSize && declared < 0 {
+ return nil, RejectedInvalid
+ }
+ if b.countExhausted || b.itemsUsed >= b.limits.MaxImagesPerRun {
+ b.countExhausted = true
+ return nil, RejectedByCount
+ }
+ item := &budgetItem{}
+ b.itemsUsed++
+ if declared != UnknownSize && !b.admitBytes(item, declared) {
+ return nil, RejectedByBytes
+ }
+ return &Reservation{budget: b, item: item}, Admitted
+}
+
+// Commit applies the actual payload size to a reservation. It is idempotent
+// for an included item and keeps byte-rejected reservations rejected.
+func (r *Reservation) Commit(actual Size) bool {
+ if r == nil || r.budget == nil || r.item == nil {
+ return false
+ }
+ if r.item.included {
+ return true
+ }
+ if r.item.charged || actual < 0 {
+ return false
+ }
+ if !r.budget.admitBytes(r.item, actual) {
+ return false
+ }
+ r.item.included = true
+ r.budget.totalBytes += int64(actual)
+ return true
+}
+
+// Count reports how many items this run has reserved.
+func (b *Budget) Count() int {
+ return b.itemsUsed
+}
+
+func (b *Budget) canAdmitBytes(size Size) bool {
+ if int64(size) > b.limits.ImageMaxBytes {
+ return false
+ }
+ return int64(size) <= b.limits.MaxBytesPerRun-b.totalBytes
+}
+
+func (b *Budget) admitBytes(item *budgetItem, size Size) bool {
+ if item.charged {
+ return false
+ }
+ if b.canAdmitBytes(size) {
+ return true
+ }
+ item.charged = true
+ b.noteByteRejection(size)
+ return false
+}
+
+func (b *Budget) noteByteRejection(size Size) {
+ if int64(size) > b.limits.ImageMaxBytes {
+ b.oversizedImages++
+ return
+ }
+ b.totalOverflows++
+}
+
+// Warnings reports the budget omissions accumulated by this run.
+func (b *Budget) Warnings() []string {
+ var warnings []string
+ if b.oversizedImages > 0 {
+ warnings = append(warnings, fmt.Sprintf(
+ "omitted %d embedded image payload(s) larger than the %d-byte per-image limit",
+ b.oversizedImages,
+ b.limits.ImageMaxBytes,
+ ))
+ }
+ if b.totalOverflows > 0 {
+ warnings = append(warnings, fmt.Sprintf(
+ "omitted %d embedded image payload(s) after reaching the %d-byte run image budget",
+ b.totalOverflows,
+ b.limits.MaxBytesPerRun,
+ ))
+ }
+ if b.countExhausted {
+ warnings = append(warnings, fmt.Sprintf(
+ "stopped extracting embedded images after the %d-item run limit",
+ b.limits.MaxImagesPerRun,
+ ))
+ }
+ return warnings
+}
diff --git a/internal/parser/media/budget_test.go b/internal/parser/media/budget_test.go
new file mode 100644
index 0000000000..e3f482ac34
--- /dev/null
+++ b/internal/parser/media/budget_test.go
@@ -0,0 +1,200 @@
+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package media
+
+import (
+ "reflect"
+ "strings"
+ "testing"
+)
+
+func testBudget(maxItems int, maxImageBytes, maxTotalBytes int64) *Budget {
+ return NewBudgetWithLimits(Limits{
+ ImageMaxBytes: maxImageBytes,
+ MaxImagesPerRun: maxItems,
+ MaxBytesPerRun: maxTotalBytes,
+ })
+}
+
+func reserveAndCommit(t *testing.T, budget *Budget, size Size) bool {
+ t.Helper()
+ reservation, admission := budget.Reserve(size)
+ if admission != Admitted {
+ return false
+ }
+ return reservation.Commit(size)
+}
+
+func TestNewBudgetUsesRunCaps(t *testing.T) {
+ t.Setenv("RAGFLOW_MEDIA_IMAGE_MAX_BYTES", "2048")
+ t.Setenv("RAGFLOW_MEDIA_MAX_IMAGES_PER_RUN", "7")
+ t.Setenv("RAGFLOW_MEDIA_MAX_BYTES_PER_RUN", "1234")
+ budget := NewBudget()
+ want := Limits{ImageMaxBytes: 2048, MaxImagesPerRun: 7, MaxBytesPerRun: 1234}
+ if budget.limits != want {
+ t.Fatalf("budget limits = %+v, want %+v", budget.limits, want)
+ }
+}
+
+func TestBudgetItemLimitExhaustion(t *testing.T) {
+ budget := testBudget(2, 16, 16)
+ for attempt := 0; attempt < 2; attempt++ {
+ reservation, admission := budget.Reserve(UnknownSize)
+ if admission != Admitted || !reservation.Commit(1) {
+ t.Fatalf("item #%d was not admitted", attempt+1)
+ }
+ }
+ if _, admission := budget.Reserve(UnknownSize); admission != RejectedByCount {
+ t.Fatalf("third item admission = %v, want count rejection", admission)
+ }
+ if !strings.Contains(strings.Join(budget.Warnings(), "\n"), "2-item run limit") {
+ t.Fatalf("warnings = %v, want the item-count warning", budget.Warnings())
+ }
+}
+
+func TestBudgetDeclaredByteRejectionsDoNotConsumeItemSlots(t *testing.T) {
+ budget := testBudget(1, 8, 5)
+ if _, admission := budget.Reserve(9); admission != RejectedByBytes {
+ t.Fatalf("per-image rejection = %v, want byte rejection", admission)
+ }
+ if _, admission := budget.Reserve(6); admission != RejectedByBytes {
+ t.Fatalf("total-byte rejection = %v, want byte rejection", admission)
+ }
+ if !reserveAndCommit(t, budget, 1) {
+ t.Fatal("byte-rejected items exhausted the item budget")
+ }
+ if budget.Count() != 1 || budget.totalBytes != 1 {
+ t.Fatalf("items=%d bytes=%d, want 1 item and 1 byte", budget.Count(), budget.totalBytes)
+ }
+ if budget.oversizedImages != 1 || budget.totalOverflows != 1 {
+ t.Fatalf("oversized=%d overflows=%d, want 1 and 1", budget.oversizedImages, budget.totalOverflows)
+ }
+}
+
+func TestBudgetCommitRejectionsAreStable(t *testing.T) {
+ budget := testBudget(4, 4, 8)
+ reservation, admission := budget.Reserve(UnknownSize)
+ if admission != Admitted {
+ t.Fatalf("Reserve = %v, want admitted", admission)
+ }
+ for attempt := 0; attempt < 3; attempt++ {
+ if reservation.Commit(5) {
+ t.Fatal("Commit admitted an oversized payload")
+ }
+ }
+ if budget.oversizedImages != 1 || budget.totalBytes != 0 {
+ t.Fatalf("oversized=%d bytes=%d, want one oversized skip and no bytes", budget.oversizedImages, budget.totalBytes)
+ }
+}
+
+func TestBudgetTotalByteLimitDoesNotRecount(t *testing.T) {
+ budget := testBudget(4, 8, 5)
+ if !reserveAndCommit(t, budget, 3) {
+ t.Fatal("first payload should fit")
+ }
+ reservation, admission := budget.Reserve(UnknownSize)
+ if admission != Admitted {
+ t.Fatalf("second item admission = %v, want admitted", admission)
+ }
+ if reservation.Commit(3) {
+ t.Fatal("Commit admitted a payload past the total byte limit")
+ }
+ if reservation.Commit(3) {
+ t.Fatal("Commit recounted the rejected item")
+ }
+ if budget.totalOverflows != 1 || budget.totalBytes != 3 {
+ t.Fatalf("overflows=%d bytes=%d, want one overflow and 3 bytes", budget.totalOverflows, budget.totalBytes)
+ }
+}
+
+func TestBudgetOccurrenceConsumesCountBeforeByteAdmission(t *testing.T) {
+ budget := testBudget(1, 8, 5)
+ if _, admission := budget.ReserveOccurrence(6); admission != RejectedByBytes {
+ t.Fatalf("byte rejection = %v, want rejected by bytes", admission)
+ }
+ if budget.Count() != 1 {
+ t.Fatalf("count = %d, want the rejected occurrence to consume one slot", budget.Count())
+ }
+ if _, admission := budget.ReserveOccurrence(UnknownSize); admission != RejectedByCount {
+ t.Fatalf("next occurrence = %v, want rejected by count", admission)
+ }
+}
+
+func TestBudgetOccurrenceKeepsWalkingAfterByteRejection(t *testing.T) {
+ budget := testBudget(3, 4, 5)
+ reservation, admission := budget.ReserveOccurrence(UnknownSize)
+ if admission != Admitted || !reservation.Commit(3) {
+ t.Fatal("first occurrence should fit")
+ }
+ reservation, admission = budget.ReserveOccurrence(UnknownSize)
+ if admission != Admitted {
+ t.Fatalf("second occurrence admission = %v, want admitted", admission)
+ }
+ if reservation.Commit(3) {
+ t.Fatal("second occurrence exceeded the byte limit")
+ }
+ reservation, admission = budget.ReserveOccurrence(UnknownSize)
+ if admission != Admitted {
+ t.Fatalf("third occurrence admission = %v, want admitted", admission)
+ }
+ if !reservation.Commit(2) {
+ t.Fatal("third occurrence should fit after the byte rejection")
+ }
+ if budget.Count() != 3 || budget.totalBytes != 5 {
+ t.Fatalf("count=%d bytes=%d, want 3 and 5", budget.Count(), budget.totalBytes)
+ }
+}
+
+func TestBudgetExactByteLimits(t *testing.T) {
+ budget := testBudget(2, 4, 4)
+ if !reserveAndCommit(t, budget, 4) {
+ t.Fatal("exact-limit image should be admitted")
+ }
+ reservation, admission := budget.ReserveOccurrence(UnknownSize)
+ if admission != Admitted || reservation.Commit(1) {
+ t.Fatal("image after exact run limit should be omitted")
+ }
+}
+
+func TestBudgetInvalidSizes(t *testing.T) {
+ budget := testBudget(2, 8, 8)
+ if _, admission := budget.Reserve(-2); admission != RejectedInvalid {
+ t.Fatalf("invalid declared size = %v, want invalid rejection", admission)
+ }
+ reservation, admission := budget.Reserve(UnknownSize)
+ if admission != Admitted {
+ t.Fatalf("valid reservation = %v, want admitted", admission)
+ }
+ if reservation.Commit(-2) {
+ t.Fatal("Commit admitted an invalid actual size")
+ }
+}
+
+func TestBudgetWarningsAreStable(t *testing.T) {
+ budget := testBudget(3, 4, 5)
+ _, _ = budget.Reserve(5)
+ _, _ = budget.ReserveOccurrence(6)
+ _, _ = budget.Reserve(UnknownSize)
+ _, _ = budget.Reserve(UnknownSize)
+ _, _ = budget.Reserve(UnknownSize)
+ _, _ = budget.Reserve(UnknownSize)
+ want := []string{
+ "omitted 2 embedded image payload(s) larger than the 4-byte per-image limit",
+ "stopped extracting embedded images after the 3-item run limit",
+ }
+ if got := budget.Warnings(); !reflect.DeepEqual(got, want) {
+ t.Fatalf("warnings = %q, want %q", got, want)
+ }
+}
diff --git a/internal/parser/media/cache.go b/internal/parser/media/cache.go
index d310fb2996..022f112ef9 100644
--- a/internal/parser/media/cache.go
+++ b/internal/parser/media/cache.go
@@ -29,6 +29,11 @@ import (
// be returned to the task rather than treated as a missing image.
var ErrMediaNotFound = errors.New("media: reference not found")
+// ErrMediaLimitExceeded reports that a run-scoped media limit rejected a
+// write. Parser paths may skip the image and warn; every other cache error is
+// fatal to the task.
+var ErrMediaLimitExceeded = errors.New("media: run media limit exceeded")
+
// Cache stores decoded image bytes for materialization by later ingestion
// components. Implementations own key scoping, lifecycle bookkeeping and
// validation of the payload size and digest before returning bytes from Get.
@@ -54,8 +59,8 @@ func CacheFromContext(ctx context.Context) Cache {
// ValidatePayload performs the shared cache integrity and size checks.
func ValidatePayload(data []byte, ref entity.MediaRef) ([]byte, error) {
- if len(data) > HTMLImageMaxBytes() {
- return nil, fmt.Errorf("media: image %s exceeds the %d-byte limit", shortMediaDigest(ref.SHA256), HTMLImageMaxBytes())
+ if len(data) > ImageMaxBytes() {
+ return nil, fmt.Errorf("media: image %s exceeds the %d-byte limit", shortMediaDigest(ref.SHA256), ImageMaxBytes())
}
sum := sha256.Sum256(data)
if got := hex.EncodeToString(sum[:]); got != ref.SHA256 {
diff --git a/internal/parser/media/html.go b/internal/parser/media/datauri.go
similarity index 62%
rename from internal/parser/media/html.go
rename to internal/parser/media/datauri.go
index 44650b5cd1..a0431688a9 100644
--- a/internal/parser/media/html.go
+++ b/internal/parser/media/datauri.go
@@ -16,41 +16,30 @@ package media
import (
"crypto/sha256"
- "os"
- "strconv"
"strings"
)
-// DecodeHTMLImageSource validates and decodes one inline HTML image. The HTML
+// DecodeDataImageSource validates and decodes one inline image data URI. The
// parser calls this once, then writes the bytes to the run media cache; later
// ingestion components materialize only the emitted MediaRef.
-func DecodeHTMLImageSource(src string) (string, []byte, bool) {
- prefix, payload, ok := splitHTMLImageSource(src)
+func DecodeDataImageSource(src string) (string, []byte, bool) {
+ prefix, payload, ok := splitDataImageSource(src)
if !ok {
return "", nil, false
}
- data, ok := decodeBase64ImagePayload(payload, HTMLImageMaxBytes())
+ data, ok := decodeBase64ImagePayload(payload, ImageMaxBytes())
if !ok {
return "", nil, false
}
return DataURIMediaType(prefix), data, true
}
-// HTMLImageMaxBytes is the per-image limit for HTML media.
-func HTMLImageMaxBytes() int {
- const defaultLimit = 32 << 20
- if n, err := strconv.Atoi(os.Getenv("RAGFLOW_HTML_IMAGE_MAX_BYTES")); err == nil && n > 0 && n <= int(^uint(0)>>1)/4*3 {
- return n
- }
- return defaultLimit
-}
-
-// HTMLImageDecodedSize validates an image data URI and returns the decoded
+// DataImageDecodedSize validates an image data URI and returns the decoded
// payload size without allocating the decoded bytes. Malformed payloads are
-// rejected; a syntactically valid payload above HTMLImageMaxBytes is still
+// rejected; a syntactically valid payload above ImageMaxBytes is still
// reported by its size so callers can apply their own limit.
-func HTMLImageDecodedSize(src string) (int, bool) {
- _, payload, ok := splitHTMLImageSource(src)
+func DataImageDecodedSize(src string) (int, bool) {
+ _, payload, ok := splitDataImageSource(src)
if !ok {
return 0, false
}
@@ -62,11 +51,11 @@ func HTMLImageDecodedSize(src string) (int, bool) {
return padded/4*3 - (padded - encoded) - padding, true
}
-// splitHTMLImageSource validates the data-URI header and returns the normalized
+// splitDataImageSource validates the data-URI header and returns the normalized
// prefix and base64 payload.
-func splitHTMLImageSource(src string) (prefix, payload string, ok bool) {
+func splitDataImageSource(src string) (prefix, payload string, ok bool) {
src = strings.TrimSpace(src)
- if !IsHTMLDataImageSource(src) {
+ if !IsDataImageSource(src) {
return "", "", false
}
separator := strings.IndexByte(src, ',')
@@ -76,16 +65,16 @@ func splitHTMLImageSource(src string) (prefix, payload string, ok bool) {
return strings.ToLower(src[:separator+1]), src[separator+1:], true
}
-// NormalizeHTMLImageSource validates an inline image while avoiding a decoded
-// byte buffer. It remains the path for HTML layouts that intentionally keep
-// the normalized data URI inline rather than storing it in the run cache.
-func NormalizeHTMLImageSource(src string) (string, int64, [sha256.Size]byte, bool) {
+// NormalizeDataImageSource validates an inline image while avoiding a decoded
+// byte buffer. It remains the path for layouts that intentionally keep the
+// normalized data URI inline rather than storing it in the run cache.
+func NormalizeDataImageSource(src string) (string, int64, [sha256.Size]byte, bool) {
var sum [sha256.Size]byte
- prefix, payload, ok := splitHTMLImageSource(src)
+ prefix, payload, ok := splitDataImageSource(src)
if !ok {
return "", 0, sum, false
}
- normalized, size, sum, ok := NormalizeBase64ImagePayload(payload, MaxImagePayloadBytes)
+ normalized, size, sum, ok := NormalizeBase64ImagePayload(payload, ImageMaxBytes())
if !ok {
return "", 0, sum, false
}
@@ -103,8 +92,8 @@ func DataURIMediaType(uri string) string {
return strings.ToLower(uri[colon+1 : semi])
}
-// IsHTMLDataImageSource reports whether src is an image data URI.
-func IsHTMLDataImageSource(src string) bool {
+// IsDataImageSource reports whether src is an image data URI.
+func IsDataImageSource(src string) bool {
const dataPrefix = "data:image/"
src = strings.TrimSpace(src)
return len(src) >= len(dataPrefix) && strings.EqualFold(src[:len(dataPrefix)], dataPrefix)
diff --git a/internal/parser/media/html_test.go b/internal/parser/media/datauri_test.go
similarity index 64%
rename from internal/parser/media/html_test.go
rename to internal/parser/media/datauri_test.go
index 902def6482..363e6ca880 100644
--- a/internal/parser/media/html_test.go
+++ b/internal/parser/media/datauri_test.go
@@ -20,10 +20,10 @@ import (
"testing"
)
-func TestDecodeHTMLImageSourceCaseAndPadding(t *testing.T) {
- mime, data, ok := DecodeHTMLImageSource("DATA:IMAGE/PNG;BASE64,aGVsbG8")
+func TestDecodeDataImageSourceCaseAndPadding(t *testing.T) {
+ mime, data, ok := DecodeDataImageSource("DATA:IMAGE/PNG;BASE64,aGVsbG8")
if !ok {
- t.Fatal("DecodeHTMLImageSource rejected a valid case-insensitive data URI")
+ t.Fatal("DecodeDataImageSource rejected a valid case-insensitive data URI")
}
if mime != "image/png" {
t.Errorf("mime = %q, want image/png", mime)
@@ -33,7 +33,7 @@ func TestDecodeHTMLImageSourceCaseAndPadding(t *testing.T) {
}
}
-func TestDecodeHTMLImageSourceRejectsInvalidAndOversizedHeaders(t *testing.T) {
+func TestDecodeDataImageSourceRejectsInvalidAndOversizedHeaders(t *testing.T) {
for _, src := range []string{
"data:image/png;base64,",
"data:image/png;base64,YQ=",
@@ -41,21 +41,21 @@ func TestDecodeHTMLImageSourceRejectsInvalidAndOversizedHeaders(t *testing.T) {
"data:image/png;base64,G0==JbjQ",
"data:image/png;" + strings.Repeat("x", 200) + ",aGVsbG8=",
} {
- if _, _, ok := DecodeHTMLImageSource(src); ok {
- t.Errorf("DecodeHTMLImageSource(%q) succeeded", src)
+ if _, _, ok := DecodeDataImageSource(src); ok {
+ t.Errorf("DecodeDataImageSource(%q) succeeded", src)
}
}
}
-func TestNormalizeHTMLImageSourceMatchesDecode(t *testing.T) {
+func TestNormalizeDataImageSourceMatchesDecode(t *testing.T) {
src := "data:image/png;base64,aGVs\nbG8"
- normalized, size, normalizedSum, ok := NormalizeHTMLImageSource(src)
+ normalized, size, normalizedSum, ok := NormalizeDataImageSource(src)
if !ok {
- t.Fatal("NormalizeHTMLImageSource rejected a valid payload")
+ t.Fatal("NormalizeDataImageSource rejected a valid payload")
}
- mime, data, ok := DecodeHTMLImageSource(src)
+ mime, data, ok := DecodeDataImageSource(src)
if !ok {
- t.Fatal("DecodeHTMLImageSource rejected a valid payload")
+ t.Fatal("DecodeDataImageSource rejected a valid payload")
}
if normalized != "data:image/png;base64,aGVsbG8=" {
t.Fatalf("normalized = %q", normalized)
@@ -67,10 +67,10 @@ func TestNormalizeHTMLImageSourceMatchesDecode(t *testing.T) {
}
}
-// TestHTMLImageDecodedSizeMatchesDecode checks that the allocation-free
-// preflight accepts exactly what DecodeHTMLImageSource accepts and reports the
+// TestDataImageDecodedSizeMatchesDecode checks that the allocation-free
+// preflight accepts exactly what DecodeDataImageSource accepts and reports the
// same decoded size, so budget admission never diverges from the actual write.
-func TestHTMLImageDecodedSizeMatchesDecode(t *testing.T) {
+func TestDataImageDecodedSizeMatchesDecode(t *testing.T) {
cases := []struct {
src string
want int // -1 means the source is malformed
@@ -90,10 +90,10 @@ func TestHTMLImageDecodedSizeMatchesDecode(t *testing.T) {
{"data:image/png;" + strings.Repeat("x", 200) + ",aGVsbG8=", -1},
}
for _, tc := range cases {
- size, ok := HTMLImageDecodedSize(tc.src)
- _, data, decodeOK := DecodeHTMLImageSource(tc.src)
+ size, ok := DataImageDecodedSize(tc.src)
+ _, data, decodeOK := DecodeDataImageSource(tc.src)
if ok != (tc.want >= 0) {
- t.Errorf("%q: HTMLImageDecodedSize ok = %v, want %v", tc.src, ok, tc.want >= 0)
+ t.Errorf("%q: DataImageDecodedSize ok = %v, want %v", tc.src, ok, tc.want >= 0)
continue
}
if ok != decodeOK {
@@ -106,27 +106,16 @@ func TestHTMLImageDecodedSizeMatchesDecode(t *testing.T) {
}
}
-// TestHTMLImageDecodedSizeIgnoresPerImageLimit checks that the preflight
+// TestDataImageDecodedSizeIgnoresPerImageLimit checks that the preflight
// reports the size of a syntactically valid payload even when it exceeds the
// configured per-image limit, leaving the limit decision to the caller.
-func TestHTMLImageDecodedSizeIgnoresPerImageLimit(t *testing.T) {
- t.Setenv("RAGFLOW_HTML_IMAGE_MAX_BYTES", "2")
- size, ok := HTMLImageDecodedSize("data:image/png;base64,aGVsbG8=")
+func TestDataImageDecodedSizeIgnoresPerImageLimit(t *testing.T) {
+ t.Setenv("RAGFLOW_MEDIA_IMAGE_MAX_BYTES", "2")
+ size, ok := DataImageDecodedSize("data:image/png;base64,aGVsbG8=")
if !ok || size != 5 {
t.Fatalf("size, ok = %d, %v, want 5, true", size, ok)
}
- if size <= HTMLImageMaxBytes() {
- t.Fatalf("size %d does not exceed the configured limit %d", size, HTMLImageMaxBytes())
- }
-}
-
-func TestHTMLImageLimitDefaultsAndConfiguration(t *testing.T) {
- t.Setenv("RAGFLOW_HTML_IMAGE_MAX_BYTES", "")
- if got := HTMLImageMaxBytes(); got != 32<<20 {
- t.Fatalf("default = %d", got)
- }
- t.Setenv("RAGFLOW_HTML_IMAGE_MAX_BYTES", "67108864")
- if got := HTMLImageMaxBytes(); got != 64<<20 {
- t.Fatalf("configured = %d", got)
+ if size <= ImageMaxBytes() {
+ t.Fatalf("size %d does not exceed the configured limit %d", size, ImageMaxBytes())
}
}
diff --git a/internal/parser/media/limits.go b/internal/parser/media/limits.go
new file mode 100644
index 0000000000..2b5eb28695
--- /dev/null
+++ b/internal/parser/media/limits.go
@@ -0,0 +1,75 @@
+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package media
+
+import (
+ "os"
+ "strconv"
+
+ "ragflow/internal/common"
+)
+
+const (
+ defaultImageMaxBytes = 32 << 20
+ defaultMediaMaxImagesPerRun = 1024
+ defaultMediaMaxBytesPerRun = int64(1 << 30)
+)
+
+// Limits is the run media quota snapshot shared by parser budgets.
+type Limits struct {
+ ImageMaxBytes int64
+ MaxImagesPerRun int
+ MaxBytesPerRun int64
+}
+
+// LoadLimits reads the configured media limits once for a parser run.
+func LoadLimits() Limits {
+ return Limits{
+ ImageMaxBytes: int64(ImageMaxBytes()),
+ MaxImagesPerRun: MediaMaxImagesPerRun(),
+ MaxBytesPerRun: MediaMaxBytesPerRun(),
+ }
+}
+
+// ImageMaxBytes is the single per-image limit shared by every media path:
+// data URI decoding, ZIP entry extraction, picture parsing, vision images and
+// the run media cache. Set RAGFLOW_MEDIA_IMAGE_MAX_BYTES to a positive byte
+// count to change it.
+func ImageMaxBytes() int {
+ if n, err := strconv.Atoi(os.Getenv(common.EnvMediaImageMaxBytes)); err == nil && n > 0 && n <= int(^uint(0)>>1)/4*3 {
+ return n
+ }
+ return defaultImageMaxBytes
+}
+
+// MediaMaxImagesPerRun bounds the number of distinct images one run may keep
+// in the media cache. Set RAGFLOW_MEDIA_MAX_IMAGES_PER_RUN to a positive count
+// to change it.
+func MediaMaxImagesPerRun() int {
+ if n, err := strconv.Atoi(os.Getenv(common.EnvMediaMaxImagesPerRun)); err == nil && n > 0 {
+ return n
+ }
+ return defaultMediaMaxImagesPerRun
+}
+
+// MediaMaxBytesPerRun bounds the total decoded image bytes one run may keep in
+// the media cache. Set RAGFLOW_MEDIA_MAX_BYTES_PER_RUN to a positive byte count
+// to change it.
+func MediaMaxBytesPerRun() int64 {
+ if n, err := strconv.ParseInt(os.Getenv(common.EnvMediaMaxBytesPerRun), 10, 64); err == nil && n > 0 {
+ return n
+ }
+ return defaultMediaMaxBytesPerRun
+}
diff --git a/internal/parser/media/limits_test.go b/internal/parser/media/limits_test.go
new file mode 100644
index 0000000000..9ef172ab65
--- /dev/null
+++ b/internal/parser/media/limits_test.go
@@ -0,0 +1,65 @@
+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package media
+
+import "testing"
+
+func TestImageMaxBytesDefaultsAndConfiguration(t *testing.T) {
+ t.Setenv("RAGFLOW_MEDIA_IMAGE_MAX_BYTES", "")
+ if got := ImageMaxBytes(); got != 32<<20 {
+ t.Fatalf("default = %d, want %d", got, 32<<20)
+ }
+ t.Setenv("RAGFLOW_MEDIA_IMAGE_MAX_BYTES", "67108864")
+ if got := ImageMaxBytes(); got != 64<<20 {
+ t.Fatalf("configured = %d, want %d", got, 64<<20)
+ }
+ for _, invalid := range []string{"0", "-1", "not-a-number"} {
+ t.Setenv("RAGFLOW_MEDIA_IMAGE_MAX_BYTES", invalid)
+ if got := ImageMaxBytes(); got != 32<<20 {
+ t.Fatalf("invalid %q = %d, want the default", invalid, got)
+ }
+ }
+}
+
+func TestMediaRunLimitsDefaultsAndConfiguration(t *testing.T) {
+ t.Setenv("RAGFLOW_MEDIA_MAX_IMAGES_PER_RUN", "")
+ t.Setenv("RAGFLOW_MEDIA_MAX_BYTES_PER_RUN", "")
+ if got := MediaMaxImagesPerRun(); got != 1024 {
+ t.Fatalf("default images = %d, want 1024", got)
+ }
+ if got := MediaMaxBytesPerRun(); got != 1<<30 {
+ t.Fatalf("default bytes = %d, want %d", got, 1<<30)
+ }
+
+ t.Setenv("RAGFLOW_MEDIA_MAX_IMAGES_PER_RUN", "7")
+ t.Setenv("RAGFLOW_MEDIA_MAX_BYTES_PER_RUN", "2048")
+ if got := MediaMaxImagesPerRun(); got != 7 {
+ t.Fatalf("configured images = %d, want 7", got)
+ }
+ if got := MediaMaxBytesPerRun(); got != 2048 {
+ t.Fatalf("configured bytes = %d, want 2048", got)
+ }
+
+ for _, invalid := range []string{"0", "-1", "not-a-number"} {
+ t.Setenv("RAGFLOW_MEDIA_MAX_IMAGES_PER_RUN", invalid)
+ t.Setenv("RAGFLOW_MEDIA_MAX_BYTES_PER_RUN", invalid)
+ if got := MediaMaxImagesPerRun(); got != 1024 {
+ t.Fatalf("invalid images %q = %d, want 1024", invalid, got)
+ }
+ if got := MediaMaxBytesPerRun(); got != 1<<30 {
+ t.Fatalf("invalid bytes %q = %d, want %d", invalid, got, int64(1<<30))
+ }
+ }
+}
diff --git a/internal/parser/media/payload.go b/internal/parser/media/payload.go
index c0aec5ef90..e626585523 100644
--- a/internal/parser/media/payload.go
+++ b/internal/parser/media/payload.go
@@ -23,12 +23,8 @@ import (
"strings"
)
-const (
- // MaxImagePayloadBytes bounds compressed image data accepted by media paths.
- MaxImagePayloadBytes = 32 << 20
- // maxStreamBuffer caps the copy buffer used while validating a payload.
- maxStreamBuffer = 32 << 10
-)
+// maxStreamBuffer caps the copy buffer used while validating a payload.
+const maxStreamBuffer = 32 << 10
// NormalizeBase64ImagePayload validates an external image payload within its
// decoded byte limit and returns standard base64 with padding, no CR/LF, the
diff --git a/internal/parser/parser/doc_parser.go b/internal/parser/parser/doc_parser.go
index 1daeb0394b..766ca90694 100644
--- a/internal/parser/parser/doc_parser.go
+++ b/internal/parser/parser/doc_parser.go
@@ -22,6 +22,7 @@ import (
"strings"
deepdocoffice "ragflow/internal/deepdoc/parser/office"
+ "ragflow/internal/parser/media"
)
// docExtract is the native office_oxide extraction seam for the DOC family.
@@ -60,12 +61,12 @@ func (p *DOCParser) ParseWithResult(ctx context.Context, filename string, data [
// usable sections, so a document whose IR is empty/unusable still keeps its
// content (PlainText / ToMarkdown) rather than silently producing 0 chunks.
if strings.TrimSpace(irJSON) != "" {
- budget := newEmbeddedMediaBudget()
+ budget := media.NewBudget()
if sections := buildDOCXJSONSections(irJSON, budget); len(sections) > 0 {
return ParseResult{
File: map[string]any{"name": filename, "format": "doc"},
JSON: MapsToItems(sections),
- Warnings: budget.warnings(),
+ Warnings: budget.Warnings(),
}
}
}
diff --git a/internal/parser/parser/docx_caption_test.go b/internal/parser/parser/docx_caption_test.go
index 362dc1c2c5..84ec46f4bd 100644
--- a/internal/parser/parser/docx_caption_test.go
+++ b/internal/parser/parser/docx_caption_test.go
@@ -39,7 +39,7 @@ func TestBuildDOCXJSONSections_TableCaption_ByHeading(t *testing.T) {
`{"type":"heading","level":1,"content":[{"type":"text","text":"设备台账"}]}`,
docxCaptionTableBody,
)
- sections := buildDOCXJSONSections(ir, newEmbeddedMediaBudget())
+ sections := buildDOCXJSONSections(ir, newMediaBudgetForTest())
if got := docxCaptionFindTableCaption(t, sections); got != "Table Location: 设备台账" {
t.Errorf("caption = %q, want %q", got, "Table Location: 设备台账")
}
@@ -55,7 +55,7 @@ func TestBuildDOCXJSONSections_TableCaption_NestedHeading(t *testing.T) {
`{"type":"heading","level":3,"content":[{"type":"text","text":"H3"}]}`,
docxCaptionTableBody,
)
- sections := buildDOCXJSONSections(ir, newEmbeddedMediaBudget())
+ sections := buildDOCXJSONSections(ir, newMediaBudgetForTest())
if got := docxCaptionFindTableCaption(t, sections); got != "Table Location: H1 > H2 > H3" {
t.Errorf("caption = %q, want %q", got, "Table Location: H1 > H2 > H3")
}
@@ -67,7 +67,7 @@ func TestBuildDOCXJSONSections_TableCaption_HeadingAfterTable(t *testing.T) {
docxCaptionTableBody,
`{"type":"heading","level":1,"content":[{"type":"text","text":"后续标题"}]}`,
)
- sections := buildDOCXJSONSections(ir, newEmbeddedMediaBudget())
+ sections := buildDOCXJSONSections(ir, newMediaBudgetForTest())
if got := docxCaptionFindTableCaption(t, sections); got != "" {
t.Errorf("table should have no caption when heading is after it, got %q", got)
}
@@ -83,7 +83,7 @@ func TestBuildDOCXJSONSections_TableCaption_SiblingHeadingSupersedes(t *testing.
`{"type":"heading","level":2,"content":[{"type":"text","text":"H2b"}]}`,
docxCaptionTableBody,
)
- sections := buildDOCXJSONSections(ir, newEmbeddedMediaBudget())
+ sections := buildDOCXJSONSections(ir, newMediaBudgetForTest())
if got := docxCaptionFindTableCaption(t, sections); got != "Table Location: H1 > H2b" {
t.Errorf("caption = %q, want %q", got, "Table Location: H1 > H2b")
}
@@ -99,7 +99,7 @@ func TestBuildDOCXJSONSections_TableCaption_DeeperHeadingAfterSibling(t *testing
`{"type":"heading","level":3,"content":[{"type":"text","text":"H3b"}]}`,
docxCaptionTableBody,
)
- sections := buildDOCXJSONSections(ir, newEmbeddedMediaBudget())
+ sections := buildDOCXJSONSections(ir, newMediaBudgetForTest())
if got := docxCaptionFindTableCaption(t, sections); got != "Table Location: H2b > H3b" {
t.Errorf("caption = %q, want %q", got, "Table Location: H2b > H3b")
}
@@ -113,7 +113,7 @@ func TestBuildDOCXJSONSections_TableCaption_EmptyHeadingIgnored(t *testing.T) {
`{"type":"heading","level":2,"content":[{"type":"text","text":" "}]}`,
docxCaptionTableBody,
)
- sections := buildDOCXJSONSections(ir, newEmbeddedMediaBudget())
+ sections := buildDOCXJSONSections(ir, newMediaBudgetForTest())
if got := docxCaptionFindTableCaption(t, sections); got != "Table Location: H1" {
t.Errorf("caption = %q, want %q", got, "Table Location: H1")
}
@@ -126,7 +126,7 @@ func TestBuildDOCXJSONSections_TableCaption_OutlineSpansSections(t *testing.T) {
{"title":"","elements":[{"type":"heading","level":1,"content":[{"type":"text","text":"第一章"}]}]},
{"title":"","elements":[` + docxCaptionTableBody + `]}
]}`
- sections := buildDOCXJSONSections(ir, newEmbeddedMediaBudget())
+ sections := buildDOCXJSONSections(ir, newMediaBudgetForTest())
if got := docxCaptionFindTableCaption(t, sections); got != "Table Location: 第一章" {
t.Errorf("caption = %q, want %q", got, "Table Location: 第一章")
}
@@ -139,7 +139,7 @@ func TestBuildDOCXJSONSections_TableCaption_PlainParagraphNotCaption(t *testing.
`{"type":"paragraph","content":[{"type":"text","text":"设备流转明细"}]}`,
docxCaptionTableBody,
)
- sections := buildDOCXJSONSections(ir, newEmbeddedMediaBudget())
+ sections := buildDOCXJSONSections(ir, newMediaBudgetForTest())
if got := docxCaptionFindTableCaption(t, sections); got != "" {
t.Errorf("plain paragraph must not become a caption, got %q", got)
}
@@ -147,7 +147,7 @@ func TestBuildDOCXJSONSections_TableCaption_PlainParagraphNotCaption(t *testing.
// Corner: empty IR yields no sections.
func TestBuildDOCXJSONSections_TableCaption_Empty(t *testing.T) {
- sections := buildDOCXJSONSections(`{"sections":[]}`, newEmbeddedMediaBudget())
+ sections := buildDOCXJSONSections(`{"sections":[]}`, newMediaBudgetForTest())
if len(sections) != 0 {
t.Errorf("empty IR should yield no sections, got %d", len(sections))
}
@@ -161,7 +161,7 @@ func TestBuildDOCXJSONSections_TableCaption_ExplicitPreferredOverLocation(t *tes
`{"type":"heading","level":1,"content":[{"type":"text","text":"设备台账"}]}`,
docxCaptionTableBody,
)
- sections := buildDOCXJSONSectionsWithCaptions(ir, newEmbeddedMediaBudget(), []string{"设备流转明细"})
+ sections := buildDOCXJSONSectionsWithCaptions(ir, newMediaBudgetForTest(), []string{"设备流转明细"})
if got := docxCaptionFindTableCaption(t, sections); got != "设备流转明细" {
t.Errorf("explicit caption = %q, want %q", got, "设备流转明细")
}
@@ -170,7 +170,7 @@ func TestBuildDOCXJSONSections_TableCaption_ExplicitPreferredOverLocation(t *tes
// An explicit caption is attached even when the table sits under no heading.
func TestBuildDOCXJSONSections_TableCaption_ExplicitWithNoHeading(t *testing.T) {
ir := docxCaptionTableIR(docxCaptionTableBody)
- sections := buildDOCXJSONSectionsWithCaptions(ir, newEmbeddedMediaBudget(), []string{"设备流转明细"})
+ sections := buildDOCXJSONSectionsWithCaptions(ir, newMediaBudgetForTest(), []string{"设备流转明细"})
if got := docxCaptionFindTableCaption(t, sections); got != "设备流转明细" {
t.Errorf("explicit caption = %q, want %q", got, "设备流转明细")
}
@@ -182,7 +182,7 @@ func TestBuildDOCXJSONSections_TableCaption_BlankDoesNotOverrideLocation(t *test
`{"type":"heading","level":1,"content":[{"type":"text","text":"设备台账"}]}`,
docxCaptionTableBody,
)
- sections := buildDOCXJSONSectionsWithCaptions(ir, newEmbeddedMediaBudget(), []string{""})
+ sections := buildDOCXJSONSectionsWithCaptions(ir, newMediaBudgetForTest(), []string{""})
if got := docxCaptionFindTableCaption(t, sections); got != "Table Location: 设备台账" {
t.Errorf("caption = %q, want %q", got, "Table Location: 设备台账")
}
diff --git a/internal/parser/parser/docx_ir.go b/internal/parser/parser/docx_ir.go
index dfe3707ad1..3edcbe39a5 100644
--- a/internal/parser/parser/docx_ir.go
+++ b/internal/parser/parser/docx_ir.go
@@ -31,6 +31,7 @@ import (
"strings"
"ragflow/internal/entity"
+ "ragflow/internal/parser/media"
"ragflow/internal/parser/tableutil"
)
@@ -382,7 +383,7 @@ func docxHeadingLocationCaption(headings []docxHeading) string {
// This is the caption-only (heading hierarchy) entry point. Use
// buildDOCXJSONSectionsWithCaptions to also attach Caption-styled paragraph
// text (a temporary stopgap; see docx_caption_scan.go).
-func buildDOCXJSONSections(irJSON string, budget *embeddedMediaBudget) []map[string]any {
+func buildDOCXJSONSections(irJSON string, budget *media.Budget) []map[string]any {
return buildDOCXJSONSectionsCore(irJSON, budget, nil)
}
@@ -392,7 +393,7 @@ func buildDOCXJSONSections(irJSON string, budget *embeddedMediaBudget) []map[str
//
// TEMPORARY: remove once office_oxide emits paragraph styles
// (yfedoseev/office_oxide#455).
-func buildDOCXJSONSectionsWithCaptions(irJSON string, budget *embeddedMediaBudget, tableCaptions []string) []map[string]any {
+func buildDOCXJSONSectionsWithCaptions(irJSON string, budget *media.Budget, tableCaptions []string) []map[string]any {
return buildDOCXJSONSectionsCore(irJSON, budget, tableCaptions)
}
@@ -403,13 +404,13 @@ func buildDOCXJSONSectionsWithCaptions(irJSON string, budget *embeddedMediaBudge
// between tableCaptions and the IR tables is approximate (document-body order
// vs office_oxide IR order) and is a known limitation of the temporary
// Caption-style stopgap in docx_caption_scan.go.
-func buildDOCXJSONSectionsCore(irJSON string, budget *embeddedMediaBudget, tableCaptions []string) []map[string]any {
+func buildDOCXJSONSectionsCore(irJSON string, budget *media.Budget, tableCaptions []string) []map[string]any {
var ir docxIRDocument
if err := json.Unmarshal([]byte(irJSON), &ir); err != nil {
return nil
}
if budget == nil {
- budget = newEmbeddedMediaBudget()
+ budget = media.NewBudget()
}
var sections []map[string]any
tableSequence := 0
@@ -539,12 +540,14 @@ func buildDOCXJSONSectionsCore(irJSON string, budget *embeddedMediaBudget, table
}
// appendOfficeTableSections emits a table and its embedded images together.
-func appendOfficeTableSections(sections []map[string]any, el docxIRElement, caption, tableID string, budget *embeddedMediaBudget) []map[string]any {
+func appendOfficeTableSections(sections []map[string]any, el docxIRElement, caption, tableID string, budget *media.Budget) []map[string]any {
table := NewTableDataJSONItem(officeIRTableData(el, caption), "", nil).ToMap()
var tableImages []docxTableImage
forEachDOCXTableImage(el, func(tableImage docxTableImage) bool {
- included, keepWalking := budget.include(tableImage.data)
+ reservation, admission := budget.ReserveOccurrence(media.UnknownSize)
+ keepWalking := admission != media.RejectedByCount
+ included := admission == media.Admitted && reservation.Commit(media.Size(len(tableImage.data)))
if included || keepWalking {
tableImage.included = included
tableImages = append(tableImages, tableImage)
@@ -572,7 +575,7 @@ func appendOfficeTableSections(sections []map[string]any, el docxIRElement, capt
// order. A paragraph is not necessarily a single text unit: office_oxide can
// place an image between two text runs, and dropping that run would make the
// downstream DOCX chunker lose both media order and adjacency context.
-func appendDOCXParagraphSections(sections []map[string]any, el docxIRElement, heading bool, budget *embeddedMediaBudget) []map[string]any {
+func appendDOCXParagraphSections(sections []map[string]any, el docxIRElement, heading bool, budget *media.Budget) []map[string]any {
var text strings.Builder
flushText := func() {
value := strings.TrimSpace(text.String())
@@ -611,11 +614,13 @@ func appendDOCXParagraphSections(sections []map[string]any, el docxIRElement, he
return sections
}
-func appendDOCXImageSection(sections []map[string]any, data []byte, budget *embeddedMediaBudget, metadata map[string]any) ([]map[string]any, bool) {
+func appendDOCXImageSection(sections []map[string]any, data []byte, budget *media.Budget, metadata map[string]any) ([]map[string]any, bool) {
if len(data) == 0 {
return sections, true
}
- included, keepWalking := budget.include(data)
+ reservation, admission := budget.ReserveOccurrence(media.UnknownSize)
+ keepWalking := admission != media.RejectedByCount
+ included := admission == media.Admitted && reservation.Commit(media.Size(len(data)))
if !included && !keepWalking {
return sections, false
}
@@ -649,13 +654,13 @@ func appendDOCXImagePayload(sections []map[string]any, data []byte, included boo
//
// Reuses the IR already obtained from the doc handle in
// ParseWithResult so the binary is not opened twice.
-func extractDOCXFiguresFromIR(irJSON string, budget *embeddedMediaBudget) []DOCXFigure {
+func extractDOCXFiguresFromIR(irJSON string, budget *media.Budget) []DOCXFigure {
var ir docxIRDocument
if err := json.Unmarshal([]byte(irJSON), &ir); err != nil {
return nil
}
if budget == nil {
- budget = newEmbeddedMediaBudget()
+ budget = media.NewBudget()
}
var flat []flatBlock
@@ -668,7 +673,9 @@ func extractDOCXFiguresFromIR(irJSON string, budget *embeddedMediaBudget) []DOCX
if len(block.imageData) == 0 {
continue
}
- included, keepWalking := budget.include(block.imageData)
+ reservation, admission := budget.ReserveOccurrence(media.UnknownSize)
+ keepWalking := admission != media.RejectedByCount
+ included := admission == media.Admitted && reservation.Commit(media.Size(len(block.imageData)))
if !included && !keepWalking {
break
}
diff --git a/internal/parser/parser/docx_ir_test.go b/internal/parser/parser/docx_ir_test.go
index 866ee36097..efe23b41c4 100644
--- a/internal/parser/parser/docx_ir_test.go
+++ b/internal/parser/parser/docx_ir_test.go
@@ -27,7 +27,7 @@ func TestBuildDOCXJSONSections_FromJSON(t *testing.T) {
{"type":"list","items":[{"content":[{"type":"paragraph","content":[{"type":"text","text":"item1"}]}]}]}
]}]}`
- got := buildDOCXJSONSections(irJSON, newEmbeddedMediaBudget())
+ got := buildDOCXJSONSections(irJSON, newMediaBudgetForTest())
want := []map[string]any{
{"text": "Title", "image": nil, "doc_type_kwd": "text", "layout": "heading"},
{"text": "Hello", "image": nil, "doc_type_kwd": "text"},
@@ -49,7 +49,7 @@ func TestBuildDOCXJSONSections_EmptyTableSkipped(t *testing.T) {
{"type":"paragraph","content":[{"type":"text","text":"keep"}]},
{"type":"table","rows":[]}
]}]}`
- got := buildDOCXJSONSections(irJSON, newEmbeddedMediaBudget())
+ got := buildDOCXJSONSections(irJSON, newMediaBudgetForTest())
if len(got) != 1 || got[0]["text"] != "keep" {
t.Fatalf("expected only the paragraph to survive, got %+v", got)
}
@@ -71,7 +71,7 @@ func TestBuildDOCXJSONSections_MergesSplitDocTable(t *testing.T) {
{"type":"table","rows":[{"cells":[%s,%s,%s]},{"cells":[%s,%s,%s]}]}
]}]}`, cell, cell, cell, cell, cell, cell, cell, cell, cell)
- got := buildDOCXJSONSections(irJSON, newEmbeddedMediaBudget())
+ got := buildDOCXJSONSections(irJSON, newMediaBudgetForTest())
var tables []map[string]any
for _, it := range got {
if dt, _ := it["doc_type_kwd"].(string); dt == "table" {
@@ -103,7 +103,7 @@ func TestBuildDOCXJSONSections_DifferentColumnTablesStaySeparate(t *testing.T) {
{"type":"table","rows":[{"cells":[%s,%s,%s]}]},
{"type":"table","rows":[{"cells":[%s,%s]},{"cells":[%s,%s]}]}
]}]}`, cell, cell, cell, cell, cell, cell, cell)
- got := buildDOCXJSONSections(irJSON, newEmbeddedMediaBudget())
+ got := buildDOCXJSONSections(irJSON, newMediaBudgetForTest())
var n int
for _, it := range got {
if dt, _ := it["doc_type_kwd"].(string); dt == "table" {
@@ -128,7 +128,7 @@ func TestBuildDOCXJSONSections_AdjacentMultiRowTablesStaySeparate(t *testing.T)
{"type":"table","rows":[{"cells":[%s,%s,%s]},{"cells":[%s,%s,%s]}]},
{"type":"table","rows":[{"cells":[%s,%s,%s]},{"cells":[%s,%s,%s]}]}
]}]}`, cell, cell, cell, cell, cell, cell, cell, cell, cell, cell, cell, cell)
- got := buildDOCXJSONSections(irJSON, newEmbeddedMediaBudget())
+ got := buildDOCXJSONSections(irJSON, newMediaBudgetForTest())
var tables []map[string]any
for _, it := range got {
if dt, _ := it["doc_type_kwd"].(string); dt == "table" {
@@ -160,7 +160,7 @@ func TestBuildDOCXJSONSections_HeaderFragmentWithParagraphBetweenStaysSplit(t *t
{"type":"paragraph","content":[{"type":"text","text":"note"}]},
{"type":"table","rows":[{"cells":[%s,%s,%s]},{"cells":[%s,%s,%s]}]}
]}]}`, cell, cell, cell, cell, cell, cell, cell, cell, cell)
- got := buildDOCXJSONSections(irJSON, newEmbeddedMediaBudget())
+ got := buildDOCXJSONSections(irJSON, newMediaBudgetForTest())
var n int
for _, it := range got {
if dt, _ := it["doc_type_kwd"].(string); dt == "table" {
@@ -180,7 +180,7 @@ func TestBuildDOCXJSONSections_PreservesInlineImageOrder(t *testing.T) {
{"type":"text","text":"after"}
]}
]}]}`
- got := buildDOCXJSONSections(irJSON, newEmbeddedMediaBudget())
+ got := buildDOCXJSONSections(irJSON, newMediaBudgetForTest())
if len(got) != 3 {
t.Fatalf("sections = %+v, want text/image/text sequence", got)
}
@@ -207,7 +207,7 @@ func TestBuildDOCXJSONSections_ExtractsTableCellImages(t *testing.T) {
]}]}
]}]}`
- got := buildDOCXJSONSections(irJSON, newEmbeddedMediaBudget())
+ got := buildDOCXJSONSections(irJSON, newMediaBudgetForTest())
if len(got) != 3 {
t.Fatalf("sections = %+v, want table plus two cell image items", got)
}
@@ -234,7 +234,7 @@ func TestBuildDOCXJSONSections_BudgetKeepsInlineTextAroundOmittedImage(t *testin
{"type":"image","data":"YWJjZA=="},
{"type":"text","text":"after"}
]}]}]}`
- budget := &embeddedMediaBudget{maxImageBytes: 3, maxTotalBytes: 5, maxItems: 10}
+ budget := newLimitedMediaBudgetForTest(t, 3, 5, 10)
items := buildDOCXJSONSections(irJSON, budget)
if len(items) != 3 {
@@ -246,7 +246,7 @@ func TestBuildDOCXJSONSections_BudgetKeepsInlineTextAroundOmittedImage(t *testin
if items[1]["doc_type_kwd"] != "image" || items[1]["image"] != nil || items[1]["media_omitted"] != true {
t.Fatalf("omitted inline image item = %+v, want metadata without payload", items[1])
}
- if len(budget.warnings()) == 0 {
+ if len(budget.Warnings()) == 0 {
t.Fatal("omitted image should produce a parser warning")
}
}
@@ -256,7 +256,7 @@ func TestBuildDOCXJSONSections_BudgetKeepsTableImageMetadata(t *testing.T) {
{"content":[{"type":"image","data":"YWJj"}]},
{"content":[{"type":"image","data":"ZGVm"}]}
]}]}]}]}`
- budget := &embeddedMediaBudget{maxImageBytes: 4, maxTotalBytes: 5, maxItems: 10}
+ budget := newLimitedMediaBudgetForTest(t, 4, 5, 10)
items := buildDOCXJSONSections(irJSON, budget)
if len(items) != 3 {
@@ -275,9 +275,11 @@ func TestBuildDOCXJSONSections_BudgetKeepsTableImageMetadata(t *testing.T) {
}
func TestBuildDOCXJSONSections_BudgetCapsMediaCount(t *testing.T) {
+ const countLimit = 8
+ budget := newLimitedMediaBudgetForTest(t, 8, 1024, countLimit)
var ir strings.Builder
ir.WriteString(`{"sections":[{"elements":[`)
- for i := 0; i < 257; i++ {
+ for i := 0; i <= countLimit; i++ {
if i > 0 {
ir.WriteByte(',')
}
@@ -285,7 +287,6 @@ func TestBuildDOCXJSONSections_BudgetCapsMediaCount(t *testing.T) {
}
ir.WriteString(`]}]}`)
- budget := newEmbeddedMediaBudget()
items := buildDOCXJSONSections(ir.String(), budget)
imageCount := 0
for _, item := range items {
@@ -293,10 +294,10 @@ func TestBuildDOCXJSONSections_BudgetCapsMediaCount(t *testing.T) {
imageCount++
}
}
- if imageCount != budget.maxItems {
- t.Fatalf("image items = %d, want count limit %d", imageCount, budget.maxItems)
+ if imageCount != countLimit {
+ t.Fatalf("image items = %d, want count limit %d", imageCount, countLimit)
}
- if len(budget.warnings()) == 0 {
+ if len(budget.Warnings()) == 0 {
t.Fatal("truncated media count should produce a parser warning")
}
}
@@ -306,7 +307,7 @@ func TestBuildPPTXJSONSections_BudgetKeepsSlideMetadata(t *testing.T) {
{"type":"paragraph","content":[{"type":"text","text":"slide text"}]},
{"type":"image","data":"YWJjZA=="}
]}]}`
- budget := &embeddedMediaBudget{maxImageBytes: 3, maxTotalBytes: 5, maxItems: 10}
+ budget := newLimitedMediaBudgetForTest(t, 3, 5, 10)
items, err := buildPPTXJSONSections(irJSON, budget)
if err != nil {
@@ -325,9 +326,11 @@ func TestBuildPPTXJSONSections_BudgetKeepsSlideMetadata(t *testing.T) {
}
func TestBuildPPTXJSONSections_BudgetCapsMediaCount(t *testing.T) {
+ const countLimit = 8
+ budget := newLimitedMediaBudgetForTest(t, 8, 1024, countLimit)
var ir strings.Builder
ir.WriteString(`{"sections":[{"elements":[`)
- for i := 0; i < 257; i++ {
+ for i := 0; i <= countLimit; i++ {
if i > 0 {
ir.WriteByte(',')
}
@@ -335,7 +338,6 @@ func TestBuildPPTXJSONSections_BudgetCapsMediaCount(t *testing.T) {
}
ir.WriteString(`]}]}`)
- budget := newEmbeddedMediaBudget()
items, err := buildPPTXJSONSections(ir.String(), budget)
if err != nil {
t.Fatalf("buildPPTXJSONSections: %v", err)
@@ -346,26 +348,26 @@ func TestBuildPPTXJSONSections_BudgetCapsMediaCount(t *testing.T) {
imageCount++
}
}
- if imageCount != budget.maxItems {
- t.Fatalf("image items = %d, want count limit %d", imageCount, budget.maxItems)
+ if imageCount != countLimit {
+ t.Fatalf("image items = %d, want count limit %d", imageCount, countLimit)
}
- if len(budget.warnings()) == 0 {
+ if len(budget.Warnings()) == 0 {
t.Fatal("truncated media count should produce a parser warning")
}
}
-func TestEmbeddedMediaBudgetRejectsPerImageAndDocumentByteLimits(t *testing.T) {
- budget := &embeddedMediaBudget{maxImageBytes: 4, maxTotalBytes: 5, maxItems: 4}
- if included, keepGoing := budget.include([]byte("abc")); !included || !keepGoing {
+func TestMediaBudgetRejectsPerImageAndDocumentByteLimits(t *testing.T) {
+ budget := newLimitedMediaBudgetForTest(t, 4, 5, 4)
+ if included, keepGoing := includeMediaForTest(budget, []byte("abc")); !included || !keepGoing {
t.Fatalf("first image admission = (%v, %v), want (true, true)", included, keepGoing)
}
- if included, keepGoing := budget.include([]byte("def")); included || !keepGoing {
+ if included, keepGoing := includeMediaForTest(budget, []byte("def")); included || !keepGoing {
t.Fatalf("aggregate overflow admission = (%v, %v), want (false, true)", included, keepGoing)
}
- if included, keepGoing := budget.include([]byte("12345")); included || !keepGoing {
+ if included, keepGoing := includeMediaForTest(budget, []byte("12345")); included || !keepGoing {
t.Fatalf("per-image overflow admission = (%v, %v), want (false, true)", included, keepGoing)
}
- if len(budget.warnings()) == 0 {
+ if len(budget.Warnings()) == 0 {
t.Fatal("omitted byte payloads should produce a parser warning")
}
}
@@ -374,7 +376,7 @@ func TestBuildDOCXJSONSectionsEncodesOnlyAcceptedImageData(t *testing.T) {
data := []byte("text is not a valid image, but the parser preserves it")
encoded := base64.StdEncoding.EncodeToString(data)
irJSON := fmt.Sprintf(`{"sections":[{"elements":[{"type":"image","data":%q}]}]}`, encoded)
- budget := &embeddedMediaBudget{maxImageBytes: 8, maxTotalBytes: 8, maxItems: 4}
+ budget := newLimitedMediaBudgetForTest(t, 8, 8, 4)
items := buildDOCXJSONSections(irJSON, budget)
if len(items) != 1 || items[0]["image"] != nil || items[0]["media_omitted"] != true {
t.Fatalf("oversized image items = %+v, want metadata without payload", items)
@@ -420,7 +422,7 @@ func TestExtractDOCXFiguresFromIR(t *testing.T) {
{"type":"paragraph","content":[{"type":"text","text":"after"}]}
]}]}`
- figs := extractDOCXFiguresFromIR(irJSON, newEmbeddedMediaBudget())
+ figs := extractDOCXFiguresFromIR(irJSON, newMediaBudgetForTest())
if len(figs) != 1 {
t.Fatalf("expected 1 figure, got %d", len(figs))
}
@@ -444,7 +446,7 @@ func TestExtractDOCXFiguresFromIR_InlineImage(t *testing.T) {
{"type":"text","text":"after"}
]}
]}]}`
- figs := extractDOCXFiguresFromIR(irJSON, newEmbeddedMediaBudget())
+ figs := extractDOCXFiguresFromIR(irJSON, newMediaBudgetForTest())
if len(figs) != 1 {
t.Fatalf("expected 1 inline figure, got %d", len(figs))
}
@@ -459,7 +461,7 @@ func TestExtractDOCXFiguresFromIR_BudgetKeepsContextWithoutPayload(t *testing.T)
{"type":"image","data":"YWJjZA=="},
{"type":"paragraph","content":[{"type":"text","text":"after"}]}
]}]}`
- budget := &embeddedMediaBudget{maxImageBytes: 3, maxTotalBytes: 5, maxItems: 10}
+ budget := newLimitedMediaBudgetForTest(t, 3, 5, 10)
figures := extractDOCXFiguresFromIR(irJSON, budget)
if len(figures) != 1 {
@@ -476,7 +478,7 @@ func TestExtractDOCXFiguresFromIR_NoImage(t *testing.T) {
irJSON := `{"sections":[{"elements":[
{"type":"paragraph","content":[{"type":"text","text":"only text"}]}
]}]}`
- if figs := extractDOCXFiguresFromIR(irJSON, newEmbeddedMediaBudget()); figs != nil {
+ if figs := extractDOCXFiguresFromIR(irJSON, newMediaBudgetForTest()); figs != nil {
t.Fatalf("expected nil, got %+v", figs)
}
}
@@ -484,7 +486,7 @@ func TestExtractDOCXFiguresFromIR_NoImage(t *testing.T) {
// TestExtractDOCXFiguresFromIR_BadJSON returns nil on unparseable IR,
// mirroring the json.Unmarshal error guard.
func TestExtractDOCXFiguresFromIR_BadJSON(t *testing.T) {
- if figs := extractDOCXFiguresFromIR("{not json", newEmbeddedMediaBudget()); figs != nil {
+ if figs := extractDOCXFiguresFromIR("{not json", newMediaBudgetForTest()); figs != nil {
t.Fatalf("expected nil for bad JSON, got %+v", figs)
}
}
@@ -502,7 +504,7 @@ func TestExtractDOCXFiguresFromIR_NonParagraphContext(t *testing.T) {
{"type":"list","items":[{"content":[{"type":"paragraph","content":[{"type":"text","text":"list item"}]}]}]},
{"type":"text_box","content":[{"type":"paragraph","content":[{"type":"text","text":"box text"}]}]}
]}]}`
- figs := extractDOCXFiguresFromIR(irJSON, newEmbeddedMediaBudget())
+ figs := extractDOCXFiguresFromIR(irJSON, newMediaBudgetForTest())
if len(figs) != 1 {
t.Fatalf("expected 1 figure, got %d", len(figs))
}
@@ -526,7 +528,7 @@ func TestExtractDOCXFiguresFromIR_NestedImages(t *testing.T) {
{"type":"table","rows":[{"cells":[{"content":[{"type":"paragraph","content":[{"type":"text","text":"cell"},{"type":"image","data":"aGVsbG8="}]}]}]}]},
{"type":"text_box","content":[{"type":"paragraph","content":[{"type":"image","data":"d29ybGQ="}]}]}
]}]}`
- figs := extractDOCXFiguresFromIR(irJSON, newEmbeddedMediaBudget())
+ figs := extractDOCXFiguresFromIR(irJSON, newMediaBudgetForTest())
if len(figs) != 2 {
t.Fatalf("expected 2 nested figures, got %d: %+v", len(figs), figs)
}
@@ -626,7 +628,7 @@ func TestExtractTextFromListItem_NestedNull(t *testing.T) {
{"content":[{"type":"paragraph","content":[{"type":"text","text":"only item"}]}],"nested":null}
]}
]}]}`
- sections := buildDOCXJSONSections(irJSON, newEmbeddedMediaBudget())
+ sections := buildDOCXJSONSections(irJSON, newMediaBudgetForTest())
if len(sections) != 1 || sections[0]["text"] != "only item" {
t.Fatalf("expected single item \"only item\", got %+v", sections)
}
@@ -648,7 +650,7 @@ func TestExtractTextFromListItem_NestedTable(t *testing.T) {
]}
]}
]}]}`
- sections := buildDOCXJSONSections(irJSON, newEmbeddedMediaBudget())
+ sections := buildDOCXJSONSections(irJSON, newMediaBudgetForTest())
if len(sections) != 2 || sections[0]["text"] != "before" {
t.Fatalf("sections=%#v", sections)
}
@@ -697,7 +699,7 @@ func TestBuildDOCXJSONSections_TextBoxTable(t *testing.T) {
]}
]}
]}]}`
- sections := buildDOCXJSONSections(irJSON, newEmbeddedMediaBudget())
+ sections := buildDOCXJSONSections(irJSON, newMediaBudgetForTest())
if len(sections) != 1 {
t.Fatalf("expected 1 section, got %d: %+v", len(sections), sections)
}
@@ -716,7 +718,7 @@ func TestBuildDOCXJSONSections_NestedList(t *testing.T) {
]}}
]}
]}]}`
- sections := buildDOCXJSONSections(irJSON, newEmbeddedMediaBudget())
+ sections := buildDOCXJSONSections(irJSON, newMediaBudgetForTest())
if len(sections) != 3 {
t.Fatalf("sections=%#v", sections)
}
@@ -744,7 +746,7 @@ func TestBuildDOCXJSONSections_EmitsListAndTextBoxImages(t *testing.T) {
{"type":"image","data":"cGljdHVyZQ=="}
]}]}
]}]}`
- sections := buildDOCXJSONSections(irJSON, newEmbeddedMediaBudget())
+ sections := buildDOCXJSONSections(irJSON, newMediaBudgetForTest())
want := []struct {
docType string
text string
diff --git a/internal/parser/parser/docx_parser.go b/internal/parser/parser/docx_parser.go
index 0642327787..805bf6e6e9 100644
--- a/internal/parser/parser/docx_parser.go
+++ b/internal/parser/parser/docx_parser.go
@@ -21,6 +21,7 @@ import (
"fmt"
deepdocoffice "ragflow/internal/deepdoc/parser/office"
+ "ragflow/internal/parser/media"
)
// docxExtract is a test seam for the native office_oxide extraction. It
@@ -98,7 +99,7 @@ func (p *DOCXParser) ParseWithResult(ctx context.Context, filename string, data
// Extract IR JSON for section building and embedded-image extraction.
// The budget bounds image payloads so the downstream VLM enhancement
// stays within limits.
- budget := newEmbeddedMediaBudget()
+ budget := media.NewBudget()
if irJSON != "" {
figures := extractDOCXFiguresFromIR(irJSON, budget)
if len(figures) > 0 {
@@ -145,7 +146,7 @@ func (p *DOCXParser) ParseWithResult(ctx context.Context, filename string, data
return ParseResult{
File: fileMeta,
JSON: MapsToItems(sections),
- Warnings: budget.warnings(),
+ Warnings: budget.Warnings(),
}
}
diff --git a/internal/parser/parser/docx_parser_test.go b/internal/parser/parser/docx_parser_test.go
index a4398a6b1b..62d9875b0c 100644
--- a/internal/parser/parser/docx_parser_test.go
+++ b/internal/parser/parser/docx_parser_test.go
@@ -30,7 +30,7 @@ func TestBuildDOCXJSONSections_Paragraphs(t *testing.T) {
ir := `{"sections":[{"title":"","elements":[
{"type":"paragraph","content":[{"type":"text","text":"Hello world"}],"style":"Normal"}
]}]}`
- sections := buildDOCXJSONSections(ir, newEmbeddedMediaBudget())
+ sections := buildDOCXJSONSections(ir, newMediaBudgetForTest())
if len(sections) != 1 {
t.Fatalf("got %d sections, want 1", len(sections))
}
@@ -47,7 +47,7 @@ func TestBuildDOCXJSONSections_Headings(t *testing.T) {
ir := `{"sections":[{"title":"","elements":[
{"type":"heading","level":1,"content":[{"type":"text","text":"Chapter 1"}]}
]}]}`
- sections := buildDOCXJSONSections(ir, newEmbeddedMediaBudget())
+ sections := buildDOCXJSONSections(ir, newMediaBudgetForTest())
if len(sections) != 1 {
t.Fatalf("got %d sections, want 1", len(sections))
}
@@ -68,7 +68,7 @@ func TestBuildDOCXJSONSections_Images(t *testing.T) {
ir := `{"sections":[{"title":"","elements":[
{"type":"image","data":"` + b64 + `"}
]}]}`
- sections := buildDOCXJSONSections(ir, newEmbeddedMediaBudget())
+ sections := buildDOCXJSONSections(ir, newMediaBudgetForTest())
if len(sections) != 1 {
t.Fatalf("got %d sections, want 1", len(sections))
}
@@ -91,7 +91,7 @@ func TestBuildDOCXJSONSections_Tables(t *testing.T) {
{"cells":[{"content":[{"type":"paragraph","content":[{"type":"text","text":"A2"}]}]},{"content":[{"type":"paragraph","content":[{"type":"text","text":"B2"}]}]}]}
]}
]}]}`
- sections := buildDOCXJSONSections(ir, newEmbeddedMediaBudget())
+ sections := buildDOCXJSONSections(ir, newMediaBudgetForTest())
if len(sections) != 1 {
t.Fatalf("got %d sections, want 1", len(sections))
}
@@ -126,7 +126,7 @@ func TestBuildDOCXJSONSections_MixedContent(t *testing.T) {
{"type":"table","rows":[{"cells":[{"content":[{"type":"paragraph","content":[{"type":"text","text":"cell"}]}]}]}]},
{"type":"heading","level":2,"content":[{"type":"text","text":"Sub title"}]}
]}]}`
- sections := buildDOCXJSONSections(ir, newEmbeddedMediaBudget())
+ sections := buildDOCXJSONSections(ir, newMediaBudgetForTest())
if len(sections) != 4 {
t.Fatalf("got %d sections, want 4", len(sections))
}
@@ -159,7 +159,7 @@ func TestBuildDOCXJSONSections_List(t *testing.T) {
{"content":[{"type":"paragraph","content":[{"type":"text","text":"Item 3"}]}]}
],"level":0}
]}]}`
- sections := buildDOCXJSONSections(ir, newEmbeddedMediaBudget())
+ sections := buildDOCXJSONSections(ir, newMediaBudgetForTest())
if len(sections) != 3 {
t.Fatalf("got %d sections, want 3 (each list item should be a section)", len(sections))
}
@@ -177,7 +177,7 @@ func TestBuildDOCXJSONSections_TextBox(t *testing.T) {
ir := `{"sections":[{"title":"","elements":[
{"type":"text_box","content":[{"type":"paragraph","content":[{"type":"text","text":"Boxed paragraph"}]}],"width_emu":null}
]}]}`
- sections := buildDOCXJSONSections(ir, newEmbeddedMediaBudget())
+ sections := buildDOCXJSONSections(ir, newMediaBudgetForTest())
if len(sections) != 1 {
t.Fatalf("got %d sections, want 1 (text_box content should become a section)", len(sections))
}
@@ -198,7 +198,7 @@ func TestBuildDOCXJSONSections_MixedWithList(t *testing.T) {
],"level":0},
{"type":"paragraph","content":[{"type":"text","text":"Trailer"}],"style":"Normal"}
]}]}`
- sections := buildDOCXJSONSections(ir, newEmbeddedMediaBudget())
+ sections := buildDOCXJSONSections(ir, newMediaBudgetForTest())
if len(sections) != 4 {
t.Fatalf("got %d sections, want 4 (para + 2 list items + para)", len(sections))
}
@@ -216,7 +216,7 @@ func TestBuildDOCXJSONSections_EmptySkipped(t *testing.T) {
{"type":"paragraph","content":[{"type":"text","text":"Only this matters"}]},
{"type":"table","rows":[]}
]}]}`
- sections := buildDOCXJSONSections(ir, newEmbeddedMediaBudget())
+ sections := buildDOCXJSONSections(ir, newMediaBudgetForTest())
if len(sections) != 1 {
t.Fatalf("got %d sections, want 1 (only non-empty paragraph)", len(sections))
}
diff --git a/internal/parser/parser/epub_parser.go b/internal/parser/parser/epub_parser.go
index 8bbda6785c..af1f08e431 100644
--- a/internal/parser/parser/epub_parser.go
+++ b/internal/parser/parser/epub_parser.go
@@ -36,6 +36,7 @@ import (
"bytes"
"context"
"encoding/xml"
+ "errors"
"fmt"
"io"
"net/url"
@@ -287,15 +288,11 @@ func extractEPUBTextItems(ctx context.Context, r *zip.Reader, opf *opfManifest,
var items []map[string]any
var failures []string
read := 0
- budget := newEmbeddedMediaBudget()
- // mediacache.Put enforces the same per-image limit, so keep the parser
- // budget aligned with it instead of the legacy fixed constant.
- budget.maxImageBytes = media.HTMLImageMaxBytes()
state := &htmlWalkState{
refOnly: true,
ctx: ctx,
mediaCache: media.CacheFromContext(ctx),
- mediaBudget: budget,
+ budget: media.NewBudget(),
mediaFormat: "epub",
skipImages: textOnly,
}
@@ -357,15 +354,16 @@ func extractEPUBTextItems(ctx context.Context, r *zip.Reader, opf *opfManifest,
// the run media cache and the caller receives only the reference.
//
// A source that cannot be mapped to a readable manifest image is skipped and
-// counted in the shared warning. Cache failures fail the task.
+// counted in the shared warning. Cache failures fail the task; a run media
+// limit skips the image with a warning instead.
func epubMediaResolver(entries map[string]*zip.File, state *htmlWalkState, chapterEntry string, images map[string]string) func(string) (*entity.MediaRef, bool) {
chapterDir := path.Dir(chapterEntry)
return func(src string) (*entity.MediaRef, bool) {
if state.err != nil {
return nil, false
}
- // Inline data URIs keep the shared HTML data-URI path.
- if media.IsHTMLDataImageSource(src) {
+ // Inline data URIs keep the shared data-URI path.
+ if media.IsDataImageSource(src) {
return nil, false
}
if state.mediaCache == nil {
@@ -377,49 +375,78 @@ func epubMediaResolver(entries map[string]*zip.File, state *htmlWalkState, chapt
state.unresolvedImages++
return nil, false
}
+ // The memo is shared by every chapter: one ZIP entry is read, hashed,
+ // and cached once per parse, even when different chapters reference it.
+ // A nil value records a source that was already rejected, so repeated
+ // references do not re-read it or inflate the warning counters.
+ if ref, seen := state.epubMediaMemo[name]; seen {
+ if ref == nil {
+ return nil, false
+ }
+ return ref, true
+ }
mime, ok := images[name]
if !ok {
state.unresolvedImages++
+ state.memoizeEPUBMedia(name, nil)
return nil, false
}
entry, ok := entries[name]
if !ok {
state.unresolvedImages++
+ state.memoizeEPUBMedia(name, nil)
return nil, false
}
- maxBytes := state.mediaBudget.maxImageBytes
- if entry.UncompressedSize64 > uint64(maxBytes) {
- // Charge the per-image limit so the skip is reported through the
- // shared budget warning instead of being silent.
- state.mediaBudget.includeReservedSize(maxBytes + 1)
- return nil, false
- }
- // Reserve only for an actual ZIP read: entries skipped above need no
- // bytes and must not exhaust the attempt budget.
- if !state.mediaBudget.reserveItem() {
- return nil, false
- }
- f, err := entry.Open()
- if err != nil {
- state.unresolvedImages++
- return nil, false
- }
- data, err := io.ReadAll(io.LimitReader(f, int64(maxBytes)+1))
- f.Close()
- if err != nil || len(data) == 0 {
- state.unresolvedImages++
- return nil, false
- }
- if !state.mediaBudget.includeReservedSize(len(data)) {
- return nil, false
- }
- ref, err := state.mediaCache.Put(state.ctx, state.cacheFormat(), mime, data)
- if err != nil {
- state.err = fmt.Errorf("%s media cache write: %w", state.cacheFormat(), err)
+ return state.loadEPUBMedia(name, mime, entry)
+ }
+}
+
+// loadEPUBMedia applies size admission, reads one ZIP entry, and writes it to
+// the run media cache. It memoizes both success and failure so a source shared
+// by multiple chapters performs the work and records warnings once.
+func (s *htmlWalkState) loadEPUBMedia(name, mime string, entry *zip.File) (*entity.MediaRef, bool) {
+ maxBytes := media.ImageMaxBytes()
+ declared := media.Size(maxBytes + 1)
+ if entry.UncompressedSize64 <= uint64(maxBytes) {
+ declared = media.Size(entry.UncompressedSize64)
+ }
+ // Check the declared uncompressed size against both byte limits before
+ // reserving an item slot or decompressing the entry.
+ reservation, admission := s.budget.Reserve(declared)
+ if admission != media.Admitted {
+ s.memoizeEPUBMedia(name, nil)
+ return nil, false
+ }
+ f, err := entry.Open()
+ if err != nil {
+ s.unresolvedImages++
+ s.memoizeEPUBMedia(name, nil)
+ return nil, false
+ }
+ data, err := io.ReadAll(io.LimitReader(f, int64(maxBytes)+1))
+ f.Close()
+ if err != nil || len(data) == 0 {
+ s.unresolvedImages++
+ s.memoizeEPUBMedia(name, nil)
+ return nil, false
+ }
+ if !reservation.Commit(media.Size(len(data))) {
+ s.memoizeEPUBMedia(name, nil)
+ return nil, false
+ }
+ ref, err := s.mediaCache.Put(s.ctx, s.cacheFormat(), mime, data)
+ if err != nil {
+ if errors.Is(err, media.ErrMediaLimitExceeded) {
+ s.mediaLimitSkips++
+ s.memoizeEPUBMedia(name, nil)
return nil, false
}
- return &ref, true
+ s.err = fmt.Errorf("%s media cache write: %w", s.cacheFormat(), err)
+ s.memoizeEPUBMedia(name, nil)
+ return nil, false
}
+ s.memoizeEPUBMedia(name, &ref)
+ return &ref, true
}
// appendEPUBChapter keeps continuous chapter prose together, splitting only
diff --git a/internal/parser/parser/epub_parser_test.go b/internal/parser/parser/epub_parser_test.go
index 0d484f0b74..1a5cc69281 100644
--- a/internal/parser/parser/epub_parser_test.go
+++ b/internal/parser/parser/epub_parser_test.go
@@ -21,6 +21,7 @@ import (
"bytes"
"context"
"crypto/sha256"
+ "encoding/base64"
"encoding/hex"
"errors"
"fmt"
@@ -143,32 +144,50 @@ func parseTestEPUBContext(t *testing.T, ctx context.Context, chapters []epubTest
return NewEPUBParser().ParseWithResult(ctx, "book.epub", buildTestEPUBWithImages(t, chapters, images))
}
-// The parser-side embedded budget temporarily bounds how many ZIP images one
-// EPUB extracts, until the shared media budget replaces it.
+// The run media budget bounds how many distinct ZIP images one EPUB extracts.
+// Distinct sources are required to reach the cap; repeated references share
+// one source (see TestEPUBParserRepeatedZipImageReadsAndWritesOnce).
func TestEPUBParserCapsZipImageExtractionAcrossChapters(t *testing.T) {
- image := `<img src="Images/plate.png">`
+ t.Setenv("RAGFLOW_MEDIA_MAX_IMAGES_PER_RUN", "256")
+ const limit = 256
+ var first, second strings.Builder
+ images := make([]epubTestImage, 0, limit)
+ for i := 0; i < limit; i++ {
+ name := fmt.Sprintf("Images/plate-%d.png", i)
+ if i < limit/2 {
+ first.WriteString(`<img src="` + name + `">`)
+ } else {
+ second.WriteString(`<img src="` + name + `">`)
+ }
+ images = append(images, epubTestImage{name: name, content: fmt.Sprintf("plate-%d", i)})
+ }
+ // One distinct source past the cap, which must be omitted.
+ const overflowName = "Images/overflow.png"
+ second.WriteString(`<img src="` + overflowName + `">`)
+ images = append(images, epubTestImage{name: overflowName, content: "overflow"})
+
ctx, cache := recordingMediaContext()
res := parseTestEPUBContext(t, ctx, []epubTestChapter{
- {name: "one.xhtml", content: epubTestHTML(strings.Repeat(image, 128))},
- {name: "two.xhtml", content: epubTestHTML(strings.Repeat(image, 129) + "<p>After images.</p>")},
- }, []epubTestImage{{name: "Images/plate.png", content: "plate-bytes"}})
+ {name: "one.xhtml", content: epubTestHTML(first.String())},
+ {name: "two.xhtml", content: epubTestHTML(second.String() + "<p>After images.</p>")},
+ }, images)
if res.Err != nil {
t.Fatal(res.Err)
}
- images := 0
+ items := 0
for _, item := range ItemsToMaps(res.JSON) {
if item["doc_type_kwd"] == DocTypeImage {
- images++
+ items++
}
}
- if images != maxEmbeddedMediaItems || !strings.Contains(epubTestText(res), "After images.") {
- t.Fatalf("images=%d text=%q", images, epubTestText(res))
+ if items != limit || !strings.Contains(epubTestText(res), "After images.") {
+ t.Fatalf("images=%d text=%q", items, epubTestText(res))
}
- if cache.putCalls != maxEmbeddedMediaItems {
- t.Fatalf("cache writes = %d, want %d", cache.putCalls, maxEmbeddedMediaItems)
+ if cache.putCalls != limit {
+ t.Fatalf("cache writes = %d, want %d", cache.putCalls, limit)
}
- if !strings.Contains(strings.Join(res.Warnings, "\n"), "256-item document limit") {
- t.Fatalf("missing document image limit warning: %v", res.Warnings)
+ if !strings.Contains(strings.Join(res.Warnings, "\n"), "256-item run limit") {
+ t.Fatalf("missing run image limit warning: %v", res.Warnings)
}
}
@@ -432,6 +451,7 @@ func TestHTMLWalkStateResolveMediaPrecedesResolveImageAndDataURI(t *testing.T) {
refOnly: true,
ctx: ctx,
mediaCache: cache,
+ budget: media.NewBudget(),
mediaFormat: "epub",
resolveMedia: func(src string) (*entity.MediaRef, bool) {
if src == "zip.png" || src == htmlMediaDataURI {
@@ -467,7 +487,7 @@ func TestHTMLWalkStateDataURIWriteUsesMediaFormat(t *testing.T) {
} {
t.Run(tc.name, func(t *testing.T) {
ctx, cache := recordingMediaContext()
- state := &htmlWalkState{refOnly: true, ctx: ctx, mediaCache: cache, mediaFormat: tc.format}
+ state := &htmlWalkState{refOnly: true, ctx: ctx, mediaCache: cache, budget: media.NewBudget(), mediaFormat: tc.format}
_, ref, ok := state.imageSource(htmlMediaDataURI)
if !ok || ref == nil {
t.Fatalf("data URI image was not cached: ok=%v ref=%+v", ok, ref)
@@ -621,27 +641,35 @@ func TestEPUBParserSkipsUnreadableImageEntries(t *testing.T) {
}
// A manifest image missing from the ZIP is skipped without reading bytes, so
-// it must not consume an image attempt and hide a later readable image.
+// it must not consume an image attempt and hide a later readable image. The
+// missing sources are distinct so the count cap would be reachable if the
+// resolver charged them.
func TestEPUBParserMissingZipEntryDoesNotConsumeImageBudget(t *testing.T) {
+ const missing = 256
+ var content strings.Builder
+ images := make([]epubTestImage, 0, missing+1)
+ for i := 0; i < missing; i++ {
+ name := fmt.Sprintf("Images/gone-%d.png", i)
+ content.WriteString(`<img src="` + name + `">`)
+ images = append(images, epubTestImage{name: name, missing: true})
+ }
+ content.WriteString(`<img src="Images/keep.png" alt="keep">`)
+ images = append(images, epubTestImage{name: "Images/keep.png", content: "keep-bytes"})
+
ctx, cache := recordingMediaContext()
- content := epubTestHTML(strings.Repeat(`<img src="Images/gone.png">`, maxEmbeddedMediaItems) +
- `<img src="Images/keep.png" alt="keep">`)
res := parseTestEPUBContext(t, ctx, []epubTestChapter{{
name: "ch1.xhtml",
- content: content,
- }}, []epubTestImage{
- {name: "Images/gone.png", missing: true},
- {name: "Images/keep.png", content: "keep-bytes"},
- })
+ content: epubTestHTML(content.String()),
+ }}, images)
if res.Err != nil {
t.Fatalf("ParseWithResult: %v", res.Err)
}
if cache.putCalls != 1 || len(cache.puts) != 1 || string(cache.puts[0].data) != "keep-bytes" {
t.Fatalf("cache puts = %+v, want only the readable image", cache.puts)
}
- images := epubTestImageItems(t, res)
- if len(images) != 1 || images[0]["text"] != "keep" {
- t.Fatalf("image items = %#v, want the readable image", images)
+ items := epubTestImageItems(t, res)
+ if len(items) != 1 || items[0]["text"] != "keep" {
+ t.Fatalf("image items = %#v, want the readable image", items)
}
if !strings.Contains(epubTestWarningText(res), "unresolved image reference") {
t.Fatalf("missing unresolved-image warning: %v", res.Warnings)
@@ -651,9 +679,10 @@ func TestEPUBParserMissingZipEntryDoesNotConsumeImageBudget(t *testing.T) {
// An image above the per-image limit is reported through the budget warning
// without reading bytes, so it must not consume an image attempt either.
func TestEPUBParserOversizedZipImageDoesNotConsumeImageBudget(t *testing.T) {
- t.Setenv("RAGFLOW_HTML_IMAGE_MAX_BYTES", "8")
+ t.Setenv("RAGFLOW_MEDIA_IMAGE_MAX_BYTES", "8")
+ t.Setenv("RAGFLOW_MEDIA_MAX_IMAGES_PER_RUN", "256")
ctx, cache := recordingMediaContext()
- content := epubTestHTML(strings.Repeat(`<img src="Images/huge.png">`, maxEmbeddedMediaItems) +
+ content := epubTestHTML(strings.Repeat(`<img src="Images/huge.png">`, 256) +
`<img src="Images/ok.png" alt="ok">`)
res := parseTestEPUBContext(t, ctx, []epubTestChapter{{
name: "ch1.xhtml",
@@ -673,27 +702,60 @@ func TestEPUBParserOversizedZipImageDoesNotConsumeImageBudget(t *testing.T) {
}
}
+func TestEPUBParserByteRejectedEntriesDoNotConsumeImageSlots(t *testing.T) {
+ t.Setenv("RAGFLOW_MEDIA_IMAGE_MAX_BYTES", "8")
+ t.Setenv("RAGFLOW_MEDIA_MAX_IMAGES_PER_RUN", "2")
+ t.Setenv("RAGFLOW_MEDIA_MAX_BYTES_PER_RUN", "5")
+ ctx, cache := recordingMediaContext()
+ res := parseTestEPUBContext(t, ctx, []epubTestChapter{{
+ name: "ch1.xhtml",
+ content: epubTestHTML(
+ `<img src="Images/large-1.png"><img src="Images/large-2.png"><img src="Images/small.png" alt="small">`),
+ }}, []epubTestImage{
+ {name: "Images/large-1.png", content: strings.Repeat("a", 6)},
+ {name: "Images/large-2.png", content: strings.Repeat("b", 6)},
+ {name: "Images/small.png", content: "c"},
+ })
+ if res.Err != nil {
+ t.Fatalf("ParseWithResult: %v", res.Err)
+ }
+ if cache.putCalls != 1 || len(cache.puts) != 1 || string(cache.puts[0].data) != "c" {
+ t.Fatalf("cache puts = %+v, want only the small image", cache.puts)
+ }
+ if !strings.Contains(epubTestWarningText(res), "5-byte run image budget") {
+ t.Fatalf("missing total-byte warning: %v", res.Warnings)
+ }
+}
+
// A read failure is still a fetch attempt, so the item budget keeps bounding
-// the reads a document full of unreadable entries can force.
+// the reads a document full of unreadable entries can force. Distinct sources
+// are required: repeating one damaged entry shares one admission.
func TestEPUBParserUnreadableImageAttemptsCountAgainstBudget(t *testing.T) {
+ t.Setenv("RAGFLOW_MEDIA_MAX_IMAGES_PER_RUN", "256")
+ const attempts = 256
+ var content strings.Builder
+ images := make([]epubTestImage, 0, attempts+1)
+ for i := 0; i < attempts; i++ {
+ name := fmt.Sprintf("Images/bad-%d.png", i)
+ content.WriteString(`<img src="` + name + `">`)
+ images = append(images, epubTestImage{name: name, damaged: true})
+ }
+ content.WriteString(`<img src="Images/keep.png" alt="keep">`)
+ images = append(images, epubTestImage{name: "Images/keep.png", content: "keep-bytes"})
+
ctx, cache := recordingMediaContext()
- content := epubTestHTML(strings.Repeat(`<img src="Images/bad.png">`, maxEmbeddedMediaItems) +
- `<img src="Images/keep.png" alt="keep">`)
res := parseTestEPUBContext(t, ctx, []epubTestChapter{{
name: "ch1.xhtml",
- content: content,
- }}, []epubTestImage{
- {name: "Images/bad.png", damaged: true},
- {name: "Images/keep.png", content: "keep-bytes"},
- })
+ content: epubTestHTML(content.String()),
+ }}, images)
if res.Err != nil {
t.Fatalf("ParseWithResult: %v", res.Err)
}
if cache.putCalls != 0 {
t.Fatalf("cache puts = %+v, want the readable image dropped after the attempt cap", cache.puts)
}
- if !strings.Contains(epubTestWarningText(res), "256-item document limit") {
- t.Fatalf("missing document image limit warning: %v", res.Warnings)
+ if !strings.Contains(epubTestWarningText(res), "256-item run limit") {
+ t.Fatalf("missing run image limit warning: %v", res.Warnings)
}
}
@@ -750,17 +812,28 @@ func TestEPUBTextParserSkipsImagesWithoutCache(t *testing.T) {
}
}
-func TestEPUBParserDataURIImagesShareEmbeddedMediaBudget(t *testing.T) {
+// Data URIs share the run budget with ZIP entries. Distinct sources reach the
+// count cap; repeated data URIs share one admission.
+func TestEPUBParserDataURIImagesShareRunBudget(t *testing.T) {
+ t.Setenv("RAGFLOW_MEDIA_MAX_IMAGES_PER_RUN", "256")
+ const limit = 256
+ var content strings.Builder
+ for i := 0; i <= limit; i++ {
+ content.WriteString(`<img src="data:image/png;base64,`)
+ content.WriteString(base64.StdEncoding.EncodeToString([]byte(fmt.Sprintf("image-%d", i))))
+ content.WriteString(`">`)
+ }
+ content.WriteString("<p>After images.</p>")
ctx, cache := recordingMediaContext()
res := parseTestEPUBContext(t, ctx, []epubTestChapter{{
name: "ch1.xhtml",
- content: epubTestHTML(strings.Repeat(`<img src="`+htmlMediaDataURI+`">`, maxEmbeddedMediaItems+1) + "<p>After images.</p>"),
+ content: epubTestHTML(content.String()),
}}, nil)
if res.Err != nil {
t.Fatalf("ParseWithResult: %v", res.Err)
}
- if cache.putCalls != maxEmbeddedMediaItems {
- t.Fatalf("cache writes = %d, want %d", cache.putCalls, maxEmbeddedMediaItems)
+ if cache.putCalls != limit {
+ t.Fatalf("cache writes = %d, want %d", cache.putCalls, limit)
}
images := 0
for _, item := range ItemsToMaps(res.JSON) {
@@ -768,11 +841,147 @@ func TestEPUBParserDataURIImagesShareEmbeddedMediaBudget(t *testing.T) {
images++
}
}
- if images != maxEmbeddedMediaItems || !strings.Contains(epubTestText(res), "After images.") {
+ if images != limit || !strings.Contains(epubTestText(res), "After images.") {
t.Fatalf("images=%d text=%q", images, epubTestText(res))
}
- if !strings.Contains(strings.Join(res.Warnings, "\n"), "256-item document limit") {
- t.Fatalf("missing document image limit warning: %v", res.Warnings)
+ if !strings.Contains(strings.Join(res.Warnings, "\n"), "256-item run limit") {
+ t.Fatalf("missing run image limit warning: %v", res.Warnings)
+ }
+}
+
+// TestEPUBParserRepeatedZipImageReadsAndWritesOnce pins source dedupe on the
+// resolver: one ZIP entry referenced repeatedly is read and cached once, while
+// every reference and its alt text stay in the result.
+func TestEPUBParserRepeatedZipImageReadsAndWritesOnce(t *testing.T) {
+ ctx, cache := recordingMediaContext()
+ res := parseTestEPUBContext(t, ctx, []epubTestChapter{{
+ name: "ch1.xhtml",
+ content: epubTestHTML(
+ `<p>one</p><img src="Images/icon.png" alt="first"><p>two</p>` +
+ `<img src="Images/icon.png" alt="second"><img src="Images/icon.png" alt="third">`),
+ }}, []epubTestImage{{name: "Images/icon.png", content: "icon-bytes"}})
+ if res.Err != nil {
+ t.Fatalf("ParseWithResult: %v", res.Err)
+ }
+ if cache.putCalls != 1 || len(cache.puts) != 1 || string(cache.puts[0].data) != "icon-bytes" {
+ t.Fatalf("cache puts = %+v, want one write for the repeated entry", cache.puts)
+ }
+ items := epubTestImageItems(t, res)
+ if len(items) != 3 {
+ t.Fatalf("image items = %#v, want all three references", items)
+ }
+ first := entity.MediaRefFromAny(items[0]["media_ref"])
+ for i, wantAlt := range []string{"first", "second", "third"} {
+ if items[i]["text"] != wantAlt {
+ t.Fatalf("image %d alt = %v, want %q", i, items[i]["text"], wantAlt)
+ }
+ ref := entity.MediaRefFromAny(items[i]["media_ref"])
+ if first == nil || ref == nil || ref.SHA256 != first.SHA256 {
+ t.Fatalf("image %d ref = %+v, want the shared reference %+v", i, ref, first)
+ }
+ }
+ if len(res.Warnings) != 0 {
+ t.Fatalf("warnings = %v, want none", res.Warnings)
+ }
+}
+
+// TestEPUBParserRepeatedZipImageAcrossChaptersReadsAndWritesOnce pins that the
+// source memo is shared by the whole parse, not recreated for each chapter.
+func TestEPUBParserRepeatedZipImageAcrossChaptersReadsAndWritesOnce(t *testing.T) {
+ ctx, cache := recordingMediaContext()
+ res := parseTestEPUBContext(t, ctx, []epubTestChapter{
+ {name: "ch1.xhtml", content: epubTestHTML(`<img src="Images/icon.png" alt="first">`)},
+ {name: "ch2.xhtml", content: epubTestHTML(`<img src="Images/icon.png" alt="second">`)},
+ }, []epubTestImage{{name: "Images/icon.png", content: "icon-bytes"}})
+ if res.Err != nil {
+ t.Fatalf("ParseWithResult: %v", res.Err)
+ }
+ if cache.putCalls != 1 || len(cache.puts) != 1 || string(cache.puts[0].data) != "icon-bytes" {
+ t.Fatalf("cache puts = %+v, want one write across chapters", cache.puts)
+ }
+ items := epubTestImageItems(t, res)
+ if len(items) != 2 {
+ t.Fatalf("image items = %#v, want one per chapter", items)
+ }
+ first := entity.MediaRefFromAny(items[0]["media_ref"])
+ second := entity.MediaRefFromAny(items[1]["media_ref"])
+ if first == nil || second == nil || first.SHA256 != second.SHA256 {
+ t.Fatalf("refs = %+v / %+v, want the shared reference", first, second)
+ }
+ if len(res.Warnings) != 0 {
+ t.Fatalf("warnings = %v, want none", res.Warnings)
+ }
+}
+
+func TestEPUBParserRepeatedRejectedZipImageAcrossChaptersWarnsOnce(t *testing.T) {
+ tests := []struct {
+ name string
+ maxImage string
+ maxTotal string
+ payload string
+ wantWarn string
+ }{
+ {
+ name: "oversized",
+ maxImage: "8",
+ maxTotal: "8",
+ payload: strings.Repeat("x", 9),
+ wantWarn: "larger than the 8-byte per-image limit",
+ },
+ {
+ name: "total bytes",
+ maxImage: "8",
+ maxTotal: "5",
+ payload: strings.Repeat("x", 6),
+ wantWarn: "5-byte run image budget",
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ t.Setenv("RAGFLOW_MEDIA_IMAGE_MAX_BYTES", tt.maxImage)
+ t.Setenv("RAGFLOW_MEDIA_MAX_BYTES_PER_RUN", tt.maxTotal)
+ ctx, cache := recordingMediaContext()
+ res := parseTestEPUBContext(t, ctx, []epubTestChapter{
+ {name: "ch1.xhtml", content: epubTestHTML(`<img src="Images/rejected.png">`)},
+ {name: "ch2.xhtml", content: epubTestHTML(`<img src="Images/rejected.png">`)},
+ }, []epubTestImage{{name: "Images/rejected.png", content: tt.payload}})
+ if res.Err != nil {
+ t.Fatalf("ParseWithResult: %v", res.Err)
+ }
+ if cache.putCalls != 0 {
+ t.Fatalf("cache writes = %+v, want none", cache.puts)
+ }
+ warnings := epubTestWarningText(res)
+ if strings.Count(warnings, tt.wantWarn) != 1 {
+ t.Fatalf("warnings = %q, want one %q entry", warnings, tt.wantWarn)
+ }
+ })
+ }
+}
+
+// TestEPUBParserSkipsImagesAfterRunCacheLimit pins that a cache limit error is
+// a skip with a warning, not a task failure.
+func TestEPUBParserSkipsImagesAfterRunCacheLimit(t *testing.T) {
+ cache := &recordingMediaCache{putErr: fmt.Errorf("mediacache: run image count limit 1 exceeded: %w", media.ErrMediaLimitExceeded)}
+ ctx := media.WithCache(context.Background(), cache)
+ res := parseTestEPUBContext(t, ctx, []epubTestChapter{{
+ name: "ch1.xhtml",
+ content: epubTestHTML(`<p>before</p><img src="Images/a.png"><img src="Images/b.png"><p>after</p>`),
+ }}, []epubTestImage{{name: "Images/a.png", content: "a"}, {name: "Images/b.png", content: "b"}})
+ if res.Err != nil {
+ t.Fatalf("ParseWithResult: %v", res.Err)
+ }
+ if cache.putCalls != 2 {
+ t.Fatalf("cache writes = %d, want both references attempted", cache.putCalls)
+ }
+ if images := epubTestImageItems(t, res); len(images) != 0 {
+ t.Fatalf("image items = %#v, want none", images)
+ }
+ if !strings.Contains(epubTestWarningText(res), "run media cache limit") {
+ t.Fatalf("missing run-limit warning: %v", res.Warnings)
+ }
+ if !strings.Contains(epubTestText(res), "after") {
+ t.Fatalf("text after skipped images lost: %q", epubTestText(res))
}
}
diff --git a/internal/parser/parser/html_media_test.go b/internal/parser/parser/html_media_test.go
index c728c0a082..9ec4d6eae7 100644
--- a/internal/parser/parser/html_media_test.go
+++ b/internal/parser/parser/html_media_test.go
@@ -7,6 +7,7 @@ import (
"encoding/hex"
"encoding/json"
"errors"
+ "fmt"
"strings"
"testing"
@@ -175,7 +176,7 @@ func TestHTMLParser_EmitsTableCellImagesWithParentAndCellOrder(t *testing.T) {
}
func TestHTMLParser_SkipsOversizedInlineImagePayload(t *testing.T) {
- t.Setenv("RAGFLOW_HTML_IMAGE_MAX_BYTES", "4")
+ t.Setenv("RAGFLOW_MEDIA_IMAGE_MAX_BYTES", "4")
for name, oversized := range map[string]string{
"inline": `<p><img src="` + htmlMediaDataURI + `"></p>`,
"table": `<table><tr><td><img src="` + htmlMediaDataURI + `"></td></tr></table>`,
@@ -193,17 +194,42 @@ func TestHTMLParser_SkipsOversizedInlineImagePayload(t *testing.T) {
if !strings.Contains(renderedText(result.JSON[len(result.JSON)-2].ToMap()), "after") {
t.Fatalf("text after oversized image lost: %+v", result.JSON)
}
- if !strings.Contains(strings.Join(result.Warnings, " "), "invalid or oversized inline image") {
- t.Fatalf("missing skipped-image warning: %v", result.Warnings)
+ if !strings.Contains(strings.Join(result.Warnings, " "), "larger than the 4-byte per-image limit") {
+ t.Fatalf("missing oversized-image warning: %v", result.Warnings)
}
})
}
}
-func TestHTMLParser_HasNoParserSideImageCountLimit(t *testing.T) {
- const count = maxEmbeddedMediaItems + 44
+func TestHTMLParserByteRejectedDataURIsDoNotConsumeSourceSlots(t *testing.T) {
+ t.Setenv("RAGFLOW_MEDIA_IMAGE_MAX_BYTES", "8")
+ t.Setenv("RAGFLOW_MEDIA_MAX_IMAGES_PER_RUN", "2")
+ t.Setenv("RAGFLOW_MEDIA_MAX_BYTES_PER_RUN", "5")
+ large1 := "data:image/png;base64," + base64.StdEncoding.EncodeToString([]byte("aaaaaa"))
+ large2 := "data:image/png;base64," + base64.StdEncoding.EncodeToString([]byte("bbbbbb"))
+ small := "data:image/png;base64," + base64.StdEncoding.EncodeToString([]byte("c"))
+ input := `<img src="` + large1 + `"><img src="` + large2 + `"><img src="` + small + `" alt="small">`
+
+ ctx, cache := recordingMediaContext()
+ result := NewHTMLParser().ParseWithResult(ctx, "byte-limit.html", []byte(input))
+ if result.Err != nil {
+ t.Fatalf("ParseWithResult: %v", result.Err)
+ }
+ if cache.putCalls != 1 || len(cache.puts) != 1 || string(cache.puts[0].data) != "c" {
+ t.Fatalf("cache puts = %+v, want only the small image", cache.puts)
+ }
+ if !strings.Contains(strings.Join(result.Warnings, "\n"), "5-byte run image budget") {
+ t.Fatalf("missing total-byte warning: %v", result.Warnings)
+ }
+}
+
+// TestHTMLParser_RepeatedDataURIsShareOneSource pins that an icon repeated
+// across a document is decoded and written once, while every reference is
+// still emitted with the same MediaRef.
+func TestHTMLParser_RepeatedDataURIsShareOneSource(t *testing.T) {
+ const count = 300
input := "<p>" + strings.Repeat(`<img src="`+htmlMediaDataURI+`">`, count) + "</p>"
- ctx, _ := recordingMediaContext()
+ ctx, cache := recordingMediaContext()
result := NewHTMLParser().ParseWithResult(ctx, "many.html", []byte(input))
if result.Err != nil {
t.Fatalf("ParseWithResult: %v", result.Err)
@@ -215,10 +241,75 @@ func TestHTMLParser_HasNoParserSideImageCountLimit(t *testing.T) {
}
}
if images != count {
- t.Fatalf("images = %d, want %d (no parser-side cap, matching PDF)", images, count)
+ t.Fatalf("images = %d, want %d (one repeated source stays unbounded)", images, count)
+ }
+ if len(result.Warnings) != 0 {
+ t.Fatalf("warnings = %v, want none", result.Warnings)
+ }
+ if cache.putCalls != 1 {
+ t.Fatalf("cache writes = %d, want one for the repeated source", cache.putCalls)
+ }
+}
+
+// TestHTMLParser_DistinctDataURIsHitSourceLimit pins that distinct sources
+// remain bounded by the run budget and report the omission.
+func TestHTMLParser_DistinctDataURIsHitSourceLimit(t *testing.T) {
+ t.Setenv("RAGFLOW_MEDIA_MAX_IMAGES_PER_RUN", "256")
+ const limit = 256
+ var body strings.Builder
+ body.WriteString("<p>")
+ for i := 0; i <= limit; i++ {
+ body.WriteString(`<img src="data:image/png;base64,`)
+ body.WriteString(base64.StdEncoding.EncodeToString([]byte(fmt.Sprintf("image-%d", i))))
+ body.WriteString(`">`)
+ }
+ body.WriteString("</p>")
+ ctx, cache := recordingMediaContext()
+ result := NewHTMLParser().ParseWithResult(ctx, "distinct.html", []byte(body.String()))
+ if result.Err != nil {
+ t.Fatalf("ParseWithResult: %v", result.Err)
+ }
+ images := 0
+ for _, item := range result.JSON {
+ if item.ToMap()["doc_type_kwd"] == "image" {
+ images++
+ }
+ }
+ if images != limit {
+ t.Fatalf("images = %d, want %d", images, limit)
+ }
+ if cache.putCalls != limit {
+ t.Fatalf("cache writes = %d, want %d", cache.putCalls, limit)
+ }
+ if !strings.Contains(strings.Join(result.Warnings, "\n"), fmt.Sprintf("%d-item run limit", limit)) {
+ t.Fatalf("missing source-count warning: %v", result.Warnings)
+ }
+}
+
+// TestHTMLParser_SkipsImagesAfterRunCacheLimit pins the parser-side behavior
+// for a cache that has exhausted its run caps: the image is skipped with a
+// warning and the rest of the document still parses.
+func TestHTMLParser_SkipsImagesAfterRunCacheLimit(t *testing.T) {
+ cache := &recordingMediaCache{putErr: fmt.Errorf("mediacache: run image count limit 1 exceeded: %w", media.ErrMediaLimitExceeded)}
+ ctx := media.WithCache(context.Background(), cache)
+ input := `<p>before</p><img src="` + htmlMediaDataURI + `"><img src="data:image/png;base64,YQ=="><p>after</p>`
+ result := NewHTMLParser().ParseWithResult(ctx, "limit.html", []byte(input))
+ if result.Err != nil {
+ t.Fatalf("ParseWithResult: %v", result.Err)
+ }
+ if cache.putCalls != 2 {
+ t.Fatalf("cache writes = %d, want both references attempted", cache.putCalls)
+ }
+ for _, item := range result.JSON {
+ if item.ToMap()["doc_type_kwd"] == "image" {
+ t.Fatalf("image item emitted after a cache-limit skip: %#v", item.ToMap())
+ }
+ }
+ if text := renderedText(result.JSON[len(result.JSON)-1].ToMap()); text != "after" {
+ t.Fatalf("text after skipped images = %q, want after", text)
}
- if strings.Contains(strings.Join(result.Warnings, "\n"), "256-item") {
- t.Fatalf("unexpected parser-side image cap warning: %v", result.Warnings)
+ if !strings.Contains(strings.Join(result.Warnings, "\n"), "run media cache limit") {
+ t.Fatalf("missing run-limit warning: %v", result.Warnings)
}
}
diff --git a/internal/parser/parser/html_parser.go b/internal/parser/parser/html_parser.go
index d319cd371e..4cfadf6c81 100644
--- a/internal/parser/parser/html_parser.go
+++ b/internal/parser/parser/html_parser.go
@@ -19,6 +19,7 @@ package parser
import (
"bytes"
"context"
+ "errors"
"fmt"
"strings"
@@ -91,6 +92,7 @@ func (p *HTMLParser) ParseWithResult(ctx context.Context, filename string, data
refOnly: true,
ctx: ctx,
mediaCache: media.CacheFromContext(ctx),
+ budget: media.NewBudget(),
}
if p.textOnly {
state.resolveImage = func(string) (string, bool) { return "", false }
@@ -131,7 +133,13 @@ type htmlWalkState struct {
mediaOrder int
skippedInlineImages int
unresolvedImages int
- mediaBudget *embeddedMediaBudget
+ mediaLimitSkips int
+ budget *media.Budget
+ // dataURIMemo and epubMediaMemo dedupe sources across one parse. A nil
+ // value means the source was already processed and skipped; a non-nil
+ // value is the reference every later occurrence reuses.
+ dataURIMemo map[string]*entity.MediaRef
+ epubMediaMemo map[string]*entity.MediaRef
// refOnly defers image bytes to content-addressed references instead of
// carrying the normalized data URI on the item. HTML and EPUB set it;
// formats that still inline image bytes leave it false.
@@ -150,7 +158,11 @@ type htmlWalkState struct {
}
func (s *htmlWalkState) warnings() []string {
- warnings := s.mediaBudget.warnings()
+ var warnings []string
+ warnings = append(warnings, s.budget.Warnings()...)
+ if s.mediaLimitSkips > 0 {
+ warnings = append(warnings, fmt.Sprintf("document parser skipped %d image(s) after the run media cache limit", s.mediaLimitSkips))
+ }
if s.skippedInlineImages > 0 {
warnings = append(warnings, fmt.Sprintf("document parser skipped %d invalid or oversized inline image(s)", s.skippedInlineImages))
}
@@ -184,56 +196,90 @@ func (s *htmlWalkState) imageSource(src string) (string, *entity.MediaRef, bool)
normalized, ok := s.resolveImage(src)
return normalized, nil, ok
}
- if !media.IsHTMLDataImageSource(src) {
+ if !media.IsDataImageSource(src) {
return "", nil, false
}
if s.refOnly {
+ // A repeated source is decoded and written once; every reference still
+ // receives the same MediaRef. A nil memo value means it was already
+ // processed and skipped, so the warning counters are not inflated.
+ if ref, seen := s.dataURIMemo[src]; seen {
+ if ref == nil {
+ return "", nil, false
+ }
+ return "", ref, true
+ }
// Validate and size the payload without allocating so budget admission
- // happens before the decode. Malformed sources are never charged
- // against the item budget.
- size, ok := media.HTMLImageDecodedSize(src)
- if !ok || size > media.HTMLImageMaxBytes() {
+ // happens before the decode. Malformed sources never spend budget.
+ size, ok := media.DataImageDecodedSize(src)
+ if !ok {
+ s.skippedInlineImages++
+ s.memoizeDataURI(src, nil)
return "", nil, false
}
if s.mediaCache == nil {
s.err = fmt.Errorf("%s media cache is not configured", s.cacheFormat())
return "", nil, false
}
- if s.mediaBudget != nil {
- if !s.mediaBudget.reserveItem() {
- return "", nil, false
- }
- if !s.mediaBudget.includeReservedSize(size) {
- return "", nil, false
- }
+ // Repeated sources share one reservation, so a document made of one
+ // repeated data URI stays bounded without capping its references.
+ reservation, admission := s.budget.Reserve(media.Size(size))
+ if admission != media.Admitted {
+ s.memoizeDataURI(src, nil)
+ return "", nil, false
}
- mime, data, ok := media.DecodeHTMLImageSource(src)
+ mime, data, ok := media.DecodeDataImageSource(src)
if !ok {
+ s.skippedInlineImages++
+ s.memoizeDataURI(src, nil)
+ return "", nil, false
+ }
+ if !reservation.Commit(media.Size(len(data))) {
+ s.memoizeDataURI(src, nil)
return "", nil, false
}
ref, err := s.mediaCache.Put(s.ctx, s.cacheFormat(), mime, data)
if err != nil {
+ if errors.Is(err, media.ErrMediaLimitExceeded) {
+ s.mediaLimitSkips++
+ s.memoizeDataURI(src, nil)
+ return "", nil, false
+ }
s.err = fmt.Errorf("%s media cache write: %w", s.cacheFormat(), err)
return "", nil, false
}
+ s.memoizeDataURI(src, &ref)
return "", &ref, true
}
- if s.mediaBudget == nil {
- s.mediaBudget = newEmbeddedMediaBudget()
- }
- if !s.mediaBudget.reserveItem() {
+ reservation, admission := s.budget.ReserveOccurrence(media.UnknownSize)
+ if admission != media.Admitted {
return "", nil, false
}
- normalized, size, _, ok := media.NormalizeHTMLImageSource(src)
+ normalized, size, _, ok := media.NormalizeDataImageSource(src)
if !ok {
+ s.skippedInlineImages++
return "", nil, false
}
- if !s.mediaBudget.includeReservedSize(int(size)) {
+ if !reservation.Commit(media.Size(size)) {
return "", nil, false
}
return normalized, nil, true
}
+func (s *htmlWalkState) memoizeDataURI(src string, ref *entity.MediaRef) {
+ if s.dataURIMemo == nil {
+ s.dataURIMemo = make(map[string]*entity.MediaRef)
+ }
+ s.dataURIMemo[src] = ref
+}
+
+func (s *htmlWalkState) memoizeEPUBMedia(name string, ref *entity.MediaRef) {
+ if s.epubMediaMemo == nil {
+ s.epubMediaMemo = make(map[string]*entity.MediaRef)
+ }
+ s.epubMediaMemo[name] = ref
+}
+
// walkHTMLBlocksWithState emits normalized text and image items in document
// order. Inline formatting is collapsed into its parent text; script, style,
// and noscript subtrees are skipped.
@@ -497,8 +543,6 @@ func walkHTMLLeaf(n *html.Node, w *leafWriter, out *[]map[string]any, layout str
if !ok {
if state.resolveImage != nil {
w.writeText(htmlAttribute(n, "alt"))
- } else if media.IsHTMLDataImageSource(src) {
- state.skippedInlineImages++
}
return
}
@@ -562,8 +606,7 @@ func emitHTMLTable(n *html.Node, out *[]map[string]any, state *htmlWalkState) {
if state.err != nil {
return
}
- images, skippedInlineImages := htmlTableImages(n, state)
- state.skippedInlineImages += skippedInlineImages
+ images := htmlTableImages(n, state)
markup := renderTableHTML(n)
if strings.TrimSpace(markup) == "" {
return
@@ -591,9 +634,8 @@ func emitHTMLTable(n *html.Node, out *[]map[string]any, state *htmlWalkState) {
}
}
-func htmlTableImages(table *html.Node, state *htmlWalkState) ([]htmlTableImage, int) {
+func htmlTableImages(table *html.Node, state *htmlWalkState) []htmlTableImage {
var images []htmlTableImage
- skippedInlineImages := 0
rowCount := 0
columnCounts := make(map[int]int)
var walk func(node *html.Node, row, column int)
@@ -626,8 +668,7 @@ func htmlTableImages(table *html.Node, state *htmlWalkState) ([]htmlTableImage,
replacement = "![" + alt + "](" + src + ")"
}
node.Type, node.Data, node.Attr = html.TextNode, replacement, nil
- } else if media.IsHTMLDataImageSource(src) {
- skippedInlineImages++
+ } else if media.IsDataImageSource(src) {
removeHTMLImageSource(node)
}
} else {
@@ -653,7 +694,7 @@ func htmlTableImages(table *html.Node, state *htmlWalkState) ([]htmlTableImage,
}
}
walk(table, 0, 0)
- return images, skippedInlineImages
+ return images
}
func removeHTMLImageSource(n *html.Node) {
diff --git a/internal/parser/parser/markdown_parser.go b/internal/parser/parser/markdown_parser.go
index 0112047a0b..891154e237 100644
--- a/internal/parser/parser/markdown_parser.go
+++ b/internal/parser/parser/markdown_parser.go
@@ -96,7 +96,7 @@ func (p *MarkdownParser) ConfigureFromSetup(setup map[string]any) {
// the base64-encoded image payload. The legacy debug-print path has
// been removed; callers consume ParseResult directly.
func (p *MarkdownParser) ParseWithResult(ctx context.Context, filename string, data []byte) ParseResult {
- state := &htmlWalkState{}
+ state := &htmlWalkState{budget: media.NewBudget()}
result := p.parseWithState(ctx, filename, data, state)
result.Warnings = append(state.warnings(), result.Warnings...)
return result
@@ -156,10 +156,7 @@ func (p *MarkdownParser) parseWithState(ctx context.Context, filename string, da
doc := blockParser.Parse(nil)
var items []map[string]any
- if state.mediaBudget == nil {
- state.mediaBudget = newEmbeddedMediaBudget()
- }
- mediaBudget := state.mediaBudget
+ mediaBudget := state.budget
imageCtx, cancel := context.WithTimeout(ctx, 60*time.Second)
defer cancel()
unresolvedImages := walkMarkdownBlocksWithImages(imageCtx, doc, &items, p.FlattenMediaToText, p.FetchRemoteImages, state)
@@ -170,7 +167,7 @@ func (p *MarkdownParser) parseWithState(ctx context.Context, filename string, da
items = []map[string]any{{"text": "", "doc_type_kwd": "text"}}
}
var warnings []string
- if mediaBudget.items > 0 && imageCtx.Err() == context.DeadlineExceeded && ctx.Err() == nil {
+ if mediaBudget.Count() > 0 && imageCtx.Err() == context.DeadlineExceeded && ctx.Err() == nil {
warnings = append(warnings, "Markdown image processing reached the document time budget")
}
if unresolvedImages > 0 {
@@ -447,10 +444,7 @@ func markdownTableCells(line string) []string {
// Tables are emitted through the HTML walker as structured items in source
// order, with embedded images linked to their parent table.
func walkMarkdownBlocksWithImages(ctx context.Context, doc ast.Node, out *[]map[string]any, flatten, fetchRemoteImages bool, htmlState *htmlWalkState) int {
- if htmlState.mediaBudget == nil {
- htmlState.mediaBudget = newEmbeddedMediaBudget()
- }
- budget := htmlState.mediaBudget
+ budget := htmlState.budget
start := len(*out)
defer func() {
if flatten {
@@ -635,8 +629,12 @@ func findBlockImage(n ast.Node) (string, bool) {
// resolveImageURL resolves one bounded Markdown image attempt. It returns
// either normalized base64 or downloaded bytes, and distinguishes a budget
// or timeout omission from an unresolved source. Local paths are never fetched.
-func resolveImageURL(ctx context.Context, imageURL string, budget *embeddedMediaBudget) (encoded string, raw []byte, omitted, resolved bool) {
- if ctx.Err() != nil || !budget.reserveItem() {
+func resolveImageURL(ctx context.Context, imageURL string, budget *media.Budget) (encoded string, raw []byte, omitted, resolved bool) {
+ if ctx.Err() != nil {
+ return "", nil, true, false
+ }
+ reservation, admission := budget.ReserveOccurrence(media.UnknownSize)
+ if admission != media.Admitted {
return "", nil, true, false
}
var size int
@@ -646,7 +644,7 @@ func resolveImageURL(ctx context.Context, imageURL string, budget *embeddedMedia
return "", nil, false, false
}
var decodedSize int64
- encoded, decodedSize, _, resolved = media.NormalizeBase64ImagePayload(imageURL[idx+len("base64,"):], media.MaxImagePayloadBytes)
+ encoded, decodedSize, _, resolved = media.NormalizeBase64ImagePayload(imageURL[idx+len("base64,"):], media.ImageMaxBytes())
if !resolved {
return "", nil, false, false
}
@@ -661,7 +659,7 @@ func resolveImageURL(ctx context.Context, imageURL string, budget *embeddedMedia
} else {
return "", nil, false, false
}
- if !budget.includeReservedSize(size) {
+ if !reservation.Commit(media.Size(size)) {
return "", nil, true, false
}
return encoded, raw, false, true
@@ -746,12 +744,12 @@ func fetchImage(ctx context.Context, rawURL string) ([]byte, error) {
if resp.StatusCode < 200 || resp.StatusCode >= 300 {
return nil, fmt.Errorf("markdown: fetch image %s: HTTP %d", rawURL, resp.StatusCode)
}
- body, err := io.ReadAll(io.LimitReader(resp.Body, media.MaxImagePayloadBytes+1))
+ body, err := io.ReadAll(io.LimitReader(resp.Body, int64(media.ImageMaxBytes())+1))
if err != nil {
return nil, fmt.Errorf("markdown: read image %s: %w", rawURL, err)
}
- if len(body) > media.MaxImagePayloadBytes {
- return nil, fmt.Errorf("markdown: image %s exceeds %d-byte limit", rawURL, media.MaxImagePayloadBytes)
+ if len(body) > media.ImageMaxBytes() {
+ return nil, fmt.Errorf("markdown: image %s exceeds %d-byte limit", rawURL, media.ImageMaxBytes())
}
if len(body) == 0 {
return nil, fmt.Errorf("markdown: image %s is empty", rawURL)
diff --git a/internal/parser/parser/markdown_parser_test.go b/internal/parser/parser/markdown_parser_test.go
index 3752aadd90..6a7e25e7b1 100644
--- a/internal/parser/parser/markdown_parser_test.go
+++ b/internal/parser/parser/markdown_parser_test.go
@@ -552,7 +552,7 @@ func TestMarkdownParser_FlattenMediaToText(t *testing.T) {
func TestResolveImageURL_DataURI(t *testing.T) {
b64 := base64.StdEncoding.EncodeToString([]byte("fakeimage"))
- budget := newEmbeddedMediaBudget()
+ budget := newLimitedMediaBudgetForTest(t, len("fakeimage"), int64(len("fakeimage")), 2)
result, raw, omitted, found := resolveImageURL(context.Background(), "data:image/png;base64,"+b64, budget)
if !found {
t.Fatal("expected image found for data URI")
@@ -560,14 +560,17 @@ func TestResolveImageURL_DataURI(t *testing.T) {
if result != b64 || raw != nil {
t.Fatalf("got %q, want %q", result, b64)
}
- if omitted || budget.totalBytes != len("fakeimage") {
- t.Errorf("charged bytes = %d, want %d", budget.totalBytes, len("fakeimage"))
+ if omitted || len(budget.Warnings()) != 0 {
+ t.Fatalf("first image omitted=%v warnings=%v", omitted, budget.Warnings())
+ }
+ if included, keepWalking := includeMediaForTest(budget, []byte("x")); included || !keepWalking {
+ t.Fatalf("payload after the exact byte cap = (%v, %v), want (false, true)", included, keepWalking)
}
}
func TestResolveImageURL_LocalPathNotFetched(t *testing.T) {
// Local / relative paths are not fetched (security); resolution fails.
- if _, _, _, found := resolveImageURL(context.Background(), "./local/image.png", newEmbeddedMediaBudget()); found {
+ if _, _, _, found := resolveImageURL(context.Background(), "./local/image.png", newMediaBudgetForTest()); found {
t.Fatal("expected no image resolved for a local path")
}
}
@@ -586,7 +589,7 @@ func TestResolveImageURL_HTTPImage(t *testing.T) {
}))
defer ts.Close()
- budget := newEmbeddedMediaBudget()
+ budget := newLimitedMediaBudgetForTest(t, len("fake-png-bytes"), int64(len("fake-png-bytes")), 2)
encoded, result, omitted, found := resolveImageURL(context.Background(), ts.URL+"/image.png", budget)
if !found {
t.Fatal("expected image found for HTTP URL")
@@ -594,8 +597,11 @@ func TestResolveImageURL_HTTPImage(t *testing.T) {
if encoded != "" || string(result) != "fake-png-bytes" {
t.Fatalf("got encoded=%q raw=%q, want raw image bytes", encoded, result)
}
- if omitted || budget.totalBytes != len("fake-png-bytes") {
- t.Errorf("charged bytes = %d, want %d", budget.totalBytes, len("fake-png-bytes"))
+ if omitted || len(budget.Warnings()) != 0 {
+ t.Fatalf("first image omitted=%v warnings=%v", omitted, budget.Warnings())
+ }
+ if included, keepWalking := includeMediaForTest(budget, []byte("x")); included || !keepWalking {
+ t.Fatalf("payload after the exact byte cap = (%v, %v), want (false, true)", included, keepWalking)
}
}
@@ -616,9 +622,9 @@ func TestMarkdownImageBudgetSkipsFetchAfterCountLimit(t *testing.T) {
"![second](" + server.URL + "/2)\n\n" +
"![third](" + server.URL + "/3)\n"
doc := markdownNew().Parse([]byte(markdown))
- budget := &embeddedMediaBudget{maxImageBytes: 4, maxTotalBytes: 4, maxItems: 2}
+ budget := newLimitedMediaBudgetForTest(t, 4, 4, 2)
var items []map[string]any
- if unresolved := walkMarkdownBlocksWithImages(t.Context(), doc, &items, false, true, &htmlWalkState{mediaBudget: budget}); unresolved != 0 {
+ if unresolved := walkMarkdownBlocksWithImages(t.Context(), doc, &items, false, true, &htmlWalkState{budget: budget}); unresolved != 0 {
t.Fatalf("unresolved images = %d, want 0", unresolved)
}
if got := fetches.Load(); got != 2 {
@@ -632,10 +638,10 @@ func TestMarkdownImageBudgetSkipsFetchAfterCountLimit(t *testing.T) {
t.Fatalf("image %d = %+v, want omitted payload", index, items[index])
}
}
- warnings := strings.Join(budget.warnings(), "\n")
- if !strings.Contains(warnings, "omitted 1 embedded image payload(s) after reaching the 4-byte document image budget") ||
- !strings.Contains(warnings, "stopped extracting embedded images after the 2-item document limit") {
- t.Fatalf("warnings = %q, want document byte and item limit warnings", warnings)
+ warnings := strings.Join(budget.Warnings(), "\n")
+ if !strings.Contains(warnings, "omitted 1 embedded image payload(s) after reaching the 4-byte run image budget") ||
+ !strings.Contains(warnings, "stopped extracting embedded images after the 2-item run limit") {
+ t.Fatalf("warnings = %q, want run byte and item limit warnings", warnings)
}
}
diff --git a/internal/parser/parser/media_budget.go b/internal/parser/parser/media_budget.go
deleted file mode 100644
index a3966bd3c5..0000000000
--- a/internal/parser/parser/media_budget.go
+++ /dev/null
@@ -1,117 +0,0 @@
-//
-// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-//
-
-package parser
-
-import (
- "fmt"
-
- "ragflow/internal/parser/media"
-)
-
-const (
- maxEmbeddedImageBytes = media.MaxImagePayloadBytes
- maxEmbeddedDocumentImageBytes = 64 << 20
- maxEmbeddedMediaItems = 256
-)
-
-type embeddedMediaBudget struct {
- maxImageBytes int
- maxTotalBytes int
- maxItems int
-
- items int
- totalBytes int
- oversizedImages int
- documentOverflows int
- countExhausted bool
-}
-
-func newEmbeddedMediaBudget() *embeddedMediaBudget {
- return &embeddedMediaBudget{
- maxImageBytes: maxEmbeddedImageBytes,
- maxTotalBytes: maxEmbeddedDocumentImageBytes,
- maxItems: maxEmbeddedMediaItems,
- }
-}
-
-// include reserves budget for one image payload. The second result is false
-// when callers should stop walking images because the document item limit has
-// been reached. Byte-limit omissions can keep walking so later text and media
-// metadata remain available.
-func (b *embeddedMediaBudget) include(data []byte) (bool, bool) {
- if !b.reserveItem() {
- return false, false
- }
- return b.includeReservedSize(len(data)), true
-}
-
-// reserveItem caps the image fetch attempts for one document. Callers reserve
-// immediately before the I/O; skips that read no bytes are not charged.
-func (b *embeddedMediaBudget) reserveItem() bool {
- if b.countExhausted {
- return false
- }
- if b.items >= b.maxItems {
- b.countExhausted = true
- return false
- }
- b.items++
- return true
-}
-
-// includeReservedSize charges a payload against the per-image and document
-// byte limits without retaining or decoding it just to measure its size.
-func (b *embeddedMediaBudget) includeReservedSize(size int) bool {
- if size > b.maxImageBytes {
- b.oversizedImages++
- return false
- }
- if size > b.maxTotalBytes-b.totalBytes {
- b.documentOverflows++
- return false
- }
- b.totalBytes += size
- return true
-}
-
-func (b *embeddedMediaBudget) warnings() []string {
- if b == nil {
- return nil
- }
- var warnings []string
- if b.oversizedImages > 0 {
- warnings = append(warnings, fmt.Sprintf(
- "omitted %d embedded image payload(s) larger than the %d-byte per-image limit",
- b.oversizedImages,
- b.maxImageBytes,
- ))
- }
- if b.documentOverflows > 0 {
- warnings = append(warnings, fmt.Sprintf(
- "omitted %d embedded image payload(s) after reaching the %d-byte document image budget",
- b.documentOverflows,
- b.maxTotalBytes,
- ))
- }
- if b.countExhausted {
- warnings = append(warnings, fmt.Sprintf(
- "stopped extracting embedded images after the %d-item document limit",
- b.maxItems,
- ))
- }
- return warnings
-}
diff --git a/internal/parser/parser/media_budget_test.go b/internal/parser/parser/media_budget_test.go
index df8f13ff6a..a47a9c5b0c 100644
--- a/internal/parser/parser/media_budget_test.go
+++ b/internal/parser/parser/media_budget_test.go
@@ -1,50 +1,45 @@
+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
package parser
import (
- "reflect"
"testing"
+
+ "ragflow/internal/parser/media"
)
-func TestEmbeddedMediaBudgetReservedItemsAndWarnings(t *testing.T) {
- budget := &embeddedMediaBudget{maxImageBytes: 4, maxTotalBytes: 5, maxItems: 3}
+func newMediaBudgetForTest() *media.Budget {
+ return media.NewBudget()
+}
- if !budget.reserveItem() || !budget.includeReservedSize(3) {
- t.Fatal("first image should fit")
- }
- if !budget.reserveItem() || budget.includeReservedSize(3) {
- t.Fatal("second image should exceed the document byte limit")
- }
- if !budget.reserveItem() || budget.includeReservedSize(5) {
- t.Fatal("third image should exceed the per-image byte limit")
- }
- if budget.reserveItem() {
- t.Fatal("fourth image should be rejected before its source is read")
- }
- if included, keepWalking := budget.include([]byte("x")); included || keepWalking {
- t.Fatalf("include after count exhaustion = (%v, %v), want (false, false)", included, keepWalking)
- }
- if budget.items != 3 || budget.totalBytes != 3 {
- t.Fatalf("budget counters = (%d items, %d bytes), want (3, 3)", budget.items, budget.totalBytes)
- }
- want := []string{
- "omitted 1 embedded image payload(s) larger than the 4-byte per-image limit",
- "omitted 1 embedded image payload(s) after reaching the 5-byte document image budget",
- "stopped extracting embedded images after the 3-item document limit",
- }
- if got := budget.warnings(); !reflect.DeepEqual(got, want) {
- t.Fatalf("warnings = %q, want %q", got, want)
- }
+func newLimitedMediaBudgetForTest(t *testing.T, maxImageBytes int, maxTotalBytes int64, maxItems int) *media.Budget {
+ t.Helper()
+ return media.NewBudgetWithLimits(media.Limits{
+ ImageMaxBytes: int64(maxImageBytes),
+ MaxImagesPerRun: maxItems,
+ MaxBytesPerRun: maxTotalBytes,
+ })
}
-func TestEmbeddedMediaBudgetIncludesExactByteLimits(t *testing.T) {
- budget := &embeddedMediaBudget{maxImageBytes: 4, maxTotalBytes: 4, maxItems: 2}
- if included, keepWalking := budget.include([]byte("abcd")); !included || !keepWalking {
- t.Fatalf("exact-limit image = (%v, %v), want (true, true)", included, keepWalking)
- }
- if included, keepWalking := budget.include([]byte("e")); included || !keepWalking {
- t.Fatalf("image after exact document limit = (%v, %v), want (false, true)", included, keepWalking)
+func includeMediaForTest(budget *media.Budget, data []byte) (included, keepWalking bool) {
+ reservation, admission := budget.ReserveOccurrence(media.UnknownSize)
+ if admission == media.RejectedByCount {
+ return false, false
}
- if budget.totalBytes != 4 {
- t.Fatalf("totalBytes = %d, want 4", budget.totalBytes)
+ if admission != media.Admitted {
+ return false, true
}
+ return reservation.Commit(media.Size(len(data))), true
}
diff --git a/internal/parser/parser/office_table_render.go b/internal/parser/parser/office_table_render.go
index 3013f7b76e..35312221d6 100644
--- a/internal/parser/parser/office_table_render.go
+++ b/internal/parser/parser/office_table_render.go
@@ -29,6 +29,7 @@ import (
"ragflow/internal/common"
"ragflow/internal/entity"
+ "ragflow/internal/parser/media"
"ragflow/internal/parser/tableutil"
)
@@ -598,7 +599,7 @@ func deprecatedHTML4Excel(setup map[string]any, parserName string) {
// worksheet as structured parser items. Excelize exposes both kinds through
// GetPictureCells/GetPictures; walking the reported anchor cells avoids
// scanning the worksheet's entire coordinate space.
-func extractXLSXImages(f *excelize.File, sheet string, budget *embeddedMediaBudget) ([]map[string]any, []string) {
+func extractXLSXImages(f *excelize.File, sheet string, budget *media.Budget) ([]map[string]any, []string) {
cells, err := f.GetPictureCells(sheet)
if err != nil {
return nil, []string{fmt.Sprintf("XLSX image discovery failed for sheet %q: %v", sheet, err)}
@@ -627,7 +628,9 @@ cellLoop:
}
included, keepWalking := true, true
if budget != nil {
- included, keepWalking = budget.include(picture.File)
+ reservation, admission := budget.ReserveOccurrence(media.UnknownSize)
+ keepWalking = admission != media.RejectedByCount
+ included = admission == media.Admitted && reservation.Commit(media.Size(len(picture.File)))
}
if !keepWalking {
break cellLoop
diff --git a/internal/parser/parser/parse_with_result_test.go b/internal/parser/parser/parse_with_result_test.go
index 4dcdea6760..8baed39852 100644
--- a/internal/parser/parser/parse_with_result_test.go
+++ b/internal/parser/parser/parse_with_result_test.go
@@ -246,7 +246,7 @@ func TestWalkHTMLBlocksWithState_SkipsHeadLooseText(t *testing.T) {
head.AppendChild(&html.Node{Type: html.TextNode, Data: "Head metadata"})
var items []map[string]any
- state := &htmlWalkState{}
+ state := &htmlWalkState{budget: newMediaBudgetForTest()}
walkHTMLBlocksWithState(head, &items, state)
if len(items) != 0 {
t.Fatalf("JSON len = %d, want 0: %#v", len(items), items)
diff --git a/internal/parser/parser/pdf_parser_somark.go b/internal/parser/parser/pdf_parser_somark.go
index cd9b5a9ee0..7c09323685 100644
--- a/internal/parser/parser/pdf_parser_somark.go
+++ b/internal/parser/parser/pdf_parser_somark.go
@@ -10,10 +10,12 @@ import (
"mime/multipart"
"net/http"
"net/url"
- "ragflow/internal/common"
"strings"
pdfreader "github.com/ledongthuc/pdf"
+
+ "ragflow/internal/common"
+ "ragflow/internal/parser/media"
)
// parsePDFWithSoMark submits a PDF and converts the task result into parser
@@ -168,7 +170,7 @@ func soMarkItems(ctx context.Context, result map[string]any, keepHeaderFooter bo
jsonPayload, _ := outputs["json"].(map[string]any)
pages, _ := jsonPayload["pages"].([]any)
items := make([]map[string]any, 0)
- state := &htmlWalkState{}
+ state := &htmlWalkState{budget: media.NewBudget()}
var warnings []string
// Positions reference real document pages: prefer the source PDF's page
diff --git a/internal/parser/parser/pdf_parser_somark_test.go b/internal/parser/parser/pdf_parser_somark_test.go
index f9b898c5f3..7588e88f45 100644
--- a/internal/parser/parser/pdf_parser_somark_test.go
+++ b/internal/parser/parser/pdf_parser_somark_test.go
@@ -98,7 +98,7 @@ func TestPDFParser_ParseWithResult_SoMarkRequiresBaseURL(t *testing.T) {
}
func TestSoMarkBlockToItem_DropsHeaderByDefault(t *testing.T) {
- if item, _ := soMarkBlockToItems(t.Context(), map[string]any{"type": "header", "content": "x"}, false, &htmlWalkState{}); item != nil {
+ if item, _ := soMarkBlockToItems(t.Context(), map[string]any{"type": "header", "content": "x"}, false, &htmlWalkState{budget: newMediaBudgetForTest()}); item != nil {
t.Fatalf("item = %#v, want nil", item)
}
}
diff --git a/internal/parser/parser/pdf_parser_table_contract_test.go b/internal/parser/parser/pdf_parser_table_contract_test.go
index b8501c8d4f..856b0335c6 100644
--- a/internal/parser/parser/pdf_parser_table_contract_test.go
+++ b/internal/parser/parser/pdf_parser_table_contract_test.go
@@ -77,11 +77,11 @@ func TestPDFRemoteParsers_TableItemsCarryTableMarkup(t *testing.T) {
return tcadpAnyToItems(map[string]any{"type": "table", "content": pipeText, "table_data": blankRows})
}, "text"},
{"somark html", func() []map[string]any {
- items, _ := soMarkBlockToItems(t.Context(), map[string]any{"type": "table", "content": tableHTML}, false, &htmlWalkState{})
+ items, _ := soMarkBlockToItems(t.Context(), map[string]any{"type": "table", "content": tableHTML}, false, &htmlWalkState{budget: newMediaBudgetForTest()})
return items
}, "table"},
{"somark markdown", func() []map[string]any {
- items, _ := soMarkBlockToItems(t.Context(), map[string]any{"type": "table", "content": markdownTable}, false, &htmlWalkState{})
+ items, _ := soMarkBlockToItems(t.Context(), map[string]any{"type": "table", "content": markdownTable}, false, &htmlWalkState{budget: newMediaBudgetForTest()})
return items
}, "table"},
}
diff --git a/internal/parser/parser/pdf_table_no_html_text_test.go b/internal/parser/parser/pdf_table_no_html_text_test.go
index ee4c88f270..99bb7ee219 100644
--- a/internal/parser/parser/pdf_table_no_html_text_test.go
+++ b/internal/parser/parser/pdf_table_no_html_text_test.go
@@ -34,11 +34,11 @@ func TestPDFTableItemsOmitHTMLText(t *testing.T) {
return tcadpAnyToItems(map[string]any{"type": "table", "content": tableHTML})
}},
{"somark html", func() []map[string]any {
- items, _ := soMarkBlockToItems(t.Context(), map[string]any{"type": "table", "content": tableHTML}, false, &htmlWalkState{})
+ items, _ := soMarkBlockToItems(t.Context(), map[string]any{"type": "table", "content": tableHTML}, false, &htmlWalkState{budget: newMediaBudgetForTest()})
return items
}},
{"docx table", func() []map[string]any {
- return buildDOCXJSONSections(docxIR, newEmbeddedMediaBudget())
+ return buildDOCXJSONSections(docxIR, newMediaBudgetForTest())
}},
}
for _, tc := range cases {
diff --git a/internal/parser/parser/picture_parser.go b/internal/parser/parser/picture_parser.go
index 5481c2bd70..80fc30b31d 100644
--- a/internal/parser/parser/picture_parser.go
+++ b/internal/parser/parser/picture_parser.go
@@ -71,8 +71,8 @@ func (p *PictureParser) ParseWithResult(ctx context.Context, filename string, da
Err: fmt.Errorf("picture: unsupported extension %q (filename: %s); accepted: .jpg/.jpeg/.png/.gif/.bmp/.tiff/.tif/.webp/.svg/.ico/.avif/.heic/...", ext, filename),
}
}
- if len(data) == 0 || len(data) > media.MaxImagePayloadBytes {
- return ParseResult{Err: fmt.Errorf("picture: image payload size %d is outside the %d-byte limit", len(data), media.MaxImagePayloadBytes)}
+ if len(data) == 0 || len(data) > media.ImageMaxBytes() {
+ return ParseResult{Err: fmt.Errorf("picture: image payload size %d is outside the %d-byte limit", len(data), media.ImageMaxBytes())}
}
release, err := AcquireImageMedia(ctx)
diff --git a/internal/parser/parser/pptx_ir.go b/internal/parser/parser/pptx_ir.go
index 677686932c..b2e6d80434 100644
--- a/internal/parser/parser/pptx_ir.go
+++ b/internal/parser/parser/pptx_ir.go
@@ -22,18 +22,19 @@ import (
"strings"
"ragflow/internal/entity"
+ "ragflow/internal/parser/media"
)
// buildPPTXJSONSections emits text runs, tables and images in slide order.
// Every item retains its slide number; a blank slide emits an empty text item.
// Container elements are traversed recursively so grouped tables remain structured.
-func buildPPTXJSONSections(irJSON string, budget *embeddedMediaBudget) ([]map[string]any, error) {
+func buildPPTXJSONSections(irJSON string, budget *media.Budget) ([]map[string]any, error) {
var ir docxIRDocument
if err := json.Unmarshal([]byte(irJSON), &ir); err != nil {
return nil, fmt.Errorf("presentation ir-json decode: %w", err)
}
if budget == nil {
- budget = newEmbeddedMediaBudget()
+ budget = media.NewBudget()
}
items := make([]map[string]any, 0, len(ir.Sections))
for i, sec := range ir.Sections {
diff --git a/internal/parser/parser/pptx_ir_test.go b/internal/parser/parser/pptx_ir_test.go
index 7c6240024f..c1d77599e9 100644
--- a/internal/parser/parser/pptx_ir_test.go
+++ b/internal/parser/parser/pptx_ir_test.go
@@ -172,7 +172,7 @@ func TestBuildPPTXJSONSections(t *testing.T) {
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
- items, err := buildPPTXJSONSections(tt.irJSON, newEmbeddedMediaBudget())
+ items, err := buildPPTXJSONSections(tt.irJSON, newMediaBudgetForTest())
if tt.wantErr {
if err == nil {
t.Fatalf("buildPPTXJSONSections: want error, got items %+v", items)
@@ -217,7 +217,7 @@ func TestBuildPPTXJSONSections_ExtractsSlideImagesInOrder(t *testing.T) {
{"type":"table","rows":[{"cells":[{"content":[{"type":"image","data":"aW1hZ2U="}]}]}]}
]},{"elements":[{"type":"image","data":"c2Vjb25k"}]}]}`
- items, err := buildPPTXJSONSections(irJSON, newEmbeddedMediaBudget())
+ items, err := buildPPTXJSONSections(irJSON, newMediaBudgetForTest())
if err != nil {
t.Fatalf("buildPPTXJSONSections: %v", err)
}
diff --git a/internal/parser/parser/pptx_parser.go b/internal/parser/parser/pptx_parser.go
index b050d87e36..a54d9d24fd 100644
--- a/internal/parser/parser/pptx_parser.go
+++ b/internal/parser/parser/pptx_parser.go
@@ -22,6 +22,7 @@ import (
"strings"
deepdocoffice "ragflow/internal/deepdoc/parser/office"
+ "ragflow/internal/parser/media"
)
// pptxExtract is a test seam for the native office_oxide extraction. It
@@ -151,7 +152,7 @@ func (p *PPTXParser) ParseWithResult(ctx context.Context, filename string, data
JSON: MapsToItems(itemsFromPlainText(plainText)),
}
}
- budget := newEmbeddedMediaBudget()
+ budget := media.NewBudget()
items, err := buildPPTXJSONSections(irJSON, budget)
if err != nil {
return ParseResult{Err: err}
@@ -174,6 +175,6 @@ func (p *PPTXParser) ParseWithResult(ctx context.Context, filename string, data
return ParseResult{
File: map[string]any{"name": filename, "format": effFormat},
JSON: MapsToItems(items),
- Warnings: budget.warnings(),
+ Warnings: budget.Warnings(),
}
}
diff --git a/internal/parser/parser/table_regression_test.go b/internal/parser/parser/table_regression_test.go
index 6320ea815a..2d7099c7e0 100644
--- a/internal/parser/parser/table_regression_test.go
+++ b/internal/parser/parser/table_regression_test.go
@@ -105,8 +105,10 @@ func TestTableRegressionSoMarkTableImageIDs(t *testing.T) {
}
func TestTableRegressionSoMarkDocumentImageBudget(t *testing.T) {
+ const limit = 256
+ t.Setenv("RAGFLOW_MEDIA_MAX_IMAGES_PER_RUN", "256")
markup := "| Field | Value |\n|---|---|\n| Logo | ![logo](data:image/png;base64,aGVsbG8=) |"
- blocks := make([]any, maxEmbeddedMediaItems+1)
+ blocks := make([]any, limit+1)
for i := range blocks {
blocks[i] = map[string]any{"type": "table", "content": markup}
}
@@ -117,8 +119,8 @@ func TestTableRegressionSoMarkDocumentImageBudget(t *testing.T) {
images++
}
}
- t.Logf("retained images=%d document item limit=%d", images, maxEmbeddedMediaItems)
- if images != maxEmbeddedMediaItems || !strings.Contains(strings.Join(warnings, "\n"), "256-item") {
+ t.Logf("retained images=%d run item limit=%d", images, limit)
+ if images != limit || !strings.Contains(strings.Join(warnings, "\n"), "256-item") {
t.Errorf("retained=%d warnings=%v, want document-wide item limit", images, warnings)
}
}
@@ -135,19 +137,21 @@ func TestTableRegressionHTMLTheadWithTDRetainsDefaultHeader(t *testing.T) {
}
func TestTableRegressionMarkdownRawHTMLImageBudget(t *testing.T) {
- source := "<table><tr><td>" + strings.Repeat("<img src=\"data:image/png;base64,aGVsbG8=\">", maxEmbeddedMediaItems+1) + "</td></tr></table>\n\n"
+ const limit = 256
+ t.Setenv("RAGFLOW_MEDIA_MAX_IMAGES_PER_RUN", "256")
+ source := "<table><tr><td>" + strings.Repeat("<img src=\"data:image/png;base64,aGVsbG8=\">", limit+1) + "</td></tr></table>\n\n"
doc := markdownNew().Parse([]byte(source))
- budget := newEmbeddedMediaBudget()
+ budget := newMediaBudgetForTest()
var items []map[string]any
- walkMarkdownBlocksWithImages(t.Context(), doc, &items, false, true, &htmlWalkState{mediaBudget: budget})
+ walkMarkdownBlocksWithImages(t.Context(), doc, &items, false, true, &htmlWalkState{budget: budget})
images := 0
for _, item := range items {
if item["doc_type_kwd"] == "image" && item["image"] != nil {
images++
}
}
- t.Logf("retained images=%d budget attempts=%d document item limit=%d", images, budget.items, maxEmbeddedMediaItems)
- if images != maxEmbeddedMediaItems || budget.items != maxEmbeddedMediaItems || !budget.countExhausted {
+ t.Logf("retained images=%d budget attempts=%d run item limit=%d", images, budget.Count(), limit)
+ if images != limit || budget.Count() != limit || !strings.Contains(strings.Join(budget.Warnings(), "\n"), "256-item run limit") {
t.Errorf("raw HTML budget not enforced: retained=%d budget=%+v", images, budget)
}
}
diff --git a/internal/parser/parser/table_structure_test.go b/internal/parser/parser/table_structure_test.go
index b15e880430..35aee1af44 100644
--- a/internal/parser/parser/table_structure_test.go
+++ b/internal/parser/parser/table_structure_test.go
@@ -96,11 +96,11 @@ func TestMarkdownTableImageTimeoutDoesNotConsumeBudget(t *testing.T) {
ctx, cancel := context.WithCancel(t.Context())
cancel()
doc := markdownNew().Parse([]byte("| A | B |\n|---|---|\n| Logo | ![logo](http://127.0.0.1/logo.png) |\n"))
- budget := newEmbeddedMediaBudget()
+ budget := newMediaBudgetForTest()
var items []map[string]any
- unresolved := walkMarkdownBlocksWithImages(ctx, doc, &items, false, true, &htmlWalkState{mediaBudget: budget})
- if budget.items != 0 || unresolved != 0 {
- t.Fatalf("timed out image consumed budget: attempts=%d unresolved=%d", budget.items, unresolved)
+ unresolved := walkMarkdownBlocksWithImages(ctx, doc, &items, false, true, &htmlWalkState{budget: budget})
+ if budget.Count() != 0 || unresolved != 0 {
+ t.Fatalf("timed out image consumed budget: attempts=%d unresolved=%d", budget.Count(), unresolved)
}
}
@@ -268,7 +268,7 @@ func TestOfficeTableRetainsDefaultHeaderCount(t *testing.T) {
func TestOfficeTablePreservesLiteralCellText(t *testing.T) {
el := docxIRElement{Type: "table", Rows: []docxIRRow{{Cells: []docxIRCell{{Content: []docxIRElement{{Type: "paragraph", Content: json.RawMessage(`[{"type":"text","text":" <b>A & B</b>\nsecond line "}]`)}}}}}}}
- items := appendOfficeTableSections(nil, el, " A & <B> ", "table-1", newEmbeddedMediaBudget())
+ items := appendOfficeTableSections(nil, el, " A & <B> ", "table-1", newMediaBudgetForTest())
if len(items) != 1 {
t.Fatalf("items=%#v", items)
}
diff --git a/internal/parser/parser/xls_parser.go b/internal/parser/parser/xls_parser.go
index 76c5dfcd8a..5820b3bce9 100644
--- a/internal/parser/parser/xls_parser.go
+++ b/internal/parser/parser/xls_parser.go
@@ -24,6 +24,8 @@ import (
"time"
excel "github.com/jloor/go-excel-reader"
+
+ "ragflow/internal/parser/media"
)
// CompoundDocumentSignature is the OLE compound-document magic
@@ -100,9 +102,9 @@ func (p *XLSParser) ParseWithResult(ctx context.Context, filename string, data [
if bytes.HasPrefix(data, CompoundDocumentSignature) {
items, sheetsCount, err = parseBIFF8Bytes(ctx, data)
} else {
- mediaBudget := newEmbeddedMediaBudget()
+ mediaBudget := media.NewBudget()
items, warnings, sheetsCount, err = parseXLSXBytes(data, mediaBudget)
- warnings = append(warnings, mediaBudget.warnings()...)
+ warnings = append(warnings, mediaBudget.Warnings()...)
}
if err := ctx.Err(); err != nil {
return ParseResult{Err: err}
diff --git a/internal/parser/parser/xls_parser_test.go b/internal/parser/parser/xls_parser_test.go
index a4c952f47e..4d8f017cdf 100644
--- a/internal/parser/parser/xls_parser_test.go
+++ b/internal/parser/parser/xls_parser_test.go
@@ -322,13 +322,15 @@ func TestXLSParser_ImagesDoNotRunOCR(t *testing.T) {
}
func TestXLSParser_CapsEmbeddedImageCount(t *testing.T) {
+ const limit = 256
+ t.Setenv("RAGFLOW_MEDIA_MAX_IMAGES_PER_RUN", "256")
var imageData bytes.Buffer
if err := png.Encode(&imageData, image.NewRGBA(image.Rect(0, 0, 2, 2))); err != nil {
t.Fatalf("encode fixture: %v", err)
}
f := excelize.NewFile()
defer f.Close()
- for i := 0; i <= maxEmbeddedMediaItems; i++ {
+ for i := 0; i <= limit; i++ {
cell, err := excelize.CoordinatesToCellName(3, i+1)
if err != nil {
t.Fatalf("cell %d: %v", i, err)
@@ -361,10 +363,10 @@ func TestXLSParser_CapsEmbeddedImageCount(t *testing.T) {
imageCount++
}
}
- if imageCount != maxEmbeddedMediaItems {
- t.Fatalf("image items = %d, want %d", imageCount, maxEmbeddedMediaItems)
+ if imageCount != limit {
+ t.Fatalf("image items = %d, want %d", imageCount, limit)
}
- if warnings := strings.Join(result.Warnings, "\n"); !strings.Contains(warnings, "stopped extracting embedded images after the 256-item document limit") {
+ if warnings := strings.Join(result.Warnings, "\n"); !strings.Contains(warnings, "stopped extracting embedded images after the 256-item run limit") {
t.Fatalf("warnings = %q, want image-count limit warning", warnings)
}
}
diff --git a/internal/parser/parser/xlsx_parser.go b/internal/parser/parser/xlsx_parser.go
index c5fde577e1..ad8061654d 100644
--- a/internal/parser/parser/xlsx_parser.go
+++ b/internal/parser/parser/xlsx_parser.go
@@ -23,6 +23,8 @@ import (
"strings"
"github.com/xuri/excelize/v2"
+
+ "ragflow/internal/parser/media"
)
type XLSXParser struct {
@@ -102,13 +104,13 @@ func (p *XLSXParser) ParseWithResult(ctx context.Context, filename string, data
// for spreadsheet processing.
}
- mediaBudget := newEmbeddedMediaBudget()
+ mediaBudget := media.NewBudget()
items, warnings, sheets, err := parseXLSXBytes(data, mediaBudget)
if err == nil {
if ctx.Err() != nil {
return ParseResult{Err: ctx.Err()}
}
- warnings = append(warnings, mediaBudget.warnings()...)
+ warnings = append(warnings, mediaBudget.Warnings()...)
return xlsxParseResult(filename, items, warnings, sheets)
}
if ctx.Err() != nil {
@@ -124,7 +126,7 @@ func (p *XLSXParser) ParseWithResult(ctx context.Context, filename string, data
}
// The first parse attempt is discarded, so its media reservations must not
// affect the normalized retry's output.
- mediaBudget = newEmbeddedMediaBudget()
+ mediaBudget = media.NewBudget()
items, warnings, sheets, retryErr := parseXLSXBytes(normalized, mediaBudget)
if retryErr != nil {
return ParseResult{Err: fmt.Errorf("xlsx parse: %w; retry after normalization: %v", err, retryErr)}
@@ -133,11 +135,11 @@ func (p *XLSXParser) ParseWithResult(ctx context.Context, filename string, data
return ParseResult{Err: ctx.Err()}
}
warnings = append(normalizeWarnings, warnings...)
- warnings = append(warnings, mediaBudget.warnings()...)
+ warnings = append(warnings, mediaBudget.Warnings()...)
return xlsxParseResult(filename, items, warnings, sheets)
}
-func parseXLSXBytes(data []byte, mediaBudget *embeddedMediaBudget) ([]map[string]any, []string, int, error) {
+func parseXLSXBytes(data []byte, mediaBudget *media.Budget) ([]map[string]any, []string, int, error) {
f, err := excelize.OpenReader(bytes.NewReader(data))
if err != nil {
return nil, nil, 0, fmt.Errorf("open XLSX: %w", err)
diff --git a/internal/service/chunk/chunk.go b/internal/service/chunk/chunk.go
index b6025c5b93..4ad9518d21 100644
--- a/internal/service/chunk/chunk.go
+++ b/internal/service/chunk/chunk.go
@@ -20,6 +20,7 @@ import (
"bytes"
"context"
"encoding/base64"
+ "errors"
"fmt"
"image"
"image/color"
@@ -657,18 +658,12 @@ func (s *ChunkService) StopParsing(ctx context.Context, userID, datasetID string
return nil, common.CodeDataError, fmt.Errorf("you don't own the document %s", docID)
}
- task, err := dao.NewIngestionTaskDAO().GetByDocumentID(ctx, dao.DB, docID)
- if err != nil {
- return nil, common.CodeServerError, fmt.Errorf("get ingestion task for %s: %w", docID, err)
- }
- if task == nil || task.Status == common.COMPLETED ||
- task.Status == common.STOPPED || task.Status == common.FAILED {
- return &service.StopParsingResponse{
- Data: map[string]interface{}{"error_code": docStopParsingInvalidStateErrorCode},
- }, common.CodeDataError, fmt.Errorf("%s", docStopParsingInvalidStateMessage)
- }
-
if err = s.cancelAllTasksOfDoc(ctx, doc); err != nil {
+ if errors.Is(err, document.ErrParseNotRunning) {
+ return &service.StopParsingResponse{
+ Data: map[string]interface{}{"error_code": docStopParsingInvalidStateErrorCode},
+ }, common.CodeDataError, fmt.Errorf("%s", docStopParsingInvalidStateMessage)
+ }
return nil, common.CodeServerError, err
}
// CancelDocParse (inside cancelAllTasksOfDoc) already issues
diff --git a/internal/service/chunk/chunk_test.go b/internal/service/chunk/chunk_test.go
index 75d62fbe22..b59791a7bd 100644
--- a/internal/service/chunk/chunk_test.go
+++ b/internal/service/chunk/chunk_test.go
@@ -301,7 +301,7 @@ func TestParseRejectsRunningDocument(t *testing.T) {
if code != common.CodeDataError {
t.Fatalf("expected CodeDataError, got %v", code)
}
- if !strings.Contains(err.Error(), "ingestion task is RUNNING") {
+ if !strings.Contains(err.Error(), "ingestion task task-1 is RUNNING") {
t.Fatalf("unexpected error: %v", err)
}
}
@@ -2755,8 +2755,6 @@ func TestStopParsing_CallsCancelIngestionTask(t *testing.T) {
pushChunkTestDB(t, db)
insertChunkTestKB(t, "kb-1", "user-1") // owner = user-1, Accessible passes
insertChunkTestDoc(t, "doc-1", "kb-1")
- // StopParsing checks the IngestionTask status.
- insertChunkTestIngestionTask(t, "task-1", "user-1", "doc-1", "kb-1", common.RUNNING)
svc := newParseTestService(t)
var calledDocID string
@@ -2783,7 +2781,10 @@ func TestStopParsing_RejectsDocumentWithoutIngestionTask(t *testing.T) {
svc := newParseTestService(t)
var called bool
- svc.cancelIngestionTaskFunc = func(ctx context.Context, doc *entity.Document) error { called = true; return nil }
+ svc.cancelIngestionTaskFunc = func(ctx context.Context, doc *entity.Document) error {
+ called = true
+ return document.ErrParseNotRunning
+ }
ctx := chunkTestContext(t, "user-1", "user-1")
_, code, err := svc.StopParsing(ctx, "user-1", "kb-1", service.StopParsingRequest{DocumentIDs: []string{"doc-1"}})
if err == nil {
@@ -2792,8 +2793,8 @@ func TestStopParsing_RejectsDocumentWithoutIngestionTask(t *testing.T) {
if code != common.CodeDataError {
t.Fatalf("expected CodeDataError, got %v", code)
}
- if called {
- t.Fatal("cancel should not be called for doc without ingestion task")
+ if !called {
+ t.Fatal("StopParsing must delegate task-state validation to CancelDocParse")
}
}
diff --git a/internal/service/document/document_crud.go b/internal/service/document/document_crud.go
index ddd2546a91..3550fbbeaf 100644
--- a/internal/service/document/document_crud.go
+++ b/internal/service/document/document_crud.go
@@ -280,6 +280,14 @@ func (s *DocumentService) deleteDocumentFull(ctx context.Context, docID string)
if err != nil {
return err
}
+ unlock := lockDocumentParse(docID)
+ defer unlock()
+
+ // Settle and delete the task before purging its cache state. A conditional
+ // delete failure leaves the document and the newly active run intact.
+ if err := s.cleanupDocumentTask(ctx, docID, taskTerminalMessageDocumentDeleted); err != nil {
+ return err
+ }
// Release an uncommitted parse hold before the document disappears.
if billingService := billing.Get(); billingService != nil && billingService.Enabled() {
@@ -288,21 +296,6 @@ func (s *DocumentService) deleteDocumentFull(ctx context.Context, docID string)
}
}
- // Delete tasks from DB
- ingestionTask, err := s.ingestionTaskDAO.GetByDocumentID(ctx, dao.DB, docID)
- if err != nil {
- common.Error(fmt.Sprintf("failed to get ingestion task by doc:%s", doc.ID), err)
- return err
- }
- if ingestionTask != nil {
- if err := s.purgeTaskStateForCleanup(ctx, ingestionTask.ID); err != nil {
- return err
- }
- if _, err := s.ingestionTaskSvc.Remove(ctx, ingestionTask.ID, &ingestionTask.UserID); err != nil {
- return err
- }
- }
-
if err := s.deleteDocEngineData(ctx, docID, kb.TenantID, doc.KbID); err != nil {
return err
}
@@ -328,21 +321,18 @@ func (s *DocumentService) RemoveDocumentKeepFile(ctx context.Context, docID stri
if err != nil {
return err
}
+ unlock := lockDocumentParse(docID)
+ defer unlock()
+
variants, taskTypes, typeErr := s.documentKnowledgeCompileTypes(ctx, kb.TenantID, kb.ID, docID)
if typeErr != nil {
return fmt.Errorf("resolve knowledge compile types for document %s: %w", docID, typeErr)
}
- ingestionTask, err := s.ingestionTaskDAO.GetByDocumentID(ctx, dao.DB, docID)
- if err != nil {
- return fmt.Errorf("failed to get ingestion task for %s: %w", docID, err)
- }
- if ingestionTask != nil {
- if err := s.purgeTaskStateForCleanup(ctx, ingestionTask.ID); err != nil {
- return err
- }
- if _, err := s.ingestionTaskSvc.Remove(ctx, ingestionTask.ID, nil); err != nil {
- return fmt.Errorf("remove ingestion task for document %s: %w", docID, err)
- }
+
+ // Settle and delete the task before purging its cache state. Active workers
+ // must be stopped before this path can run.
+ if err := s.cleanupDocumentTask(ctx, docID, taskTerminalMessageDocumentRemoved); err != nil {
+ return err
}
if _, delErr := s.taskDAO.DeleteByDocIDs(ctx, dao.DB, []string{docID}); delErr != nil {
if errors.Is(delErr, context.Canceled) || errors.Is(delErr, context.DeadlineExceeded) {
diff --git a/internal/service/document/document_ingest.go b/internal/service/document/document_ingest.go
index 02835fedf1..15d839c582 100644
--- a/internal/service/document/document_ingest.go
+++ b/internal/service/document/document_ingest.go
@@ -135,7 +135,7 @@ func (s *DocumentService) Ingest(ctx context.Context, userID string, req *Ingest
if run == string(entity.TaskStatusCancel) {
if err = s.CancelDocParse(ctx, doc); err != nil {
common.Error(fmt.Sprintf("go side, start to process %s, run is cancel", doc.ID), err)
- if errors.Is(err, errParseNotRunning) {
+ if errors.Is(err, ErrParseNotRunning) {
// Mirror the Python /documents/ingest endpoint's message.
return common.CodeDataError, errors.New("Cannot cancel a task that is not in RUNNING status")
}
@@ -150,20 +150,33 @@ func (s *DocumentService) Ingest(ctx context.Context, userID string, req *Ingest
continue
}
- // Delete-only: user asked to remove prior parse results without
- // starting a new parse. RUNNING already continued above.
+ if req.Delete {
+ // Delete-only: user asked to remove prior parse results without
+ // starting a new parse. Serialize the counter reset and cleanup with
+ // StartParseDocuments so a concurrent parse cannot reuse the task
+ // while its old cache state is being removed.
+ unlock := lockDocumentParse(doc.ID)
+ if err = s.documentDAO.UpdateByID(ctx, dao.DB, doc.ID, map[string]interface{}{
+ "progress": 0,
+ }); err != nil {
+ unlock()
+ common.Error(fmt.Sprintf("go side, doc %s, UpdateByID failed", doc.ID), err)
+ return common.CodeExceptionError, err
+ }
+ if err := s.clearDocumentParseResults(ctx, doc, kb.TenantID, taskTerminalMessageParseResultsDeleted); err != nil {
+ unlock()
+ return common.CodeExceptionError, err
+ }
+ unlock()
+ continue
+ }
+
if err = s.documentDAO.UpdateByID(ctx, dao.DB, doc.ID, map[string]interface{}{
"progress": 0,
}); err != nil {
common.Error(fmt.Sprintf("go side, doc %s, UpdateByID failed", doc.ID), err)
return common.CodeExceptionError, err
}
-
- if req.Delete {
- if err := s.clearDocumentParseResults(ctx, doc, kb.TenantID); err != nil {
- return common.CodeExceptionError, err
- }
- }
}
return common.CodeSuccess, nil
diff --git a/internal/service/document/document_parse.go b/internal/service/document/document_parse.go
index 3992f8b4a3..98d248a042 100644
--- a/internal/service/document/document_parse.go
+++ b/internal/service/document/document_parse.go
@@ -34,7 +34,11 @@ var documentParseLocks = struct {
}{locks: make(map[string]*documentParseLock)}
const (
- cleanupBatchTimeout = 30 * time.Second
+ cleanupBatchTimeout = 30 * time.Second
+ taskTerminalMessageSuperseded = "Task superseded by a new parse request."
+ taskTerminalMessageDocumentDeleted = "Task cancelled by document deletion."
+ taskTerminalMessageDocumentRemoved = "Task cancelled by document removal."
+ taskTerminalMessageParseResultsDeleted = "Task cancelled by parse result deletion."
)
func lockDocumentParse(docID string) func() {
@@ -59,18 +63,48 @@ func lockDocumentParse(docID string) func() {
}
}
-// purgeTaskStateForCleanup removes the per-chunk cache state for a task before
-// its row is deleted. The bounded context makes Redis outages and hung clients
-// observable to the caller instead of allowing a destructive task/document
-// delete to proceed with stale cache state.
-func (s *DocumentService) purgeTaskStateForCleanup(ctx context.Context, taskID string) error {
+// cleanupDocumentTask settles and conditionally deletes the document's task
+// before purging its run-scoped cache state. If the task starts running between
+// settlement and deletion, the conditional delete fails and no cache state is
+// removed for the new run.
+func (s *DocumentService) cleanupDocumentTask(ctx context.Context, docID, terminalMessage string) error {
+ task, err := s.ingestionTaskSvc.PrepareDocumentTaskForCleanup(ctx, docID, terminalMessage)
+ if err != nil {
+ return fmt.Errorf("prepare ingestion task for document %s cleanup: %w", docID, err)
+ }
+ if err := s.deletePreparedDocumentTask(ctx, docID, task); err != nil {
+ return err
+ }
+ if task == nil {
+ return nil
+ }
+
purgeTaskState := s.purgeTaskState
if purgeTaskState == nil {
purgeTaskState = ingestionpipeline.PurgeTaskState
}
- batchCtx, cancel := context.WithTimeout(ctx, cleanupBatchTimeout)
+ // The task row is gone, so a request cancellation must not skip cleanup that
+ // is now retryable only through the 24-hour TTL.
+ batchCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), cleanupBatchTimeout)
defer cancel()
- return purgeTaskState(batchCtx, taskID)
+ if err := purgeTaskState(batchCtx, task.ID); err != nil {
+ return fmt.Errorf("purge task state for document %s task %s: %w", docID, task.ID, err)
+ }
+ return nil
+}
+
+func (s *DocumentService) deletePreparedDocumentTask(ctx context.Context, docID string, task *entity.IngestionTask) error {
+ if task == nil {
+ return nil
+ }
+ deleted, err := s.ingestionTaskDAO.DeleteIfTerminal(ctx, dao.DB, task.ID)
+ if err != nil {
+ return fmt.Errorf("delete ingestion task %s for document %s: %w", task.ID, docID, err)
+ }
+ if deleted != 1 {
+ return fmt.Errorf("document %s ingestion task %s started running; stop it before cleanup", docID, task.ID)
+ }
+ return nil
}
// StartParseDocuments starts parsing a document via the DSL ingestion
@@ -95,7 +129,7 @@ func (s *DocumentService) StartParseDocuments(ctx context.Context, doc *entity.D
}
if opts.RerunWithDelete {
- if err := s.clearDocumentParseResults(ctx, doc, kb.TenantID); err != nil {
+ if err := s.clearDocumentParseResults(ctx, doc, kb.TenantID, taskTerminalMessageSuperseded); err != nil {
return err
}
}
@@ -285,60 +319,24 @@ func (s *DocumentService) AssertIngestionTasksTerminal(ctx context.Context, docI
if err != nil {
return fmt.Errorf("check ingestion task for %s: %w", docID, err)
}
- if task == nil {
- continue
- }
- if common.IsRunningOrStopping(task.Status) {
- return fmt.Errorf("document %s ingestion task is %s; stop it and wait for a terminal state before re-parsing", docID, task.Status)
+ if task != nil && common.IsRunningOrStopping(task.Status) {
+ return fmt.Errorf("document %s ingestion task %s is %s; stop it and wait for a terminal state before re-parsing", docID, task.ID, task.Status)
}
}
return nil
}
-func (s *DocumentService) clearDocumentParseResults(ctx context.Context, doc *entity.Document, tenantID string) error {
+func (s *DocumentService) clearDocumentParseResults(ctx context.Context, doc *entity.Document, tenantID, terminalMessage string) error {
if doc == nil {
return fmt.Errorf("document is nil")
}
- // Refuse to clear a non-terminal ingestion task. An in-flight worker
- // (RUNNING) or one mid-stop (STOPPING) would keep writing chunks and
- // corrupt the new run's results. The caller must stop the task first
- // and wait for a terminal state (COMPLETED/STOPPED/FAILED), CREATED, or
- // SCHEDULED.
- task, err := s.ingestionTaskDAO.GetByDocumentID(ctx, dao.DB, doc.ID)
- if err != nil {
- return fmt.Errorf("get ingestion task for document %s: %w", doc.ID, err)
- }
- taskExisted := task != nil
- if task != nil {
- if task.Status == common.RUNNING || task.Status == common.STOPPING {
- return fmt.Errorf("document %s ingestion task is %s; stop it and wait for a terminal state before re-parsing", doc.ID, task.Status)
- }
- if task.Status == common.CREATED || task.Status == common.SCHEDULED {
- if err := s.ingestionTaskSvc.SupersedeUnstartedTask(ctx, task.ID); err != nil {
- return fmt.Errorf("supersede queued ingestion task for document %s: %w", doc.ID, err)
- }
- task.Status = common.STOPPED
- }
- taskExisted = true
- }
- if task != nil {
- if err := s.purgeTaskStateForCleanup(ctx, task.ID); err != nil {
- return fmt.Errorf("purge task state for document %s: %w", doc.ID, err)
- }
- }
-
- // Delete terminal, CREATED, and SCHEDULED ingestion tasks atomically, leaving
- // RUNNING/STOPPING tasks untouched so the check-then-delete window
- // between GetByDocumentID and the delete cannot delete a task that just
- // transitioned to RUNNING. In that case, do not clear its parse results.
- deleted, err := s.ingestionTaskDAO.DeleteIfTerminal(ctx, dao.DB, doc.ID)
- if err != nil {
+ // Settle and conditionally delete the task before touching cache state or
+ // durable results. A task that starts running after preparation blocks the
+ // delete and leaves both the document and its results intact.
+ if err := s.cleanupDocumentTask(ctx, doc.ID, terminalMessage); err != nil {
return err
}
- if taskExisted && deleted == 0 {
- return fmt.Errorf("document %s ingestion task started running; stop it before re-parsing", doc.ID)
- }
if err := s.clearDocumentAndKBCountersForRerun(doc.ID, doc.KbID); err != nil {
return err
}
@@ -504,7 +502,7 @@ func (s *DocumentService) StopParseDocuments(ctx context.Context, datasetID stri
successCount := 0
for _, doc := range docs {
if cancelErr := s.CancelDocParse(ctx, doc); cancelErr != nil {
- if errors.Is(cancelErr, errParseNotRunning) {
+ if errors.Is(cancelErr, ErrParseNotRunning) {
// Mirror the Python /documents/stop endpoint's message.
errs = append(errs, "Can't stop parsing document that has not started or already completed")
} else {
@@ -566,35 +564,34 @@ func (s *DocumentService) validateDocsInDataset(ctx context.Context, docIDs []st
return docs, nil
}
-// errParseNotRunning is returned by CancelDocParse when the document has no
+// ErrParseNotRunning is returned by CancelDocParse when the document has no
// in-flight ingestion task and is not already stopped. Callers map it to their
// endpoint-specific message (the Python /documents/ingest and /documents/stop messages differ).
-var errParseNotRunning = errors.New("parse task is not in running status")
+var ErrParseNotRunning = errors.New("parse task is not in running status")
-// CancelDocParse stops the ingestion task for the document by calling
-// RequestStop (STOPPING).
-// It returns errParseNotRunning if the document has neither an in-flight
-// ingestion task (CREATED/SCHEDULED/RUNNING/STOPPING) nor an already stopped
-// task (STOPPED).
+// CancelDocParse stops the document's active ingestion task. It returns
+// ErrParseNotRunning when the task is absent or already completed/failed.
func (s *DocumentService) CancelDocParse(ctx context.Context, doc *entity.Document) error {
task, err := s.ingestionTaskDAO.GetByDocumentID(ctx, dao.DB, doc.ID)
if err != nil {
return fmt.Errorf("failed to get ingestion task for %s: %w", doc.ID, err)
}
-
- inFlight := task != nil && common.IsActiveTaskStatus(task.Status)
- isStopped := task != nil && task.Status == common.STOPPED
- if !inFlight && !isStopped {
- return errParseNotRunning
+ if task == nil {
+ return ErrParseNotRunning
}
-
- if inFlight {
- if _, err = s.ingestionTaskSvc.RequestStop(ctx, task.ID); err != nil {
+ switch {
+ case common.IsActiveTaskStatus(task.Status):
+ if _, err := s.ingestionTaskSvc.RequestStop(ctx, task.ID); err != nil {
return fmt.Errorf("failed to stop ingestion task %s: %w", task.ID, err)
}
+ return nil
+ case task.Status == common.STOPPED:
+ return nil
+ case task.Status == common.COMPLETED || task.Status == common.FAILED:
+ return ErrParseNotRunning
+ default:
+ return fmt.Errorf("ingestion task %s has unrecognized status %q", task.ID, task.Status)
}
-
- return nil
}
// deleteDocumentChunkImages removes source chunk image objects in bounded
diff --git a/internal/service/document/document_test.go b/internal/service/document/document_test.go
index a6de761893..3d361f6b04 100644
--- a/internal/service/document/document_test.go
+++ b/internal/service/document/document_test.go
@@ -789,6 +789,29 @@ func insertTestIngestionTaskWithStatus(t *testing.T, id, userID, docID, datasetI
}
}
+func insertTestOpenTaskRun(t *testing.T, taskID, logID, userID, docID, datasetID string) {
+ t.Helper()
+ insertTestIngestionTaskWithStatus(t, taskID, userID, docID, datasetID, common.CREATED)
+
+ runCount := 1
+ if err := dao.DB.Create(&entity.PipelineOperationLog{
+ ID: logID,
+ DocumentID: docID,
+ TenantID: "tenant-1",
+ KbID: datasetID,
+ ParserID: "naive",
+ TaskType: string(entity.PipelineTaskTypeParse),
+ OperationStatus: string(entity.TaskStatusUnstart),
+ RunCount: &runCount,
+ }).Error; err != nil {
+ t.Fatalf("insert open pipeline run: %v", err)
+ }
+ if err := dao.DB.Model(&entity.IngestionTask{}).Where("id = ?", taskID).
+ Update("pipeline_log_id", logID).Error; err != nil {
+ t.Fatalf("bind open pipeline run: %v", err)
+ }
+}
+
func insertTestFile2Document(t *testing.T, id, fileID, docID string) {
t.Helper()
f2d := &entity.File2Document{
@@ -863,7 +886,7 @@ func TestDeleteDocumentFull_Basic(t *testing.T) {
}
}
-func TestDeleteDocumentFullPurgesTaskStateBeforeDeletingTask(t *testing.T) {
+func TestDeleteDocumentFullDeletesTaskBeforePurgingState(t *testing.T) {
db := setupServiceTestDB(t)
pushServiceDB(t, db)
insertTestKB(t, "kb-1", "tenant-1", 1, 0, 0)
@@ -879,11 +902,11 @@ func TestDeleteDocumentFullPurgesTaskStateBeforeDeletingTask(t *testing.T) {
}
if task, err := svc.ingestionTaskDAO.GetByDocumentID(t.Context(), db, "doc-1"); err != nil {
t.Fatalf("reload ingestion task: %v", err)
- } else if task == nil {
- t.Fatal("ingestion task was deleted despite resumable state cleanup failure")
+ } else if task != nil {
+ t.Fatal("ingestion task was not deleted before state cleanup")
}
if _, err := svc.documentDAO.GetByID(t.Context(), db, "doc-1"); err != nil {
- t.Fatalf("document was deleted despite resumable state cleanup failure: %v", err)
+ t.Fatalf("document was deleted before state cleanup succeeded: %v", err)
}
}
@@ -905,7 +928,7 @@ func TestDeleteDocumentFullKeepsDocumentWhenChunkDeletionFails(t *testing.T) {
}
}
-func TestRemoveDocumentKeepFilePurgesTaskStateBeforeDeletingDocument(t *testing.T) {
+func TestRemoveDocumentKeepFileDeletesTaskBeforePurgingState(t *testing.T) {
db := setupServiceTestDB(t)
pushServiceDB(t, db)
insertTestKB(t, "kb-1", "tenant-1", 1, 0, 0)
@@ -921,11 +944,133 @@ func TestRemoveDocumentKeepFilePurgesTaskStateBeforeDeletingDocument(t *testing.
}
if task, err := svc.ingestionTaskDAO.GetByDocumentID(t.Context(), db, "doc-1"); err != nil {
t.Fatalf("reload ingestion task: %v", err)
- } else if task == nil {
- t.Fatal("ingestion task was deleted despite resumable state cleanup failure")
+ } else if task != nil {
+ t.Fatal("ingestion task was not deleted before state cleanup")
}
if _, err := svc.documentDAO.GetByID(t.Context(), db, "doc-1"); err != nil {
- t.Fatalf("document was deleted despite resumable state cleanup failure: %v", err)
+ t.Fatalf("document was deleted before state cleanup succeeded: %v", err)
+ }
+}
+
+func TestDocumentCleanupRecordsCallerSpecificTaskCancellationReason(t *testing.T) {
+ tests := []struct {
+ name string
+ run func(*DocumentService, *entity.Document) error
+ want string
+ }{
+ {
+ name: "delete document",
+ run: func(svc *DocumentService, _ *entity.Document) error {
+ return svc.deleteDocumentFull(t.Context(), "doc-1")
+ },
+ want: taskTerminalMessageDocumentDeleted,
+ },
+ {
+ name: "remove document keep file",
+ run: func(svc *DocumentService, _ *entity.Document) error {
+ return svc.RemoveDocumentKeepFile(t.Context(), "doc-1")
+ },
+ want: taskTerminalMessageDocumentRemoved,
+ },
+ {
+ name: "clear parse results",
+ run: func(svc *DocumentService, doc *entity.Document) error {
+ return svc.clearDocumentParseResults(t.Context(), doc, "tenant-1", taskTerminalMessageSuperseded)
+ },
+ want: taskTerminalMessageSuperseded,
+ },
+ }
+
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ db := setupServiceTestDB(t)
+ pushServiceDB(t, db)
+ insertTestKB(t, "kb-1", "tenant-1", 1, 0, 0)
+ insertTestDoc(t, "doc-1", "kb-1", 0, 0)
+ insertTestOpenTaskRun(t, "task-1", "parse-run-1", "user-1", "doc-1", "kb-1")
+
+ svc := testDocumentService(t)
+ svc.purgeTaskState = func(context.Context, string) error { return nil }
+ doc, err := svc.documentDAO.GetByID(t.Context(), db, "doc-1")
+ if err != nil {
+ t.Fatalf("load document: %v", err)
+ }
+ if err := tt.run(svc, doc); err != nil {
+ t.Fatalf("%s: %v", tt.name, err)
+ }
+
+ var terminal entity.IngestionTaskLog
+ if err := db.Where("pipeline_log_id = ? AND event_type = ?", "parse-run-1", dao.EventTypeTerminal).
+ First(&terminal).Error; err != nil {
+ t.Fatalf("load terminal event: %v", err)
+ }
+ if terminal.Message != tt.want {
+ t.Fatalf("terminal message = %q, want %q", terminal.Message, tt.want)
+ }
+ })
+ }
+}
+
+func TestDocumentCleanupRejectsActiveTask(t *testing.T) {
+ for _, status := range []string{common.RUNNING, common.STOPPING} {
+ t.Run(status, func(t *testing.T) {
+ db := setupServiceTestDB(t)
+ pushServiceDB(t, db)
+ insertTestKB(t, "kb-1", "tenant-1", 1, 10, 5)
+ insertTestDoc(t, "doc-1", "kb-1", 10, 5)
+ insertTestIngestionTaskWithStatus(t, "task-active", "user-1", "doc-1", "kb-1", status)
+
+ svc := testDocumentService(t)
+ doc, err := svc.documentDAO.GetByID(t.Context(), db, "doc-1")
+ if err != nil {
+ t.Fatalf("load document: %v", err)
+ }
+
+ tests := []struct {
+ name string
+ run func() error
+ }{
+ {
+ name: "delete document",
+ run: func() error { return svc.deleteDocumentFull(t.Context(), "doc-1") },
+ },
+ {
+ name: "remove document keep file",
+ run: func() error { return svc.RemoveDocumentKeepFile(t.Context(), "doc-1") },
+ },
+ {
+ name: "clear parse results",
+ run: func() error {
+ return svc.clearDocumentParseResults(t.Context(), doc, "tenant-1", taskTerminalMessageSuperseded)
+ },
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ purged := map[string]bool{}
+ svc.purgeTaskState = func(_ context.Context, taskID string) error {
+ purged[taskID] = true
+ return nil
+ }
+ if err := tt.run(); err == nil {
+ t.Fatalf("cleanup succeeded with an active %s task", status)
+ }
+ if len(purged) != 0 {
+ t.Fatalf("purged task state %v before rejecting an active task", purged)
+ }
+ if _, err := svc.ingestionTaskDAO.GetByID(t.Context(), db, "task-active"); err != nil {
+ t.Fatalf("task was removed after rejected cleanup: %v", err)
+ }
+ stored, err := svc.documentDAO.GetByID(t.Context(), db, "doc-1")
+ if err != nil {
+ t.Fatalf("document was removed after rejected cleanup: %v", err)
+ }
+ if stored.TokenNum != 10 || stored.ChunkNum != 5 {
+ t.Fatalf("document counters changed after rejected cleanup: token=%d chunk=%d", stored.TokenNum, stored.ChunkNum)
+ }
+ })
+ }
+ })
}
}
@@ -3311,7 +3456,7 @@ func TestClearDocumentParseResultsClearsCountersTasksAndChunks(t *testing.T) {
svc := testDocumentService(t)
svc.docEngine = engine
- if err = svc.clearDocumentParseResults(ctx, doc, "tenant-1"); err != nil {
+ if err = svc.clearDocumentParseResults(ctx, doc, "tenant-1", taskTerminalMessageSuperseded); err != nil {
t.Fatalf("clearDocumentParseResults failed: %v", err)
}
@@ -3353,7 +3498,7 @@ func TestClearDocumentParseResultsDeletesDocumentGeneratedChunks(t *testing.T) {
t.Fatalf("load document: %v", err)
}
- if err := svc.clearDocumentParseResults(t.Context(), doc, "tenant-1"); err != nil {
+ if err := svc.clearDocumentParseResults(t.Context(), doc, "tenant-1", taskTerminalMessageSuperseded); err != nil {
t.Fatalf("clearDocumentParseResults failed: %v", err)
}
if !engine.generatedDeleted {
@@ -3386,7 +3531,7 @@ func TestClearDocumentParseResultsDeletesSourceChunkImages(t *testing.T) {
t.Fatalf("load document: %v", err)
}
- if err := svc.clearDocumentParseResults(t.Context(), doc, "tenant-1"); err != nil {
+ if err := svc.clearDocumentParseResults(t.Context(), doc, "tenant-1", taskTerminalMessageSuperseded); err != nil {
t.Fatalf("clearDocumentParseResults failed: %v", err)
}
if imageStore.ObjExist(t.Context(), "kb-1", "image-001") {
@@ -3394,7 +3539,7 @@ func TestClearDocumentParseResultsDeletesSourceChunkImages(t *testing.T) {
}
}
-func TestClearDocumentParseResultsPurgesTaskStateBeforeDeletingTask(t *testing.T) {
+func TestClearDocumentParseResultsDeletesTaskBeforePurgingState(t *testing.T) {
db := setupServiceTestDB(t)
pushServiceDB(t, db)
insertTestKB(t, "kb-1", "tenant-1", 0, 0, 0)
@@ -3409,15 +3554,15 @@ func TestClearDocumentParseResultsPurgesTaskStateBeforeDeletingTask(t *testing.T
t.Fatalf("load document: %v", err)
}
- if err := svc.clearDocumentParseResults(t.Context(), doc, "tenant-1"); !errors.Is(err, purgeErr) {
+ if err := svc.clearDocumentParseResults(t.Context(), doc, "tenant-1", taskTerminalMessageSuperseded); !errors.Is(err, purgeErr) {
t.Fatalf("clearDocumentParseResults error = %v, want purge error", err)
}
remaining, err := svc.ingestionTaskDAO.GetByDocumentID(t.Context(), db, "doc-1")
if err != nil {
t.Fatalf("reload ingestion task: %v", err)
}
- if remaining == nil {
- t.Fatal("task was deleted despite resumable state cleanup failure")
+ if remaining != nil {
+ t.Fatal("task was not deleted before state cleanup")
}
}
@@ -3487,10 +3632,10 @@ func TestClearDocumentParseResultsIsIdempotentForStaleDocSnapshot(t *testing.T)
}
svc := testDocumentService(t)
- if err = svc.clearDocumentParseResults(ctx, staleDoc, "tenant-1"); err != nil {
+ if err = svc.clearDocumentParseResults(ctx, staleDoc, "tenant-1", taskTerminalMessageSuperseded); err != nil {
t.Fatalf("first clearDocumentParseResults failed: %v", err)
}
- if err = svc.clearDocumentParseResults(ctx, staleDoc, "tenant-1"); err != nil {
+ if err = svc.clearDocumentParseResults(ctx, staleDoc, "tenant-1", taskTerminalMessageSuperseded); err != nil {
t.Fatalf("second clearDocumentParseResults failed: %v", err)
}
@@ -3523,7 +3668,7 @@ func TestClearDocumentParseResults_RejectsNonTerminalIngestionTask(t *testing.T)
}
svc := testDocumentService(t)
- if err = svc.clearDocumentParseResults(ctx, doc, "tenant-1"); err == nil {
+ if err = svc.clearDocumentParseResults(ctx, doc, "tenant-1", taskTerminalMessageSuperseded); err == nil {
t.Fatalf("expected error for %s ingestion task, got nil", status)
}
// The non-terminal task must NOT be deleted.
@@ -3560,13 +3705,24 @@ func TestClearDocumentParseResultsDoesNotClearResultsWhenTaskStartsRunning(t *te
if err != nil {
t.Fatalf("get document: %v", err)
}
- if err = testDocumentService(t).clearDocumentParseResults(ctx, doc, "tenant-1"); err == nil || !strings.Contains(err.Error(), "started running") {
+ purgeCalls := 0
+ svc := testDocumentService(t)
+ svc.purgeTaskState = func(context.Context, string) error {
+ purgeCalls++
+ return nil
+ }
+ if err = svc.clearDocumentParseResults(ctx, doc, "tenant-1", taskTerminalMessageSuperseded); err == nil || !strings.Contains(err.Error(), "started running") {
t.Fatalf("expected reparse cleanup to reject a task that started running, got %v", err)
}
+ if purgeCalls != 0 {
+ t.Fatalf("purged task state %d time(s) after the conditional task delete failed", purgeCalls)
+ }
task, err := dao.NewIngestionTaskDAO().GetByID(ctx, db, "task-1")
if err != nil {
t.Fatalf("reload task: %v", err)
}
+ // The task won the race and remains active, so the conditional delete did
+ // not remove it.
if task.Status != common.RUNNING {
t.Fatalf("task status = %q, want %q", task.Status, common.RUNNING)
}
@@ -3605,7 +3761,7 @@ func TestClearDocumentParseResults_DeletesTerminalIngestionTask(t *testing.T) {
}
svc := testDocumentService(t)
- if err = svc.clearDocumentParseResults(ctx, doc, "tenant-1"); err != nil {
+ if err = svc.clearDocumentParseResults(ctx, doc, "tenant-1", taskTerminalMessageSuperseded); err != nil {
t.Fatalf("clearDocumentParseResults for %s task: %v", status, err)
}
task, _ := svc.ingestionTaskDAO.GetByDocumentID(ctx, db, "doc-1")
@@ -5115,9 +5271,10 @@ func TestIngest_DeleteOnlyCleansTasks(t *testing.T) {
insertUserTenantForAccessCheck(t, "user-1", "tenant-1")
insertTestKB(t, "kb-1", "tenant-1", 0, 0, 0)
insertTestDoc(t, "doc-1", "kb-1", 10, 5)
- insertTestIngestionTaskWithStatus(t, "task-1", "user-1", "doc-1", "kb-1", common.COMPLETED)
+ insertTestOpenTaskRun(t, "task-1", "parse-run-1", "user-1", "doc-1", "kb-1")
svc := testDocumentService(t)
+ svc.purgeTaskState = func(context.Context, string) error { return nil }
ctx := documentTestContext(t.Context())
code, err := svc.Ingest(ctx, "user-1", &IngestDocumentRequest{
DocIDs: []string{"doc-1"},
@@ -5139,6 +5296,15 @@ func TestIngest_DeleteOnlyCleansTasks(t *testing.T) {
t.Fatal("expected task to be deleted in delete-only mode")
}
+ var terminal entity.IngestionTaskLog
+ if err := db.Where("pipeline_log_id = ? AND event_type = ?", "parse-run-1", dao.EventTypeTerminal).
+ First(&terminal).Error; err != nil {
+ t.Fatalf("load terminal event: %v", err)
+ }
+ if terminal.Message != taskTerminalMessageParseResultsDeleted {
+ t.Fatalf("terminal message = %q, want delete-only reason", terminal.Message)
+ }
+
doc, err := dao.NewDocumentDAO().GetByID(ctx, db, "doc-1")
if err != nil {
t.Fatalf("query doc: %v", err)
diff --git a/internal/service/ingestion_task_service.go b/internal/service/ingestion_task_service.go
index 8f87c600f3..c427daba90 100644
--- a/internal/service/ingestion_task_service.go
+++ b/internal/service/ingestion_task_service.go
@@ -5,15 +5,18 @@ import (
"errors"
"fmt"
"strings"
+ "sync"
"time"
"ragflow/internal/common"
"ragflow/internal/dao"
"ragflow/internal/engine/kvrocks"
"ragflow/internal/entity"
+ ingestionpipeline "ragflow/internal/ingestion/pipeline"
"go.uber.org/zap"
"gorm.io/gorm"
+ "gorm.io/gorm/clause"
)
// ingestionEventKind intentionally has an invalid zero value. The persisted
@@ -29,6 +32,58 @@ const (
ingestionEventSystem
)
+const (
+ taskStateCleanupTimeout = 5 * time.Second
+ taskStateCleanupCapacity = 1024
+ taskTerminalMessageComplete = "Task completed."
+ taskTerminalMessageFailed = "Task failed."
+)
+
+type taskStatePurgeRequest struct {
+ ctx context.Context
+ taskID string
+ purgeTask func(context.Context, string) error
+}
+
+var taskStateCleanupQueue = struct {
+ once sync.Once
+ queue chan taskStatePurgeRequest
+}{
+ queue: make(chan taskStatePurgeRequest, taskStateCleanupCapacity),
+}
+
+func scheduleTaskStatePurge(req taskStatePurgeRequest) bool {
+ taskStateCleanupQueue.once.Do(func() {
+ go func() {
+ for req := range taskStateCleanupQueue.queue {
+ started := time.Now()
+ ctx, cancel := context.WithTimeout(context.WithoutCancel(req.ctx), taskStateCleanupTimeout)
+ err := req.purgeTask(ctx, req.taskID)
+ cancel()
+ elapsed := time.Since(started)
+ if err != nil {
+ common.Warn("purge task state after delete",
+ zap.String("task_id", req.taskID),
+ zap.Duration("elapsed", elapsed),
+ zap.Error(err),
+ )
+ continue
+ }
+ common.Debug("purged task state after delete",
+ zap.String("task_id", req.taskID),
+ zap.Duration("elapsed", elapsed),
+ )
+ }
+ }()
+ })
+ select {
+ case taskStateCleanupQueue.queue <- req:
+ return true
+ default:
+ return false
+ }
+}
+
type InvalidTaskTransitionError struct {
TaskID string
From string
@@ -70,6 +125,9 @@ type IngestionTaskService struct {
userCanvasDAO *dao.UserCanvasDAO
taskPublisher TaskPublisher
logSettings IngestionLogSettings
+ // purgeTaskState removes the deleted task's run-scoped chunk and media
+ // cache state. Nil resolves to the process-wide implementation.
+ purgeTaskState func(context.Context, string) error
}
func NewIngestionTaskService() *IngestionTaskService {
@@ -272,23 +330,65 @@ func (s *IngestionTaskService) RequestStop(ctx context.Context, taskID string) (
// created. A numbered pipeline log is an immutable run ledger, so this must
// record its cancellation rather than deleting the row with the task.
func (s *IngestionTaskService) SupersedeUnstartedTask(ctx context.Context, taskID string) error {
+ _, err := s.supersedeUnstartedTask(ctx, taskID, "Task superseded by a new parse request.")
+ return err
+}
+
+func (s *IngestionTaskService) supersedeUnstartedTask(ctx context.Context, taskID, terminalMessage string) (*entity.IngestionTask, error) {
task, err := s.GetTask(ctx, taskID)
if err != nil {
- return err
+ return nil, err
}
switch task.Status {
case common.CREATED, common.SCHEDULED:
stopped, err := s.transition(ctx, taskID, common.STOPPED)
if err != nil {
- return err
+ return nil, err
}
- s.advanceOpenLog(ctx, stopped, dao.OpenPipelineOperationStatuses(), string(entity.TaskStatusCancel))
- s.recordRunTerminal(ctx, stopped, "Task superseded by a new parse request.")
- return nil
+ if err := s.settleTaskRun(ctx, stopped, entity.TaskStatusCancel, terminalMessage); err != nil {
+ return nil, err
+ }
+ return stopped, nil
case common.RUNNING, common.STOPPING:
- return fmt.Errorf("task %s is %s and cannot be superseded", taskID, task.Status)
+ return nil, fmt.Errorf("task %s is %s and cannot be superseded", taskID, task.Status)
default:
- return nil
+ return task, nil
+ }
+}
+
+// PrepareDocumentTaskForCleanup rejects an active task and freezes a queued
+// task before destructive document cleanup. The caller deletes the returned
+// terminal task after purging its run-scoped cache state.
+func (s *IngestionTaskService) PrepareDocumentTaskForCleanup(ctx context.Context, documentID, terminalMessage string) (*entity.IngestionTask, error) {
+ task, err := s.ingestionTaskDAO.GetByDocumentID(ctx, dao.DB, documentID)
+ if err != nil {
+ return nil, err
+ }
+ if task == nil {
+ return nil, nil
+ }
+ switch task.Status {
+ case common.CREATED, common.SCHEDULED:
+ return s.supersedeUnstartedTask(ctx, task.ID, terminalMessage)
+ case common.RUNNING, common.STOPPING:
+ return nil, fmt.Errorf("document %s ingestion task %s is %s; stop it and wait for a terminal state before cleanup", documentID, task.ID, task.Status)
+ case common.STOPPED:
+ if err := s.settleTaskRun(ctx, task, entity.TaskStatusCancel, terminalMessage); err != nil {
+ return nil, err
+ }
+ return task, nil
+ case common.COMPLETED:
+ if err := s.settleTaskRun(ctx, task, entity.TaskStatusDone, taskTerminalMessageComplete); err != nil {
+ return nil, err
+ }
+ return task, nil
+ case common.FAILED:
+ if err := s.settleTaskRun(ctx, task, entity.TaskStatusFail, taskTerminalMessageFailed); err != nil {
+ return nil, err
+ }
+ return task, nil
+ default:
+ return nil, fmt.Errorf("document %s ingestion task %s has unrecognized status %q", documentID, task.ID, task.Status)
}
}
@@ -355,10 +455,40 @@ func (s *IngestionTaskService) Remove(ctx context.Context, taskID string, userID
}
}
}
+ // A queued task becomes STOPPED before its run is settled. If settlement
+ // failed, a retry must finish it before the task row is deleted.
+ if task.Status == common.STOPPED && task.PipelineLogID != nil && *task.PipelineLogID != "" {
+ run, runErr := s.pipelineLogDAO.GetByID(ctx, dao.DB, *task.PipelineLogID)
+ if runErr != nil {
+ return nil, runErr
+ }
+ if run.RunCount != nil && *run.RunCount > 0 {
+ if err := s.settleTaskRun(ctx, task, entity.TaskStatusCancel, "Task superseded by a new parse request."); err != nil {
+ return nil, err
+ }
+ }
+ }
info, err := s.ingestionTaskDAO.Delete(ctx, dao.DB, taskID, userID)
if err != nil {
return nil, err
}
+ // The row is gone, so purge failures cannot be retried by the caller and
+ // TTL remains the backstop. Keep backend I/O off the request path; RemoveMany
+ // may otherwise accumulate one slow Kvrocks round-trip per deleted task.
+ purgeTaskState := s.purgeTaskState
+ if purgeTaskState == nil {
+ purgeTaskState = ingestionpipeline.PurgeTaskState
+ }
+ if !scheduleTaskStatePurge(taskStatePurgeRequest{
+ ctx: ctx,
+ taskID: taskID,
+ purgeTask: purgeTaskState,
+ }) {
+ common.Warn("task-state cleanup queue is full",
+ zap.String("task_id", taskID),
+ zap.Int("capacity", taskStateCleanupCapacity),
+ )
+ }
return info, nil
}
@@ -768,12 +898,17 @@ func isTerminalIngestionTask(status string) bool {
return status == common.COMPLETED || status == common.STOPPED || status == common.FAILED
}
-// advanceOpenLog moves a task's already-bound row to operationStatus. It never
-// creates or adopts a row, so a worker cannot repair a missing identity or
-// affect a newer run.
+// advanceOpenLog performs a best-effort ledger advance for non-cleanup flows.
func (s *IngestionTaskService) advanceOpenLog(ctx context.Context, task *entity.IngestionTask, fromStatuses []string, operationStatus string) {
+ _, _ = s.advanceOpenLogIfOpen(ctx, task, fromStatuses, operationStatus)
+}
+
+// advanceOpenLogIfOpen moves a task's already-bound row to operationStatus and
+// reports whether this call changed it. It never creates or adopts a row, so a
+// worker cannot repair a missing identity or affect a newer run.
+func (s *IngestionTaskService) advanceOpenLogIfOpen(ctx context.Context, task *entity.IngestionTask, fromStatuses []string, operationStatus string) (bool, error) {
if task == nil || task.DocumentID == "" || s.pipelineLogDAO == nil {
- return
+ return false, nil
}
logID := ""
if task.PipelineLogID != nil {
@@ -781,11 +916,29 @@ func (s *IngestionTaskService) advanceOpenLog(ctx context.Context, task *entity.
}
if logID == "" {
common.Warn(fmt.Sprintf("advance open pipeline log for task %s: missing run identity", task.ID))
- return
+ return false, nil
}
- if err := s.pipelineLogDAO.AdvanceOpenLog(ctx, dao.DB, logID, fromStatuses, operationStatus); err != nil {
+ advanced, err := s.pipelineLogDAO.AdvanceOpenLog(ctx, dao.DB, logID, fromStatuses, operationStatus)
+ if err != nil {
common.Warn(fmt.Sprintf("advance open pipeline log for document %s to %s: %v", task.DocumentID, operationStatus, err))
}
+ return advanced, err
+}
+
+// settleTaskRun closes a terminal task's run ledger and ensures its terminal
+// event exists. The event write is retried even when a previous attempt already
+// advanced the operation row, so a partial settlement never becomes permanent.
+func (s *IngestionTaskService) settleTaskRun(ctx context.Context, task *entity.IngestionTask, operationStatus entity.TaskStatus, terminalMessage string) error {
+ if task == nil || task.PipelineLogID == nil || *task.PipelineLogID == "" {
+ return nil
+ }
+ if _, err := s.advanceOpenLogIfOpen(ctx, task, dao.OpenPipelineOperationStatuses(), string(operationStatus)); err != nil {
+ return err
+ }
+ if err := s.RecordTerminal(ctx, *task.PipelineLogID, task.ID, terminalMessage); err != nil {
+ return fmt.Errorf("record terminal event for task %s run %s: %w", task.ID, *task.PipelineLogID, err)
+ }
+ return nil
}
func (s *IngestionTaskService) recordRunMessage(ctx context.Context, task *entity.IngestionTask, message string) {
@@ -891,7 +1044,7 @@ func (s *IngestionTaskService) RecordMessage(ctx context.Context, pipelineLogID,
// RecordTerminal writes the terminal explanation associated with a run.
func (s *IngestionTaskService) RecordTerminal(ctx context.Context, pipelineLogID, taskID, message string) error {
- if err := s.insertEvent(ctx, ingestionEventTerminal, pipelineLogID, taskID, "", 0, message); err != nil {
+ if err := s.ensureTerminalEvent(ctx, pipelineLogID, taskID, message); err != nil {
return err
}
foldContext := context.WithoutCancel(ctx)
@@ -916,6 +1069,32 @@ func (s *IngestionTaskService) RecordTerminal(ctx context.Context, pipelineLogID
return nil
}
+// ensureTerminalEvent serializes terminal writers on the run row and inserts
+// the terminal event only when one is not already present. This makes retrying
+// settlement safe after the operation status has already advanced.
+func (s *IngestionTaskService) ensureTerminalEvent(ctx context.Context, pipelineLogID, taskID, message string) error {
+ if dao.DB == nil {
+ return errors.New("ingestion task: nil database")
+ }
+ return dao.DB.WithContext(ctx).Transaction(func(tx *gorm.DB) error {
+ var run entity.PipelineOperationLog
+ if err := tx.Clauses(clause.Locking{Strength: "UPDATE"}).
+ Take(&run, "id = ?", pipelineLogID).Error; err != nil {
+ return err
+ }
+ var count int64
+ if err := tx.Model(&entity.IngestionTaskLog{}).
+ Where("pipeline_log_id = ? AND event_type = ?", pipelineLogID, dao.EventTypeTerminal).
+ Count(&count).Error; err != nil {
+ return err
+ }
+ if count > 0 {
+ return nil
+ }
+ return s.insertEventOnDB(ctx, tx, ingestionEventTerminal, pipelineLogID, taskID, "", 0, message)
+ })
+}
+
func rejectIngestionEvent(reason, pipelineLogID, taskID string, err error) error {
common.Warn("ingestion_log_event_rejected",
zap.String("event", "ingestion_log_event_rejected"),
@@ -927,12 +1106,19 @@ func rejectIngestionEvent(reason, pipelineLogID, taskID string, err error) error
}
func (s *IngestionTaskService) insertEvent(ctx context.Context, kind ingestionEventKind, pipelineLogID, taskID, component string, phase int, message string) error {
+ return s.insertEventOnDB(ctx, dao.DB, kind, pipelineLogID, taskID, component, phase, message)
+}
+
+func (s *IngestionTaskService) insertEventOnDB(ctx context.Context, db *gorm.DB, kind ingestionEventKind, pipelineLogID, taskID, component string, phase int, message string) error {
if pipelineLogID == "" {
return rejectIngestionEvent("missing_pipeline_log_id", pipelineLogID, taskID, errors.New("ingestion event requires a pipeline log id"))
}
if taskID == "" {
return rejectIngestionEvent("missing_task_id", pipelineLogID, taskID, errors.New("ingestion event requires a task id"))
}
+ if db == nil {
+ return errors.New("ingestion task: nil database")
+ }
eventType := 0
switch kind {
case ingestionEventLifecycle:
@@ -955,7 +1141,7 @@ func (s *IngestionTaskService) insertEvent(ctx context.Context, kind ingestionEv
// EventTypeLifecycle is zero while the database default is the defensive
// legacy value. A map keeps the explicitly mapped protocol value intact;
// GORM otherwise substitutes a default-tag value for a zero struct field.
- return dao.DB.WithContext(ctx).Model(&entity.IngestionTaskLog{}).Create(map[string]interface{}{
+ return db.WithContext(ctx).Model(&entity.IngestionTaskLog{}).Create(map[string]interface{}{
"task_id": taskID,
"pipeline_log_id": pipelineLogID,
"checkpoint": entity.JSONMap{},
diff --git a/internal/service/ingestion_task_service_test.go b/internal/service/ingestion_task_service_test.go
index 2fee206a6d..0f07d18771 100644
--- a/internal/service/ingestion_task_service_test.go
+++ b/internal/service/ingestion_task_service_test.go
@@ -7,6 +7,7 @@ import (
"strings"
"sync"
"testing"
+ "time"
"unicode/utf8"
"gorm.io/gorm"
@@ -1072,6 +1073,485 @@ func TestIngestionTaskServiceRemoveSettlesNumberedQueuedRun(t *testing.T) {
}
}
+func TestIngestionTaskServiceRemoveRetriesStoppedRunSettlement(t *testing.T) {
+ db := setupServiceTestDB(t)
+ pushServiceDB(t, db)
+ insertTestKB(t, "kb-1", "tenant-1", 1, 0, 0)
+ insertTestDoc(t, "doc-1", "kb-1", 0, 0)
+ insertTestIngestionTaskWithStatus(t, "task-1", "user-1", "doc-1", "kb-1", common.CREATED)
+
+ runCount := 1
+ if err := db.Create(&entity.PipelineOperationLog{
+ ID: "remove-open-log",
+ DocumentID: "doc-1",
+ TenantID: "tenant-1",
+ KbID: "kb-1",
+ ParserID: "naive",
+ TaskType: string(entity.PipelineTaskTypeParse),
+ OperationStatus: string(entity.TaskStatusUnstart),
+ RunCount: &runCount,
+ }).Error; err != nil {
+ t.Fatalf("seed numbered pipeline log: %v", err)
+ }
+ if err := db.Model(&entity.IngestionTask{}).Where("id = ?", "task-1").
+ Update("pipeline_log_id", "remove-open-log").Error; err != nil {
+ t.Fatalf("bind numbered pipeline log: %v", err)
+ }
+
+ settlementErr := errors.New("pipeline ledger unavailable")
+ const callbackName = "test:fail-remove-ledger-settlement"
+ if err := db.Callback().Update().Before("gorm:update").Register(callbackName, func(tx *gorm.DB) {
+ if tx.Statement.Table == (&entity.PipelineOperationLog{}).TableName() {
+ _ = tx.AddError(settlementErr)
+ }
+ }); err != nil {
+ t.Fatalf("register update failure: %v", err)
+ }
+
+ svc := NewIngestionTaskService()
+ if _, err := svc.Remove(t.Context(), "task-1", sptr("user-1")); !errors.Is(err, settlementErr) {
+ t.Fatalf("first Remove error = %v, want settlement failure", err)
+ }
+ task, err := svc.ingestionTaskDAO.GetByID(t.Context(), db, "task-1")
+ if err != nil {
+ t.Fatalf("reload task after failed settlement: %v", err)
+ }
+ if task.Status != common.STOPPED {
+ t.Fatalf("task status = %q, want %q", task.Status, common.STOPPED)
+ }
+ var run entity.PipelineOperationLog
+ if err := db.First(&run, "id = ?", "remove-open-log").Error; err != nil {
+ t.Fatalf("load pipeline log after failed settlement: %v", err)
+ }
+ if run.OperationStatus != string(entity.TaskStatusUnstart) {
+ t.Fatalf("pipeline run status = %q, want %q", run.OperationStatus, entity.TaskStatusUnstart)
+ }
+
+ if err := db.Callback().Update().Remove(callbackName); err != nil {
+ t.Fatalf("remove update failure: %v", err)
+ }
+ if _, err := svc.Remove(t.Context(), "task-1", sptr("user-1")); err != nil {
+ t.Fatalf("retry Remove: %v", err)
+ }
+ if _, err := svc.ingestionTaskDAO.GetByID(t.Context(), db, "task-1"); !errors.Is(err, gorm.ErrRecordNotFound) {
+ t.Fatalf("deleted task reload error = %v, want record not found", err)
+ }
+ if err := db.First(&run, "id = ?", "remove-open-log").Error; err != nil {
+ t.Fatalf("load pipeline log after retry: %v", err)
+ }
+ if run.OperationStatus != string(entity.TaskStatusCancel) {
+ t.Fatalf("pipeline run status after retry = %q, want %q", run.OperationStatus, entity.TaskStatusCancel)
+ }
+ var terminal entity.IngestionTaskLog
+ if err := db.Where("pipeline_log_id = ? AND event_type = ?", "remove-open-log", dao.EventTypeTerminal).
+ First(&terminal).Error; err != nil {
+ t.Fatalf("load terminal event after retry: %v", err)
+ }
+ if terminal.Message != "Task superseded by a new parse request." {
+ t.Fatalf("terminal message after retry = %q, want supersede reason", terminal.Message)
+ }
+}
+
+func TestIngestionTaskServicePrepareDocumentTaskForCleanupRecordsCleanupReason(t *testing.T) {
+ db := setupServiceTestDB(t)
+ pushServiceDB(t, db)
+ insertTestKB(t, "kb-1", "tenant-1", 1, 0, 0)
+ insertTestDoc(t, "doc-1", "kb-1", 0, 0)
+ insertTestIngestionTaskWithStatus(t, "task-1", "user-1", "doc-1", "kb-1", common.CREATED)
+
+ runCount := 1
+ if err := db.Create(&entity.PipelineOperationLog{
+ ID: "cleanup-open-log",
+ DocumentID: "doc-1",
+ TenantID: "tenant-1",
+ KbID: "kb-1",
+ ParserID: "naive",
+ TaskType: string(entity.PipelineTaskTypeParse),
+ OperationStatus: string(entity.TaskStatusUnstart),
+ RunCount: &runCount,
+ }).Error; err != nil {
+ t.Fatalf("seed numbered pipeline log: %v", err)
+ }
+ if err := db.Model(&entity.IngestionTask{}).Where("id = ?", "task-1").
+ Update("pipeline_log_id", "cleanup-open-log").Error; err != nil {
+ t.Fatalf("bind numbered pipeline log: %v", err)
+ }
+
+ task, err := NewIngestionTaskService().PrepareDocumentTaskForCleanup(
+ t.Context(),
+ "doc-1",
+ "Task cancelled by document cleanup.",
+ )
+ if err != nil {
+ t.Fatalf("PrepareDocumentTaskForCleanup failed: %v", err)
+ }
+ if task == nil || task.Status != common.STOPPED {
+ t.Fatalf("cleanup task = %+v, want one STOPPED task", task)
+ }
+
+ var run entity.PipelineOperationLog
+ if err := db.First(&run, "id = ?", "cleanup-open-log").Error; err != nil {
+ t.Fatalf("load pipeline log: %v", err)
+ }
+ if run.OperationStatus != string(entity.TaskStatusCancel) {
+ t.Fatalf("pipeline run status = %q, want %q", run.OperationStatus, entity.TaskStatusCancel)
+ }
+
+ var terminal entity.IngestionTaskLog
+ if err := db.Where("pipeline_log_id = ? AND event_type = ?", "cleanup-open-log", dao.EventTypeTerminal).
+ First(&terminal).Error; err != nil {
+ t.Fatalf("load terminal event: %v", err)
+ }
+ if terminal.Message != "Task cancelled by document cleanup." {
+ t.Fatalf("terminal message = %q, want cleanup reason", terminal.Message)
+ }
+}
+
+func TestIngestionTaskServicePrepareDocumentTaskForCleanupRetriesOpenLedgerSettlement(t *testing.T) {
+ db := setupServiceTestDB(t)
+ pushServiceDB(t, db)
+ insertTestKB(t, "kb-1", "tenant-1", 1, 0, 0)
+ insertTestDoc(t, "doc-1", "kb-1", 0, 0)
+ insertTestIngestionTaskWithStatus(t, "task-1", "user-1", "doc-1", "kb-1", common.CREATED)
+
+ runCount := 1
+ if err := db.Create(&entity.PipelineOperationLog{
+ ID: "cleanup-open-log",
+ DocumentID: "doc-1",
+ TenantID: "tenant-1",
+ KbID: "kb-1",
+ ParserID: "naive",
+ TaskType: string(entity.PipelineTaskTypeParse),
+ OperationStatus: string(entity.TaskStatusUnstart),
+ RunCount: &runCount,
+ }).Error; err != nil {
+ t.Fatalf("seed numbered pipeline log: %v", err)
+ }
+ if err := db.Model(&entity.IngestionTask{}).Where("id = ?", "task-1").
+ Update("pipeline_log_id", "cleanup-open-log").Error; err != nil {
+ t.Fatalf("bind numbered pipeline log: %v", err)
+ }
+
+ settlementErr := errors.New("pipeline ledger unavailable")
+ const callbackName = "test:fail-cleanup-ledger-settlement"
+ if err := db.Callback().Update().Before("gorm:update").Register(callbackName, func(tx *gorm.DB) {
+ if tx.Statement.Table == (&entity.PipelineOperationLog{}).TableName() {
+ _ = tx.AddError(settlementErr)
+ }
+ }); err != nil {
+ t.Fatalf("register update failure: %v", err)
+ }
+
+ svc := NewIngestionTaskService()
+ if _, err := svc.PrepareDocumentTaskForCleanup(
+ t.Context(),
+ "doc-1",
+ "Task cancelled by document cleanup.",
+ ); !errors.Is(err, settlementErr) {
+ t.Fatalf("first cleanup error = %v, want settlement failure", err)
+ }
+
+ task, err := svc.ingestionTaskDAO.GetByID(t.Context(), db, "task-1")
+ if err != nil {
+ t.Fatalf("reload task after failed settlement: %v", err)
+ }
+ if task.Status != common.STOPPED {
+ t.Fatalf("task status = %q, want %q", task.Status, common.STOPPED)
+ }
+ var run entity.PipelineOperationLog
+ if err := db.First(&run, "id = ?", "cleanup-open-log").Error; err != nil {
+ t.Fatalf("load pipeline log after failed settlement: %v", err)
+ }
+ if run.OperationStatus != string(entity.TaskStatusUnstart) {
+ t.Fatalf("pipeline run status = %q, want %q", run.OperationStatus, entity.TaskStatusUnstart)
+ }
+ var terminalCount int64
+ if err := db.Model(&entity.IngestionTaskLog{}).
+ Where("pipeline_log_id = ? AND event_type = ?", "cleanup-open-log", dao.EventTypeTerminal).
+ Count(&terminalCount).Error; err != nil {
+ t.Fatalf("count terminal events after failed settlement: %v", err)
+ }
+ if terminalCount != 0 {
+ t.Fatalf("terminal events after failed settlement = %d, want 0", terminalCount)
+ }
+
+ if err := db.Callback().Update().Remove(callbackName); err != nil {
+ t.Fatalf("remove update failure: %v", err)
+ }
+ task, err = svc.PrepareDocumentTaskForCleanup(
+ t.Context(),
+ "doc-1",
+ "Task cancelled by document cleanup.",
+ )
+ if err != nil {
+ t.Fatalf("retry PrepareDocumentTaskForCleanup: %v", err)
+ }
+ if task == nil || task.Status != common.STOPPED {
+ t.Fatalf("retried cleanup task = %+v, want one STOPPED task", task)
+ }
+ if err := db.First(&run, "id = ?", "cleanup-open-log").Error; err != nil {
+ t.Fatalf("load pipeline log after retry: %v", err)
+ }
+ if run.OperationStatus != string(entity.TaskStatusCancel) {
+ t.Fatalf("pipeline run status after retry = %q, want %q", run.OperationStatus, entity.TaskStatusCancel)
+ }
+ var terminal entity.IngestionTaskLog
+ if err := db.Where("pipeline_log_id = ? AND event_type = ?", "cleanup-open-log", dao.EventTypeTerminal).
+ First(&terminal).Error; err != nil {
+ t.Fatalf("load terminal event after retry: %v", err)
+ }
+ if terminal.Message != "Task cancelled by document cleanup." {
+ t.Fatalf("terminal message after retry = %q, want cleanup reason", terminal.Message)
+ }
+ if err := db.Model(&entity.IngestionTaskLog{}).
+ Where("pipeline_log_id = ? AND event_type = ?", "cleanup-open-log", dao.EventTypeTerminal).
+ Count(&terminalCount).Error; err != nil {
+ t.Fatalf("count terminal events after retry: %v", err)
+ }
+ if terminalCount != 1 {
+ t.Fatalf("terminal events after retry = %d, want 1", terminalCount)
+ }
+}
+
+func TestIngestionTaskServicePrepareDocumentTaskForCleanupSettlesTerminalTaskLedger(t *testing.T) {
+ tests := []struct {
+ name string
+ taskStatus string
+ operationStatus entity.TaskStatus
+ wantMessage string
+ }{
+ {
+ name: "completed",
+ taskStatus: common.COMPLETED,
+ operationStatus: entity.TaskStatusDone,
+ wantMessage: "Task completed.",
+ },
+ {
+ name: "failed",
+ taskStatus: common.FAILED,
+ operationStatus: entity.TaskStatusFail,
+ wantMessage: "Task failed.",
+ },
+ }
+ for _, tc := range tests {
+ t.Run(tc.name, func(t *testing.T) {
+ db := setupServiceTestDB(t)
+ pushServiceDB(t, db)
+ insertTestKB(t, "kb-1", "tenant-1", 1, 0, 0)
+ insertTestDoc(t, "doc-1", "kb-1", 0, 0)
+ insertTestIngestionTaskWithStatus(t, "task-1", "user-1", "doc-1", "kb-1", tc.taskStatus)
+
+ runCount := 1
+ run := &entity.PipelineOperationLog{
+ ID: "cleanup-terminal-log",
+ DocumentID: "doc-1",
+ TenantID: "tenant-1",
+ KbID: "kb-1",
+ ParserID: "naive",
+ TaskType: string(entity.PipelineTaskTypeParse),
+ OperationStatus: string(entity.TaskStatusRunning),
+ RunCount: &runCount,
+ }
+ if err := db.Create(run).Error; err != nil {
+ t.Fatalf("seed numbered pipeline log: %v", err)
+ }
+ if err := db.Model(&entity.IngestionTask{}).Where("id = ?", "task-1").
+ Update("pipeline_log_id", run.ID).Error; err != nil {
+ t.Fatalf("bind numbered pipeline log: %v", err)
+ }
+
+ task, err := NewIngestionTaskService().PrepareDocumentTaskForCleanup(
+ t.Context(),
+ "doc-1",
+ "Task cancelled by document cleanup.",
+ )
+ if err != nil {
+ t.Fatalf("PrepareDocumentTaskForCleanup failed: %v", err)
+ }
+ if task == nil || task.Status != tc.taskStatus {
+ t.Fatalf("cleanup task = %+v, want status %q", task, tc.taskStatus)
+ }
+ if err := db.First(run, "id = ?", run.ID).Error; err != nil {
+ t.Fatalf("load pipeline log: %v", err)
+ }
+ if run.OperationStatus != string(tc.operationStatus) {
+ t.Fatalf("pipeline run status = %q, want %q", run.OperationStatus, tc.operationStatus)
+ }
+ var terminal entity.IngestionTaskLog
+ if err := db.Where("pipeline_log_id = ? AND event_type = ?", run.ID, dao.EventTypeTerminal).
+ First(&terminal).Error; err != nil {
+ t.Fatalf("load terminal event: %v", err)
+ }
+ if terminal.Message != tc.wantMessage {
+ t.Fatalf("terminal message = %q, want %q", terminal.Message, tc.wantMessage)
+ }
+ })
+ }
+}
+
+func TestIngestionTaskServicePrepareDocumentTaskForCleanupRetriesMissingTerminalEvent(t *testing.T) {
+ db := setupServiceTestDB(t)
+ pushServiceDB(t, db)
+ insertTestKB(t, "kb-1", "tenant-1", 1, 0, 0)
+ insertTestDoc(t, "doc-1", "kb-1", 0, 0)
+ insertTestIngestionTaskWithStatus(t, "task-1", "user-1", "doc-1", "kb-1", common.COMPLETED)
+
+ runCount := 1
+ run := &entity.PipelineOperationLog{
+ ID: "cleanup-event-log",
+ DocumentID: "doc-1",
+ TenantID: "tenant-1",
+ KbID: "kb-1",
+ ParserID: "naive",
+ TaskType: string(entity.PipelineTaskTypeParse),
+ OperationStatus: string(entity.TaskStatusDone),
+ RunCount: &runCount,
+ }
+ if err := db.Create(run).Error; err != nil {
+ t.Fatalf("seed numbered pipeline log: %v", err)
+ }
+ if err := db.Model(&entity.IngestionTask{}).Where("id = ?", "task-1").
+ Update("pipeline_log_id", run.ID).Error; err != nil {
+ t.Fatalf("bind numbered pipeline log: %v", err)
+ }
+
+ insertErr := errors.New("terminal event unavailable")
+ failInsert := true
+ const callbackName = "test:fail-cleanup-terminal-event"
+ if err := db.Callback().Create().Before("gorm:create").Register(callbackName, func(tx *gorm.DB) {
+ if !failInsert || tx.Statement.Table != (&entity.IngestionTaskLog{}).TableName() {
+ return
+ }
+ eventType, ok := tx.Statement.Dest.(map[string]interface{})["event_type"].(int)
+ if ok && eventType == dao.EventTypeTerminal {
+ _ = tx.AddError(insertErr)
+ }
+ }); err != nil {
+ t.Fatalf("register terminal insert failure: %v", err)
+ }
+
+ svc := NewIngestionTaskService()
+ if _, err := svc.PrepareDocumentTaskForCleanup(
+ t.Context(),
+ "doc-1",
+ "Task cancelled by document cleanup.",
+ ); !errors.Is(err, insertErr) {
+ t.Fatalf("first cleanup error = %v, want terminal insert failure", err)
+ }
+ var terminalCount int64
+ if err := db.Model(&entity.IngestionTaskLog{}).
+ Where("pipeline_log_id = ? AND event_type = ?", run.ID, dao.EventTypeTerminal).
+ Count(&terminalCount).Error; err != nil {
+ t.Fatalf("count terminal events after failed insert: %v", err)
+ }
+ if terminalCount != 0 {
+ t.Fatalf("terminal events after failed insert = %d, want 0", terminalCount)
+ }
+
+ failInsert = false
+ if _, err := svc.PrepareDocumentTaskForCleanup(
+ t.Context(),
+ "doc-1",
+ "Task cancelled by document cleanup.",
+ ); err != nil {
+ t.Fatalf("retry PrepareDocumentTaskForCleanup: %v", err)
+ }
+ if err := db.Model(&entity.IngestionTaskLog{}).
+ Where("pipeline_log_id = ? AND event_type = ?", run.ID, dao.EventTypeTerminal).
+ Count(&terminalCount).Error; err != nil {
+ t.Fatalf("count terminal events after retry: %v", err)
+ }
+ if terminalCount != 1 {
+ t.Fatalf("terminal events after retry = %d, want 1", terminalCount)
+ }
+}
+
+func TestIngestionTaskServiceRemovePurgesTaskState(t *testing.T) {
+ db := setupServiceTestDB(t)
+ pushServiceDB(t, db)
+ insertTestKB(t, "kb-1", "tenant-1", 1, 0, 0)
+ insertTestDoc(t, "doc-1", "kb-1", 0, 0)
+ insertTestIngestionTaskWithStatus(t, "task-1", "user-1", "doc-1", "kb-1", common.COMPLETED)
+
+ purged := make(chan string, 1)
+ svc := NewIngestionTaskService()
+ svc.purgeTaskState = func(_ context.Context, taskID string) error {
+ purged <- taskID
+ return nil
+ }
+ if _, err := svc.Remove(t.Context(), "task-1", sptr("user-1")); err != nil {
+ t.Fatalf("Remove failed: %v", err)
+ }
+ select {
+ case taskID := <-purged:
+ if taskID != "task-1" {
+ t.Fatalf("purged task = %q, want task-1", taskID)
+ }
+ case <-time.After(2 * time.Second):
+ t.Fatal("Remove did not schedule task-state purge")
+ }
+}
+
+func TestIngestionTaskServiceRemoveDoesNotWaitForTaskStatePurge(t *testing.T) {
+ db := setupServiceTestDB(t)
+ pushServiceDB(t, db)
+ insertTestKB(t, "kb-1", "tenant-1", 1, 0, 0)
+ insertTestDoc(t, "doc-1", "kb-1", 0, 0)
+ insertTestIngestionTaskWithStatus(t, "task-remove-async", "user-1", "doc-1", "kb-1", common.COMPLETED)
+
+ started := make(chan struct{})
+ release := make(chan struct{})
+ svc := NewIngestionTaskService()
+ svc.purgeTaskState = func(ctx context.Context, _ string) error {
+ close(started)
+ select {
+ case <-release:
+ return nil
+ case <-ctx.Done():
+ return ctx.Err()
+ }
+ }
+
+ done := make(chan error, 1)
+ go func() {
+ _, err := svc.Remove(t.Context(), "task-remove-async", sptr("user-1"))
+ done <- err
+ }()
+ select {
+ case err := <-done:
+ if err != nil {
+ t.Fatalf("Remove failed: %v", err)
+ }
+ case <-time.After(2 * time.Second):
+ t.Fatal("Remove blocked on task-state purge")
+ }
+ select {
+ case <-started:
+ case <-time.After(2 * time.Second):
+ t.Fatal("background task-state purge did not start")
+ }
+ close(release)
+}
+
+func TestIngestionTaskServiceRemoveKeepsDeleteWhenPurgeFails(t *testing.T) {
+ db := setupServiceTestDB(t)
+ pushServiceDB(t, db)
+ insertTestKB(t, "kb-1", "tenant-1", 1, 0, 0)
+ insertTestDoc(t, "doc-1", "kb-1", 0, 0)
+ insertTestIngestionTaskWithStatus(t, "task-1", "user-1", "doc-1", "kb-1", common.COMPLETED)
+
+ svc := NewIngestionTaskService()
+ svc.purgeTaskState = func(_ context.Context, _ string) error {
+ return errors.New("kvrocks unavailable")
+ }
+ if _, err := svc.Remove(t.Context(), "task-1", sptr("user-1")); err != nil {
+ t.Fatalf("Remove failed after a purge error: %v", err)
+ }
+ if _, err := dao.NewIngestionTaskDAO().GetByID(t.Context(), db, "task-1"); !errors.Is(err, gorm.ErrRecordNotFound) {
+ t.Fatalf("deleted task reload error = %v, want record not found", err)
+ }
+}
+
func TestIngestionTaskServiceUpdateComponentTotalPersistsDenominator(t *testing.T) {
db := setupServiceTestDB(t)
pushServiceDB(t, db)