RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
feat(parser): hand off Office images as media refs (#21700)
jay77721aaa8e05354f1
diff --git a/internal/cli/user_command.go b/internal/cli/user_command.goindex 4d80e1041f..80594d0c9a 100644--- a/internal/cli/user_command.go+++ b/internal/cli/user_command.go@@ -3023,16 +3023,10 @@ func (c *CLI) APIParseLocalFileCommand(commandCount int, cmd *Command) (Response}fileType := utility.GetFileType(filename)- fileParser, err := parser.GetParser(fileType)+ fileParser, err := parser.GetTextOnlyParser(fileType)if err != nil {return nil, err}- if fileType == utility.FileTypeHTML {- fileParser = parser.NewHTMLTextParser()- }- if fileType == utility.FileTypeEPUB {- fileParser = parser.NewEPUBTextParser()- }fileContent, err := os.ReadFile(filename)if err != nil {diff --git a/internal/development.md b/internal/development.mdindex 8f857b44a8..3174f120ff 100644--- a/internal/development.md+++ b/internal/development.md@@ -300,26 +300,45 @@ docker compose -f docker/docker-compose-base.yml --profile infinity up -ddeployment disables the Valkey/Redis service so there is no clash). Export thesebefore running `./bin/ragflow_server ...` on the host, or load them from`docker/.env-go`:-```bash-export KVROCKS_HOST=127.0.0.1-export KVROCKS_PORT=6379-```+ ```bash+ export KVROCKS_HOST=127.0.0.1+ export KVROCKS_PORT=6379+ ```In docker the Go services override these to `kvrocks:6379` automatically(see `docker-compose-go.yml`). If Kvrocks is unreachable at startup theprocess fails fast with `failed to connect to Kvrocks`.- HTML and EPUB ingestion use Kvrocks to hand inline images from the parser to- the chunker; the ingestor also validates the injected Kvrocks client at startup.- Inline image payloads default to a 32 MiB per-image limit; set- `RAGFLOW_MEDIA_IMAGE_MAX_BYTES` to a positive byte count to change it.+ HTML, EPUB, DOCX, PPTX and XLSX ingestion use Kvrocks to hand embedded images+ from the parser to the chunker; the ingestor also validates the injected+ Kvrocks client at startup.+ Image payloads share a default 32 MiB per-image limit across data URIs,+ ZIP-entry extraction, picture parsing, vision encoding and media-cache writes.+ Set `RAGFLOW_MEDIA_IMAGE_MAX_BYTES` to a positive byte count to change it.+ Native PDF region cropping retains its separate page/raster limits.The parser budget and media cache apply the same numeric run caps: 1024 itemsand 1 GiB of decoded image bytes per run. The parser counts distinct sourcereferences; the cache counts distinct SHA-256 payloads. Set`RAGFLOW_MEDIA_MAX_IMAGES_PER_RUN` and `RAGFLOW_MEDIA_MAX_BYTES_PER_RUN` topositive values to change them. Quota state is keyed by ingestion task andpersists across retries of that task until cleanup or the 24-hour media TTL.+ Office parsers deduplicate sources by payload digest. DOC/PPT share their+ Office family path; BIFF8 XLS does not extract embedded pictures. Office IR+ and Excelize expose already materialized image bytes, so these budgets bound+ emitted/cached payloads, not native extraction or ZIP decoding peak memory.+ DOC/DOCX and PPT/PPTX identify image MIME types from payload signatures;+ XLSX-content spreadsheets use an extension allowlist, including EMZ/WMZ+ which the byte-only Office detector does not recognize. Unsupported images+ are skipped with warnings.+ File text extraction and email attachment re-chunking use text-only parsers;+ Office images skipped by limits produce warnings without placeholder items.Debug previews use a temporary `pipeline_<document-id>` run scope and purgetheir cache state when the preview finishes.+ Validate Office media handoff against the services configured in+ `conf/service_conf.yaml` (real MySQL, Kvrocks and MinIO):++ ```bash+ RAGFLOW_REAL_OFFICE_MEDIA_TEST=1 bash build.sh --test-integration-go ./internal/ingestion/pipeline -run '^TestOfficeMediaRealServices$' -v+ ```- Start RAGFlowNote: Database migrations must complete before starting any server mode.diff --git a/internal/ingestion/pipeline/html_media_real_integration_test.go b/internal/ingestion/pipeline/html_media_real_integration_test.goindex cfab629bdf..4f64858767 100644--- a/internal/ingestion/pipeline/html_media_real_integration_test.go+++ b/internal/ingestion/pipeline/html_media_real_integration_test.go@@ -3,264 +3,20 @@package pipelineimport (- "bytes"- "context""encoding/base64"- "errors"- "fmt""os"- "path/filepath""testing"- "time"-- "github.com/cespare/xxhash/v2"- "gopkg.in/yaml.v3"- "ragflow/internal/common"- "ragflow/internal/dao"- "ragflow/internal/engine/kvrocks"- "ragflow/internal/entity"- component "ragflow/internal/ingestion/component"- "ragflow/internal/ingestion/component/schema"- "ragflow/internal/ingestion/mediacache"- "ragflow/internal/parser/media"- "ragflow/internal/parser/parser"- "ragflow/internal/server"- "ragflow/internal/server/config"- "ragflow/internal/storage"- "ragflow/internal/tokenizer")-// TestHTMLMediaRealServices exercises the normal ingestion canvas using real-// MySQL, Kvrocks and MinIO. The source HTML contains a one-pixel PNG.func TestHTMLMediaRealServices(t *testing.T) {if os.Getenv("RAGFLOW_REAL_HTML_MEDIA_TEST") != "1" {t.Skip("set RAGFLOW_REAL_HTML_MEDIA_TEST=1 with configured MySQL, Kvrocks and MinIO")}- if common.GetEnv(common.EnvRAGFlowDictPath) == "" {- const dictPath = "/usr/share/infinity/resource"- if _, err := os.Stat(filepath.Join(dictPath, "rag", "huqie.txt")); err != nil {- t.Fatalf("tokenizer resource: %v", err)- }- t.Setenv(common.EnvRAGFlowDictPath, dictPath)- }- if err := tokenizer.Init(&tokenizer.PoolConfig{DictPath: "/usr/share/infinity/resource", MinSize: 1, MaxSize: 2, IdleTimeout: 30 * time.Second, AcquireTimeout: 5 * time.Second}); err != nil {- t.Fatalf("tokenizer initialization: %v", err)- }- cfg := loadHTMLMediaIntegrationConfig(t)- db := mustOpenRealMySQL(t, cfg)- sqlDB, err := db.DB()- if err != nil {- t.Fatal(err)- }- sqlDB.SetMaxOpenConns(1)- if err := db.Exec("SET SESSION sql_mode = ''").Error; err != nil {- t.Fatal(err)- }- t.Cleanup(func() { _ = sqlDB.Close() })- if err := db.AutoMigrate(&entity.Tenant{}, &entity.UserTenant{}, &entity.Knowledgebase{}, &entity.Document{}, &entity.File{}, &entity.File2Document{}); err != nil {- t.Fatal(err)- }- if err := kvrocks.Init(t.Context()); err != nil {- t.Fatal(err)- }- kv := kvrocks.Get()- stg, err := storage.NewMinioStorage(cfg.GetMinioConfig())- if err != nil {- t.Fatal(err)- }- readyCtx, cancel := context.WithTimeout(t.Context(), 30*time.Second)- defer cancel()- for {- _, err := stg.BucketExistsWithError(readyCtx, "ragflow-html-media-probe")- if err == nil {- break- }- select {- case <-readyCtx.Done():- t.Fatalf("MinIO S3 not ready: %v", err)- case <-time.After(250 * time.Millisecond):- }- }- originalDB := dao.DB- originalStorage := storage.GetStorageFactory().GetStorage()- originalResolver := component.ResolveDocumentStorageOverride- dao.DB = db- storage.GetStorageFactory().SetStorage(stg)- component.ResolveDocumentStorageOverride = nil- t.Cleanup(func() {- dao.DB = originalDB- storage.GetStorageFactory().SetStorage(originalStorage)- component.ResolveDocumentStorageOverride = originalResolver- })- template, err := os.ReadFile(filepath.Join(repoRootFromPipelineTest(t), "internal/ingestion/pipeline/template/ingestion_pipeline_general.json"))- if err != nil {- t.Fatal(err)- }- template = disableTokenizerEmbeddingForTemplate(t, template)- suffix := fmt.Sprintf("%d", time.Now().UnixNano())- tenantID, kbID := limit32("it_tenant_"+suffix), s3SafeBucketName("it-kb-"+suffix)- docID, fileID := limit32("it_doc_"+suffix), limit32("it_file_"+suffix)- taskID := "it-html-" + suffix- bucket := s3SafeBucketName(kbID)- objectPath := "integration/" + taskID + "/image.html"const png64 = "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mP8/x8AAwMCAO+jRZkAAAAASUVORK5CYII="png, err := base64.StdEncoding.DecodeString(png64)if err != nil {t.Fatal(err)}- html := `<p>HTML image fixture</p><img alt="sample figure" src="data:image/png;base64,` + png64 + `">`- mustSeedRealPipelineDocument(t, db, stg, tenantID, kbID, docID, fileID, bucket, objectPath, "image.html", html)- t.Cleanup(func() {- _ = mediacache.PurgeTask(context.Background(), kv, taskID)- cleanupRealPipelineDocument(db, stg, tenantID, kbID, docID, fileID, bucket, objectPath)- })- if err := db.Model(&entity.Document{}).Where("id = ?", docID).Updates(map[string]any{"type": "html", "suffix": ".html"}).Error; err != nil {- t.Fatal(err)- }- if err := db.Model(&entity.File{}).Where("id = ?", fileID).Update("type", "html").Error; err != nil {- t.Fatal(err)- }- pipe, err := NewPipelineFromDSL(template, taskID, WithDocumentID(docID), WithKV(kv))- if err != nil {- t.Fatal(err)- }- out, err := pipe.Run(t.Context(), map[string]any{"doc_id": docID, "kb_id": kbID, "tenant_id": tenantID}, nil)- if err != nil {- t.Fatal(err)- }- state := stateFromRunOutput(t, out)- docs, ok := state["Parser:HipSignsRhyme"]["json"].([]parser.DocItem)- if !ok {- t.Fatalf("parser output = %T", state["Parser:HipSignsRhyme"]["json"])- }- var ref *entity.MediaRef- for _, item := range docs {- m := item.ToMap()- if m["media_ref"] != nil {- ref = entity.MediaRefFromAny(m["media_ref"])- if m["image"] != nil {- t.Fatal("parser retained inline image")- }- }- }- if ref == nil {- t.Fatal("parser did not emit media_ref")- }- cached, err := mediacache.New(kv, taskID).Get(t.Context(), *ref)- if err != nil || !bytes.Equal(cached, png) {- t.Fatalf("cached bytes differ: %v", err)- }- chunks, ok := schema.ChunkCarrierToMaps(state["GeneralChunker:SixApplesFall"]["chunks"])- if !ok {- t.Fatal("missing chunker output")- }- foundImage := false- for _, ck := range chunks {- id, _ := ck["id"].(string)- imgID, _ := ck["img_id"].(string)- if imgID == "" {- continue- }- if imgID != kbID+"-"+id || ck["media_ref"] != nil {- t.Fatalf("invalid image chunk: %v", ck)- }- got, err := stg.Get(t.Context(), kbID, id)- if err != nil {- t.Fatal(err)- }- t.Cleanup(func() { _ = stg.Remove(context.Background(), kbID, id) })- if !bytes.Equal(got, png) {- t.Fatal("stored image does not match source")- }- foundImage = true- }- if !foundImage {- t.Fatalf("chunker produced no uploaded image: chunks=%#v state=%#v", chunks, state["GeneralChunker:SixApplesFall"])- }- key := fmt.Sprintf("mc:media:%x:%s", xxhash.Sum64String(taskID), ref.SHA256)- ttl, err := kv.GetClient().TTL(t.Context(), key).Result()- if err != nil || ttl <= 23*time.Hour || ttl > mediacache.TTL {- t.Fatalf("media TTL = %v, %v", ttl, err)- }- if err := kv.GetClient().Expire(t.Context(), key, time.Millisecond).Err(); err != nil {- t.Fatal(err)- }- deadline := time.Now().Add(2 * time.Second)- for {- _, err = mediacache.New(kv, taskID).Get(t.Context(), *ref)- if errors.Is(err, media.ErrMediaNotFound) {- break- }- if err != nil || time.Now().After(deadline) {- t.Fatalf("media did not expire: %v", err)- }- time.Sleep(10 * time.Millisecond)- }- retry := mediacache.New(kv, taskID)- retryRef, err := retry.Put(t.Context(), "html", "image/png", png)- if err != nil {- t.Fatal(err)- }- if got, err := retry.Get(t.Context(), retryRef); err != nil || !bytes.Equal(got, png) {- t.Fatalf("retry cache bytes: %v", err)- }- scopeHash := xxhash.Sum64String(taskID)- digestKey := fmt.Sprintf("mc:quota:%x:digests", scopeHash)- stateKey := fmt.Sprintf("mc:quota:%x:state", scopeHash)- manifestKey := fmt.Sprintf("mc:manifest:%x", scopeHash)- if member, err := kv.GetClient().SIsMember(t.Context(), digestKey, retryRef.SHA256).Result(); err != nil || !member {- t.Fatalf("quota digest membership = %v, %v; want true", member, err)- }- images, err := kv.GetClient().SCard(t.Context(), digestKey).Result()- if err != nil || images != 1 {- t.Fatalf("quota images = %d, %v; want 1", images, err)- }- bytesUsed, err := kv.GetClient().HGet(t.Context(), stateKey, "bytes").Result()- if err != nil || bytesUsed != fmt.Sprint(len(png)) {- t.Fatalf("quota bytes = %q, %v; want %d", bytesUsed, err, len(png))- }- if member, err := kv.GetClient().SIsMember(t.Context(), manifestKey, key).Result(); err != nil || !member {- t.Fatalf("manifest membership = %v, %v; want true", member, err)- }- if err := mediacache.PurgeTask(t.Context(), kv, taskID); err != nil {- t.Fatal(err)- }- if _, err := retry.Get(t.Context(), retryRef); !errors.Is(err, media.ErrMediaNotFound) {- t.Fatalf("purged cache read: %v", err)- }- for _, purgedKey := range []string{key, digestKey, stateKey, manifestKey} {- if exists, err := kv.GetClient().Exists(t.Context(), purgedKey).Result(); err != nil || exists != 0 {- t.Fatalf("purged key %s exists=%d, err=%v", purgedKey, exists, err)- }- }-}--// loadHTMLMediaIntegrationConfig uses the CI service addresses when set.-func loadHTMLMediaIntegrationConfig(t *testing.T) *config.Config {- t.Helper()- kvAddr := os.Getenv("RAGFLOW_TEST_KVROCKS_ADDR")- if kvAddr == "" {- return mustLoadRealIntegrationConfig(t)- }- cfg := map[string]any{- "general": map[string]any{"cache_engine": "kvrocks"},- "mysql": map[string]any{"host": os.Getenv("RAGFLOW_MYSQL_HOST"), "port": os.Getenv("RAGFLOW_MYSQL_PORT"), "password": os.Getenv("MYSQL_PASSWORD")},- "minio": map[string]any{"host": os.Getenv("RAGFLOW_MINIO_HOST"), "user": os.Getenv("MINIO_USER"), "password": os.Getenv("MINIO_PASSWORD")},- "kvrocks": map[string]any{"host": kvAddr, "password": os.Getenv("REDIS_PASSWORD"), "db": 1},- }- data, err := yaml.Marshal(cfg)- if err != nil {- t.Fatal(err)- }- path := filepath.Join(t.TempDir(), "service_conf.yaml")- if err := os.WriteFile(path, data, 0600); err != nil {- t.Fatal(err)- }- if err := common.InitLogger("info", common.FileOutput{}, ""); err != nil {- t.Fatal(err)- }- if err := server.Init(path); err != nil {- t.Fatal(err)- }- return server.GetConfig()+ source := []byte("<p>HTML image fixture</p><img alt='sample figure' src='data:image/png;base64," + png64 + "'>")+ runMediaRealServices(t, "html", source, png)}diff --git a/internal/ingestion/pipeline/media_real_integration_test.go b/internal/ingestion/pipeline/media_real_integration_test.gonew file mode 100644index 0000000000..399a1cc4ea--- /dev/null+++ b/internal/ingestion/pipeline/media_real_integration_test.go@@ -0,0 +1,270 @@+//go:build integration++package pipeline++import (+ "bytes"+ "context"+ "errors"+ "fmt"+ "os"+ "path/filepath"+ "testing"+ "time"++ "github.com/cespare/xxhash/v2"+ "gopkg.in/yaml.v3"+ "ragflow/internal/common"+ "ragflow/internal/dao"+ "ragflow/internal/engine/kvrocks"+ "ragflow/internal/entity"+ component "ragflow/internal/ingestion/component"+ "ragflow/internal/ingestion/component/schema"+ "ragflow/internal/ingestion/mediacache"+ "ragflow/internal/parser/media"+ "ragflow/internal/parser/parser"+ "ragflow/internal/server"+ "ragflow/internal/server/config"+ "ragflow/internal/storage"+ "ragflow/internal/tokenizer"+)++// runMediaRealServices exercises the normal canvas with real MySQL, Kvrocks+// and MinIO, checking parser references, uploaded bytes and cache lifecycle.+func runMediaRealServices(t *testing.T, format string, source, png []byte) {+ if common.GetEnv(common.EnvRAGFlowDictPath) == "" {+ const dictPath = "/usr/share/infinity/resource"+ if _, err := os.Stat(filepath.Join(dictPath, "rag", "huqie.txt")); err != nil {+ t.Fatalf("tokenizer resource: %v", err)+ }+ t.Setenv(common.EnvRAGFlowDictPath, dictPath)+ }+ if err := tokenizer.Init(&tokenizer.PoolConfig{DictPath: common.GetEnv(common.EnvRAGFlowDictPath), MinSize: 1, MaxSize: 2, IdleTimeout: 30 * time.Second, AcquireTimeout: 5 * time.Second}); err != nil {+ t.Fatalf("tokenizer initialization: %v", err)+ }+ cfg := loadMediaIntegrationConfig(t)+ db := mustOpenRealMySQL(t, cfg)+ sqlDB, err := db.DB()+ if err != nil {+ t.Fatal(err)+ }+ sqlDB.SetMaxOpenConns(1)+ if err := db.Exec("SET SESSION sql_mode = ''").Error; err != nil {+ t.Fatal(err)+ }+ t.Cleanup(func() { _ = sqlDB.Close() })+ if err := db.AutoMigrate(&entity.Tenant{}, &entity.UserTenant{}, &entity.Knowledgebase{}, &entity.Document{}, &entity.File{}, &entity.File2Document{}); err != nil {+ t.Fatal(err)+ }+ if err := kvrocks.Init(t.Context()); err != nil {+ t.Fatal(err)+ }+ kv := kvrocks.Get()+ stg, err := storage.NewMinioStorage(cfg.GetMinioConfig())+ if err != nil {+ t.Fatal(err)+ }+ readyCtx, cancel := context.WithTimeout(t.Context(), 30*time.Second)+ defer cancel()+ for {+ _, err := stg.BucketExistsWithError(readyCtx, "ragflow-html-media-probe")+ if err == nil {+ break+ }+ select {+ case <-readyCtx.Done():+ t.Fatalf("MinIO S3 not ready: %v", err)+ case <-time.After(250 * time.Millisecond):+ }+ }+ originalDB := dao.DB+ originalStorage := storage.GetStorageFactory().GetStorage()+ originalResolver := component.ResolveDocumentStorageOverride+ dao.DB = db+ storage.GetStorageFactory().SetStorage(stg)+ component.ResolveDocumentStorageOverride = nil+ t.Cleanup(func() {+ dao.DB = originalDB+ storage.GetStorageFactory().SetStorage(originalStorage)+ component.ResolveDocumentStorageOverride = originalResolver+ })+ template, err := os.ReadFile(filepath.Join(repoRootFromPipelineTest(t), "internal/ingestion/pipeline/template/ingestion_pipeline_general.json"))+ if err != nil {+ t.Fatal(err)+ }+ template = disableTokenizerEmbeddingForTemplate(t, template)+ suffix := fmt.Sprintf("%d", time.Now().UnixNano())+ tenantID, kbID := limit32("it_tenant_"+suffix), s3SafeBucketName("it-kb-"+suffix)+ docID, fileID := limit32("it_doc_"+suffix), limit32("it_file_"+suffix)+ taskID := "it-" + format + "-" + suffix+ bucket := s3SafeBucketName(kbID)+ name := "image." + format+ objectPath := "integration/" + taskID + "/" + name+ mustSeedRealPipelineDocument(t, db, stg, tenantID, kbID, docID, fileID, bucket, objectPath, name, string(source))+ t.Cleanup(func() {+ _ = mediacache.PurgeTask(context.Background(), kv, taskID)+ cleanupRealPipelineDocument(db, stg, tenantID, kbID, docID, fileID, bucket, objectPath)+ })+ if err := db.Model(&entity.Document{}).Where("id = ?", docID).Updates(map[string]any{"type": format, "suffix": "." + format}).Error; err != nil {+ t.Fatal(err)+ }+ if err := db.Model(&entity.File{}).Where("id = ?", fileID).Update("type", format).Error; err != nil {+ t.Fatal(err)+ }+ pipe, err := NewPipelineFromDSL(template, taskID, WithDocumentID(docID), WithKV(kv))+ if err != nil {+ t.Fatal(err)+ }+ out, err := pipe.Run(t.Context(), map[string]any{"doc_id": docID, "kb_id": kbID, "tenant_id": tenantID}, nil)+ if err != nil {+ t.Fatal(err)+ }+ state := stateFromRunOutput(t, out)+ docs, ok := state["Parser:HipSignsRhyme"]["json"].([]parser.DocItem)+ if !ok {+ t.Fatalf("parser output = %T", state["Parser:HipSignsRhyme"]["json"])+ }+ var ref *entity.MediaRef+ for _, item := range docs {+ m := item.ToMap()+ if m["media_ref"] != nil {+ ref = entity.MediaRefFromAny(m["media_ref"])+ if m["image"] != nil {+ t.Fatal("parser retained inline image")+ }+ }+ }+ if ref == nil || ref.Format != format {+ t.Fatal("parser did not emit media_ref")+ }+ cached, err := mediacache.New(kv, taskID).Get(t.Context(), *ref)+ if err != nil || !bytes.Equal(cached, png) {+ t.Fatalf("cached bytes differ: %v", err)+ }+ chunks, ok := schema.ChunkCarrierToMaps(state["GeneralChunker:SixApplesFall"]["chunks"])+ if !ok {+ t.Fatal("missing chunker output")+ }+ foundImage := false+ for _, ck := range chunks {+ id, _ := ck["id"].(string)+ imgID, _ := ck["img_id"].(string)+ if imgID == "" {+ continue+ }+ if imgID != kbID+"-"+id || ck["media_ref"] != nil {+ t.Fatalf("invalid image chunk: %v", ck)+ }+ got, err := stg.Get(t.Context(), kbID, id)+ if err != nil {+ t.Fatal(err)+ }+ t.Cleanup(func() { _ = stg.Remove(context.Background(), kbID, id) })+ if !bytes.Equal(got, png) {+ t.Fatal("stored image does not match source")+ }+ foundImage = true+ }+ if !foundImage {+ t.Fatalf("chunker produced no uploaded image: chunks=%#v state=%#v", chunks, state["GeneralChunker:SixApplesFall"])+ }+ key := fmt.Sprintf("mc:media:%x:%s", xxhash.Sum64String(taskID), ref.SHA256)+ ttl, err := kv.GetClient().TTL(t.Context(), key).Result()+ if err != nil || ttl <= 23*time.Hour || ttl > mediacache.TTL {+ t.Fatalf("media TTL = %v, %v", ttl, err)+ }+ resolveCtx := media.WithCache(t.Context(), mediacache.New(kv, taskID))+ if err := kv.GetClient().Set(t.Context(), key, "corrupt", mediacache.TTL).Err(); err != nil {+ t.Fatal(err)+ }+ if _, _, err := component.ResolveMediaRef(resolveCtx, map[string]any{"media_ref": ref}); err == nil || errors.Is(err, media.ErrMediaNotFound) {+ t.Fatalf("corrupt payload must propagate an integrity error: %v", err)+ }+ if err := kv.GetClient().Set(t.Context(), key, png, mediacache.TTL).Err(); err != nil {+ t.Fatal(err)+ }+ if err := kv.GetClient().Expire(t.Context(), key, time.Second).Err(); err != nil {+ t.Fatal(err)+ }+ deadline := time.Now().Add(2 * time.Second)+ for {+ _, err = mediacache.New(kv, taskID).Get(t.Context(), *ref)+ if errors.Is(err, media.ErrMediaNotFound) {+ break+ }+ if err != nil || time.Now().After(deadline) {+ t.Fatalf("media did not expire: %v", err)+ }+ time.Sleep(10 * time.Millisecond)+ }+ if _, _, err := component.ResolveMediaRef(resolveCtx, map[string]any{"media_ref": ref}); !errors.Is(err, media.ErrMediaNotFound) {+ t.Fatalf("expired reference must propagate a cache miss: %v", err)+ }+ retry := mediacache.New(kv, taskID)+ retryRef, err := retry.Put(t.Context(), format, "image/png", png)+ if err != nil {+ t.Fatal(err)+ }+ if got, err := retry.Get(t.Context(), retryRef); err != nil || !bytes.Equal(got, png) {+ t.Fatalf("retry cache bytes: %v", err)+ }+ scopeHash := xxhash.Sum64String(taskID)+ digestKey := fmt.Sprintf("mc:quota:%x:digests", scopeHash)+ stateKey := fmt.Sprintf("mc:quota:%x:state", scopeHash)+ manifestKey := fmt.Sprintf("mc:manifest:%x", scopeHash)+ if member, err := kv.GetClient().SIsMember(t.Context(), digestKey, retryRef.SHA256).Result(); err != nil || !member {+ t.Fatalf("quota digest membership = %v, %v; want true", member, err)+ }+ images, err := kv.GetClient().SCard(t.Context(), digestKey).Result()+ if err != nil || images != 1 {+ t.Fatalf("quota images = %d, %v; want 1", images, err)+ }+ bytesUsed, err := kv.GetClient().HGet(t.Context(), stateKey, "bytes").Result()+ if err != nil || bytesUsed != fmt.Sprint(len(png)) {+ t.Fatalf("quota bytes = %q, %v; want %d", bytesUsed, err, len(png))+ }+ if member, err := kv.GetClient().SIsMember(t.Context(), manifestKey, key).Result(); err != nil || !member {+ t.Fatalf("manifest membership = %v, %v; want true", member, err)+ }+ if err := mediacache.PurgeTask(t.Context(), kv, taskID); err != nil {+ t.Fatal(err)+ }+ if _, err := retry.Get(t.Context(), retryRef); !errors.Is(err, media.ErrMediaNotFound) {+ t.Fatalf("purged cache read: %v", err)+ }+ for _, purgedKey := range []string{key, digestKey, stateKey, manifestKey} {+ if exists, err := kv.GetClient().Exists(t.Context(), purgedKey).Result(); err != nil || exists != 0 {+ t.Fatalf("purged key %s exists=%d, err=%v", purgedKey, exists, err)+ }+ }+}++// loadMediaIntegrationConfig uses the CI service addresses when set.+func loadMediaIntegrationConfig(t *testing.T) *config.Config {+ t.Helper()+ kvAddr := os.Getenv("RAGFLOW_TEST_KVROCKS_ADDR")+ if kvAddr == "" {+ return mustLoadRealIntegrationConfig(t)+ }+ cfg := map[string]any{+ "general": map[string]any{"cache_engine": "kvrocks"},+ "mysql": map[string]any{"host": os.Getenv("RAGFLOW_MYSQL_HOST"), "port": os.Getenv("RAGFLOW_MYSQL_PORT"), "password": os.Getenv("MYSQL_PASSWORD")},+ "minio": map[string]any{"host": os.Getenv("RAGFLOW_MINIO_HOST"), "user": os.Getenv("MINIO_USER"), "password": os.Getenv("MINIO_PASSWORD")},+ "kvrocks": map[string]any{"host": kvAddr, "password": os.Getenv("REDIS_PASSWORD"), "db": 1},+ }+ data, err := yaml.Marshal(cfg)+ if err != nil {+ t.Fatal(err)+ }+ path := filepath.Join(t.TempDir(), "service_conf.yaml")+ if err := os.WriteFile(path, data, 0600); err != nil {+ t.Fatal(err)+ }+ if err := common.InitLogger("info", common.FileOutput{}, ""); err != nil {+ t.Fatal(err)+ }+ if err := server.Init(path); err != nil {+ t.Fatal(err)+ }+ return server.GetConfig()+}diff --git a/internal/ingestion/pipeline/office_media_real_integration_test.go b/internal/ingestion/pipeline/office_media_real_integration_test.gonew file mode 100644index 0000000000..803a0cf085--- /dev/null+++ b/internal/ingestion/pipeline/office_media_real_integration_test.go@@ -0,0 +1,79 @@+//go:build cgo && integration++package pipeline++import (+ "archive/zip"+ "bytes"+ "io"+ "os"+ "path/filepath"+ "strings"+ "testing"++ "github.com/xuri/excelize/v2"+ officeOxide "github.com/yfedoseev/office_oxide/go"+)++// TestOfficeMediaRealServices uses the production native/Excelize parsers and+// the ingestion canvas, never a parser or storage double.+func TestOfficeMediaRealServices(t *testing.T) {+ if os.Getenv("RAGFLOW_REAL_OFFICE_MEDIA_TEST") != "1" {+ t.Skip("set RAGFLOW_REAL_OFFICE_MEDIA_TEST=1 with configured MySQL, Kvrocks and MinIO")+ }+ docx, err := os.ReadFile(filepath.Join(repoRootFromPipelineTest(t), "internal/parser/parser/testdata/docx_table_image.docx"))+ if err != nil {+ t.Fatal(err)+ }+ archive, err := zip.NewReader(bytes.NewReader(docx), int64(len(docx)))+ if err != nil {+ t.Fatal(err)+ }+ var png []byte+ for _, entry := range archive.File {+ if !strings.HasPrefix(entry.Name, "word/media/") || !strings.HasSuffix(entry.Name, ".png") {+ continue+ }+ f, err := entry.Open()+ if err != nil {+ t.Fatal(err)+ }+ png, err = io.ReadAll(f)+ f.Close()+ if err != nil {+ t.Fatal(err)+ }+ break+ }+ if len(png) == 0 {+ t.Fatal("DOCX fixture has no PNG")+ }+ t.Run("docx_table_image", func(t *testing.T) { runMediaRealServices(t, "docx", docx, png) })+ t.Run("pptx_slide_image", func(t *testing.T) {+ w := officeOxide.NewPptxWriter()+ defer w.Close()+ slide := w.AddSlide()+ w.AddSlideText(slide, "Office media fixture")+ w.AddSlideImage(slide, png, "png", 914400, 914400, 914400, 457200)+ data, err := w.ToBytes()+ if err != nil {+ t.Fatal(err)+ }+ runMediaRealServices(t, "pptx", data, png)+ })+ t.Run("xlsx_cell_image", func(t *testing.T) {+ f := excelize.NewFile()+ defer f.Close()+ if err := f.SetCellValue("Sheet1", "A1", "Office media fixture"); err != nil {+ t.Fatal(err)+ }+ if err := f.AddPictureFromBytes("Sheet1", "C3", &excelize.Picture{Extension: ".png", File: png}); err != nil {+ t.Fatal(err)+ }+ buf, err := f.WriteToBuffer()+ if err != nil {+ t.Fatal(err)+ }+ runMediaRealServices(t, "xlsx", buf.Bytes(), png)+ })+}diff --git a/internal/parser/media/sink.go b/internal/parser/media/sink.gonew file mode 100644index 0000000000..bc014e6fc5--- /dev/null+++ b/internal/parser/media/sink.go@@ -0,0 +1,95 @@+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.++package media++import (+ "context"+ "errors"+ "fmt"++ "ragflow/internal/entity"+)++// Sink admits and caches media sources for one serial parser invocation.+// Source identity and payload extraction belong to the caller; storage scope,+// atomic quotas and lifecycle belong to Cache.+type Sink struct {+ ctx context.Context+ cache Cache+ format string+ budget *Budget+ refs map[string]*entity.MediaRef+ limitSkips int+}++// NewSink binds a parser invocation to its injected cache and shared budget.+// Both ctx and budget must be non-nil.+func NewSink(ctx context.Context, format string, budget *Budget) *Sink {+ return &Sink{ctx: ctx, cache: CacheFromContext(ctx), format: format, budget: budget, refs: make(map[string]*entity.MediaRef)}+}++// Resolve checks the declared size before loading a new source. A false load+// result records a source omission; the caller reports its format-specific+// warning. Limits also return nil without an error. Every cache failure other+// than ErrMediaLimitExceeded is returned to the caller.+// Repeated sources reuse both accepted references and recorded omissions.+func (s *Sink) Resolve(source string, declared Size, load func() (mime string, data []byte, ok bool)) (*entity.MediaRef, error) {+ if err := s.ctx.Err(); err != nil {+ return nil, err+ }+ if ref, seen := s.refs[source]; seen {+ return ref, nil+ }+ if s.cache == nil {+ return nil, fmt.Errorf("%s media cache is not configured", s.format)+ }+ s.refs[source] = nil+ reservation, admission := s.budget.Reserve(declared)+ if admission != Admitted {+ return nil, nil+ }+ mime, data, ok := load()+ if !ok || !reservation.Commit(Size(len(data))) {+ return nil, nil+ }+ ref, err := s.cache.Put(s.ctx, s.format, mime, data)+ if errors.Is(err, ErrMediaLimitExceeded) {+ s.limitSkips+++ return nil, nil+ }+ if err != nil {+ delete(s.refs, source) // A cache failure is not a memoized source omission.+ return nil, fmt.Errorf("%s media cache write: %w", s.format, err)+ }+ s.refs[source] = &ref+ return &ref, nil+}++func (s *Sink) Warnings() []string {+ warnings := s.budget.Warnings()+ if s.limitSkips > 0 {+ warnings = append(warnings, fmt.Sprintf("%s parser skipped %d image(s) after the run media cache limit", s.format, s.limitSkips))+ }+ return warnings+}++// Lookup lets parsers avoid revalidating source metadata on repeated references.+func (s *Sink) Lookup(source string) (*entity.MediaRef, bool) {+ ref, seen := s.refs[source]+ return ref, seen+}++// Skip records an unreadable or invalid source whose warning the parser owns.+func (s *Sink) Skip(source string) { s.refs[source] = nil }diff --git a/internal/parser/media/sink_test.go b/internal/parser/media/sink_test.gonew file mode 100644index 0000000000..a674125736--- /dev/null+++ b/internal/parser/media/sink_test.go@@ -0,0 +1,116 @@+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.++package media++import (+ "context"+ "errors"+ "strings"+ "testing"++ "ragflow/internal/entity"+)++// These tests pin admission before loading, memoized omissions, and the+// distinction between resource limits and fatal cache failures.+func TestSinkResolvesSourceOnceAndKeepsReferencesAtLimit(t *testing.T) {+ t.Setenv("RAGFLOW_MEDIA_MAX_IMAGES_PER_RUN", "1")+ cache := &sinkTestCache{}+ sink := NewSink(WithCache(t.Context(), cache), "docx", NewBudget())+ loads := 0+ load := func() (string, []byte, bool) { loads++; return "image/png", []byte("image"), true }+ first, err := sink.Resolve("first", 5, load)+ if err != nil || first == nil {+ t.Fatalf("first: %v, %v", first, err)+ }+ if ref, err := sink.Resolve("second", 5, load); ref != nil || err != nil {+ t.Fatalf("limit: %v, %v", ref, err)+ }+ repeated, err := sink.Resolve("first", 5, load)+ if err != nil || repeated == nil || *first != *repeated {+ t.Fatalf("repeat: %v, %v", repeated, err)+ }+ if loads != 1 || cache.writes != 1 {+ t.Fatalf("loads=%d writes=%d; want 1 each", loads, cache.writes)+ }+ if len(sink.Warnings()) == 0 {+ t.Fatal("missing limit warning")+ }+}++func TestSinkSkipsOversizedSourceBeforeLoading(t *testing.T) {+ t.Setenv("RAGFLOW_MEDIA_IMAGE_MAX_BYTES", "4")+ sink := NewSink(WithCache(t.Context(), &sinkTestCache{}), "epub", NewBudget())+ loaded := false+ ref, err := sink.Resolve("large", 5, func() (string, []byte, bool) { loaded = true; return "image/png", []byte("large"), true })+ if ref != nil || err != nil || loaded {+ t.Fatalf("oversized: %v %v loaded=%v", ref, err, loaded)+ }+ if len(sink.Warnings()) != 1 {+ t.Fatalf("warnings=%v", sink.Warnings())+ }+}++func TestSinkMemoizesLoadAndCacheLimitOmissions(t *testing.T) {+ for _, cacheLimit := range []bool{false, true} {+ t.Run(map[bool]string{false: "source", true: "cache_limit"}[cacheLimit], func(t *testing.T) {+ cache := &sinkTestCache{}+ if cacheLimit {+ cache.err = ErrMediaLimitExceeded+ }+ sink := NewSink(WithCache(t.Context(), cache), "html", NewBudget())+ loads := 0+ for range 2 {+ ref, err := sink.Resolve("source", 5, func() (string, []byte, bool) { loads++; return "image/png", []byte("image"), cacheLimit })+ if ref != nil || err != nil {+ t.Fatalf("omission: %v %v", ref, err)+ }+ }+ if loads != 1 {+ t.Fatalf("loads=%d; want 1", loads)+ }+ if cacheLimit && len(sink.Warnings()) != 1 {+ t.Fatalf("warnings=%v", sink.Warnings())+ }+ })+ }+}++func TestSinkPropagatesCacheFailureAndMissingCache(t *testing.T) {+ failure := errors.New("cache unavailable")+ load := func() (string, []byte, bool) { return "image/png", []byte("image"), true }+ sink := NewSink(WithCache(t.Context(), &sinkTestCache{err: failure}), "pptx", NewBudget())+ if _, err := sink.Resolve("source", 5, load); !errors.Is(err, failure) {+ t.Fatalf("error=%v", err)+ }+ if _, err := sink.Resolve("source", 5, load); !errors.Is(err, failure) {+ t.Fatalf("repeated resolution swallowed the fatal cache failure: %v", err)+ }+ sink = NewSink(t.Context(), "pptx", NewBudget())+ if _, err := sink.Resolve("source", 5, load); err == nil || !strings.Contains(err.Error(), "cache is not configured") {+ t.Fatalf("error=%v", err)+ }+}++type sinkTestCache struct {+ writes int+ err error+}++func (c *sinkTestCache) Put(_ context.Context, format, mime string, data []byte) (entity.MediaRef, error) {+ c.writes+++ return entity.MediaRef{Format: format, MIME: mime, Size: int64(len(data)), SHA256: strings.Repeat("a", 64), Kind: entity.MediaKindKV, Scope: "test"}, c.err+}+func (*sinkTestCache) Get(context.Context, entity.MediaRef) ([]byte, error) { panic("unused") }diff --git a/internal/parser/parser/doc_parser.go b/internal/parser/parser/doc_parser.goindex 766ca90694..308704d842 100644--- a/internal/parser/parser/doc_parser.go+++ b/internal/parser/parser/doc_parser.go@@ -22,13 +22,14 @@ import ("strings"deepdocoffice "ragflow/internal/deepdoc/parser/office"- "ragflow/internal/parser/media")// docExtract is the native office_oxide extraction seam for the DOC family.var docExtract = deepdocoffice.OpenAndExtract-type DOCParser struct{}+type DOCParser struct {+ textOnly bool+}func NewDOCParser() *DOCParser {return &DOCParser{}@@ -61,14 +62,23 @@ func (p *DOCParser) ParseWithResult(ctx context.Context, filename string, data [// usable sections, so a document whose IR is empty/unusable still keeps its// content (PlainText / ToMarkdown) rather than silently producing 0 chunks.if strings.TrimSpace(irJSON) != "" {- budget := media.NewBudget()- if sections := buildDOCXJSONSections(irJSON, budget); len(sections) > 0 {+ var sink *officeMediaSink+ if !p.textOnly {+ sink = newOfficeMediaSink(ctx, officeMediaFormatDOCX)+ }+ if sections := buildDOCXJSONSections(irJSON, sink); len(sections) > 0 {+ if sink != nil && sink.err != nil {+ return ParseResult{Err: sink.err}+ }return ParseResult{File: map[string]any{"name": filename, "format": "doc"},JSON: MapsToItems(sections),- Warnings: budget.Warnings(),+ Warnings: sink.warnings(),}}+ if sink != nil && sink.err != nil {+ return ParseResult{Err: sink.err}+ }}if strings.TrimSpace(text) == "" {return ParseResult{Err: fmt.Errorf("doc extract: no text content")}diff --git a/internal/parser/parser/docx_caption_test.go b/internal/parser/parser/docx_caption_test.goindex 84ec46f4bd..74f75f46f4 100644--- a/internal/parser/parser/docx_caption_test.go+++ b/internal/parser/parser/docx_caption_test.go@@ -39,7 +39,7 @@ func TestBuildDOCXJSONSections_TableCaption_ByHeading(t *testing.T) {`{"type":"heading","level":1,"content":[{"type":"text","text":"设备台账"}]}`,docxCaptionTableBody,)- sections := buildDOCXJSONSections(ir, newMediaBudgetForTest())+ sections := buildDOCXJSONSections(ir, nil)if got := docxCaptionFindTableCaption(t, sections); got != "Table Location: 设备台账" {t.Errorf("caption = %q, want %q", got, "Table Location: 设备台账")}@@ -55,7 +55,7 @@ func TestBuildDOCXJSONSections_TableCaption_NestedHeading(t *testing.T) {`{"type":"heading","level":3,"content":[{"type":"text","text":"H3"}]}`,docxCaptionTableBody,)- sections := buildDOCXJSONSections(ir, newMediaBudgetForTest())+ sections := buildDOCXJSONSections(ir, nil)if got := docxCaptionFindTableCaption(t, sections); got != "Table Location: H1 > H2 > H3" {t.Errorf("caption = %q, want %q", got, "Table Location: H1 > H2 > H3")}@@ -67,7 +67,7 @@ func TestBuildDOCXJSONSections_TableCaption_HeadingAfterTable(t *testing.T) {docxCaptionTableBody,`{"type":"heading","level":1,"content":[{"type":"text","text":"后续标题"}]}`,)- sections := buildDOCXJSONSections(ir, newMediaBudgetForTest())+ sections := buildDOCXJSONSections(ir, nil)if got := docxCaptionFindTableCaption(t, sections); got != "" {t.Errorf("table should have no caption when heading is after it, got %q", got)}@@ -83,7 +83,7 @@ func TestBuildDOCXJSONSections_TableCaption_SiblingHeadingSupersedes(t *testing.`{"type":"heading","level":2,"content":[{"type":"text","text":"H2b"}]}`,docxCaptionTableBody,)- sections := buildDOCXJSONSections(ir, newMediaBudgetForTest())+ sections := buildDOCXJSONSections(ir, nil)if got := docxCaptionFindTableCaption(t, sections); got != "Table Location: H1 > H2b" {t.Errorf("caption = %q, want %q", got, "Table Location: H1 > H2b")}@@ -99,7 +99,7 @@ func TestBuildDOCXJSONSections_TableCaption_DeeperHeadingAfterSibling(t *testing`{"type":"heading","level":3,"content":[{"type":"text","text":"H3b"}]}`,docxCaptionTableBody,)- sections := buildDOCXJSONSections(ir, newMediaBudgetForTest())+ sections := buildDOCXJSONSections(ir, nil)if got := docxCaptionFindTableCaption(t, sections); got != "Table Location: H2b > H3b" {t.Errorf("caption = %q, want %q", got, "Table Location: H2b > H3b")}@@ -113,7 +113,7 @@ func TestBuildDOCXJSONSections_TableCaption_EmptyHeadingIgnored(t *testing.T) {`{"type":"heading","level":2,"content":[{"type":"text","text":" "}]}`,docxCaptionTableBody,)- sections := buildDOCXJSONSections(ir, newMediaBudgetForTest())+ sections := buildDOCXJSONSections(ir, nil)if got := docxCaptionFindTableCaption(t, sections); got != "Table Location: H1" {t.Errorf("caption = %q, want %q", got, "Table Location: H1")}@@ -126,7 +126,7 @@ func TestBuildDOCXJSONSections_TableCaption_OutlineSpansSections(t *testing.T) {{"title":"","elements":[{"type":"heading","level":1,"content":[{"type":"text","text":"第一章"}]}]},{"title":"","elements":[` + docxCaptionTableBody + `]}]}`- sections := buildDOCXJSONSections(ir, newMediaBudgetForTest())+ sections := buildDOCXJSONSections(ir, nil)if got := docxCaptionFindTableCaption(t, sections); got != "Table Location: 第一章" {t.Errorf("caption = %q, want %q", got, "Table Location: 第一章")}@@ -139,7 +139,7 @@ func TestBuildDOCXJSONSections_TableCaption_PlainParagraphNotCaption(t *testing.`{"type":"paragraph","content":[{"type":"text","text":"设备流转明细"}]}`,docxCaptionTableBody,)- sections := buildDOCXJSONSections(ir, newMediaBudgetForTest())+ sections := buildDOCXJSONSections(ir, nil)if got := docxCaptionFindTableCaption(t, sections); got != "" {t.Errorf("plain paragraph must not become a caption, got %q", got)}@@ -147,7 +147,7 @@ func TestBuildDOCXJSONSections_TableCaption_PlainParagraphNotCaption(t *testing.// Corner: empty IR yields no sections.func TestBuildDOCXJSONSections_TableCaption_Empty(t *testing.T) {- sections := buildDOCXJSONSections(`{"sections":[]}`, newMediaBudgetForTest())+ sections := buildDOCXJSONSections(`{"sections":[]}`, nil)if len(sections) != 0 {t.Errorf("empty IR should yield no sections, got %d", len(sections))}@@ -161,7 +161,7 @@ func TestBuildDOCXJSONSections_TableCaption_ExplicitPreferredOverLocation(t *tes`{"type":"heading","level":1,"content":[{"type":"text","text":"设备台账"}]}`,docxCaptionTableBody,)- sections := buildDOCXJSONSectionsWithCaptions(ir, newMediaBudgetForTest(), []string{"设备流转明细"})+ sections := buildDOCXJSONSectionsWithCaptions(ir, nil, []string{"设备流转明细"})if got := docxCaptionFindTableCaption(t, sections); got != "设备流转明细" {t.Errorf("explicit caption = %q, want %q", got, "设备流转明细")}@@ -170,7 +170,7 @@ func TestBuildDOCXJSONSections_TableCaption_ExplicitPreferredOverLocation(t *tes// An explicit caption is attached even when the table sits under no heading.func TestBuildDOCXJSONSections_TableCaption_ExplicitWithNoHeading(t *testing.T) {ir := docxCaptionTableIR(docxCaptionTableBody)- sections := buildDOCXJSONSectionsWithCaptions(ir, newMediaBudgetForTest(), []string{"设备流转明细"})+ sections := buildDOCXJSONSectionsWithCaptions(ir, nil, []string{"设备流转明细"})if got := docxCaptionFindTableCaption(t, sections); got != "设备流转明细" {t.Errorf("explicit caption = %q, want %q", got, "设备流转明细")}@@ -182,7 +182,7 @@ func TestBuildDOCXJSONSections_TableCaption_BlankDoesNotOverrideLocation(t *test`{"type":"heading","level":1,"content":[{"type":"text","text":"设备台账"}]}`,docxCaptionTableBody,)- sections := buildDOCXJSONSectionsWithCaptions(ir, newMediaBudgetForTest(), []string{""})+ sections := buildDOCXJSONSectionsWithCaptions(ir, nil, []string{""})if got := docxCaptionFindTableCaption(t, sections); got != "Table Location: 设备台账" {t.Errorf("caption = %q, want %q", got, "Table Location: 设备台账")}diff --git a/internal/parser/parser/docx_ir.go b/internal/parser/parser/docx_ir.goindex 3edcbe39a5..9bc9bd144d 100644--- a/internal/parser/parser/docx_ir.go+++ b/internal/parser/parser/docx_ir.go@@ -25,26 +25,14 @@package parserimport (- "encoding/base64""encoding/json""fmt""strings""ragflow/internal/entity"- "ragflow/internal/parser/media""ragflow/internal/parser/tableutil")-// DOCXFigure represents one embedded image plus its surrounding text-// context, mirroring the chunk-level shape that Python's-// naive_merge_docx produces for vision_figure_parser_docx_wrapper_naive.-type DOCXFigure struct {- Image string `json:"image"` // base64-encoded image bytes- ContextAbove string `json:"context_above"` // text before the image block- ContextBelow string `json:"context_below"` // text after the image block- Marker string `json:"marker"` // substring to locate image position in Markdown-}-// --- office_oxide IR types (local copy, independent of deepdoc) ---type docxIRDocument struct {@@ -205,19 +193,18 @@ func officeIRTableData(el docxIRElement, caption string) *entity.TableData {}type docxTableImage struct {- data []byte- row int- column int- included bool+ data []byte+ ref *entity.MediaRef+ row int+ column int}func forEachDOCXTableImage(el docxIRElement, visit func(docxTableImage) bool) bool {for rowIndex, row := range el.Rows {for columnIndex, cell := range row.Cells {- completed := forEachDOCXIRImage(cell.Content, func(data []byte) bool {+ if !forEachDOCXIRImage(cell.Content, func(data []byte) bool {return visit(docxTableImage{data: data, row: rowIndex + 1, column: columnIndex + 1})- })- if !completed {+ }) {return false}}@@ -225,9 +212,8 @@ func forEachDOCXTableImage(el docxIRElement, visit func(docxTableImage) bool) boreturn true}-// forEachDOCXIRImage visits image payloads without first collecting a second-// document-sized slice of image references. Returning false from visit stops-// traversal as soon as the caller's media budget is exhausted.+// forEachDOCXIRImage stops when a visitor reports a fatal failure. Resource+// omissions keep walking so later text and repeated admitted images survive.func forEachDOCXIRImage(elements []docxIRElement, visit func([]byte) bool) bool {for _, el := range elements {switch el.Type {@@ -245,12 +231,8 @@ func forEachDOCXIRImage(elements []docxIRElement, visit func([]byte) bool) bool}}case "table":- for _, row := range el.Rows {- for _, cell := range row.Cells {- if !forEachDOCXIRImage(cell.Content, visit) {- return false- }- }+ if !forEachDOCXTableImage(el, func(image docxTableImage) bool { return visit(image.data) }) {+ return false}case "list":if !forEachDOCXIRListImage(docxIRList{Items: el.Items}, visit) {@@ -270,10 +252,8 @@ func forEachDOCXIRListImage(list docxIRList, visit func([]byte) bool) bool {if !forEachDOCXIRImage(item.Content, visit) {return false}- if item.Nested != nil {- if !forEachDOCXIRListImage(*item.Nested, visit) {- return false- }+ if item.Nested != nil && !forEachDOCXIRListImage(*item.Nested, visit) {+ return false}}return true@@ -281,10 +261,7 @@ func forEachDOCXIRListImage(list docxIRList, visit func([]byte) bool) bool {// docxElementText returns the plain-text rendering of any supported// IR element type. Used by buildPPTXJSONSections to flatten one slide-// section, and by extractDOCXFiguresFromIR so that tables, lists, and-// text boxes contribute to image surrounding context instead of-// becoming empty flatBlocks (which would drop adjacent VLM context).-// A bare "text" block is not part of the current IR schema, but is+// section. A bare "text" block is not part of the current IR schema, but is// passed through so its payload is never silently dropped. Returns ""// for image and unknown types.//@@ -383,8 +360,11 @@ func docxHeadingLocationCaption(headings []docxHeading) string {// This is the caption-only (heading hierarchy) entry point. Use// buildDOCXJSONSectionsWithCaptions to also attach Caption-styled paragraph// text (a temporary stopgap; see docx_caption_scan.go).-func buildDOCXJSONSections(irJSON string, budget *media.Budget) []map[string]any {- return buildDOCXJSONSectionsCore(irJSON, budget, nil)+//+// A nil sink means a text-only parse: embedded images are skipped and no+// media cache is required. Normal DOCX/DOC parsing always passes a sink.+func buildDOCXJSONSections(irJSON string, sink *officeMediaSink) []map[string]any {+ return buildDOCXJSONSectionsCore(irJSON, sink, nil)}// buildDOCXJSONSectionsWithCaptions is buildDOCXJSONSections plus the@@ -393,8 +373,8 @@ func buildDOCXJSONSections(irJSON string, budget *media.Budget) []map[string]any//// TEMPORARY: remove once office_oxide emits paragraph styles// (yfedoseev/office_oxide#455).-func buildDOCXJSONSectionsWithCaptions(irJSON string, budget *media.Budget, tableCaptions []string) []map[string]any {- return buildDOCXJSONSectionsCore(irJSON, budget, tableCaptions)+func buildDOCXJSONSectionsWithCaptions(irJSON string, sink *officeMediaSink, tableCaptions []string) []map[string]any {+ return buildDOCXJSONSectionsCore(irJSON, sink, tableCaptions)}// buildDOCXJSONSectionsCore is the shared implementation of the two entry@@ -404,14 +384,11 @@ func buildDOCXJSONSectionsWithCaptions(irJSON string, budget *media.Budget, tabl// between tableCaptions and the IR tables is approximate (document-body order// vs office_oxide IR order) and is a known limitation of the temporary// Caption-style stopgap in docx_caption_scan.go.-func buildDOCXJSONSectionsCore(irJSON string, budget *media.Budget, tableCaptions []string) []map[string]any {+func buildDOCXJSONSectionsCore(irJSON string, sink *officeMediaSink, tableCaptions []string) []map[string]any {var ir docxIRDocumentif err := json.Unmarshal([]byte(irJSON), &ir); err != nil {return nil}- if budget == nil {- budget = media.NewBudget()- }var sections []map[string]anytableSequence := 0captionSequence := 0@@ -424,6 +401,9 @@ func buildDOCXJSONSectionsCore(irJSON string, budget *media.Budget, tableCaptiondefer func() { headings = parentHeadings }()}for ei := 0; ei < len(elements); ei++ {+ if sink != nil && sink.err != nil {+ return+ }el := elements[ei]// TEMPORARY: this merge is a stopgap. Revisit it once docxIRRow// carries a three-state header field and office_oxide populates@@ -497,10 +477,10 @@ func buildDOCXJSONSectionsCore(irJSON string, budget *media.Budget, tableCaptioncontinue}}- sections = appendDOCXParagraphSections(sections, el, el.Type == "heading", budget)+ sections = appendDOCXParagraphSections(sections, el, el.Type == "heading", sink)case "image":- sections, _ = appendDOCXImageSection(sections, el.Data, budget, nil)+ sections = appendDOCXImageSection(sections, el.Data, sink, nil)case "table":tableSequence++@@ -519,7 +499,7 @@ func buildDOCXJSONSectionsCore(irJSON string, budget *media.Budget, tableCaptioncaption = explicit}}- sections = appendOfficeTableSections(sections, el, caption, fmt.Sprintf("docx-table-%d", tableSequence), budget)+ sections = appendOfficeTableSections(sections, el, caption, fmt.Sprintf("docx-table-%d", tableSequence), sink)case "list":for _, item := range el.Items {@@ -540,19 +520,20 @@ func buildDOCXJSONSectionsCore(irJSON string, budget *media.Budget, tableCaption}// appendOfficeTableSections emits a table and its embedded images together.-func appendOfficeTableSections(sections []map[string]any, el docxIRElement, caption, tableID string, budget *media.Budget) []map[string]any {+// Images skipped by the media sink are not emitted; the table and its text+// remain so a cache-limit omission never loses surrounding content.+func appendOfficeTableSections(sections []map[string]any, el docxIRElement, caption, tableID string, sink *officeMediaSink) []map[string]any {table := NewTableDataJSONItem(officeIRTableData(el, caption), "", nil).ToMap()var tableImages []docxTableImageforEachDOCXTableImage(el, func(tableImage docxTableImage) bool {- reservation, admission := budget.ReserveOccurrence(media.UnknownSize)- keepWalking := admission != media.RejectedByCount- included := admission == media.Admitted && reservation.Commit(media.Size(len(tableImage.data)))- if included || keepWalking {- tableImage.included = included- tableImages = append(tableImages, tableImage)+ ref, ok := sink.put(tableImage.data, "")+ if !ok {+ return sink == nil || sink.err == nil}- return keepWalking+ tableImage.ref = ref+ tableImages = append(tableImages, tableImage)+ return true})if len(tableImages) > 0 {table["source_table_id"] = tableID@@ -565,7 +546,7 @@ func appendOfficeTableSections(sections []map[string]any, el docxIRElement, capt"column_index": tableImage.column,"media_order": mediaOrder + 1,}- sections = appendDOCXImagePayload(sections, tableImage.data, tableImage.included, metadata)+ sections = appendDOCXImagePayload(sections, tableImage.ref, metadata)}return sections@@ -575,7 +556,7 @@ func appendOfficeTableSections(sections []map[string]any, el docxIRElement, capt// order. A paragraph is not necessarily a single text unit: office_oxide can// place an image between two text runs, and dropping that run would make the// downstream DOCX chunker lose both media order and adjacency context.-func appendDOCXParagraphSections(sections []map[string]any, el docxIRElement, heading bool, budget *media.Budget) []map[string]any {+func appendDOCXParagraphSections(sections []map[string]any, el docxIRElement, heading bool, sink *officeMediaSink) []map[string]any {var text strings.BuilderflushText := func() {value := strings.TrimSpace(text.String())@@ -585,7 +566,6 @@ func appendDOCXParagraphSections(sections []map[string]any, el docxIRElement, he}item := map[string]any{"text": value,- "image": nil,"doc_type_kwd": "text",}if heading {@@ -593,12 +573,13 @@ func appendDOCXParagraphSections(sections []map[string]any, el docxIRElement, he}sections = append(sections, item)}- appendImage := func(data []byte) bool {- var keepWalking bool- sections, keepWalking = appendDOCXImageSection(sections, data, budget, nil)- return keepWalking+ appendImage := func(data []byte) {+ sections = appendDOCXImageSection(sections, data, sink, nil)}for _, run := range el.contentRuns() {+ if sink != nil && sink.err != nil {+ return sections+ }switch run.Type {case "text":text.WriteString(run.Text)@@ -608,263 +589,30 @@ func appendDOCXParagraphSections(sections []map[string]any, el docxIRElement, heflushText()appendImage(run.Data)}- forEachDOCXIRImage(run.Content, func(data []byte) bool { flushText(); return appendImage(data) })+ if !forEachDOCXIRImage(run.Content, func(data []byte) bool { flushText(); appendImage(data); return sink == nil || sink.err == nil }) {+ return sections+ }}flushText()return sections}-func appendDOCXImageSection(sections []map[string]any, data []byte, budget *media.Budget, metadata map[string]any) ([]map[string]any, bool) {- if len(data) == 0 {- return sections, true+func appendDOCXImageSection(sections []map[string]any, data []byte, sink *officeMediaSink, metadata map[string]any) []map[string]any {+ ref, ok := sink.put(data, "")+ if !ok {+ return sections}- reservation, admission := budget.ReserveOccurrence(media.UnknownSize)- keepWalking := admission != media.RejectedByCount- included := admission == media.Admitted && reservation.Commit(media.Size(len(data)))- if !included && !keepWalking {- return sections, false- }- return appendDOCXImagePayload(sections, data, included, metadata), keepWalking+ return appendDOCXImagePayload(sections, ref, metadata)}-func appendDOCXImagePayload(sections []map[string]any, data []byte, included bool, metadata map[string]any) []map[string]any {+func appendDOCXImagePayload(sections []map[string]any, ref *entity.MediaRef, metadata map[string]any) []map[string]any {item := map[string]any{"text": "",- "image": nil,"doc_type_kwd": "image",- }- if included {- item["image"] = base64.StdEncoding.EncodeToString(data)- } else {- item["media_omitted"] = true+ "media_ref": ref,}for key, value := range metadata {item[key] = value}return append(sections, item)}--// --- figure extraction (used by the cgo parser path) -----// extractDOCXFiguresFromIR parses the office_oxide IR JSON and-// returns the images admitted by budget together with the plain text-// immediately surrounding each image. The context matches what Python's-// naive_merge_docx attaches as context_above / context_below on-// each chunk that carries an image.-//-// Reuses the IR already obtained from the doc handle in-// ParseWithResult so the binary is not opened twice.-func extractDOCXFiguresFromIR(irJSON string, budget *media.Budget) []DOCXFigure {- var ir docxIRDocument- if err := json.Unmarshal([]byte(irJSON), &ir); err != nil {- return nil- }- if budget == nil {- budget = media.NewBudget()- }-- var flat []flatBlock- for _, sec := range ir.Sections {- flat = appendDOCXFigureBlocks(flat, sec.Elements)- }-- var figures []DOCXFigure- for i, block := range flat {- if len(block.imageData) == 0 {- continue- }- reservation, admission := budget.ReserveOccurrence(media.UnknownSize)- keepWalking := admission != media.RejectedByCount- included := admission == media.Admitted && reservation.Commit(media.Size(len(block.imageData)))- if !included && !keepWalking {- break- }- fig := DOCXFigure{}- if included {- fig.Image = base64.StdEncoding.EncodeToString(block.imageData)- }-- // Collect text above (backward scan up to docxContextWindow- // chars, or until another image is hit).- above := collectDOCXPrevText(flat, i, 512)- fig.ContextAbove = strings.TrimSpace(above)-- // Collect text below (forward scan up to docxContextWindow- // chars, or until another image is hit).- below := collectDOCXNextText(flat, i, 512)- fig.ContextBelow = strings.TrimSpace(below)-- // Marker: text of the immediately preceding flat block,- // used by the vision dispatcher to locate the image position- // in the rendered Markdown for inline insertion.- for j := i - 1; j >= 0; j-- {- if flat[j].text != "" {- fig.Marker = flat[j].text- break- }- }-- figures = append(figures, fig)- if !keepWalking {- break- }- }- return figures-}--// appendDOCXFigureBlocks flattens IR elements in document order, including-// images nested in tables, lists and text boxes. Section emission recurses-// through the same containers, so figure metadata must use the same traversal-// instead of recording only top-level images.-func appendDOCXFigureBlocks(flat []flatBlock, elements []docxIRElement) []flatBlock {- for _, el := range elements {- switch el.Type {- case "image":- if len(el.Data) > 0 {- flat = append(flat, flatBlock{imageData: el.Data})- }- case "paragraph", "heading":- flat = append(flat, docxInlineFlatBlocks(el)...)- case "table":- for _, row := range el.Rows {- for _, cell := range row.Cells {- flat = appendDOCXFigureBlocks(flat, cell.Content)- }- }- case "list":- for _, item := range el.Items {- flat = appendDOCXFigureBlocks(flat, item.Content)- if item.Nested != nil {- flat = appendDOCXFigureBlocks(flat, []docxIRElement{{Type: "list", Items: item.Nested.Items}})- }- }- case "text_box":- flat = appendDOCXFigureBlocks(flat, el.contentBlocks())- default:- if text := docxElementText(el, "\n"); text != "" {- flat = append(flat, flatBlock{text: text})- }- }- }- return flat-}--// docxInlineFlatBlocks preserves the text/image sequence inside a paragraph-// for figure context extraction. Top-level element order alone is not enough-// for DOCX because office_oxide can encode an inline image as a paragraph run.-func docxInlineFlatBlocks(el docxIRElement) []flatBlock {- var blocks []flatBlock- var text strings.Builder- flushText := func() {- if text.Len() == 0 {- return- }- blocks = append(blocks, flatBlock{text: text.String()})- text.Reset()- }- appendImage := func(data []byte) {- if len(data) == 0 {- return- }- blocks = append(blocks, flatBlock{imageData: data})- }- for _, run := range el.contentRuns() {- switch run.Type {- case "text":- text.WriteString(run.Text)- case "line_break":- text.WriteByte('\n')- case "image":- flushText()- appendImage(run.Data)- for _, nested := range run.Content {- if nested.Type == "image" {- appendImage(nested.Data)- }- }- }- }- flushText()- return blocks-}--// --- internal types -----// flatBlock is a flattened IR element used internally to collect-// text / image context around embedded figures.-type flatBlock struct {- text string- imageData []byte-}--const docxContextWindow = 512--func collectDOCXPrevText(flat []flatBlock, idx, maxLen int) string {- var parts []string- remaining := maxLen- for i := idx - 1; i >= 0 && remaining > 0; i-- {- if len(flat[i].imageData) > 0 {- break // stop at previous image- }- if flat[i].text == "" {- continue- }- r := []rune(flat[i].text)- if len(r) > remaining {- r = r[len(r)-remaining:]- }- parts = append([]string{string(r)}, parts...)- remaining -= len(r)- }- // parts are in document order (farthest first, closest last);- // keep the tail = text nearest the image when truncating for the- // newline separators that Join inserts (they are not counted by- // `remaining`, so the joined length can otherwise exceed maxLen).- joined := strings.Join(parts, "\n")- if r := []rune(joined); len(r) > maxLen {- joined = string(r[len(r)-maxLen:])- }- return joined-}--func collectDOCXNextText(flat []flatBlock, idx, maxLen int) string {- var parts []string- remaining := maxLen- for i := idx + 1; i < len(flat) && remaining > 0; i++ {- if len(flat[i].imageData) > 0 {- break // stop at next image- }- if flat[i].text == "" {- continue- }- r := []rune(flat[i].text)- if len(r) > remaining {- r = r[:remaining]- }- parts = append(parts, string(r))- remaining -= len(r)- }- // parts are in document order (closest first, farthest last);- // keep the head = text nearest the image when truncating for the- // newline separators that Join inserts.- joined := strings.Join(parts, "\n")- if r := []rune(joined); len(r) > maxLen {- joined = string(r[:maxLen])- }- return joined-}--// buildFiguresMap converts the internal DOCXFigure slice to the-// map form attached to fileMeta["figures"].-func buildFiguresMap(figures []DOCXFigure) []map[string]any {- figs := make([]map[string]any, 0, len(figures))- for _, f := range figures {- figs = append(figs, map[string]any{- "image": f.Image,- "context_above": f.ContextAbove,- "context_below": f.ContextBelow,- "marker": f.Marker,- })- }- return figs-}diff --git a/internal/parser/parser/docx_ir_test.go b/internal/parser/parser/docx_ir_test.goindex efe23b41c4..6ef50aaea8 100644--- a/internal/parser/parser/docx_ir_test.go+++ b/internal/parser/parser/docx_ir_test.go@@ -1,11 +1,11 @@package parserimport (+ "bytes""encoding/base64""encoding/json""fmt""reflect"- "strings""testing""ragflow/internal/entity"@@ -17,8 +17,8 @@ import (// pure-Go IR → sections transform without requiring the office_oxide// native library, so it runs under CGO_ENABLED=0.func TestBuildDOCXJSONSections_FromJSON(t *testing.T) {- // "data":"aGVsbG8=" is base64 for "hello"; buildDOCXJSONSections- // re-encodes el.Data, so the item's "image" must equal "aGVsbG8=".+ // A nil sink is the text-only path: the embedded image is skipped, so+ // no image item is emitted.irJSON := `{"sections":[{"title":"","elements":[{"type":"heading","level":1,"content":[{"type":"text","text":"Title"}]},{"type":"paragraph","content":[{"type":"text","text":"Hello"}]},@@ -27,15 +27,14 @@ func TestBuildDOCXJSONSections_FromJSON(t *testing.T) {{"type":"list","items":[{"content":[{"type":"paragraph","content":[{"type":"text","text":"item1"}]}]}]}]}]}`- got := buildDOCXJSONSections(irJSON, newMediaBudgetForTest())+ got := buildDOCXJSONSections(irJSON, nil)want := []map[string]any{- {"text": "Title", "image": nil, "doc_type_kwd": "text", "layout": "heading"},- {"text": "Hello", "image": nil, "doc_type_kwd": "text"},- {"text": "", "image": "aGVsbG8=", "doc_type_kwd": "image"},+ {"text": "Title", "doc_type_kwd": "text", "layout": "heading"},+ {"text": "Hello", "doc_type_kwd": "text"},// The table sits under heading "Title", so it carries the// heading-location caption (mirrors Python __get_nearest_title).{"doc_type_kwd": "table", "table": &entity.TableData{Grid: [][]entity.TableCell{{{Text: "cell", ColSpan: 1, RowSpan: 1}}}, HeaderRows: 1, Caption: "Table Location: Title"}},- {"text": "item1", "image": nil, "doc_type_kwd": "text"},+ {"text": "item1", "doc_type_kwd": "text"},}if !reflect.DeepEqual(got, want) {t.Fatalf("buildDOCXJSONSections mismatch:\n got: %+v\nwant: %+v", got, want)@@ -49,7 +48,7 @@ func TestBuildDOCXJSONSections_EmptyTableSkipped(t *testing.T) {{"type":"paragraph","content":[{"type":"text","text":"keep"}]},{"type":"table","rows":[]}]}]}`- got := buildDOCXJSONSections(irJSON, newMediaBudgetForTest())+ got := buildDOCXJSONSections(irJSON, nil)if len(got) != 1 || got[0]["text"] != "keep" {t.Fatalf("expected only the paragraph to survive, got %+v", got)}@@ -71,7 +70,7 @@ func TestBuildDOCXJSONSections_MergesSplitDocTable(t *testing.T) {{"type":"table","rows":[{"cells":[%s,%s,%s]},{"cells":[%s,%s,%s]}]}]}]}`, cell, cell, cell, cell, cell, cell, cell, cell, cell)- got := buildDOCXJSONSections(irJSON, newMediaBudgetForTest())+ got := buildDOCXJSONSections(irJSON, nil)var tables []map[string]anyfor _, it := range got {if dt, _ := it["doc_type_kwd"].(string); dt == "table" {@@ -103,7 +102,7 @@ func TestBuildDOCXJSONSections_DifferentColumnTablesStaySeparate(t *testing.T) {{"type":"table","rows":[{"cells":[%s,%s,%s]}]},{"type":"table","rows":[{"cells":[%s,%s]},{"cells":[%s,%s]}]}]}]}`, cell, cell, cell, cell, cell, cell, cell)- got := buildDOCXJSONSections(irJSON, newMediaBudgetForTest())+ got := buildDOCXJSONSections(irJSON, nil)var n intfor _, it := range got {if dt, _ := it["doc_type_kwd"].(string); dt == "table" {@@ -128,7 +127,7 @@ func TestBuildDOCXJSONSections_AdjacentMultiRowTablesStaySeparate(t *testing.T){"type":"table","rows":[{"cells":[%s,%s,%s]},{"cells":[%s,%s,%s]}]},{"type":"table","rows":[{"cells":[%s,%s,%s]},{"cells":[%s,%s,%s]}]}]}]}`, cell, cell, cell, cell, cell, cell, cell, cell, cell, cell, cell, cell)- got := buildDOCXJSONSections(irJSON, newMediaBudgetForTest())+ got := buildDOCXJSONSections(irJSON, nil)var tables []map[string]anyfor _, it := range got {if dt, _ := it["doc_type_kwd"].(string); dt == "table" {@@ -160,7 +159,7 @@ func TestBuildDOCXJSONSections_HeaderFragmentWithParagraphBetweenStaysSplit(t *t{"type":"paragraph","content":[{"type":"text","text":"note"}]},{"type":"table","rows":[{"cells":[%s,%s,%s]},{"cells":[%s,%s,%s]}]}]}]}`, cell, cell, cell, cell, cell, cell, cell, cell, cell)- got := buildDOCXJSONSections(irJSON, newMediaBudgetForTest())+ got := buildDOCXJSONSections(irJSON, nil)var n intfor _, it := range got {if dt, _ := it["doc_type_kwd"].(string); dt == "table" {@@ -173,22 +172,31 @@ func TestBuildDOCXJSONSections_HeaderFragmentWithParagraphBetweenStaysSplit(t *t}func TestBuildDOCXJSONSections_PreservesInlineImageOrder(t *testing.T) {+ imageData := officeTestPNG(t, 1)irJSON := `{"sections":[{"elements":[{"type":"paragraph","content":[{"type":"text","text":"before"},- {"type":"image","data":"aGVsbG8="},+ {"type":"image","data":"` + base64.StdEncoding.EncodeToString(imageData) + `"},{"type":"text","text":"after"}]}]}]}`- got := buildDOCXJSONSections(irJSON, newMediaBudgetForTest())+ sink, cache := officeMediaTestSink(t, officeMediaFormatDOCX)+ got := buildDOCXJSONSections(irJSON, sink)+ if sink.err != nil {+ t.Fatalf("sink error: %v", sink.err)+ }if len(got) != 3 {t.Fatalf("sections = %+v, want text/image/text sequence", got)}if got[0]["text"] != "before" || got[0]["doc_type_kwd"] != "text" {t.Errorf("first item = %+v, want text before", got[0])}- if got[1]["image"] != "aGVsbG8=" || got[1]["doc_type_kwd"] != "image" {- t.Errorf("inline image item = %+v", got[1])+ ref := requireOfficeMediaRef(t, got[1])+ if ref.Format != officeMediaFormatDOCX || ref.MIME != "image/png" {+ t.Errorf("media_ref = %+v, want docx image/png", ref)+ }+ if len(cache.puts) != 1 || !bytes.Equal(cache.puts[0].data, imageData) {+ t.Fatalf("cache writes = %+v, want the inline image bytes", cache.puts)}if got[2]["text"] != "after" || got[2]["doc_type_kwd"] != "text" {t.Errorf("last item = %+v, want text after", got[2])@@ -196,18 +204,23 @@ func TestBuildDOCXJSONSections_PreservesInlineImageOrder(t *testing.T) {}func TestBuildDOCXJSONSections_ExtractsTableCellImages(t *testing.T) {+ first, second := officeTestPNG(t, 1), officeTestPNG(t, 2)irJSON := `{"sections":[{"elements":[{"type":"table","rows":[{"cells":[{"content":[{"type":"paragraph","content":[{"type":"text","text":"before"},- {"type":"image","data":"aGVsbG8="},+ {"type":"image","data":"` + base64.StdEncoding.EncodeToString(first) + `"},{"type":"text","text":"after"}]}]},- {"content":[{"type":"image","data":"aW1hZ2U="}]}+ {"content":[{"type":"image","data":"` + base64.StdEncoding.EncodeToString(second) + `"}]}]}]}]}]}`- got := buildDOCXJSONSections(irJSON, newMediaBudgetForTest())+ sink, cache := officeMediaTestSink(t, officeMediaFormatDOCX)+ got := buildDOCXJSONSections(irJSON, sink)+ if sink.err != nil {+ t.Fatalf("sink error: %v", sink.err)+ }if len(got) != 3 {t.Fatalf("sections = %+v, want table plus two cell image items", got)}@@ -217,10 +230,17 @@ func TestBuildDOCXJSONSections_ExtractsTableCellImages(t *testing.T) {if _, ok := got[0]["table"].(*entity.TableData); !ok {t.Fatalf("table item missing structured TableData: %+v", got[0])}- for i, want := range []string{"aGVsbG8=", "aW1hZ2U="} {+ if len(cache.puts) != 2 {+ t.Fatalf("cache writes = %d, want one per cell image", len(cache.puts))+ }+ for i, want := range [][]byte{first, second} {item := got[i+1]- if item["doc_type_kwd"] != "image" || item["image"] != want {- t.Errorf("image item %d = %+v, want payload %q", i, item, want)+ if item["doc_type_kwd"] != "image" {+ t.Errorf("image item %d = %+v", i, item)+ }+ requireOfficeMediaRef(t, item)+ if !bytes.Equal(cache.puts[i].data, want) {+ t.Errorf("cache write %d payload mismatch", i)}if item["parent_table_id"] != "docx-table-1" || item["row_index"] != 1 || item["column_index"] != i+1 || item["media_order"] != i+1 {t.Errorf("image item %d metadata = %+v", i, item)@@ -228,161 +248,6 @@ func TestBuildDOCXJSONSections_ExtractsTableCellImages(t *testing.T) {}}-func TestBuildDOCXJSONSections_BudgetKeepsInlineTextAroundOmittedImage(t *testing.T) {- irJSON := `{"sections":[{"elements":[{"type":"paragraph","content":[- {"type":"text","text":"before"},- {"type":"image","data":"YWJjZA=="},- {"type":"text","text":"after"}- ]}]}]}`- budget := newLimitedMediaBudgetForTest(t, 3, 5, 10)-- items := buildDOCXJSONSections(irJSON, budget)- if len(items) != 3 {- t.Fatalf("items = %+v, want text/image/text with omitted image metadata", items)- }- if items[0]["text"] != "before" || items[2]["text"] != "after" {- t.Fatalf("inline text was lost around omitted image: %+v", items)- }- if items[1]["doc_type_kwd"] != "image" || items[1]["image"] != nil || items[1]["media_omitted"] != true {- t.Fatalf("omitted inline image item = %+v, want metadata without payload", items[1])- }- if len(budget.Warnings()) == 0 {- t.Fatal("omitted image should produce a parser warning")- }-}--func TestBuildDOCXJSONSections_BudgetKeepsTableImageMetadata(t *testing.T) {- irJSON := `{"sections":[{"elements":[{"type":"table","rows":[{"cells":[- {"content":[{"type":"image","data":"YWJj"}]},- {"content":[{"type":"image","data":"ZGVm"}]}- ]}]}]}]}`- budget := newLimitedMediaBudgetForTest(t, 4, 5, 10)-- items := buildDOCXJSONSections(irJSON, budget)- if len(items) != 3 {- t.Fatalf("items = %+v, want table and two image metadata items", items)- }- if items[1]["image"] != "YWJj" || items[1]["media_omitted"] == true {- t.Fatalf("first table image = %+v, want accepted payload", items[1])- }- omitted := items[2]- if omitted["image"] != nil || omitted["media_omitted"] != true {- t.Fatalf("second table image = %+v, want omitted payload", omitted)- }- if omitted["parent_table_id"] != items[0]["source_table_id"] || omitted["row_index"] != 1 || omitted["column_index"] != 2 || omitted["media_order"] != 2 {- t.Fatalf("omitted table image lost location metadata: %+v", omitted)- }-}--func TestBuildDOCXJSONSections_BudgetCapsMediaCount(t *testing.T) {- const countLimit = 8- budget := newLimitedMediaBudgetForTest(t, 8, 1024, countLimit)- var ir strings.Builder- ir.WriteString(`{"sections":[{"elements":[`)- for i := 0; i <= countLimit; i++ {- if i > 0 {- ir.WriteByte(',')- }- ir.WriteString(`{"type":"image","data":"YQ=="}`)- }- ir.WriteString(`]}]}`)-- items := buildDOCXJSONSections(ir.String(), budget)- imageCount := 0- for _, item := range items {- if item["doc_type_kwd"] == "image" {- imageCount++- }- }- if imageCount != countLimit {- t.Fatalf("image items = %d, want count limit %d", imageCount, countLimit)- }- if len(budget.Warnings()) == 0 {- t.Fatal("truncated media count should produce a parser warning")- }-}--func TestBuildPPTXJSONSections_BudgetKeepsSlideMetadata(t *testing.T) {- irJSON := `{"sections":[{"elements":[- {"type":"paragraph","content":[{"type":"text","text":"slide text"}]},- {"type":"image","data":"YWJjZA=="}- ]}]}`- budget := newLimitedMediaBudgetForTest(t, 3, 5, 10)-- items, err := buildPPTXJSONSections(irJSON, budget)- if err != nil {- t.Fatalf("buildPPTXJSONSections: %v", err)- }- if len(items) != 2 || items[0]["text"] != "slide text" {- t.Fatalf("items = %+v, want text and omitted image items", items)- }- omitted := items[1]- if omitted["image"] != nil || omitted["media_omitted"] != true || omitted["slide_number"] != 1 || omitted["media_order"] != 1 {- t.Fatalf("omitted slide image lost metadata: %+v", omitted)- }- if itemsAllEmpty([]map[string]any{omitted}) {- t.Fatal("an image-only slide with a budget-omitted image must not be replaced by whole-deck fallback")- }-}--func TestBuildPPTXJSONSections_BudgetCapsMediaCount(t *testing.T) {- const countLimit = 8- budget := newLimitedMediaBudgetForTest(t, 8, 1024, countLimit)- var ir strings.Builder- ir.WriteString(`{"sections":[{"elements":[`)- for i := 0; i <= countLimit; i++ {- if i > 0 {- ir.WriteByte(',')- }- ir.WriteString(`{"type":"image","data":"YQ=="}`)- }- ir.WriteString(`]}]}`)-- items, err := buildPPTXJSONSections(ir.String(), budget)- if err != nil {- t.Fatalf("buildPPTXJSONSections: %v", err)- }- imageCount := 0- for _, item := range items {- if item["doc_type_kwd"] == "image" {- imageCount++- }- }- if imageCount != countLimit {- t.Fatalf("image items = %d, want count limit %d", imageCount, countLimit)- }- if len(budget.Warnings()) == 0 {- t.Fatal("truncated media count should produce a parser warning")- }-}--func TestMediaBudgetRejectsPerImageAndDocumentByteLimits(t *testing.T) {- budget := newLimitedMediaBudgetForTest(t, 4, 5, 4)- if included, keepGoing := includeMediaForTest(budget, []byte("abc")); !included || !keepGoing {- t.Fatalf("first image admission = (%v, %v), want (true, true)", included, keepGoing)- }- if included, keepGoing := includeMediaForTest(budget, []byte("def")); included || !keepGoing {- t.Fatalf("aggregate overflow admission = (%v, %v), want (false, true)", included, keepGoing)- }- if included, keepGoing := includeMediaForTest(budget, []byte("12345")); included || !keepGoing {- t.Fatalf("per-image overflow admission = (%v, %v), want (false, true)", included, keepGoing)- }- if len(budget.Warnings()) == 0 {- t.Fatal("omitted byte payloads should produce a parser warning")- }-}--func TestBuildDOCXJSONSectionsEncodesOnlyAcceptedImageData(t *testing.T) {- data := []byte("text is not a valid image, but the parser preserves it")- encoded := base64.StdEncoding.EncodeToString(data)- irJSON := fmt.Sprintf(`{"sections":[{"elements":[{"type":"image","data":%q}]}]}`, encoded)- budget := newLimitedMediaBudgetForTest(t, 8, 8, 4)- items := buildDOCXJSONSections(irJSON, budget)- if len(items) != 1 || items[0]["image"] != nil || items[0]["media_omitted"] != true {- t.Fatalf("oversized image items = %+v, want metadata without payload", items)- }-}-// TestJoinDOCXIRRuns pins that only text-type runs are concatenated;// non-text runs (e.g. nested image runs) are skipped.func TestJoinDOCXIRRuns(t *testing.T) {@@ -412,169 +277,6 @@ func TestJoinDOCXIRRuns_LineBreak(t *testing.T) {}}-// TestExtractDOCXFiguresFromIR verifies the image-figure context-// extraction: an image block carries the immediately surrounding text-// as ContextAbove / ContextBelow / Marker. Pure-Go, runs under !cgo.-func TestExtractDOCXFiguresFromIR(t *testing.T) {- irJSON := `{"sections":[{"elements":[- {"type":"paragraph","content":[{"type":"text","text":"before"}]},- {"type":"image","data":"aGVsbG8="},- {"type":"paragraph","content":[{"type":"text","text":"after"}]}- ]}]}`-- figs := extractDOCXFiguresFromIR(irJSON, newMediaBudgetForTest())- if len(figs) != 1 {- t.Fatalf("expected 1 figure, got %d", len(figs))- }- fig := figs[0]- want := DOCXFigure{- Image: "aGVsbG8=",- ContextAbove: "before",- ContextBelow: "after",- Marker: "before",- }- if !reflect.DeepEqual(fig, want) {- t.Fatalf("figure mismatch:\n got: %+v\nwant: %+v", fig, want)- }-}--func TestExtractDOCXFiguresFromIR_InlineImage(t *testing.T) {- irJSON := `{"sections":[{"elements":[- {"type":"paragraph","content":[- {"type":"text","text":"before"},- {"type":"image","data":"aGVsbG8="},- {"type":"text","text":"after"}- ]}- ]}]}`- figs := extractDOCXFiguresFromIR(irJSON, newMediaBudgetForTest())- if len(figs) != 1 {- t.Fatalf("expected 1 inline figure, got %d", len(figs))- }- if figs[0].Image != "aGVsbG8=" || figs[0].ContextAbove != "before" || figs[0].ContextBelow != "after" {- t.Fatalf("inline figure = %+v", figs[0])- }-}--func TestExtractDOCXFiguresFromIR_BudgetKeepsContextWithoutPayload(t *testing.T) {- irJSON := `{"sections":[{"elements":[- {"type":"paragraph","content":[{"type":"text","text":"before"}]},- {"type":"image","data":"YWJjZA=="},- {"type":"paragraph","content":[{"type":"text","text":"after"}]}- ]}]}`- budget := newLimitedMediaBudgetForTest(t, 3, 5, 10)-- figures := extractDOCXFiguresFromIR(irJSON, budget)- if len(figures) != 1 {- t.Fatalf("figures = %+v, want one metadata record", figures)- }- if figures[0].Image != "" || figures[0].ContextAbove != "before" || figures[0].ContextBelow != "after" || figures[0].Marker != "before" {- t.Fatalf("budgeted figure = %+v, want context metadata without payload", figures[0])- }-}--// TestExtractDOCXFiguresFromIR_NoImage returns nil when the IR has no-// image blocks.-func TestExtractDOCXFiguresFromIR_NoImage(t *testing.T) {- irJSON := `{"sections":[{"elements":[- {"type":"paragraph","content":[{"type":"text","text":"only text"}]}- ]}]}`- if figs := extractDOCXFiguresFromIR(irJSON, newMediaBudgetForTest()); figs != nil {- t.Fatalf("expected nil, got %+v", figs)- }-}--// TestExtractDOCXFiguresFromIR_BadJSON returns nil on unparseable IR,-// mirroring the json.Unmarshal error guard.-func TestExtractDOCXFiguresFromIR_BadJSON(t *testing.T) {- if figs := extractDOCXFiguresFromIR("{not json", newMediaBudgetForTest()); figs != nil {- t.Fatalf("expected nil for bad JSON, got %+v", figs)- }-}--// TestExtractDOCXFiguresFromIR_NonParagraphContext verifies that-// tables, lists, and text boxes adjacent to an image contribute their-// text to ContextAbove / ContextBelow. Previously the flatten loop-// called joinDOCXIRRuns(el.contentRuns()) for every non-image element,-// which returned "" for table/list/text_box (their Content is not text-// runs), silently dropping the surrounding VLM context.-func TestExtractDOCXFiguresFromIR_NonParagraphContext(t *testing.T) {- irJSON := `{"sections":[{"elements":[- {"type":"table","rows":[{"cells":[{"content":[{"type":"paragraph","content":[{"type":"text","text":"table cell"}]}]}]}]},- {"type":"image","data":"aGVsbG8="},- {"type":"list","items":[{"content":[{"type":"paragraph","content":[{"type":"text","text":"list item"}]}]}]},- {"type":"text_box","content":[{"type":"paragraph","content":[{"type":"text","text":"box text"}]}]}- ]}]}`- figs := extractDOCXFiguresFromIR(irJSON, newMediaBudgetForTest())- if len(figs) != 1 {- t.Fatalf("expected 1 figure, got %d", len(figs))- }- fig := figs[0]- if fig.ContextAbove != "table cell" {- t.Errorf("ContextAbove = %q, want %q", fig.ContextAbove, "table cell")- }- // Below the image: list item then text_box, joined by newline.- if fig.ContextBelow != "list item\nbox text" {- t.Errorf("ContextBelow = %q, want %q", fig.ContextBelow, "list item\nbox text")- }-}--// TestExtractDOCXFiguresFromIR_NestedImages verifies that images inside-// tables and text boxes reach the figure metadata. Section emission recurses-// into these containers, so a top-level-only figure walk would omit images-// the parser actually emits.-func TestExtractDOCXFiguresFromIR_NestedImages(t *testing.T) {- irJSON := `{"sections":[{"elements":[- {"type":"paragraph","content":[{"type":"text","text":"before"}]},- {"type":"table","rows":[{"cells":[{"content":[{"type":"paragraph","content":[{"type":"text","text":"cell"},{"type":"image","data":"aGVsbG8="}]}]}]}]},- {"type":"text_box","content":[{"type":"paragraph","content":[{"type":"image","data":"d29ybGQ="}]}]}- ]}]}`- figs := extractDOCXFiguresFromIR(irJSON, newMediaBudgetForTest())- if len(figs) != 2 {- t.Fatalf("expected 2 nested figures, got %d: %+v", len(figs), figs)- }- if figs[0].Image != "aGVsbG8=" || figs[1].Image != "d29ybGQ=" {- t.Fatalf("nested figures = %+v", figs)- }- if !strings.Contains(figs[0].ContextAbove, "cell") {- t.Errorf("ContextAbove = %q, want the nested cell text", figs[0].ContextAbove)- }-}--// TestCollectDOCXText_BoundedByMaxLen pins that the joined context-// never exceeds maxLen runes, even though newline separators are-// inserted between blocks (they are not counted by the per-block-// `remaining` decrement). Previously 512 one-rune blocks produced-// 1023 chars (512 runes + 511 separators).-func TestCollectDOCXText_BoundedByMaxLen(t *testing.T) {- // 600 one-rune text blocks before and after an image block.- const maxLen = 512- var flat []flatBlock- for i := 0; i < 600; i++ {- flat = append(flat, flatBlock{text: "x"})- }- imgIdx := len(flat)- flat = append(flat, flatBlock{imageData: []byte("img")})- for i := 0; i < 600; i++ {- flat = append(flat, flatBlock{text: "x"})- }-- prev := collectDOCXPrevText(flat, imgIdx, maxLen)- next := collectDOCXNextText(flat, imgIdx, maxLen)- if r := []rune(prev); len(r) > maxLen {- t.Errorf("prev context = %d runes, want <= %d", len(r), maxLen)- }- if r := []rune(next); len(r) > maxLen {- t.Errorf("next context = %d runes, want <= %d", len(r), maxLen)- }- // Both must still carry content (the closest blocks survive).- if prev == "" {- t.Error("prev context empty; expected some text")- }- if next == "" {- t.Error("next context empty; expected some text")- }-}-// rawJSON marshals v and returns it as json.RawMessage, for populating// docxIRElement.Content in tests. Defined here (tagless) so it is// available to both cgo and !cgo test files in package parser.@@ -628,7 +330,7 @@ func TestExtractTextFromListItem_NestedNull(t *testing.T) {{"content":[{"type":"paragraph","content":[{"type":"text","text":"only item"}]}],"nested":null}]}]}]}`- sections := buildDOCXJSONSections(irJSON, newMediaBudgetForTest())+ sections := buildDOCXJSONSections(irJSON, nil)if len(sections) != 1 || sections[0]["text"] != "only item" {t.Fatalf("expected single item \"only item\", got %+v", sections)}@@ -650,7 +352,7 @@ func TestExtractTextFromListItem_NestedTable(t *testing.T) {]}]}]}]}`- sections := buildDOCXJSONSections(irJSON, newMediaBudgetForTest())+ sections := buildDOCXJSONSections(irJSON, nil)if len(sections) != 2 || sections[0]["text"] != "before" {t.Fatalf("sections=%#v", sections)}@@ -699,7 +401,7 @@ func TestBuildDOCXJSONSections_TextBoxTable(t *testing.T) {]}]}]}]}`- sections := buildDOCXJSONSections(irJSON, newMediaBudgetForTest())+ sections := buildDOCXJSONSections(irJSON, nil)if len(sections) != 1 {t.Fatalf("expected 1 section, got %d: %+v", len(sections), sections)}@@ -718,7 +420,7 @@ func TestBuildDOCXJSONSections_NestedList(t *testing.T) {]}}]}]}]}`- sections := buildDOCXJSONSections(irJSON, newMediaBudgetForTest())+ sections := buildDOCXJSONSections(irJSON, nil)if len(sections) != 3 {t.Fatalf("sections=%#v", sections)}@@ -730,33 +432,38 @@ func TestBuildDOCXJSONSections_NestedList(t *testing.T) {}func TestBuildDOCXJSONSections_EmitsListAndTextBoxImages(t *testing.T) {+ images := [][]byte{officeTestPNG(t, 1), officeTestPNG(t, 2), officeTestPNG(t, 3)}irJSON := `{"sections":[{"elements":[{"type":"list","items":[{"content":[{"type":"paragraph","content":[{"type":"text","text":"list text"},- {"type":"image","data":"aGVsbG8="}+ {"type":"image","data":"` + base64.StdEncoding.EncodeToString(images[0]) + `"}]}],"nested":{"items":[{"content":[{"type":"paragraph","content":[- {"type":"image","data":"d29ybGQ="}+ {"type":"image","data":"` + base64.StdEncoding.EncodeToString(images[1]) + `"}]}]}]}}]},{"type":"text_box","content":[{"type":"paragraph","content":[{"type":"text","text":"text box text"},- {"type":"image","data":"cGljdHVyZQ=="}+ {"type":"image","data":"` + base64.StdEncoding.EncodeToString(images[2]) + `"}]}]}]}]}`- sections := buildDOCXJSONSections(irJSON, newMediaBudgetForTest())+ sink, cache := officeMediaTestSink(t, officeMediaFormatDOCX)+ sections := buildDOCXJSONSections(irJSON, sink)+ if sink.err != nil {+ t.Fatalf("sink error: %v", sink.err)+ }want := []struct {docType stringtext string- image string+ image int}{{docType: "text", text: "list text"},- {docType: "image", image: "aGVsbG8="},- {docType: "image", image: "d29ybGQ="},+ {docType: "image", image: 0},+ {docType: "image", image: 1},{docType: "text", text: "text box text"},- {docType: "image", image: "cGljdHVyZQ=="},+ {docType: "image", image: 2},}if len(sections) != len(want) {t.Fatalf("sections = %+v, want %d items", sections, len(want))@@ -768,8 +475,23 @@ func TestBuildDOCXJSONSections_EmitsListAndTextBoxImages(t *testing.T) {if expected.text != "" && sections[i]["text"] != expected.text {t.Errorf("section %d text = %v, want %q", i, sections[i]["text"], expected.text)}- if expected.image != "" && sections[i]["image"] != expected.image {- t.Errorf("section %d image = %v, want %q", i, sections[i]["image"], expected.image)+ if expected.docType == "image" {+ requireOfficeMediaRef(t, sections[i])+ if !bytes.Equal(cache.puts[expected.image].data, images[expected.image]) {+ t.Errorf("section %d image payload mismatch", i)+ }}}}++func TestDOCXImageTraversalStopsAfterFatalFailure(t *testing.T) {+ elements := []docxIRElement{+ {Type: "table", Rows: []docxIRRow{{Cells: []docxIRCell{{Content: []docxIRElement{{Type: "image", Data: []byte("first")}, {Type: "image", Data: []byte("second")}}}}}}},+ {Type: "image", Data: []byte("third")},+ }+ visits := 0+ completed := forEachDOCXIRImage(elements, func([]byte) bool { visits++; return false })+ if completed || visits != 1 {+ t.Fatalf("completed=%v visits=%d; want false, 1", completed, visits)+ }+}diff --git a/internal/parser/parser/docx_parser.go b/internal/parser/parser/docx_parser.goindex 805bf6e6e9..3d5a9e59c8 100644--- a/internal/parser/parser/docx_parser.go+++ b/internal/parser/parser/docx_parser.go@@ -21,7 +21,6 @@ import ("fmt"deepdocoffice "ragflow/internal/deepdoc/parser/office"- "ragflow/internal/parser/media")// docxExtract is a test seam for the native office_oxide extraction. It@@ -37,6 +36,7 @@ var docxExtract = deepdocoffice.OpenAndExtract// so they compile and test without native libraries.type DOCXParser struct {libType string+ textOnly boolRemoveTOC boolRemoveHeaderFooter bool}@@ -60,8 +60,8 @@ func (p *DOCXParser) ConfigureFromSetup(setup map[string]any) {}// ParseWithResult produces structured JSON items from a docx document.-// It emits bounded image items and attaches figure metadata to the file-// result.+// Embedded images are stored in the run media cache and emitted as media_ref;+// text-only parsers skip them instead.//// File["format"] in the returned ParseResult reflects the effective// container format after magic-byte sniffing (e.g. "doc" for an OLE@@ -96,15 +96,9 @@ func (p *DOCXParser) ParseWithResult(ctx context.Context, filename string, data"format": format,}- // Extract IR JSON for section building and embedded-image extraction.- // The budget bounds image payloads so the downstream VLM enhancement- // stays within limits.- budget := media.NewBudget()- if irJSON != "" {- figures := extractDOCXFiguresFromIR(irJSON, budget)- if len(figures) > 0 {- fileMeta["figures"] = buildFiguresMap(figures)- }+ var sink *officeMediaSink+ if !p.textOnly {+ sink = newOfficeMediaSink(ctx, officeMediaFormatDOCX)}// The result is built entirely from the structured IR, so an empty IR@@ -122,7 +116,10 @@ func (p *DOCXParser) ParseWithResult(ctx context.Context, filename string, dataif capErr != nil {tableCaptions = nil}- sections := buildDOCXJSONSectionsWithCaptions(irJSON, budget, tableCaptions)+ sections := buildDOCXJSONSectionsWithCaptions(irJSON, sink, tableCaptions)+ if sink != nil && sink.err != nil {+ return ParseResult{Err: sink.err}+ }if err := ctx.Err(); err != nil {return ParseResult{Err: err}}@@ -146,7 +143,7 @@ func (p *DOCXParser) ParseWithResult(ctx context.Context, filename string, datareturn ParseResult{File: fileMeta,JSON: MapsToItems(sections),- Warnings: budget.Warnings(),+ Warnings: sink.warnings(),}}diff --git a/internal/parser/parser/docx_parser_test.go b/internal/parser/parser/docx_parser_test.goindex 62d9875b0c..945447b3c5 100644--- a/internal/parser/parser/docx_parser_test.go+++ b/internal/parser/parser/docx_parser_test.go@@ -30,7 +30,7 @@ func TestBuildDOCXJSONSections_Paragraphs(t *testing.T) {ir := `{"sections":[{"title":"","elements":[{"type":"paragraph","content":[{"type":"text","text":"Hello world"}],"style":"Normal"}]}]}`- sections := buildDOCXJSONSections(ir, newMediaBudgetForTest())+ sections := buildDOCXJSONSections(ir, nil)if len(sections) != 1 {t.Fatalf("got %d sections, want 1", len(sections))}@@ -47,7 +47,7 @@ func TestBuildDOCXJSONSections_Headings(t *testing.T) {ir := `{"sections":[{"title":"","elements":[{"type":"heading","level":1,"content":[{"type":"text","text":"Chapter 1"}]}]}]}`- sections := buildDOCXJSONSections(ir, newMediaBudgetForTest())+ sections := buildDOCXJSONSections(ir, nil)if len(sections) != 1 {t.Fatalf("got %d sections, want 1", len(sections))}@@ -64,11 +64,16 @@ func TestBuildDOCXJSONSections_Headings(t *testing.T) {}func TestBuildDOCXJSONSections_Images(t *testing.T) {- b64 := base64.StdEncoding.EncodeToString([]byte("fake-image-data"))+ imageData := officeTestPNG(t, 21)+ b64 := base64.StdEncoding.EncodeToString(imageData)ir := `{"sections":[{"title":"","elements":[{"type":"image","data":"` + b64 + `"}]}]}`- sections := buildDOCXJSONSections(ir, newMediaBudgetForTest())+ sink, cache := officeMediaTestSink(t, officeMediaFormatDOCX)+ sections := buildDOCXJSONSections(ir, sink)+ if sink.err != nil {+ t.Fatalf("sink error: %v", sink.err)+ }if len(sections) != 1 {t.Fatalf("got %d sections, want 1", len(sections))}@@ -76,8 +81,12 @@ func TestBuildDOCXJSONSections_Images(t *testing.T) {if got, ok := item["text"].(string); !ok || got != "" {t.Errorf("text = %q, want empty", got)}- if got, ok := item["image"].(string); !ok || got != b64 {- t.Errorf("image = %q, want %q", got, b64)+ ref := requireOfficeMediaRef(t, item)+ if ref.Format != officeMediaFormatDOCX || ref.MIME != "image/png" {+ t.Errorf("media_ref = %+v, want docx image/png", ref)+ }+ if len(cache.puts) != 1 || string(cache.puts[0].data) != string(imageData) {+ t.Fatalf("cache writes = %+v, want the image bytes", cache.puts)}if got, ok := item["doc_type_kwd"].(string); !ok || got != "image" {t.Errorf("doc_type_kwd = %q, want %q", got, "image")@@ -91,7 +100,7 @@ func TestBuildDOCXJSONSections_Tables(t *testing.T) {{"cells":[{"content":[{"type":"paragraph","content":[{"type":"text","text":"A2"}]}]},{"content":[{"type":"paragraph","content":[{"type":"text","text":"B2"}]}]}]}]}]}]}`- sections := buildDOCXJSONSections(ir, newMediaBudgetForTest())+ sections := buildDOCXJSONSections(ir, nil)if len(sections) != 1 {t.Fatalf("got %d sections, want 1", len(sections))}@@ -122,11 +131,15 @@ func TestBuildDOCXJSONSections_Tables(t *testing.T) {func TestBuildDOCXJSONSections_MixedContent(t *testing.T) {ir := `{"sections":[{"title":"","elements":[{"type":"paragraph","content":[{"type":"text","text":"First para"}]},- {"type":"image","data":"aW1hZ2U="},+ {"type":"image","data":"` + officeTestImageB64(t, 22) + `"},{"type":"table","rows":[{"cells":[{"content":[{"type":"paragraph","content":[{"type":"text","text":"cell"}]}]}]}]},{"type":"heading","level":2,"content":[{"type":"text","text":"Sub title"}]}]}]}`- sections := buildDOCXJSONSections(ir, newMediaBudgetForTest())+ sink, _ := officeMediaTestSink(t, officeMediaFormatDOCX)+ sections := buildDOCXJSONSections(ir, sink)+ if sink.err != nil {+ t.Fatalf("sink error: %v", sink.err)+ }if len(sections) != 4 {t.Fatalf("got %d sections, want 4", len(sections))}@@ -138,6 +151,7 @@ func TestBuildDOCXJSONSections_MixedContent(t *testing.T) {if got, _ := sections[1]["doc_type_kwd"].(string); got != "image" {t.Errorf("item[1].doc_type_kwd = %q, want %q", got, "image")}+ requireOfficeMediaRef(t, sections[1])// table thirdif got, _ := sections[2]["doc_type_kwd"].(string); got != "table" {t.Errorf("item[2].doc_type_kwd = %q, want %q", got, "table")@@ -159,7 +173,7 @@ func TestBuildDOCXJSONSections_List(t *testing.T) {{"content":[{"type":"paragraph","content":[{"type":"text","text":"Item 3"}]}]}],"level":0}]}]}`- sections := buildDOCXJSONSections(ir, newMediaBudgetForTest())+ sections := buildDOCXJSONSections(ir, nil)if len(sections) != 3 {t.Fatalf("got %d sections, want 3 (each list item should be a section)", len(sections))}@@ -177,7 +191,7 @@ func TestBuildDOCXJSONSections_TextBox(t *testing.T) {ir := `{"sections":[{"title":"","elements":[{"type":"text_box","content":[{"type":"paragraph","content":[{"type":"text","text":"Boxed paragraph"}]}],"width_emu":null}]}]}`- sections := buildDOCXJSONSections(ir, newMediaBudgetForTest())+ sections := buildDOCXJSONSections(ir, nil)if len(sections) != 1 {t.Fatalf("got %d sections, want 1 (text_box content should become a section)", len(sections))}@@ -198,7 +212,7 @@ func TestBuildDOCXJSONSections_MixedWithList(t *testing.T) {],"level":0},{"type":"paragraph","content":[{"type":"text","text":"Trailer"}],"style":"Normal"}]}]}`- sections := buildDOCXJSONSections(ir, newMediaBudgetForTest())+ sections := buildDOCXJSONSections(ir, nil)if len(sections) != 4 {t.Fatalf("got %d sections, want 4 (para + 2 list items + para)", len(sections))}@@ -216,7 +230,7 @@ func TestBuildDOCXJSONSections_EmptySkipped(t *testing.T) {{"type":"paragraph","content":[{"type":"text","text":"Only this matters"}]},{"type":"table","rows":[]}]}]}`- sections := buildDOCXJSONSections(ir, newMediaBudgetForTest())+ sections := buildDOCXJSONSections(ir, nil)if len(sections) != 1 {t.Fatalf("got %d sections, want 1 (only non-empty paragraph)", len(sections))}diff --git a/internal/parser/parser/email_parser.go b/internal/parser/parser/email_parser.goindex 158b7a26d6..8a6bd91e3e 100644--- a/internal/parser/parser/email_parser.go+++ b/internal/parser/parser/email_parser.go@@ -605,15 +605,16 @@ func recoverParse(fn func() ParseResult) (res ParseResult, panicked bool) {// attachment via naive_chunk and merges the resulting chunks into the same// document.//-// KNOWN LIMITATION: re-chunk uses the default-config parser returned by-// GetParser(ft) (and, for a nested email, only the top-level p.fields — no-// tenant/setup language, parse_method, or OCR/VLM model). The extracted-// attachment text may therefore differ from ingesting the same file as a-// standalone document through the pipeline's tenant/setup-configured-// parser. This is a deliberate, best-effort approximation for making-// attachment content retrievable (the same simplification Python's-// email.py naive_chunk makes), not a correctness bug; threading the full-// tenant/setup config into re-chunk is intentionally out of scope here.+// KNOWN LIMITATION: re-chunk uses the text-only, default-config parser+// returned by GetTextOnlyParser(ft) (and, for a nested email, only the+// top-level p.fields — no tenant/setup language, parse_method, or OCR/VLM+// model). The extracted attachment text may therefore differ from ingesting+// the same file as a standalone document through the pipeline's+// tenant/setup-configured parser. This is a deliberate, best-effort+// approximation for making attachment content retrievable (the same+// simplification Python's email.py naive_chunk makes), not a correctness+// bug; threading the full tenant/setup config into re-chunk is intentionally+// out of scope here.//// Attachments whose extension has no text-oriented parser (images, audio,// video) are skipped: they carry no plain text in this pipeline and would@@ -687,16 +688,10 @@ func (p *EmailParser) rechunkEmailAttachments(ctx context.Context, content map[sreturn ep.parseEmail(ctx, fn, []byte(payload), depth+1)})} else {- np, err := GetParser(ft)+ np, err := GetTextOnlyParser(ft)if err != nil {continue}- if ft == utility.FileTypeHTML {- np = NewHTMLTextParser()- }- if ft == utility.FileTypeEPUB {- np = NewEPUBTextParser()- }res, panicked = recoverParse(func() ParseResult {return np.ParseWithResult(ctx, fn, []byte(payload))})diff --git a/internal/parser/parser/epub_parser.go b/internal/parser/parser/epub_parser.goindex af1f08e431..3c771bbc69 100644--- a/internal/parser/parser/epub_parser.go+++ b/internal/parser/parser/epub_parser.go@@ -36,7 +36,6 @@ import ("bytes""context""encoding/xml"- "errors""fmt""io""net/url"@@ -379,7 +378,7 @@ func epubMediaResolver(entries map[string]*zip.File, state *htmlWalkState, chapt// and cached once per parse, even when different chapters reference it.// A nil value records a source that was already rejected, so repeated// references do not re-read it or inflate the warning counters.- if ref, seen := state.epubMediaMemo[name]; seen {+ if ref, seen := state.mediaOutput().Lookup("entry:" + name); seen {if ref == nil {return nil, false}@@ -388,13 +387,13 @@ func epubMediaResolver(entries map[string]*zip.File, state *htmlWalkState, chaptmime, ok := images[name]if !ok {state.unresolvedImages++- state.memoizeEPUBMedia(name, nil)+ state.mediaOutput().Skip("entry:" + name)return nil, false}entry, ok := entries[name]if !ok {state.unresolvedImages++- state.memoizeEPUBMedia(name, nil)+ state.mediaOutput().Skip("entry:" + name)return nil, false}return state.loadEPUBMedia(name, mime, entry)@@ -410,43 +409,22 @@ func (s *htmlWalkState) loadEPUBMedia(name, mime string, entry *zip.File) (*entiif entry.UncompressedSize64 <= uint64(maxBytes) {declared = media.Size(entry.UncompressedSize64)}- // Check the declared uncompressed size against both byte limits before- // reserving an item slot or decompressing the entry.- reservation, admission := s.budget.Reserve(declared)- if admission != media.Admitted {- s.memoizeEPUBMedia(name, nil)- return nil, false- }- f, err := entry.Open()- if err != nil {- s.unresolvedImages++- s.memoizeEPUBMedia(name, nil)- return nil, false- }- data, err := io.ReadAll(io.LimitReader(f, int64(maxBytes)+1))- f.Close()- if err != nil || len(data) == 0 {- s.unresolvedImages++- s.memoizeEPUBMedia(name, nil)- return nil, false- }- if !reservation.Commit(media.Size(len(data))) {- s.memoizeEPUBMedia(name, nil)- return nil, false- }- ref, err := s.mediaCache.Put(s.ctx, s.cacheFormat(), mime, data)- if err != nil {- if errors.Is(err, media.ErrMediaLimitExceeded) {- s.mediaLimitSkips++- s.memoizeEPUBMedia(name, nil)- return nil, false+ ref, err := s.mediaOutput().Resolve("entry:"+name, declared, func() (string, []byte, bool) {+ f, err := entry.Open()+ if err != nil {+ s.unresolvedImages+++ return "", nil, false}- s.err = fmt.Errorf("%s media cache write: %w", s.cacheFormat(), err)- s.memoizeEPUBMedia(name, nil)- return nil, false- }- s.memoizeEPUBMedia(name, &ref)- return &ref, true+ defer f.Close()+ data, err := io.ReadAll(io.LimitReader(f, int64(maxBytes)+1))+ if err != nil || len(data) == 0 {+ s.unresolvedImages+++ return "", nil, false+ }+ return mime, data, true+ })+ s.err = err+ return ref, ref != nil}// appendEPUBChapter keeps continuous chapter prose together, splitting onlydiff --git a/internal/parser/parser/html_parser.go b/internal/parser/parser/html_parser.goindex 4cfadf6c81..54d385908c 100644--- a/internal/parser/parser/html_parser.go+++ b/internal/parser/parser/html_parser.go@@ -19,7 +19,6 @@ package parserimport ("bytes""context"- "errors""fmt""strings"@@ -133,13 +132,8 @@ type htmlWalkState struct {mediaOrder intskippedInlineImages intunresolvedImages int- mediaLimitSkips intbudget *media.Budget- // dataURIMemo and epubMediaMemo dedupe sources across one parse. A nil- // value means the source was already processed and skipped; a non-nil- // value is the reference every later occurrence reuses.- dataURIMemo map[string]*entity.MediaRef- epubMediaMemo map[string]*entity.MediaRef+ mediaSink *media.Sink// refOnly defers image bytes to content-addressed references instead of// carrying the normalized data URI on the item. HTML and EPUB set it;// formats that still inline image bytes leave it false.@@ -159,9 +153,10 @@ type htmlWalkState struct {func (s *htmlWalkState) warnings() []string {var warnings []string- warnings = append(warnings, s.budget.Warnings()...)- if s.mediaLimitSkips > 0 {- warnings = append(warnings, fmt.Sprintf("document parser skipped %d image(s) after the run media cache limit", s.mediaLimitSkips))+ if s.mediaSink != nil {+ warnings = append(warnings, s.mediaSink.Warnings()...)+ } else {+ warnings = append(warnings, s.budget.Warnings()...)}if s.skippedInlineImages > 0 {warnings = append(warnings, fmt.Sprintf("document parser skipped %d invalid or oversized inline image(s)", s.skippedInlineImages))@@ -200,56 +195,26 @@ func (s *htmlWalkState) imageSource(src string) (string, *entity.MediaRef, bool)return "", nil, false}if s.refOnly {- // A repeated source is decoded and written once; every reference still- // receives the same MediaRef. A nil memo value means it was already- // processed and skipped, so the warning counters are not inflated.- if ref, seen := s.dataURIMemo[src]; seen {- if ref == nil {- return "", nil, false- }- return "", ref, true+ sink := s.mediaOutput()+ key := "data:" + src+ if ref, seen := sink.Lookup(key); seen {+ return "", ref, ref != nil}- // Validate and size the payload without allocating so budget admission- // happens before the decode. Malformed sources never spend budget.size, ok := media.DataImageDecodedSize(src)if !ok {s.skippedInlineImages++- s.memoizeDataURI(src, nil)- return "", nil, false- }- if s.mediaCache == nil {- s.err = fmt.Errorf("%s media cache is not configured", s.cacheFormat())- return "", nil, false- }- // Repeated sources share one reservation, so a document made of one- // repeated data URI stays bounded without capping its references.- reservation, admission := s.budget.Reserve(media.Size(size))- if admission != media.Admitted {- s.memoizeDataURI(src, nil)- return "", nil, false- }- mime, data, ok := media.DecodeDataImageSource(src)- if !ok {- s.skippedInlineImages++- s.memoizeDataURI(src, nil)+ sink.Skip(key)return "", nil, false}- if !reservation.Commit(media.Size(len(data))) {- s.memoizeDataURI(src, nil)- return "", nil, false- }- ref, err := s.mediaCache.Put(s.ctx, s.cacheFormat(), mime, data)- if err != nil {- if errors.Is(err, media.ErrMediaLimitExceeded) {- s.mediaLimitSkips++- s.memoizeDataURI(src, nil)- return "", nil, false+ ref, err := sink.Resolve(key, media.Size(size), func() (string, []byte, bool) {+ mime, data, ok := media.DecodeDataImageSource(src)+ if !ok {+ s.skippedInlineImages++}- s.err = fmt.Errorf("%s media cache write: %w", s.cacheFormat(), err)- return "", nil, false- }- s.memoizeDataURI(src, &ref)- return "", &ref, true+ return mime, data, ok+ })+ s.err = err+ return "", ref, ref != nil}reservation, admission := s.budget.ReserveOccurrence(media.UnknownSize)if admission != media.Admitted {@@ -266,18 +231,12 @@ func (s *htmlWalkState) imageSource(src string) (string, *entity.MediaRef, bool)return normalized, nil, true}-func (s *htmlWalkState) memoizeDataURI(src string, ref *entity.MediaRef) {- if s.dataURIMemo == nil {- s.dataURIMemo = make(map[string]*entity.MediaRef)- }- s.dataURIMemo[src] = ref-}--func (s *htmlWalkState) memoizeEPUBMedia(name string, ref *entity.MediaRef) {- if s.epubMediaMemo == nil {- s.epubMediaMemo = make(map[string]*entity.MediaRef)+// mediaOutput shares source admission and writes across data URIs and EPUB entries.+func (s *htmlWalkState) mediaOutput() *media.Sink {+ if s.mediaSink == nil {+ s.mediaSink = media.NewSink(media.WithCache(s.ctx, s.mediaCache), s.cacheFormat(), s.budget)}- s.epubMediaMemo[name] = ref+ return s.mediaSink}// walkHTMLBlocksWithState emits normalized text and image items in documentdiff --git a/internal/parser/parser/office_image_fixture_cgo_test.go b/internal/parser/parser/office_image_fixture_cgo_test.goindex dedececab0..7ee24d4ef2 100644--- a/internal/parser/parser/office_image_fixture_cgo_test.go+++ b/internal/parser/parser/office_image_fixture_cgo_test.go@@ -4,7 +4,6 @@ package parserimport ("bytes"- "encoding/base64""errors""image""image/color"@@ -34,7 +33,8 @@ func TestDOCXParser_RealTableImage(t *testing.T) {t.Fatal(err)}p := NewDOCXParser()- result := p.ParseWithResult(t.Context(), "docx_table_image.docx", data)+ ctx, cache := officeMediaTestContext()+ result := p.ParseWithResult(ctx, "docx_table_image.docx", data)if result.Err != nil {t.Fatal(result.Err)}@@ -54,10 +54,13 @@ func TestDOCXParser_RealTableImage(t *testing.T) {if item["doc_type_kwd"] != "image" || item["parent_table_id"] != tableID {continue}- encoded, _ := item["image"].(string)- decoded, err := base64.StdEncoding.DecodeString(encoded)+ ref := requireOfficeMediaRef(t, item)+ if ref.Format != officeMediaFormatDOCX || ref.MIME != "image/png" {+ t.Fatalf("media_ref = %+v, want docx image/png", ref)+ }+ decoded, err := cache.Get(ctx, *ref)if err != nil {- t.Fatalf("table image payload: %v", err)+ t.Fatalf("cache Get: %v", err)}if _, err := png.Decode(bytes.NewReader(decoded)); err != nil {t.Fatalf("table image is not PNG: %v", err)@@ -91,7 +94,8 @@ func TestPPTXParser_RealSlideImage(t *testing.T) {if err != nil {t.Fatal(err)}- result := NewPPTXParser().ParseWithResult(t.Context(), "slide_image.pptx", data)+ ctx, cache := officeMediaTestContext()+ result := NewPPTXParser().ParseWithResult(ctx, "slide_image.pptx", data)if result.Err != nil {t.Fatal(result.Err)}@@ -102,10 +106,13 @@ func TestPPTXParser_RealSlideImage(t *testing.T) {if item["doc_type_kwd"] != "image" {continue}- encoded, _ := item["image"].(string)- decoded, err := base64.StdEncoding.DecodeString(encoded)+ ref := requireOfficeMediaRef(t, item)+ if ref.Format != officeMediaFormatPPTX || ref.MIME != "image/png" {+ t.Fatalf("media_ref = %+v, want pptx image/png", ref)+ }+ decoded, err := cache.Get(ctx, *ref)if err != nil {- t.Fatalf("slide image payload: %v", err)+ t.Fatalf("cache Get: %v", err)}if _, err := png.Decode(bytes.NewReader(decoded)); err != nil {t.Fatalf("slide image is not PNG: %v", err)diff --git a/internal/parser/parser/office_media.go b/internal/parser/parser/office_media.gonew file mode 100644index 0000000000..4456639967--- /dev/null+++ b/internal/parser/parser/office_media.go@@ -0,0 +1,171 @@+//+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.++package parser++import (+ "bytes"+ "context"+ "crypto/sha256"+ "encoding/binary"+ "encoding/hex"+ "fmt"+ "net/http"+ "strings"++ "ragflow/internal/entity"+ "ragflow/internal/parser/media"+)++// Office media formats recorded on cache writes. DOC and PPT share the DOCX+// and PPTX parser families, so the family name is the format.+const (+ officeMediaFormatDOCX = "docx"+ officeMediaFormatPPTX = "pptx"+ officeMediaFormatXLSX = "xlsx"+)++// officeMediaSink hands Office image bytes to the run media cache and tracks+// per-document budget omissions. A nil sink means a text-only parse: image+// bytes are skipped without error because no run media cache exists. The+// normal DOCX/PPTX/XLSX parsers always construct a sink, so a missing cache+// fails the parse instead of silently dropping images.+type officeMediaSink struct {+ sink *media.Sink+ format string+ err error+ invalidImages int+}++func newOfficeMediaSink(ctx context.Context, format string) *officeMediaSink {+ return &officeMediaSink{sink: media.NewSink(ctx, format, media.NewBudget()), format: format}+}++// put keeps Office payload identification local and delegates source admission,+// deduplication and cache failures to the shared media sink.+func (s *officeMediaSink) put(data []byte, mime string) (*entity.MediaRef, bool) {+ if s == nil || s.err != nil {+ return nil, false+ }+ // The IR supplies bytes rather than source paths. Memoize every digest,+ // including rejected sources, so repeated omissions are counted once.+ source := officeMediaSource(data)+ if ref, seen := s.sink.Lookup(source); seen && ref == nil {+ return nil, false+ }+ if len(data) == 0 {+ s.sink.Skip(source)+ s.invalidImages+++ return nil, false+ }+ if mime == "" {+ mime = officeImageMIME(data)+ }+ if mime == "" {+ s.sink.Skip(source)+ s.invalidImages+++ return nil, false+ }+ ref, err := s.sink.Resolve(source, media.Size(len(data)), func() (string, []byte, bool) {+ return mime, data, true+ })+ s.err = err+ return ref, ref != nil+}++func (s *officeMediaSink) warnings() []string {+ if s == nil {+ return nil+ }+ warnings := s.sink.Warnings()+ if s.invalidImages > 0 {+ warnings = append(warnings, fmt.Sprintf("%s parser skipped %d invalid or unsupported embedded image(s)", s.format, s.invalidImages))+ }+ return warnings+}++func officeMediaSource(data []byte) string {+ sum := sha256.Sum256(data)+ return hex.EncodeToString(sum[:])+}++var (+ tiffLittleEndian = []byte("II*\x00")+ tiffBigEndian = []byte("MM\x00*")+ emfSignature = []byte(" EMF")+ utf8BOM = []byte{0xEF, 0xBB, 0xBF}+)++// officeImageMIME identifies an embedded Office image from its bytes. The IR+// carries no filename or media type, so the payload itself must supply the+// real image MIME the cache stores. Returns "" when the bytes are not a+// recognised image; the parser skips that source with a warning.+func officeImageMIME(data []byte) string {+ if len(data) == 0 {+ return ""+ }+ switch {+ case bytes.HasPrefix(data, tiffLittleEndian), bytes.HasPrefix(data, tiffBigEndian):+ return "image/tiff"+ case isEMFImage(data):+ return "image/x-emf"+ case isWMFImage(data):+ return "image/x-wmf"+ }+ if mime := http.DetectContentType(data); strings.HasPrefix(mime, "image/") {+ return mime+ }+ if isSVGImage(data) {+ return "image/svg+xml"+ }+ return ""+}++// isEMFImage matches the EMF header: a type DWORD of 1 followed by the+// " EMF" signature at the fixed header offset.+func isEMFImage(data []byte) bool {+ return len(data) >= 44 && binary.LittleEndian.Uint32(data) == 1 && bytes.Equal(data[40:44], emfSignature)+}++// isWMFImage matches the placeable-metafile magic and the standard WMF+// header (type 1 or 2, header size 9 words).+func isWMFImage(data []byte) bool {+ if bytes.HasPrefix(data, []byte{0xD7, 0xCD, 0xC6, 0x9A}) {+ return true+ }+ return len(data) >= 4 && (data[0] == 1 || data[0] == 2) && data[1] == 0 && data[2] == 9 && data[3] == 0+}++// isSVGImage matches an XML/SVG document without allocating a copy of the+// payload: DetectContentType reports text for SVG, and the Office IR does not+// carry the original extension.+func isSVGImage(data []byte) bool {+ trimmed := bytes.TrimSpace(bytes.TrimPrefix(data, utf8BOM))+ if len(trimmed) > 1024 {+ trimmed = trimmed[:1024]+ }+ if len(trimmed) >= 4 && bytes.EqualFold(trimmed[:4], []byte("<svg")) {+ return true+ }+ if len(trimmed) < 5 || !bytes.EqualFold(trimmed[:5], []byte("<?xml")) {+ return false+ }+ for i := 5; i+4 <= len(trimmed); i++ {+ if trimmed[i] == '<' && bytes.EqualFold(trimmed[i:i+4], []byte("<svg")) {+ return true+ }+ }+ return false+}diff --git a/internal/parser/parser/office_media_test.go b/internal/parser/parser/office_media_test.gonew file mode 100644index 0000000000..5a736e6eb1--- /dev/null+++ b/internal/parser/parser/office_media_test.go@@ -0,0 +1,473 @@+//+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.++package parser++import (+ "bytes"+ "context"+ "crypto/sha256"+ "encoding/base64"+ "encoding/binary"+ "encoding/hex"+ "errors"+ "fmt"+ "image"+ "image/color"+ "image/png"+ "strings"+ "testing"++ "ragflow/internal/entity"+ "ragflow/internal/parser/media"+ "ragflow/internal/utility"+)++// officeTestPNG returns a small deterministic PNG. Distinct seeds produce+// distinct bytes, so tests can use the digest as the image identity.+func officeTestPNG(t *testing.T, seed int) []byte {+ t.Helper()+ img := image.NewGray(image.Rect(0, 0, 17, 17))+ img.SetGray(seed%17, (seed/17)%17, color.Gray{Y: 1 + uint8(seed%255)})+ var buf bytes.Buffer+ if err := png.Encode(&buf, img); err != nil {+ t.Fatalf("encode test PNG: %v", err)+ }+ return buf.Bytes()+}++// officeTestImageB64 is the base64 encoding the Office IR uses for image data.+func officeTestImageB64(t *testing.T, seed int) string {+ t.Helper()+ return base64.StdEncoding.EncodeToString(officeTestPNG(t, seed))+}++// officeMediaTestContext returns a context carrying a recording media cache.+func officeMediaTestContext() (context.Context, *recordingMediaCache) {+ cache := &recordingMediaCache{}+ return media.WithCache(context.Background(), cache), cache+}++// officeMediaTestSink builds a sink over a recording media cache.+func officeMediaTestSink(t *testing.T, format string) (*officeMediaSink, *recordingMediaCache) {+ t.Helper()+ ctx, cache := officeMediaTestContext()+ return newOfficeMediaSink(ctx, format), cache+}++// officeMediaTestSinkWithCache builds a sink over the given cache.+func officeMediaTestSinkWithCache(format string, cache media.Cache) *officeMediaSink {+ return newOfficeMediaSink(media.WithCache(context.Background(), cache), format)+}++// requireOfficeMediaRef asserts that an item carries a valid media_ref and no+// legacy base64 image payload.+func requireOfficeMediaRef(t *testing.T, item map[string]any) *entity.MediaRef {+ t.Helper()+ if _, ok := item["image"]; ok {+ t.Fatalf("legacy image payload retained: %#v", item)+ }+ ref := entity.MediaRefFromAny(item["media_ref"])+ if ref == nil || !ref.Valid() {+ t.Fatalf("item has no valid media_ref: %#v", item)+ }+ return ref+}++// dedupingMediaCache mirrors the run media cache's digest dedupe: repeated+// writes of one payload reach the backend once and share its reference.+type dedupingMediaCache struct {+ recordingMediaCache+ backendWrites int+ refs map[string]entity.MediaRef+}++func (c *dedupingMediaCache) Put(ctx context.Context, format, mime string, data []byte) (entity.MediaRef, error) {+ sum := sha256.Sum256(data)+ key := hex.EncodeToString(sum[:])+ if ref, ok := c.refs[key]; ok {+ return ref, nil+ }+ ref, err := c.recordingMediaCache.Put(ctx, format, mime, data)+ if err != nil {+ return entity.MediaRef{}, err+ }+ if c.refs == nil {+ c.refs = make(map[string]entity.MediaRef)+ }+ c.refs[key] = ref+ c.backendWrites+++ return ref, nil+}++func TestOfficeMediaSinkWritesImageToRunCache(t *testing.T) {+ sink, cache := officeMediaTestSink(t, officeMediaFormatDOCX)+ imageData := officeTestPNG(t, 3)+ ref, ok := sink.put(imageData, "")+ if !ok || ref == nil {+ t.Fatalf("put = (%+v, %v), want an accepted image", ref, ok)+ }+ if sink.err != nil {+ t.Fatalf("sink error: %v", sink.err)+ }+ if len(cache.puts) != 1 {+ t.Fatalf("cache writes = %d, want 1", len(cache.puts))+ }+ put := cache.puts[0]+ if put.format != officeMediaFormatDOCX || put.mime != "image/png" || !bytes.Equal(put.data, imageData) {+ t.Fatalf("cache write = %+v, want docx image/png with the image bytes", put)+ }+ sum := sha256.Sum256(imageData)+ if ref.Kind != entity.MediaKindKV || ref.Scope != "run-1" ||+ ref.SHA256 != hex.EncodeToString(sum[:]) || ref.MIME != "image/png" ||+ ref.Size != int64(len(imageData)) {+ t.Fatalf("media_ref = %+v", ref)+ }+ if warnings := sink.warnings(); len(warnings) != 0 {+ t.Fatalf("warnings = %v, want none", warnings)+ }+}++// TestOfficeMediaSinkReusesOneBackendWriteForRepeatedImage pins that repeating+// one image does not charge the document budget twice and reaches the media+// cache backend once, while every occurrence still gets the same reference.+func TestOfficeMediaSinkReusesOneBackendWriteForRepeatedImage(t *testing.T) {+ cache := &dedupingMediaCache{}+ sink := officeMediaTestSinkWithCache(officeMediaFormatPPTX, cache)+ imageData := officeTestPNG(t, 4)++ first, ok := sink.put(imageData, "image/png")+ if !ok {+ t.Fatal("first put rejected")+ }+ second, ok := sink.put(imageData, "image/png")+ if !ok {+ t.Fatal("repeated put rejected")+ }+ if *first != *second {+ t.Fatalf("references differ: %+v vs %+v", *first, *second)+ }+ if cache.backendWrites != 1 {+ t.Fatalf("backend writes = %d, want 1", cache.backendWrites)+ }+ if warnings := sink.warnings(); len(warnings) != 0 {+ t.Fatalf("warnings = %v, want none", warnings)+ }+}++func TestOfficeMediaSinkSkipsInvalidImages(t *testing.T) {+ sink, cache := officeMediaTestSink(t, officeMediaFormatDOCX)+ for _, data := range [][]byte{nil, []byte("not an image")} {+ if ref, ok := sink.put(data, ""); ok || ref != nil {+ t.Fatalf("put(%q) = (%+v, %v), want skipped", data, ref, ok)+ }+ }+ if sink.err != nil {+ t.Fatalf("sink error: %v", sink.err)+ }+ if cache.putCalls != 0 {+ t.Fatalf("cache writes = %d, want none for invalid images", cache.putCalls)+ }+ warnings := strings.Join(sink.warnings(), "\n")+ if !strings.Contains(warnings, "skipped 2 invalid or unsupported embedded image(s)") {+ t.Fatalf("warnings = %q, want invalid-image warning", warnings)+ }+}++func TestOfficeMediaSinkFailsWithoutCache(t *testing.T) {+ sink := newOfficeMediaSink(context.Background(), officeMediaFormatDOCX)+ ref, ok := sink.put(officeTestPNG(t, 5), "")+ if ok || ref != nil {+ t.Fatalf("put = (%+v, %v), want failure without a cache", ref, ok)+ }+ if sink.err == nil || !strings.Contains(sink.err.Error(), "media cache is not configured") {+ t.Fatalf("sink error = %v, want missing-cache failure", sink.err)+ }+}++func TestOfficeMediaSinkFailsOnCacheWriteError(t *testing.T) {+ writeErr := errors.New("kvrocks unavailable")+ cache := &recordingMediaCache{putErr: writeErr}+ sink := officeMediaTestSinkWithCache(officeMediaFormatDOCX, cache)++ if ref, ok := sink.put(officeTestPNG(t, 6), ""); ok || ref != nil {+ t.Fatalf("put = (%+v, %v), want failure", ref, ok)+ }+ if !errors.Is(sink.err, writeErr) {+ t.Fatalf("sink error = %v, want %v", sink.err, writeErr)+ }+ if cache.putCalls != 1 {+ t.Fatalf("cache Put calls = %d, want 1", cache.putCalls)+ }+ if warnings := sink.warnings(); len(warnings) != 0 {+ t.Fatalf("warnings = %v, want the failure reported as an error, not a skip", warnings)+ }+ if ref, ok := sink.put(officeTestPNG(t, 7), ""); ok || ref != nil {+ t.Fatalf("put after failure = (%+v, %v), want skipped", ref, ok)+ }+ if cache.putCalls != 1 {+ t.Fatalf("cache Put calls = %d, want traversal to stop after the failure", cache.putCalls)+ }+}++func TestOfficeMediaSinkSkipsImageAtRunCacheLimit(t *testing.T) {+ cache := &recordingMediaCache{putErr: fmt.Errorf(+ "mediacache: run image count limit 1 exceeded: %w", media.ErrMediaLimitExceeded)}+ sink := officeMediaTestSinkWithCache(officeMediaFormatDOCX, cache)++ if ref, ok := sink.put(officeTestPNG(t, 8), ""); ok || ref != nil {+ t.Fatalf("put = (%+v, %v), want skipped at the run limit", ref, ok)+ }+ if sink.err != nil {+ t.Fatalf("sink error = %v, want a warning, not a parse failure", sink.err)+ }+ warnings := strings.Join(sink.warnings(), "\n")+ if !strings.Contains(warnings, "skipped 1 image(s) after the run media cache limit") {+ t.Fatalf("warnings = %q, want run-limit warning", warnings)+ }+}++func TestOfficeMediaSinkSkipsOversizedImage(t *testing.T) {+ t.Setenv("RAGFLOW_MEDIA_IMAGE_MAX_BYTES", "16")+ sink, cache := officeMediaTestSink(t, officeMediaFormatDOCX)+ if ref, ok := sink.put(officeTestPNG(t, 9), ""); ok || ref != nil {+ t.Fatalf("put = (%+v, %v), want skipped for exceeding the per-image limit", ref, ok)+ }+ if sink.err != nil {+ t.Fatalf("sink error = %v, want a warning", sink.err)+ }+ if cache.putCalls != 0 {+ t.Fatalf("cache writes = %d, want none for an oversized image", cache.putCalls)+ }+ warnings := strings.Join(sink.warnings(), "\n")+ if !strings.Contains(warnings, "larger than the 16-byte per-image limit") {+ t.Fatalf("warnings = %q, want oversized-image warning", warnings)+ }+}++// TestOfficeMediaSinkCapsDistinctImages pins the run source limit: the first+// 256 distinct images are admitted and later sources are skipped with a+// warning instead of failing the parse.+func TestOfficeMediaSinkCapsDistinctImages(t *testing.T) {+ t.Setenv("RAGFLOW_MEDIA_MAX_IMAGES_PER_RUN", "256")+ sink, cache := officeMediaTestSink(t, officeMediaFormatXLSX)+ emitted := 0+ for i := 0; i <= 256; i++ {+ if _, ok := sink.put(officeTestPNG(t, i), ""); ok {+ emitted+++ }+ }+ if emitted != 256 {+ t.Fatalf("admitted images = %d, want 256", emitted)+ }+ if cache.putCalls != 256 {+ t.Fatalf("cache writes = %d, want 256", cache.putCalls)+ }+ if sink.err != nil {+ t.Fatalf("sink error = %v, want a warning", sink.err)+ }+ warnings := strings.Join(sink.warnings(), "\n")+ if !strings.Contains(warnings, "stopped extracting embedded images after the 256-item run limit") {+ t.Fatalf("warnings = %q, want run-limit warning", warnings)+ }+}++// TestBuildDOCXJSONSections_KeepsTextWhenRunLimitSkipsImage verifies that a+// cache run-limit skip removes only the image item and warns; surrounding text+// and the rest of the document survive.+func TestBuildDOCXJSONSections_KeepsTextWhenRunLimitSkipsImage(t *testing.T) {+ cache := &recordingMediaCache{putErr: fmt.Errorf(+ "mediacache: run image count limit 1 exceeded: %w", media.ErrMediaLimitExceeded)}+ sink := officeMediaTestSinkWithCache(officeMediaFormatDOCX, cache)+ irJSON := `{"sections":[{"elements":[+ {"type":"paragraph","content":[{"type":"text","text":"before"}]},+ {"type":"image","data":"` + officeTestImageB64(t, 10) + `"},+ {"type":"paragraph","content":[{"type":"text","text":"after"}]}+ ]}]}`++ sections := buildDOCXJSONSections(irJSON, sink)+ if sink.err != nil {+ t.Fatalf("sink error: %v", sink.err)+ }+ if len(sections) != 2 || sections[0]["text"] != "before" || sections[1]["text"] != "after" {+ t.Fatalf("sections = %+v, want the surrounding text without the skipped image", sections)+ }+ if !strings.Contains(strings.Join(sink.warnings(), "\n"), "after the run media cache limit") {+ t.Fatalf("warnings = %v, want run-limit warning", sink.warnings())+ }+}++func TestOfficeImageMIME(t *testing.T) {+ emf := make([]byte, 44)+ binary.LittleEndian.PutUint32(emf, 1)+ copy(emf[40:44], " EMF")++ tests := []struct {+ name string+ data []byte+ want string+ }{+ {name: "png", data: officeTestPNG(t, 0), want: "image/png"},+ {name: "jpeg", data: []byte{0xFF, 0xD8, 0xFF, 0xE0, 0x00, 0x10, 'J', 'F', 'I', 'F'}, want: "image/jpeg"},+ {name: "gif", data: []byte("GIF89a\x01\x00\x01\x00"), want: "image/gif"},+ {name: "bmp", data: []byte("BM\x00\x00\x00\x00"), want: "image/bmp"},+ {name: "tiff little endian", data: []byte("II*\x00\x08\x00\x00\x00"), want: "image/tiff"},+ {name: "tiff big endian", data: []byte("MM\x00*\x00\x00\x00\x08"), want: "image/tiff"},+ {name: "emf", data: emf, want: "image/x-emf"},+ {name: "wmf placeable", data: []byte{0xD7, 0xCD, 0xC6, 0x9A, 0x00, 0x00}, want: "image/x-wmf"},+ {name: "wmf standard", data: []byte{1, 0, 9, 0, 0, 3}, want: "image/x-wmf"},+ {name: "svg", data: []byte(`<svg xmlns="http://www.w3.org/2000/svg"/>`), want: "image/svg+xml"},+ {name: "xml svg", data: []byte("\xef\xbb\xbf<?xml version=\"1.0\"?><svg xmlns=\"http://www.w3.org/2000/svg\"/>"), want: "image/svg+xml"},+ {name: "unknown", data: []byte("not an image"), want: ""},+ {name: "empty", data: nil, want: ""},+ }+ for _, tc := range tests {+ t.Run(tc.name, func(t *testing.T) {+ if got := officeImageMIME(tc.data); got != tc.want {+ t.Errorf("officeImageMIME = %q, want %q", got, tc.want)+ }+ })+ }+}++// TestDOCXParserFailsWhenMediaCacheIsMissing pins that the normal DOCX parser+// fails the parse instead of silently dropping an embedded image when no run+// media cache is attached.+func TestDOCXParserFailsWhenMediaCacheIsMissing(t *testing.T) {+ orig := docxExtract+ t.Cleanup(func() { docxExtract = orig })+ irJSON := `{"sections":[{"elements":[+ {"type":"paragraph","content":[{"type":"text","text":"keep"}]},+ {"type":"image","data":"` + officeTestImageB64(t, 11) + `"}+ ]}]}`+ docxExtract = func([]byte, string) (string, string, string, error, error) {+ return irJSON, "", "", nil, nil+ }++ res := NewDOCXParser().ParseWithResult(context.Background(), "missing.docx", []byte("PK\x03\x04"))+ if res.Err == nil || !strings.Contains(res.Err.Error(), "media cache is not configured") {+ t.Fatalf("ParseWithResult error = %v, want missing-cache failure", res.Err)+ }+}++func TestDOCXTextParserSkipsImagesWithoutCache(t *testing.T) {+ orig := docxExtract+ t.Cleanup(func() { docxExtract = orig })+ irJSON := `{"sections":[{"elements":[+ {"type":"paragraph","content":[{"type":"text","text":"keep"}]},+ {"type":"image","data":"` + officeTestImageB64(t, 12) + `"}+ ]}]}`+ docxExtract = func([]byte, string) (string, string, string, error, error) {+ return irJSON, "", "", nil, nil+ }++ p, err := GetTextOnlyParser(utility.FileTypeDOCX)+ if err != nil {+ t.Fatal(err)+ }+ res := p.ParseWithResult(context.Background(), "text.docx", []byte("PK\x03\x04"))+ if res.Err != nil {+ t.Fatalf("ParseWithResult: %v", res.Err)+ }+ items := ItemsToMaps(res.JSON)+ if len(items) != 1 || items[0]["text"] != "keep" {+ t.Fatalf("items = %+v, want the text without the skipped image", items)+ }+ if len(res.Warnings) != 0 {+ t.Fatalf("warnings = %v, want none for a text-only parse", res.Warnings)+ }+}++func TestPPTXParserFailsWhenMediaCacheIsMissing(t *testing.T) {+ orig := pptxExtract+ t.Cleanup(func() { pptxExtract = orig })+ irJSON := `{"sections":[{"elements":[+ {"type":"paragraph","content":[{"type":"text","text":"keep"}]},+ {"type":"image","data":"` + officeTestImageB64(t, 13) + `"}+ ]}]}`+ pptxExtract = func([]byte, string) (string, string, string, error, error) {+ return irJSON, "", "", nil, nil+ }++ res := NewPPTXParser().ParseWithResult(context.Background(), "missing.pptx", []byte("PK\x03\x04"))+ if res.Err == nil || !strings.Contains(res.Err.Error(), "media cache is not configured") {+ t.Fatalf("ParseWithResult error = %v, want missing-cache failure", res.Err)+ }+}++func TestPPTXTextParserSkipsImagesWithoutCache(t *testing.T) {+ orig := pptxExtract+ t.Cleanup(func() { pptxExtract = orig })+ irJSON := `{"sections":[{"elements":[+ {"type":"paragraph","content":[{"type":"text","text":"keep"}]},+ {"type":"image","data":"` + officeTestImageB64(t, 14) + `"}+ ]}]}`+ pptxExtract = func([]byte, string) (string, string, string, error, error) {+ return irJSON, "", "", nil, nil+ }++ p, err := GetTextOnlyParser(utility.FileTypePPTX)+ if err != nil {+ t.Fatal(err)+ }+ res := p.ParseWithResult(context.Background(), "text.pptx", []byte("PK\x03\x04"))+ if res.Err != nil {+ t.Fatalf("ParseWithResult: %v", res.Err)+ }+ items := ItemsToMaps(res.JSON)+ if len(items) != 1 || items[0]["text"] != "keep" {+ t.Fatalf("items = %+v, want the text without the skipped image", items)+ }+ if len(res.Warnings) != 0 {+ t.Fatalf("warnings = %v, want none for a text-only parse", res.Warnings)+ }+}++// SVG sniffing must inspect a bounded prefix without copying a large payload.+func TestOfficeSVGSniffDoesNotCopyPayload(t *testing.T) {+ data := append([]byte(`<?xml version="1.0"?><SVG xmlns="http://www.w3.org/2000/svg">`), bytes.Repeat([]byte("X"), 1<<20)...)+ if !isSVGImage(data) {+ t.Fatal("SVG not recognized")+ }+ if allocations := testing.AllocsPerRun(10, func() { _ = isSVGImage(data) }); allocations != 0 {+ t.Fatalf("SVG sniff allocated %v times; want no payload copy", allocations)+ }+}++func TestOfficeMediaSinkCountsDistinctInvalidPayloadsWithoutSpendingBudget(t *testing.T) {+ t.Setenv("RAGFLOW_MEDIA_MAX_IMAGES_PER_RUN", "1")+ sink, cache := officeMediaTestSink(t, officeMediaFormatDOCX)+ for _, data := range [][]byte{nil, {}, []byte("unrecognized-a"), []byte("unrecognized-a"), []byte("unrecognized-b"), []byte("unrecognized-b")} {+ if ref, ok := sink.put(data, ""); ok || ref != nil {+ t.Fatalf("invalid payload accepted: %q", data)+ }+ }+ if cache.putCalls != 0 {+ t.Fatalf("invalid payload cache writes = %d, want 0", cache.putCalls)+ }+ if ref, ok := sink.put(officeTestPNG(t, 2), ""); !ok || ref == nil {+ t.Fatal("invalid payloads consumed the only image slot")+ }+ if sink.err != nil {+ t.Fatalf("sink error: %v", sink.err)+ }+ warnings := sink.warnings()+ if len(warnings) != 1 || !strings.Contains(warnings[0], "skipped 3 invalid or unsupported embedded image(s)") {+ t.Fatalf("warnings = %v, want one warning for three distinct invalid payloads", warnings)+ }+ if cache.putCalls != 1 {+ t.Fatalf("cache writes = %d, want only the valid image", cache.putCalls)+ }+}diff --git a/internal/parser/parser/office_table_render.go b/internal/parser/parser/office_table_render.goindex 35312221d6..9d76f2d090 100644--- a/internal/parser/parser/office_table_render.go+++ b/internal/parser/parser/office_table_render.go@@ -17,7 +17,6 @@package parserimport (- "encoding/base64""fmt""regexp""sort"@@ -29,7 +28,6 @@ import ("ragflow/internal/common""ragflow/internal/entity"- "ragflow/internal/parser/media""ragflow/internal/parser/tableutil")@@ -599,7 +597,16 @@ func deprecatedHTML4Excel(setup map[string]any, parserName string) {// worksheet as structured parser items. Excelize exposes both kinds through// GetPictureCells/GetPictures; walking the reported anchor cells avoids// scanning the worksheet's entire coordinate space.-func extractXLSXImages(f *excelize.File, sheet string, budget *media.Budget) ([]map[string]any, []string) {+//+// Image bytes are written to the run media cache and the item carries only a+// media_ref. A nil sink is the text-only path: images and their anchors are+// skipped. An unsupported extension or a payload rejected by the budget or the+// cache limit skips that image with a warning, leaving the surrounding sheet+// items intact.+func extractXLSXImages(f *excelize.File, sheet string, sink *officeMediaSink) ([]map[string]any, []string) {+ if sink == nil {+ return nil, nil+ }cells, err := f.GetPictureCells(sheet)if err != nil {return nil, []string{fmt.Sprintf("XLSX image discovery failed for sheet %q: %v", sheet, err)}@@ -610,8 +617,10 @@ func extractXLSXImages(f *excelize.File, sheet string, budget *media.Budget) ([]items := make([]map[string]any, 0, len(cells))warnings := make([]string, 0)-cellLoop:for _, cell := range cells {+ if sink.err != nil {+ break+ }pictures, err := f.GetPictures(sheet, cell)if err != nil {warnings = append(warnings, fmt.Sprintf("XLSX image extraction failed for sheet %q cell %s: %v", sheet, cell, err))@@ -626,38 +635,26 @@ cellLoop:warnings = append(warnings, fmt.Sprintf("XLSX image skipped for sheet %q cell %s: unsupported extension %q", sheet, cell, picture.Extension))continue}- included, keepWalking := true, true- if budget != nil {- reservation, admission := budget.ReserveOccurrence(media.UnknownSize)- keepWalking = admission != media.RejectedByCount- included = admission == media.Admitted && reservation.Commit(media.Size(len(picture.File)))- }- if !keepWalking {- break cellLoop+ ref, ok := sink.put(picture.File, mimeType)+ if !ok {+ continue}alt := cellif picture.Format != nil && picture.Format.AltText != "" {alt = picture.Format.AltText}row, col := axisToRC(cell)- item := map[string]any{+ items = append(items, map[string]any{"text": alt,"doc_type_kwd": "image",+ "media_ref": ref,"sheet": sheet,"cell": cell,"row_start": row,"row_end": row,"col_start": col,"col_end": col,- }- if included {- encoded := base64.StdEncoding.EncodeToString(picture.File)- item["image"] = "data:" + mimeType + ";base64," + encoded- } else {- item["image"] = nil- item["media_omitted"] = true- }- items = append(items, item)+ })}}return items, warningsdiff --git a/internal/parser/parser/office_table_render_test.go b/internal/parser/parser/office_table_render_test.goindex 8b487a68c2..0599c56384 100644--- a/internal/parser/parser/office_table_render_test.go+++ b/internal/parser/parser/office_table_render_test.go@@ -229,7 +229,8 @@ func TestXLSXParserImageAnchorsSplitSegments(t *testing.T) {}})p, _ := NewXLSXParser("")- res := p.ParseWithResult(t.Context(), "anchors.xlsx", data)+ ctx, _ := officeMediaTestContext()+ res := p.ParseWithResult(ctx, "anchors.xlsx", data)if res.Err != nil {t.Fatalf("ParseWithResult: %v", res.Err)}@@ -244,6 +245,7 @@ func TestXLSXParserImageAnchorsSplitSegments(t *testing.T) {if items[1]["doc_type_kwd"] != "image" || items[1]["row_start"] != 4 {t.Fatalf("image item = %#v", items[1])}+ requireOfficeMediaRef(t, items[1])for segIdx, want := range [][]string{wantFirst, wantSecond} {item := items[segIdx*2]text := tableItemHTML(t, item)@@ -310,7 +312,8 @@ func TestXLSXParserImageAnchorDoesNotSplitMergedGroup(t *testing.T) {}})p, _ := NewXLSXParser("")- res := p.ParseWithResult(t.Context(), "merge-anchor.xlsx", data)+ ctx, _ := officeMediaTestContext()+ res := p.ParseWithResult(ctx, "merge-anchor.xlsx", data)if res.Err != nil {t.Fatalf("ParseWithResult: %v", res.Err)}@@ -321,6 +324,7 @@ func TestXLSXParserImageAnchorDoesNotSplitMergedGroup(t *testing.T) {if items[1]["doc_type_kwd"] != "image" {t.Fatalf("middle item = %#v, want image", items[1])}+ requireOfficeMediaRef(t, items[1])first := items[0]["table"].(*entity.TableData)if len(first.Grid) != 3 {t.Fatalf("first segment rows = %d, want header + two merged rows", len(first.Grid))@@ -358,7 +362,8 @@ func TestXLSXParserImageAnchorAtLastMergedRow(t *testing.T) {}})p, _ := NewXLSXParser("")- res := p.ParseWithResult(t.Context(), "merge-anchor-last.xlsx", data)+ ctx, _ := officeMediaTestContext()+ res := p.ParseWithResult(ctx, "merge-anchor-last.xlsx", data)if res.Err != nil {t.Fatalf("ParseWithResult: %v", res.Err)}@@ -485,7 +490,8 @@ func TestXLSXImageMIMEType(t *testing.T) {func TestExtractXLSXImagesWarningForInvalidSheet(t *testing.T) {f := excelize.NewFile()defer f.Close()- _, warnings := extractXLSXImages(f, "MissingSheet", nil)+ sink, _ := officeMediaTestSink(t, officeMediaFormatXLSX)+ _, warnings := extractXLSXImages(f, "MissingSheet", sink)if len(warnings) != 1 || !strings.Contains(warnings[0], "image discovery failed") {t.Fatalf("warnings = %v, want image discovery warning", warnings)}@@ -816,7 +822,8 @@ func TestXLSXParserHeaderSpanClampedPerSegment(t *testing.T) {}})p, _ := NewXLSXParser("")- res := p.ParseWithResult(t.Context(), "header-span.xlsx", data)+ ctx, _ := officeMediaTestContext()+ res := p.ParseWithResult(ctx, "header-span.xlsx", data)if res.Err != nil {t.Fatalf("ParseWithResult: %v", res.Err)}@@ -824,6 +831,7 @@ func TestXLSXParserHeaderSpanClampedPerSegment(t *testing.T) {if len(items) != 3 {t.Fatalf("items = %d, want table, image, table", len(items))}+ requireOfficeMediaRef(t, items[1])first := items[0]["table"].(*entity.TableData)if first.Grid[0][1].RowSpan != 2 || !first.Grid[1][1].Covered {t.Fatalf("first segment header span = %+v, want rowspan 2 with a covered row", first.Grid)diff --git a/internal/parser/parser/office_text_only_test.go b/internal/parser/parser/office_text_only_test.gonew file mode 100644index 0000000000..34348d3c08--- /dev/null+++ b/internal/parser/parser/office_text_only_test.go@@ -0,0 +1,65 @@+//+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.++package parser++import (+ "testing"++ "ragflow/internal/utility"+)++func TestGetTextOnlyParserRoutesOfficeFamilies(t *testing.T) {+ tests := []struct {+ fileType utility.FileType+ isText func(ParseResultProducer) bool+ }{+ {utility.FileTypeDOCX, func(p ParseResultProducer) bool {+ parser, ok := p.(*DOCXParser)+ return ok && parser.textOnly+ }},+ {utility.FileTypeDOC, func(p ParseResultProducer) bool {+ parser, ok := p.(*DOCParser)+ return ok && parser.textOnly+ }},+ {utility.FileTypePPTX, func(p ParseResultProducer) bool {+ parser, ok := p.(*PPTXParser)+ return ok && parser.textOnly+ }},+ {utility.FileTypePPT, func(p ParseResultProducer) bool {+ parser, ok := p.(*PPTParser)+ return ok && parser.pptx.textOnly+ }},+ {utility.FileTypeXLSX, func(p ParseResultProducer) bool {+ parser, ok := p.(*XLSXParser)+ return ok && parser.textOnly+ }},+ {utility.FileTypeXLS, func(p ParseResultProducer) bool {+ parser, ok := p.(*XLSParser)+ return ok && parser.textOnly+ }},+ }+ for _, tc := range tests {+ t.Run(string(tc.fileType), func(t *testing.T) {+ p, err := GetTextOnlyParser(tc.fileType)+ if err != nil {+ t.Fatalf("GetTextOnlyParser: %v", err)+ }+ if !tc.isText(p) {+ t.Fatalf("GetTextOnlyParser(%q) = %T, want the text-only parser", tc.fileType, p)+ }+ })+ }+}diff --git a/internal/parser/parser/parser_type.go b/internal/parser/parser/parser_type.goindex 5136fe542d..30d8490ab8 100644--- a/internal/parser/parser/parser_type.go+++ b/internal/parser/parser/parser_type.go@@ -22,38 +22,68 @@ import ("ragflow/internal/utility")-// GetParser returns the ParseResultProducer for the given file type.-// Each format picks a single backend — the Python dispatcher does-// not pass any lib_type knob for these families, so the Go side-// mirrors that with one constructor per format. Constructors mirror-// the NewPDFParser shape (no error return): none of them have inputs-// that could fail, so the error slot is dead weight.+// GetTextOnlyParser returns a parser for readable-text extraction: previews,+// attachments, agent uploads and CLI parsing. Embedded media is skipped even+// when the extraction is nested inside an ingestion run (email attachments).+// These paths also work without an injected run media cache.+func GetTextOnlyParser(fileType utility.FileType) (ParseResultProducer, error) {+ return newParser(fileType, true)+}++// GetParser returns the ingestion parser for the given file type.func GetParser(fileType utility.FileType) (ParseResultProducer, error) {+ return newParser(fileType, false)+}++// newParser owns the format routes for both ingestion and text extraction.+func newParser(fileType utility.FileType, textOnly bool) (ParseResultProducer, error) {switch fileType {case utility.FileTypePPTX:- return NewPPTXParser(), nil+ p := NewPPTXParser()+ p.textOnly = textOnly+ return p, nilcase utility.FileTypePPT:- return NewPPTParser(), nil+ p := NewPPTParser()+ p.pptx.textOnly = textOnly+ return p, nilcase utility.FileTypeXLSX:- return NewXLSXParser("")+ p, err := NewXLSXParser("")+ if err != nil {+ return nil, err+ }+ p.textOnly = textOnly+ return p, nilcase utility.FileTypeXLS:- return NewXLSParser("")+ p, err := NewXLSParser("")+ if err != nil {+ return nil, err+ }+ p.textOnly = textOnly+ return p, nilcase utility.FileTypeCSV:return NewCSVParser(), nilcase utility.FileTypeDOCX:- return NewDOCXParser(), nil+ p := NewDOCXParser()+ p.textOnly = textOnly+ return p, nilcase utility.FileTypeDOC:- return NewDOCParser(), nil+ p := NewDOCParser()+ p.textOnly = textOnly+ return p, nilcase utility.FileTypePDF:return NewPDFParser(), nilcase utility.FileTypeHTML:- return NewHTMLParser(), nil+ p := NewHTMLParser()+ p.textOnly = textOnly+ return p, nilcase utility.FileTypeMarkdown:return NewMarkdownParser(GoMarkdown)case utility.FileTypeTXT:return NewTextParser(), nilcase utility.FileTypeEPUB:- return NewEPUBParser(), nil+ p := NewEPUBParser()+ p.textOnly = textOnly+ return p, nilcase utility.FileTypeJSON:return NewJSONParser(), nilcase utility.FileTypeEMAIL:diff --git a/internal/parser/parser/pdf_table_no_html_text_test.go b/internal/parser/parser/pdf_table_no_html_text_test.goindex 99bb7ee219..105e4bd76a 100644--- a/internal/parser/parser/pdf_table_no_html_text_test.go+++ b/internal/parser/parser/pdf_table_no_html_text_test.go@@ -38,7 +38,7 @@ func TestPDFTableItemsOmitHTMLText(t *testing.T) {return items}},{"docx table", func() []map[string]any {- return buildDOCXJSONSections(docxIR, newMediaBudgetForTest())+ return buildDOCXJSONSections(docxIR, nil)}},}for _, tc := range cases {diff --git a/internal/parser/parser/pptx_ir.go b/internal/parser/parser/pptx_ir.goindex b2e6d80434..dc063bc57b 100644--- a/internal/parser/parser/pptx_ir.go+++ b/internal/parser/parser/pptx_ir.go@@ -22,20 +22,18 @@ import ("strings""ragflow/internal/entity"- "ragflow/internal/parser/media")// buildPPTXJSONSections emits text runs, tables and images in slide order.// Every item retains its slide number; a blank slide emits an empty text item.// Container elements are traversed recursively so grouped tables remain structured.-func buildPPTXJSONSections(irJSON string, budget *media.Budget) ([]map[string]any, error) {+// Image bytes are stored in the run media cache and emitted as media_ref; a+// nil sink is the text-only path and skips images.+func buildPPTXJSONSections(irJSON string, sink *officeMediaSink) ([]map[string]any, error) {var ir docxIRDocumentif err := json.Unmarshal([]byte(irJSON), &ir); err != nil {return nil, fmt.Errorf("presentation ir-json decode: %w", err)}- if budget == nil {- budget = media.NewBudget()- }items := make([]map[string]any, 0, len(ir.Sections))for i, sec := range ir.Sections {start := len(items)@@ -50,6 +48,9 @@ func buildPPTXJSONSections(irJSON string, budget *media.Budget) ([]map[string]anvar emitElements func([]docxIRElement)emitElements = func(elements []docxIRElement) {for _, el := range elements {+ if sink != nil && sink.err != nil {+ return+ }var units []map[string]anyswitch el.Type {case "table":@@ -58,7 +59,7 @@ func buildPPTXJSONSections(irJSON string, budget *media.Budget) ([]map[string]an}flushText()tableSequence++- items = appendOfficeTableSections(items, el, "", fmt.Sprintf("pptx-slide-%d-table-%d", i+1, tableSequence), budget)+ items = appendOfficeTableSections(items, el, "", fmt.Sprintf("pptx-slide-%d-table-%d", i+1, tableSequence), sink)continuecase "text_box":emitElements(el.contentBlocks())@@ -72,9 +73,9 @@ func buildPPTXJSONSections(irJSON string, budget *media.Budget) ([]map[string]an}continuecase "paragraph", "heading":- units = appendDOCXParagraphSections(nil, el, false, budget)+ units = appendDOCXParagraphSections(nil, el, false, sink)case "image":- units, _ = appendDOCXImageSection(nil, el.Data, budget, nil)+ units = appendDOCXImageSection(nil, el.Data, sink, nil)default:if text := strings.TrimSpace(docxElementText(el, "\n")); text != "" {lines = append(lines, text)@@ -116,10 +117,7 @@ func itemsAllEmpty(items []map[string]any) bool {if text, _ := it["text"].(string); strings.TrimSpace(text) != "" {return false}- if image, _ := it["image"].(string); strings.TrimSpace(image) != "" {- return false- }- if omitted, _ := it["media_omitted"].(bool); omitted {+ if entity.MediaRefFromAny(it["media_ref"]) != nil {return false}}diff --git a/internal/parser/parser/pptx_ir_test.go b/internal/parser/parser/pptx_ir_test.goindex c1d77599e9..26a089b776 100644--- a/internal/parser/parser/pptx_ir_test.go+++ b/internal/parser/parser/pptx_ir_test.go@@ -1,6 +1,12 @@package parser-import "testing"+import (+ "encoding/base64"+ "strings"+ "testing"++ "ragflow/internal/entity"+)// TestBuildPPTXJSONSections feeds office_oxide presentation IR JSON// directly into buildPPTXJSONSections and asserts one JSON item per@@ -172,7 +178,7 @@ func TestBuildPPTXJSONSections(t *testing.T) {}for _, tt := range tests {t.Run(tt.name, func(t *testing.T) {- items, err := buildPPTXJSONSections(tt.irJSON, newMediaBudgetForTest())+ items, err := buildPPTXJSONSections(tt.irJSON, nil)if tt.wantErr {if err == nil {t.Fatalf("buildPPTXJSONSections: want error, got items %+v", items)@@ -211,39 +217,53 @@ func TestBuildPPTXJSONSections(t *testing.T) {}func TestBuildPPTXJSONSections_ExtractsSlideImagesInOrder(t *testing.T) {+ images := [][]byte{officeTestPNG(t, 1), officeTestPNG(t, 2), officeTestPNG(t, 3)}irJSON := `{"sections":[{"elements":[{"type":"paragraph","content":[{"type":"text","text":"slide text"}]},- {"type":"image","data":"aGVsbG8="},- {"type":"table","rows":[{"cells":[{"content":[{"type":"image","data":"aW1hZ2U="}]}]}]}- ]},{"elements":[{"type":"image","data":"c2Vjb25k"}]}]}`+ {"type":"image","data":"` + base64.StdEncoding.EncodeToString(images[0]) + `"},+ {"type":"table","rows":[{"cells":[{"content":[{"type":"image","data":"` + base64.StdEncoding.EncodeToString(images[1]) + `"}]}]}]}+ ]},{"elements":[{"type":"image","data":"` + base64.StdEncoding.EncodeToString(images[2]) + `"}]}]}`- items, err := buildPPTXJSONSections(irJSON, newMediaBudgetForTest())+ sink, cache := officeMediaTestSink(t, officeMediaFormatPPTX)+ items, err := buildPPTXJSONSections(irJSON, sink)if err != nil {t.Fatalf("buildPPTXJSONSections: %v", err)}+ if sink.err != nil {+ t.Fatalf("sink error: %v", sink.err)+ }want := []struct {docType stringtext string- image string+ image intslide intorder int}{{docType: "text", text: "slide text", slide: 1},- {docType: "image", image: "aGVsbG8=", slide: 1, order: 1},+ {docType: "image", image: 0, slide: 1, order: 1},{docType: "table", slide: 1},- {docType: "image", image: "aW1hZ2U=", slide: 1, order: 2},- {docType: "image", image: "c2Vjb25k", slide: 2, order: 1},+ {docType: "image", image: 1, slide: 1, order: 2},+ {docType: "image", image: 2, slide: 2, order: 1},}if len(items) != len(want) {t.Fatalf("items = %+v, want %d slide/text and image items", items, len(want))}+ if len(cache.puts) != len(images) {+ t.Fatalf("cache writes = %d, want %d", len(cache.puts), len(images))+ }for i, w := range want {item := items[i]if item["doc_type_kwd"] != w.docType || item["slide_number"] != w.slide {t.Errorf("item %d identity = %+v, want type %q slide %d", i, item, w.docType, w.slide)}- if w.docType == "image" && (item["image"] != w.image || item["media_order"] != w.order) {- t.Errorf("item %d image = %+v, want payload %q order %d", i, item, w.image, w.order)+ if w.docType == "image" {+ requireOfficeMediaRef(t, item)+ if item["media_order"] != w.order {+ t.Errorf("item %d media_order = %v, want %d", i, item["media_order"], w.order)+ }+ if string(cache.puts[w.image].data) != string(images[w.image]) {+ t.Errorf("item %d cache payload mismatch", i)+ }}if w.docType == "text" && item["text"] != w.text {t.Errorf("item %d text = %v, want %q", i, item["text"], w.text)@@ -272,9 +292,14 @@ func TestItemsAllEmpty(t *testing.T) {want: false,},{- name: "image payload",- items: []map[string]any{{"text": "", "image": "aGVsbG8=", "doc_type_kwd": "image"}},- want: false,+ name: "image reference",+ items: []map[string]any{{"text": "", "doc_type_kwd": "image", "media_ref": &entity.MediaRef{+ Format: "pptx",+ Kind: entity.MediaKindKV,+ Scope: "run-1",+ SHA256: strings.Repeat("a", 64),+ }}},+ want: false,},}for _, tt := range tests {diff --git a/internal/parser/parser/pptx_parser.go b/internal/parser/parser/pptx_parser.goindex a54d9d24fd..3d97430e4e 100644--- a/internal/parser/parser/pptx_parser.go+++ b/internal/parser/parser/pptx_parser.go@@ -22,7 +22,6 @@ import ("strings"deepdocoffice "ragflow/internal/deepdoc/parser/office"- "ragflow/internal/parser/media")// pptxExtract is a test seam for the native office_oxide extraction. It@@ -36,7 +35,8 @@ var pptxExtract = deepdocoffice.OpenAndExtract// ZIP-based OOXML presentations and "ppt" for the legacy binary// OLE format.type PPTXParser struct {- format string+ format string+ textOnly bool// TCADP cloud-parsing configurationParseMethod string@@ -77,8 +77,8 @@ func (p *PPTXParser) ConfigureFromSetup(setup map[string]any) {}}-// ParseWithResult emits one JSON item per slide with the slide's-// plain text.+// ParseWithResult emits one JSON item per slide with the slide's plain text,+// tables and media_ref image items. A text-only parser skips images.func (p *PPTXParser) ParseWithResult(ctx context.Context, filename string, data []byte) ParseResult {// p == nil guard: the struct is embedded by value in PPTParser and// always created via NewPPTXParser or the "ppt"-format constructor in@@ -152,11 +152,17 @@ func (p *PPTXParser) ParseWithResult(ctx context.Context, filename string, dataJSON: MapsToItems(itemsFromPlainText(plainText)),}}- budget := media.NewBudget()- items, err := buildPPTXJSONSections(irJSON, budget)+ var sink *officeMediaSink+ if !p.textOnly {+ sink = newOfficeMediaSink(ctx, officeMediaFormatPPTX)+ }+ items, err := buildPPTXJSONSections(irJSON, sink)if err != nil {return ParseResult{Err: err}}+ if sink != nil && sink.err != nil {+ return ParseResult{Err: sink.err}+ }if err := ctx.Err(); err != nil {return ParseResult{Err: err}}@@ -175,6 +181,6 @@ func (p *PPTXParser) ParseWithResult(ctx context.Context, filename string, datareturn ParseResult{File: map[string]any{"name": filename, "format": effFormat},JSON: MapsToItems(items),- Warnings: budget.Warnings(),+ Warnings: sink.warnings(),}}diff --git a/internal/parser/parser/table_regression_test.go b/internal/parser/parser/table_regression_test.goindex 2d7099c7e0..216d2435b1 100644--- a/internal/parser/parser/table_regression_test.go+++ b/internal/parser/parser/table_regression_test.go@@ -67,9 +67,10 @@ func TestTableRegressionMarkdownFlattenBesideTable(t *testing.T) {}func TestTableRegressionPPTXNestedImageRun(t *testing.T) {- ir := `{"sections":[{"elements":[{"type":"paragraph","content":[{"type":"text","text":"before"},{"type":"image","content":[{"type":"text_box","content":[{"type":"image","data":"aGVsbG8="}]}]},{"type":"text","text":"after"}]}]}]}`+ ir := `{"sections":[{"elements":[{"type":"paragraph","content":[{"type":"text","text":"before"},{"type":"image","content":[{"type":"text_box","content":[{"type":"image","data":"` + officeTestImageB64(t, 23) + `"}]}]},{"type":"text","text":"after"}]}]}]}`- after, err := buildPPTXJSONSections(ir, nil)+ sink, _ := officeMediaTestSink(t, officeMediaFormatPPTX)+ after, err := buildPPTXJSONSections(ir, sink)if err != nil || len(after) != 3 || after[0]["text"] != "before" || after[2]["text"] != "after" {t.Fatalf("text/image order lost: %#v err=%v", after, err)}@@ -77,11 +78,15 @@ func TestTableRegressionPPTXNestedImageRun(t *testing.T) {for _, item := range after {if item["doc_type_kwd"] == "image" {images+++ requireOfficeMediaRef(t, item)}}if images != 1 {t.Error("image inside a nested run was dropped")}+ if sink.err != nil {+ t.Fatalf("sink error: %v", sink.err)+ }}func TestTableRegressionSoMarkTableImageIDs(t *testing.T) {diff --git a/internal/parser/parser/table_structure_test.go b/internal/parser/parser/table_structure_test.goindex 35aee1af44..03dfac66a2 100644--- a/internal/parser/parser/table_structure_test.go+++ b/internal/parser/parser/table_structure_test.go@@ -268,7 +268,7 @@ func TestOfficeTableRetainsDefaultHeaderCount(t *testing.T) {func TestOfficeTablePreservesLiteralCellText(t *testing.T) {el := docxIRElement{Type: "table", Rows: []docxIRRow{{Cells: []docxIRCell{{Content: []docxIRElement{{Type: "paragraph", Content: json.RawMessage(`[{"type":"text","text":" <b>A & B</b>\nsecond line "}]`)}}}}}}}- items := appendOfficeTableSections(nil, el, " A & <B> ", "table-1", newMediaBudgetForTest())+ items := appendOfficeTableSections(nil, el, " A & <B> ", "table-1", nil)if len(items) != 1 {t.Fatalf("items=%#v", items)}diff --git a/internal/parser/parser/xls_parser.go b/internal/parser/parser/xls_parser.goindex 5820b3bce9..46f0421457 100644--- a/internal/parser/parser/xls_parser.go+++ b/internal/parser/parser/xls_parser.go@@ -24,8 +24,6 @@ import ("time"excel "github.com/jloor/go-excel-reader"-- "ragflow/internal/parser/media")// CompoundDocumentSignature is the OLE compound-document magic@@ -35,6 +33,7 @@ import (var CompoundDocumentSignature = []byte{0xd0, 0xcf, 0x11, 0xe0, 0xa1, 0xb1, 0x1a, 0xe1}type XLSParser struct {+ textOnly boolParseMethod stringTCADPAPIServer stringTCADPAPIKey string@@ -100,11 +99,19 @@ func (p *XLSParser) ParseWithResult(ctx context.Context, filename string, data [err error)if bytes.HasPrefix(data, CompoundDocumentSignature) {+ // BIFF8 extraction reads cell text only; the drawing layer is not+ // parsed, so no image can reach the media cache from this branch.items, sheetsCount, err = parseBIFF8Bytes(ctx, data)} else {- mediaBudget := media.NewBudget()- items, warnings, sheetsCount, err = parseXLSXBytes(data, mediaBudget)- warnings = append(warnings, mediaBudget.Warnings()...)+ var sink *officeMediaSink+ if !p.textOnly {+ sink = newOfficeMediaSink(ctx, officeMediaFormatXLSX)+ }+ items, warnings, sheetsCount, err = parseXLSXBytes(data, sink)+ if err == nil && sink != nil && sink.err != nil {+ return ParseResult{Err: sink.err}+ }+ warnings = append(warnings, sink.warnings()...)}if err := ctx.Err(); err != nil {return ParseResult{Err: err}diff --git a/internal/parser/parser/xls_parser_test.go b/internal/parser/parser/xls_parser_test.goindex 4d8f017cdf..31c0b065b9 100644--- a/internal/parser/parser/xls_parser_test.go+++ b/internal/parser/parser/xls_parser_test.go@@ -302,7 +302,8 @@ func TestXLSParser_ImagesDoNotRunOCR(t *testing.T) {if err != nil {t.Fatalf("NewXLSParser: %v", err)}- result := p.ParseWithResult(t.Context(), "with-image.xls", workbook.Bytes())+ ctx, cache := officeMediaTestContext()+ result := p.ParseWithResult(ctx, "with-image.xls", workbook.Bytes())if result.Err != nil {t.Fatalf("ParseWithResult: %v", result.Err)}@@ -313,8 +314,15 @@ func TestXLSParser_ImagesDoNotRunOCR(t *testing.T) {break}}- if imageItem == nil || imageItem["image"] != "data:image/png;base64,"+base64.StdEncoding.EncodeToString(imageBytes) {- t.Fatalf("image item = %+v, want retained picture payload", imageItem)+ if imageItem == nil {+ t.Fatalf("image item missing: %+v", ItemsToMaps(result.JSON))+ }+ ref := requireOfficeMediaRef(t, imageItem)+ if ref.Format != officeMediaFormatXLSX || ref.MIME != "image/png" {+ t.Fatalf("media_ref = %+v, want xlsx image/png", ref)+ }+ if len(cache.puts) != 1 || string(cache.puts[0].data) != string(imageBytes) {+ t.Fatalf("cache writes = %+v, want the picture bytes", cache.puts)}if len(result.Warnings) != 0 {t.Fatalf("warnings = %v, want none for an admitted image", result.Warnings)@@ -324,10 +332,6 @@ func TestXLSParser_ImagesDoNotRunOCR(t *testing.T) {func TestXLSParser_CapsEmbeddedImageCount(t *testing.T) {const limit = 256t.Setenv("RAGFLOW_MEDIA_MAX_IMAGES_PER_RUN", "256")- var imageData bytes.Buffer- if err := png.Encode(&imageData, image.NewRGBA(image.Rect(0, 0, 2, 2))); err != nil {- t.Fatalf("encode fixture: %v", err)- }f := excelize.NewFile()defer f.Close()for i := 0; i <= limit; i++ {@@ -335,7 +339,7 @@ func TestXLSParser_CapsEmbeddedImageCount(t *testing.T) {if err != nil {t.Fatalf("cell %d: %v", i, err)}- if err := f.AddPictureFromBytes("Sheet1", cell, &excelize.Picture{Extension: ".png", File: imageData.Bytes()}); err != nil {+ if err := f.AddPictureFromBytes("Sheet1", cell, &excelize.Picture{Extension: ".png", File: officeTestPNG(t, i)}); err != nil {t.Fatalf("AddPictureFromBytes(%s): %v", cell, err)}}@@ -353,7 +357,8 @@ func TestXLSParser_CapsEmbeddedImageCount(t *testing.T) {if err != nil {t.Fatalf("NewXLSParser: %v", err)}- result := p.ParseWithResult(t.Context(), "many-images.xls", workbook.Bytes())+ ctx, _ := officeMediaTestContext()+ result := p.ParseWithResult(ctx, "many-images.xls", workbook.Bytes())if result.Err != nil {t.Fatalf("ParseWithResult: %v", result.Err)}@@ -361,6 +366,7 @@ func TestXLSParser_CapsEmbeddedImageCount(t *testing.T) {for _, item := range ItemsToMaps(result.JSON) {if item["doc_type_kwd"] == "image" {imageCount+++ requireOfficeMediaRef(t, item)}}if imageCount != limit {diff --git a/internal/parser/parser/xlsx_media_test.go b/internal/parser/parser/xlsx_media_test.gonew file mode 100644index 0000000000..a66578094c--- /dev/null+++ b/internal/parser/parser/xlsx_media_test.go@@ -0,0 +1,136 @@+//+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.++package parser++import (+ "context"+ "strings"+ "testing"++ "github.com/xuri/excelize/v2"+ "ragflow/internal/utility"+)++func TestXLSXParserFailsWhenMediaCacheIsMissing(t *testing.T) {+ data := newTestXLSX(t, func(f *excelize.File) {+ mustSetCell(t, f, "Sheet1", "A1", "Name")+ if err := f.AddPictureFromBytes("Sheet1", "C3", &excelize.Picture{+ Extension: ".png",+ File: officeTestPNG(t, 15),+ }); err != nil {+ t.Fatalf("AddPictureFromBytes: %v", err)+ }+ })+ p, err := NewXLSXParser("")+ if err != nil {+ t.Fatalf("NewXLSXParser: %v", err)+ }+ res := p.ParseWithResult(context.Background(), "missing.xlsx", data)+ if res.Err == nil || !strings.Contains(res.Err.Error(), "media cache is not configured") {+ t.Fatalf("ParseWithResult error = %v, want missing-cache failure", res.Err)+ }+}++func TestXLSXTextParserSkipsImagesWithoutCache(t *testing.T) {+ data := newTestXLSX(t, func(f *excelize.File) {+ mustSetCell(t, f, "Sheet1", "A1", "Name")+ mustSetCell(t, f, "Sheet1", "A2", "Alice")+ if err := f.AddPictureFromBytes("Sheet1", "C3", &excelize.Picture{+ Extension: ".png",+ File: officeTestPNG(t, 16),+ Format: &excelize.GraphicOptions{AltText: "figure"},+ }); err != nil {+ t.Fatalf("AddPictureFromBytes: %v", err)+ }+ })+ p, err := GetTextOnlyParser(utility.FileTypeXLSX)+ if err != nil {+ t.Fatal(err)+ }+ res := p.ParseWithResult(context.Background(), "text.xlsx", data)+ if res.Err != nil {+ t.Fatalf("ParseWithResult: %v", res.Err)+ }+ if len(res.JSON) != 1 || res.JSON[0].ToMap()["doc_type_kwd"] != "table" {+ t.Fatalf("items = %+v, want the sheet table without the skipped image", ItemsToMaps(res.JSON))+ }+ if !strings.Contains(spreadsheetText(res), "Alice") {+ t.Fatalf("parsed rows = %q, want sheet content", spreadsheetText(res))+ }+ if len(res.Warnings) != 0 {+ t.Fatalf("warnings = %v, want none for a text-only parse", res.Warnings)+ }+}++// TestXLSXParserNormalizationRetryEmitsImage covers a workbook whose second+// sheet needs name normalization: the first attempt parses the first sheet+// (including its image) and then fails; the retry must reuse the admitted image+// reference without loading or writing the same source a second time.+func TestXLSXParserNormalizationRetryEmitsImage(t *testing.T) {+ data := newTestXLSX(t, func(f *excelize.File) {+ mustSetCell(t, f, "Sheet1", "A1", "Name")+ mustSetCell(t, f, "Sheet1", "A2", "Alice")+ if err := f.AddPictureFromBytes("Sheet1", "C3", &excelize.Picture{+ Extension: ".png",+ File: officeTestPNG(t, 17),+ Format: &excelize.GraphicOptions{AltText: "figure"},+ }); err != nil {+ t.Fatalf("AddPictureFromBytes: %v", err)+ }+ if _, err := f.NewSheet("Data"); err != nil {+ t.Fatalf("NewSheet: %v", err)+ }+ })+ data = rewriteXLSXEntry(t, data, "xl/workbook.xml", func(content []byte) []byte {+ updated := strings.Replace(string(content), `name="Data"`, `name="Visible:Data"`, 1)+ if updated == string(content) {+ t.Fatal("workbook.xml did not contain the Data sheet")+ }+ return []byte(updated)+ })++ ctx, cache := officeMediaTestContext()+ p, err := NewXLSXParser("")+ if err != nil {+ t.Fatalf("NewXLSXParser: %v", err)+ }+ res := p.ParseWithResult(ctx, "normalize-image.xlsx", data)+ if res.Err != nil {+ t.Fatalf("ParseWithResult: %v", res.Err)+ }+ var imageItem map[string]any+ for _, item := range ItemsToMaps(res.JSON) {+ if item["doc_type_kwd"] == "image" {+ imageItem = item+ }+ }+ if imageItem == nil {+ t.Fatalf("items = %+v, want the sheet image", ItemsToMaps(res.JSON))+ }+ ref := requireOfficeMediaRef(t, imageItem)+ if ref.Format != officeMediaFormatXLSX || ref.MIME != "image/png" {+ t.Fatalf("media_ref = %+v, want xlsx image/png", ref)+ }+ if imageItem["sheet"] != "Sheet1" || imageItem["cell"] != "C3" {+ t.Fatalf("image identity = %+v, want Sheet1!C3", imageItem)+ }+ if cache.putCalls != 1 {+ t.Fatalf("cache writes = %d, want one across both parse attempts", cache.putCalls)+ }+ if !strings.Contains(strings.Join(res.Warnings, "\n"), "Visible_Data") {+ t.Fatalf("warnings = %v, want sheet-name normalization warning", res.Warnings)+ }+}diff --git a/internal/parser/parser/xlsx_parse_method_test.go b/internal/parser/parser/xlsx_parse_method_test.goindex 3b037d389e..0f715b406b 100644--- a/internal/parser/parser/xlsx_parse_method_test.go+++ b/internal/parser/parser/xlsx_parse_method_test.go@@ -106,7 +106,8 @@ func TestXLSXParser_ExtractsFloatingImages(t *testing.T) {if err != nil {t.Fatalf("NewXLSXParser: %v", err)}- res := p.ParseWithResult(t.Context(), "with-image.xlsx", data)+ ctx, cache := officeMediaTestContext()+ res := p.ParseWithResult(ctx, "with-image.xlsx", data)if res.Err != nil {t.Fatalf("ParseWithResult: %v", res.Err)}@@ -117,12 +118,19 @@ func TestXLSXParser_ExtractsFloatingImages(t *testing.T) {if image["text"] != "sheet image" || image["doc_type_kwd"] != "image" {t.Fatalf("unexpected image item: %#v", image)}- if image["image"] != "data:image/png;base64,"+pngBase64 {- t.Fatalf("image data = %v, want data URL", image["image"])+ ref := requireOfficeMediaRef(t, image)+ if ref.Format != officeMediaFormatXLSX || ref.MIME != "image/png" {+ t.Fatalf("media_ref = %+v, want xlsx image/png", ref)+ }+ if len(cache.puts) != 1 || string(cache.puts[0].data) != string(mustDecodeBase64(t, pngBase64)) {+ t.Fatalf("cache writes = %+v, want the picture bytes", cache.puts)}if image["sheet_index"] != 1 || image["row_start"] != 3 || image["row_end"] != 3 || image["col_start"] != 3 || image["col_end"] != 3 {t.Fatalf("image coordinates = %#v, want Sheet1!C3", image)}+ if image["sheet"] != "Sheet1" || image["cell"] != "C3" {+ t.Fatalf("image identity = %#v, want Sheet1!C3", image)+ }}func TestXLSXParser_ImageWithoutAltTextUsesAnchorCell(t *testing.T) {@@ -141,13 +149,15 @@ func TestXLSXParser_ImageWithoutAltTextUsesAnchorCell(t *testing.T) {if err != nil {t.Fatalf("NewXLSXParser: %v", err)}- res := p.ParseWithResult(t.Context(), "without-alt.xlsx", data)+ ctx, _ := officeMediaTestContext()+ res := p.ParseWithResult(ctx, "without-alt.xlsx", data)if res.Err != nil {t.Fatalf("ParseWithResult: %v", res.Err)}if len(res.JSON) != 1 || res.JSON[0].ToMap()["text"] != "C3" {t.Fatalf("image item = %#v, want anchor-cell text C3", res.JSON)}+ requireOfficeMediaRef(t, res.JSON[0].ToMap())}func mustDecodeBase64(t *testing.T, encoded string) []byte {diff --git a/internal/parser/parser/xlsx_parser.go b/internal/parser/parser/xlsx_parser.goindex ad8061654d..939a9345f5 100644--- a/internal/parser/parser/xlsx_parser.go+++ b/internal/parser/parser/xlsx_parser.go@@ -23,11 +23,10 @@ import ("strings""github.com/xuri/excelize/v2"-- "ragflow/internal/parser/media")type XLSXParser struct {+ textOnly boolParseMethod stringTCADPAPIServer stringTCADPAPIKey string@@ -104,13 +103,19 @@ func (p *XLSXParser) ParseWithResult(ctx context.Context, filename string, data// for spreadsheet processing.}- mediaBudget := media.NewBudget()- items, warnings, sheets, err := parseXLSXBytes(data, mediaBudget)+ var sink *officeMediaSink+ if !p.textOnly {+ sink = newOfficeMediaSink(ctx, officeMediaFormatXLSX)+ }+ items, warnings, sheets, err := parseXLSXBytes(data, sink)if err == nil {+ if sink != nil && sink.err != nil {+ return ParseResult{Err: sink.err}+ }if ctx.Err() != nil {return ParseResult{Err: ctx.Err()}}- warnings = append(warnings, mediaBudget.Warnings()...)+ warnings = append(warnings, sink.warnings()...)return xlsxParseResult(filename, items, warnings, sheets)}if ctx.Err() != nil {@@ -124,22 +129,24 @@ func (p *XLSXParser) ParseWithResult(ctx context.Context, filename string, dataif !changed {return ParseResult{Err: fmt.Errorf("xlsx parse: %w", err)}}- // The first parse attempt is discarded, so its media reservations must not- // affect the normalized retry's output.- mediaBudget = media.NewBudget()- items, warnings, sheets, retryErr := parseXLSXBytes(normalized, mediaBudget)+ // Normalization changes only workbook sheet names. Reuse source memo and+ // admissions: already cached image bytes are identical on the second pass.+ items, warnings, sheets, retryErr := parseXLSXBytes(normalized, sink)if retryErr != nil {return ParseResult{Err: fmt.Errorf("xlsx parse: %w; retry after normalization: %v", err, retryErr)}}+ if sink != nil && sink.err != nil {+ return ParseResult{Err: sink.err}+ }if ctx.Err() != nil {return ParseResult{Err: ctx.Err()}}warnings = append(normalizeWarnings, warnings...)- warnings = append(warnings, mediaBudget.Warnings()...)+ warnings = append(warnings, sink.warnings()...)return xlsxParseResult(filename, items, warnings, sheets)}-func parseXLSXBytes(data []byte, mediaBudget *media.Budget) ([]map[string]any, []string, int, error) {+func parseXLSXBytes(data []byte, sink *officeMediaSink) ([]map[string]any, []string, int, error) {f, err := excelize.OpenReader(bytes.NewReader(data))if err != nil {return nil, nil, 0, fmt.Errorf("open XLSX: %w", err)@@ -155,7 +162,7 @@ func parseXLSXBytes(data []byte, mediaBudget *media.Budget) ([]map[string]any, [return nil, warnings, len(sheets), err}warnings = append(warnings, sheetWarnings...)- images, imageWarnings := extractXLSXImages(f, sheet, mediaBudget)+ images, imageWarnings := extractXLSXImages(f, sheet, sink)for _, image := range images {row, _ := numericItemInt(image["row_start"])col, _ := numericItemInt(image["col_start"])diff --git a/internal/service/document/document_artifact.go b/internal/service/document/document_artifact.goindex b0e506dfed..75a6f02826 100644--- a/internal/service/document/document_artifact.go+++ b/internal/service/document/document_artifact.go@@ -319,7 +319,11 @@ func (s *DocumentService) GetDocumentPreview(ctx context.Context, userID, docID// document via the office_oxide-backed DOCParser. The parser only emits JSON// payloads, so the preview text is flattened from the JSON items.func extractDOCPreviewText(ctx context.Context, filename string, data []byte) (string, error) {- res := parser.NewDOCParser().ParseWithResult(ctx, filename, data)+ p, err := parser.GetTextOnlyParser(utility.FileTypeDOC)+ if err != nil {+ return "", err+ }+ res := p.ParseWithResult(ctx, filename, data)if res.Err != nil {return "", res.Err}diff --git a/internal/service/file/file_content.go b/internal/service/file/file_content.goindex 13453561aa..d53da9f5c2 100644--- a/internal/service/file/file_content.go+++ b/internal/service/file/file_content.go@@ -217,16 +217,10 @@ func parseAgentUploadContent(ctx context.Context, filename string, data []byte,content := string(data)fileType := utility.GetFileType(filename)if fileType != utility.FileTypeOTHER {- fp, err := parser.GetParser(fileType)+ fp, err := parser.GetTextOnlyParser(fileType)if err != nil {return "", err}- if fileType == utility.FileTypeHTML {- fp = parser.NewHTMLTextParser()- }- if fileType == utility.FileTypeEPUB {- fp = parser.NewEPUBTextParser()- }if configurable, ok := fp.(interface{ ConfigureFromSetup(map[string]any) }); ok {configurable.ConfigureFromSetup(map[string]any{"layout_recognize": layoutRecognize})}@@ -275,16 +269,10 @@ func parseFileContent(ctx context.Context, filename string, data []byte) stringif fileType == utility.FileTypeOTHER {return string(data)}- fp, err := parser.GetParser(fileType)+ fp, err := parser.GetTextOnlyParser(fileType)if err != nil {return string(data)}- if fileType == utility.FileTypeHTML {- fp = parser.NewHTMLTextParser()- }- if fileType == utility.FileTypeEPUB {- fp = parser.NewEPUBTextParser()- }res := fp.ParseWithResult(ctx, filename, data)if res.Err != nil {return string(data)diff --git a/internal/service/file/file_content_cgo_test.go b/internal/service/file/file_content_cgo_test.goindex 60a3957823..5687ed36cd 100644--- a/internal/service/file/file_content_cgo_test.go+++ b/internal/service/file/file_content_cgo_test.go@@ -21,6 +21,8 @@ package fileimport ("archive/zip""bytes"+ "os"+ "path/filepath""strings""testing")@@ -79,3 +81,34 @@ func TestParseFileContent_DOCXReadsJSONItems(t *testing.T) {t.Fatal("content fell back to raw docx bytes instead of parsed items")}}++// TestOfficeContentExtractionWithoutMediaCache pins that previews, uploads and+// other non-ingestion callers use the text-only Office parsers: an+// image-bearing .docx must yield readable text instead of failing because no+// run media cache is attached.+func TestOfficeContentExtractionWithoutMediaCache(t *testing.T) {+ data, err := os.ReadFile(filepath.Join("..", "..", "parser", "parser", "testdata", "docx_table_image.docx"))+ if err != nil {+ t.Fatal(err)+ }++ for name, parse := range map[string]func() (string, error){+ "file": func() (string, error) { return parseFileContent(t.Context(), "image.docx", data), nil },+ "agent": func() (string, error) { return parseAgentUploadContent(t.Context(), "image.docx", data, "") },+ } {+ t.Run(name, func(t *testing.T) {+ got, err := parse()+ if err != nil {+ t.Fatal(err)+ }+ for _, want := range []string{"intro", "before"} {+ if !strings.Contains(got, want) {+ t.Errorf("missing %q: %q", want, got)+ }+ }+ if strings.Contains(got, "media_ref") || strings.Contains(got, "base64,") {+ t.Fatalf("media payload leaked into text-only content: %q", got)+ }+ })+ }+}diff --git a/internal/service/ingestion_task_service.go b/internal/service/ingestion_task_service.goindex c427daba90..fcc2f698aa 100644--- a/internal/service/ingestion_task_service.go+++ b/internal/service/ingestion_task_service.go@@ -372,18 +372,20 @@ func (s *IngestionTaskService) PrepareDocumentTaskForCleanup(ctx context.Contextreturn s.supersedeUnstartedTask(ctx, task.ID, terminalMessage)case common.RUNNING, common.STOPPING:return nil, fmt.Errorf("document %s ingestion task %s is %s; stop it and wait for a terminal state before cleanup", documentID, task.ID, task.Status)+ // Terminal cleanup can proceed when its ledger has already been deleted.+ // Queued tasks above must still settle successfully before becoming removable.case common.STOPPED:- if err := s.settleTaskRun(ctx, task, entity.TaskStatusCancel, terminalMessage); err != nil {+ if err := s.settleTaskRun(ctx, task, entity.TaskStatusCancel, terminalMessage); err != nil && !errors.Is(err, gorm.ErrRecordNotFound) {return nil, err}return task, nilcase common.COMPLETED:- if err := s.settleTaskRun(ctx, task, entity.TaskStatusDone, taskTerminalMessageComplete); err != nil {+ if err := s.settleTaskRun(ctx, task, entity.TaskStatusDone, taskTerminalMessageComplete); err != nil && !errors.Is(err, gorm.ErrRecordNotFound) {return nil, err}return task, nilcase common.FAILED:- if err := s.settleTaskRun(ctx, task, entity.TaskStatusFail, taskTerminalMessageFailed); err != nil {+ if err := s.settleTaskRun(ctx, task, entity.TaskStatusFail, taskTerminalMessageFailed); err != nil && !errors.Is(err, gorm.ErrRecordNotFound) {return nil, err}return task, nil@@ -459,10 +461,10 @@ func (s *IngestionTaskService) Remove(ctx context.Context, taskID string, userID// failed, a retry must finish it before the task row is deleted.if task.Status == common.STOPPED && task.PipelineLogID != nil && *task.PipelineLogID != "" {run, runErr := s.pipelineLogDAO.GetByID(ctx, dao.DB, *task.PipelineLogID)- if runErr != nil {+ if runErr != nil && !errors.Is(runErr, gorm.ErrRecordNotFound) {return nil, runErr}- if run.RunCount != nil && *run.RunCount > 0 {+ if run != nil && run.RunCount != nil && *run.RunCount > 0 {if err := s.settleTaskRun(ctx, task, entity.TaskStatusCancel, "Task superseded by a new parse request."); err != nil {return nil, err}diff --git a/internal/service/ingestion_task_service_test.go b/internal/service/ingestion_task_service_test.goindex 0f07d18771..f80485b465 100644--- a/internal/service/ingestion_task_service_test.go+++ b/internal/service/ingestion_task_service_test.go@@ -2413,3 +2413,54 @@ func TestIngestionTaskServiceAdvanceRejectsMissingIdentity(t *testing.T) {t.Fatalf("missing identity created %d pipeline log rows, want 0", got)}}++// A deleted ledger must not strand a terminal task during document cleanup.+// Active/queued tasks still require valid identity; runtime event writes remain strict.+func TestIngestionTaskServiceCleanupMissingRun(t *testing.T) {+ for _, status := range []string{common.STOPPED, common.COMPLETED, common.FAILED, common.CREATED, common.SCHEDULED, common.RUNNING, common.STOPPING} {+ t.Run(status, func(t *testing.T) {+ db := setupServiceTestDB(t)+ pushServiceDB(t, db)+ insertTestIngestionTaskWithStatus(t, "task-1", "user-1", "doc-1", "kb-1", status)+ if err := db.Model(&entity.IngestionTask{}).Where("id = ?", "task-1").Update("pipeline_log_id", "deleted-run").Error; err != nil {+ t.Fatal(err)+ }+ svc := NewIngestionTaskService()+ svc.purgeTaskState = func(context.Context, string) error { return nil }+ terminal := common.IsTerminalTaskStatus(status)+ prepared, err := svc.PrepareDocumentTaskForCleanup(t.Context(), "doc-1", "Document cleanup")+ if terminal {+ if err != nil || prepared == nil || prepared.ID != "task-1" {+ t.Fatalf("terminal cleanup: task=%v err=%v", prepared, err)+ }+ } else if err == nil {+ t.Fatal("nonterminal cleanup accepted missing run identity")+ }+ // Recreate the original state: preparing a queued task may freeze it.+ if err := db.Model(&entity.IngestionTask{}).Where("id = ?", "task-1").Update("status", status).Error; err != nil {+ t.Fatal(err)+ }+ _, err = svc.Remove(t.Context(), "task-1", sptr("user-1"))+ if terminal {+ if err != nil {+ t.Fatalf("Remove terminal task with deleted run: %v", err)+ }+ if _, err := svc.GetTask(t.Context(), "task-1"); !errors.Is(err, common.ErrTaskNotFound) {+ t.Fatalf("task remains after removal: %v", err)+ }+ } else if err == nil {+ t.Fatal("removed nonterminal task with missing run identity")+ }+ if err := svc.RecordTerminal(t.Context(), "deleted-run", "task-1", "runtime write"); !errors.Is(err, gorm.ErrRecordNotFound) {+ t.Fatalf("runtime event error = %v, want missing run failure", err)+ }+ var events int64+ if err := db.Model(&entity.IngestionTaskLog{}).Where("pipeline_log_id = ?", "deleted-run").Count(&events).Error; err != nil {+ t.Fatal(err)+ }+ if events != 0 {+ t.Fatalf("orphan terminal events = %d", events)+ }+ })+ }+}