RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
refactor(parser): consolidate vision figure enhancement across document formats (#18775)
jay77721959c8e8b4ab8
diff --git a/internal/ingestion/component/docx_vision_dispatch.go b/internal/ingestion/component/docx_vision_dispatch.godeleted file mode 100644index 4f7b2622f3..0000000000--- a/internal/ingestion/component/docx_vision_dispatch.go+++ /dev/null@@ -1,282 +0,0 @@-//-// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.-//-// Licensed under the Apache License, Version 2.0 (the "License");-// you may not use this file except in compliance with the License.-// You may obtain a copy of the License at-//-// http://www.apache.org/licenses/LICENSE-2.0-//-// Unless required by applicable law or agreed to in writing, software-// distributed under the License is distributed on an "AS IS" BASIS,-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.-// See the License for the specific language governing permissions and-// limitations under the License.-//--// DOCX vision figure dispatch: enriches the parse result with-// LLM-generated descriptions of embedded images, mirroring-// Python's enhance_media_sections_with_vision in-// rag/flow/parser/utils.py (invoked from parser.py:_doc's JSON branch).-//-// Unlike the PDF vision path (which replaces dispatchParse entirely),-// DOCX vision is a post-processing step. It mirrors Python exactly:-// vision enrichment happens ONLY on the JSON output path, where each-// item carries a doc_type_kwd and an optional image. The Markdown path-// performs no vision enrichment in Python, so it must not be here either.--package component--import (- "context"- "fmt"- "os"- "path/filepath"- "strings"- "sync"-- pdflayout "ragflow/internal/deepdoc/parser/pdf/layout"- "ragflow/internal/entity"- modelModule "ragflow/internal/entity/models"- "ragflow/internal/ingestion/component/schema"- "ragflow/internal/utility"-- "gorm.io/gorm"-)--var (- figureVisionPromptBuilder = buildFigureVisionPrompt- visionChatInvoker = defaultVisionChatInvoker- docxVisionConcurrency uint = 10-)--const (- figureVisionPromptFile = "vision_llm_figure_describe_prompt.md"- figureVisionPromptWithContextFile = "vision_llm_figure_describe_prompt_with_context.md"-)--var (- figureVisionPromptsBase string- figureVisionPromptsOnce sync.Once- figureVisionPromptCache = make(map[string]string)- figureVisionPromptMu sync.RWMutex-)--// maybeDispatchDOCXVision enriches a DOCX parse result with vision-model-// descriptions of embedded images. It mirrors Python's-// enhance_media_sections_with_vision (rag/flow/parser/utils.py:162), which-// runs only in the JSON output branch of parser.py:_doc.-//-// For each JSON item whose doc_type_kwd is "image" or "table" AND that-// carries a non-empty "image" field, the vision model describes the image-// and the description is appended to the item's text (Python:-// item["text"] = f"{text}\n{parsed_text}" if text else parsed_text). Items-// without an image (e.g. DOCX tables) are left untouched, exactly as Python-// skips them via `if item.get("image") is None: continue`.-//-// The Markdown output path receives no vision enrichment — Python's DOCX-// Markdown branch only concatenates text and never calls the vision model.-func maybeDispatchDOCXVision(- ctx context.Context,- db *gorm.DB,- fileType utility.FileType,- dispatched parserDispatchResult,- inputs map[string]any,- setups map[string]schema.ParserSetup,-) (parserDispatchResult, bool, error) {- if fileType != utility.FileTypeDOCX {- return dispatched, false, nil- }- // Python triggers vision enrichment only on the JSON path- // (parser.py:_doc → enhance_media_sections_with_vision).- if dispatched.Err != nil || dispatched.OutputFormat != "json" || len(dispatched.JSON) == 0 {- return dispatched, false, nil- }-- tenantID := getStringOr(inputs, "tenant_id", "")- if tenantID == "" {- return dispatched, false, nil- }- language := resolveVisionLanguage(inputs, "")-- // Resolve the tenant's IMAGE2TEXT model.- driver, modelName, apiConfig, _, err := resolveTenantModelByType(ctx, db, tenantID, entity.ModelTypeImage2Text)- if err != nil {- // Model not available — skip vision enhancement silently,- // matching Python's try/except pass behaviour.- return dispatched, false, nil- }-- // Collect the indices of JSON items that carry an embeddable image.- type target struct {- idx int- }- var targets []target- for i, item := range dispatched.JSON {- kd, _ := item["doc_type_kwd"].(string)- if kd != "image" && kd != "table" {- continue- }- img, _ := item["image"].(string)- if img == "" {- continue- }- targets = append(targets, target{idx: i})- }- if len(targets) == 0 {- return dispatched, false, nil- }-- descriptions := make([]string, len(targets))- var wg sync.WaitGroup- sem := make(chan struct{}, docxVisionConcurrency)-- for slot, tg := range targets {- wg.Add(1)- go func(slot int, itemIdx int) {- defer wg.Done()- sem <- struct{}{}- defer func() { <-sem }()-- img, _ := dispatched.JSON[itemIdx]["image"].(string)- if img == "" {- return- }- // DOCX JSON items have no surrounding context (unlike the- // former Markdown path), so use the bare figure prompt —- // matching Python's VisionFigureParser(context_size=0).- prompt, perr := figureVisionPromptBuilder("", "", language)- if perr != nil {- return- }- messages := buildVisionMessages(prompt, img)- resp, ierr := visionChatInvoker(ctx, driver, modelName, messages, apiConfig)- if ierr != nil {- return- }- descriptions[slot] = extractDOCXVisionAnswer(resp)- }(slot, tg.idx)- }- wg.Wait()-- modified := false- for slot, tg := range targets {- desc := strings.TrimSpace(descriptions[slot])- if desc == "" {- continue- }- existing, _ := dispatched.JSON[tg.idx]["text"].(string)- if existing != "" {- dispatched.JSON[tg.idx]["text"] = existing + "\n" + desc- } else {- dispatched.JSON[tg.idx]["text"] = desc- }- modified = true- }-- return dispatched, modified, nil-}--// buildFigureVisionPrompt loads the figure-describe prompt template-// and, when context text is available, renders it with the-// with-context variant. Mirrors Python:-//-// if context_above or context_below:-// prompt = vision_llm_figure_describe_prompt_with_context(-// context_above, context_below, language=language)-// else:-// prompt = vision_llm_figure_describe_prompt(language=language)-func buildFigureVisionPrompt(contextAbove, contextBelow, language string) (string, error) {- hasContext := strings.TrimSpace(contextAbove) != "" || strings.TrimSpace(contextBelow) != ""-- var templateName string- if hasContext {- templateName = figureVisionPromptWithContextFile- } else {- templateName = figureVisionPromptFile- }-- template, err := loadFigureVisionPromptFile(templateName)- if err != nil {- return "", err- }- template = renderFigureVisionLanguage(template, language)-- if hasContext {- template = strings.ReplaceAll(template, "{{ context_above }}", contextAbove)- template = strings.ReplaceAll(template, "{{ context_below }}", contextBelow)- }- return template, nil-}--func loadFigureVisionPromptFile(filename string) (string, error) {- figureVisionPromptMu.RLock()- if cached, ok := figureVisionPromptCache[filename]; ok {- figureVisionPromptMu.RUnlock()- return cached, nil- }- figureVisionPromptMu.RUnlock()-- baseDir, err := figureVisionPromptsBaseDir()- if err != nil {- return "", err- }- promptPath := filepath.Join(baseDir, "rag", "prompts", filename)- content, err := os.ReadFile(promptPath)- if err != nil {- return "", fmt.Errorf("figure vision prompt %q: %w", filename, err)- }- cached := strings.TrimSpace(string(content))- figureVisionPromptMu.Lock()- figureVisionPromptCache[filename] = cached- figureVisionPromptMu.Unlock()- return cached, nil-}--func figureVisionPromptsBaseDir() (string, error) {- var initErr error- figureVisionPromptsOnce.Do(func() {- root := utility.GetProjectRoot()- if _, statErr := os.Stat(filepath.Join(root, "rag", "prompts")); statErr == nil {- figureVisionPromptsBase = root- return- }- initErr = fmt.Errorf("rag/prompts not found under project root %q", root)- })- if initErr != nil {- return "", initErr- }- return figureVisionPromptsBase, nil-}--func buildVisionMessages(prompt, imageBase64 string) []modelModule.Message {- dataURI := pdflayout.InlinePNGDataURL(imageBase64)- if dataURI == "" {- dataURI = strings.TrimSpace(imageBase64)- }- return []modelModule.Message{{- Role: "user",- Content: []interface{}{- map[string]any{"type": "text", "text": prompt},- map[string]any{"type": "image_url", "image_url": map[string]any{"url": dataURI}},- },- }}-}--func extractDOCXVisionAnswer(resp *modelModule.ChatResponse) string {- if resp == nil || resp.Answer == nil {- return ""- }- return strings.TrimSpace(*resp.Answer)-}--func defaultVisionChatInvoker(- ctx context.Context,- driver modelModule.ModelDriver,- modelName string,- messages []modelModule.Message,- apiConfig *modelModule.APIConfig,-) (*modelModule.ChatResponse, error) {- vision := true- return driver.ChatWithMessages(ctx, modelName, messages, apiConfig, &modelModule.ChatConfig{Vision: &vision}, nil)-}diff --git a/internal/ingestion/component/docx_vision_dispatch_test.go b/internal/ingestion/component/docx_vision_dispatch_test.godeleted file mode 100644index 1c169d3016..0000000000--- a/internal/ingestion/component/docx_vision_dispatch_test.go+++ /dev/null@@ -1,254 +0,0 @@-//-// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.-//-// Licensed under the Apache License, Version 2.0 (the "License");-// you may not use this file except in compliance with the License.-// You may obtain a copy of the License at-//-// http://www.apache.org/licenses/LICENSE-2.0-//-// Unless required by applicable law or agreed to in writing, software-// distributed under the License is distributed on an "AS IS" BASIS,-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.-// See the License for the specific language governing permissions and-// limitations under the License.--package component--import (- "context"- "ragflow/internal/dao"- "sync"- "testing"-- "ragflow/internal/entity"- modelModule "ragflow/internal/entity/models"- "ragflow/internal/utility"-- "gorm.io/gorm"-)--// docxVisionFakeDriver satisfies modelModule.ModelDriver but never reaches the-// network: docxVisionCaptureInvoker intercepts the call before the driver.-type docxVisionFakeDriver struct {- modelModule.ModelDriver-}--// docxVisionCaptureInvoker records the requested image and returns a fixed-// description, mirroring the markdown-vision test's capture driver.-type docxVisionCaptureInvoker struct {- mu sync.Mutex- images []string- captured []modelModule.Message-}--func (c *docxVisionCaptureInvoker) invoke(- ctx context.Context,- driver modelModule.ModelDriver,- modelName string,- messages []modelModule.Message,- apiConfig *modelModule.APIConfig,-) (*modelModule.ChatResponse, error) {- c.mu.Lock()- c.captured = append(c.captured, messages...)- // Pull the data URI out of the second content part.- if parts, ok := messages[0].Content.([]interface{}); ok && len(parts) >= 2 {- if img, ok := parts[1].(map[string]any); ok {- if url, ok := img["image_url"].(map[string]any); ok {- if u, ok := url["url"].(string); ok {- c.images = append(c.images, u)- }- }- }- }- c.mu.Unlock()- ans := "a diagram of a pipeline"- return &modelModule.ChatResponse{Answer: &ans}, nil-}--// TestMaybeDispatchDOCXVision_EnhancesJSONImages verifies Diff 2.4: DOCX vision-// enhancement must trigger on the JSON output path (like Python's-// enhance_media_sections_with_vision in parser.py:_doc) and must NOT trigger on-// the Markdown path. Image items with a non-empty `image` field get their VLM-// description appended to `text`; table items (no image) and text items are-// left untouched.-func TestMaybeDispatchDOCXVision_EnhancesJSONImages(t *testing.T) {- origResolver := resolveTenantModelByType- origInvoker := visionChatInvoker- origPrompt := figureVisionPromptBuilder- defer func() {- resolveTenantModelByType = origResolver- visionChatInvoker = origInvoker- figureVisionPromptBuilder = origPrompt- }()-- resolveTenantModelByType = func(ctx context.Context, db *gorm.DB, tenantID string, modelType entity.ModelType) (modelModule.ModelDriver, string, *modelModule.APIConfig, int, error) {- return &docxVisionFakeDriver{}, "docx-vision-model", &modelModule.APIConfig{}, 0, nil- }- invoker := &docxVisionCaptureInvoker{}- visionChatInvoker = invoker.invoke- var capturedLanguage string- figureVisionPromptBuilder = func(_, _, language string) (string, error) {- capturedLanguage = language- return "describe the figure", nil- }-- dispatched := parserDispatchResult{- OutputFormat: "json",- DocType: "docx",- JSON: []map[string]any{- {"text": "Intro paragraph", "image": nil, "doc_type_kwd": "text"},- {"text": "", "image": "aGVsbG8taW1hZ2U=", "doc_type_kwd": "image"},- {"text": "<table></table>", "image": nil, "doc_type_kwd": "table"},- },- }- ctx := t.Context()-- res, handled, err := maybeDispatchDOCXVision(- ctx,- dao.DB,- utility.FileTypeDOCX,- dispatched,- map[string]any{"tenant_id": "t1", "lang": "Japanese"},- defaultSetups(),- )- if err != nil {- t.Fatalf("maybeDispatchDOCXVision: unexpected error: %v", err)- }- if !handled {- t.Fatal("handled = false, want true (JSON image item should be enhanced)")- }- if len(res.JSON) != 3 {- t.Fatalf("JSON len = %d, want 3", len(res.JSON))- }- if got := res.JSON[0]["text"].(string); got != "Intro paragraph" {- t.Errorf("text item text = %q, want unchanged", got)- }- // image item: VLM description appended to (empty) text.- if got, _ := res.JSON[1]["text"].(string); got != "a diagram of a pipeline" {- t.Errorf("image item text = %q, want appended VLM description", got)- }- if got, _ := res.JSON[2]["text"].(string); got != "<table></table>" {- t.Errorf("table item text = %q, want unchanged (no image)", got)- }- if len(invoker.images) != 1 {- t.Fatalf("vision invoker called %d times, want 1 (only the image item)", len(invoker.images))- }- if capturedLanguage != "Japanese" {- t.Errorf("figure prompt language = %q, want Japanese", capturedLanguage)- }- if want := "data:image/png;base64,aGVsbG8taW1hZ2U="; invoker.images[0] != want {- t.Errorf("vision image data URI = %q, want %q", invoker.images[0], want)- }-}--// TestMaybeDispatchDOCXVision_JSONOnly verifies Diff 2.4: the Markdown output-// path must NOT be enhanced (Python's markdown/docx branch performs no vision-// enrichment). A Markdown result with embedded figures is returned untouched.-func TestMaybeDispatchDOCXVision_JSONOnly(t *testing.T) {- origResolver := resolveTenantModelByType- origInvoker := visionChatInvoker- defer func() {- resolveTenantModelByType = origResolver- visionChatInvoker = origInvoker- }()-- called := false- resolveTenantModelByType = func(ctx context.Context, db *gorm.DB, tenantID string, modelType entity.ModelType) (modelModule.ModelDriver, string, *modelModule.APIConfig, int, error) {- called = true- return &docxVisionFakeDriver{}, "m", &modelModule.APIConfig{}, 0, nil- }- visionChatInvoker = func(ctx context.Context, d modelModule.ModelDriver, m string, msgs []modelModule.Message, c *modelModule.APIConfig) (*modelModule.ChatResponse, error) {- called = true- ans := "x"- return &modelModule.ChatResponse{Answer: &ans}, nil- }-- dispatched := parserDispatchResult{- OutputFormat: "markdown",- DocType: "docx",- Markdown: "",- File: map[string]any{"figures": []map[string]any{{"image": "abc", "marker": "x"}}},- }- ctx := t.Context()-- res, handled, err := maybeDispatchDOCXVision(- ctx,- dao.DB,- utility.FileTypeDOCX,- dispatched,- map[string]any{"tenant_id": "t1"},- defaultSetups(),- )- if err != nil {- t.Fatalf("maybeDispatchDOCXVision: unexpected error: %v", err)- }- if handled {- t.Error("handled = true, want false (markdown path must not be enhanced)")- }- if called {- t.Error("vision model was resolved/invoked on the markdown path")- }- if res.Markdown != "" {- t.Errorf("markdown mutated: %q", res.Markdown)- }-}--func TestBuildVisionMessages_PreventsDoublePrefix(t *testing.T) {- cases := []struct {- name string- input string- wantURL string- }{- {- name: "raw base64 gets png prefix",- input: "aGVsbG8=",- wantURL: "data:image/png;base64,aGVsbG8=",- },- {- name: "already png data uri is preserved without double prefix",- input: "data:image/png;base64,aGVsbG8=",- wantURL: "data:image/png;base64,aGVsbG8=",- },- {- name: "jpeg data uri is preserved as jpeg",- input: "data:image/jpeg;base64,/9j/4AAQ",- wantURL: "data:image/jpeg;base64,/9j/4AAQ",- },- {- name: "https url is preserved",- input: "https://example.com/figure.png",- wantURL: "https://example.com/figure.png",- },- {- name: "whitespace trimmed",- input: " data:image/png;base64,abc ",- wantURL: "data:image/png;base64,abc",- },- }-- for _, tc := range cases {- t.Run(tc.name, func(t *testing.T) {- msgs := buildVisionMessages("describe", tc.input)- if len(msgs) != 1 {- t.Fatalf("len(msgs) = %d, want 1", len(msgs))- }- parts, ok := msgs[0].Content.([]interface{})- if !ok || len(parts) < 2 {- t.Fatalf("Content parts = %+v, want >= 2", msgs[0].Content)- }- img, ok := parts[1].(map[string]any)- if !ok {- t.Fatalf("parts[1] = %+v, want map[string]any", parts[1])- }- imgURL, ok := img["image_url"].(map[string]any)- if !ok {- t.Fatalf("img[image_url] = %+v, want map[string]any", img["image_url"])- }- got, _ := imgURL["url"].(string)- if got != tc.wantURL {- t.Errorf("image_url.url = %q, want %q", got, tc.wantURL)- }- })- }-}diff --git a/internal/ingestion/component/markdown_vision_dispatch.go b/internal/ingestion/component/markdown_vision_dispatch.godeleted file mode 100644index ff0add6b03..0000000000--- a/internal/ingestion/component/markdown_vision_dispatch.go+++ /dev/null@@ -1,159 +0,0 @@-//-// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.-//-// Licensed under the Apache License, Version 2.0 (the "License");-// you may not use this file except in compliance with the License.-// You may obtain a copy of the License at-//-// http://www.apache.org/licenses/LICENSE-2.0-//-// Unless required by applicable law or agreed to in writing, software-// distributed under the License is distributed on an "AS IS" BASIS,-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.-// See the License for the specific language governing permissions and-// limitations under the License.-//--// Markdown vision figure dispatch: enriches parsed Markdown JSON-// items with LLM-generated descriptions of embedded images,-// mirroring Python's enhance_media_sections_with_vision in-// rag/flow/parser/utils.py, called from the _markdown path.-//-// Unlike the DOCX vision path (which processes a separate figures-// array), Markdown vision iterates over the JSON items produced by-// MarkdownParser.ParseWithResult and enhances items whose-// doc_type_kwd == "image" and whose "image" field contains a-// base64-encoded image.--package component--import (- "context"- "strings"- "sync"-- "ragflow/internal/entity"- "ragflow/internal/utility"-- "gorm.io/gorm"-)--var (- markdownVisionConcurrency uint = 10-)--// maybeDispatchMarkdownVision checks whether the Markdown parse result-// contains JSON items with embedded images and, when a vision model is-// available, enriches those items with AI-generated figure descriptions.-//-// Mirrors the Python flow:-//-// 1. _markdown → sections + section_images (parser.py:1005)-// 2. enhance_media_sections_with_vision (parser.py:1054)-//-// The function is called AFTER dispatchParse so the normal parse-// path produces JSON items with doc_type_kwd == "image" and an-// "image" base64 field. It returns (result, handled, error).-func maybeDispatchMarkdownVision(- ctx context.Context,- db *gorm.DB,- fileType utility.FileType,- dispatched parserDispatchResult,- inputs map[string]any,-) (parserDispatchResult, bool, error) {- if fileType != utility.FileTypeMarkdown {- return dispatched, false, nil- }- if dispatched.Err != nil || dispatched.OutputFormat != "json" {- return dispatched, false, nil- }- if len(dispatched.JSON) == 0 {- return dispatched, false, nil- }-- // Collect indices of image items.- type imgItem struct {- idx int- imageB64 string- text string- }- var images []imgItem- for i, item := range dispatched.JSON {- kd, _ := item["doc_type_kwd"].(string)- // Diff 2.5: Python enhances both image and table items- // (parser/utils.py:181 checks {"image","table"}); only items- // carrying an image are sent to the VLM (utils.py:183).- if kd != "image" && kd != "table" {- continue- }- img, _ := item["image"].(string)- if img == "" {- continue- }- text, _ := item["text"].(string)- images = append(images, imgItem{idx: i, imageB64: img, text: text})- }- if len(images) == 0 {- return dispatched, false, nil- }-- tenantID := getStringOr(inputs, "tenant_id", "")- if tenantID == "" {- return dispatched, false, nil- }- language := resolveVisionLanguage(inputs, "")-- // Resolve the tenant's IMAGE2TEXT model.- driver, modelName, apiConfig, _, err := resolveTenantModelByType(ctx, db, tenantID, entity.ModelTypeImage2Text)- if err != nil {- // Model not available — skip vision enhancement silently,- // matching Python's try/except pass behaviour.- return dispatched, false, nil- }-- descriptions := make([]string, len(images))- var wg sync.WaitGroup- sem := make(chan struct{}, markdownVisionConcurrency)-- for i, img := range images {- wg.Add(1)- go func(pos int, item imgItem) {- defer wg.Done()- sem <- struct{}{}- defer func() { <-sem }()-- // Markdown images have no context — use the- // default (no-context) prompt template.- prompt, err := figureVisionPromptBuilder("", "", language)- if err != nil {- return- }-- messages := buildVisionMessages(prompt, item.imageB64)- resp, err := visionChatInvoker(ctx, driver, modelName, messages, apiConfig)- if err != nil {- return- }- descriptions[pos] = extractDOCXVisionAnswer(resp)- }(i, img)- }- wg.Wait()-- // Append vision descriptions to each image item's text field,- // matching Python's `item["text"] = f"{text}\n{parsed_text}"`.- for pos, img := range images {- desc := strings.TrimSpace(descriptions[pos])- if desc == "" {- continue- }- item := dispatched.JSON[img.idx]- existing, _ := item["text"].(string)- if existing != "" {- item["text"] = existing + "\n\n" + desc- } else {- item["text"] = desc- }- }-- return dispatched, true, nil-}diff --git a/internal/ingestion/component/media_dispatch_test.go b/internal/ingestion/component/media_dispatch_test.goindex ee98c2bf73..b86cdb55a9 100644--- a/internal/ingestion/component/media_dispatch_test.go+++ b/internal/ingestion/component/media_dispatch_test.go@@ -401,60 +401,6 @@ func TestMaybeDispatchAudio_DefaultOutputFormatJson(t *testing.T) {}}-// TestMaybeDispatchMarkdownVision_EnhancesTables pins diff 2.5: Markdown-// vision enhancement must also process items whose doc_type_kwd is "table"-// (Python checks {"image","table"} in parser/utils.py:181), not only "image".-// Before the fix the table item was skipped and never sent to the VLM.-func TestMaybeDispatchMarkdownVision_EnhancesTables(t *testing.T) {- origResolver := resolveTenantModelByType- origPrompt := figureVisionPromptBuilder- defer func() {- resolveTenantModelByType = origResolver- figureVisionPromptBuilder = origPrompt- }()-- drv := &imagePromptCaptureDriver{}- resolveTenantModelByType = func(ctx context.Context, db *gorm.DB, tenantID string, modelType entity.ModelType) (modelModule.ModelDriver, string, *modelModule.APIConfig, int, error) {- return drv, "img-model", &modelModule.APIConfig{}, 0, nil- }- figureVisionPromptBuilder = func(_, _, language string) (string, error) {- return "describe in " + language, nil- }-- dispatched := parserDispatchResult{- OutputFormat: "json",- JSON: []map[string]any{- {"doc_type_kwd": "table", "image": "base64table", "text": ""},- },- }-- ctx := t.Context()- res, handled, err := maybeDispatchMarkdownVision(- ctx,- dao.DB,- utility.FileTypeMarkdown,- dispatched,- map[string]any{"tenant_id": "t1", "lang": "Korean"},- )- if err != nil {- t.Fatalf("maybeDispatchMarkdownVision: %v", err)- }- if !handled {- t.Fatalf("expected handled=true for markdown with a table image")- }- if len(res.JSON) != 1 {- t.Fatalf("JSON len = %d, want 1", len(res.JSON))- }- // The table item must have been sent to the VLM and its description appended.- if got, _ := res.JSON[0]["text"].(string); got != "captured" {- t.Fatalf("table item text = %q, want %q (table items must be vision-enhanced)", got, "captured")- }- gotPrompt, ok := firstUserText(drv.captured)- if !ok || gotPrompt != "describe in Korean" {- t.Fatalf("VLM user text = %q, want dataset language propagated", gotPrompt)- }-}-// TestDefaultEmailOutputFormatIsJSON pins diff 2.2: the email family default// output_format must be "json" (matching Python parser.py:212), not "text".// With "text" the structured email fields (from/to/subject/attachments/...) arediff --git a/internal/ingestion/component/parser.go b/internal/ingestion/component/parser.goindex 19c74443f0..836a8d9d9c 100644--- a/internal/ingestion/component/parser.go+++ b/internal/ingestion/component/parser.go@@ -482,24 +482,14 @@ func (c *ParserComponent) Invoke(ctx context.Context, db *gorm.DB, inputs map[stdispatched = dispatchParse(ctx, fileTypeExt, filename, binary, c.Setups)dispatched = hydrateEmptyDispatchPayload(dispatched, binary)- // DOCX vision figure enhancement: on the JSON output path,- // append vision-model descriptions to embedded image items- // (doc_type_kwd "image"). Mirrors Python's- // enhance_media_sections_with_vision in parser.py:_doc.- dispatched, _, _ = maybeDispatchDOCXVision(ctx, db, fileTypeExt, dispatched, inputs, c.Setups)-- // Markdown vision figure enhancement: enrich parsed- // Markdown JSON items with LLM-generated descriptions of- // referenced images (). Mirrors Python's- // enhance_media_sections_with_vision in _Markdown.- dispatched, _, _ = maybeDispatchMarkdownVision(ctx, db, fileTypeExt, dispatched, inputs)-- // PDF vision figure enhancement: enrich parsed PDF JSON- // items with vision-model descriptions of embedded- // images/tables (doc_type_kwd "image"/"table" with non-empty- // image field). Mirrors Python's enhance_media_sections_with_vision- // in parser.py:_pdf- dispatched, _, _ = maybeDispatchPDFVisionEnhancement(ctx, db, fileTypeExt, dispatched, inputs)+ // Vision figure enhancement: on the JSON output path,+ // append vision-model descriptions to embedded image and+ // table items. Mirrors Python's enhance_media_sections_with_vision+ // (rag/flow/parser/utils.py:162, called at parser.py:772/978/1115).+ // Errors (including context cancellation) are intentionally+ // discarded — enhancement is best-effort, matching Python's+ // try/except pass pattern.+ dispatched, _, _ = maybeDispatchVisionEnhancement(ctx, db, fileTypeExt, dispatched, inputs)}// Known/supported families must fail loudly when dispatch or// parsing breaks. Only unknown families keep the raw-text fallback.diff --git a/internal/ingestion/component/pdf_vision_dispatch.go b/internal/ingestion/component/pdf_vision_dispatch.goindex 43f5d074e2..9eb7fc7d74 100644--- a/internal/ingestion/component/pdf_vision_dispatch.go+++ b/internal/ingestion/component/pdf_vision_dispatch.go@@ -602,7 +602,7 @@ func dispatchPDFVision(if err != nil {return parserDispatchResult{}, fmt.Errorf("parser: pdf vision page %d: %w", page.PageNumber, err)}- text := extractPDFVisionAnswer(resp)+ text := extractVisionAnswer(resp)positions := [][]any{{page.PageNumber, 0.0, page.WidthPts, 0.0, page.HeightPts}}items = append(items, map[string]any{"text": text,@@ -654,13 +654,6 @@ func buildPDFVisionMessages(prompt string, imageURL string) []modelModule.Messag}}}-func extractPDFVisionAnswer(resp *modelModule.ChatResponse) string {- if resp == nil || resp.Answer == nil {- return ""- }- return strings.TrimSpace(*resp.Answer)-}-func defaultPDFVisionModelResolver(ctx context.Context,db *gorm.DB,@@ -729,7 +722,7 @@ func pdfVisionPromptsBaseDir() (string, error) {// renderPDFVisionPrompt only renders page metadata. The full-page PDF vision// prompt is a transcription contract that preserves the document's original// language; dataset-language instructions apply to figure descriptions in-// maybeDispatchPDFVisionEnhancement instead.+// maybeDispatchVisionEnhancement instead.func renderPDFVisionPrompt(template string, page int) string {rendered := strings.ReplaceAll(template, "{{ page }}", fmt.Sprintf("%d", page))rendered = strings.ReplaceAll(rendered, "{{page}}", fmt.Sprintf("%d", page))@@ -779,97 +772,3 @@ func defaultIsPaddleOCRLayoutModelID(ctx context.Context, db *gorm.DB, tenantID,}return isPaddleOCRDriver(driver)}--// maybeDispatchPDFVisionEnhancement mirrors Python's-// enhance_media_sections_with_vision for PDF-// After the normal PDF parser produces JSON items, this function-// enriches image/table items by calling the tenant's IMAGE2TEXT-// model and appending vision descriptions to each item's text field.-// The markdown/text output paths are not enhanced (Python does the same).-//-// This follows the exact same convention as maybeDispatchDOCXVision and-// maybeDispatchMarkdownVision: it is not gated by a separate setup flag,-// it simply resolves the tenant's IMAGE2TEXT model and skips silently-// when none is configured — matching Python's try/except pass behaviour.-func maybeDispatchPDFVisionEnhancement(- ctx context.Context,- db *gorm.DB,- fileType utility.FileType,- dispatched parserDispatchResult,- inputs map[string]any,-) (parserDispatchResult, bool, error) {- if fileType != utility.FileTypePDF {- return dispatched, false, nil- }- if dispatched.Err != nil || dispatched.OutputFormat != "json" || len(dispatched.JSON) == 0 {- return dispatched, false, nil- }- tenantID := getStringOr(inputs, "tenant_id", "")- if tenantID == "" {- return dispatched, false, nil- }- language := resolveVisionLanguage(inputs, "")- driver, modelName, apiConfig, _, err := resolveTenantModelByType(ctx, db, tenantID, entity.ModelTypeImage2Text)- if err != nil {- return dispatched, false, nil- }- type target struct{ idx int }- var targets []target- for i, item := range dispatched.JSON {- kd, _ := item["doc_type_kwd"].(string)- if kd != "image" && kd != "table" {- continue- }- img, _ := item["image"].(string)- if img == "" {- continue- }- targets = append(targets, target{idx: i})- }- if len(targets) == 0 {- return dispatched, false, nil- }- descriptions := make([]string, len(targets))- var wg sync.WaitGroup- sem := make(chan struct{}, pdfVisionEnhanceConcurrency)- for slot, tg := range targets {- sem <- struct{}{}- wg.Add(1)- go func(slot int, itemIdx int) {- defer wg.Done()- defer func() { <-sem }()- img, _ := dispatched.JSON[itemIdx]["image"].(string)- if img == "" {- return- }- prompt, perr := figureVisionPromptBuilder("", "", language)- if perr != nil {- return- }- messages := buildVisionMessages(prompt, img)- resp, ierr := visionChatInvoker(ctx, driver, modelName, messages, apiConfig)- if ierr != nil {- return- }- descriptions[slot] = extractDOCXVisionAnswer(resp)- }(slot, tg.idx)- }- wg.Wait()- modified := false- for slot, tg := range targets {- desc := strings.TrimSpace(descriptions[slot])- if desc == "" {- continue- }- existing, _ := dispatched.JSON[tg.idx]["text"].(string)- if existing != "" {- dispatched.JSON[tg.idx]["text"] = existing + "\n" + desc- } else {- dispatched.JSON[tg.idx]["text"] = desc- }- modified = true- }- return dispatched, modified, nil-}--var pdfVisionEnhanceConcurrency = 10diff --git a/internal/ingestion/component/pdf_vision_dispatch_test.go b/internal/ingestion/component/pdf_vision_dispatch_test.goindex 89df090cd7..9008db90ee 100644--- a/internal/ingestion/component/pdf_vision_dispatch_test.go+++ b/internal/ingestion/component/pdf_vision_dispatch_test.go@@ -21,10 +21,8 @@ import ("ragflow/internal/common""ragflow/internal/dao"- "ragflow/internal/entity"modelModule "ragflow/internal/entity/models""ragflow/internal/ingestion/component/schema"- "ragflow/internal/utility""gorm.io/gorm")@@ -108,62 +106,6 @@ func TestDispatchPaddleOCRPdfEmptyTextFails(t *testing.T) {}}-func TestMaybeDispatchPDFVisionEnhancementForwardsDatasetLanguage(t *testing.T) {- origResolver := resolveTenantModelByType- origInvoker := visionChatInvoker- origPrompt := figureVisionPromptBuilder- t.Cleanup(func() {- resolveTenantModelByType = origResolver- visionChatInvoker = origInvoker- figureVisionPromptBuilder = origPrompt- })-- resolveTenantModelByType = func(context.Context, *gorm.DB, string, entity.ModelType) (modelModule.ModelDriver, string, *modelModule.APIConfig, int, error) {- return &docxVisionFakeDriver{}, "pdf-vision-model", &modelModule.APIConfig{}, 0, nil- }- invoker := &docxVisionCaptureInvoker{}- visionChatInvoker = invoker.invoke- capturedLanguage := ""- figureVisionPromptBuilder = func(_, _, language string) (string, error) {- capturedLanguage = language- return "describe the figure", nil- }-- dispatched := parserDispatchResult{- OutputFormat: "json",- DocType: "pdf",- JSON: []map[string]any{- {"text": "caption", "image": "data:image/png;base64,aW1hZ2U=", "doc_type_kwd": "image"},- },- }-- res, modified, err := maybeDispatchPDFVisionEnhancement(- t.Context(),- dao.DB,- utility.FileTypePDF,- dispatched,- map[string]any{"tenant_id": "t1", "lang": "Dutch"},- )- if err != nil {- t.Fatalf("maybeDispatchPDFVisionEnhancement: %v", err)- }- if !modified {- t.Fatal("modified = false, want true")- }- if capturedLanguage != "Dutch" {- t.Fatalf("figure prompt language = %q, want Dutch", capturedLanguage)- }- if got := res.JSON[0]["text"]; got != "caption\na diagram of a pipeline" {- t.Fatalf("enhanced text = %q", got)- }- if len(invoker.images) != 1 {- t.Fatalf("vision invoker called %d times, want 1", len(invoker.images))- }- if want := "data:image/png;base64,aW1hZ2U="; invoker.images[0] != want {- t.Fatalf("vision image data URI = %q, want %q (no double data: prefix)", invoker.images[0], want)- }-}-// TestIsNamedPDFParseMethodWhitelistAligned verifies that the runtime// "named parse_method" classifier agrees with (*ParserComponent).Check()'s// PDF whitelist (parser.go:200-203):diff --git a/internal/ingestion/component/vision_enhancement.go b/internal/ingestion/component/vision_enhancement.gonew file mode 100644index 0000000000..f7b5167d8b--- /dev/null+++ b/internal/ingestion/component/vision_enhancement.go@@ -0,0 +1,288 @@+//+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.+//++// Package component — vision figure enhancement: enriches parsed JSON items+// (PDF / DOCX / Markdown, OutputFormat=="json") with vision-model descriptions+// of embedded images/tables. Mirrors Python's enhance_media_sections_with_vision+// (rag/flow/parser/utils.py:162, called at parser.py:772/978/1115).++package component++import (+ "context"+ "fmt"+ "os"+ "path/filepath"+ "regexp"+ "strings"+ "sync"++ pdflayout "ragflow/internal/deepdoc/parser/pdf/layout"+ "ragflow/internal/entity"+ modelModule "ragflow/internal/entity/models"+ "ragflow/internal/utility"++ "gorm.io/gorm"+)++var (+ figureVisionPromptBuilder func(language string) (string, error) = buildFigureVisionPrompt+ visionChatInvoker = defaultVisionChatInvoker+)++const (+ figureVisionPromptFile = "vision_llm_figure_describe_prompt.md"+ visionEnhancementConcurrency int = 10+)++var (+ figureVisionPromptsBase string+ figureVisionPromptsOnce sync.Once+ figureVisionPromptCache = make(map[string]string)+ figureVisionPromptMu sync.RWMutex+)++// cleanMarkdownBlock mirrors Python common/string_utils.py:clean_markdown_block+//+// re.sub(r"^\s*```markdown\s*\n?", "", text)+// re.sub(r"\n?\s*```\s*$", "", text)+//+// Matches Python without re.MULTILINE so ^/$ anchor only the whole text.+var (+ reMarkdownOpen = regexp.MustCompile(`^\s*` + "```" + `markdown\s*\n?`)+ reMarkdownClose = regexp.MustCompile(`\n?\s*` + "```" + `\s*$`)+)++func cleanMarkdownBlock(s string) string {+ s = reMarkdownOpen.ReplaceAllString(s, "")+ s = reMarkdownClose.ReplaceAllString(s, "")+ return strings.TrimSpace(s)+}++// isVisionEnhancementAllowed mirrors Python's 3 call sites in+// rag/flow/parser/parser.py:772/978/1115 (PDF/DOCX/Markdown JSON branches).+func isVisionEnhancementAllowed(fileType utility.FileType) bool {+ switch fileType {+ case utility.FileTypePDF, utility.FileTypeDOCX, utility.FileTypeMarkdown:+ return true+ default:+ return false+ }+}++// maybeDispatchVisionEnhancement enriches parsed JSON items with vision-model+// descriptions of embedded images and tables (doc_type_kwd in {"image", "table"}+// with non-empty image field).+// Mirrors Python's enhance_media_sections_with_vision in rag/flow/parser/utils.py:162.+func maybeDispatchVisionEnhancement(+ ctx context.Context,+ db *gorm.DB,+ fileType utility.FileType,+ dispatched parserDispatchResult,+ inputs map[string]any,+) (parserDispatchResult, bool, error) {+ // 0. FileType allowlist guard.+ if !isVisionEnhancementAllowed(fileType) {+ return dispatched, false, nil+ }+ // Only enhance successful JSON output format containing items.+ if dispatched.Err != nil || dispatched.OutputFormat != "json" || len(dispatched.JSON) == 0 {+ return dispatched, false, nil+ }++ tenantID := getStringOr(inputs, "tenant_id", "")+ if tenantID == "" {+ return dispatched, false, nil+ }+ language := resolveVisionLanguage(inputs, "")++ // 1. Collect target items (images/tables with non-empty image data).+ type target struct {+ idx int+ }+ var targets []target+ for i, item := range dispatched.JSON {+ kd, _ := item["doc_type_kwd"].(string)+ if kd != "image" && kd != "table" {+ continue+ }+ img, _ := item["image"].(string)+ if img == "" {+ continue+ }+ targets = append(targets, target{idx: i})+ }+ if len(targets) == 0 {+ return dispatched, false, nil+ }++ // 2. Resolve tenant's IMAGE2TEXT model.+ driver, modelName, apiConfig, _, err := resolveTenantModelByType(ctx, db, tenantID, entity.ModelTypeImage2Text)+ if err != nil {+ // Model not available — skip vision enhancement silently, matching Python's try/except pass.+ return dispatched, false, nil+ }++ // Hoist prompt once: language is invariant across items.+ prompt, perr := figureVisionPromptBuilder(language)+ if perr != nil {+ //nolint:nilerr // Vision enhancement is best-effort.+ return dispatched, false, nil+ }++ // 3. Concurrently invoke VLM — acquire semaphore before launching goroutine+ // so live goroutine count is bounded by visionEnhancementConcurrency.+ // Stop scheduling new VLM calls after cancellation; already-running calls+ // finish via wg.Wait() and cancellation propagates via ctx through+ // visionChatInvoker.+ descriptions := make([]string, len(targets))+ var wg sync.WaitGroup+ sem := make(chan struct{}, visionEnhancementConcurrency)++dispatch:+ for slot, tg := range targets {+ if ctx.Err() != nil {+ break dispatch+ }+ select {+ case sem <- struct{}{}:+ case <-ctx.Done():+ break dispatch+ }+ wg.Add(1)+ go func(slot int, itemIdx int) {+ defer wg.Done()+ defer func() { <-sem }()++ img, _ := dispatched.JSON[itemIdx]["image"].(string)+ if img == "" {+ return+ }+ messages := buildVisionMessages(prompt, img)+ resp, ierr := visionChatInvoker(ctx, driver, modelName, messages, apiConfig)+ if ierr != nil {+ return+ }+ descriptions[slot] = extractVisionAnswer(resp)+ }(slot, tg.idx)+ }+ wg.Wait()+ if err := ctx.Err(); err != nil {+ return dispatched, false, err+ }++ // 4. Append descriptions to item text (single newline \n, matching Python).+ modified := false+ for slot, tg := range targets {+ desc := strings.TrimSpace(descriptions[slot])+ if desc == "" {+ continue+ }+ existing, _ := dispatched.JSON[tg.idx]["text"].(string)+ if existing != "" {+ dispatched.JSON[tg.idx]["text"] = existing + "\n" + desc+ } else {+ dispatched.JSON[tg.idx]["text"] = desc+ }+ modified = true+ }++ return dispatched, modified, nil+}++func buildFigureVisionPrompt(language string) (string, error) {+ template, err := loadFigureVisionPromptFile(figureVisionPromptFile)+ if err != nil {+ return "", err+ }+ return renderFigureVisionLanguage(template, language), nil+}++func loadFigureVisionPromptFile(filename string) (string, error) {+ figureVisionPromptMu.RLock()+ if cached, ok := figureVisionPromptCache[filename]; ok {+ figureVisionPromptMu.RUnlock()+ return cached, nil+ }+ figureVisionPromptMu.RUnlock()++ baseDir, err := figureVisionPromptsBaseDir()+ if err != nil {+ return "", err+ }+ promptPath := filepath.Join(baseDir, "rag", "prompts", filename)+ content, err := os.ReadFile(promptPath)+ if err != nil {+ return "", fmt.Errorf("figure vision prompt %q: %w", filename, err)+ }+ cached := strings.TrimSpace(string(content))+ figureVisionPromptMu.Lock()+ figureVisionPromptCache[filename] = cached+ figureVisionPromptMu.Unlock()+ return cached, nil+}++func figureVisionPromptsBaseDir() (string, error) {+ var initErr error+ figureVisionPromptsOnce.Do(func() {+ root := utility.GetProjectRoot()+ if _, statErr := os.Stat(filepath.Join(root, "rag", "prompts")); statErr == nil {+ figureVisionPromptsBase = root+ return+ }+ initErr = fmt.Errorf("rag/prompts not found under project root %q", root)+ })+ if initErr != nil {+ return "", initErr+ }+ return figureVisionPromptsBase, nil+}++func buildVisionMessages(prompt, imageBase64 string) []modelModule.Message {+ dataURI := pdflayout.InlinePNGDataURL(imageBase64)+ if dataURI == "" {+ dataURI = strings.TrimSpace(imageBase64)+ }+ return []modelModule.Message{{+ Role: "user",+ Content: []interface{}{+ map[string]any{"type": "text", "text": prompt},+ map[string]any{"type": "image_url", "image_url": map[string]any{"url": dataURI}},+ },+ }}+}++// extractVisionAnswer strips ```markdown fences from the VLM response.+// Python calls clean_markdown_block inside vision_llm_chunk (rag/app/picture.py:198,+// i.e., inside the invoker wrapper). Go applies it here post-invoker so that+// visionChatInvoker stays a pure, swappable test seam — output is equivalent.+func extractVisionAnswer(resp *modelModule.ChatResponse) string {+ if resp == nil || resp.Answer == nil {+ return ""+ }+ return cleanMarkdownBlock(*resp.Answer)+}++func defaultVisionChatInvoker(+ ctx context.Context,+ driver modelModule.ModelDriver,+ modelName string,+ messages []modelModule.Message,+ apiConfig *modelModule.APIConfig,+) (*modelModule.ChatResponse, error) {+ vision := true+ return driver.ChatWithMessages(ctx, modelName, messages, apiConfig, &modelModule.ChatConfig{Vision: &vision}, nil)+}diff --git a/internal/ingestion/component/vision_enhancement_test.go b/internal/ingestion/component/vision_enhancement_test.gonew file mode 100644index 0000000000..aa55008eb1--- /dev/null+++ b/internal/ingestion/component/vision_enhancement_test.go@@ -0,0 +1,711 @@+//+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.+//++package component++import (+ "context"+ "errors"+ "fmt"+ "sync"+ "testing"++ "ragflow/internal/dao"+ "ragflow/internal/entity"+ modelModule "ragflow/internal/entity/models"+ "ragflow/internal/utility"++ "gorm.io/gorm"+)++type visionEnhanceFakeDriver struct {+ modelModule.ModelDriver+}++type visionEnhanceCaptureInvoker struct {+ mu sync.Mutex+ images []string+ captured []modelModule.Message+}++func (c *visionEnhanceCaptureInvoker) invoke(+ ctx context.Context,+ driver modelModule.ModelDriver,+ modelName string,+ messages []modelModule.Message,+ apiConfig *modelModule.APIConfig,+) (*modelModule.ChatResponse, error) {+ c.mu.Lock()+ c.captured = append(c.captured, messages...)+ if parts, ok := messages[0].Content.([]interface{}); ok && len(parts) >= 2 {+ if img, ok := parts[1].(map[string]any); ok {+ if url, ok := img["image_url"].(map[string]any); ok {+ if u, ok := url["url"].(string); ok {+ c.images = append(c.images, u)+ }+ }+ }+ }+ c.mu.Unlock()+ ans := "```markdown\na diagram of a pipeline\n```"+ return &modelModule.ChatResponse{Answer: &ans}, nil+}++// swapVisionGlobals replaces injectable vars and restores them on t.Cleanup.+func swapVisionGlobals(+ t *testing.T,+ resolver func(context.Context, *gorm.DB, string, entity.ModelType) (modelModule.ModelDriver, string, *modelModule.APIConfig, int, error),+ invoker func(context.Context, modelModule.ModelDriver, string, []modelModule.Message, *modelModule.APIConfig) (*modelModule.ChatResponse, error),+ prompt func(string) (string, error),+) {+ t.Helper()+ origResolver := resolveTenantModelByType+ origInvoker := visionChatInvoker+ origPrompt := figureVisionPromptBuilder+ t.Cleanup(func() {+ resolveTenantModelByType = origResolver+ visionChatInvoker = origInvoker+ figureVisionPromptBuilder = origPrompt+ })+ if resolver != nil {+ resolveTenantModelByType = resolver+ }+ if invoker != nil {+ visionChatInvoker = invoker+ }+ if prompt != nil {+ figureVisionPromptBuilder = prompt+ }+}++func fakeResolver(_ context.Context, _ *gorm.DB, _ string, _ entity.ModelType) (modelModule.ModelDriver, string, *modelModule.APIConfig, int, error) {+ return &visionEnhanceFakeDriver{}, "vision-model", &modelModule.APIConfig{}, 0, nil+}++func fakePrompt(language string) (string, error) {+ return "describe the figure in " + language, nil+}++func TestVisionEnhancement_EnhancesJSONImagesAndTables(t *testing.T) {+ testCases := []struct {+ name string+ fileType utility.FileType+ }{+ {"DOCX", utility.FileTypeDOCX},+ {"PDF", utility.FileTypePDF},+ {"Markdown", utility.FileTypeMarkdown},+ }++ for _, tc := range testCases {+ t.Run(tc.name, func(t *testing.T) {+ var capturedLanguage string+ // Per-subtest invoker and prompt — avoids implicit coupling between subtests.+ invoker := &visionEnhanceCaptureInvoker{}+ swapVisionGlobals(t, fakeResolver, invoker.invoke, func(language string) (string, error) {+ capturedLanguage = language+ return "describe the figure in " + language, nil+ })++ dispatched := parserDispatchResult{+ OutputFormat: "json",+ DocType: string(tc.fileType),+ JSON: []map[string]any{+ {"text": "Intro paragraph", "image": nil, "doc_type_kwd": "text"},+ {"text": "", "image": "aGVsbG8taW1hZ2U=", "doc_type_kwd": "image"},+ {"text": "existing table", "image": "dGFibGUtaW1hZ2U=", "doc_type_kwd": "table"},+ {"text": "<table></table>", "image": nil, "doc_type_kwd": "table"},+ },+ }++ res, handled, err := maybeDispatchVisionEnhancement(+ t.Context(),+ dao.DB,+ tc.fileType,+ dispatched,+ map[string]any{"tenant_id": "t1", "lang": "Japanese"},+ )+ if err != nil {+ t.Fatalf("unexpected error: %v", err)+ }+ if !handled {+ t.Fatal("handled = false, want true")+ }+ if len(res.JSON) != 4 {+ t.Fatalf("JSON len = %d, want 4", len(res.JSON))+ }+ if got := res.JSON[0]["text"].(string); got != "Intro paragraph" {+ t.Errorf("text item text = %q, want unchanged", got)+ }+ // image item: VLM description cleaned of ```markdown and appended+ if got, _ := res.JSON[1]["text"].(string); got != "a diagram of a pipeline" {+ t.Errorf("image item text = %q, want 'a diagram of a pipeline'", got)+ }+ // table item with image: VLM description appended with single \n+ if got, _ := res.JSON[2]["text"].(string); got != "existing table\na diagram of a pipeline" {+ t.Errorf("table item text = %q, want 'existing table\\na diagram of a pipeline'", got)+ }+ if got, _ := res.JSON[3]["text"].(string); got != "<table></table>" {+ t.Errorf("table item without image text = %q, want unchanged", got)+ }+ if capturedLanguage != "Japanese" {+ t.Errorf("figure prompt language = %q, want Japanese", capturedLanguage)+ }+ })+ }+}++func TestVisionEnhancement_MarkdownOutputUntouched(t *testing.T) {+ called := false+ swapVisionGlobals(t,+ func(_ context.Context, _ *gorm.DB, _ string, _ entity.ModelType) (modelModule.ModelDriver, string, *modelModule.APIConfig, int, error) {+ called = true+ return &visionEnhanceFakeDriver{}, "m", &modelModule.APIConfig{}, 0, nil+ },+ func(_ context.Context, _ modelModule.ModelDriver, _ string, _ []modelModule.Message, _ *modelModule.APIConfig) (*modelModule.ChatResponse, error) {+ called = true+ ans := "x"+ return &modelModule.ChatResponse{Answer: &ans}, nil+ },+ nil,+ )++ dispatched := parserDispatchResult{+ OutputFormat: "markdown",+ DocType: "docx",+ Markdown: "",+ File: map[string]any{"figures": []map[string]any{{"image": "abc", "marker": "x"}}},+ }++ res, handled, err := maybeDispatchVisionEnhancement(+ t.Context(),+ dao.DB,+ utility.FileTypeDOCX,+ dispatched,+ map[string]any{"tenant_id": "t1"},+ )+ if err != nil {+ t.Fatalf("unexpected error: %v", err)+ }+ if handled {+ t.Error("handled = true, want false (markdown path must not be enhanced)")+ }+ if called {+ t.Error("vision model was resolved/invoked on the markdown path")+ }+ if res.Markdown != "" {+ t.Errorf("markdown mutated: %q", res.Markdown)+ }+}++func TestVisionEnhancement_NonAllowedFileTypeSkipped(t *testing.T) {+ dispatched := parserDispatchResult{+ OutputFormat: "json",+ DocType: "other",+ JSON: []map[string]any{+ {"text": "", "image": "aGVsbG8=", "doc_type_kwd": "image"},+ },+ }++ res, handled, err := maybeDispatchVisionEnhancement(+ t.Context(),+ dao.DB,+ utility.FileTypeOTHER,+ dispatched,+ map[string]any{"tenant_id": "t1"},+ )+ if err != nil {+ t.Fatalf("unexpected error: %v", err)+ }+ if handled {+ t.Error("handled = true, want false for FileTypeOTHER")+ }+ if res.JSON[0]["text"] != "" {+ t.Errorf("text = %q, want empty", res.JSON[0]["text"])+ }+}++func TestVisionEnhancement_EmptyOrNoTenantSkipped(t *testing.T) {+ dispatched := parserDispatchResult{+ OutputFormat: "json",+ DocType: "docx",+ JSON: []map[string]any{+ {"text": "", "image": "aGVsbG8=", "doc_type_kwd": "image"},+ },+ }++ // No tenant_id+ res, handled, err := maybeDispatchVisionEnhancement(+ t.Context(),+ dao.DB,+ utility.FileTypeDOCX,+ dispatched,+ map[string]any{},+ )+ if err != nil || handled {+ t.Errorf("handled=%v, err=%v, want false, nil for missing tenant_id", handled, err)+ }+ if res.JSON[0]["text"] != "" {+ t.Errorf("text = %q, want untouched", res.JSON[0]["text"])+ }+}++// TestVisionEnhancement_DispatchedErrSkipped ensures a failed parse result is+// returned unchanged — enhancement must not touch items when dispatched.Err != nil.+func TestVisionEnhancement_DispatchedErrSkipped(t *testing.T) {+ parseErr := errors.New("parse failed")+ dispatched := parserDispatchResult{+ Err: parseErr,+ OutputFormat: "json",+ JSON: []map[string]any{+ {"text": "", "image": "aGVsbG8=", "doc_type_kwd": "image"},+ },+ }++ res, handled, err := maybeDispatchVisionEnhancement(+ t.Context(),+ dao.DB,+ utility.FileTypePDF,+ dispatched,+ map[string]any{"tenant_id": "t1"},+ )+ if err != nil {+ t.Fatalf("unexpected error: %v", err)+ }+ if handled {+ t.Error("handled = true, want false when dispatched.Err is set")+ }+ if res.Err != parseErr {+ t.Errorf("res.Err = %v, want original parse error preserved", res.Err)+ }+}++// TestVisionEnhancement_ContextCancellation verifies that a cancelled context+// propagates through visionChatInvoker and the enhancement is skipped without panic.+func TestVisionEnhancement_ContextCancellation(t *testing.T) {+ swapVisionGlobals(t, fakeResolver,+ func(ctx context.Context, _ modelModule.ModelDriver, _ string, _ []modelModule.Message, _ *modelModule.APIConfig) (*modelModule.ChatResponse, error) {+ return nil, ctx.Err()+ },+ fakePrompt,+ )++ ctx, cancel := context.WithCancel(t.Context())+ cancel() // pre-cancel++ dispatched := parserDispatchResult{+ OutputFormat: "json",+ JSON: []map[string]any{+ {"text": "", "image": "aGVsbG8=", "doc_type_kwd": "image"},+ },+ }++ res, handled, err := maybeDispatchVisionEnhancement(+ ctx,+ dao.DB,+ utility.FileTypePDF,+ dispatched,+ map[string]any{"tenant_id": "t1"},+ )+ if !errors.Is(err, context.Canceled) {+ t.Fatalf("err = %v, want context.Canceled", err)+ }+ if handled {+ t.Error("handled = true with cancelled context, want false")+ }+ // text field should be untouched since invoker returned no answer.+ if got, _ := res.JSON[0]["text"].(string); got != "" {+ t.Errorf("text = %q, want empty (no successful VLM response)", got)+ }+}++// TestVisionEnhancement_NonStringImageFieldFiltered verifies that items with a+// non-string image field (e.g., int) are silently skipped during target collection.+func TestVisionEnhancement_NonStringImageFieldFiltered(t *testing.T) {+ invokerCalled := false+ swapVisionGlobals(t, fakeResolver,+ func(_ context.Context, _ modelModule.ModelDriver, _ string, _ []modelModule.Message, _ *modelModule.APIConfig) (*modelModule.ChatResponse, error) {+ invokerCalled = true+ ans := "description"+ return &modelModule.ChatResponse{Answer: &ans}, nil+ },+ fakePrompt,+ )++ dispatched := parserDispatchResult{+ OutputFormat: "json",+ JSON: []map[string]any{+ // non-string image field — filtered by target collector+ {"text": "para", "image": 12345, "doc_type_kwd": "image"},+ // nil image — also filtered+ {"text": "para2", "image": nil, "doc_type_kwd": "image"},+ },+ }++ _, handled, err := maybeDispatchVisionEnhancement(+ t.Context(),+ dao.DB,+ utility.FileTypePDF,+ dispatched,+ map[string]any{"tenant_id": "t1"},+ )+ if err != nil {+ t.Fatalf("unexpected error: %v", err)+ }+ if handled {+ t.Error("handled = true, want false — no valid image targets")+ }+ if invokerCalled {+ t.Error("invoker was called despite no valid image targets")+ }+}++// TestVisionEnhancement_MoreThanConcurrencyItems verifies that N > visionEnhancementConcurrency+// items are all processed without deadlock.+func TestVisionEnhancement_MoreThanConcurrencyItems(t *testing.T) {+ n := visionEnhancementConcurrency + 5+ swapVisionGlobals(t, fakeResolver,+ func(_ context.Context, _ modelModule.ModelDriver, _ string, _ []modelModule.Message, _ *modelModule.APIConfig) (*modelModule.ChatResponse, error) {+ ans := "desc"+ return &modelModule.ChatResponse{Answer: &ans}, nil+ },+ fakePrompt,+ )++ items := make([]map[string]any, n)+ for i := range items {+ items[i] = map[string]any{+ "text": fmt.Sprintf("item%d", i),+ "image": "aGVsbG8=",+ "doc_type_kwd": "image",+ }+ }+ dispatched := parserDispatchResult{+ OutputFormat: "json",+ JSON: items,+ }++ res, handled, err := maybeDispatchVisionEnhancement(+ t.Context(),+ dao.DB,+ utility.FileTypePDF,+ dispatched,+ map[string]any{"tenant_id": "t1"},+ )+ if err != nil {+ t.Fatalf("unexpected error: %v", err)+ }+ if !handled {+ t.Error("handled = false, want true")+ }+ // All n items should have received a description appended.+ for i, item := range res.JSON {+ got, _ := item["text"].(string)+ if got == fmt.Sprintf("item%d", i) {+ t.Errorf("item[%d] text unchanged %q, expected description appended", i, got)+ }+ }+}++// TestVisionEnhancement_PlainTextResponseNotTruncated verifies that a plain-text+// (no fences) VLM response is returned verbatim — cleanMarkdownBlock must not truncate it.+func TestVisionEnhancement_PlainTextResponseNotTruncated(t *testing.T) {+ plain := "A pipeline diagram showing three stages."+ swapVisionGlobals(t, fakeResolver,+ func(_ context.Context, _ modelModule.ModelDriver, _ string, _ []modelModule.Message, _ *modelModule.APIConfig) (*modelModule.ChatResponse, error) {+ return &modelModule.ChatResponse{Answer: &plain}, nil+ },+ fakePrompt,+ )++ dispatched := parserDispatchResult{+ OutputFormat: "json",+ JSON: []map[string]any{+ {"text": "", "image": "aGVsbG8=", "doc_type_kwd": "image"},+ },+ }++ res, handled, err := maybeDispatchVisionEnhancement(+ t.Context(),+ dao.DB,+ utility.FileTypePDF,+ dispatched,+ map[string]any{"tenant_id": "t1"},+ )+ if err != nil {+ t.Fatalf("unexpected error: %v", err)+ }+ if !handled {+ t.Error("handled = false, want true")+ }+ if got, _ := res.JSON[0]["text"].(string); got != plain {+ t.Errorf("plain text response = %q, want %q (must not be truncated)", got, plain)+ }+}++// TestVisionEnhancement_PromptBuilderErrorSkipped verifies that a prompt build+// failure skips enhancement silently (best-effort, nilerr is intentional).+func TestVisionEnhancement_PromptBuilderErrorSkipped(t *testing.T) {+ swapVisionGlobals(t,+ fakeResolver,+ func(_ context.Context, _ modelModule.ModelDriver, _ string, _ []modelModule.Message, _ *modelModule.APIConfig) (*modelModule.ChatResponse, error) {+ t.Error("invoker should not be called when prompt builder fails")+ return nil, nil+ },+ func(_ string) (string, error) {+ return "", errors.New("prompt load failed")+ },+ )++ dispatched := parserDispatchResult{+ OutputFormat: "json",+ JSON: []map[string]any{+ {"text": "", "image": "aGVsbG8=", "doc_type_kwd": "image"},+ },+ }++ res, handled, err := maybeDispatchVisionEnhancement(+ t.Context(),+ dao.DB,+ utility.FileTypePDF,+ dispatched,+ map[string]any{"tenant_id": "t1"},+ )+ if err != nil {+ t.Fatalf("unexpected error: %v", err)+ }+ if handled {+ t.Error("handled = true, want false when prompt builder fails")+ }+ if got, _ := res.JSON[0]["text"].(string); got != "" {+ t.Errorf("text = %q, want empty", got)+ }+}++// TestVisionEnhancement_ModelResolveFailureSkipped verifies that a resolver error+// (e.g., tenant has no IMAGE2TEXT model) skips enhancement silently.+func TestVisionEnhancement_ModelResolveFailureSkipped(t *testing.T) {+ swapVisionGlobals(t,+ func(_ context.Context, _ *gorm.DB, _ string, _ entity.ModelType) (modelModule.ModelDriver, string, *modelModule.APIConfig, int, error) {+ return nil, "", nil, 0, errors.New("no model")+ },+ func(_ context.Context, _ modelModule.ModelDriver, _ string, _ []modelModule.Message, _ *modelModule.APIConfig) (*modelModule.ChatResponse, error) {+ t.Error("invoker should not be called when model resolve fails")+ return nil, nil+ },+ fakePrompt,+ )++ dispatched := parserDispatchResult{+ OutputFormat: "json",+ JSON: []map[string]any{+ {"text": "", "image": "aGVsbG8=", "doc_type_kwd": "image"},+ },+ }++ res, handled, err := maybeDispatchVisionEnhancement(+ t.Context(),+ dao.DB,+ utility.FileTypePDF,+ dispatched,+ map[string]any{"tenant_id": "t1"},+ )+ if err != nil {+ t.Fatalf("unexpected error: %v", err)+ }+ if handled {+ t.Error("handled = true, want false when model resolve fails")+ }+ if got, _ := res.JSON[0]["text"].(string); got != "" {+ t.Errorf("text = %q, want empty", got)+ }+}++// TestVisionEnhancement_CancellationStopsSchedulingWithManyItems verifies that+// cancellation stops scheduling when N > visionEnhancementConcurrency, without deadlock.+func TestVisionEnhancement_CancellationStopsSchedulingWithManyItems(t *testing.T) {+ n := visionEnhancementConcurrency + 5+ swapVisionGlobals(t, fakeResolver,+ func(ctx context.Context, _ modelModule.ModelDriver, _ string, _ []modelModule.Message, _ *modelModule.APIConfig) (*modelModule.ChatResponse, error) {+ // Simulate work that respects ctx.+ select {+ case <-ctx.Done():+ return nil, ctx.Err()+ default:+ }+ ans := "desc"+ return &modelModule.ChatResponse{Answer: &ans}, nil+ },+ fakePrompt,+ )++ items := make([]map[string]any, n)+ for i := range items {+ items[i] = map[string]any{+ "text": fmt.Sprintf("item%d", i),+ "image": "aGVsbG8=",+ "doc_type_kwd": "image",+ }+ }+ dispatched := parserDispatchResult{+ OutputFormat: "json",+ JSON: items,+ }++ ctx, cancel := context.WithCancel(t.Context())+ cancel() // pre-cancel — dispatch loop should break immediately++ res, handled, err := maybeDispatchVisionEnhancement(+ ctx,+ dao.DB,+ utility.FileTypePDF,+ dispatched,+ map[string]any{"tenant_id": "t1"},+ )+ if !errors.Is(err, context.Canceled) {+ t.Fatalf("err = %v, want context.Canceled", err)+ }+ if handled {+ t.Error("handled = true with cancelled context, want false")+ }+ // No item should be modified when dispatch is cancelled before scheduling.+ for i, item := range res.JSON {+ if got, _ := item["text"].(string); got != fmt.Sprintf("item%d", i) {+ t.Errorf("item[%d] text = %q, want untouched after cancellation", i, got)+ }+ }+}++func TestBuildVisionMessages_PreventsDoublePrefix(t *testing.T) {+ cases := []struct {+ name string+ input string+ wantURL string+ }{+ {+ name: "raw base64 gets png prefix",+ input: "aGVsbG8=",+ wantURL: "data:image/png;base64,aGVsbG8=",+ },+ {+ name: "already png data uri is preserved without double prefix",+ input: "data:image/png;base64,aGVsbG8=",+ wantURL: "data:image/png;base64,aGVsbG8=",+ },+ {+ name: "jpeg data uri is preserved as jpeg",+ input: "data:image/jpeg;base64,/9j/4AAQ",+ wantURL: "data:image/jpeg;base64,/9j/4AAQ",+ },+ {+ name: "https url is preserved",+ input: "https://example.com/figure.png",+ wantURL: "https://example.com/figure.png",+ },+ {+ name: "whitespace trimmed",+ input: " data:image/png;base64,abc ",+ wantURL: "data:image/png;base64,abc",+ },+ }++ for _, tc := range cases {+ t.Run(tc.name, func(t *testing.T) {+ msgs := buildVisionMessages("describe", tc.input)+ if len(msgs) != 1 {+ t.Fatalf("len(msgs) = %d, want 1", len(msgs))+ }+ parts, ok := msgs[0].Content.([]interface{})+ if !ok || len(parts) < 2 {+ t.Fatalf("Content parts = %+v, want >= 2", msgs[0].Content)+ }+ img, ok := parts[1].(map[string]any)+ if !ok {+ t.Fatalf("parts[1] = %+v, want map[string]any", parts[1])+ }+ imgURL, ok := img["image_url"].(map[string]any)+ if !ok {+ t.Fatalf("img[image_url] = %+v, want map[string]any", img["image_url"])+ }+ got, _ := imgURL["url"].(string)+ if got != tc.wantURL {+ t.Errorf("image_url.url = %q, want %q", got, tc.wantURL)+ }+ })+ }+}++func TestExtractVisionAnswer_CleansMarkdownBlock(t *testing.T) {+ ans := "```markdown\n| Header 1 | Header 2 |\n| --- | --- |\n| Cell 1 | Cell 2 |\n```"+ resp := &modelModule.ChatResponse{Answer: &ans}+ got := extractVisionAnswer(resp)+ want := "| Header 1 | Header 2 |\n| --- | --- |\n| Cell 1 | Cell 2 |"+ if got != want {+ t.Errorf("extractVisionAnswer = %q, want %q", got, want)+ }+}++func TestCleanMarkdownBlock_EdgeCases(t *testing.T) {+ cases := []struct {+ name string+ input string+ want string+ }{+ {+ name: "standard markdown block",+ input: "```markdown\nHello world\n```",+ want: "Hello world",+ },+ {+ name: "windows CRLF endings",+ input: "```markdown\r\nHello world\r\n```",+ want: "Hello world",+ },+ {+ name: "nested code blocks preserved",+ input: "```markdown\nText with ```nested``` blocks\n```",+ want: "Text with ```nested``` blocks",+ },+ {+ name: "mixed whitespace tabs",+ input: "\t```markdown\t\n\tContent with tabs\n\t```\t",+ want: "Content with tabs",+ },+ {+ name: "empty string",+ input: "",+ want: "",+ },+ {+ name: "only markdown tags",+ input: "```markdown```",+ want: "",+ },+ {+ name: "unwrapped text unchanged",+ input: "Just plain text without markdown block",+ want: "Just plain text without markdown block",+ },+ }++ for _, tc := range cases {+ t.Run(tc.name, func(t *testing.T) {+ got := cleanMarkdownBlock(tc.input)+ if got != tc.want {+ t.Errorf("cleanMarkdownBlock(%q) = %q, want %q", tc.input, got, tc.want)+ }+ })+ }+}diff --git a/internal/ingestion/component/vision_language_test.go b/internal/ingestion/component/vision_language_test.goindex 8b0cd9705e..deadc898b1 100644--- a/internal/ingestion/component/vision_language_test.go+++ b/internal/ingestion/component/vision_language_test.go@@ -69,28 +69,23 @@ func TestRenderFigureVisionLanguage(t *testing.T) {}}-func TestBuildFigureVisionPromptRendersLanguageAndContext(t *testing.T) {- prompt, err := buildFigureVisionPrompt("Context above", "Context below", "Japanese")+func TestBuildFigureVisionPromptRendersLanguage(t *testing.T) {+ prompt, err := buildFigureVisionPrompt("Japanese")if err != nil {t.Fatalf("buildFigureVisionPrompt: %v", err)}- for _, want := range []string{- "Write all descriptions and field values in Japanese.",- "Context above",- "Context below",- } {- if !strings.Contains(prompt, want) {- t.Fatalf("prompt does not contain %q", want)- }+ if !strings.Contains(prompt, "Write all descriptions and field values in Japanese.") {+ t.Fatalf("prompt does not contain language directive, got: %q", prompt)}- for _, unresolved := range []string{"{{ language }}", "{{ context_above }}", "{{ context_below }}"} {+ for _, unresolved := range []string{"{{ language }}", "{{language}}"} {if strings.Contains(prompt, unresolved) {t.Fatalf("prompt contains unresolved placeholder %q", unresolved)}}- secondPrompt, err := buildFigureVisionPrompt("", "", "Chinese")+ // Second call uses cache — verify language is re-rendered correctly per call.+ secondPrompt, err := buildFigureVisionPrompt("Chinese")if err != nil {t.Fatalf("buildFigureVisionPrompt with cached template: %v", err)}