Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

refactor(parser): consolidate vision figure enhancement across document formats (#18775)

jay77721959c8e8b4ab8
diff --git a/internal/ingestion/component/docx_vision_dispatch.go b/internal/ingestion/component/docx_vision_dispatch.go
deleted file mode 100644
index 4f7b2622f3..0000000000
--- a/internal/ingestion/component/docx_vision_dispatch.go
+++ /dev/null
@@ -1,282 +0,0 @@
-//
-// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-//
-
-// DOCX vision figure dispatch: enriches the parse result with
-// LLM-generated descriptions of embedded images, mirroring
-// Python's enhance_media_sections_with_vision in
-// rag/flow/parser/utils.py (invoked from parser.py:_doc's JSON branch).
-//
-// Unlike the PDF vision path (which replaces dispatchParse entirely),
-// DOCX vision is a post-processing step. It mirrors Python exactly:
-// vision enrichment happens ONLY on the JSON output path, where each
-// item carries a doc_type_kwd and an optional image. The Markdown path
-// performs no vision enrichment in Python, so it must not be here either.
-
-package component
-
-import (
- "context"
- "fmt"
- "os"
- "path/filepath"
- "strings"
- "sync"
-
- pdflayout "ragflow/internal/deepdoc/parser/pdf/layout"
- "ragflow/internal/entity"
- modelModule "ragflow/internal/entity/models"
- "ragflow/internal/ingestion/component/schema"
- "ragflow/internal/utility"
-
- "gorm.io/gorm"
-)
-
-var (
- figureVisionPromptBuilder = buildFigureVisionPrompt
- visionChatInvoker = defaultVisionChatInvoker
- docxVisionConcurrency uint = 10
-)
-
-const (
- figureVisionPromptFile = "vision_llm_figure_describe_prompt.md"
- figureVisionPromptWithContextFile = "vision_llm_figure_describe_prompt_with_context.md"
-)
-
-var (
- figureVisionPromptsBase string
- figureVisionPromptsOnce sync.Once
- figureVisionPromptCache = make(map[string]string)
- figureVisionPromptMu sync.RWMutex
-)
-
-// maybeDispatchDOCXVision enriches a DOCX parse result with vision-model
-// descriptions of embedded images. It mirrors Python's
-// enhance_media_sections_with_vision (rag/flow/parser/utils.py:162), which
-// runs only in the JSON output branch of parser.py:_doc.
-//
-// For each JSON item whose doc_type_kwd is "image" or "table" AND that
-// carries a non-empty "image" field, the vision model describes the image
-// and the description is appended to the item's text (Python:
-// item["text"] = f"{text}\n{parsed_text}" if text else parsed_text). Items
-// without an image (e.g. DOCX tables) are left untouched, exactly as Python
-// skips them via `if item.get("image") is None: continue`.
-//
-// The Markdown output path receives no vision enrichment — Python's DOCX
-// Markdown branch only concatenates text and never calls the vision model.
-func maybeDispatchDOCXVision(
- ctx context.Context,
- db *gorm.DB,
- fileType utility.FileType,
- dispatched parserDispatchResult,
- inputs map[string]any,
- setups map[string]schema.ParserSetup,
-) (parserDispatchResult, bool, error) {
- if fileType != utility.FileTypeDOCX {
- return dispatched, false, nil
- }
- // Python triggers vision enrichment only on the JSON path
- // (parser.py:_doc → enhance_media_sections_with_vision).
- if dispatched.Err != nil || dispatched.OutputFormat != "json" || len(dispatched.JSON) == 0 {
- return dispatched, false, nil
- }
-
- tenantID := getStringOr(inputs, "tenant_id", "")
- if tenantID == "" {
- return dispatched, false, nil
- }
- language := resolveVisionLanguage(inputs, "")
-
- // Resolve the tenant's IMAGE2TEXT model.
- driver, modelName, apiConfig, _, err := resolveTenantModelByType(ctx, db, tenantID, entity.ModelTypeImage2Text)
- if err != nil {
- // Model not available — skip vision enhancement silently,
- // matching Python's try/except pass behaviour.
- return dispatched, false, nil
- }
-
- // Collect the indices of JSON items that carry an embeddable image.
- type target struct {
- idx int
- }
- var targets []target
- for i, item := range dispatched.JSON {
- kd, _ := item["doc_type_kwd"].(string)
- if kd != "image" && kd != "table" {
- continue
- }
- img, _ := item["image"].(string)
- if img == "" {
- continue
- }
- targets = append(targets, target{idx: i})
- }
- if len(targets) == 0 {
- return dispatched, false, nil
- }
-
- descriptions := make([]string, len(targets))
- var wg sync.WaitGroup
- sem := make(chan struct{}, docxVisionConcurrency)
-
- for slot, tg := range targets {
- wg.Add(1)
- go func(slot int, itemIdx int) {
- defer wg.Done()
- sem <- struct{}{}
- defer func() { <-sem }()
-
- img, _ := dispatched.JSON[itemIdx]["image"].(string)
- if img == "" {
- return
- }
- // DOCX JSON items have no surrounding context (unlike the
- // former Markdown path), so use the bare figure prompt —
- // matching Python's VisionFigureParser(context_size=0).
- prompt, perr := figureVisionPromptBuilder("", "", language)
- if perr != nil {
- return
- }
- messages := buildVisionMessages(prompt, img)
- resp, ierr := visionChatInvoker(ctx, driver, modelName, messages, apiConfig)
- if ierr != nil {
- return
- }
- descriptions[slot] = extractDOCXVisionAnswer(resp)
- }(slot, tg.idx)
- }
- wg.Wait()
-
- modified := false
- for slot, tg := range targets {
- desc := strings.TrimSpace(descriptions[slot])
- if desc == "" {
- continue
- }
- existing, _ := dispatched.JSON[tg.idx]["text"].(string)
- if existing != "" {
- dispatched.JSON[tg.idx]["text"] = existing + "\n" + desc
- } else {
- dispatched.JSON[tg.idx]["text"] = desc
- }
- modified = true
- }
-
- return dispatched, modified, nil
-}
-
-// buildFigureVisionPrompt loads the figure-describe prompt template
-// and, when context text is available, renders it with the
-// with-context variant. Mirrors Python:
-//
-// if context_above or context_below:
-// prompt = vision_llm_figure_describe_prompt_with_context(
-// context_above, context_below, language=language)
-// else:
-// prompt = vision_llm_figure_describe_prompt(language=language)
-func buildFigureVisionPrompt(contextAbove, contextBelow, language string) (string, error) {
- hasContext := strings.TrimSpace(contextAbove) != "" || strings.TrimSpace(contextBelow) != ""
-
- var templateName string
- if hasContext {
- templateName = figureVisionPromptWithContextFile
- } else {
- templateName = figureVisionPromptFile
- }
-
- template, err := loadFigureVisionPromptFile(templateName)
- if err != nil {
- return "", err
- }
- template = renderFigureVisionLanguage(template, language)
-
- if hasContext {
- template = strings.ReplaceAll(template, "{{ context_above }}", contextAbove)
- template = strings.ReplaceAll(template, "{{ context_below }}", contextBelow)
- }
- return template, nil
-}
-
-func loadFigureVisionPromptFile(filename string) (string, error) {
- figureVisionPromptMu.RLock()
- if cached, ok := figureVisionPromptCache[filename]; ok {
- figureVisionPromptMu.RUnlock()
- return cached, nil
- }
- figureVisionPromptMu.RUnlock()
-
- baseDir, err := figureVisionPromptsBaseDir()
- if err != nil {
- return "", err
- }
- promptPath := filepath.Join(baseDir, "rag", "prompts", filename)
- content, err := os.ReadFile(promptPath)
- if err != nil {
- return "", fmt.Errorf("figure vision prompt %q: %w", filename, err)
- }
- cached := strings.TrimSpace(string(content))
- figureVisionPromptMu.Lock()
- figureVisionPromptCache[filename] = cached
- figureVisionPromptMu.Unlock()
- return cached, nil
-}
-
-func figureVisionPromptsBaseDir() (string, error) {
- var initErr error
- figureVisionPromptsOnce.Do(func() {
- root := utility.GetProjectRoot()
- if _, statErr := os.Stat(filepath.Join(root, "rag", "prompts")); statErr == nil {
- figureVisionPromptsBase = root
- return
- }
- initErr = fmt.Errorf("rag/prompts not found under project root %q", root)
- })
- if initErr != nil {
- return "", initErr
- }
- return figureVisionPromptsBase, nil
-}
-
-func buildVisionMessages(prompt, imageBase64 string) []modelModule.Message {
- dataURI := pdflayout.InlinePNGDataURL(imageBase64)
- if dataURI == "" {
- dataURI = strings.TrimSpace(imageBase64)
- }
- return []modelModule.Message{{
- Role: "user",
- Content: []interface{}{
- map[string]any{"type": "text", "text": prompt},
- map[string]any{"type": "image_url", "image_url": map[string]any{"url": dataURI}},
- },
- }}
-}
-
-func extractDOCXVisionAnswer(resp *modelModule.ChatResponse) string {
- if resp == nil || resp.Answer == nil {
- return ""
- }
- return strings.TrimSpace(*resp.Answer)
-}
-
-func defaultVisionChatInvoker(
- ctx context.Context,
- driver modelModule.ModelDriver,
- modelName string,
- messages []modelModule.Message,
- apiConfig *modelModule.APIConfig,
-) (*modelModule.ChatResponse, error) {
- vision := true
- return driver.ChatWithMessages(ctx, modelName, messages, apiConfig, &modelModule.ChatConfig{Vision: &vision}, nil)
-}
diff --git a/internal/ingestion/component/docx_vision_dispatch_test.go b/internal/ingestion/component/docx_vision_dispatch_test.go
deleted file mode 100644
index 1c169d3016..0000000000
--- a/internal/ingestion/component/docx_vision_dispatch_test.go
+++ /dev/null
@@ -1,254 +0,0 @@
-//
-// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package component
-
-import (
- "context"
- "ragflow/internal/dao"
- "sync"
- "testing"
-
- "ragflow/internal/entity"
- modelModule "ragflow/internal/entity/models"
- "ragflow/internal/utility"
-
- "gorm.io/gorm"
-)
-
-// docxVisionFakeDriver satisfies modelModule.ModelDriver but never reaches the
-// network: docxVisionCaptureInvoker intercepts the call before the driver.
-type docxVisionFakeDriver struct {
- modelModule.ModelDriver
-}
-
-// docxVisionCaptureInvoker records the requested image and returns a fixed
-// description, mirroring the markdown-vision test's capture driver.
-type docxVisionCaptureInvoker struct {
- mu sync.Mutex
- images []string
- captured []modelModule.Message
-}
-
-func (c *docxVisionCaptureInvoker) invoke(
- ctx context.Context,
- driver modelModule.ModelDriver,
- modelName string,
- messages []modelModule.Message,
- apiConfig *modelModule.APIConfig,
-) (*modelModule.ChatResponse, error) {
- c.mu.Lock()
- c.captured = append(c.captured, messages...)
- // Pull the data URI out of the second content part.
- if parts, ok := messages[0].Content.([]interface{}); ok && len(parts) >= 2 {
- if img, ok := parts[1].(map[string]any); ok {
- if url, ok := img["image_url"].(map[string]any); ok {
- if u, ok := url["url"].(string); ok {
- c.images = append(c.images, u)
- }
- }
- }
- }
- c.mu.Unlock()
- ans := "a diagram of a pipeline"
- return &modelModule.ChatResponse{Answer: &ans}, nil
-}
-
-// TestMaybeDispatchDOCXVision_EnhancesJSONImages verifies Diff 2.4: DOCX vision
-// enhancement must trigger on the JSON output path (like Python's
-// enhance_media_sections_with_vision in parser.py:_doc) and must NOT trigger on
-// the Markdown path. Image items with a non-empty `image` field get their VLM
-// description appended to `text`; table items (no image) and text items are
-// left untouched.
-func TestMaybeDispatchDOCXVision_EnhancesJSONImages(t *testing.T) {
- origResolver := resolveTenantModelByType
- origInvoker := visionChatInvoker
- origPrompt := figureVisionPromptBuilder
- defer func() {
- resolveTenantModelByType = origResolver
- visionChatInvoker = origInvoker
- figureVisionPromptBuilder = origPrompt
- }()
-
- resolveTenantModelByType = func(ctx context.Context, db *gorm.DB, tenantID string, modelType entity.ModelType) (modelModule.ModelDriver, string, *modelModule.APIConfig, int, error) {
- return &docxVisionFakeDriver{}, "docx-vision-model", &modelModule.APIConfig{}, 0, nil
- }
- invoker := &docxVisionCaptureInvoker{}
- visionChatInvoker = invoker.invoke
- var capturedLanguage string
- figureVisionPromptBuilder = func(_, _, language string) (string, error) {
- capturedLanguage = language
- return "describe the figure", nil
- }
-
- dispatched := parserDispatchResult{
- OutputFormat: "json",
- DocType: "docx",
- JSON: []map[string]any{
- {"text": "Intro paragraph", "image": nil, "doc_type_kwd": "text"},
- {"text": "", "image": "aGVsbG8taW1hZ2U=", "doc_type_kwd": "image"},
- {"text": "<table></table>", "image": nil, "doc_type_kwd": "table"},
- },
- }
- ctx := t.Context()
-
- res, handled, err := maybeDispatchDOCXVision(
- ctx,
- dao.DB,
- utility.FileTypeDOCX,
- dispatched,
- map[string]any{"tenant_id": "t1", "lang": "Japanese"},
- defaultSetups(),
- )
- if err != nil {
- t.Fatalf("maybeDispatchDOCXVision: unexpected error: %v", err)
- }
- if !handled {
- t.Fatal("handled = false, want true (JSON image item should be enhanced)")
- }
- if len(res.JSON) != 3 {
- t.Fatalf("JSON len = %d, want 3", len(res.JSON))
- }
- if got := res.JSON[0]["text"].(string); got != "Intro paragraph" {
- t.Errorf("text item text = %q, want unchanged", got)
- }
- // image item: VLM description appended to (empty) text.
- if got, _ := res.JSON[1]["text"].(string); got != "a diagram of a pipeline" {
- t.Errorf("image item text = %q, want appended VLM description", got)
- }
- if got, _ := res.JSON[2]["text"].(string); got != "<table></table>" {
- t.Errorf("table item text = %q, want unchanged (no image)", got)
- }
- if len(invoker.images) != 1 {
- t.Fatalf("vision invoker called %d times, want 1 (only the image item)", len(invoker.images))
- }
- if capturedLanguage != "Japanese" {
- t.Errorf("figure prompt language = %q, want Japanese", capturedLanguage)
- }
- if want := "data:image/png;base64,aGVsbG8taW1hZ2U="; invoker.images[0] != want {
- t.Errorf("vision image data URI = %q, want %q", invoker.images[0], want)
- }
-}
-
-// TestMaybeDispatchDOCXVision_JSONOnly verifies Diff 2.4: the Markdown output
-// path must NOT be enhanced (Python's markdown/docx branch performs no vision
-// enrichment). A Markdown result with embedded figures is returned untouched.
-func TestMaybeDispatchDOCXVision_JSONOnly(t *testing.T) {
- origResolver := resolveTenantModelByType
- origInvoker := visionChatInvoker
- defer func() {
- resolveTenantModelByType = origResolver
- visionChatInvoker = origInvoker
- }()
-
- called := false
- resolveTenantModelByType = func(ctx context.Context, db *gorm.DB, tenantID string, modelType entity.ModelType) (modelModule.ModelDriver, string, *modelModule.APIConfig, int, error) {
- called = true
- return &docxVisionFakeDriver{}, "m", &modelModule.APIConfig{}, 0, nil
- }
- visionChatInvoker = func(ctx context.Context, d modelModule.ModelDriver, m string, msgs []modelModule.Message, c *modelModule.APIConfig) (*modelModule.ChatResponse, error) {
- called = true
- ans := "x"
- return &modelModule.ChatResponse{Answer: &ans}, nil
- }
-
- dispatched := parserDispatchResult{
- OutputFormat: "markdown",
- DocType: "docx",
- Markdown: "![Image](data:image/png;base64,abc)",
- File: map[string]any{"figures": []map[string]any{{"image": "abc", "marker": "x"}}},
- }
- ctx := t.Context()
-
- res, handled, err := maybeDispatchDOCXVision(
- ctx,
- dao.DB,
- utility.FileTypeDOCX,
- dispatched,
- map[string]any{"tenant_id": "t1"},
- defaultSetups(),
- )
- if err != nil {
- t.Fatalf("maybeDispatchDOCXVision: unexpected error: %v", err)
- }
- if handled {
- t.Error("handled = true, want false (markdown path must not be enhanced)")
- }
- if called {
- t.Error("vision model was resolved/invoked on the markdown path")
- }
- if res.Markdown != "![Image](data:image/png;base64,abc)" {
- t.Errorf("markdown mutated: %q", res.Markdown)
- }
-}
-
-func TestBuildVisionMessages_PreventsDoublePrefix(t *testing.T) {
- cases := []struct {
- name string
- input string
- wantURL string
- }{
- {
- name: "raw base64 gets png prefix",
- input: "aGVsbG8=",
- wantURL: "data:image/png;base64,aGVsbG8=",
- },
- {
- name: "already png data uri is preserved without double prefix",
- input: "data:image/png;base64,aGVsbG8=",
- wantURL: "data:image/png;base64,aGVsbG8=",
- },
- {
- name: "jpeg data uri is preserved as jpeg",
- input: "data:image/jpeg;base64,/9j/4AAQ",
- wantURL: "data:image/jpeg;base64,/9j/4AAQ",
- },
- {
- name: "https url is preserved",
- input: "https://example.com/figure.png",
- wantURL: "https://example.com/figure.png",
- },
- {
- name: "whitespace trimmed",
- input: " data:image/png;base64,abc ",
- wantURL: "data:image/png;base64,abc",
- },
- }
-
- for _, tc := range cases {
- t.Run(tc.name, func(t *testing.T) {
- msgs := buildVisionMessages("describe", tc.input)
- if len(msgs) != 1 {
- t.Fatalf("len(msgs) = %d, want 1", len(msgs))
- }
- parts, ok := msgs[0].Content.([]interface{})
- if !ok || len(parts) < 2 {
- t.Fatalf("Content parts = %+v, want >= 2", msgs[0].Content)
- }
- img, ok := parts[1].(map[string]any)
- if !ok {
- t.Fatalf("parts[1] = %+v, want map[string]any", parts[1])
- }
- imgURL, ok := img["image_url"].(map[string]any)
- if !ok {
- t.Fatalf("img[image_url] = %+v, want map[string]any", img["image_url"])
- }
- got, _ := imgURL["url"].(string)
- if got != tc.wantURL {
- t.Errorf("image_url.url = %q, want %q", got, tc.wantURL)
- }
- })
- }
-}
diff --git a/internal/ingestion/component/markdown_vision_dispatch.go b/internal/ingestion/component/markdown_vision_dispatch.go
deleted file mode 100644
index ff0add6b03..0000000000
--- a/internal/ingestion/component/markdown_vision_dispatch.go
+++ /dev/null
@@ -1,159 +0,0 @@
-//
-// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-//
-
-// Markdown vision figure dispatch: enriches parsed Markdown JSON
-// items with LLM-generated descriptions of embedded images,
-// mirroring Python's enhance_media_sections_with_vision in
-// rag/flow/parser/utils.py, called from the _markdown path.
-//
-// Unlike the DOCX vision path (which processes a separate figures
-// array), Markdown vision iterates over the JSON items produced by
-// MarkdownParser.ParseWithResult and enhances items whose
-// doc_type_kwd == "image" and whose "image" field contains a
-// base64-encoded image.
-
-package component
-
-import (
- "context"
- "strings"
- "sync"
-
- "ragflow/internal/entity"
- "ragflow/internal/utility"
-
- "gorm.io/gorm"
-)
-
-var (
- markdownVisionConcurrency uint = 10
-)
-
-// maybeDispatchMarkdownVision checks whether the Markdown parse result
-// contains JSON items with embedded images and, when a vision model is
-// available, enriches those items with AI-generated figure descriptions.
-//
-// Mirrors the Python flow:
-//
-// 1. _markdown → sections + section_images (parser.py:1005)
-// 2. enhance_media_sections_with_vision (parser.py:1054)
-//
-// The function is called AFTER dispatchParse so the normal parse
-// path produces JSON items with doc_type_kwd == "image" and an
-// "image" base64 field. It returns (result, handled, error).
-func maybeDispatchMarkdownVision(
- ctx context.Context,
- db *gorm.DB,
- fileType utility.FileType,
- dispatched parserDispatchResult,
- inputs map[string]any,
-) (parserDispatchResult, bool, error) {
- if fileType != utility.FileTypeMarkdown {
- return dispatched, false, nil
- }
- if dispatched.Err != nil || dispatched.OutputFormat != "json" {
- return dispatched, false, nil
- }
- if len(dispatched.JSON) == 0 {
- return dispatched, false, nil
- }
-
- // Collect indices of image items.
- type imgItem struct {
- idx int
- imageB64 string
- text string
- }
- var images []imgItem
- for i, item := range dispatched.JSON {
- kd, _ := item["doc_type_kwd"].(string)
- // Diff 2.5: Python enhances both image and table items
- // (parser/utils.py:181 checks {"image","table"}); only items
- // carrying an image are sent to the VLM (utils.py:183).
- if kd != "image" && kd != "table" {
- continue
- }
- img, _ := item["image"].(string)
- if img == "" {
- continue
- }
- text, _ := item["text"].(string)
- images = append(images, imgItem{idx: i, imageB64: img, text: text})
- }
- if len(images) == 0 {
- return dispatched, false, nil
- }
-
- tenantID := getStringOr(inputs, "tenant_id", "")
- if tenantID == "" {
- return dispatched, false, nil
- }
- language := resolveVisionLanguage(inputs, "")
-
- // Resolve the tenant's IMAGE2TEXT model.
- driver, modelName, apiConfig, _, err := resolveTenantModelByType(ctx, db, tenantID, entity.ModelTypeImage2Text)
- if err != nil {
- // Model not available — skip vision enhancement silently,
- // matching Python's try/except pass behaviour.
- return dispatched, false, nil
- }
-
- descriptions := make([]string, len(images))
- var wg sync.WaitGroup
- sem := make(chan struct{}, markdownVisionConcurrency)
-
- for i, img := range images {
- wg.Add(1)
- go func(pos int, item imgItem) {
- defer wg.Done()
- sem <- struct{}{}
- defer func() { <-sem }()
-
- // Markdown images have no context — use the
- // default (no-context) prompt template.
- prompt, err := figureVisionPromptBuilder("", "", language)
- if err != nil {
- return
- }
-
- messages := buildVisionMessages(prompt, item.imageB64)
- resp, err := visionChatInvoker(ctx, driver, modelName, messages, apiConfig)
- if err != nil {
- return
- }
- descriptions[pos] = extractDOCXVisionAnswer(resp)
- }(i, img)
- }
- wg.Wait()
-
- // Append vision descriptions to each image item's text field,
- // matching Python's `item["text"] = f"{text}\n{parsed_text}"`.
- for pos, img := range images {
- desc := strings.TrimSpace(descriptions[pos])
- if desc == "" {
- continue
- }
- item := dispatched.JSON[img.idx]
- existing, _ := item["text"].(string)
- if existing != "" {
- item["text"] = existing + "\n\n" + desc
- } else {
- item["text"] = desc
- }
- }
-
- return dispatched, true, nil
-}
diff --git a/internal/ingestion/component/media_dispatch_test.go b/internal/ingestion/component/media_dispatch_test.go
index ee98c2bf73..b86cdb55a9 100644
--- a/internal/ingestion/component/media_dispatch_test.go
+++ b/internal/ingestion/component/media_dispatch_test.go
@@ -401,60 +401,6 @@ func TestMaybeDispatchAudio_DefaultOutputFormatJson(t *testing.T) {
}
}
-// TestMaybeDispatchMarkdownVision_EnhancesTables pins diff 2.5: Markdown
-// vision enhancement must also process items whose doc_type_kwd is "table"
-// (Python checks {"image","table"} in parser/utils.py:181), not only "image".
-// Before the fix the table item was skipped and never sent to the VLM.
-func TestMaybeDispatchMarkdownVision_EnhancesTables(t *testing.T) {
- origResolver := resolveTenantModelByType
- origPrompt := figureVisionPromptBuilder
- defer func() {
- resolveTenantModelByType = origResolver
- figureVisionPromptBuilder = origPrompt
- }()
-
- drv := &imagePromptCaptureDriver{}
- resolveTenantModelByType = func(ctx context.Context, db *gorm.DB, tenantID string, modelType entity.ModelType) (modelModule.ModelDriver, string, *modelModule.APIConfig, int, error) {
- return drv, "img-model", &modelModule.APIConfig{}, 0, nil
- }
- figureVisionPromptBuilder = func(_, _, language string) (string, error) {
- return "describe in " + language, nil
- }
-
- dispatched := parserDispatchResult{
- OutputFormat: "json",
- JSON: []map[string]any{
- {"doc_type_kwd": "table", "image": "base64table", "text": ""},
- },
- }
-
- ctx := t.Context()
- res, handled, err := maybeDispatchMarkdownVision(
- ctx,
- dao.DB,
- utility.FileTypeMarkdown,
- dispatched,
- map[string]any{"tenant_id": "t1", "lang": "Korean"},
- )
- if err != nil {
- t.Fatalf("maybeDispatchMarkdownVision: %v", err)
- }
- if !handled {
- t.Fatalf("expected handled=true for markdown with a table image")
- }
- if len(res.JSON) != 1 {
- t.Fatalf("JSON len = %d, want 1", len(res.JSON))
- }
- // The table item must have been sent to the VLM and its description appended.
- if got, _ := res.JSON[0]["text"].(string); got != "captured" {
- t.Fatalf("table item text = %q, want %q (table items must be vision-enhanced)", got, "captured")
- }
- gotPrompt, ok := firstUserText(drv.captured)
- if !ok || gotPrompt != "describe in Korean" {
- t.Fatalf("VLM user text = %q, want dataset language propagated", gotPrompt)
- }
-}
-
// TestDefaultEmailOutputFormatIsJSON pins diff 2.2: the email family default
// output_format must be "json" (matching Python parser.py:212), not "text".
// With "text" the structured email fields (from/to/subject/attachments/...) are
diff --git a/internal/ingestion/component/parser.go b/internal/ingestion/component/parser.go
index 19c74443f0..836a8d9d9c 100644
--- a/internal/ingestion/component/parser.go
+++ b/internal/ingestion/component/parser.go
@@ -482,24 +482,14 @@ func (c *ParserComponent) Invoke(ctx context.Context, db *gorm.DB, inputs map[st
dispatched = dispatchParse(ctx, fileTypeExt, filename, binary, c.Setups)
dispatched = hydrateEmptyDispatchPayload(dispatched, binary)
- // DOCX vision figure enhancement: on the JSON output path,
- // append vision-model descriptions to embedded image items
- // (doc_type_kwd "image"). Mirrors Python's
- // enhance_media_sections_with_vision in parser.py:_doc.
- dispatched, _, _ = maybeDispatchDOCXVision(ctx, db, fileTypeExt, dispatched, inputs, c.Setups)
-
- // Markdown vision figure enhancement: enrich parsed
- // Markdown JSON items with LLM-generated descriptions of
- // referenced images (![alt](url)). Mirrors Python's
- // enhance_media_sections_with_vision in _Markdown.
- dispatched, _, _ = maybeDispatchMarkdownVision(ctx, db, fileTypeExt, dispatched, inputs)
-
- // PDF vision figure enhancement: enrich parsed PDF JSON
- // items with vision-model descriptions of embedded
- // images/tables (doc_type_kwd "image"/"table" with non-empty
- // image field). Mirrors Python's enhance_media_sections_with_vision
- // in parser.py:_pdf
- dispatched, _, _ = maybeDispatchPDFVisionEnhancement(ctx, db, fileTypeExt, dispatched, inputs)
+ // Vision figure enhancement: on the JSON output path,
+ // append vision-model descriptions to embedded image and
+ // table items. Mirrors Python's enhance_media_sections_with_vision
+ // (rag/flow/parser/utils.py:162, called at parser.py:772/978/1115).
+ // Errors (including context cancellation) are intentionally
+ // discarded — enhancement is best-effort, matching Python's
+ // try/except pass pattern.
+ dispatched, _, _ = maybeDispatchVisionEnhancement(ctx, db, fileTypeExt, dispatched, inputs)
}
// Known/supported families must fail loudly when dispatch or
// parsing breaks. Only unknown families keep the raw-text fallback.
diff --git a/internal/ingestion/component/pdf_vision_dispatch.go b/internal/ingestion/component/pdf_vision_dispatch.go
index 43f5d074e2..9eb7fc7d74 100644
--- a/internal/ingestion/component/pdf_vision_dispatch.go
+++ b/internal/ingestion/component/pdf_vision_dispatch.go
@@ -602,7 +602,7 @@ func dispatchPDFVision(
if err != nil {
return parserDispatchResult{}, fmt.Errorf("parser: pdf vision page %d: %w", page.PageNumber, err)
}
- text := extractPDFVisionAnswer(resp)
+ text := extractVisionAnswer(resp)
positions := [][]any{{page.PageNumber, 0.0, page.WidthPts, 0.0, page.HeightPts}}
items = append(items, map[string]any{
"text": text,
@@ -654,13 +654,6 @@ func buildPDFVisionMessages(prompt string, imageURL string) []modelModule.Messag
}}
}
-func extractPDFVisionAnswer(resp *modelModule.ChatResponse) string {
- if resp == nil || resp.Answer == nil {
- return ""
- }
- return strings.TrimSpace(*resp.Answer)
-}
-
func defaultPDFVisionModelResolver(
ctx context.Context,
db *gorm.DB,
@@ -729,7 +722,7 @@ func pdfVisionPromptsBaseDir() (string, error) {
// renderPDFVisionPrompt only renders page metadata. The full-page PDF vision
// prompt is a transcription contract that preserves the document's original
// language; dataset-language instructions apply to figure descriptions in
-// maybeDispatchPDFVisionEnhancement instead.
+// maybeDispatchVisionEnhancement instead.
func renderPDFVisionPrompt(template string, page int) string {
rendered := strings.ReplaceAll(template, "{{ page }}", fmt.Sprintf("%d", page))
rendered = strings.ReplaceAll(rendered, "{{page}}", fmt.Sprintf("%d", page))
@@ -779,97 +772,3 @@ func defaultIsPaddleOCRLayoutModelID(ctx context.Context, db *gorm.DB, tenantID,
}
return isPaddleOCRDriver(driver)
}
-
-// maybeDispatchPDFVisionEnhancement mirrors Python's
-// enhance_media_sections_with_vision for PDF
-// After the normal PDF parser produces JSON items, this function
-// enriches image/table items by calling the tenant's IMAGE2TEXT
-// model and appending vision descriptions to each item's text field.
-// The markdown/text output paths are not enhanced (Python does the same).
-//
-// This follows the exact same convention as maybeDispatchDOCXVision and
-// maybeDispatchMarkdownVision: it is not gated by a separate setup flag,
-// it simply resolves the tenant's IMAGE2TEXT model and skips silently
-// when none is configured — matching Python's try/except pass behaviour.
-func maybeDispatchPDFVisionEnhancement(
- ctx context.Context,
- db *gorm.DB,
- fileType utility.FileType,
- dispatched parserDispatchResult,
- inputs map[string]any,
-) (parserDispatchResult, bool, error) {
- if fileType != utility.FileTypePDF {
- return dispatched, false, nil
- }
- if dispatched.Err != nil || dispatched.OutputFormat != "json" || len(dispatched.JSON) == 0 {
- return dispatched, false, nil
- }
- tenantID := getStringOr(inputs, "tenant_id", "")
- if tenantID == "" {
- return dispatched, false, nil
- }
- language := resolveVisionLanguage(inputs, "")
- driver, modelName, apiConfig, _, err := resolveTenantModelByType(ctx, db, tenantID, entity.ModelTypeImage2Text)
- if err != nil {
- return dispatched, false, nil
- }
- type target struct{ idx int }
- var targets []target
- for i, item := range dispatched.JSON {
- kd, _ := item["doc_type_kwd"].(string)
- if kd != "image" && kd != "table" {
- continue
- }
- img, _ := item["image"].(string)
- if img == "" {
- continue
- }
- targets = append(targets, target{idx: i})
- }
- if len(targets) == 0 {
- return dispatched, false, nil
- }
- descriptions := make([]string, len(targets))
- var wg sync.WaitGroup
- sem := make(chan struct{}, pdfVisionEnhanceConcurrency)
- for slot, tg := range targets {
- sem <- struct{}{}
- wg.Add(1)
- go func(slot int, itemIdx int) {
- defer wg.Done()
- defer func() { <-sem }()
- img, _ := dispatched.JSON[itemIdx]["image"].(string)
- if img == "" {
- return
- }
- prompt, perr := figureVisionPromptBuilder("", "", language)
- if perr != nil {
- return
- }
- messages := buildVisionMessages(prompt, img)
- resp, ierr := visionChatInvoker(ctx, driver, modelName, messages, apiConfig)
- if ierr != nil {
- return
- }
- descriptions[slot] = extractDOCXVisionAnswer(resp)
- }(slot, tg.idx)
- }
- wg.Wait()
- modified := false
- for slot, tg := range targets {
- desc := strings.TrimSpace(descriptions[slot])
- if desc == "" {
- continue
- }
- existing, _ := dispatched.JSON[tg.idx]["text"].(string)
- if existing != "" {
- dispatched.JSON[tg.idx]["text"] = existing + "\n" + desc
- } else {
- dispatched.JSON[tg.idx]["text"] = desc
- }
- modified = true
- }
- return dispatched, modified, nil
-}
-
-var pdfVisionEnhanceConcurrency = 10
diff --git a/internal/ingestion/component/pdf_vision_dispatch_test.go b/internal/ingestion/component/pdf_vision_dispatch_test.go
index 89df090cd7..9008db90ee 100644
--- a/internal/ingestion/component/pdf_vision_dispatch_test.go
+++ b/internal/ingestion/component/pdf_vision_dispatch_test.go
@@ -21,10 +21,8 @@ import (
"ragflow/internal/common"
"ragflow/internal/dao"
- "ragflow/internal/entity"
modelModule "ragflow/internal/entity/models"
"ragflow/internal/ingestion/component/schema"
- "ragflow/internal/utility"
"gorm.io/gorm"
)
@@ -108,62 +106,6 @@ func TestDispatchPaddleOCRPdfEmptyTextFails(t *testing.T) {
}
}
-func TestMaybeDispatchPDFVisionEnhancementForwardsDatasetLanguage(t *testing.T) {
- origResolver := resolveTenantModelByType
- origInvoker := visionChatInvoker
- origPrompt := figureVisionPromptBuilder
- t.Cleanup(func() {
- resolveTenantModelByType = origResolver
- visionChatInvoker = origInvoker
- figureVisionPromptBuilder = origPrompt
- })
-
- resolveTenantModelByType = func(context.Context, *gorm.DB, string, entity.ModelType) (modelModule.ModelDriver, string, *modelModule.APIConfig, int, error) {
- return &docxVisionFakeDriver{}, "pdf-vision-model", &modelModule.APIConfig{}, 0, nil
- }
- invoker := &docxVisionCaptureInvoker{}
- visionChatInvoker = invoker.invoke
- capturedLanguage := ""
- figureVisionPromptBuilder = func(_, _, language string) (string, error) {
- capturedLanguage = language
- return "describe the figure", nil
- }
-
- dispatched := parserDispatchResult{
- OutputFormat: "json",
- DocType: "pdf",
- JSON: []map[string]any{
- {"text": "caption", "image": "data:image/png;base64,aW1hZ2U=", "doc_type_kwd": "image"},
- },
- }
-
- res, modified, err := maybeDispatchPDFVisionEnhancement(
- t.Context(),
- dao.DB,
- utility.FileTypePDF,
- dispatched,
- map[string]any{"tenant_id": "t1", "lang": "Dutch"},
- )
- if err != nil {
- t.Fatalf("maybeDispatchPDFVisionEnhancement: %v", err)
- }
- if !modified {
- t.Fatal("modified = false, want true")
- }
- if capturedLanguage != "Dutch" {
- t.Fatalf("figure prompt language = %q, want Dutch", capturedLanguage)
- }
- if got := res.JSON[0]["text"]; got != "caption\na diagram of a pipeline" {
- t.Fatalf("enhanced text = %q", got)
- }
- if len(invoker.images) != 1 {
- t.Fatalf("vision invoker called %d times, want 1", len(invoker.images))
- }
- if want := "data:image/png;base64,aW1hZ2U="; invoker.images[0] != want {
- t.Fatalf("vision image data URI = %q, want %q (no double data: prefix)", invoker.images[0], want)
- }
-}
-
// TestIsNamedPDFParseMethodWhitelistAligned verifies that the runtime
// "named parse_method" classifier agrees with (*ParserComponent).Check()'s
// PDF whitelist (parser.go:200-203):
diff --git a/internal/ingestion/component/vision_enhancement.go b/internal/ingestion/component/vision_enhancement.go
new file mode 100644
index 0000000000..f7b5167d8b
--- /dev/null
+++ b/internal/ingestion/component/vision_enhancement.go
@@ -0,0 +1,288 @@
+//
+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+// Package component — vision figure enhancement: enriches parsed JSON items
+// (PDF / DOCX / Markdown, OutputFormat=="json") with vision-model descriptions
+// of embedded images/tables. Mirrors Python's enhance_media_sections_with_vision
+// (rag/flow/parser/utils.py:162, called at parser.py:772/978/1115).
+
+package component
+
+import (
+ "context"
+ "fmt"
+ "os"
+ "path/filepath"
+ "regexp"
+ "strings"
+ "sync"
+
+ pdflayout "ragflow/internal/deepdoc/parser/pdf/layout"
+ "ragflow/internal/entity"
+ modelModule "ragflow/internal/entity/models"
+ "ragflow/internal/utility"
+
+ "gorm.io/gorm"
+)
+
+var (
+ figureVisionPromptBuilder func(language string) (string, error) = buildFigureVisionPrompt
+ visionChatInvoker = defaultVisionChatInvoker
+)
+
+const (
+ figureVisionPromptFile = "vision_llm_figure_describe_prompt.md"
+ visionEnhancementConcurrency int = 10
+)
+
+var (
+ figureVisionPromptsBase string
+ figureVisionPromptsOnce sync.Once
+ figureVisionPromptCache = make(map[string]string)
+ figureVisionPromptMu sync.RWMutex
+)
+
+// cleanMarkdownBlock mirrors Python common/string_utils.py:clean_markdown_block
+//
+// re.sub(r"^\s*```markdown\s*\n?", "", text)
+// re.sub(r"\n?\s*```\s*$", "", text)
+//
+// Matches Python without re.MULTILINE so ^/$ anchor only the whole text.
+var (
+ reMarkdownOpen = regexp.MustCompile(`^\s*` + "```" + `markdown\s*\n?`)
+ reMarkdownClose = regexp.MustCompile(`\n?\s*` + "```" + `\s*$`)
+)
+
+func cleanMarkdownBlock(s string) string {
+ s = reMarkdownOpen.ReplaceAllString(s, "")
+ s = reMarkdownClose.ReplaceAllString(s, "")
+ return strings.TrimSpace(s)
+}
+
+// isVisionEnhancementAllowed mirrors Python's 3 call sites in
+// rag/flow/parser/parser.py:772/978/1115 (PDF/DOCX/Markdown JSON branches).
+func isVisionEnhancementAllowed(fileType utility.FileType) bool {
+ switch fileType {
+ case utility.FileTypePDF, utility.FileTypeDOCX, utility.FileTypeMarkdown:
+ return true
+ default:
+ return false
+ }
+}
+
+// maybeDispatchVisionEnhancement enriches parsed JSON items with vision-model
+// descriptions of embedded images and tables (doc_type_kwd in {"image", "table"}
+// with non-empty image field).
+// Mirrors Python's enhance_media_sections_with_vision in rag/flow/parser/utils.py:162.
+func maybeDispatchVisionEnhancement(
+ ctx context.Context,
+ db *gorm.DB,
+ fileType utility.FileType,
+ dispatched parserDispatchResult,
+ inputs map[string]any,
+) (parserDispatchResult, bool, error) {
+ // 0. FileType allowlist guard.
+ if !isVisionEnhancementAllowed(fileType) {
+ return dispatched, false, nil
+ }
+ // Only enhance successful JSON output format containing items.
+ if dispatched.Err != nil || dispatched.OutputFormat != "json" || len(dispatched.JSON) == 0 {
+ return dispatched, false, nil
+ }
+
+ tenantID := getStringOr(inputs, "tenant_id", "")
+ if tenantID == "" {
+ return dispatched, false, nil
+ }
+ language := resolveVisionLanguage(inputs, "")
+
+ // 1. Collect target items (images/tables with non-empty image data).
+ type target struct {
+ idx int
+ }
+ var targets []target
+ for i, item := range dispatched.JSON {
+ kd, _ := item["doc_type_kwd"].(string)
+ if kd != "image" && kd != "table" {
+ continue
+ }
+ img, _ := item["image"].(string)
+ if img == "" {
+ continue
+ }
+ targets = append(targets, target{idx: i})
+ }
+ if len(targets) == 0 {
+ return dispatched, false, nil
+ }
+
+ // 2. Resolve tenant's IMAGE2TEXT model.
+ driver, modelName, apiConfig, _, err := resolveTenantModelByType(ctx, db, tenantID, entity.ModelTypeImage2Text)
+ if err != nil {
+ // Model not available — skip vision enhancement silently, matching Python's try/except pass.
+ return dispatched, false, nil
+ }
+
+ // Hoist prompt once: language is invariant across items.
+ prompt, perr := figureVisionPromptBuilder(language)
+ if perr != nil {
+ //nolint:nilerr // Vision enhancement is best-effort.
+ return dispatched, false, nil
+ }
+
+ // 3. Concurrently invoke VLM — acquire semaphore before launching goroutine
+ // so live goroutine count is bounded by visionEnhancementConcurrency.
+ // Stop scheduling new VLM calls after cancellation; already-running calls
+ // finish via wg.Wait() and cancellation propagates via ctx through
+ // visionChatInvoker.
+ descriptions := make([]string, len(targets))
+ var wg sync.WaitGroup
+ sem := make(chan struct{}, visionEnhancementConcurrency)
+
+dispatch:
+ for slot, tg := range targets {
+ if ctx.Err() != nil {
+ break dispatch
+ }
+ select {
+ case sem <- struct{}{}:
+ case <-ctx.Done():
+ break dispatch
+ }
+ wg.Add(1)
+ go func(slot int, itemIdx int) {
+ defer wg.Done()
+ defer func() { <-sem }()
+
+ img, _ := dispatched.JSON[itemIdx]["image"].(string)
+ if img == "" {
+ return
+ }
+ messages := buildVisionMessages(prompt, img)
+ resp, ierr := visionChatInvoker(ctx, driver, modelName, messages, apiConfig)
+ if ierr != nil {
+ return
+ }
+ descriptions[slot] = extractVisionAnswer(resp)
+ }(slot, tg.idx)
+ }
+ wg.Wait()
+ if err := ctx.Err(); err != nil {
+ return dispatched, false, err
+ }
+
+ // 4. Append descriptions to item text (single newline \n, matching Python).
+ modified := false
+ for slot, tg := range targets {
+ desc := strings.TrimSpace(descriptions[slot])
+ if desc == "" {
+ continue
+ }
+ existing, _ := dispatched.JSON[tg.idx]["text"].(string)
+ if existing != "" {
+ dispatched.JSON[tg.idx]["text"] = existing + "\n" + desc
+ } else {
+ dispatched.JSON[tg.idx]["text"] = desc
+ }
+ modified = true
+ }
+
+ return dispatched, modified, nil
+}
+
+func buildFigureVisionPrompt(language string) (string, error) {
+ template, err := loadFigureVisionPromptFile(figureVisionPromptFile)
+ if err != nil {
+ return "", err
+ }
+ return renderFigureVisionLanguage(template, language), nil
+}
+
+func loadFigureVisionPromptFile(filename string) (string, error) {
+ figureVisionPromptMu.RLock()
+ if cached, ok := figureVisionPromptCache[filename]; ok {
+ figureVisionPromptMu.RUnlock()
+ return cached, nil
+ }
+ figureVisionPromptMu.RUnlock()
+
+ baseDir, err := figureVisionPromptsBaseDir()
+ if err != nil {
+ return "", err
+ }
+ promptPath := filepath.Join(baseDir, "rag", "prompts", filename)
+ content, err := os.ReadFile(promptPath)
+ if err != nil {
+ return "", fmt.Errorf("figure vision prompt %q: %w", filename, err)
+ }
+ cached := strings.TrimSpace(string(content))
+ figureVisionPromptMu.Lock()
+ figureVisionPromptCache[filename] = cached
+ figureVisionPromptMu.Unlock()
+ return cached, nil
+}
+
+func figureVisionPromptsBaseDir() (string, error) {
+ var initErr error
+ figureVisionPromptsOnce.Do(func() {
+ root := utility.GetProjectRoot()
+ if _, statErr := os.Stat(filepath.Join(root, "rag", "prompts")); statErr == nil {
+ figureVisionPromptsBase = root
+ return
+ }
+ initErr = fmt.Errorf("rag/prompts not found under project root %q", root)
+ })
+ if initErr != nil {
+ return "", initErr
+ }
+ return figureVisionPromptsBase, nil
+}
+
+func buildVisionMessages(prompt, imageBase64 string) []modelModule.Message {
+ dataURI := pdflayout.InlinePNGDataURL(imageBase64)
+ if dataURI == "" {
+ dataURI = strings.TrimSpace(imageBase64)
+ }
+ return []modelModule.Message{{
+ Role: "user",
+ Content: []interface{}{
+ map[string]any{"type": "text", "text": prompt},
+ map[string]any{"type": "image_url", "image_url": map[string]any{"url": dataURI}},
+ },
+ }}
+}
+
+// extractVisionAnswer strips ```markdown fences from the VLM response.
+// Python calls clean_markdown_block inside vision_llm_chunk (rag/app/picture.py:198,
+// i.e., inside the invoker wrapper). Go applies it here post-invoker so that
+// visionChatInvoker stays a pure, swappable test seam — output is equivalent.
+func extractVisionAnswer(resp *modelModule.ChatResponse) string {
+ if resp == nil || resp.Answer == nil {
+ return ""
+ }
+ return cleanMarkdownBlock(*resp.Answer)
+}
+
+func defaultVisionChatInvoker(
+ ctx context.Context,
+ driver modelModule.ModelDriver,
+ modelName string,
+ messages []modelModule.Message,
+ apiConfig *modelModule.APIConfig,
+) (*modelModule.ChatResponse, error) {
+ vision := true
+ return driver.ChatWithMessages(ctx, modelName, messages, apiConfig, &modelModule.ChatConfig{Vision: &vision}, nil)
+}
diff --git a/internal/ingestion/component/vision_enhancement_test.go b/internal/ingestion/component/vision_enhancement_test.go
new file mode 100644
index 0000000000..aa55008eb1
--- /dev/null
+++ b/internal/ingestion/component/vision_enhancement_test.go
@@ -0,0 +1,711 @@
+//
+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+package component
+
+import (
+ "context"
+ "errors"
+ "fmt"
+ "sync"
+ "testing"
+
+ "ragflow/internal/dao"
+ "ragflow/internal/entity"
+ modelModule "ragflow/internal/entity/models"
+ "ragflow/internal/utility"
+
+ "gorm.io/gorm"
+)
+
+type visionEnhanceFakeDriver struct {
+ modelModule.ModelDriver
+}
+
+type visionEnhanceCaptureInvoker struct {
+ mu sync.Mutex
+ images []string
+ captured []modelModule.Message
+}
+
+func (c *visionEnhanceCaptureInvoker) invoke(
+ ctx context.Context,
+ driver modelModule.ModelDriver,
+ modelName string,
+ messages []modelModule.Message,
+ apiConfig *modelModule.APIConfig,
+) (*modelModule.ChatResponse, error) {
+ c.mu.Lock()
+ c.captured = append(c.captured, messages...)
+ if parts, ok := messages[0].Content.([]interface{}); ok && len(parts) >= 2 {
+ if img, ok := parts[1].(map[string]any); ok {
+ if url, ok := img["image_url"].(map[string]any); ok {
+ if u, ok := url["url"].(string); ok {
+ c.images = append(c.images, u)
+ }
+ }
+ }
+ }
+ c.mu.Unlock()
+ ans := "```markdown\na diagram of a pipeline\n```"
+ return &modelModule.ChatResponse{Answer: &ans}, nil
+}
+
+// swapVisionGlobals replaces injectable vars and restores them on t.Cleanup.
+func swapVisionGlobals(
+ t *testing.T,
+ resolver func(context.Context, *gorm.DB, string, entity.ModelType) (modelModule.ModelDriver, string, *modelModule.APIConfig, int, error),
+ invoker func(context.Context, modelModule.ModelDriver, string, []modelModule.Message, *modelModule.APIConfig) (*modelModule.ChatResponse, error),
+ prompt func(string) (string, error),
+) {
+ t.Helper()
+ origResolver := resolveTenantModelByType
+ origInvoker := visionChatInvoker
+ origPrompt := figureVisionPromptBuilder
+ t.Cleanup(func() {
+ resolveTenantModelByType = origResolver
+ visionChatInvoker = origInvoker
+ figureVisionPromptBuilder = origPrompt
+ })
+ if resolver != nil {
+ resolveTenantModelByType = resolver
+ }
+ if invoker != nil {
+ visionChatInvoker = invoker
+ }
+ if prompt != nil {
+ figureVisionPromptBuilder = prompt
+ }
+}
+
+func fakeResolver(_ context.Context, _ *gorm.DB, _ string, _ entity.ModelType) (modelModule.ModelDriver, string, *modelModule.APIConfig, int, error) {
+ return &visionEnhanceFakeDriver{}, "vision-model", &modelModule.APIConfig{}, 0, nil
+}
+
+func fakePrompt(language string) (string, error) {
+ return "describe the figure in " + language, nil
+}
+
+func TestVisionEnhancement_EnhancesJSONImagesAndTables(t *testing.T) {
+ testCases := []struct {
+ name string
+ fileType utility.FileType
+ }{
+ {"DOCX", utility.FileTypeDOCX},
+ {"PDF", utility.FileTypePDF},
+ {"Markdown", utility.FileTypeMarkdown},
+ }
+
+ for _, tc := range testCases {
+ t.Run(tc.name, func(t *testing.T) {
+ var capturedLanguage string
+ // Per-subtest invoker and prompt — avoids implicit coupling between subtests.
+ invoker := &visionEnhanceCaptureInvoker{}
+ swapVisionGlobals(t, fakeResolver, invoker.invoke, func(language string) (string, error) {
+ capturedLanguage = language
+ return "describe the figure in " + language, nil
+ })
+
+ dispatched := parserDispatchResult{
+ OutputFormat: "json",
+ DocType: string(tc.fileType),
+ JSON: []map[string]any{
+ {"text": "Intro paragraph", "image": nil, "doc_type_kwd": "text"},
+ {"text": "", "image": "aGVsbG8taW1hZ2U=", "doc_type_kwd": "image"},
+ {"text": "existing table", "image": "dGFibGUtaW1hZ2U=", "doc_type_kwd": "table"},
+ {"text": "<table></table>", "image": nil, "doc_type_kwd": "table"},
+ },
+ }
+
+ res, handled, err := maybeDispatchVisionEnhancement(
+ t.Context(),
+ dao.DB,
+ tc.fileType,
+ dispatched,
+ map[string]any{"tenant_id": "t1", "lang": "Japanese"},
+ )
+ if err != nil {
+ t.Fatalf("unexpected error: %v", err)
+ }
+ if !handled {
+ t.Fatal("handled = false, want true")
+ }
+ if len(res.JSON) != 4 {
+ t.Fatalf("JSON len = %d, want 4", len(res.JSON))
+ }
+ if got := res.JSON[0]["text"].(string); got != "Intro paragraph" {
+ t.Errorf("text item text = %q, want unchanged", got)
+ }
+ // image item: VLM description cleaned of ```markdown and appended
+ if got, _ := res.JSON[1]["text"].(string); got != "a diagram of a pipeline" {
+ t.Errorf("image item text = %q, want 'a diagram of a pipeline'", got)
+ }
+ // table item with image: VLM description appended with single \n
+ if got, _ := res.JSON[2]["text"].(string); got != "existing table\na diagram of a pipeline" {
+ t.Errorf("table item text = %q, want 'existing table\\na diagram of a pipeline'", got)
+ }
+ if got, _ := res.JSON[3]["text"].(string); got != "<table></table>" {
+ t.Errorf("table item without image text = %q, want unchanged", got)
+ }
+ if capturedLanguage != "Japanese" {
+ t.Errorf("figure prompt language = %q, want Japanese", capturedLanguage)
+ }
+ })
+ }
+}
+
+func TestVisionEnhancement_MarkdownOutputUntouched(t *testing.T) {
+ called := false
+ swapVisionGlobals(t,
+ func(_ context.Context, _ *gorm.DB, _ string, _ entity.ModelType) (modelModule.ModelDriver, string, *modelModule.APIConfig, int, error) {
+ called = true
+ return &visionEnhanceFakeDriver{}, "m", &modelModule.APIConfig{}, 0, nil
+ },
+ func(_ context.Context, _ modelModule.ModelDriver, _ string, _ []modelModule.Message, _ *modelModule.APIConfig) (*modelModule.ChatResponse, error) {
+ called = true
+ ans := "x"
+ return &modelModule.ChatResponse{Answer: &ans}, nil
+ },
+ nil,
+ )
+
+ dispatched := parserDispatchResult{
+ OutputFormat: "markdown",
+ DocType: "docx",
+ Markdown: "![Image](data:image/png;base64,abc)",
+ File: map[string]any{"figures": []map[string]any{{"image": "abc", "marker": "x"}}},
+ }
+
+ res, handled, err := maybeDispatchVisionEnhancement(
+ t.Context(),
+ dao.DB,
+ utility.FileTypeDOCX,
+ dispatched,
+ map[string]any{"tenant_id": "t1"},
+ )
+ if err != nil {
+ t.Fatalf("unexpected error: %v", err)
+ }
+ if handled {
+ t.Error("handled = true, want false (markdown path must not be enhanced)")
+ }
+ if called {
+ t.Error("vision model was resolved/invoked on the markdown path")
+ }
+ if res.Markdown != "![Image](data:image/png;base64,abc)" {
+ t.Errorf("markdown mutated: %q", res.Markdown)
+ }
+}
+
+func TestVisionEnhancement_NonAllowedFileTypeSkipped(t *testing.T) {
+ dispatched := parserDispatchResult{
+ OutputFormat: "json",
+ DocType: "other",
+ JSON: []map[string]any{
+ {"text": "", "image": "aGVsbG8=", "doc_type_kwd": "image"},
+ },
+ }
+
+ res, handled, err := maybeDispatchVisionEnhancement(
+ t.Context(),
+ dao.DB,
+ utility.FileTypeOTHER,
+ dispatched,
+ map[string]any{"tenant_id": "t1"},
+ )
+ if err != nil {
+ t.Fatalf("unexpected error: %v", err)
+ }
+ if handled {
+ t.Error("handled = true, want false for FileTypeOTHER")
+ }
+ if res.JSON[0]["text"] != "" {
+ t.Errorf("text = %q, want empty", res.JSON[0]["text"])
+ }
+}
+
+func TestVisionEnhancement_EmptyOrNoTenantSkipped(t *testing.T) {
+ dispatched := parserDispatchResult{
+ OutputFormat: "json",
+ DocType: "docx",
+ JSON: []map[string]any{
+ {"text": "", "image": "aGVsbG8=", "doc_type_kwd": "image"},
+ },
+ }
+
+ // No tenant_id
+ res, handled, err := maybeDispatchVisionEnhancement(
+ t.Context(),
+ dao.DB,
+ utility.FileTypeDOCX,
+ dispatched,
+ map[string]any{},
+ )
+ if err != nil || handled {
+ t.Errorf("handled=%v, err=%v, want false, nil for missing tenant_id", handled, err)
+ }
+ if res.JSON[0]["text"] != "" {
+ t.Errorf("text = %q, want untouched", res.JSON[0]["text"])
+ }
+}
+
+// TestVisionEnhancement_DispatchedErrSkipped ensures a failed parse result is
+// returned unchanged — enhancement must not touch items when dispatched.Err != nil.
+func TestVisionEnhancement_DispatchedErrSkipped(t *testing.T) {
+ parseErr := errors.New("parse failed")
+ dispatched := parserDispatchResult{
+ Err: parseErr,
+ OutputFormat: "json",
+ JSON: []map[string]any{
+ {"text": "", "image": "aGVsbG8=", "doc_type_kwd": "image"},
+ },
+ }
+
+ res, handled, err := maybeDispatchVisionEnhancement(
+ t.Context(),
+ dao.DB,
+ utility.FileTypePDF,
+ dispatched,
+ map[string]any{"tenant_id": "t1"},
+ )
+ if err != nil {
+ t.Fatalf("unexpected error: %v", err)
+ }
+ if handled {
+ t.Error("handled = true, want false when dispatched.Err is set")
+ }
+ if res.Err != parseErr {
+ t.Errorf("res.Err = %v, want original parse error preserved", res.Err)
+ }
+}
+
+// TestVisionEnhancement_ContextCancellation verifies that a cancelled context
+// propagates through visionChatInvoker and the enhancement is skipped without panic.
+func TestVisionEnhancement_ContextCancellation(t *testing.T) {
+ swapVisionGlobals(t, fakeResolver,
+ func(ctx context.Context, _ modelModule.ModelDriver, _ string, _ []modelModule.Message, _ *modelModule.APIConfig) (*modelModule.ChatResponse, error) {
+ return nil, ctx.Err()
+ },
+ fakePrompt,
+ )
+
+ ctx, cancel := context.WithCancel(t.Context())
+ cancel() // pre-cancel
+
+ dispatched := parserDispatchResult{
+ OutputFormat: "json",
+ JSON: []map[string]any{
+ {"text": "", "image": "aGVsbG8=", "doc_type_kwd": "image"},
+ },
+ }
+
+ res, handled, err := maybeDispatchVisionEnhancement(
+ ctx,
+ dao.DB,
+ utility.FileTypePDF,
+ dispatched,
+ map[string]any{"tenant_id": "t1"},
+ )
+ if !errors.Is(err, context.Canceled) {
+ t.Fatalf("err = %v, want context.Canceled", err)
+ }
+ if handled {
+ t.Error("handled = true with cancelled context, want false")
+ }
+ // text field should be untouched since invoker returned no answer.
+ if got, _ := res.JSON[0]["text"].(string); got != "" {
+ t.Errorf("text = %q, want empty (no successful VLM response)", got)
+ }
+}
+
+// TestVisionEnhancement_NonStringImageFieldFiltered verifies that items with a
+// non-string image field (e.g., int) are silently skipped during target collection.
+func TestVisionEnhancement_NonStringImageFieldFiltered(t *testing.T) {
+ invokerCalled := false
+ swapVisionGlobals(t, fakeResolver,
+ func(_ context.Context, _ modelModule.ModelDriver, _ string, _ []modelModule.Message, _ *modelModule.APIConfig) (*modelModule.ChatResponse, error) {
+ invokerCalled = true
+ ans := "description"
+ return &modelModule.ChatResponse{Answer: &ans}, nil
+ },
+ fakePrompt,
+ )
+
+ dispatched := parserDispatchResult{
+ OutputFormat: "json",
+ JSON: []map[string]any{
+ // non-string image field — filtered by target collector
+ {"text": "para", "image": 12345, "doc_type_kwd": "image"},
+ // nil image — also filtered
+ {"text": "para2", "image": nil, "doc_type_kwd": "image"},
+ },
+ }
+
+ _, handled, err := maybeDispatchVisionEnhancement(
+ t.Context(),
+ dao.DB,
+ utility.FileTypePDF,
+ dispatched,
+ map[string]any{"tenant_id": "t1"},
+ )
+ if err != nil {
+ t.Fatalf("unexpected error: %v", err)
+ }
+ if handled {
+ t.Error("handled = true, want false — no valid image targets")
+ }
+ if invokerCalled {
+ t.Error("invoker was called despite no valid image targets")
+ }
+}
+
+// TestVisionEnhancement_MoreThanConcurrencyItems verifies that N > visionEnhancementConcurrency
+// items are all processed without deadlock.
+func TestVisionEnhancement_MoreThanConcurrencyItems(t *testing.T) {
+ n := visionEnhancementConcurrency + 5
+ swapVisionGlobals(t, fakeResolver,
+ func(_ context.Context, _ modelModule.ModelDriver, _ string, _ []modelModule.Message, _ *modelModule.APIConfig) (*modelModule.ChatResponse, error) {
+ ans := "desc"
+ return &modelModule.ChatResponse{Answer: &ans}, nil
+ },
+ fakePrompt,
+ )
+
+ items := make([]map[string]any, n)
+ for i := range items {
+ items[i] = map[string]any{
+ "text": fmt.Sprintf("item%d", i),
+ "image": "aGVsbG8=",
+ "doc_type_kwd": "image",
+ }
+ }
+ dispatched := parserDispatchResult{
+ OutputFormat: "json",
+ JSON: items,
+ }
+
+ res, handled, err := maybeDispatchVisionEnhancement(
+ t.Context(),
+ dao.DB,
+ utility.FileTypePDF,
+ dispatched,
+ map[string]any{"tenant_id": "t1"},
+ )
+ if err != nil {
+ t.Fatalf("unexpected error: %v", err)
+ }
+ if !handled {
+ t.Error("handled = false, want true")
+ }
+ // All n items should have received a description appended.
+ for i, item := range res.JSON {
+ got, _ := item["text"].(string)
+ if got == fmt.Sprintf("item%d", i) {
+ t.Errorf("item[%d] text unchanged %q, expected description appended", i, got)
+ }
+ }
+}
+
+// TestVisionEnhancement_PlainTextResponseNotTruncated verifies that a plain-text
+// (no fences) VLM response is returned verbatim — cleanMarkdownBlock must not truncate it.
+func TestVisionEnhancement_PlainTextResponseNotTruncated(t *testing.T) {
+ plain := "A pipeline diagram showing three stages."
+ swapVisionGlobals(t, fakeResolver,
+ func(_ context.Context, _ modelModule.ModelDriver, _ string, _ []modelModule.Message, _ *modelModule.APIConfig) (*modelModule.ChatResponse, error) {
+ return &modelModule.ChatResponse{Answer: &plain}, nil
+ },
+ fakePrompt,
+ )
+
+ dispatched := parserDispatchResult{
+ OutputFormat: "json",
+ JSON: []map[string]any{
+ {"text": "", "image": "aGVsbG8=", "doc_type_kwd": "image"},
+ },
+ }
+
+ res, handled, err := maybeDispatchVisionEnhancement(
+ t.Context(),
+ dao.DB,
+ utility.FileTypePDF,
+ dispatched,
+ map[string]any{"tenant_id": "t1"},
+ )
+ if err != nil {
+ t.Fatalf("unexpected error: %v", err)
+ }
+ if !handled {
+ t.Error("handled = false, want true")
+ }
+ if got, _ := res.JSON[0]["text"].(string); got != plain {
+ t.Errorf("plain text response = %q, want %q (must not be truncated)", got, plain)
+ }
+}
+
+// TestVisionEnhancement_PromptBuilderErrorSkipped verifies that a prompt build
+// failure skips enhancement silently (best-effort, nilerr is intentional).
+func TestVisionEnhancement_PromptBuilderErrorSkipped(t *testing.T) {
+ swapVisionGlobals(t,
+ fakeResolver,
+ func(_ context.Context, _ modelModule.ModelDriver, _ string, _ []modelModule.Message, _ *modelModule.APIConfig) (*modelModule.ChatResponse, error) {
+ t.Error("invoker should not be called when prompt builder fails")
+ return nil, nil
+ },
+ func(_ string) (string, error) {
+ return "", errors.New("prompt load failed")
+ },
+ )
+
+ dispatched := parserDispatchResult{
+ OutputFormat: "json",
+ JSON: []map[string]any{
+ {"text": "", "image": "aGVsbG8=", "doc_type_kwd": "image"},
+ },
+ }
+
+ res, handled, err := maybeDispatchVisionEnhancement(
+ t.Context(),
+ dao.DB,
+ utility.FileTypePDF,
+ dispatched,
+ map[string]any{"tenant_id": "t1"},
+ )
+ if err != nil {
+ t.Fatalf("unexpected error: %v", err)
+ }
+ if handled {
+ t.Error("handled = true, want false when prompt builder fails")
+ }
+ if got, _ := res.JSON[0]["text"].(string); got != "" {
+ t.Errorf("text = %q, want empty", got)
+ }
+}
+
+// TestVisionEnhancement_ModelResolveFailureSkipped verifies that a resolver error
+// (e.g., tenant has no IMAGE2TEXT model) skips enhancement silently.
+func TestVisionEnhancement_ModelResolveFailureSkipped(t *testing.T) {
+ swapVisionGlobals(t,
+ func(_ context.Context, _ *gorm.DB, _ string, _ entity.ModelType) (modelModule.ModelDriver, string, *modelModule.APIConfig, int, error) {
+ return nil, "", nil, 0, errors.New("no model")
+ },
+ func(_ context.Context, _ modelModule.ModelDriver, _ string, _ []modelModule.Message, _ *modelModule.APIConfig) (*modelModule.ChatResponse, error) {
+ t.Error("invoker should not be called when model resolve fails")
+ return nil, nil
+ },
+ fakePrompt,
+ )
+
+ dispatched := parserDispatchResult{
+ OutputFormat: "json",
+ JSON: []map[string]any{
+ {"text": "", "image": "aGVsbG8=", "doc_type_kwd": "image"},
+ },
+ }
+
+ res, handled, err := maybeDispatchVisionEnhancement(
+ t.Context(),
+ dao.DB,
+ utility.FileTypePDF,
+ dispatched,
+ map[string]any{"tenant_id": "t1"},
+ )
+ if err != nil {
+ t.Fatalf("unexpected error: %v", err)
+ }
+ if handled {
+ t.Error("handled = true, want false when model resolve fails")
+ }
+ if got, _ := res.JSON[0]["text"].(string); got != "" {
+ t.Errorf("text = %q, want empty", got)
+ }
+}
+
+// TestVisionEnhancement_CancellationStopsSchedulingWithManyItems verifies that
+// cancellation stops scheduling when N > visionEnhancementConcurrency, without deadlock.
+func TestVisionEnhancement_CancellationStopsSchedulingWithManyItems(t *testing.T) {
+ n := visionEnhancementConcurrency + 5
+ swapVisionGlobals(t, fakeResolver,
+ func(ctx context.Context, _ modelModule.ModelDriver, _ string, _ []modelModule.Message, _ *modelModule.APIConfig) (*modelModule.ChatResponse, error) {
+ // Simulate work that respects ctx.
+ select {
+ case <-ctx.Done():
+ return nil, ctx.Err()
+ default:
+ }
+ ans := "desc"
+ return &modelModule.ChatResponse{Answer: &ans}, nil
+ },
+ fakePrompt,
+ )
+
+ items := make([]map[string]any, n)
+ for i := range items {
+ items[i] = map[string]any{
+ "text": fmt.Sprintf("item%d", i),
+ "image": "aGVsbG8=",
+ "doc_type_kwd": "image",
+ }
+ }
+ dispatched := parserDispatchResult{
+ OutputFormat: "json",
+ JSON: items,
+ }
+
+ ctx, cancel := context.WithCancel(t.Context())
+ cancel() // pre-cancel — dispatch loop should break immediately
+
+ res, handled, err := maybeDispatchVisionEnhancement(
+ ctx,
+ dao.DB,
+ utility.FileTypePDF,
+ dispatched,
+ map[string]any{"tenant_id": "t1"},
+ )
+ if !errors.Is(err, context.Canceled) {
+ t.Fatalf("err = %v, want context.Canceled", err)
+ }
+ if handled {
+ t.Error("handled = true with cancelled context, want false")
+ }
+ // No item should be modified when dispatch is cancelled before scheduling.
+ for i, item := range res.JSON {
+ if got, _ := item["text"].(string); got != fmt.Sprintf("item%d", i) {
+ t.Errorf("item[%d] text = %q, want untouched after cancellation", i, got)
+ }
+ }
+}
+
+func TestBuildVisionMessages_PreventsDoublePrefix(t *testing.T) {
+ cases := []struct {
+ name string
+ input string
+ wantURL string
+ }{
+ {
+ name: "raw base64 gets png prefix",
+ input: "aGVsbG8=",
+ wantURL: "data:image/png;base64,aGVsbG8=",
+ },
+ {
+ name: "already png data uri is preserved without double prefix",
+ input: "data:image/png;base64,aGVsbG8=",
+ wantURL: "data:image/png;base64,aGVsbG8=",
+ },
+ {
+ name: "jpeg data uri is preserved as jpeg",
+ input: "data:image/jpeg;base64,/9j/4AAQ",
+ wantURL: "data:image/jpeg;base64,/9j/4AAQ",
+ },
+ {
+ name: "https url is preserved",
+ input: "https://example.com/figure.png",
+ wantURL: "https://example.com/figure.png",
+ },
+ {
+ name: "whitespace trimmed",
+ input: " data:image/png;base64,abc ",
+ wantURL: "data:image/png;base64,abc",
+ },
+ }
+
+ for _, tc := range cases {
+ t.Run(tc.name, func(t *testing.T) {
+ msgs := buildVisionMessages("describe", tc.input)
+ if len(msgs) != 1 {
+ t.Fatalf("len(msgs) = %d, want 1", len(msgs))
+ }
+ parts, ok := msgs[0].Content.([]interface{})
+ if !ok || len(parts) < 2 {
+ t.Fatalf("Content parts = %+v, want >= 2", msgs[0].Content)
+ }
+ img, ok := parts[1].(map[string]any)
+ if !ok {
+ t.Fatalf("parts[1] = %+v, want map[string]any", parts[1])
+ }
+ imgURL, ok := img["image_url"].(map[string]any)
+ if !ok {
+ t.Fatalf("img[image_url] = %+v, want map[string]any", img["image_url"])
+ }
+ got, _ := imgURL["url"].(string)
+ if got != tc.wantURL {
+ t.Errorf("image_url.url = %q, want %q", got, tc.wantURL)
+ }
+ })
+ }
+}
+
+func TestExtractVisionAnswer_CleansMarkdownBlock(t *testing.T) {
+ ans := "```markdown\n| Header 1 | Header 2 |\n| --- | --- |\n| Cell 1 | Cell 2 |\n```"
+ resp := &modelModule.ChatResponse{Answer: &ans}
+ got := extractVisionAnswer(resp)
+ want := "| Header 1 | Header 2 |\n| --- | --- |\n| Cell 1 | Cell 2 |"
+ if got != want {
+ t.Errorf("extractVisionAnswer = %q, want %q", got, want)
+ }
+}
+
+func TestCleanMarkdownBlock_EdgeCases(t *testing.T) {
+ cases := []struct {
+ name string
+ input string
+ want string
+ }{
+ {
+ name: "standard markdown block",
+ input: "```markdown\nHello world\n```",
+ want: "Hello world",
+ },
+ {
+ name: "windows CRLF endings",
+ input: "```markdown\r\nHello world\r\n```",
+ want: "Hello world",
+ },
+ {
+ name: "nested code blocks preserved",
+ input: "```markdown\nText with ```nested``` blocks\n```",
+ want: "Text with ```nested``` blocks",
+ },
+ {
+ name: "mixed whitespace tabs",
+ input: "\t```markdown\t\n\tContent with tabs\n\t```\t",
+ want: "Content with tabs",
+ },
+ {
+ name: "empty string",
+ input: "",
+ want: "",
+ },
+ {
+ name: "only markdown tags",
+ input: "```markdown```",
+ want: "",
+ },
+ {
+ name: "unwrapped text unchanged",
+ input: "Just plain text without markdown block",
+ want: "Just plain text without markdown block",
+ },
+ }
+
+ for _, tc := range cases {
+ t.Run(tc.name, func(t *testing.T) {
+ got := cleanMarkdownBlock(tc.input)
+ if got != tc.want {
+ t.Errorf("cleanMarkdownBlock(%q) = %q, want %q", tc.input, got, tc.want)
+ }
+ })
+ }
+}
diff --git a/internal/ingestion/component/vision_language_test.go b/internal/ingestion/component/vision_language_test.go
index 8b0cd9705e..deadc898b1 100644
--- a/internal/ingestion/component/vision_language_test.go
+++ b/internal/ingestion/component/vision_language_test.go
@@ -69,28 +69,23 @@ func TestRenderFigureVisionLanguage(t *testing.T) {
}
}
-func TestBuildFigureVisionPromptRendersLanguageAndContext(t *testing.T) {
- prompt, err := buildFigureVisionPrompt("Context above", "Context below", "Japanese")
+func TestBuildFigureVisionPromptRendersLanguage(t *testing.T) {
+ prompt, err := buildFigureVisionPrompt("Japanese")
if err != nil {
t.Fatalf("buildFigureVisionPrompt: %v", err)
}
- for _, want := range []string{
- "Write all descriptions and field values in Japanese.",
- "Context above",
- "Context below",
- } {
- if !strings.Contains(prompt, want) {
- t.Fatalf("prompt does not contain %q", want)
- }
+ if !strings.Contains(prompt, "Write all descriptions and field values in Japanese.") {
+ t.Fatalf("prompt does not contain language directive, got: %q", prompt)
}
- for _, unresolved := range []string{"{{ language }}", "{{ context_above }}", "{{ context_below }}"} {
+ for _, unresolved := range []string{"{{ language }}", "{{language}}"} {
if strings.Contains(prompt, unresolved) {
t.Fatalf("prompt contains unresolved placeholder %q", unresolved)
}
}
- secondPrompt, err := buildFigureVisionPrompt("", "", "Chinese")
+ // Second call uses cache — verify language is re-rendered correctly per call.
+ secondPrompt, err := buildFigureVisionPrompt("Chinese")
if err != nil {
t.Fatalf("buildFigureVisionPrompt with cached template: %v", err)
}