RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
fix: honor dataset language in Go vision dispatch (#17892)
taek105492d6d81a9f6
diff --git a/internal/ingestion/component/docx_vision_dispatch.go b/internal/ingestion/component/docx_vision_dispatch.goindex 795ff1bb7e..0e16fc5a90 100644--- a/internal/ingestion/component/docx_vision_dispatch.go+++ b/internal/ingestion/component/docx_vision_dispatch.go@@ -44,21 +44,21 @@ import ()var (- docxVisionPromptBuilder = buildDOCXVisionPrompt- visionChatInvoker = defaultVisionChatInvoker- docxVisionConcurrency uint = 10+ figureVisionPromptBuilder = buildFigureVisionPrompt+ visionChatInvoker = defaultVisionChatInvoker+ docxVisionConcurrency uint = 10)const (- docxVisionPromptFile = "vision_llm_figure_describe_prompt.md"- docxVisionPromptWithContextFile = "vision_llm_figure_describe_prompt_with_context.md"+ figureVisionPromptFile = "vision_llm_figure_describe_prompt.md"+ figureVisionPromptWithContextFile = "vision_llm_figure_describe_prompt_with_context.md")var (- docxVisionPromptsBase string- docxVisionPromptsOnce sync.Once- docxVisionPromptCache = make(map[string]string)- docxVisionPromptMu sync.RWMutex+ figureVisionPromptsBase string+ figureVisionPromptsOnce sync.Once+ figureVisionPromptCache = make(map[string]string)+ figureVisionPromptMu sync.RWMutex)// maybeDispatchDOCXVision enriches a DOCX parse result with vision-model@@ -96,6 +96,7 @@ func maybeDispatchDOCXVision(if tenantID == "" {return dispatched, false, nil}+ language := resolveVisionLanguage(inputs, "")// Resolve the tenant's IMAGE2TEXT model.driver, modelName, apiConfig, _, err := resolveTenantModelByType(ctx, db, tenantID, entity.ModelTypeImage2Text)@@ -143,7 +144,7 @@ func maybeDispatchDOCXVision(// DOCX JSON items have no surrounding context (unlike the// former Markdown path), so use the bare figure prompt —// matching Python's VisionFigureParser(context_size=0).- prompt, perr := docxVisionPromptBuilder("", "")+ prompt, perr := figureVisionPromptBuilder("", "", language)if perr != nil {return}@@ -175,28 +176,30 @@ func maybeDispatchDOCXVision(return dispatched, modified, nil}-// buildDOCXVisionPrompt loads the figure-describe prompt template+// buildFigureVisionPrompt loads the figure-describe prompt template// and, when context text is available, renders it with the// with-context variant. Mirrors Python://// if context_above or context_below:-// prompt = vision_llm_figure_describe_prompt_with_context(context_above, context_below)+// prompt = vision_llm_figure_describe_prompt_with_context(+// context_above, context_below, language=language)// else:-// prompt = vision_llm_figure_describe_prompt()-func buildDOCXVisionPrompt(contextAbove, contextBelow string) (string, error) {+// prompt = vision_llm_figure_describe_prompt(language=language)+func buildFigureVisionPrompt(contextAbove, contextBelow, language string) (string, error) {hasContext := strings.TrimSpace(contextAbove) != "" || strings.TrimSpace(contextBelow) != ""var templateName stringif hasContext {- templateName = docxVisionPromptWithContextFile+ templateName = figureVisionPromptWithContextFile} else {- templateName = docxVisionPromptFile+ templateName = figureVisionPromptFile}- template, err := loadDOCXVisionPromptFile(templateName)+ template, err := loadFigureVisionPromptFile(templateName)if err != nil {return "", err}+ template = renderFigureVisionLanguage(template, language)if hasContext {template = strings.ReplaceAll(template, "{{ context_above }}", contextAbove)@@ -205,36 +208,36 @@ func buildDOCXVisionPrompt(contextAbove, contextBelow string) (string, error) {return template, nil}-func loadDOCXVisionPromptFile(filename string) (string, error) {- docxVisionPromptMu.RLock()- if cached, ok := docxVisionPromptCache[filename]; ok {- docxVisionPromptMu.RUnlock()+func loadFigureVisionPromptFile(filename string) (string, error) {+ figureVisionPromptMu.RLock()+ if cached, ok := figureVisionPromptCache[filename]; ok {+ figureVisionPromptMu.RUnlock()return cached, nil}- docxVisionPromptMu.RUnlock()+ figureVisionPromptMu.RUnlock()- baseDir, err := docxVisionPromptsBaseDir()+ baseDir, err := figureVisionPromptsBaseDir()if err != nil {return "", err}promptPath := filepath.Join(baseDir, "rag", "prompts", filename)content, err := os.ReadFile(promptPath)if err != nil {- return "", fmt.Errorf("docx vision prompt %q: %w", filename, err)+ return "", fmt.Errorf("figure vision prompt %q: %w", filename, err)}cached := strings.TrimSpace(string(content))- docxVisionPromptMu.Lock()- docxVisionPromptCache[filename] = cached- docxVisionPromptMu.Unlock()+ figureVisionPromptMu.Lock()+ figureVisionPromptCache[filename] = cached+ figureVisionPromptMu.Unlock()return cached, nil}-func docxVisionPromptsBaseDir() (string, error) {+func figureVisionPromptsBaseDir() (string, error) {var initErr error- docxVisionPromptsOnce.Do(func() {+ figureVisionPromptsOnce.Do(func() {root := utility.GetProjectRoot()if _, statErr := os.Stat(filepath.Join(root, "rag", "prompts")); statErr == nil {- docxVisionPromptsBase = root+ figureVisionPromptsBase = rootreturn}initErr = fmt.Errorf("rag/prompts not found under project root %q", root)@@ -242,7 +245,7 @@ func docxVisionPromptsBaseDir() (string, error) {if initErr != nil {return "", initErr}- return docxVisionPromptsBase, nil+ return figureVisionPromptsBase, nil}func buildVisionMessages(prompt, imageBase64 string) []modelModule.Message {diff --git a/internal/ingestion/component/docx_vision_dispatch_test.go b/internal/ingestion/component/docx_vision_dispatch_test.goindex 4b38f74d7f..8dab40df54 100644--- a/internal/ingestion/component/docx_vision_dispatch_test.go+++ b/internal/ingestion/component/docx_vision_dispatch_test.go@@ -75,11 +75,11 @@ func (c *docxVisionCaptureInvoker) invoke(func TestMaybeDispatchDOCXVision_EnhancesJSONImages(t *testing.T) {origResolver := resolveTenantModelByTypeorigInvoker := visionChatInvoker- origPrompt := docxVisionPromptBuilder+ origPrompt := figureVisionPromptBuilderdefer func() {resolveTenantModelByType = origResolvervisionChatInvoker = origInvoker- docxVisionPromptBuilder = origPrompt+ figureVisionPromptBuilder = origPrompt}()resolveTenantModelByType = func(ctx context.Context, db *gorm.DB, tenantID string, modelType entity.ModelType) (modelModule.ModelDriver, string, *modelModule.APIConfig, int, error) {@@ -87,7 +87,11 @@ func TestMaybeDispatchDOCXVision_EnhancesJSONImages(t *testing.T) {}invoker := &docxVisionCaptureInvoker{}visionChatInvoker = invoker.invoke- docxVisionPromptBuilder = func(string, string) (string, error) { return "describe the figure", nil }+ var capturedLanguage string+ figureVisionPromptBuilder = func(_, _, language string) (string, error) {+ capturedLanguage = language+ return "describe the figure", nil+ }dispatched := parserDispatchResult{OutputFormat: "json",@@ -105,7 +109,7 @@ func TestMaybeDispatchDOCXVision_EnhancesJSONImages(t *testing.T) {dao.DB,utility.FileTypeDOCX,dispatched,- map[string]any{"tenant_id": "t1"},+ map[string]any{"tenant_id": "t1", "lang": "Japanese"},defaultSetups(),)if err != nil {@@ -130,6 +134,9 @@ func TestMaybeDispatchDOCXVision_EnhancesJSONImages(t *testing.T) {if len(invoker.images) != 1 {t.Fatalf("vision invoker called %d times, want 1 (only the image item)", len(invoker.images))}+ if capturedLanguage != "Japanese" {+ t.Errorf("figure prompt language = %q, want Japanese", capturedLanguage)+ }if want := "data:image/png;base64,aGVsbG8taW1hZ2U="; invoker.images[0] != want {t.Errorf("vision image data URI = %q, want %q", invoker.images[0], want)}diff --git a/internal/ingestion/component/markdown_vision_dispatch.go b/internal/ingestion/component/markdown_vision_dispatch.goindex a70a3e846d..ff0add6b03 100644--- a/internal/ingestion/component/markdown_vision_dispatch.go+++ b/internal/ingestion/component/markdown_vision_dispatch.go@@ -29,7 +29,6 @@ package componentimport ("context"- "fmt""strings""sync"@@ -102,6 +101,7 @@ func maybeDispatchMarkdownVision(if tenantID == "" {return dispatched, false, nil}+ language := resolveVisionLanguage(inputs, "")// Resolve the tenant's IMAGE2TEXT model.driver, modelName, apiConfig, _, err := resolveTenantModelByType(ctx, db, tenantID, entity.ModelTypeImage2Text)@@ -124,7 +124,7 @@ func maybeDispatchMarkdownVision(// Markdown images have no context — use the// default (no-context) prompt template.- prompt, err := buildMarkdownVisionPrompt()+ prompt, err := figureVisionPromptBuilder("", "", language)if err != nil {return}@@ -157,14 +157,3 @@ func maybeDispatchMarkdownVision(return dispatched, true, nil}--// buildMarkdownVisionPrompt loads the default (no-context) figure-// describe prompt template, mirroring Python's-// vision_llm_figure_describe_prompt().-func buildMarkdownVisionPrompt() (string, error) {- template, err := loadDOCXVisionPromptFile(docxVisionPromptFile)- if err != nil {- return "", fmt.Errorf("markdown vision prompt: %w", err)- }- return template, nil-}diff --git a/internal/ingestion/component/media_dispatch.go b/internal/ingestion/component/media_dispatch.goindex 7b629b7607..7b17d21d8b 100644--- a/internal/ingestion/component/media_dispatch.go+++ b/internal/ingestion/component/media_dispatch.go@@ -151,7 +151,7 @@ func maybeDispatchImage(// --- Phase 2: VLM description (when OCR text is short) ---// Mirrors Python's check: if (eng and len(txt.split()) > 32) or len(txt) > 32// then use OCR text only; otherwise call cv_mdl.describe().- lang := getStringOr(setup, "lang", "")+ lang := resolveVisionLanguage(inputs, getStringOr(setup, "lang", ""))eng := strings.EqualFold(lang, "english")if ocrText != "" {@@ -174,7 +174,7 @@ func maybeDispatchImage(fmt.Errorf("parser: picture image2text model: %w", err)}- prompt := "Describe this image in detail."+ prompt := defaultImageVisionPrompt(lang)// image family's contract key is system_prompt (parser.go:295),// mirroring Python parser.py:1119. Do NOT read setup["prompt"]// here — that key is for the video family, not image.diff --git a/internal/ingestion/component/media_dispatch_test.go b/internal/ingestion/component/media_dispatch_test.goindex 46213b657d..2c465616cc 100644--- a/internal/ingestion/component/media_dispatch_test.go+++ b/internal/ingestion/component/media_dispatch_test.go@@ -133,6 +133,44 @@ func TestMaybeDispatchImage_UsesSystemPrompt(t *testing.T) {}}+func TestMaybeDispatchImage_DefaultPromptUsesDatasetLanguage(t *testing.T) {+ origResolver := resolveTenantModelByType+ defer func() { resolveTenantModelByType = origResolver }()++ drv := &imagePromptCaptureDriver{}+ resolveTenantModelByType = func(ctx context.Context, db *gorm.DB, tenantID string, modelType entity.ModelType) (modelModule.ModelDriver, string, *modelModule.APIConfig, int, error) {+ return drv, "img-model", &modelModule.APIConfig{}, 0, nil+ }++ setups := defaultSetups()+ setups["image"]["lang"] = "Chinese"+ setups["image"]["system_prompt"] = ""++ _, _, err := maybeDispatchImage(+ t.Context(),+ dao.DB,+ utility.FileTypeVISUAL,+ "test.png",+ []byte("not-a-real-image"),+ map[string]any{"tenant_id": "t1", "lang": "Japanese"},+ setups,+ )+ if err != nil {+ t.Fatalf("maybeDispatchImage: %v", err)+ }++ got, ok := firstUserText(drv.captured)+ if !ok {+ t.Fatalf("no user text captured in VLM messages: %#v", drv.captured)+ }+ if !strings.Contains(got, "Respond in Japanese.") {+ t.Fatalf("VLM user text = %q, want dataset language instruction", got)+ }+ if strings.Contains(got, "Respond in Chinese.") {+ t.Fatalf("VLM user text = %q, setup fallback overrode dataset language", got)+ }+}+// TestMaybeDispatchImage_ReturnsJSONWithImage pins the output-shape fix:// the image branch must return a JSON item carrying the `image` attachment// (data URI) and `doc_type_kwd:"image"`, mirroring Python@@ -363,12 +401,19 @@ func TestMaybeDispatchAudio_DefaultOutputFormatJson(t *testing.T) {// Before the fix the table item was skipped and never sent to the VLM.func TestMaybeDispatchMarkdownVision_EnhancesTables(t *testing.T) {origResolver := resolveTenantModelByType- defer func() { resolveTenantModelByType = origResolver }()+ origPrompt := figureVisionPromptBuilder+ defer func() {+ resolveTenantModelByType = origResolver+ figureVisionPromptBuilder = origPrompt+ }()drv := &imagePromptCaptureDriver{}resolveTenantModelByType = func(ctx context.Context, db *gorm.DB, tenantID string, modelType entity.ModelType) (modelModule.ModelDriver, string, *modelModule.APIConfig, int, error) {return drv, "img-model", &modelModule.APIConfig{}, 0, nil}+ figureVisionPromptBuilder = func(_, _, language string) (string, error) {+ return "describe in " + language, nil+ }dispatched := parserDispatchResult{OutputFormat: "json",@@ -383,7 +428,7 @@ func TestMaybeDispatchMarkdownVision_EnhancesTables(t *testing.T) {dao.DB,utility.FileTypeMarkdown,dispatched,- map[string]any{"tenant_id": "t1"},+ map[string]any{"tenant_id": "t1", "lang": "Korean"},)if err != nil {t.Fatalf("maybeDispatchMarkdownVision: %v", err)@@ -398,6 +443,10 @@ func TestMaybeDispatchMarkdownVision_EnhancesTables(t *testing.T) {if got, _ := res.JSON[0]["text"].(string); got != "captured" {t.Fatalf("table item text = %q, want %q (table items must be vision-enhanced)", got, "captured")}+ gotPrompt, ok := firstUserText(drv.captured)+ if !ok || gotPrompt != "describe in Korean" {+ t.Fatalf("VLM user text = %q, want dataset language propagated", gotPrompt)+ }}// TestDefaultEmailOutputFormatIsJSON pins diff 2.2: the email family defaultdiff --git a/internal/ingestion/component/pdf_vision_dispatch.go b/internal/ingestion/component/pdf_vision_dispatch.goindex ec9a67e5db..f844990a0d 100644--- a/internal/ingestion/component/pdf_vision_dispatch.go+++ b/internal/ingestion/component/pdf_vision_dispatch.go@@ -543,6 +543,10 @@ func pdfVisionPromptsBaseDir() (string, error) {return pdfVisionPromptsBase, nil}+// renderPDFVisionPrompt only renders page metadata. The full-page PDF vision+// prompt is a transcription contract that preserves the document's original+// language; dataset-language instructions apply to figure descriptions in+// maybeDispatchPDFVisionEnhancement instead.func renderPDFVisionPrompt(template string, page int) string {rendered := strings.ReplaceAll(template, "{{ page }}", fmt.Sprintf("%d", page))rendered = strings.ReplaceAll(rendered, "{{page}}", fmt.Sprintf("%d", page))@@ -587,6 +591,7 @@ func maybeDispatchPDFVisionEnhancement(if tenantID == "" {return dispatched, false, nil}+ language := resolveVisionLanguage(inputs, "")driver, modelName, apiConfig, _, err := resolveTenantModelByType(ctx, db, tenantID, entity.ModelTypeImage2Text)if err != nil {return dispatched, false, nil@@ -620,7 +625,7 @@ func maybeDispatchPDFVisionEnhancement(if img == "" {return}- prompt, perr := docxVisionPromptBuilder("", "")+ prompt, perr := figureVisionPromptBuilder("", "", language)if perr != nil {return}diff --git a/internal/ingestion/component/pdf_vision_dispatch_test.go b/internal/ingestion/component/pdf_vision_dispatch_test.goindex de543b5db8..3cbf5e22d0 100644--- a/internal/ingestion/component/pdf_vision_dispatch_test.go+++ b/internal/ingestion/component/pdf_vision_dispatch_test.go@@ -15,7 +15,67 @@package component-import "testing"+import (+ "context"+ "testing"++ "ragflow/internal/dao"+ "ragflow/internal/entity"+ modelModule "ragflow/internal/entity/models"+ "ragflow/internal/utility"++ "gorm.io/gorm"+)++func TestMaybeDispatchPDFVisionEnhancementForwardsDatasetLanguage(t *testing.T) {+ origResolver := resolveTenantModelByType+ origInvoker := visionChatInvoker+ origPrompt := figureVisionPromptBuilder+ t.Cleanup(func() {+ resolveTenantModelByType = origResolver+ visionChatInvoker = origInvoker+ figureVisionPromptBuilder = origPrompt+ })++ resolveTenantModelByType = func(context.Context, *gorm.DB, string, entity.ModelType) (modelModule.ModelDriver, string, *modelModule.APIConfig, int, error) {+ return &docxVisionFakeDriver{}, "pdf-vision-model", &modelModule.APIConfig{}, 0, nil+ }+ invoker := &docxVisionCaptureInvoker{}+ visionChatInvoker = invoker.invoke+ capturedLanguage := ""+ figureVisionPromptBuilder = func(_, _, language string) (string, error) {+ capturedLanguage = language+ return "describe the figure", nil+ }++ dispatched := parserDispatchResult{+ OutputFormat: "json",+ DocType: "pdf",+ JSON: []map[string]any{+ {"text": "caption", "image": "data:image/png;base64,aW1hZ2U=", "doc_type_kwd": "image"},+ },+ }++ res, modified, err := maybeDispatchPDFVisionEnhancement(+ t.Context(),+ dao.DB,+ utility.FileTypePDF,+ dispatched,+ map[string]any{"tenant_id": "t1", "lang": "Dutch"},+ )+ if err != nil {+ t.Fatalf("maybeDispatchPDFVisionEnhancement: %v", err)+ }+ if !modified {+ t.Fatal("modified = false, want true")+ }+ if capturedLanguage != "Dutch" {+ t.Fatalf("figure prompt language = %q, want Dutch", capturedLanguage)+ }+ if got := res.JSON[0]["text"]; got != "caption\na diagram of a pipeline" {+ t.Fatalf("enhanced text = %q", got)+ }+}// TestIsNamedPDFParseMethodWhitelistAligned verifies that the runtime// "named parse_method" classifier agrees with (*ParserComponent).Check()'sdiff --git a/internal/ingestion/component/vision_language.go b/internal/ingestion/component/vision_language.gonew file mode 100644index 0000000000..5f9b3078b4--- /dev/null+++ b/internal/ingestion/component/vision_language.go@@ -0,0 +1,46 @@+//+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.+//++package component++import "strings"++const defaultVisionLanguage = "English"++func normalizeVisionLanguage(language string) string {+ language = strings.TrimSpace(language)+ if language == "" {+ return defaultVisionLanguage+ }+ return language+}++func resolveVisionLanguage(inputs map[string]any, fallback string) string {+ if language := strings.TrimSpace(getStringOr(inputs, "lang", "")); language != "" {+ return language+ }+ return normalizeVisionLanguage(fallback)+}++func renderFigureVisionLanguage(prompt, language string) string {+ language = normalizeVisionLanguage(language)+ prompt = strings.ReplaceAll(prompt, "{{ language }}", language)+ return strings.ReplaceAll(prompt, "{{language}}", language)+}++func defaultImageVisionPrompt(language string) string {+ return "Describe this image in detail. Respond in " + normalizeVisionLanguage(language) + ". Preserve all visible text in its original language; do not translate it."+}diff --git a/internal/ingestion/component/vision_language_test.go b/internal/ingestion/component/vision_language_test.gonew file mode 100644index 0000000000..8b0cd9705e--- /dev/null+++ b/internal/ingestion/component/vision_language_test.go@@ -0,0 +1,103 @@+//+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.+//++package component++import (+ "strings"+ "testing"+)++func TestResolveVisionLanguage(t *testing.T) {+ tests := []struct {+ name string+ inputs map[string]any+ fallback string+ want string+ }{+ {+ name: "dataset language takes precedence",+ inputs: map[string]any{"lang": " Japanese "},+ fallback: "Chinese",+ want: "Japanese",+ },+ {+ name: "configured fallback is used",+ inputs: map[string]any{"lang": ""},+ fallback: " Korean ",+ want: "Korean",+ },+ {+ name: "empty values default to English",+ inputs: nil,+ fallback: "",+ want: defaultVisionLanguage,+ },+ }++ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ if got := resolveVisionLanguage(tt.inputs, tt.fallback); got != tt.want {+ t.Fatalf("resolveVisionLanguage() = %q, want %q", got, tt.want)+ }+ })+ }+}++func TestRenderFigureVisionLanguage(t *testing.T) {+ prompt := renderFigureVisionLanguage("spaced={{ language }} compact={{language}}", "Japanese")+ if prompt != "spaced=Japanese compact=Japanese" {+ t.Fatalf("renderFigureVisionLanguage() = %q", prompt)+ }++ prompt = renderFigureVisionLanguage("language={{ language }}", "")+ if prompt != "language=English" {+ t.Fatalf("empty-language rendering = %q, want English fallback", prompt)+ }+}++func TestBuildFigureVisionPromptRendersLanguageAndContext(t *testing.T) {+ prompt, err := buildFigureVisionPrompt("Context above", "Context below", "Japanese")+ if err != nil {+ t.Fatalf("buildFigureVisionPrompt: %v", err)+ }++ for _, want := range []string{+ "Write all descriptions and field values in Japanese.",+ "Context above",+ "Context below",+ } {+ if !strings.Contains(prompt, want) {+ t.Fatalf("prompt does not contain %q", want)+ }+ }+ for _, unresolved := range []string{"{{ language }}", "{{ context_above }}", "{{ context_below }}"} {+ if strings.Contains(prompt, unresolved) {+ t.Fatalf("prompt contains unresolved placeholder %q", unresolved)+ }+ }++ secondPrompt, err := buildFigureVisionPrompt("", "", "Chinese")+ if err != nil {+ t.Fatalf("buildFigureVisionPrompt with cached template: %v", err)+ }+ if !strings.Contains(secondPrompt, "Write all descriptions and field values in Chinese.") {+ t.Fatalf("cached prompt did not render the second request language")+ }+ if strings.Contains(secondPrompt, "in Japanese.") {+ t.Fatalf("cached prompt leaked the previous request language")+ }+}