RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
fix(parser): wire per-call VLM model config, harden vision dispatch (#18846)
jay7772134870e297d72
diff --git a/internal/ingestion/component/dispatch_model.go b/internal/ingestion/component/dispatch_model.goindex cc0a291b5c..6560f9762f 100644--- a/internal/ingestion/component/dispatch_model.go+++ b/internal/ingestion/component/dispatch_model.go@@ -29,6 +29,7 @@ import ("ragflow/internal/dao""ragflow/internal/entity"modelModule "ragflow/internal/entity/models"+ "ragflow/internal/ingestion/component/schema""gorm.io/gorm")@@ -39,6 +40,29 @@ type tenantModelExtra struct {var resolveTenantModelByType = defaultResolveTenantModelByType+// resolveModelConfig resolves a specific model reference (tenant-model ID or+// "name@instance@provider" composite) to a driver. Exposed as a package var so+// per-call model-selection tests can inject a fake without a live MySQL.+var resolveModelConfig = defaultResolveModelConfig++// configuredMediaModelID extracts a per-call model reference from a parser setup.+// Image parsing stores the VLM model reference in parse_method when it is not+// "ocr" (mirroring Python rag/flow/parser/parser.py:_image). Other media+// families use vlm.llm_id, matching the frontend parser form.+func configuredMediaModelID(setup schema.ParserSetup, family string) string {+ if family == "image" {+ if ref := getStringOr(setup, "parse_method", ""); ref != "" && !strings.EqualFold(ref, "ocr") {+ return ref+ }+ }+ if vlm, ok := setup["vlm"].(map[string]any); ok {+ if ref, _ := vlm["llm_id"].(string); ref != "" {+ return ref+ }+ }+ return getStringOr(setup, "llm_id", "")+}+func defaultResolveTenantModelByType(ctx context.Context, db *gorm.DB, tenantID string, modelType entity.ModelType) (modelModule.ModelDriver, string, *modelModule.APIConfig, int, error) {tenantDAO := dao.NewTenantDAO()tenant, err := tenantDAO.GetByID(ctx, db, tenantID)@@ -164,7 +188,7 @@ func stringValue(value *string) string {return *value}-func resolveModelConfig(ctx context.Context, db *gorm.DB, tenantID string, modelType entity.ModelType, modelRef string) (modelModule.ModelDriver, string, *modelModule.APIConfig, int, error) {+func defaultResolveModelConfig(ctx context.Context, db *gorm.DB, tenantID string, modelType entity.ModelType, modelRef string) (modelModule.ModelDriver, string, *modelModule.APIConfig, int, error) {modelDAO := dao.NewTenantModelDAO()if _, err := modelDAO.GetByID(ctx, db, modelRef); err == nil {return resolveModelConfigByID(ctx, db, tenantID, modelType, modelRef)diff --git a/internal/ingestion/component/dispatch_model_test.go b/internal/ingestion/component/dispatch_model_test.gonew file mode 100644index 0000000000..480ea64714--- /dev/null+++ b/internal/ingestion/component/dispatch_model_test.go@@ -0,0 +1,82 @@+//+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.++package component++import (+ "testing"++ "ragflow/internal/ingestion/component/schema"+)++func TestConfiguredMediaModelID(t *testing.T) {+ tests := []struct {+ name string+ family string+ setup schema.ParserSetup+ want string+ }{+ {+ name: "image parse_method is the VLM model ref",+ family: "image",+ setup: schema.ParserSetup{"parse_method": "gpt-4-vision@openai"},+ want: "gpt-4-vision@openai",+ },+ {+ name: "image ocr falls back to vlm.llm_id",+ family: "image",+ setup: schema.ParserSetup{+ "parse_method": "ocr",+ "vlm": map[string]any{"llm_id": "gpt-4-vision@openai"},+ },+ want: "gpt-4-vision@openai",+ },+ {+ name: "image ocr falls back to top-level llm_id",+ family: "image",+ setup: schema.ParserSetup{"parse_method": "ocr", "llm_id": "qwen-vl@dashscope"},+ want: "qwen-vl@dashscope",+ },+ {+ name: "pdf uses vlm.llm_id",+ family: "pdf",+ setup: schema.ParserSetup{+ "parse_method": "deepdoc",+ "vlm": map[string]any{"llm_id": "custom-vlm@provider"},+ },+ want: "custom-vlm@provider",+ },+ {+ name: "audio uses vlm.llm_id",+ family: "audio",+ setup: schema.ParserSetup{"vlm": map[string]any{"llm_id": "whisper@openai"}},+ want: "whisper@openai",+ },+ {+ name: "empty setup",+ family: "pdf",+ setup: schema.ParserSetup{},+ want: "",+ },+ }++ for _, tc := range tests {+ t.Run(tc.name, func(t *testing.T) {+ if got := configuredMediaModelID(tc.setup, tc.family); got != tc.want {+ t.Fatalf("configuredMediaModelID() = %q, want %q", got, tc.want)+ }+ })+ }+}diff --git a/internal/ingestion/component/media_dispatch.go b/internal/ingestion/component/media_dispatch.goindex 16970d6a87..864a864210 100644--- a/internal/ingestion/component/media_dispatch.go+++ b/internal/ingestion/component/media_dispatch.go@@ -31,6 +31,7 @@ import ("encoding/base64""fmt""image"+ "log/slog""ragflow/internal/common"// Import image decoders for common formats.@@ -165,7 +166,21 @@ func maybeDispatchImage(}// Short OCR text (or no text): supplement with VLM describe.- driver, modelName, apiConfig, _, err := resolveTenantModelByType(ctx, db, tenantID, entity.ModelTypeImage2Text)+ modelRef := configuredMediaModelID(setup, "image")+ var driver modelModule.ModelDriver+ var modelName string+ var apiConfig *modelModule.APIConfig+ var err error+ if modelRef != "" {+ driver, modelName, apiConfig, _, err = resolveModelConfig(ctx, db, tenantID, entity.ModelTypeImage2Text, modelRef)+ if err != nil {+ slog.Warn("media dispatch image: per-call VLM resolve failed, falling back to tenant default",+ "modelRef", modelRef, "tenant", tenantID, "err", err)+ driver, modelName, apiConfig, _, err = resolveTenantModelByType(ctx, db, tenantID, entity.ModelTypeImage2Text)+ }+ } else {+ driver, modelName, apiConfig, _, err = resolveTenantModelByType(ctx, db, tenantID, entity.ModelTypeImage2Text)+ }if err != nil {// If VLM is unavailable, but we have OCR text, return it.if ocrText != "" {@@ -260,7 +275,21 @@ func maybeDispatchAudio(fmt.Errorf("parser: audio requires tenant_id")}- driver, modelName, apiConfig, _, err := resolveTenantModelByType(ctx, db, tenantID, entity.ModelTypeSpeech2Text)+ modelRef := configuredMediaModelID(setup, "audio")+ var driver modelModule.ModelDriver+ var modelName string+ var apiConfig *modelModule.APIConfig+ var err error+ if modelRef != "" {+ driver, modelName, apiConfig, _, err = resolveModelConfig(ctx, db, tenantID, entity.ModelTypeSpeech2Text, modelRef)+ if err != nil {+ slog.Warn("media dispatch audio: per-call VLM resolve failed, falling back to tenant default",+ "modelRef", modelRef, "tenant", tenantID, "err", err)+ driver, modelName, apiConfig, _, err = resolveTenantModelByType(ctx, db, tenantID, entity.ModelTypeSpeech2Text)+ }+ } else {+ driver, modelName, apiConfig, _, err = resolveTenantModelByType(ctx, db, tenantID, entity.ModelTypeSpeech2Text)+ }if err != nil {return parserDispatchResult{}, true,fmt.Errorf("parser: audio speech2text model: %w", err)diff --git a/internal/ingestion/component/media_dispatch_test.go b/internal/ingestion/component/media_dispatch_test.goindex b86cdb55a9..5f80952114 100644--- a/internal/ingestion/component/media_dispatch_test.go+++ b/internal/ingestion/component/media_dispatch_test.go@@ -411,3 +411,110 @@ func TestDefaultEmailOutputFormatIsJSON(t *testing.T) {t.Fatalf("email default output_format = %q, want json", got)}}++func TestMaybeDispatchImage_UsesConfiguredVLMModel(t *testing.T) {+ origTenantResolver := resolveTenantModelByType+ origModelResolver := resolveModelConfig+ t.Cleanup(func() {+ resolveTenantModelByType = origTenantResolver+ resolveModelConfig = origModelResolver+ })++ tenantResolverCalled := false+ resolveTenantModelByType = func(_ context.Context, _ *gorm.DB, _ string, _ entity.ModelType) (modelModule.ModelDriver, string, *modelModule.APIConfig, int, error) {+ tenantResolverCalled = true+ return nil, "", nil, 0, nil+ }++ var gotRef string+ var gotType entity.ModelType+ drv := &imagePromptCaptureDriver{}+ resolveModelConfig = func(_ context.Context, _ *gorm.DB, _ string, modelType entity.ModelType, ref string) (modelModule.ModelDriver, string, *modelModule.APIConfig, int, error) {+ gotRef = ref+ gotType = modelType+ return drv, "custom-vlm", &modelModule.APIConfig{}, 0, nil+ }++ setups := defaultSetups()+ setups["image"]["parse_method"] = "custom-vlm@provider"++ _, dispatched, err := maybeDispatchImage(+ t.Context(),+ dao.DB,+ utility.FileTypeVISUAL,+ "test.png",+ []byte("not-a-real-image"),+ map[string]any{"tenant_id": "t1"},+ setups,+ )+ if err != nil {+ t.Fatalf("maybeDispatchImage: %v", err)+ }+ if !dispatched {+ t.Fatal("expected dispatched=true for VISUAL file")+ }+ if gotRef != "custom-vlm@provider" {+ t.Errorf("model ref = %q, want %q", gotRef, "custom-vlm@provider")+ }+ if gotType != entity.ModelTypeImage2Text {+ t.Errorf("model type = %q, want %q", gotType, entity.ModelTypeImage2Text)+ }+ if tenantResolverCalled {+ t.Error("tenant default resolver must not be called when image parse_method names a VLM model")+ }+}++func TestMaybeDispatchAudio_UsesConfiguredModel(t *testing.T) {+ origTenantResolver := resolveTenantModelByType+ origModelResolver := resolveModelConfig+ t.Cleanup(func() {+ resolveTenantModelByType = origTenantResolver+ resolveModelConfig = origModelResolver+ })++ tenantResolverCalled := false+ resolveTenantModelByType = func(_ context.Context, _ *gorm.DB, _ string, _ entity.ModelType) (modelModule.ModelDriver, string, *modelModule.APIConfig, int, error) {+ tenantResolverCalled = true+ return nil, "", nil, 0, nil+ }++ var gotRef string+ var gotType entity.ModelType+ drv := &audioTranscribeDriver{transcription: "transcribed"}+ resolveModelConfig = func(_ context.Context, _ *gorm.DB, _ string, modelType entity.ModelType, ref string) (modelModule.ModelDriver, string, *modelModule.APIConfig, int, error) {+ gotRef = ref+ gotType = modelType+ return drv, "custom-asr", &modelModule.APIConfig{}, 0, nil+ }++ setups := map[string]schema.ParserSetup{+ "audio": {"vlm": map[string]any{"llm_id": "custom-asr@provider"}},+ }+ res, dispatched, err := maybeDispatchAudio(+ t.Context(),+ dao.DB,+ utility.FileTypeAURAL,+ "test.mp3",+ []byte("fake-audio"),+ map[string]any{"tenant_id": "t1"},+ setups,+ )+ if err != nil {+ t.Fatalf("maybeDispatchAudio: %v", err)+ }+ if !dispatched {+ t.Fatal("expected dispatched=true for AURAL file")+ }+ if gotRef != "custom-asr@provider" {+ t.Errorf("model ref = %q, want %q", gotRef, "custom-asr@provider")+ }+ if gotType != entity.ModelTypeSpeech2Text {+ t.Errorf("model type = %q, want %q", gotType, entity.ModelTypeSpeech2Text)+ }+ if tenantResolverCalled {+ t.Error("tenant default resolver must not be called when audio setup vlm.llm_id is set")+ }+ if len(res.JSON) != 1 || res.JSON[0]["text"] != "transcribed" {+ t.Fatalf("unexpected audio result: %#v", res.JSON)+ }+}diff --git a/internal/ingestion/component/parser.go b/internal/ingestion/component/parser.goindex a25c5b3d0c..bcf881ad5e 100644--- a/internal/ingestion/component/parser.go+++ b/internal/ingestion/component/parser.go@@ -489,7 +489,7 @@ func (c *ParserComponent) Invoke(ctx context.Context, db *gorm.DB, inputs map[st// Errors (including context cancellation) are intentionally// discarded — enhancement is best-effort, matching Python's// try/except pass pattern.- dispatched, _, _ = maybeDispatchVisionEnhancement(ctx, db, fileTypeExt, dispatched, inputs)+ dispatched, _, _ = maybeDispatchVisionEnhancement(ctx, db, fileTypeExt, dispatched, inputs, c.Setups)}// Known/supported families must fail loudly when dispatch or// parsing breaks. Only unknown families keep the raw-text fallback.diff --git a/internal/ingestion/component/pdf_vision_dispatch.go b/internal/ingestion/component/pdf_vision_dispatch.goindex 9eb7fc7d74..3c20d01a7a 100644--- a/internal/ingestion/component/pdf_vision_dispatch.go+++ b/internal/ingestion/component/pdf_vision_dispatch.go@@ -42,8 +42,7 @@ var (var (pdfVisionPromptCache = make(map[string]string)pdfVisionPromptCacheMu sync.RWMutex- pdfVisionPromptsBase string- pdfVisionPromptsOnce sync.Once+ pdfVisionPrompts promptDirState)func maybeDispatchPDFVision(@@ -704,19 +703,7 @@ func loadPDFVisionPrompt(name string) (string, error) {}func pdfVisionPromptsBaseDir() (string, error) {- var initErr error- pdfVisionPromptsOnce.Do(func() {- root := utility.GetProjectRoot()- if _, statErr := os.Stat(filepath.Join(root, "rag", "prompts")); statErr == nil {- pdfVisionPromptsBase = root- return- }- initErr = fmt.Errorf("rag/prompts not found under project root %q", root)- })- if initErr != nil {- return "", initErr- }- return pdfVisionPromptsBase, nil+ return pdfVisionPrompts.resolve(utility.GetProjectRoot())}// renderPDFVisionPrompt only renders page metadata. The full-page PDF visiondiff --git a/internal/ingestion/component/vision_enhancement.go b/internal/ingestion/component/vision_enhancement.goindex f7b5167d8b..dc2f81308c 100644--- a/internal/ingestion/component/vision_enhancement.go+++ b/internal/ingestion/component/vision_enhancement.go@@ -23,16 +23,21 @@ package componentimport ("context"+ "encoding/base64"+ "errors""fmt"+ "log/slog""os""path/filepath""regexp""strings""sync"+ "time"pdflayout "ragflow/internal/deepdoc/parser/pdf/layout""ragflow/internal/entity"modelModule "ragflow/internal/entity/models"+ "ragflow/internal/ingestion/component/schema""ragflow/internal/utility""gorm.io/gorm"@@ -46,15 +51,44 @@ var (const (figureVisionPromptFile = "vision_llm_figure_describe_prompt.md"visionEnhancementConcurrency int = 10+ // visionChatTimeout bounds a single VLM call so a hung endpoint cannot+ // occupy one of the concurrency slots indefinitely. Python wraps the+ // per-image call in @timeout(30, 3) (deepdoc/parser/figure_parser.py).+ visionChatTimeout = 30 * time.Second)var (- figureVisionPromptsBase string- figureVisionPromptsOnce sync.Once+ figureVisionPrompts promptDirStatefigureVisionPromptCache = make(map[string]string)figureVisionPromptMu sync.RWMutex)+// promptDirState caches the project root that contains rag/prompts. The once+// guard makes initialization sticky; unlike a function-local initErr, an+// initialization failure is preserved for every subsequent call.+type promptDirState struct {+ once sync.Once+ root string+ initErr error+}++func (s *promptDirState) resolve(requestedRoot string) (string, error) {+ s.once.Do(func() {+ if info, statErr := os.Stat(filepath.Join(requestedRoot, "rag", "prompts")); statErr == nil && info.IsDir() {+ s.root = requestedRoot+ return+ }+ s.initErr = fmt.Errorf("rag/prompts not found under project root %q", requestedRoot)+ })+ if s.initErr != nil {+ return "", s.initErr+ }+ if s.root == "" {+ return "", errors.New("prompts base dir not initialized")+ }+ return s.root, nil+}+// cleanMarkdownBlock mirrors Python common/string_utils.py:clean_markdown_block//// re.sub(r"^\s*```markdown\s*\n?", "", text)@@ -72,6 +106,43 @@ func cleanMarkdownBlock(s string) string {return strings.TrimSpace(s)}+// isUsableVisionImage reports whether raw carries a valid image data URI or a+// base64 payload that can be sent to a vision model.+func isUsableVisionImage(raw string) bool {+ raw = strings.TrimSpace(raw)+ if raw == "" {+ return false+ }+ if strings.HasPrefix(raw, "data:image/") {+ idx := strings.Index(raw, "base64,")+ if idx < 0 {+ return false+ }+ return isValidBase64(raw[idx+len("base64,"):])+ }+ if strings.HasPrefix(raw, "http://") || strings.HasPrefix(raw, "https://") {+ return true+ }+ cleaned := strings.Map(func(r rune) rune {+ if r == '\r' || r == '\n' || r == ' ' || r == '\t' {+ return -1+ }+ return r+ }, raw)+ return isValidBase64(cleaned)+}++func isValidBase64(s string) bool {+ if s == "" {+ return false+ }+ if _, err := base64.StdEncoding.DecodeString(s); err == nil {+ return true+ }+ _, err := base64.RawStdEncoding.DecodeString(s)+ return err == nil+}+// isVisionEnhancementAllowed mirrors Python's 3 call sites in// rag/flow/parser/parser.py:772/978/1115 (PDF/DOCX/Markdown JSON branches).func isVisionEnhancementAllowed(fileType utility.FileType) bool {@@ -93,6 +164,7 @@ func maybeDispatchVisionEnhancement(fileType utility.FileType,dispatched parserDispatchResult,inputs map[string]any,+ setups map[string]schema.ParserSetup,) (parserDispatchResult, bool, error) {// 0. FileType allowlist guard.if !isVisionEnhancementAllowed(fileType) {@@ -129,8 +201,25 @@ func maybeDispatchVisionEnhancement(return dispatched, false, nil}- // 2. Resolve tenant's IMAGE2TEXT model.- driver, modelName, apiConfig, _, err := resolveTenantModelByType(ctx, db, tenantID, entity.ModelTypeImage2Text)+ // 2. Resolve the per-call IMAGE2TEXT model, then fall back to the tenant+ // default. Mirror Python's vlm_conf["llm_id"] preference.+ family := resolveParserFamily(fileType)+ setup := setups[family]+ modelRef := configuredMediaModelID(setup, family)+ var driver modelModule.ModelDriver+ var modelName string+ var apiConfig *modelModule.APIConfig+ var err error+ if modelRef != "" {+ driver, modelName, apiConfig, _, err = resolveModelConfig(ctx, db, tenantID, entity.ModelTypeImage2Text, modelRef)+ if err != nil {+ slog.Warn("vision enhancement: per-call VLM resolve failed, falling back to tenant default",+ "family", family, "modelRef", modelRef, "tenant", tenantID, "err", err)+ driver, modelName, apiConfig, _, err = resolveTenantModelByType(ctx, db, tenantID, entity.ModelTypeImage2Text)+ }+ } else {+ driver, modelName, apiConfig, _, err = resolveTenantModelByType(ctx, db, tenantID, entity.ModelTypeImage2Text)+ }if err != nil {// Model not available — skip vision enhancement silently, matching Python's try/except pass.return dispatched, false, nil@@ -168,10 +257,15 @@ dispatch:defer func() { <-sem }()img, _ := dispatched.JSON[itemIdx]["image"].(string)- if img == "" {+ if !isUsableVisionImage(img) {+ slog.Warn("vision enhancement: invalid image data skipped",+ "item", itemIdx)return}messages := buildVisionMessages(prompt, img)+ if len(messages) == 0 {+ return+ }resp, ierr := visionChatInvoker(ctx, driver, modelName, messages, apiConfig)if ierr != nil {return@@ -236,25 +330,13 @@ func loadFigureVisionPromptFile(filename string) (string, error) {}func figureVisionPromptsBaseDir() (string, error) {- var initErr error- figureVisionPromptsOnce.Do(func() {- root := utility.GetProjectRoot()- if _, statErr := os.Stat(filepath.Join(root, "rag", "prompts")); statErr == nil {- figureVisionPromptsBase = root- return- }- initErr = fmt.Errorf("rag/prompts not found under project root %q", root)- })- if initErr != nil {- return "", initErr- }- return figureVisionPromptsBase, nil+ return figureVisionPrompts.resolve(utility.GetProjectRoot())}func buildVisionMessages(prompt, imageBase64 string) []modelModule.Message {dataURI := pdflayout.InlinePNGDataURL(imageBase64)if dataURI == "" {- dataURI = strings.TrimSpace(imageBase64)+ return nil}return []modelModule.Message{{Role: "user",@@ -283,6 +365,8 @@ func defaultVisionChatInvoker(messages []modelModule.Message,apiConfig *modelModule.APIConfig,) (*modelModule.ChatResponse, error) {+ chatCtx, cancel := context.WithTimeout(ctx, visionChatTimeout)+ defer cancel()vision := true- return driver.ChatWithMessages(ctx, modelName, messages, apiConfig, &modelModule.ChatConfig{Vision: &vision}, nil)+ return driver.ChatWithMessages(chatCtx, modelName, messages, apiConfig, &modelModule.ChatConfig{Vision: &vision}, nil)}diff --git a/internal/ingestion/component/vision_enhancement_test.go b/internal/ingestion/component/vision_enhancement_test.goindex aa55008eb1..84b107f0cd 100644--- a/internal/ingestion/component/vision_enhancement_test.go+++ b/internal/ingestion/component/vision_enhancement_test.go@@ -20,12 +20,17 @@ import ("context""errors""fmt"+ "os"+ "path/filepath""sync""testing"+ "time"+ "ragflow/internal/common""ragflow/internal/dao""ragflow/internal/entity"modelModule "ragflow/internal/entity/models"+ "ragflow/internal/ingestion/component/schema""ragflow/internal/utility""gorm.io/gorm"@@ -135,8 +140,7 @@ func TestVisionEnhancement_EnhancesJSONImagesAndTables(t *testing.T) {dao.DB,tc.fileType,dispatched,- map[string]any{"tenant_id": "t1", "lang": "Japanese"},- )+ map[string]any{"tenant_id": "t1", "lang": "Japanese"}, nil)if err != nil {t.Fatalf("unexpected error: %v", err)}@@ -194,8 +198,7 @@ func TestVisionEnhancement_MarkdownOutputUntouched(t *testing.T) {dao.DB,utility.FileTypeDOCX,dispatched,- map[string]any{"tenant_id": "t1"},- )+ map[string]any{"tenant_id": "t1"}, nil)if err != nil {t.Fatalf("unexpected error: %v", err)}@@ -224,8 +227,7 @@ func TestVisionEnhancement_NonAllowedFileTypeSkipped(t *testing.T) {dao.DB,utility.FileTypeOTHER,dispatched,- map[string]any{"tenant_id": "t1"},- )+ map[string]any{"tenant_id": "t1"}, nil)if err != nil {t.Fatalf("unexpected error: %v", err)}@@ -252,8 +254,7 @@ func TestVisionEnhancement_EmptyOrNoTenantSkipped(t *testing.T) {dao.DB,utility.FileTypeDOCX,dispatched,- map[string]any{},- )+ map[string]any{}, nil)if err != nil || handled {t.Errorf("handled=%v, err=%v, want false, nil for missing tenant_id", handled, err)}@@ -279,8 +280,7 @@ func TestVisionEnhancement_DispatchedErrSkipped(t *testing.T) {dao.DB,utility.FileTypePDF,dispatched,- map[string]any{"tenant_id": "t1"},- )+ map[string]any{"tenant_id": "t1"}, nil)if err != nil {t.Fatalf("unexpected error: %v", err)}@@ -317,8 +317,7 @@ func TestVisionEnhancement_ContextCancellation(t *testing.T) {dao.DB,utility.FileTypePDF,dispatched,- map[string]any{"tenant_id": "t1"},- )+ map[string]any{"tenant_id": "t1"}, nil)if !errors.Is(err, context.Canceled) {t.Fatalf("err = %v, want context.Canceled", err)}@@ -359,8 +358,7 @@ func TestVisionEnhancement_NonStringImageFieldFiltered(t *testing.T) {dao.DB,utility.FileTypePDF,dispatched,- map[string]any{"tenant_id": "t1"},- )+ map[string]any{"tenant_id": "t1"}, nil)if err != nil {t.Fatalf("unexpected error: %v", err)}@@ -402,8 +400,7 @@ func TestVisionEnhancement_MoreThanConcurrencyItems(t *testing.T) {dao.DB,utility.FileTypePDF,dispatched,- map[string]any{"tenant_id": "t1"},- )+ map[string]any{"tenant_id": "t1"}, nil)if err != nil {t.Fatalf("unexpected error: %v", err)}@@ -442,8 +439,7 @@ func TestVisionEnhancement_PlainTextResponseNotTruncated(t *testing.T) {dao.DB,utility.FileTypePDF,dispatched,- map[string]any{"tenant_id": "t1"},- )+ map[string]any{"tenant_id": "t1"}, nil)if err != nil {t.Fatalf("unexpected error: %v", err)}@@ -481,8 +477,7 @@ func TestVisionEnhancement_PromptBuilderErrorSkipped(t *testing.T) {dao.DB,utility.FileTypePDF,dispatched,- map[string]any{"tenant_id": "t1"},- )+ map[string]any{"tenant_id": "t1"}, nil)if err != nil {t.Fatalf("unexpected error: %v", err)}@@ -520,8 +515,7 @@ func TestVisionEnhancement_ModelResolveFailureSkipped(t *testing.T) {dao.DB,utility.FileTypePDF,dispatched,- map[string]any{"tenant_id": "t1"},- )+ map[string]any{"tenant_id": "t1"}, nil)if err != nil {t.Fatalf("unexpected error: %v", err)}@@ -572,8 +566,7 @@ func TestVisionEnhancement_CancellationStopsSchedulingWithManyItems(t *testing.Tdao.DB,utility.FileTypePDF,dispatched,- map[string]any{"tenant_id": "t1"},- )+ map[string]any{"tenant_id": "t1"}, nil)if !errors.Is(err, context.Canceled) {t.Fatalf("err = %v, want context.Canceled", err)}@@ -709,3 +702,171 @@ func TestCleanMarkdownBlock_EdgeCases(t *testing.T) {})}}++func TestPromptDirState_FailureIsSticky(t *testing.T) {+ dir := t.TempDir()+ var state promptDirState+ for i := 0; i < 2; i++ {+ if _, err := state.resolve(dir); err == nil {+ t.Fatalf("call %d: resolve() = nil error, want sticky init error", i+1)+ }+ }+}++func TestPromptDirState_SuccessIsCached(t *testing.T) {+ dir := t.TempDir()+ if err := os.MkdirAll(filepath.Join(dir, "rag", "prompts"), 0o755); err != nil {+ t.Fatal(err)+ }+ var state promptDirState+ for i := 0; i < 2; i++ {+ got, err := state.resolve(dir)+ if err != nil {+ t.Fatalf("call %d: resolve() error: %v", i+1, err)+ }+ if got != dir {+ t.Fatalf("call %d: resolve() = %q, want %q", i+1, got, dir)+ }+ }+}++// TestVisionEnhancement_PerCallModelPreferred verifies that setups vlm.llm_id+// takes precedence over the tenant default IMAGE2TEXT model.+func TestVisionEnhancement_PerCallModelPreferred(t *testing.T) {+ origTenantResolver := resolveTenantModelByType+ origModelResolver := resolveModelConfig+ origInvoker := visionChatInvoker+ origPrompt := figureVisionPromptBuilder+ t.Cleanup(func() {+ resolveTenantModelByType = origTenantResolver+ resolveModelConfig = origModelResolver+ visionChatInvoker = origInvoker+ figureVisionPromptBuilder = origPrompt+ })++ tenantResolverCalled := false+ resolveTenantModelByType = func(context.Context, *gorm.DB, string, entity.ModelType) (modelModule.ModelDriver, string, *modelModule.APIConfig, int, error) {+ tenantResolverCalled = true+ return &visionEnhanceFakeDriver{}, "tenant-model", &modelModule.APIConfig{}, 0, nil+ }++ var gotRef string+ var gotType entity.ModelType+ resolveModelConfig = func(_ context.Context, _ *gorm.DB, _ string, modelType entity.ModelType, ref string) (modelModule.ModelDriver, string, *modelModule.APIConfig, int, error) {+ gotRef = ref+ gotType = modelType+ return &visionEnhanceFakeDriver{}, "custom-model", &modelModule.APIConfig{}, 0, nil+ }++ invoker := &visionEnhanceCaptureInvoker{}+ visionChatInvoker = invoker.invoke+ figureVisionPromptBuilder = fakePrompt++ setups := map[string]schema.ParserSetup{+ "pdf": {"vlm": map[string]any{"llm_id": "custom-vlm@provider"}},+ }+ dispatched := parserDispatchResult{+ OutputFormat: "json",+ JSON: []map[string]any{+ {"text": "", "image": "aGVsbG8=", "doc_type_kwd": "image"},+ },+ }++ _, handled, err := maybeDispatchVisionEnhancement(+ t.Context(),+ dao.DB,+ utility.FileTypePDF,+ dispatched,+ map[string]any{"tenant_id": "t1"},+ setups,+ )+ if err != nil {+ t.Fatalf("maybeDispatchVisionEnhancement: %v", err)+ }+ if !handled {+ t.Fatal("handled = false, want true")+ }+ if gotRef != "custom-vlm@provider" {+ t.Errorf("resolveModelConfig modelRef = %q, want %q", gotRef, "custom-vlm@provider")+ }+ if gotType != entity.ModelTypeImage2Text {+ t.Errorf("resolveModelConfig modelType = %q, want %q", gotType, entity.ModelTypeImage2Text)+ }+ if tenantResolverCalled {+ t.Error("tenant default resolver must not be called when setup vlm.llm_id is set")+ }+}++func TestVisionEnhancement_InvalidImageDataSkipped(t *testing.T) {+ invokerCalled := false+ swapVisionGlobals(t,+ fakeResolver,+ func(_ context.Context, _ modelModule.ModelDriver, _ string, _ []modelModule.Message, _ *modelModule.APIConfig) (*modelModule.ChatResponse, error) {+ invokerCalled = true+ return nil, nil+ },+ fakePrompt,+ )++ dispatched := parserDispatchResult{+ OutputFormat: "json",+ JSON: []map[string]any{+ {"text": "keep", "image": "!!!not-base64!!!", "doc_type_kwd": "image"},+ },+ }++ res, handled, err := maybeDispatchVisionEnhancement(+ t.Context(),+ dao.DB,+ utility.FileTypePDF,+ dispatched,+ map[string]any{"tenant_id": "t1"}, nil)+ if err != nil {+ t.Fatalf("maybeDispatchVisionEnhancement: %v", err)+ }+ if handled {+ t.Error("handled = true, want false when invalid image data is skipped")+ }+ if invokerCalled {+ t.Error("invoker must not be called for invalid image data")+ }+ if got, _ := res.JSON[0]["text"].(string); got != "keep" {+ t.Errorf("text = %q, want unchanged %q", got, "keep")+ }+}++type deadlineCaptureDriver struct {+ modelModule.ModelDriver+ hasDeadline bool+ remaining time.Duration+}++func (d *deadlineCaptureDriver) ChatWithMessages(+ ctx context.Context,+ _ string,+ _ []modelModule.Message,+ _ *modelModule.APIConfig,+ _ *modelModule.ChatConfig,+ _ *common.ModelUsage,+) (*modelModule.ChatResponse, error) {+ deadline, ok := ctx.Deadline()+ d.hasDeadline = ok+ if ok {+ d.remaining = time.Until(deadline)+ }+ ans := "ok"+ return &modelModule.ChatResponse{Answer: &ans}, nil+}++func TestDefaultVisionChatInvoker_AppliesDeadline(t *testing.T) {+ drv := &deadlineCaptureDriver{}+ if _, err := defaultVisionChatInvoker(context.Background(), drv, "m", nil, nil); err != nil {+ t.Fatalf("defaultVisionChatInvoker: %v", err)+ }+ if !drv.hasDeadline {+ t.Fatal("vision chat context must have a deadline")+ }+ if drv.remaining <= 0 || drv.remaining > visionChatTimeout+time.Second {+ t.Fatalf("deadline remaining = %v, want ~%v", drv.remaining, visionChatTimeout)+ }+}diff --git a/internal/parser/parser/pdf_parser_common.go b/internal/parser/parser/pdf_parser_common.goindex c020d8bfdb..d366ea8427 100644--- a/internal/parser/parser/pdf_parser_common.go+++ b/internal/parser/parser/pdf_parser_common.go@@ -458,6 +458,8 @@ func cropMediaSections(result *deepdoctype.ParseResult) {// current section's page span (typically one page, a few at most for a// cross-page section) instead of caching the whole PDF.pageCache := make(map[int]image.Image)+ var lastMinPage = -1+ sectionsOrdered := truerenderPage := func(pn int) image.Image {if img, ok := pageCache[pn]; ok {return img@@ -499,11 +501,21 @@ func cropMediaSections(result *deepdoctype.ParseResult) {}}}+ if lastMinPage >= 0 && minPage < lastMinPage {+ if sectionsOrdered {+ slog.Warn("cropMediaSections: sections out of page order; disabling page cache eviction",+ "min_page", minPage, "last_min_page", lastMinPage)+ }+ sectionsOrdered = false+ }+ lastMinPage = minPage// Evict page images that no later section can reference (all future// sections start at page >= minPage).- for pn := range pageCache {- if pn < minPage {- delete(pageCache, pn)+ if sectionsOrdered {+ for pn := range pageCache {+ if pn < minPage {+ delete(pageCache, pn)+ }}}// Collect every distinct page this section spans so CropSectionByDLAdiff --git a/internal/parser/parser/picture_parser.go b/internal/parser/parser/picture_parser.goindex 3c4e53988b..47406db6f6 100644--- a/internal/parser/parser/picture_parser.go+++ b/internal/parser/parser/picture_parser.go@@ -52,12 +52,12 @@ var imageExtensions = map[string]bool{// PictureParser handles image files for OCR and VLM description.// Mirrors the configuration from setups["picture"]:-// - vlm.llm_id → VLMModelID (IMAGE2TEXT model for describe)+// - VLMModelID ← setup parse_method (non-"ocr"), vlm.llm_id, or top-level llm_id (in order)// - output_format// - image_context_size// - layout_recognize → ("@PaddleOCR:model_name" or empty)type PictureParser struct {- VLMModelID string // vlm.llm_id — the IMAGE2TEXT model+ VLMModelID string // per-call IMAGE2TEXT model reference (parse_method / vlm.llm_id / llm_id)OutputFormat stringImageContextSize int // default 0LayoutRecognize string // layout_recognize (e.g. "@PaddleOCR")@@ -70,17 +70,25 @@ func NewPictureParser() *PictureParser {}// ConfigureFromSetup reads picture-specific configuration from the-// parser setup map. Extracts vlm.llm_id, output_format,+// parser setup map. Extracts the per-call IMAGE2TEXT model reference+// (parse_method non-"ocr" > vlm.llm_id > llm_id), output_format,// image_context_size, layout_recognize, and video_prompt.func (p *PictureParser) ConfigureFromSetup(setup map[string]any) {if p == nil || setup == nil {return}- if vlm, ok := setup["vlm"].(map[string]any); ok {+ if pm, ok := setup["parse_method"].(string); ok && pm != "" && !strings.EqualFold(pm, "ocr") {+ p.VLMModelID = pm+ } else if vlm, ok := setup["vlm"].(map[string]any); ok {if llmID, ok := vlm["llm_id"].(string); ok && llmID != "" {p.VLMModelID = llmID}}+ if p.VLMModelID == "" {+ if llmID, ok := setup["llm_id"].(string); ok && llmID != "" {+ p.VLMModelID = llmID+ }+ }if v, ok := setup["output_format"].(string); ok && v != "" {p.OutputFormat = v}