Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

fix(parser): wire per-call VLM model config, harden vision dispatch (#18846)

jay7772134870e297d72
diff --git a/internal/ingestion/component/dispatch_model.go b/internal/ingestion/component/dispatch_model.go
index cc0a291b5c..6560f9762f 100644
--- a/internal/ingestion/component/dispatch_model.go
+++ b/internal/ingestion/component/dispatch_model.go
@@ -29,6 +29,7 @@ import (
"ragflow/internal/dao"
"ragflow/internal/entity"
modelModule "ragflow/internal/entity/models"
+ "ragflow/internal/ingestion/component/schema"
"gorm.io/gorm"
)
@@ -39,6 +40,29 @@ type tenantModelExtra struct {
var resolveTenantModelByType = defaultResolveTenantModelByType
+// resolveModelConfig resolves a specific model reference (tenant-model ID or
+// "name@instance@provider" composite) to a driver. Exposed as a package var so
+// per-call model-selection tests can inject a fake without a live MySQL.
+var resolveModelConfig = defaultResolveModelConfig
+
+// configuredMediaModelID extracts a per-call model reference from a parser setup.
+// Image parsing stores the VLM model reference in parse_method when it is not
+// "ocr" (mirroring Python rag/flow/parser/parser.py:_image). Other media
+// families use vlm.llm_id, matching the frontend parser form.
+func configuredMediaModelID(setup schema.ParserSetup, family string) string {
+ if family == "image" {
+ if ref := getStringOr(setup, "parse_method", ""); ref != "" && !strings.EqualFold(ref, "ocr") {
+ return ref
+ }
+ }
+ if vlm, ok := setup["vlm"].(map[string]any); ok {
+ if ref, _ := vlm["llm_id"].(string); ref != "" {
+ return ref
+ }
+ }
+ return getStringOr(setup, "llm_id", "")
+}
+
func defaultResolveTenantModelByType(ctx context.Context, db *gorm.DB, tenantID string, modelType entity.ModelType) (modelModule.ModelDriver, string, *modelModule.APIConfig, int, error) {
tenantDAO := dao.NewTenantDAO()
tenant, err := tenantDAO.GetByID(ctx, db, tenantID)
@@ -164,7 +188,7 @@ func stringValue(value *string) string {
return *value
}
-func resolveModelConfig(ctx context.Context, db *gorm.DB, tenantID string, modelType entity.ModelType, modelRef string) (modelModule.ModelDriver, string, *modelModule.APIConfig, int, error) {
+func defaultResolveModelConfig(ctx context.Context, db *gorm.DB, tenantID string, modelType entity.ModelType, modelRef string) (modelModule.ModelDriver, string, *modelModule.APIConfig, int, error) {
modelDAO := dao.NewTenantModelDAO()
if _, err := modelDAO.GetByID(ctx, db, modelRef); err == nil {
return resolveModelConfigByID(ctx, db, tenantID, modelType, modelRef)
diff --git a/internal/ingestion/component/dispatch_model_test.go b/internal/ingestion/component/dispatch_model_test.go
new file mode 100644
index 0000000000..480ea64714
--- /dev/null
+++ b/internal/ingestion/component/dispatch_model_test.go
@@ -0,0 +1,82 @@
+//
+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package component
+
+import (
+ "testing"
+
+ "ragflow/internal/ingestion/component/schema"
+)
+
+func TestConfiguredMediaModelID(t *testing.T) {
+ tests := []struct {
+ name string
+ family string
+ setup schema.ParserSetup
+ want string
+ }{
+ {
+ name: "image parse_method is the VLM model ref",
+ family: "image",
+ setup: schema.ParserSetup{"parse_method": "gpt-4-vision@openai"},
+ want: "gpt-4-vision@openai",
+ },
+ {
+ name: "image ocr falls back to vlm.llm_id",
+ family: "image",
+ setup: schema.ParserSetup{
+ "parse_method": "ocr",
+ "vlm": map[string]any{"llm_id": "gpt-4-vision@openai"},
+ },
+ want: "gpt-4-vision@openai",
+ },
+ {
+ name: "image ocr falls back to top-level llm_id",
+ family: "image",
+ setup: schema.ParserSetup{"parse_method": "ocr", "llm_id": "qwen-vl@dashscope"},
+ want: "qwen-vl@dashscope",
+ },
+ {
+ name: "pdf uses vlm.llm_id",
+ family: "pdf",
+ setup: schema.ParserSetup{
+ "parse_method": "deepdoc",
+ "vlm": map[string]any{"llm_id": "custom-vlm@provider"},
+ },
+ want: "custom-vlm@provider",
+ },
+ {
+ name: "audio uses vlm.llm_id",
+ family: "audio",
+ setup: schema.ParserSetup{"vlm": map[string]any{"llm_id": "whisper@openai"}},
+ want: "whisper@openai",
+ },
+ {
+ name: "empty setup",
+ family: "pdf",
+ setup: schema.ParserSetup{},
+ want: "",
+ },
+ }
+
+ for _, tc := range tests {
+ t.Run(tc.name, func(t *testing.T) {
+ if got := configuredMediaModelID(tc.setup, tc.family); got != tc.want {
+ t.Fatalf("configuredMediaModelID() = %q, want %q", got, tc.want)
+ }
+ })
+ }
+}
diff --git a/internal/ingestion/component/media_dispatch.go b/internal/ingestion/component/media_dispatch.go
index 16970d6a87..864a864210 100644
--- a/internal/ingestion/component/media_dispatch.go
+++ b/internal/ingestion/component/media_dispatch.go
@@ -31,6 +31,7 @@ import (
"encoding/base64"
"fmt"
"image"
+ "log/slog"
"ragflow/internal/common"
// Import image decoders for common formats.
@@ -165,7 +166,21 @@ func maybeDispatchImage(
}
// Short OCR text (or no text): supplement with VLM describe.
- driver, modelName, apiConfig, _, err := resolveTenantModelByType(ctx, db, tenantID, entity.ModelTypeImage2Text)
+ modelRef := configuredMediaModelID(setup, "image")
+ var driver modelModule.ModelDriver
+ var modelName string
+ var apiConfig *modelModule.APIConfig
+ var err error
+ if modelRef != "" {
+ driver, modelName, apiConfig, _, err = resolveModelConfig(ctx, db, tenantID, entity.ModelTypeImage2Text, modelRef)
+ if err != nil {
+ slog.Warn("media dispatch image: per-call VLM resolve failed, falling back to tenant default",
+ "modelRef", modelRef, "tenant", tenantID, "err", err)
+ driver, modelName, apiConfig, _, err = resolveTenantModelByType(ctx, db, tenantID, entity.ModelTypeImage2Text)
+ }
+ } else {
+ driver, modelName, apiConfig, _, err = resolveTenantModelByType(ctx, db, tenantID, entity.ModelTypeImage2Text)
+ }
if err != nil {
// If VLM is unavailable, but we have OCR text, return it.
if ocrText != "" {
@@ -260,7 +275,21 @@ func maybeDispatchAudio(
fmt.Errorf("parser: audio requires tenant_id")
}
- driver, modelName, apiConfig, _, err := resolveTenantModelByType(ctx, db, tenantID, entity.ModelTypeSpeech2Text)
+ modelRef := configuredMediaModelID(setup, "audio")
+ var driver modelModule.ModelDriver
+ var modelName string
+ var apiConfig *modelModule.APIConfig
+ var err error
+ if modelRef != "" {
+ driver, modelName, apiConfig, _, err = resolveModelConfig(ctx, db, tenantID, entity.ModelTypeSpeech2Text, modelRef)
+ if err != nil {
+ slog.Warn("media dispatch audio: per-call VLM resolve failed, falling back to tenant default",
+ "modelRef", modelRef, "tenant", tenantID, "err", err)
+ driver, modelName, apiConfig, _, err = resolveTenantModelByType(ctx, db, tenantID, entity.ModelTypeSpeech2Text)
+ }
+ } else {
+ driver, modelName, apiConfig, _, err = resolveTenantModelByType(ctx, db, tenantID, entity.ModelTypeSpeech2Text)
+ }
if err != nil {
return parserDispatchResult{}, true,
fmt.Errorf("parser: audio speech2text model: %w", err)
diff --git a/internal/ingestion/component/media_dispatch_test.go b/internal/ingestion/component/media_dispatch_test.go
index b86cdb55a9..5f80952114 100644
--- a/internal/ingestion/component/media_dispatch_test.go
+++ b/internal/ingestion/component/media_dispatch_test.go
@@ -411,3 +411,110 @@ func TestDefaultEmailOutputFormatIsJSON(t *testing.T) {
t.Fatalf("email default output_format = %q, want json", got)
}
}
+
+func TestMaybeDispatchImage_UsesConfiguredVLMModel(t *testing.T) {
+ origTenantResolver := resolveTenantModelByType
+ origModelResolver := resolveModelConfig
+ t.Cleanup(func() {
+ resolveTenantModelByType = origTenantResolver
+ resolveModelConfig = origModelResolver
+ })
+
+ tenantResolverCalled := false
+ resolveTenantModelByType = func(_ context.Context, _ *gorm.DB, _ string, _ entity.ModelType) (modelModule.ModelDriver, string, *modelModule.APIConfig, int, error) {
+ tenantResolverCalled = true
+ return nil, "", nil, 0, nil
+ }
+
+ var gotRef string
+ var gotType entity.ModelType
+ drv := &imagePromptCaptureDriver{}
+ resolveModelConfig = func(_ context.Context, _ *gorm.DB, _ string, modelType entity.ModelType, ref string) (modelModule.ModelDriver, string, *modelModule.APIConfig, int, error) {
+ gotRef = ref
+ gotType = modelType
+ return drv, "custom-vlm", &modelModule.APIConfig{}, 0, nil
+ }
+
+ setups := defaultSetups()
+ setups["image"]["parse_method"] = "custom-vlm@provider"
+
+ _, dispatched, err := maybeDispatchImage(
+ t.Context(),
+ dao.DB,
+ utility.FileTypeVISUAL,
+ "test.png",
+ []byte("not-a-real-image"),
+ map[string]any{"tenant_id": "t1"},
+ setups,
+ )
+ if err != nil {
+ t.Fatalf("maybeDispatchImage: %v", err)
+ }
+ if !dispatched {
+ t.Fatal("expected dispatched=true for VISUAL file")
+ }
+ if gotRef != "custom-vlm@provider" {
+ t.Errorf("model ref = %q, want %q", gotRef, "custom-vlm@provider")
+ }
+ if gotType != entity.ModelTypeImage2Text {
+ t.Errorf("model type = %q, want %q", gotType, entity.ModelTypeImage2Text)
+ }
+ if tenantResolverCalled {
+ t.Error("tenant default resolver must not be called when image parse_method names a VLM model")
+ }
+}
+
+func TestMaybeDispatchAudio_UsesConfiguredModel(t *testing.T) {
+ origTenantResolver := resolveTenantModelByType
+ origModelResolver := resolveModelConfig
+ t.Cleanup(func() {
+ resolveTenantModelByType = origTenantResolver
+ resolveModelConfig = origModelResolver
+ })
+
+ tenantResolverCalled := false
+ resolveTenantModelByType = func(_ context.Context, _ *gorm.DB, _ string, _ entity.ModelType) (modelModule.ModelDriver, string, *modelModule.APIConfig, int, error) {
+ tenantResolverCalled = true
+ return nil, "", nil, 0, nil
+ }
+
+ var gotRef string
+ var gotType entity.ModelType
+ drv := &audioTranscribeDriver{transcription: "transcribed"}
+ resolveModelConfig = func(_ context.Context, _ *gorm.DB, _ string, modelType entity.ModelType, ref string) (modelModule.ModelDriver, string, *modelModule.APIConfig, int, error) {
+ gotRef = ref
+ gotType = modelType
+ return drv, "custom-asr", &modelModule.APIConfig{}, 0, nil
+ }
+
+ setups := map[string]schema.ParserSetup{
+ "audio": {"vlm": map[string]any{"llm_id": "custom-asr@provider"}},
+ }
+ res, dispatched, err := maybeDispatchAudio(
+ t.Context(),
+ dao.DB,
+ utility.FileTypeAURAL,
+ "test.mp3",
+ []byte("fake-audio"),
+ map[string]any{"tenant_id": "t1"},
+ setups,
+ )
+ if err != nil {
+ t.Fatalf("maybeDispatchAudio: %v", err)
+ }
+ if !dispatched {
+ t.Fatal("expected dispatched=true for AURAL file")
+ }
+ if gotRef != "custom-asr@provider" {
+ t.Errorf("model ref = %q, want %q", gotRef, "custom-asr@provider")
+ }
+ if gotType != entity.ModelTypeSpeech2Text {
+ t.Errorf("model type = %q, want %q", gotType, entity.ModelTypeSpeech2Text)
+ }
+ if tenantResolverCalled {
+ t.Error("tenant default resolver must not be called when audio setup vlm.llm_id is set")
+ }
+ if len(res.JSON) != 1 || res.JSON[0]["text"] != "transcribed" {
+ t.Fatalf("unexpected audio result: %#v", res.JSON)
+ }
+}
diff --git a/internal/ingestion/component/parser.go b/internal/ingestion/component/parser.go
index a25c5b3d0c..bcf881ad5e 100644
--- a/internal/ingestion/component/parser.go
+++ b/internal/ingestion/component/parser.go
@@ -489,7 +489,7 @@ func (c *ParserComponent) Invoke(ctx context.Context, db *gorm.DB, inputs map[st
// Errors (including context cancellation) are intentionally
// discarded — enhancement is best-effort, matching Python's
// try/except pass pattern.
- dispatched, _, _ = maybeDispatchVisionEnhancement(ctx, db, fileTypeExt, dispatched, inputs)
+ dispatched, _, _ = maybeDispatchVisionEnhancement(ctx, db, fileTypeExt, dispatched, inputs, c.Setups)
}
// Known/supported families must fail loudly when dispatch or
// parsing breaks. Only unknown families keep the raw-text fallback.
diff --git a/internal/ingestion/component/pdf_vision_dispatch.go b/internal/ingestion/component/pdf_vision_dispatch.go
index 9eb7fc7d74..3c20d01a7a 100644
--- a/internal/ingestion/component/pdf_vision_dispatch.go
+++ b/internal/ingestion/component/pdf_vision_dispatch.go
@@ -42,8 +42,7 @@ var (
var (
pdfVisionPromptCache = make(map[string]string)
pdfVisionPromptCacheMu sync.RWMutex
- pdfVisionPromptsBase string
- pdfVisionPromptsOnce sync.Once
+ pdfVisionPrompts promptDirState
)
func maybeDispatchPDFVision(
@@ -704,19 +703,7 @@ func loadPDFVisionPrompt(name string) (string, error) {
}
func pdfVisionPromptsBaseDir() (string, error) {
- var initErr error
- pdfVisionPromptsOnce.Do(func() {
- root := utility.GetProjectRoot()
- if _, statErr := os.Stat(filepath.Join(root, "rag", "prompts")); statErr == nil {
- pdfVisionPromptsBase = root
- return
- }
- initErr = fmt.Errorf("rag/prompts not found under project root %q", root)
- })
- if initErr != nil {
- return "", initErr
- }
- return pdfVisionPromptsBase, nil
+ return pdfVisionPrompts.resolve(utility.GetProjectRoot())
}
// renderPDFVisionPrompt only renders page metadata. The full-page PDF vision
diff --git a/internal/ingestion/component/vision_enhancement.go b/internal/ingestion/component/vision_enhancement.go
index f7b5167d8b..dc2f81308c 100644
--- a/internal/ingestion/component/vision_enhancement.go
+++ b/internal/ingestion/component/vision_enhancement.go
@@ -23,16 +23,21 @@ package component
import (
"context"
+ "encoding/base64"
+ "errors"
"fmt"
+ "log/slog"
"os"
"path/filepath"
"regexp"
"strings"
"sync"
+ "time"
pdflayout "ragflow/internal/deepdoc/parser/pdf/layout"
"ragflow/internal/entity"
modelModule "ragflow/internal/entity/models"
+ "ragflow/internal/ingestion/component/schema"
"ragflow/internal/utility"
"gorm.io/gorm"
@@ -46,15 +51,44 @@ var (
const (
figureVisionPromptFile = "vision_llm_figure_describe_prompt.md"
visionEnhancementConcurrency int = 10
+ // visionChatTimeout bounds a single VLM call so a hung endpoint cannot
+ // occupy one of the concurrency slots indefinitely. Python wraps the
+ // per-image call in @timeout(30, 3) (deepdoc/parser/figure_parser.py).
+ visionChatTimeout = 30 * time.Second
)
var (
- figureVisionPromptsBase string
- figureVisionPromptsOnce sync.Once
+ figureVisionPrompts promptDirState
figureVisionPromptCache = make(map[string]string)
figureVisionPromptMu sync.RWMutex
)
+// promptDirState caches the project root that contains rag/prompts. The once
+// guard makes initialization sticky; unlike a function-local initErr, an
+// initialization failure is preserved for every subsequent call.
+type promptDirState struct {
+ once sync.Once
+ root string
+ initErr error
+}
+
+func (s *promptDirState) resolve(requestedRoot string) (string, error) {
+ s.once.Do(func() {
+ if info, statErr := os.Stat(filepath.Join(requestedRoot, "rag", "prompts")); statErr == nil && info.IsDir() {
+ s.root = requestedRoot
+ return
+ }
+ s.initErr = fmt.Errorf("rag/prompts not found under project root %q", requestedRoot)
+ })
+ if s.initErr != nil {
+ return "", s.initErr
+ }
+ if s.root == "" {
+ return "", errors.New("prompts base dir not initialized")
+ }
+ return s.root, nil
+}
+
// cleanMarkdownBlock mirrors Python common/string_utils.py:clean_markdown_block
//
// re.sub(r"^\s*```markdown\s*\n?", "", text)
@@ -72,6 +106,43 @@ func cleanMarkdownBlock(s string) string {
return strings.TrimSpace(s)
}
+// isUsableVisionImage reports whether raw carries a valid image data URI or a
+// base64 payload that can be sent to a vision model.
+func isUsableVisionImage(raw string) bool {
+ raw = strings.TrimSpace(raw)
+ if raw == "" {
+ return false
+ }
+ if strings.HasPrefix(raw, "data:image/") {
+ idx := strings.Index(raw, "base64,")
+ if idx < 0 {
+ return false
+ }
+ return isValidBase64(raw[idx+len("base64,"):])
+ }
+ if strings.HasPrefix(raw, "http://") || strings.HasPrefix(raw, "https://") {
+ return true
+ }
+ cleaned := strings.Map(func(r rune) rune {
+ if r == '\r' || r == '\n' || r == ' ' || r == '\t' {
+ return -1
+ }
+ return r
+ }, raw)
+ return isValidBase64(cleaned)
+}
+
+func isValidBase64(s string) bool {
+ if s == "" {
+ return false
+ }
+ if _, err := base64.StdEncoding.DecodeString(s); err == nil {
+ return true
+ }
+ _, err := base64.RawStdEncoding.DecodeString(s)
+ return err == nil
+}
+
// isVisionEnhancementAllowed mirrors Python's 3 call sites in
// rag/flow/parser/parser.py:772/978/1115 (PDF/DOCX/Markdown JSON branches).
func isVisionEnhancementAllowed(fileType utility.FileType) bool {
@@ -93,6 +164,7 @@ func maybeDispatchVisionEnhancement(
fileType utility.FileType,
dispatched parserDispatchResult,
inputs map[string]any,
+ setups map[string]schema.ParserSetup,
) (parserDispatchResult, bool, error) {
// 0. FileType allowlist guard.
if !isVisionEnhancementAllowed(fileType) {
@@ -129,8 +201,25 @@ func maybeDispatchVisionEnhancement(
return dispatched, false, nil
}
- // 2. Resolve tenant's IMAGE2TEXT model.
- driver, modelName, apiConfig, _, err := resolveTenantModelByType(ctx, db, tenantID, entity.ModelTypeImage2Text)
+ // 2. Resolve the per-call IMAGE2TEXT model, then fall back to the tenant
+ // default. Mirror Python's vlm_conf["llm_id"] preference.
+ family := resolveParserFamily(fileType)
+ setup := setups[family]
+ modelRef := configuredMediaModelID(setup, family)
+ var driver modelModule.ModelDriver
+ var modelName string
+ var apiConfig *modelModule.APIConfig
+ var err error
+ if modelRef != "" {
+ driver, modelName, apiConfig, _, err = resolveModelConfig(ctx, db, tenantID, entity.ModelTypeImage2Text, modelRef)
+ if err != nil {
+ slog.Warn("vision enhancement: per-call VLM resolve failed, falling back to tenant default",
+ "family", family, "modelRef", modelRef, "tenant", tenantID, "err", err)
+ driver, modelName, apiConfig, _, err = resolveTenantModelByType(ctx, db, tenantID, entity.ModelTypeImage2Text)
+ }
+ } else {
+ driver, modelName, apiConfig, _, err = resolveTenantModelByType(ctx, db, tenantID, entity.ModelTypeImage2Text)
+ }
if err != nil {
// Model not available — skip vision enhancement silently, matching Python's try/except pass.
return dispatched, false, nil
@@ -168,10 +257,15 @@ dispatch:
defer func() { <-sem }()
img, _ := dispatched.JSON[itemIdx]["image"].(string)
- if img == "" {
+ if !isUsableVisionImage(img) {
+ slog.Warn("vision enhancement: invalid image data skipped",
+ "item", itemIdx)
return
}
messages := buildVisionMessages(prompt, img)
+ if len(messages) == 0 {
+ return
+ }
resp, ierr := visionChatInvoker(ctx, driver, modelName, messages, apiConfig)
if ierr != nil {
return
@@ -236,25 +330,13 @@ func loadFigureVisionPromptFile(filename string) (string, error) {
}
func figureVisionPromptsBaseDir() (string, error) {
- var initErr error
- figureVisionPromptsOnce.Do(func() {
- root := utility.GetProjectRoot()
- if _, statErr := os.Stat(filepath.Join(root, "rag", "prompts")); statErr == nil {
- figureVisionPromptsBase = root
- return
- }
- initErr = fmt.Errorf("rag/prompts not found under project root %q", root)
- })
- if initErr != nil {
- return "", initErr
- }
- return figureVisionPromptsBase, nil
+ return figureVisionPrompts.resolve(utility.GetProjectRoot())
}
func buildVisionMessages(prompt, imageBase64 string) []modelModule.Message {
dataURI := pdflayout.InlinePNGDataURL(imageBase64)
if dataURI == "" {
- dataURI = strings.TrimSpace(imageBase64)
+ return nil
}
return []modelModule.Message{{
Role: "user",
@@ -283,6 +365,8 @@ func defaultVisionChatInvoker(
messages []modelModule.Message,
apiConfig *modelModule.APIConfig,
) (*modelModule.ChatResponse, error) {
+ chatCtx, cancel := context.WithTimeout(ctx, visionChatTimeout)
+ defer cancel()
vision := true
- return driver.ChatWithMessages(ctx, modelName, messages, apiConfig, &modelModule.ChatConfig{Vision: &vision}, nil)
+ return driver.ChatWithMessages(chatCtx, modelName, messages, apiConfig, &modelModule.ChatConfig{Vision: &vision}, nil)
}
diff --git a/internal/ingestion/component/vision_enhancement_test.go b/internal/ingestion/component/vision_enhancement_test.go
index aa55008eb1..84b107f0cd 100644
--- a/internal/ingestion/component/vision_enhancement_test.go
+++ b/internal/ingestion/component/vision_enhancement_test.go
@@ -20,12 +20,17 @@ import (
"context"
"errors"
"fmt"
+ "os"
+ "path/filepath"
"sync"
"testing"
+ "time"
+ "ragflow/internal/common"
"ragflow/internal/dao"
"ragflow/internal/entity"
modelModule "ragflow/internal/entity/models"
+ "ragflow/internal/ingestion/component/schema"
"ragflow/internal/utility"
"gorm.io/gorm"
@@ -135,8 +140,7 @@ func TestVisionEnhancement_EnhancesJSONImagesAndTables(t *testing.T) {
dao.DB,
tc.fileType,
dispatched,
- map[string]any{"tenant_id": "t1", "lang": "Japanese"},
- )
+ map[string]any{"tenant_id": "t1", "lang": "Japanese"}, nil)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
@@ -194,8 +198,7 @@ func TestVisionEnhancement_MarkdownOutputUntouched(t *testing.T) {
dao.DB,
utility.FileTypeDOCX,
dispatched,
- map[string]any{"tenant_id": "t1"},
- )
+ map[string]any{"tenant_id": "t1"}, nil)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
@@ -224,8 +227,7 @@ func TestVisionEnhancement_NonAllowedFileTypeSkipped(t *testing.T) {
dao.DB,
utility.FileTypeOTHER,
dispatched,
- map[string]any{"tenant_id": "t1"},
- )
+ map[string]any{"tenant_id": "t1"}, nil)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
@@ -252,8 +254,7 @@ func TestVisionEnhancement_EmptyOrNoTenantSkipped(t *testing.T) {
dao.DB,
utility.FileTypeDOCX,
dispatched,
- map[string]any{},
- )
+ map[string]any{}, nil)
if err != nil || handled {
t.Errorf("handled=%v, err=%v, want false, nil for missing tenant_id", handled, err)
}
@@ -279,8 +280,7 @@ func TestVisionEnhancement_DispatchedErrSkipped(t *testing.T) {
dao.DB,
utility.FileTypePDF,
dispatched,
- map[string]any{"tenant_id": "t1"},
- )
+ map[string]any{"tenant_id": "t1"}, nil)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
@@ -317,8 +317,7 @@ func TestVisionEnhancement_ContextCancellation(t *testing.T) {
dao.DB,
utility.FileTypePDF,
dispatched,
- map[string]any{"tenant_id": "t1"},
- )
+ map[string]any{"tenant_id": "t1"}, nil)
if !errors.Is(err, context.Canceled) {
t.Fatalf("err = %v, want context.Canceled", err)
}
@@ -359,8 +358,7 @@ func TestVisionEnhancement_NonStringImageFieldFiltered(t *testing.T) {
dao.DB,
utility.FileTypePDF,
dispatched,
- map[string]any{"tenant_id": "t1"},
- )
+ map[string]any{"tenant_id": "t1"}, nil)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
@@ -402,8 +400,7 @@ func TestVisionEnhancement_MoreThanConcurrencyItems(t *testing.T) {
dao.DB,
utility.FileTypePDF,
dispatched,
- map[string]any{"tenant_id": "t1"},
- )
+ map[string]any{"tenant_id": "t1"}, nil)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
@@ -442,8 +439,7 @@ func TestVisionEnhancement_PlainTextResponseNotTruncated(t *testing.T) {
dao.DB,
utility.FileTypePDF,
dispatched,
- map[string]any{"tenant_id": "t1"},
- )
+ map[string]any{"tenant_id": "t1"}, nil)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
@@ -481,8 +477,7 @@ func TestVisionEnhancement_PromptBuilderErrorSkipped(t *testing.T) {
dao.DB,
utility.FileTypePDF,
dispatched,
- map[string]any{"tenant_id": "t1"},
- )
+ map[string]any{"tenant_id": "t1"}, nil)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
@@ -520,8 +515,7 @@ func TestVisionEnhancement_ModelResolveFailureSkipped(t *testing.T) {
dao.DB,
utility.FileTypePDF,
dispatched,
- map[string]any{"tenant_id": "t1"},
- )
+ map[string]any{"tenant_id": "t1"}, nil)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
@@ -572,8 +566,7 @@ func TestVisionEnhancement_CancellationStopsSchedulingWithManyItems(t *testing.T
dao.DB,
utility.FileTypePDF,
dispatched,
- map[string]any{"tenant_id": "t1"},
- )
+ map[string]any{"tenant_id": "t1"}, nil)
if !errors.Is(err, context.Canceled) {
t.Fatalf("err = %v, want context.Canceled", err)
}
@@ -709,3 +702,171 @@ func TestCleanMarkdownBlock_EdgeCases(t *testing.T) {
})
}
}
+
+func TestPromptDirState_FailureIsSticky(t *testing.T) {
+ dir := t.TempDir()
+ var state promptDirState
+ for i := 0; i < 2; i++ {
+ if _, err := state.resolve(dir); err == nil {
+ t.Fatalf("call %d: resolve() = nil error, want sticky init error", i+1)
+ }
+ }
+}
+
+func TestPromptDirState_SuccessIsCached(t *testing.T) {
+ dir := t.TempDir()
+ if err := os.MkdirAll(filepath.Join(dir, "rag", "prompts"), 0o755); err != nil {
+ t.Fatal(err)
+ }
+ var state promptDirState
+ for i := 0; i < 2; i++ {
+ got, err := state.resolve(dir)
+ if err != nil {
+ t.Fatalf("call %d: resolve() error: %v", i+1, err)
+ }
+ if got != dir {
+ t.Fatalf("call %d: resolve() = %q, want %q", i+1, got, dir)
+ }
+ }
+}
+
+// TestVisionEnhancement_PerCallModelPreferred verifies that setups vlm.llm_id
+// takes precedence over the tenant default IMAGE2TEXT model.
+func TestVisionEnhancement_PerCallModelPreferred(t *testing.T) {
+ origTenantResolver := resolveTenantModelByType
+ origModelResolver := resolveModelConfig
+ origInvoker := visionChatInvoker
+ origPrompt := figureVisionPromptBuilder
+ t.Cleanup(func() {
+ resolveTenantModelByType = origTenantResolver
+ resolveModelConfig = origModelResolver
+ visionChatInvoker = origInvoker
+ figureVisionPromptBuilder = origPrompt
+ })
+
+ tenantResolverCalled := false
+ resolveTenantModelByType = func(context.Context, *gorm.DB, string, entity.ModelType) (modelModule.ModelDriver, string, *modelModule.APIConfig, int, error) {
+ tenantResolverCalled = true
+ return &visionEnhanceFakeDriver{}, "tenant-model", &modelModule.APIConfig{}, 0, nil
+ }
+
+ var gotRef string
+ var gotType entity.ModelType
+ resolveModelConfig = func(_ context.Context, _ *gorm.DB, _ string, modelType entity.ModelType, ref string) (modelModule.ModelDriver, string, *modelModule.APIConfig, int, error) {
+ gotRef = ref
+ gotType = modelType
+ return &visionEnhanceFakeDriver{}, "custom-model", &modelModule.APIConfig{}, 0, nil
+ }
+
+ invoker := &visionEnhanceCaptureInvoker{}
+ visionChatInvoker = invoker.invoke
+ figureVisionPromptBuilder = fakePrompt
+
+ setups := map[string]schema.ParserSetup{
+ "pdf": {"vlm": map[string]any{"llm_id": "custom-vlm@provider"}},
+ }
+ dispatched := parserDispatchResult{
+ OutputFormat: "json",
+ JSON: []map[string]any{
+ {"text": "", "image": "aGVsbG8=", "doc_type_kwd": "image"},
+ },
+ }
+
+ _, handled, err := maybeDispatchVisionEnhancement(
+ t.Context(),
+ dao.DB,
+ utility.FileTypePDF,
+ dispatched,
+ map[string]any{"tenant_id": "t1"},
+ setups,
+ )
+ if err != nil {
+ t.Fatalf("maybeDispatchVisionEnhancement: %v", err)
+ }
+ if !handled {
+ t.Fatal("handled = false, want true")
+ }
+ if gotRef != "custom-vlm@provider" {
+ t.Errorf("resolveModelConfig modelRef = %q, want %q", gotRef, "custom-vlm@provider")
+ }
+ if gotType != entity.ModelTypeImage2Text {
+ t.Errorf("resolveModelConfig modelType = %q, want %q", gotType, entity.ModelTypeImage2Text)
+ }
+ if tenantResolverCalled {
+ t.Error("tenant default resolver must not be called when setup vlm.llm_id is set")
+ }
+}
+
+func TestVisionEnhancement_InvalidImageDataSkipped(t *testing.T) {
+ invokerCalled := false
+ swapVisionGlobals(t,
+ fakeResolver,
+ func(_ context.Context, _ modelModule.ModelDriver, _ string, _ []modelModule.Message, _ *modelModule.APIConfig) (*modelModule.ChatResponse, error) {
+ invokerCalled = true
+ return nil, nil
+ },
+ fakePrompt,
+ )
+
+ dispatched := parserDispatchResult{
+ OutputFormat: "json",
+ JSON: []map[string]any{
+ {"text": "keep", "image": "!!!not-base64!!!", "doc_type_kwd": "image"},
+ },
+ }
+
+ res, handled, err := maybeDispatchVisionEnhancement(
+ t.Context(),
+ dao.DB,
+ utility.FileTypePDF,
+ dispatched,
+ map[string]any{"tenant_id": "t1"}, nil)
+ if err != nil {
+ t.Fatalf("maybeDispatchVisionEnhancement: %v", err)
+ }
+ if handled {
+ t.Error("handled = true, want false when invalid image data is skipped")
+ }
+ if invokerCalled {
+ t.Error("invoker must not be called for invalid image data")
+ }
+ if got, _ := res.JSON[0]["text"].(string); got != "keep" {
+ t.Errorf("text = %q, want unchanged %q", got, "keep")
+ }
+}
+
+type deadlineCaptureDriver struct {
+ modelModule.ModelDriver
+ hasDeadline bool
+ remaining time.Duration
+}
+
+func (d *deadlineCaptureDriver) ChatWithMessages(
+ ctx context.Context,
+ _ string,
+ _ []modelModule.Message,
+ _ *modelModule.APIConfig,
+ _ *modelModule.ChatConfig,
+ _ *common.ModelUsage,
+) (*modelModule.ChatResponse, error) {
+ deadline, ok := ctx.Deadline()
+ d.hasDeadline = ok
+ if ok {
+ d.remaining = time.Until(deadline)
+ }
+ ans := "ok"
+ return &modelModule.ChatResponse{Answer: &ans}, nil
+}
+
+func TestDefaultVisionChatInvoker_AppliesDeadline(t *testing.T) {
+ drv := &deadlineCaptureDriver{}
+ if _, err := defaultVisionChatInvoker(context.Background(), drv, "m", nil, nil); err != nil {
+ t.Fatalf("defaultVisionChatInvoker: %v", err)
+ }
+ if !drv.hasDeadline {
+ t.Fatal("vision chat context must have a deadline")
+ }
+ if drv.remaining <= 0 || drv.remaining > visionChatTimeout+time.Second {
+ t.Fatalf("deadline remaining = %v, want ~%v", drv.remaining, visionChatTimeout)
+ }
+}
diff --git a/internal/parser/parser/pdf_parser_common.go b/internal/parser/parser/pdf_parser_common.go
index c020d8bfdb..d366ea8427 100644
--- a/internal/parser/parser/pdf_parser_common.go
+++ b/internal/parser/parser/pdf_parser_common.go
@@ -458,6 +458,8 @@ func cropMediaSections(result *deepdoctype.ParseResult) {
// current section's page span (typically one page, a few at most for a
// cross-page section) instead of caching the whole PDF.
pageCache := make(map[int]image.Image)
+ var lastMinPage = -1
+ sectionsOrdered := true
renderPage := func(pn int) image.Image {
if img, ok := pageCache[pn]; ok {
return img
@@ -499,11 +501,21 @@ func cropMediaSections(result *deepdoctype.ParseResult) {
}
}
}
+ if lastMinPage >= 0 && minPage < lastMinPage {
+ if sectionsOrdered {
+ slog.Warn("cropMediaSections: sections out of page order; disabling page cache eviction",
+ "min_page", minPage, "last_min_page", lastMinPage)
+ }
+ sectionsOrdered = false
+ }
+ lastMinPage = minPage
// Evict page images that no later section can reference (all future
// sections start at page >= minPage).
- for pn := range pageCache {
- if pn < minPage {
- delete(pageCache, pn)
+ if sectionsOrdered {
+ for pn := range pageCache {
+ if pn < minPage {
+ delete(pageCache, pn)
+ }
}
}
// Collect every distinct page this section spans so CropSectionByDLA
diff --git a/internal/parser/parser/picture_parser.go b/internal/parser/parser/picture_parser.go
index 3c4e53988b..47406db6f6 100644
--- a/internal/parser/parser/picture_parser.go
+++ b/internal/parser/parser/picture_parser.go
@@ -52,12 +52,12 @@ var imageExtensions = map[string]bool{
// PictureParser handles image files for OCR and VLM description.
// Mirrors the configuration from setups["picture"]:
-// - vlm.llm_id → VLMModelID (IMAGE2TEXT model for describe)
+// - VLMModelID ← setup parse_method (non-"ocr"), vlm.llm_id, or top-level llm_id (in order)
// - output_format
// - image_context_size
// - layout_recognize → ("@PaddleOCR:model_name" or empty)
type PictureParser struct {
- VLMModelID string // vlm.llm_id — the IMAGE2TEXT model
+ VLMModelID string // per-call IMAGE2TEXT model reference (parse_method / vlm.llm_id / llm_id)
OutputFormat string
ImageContextSize int // default 0
LayoutRecognize string // layout_recognize (e.g. "@PaddleOCR")
@@ -70,17 +70,25 @@ func NewPictureParser() *PictureParser {
}
// ConfigureFromSetup reads picture-specific configuration from the
-// parser setup map. Extracts vlm.llm_id, output_format,
+// parser setup map. Extracts the per-call IMAGE2TEXT model reference
+// (parse_method non-"ocr" > vlm.llm_id > llm_id), output_format,
// image_context_size, layout_recognize, and video_prompt.
func (p *PictureParser) ConfigureFromSetup(setup map[string]any) {
if p == nil || setup == nil {
return
}
- if vlm, ok := setup["vlm"].(map[string]any); ok {
+ if pm, ok := setup["parse_method"].(string); ok && pm != "" && !strings.EqualFold(pm, "ocr") {
+ p.VLMModelID = pm
+ } else if vlm, ok := setup["vlm"].(map[string]any); ok {
if llmID, ok := vlm["llm_id"].(string); ok && llmID != "" {
p.VLMModelID = llmID
}
}
+ if p.VLMModelID == "" {
+ if llmID, ok := setup["llm_id"].(string); ok && llmID != "" {
+ p.VLMModelID = llmID
+ }
+ }
if v, ok := setup["output_format"].(string); ok && v != "" {
p.OutputFormat = v
}