Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

fix(parser): run image VLM when selected as parse_method (#20668)

ciel143e74dc863c
diff --git a/internal/ingestion/component/media_dispatch.go b/internal/ingestion/component/media_dispatch.go
index 3cb76bf8e9..94c0c40654 100644
--- a/internal/ingestion/component/media_dispatch.go
+++ b/internal/ingestion/component/media_dispatch.go
@@ -97,6 +97,15 @@ func maybeDispatchImage(
}
method := getStringOr(setup, "parse_method", "")
useOCR := method == "" || strings.EqualFold(method, "ocr")
+ // A model named as parse_method is an explicit VLM request (mirrors
+ // Python rag/flow/parser/parser.py:_image: "ocr" runs OCR, anything
+ // else is the vision model). It must run the description even when the
+ // global enhancement switch is off, otherwise the image item carries no
+ // text and the Tokenizer's retrievability filter drops the chunk.
+ modelFromParseMethod := ""
+ if !useOCR {
+ modelFromParseMethod = method
+ }
release, err := parser.AcquireImageMedia(ctx)
if err != nil {
return parser.ParseResult{}, true, err
@@ -129,10 +138,10 @@ func maybeDispatchImage(
if err := ctx.Err(); err != nil {
return parsed, true, err
}
- if enableVisionEnhancement {
+ if enableVisionEnhancement || modelFromParseMethod != "" {
modelRef := visionModelID
- if !useOCR {
- modelRef = method
+ if modelFromParseMethod != "" {
+ modelRef = modelFromParseMethod
}
description, warnings := describeImage(ctx, db, imageData, getStringOr(inputs, "tenant_id", ""), setup, inputs, modelRef)
parsed.Warnings = append(parsed.Warnings, warnings...)
diff --git a/internal/ingestion/component/media_dispatch_test.go b/internal/ingestion/component/media_dispatch_test.go
index 23572ed780..f69265e4a7 100644
--- a/internal/ingestion/component/media_dispatch_test.go
+++ b/internal/ingestion/component/media_dispatch_test.go
@@ -92,7 +92,7 @@ func TestMaybeDispatchImageOCRIndependentOfVision(t *testing.T) {
{"enabled", "ocr", true, "t1", "OCR text\ncaptured"},
{"unavailable", "ocr", true, "", "OCR text"},
{"default", "", false, "", "OCR text"},
- {"vlm-only", "custom-vlm", false, "", ""},
+ {"vlm-only", "custom-vlm", false, "t1", "captured"},
} {
t.Run(tc.name, func(t *testing.T) {
originalResolver := resolveTenantModelByType
@@ -100,6 +100,13 @@ func TestMaybeDispatchImageOCRIndependentOfVision(t *testing.T) {
return &imagePromptCaptureDriver{}, "vision", &modelModule.APIConfig{}, 0, nil
}
t.Cleanup(func() { resolveTenantModelByType = originalResolver })
+ // A model selected via parse_method resolves through resolveModelConfig;
+ // stub it so the forced VLM path does not touch the database.
+ originalModelResolver := resolveModelConfig
+ resolveModelConfig = func(context.Context, *gorm.DB, string, entity.ModelType, string) (modelModule.ModelDriver, string, *modelModule.APIConfig, int, error) {
+ return &imagePromptCaptureDriver{}, "custom-vlm", &modelModule.APIConfig{}, 0, nil
+ }
+ t.Cleanup(func() { resolveModelConfig = originalModelResolver })
params["image"].(map[string]any)["parse_method"] = tc.method
component, err := NewParserComponent(params)
if err != nil {
@@ -122,6 +129,101 @@ func TestMaybeDispatchImageOCRIndependentOfVision(t *testing.T) {
}
}
+// TestMaybeDispatchImage_ParseMethodModelRunsVisionWithoutGlobalEnhancement
+// pins that a model selected as the image parse_method is itself an explicit
+// VLM request: the vision description must run even when the global
+// enable_vision_enhancement switch is off. Before the fix the selected model
+// was ignored unless the switch was on, leaving the image item text-less so
+// the Tokenizer's retrievability filter dropped the only chunk.
+func TestMaybeDispatchImage_ParseMethodModelRunsVisionWithoutGlobalEnhancement(t *testing.T) {
+ originalModelResolver := resolveModelConfig
+ originalTenantResolver := resolveTenantModelByType
+ t.Cleanup(func() {
+ resolveModelConfig = originalModelResolver
+ resolveTenantModelByType = originalTenantResolver
+ })
+
+ tenantResolverCalled := false
+ resolveTenantModelByType = func(context.Context, *gorm.DB, string, entity.ModelType) (modelModule.ModelDriver, string, *modelModule.APIConfig, int, error) {
+ tenantResolverCalled = true
+ return nil, "", nil, 0, nil
+ }
+
+ var gotRef string
+ var gotType entity.ModelType
+ resolveModelConfig = func(_ context.Context, _ *gorm.DB, _ string, modelType entity.ModelType, ref string) (modelModule.ModelDriver, string, *modelModule.APIConfig, int, error) {
+ gotRef = ref
+ gotType = modelType
+ return &imagePromptCaptureDriver{}, "custom-vlm", &modelModule.APIConfig{}, 0, nil
+ }
+
+ setups := defaultSetups()
+ setups["image"]["parse_method"] = "custom-vlm@provider"
+
+ res, dispatched, err := maybeDispatchImage(
+ t.Context(),
+ dao.DB,
+ utility.FileTypeVISUAL,
+ "photo.png",
+ picturePNG(t),
+ map[string]any{"tenant_id": "t1"},
+ setups,
+ false, "",
+ )
+ if err != nil {
+ t.Fatalf("maybeDispatchImage: %v", err)
+ }
+ if !dispatched {
+ t.Fatal("expected dispatched=true for VISUAL file")
+ }
+ if gotRef != "custom-vlm@provider" || gotType != entity.ModelTypeImage2Text {
+ t.Fatalf("resolved model = %q / %v, want custom-vlm@provider / %v", gotRef, gotType, entity.ModelTypeImage2Text)
+ }
+ if tenantResolverCalled {
+ t.Error("tenant default resolver must not be called when parse_method names a VLM model")
+ }
+ if len(res.JSON) != 1 {
+ t.Fatalf("JSON len = %d, want 1", len(res.JSON))
+ }
+ text, _ := res.JSON[0]["text"].(string)
+ if strings.TrimSpace(text) == "" {
+ t.Fatalf("image item text is empty; VLM description was not applied: %+v", res.JSON[0])
+ }
+ if res.JSON[0]["image"] == "" || res.JSON[0]["doc_type_kwd"] != "image" {
+ t.Fatalf("image attachment lost: %+v", res.JSON[0])
+ }
+}
+
+// TestMaybeDispatchImage_OCRWithoutEnhancementSkipsVision pins the other side
+// of the contract: parse_method "ocr" with enhancement off keeps running OCR
+// and must not resolve any vision model.
+func TestMaybeDispatchImage_OCRWithoutEnhancementSkipsVision(t *testing.T) {
+ originalAnalyzer := deepdoctype.NativeDocAnalyzerFactory
+ deepdoctype.NativeDocAnalyzerFactory = func() (deepdoctype.DocAnalyzer, bool) { return &pictureOCRAnalyzer{}, true }
+ t.Cleanup(func() { deepdoctype.NativeDocAnalyzerFactory = originalAnalyzer })
+
+ originalModelResolver := resolveModelConfig
+ t.Cleanup(func() { resolveModelConfig = originalModelResolver })
+ resolveModelConfigCalled := false
+ resolveModelConfig = func(context.Context, *gorm.DB, string, entity.ModelType, string) (modelModule.ModelDriver, string, *modelModule.APIConfig, int, error) {
+ resolveModelConfigCalled = true
+ return nil, "", nil, 0, nil
+ }
+
+ setups := defaultSetups()
+ setups["image"]["parse_method"] = "ocr"
+ res, _, err := maybeDispatchImage(t.Context(), dao.DB, utility.FileTypeVISUAL, "photo.png", picturePNG(t), map[string]any{"tenant_id": "t1"}, setups, false, "")
+ if err != nil {
+ t.Fatalf("maybeDispatchImage: %v", err)
+ }
+ if resolveModelConfigCalled {
+ t.Fatal("resolveModelConfig must not be called for parse_method=ocr without enhancement")
+ }
+ if len(res.JSON) != 1 || res.JSON[0]["text"] != "OCR text" {
+ t.Fatalf("res = %+v, want OCR text", res.JSON)
+ }
+}
+
// imagePromptCaptureDriver embeds ModelDriver so it satisfies the interface
// without listing every method; only ChatWithMessages is overridden to record
// the messages the image branch selected.