RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
fix(parser): run image VLM when selected as parse_method (#20668)
ciel143e74dc863c
diff --git a/internal/ingestion/component/media_dispatch.go b/internal/ingestion/component/media_dispatch.goindex 3cb76bf8e9..94c0c40654 100644--- a/internal/ingestion/component/media_dispatch.go+++ b/internal/ingestion/component/media_dispatch.go@@ -97,6 +97,15 @@ func maybeDispatchImage(}method := getStringOr(setup, "parse_method", "")useOCR := method == "" || strings.EqualFold(method, "ocr")+ // A model named as parse_method is an explicit VLM request (mirrors+ // Python rag/flow/parser/parser.py:_image: "ocr" runs OCR, anything+ // else is the vision model). It must run the description even when the+ // global enhancement switch is off, otherwise the image item carries no+ // text and the Tokenizer's retrievability filter drops the chunk.+ modelFromParseMethod := ""+ if !useOCR {+ modelFromParseMethod = method+ }release, err := parser.AcquireImageMedia(ctx)if err != nil {return parser.ParseResult{}, true, err@@ -129,10 +138,10 @@ func maybeDispatchImage(if err := ctx.Err(); err != nil {return parsed, true, err}- if enableVisionEnhancement {+ if enableVisionEnhancement || modelFromParseMethod != "" {modelRef := visionModelID- if !useOCR {- modelRef = method+ if modelFromParseMethod != "" {+ modelRef = modelFromParseMethod}description, warnings := describeImage(ctx, db, imageData, getStringOr(inputs, "tenant_id", ""), setup, inputs, modelRef)parsed.Warnings = append(parsed.Warnings, warnings...)diff --git a/internal/ingestion/component/media_dispatch_test.go b/internal/ingestion/component/media_dispatch_test.goindex 23572ed780..f69265e4a7 100644--- a/internal/ingestion/component/media_dispatch_test.go+++ b/internal/ingestion/component/media_dispatch_test.go@@ -92,7 +92,7 @@ func TestMaybeDispatchImageOCRIndependentOfVision(t *testing.T) {{"enabled", "ocr", true, "t1", "OCR text\ncaptured"},{"unavailable", "ocr", true, "", "OCR text"},{"default", "", false, "", "OCR text"},- {"vlm-only", "custom-vlm", false, "", ""},+ {"vlm-only", "custom-vlm", false, "t1", "captured"},} {t.Run(tc.name, func(t *testing.T) {originalResolver := resolveTenantModelByType@@ -100,6 +100,13 @@ func TestMaybeDispatchImageOCRIndependentOfVision(t *testing.T) {return &imagePromptCaptureDriver{}, "vision", &modelModule.APIConfig{}, 0, nil}t.Cleanup(func() { resolveTenantModelByType = originalResolver })+ // A model selected via parse_method resolves through resolveModelConfig;+ // stub it so the forced VLM path does not touch the database.+ originalModelResolver := resolveModelConfig+ resolveModelConfig = func(context.Context, *gorm.DB, string, entity.ModelType, string) (modelModule.ModelDriver, string, *modelModule.APIConfig, int, error) {+ return &imagePromptCaptureDriver{}, "custom-vlm", &modelModule.APIConfig{}, 0, nil+ }+ t.Cleanup(func() { resolveModelConfig = originalModelResolver })params["image"].(map[string]any)["parse_method"] = tc.methodcomponent, err := NewParserComponent(params)if err != nil {@@ -122,6 +129,101 @@ func TestMaybeDispatchImageOCRIndependentOfVision(t *testing.T) {}}+// TestMaybeDispatchImage_ParseMethodModelRunsVisionWithoutGlobalEnhancement+// pins that a model selected as the image parse_method is itself an explicit+// VLM request: the vision description must run even when the global+// enable_vision_enhancement switch is off. Before the fix the selected model+// was ignored unless the switch was on, leaving the image item text-less so+// the Tokenizer's retrievability filter dropped the only chunk.+func TestMaybeDispatchImage_ParseMethodModelRunsVisionWithoutGlobalEnhancement(t *testing.T) {+ originalModelResolver := resolveModelConfig+ originalTenantResolver := resolveTenantModelByType+ t.Cleanup(func() {+ resolveModelConfig = originalModelResolver+ resolveTenantModelByType = originalTenantResolver+ })++ tenantResolverCalled := false+ resolveTenantModelByType = func(context.Context, *gorm.DB, string, entity.ModelType) (modelModule.ModelDriver, string, *modelModule.APIConfig, int, error) {+ tenantResolverCalled = true+ return nil, "", nil, 0, nil+ }++ var gotRef string+ var gotType entity.ModelType+ resolveModelConfig = func(_ context.Context, _ *gorm.DB, _ string, modelType entity.ModelType, ref string) (modelModule.ModelDriver, string, *modelModule.APIConfig, int, error) {+ gotRef = ref+ gotType = modelType+ return &imagePromptCaptureDriver{}, "custom-vlm", &modelModule.APIConfig{}, 0, nil+ }++ setups := defaultSetups()+ setups["image"]["parse_method"] = "custom-vlm@provider"++ res, dispatched, err := maybeDispatchImage(+ t.Context(),+ dao.DB,+ utility.FileTypeVISUAL,+ "photo.png",+ picturePNG(t),+ map[string]any{"tenant_id": "t1"},+ setups,+ false, "",+ )+ if err != nil {+ t.Fatalf("maybeDispatchImage: %v", err)+ }+ if !dispatched {+ t.Fatal("expected dispatched=true for VISUAL file")+ }+ if gotRef != "custom-vlm@provider" || gotType != entity.ModelTypeImage2Text {+ t.Fatalf("resolved model = %q / %v, want custom-vlm@provider / %v", gotRef, gotType, entity.ModelTypeImage2Text)+ }+ if tenantResolverCalled {+ t.Error("tenant default resolver must not be called when parse_method names a VLM model")+ }+ if len(res.JSON) != 1 {+ t.Fatalf("JSON len = %d, want 1", len(res.JSON))+ }+ text, _ := res.JSON[0]["text"].(string)+ if strings.TrimSpace(text) == "" {+ t.Fatalf("image item text is empty; VLM description was not applied: %+v", res.JSON[0])+ }+ if res.JSON[0]["image"] == "" || res.JSON[0]["doc_type_kwd"] != "image" {+ t.Fatalf("image attachment lost: %+v", res.JSON[0])+ }+}++// TestMaybeDispatchImage_OCRWithoutEnhancementSkipsVision pins the other side+// of the contract: parse_method "ocr" with enhancement off keeps running OCR+// and must not resolve any vision model.+func TestMaybeDispatchImage_OCRWithoutEnhancementSkipsVision(t *testing.T) {+ originalAnalyzer := deepdoctype.NativeDocAnalyzerFactory+ deepdoctype.NativeDocAnalyzerFactory = func() (deepdoctype.DocAnalyzer, bool) { return &pictureOCRAnalyzer{}, true }+ t.Cleanup(func() { deepdoctype.NativeDocAnalyzerFactory = originalAnalyzer })++ originalModelResolver := resolveModelConfig+ t.Cleanup(func() { resolveModelConfig = originalModelResolver })+ resolveModelConfigCalled := false+ resolveModelConfig = func(context.Context, *gorm.DB, string, entity.ModelType, string) (modelModule.ModelDriver, string, *modelModule.APIConfig, int, error) {+ resolveModelConfigCalled = true+ return nil, "", nil, 0, nil+ }++ setups := defaultSetups()+ setups["image"]["parse_method"] = "ocr"+ res, _, err := maybeDispatchImage(t.Context(), dao.DB, utility.FileTypeVISUAL, "photo.png", picturePNG(t), map[string]any{"tenant_id": "t1"}, setups, false, "")+ if err != nil {+ t.Fatalf("maybeDispatchImage: %v", err)+ }+ if resolveModelConfigCalled {+ t.Fatal("resolveModelConfig must not be called for parse_method=ocr without enhancement")+ }+ if len(res.JSON) != 1 || res.JSON[0]["text"] != "OCR text" {+ t.Fatalf("res = %+v, want OCR text", res.JSON)+ }+}+// imagePromptCaptureDriver embeds ModelDriver so it satisfies the interface// without listing every method; only ChatWithMessages is overridden to record// the messages the image branch selected.