Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

refactor(parser): split image OCR into an independent switch (#20683)

jay7772195df031cb06a
diff --git a/internal/ingestion/component/media_dispatch.go b/internal/ingestion/component/media_dispatch.go
index 1fce20f46d..044c9be8f8 100644
--- a/internal/ingestion/component/media_dispatch.go
+++ b/internal/ingestion/component/media_dispatch.go
@@ -86,7 +86,7 @@ func maybeDispatchImage(
inputs map[string]any,
setups map[string]schema.ParserSetup,
enableVisionEnhancement bool,
- visionModelID string,
+ vision visionSettings,
) (parser.ParseResult, bool, error) {
if fileType != utility.FileTypeVISUAL {
return parser.ParseResult{}, false, nil
@@ -96,16 +96,11 @@ func maybeDispatchImage(
return parser.ParseResult{}, false, nil
}
method := getStringOr(setup, "parse_method", "")
- useOCR := method == "" || strings.EqualFold(method, "ocr")
- // A model named as parse_method is an explicit VLM request (mirrors
- // Python rag/flow/parser/parser.py:_image: "ocr" runs OCR, anything
- // else is the vision model). It must run the description even when the
- // global enhancement switch is off, otherwise the image item carries no
- // text and the Tokenizer's retrievability filter drops the chunk.
- modelFromParseMethod := ""
- if !useOCR {
- modelFromParseMethod = method
- }
+ ocrEnabled, hasOCRFlag := setup["ocr_enabled"].(bool)
+ // ocr_enabled is the OCR switch. Legacy setups omit it and encode the same
+ // choice in parse_method: "ocr"/empty selects local OCR, any other value is
+ // a VLM model reference that also implies OCR is off.
+ useOCR := (hasOCRFlag && ocrEnabled) || (!hasOCRFlag && (method == "" || strings.EqualFold(method, "ocr")))
release, err := parser.AcquireImageMedia(ctx)
if err != nil {
return parser.ParseResult{}, true, err
@@ -135,18 +130,37 @@ func maybeDispatchImage(
} else if useOCR && strings.TrimSpace(text) == "" {
parsed.Warnings = append(parsed.Warnings, "image OCR returned no text")
}
+ if !useOCR && !enableVisionEnhancement {
+ // Neither text source is enabled: local OCR is off and the vision
+ // description is not permitted. The item still carries the image, but the
+ // Tokenizer's retrievability filter drops chunks that have no text and no
+ // surrounding context, so this document indexes nothing. Say so instead of
+ // silently returning an empty result.
+ parsed.Warnings = append(parsed.Warnings,
+ "image has no text source: OCR is off and vision enhancement is disabled, so the Tokenizer will discard this item")
+ }
if err := ctx.Err(); err != nil {
return parsed, true, err
}
- if enableVisionEnhancement || modelFromParseMethod != "" {
- modelRef := visionModelID
- if modelFromParseMethod != "" {
- modelRef = modelFromParseMethod
+ if enableVisionEnhancement {
+ // A setup with an explicit ocr_enabled switch always uses the global
+ // vision settings. Only legacy setups (no switch) carry the VLM model
+ // reference in parse_method, which also disabled OCR by construction.
+ imageVision := vision
+ if !hasOCRFlag && !useOCR {
+ imageVision.modelID = method
}
- description, warnings := describeImage(ctx, db, imageData, getStringOr(inputs, "tenant_id", ""), setup, inputs, modelRef)
+ description, warnings := describeImage(ctx, db, imageData, getStringOr(inputs, "tenant_id", ""), setup, inputs, imageVision)
parsed.Warnings = append(parsed.Warnings, warnings...)
if description != "" {
appendItemText(parsed.JSON[0], description)
+ } else if strings.TrimSpace(text) == "" {
+ // Enhancement was permitted and the model produced nothing usable,
+ // so the item still has no text at all. Without this the chunk is
+ // dropped downstream with no trace, which is what the two other
+ // empty-text warnings above exist to prevent.
+ parsed.Warnings = append(parsed.Warnings,
+ "image has no searchable text: the vision model returned no description, so the Tokenizer will discard this item")
}
}
return parsed, true, nil
@@ -211,15 +225,21 @@ func describeImage(
tenantID string,
setup schema.ParserSetup,
inputs map[string]any,
- modelRef string,
+ vision visionSettings,
) (string, []string) {
// --- Optional VLM description ---
- lang := resolveVisionLanguage(inputs, getStringOr(setup, "lang", ""))
+ // Captions follow the knowledge base language, which also drives
+ // tokenization, so indexed text and analysis can never disagree. The family
+ // setup's lang is not consulted here: it is an OCR engine input (see
+ // pdf_vision_dispatch.go) and its old implicit default was the subject of
+ // issue #20727.
+ lang := resolveVisionLanguage(inputs)
if tenantID == "" {
return "", []string{"image VLM enhancement skipped: tenant ID is missing"}
}
// Use the selected description model or the tenant default.
+ modelRef := vision.modelID
var driver modelModule.ModelDriver
var modelName string
var apiConfig *modelModule.APIConfig
@@ -242,10 +262,11 @@ func describeImage(
}
prompt := defaultImageVisionPrompt(lang)
- // image family's contract key is system_prompt (parser.go:295),
- // mirroring Python parser.py:1119. Do NOT read setup["prompt"]
- // here — that key is for the video family, not image.
- if v, ok := setup["system_prompt"].(string); ok && v != "" {
+ // The global enhancement prompt wins; the image family's legacy system_prompt
+ // (the key the Python parser's image branch used) stays readable for canvases
+ // saved before the move. Do NOT read setup["prompt"] here — that key is for
+ // the video family, not image.
+ if v := firstNonEmpty(vision.systemPrompt, getStringOr(setup, "system_prompt", "")); v != "" {
prompt = v
}
messages := []modelModule.Message{{
@@ -255,9 +276,9 @@ func describeImage(
map[string]any{"type": "image_url", "image_url": map[string]any{"url": dataURI}},
},
}}
- vision := true
+ visionFlag := true
chatModel := modelModule.NewChatModel(driver, &modelName, apiConfig)
- resp, err := chatModel.ChatWithMessages(ctx, messages, &modelModule.ChatConfig{Vision: &vision}, nil)
+ resp, err := chatModel.ChatWithMessages(ctx, messages, &modelModule.ChatConfig{Vision: &visionFlag}, nil)
if err != nil {
return "", []string{fmt.Sprintf("image VLM enhancement failed: %v", err)}
}
diff --git a/internal/ingestion/component/media_dispatch_integration_test.go b/internal/ingestion/component/media_dispatch_integration_test.go
index 8e4ce4f4fa..2e6e53b490 100644
--- a/internal/ingestion/component/media_dispatch_integration_test.go
+++ b/internal/ingestion/component/media_dispatch_integration_test.go
@@ -10,13 +10,20 @@ import (
nativeanalyzer "ragflow/internal/deepdoc/parser/pdf/inference/native_analyzer"
deepdoctype "ragflow/internal/deepdoc/parser/type"
+ "ragflow/internal/ingestion/component/schema"
+ "ragflow/internal/parser/parser"
"ragflow/internal/utility"
)
//go:embed testdata/picture_ocr.png
var pictureOCRFixture []byte
-func TestMaybeDispatchImageNativeOCRWithoutVision(t *testing.T) {
+const pictureOCRExpectedText = "HELLOWORLD"
+
+// registerNativeOCRModels installs the in-process DeepDoc analyzer backed by the
+// real ONNX models. MODEL_DIR is supplied by the CI integration jobs.
+func registerNativeOCRModels(t *testing.T) {
+ t.Helper()
modelDir := os.Getenv("MODEL_DIR")
if modelDir == "" {
t.Skip("MODEL_DIR required for native OCR integration")
@@ -26,19 +33,81 @@ func TestMaybeDispatchImageNativeOCRWithoutVision(t *testing.T) {
if err := nativeanalyzer.Register(modelDir, nativeanalyzer.DefaultDropScore); err != nil {
t.Fatal(err)
}
+}
+
+// imageSetupsWithSwitch returns the new contract shape: the image family states
+// its OCR choice with ocr_enabled and carries no parse_method at all.
+func imageSetupsWithSwitch(ocrEnabled bool) map[string]schema.ParserSetup {
+ setups := defaultSetups()
+ delete(setups["image"], "parse_method")
+ setups["image"]["ocr_enabled"] = ocrEnabled
+ return setups
+}
+
+func assertNativeOCRTextAndImage(t *testing.T, result parser.ParseResult) {
+ t.Helper()
+ text, _ := result.JSON[0]["text"].(string)
+ if strings.Join(strings.Fields(strings.ToUpper(text)), "") != pictureOCRExpectedText {
+ t.Fatalf("native OCR text = %q, want HELLO WORLD: warnings = %v", text, result.Warnings)
+ }
+ if result.JSON[0]["image"] == "" || result.JSON[0]["doc_type_kwd"] != "image" {
+ t.Fatalf("missing image attachment: %+v", result.JSON[0])
+ }
+}
+
+// TestMaybeDispatchImageNativeOCRWithoutVision runs the real recognizer through
+// the legacy parse_method shape, so it also guards the backward-compatible
+// reading of canvases saved before the ocr_enabled switch existed.
+func TestMaybeDispatchImageNativeOCRWithoutVision(t *testing.T) {
+ registerNativeOCRModels(t)
result, handled, err := maybeDispatchImage(t.Context(), nil, utility.FileTypeVISUAL,
- "picture_ocr.png", pictureOCRFixture, nil, defaultSetups(), false, "")
+ "picture_ocr.png", pictureOCRFixture, nil, defaultSetups(), false, visionSettings{})
if err != nil {
t.Fatal(err)
}
if !handled || len(result.JSON) != 1 {
t.Fatalf("result = %+v, handled = %v", result, handled)
}
- text, _ := result.JSON[0]["text"].(string)
- if strings.Join(strings.Fields(strings.ToUpper(text)), "") != "HELLOWORLD" {
- t.Fatalf("native OCR text = %q, want HELLO WORLD: warnings = %v", text, result.Warnings)
+ assertNativeOCRTextAndImage(t, result)
+}
+
+// TestMaybeDispatchImageNativeOCRWithSwitch drives the same real models through
+// the ocr_enabled switch that new canvases send. Without it the switch shape
+// would only ever be exercised against a fake analyzer.
+func TestMaybeDispatchImageNativeOCRWithSwitch(t *testing.T) {
+ registerNativeOCRModels(t)
+ result, handled, err := maybeDispatchImage(t.Context(), nil, utility.FileTypeVISUAL,
+ "picture_ocr.png", pictureOCRFixture, nil, imageSetupsWithSwitch(true), false, visionSettings{})
+ if err != nil {
+ t.Fatal(err)
}
- if result.JSON[0]["image"] == "" || result.JSON[0]["doc_type_kwd"] != "image" {
- t.Fatalf("missing image attachment: %+v", result.JSON[0])
+ if !handled || len(result.JSON) != 1 {
+ t.Fatalf("result = %+v, handled = %v", result, handled)
+ }
+ assertNativeOCRTextAndImage(t, result)
+}
+
+// TestMaybeDispatchImageNativeSwitchOffRunsNoOCR pins the supersession of #20668
+// against the real backend: with the switch off and enhancement off the analyzer
+// is never consulted, so the item keeps its image with empty text, and the
+// dispatch says so instead of failing silently.
+func TestMaybeDispatchImageNativeSwitchOffRunsNoOCR(t *testing.T) {
+ registerNativeOCRModels(t)
+ result, handled, err := maybeDispatchImage(t.Context(), nil, utility.FileTypeVISUAL,
+ "picture_ocr.png", pictureOCRFixture, nil, imageSetupsWithSwitch(false), false, visionSettings{})
+ if err != nil || result.Err != nil {
+ t.Fatalf("degraded config must not error: err = %v, result.Err = %v", err, result.Err)
+ }
+ if !handled || len(result.JSON) != 1 {
+ t.Fatalf("result = %+v, handled = %v", result, handled)
+ }
+ if text, _ := result.JSON[0]["text"].(string); text != "" {
+ t.Fatalf("OCR must not run with the switch off, got text %q", text)
+ }
+ if result.JSON[0]["image"] == "" {
+ t.Fatalf("image attachment must survive: %+v", result.JSON[0])
+ }
+ if !strings.Contains(strings.Join(result.Warnings, "\n"), "the Tokenizer will discard this item") {
+ t.Fatalf("expected a no-text-source warning, got %v", result.Warnings)
}
}
diff --git a/internal/ingestion/component/media_dispatch_test.go b/internal/ingestion/component/media_dispatch_test.go
index f69265e4a7..2c737d164f 100644
--- a/internal/ingestion/component/media_dispatch_test.go
+++ b/internal/ingestion/component/media_dispatch_test.go
@@ -77,22 +77,33 @@ func TestMaybeDispatchImageOCRIndependentOfVision(t *testing.T) {
t.Fatal(err)
}
params := template.DSL.Components["Parser:ViewsCaptureLight"].Obj.Params
+ imageSetup, ok := params["image"].(map[string]any)
+ if !ok {
+ t.Fatal("picture template parser params carry no image setup")
+ }
+ if _, has := imageSetup["parse_method"]; has {
+ t.Fatal("picture template image setup must use ocr_enabled, not parse_method")
+ }
original := deepdoctype.NativeDocAnalyzerFactory
deepdoctype.NativeDocAnalyzerFactory = func() (deepdoctype.DocAnalyzer, bool) { return &pictureOCRAnalyzer{}, true }
t.Cleanup(func() { deepdoctype.NativeDocAnalyzerFactory = original })
data := picturePNG(t)
+ // ocr_enabled is the OCR switch and vision enhancement is an independent
+ // toggle. The picture template ships with the switch shape, so the merged
+ // setup carries ocr_enabled and parse_method is ignored even though the
+ // defaults still list parse_method:"ocr".
for _, tc := range []struct {
name string
- method string
+ ocr bool
enabled bool
tenant string
want string
}{
- {"disabled", "ocr", false, "", "OCR text"},
- {"enabled", "ocr", true, "t1", "OCR text\ncaptured"},
- {"unavailable", "ocr", true, "", "OCR text"},
- {"default", "", false, "", "OCR text"},
- {"vlm-only", "custom-vlm", false, "t1", "captured"},
+ {"ocr-only", true, false, "", "OCR text"},
+ {"ocr-and-vision", true, true, "t1", "OCR text\ncaptured"},
+ {"ocr-vision-unavailable", true, true, "", "OCR text"},
+ {"ocr-off", false, false, "", ""},
+ {"vision-only", false, true, "t1", "captured"},
} {
t.Run(tc.name, func(t *testing.T) {
originalResolver := resolveTenantModelByType
@@ -100,14 +111,7 @@ func TestMaybeDispatchImageOCRIndependentOfVision(t *testing.T) {
return &imagePromptCaptureDriver{}, "vision", &modelModule.APIConfig{}, 0, nil
}
t.Cleanup(func() { resolveTenantModelByType = originalResolver })
- // A model selected via parse_method resolves through resolveModelConfig;
- // stub it so the forced VLM path does not touch the database.
- originalModelResolver := resolveModelConfig
- resolveModelConfig = func(context.Context, *gorm.DB, string, entity.ModelType, string) (modelModule.ModelDriver, string, *modelModule.APIConfig, int, error) {
- return &imagePromptCaptureDriver{}, "custom-vlm", &modelModule.APIConfig{}, 0, nil
- }
- t.Cleanup(func() { resolveModelConfig = originalModelResolver })
- params["image"].(map[string]any)["parse_method"] = tc.method
+ imageSetup["ocr_enabled"] = tc.ocr
component, err := NewParserComponent(params)
if err != nil {
t.Fatal(err)
@@ -129,13 +133,15 @@ func TestMaybeDispatchImageOCRIndependentOfVision(t *testing.T) {
}
}
-// TestMaybeDispatchImage_ParseMethodModelRunsVisionWithoutGlobalEnhancement
-// pins that a model selected as the image parse_method is itself an explicit
-// VLM request: the vision description must run even when the global
-// enable_vision_enhancement switch is off. Before the fix the selected model
-// was ignored unless the switch was on, leaving the image item text-less so
-// the Tokenizer's retrievability filter dropped the only chunk.
-func TestMaybeDispatchImage_ParseMethodModelRunsVisionWithoutGlobalEnhancement(t *testing.T) {
+// TestMaybeDispatchImage_ParseMethodModelDoesNotOverrideGlobalEnhancement
+// supersedes the #20668 behaviour, which treated a model named in the image
+// parse_method as its own VLM permission and ran the description even with
+// enable_vision_enhancement off. That bypass existed only because the old
+// contract offered no other way to ask for a description. The image dropdown and
+// its model picker are gone now: the switch is the single permission, and
+// parse_method names at most which model to use once that permission is granted
+// (see TestMaybeDispatchImage_UsesConfiguredVLMModel).
+func TestMaybeDispatchImage_ParseMethodModelDoesNotOverrideGlobalEnhancement(t *testing.T) {
originalModelResolver := resolveModelConfig
originalTenantResolver := resolveTenantModelByType
t.Cleanup(func() {
@@ -149,11 +155,12 @@ func TestMaybeDispatchImage_ParseMethodModelRunsVisionWithoutGlobalEnhancement(t
return nil, "", nil, 0, nil
}
- var gotRef string
- var gotType entity.ModelType
- resolveModelConfig = func(_ context.Context, _ *gorm.DB, _ string, modelType entity.ModelType, ref string) (modelModule.ModelDriver, string, *modelModule.APIConfig, int, error) {
- gotRef = ref
- gotType = modelType
+ // The VLM model resolver must not be consulted at all: with enhancement off
+ // nothing is permitted to call a vision model, whichever model parse_method
+ // happens to name.
+ resolveModelConfigCalled := false
+ resolveModelConfig = func(_ context.Context, _ *gorm.DB, _ string, _ entity.ModelType, _ string) (modelModule.ModelDriver, string, *modelModule.APIConfig, int, error) {
+ resolveModelConfigCalled = true
return &imagePromptCaptureDriver{}, "custom-vlm", &modelModule.APIConfig{}, 0, nil
}
@@ -168,7 +175,7 @@ func TestMaybeDispatchImage_ParseMethodModelRunsVisionWithoutGlobalEnhancement(t
picturePNG(t),
map[string]any{"tenant_id": "t1"},
setups,
- false, "",
+ false, visionSettings{},
)
if err != nil {
t.Fatalf("maybeDispatchImage: %v", err)
@@ -176,22 +183,29 @@ func TestMaybeDispatchImage_ParseMethodModelRunsVisionWithoutGlobalEnhancement(t
if !dispatched {
t.Fatal("expected dispatched=true for VISUAL file")
}
- if gotRef != "custom-vlm@provider" || gotType != entity.ModelTypeImage2Text {
- t.Fatalf("resolved model = %q / %v, want custom-vlm@provider / %v", gotRef, gotType, entity.ModelTypeImage2Text)
+ if resolveModelConfigCalled {
+ t.Error("model resolution happened with enhancement off")
}
if tenantResolverCalled {
- t.Error("tenant default resolver must not be called when parse_method names a VLM model")
+ t.Error("tenant default resolver must not be called when enhancement is off")
}
if len(res.JSON) != 1 {
t.Fatalf("JSON len = %d, want 1", len(res.JSON))
}
text, _ := res.JSON[0]["text"].(string)
- if strings.TrimSpace(text) == "" {
- t.Fatalf("image item text is empty; VLM description was not applied: %+v", res.JSON[0])
+ if strings.TrimSpace(text) != "" {
+ t.Fatalf("image item text = %q, want empty: enhancement is off and OCR is off", text)
}
+ // The image itself is still carried; only the description is withheld, so
+ // turning the switch on later recovers the caption without a re-upload.
if res.JSON[0]["image"] == "" || res.JSON[0]["doc_type_kwd"] != "image" {
t.Fatalf("image attachment lost: %+v", res.JSON[0])
}
+ // #20668 reported this exact configuration producing zero chunks with no
+ // explanation. The answer is now a warning rather than a permission bypass.
+ if !strings.Contains(strings.Join(res.Warnings, "\n"), "the Tokenizer will discard this item") {
+ t.Fatalf("silent empty result: expected a no-text-source warning, got %v", res.Warnings)
+ }
}
// TestMaybeDispatchImage_OCRWithoutEnhancementSkipsVision pins the other side
@@ -212,7 +226,7 @@ func TestMaybeDispatchImage_OCRWithoutEnhancementSkipsVision(t *testing.T) {
setups := defaultSetups()
setups["image"]["parse_method"] = "ocr"
- res, _, err := maybeDispatchImage(t.Context(), dao.DB, utility.FileTypeVISUAL, "photo.png", picturePNG(t), map[string]any{"tenant_id": "t1"}, setups, false, "")
+ res, _, err := maybeDispatchImage(t.Context(), dao.DB, utility.FileTypeVISUAL, "photo.png", picturePNG(t), map[string]any{"tenant_id": "t1"}, setups, false, visionSettings{})
if err != nil {
t.Fatalf("maybeDispatchImage: %v", err)
}
@@ -243,6 +257,17 @@ func (d *imagePromptCaptureDriver) ChatWithMessages(ctx context.Context, modelNa
return &modelModule.ChatResponse{Answer: &ans}, nil
}
+// blankAnswerVisionDriver models a vision model that answers successfully but
+// with no usable text, which must still leave a trace behind.
+type blankAnswerVisionDriver struct {
+ modelModule.ModelDriver
+}
+
+func (blankAnswerVisionDriver) ChatWithMessages(context.Context, string, []modelModule.Message, *modelModule.APIConfig, *modelModule.ChatConfig, *common.ModelUsage) (*modelModule.ChatResponse, error) {
+ blank := " "
+ return &modelModule.ChatResponse{Answer: &blank}, nil
+}
+
// firstUserText extracts the text of the first "text" content part from the
// first captured user message. It scans parts by the "type" discriminator so
// the test stays valid regardless of part ordering (image_url may precede text).
@@ -283,7 +308,7 @@ func TestMaybeDispatchImage_UsesSystemPrompt(t *testing.T) {
}
setups := defaultSetups()
- // image family's contract key is system_prompt (parser.go:295).
+ // The image family's contract key is system_prompt.
// Also set a legacy `prompt` sentinel to assert system_prompt wins
// when both keys are present (regression guard against re-reading
// setup["prompt"], which is the video-family key, not image).
@@ -299,7 +324,7 @@ func TestMaybeDispatchImage_UsesSystemPrompt(t *testing.T) {
picturePNG(t),
map[string]any{"tenant_id": "t1"},
setups,
- true, "",
+ true, visionSettings{},
)
if err != nil {
t.Fatalf("maybeDispatchImage: %v", err)
@@ -351,7 +376,7 @@ func TestMaybeDispatchImage_DefaultPromptUsesDatasetLanguage(t *testing.T) {
picturePNG(t),
map[string]any{"tenant_id": "t1", "lang": "Japanese"},
setups,
- true, "",
+ true, visionSettings{},
)
if err != nil {
t.Fatalf("maybeDispatchImage: %v", err)
@@ -369,6 +394,70 @@ func TestMaybeDispatchImage_DefaultPromptUsesDatasetLanguage(t *testing.T) {
}
}
+// TestMaybeDispatchImage_GlobalVisionPromptDrivesImage locks the moved
+// contract: the image description reads its prompt from the global vlm block,
+// and a legacy per-family value only applies when the global one is unset.
+// Language is not part of the enhancement settings — captions follow the
+// knowledge base language (see TestMaybeDispatchImage_DefaultPromptUsesDatasetLanguage).
+func TestMaybeDispatchImage_GlobalVisionPromptDrivesImage(t *testing.T) {
+ origResolver := resolveTenantModelByType
+ defer func() { resolveTenantModelByType = origResolver }()
+
+ for _, tc := range []struct {
+ name string
+ inputs map[string]any
+ familyLang string
+ globalPrompt string
+ familyPrompt string
+ want string
+ }{
+ {
+ name: "global prompt wins over the family value",
+ inputs: map[string]any{"tenant_id": "t1"},
+ globalPrompt: "global prompt",
+ familyPrompt: "family prompt",
+ want: "global prompt",
+ },
+ {
+ name: "family prompt still applies when global is unset",
+ inputs: map[string]any{"tenant_id": "t1"},
+ familyPrompt: "family prompt",
+ want: "family prompt",
+ },
+ {
+ name: "language comes from the dataset, not the enhancement settings",
+ inputs: map[string]any{"tenant_id": "t1", "lang": "Japanese"},
+ familyLang: "Chinese",
+ want: "Respond in Japanese.",
+ },
+ } {
+ t.Run(tc.name, func(t *testing.T) {
+ drv := &imagePromptCaptureDriver{}
+ resolveTenantModelByType = func(context.Context, *gorm.DB, string, entity.ModelType) (modelModule.ModelDriver, string, *modelModule.APIConfig, int, error) {
+ return drv, "img-model", &modelModule.APIConfig{}, 0, nil
+ }
+ setups := defaultSetups()
+ setups["image"]["lang"] = tc.familyLang
+ setups["image"]["system_prompt"] = tc.familyPrompt
+ _, _, err := maybeDispatchImage(
+ t.Context(), dao.DB, utility.FileTypeVISUAL, "test.png", picturePNG(t),
+ tc.inputs, setups, true,
+ visionSettings{systemPrompt: tc.globalPrompt},
+ )
+ if err != nil {
+ t.Fatalf("maybeDispatchImage: %v", err)
+ }
+ got, ok := firstUserText(drv.captured)
+ if !ok {
+ t.Fatalf("no user text captured: %#v", drv.captured)
+ }
+ if !strings.Contains(got, tc.want) {
+ t.Fatalf("VLM user text = %q, want %q", got, tc.want)
+ }
+ })
+ }
+}
+
// TestMaybeDispatchImage_ReturnsJSONWithImage pins the output-shape fix:
// the image branch must return a JSON item carrying the `image` attachment
// (data URI) and `doc_type_kwd:"image"`, mirroring Python
@@ -394,7 +483,7 @@ func TestMaybeDispatchImage_ReturnsJSONWithImage(t *testing.T) {
picturePNG(t),
map[string]any{"tenant_id": "t1"},
setups,
- true, "",
+ true, visionSettings{},
)
if err != nil {
t.Fatalf("maybeDispatchImage: %v", err)
@@ -446,7 +535,7 @@ func TestMaybeDispatchImage_HardcodesJSONOutput(t *testing.T) {
picturePNG(t),
map[string]any{"tenant_id": "t1"},
setups,
- true, "",
+ true, visionSettings{},
)
if err != nil {
t.Fatalf("maybeDispatchImage: %v", err)
@@ -638,7 +727,7 @@ func TestMaybeDispatchImage_UsesConfiguredVLMModel(t *testing.T) {
picturePNG(t),
map[string]any{"tenant_id": "t1"},
setups,
- true, "global-vision-model",
+ true, visionSettings{modelID: "global-vision-model"},
)
if err != nil {
t.Fatalf("maybeDispatchImage: %v", err)
@@ -765,7 +854,7 @@ func TestMaybeDispatchImageWithoutOCRTextKeepsImage(t *testing.T) {
for _, method := range []string{"ocr", ""} {
setups := defaultSetups()
setups["image"]["parse_method"] = method
- result, handled, err := maybeDispatchImage(t.Context(), nil, utility.FileTypeVISUAL, "photo.png", picturePNG(t), nil, setups, false, "")
+ result, handled, err := maybeDispatchImage(t.Context(), nil, utility.FileTypeVISUAL, "photo.png", picturePNG(t), nil, setups, false, visionSettings{})
if err != nil {
t.Fatal(err)
}
@@ -789,7 +878,7 @@ func TestMaybeDispatchImageUnhealthyOCRKeepsImage(t *testing.T) {
deepdoctype.NativeDocAnalyzerFactory = func() (deepdoctype.DocAnalyzer, bool) {
return &unhealthyPictureOCRAnalyzer{}, true
}
- result, handled, err := maybeDispatchImage(t.Context(), nil, utility.FileTypeVISUAL, "photo.png", picturePNG(t), nil, defaultSetups(), false, "")
+ result, handled, err := maybeDispatchImage(t.Context(), nil, utility.FileTypeVISUAL, "photo.png", picturePNG(t), nil, defaultSetups(), false, visionSettings{})
if err != nil || result.Err != nil {
t.Fatalf("unhealthy OCR should degrade without an error: err = %v, result.Err = %v", err, result.Err)
}
@@ -805,7 +894,7 @@ func TestMaybeDispatchImageRejectsUndecodableBytes(t *testing.T) {
for _, method := range []string{"ocr", "custom-vlm"} {
setups := defaultSetups()
setups["image"]["parse_method"] = method
- _, _, err := maybeDispatchImage(t.Context(), nil, utility.FileTypeVISUAL, "photo.png", []byte("bad image"), nil, setups, false, "")
+ _, _, err := maybeDispatchImage(t.Context(), nil, utility.FileTypeVISUAL, "photo.png", []byte("bad image"), nil, setups, false, visionSettings{})
if err == nil || !strings.Contains(err.Error(), "decode") {
t.Fatalf("method %q: error = %v, want decode error", method, err)
}
@@ -825,7 +914,7 @@ func TestMaybeDispatchImageReleasesAdmissionAfterPanic(t *testing.T) {
t.Error("expected OCR analyzer panic")
}
}()
- _, _, _ = maybeDispatchImage(t.Context(), nil, utility.FileTypeVISUAL, "photo.png", data, nil, defaultSetups(), false, "")
+ _, _, _ = maybeDispatchImage(t.Context(), nil, utility.FileTypeVISUAL, "photo.png", data, nil, defaultSetups(), false, visionSettings{})
}()
ctx, cancel := context.WithTimeout(t.Context(), time.Second)
defer cancel()
@@ -853,7 +942,7 @@ func TestMaybeDispatchImageDecodesRasterOnlyForOCR(t *testing.T) {
setups := defaultSetups()
setups["image"]["parse_method"] = method
result, handled, err := maybeDispatchImage(t.Context(), nil, utility.FileTypeVISUAL,
- "photo.png", []byte(magic), map[string]any{"tenant_id": "t1"}, setups, true, "")
+ "photo.png", []byte(magic), map[string]any{"tenant_id": "t1"}, setups, true, visionSettings{})
if method == "ocr" {
if err == nil || !strings.Contains(err.Error(), "raster decoding requested") {
t.Fatalf("error = %v, want raster decode error", err)
@@ -870,6 +959,121 @@ func TestMaybeDispatchImageDecodesRasterOnlyForOCR(t *testing.T) {
}
}
+// TestMaybeDispatchImageOCRSwitch drives the new ocr_enabled shape (no
+// parse_method) directly. It locks the switch semantics the dropdown used to
+// encode: OCR on runs local OCR regardless of vision enhancement; OCR off with
+// enhancement off yields an empty-text image that is still kept, plus a warning
+// naming the dead configuration (the #20424 degradation, now diagnosable); OCR
+// off with enhancement on runs only the VLM.
+// The presence of ocr_enabled must also override a stale parse_method:"ocr".
+func TestMaybeDispatchImageOCRSwitch(t *testing.T) {
+ original := deepdoctype.NativeDocAnalyzerFactory
+ t.Cleanup(func() { deepdoctype.NativeDocAnalyzerFactory = original })
+ origResolver := resolveTenantModelByType
+ t.Cleanup(func() { resolveTenantModelByType = origResolver })
+ deepdoctype.NativeDocAnalyzerFactory = func() (deepdoctype.DocAnalyzer, bool) { return &pictureOCRAnalyzer{}, true }
+ resolveTenantModelByType = func(context.Context, *gorm.DB, string, entity.ModelType) (modelModule.ModelDriver, string, *modelModule.APIConfig, int, error) {
+ return &imagePromptCaptureDriver{}, "vision", &modelModule.APIConfig{}, 0, nil
+ }
+
+ ocrSetups := func(on bool) map[string]schema.ParserSetup {
+ setups := defaultSetups()
+ delete(setups["image"], "parse_method")
+ setups["image"]["ocr_enabled"] = on
+ return setups
+ }
+
+ t.Run("ocr-on", func(t *testing.T) {
+ res, handled, err := maybeDispatchImage(t.Context(), nil, utility.FileTypeVISUAL, "photo.png", picturePNG(t),
+ map[string]any{"tenant_id": "t1"}, ocrSetups(true), false, visionSettings{})
+ if err != nil || !handled {
+ t.Fatalf("err = %v, handled = %v", err, handled)
+ }
+ if res.JSON[0]["text"] != "OCR text" {
+ t.Fatalf("text = %v, want OCR text", res.JSON[0]["text"])
+ }
+ })
+ t.Run("ocr-off-no-enhancement-keeps-empty", func(t *testing.T) {
+ res, handled, err := maybeDispatchImage(t.Context(), nil, utility.FileTypeVISUAL, "photo.png", picturePNG(t),
+ map[string]any{"tenant_id": "t1"}, ocrSetups(false), false, visionSettings{})
+ if err != nil || !handled {
+ t.Fatalf("err = %v, handled = %v", err, handled)
+ }
+ if res.JSON[0]["text"] != "" || res.JSON[0]["image"] == "" {
+ t.Fatalf("want empty text with image retained, got %#v", res.JSON[0])
+ }
+ if len(res.Warnings) == 0 {
+ t.Fatalf("OCR off + enhancement off must warn that the item has no text source, got %v", res.Warnings)
+ }
+ if !strings.Contains(strings.Join(res.Warnings, "\n"), "the Tokenizer will discard this item") {
+ t.Fatalf("missing no-text-source warning: %v", res.Warnings)
+ }
+ })
+ t.Run("ocr-off-vision-only", func(t *testing.T) {
+ res, handled, err := maybeDispatchImage(t.Context(), nil, utility.FileTypeVISUAL, "photo.png", picturePNG(t),
+ map[string]any{"tenant_id": "t1"}, ocrSetups(false), true, visionSettings{})
+ if err != nil || !handled {
+ t.Fatalf("err = %v, handled = %v", err, handled)
+ }
+ if res.JSON[0]["text"] != "captured" {
+ t.Fatalf("text = %v, want captured (VLM only)", res.JSON[0]["text"])
+ }
+ })
+ t.Run("switch-beats-stale-parse-method", func(t *testing.T) {
+ setups := defaultSetups() // image still carries parse_method:"ocr"
+ setups["image"]["ocr_enabled"] = false
+ res, _, err := maybeDispatchImage(t.Context(), nil, utility.FileTypeVISUAL, "photo.png", picturePNG(t),
+ map[string]any{"tenant_id": "t1"}, setups, false, visionSettings{})
+ if err != nil {
+ t.Fatal(err)
+ }
+ if res.JSON[0]["text"] != "" {
+ t.Fatalf("ocr_enabled:false must override stale parse_method, got text %v", res.JSON[0]["text"])
+ }
+ })
+}
+
+// TestMaybeDispatchImageBlankVisionDescriptionWarns covers the third way an
+// image can end up with no text: enhancement is permitted, OCR is off, and the
+// model answers with nothing usable. The chunk is discarded downstream, so the
+// dispatch has to say so rather than return a silent empty item.
+func TestMaybeDispatchImageBlankVisionDescriptionWarns(t *testing.T) {
+ origResolver := resolveTenantModelByType
+ origAnalyzer := deepdoctype.NativeDocAnalyzerFactory
+ t.Cleanup(func() {
+ resolveTenantModelByType = origResolver
+ deepdoctype.NativeDocAnalyzerFactory = origAnalyzer
+ })
+ // OCR is off, so the analyzer must never run.
+ deepdoctype.NativeDocAnalyzerFactory = func() (deepdoctype.DocAnalyzer, bool) {
+ panic("OCR analyzer must not be used with the switch off")
+ }
+ resolveTenantModelByType = func(context.Context, *gorm.DB, string, entity.ModelType) (modelModule.ModelDriver, string, *modelModule.APIConfig, int, error) {
+ return blankAnswerVisionDriver{}, "vision", &modelModule.APIConfig{}, 0, nil
+ }
+
+ res, handled, err := maybeDispatchImage(t.Context(), nil, utility.FileTypeVISUAL, "photo.png", picturePNG(t),
+ map[string]any{"tenant_id": "t1"}, imageSwitchSetups(t, false), true, visionSettings{})
+ if err != nil || !handled {
+ t.Fatalf("err = %v, handled = %v", err, handled)
+ }
+ if text, _ := res.JSON[0]["text"].(string); strings.TrimSpace(text) != "" {
+ t.Fatalf("text = %q, want empty", text)
+ }
+ if !strings.Contains(strings.Join(res.Warnings, "\n"), "the Tokenizer will discard this item") {
+ t.Fatalf("blank description must warn, got %v", res.Warnings)
+ }
+}
+
+// imageSwitchSetups returns the image family in the new contract shape: the
+// OCR choice lives in ocr_enabled and parse_method is gone.
+func imageSwitchSetups(_ *testing.T, ocrEnabled bool) map[string]schema.ParserSetup {
+ setups := defaultSetups()
+ delete(setups["image"], "parse_method")
+ setups["image"]["ocr_enabled"] = ocrEnabled
+ return setups
+}
+
func picturePNG(t *testing.T) []byte {
t.Helper()
var data bytes.Buffer
diff --git a/internal/ingestion/component/parser.go b/internal/ingestion/component/parser.go
index b86edf62cf..90ea5c15ab 100644
--- a/internal/ingestion/component/parser.go
+++ b/internal/ingestion/component/parser.go
@@ -64,9 +64,8 @@
// the upload step is the responsibility of a separate
// side-effect component (out of scope for Phase 2.2).
//
-// - The Python _param.check() business validation
-// (parse_method whitelist, conditional lang checks) is mirrored
-// by (*ParserComponent).Check() below, which NewParserComponent
+// - The Python _param.check() business validation (parse_method whitelist)
+// is mirrored by (*ParserComponent).Check() below, which NewParserComponent
// runs at construction time. Neither backend validates
// audio/video vlm.llm_id: Python's check() has no such branch,
// and the audio model is resolved at dispatch time with a
@@ -112,7 +111,17 @@ const pageFormFeed = '\f'
type ParserComponent struct {
setups map[string]schema.ParserSetup
enableVisionEnhancement bool
- visionModelID string
+ vision visionSettings
+}
+
+// visionSettings carries the global vision-enhancement choices. They sit at the
+// Parser params top level (vlm) instead of a per-family setup, so one model and
+// prompt serve the enhanced file types rather than each family repeating them.
+// Response language is deliberately absent: it belongs to the knowledge base,
+// which also drives tokenization, so the two can never disagree.
+type visionSettings struct {
+ modelID string
+ systemPrompt string
}
// NewParserComponent constructs a Parser from a DSL param map.
@@ -126,7 +135,7 @@ type ParserComponent struct {
//
// {
// "enable_vision_enhancement": bool,
-// "vlm": {"llm_id": string},
+// "vlm": {"llm_id": string, "system_prompt": string},
// "pdf": map[string]any,
// "docx": map[string]any,
// ...
@@ -151,18 +160,24 @@ func NewParserComponent(params map[string]any) (runtime.Component, error) {
return nil, errors.New("parser: enable_vision_enhancement must be a boolean")
}
}
- var visionModelID string
+ var vision visionSettings
if raw, exists := params["vlm"]; exists {
vlm, ok := raw.(map[string]any)
if !ok {
return nil, errors.New("parser: vlm must be an object")
}
if rawID, exists := vlm["llm_id"]; exists {
- visionModelID, ok = rawID.(string)
+ vision.modelID, ok = rawID.(string)
if !ok {
return nil, errors.New("parser: vlm.llm_id must be a string")
}
}
+ if rawPrompt, exists := vlm["system_prompt"]; exists {
+ vision.systemPrompt, ok = rawPrompt.(string)
+ if !ok {
+ return nil, errors.New("parser: vlm.system_prompt must be a string")
+ }
+ }
}
for k, raw := range params {
if k == "outputs" || k == "allowed_output_format" || k == "enable_vision_enhancement" || k == "vlm" {
@@ -180,7 +195,7 @@ func NewParserComponent(params map[string]any) (runtime.Component, error) {
}
}
normalizeParserOutputFormats(s)
- pc := &ParserComponent{setups: s, enableVisionEnhancement: enableVisionEnhancement, visionModelID: visionModelID}
+ pc := &ParserComponent{setups: s, enableVisionEnhancement: enableVisionEnhancement, vision: vision}
if err := pc.Check(); err != nil {
return nil, fmt.Errorf("parser: %w", err)
}
@@ -239,30 +254,24 @@ func cloneParserSetupValue(value any) any {
// either, and audio dispatch resolves a missing/empty model to
// the tenant default, so validating it here would only block
// otherwise valid pipelines (see ingestion_pipeline_audio.json).
+// - any language: a missing language is no longer fatal downstream —
+// captions fall back to the knowledge base language and MinerU to its own
+// default — and Check runs before run inputs exist, so the knowledge base
+// value cannot be consulted here. Rejecting stored canvases for it would
+// only break pipelines that resolve fine at run time.
func (c *ParserComponent) Check() error {
- // PDF family (parser.py:252-261).
if pdf, ok := c.setups["pdf"]; ok {
- pm, _ := pdf["parse_method"].(string)
- if pm == "" {
+ if pm, _ := pdf["parse_method"].(string); pm == "" {
return errors.New("parse method abnormal. does not support empty value")
}
- if !parser.IsPDFParseMethod(pm) {
- // A parse_method outside the known vocabulary is treated as a
- // VLM model reference, which requires lang (Python
- // parser.py:257-258).
- if lang, _ := pdf["lang"].(string); lang == "" {
- return errors.New("PDF VLM language does not support empty value")
- }
- }
}
- // Image OCR runs independently of optional vision enhancement.
+ // The whole image contract keys off whether ocr_enabled is present, so a
+ // value of the wrong type must not quietly read as "absent" and fall back to
+ // the legacy parse_method inference.
if img, ok := c.setups["image"]; ok {
- pm, _ := img["parse_method"].(string)
- // A model selected for optional image enhancement needs a language
- // only when enhancement is enabled.
- if c.enableVisionEnhancement && !strings.EqualFold(pm, "ocr") && pm != "" {
- if lang, _ := img["lang"].(string); lang == "" {
- return errors.New("image VLM language does not support empty value")
+ if raw, exists := img["ocr_enabled"]; exists {
+ if _, isBool := raw.(bool); !isBool {
+ return errors.New("parser: image ocr_enabled must be a boolean")
}
}
}
@@ -273,7 +282,6 @@ func defaultSetups() map[string]schema.ParserSetup {
return map[string]schema.ParserSetup{
"pdf": {
"parse_method": "deepdoc",
- "lang": "Chinese",
"flatten_media_to_text": false,
"remove_toc": false,
"remove_header_footer": false,
@@ -325,9 +333,13 @@ func defaultSetups() map[string]schema.ParserSetup {
"output_format": "json",
},
"image": {
+ // Default stays on the legacy parse_method shape. ocr_enabled is
+ // deliberately NOT set here: NewParserComponent overlays params on
+ // top of these defaults, so the mere presence of ocr_enabled in the
+ // merged setup proves the caller chose the switch shape. A default
+ // value would leak the flag onto legacy (parse_method-only) canvases.
"parse_method": "ocr",
"llm_id": "",
- "lang": "Chinese",
"system_prompt": "",
"suffix": []string{"jpg", "jpeg", "png", "gif", "bmp", "tif", "tiff", "webp"},
"output_format": "json",
@@ -496,7 +508,7 @@ func (c *ParserComponent) Invoke(ctx context.Context, db *gorm.DB, inputs map[st
var handledImage bool
if !handledVision && !handledMedia {
// Image dispatch: OCR with independently controlled VLM enhancement.
- dispatched, handledImage, visionErr = maybeDispatchImage(ctx, db, fileTypeExt, filename, binary, inputs, setups, c.enableVisionEnhancement, c.visionModelID)
+ dispatched, handledImage, visionErr = maybeDispatchImage(ctx, db, fileTypeExt, filename, binary, inputs, setups, c.enableVisionEnhancement, c.vision)
if visionErr != nil {
return nil, visionErr
}
@@ -516,7 +528,7 @@ func (c *ParserComponent) Invoke(ctx context.Context, db *gorm.DB, inputs map[st
if c.enableVisionEnhancement {
// Enhancement is optional; parser-provided text and image metadata
// remain available if a vision model cannot describe an image.
- dispatched, _, _ = maybeDispatchVisionEnhancement(ctx, db, fileTypeExt, dispatched, inputs, setups, c.visionModelID)
+ dispatched, _, _ = maybeDispatchVisionEnhancement(ctx, db, fileTypeExt, dispatched, inputs, c.vision)
}
}
// Known/supported families must fail loudly when dispatch or
diff --git a/internal/ingestion/component/parser_check_test.go b/internal/ingestion/component/parser_check_test.go
index afffa883c0..baa53134a1 100644
--- a/internal/ingestion/component/parser_check_test.go
+++ b/internal/ingestion/component/parser_check_test.go
@@ -68,9 +68,11 @@ func TestParserComponent_Check(t *testing.T) {
setups: map[string]schema.ParserSetup{"pdf": {"parse_method": "tcadp parser"}},
},
{
- name: "pdf: unknown VLM method without lang → error",
- setups: map[string]schema.ParserSetup{"pdf": {"parse_method": "some_vlm", "lang": ""}},
- wantErr: "PDF VLM language",
+ // A stored canvas that names a model in parse_method but carries no
+ // language is valid: the engine and the caption path each resolve
+ // their own default at run time, and Check has no run inputs yet.
+ name: "pdf: unknown VLM method without lang → pass",
+ setups: map[string]schema.ParserSetup{"pdf": {"parse_method": "some_vlm", "lang": ""}},
},
{
name: "pdf: unknown VLM method with lang → pass",
@@ -87,27 +89,43 @@ func TestParserComponent_Check(t *testing.T) {
// --- image family (parser.py:283-287) ---
{
- name: "image: ocr without lang → pass (no lang check for OCR)",
+ name: "image: legacy ocr shape → pass",
setups: map[string]schema.ParserSetup{"image": {"parse_method": "ocr"}},
},
{
- name: "image: ocr with empty lang → pass (OCR skips lang)",
- setups: map[string]schema.ParserSetup{"image": {"parse_method": "ocr", "lang": ""}},
- },
- {
- name: "image: non-ocr without lang → error",
+ // Language is not validated anywhere: it is no longer an input to
+ // captions, and Check runs before run inputs exist so it cannot see
+ // the knowledge base value. These two cases only differ in lang.
+ name: "image: model reference with enhancement → pass regardless of lang",
setups: map[string]schema.ParserSetup{"image": {"parse_method": "vlm_xyz", "lang": ""}},
enhance: true,
- wantErr: "image VLM language",
},
{
- name: "image: non-ocr with lang → pass",
+ name: "image: model reference with lang → pass",
setups: map[string]schema.ParserSetup{"image": {"parse_method": "vlm_xyz", "lang": "English"}},
},
{
- name: "image: missing parse_method → pass (treated as non-ocr, but lang defaults empty in DSL)",
+ // Absent parse_method matches the backend's legacy inference, which
+ // reads it as "run local OCR".
+ name: "image: no parse_method → pass",
setups: map[string]schema.ParserSetup{"image": {"lang": "English"}},
},
+ {
+ // The image contract keys off the presence of ocr_enabled, so a
+ // wrong-typed value must not read as "absent" and fall back to the
+ // legacy parse_method inference.
+ name: "image: non-boolean ocr_enabled → error",
+ setups: map[string]schema.ParserSetup{"image": {"ocr_enabled": "false"}},
+ wantErr: "ocr_enabled must be a boolean",
+ },
+ {
+ name: "image: boolean ocr_enabled → pass",
+ setups: map[string]schema.ParserSetup{"image": {"ocr_enabled": false}},
+ },
+ {
+ name: "image: ocr_enabled alongside a stale parse_method → pass (switch wins)",
+ setups: map[string]schema.ParserSetup{"image": {"ocr_enabled": true, "parse_method": "ocr"}},
+ },
// --- audio/video: vlm.llm_id not validated (matches Python check()) ---
{
diff --git a/internal/ingestion/component/pdf_vision_dispatch.go b/internal/ingestion/component/pdf_vision_dispatch.go
index b733ae4fdd..e5c603ffb8 100644
--- a/internal/ingestion/component/pdf_vision_dispatch.go
+++ b/internal/ingestion/component/pdf_vision_dispatch.go
@@ -31,6 +31,11 @@ import (
"gorm.io/gorm"
)
+// minerUDefaultLanguage is MinerU's own OCR language default, applied when
+// neither mineru_lang nor the family setup names one. It deliberately does not
+// consult the knowledge base: it is an engine input, not a caption setting.
+const minerUDefaultLanguage = "Chinese"
+
type pdfVisionPage struct {
PageNumber int
WidthPts float64
@@ -518,13 +523,14 @@ func dispatchMinerUPDF(
apiURL := strings.TrimRight(baseURL, "/") + "/file_parse"
parseMethod := mineruAPIParseMethod(getStringOr(setup, "mineru_parse_method", ""))
- // Language chain mirrors Python's mineru_parser.py:1181
- // (mineru_lang → lang → "English"): the dedicated MinerU option wins,
- // then the setup language, then English. The pdf setup's lang default
- // ("Chinese") makes the unconfigured lang_list match Python's ch.
- lang := getStringOr(setup, "mineru_lang", "")
+ // MinerU's language is an OCR engine parameter, not the caption language:
+ // mineru_lang wins, then the family lang, then the engine's own default.
+ // That default used to arrive implicitly through defaultSetups() — which
+ // also fed captions, so a knowledge base that never chose a language got
+ // Chinese captions (issue #20727). Keeping it here confines it to the engine.
+ lang := firstNonEmpty(getStringOr(setup, "mineru_lang", ""), getStringOr(setup, "lang", ""))
if lang == "" {
- lang = getStringOr(setup, "lang", "English")
+ lang = minerUDefaultLanguage
}
mineruLang := mineruLangCode(lang)
backend := modelModule.ResolveMinerUBackend(getStringOr(setup, "mineru_backend", ""), apiKeyRaw)
diff --git a/internal/ingestion/component/tokenizer.go b/internal/ingestion/component/tokenizer.go
index 185f46222a..f19a2dbb05 100644
--- a/internal/ingestion/component/tokenizer.go
+++ b/internal/ingestion/component/tokenizer.go
@@ -1103,6 +1103,18 @@ func getStringOr(m map[string]any, key, def string) string {
return def
}
+// firstNonEmpty returns the first non-empty value, or "" when all are empty.
+// Used to walk a settings precedence chain (global choice → legacy per-family
+// choice).
+func firstNonEmpty(values ...string) string {
+ for _, v := range values {
+ if v != "" {
+ return v
+ }
+ }
+ return ""
+}
+
// cloneInputs returns a shallow copy of m with room for one extra key.
// Used to inject the Globals-resolved `name` into the decode input without
// mutating the caller's input snapshot.
diff --git a/internal/ingestion/component/vision_enhancement.go b/internal/ingestion/component/vision_enhancement.go
index fc7727169d..bcdae1d68a 100644
--- a/internal/ingestion/component/vision_enhancement.go
+++ b/internal/ingestion/component/vision_enhancement.go
@@ -40,7 +40,6 @@ import (
pdflayout "ragflow/internal/deepdoc/parser/pdf/layout"
"ragflow/internal/entity"
modelModule "ragflow/internal/entity/models"
- "ragflow/internal/ingestion/component/schema"
"ragflow/internal/parser/parser"
"ragflow/internal/utility"
@@ -166,8 +165,7 @@ func maybeDispatchVisionEnhancement(
fileType utility.FileType,
dispatched parser.ParseResult,
inputs map[string]any,
- setups map[string]schema.ParserSetup,
- modelRef string,
+ vision visionSettings,
) (parser.ParseResult, bool, error) {
// Only enhance successful JSON output format containing items.
if dispatched.Err != nil || dispatched.OutputFormat != "json" || len(dispatched.JSON) == 0 {
@@ -176,8 +174,10 @@ func maybeDispatchVisionEnhancement(
tenantID := getStringOr(inputs, "tenant_id", "")
family := resolveParserFamily(fileType)
- setup := setups[family]
- language := resolveVisionLanguage(inputs, getStringOr(setup, "lang", ""))
+ // Captions follow the knowledge base language, which also drives
+ // tokenization. The family setup's lang is an OCR engine parameter
+ // (pdf_vision_dispatch.go), not an input here — see issue #20727.
+ language := resolveVisionLanguage(inputs)
// Collect visual resources, including Markdown images whose type is text
// because flatten_media_to_text is enabled.
@@ -207,11 +207,11 @@ func maybeDispatchVisionEnhancement(
var resolveErr error
if tenantID != "" {
var err error
- if modelRef != "" {
- driver, modelName, apiConfig, _, err = resolveModelConfig(ctx, db, tenantID, entity.ModelTypeImage2Text, modelRef)
+ if vision.modelID != "" {
+ driver, modelName, apiConfig, _, err = resolveModelConfig(ctx, db, tenantID, entity.ModelTypeImage2Text, vision.modelID)
if err != nil {
common.Warn("vision enhancement: per-call VLM resolve failed, falling back to tenant default",
- zap.String("family", family), zap.String("modelRef", modelRef), zap.String("tenant", tenantID), zap.Error(err))
+ zap.String("family", family), zap.String("modelRef", vision.modelID), zap.String("tenant", tenantID), zap.Error(err))
driver, modelName, apiConfig, _, err = resolveTenantModelByType(ctx, db, tenantID, entity.ModelTypeImage2Text)
}
} else {
diff --git a/internal/ingestion/component/vision_enhancement_test.go b/internal/ingestion/component/vision_enhancement_test.go
index e38005b591..ad948aa985 100644
--- a/internal/ingestion/component/vision_enhancement_test.go
+++ b/internal/ingestion/component/vision_enhancement_test.go
@@ -203,7 +203,8 @@ func TestVisionEnhancement_AppendsVLMToExistingText(t *testing.T) {
}
res, handled, err := maybeDispatchVisionEnhancement(
t.Context(), dao.DB, utility.FileTypeXLSX, dispatched,
- map[string]any{"tenant_id": "t1"}, nil, "")
+ map[string]any{"tenant_id": "t1"},
+ visionSettings{})
if err != nil {
t.Fatalf("maybeDispatchVisionEnhancement: %v", err)
}
@@ -232,7 +233,8 @@ func TestVisionEnhancement_ReportsVLMInvocationFailure(t *testing.T) {
}},
}
result, _, err := maybeDispatchVisionEnhancement(t.Context(), dao.DB, utility.FileTypeDOCX, dispatched,
- map[string]any{"tenant_id": "t1"}, nil, "")
+ map[string]any{"tenant_id": "t1"},
+ visionSettings{})
if err != nil {
t.Fatalf("maybeDispatchVisionEnhancement: %v", err)
}
@@ -260,7 +262,7 @@ func TestVisionEnhancement_CropFailureFallsBackToInlineImage(t *testing.T) {
}
result, handled, err := maybeDispatchVisionEnhancement(
t.Context(), dao.DB, utility.FileTypePDF, dispatched,
- map[string]any{"tenant_id": "t1"}, map[string]schema.ParserSetup{"pdf": {"parse_method": "mineru"}}, "",
+ map[string]any{"tenant_id": "t1"}, visionSettings{},
)
if err != nil {
t.Fatalf("maybeDispatchVisionEnhancement: %v", err)
@@ -296,7 +298,7 @@ func TestVisionEnhancement_EnhancesPDFTablesRegardlessOfOCRProvider(t *testing.T
res, handled, err := maybeDispatchVisionEnhancement(
t.Context(), dao.DB, utility.FileTypePDF, dispatched,
map[string]any{"tenant_id": "t1"},
- map[string]schema.ParserSetup{"pdf": {"parse_method": parseMethod}}, "",
+ visionSettings{},
)
if err != nil {
t.Fatalf("maybeDispatchVisionEnhancement: %v", err)
@@ -333,7 +335,7 @@ func TestVisionEnhancement_EnhancesTableAndCellImages(t *testing.T) {
result, handled, err := maybeDispatchVisionEnhancement(
t.Context(), dao.DB, utility.FileTypePDF, dispatched,
- map[string]any{"tenant_id": "t1"}, map[string]schema.ParserSetup{"pdf": {"parse_method": "deepdoc"}}, "",
+ map[string]any{"tenant_id": "t1"}, visionSettings{},
)
if err != nil {
t.Fatalf("maybeDispatchVisionEnhancement: %v", err)
@@ -363,7 +365,7 @@ func TestVisionEnhancement_EnhancesExternalPDFImage(t *testing.T) {
result, handled, err := maybeDispatchVisionEnhancement(
t.Context(), dao.DB, utility.FileTypePDF, dispatched,
- map[string]any{"tenant_id": "t1"}, map[string]schema.ParserSetup{"pdf": {"parse_method": "mineru"}}, "",
+ map[string]any{"tenant_id": "t1"}, visionSettings{},
)
if err != nil {
t.Fatalf("maybeDispatchVisionEnhancement: %v", err)
@@ -414,7 +416,8 @@ func TestVisionEnhancement_EnhancesJSONImagesAndTables(t *testing.T) {
dao.DB,
tc.fileType,
dispatched,
- map[string]any{"tenant_id": "t1", "lang": "Japanese"}, nil, "")
+ map[string]any{"tenant_id": "t1", "lang": "Japanese"},
+ visionSettings{})
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
@@ -463,10 +466,10 @@ func TestVisionEnhancement_LanguagePriority(t *testing.T) {
want: "Japanese",
},
{
- name: "setup lang used when inputs carry no lang",
+ name: "family lang no longer drives captions; without a dataset language it is English",
inputs: map[string]any{"tenant_id": "t1"},
setups: map[string]schema.ParserSetup{"pdf": {"lang": "Chinese"}},
- want: "Chinese",
+ want: "English",
},
{
name: "english when neither is set",
@@ -493,7 +496,8 @@ func TestVisionEnhancement_LanguagePriority(t *testing.T) {
}
_, handled, err := maybeDispatchVisionEnhancement(
- t.Context(), dao.DB, utility.FileTypePDF, dispatched, tc.inputs, tc.setups, "")
+ t.Context(), dao.DB, utility.FileTypePDF, dispatched, tc.inputs,
+ visionSettings{})
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
@@ -533,7 +537,8 @@ func TestVisionEnhancement_MarkdownOutputUntouched(t *testing.T) {
dao.DB,
utility.FileTypeDOCX,
dispatched,
- map[string]any{"tenant_id": "t1"}, nil, "")
+ map[string]any{"tenant_id": "t1"},
+ visionSettings{})
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
@@ -563,7 +568,8 @@ func TestVisionEnhancement_UsesVisualPayloadRegardlessOfFileType(t *testing.T) {
dao.DB,
utility.FileTypeOTHER,
dispatched,
- map[string]any{"tenant_id": "t1"}, nil, "")
+ map[string]any{"tenant_id": "t1"},
+ visionSettings{})
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
@@ -592,7 +598,8 @@ func TestVisionEnhancement_EmptyOrNoTenantSkipped(t *testing.T) {
dao.DB,
utility.FileTypeDOCX,
dispatched,
- map[string]any{}, nil, "")
+ map[string]any{},
+ visionSettings{})
if err != nil || handled {
t.Errorf("handled=%v, err=%v, want false, nil for missing tenant_id", handled, err)
}
@@ -618,7 +625,8 @@ func TestVisionEnhancement_DispatchedErrSkipped(t *testing.T) {
dao.DB,
utility.FileTypePDF,
dispatched,
- map[string]any{"tenant_id": "t1"}, nil, "")
+ map[string]any{"tenant_id": "t1"},
+ visionSettings{})
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
@@ -655,7 +663,8 @@ func TestVisionEnhancement_ContextCancellation(t *testing.T) {
dao.DB,
utility.FileTypePDF,
dispatched,
- map[string]any{"tenant_id": "t1"}, nil, "")
+ map[string]any{"tenant_id": "t1"},
+ visionSettings{})
if !errors.Is(err, context.Canceled) {
t.Fatalf("err = %v, want context.Canceled", err)
}
@@ -696,7 +705,8 @@ func TestVisionEnhancement_NonStringImageFieldFiltered(t *testing.T) {
dao.DB,
utility.FileTypePDF,
dispatched,
- map[string]any{"tenant_id": "t1"}, nil, "")
+ map[string]any{"tenant_id": "t1"},
+ visionSettings{})
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
@@ -738,7 +748,8 @@ func TestVisionEnhancement_MoreThanConcurrencyItems(t *testing.T) {
dao.DB,
utility.FileTypePDF,
dispatched,
- map[string]any{"tenant_id": "t1"}, nil, "")
+ map[string]any{"tenant_id": "t1"},
+ visionSettings{})
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
@@ -778,7 +789,8 @@ func TestVisionEnhancement_PlainTextResponseNotTruncated(t *testing.T) {
dao.DB,
utility.FileTypePDF,
dispatched,
- map[string]any{"tenant_id": "t1"}, nil, "")
+ map[string]any{"tenant_id": "t1"},
+ visionSettings{})
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
@@ -816,7 +828,8 @@ func TestVisionEnhancement_PromptBuilderErrorSkipped(t *testing.T) {
dao.DB,
utility.FileTypePDF,
dispatched,
- map[string]any{"tenant_id": "t1"}, nil, "")
+ map[string]any{"tenant_id": "t1"},
+ visionSettings{})
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
@@ -854,7 +867,8 @@ func TestVisionEnhancement_ModelResolveFailureSkipped(t *testing.T) {
dao.DB,
utility.FileTypePDF,
dispatched,
- map[string]any{"tenant_id": "t1"}, nil, "")
+ map[string]any{"tenant_id": "t1"},
+ visionSettings{})
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
@@ -905,7 +919,8 @@ func TestVisionEnhancement_CancellationStopsSchedulingWithManyItems(t *testing.T
dao.DB,
utility.FileTypePDF,
dispatched,
- map[string]any{"tenant_id": "t1"}, nil, "")
+ map[string]any{"tenant_id": "t1"},
+ visionSettings{})
if !errors.Is(err, context.Canceled) {
t.Fatalf("err = %v, want context.Canceled", err)
}
@@ -1142,9 +1157,6 @@ func TestVisionEnhancement_GlobalModelPreferredOverPDFParser(t *testing.T) {
visionChatInvoker = invoker.invoke
figureVisionPromptBuilder = fakePrompt
- setups := map[string]schema.ParserSetup{
- "pdf": {"parse_method": "pdf-parser-model", "vlm": map[string]any{"llm_id": "stale-family-model"}},
- }
dispatched := parser.ParseResult{
OutputFormat: "json",
JSON: []map[string]any{
@@ -1158,7 +1170,7 @@ func TestVisionEnhancement_GlobalModelPreferredOverPDFParser(t *testing.T) {
utility.FileTypePDF,
dispatched,
map[string]any{"tenant_id": "t1"},
- setups, "custom-vlm@provider",
+ visionSettings{modelID: "custom-vlm@provider"},
)
if err != nil {
t.Fatalf("maybeDispatchVisionEnhancement: %v", err)
@@ -1200,7 +1212,8 @@ func TestVisionEnhancement_InvalidImageDataSkipped(t *testing.T) {
dao.DB,
utility.FileTypePDF,
dispatched,
- map[string]any{"tenant_id": "t1"}, nil, "")
+ map[string]any{"tenant_id": "t1"},
+ visionSettings{})
if err != nil {
t.Fatalf("maybeDispatchVisionEnhancement: %v", err)
}
diff --git a/internal/ingestion/component/vision_language.go b/internal/ingestion/component/vision_language.go
index 5f9b3078b4..7682225332 100644
--- a/internal/ingestion/component/vision_language.go
+++ b/internal/ingestion/component/vision_language.go
@@ -28,11 +28,14 @@ func normalizeVisionLanguage(language string) string {
return language
}
-func resolveVisionLanguage(inputs map[string]any, fallback string) string {
+// resolveVisionLanguage picks the language a caption answers in. The knowledge
+// base value is the only input, and it is also what the tokenizer analyses
+// with, so caption text and indexed tokens can never disagree.
+func resolveVisionLanguage(inputs map[string]any) string {
if language := strings.TrimSpace(getStringOr(inputs, "lang", "")); language != "" {
return language
}
- return normalizeVisionLanguage(fallback)
+ return defaultVisionLanguage
}
func renderFigureVisionLanguage(prompt, language string) string {
diff --git a/internal/ingestion/component/vision_language_test.go b/internal/ingestion/component/vision_language_test.go
index deadc898b1..c91394743b 100644
--- a/internal/ingestion/component/vision_language_test.go
+++ b/internal/ingestion/component/vision_language_test.go
@@ -23,34 +23,30 @@ import (
func TestResolveVisionLanguage(t *testing.T) {
tests := []struct {
- name string
- inputs map[string]any
- fallback string
- want string
+ name string
+ inputs map[string]any
+ want string
}{
{
- name: "dataset language takes precedence",
- inputs: map[string]any{"lang": " Japanese "},
- fallback: "Chinese",
- want: "Japanese",
+ name: "dataset language wins and is trimmed",
+ inputs: map[string]any{"lang": " Japanese "},
+ want: "Japanese",
},
{
- name: "configured fallback is used",
- inputs: map[string]any{"lang": ""},
- fallback: " Korean ",
- want: "Korean",
+ name: "empty values default to English",
+ inputs: map[string]any{"lang": ""},
+ want: defaultVisionLanguage,
},
{
- name: "empty values default to English",
- inputs: nil,
- fallback: "",
- want: defaultVisionLanguage,
+ name: "absent language defaults to English",
+ inputs: nil,
+ want: defaultVisionLanguage,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
- if got := resolveVisionLanguage(tt.inputs, tt.fallback); got != tt.want {
+ if got := resolveVisionLanguage(tt.inputs); got != tt.want {
t.Fatalf("resolveVisionLanguage() = %q, want %q", got, tt.want)
}
})
diff --git a/internal/ingestion/pipeline/template/ingestion_pipeline_picture.json b/internal/ingestion/pipeline/template/ingestion_pipeline_picture.json
index c26705b2a1..fc03a08fdc 100644
--- a/internal/ingestion/pipeline/template/ingestion_pipeline_picture.json
+++ b/internal/ingestion/pipeline/template/ingestion_pipeline_picture.json
@@ -54,8 +54,8 @@
}
},
"image": {
+ "ocr_enabled": true,
"output_format": "json",
- "parse_method": "ocr",
"preprocess": [
"main_content"
],
@@ -247,8 +247,8 @@
"setups": [
{
"fileFormat": "image",
+ "ocr_enabled": true,
"output_format": "json",
- "parse_method": "ocr",
"preprocess": [
"main_content"
]
diff --git a/internal/ingestion/task/pipeline_executor_defaults_test.go b/internal/ingestion/task/pipeline_executor_defaults_test.go
index 42a2836c20..034abe563a 100644
--- a/internal/ingestion/task/pipeline_executor_defaults_test.go
+++ b/internal/ingestion/task/pipeline_executor_defaults_test.go
@@ -46,7 +46,7 @@ var builtinComponentParamsGolden = map[string]string{
"manual": "{\"File\": {}, \"Parser:HipSignsRhyme\": {\"enable_vision_enhancement\": false, \"vlm\": {\"llm_id\": \"\"}, \"doc\": {\"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"doc\"]}, \"docx\": {\"flatten_media_to_text\": false, \"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"docx\"], \"vlm\": {}}, \"pdf\": {\"flatten_media_to_text\": false, \"output_format\": \"json\", \"parse_method\": \"DeepDOC\", \"preprocess\": [\"main_content\"], \"suffix\": [\"pdf\"], \"vlm\": {}}}, \"ManualChunker:NineInsectsFind\": {\"hierarchy\": 0, \"include_heading_content\": false, \"levels\": [[\"^#[^#]\", \"^##[^#]\", \"^###[^#]\", \"^####[^#]\"], [\"第[零一二三四五六七八九十百0-9]+(分?编|部分)\", \"第[零一二三四五六七八九十百0-9]+章\", \"第[零一二三四五六七八九十百0-9]+节\", \"第[零一二三四五六七八九十百0-9]+条\", \"[\\\\((][零一二三四五六七八九十百]+[\\\\))]\"], [\"第[0-9]+章\", \"第[0-9]+节\", \"[0-9]{1,2}[\\\\. 、]\", \"[0-9]{1,2}\\\\.[0-9]{1,2}($|[^a-zA-Z/%~.-])\", \"[0-9]{1,2}\\\\.[0-9]{1,2}\\\\.[0-9]{1,2}\"], [\"第[零一二三四五六七八九十百0-9]+章\", \"第[零一二三四五六七八九十百0-9]+节\", \"[零一二三四五六七八九十百]+[ 、]\", \"[\\\\((][零一二三四五六七八九十百]+[\\\\))]\", \"[\\\\((][0-9]{,2}[\\\\))]\"], [\"PART (ONE|TWO|THREE|FOUR|FIVE|SIX|SEVEN|EIGHT|NINE|TEN)\", \"Chapter (I+V?|VI*|XI|IX|X)\", \"Section [0-9]+\", \"Article [0-9]+\"]], \"method\": \"group\"}, \"Tokenizer:FunnyBalloonsGrin\": {\"fields\": \"text\", \"filename_embd_weight\": 0.1, \"search_method\": [\"embedding\", \"full_text\"]}, \"Extractor:AutoExtractDefault\": {\"llm_id\": \"\", \"keywords\": {\"top_n\": 0}, \"questions\": {\"top_n\": 0}, \"tags\": {\"top_n\": 0, \"tag_file_id\": \"\"}, \"summary\": {\"enabled\": false}, \"metadata\": {\"enabled\": false, \"metadata\": [], \"built_in_metadata\": []}}}",
"one": "{\"File\": {}, \"Parser:HipSignsRhyme\": {\"enable_vision_enhancement\": false, \"vlm\": {\"llm_id\": \"\"}, \"doc\": {\"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"doc\"]}, \"docx\": {\"flatten_media_to_text\": false, \"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"docx\"], \"vlm\": {}}, \"html\": {\"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"htm\", \"html\"]}, \"markdown\": {\"flatten_media_to_text\": false, \"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"md\", \"markdown\", \"mdx\"], \"vlm\": {}}, \"pdf\": {\"flatten_media_to_text\": false, \"output_format\": \"json\", \"parse_method\": \"DeepDOC\", \"preprocess\": [\"main_content\"], \"suffix\": [\"pdf\"], \"vlm\": {}}, \"spreadsheet\": {\"flatten_media_to_text\": false, \"output_format\": \"html\", \"parse_method\": \"DeepDOC\", \"preprocess\": [\"main_content\"], \"suffix\": [\"xls\", \"xlsx\"], \"vlm\": {}}, \"text&code\": {\"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"txt\"]}}, \"OneChunker:DryDrinksVisit\": {}, \"Tokenizer:FrankWeeksListen\": {\"fields\": \"text\", \"filename_embd_weight\": 0.1, \"search_method\": [\"embedding\", \"full_text\"]}, \"Extractor:AutoExtractDefault\": {\"llm_id\": \"\", \"keywords\": {\"top_n\": 0}, \"questions\": {\"top_n\": 0}, \"tags\": {\"top_n\": 0, \"tag_file_id\": \"\"}, \"summary\": {\"enabled\": false}, \"metadata\": {\"enabled\": false, \"metadata\": [], \"built_in_metadata\": []}}}",
"paper": "{\"File\": {}, \"Parser:HipSignsRhyme\": {\"enable_vision_enhancement\": false, \"vlm\": {\"llm_id\": \"\"}, \"pdf\": {\"enable_multi_column\": true, \"flatten_media_to_text\": false, \"output_format\": \"json\", \"parse_method\": \"DeepDOC\", \"preprocess\": [\"main_content\"], \"suffix\": [\"pdf\"], \"vlm\": {}}}, \"TitleChunker:SparklySchoolsTravel\": {\"hierarchy\": 0, \"include_heading_content\": false, \"levels\": [[\"^#[^#]\", \"^##[^#]\", \"^###[^#]\", \"^####[^#]\"], [\"第[零一二三四五六七八九十百0-9]+(分?编|部分)\", \"第[零一二三四五六七八九十百0-9]+章\", \"第[零一二三四五六七八九十百0-9]+节\", \"第[零一二三四五六七八九十百0-9]+条\", \"[\\\\((][零一二三四五六七八九十百]+[\\\\))]\"], [\"第[0-9]+章\", \"第[0-9]+节\", \"[0-9]{1,2}[\\\\. 、]\", \"[0-9]{1,2}\\\\.[0-9]{1,2}($|[^a-zA-Z/%~.-])\", \"[0-9]{1,2}\\\\.[0-9]{1,2}\\\\.[0-9]{1,2}\"], [\"第[零一二三四五六七八九十百0-9]+章\", \"第[零一二三四五六七八九十百0-9]+节\", \"[零一二三四五六七八九十百]+[ 、]\", \"[\\\\((][零一二三四五六七八九十百]+[\\\\))]\", \"[\\\\((][0-9]{,2}[\\\\))]\"], [\"PART (ONE|TWO|THREE|FOUR|FIVE|SIX|SEVEN|EIGHT|NINE|TEN)\", \"Chapter (I+V?|VI*|XI|IX|X)\", \"Section [0-9]+\", \"Article [0-9]+\"]], \"method\": \"group\"}, \"Tokenizer:GreatCarsWash\": {\"fields\": \"text\", \"filename_embd_weight\": 0.1, \"search_method\": [\"embedding\", \"full_text\"]}, \"Extractor:AutoExtractDefault\": {\"llm_id\": \"\", \"keywords\": {\"top_n\": 0}, \"questions\": {\"top_n\": 0}, \"tags\": {\"top_n\": 0, \"tag_file_id\": \"\"}, \"summary\": {\"enabled\": false}, \"metadata\": {\"enabled\": false, \"metadata\": [], \"built_in_metadata\": []}}}",
- "picture": "{\"File\": {}, \"Parser:ViewsCaptureLight\": {\"enable_vision_enhancement\": false, \"vlm\": {\"llm_id\": \"\"}, \"image\": {\"output_format\": \"json\", \"parse_method\": \"ocr\", \"preprocess\": [\"main_content\"], \"suffix\": [\"bmp\", \"gif\", \"jpeg\", \"jpg\", \"png\", \"svg\", \"tif\", \"tiff\", \"webp\"]}, \"video\": {\"output_format\": \"text\", \"preprocess\": [\"main_content\"], \"suffix\": [\"3gp\", \"3gpp\", \"avi\", \"flv\", \"mkv\", \"mov\", \"mp4\", \"mpeg\", \"mpg\", \"webm\", \"wmv\"]}}, \"OneChunker:BrightColorsGlow\": {}, \"Tokenizer:SharpLensFocus\": {\"fields\": \"text\", \"filename_embd_weight\": 0.1, \"search_method\": [\"embedding\", \"full_text\"]}, \"Extractor:AutoExtractDefault\": {\"llm_id\": \"\", \"keywords\": {\"top_n\": 0}, \"questions\": {\"top_n\": 0}, \"tags\": {\"top_n\": 0, \"tag_file_id\": \"\"}, \"summary\": {\"enabled\": false}, \"metadata\": {\"enabled\": false, \"metadata\": [], \"built_in_metadata\": []}}}",
+ "picture": "{\"File\": {}, \"Parser:ViewsCaptureLight\": {\"enable_vision_enhancement\": false, \"vlm\": {\"llm_id\": \"\"}, \"image\": {\"ocr_enabled\": true, \"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"bmp\", \"gif\", \"jpeg\", \"jpg\", \"png\", \"svg\", \"tif\", \"tiff\", \"webp\"]}, \"video\": {\"output_format\": \"text\", \"preprocess\": [\"main_content\"], \"suffix\": [\"3gp\", \"3gpp\", \"avi\", \"flv\", \"mkv\", \"mov\", \"mp4\", \"mpeg\", \"mpg\", \"webm\", \"wmv\"]}}, \"OneChunker:BrightColorsGlow\": {}, \"Tokenizer:SharpLensFocus\": {\"fields\": \"text\", \"filename_embd_weight\": 0.1, \"search_method\": [\"embedding\", \"full_text\"]}, \"Extractor:AutoExtractDefault\": {\"llm_id\": \"\", \"keywords\": {\"top_n\": 0}, \"questions\": {\"top_n\": 0}, \"tags\": {\"top_n\": 0, \"tag_file_id\": \"\"}, \"summary\": {\"enabled\": false}, \"metadata\": {\"enabled\": false, \"metadata\": [], \"built_in_metadata\": []}}}",
"presentation": "{\"File\": {}, \"Parser:HipSignsRhyme\": {\"enable_vision_enhancement\": false, \"vlm\": {\"llm_id\": \"\"}, \"pdf\": {\"flatten_media_to_text\": false, \"output_format\": \"json\", \"parse_method\": \"DeepDOC\", \"preprocess\": [\"main_content\"], \"suffix\": [\"pdf\"], \"vlm\": {}}, \"slides\": {\"output_format\": \"json\", \"parse_method\": \"DeepDOC\", \"preprocess\": [\"main_content\"], \"suffix\": [\"pptx\", \"ppt\"]}}, \"Tokenizer:TallTreesDance\": {\"fields\": \"text\", \"filename_embd_weight\": 0.1, \"search_method\": [\"embedding\", \"full_text\"]}, \"PageChunker:HappyHillsGlow\": {}, \"Extractor:AutoExtractDefault\": {\"llm_id\": \"\", \"keywords\": {\"top_n\": 0}, \"questions\": {\"top_n\": 0}, \"tags\": {\"top_n\": 0, \"tag_file_id\": \"\"}, \"summary\": {\"enabled\": false}, \"metadata\": {\"enabled\": false, \"metadata\": [], \"built_in_metadata\": []}}}",
"qa": "{\"File\": {}, \"Parser:HipSignsRhyme\": {\"enable_vision_enhancement\": false, \"vlm\": {\"llm_id\": \"\"}, \"docx\": {\"flatten_media_to_text\": false, \"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"docx\"], \"vlm\": {}}, \"markdown\": {\"flatten_media_to_text\": false, \"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"md\", \"markdown\", \"mdx\"], \"vlm\": {}}, \"pdf\": {\"flatten_media_to_text\": false, \"output_format\": \"json\", \"parse_method\": \"DeepDOC\", \"preprocess\": [\"main_content\"], \"suffix\": [\"pdf\"], \"vlm\": {}}, \"spreadsheet\": {\"flatten_media_to_text\": false, \"output_format\": \"html\", \"parse_method\": \"DeepDOC\", \"preprocess\": [\"main_content\"], \"suffix\": [\"xls\", \"xlsx\", \"csv\"], \"vlm\": {}}, \"text&code\": {\"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"txt\"]}}, \"Tokenizer:ColdCloudsDream\": {\"fields\": \"text\", \"filename_embd_weight\": 0.1, \"search_method\": [\"embedding\", \"full_text\"]}, \"QAChunker:TidyCloudsThink\": {}}",
"table": "{\"File\": {}, \"Parser:HipSignsRhyme\": {\"enable_vision_enhancement\": false, \"vlm\": {\"llm_id\": \"\"}, \"spreadsheet\": {\"flatten_media_to_text\": false, \"output_format\": \"html\", \"parse_method\": \"DeepDOC\", \"preprocess\": [\"main_content\"], \"suffix\": [\"xls\", \"xlsx\", \"csv\"], \"vlm\": {}, \"column_mode\": \"auto\", \"column_roles\": {}, \"column_names\": []}, \"text&code\": {\"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"txt\"]}}, \"TableChunker:FastFoxesJump\": {}, \"Tokenizer:DeepLakesShine\": {\"fields\": \"text\", \"filename_embd_weight\": 0.1, \"search_method\": [\"embedding\", \"full_text\"]}}",
diff --git a/internal/service/document/document_parser_selection_test.go b/internal/service/document/document_parser_selection_test.go
index 041aa0dacc..9ac7f58e99 100644
--- a/internal/service/document/document_parser_selection_test.go
+++ b/internal/service/document/document_parser_selection_test.go
@@ -105,7 +105,7 @@ func TestUploadLocalDocuments_RoutesFileTypesToDedicatedParsers(t *testing.T) {
t.Fatalf("jpg parser_id = %q, want picture", parserID)
}
imageSetup := parserFamilySetup(t, got[0], "image")
- if imageSetup["parse_method"] != "ocr" {
+ if imageSetup["ocr_enabled"] != true {
t.Fatalf("jpg image setup = %#v, want the picture pipeline defaults", imageSetup)
}
diff --git a/web/src/locales/en.ts b/web/src/locales/en.ts
index d061f6e688..d1599a1e71 100644
--- a/web/src/locales/en.ts
+++ b/web/src/locales/en.ts
@@ -2776,6 +2776,11 @@ Best for: Documents with flowing, contextually connected content — such as boo
enableVisionEnhancement: 'Enable vision enhancement',
enableVisionEnhancementTip:
'Use a vision model to parse image and table blocks; when off, they are treated as plain text.',
+ imageOcr: 'OCR',
+ imageOcrTip:
+ 'Extract text from uploaded images locally. Vision enhancement adds descriptions independently of this switch.',
+ visionEnhancementPromptTip:
+ 'Custom instruction for the vision model. Leave empty to use the built-in prompt.',
enableChildrenDelimiters: 'Child chunk are used for retrieval',
merge: 'Merge',
split: 'Split',
diff --git a/web/src/locales/zh.ts b/web/src/locales/zh.ts
index c194c9db3b..967a742a85 100644
--- a/web/src/locales/zh.ts
+++ b/web/src/locales/zh.ts
@@ -2353,6 +2353,11 @@ NER:使用 spaCy NER 和基于规则的关键词提取来抽取 Entities 和 R
oneChunkTitle: 'Note',
oneChunkDescription:
'所有解析后的 sections 会按原始顺序合并为 1 个 chunk。',
+ imageOcr: 'OCR',
+ imageOcrTip:
+ '从上传的图片中本地提取文字。视觉增强独立于该开关,为图片补充描述。',
+ visionEnhancementPromptTip:
+ '自定义视觉模型的指令。留空则使用内置提示词。',
enableVisionEnhancement: '启用视觉增强',
enableVisionEnhancementTip:
'使用视觉模型解析图片和表格区块,关闭时按普通文本处理。',
diff --git a/web/src/pages/agent/constant/pipeline.tsx b/web/src/pages/agent/constant/pipeline.tsx
index 84dae1a36f..ca4620c314 100644
--- a/web/src/pages/agent/constant/pipeline.tsx
+++ b/web/src/pages/agent/constant/pipeline.tsx
@@ -85,10 +85,6 @@ export enum TokenizerSearchMethod {
FullText = 'full_text',
}
-export enum ImageParseMethod {
- OCR = 'ocr',
-}
-
export enum TokenizerFields {
Text = 'text',
Questions = 'questions',
@@ -168,8 +164,9 @@ export const initialParserValues = {
json: { type: 'Array<object>', value: [] },
},
// Global vision enhancement, off by default; the model id is prefilled from
- // the tenant's image2text default when a node is created.
- vlm: { llm_id: '' },
+ // the tenant's image2text default when a node is created. The prompt is shared
+ // by the vision paths here; response language belongs to the knowledge base.
+ vlm: { llm_id: '', system_prompt: '' },
enable_vision_enhancement: false,
setups: [
{
@@ -189,9 +186,8 @@ export const initialParserValues = {
{
fileFormat: FileType.Image,
output_format: ImageOutputFormat.Json,
- parse_method: ImageParseMethod.OCR,
+ ocr_enabled: true,
preprocess: PreprocessValue.main_content,
- system_prompt: '',
},
{
fileFormat: FileType.Email,
diff --git a/web/src/pages/agent/form/parser-form/image-form-fields.tsx b/web/src/pages/agent/form/parser-form/image-form-fields.tsx
index 7a669fb906..e95d3da7c8 100644
--- a/web/src/pages/agent/form/parser-form/image-form-fields.tsx
+++ b/web/src/pages/agent/form/parser-form/image-form-fields.tsx
@@ -1,50 +1,28 @@
import { RAGFlowFormItem } from '@/components/ragflow-form';
-import { Textarea } from '@/components/ui/textarea';
-import { buildOptions } from '@/utils/form';
-import { isEmpty } from 'lodash';
-import { useMemo } from 'react';
-import { useWatch } from 'react-hook-form';
+import { Switch } from '@/components/ui/switch';
import { useTranslation } from 'react-i18next';
-import { ImageParseMethod } from '../../constant/pipeline';
-import { LanguageFormField, ParserMethodFormField } from './common-form-fields';
import { CommonProps } from './interface';
-import { useSetInitialLanguage } from './use-set-initial-language';
import { buildFieldNameWithPrefix } from './utils';
+// The image setup carries a single switch: run local OCR or not. The vision
+// model and its description prompt belong to the global enhancement block (see
+// VisionEnhancementFormFields); response language is not a setting anywhere and
+// follows the knowledge base, which also drives tokenization.
export function ImageFormFields({ prefix }: CommonProps) {
const { t } = useTranslation();
- const options = buildOptions(
- ImageParseMethod,
- t,
- 'flow.imageParseMethodOptions',
- );
- const parseMethodName = buildFieldNameWithPrefix('parse_method', prefix);
-
- const parseMethod = useWatch({
- name: parseMethodName,
- });
-
- const languageShown = useMemo(() => {
- return !isEmpty(parseMethod) && parseMethod !== ImageParseMethod.OCR;
- }, [parseMethod]);
-
- useSetInitialLanguage({ prefix, languageShown });
return (
- <>
- <ParserMethodFormField
- prefix={prefix}
- optionsWithoutLLM={options}
- ></ParserMethodFormField>
- {languageShown && <LanguageFormField prefix={prefix}></LanguageFormField>}
- {languageShown && (
- <RAGFlowFormItem
- name={buildFieldNameWithPrefix('system_prompt', prefix)}
- label={t('flow.systemPrompt')}
- >
- <Textarea placeholder={t('flow.systemPromptPlaceholder')} />
- </RAGFlowFormItem>
+ <RAGFlowFormItem
+ name={buildFieldNameWithPrefix('ocr_enabled', prefix)}
+ label={t('flow.imageOcr')}
+ tooltip={t('flow.imageOcrTip')}
+ horizontal={true}
+ labelClassName="w-full"
+ valueClassName="w-8"
+ >
+ {(field) => (
+ <Switch checked={!!field.value} onCheckedChange={field.onChange} />
)}
- </>
+ </RAGFlowFormItem>
);
}
diff --git a/web/src/pages/agent/form/parser-form/schema.ts b/web/src/pages/agent/form/parser-form/schema.ts
index 0ef1b39f89..893e47c304 100644
--- a/web/src/pages/agent/form/parser-form/schema.ts
+++ b/web/src/pages/agent/form/parser-form/schema.ts
@@ -8,6 +8,11 @@ export const SetupSchema = z
// preprocess: z.array(z.string()).optional(),
output_format: z.string().optional(),
parse_method: z.string().optional(),
+ // Image OCR runs independently of the global vision-enhancement switch.
+ // The dropdown used to encode this choice in parse_method; ocr_enabled is
+ // the new boolean contract. Legacy setups may still carry a parse_method
+ // only; normalizeParserFormValues migrates them on load.
+ ocr_enabled: z.boolean().optional(),
lang: z.string().optional(),
fields: z.array(z.string()).optional(),
// Per-setup vlm is only used by Audio (its ASR model); the vision model
@@ -55,10 +60,17 @@ export const SetupSchema = z
export const FormSchema = z.object({
setups: z.array(SetupSchema).min(1, i18n.t('flow.atLeastOneFileType')),
- // Global vision enhancement: one switch + one img2txt model shared by every
- // vision-capable file type, sitting at the params top level alongside the
- // per-family setups (see the backend Parser component contract).
- vlm: z.object({ llm_id: z.string().optional() }).optional(),
+ // Global vision enhancement: one switch + img2txt model + description prompt,
+ // sitting at the params top level alongside the per-family setups (see the
+ // backend Parser component contract). Response language is intentionally not
+ // here: captions are indexed text, so they follow the knowledge base language
+ // that also drives tokenization, and can never disagree with it.
+ vlm: z
+ .object({
+ llm_id: z.string().optional(),
+ system_prompt: z.string().optional(),
+ })
+ .optional(),
enable_vision_enhancement: z.boolean().optional(),
});
diff --git a/web/src/pages/agent/form/parser-form/utils.test.ts b/web/src/pages/agent/form/parser-form/utils.test.ts
index 93774e3a41..b4d1dfedc9 100644
--- a/web/src/pages/agent/form/parser-form/utils.test.ts
+++ b/web/src/pages/agent/form/parser-form/utils.test.ts
@@ -62,7 +62,10 @@ describe('parser-form utils', () => {
[ModelTypeToField.asr]: 'whisper@OpenAI',
});
- expect(values.vlm).toEqual({ llm_id: 'gpt-4o@OpenAI' });
+ expect(values.vlm).toEqual({
+ llm_id: 'gpt-4o@OpenAI',
+ system_prompt: '',
+ });
expect(values.enable_vision_enhancement).toBe(false);
const byFileType = new Map(
@@ -98,7 +101,10 @@ describe('parser-form utils', () => {
});
expect(values.enable_vision_enhancement).toBe(true);
- expect(values.vlm).toEqual({ llm_id: 'gpt-4o@OpenAI' });
+ expect(values.vlm).toEqual({
+ llm_id: 'gpt-4o@OpenAI',
+ system_prompt: '',
+ });
expect(values.setups[0]).toEqual({ fileFormat: FileType.PDF });
// Audio keeps its per-setup ASR model.
expect(values.setups[1]).toEqual({
@@ -116,7 +122,7 @@ describe('parser-form utils', () => {
});
expect(values.enable_vision_enhancement).toBe(false);
- expect(values.vlm).toEqual({ llm_id: '' });
+ expect(values.vlm).toEqual({ llm_id: '', system_prompt: '' });
});
it('treats a non-empty per-setup model as enabled enhancement', () => {
@@ -127,7 +133,10 @@ describe('parser-form utils', () => {
});
expect(values.enable_vision_enhancement).toBe(true);
- expect(values.vlm).toEqual({ llm_id: 'gpt-4o@OpenAI' });
+ expect(values.vlm).toEqual({
+ llm_id: 'gpt-4o@OpenAI',
+ system_prompt: '',
+ });
expect(values.setups[0]).toEqual({ fileFormat: FileType.Video });
});
@@ -147,7 +156,7 @@ describe('parser-form utils', () => {
const normalized = normalizeParserFormValues(values);
expect(normalized).toEqual({
- vlm: { llm_id: 'gpt-4o@OpenAI' },
+ vlm: { llm_id: 'gpt-4o@OpenAI', system_prompt: '' },
enable_vision_enhancement: true,
setups: [
{ fileFormat: FileType.PDF },
@@ -164,7 +173,143 @@ describe('parser-form utils', () => {
setups: [{ fileFormat: FileType.PDF, vlm: { llm_id: 'stale@Model' } }],
});
- expect(values.vlm).toEqual({ llm_id: '' });
+ expect(values.vlm).toEqual({ llm_id: '', system_prompt: '' });
+ });
+
+ it('migrates a legacy image ocr parse_method onto ocr_enabled', () => {
+ const normalized = normalizeParserFormValues({
+ setups: [{ fileFormat: FileType.Image, parse_method: 'ocr' }],
+ });
+ expect(normalized.setups[0]).toEqual({
+ fileFormat: FileType.Image,
+ ocr_enabled: true,
+ });
+ // An empty parse_method also selects OCR.
+ expect(
+ normalizeParserFormValues({
+ setups: [{ fileFormat: FileType.Image, parse_method: '' }],
+ }).setups[0],
+ ).toEqual({ fileFormat: FileType.Image, ocr_enabled: true });
+ });
+
+ it('ignores a stale parse_method once the switch shape is present', () => {
+ // Mirrors the backend: ocr_enabled present means parse_method is dead
+ // data. Hoisting it would overwrite the model the user picked globally.
+ const normalized = normalizeParserFormValues({
+ vlm: { llm_id: 'qwen-vl@DashScope', system_prompt: '' },
+ setups: [
+ {
+ fileFormat: FileType.Image,
+ ocr_enabled: true,
+ parse_method: 'abandoned-model@provider',
+ },
+ ],
+ });
+ expect(normalized.vlm).toEqual({
+ llm_id: 'qwen-vl@DashScope',
+ system_prompt: '',
+ });
+ expect(normalized.setups[0]).toEqual({
+ fileFormat: FileType.Image,
+ ocr_enabled: true,
+ });
+ });
+
+ it('lifts a legacy image model reference onto the global vlm and turns OCR off', () => {
+ const normalized = normalizeParserFormValues({
+ setups: [
+ { fileFormat: FileType.Image, parse_method: 'gpt-4o@OpenAI' },
+ { fileFormat: FileType.PDF },
+ ],
+ });
+ // Deliberate precedence: the canvas encodes a model choice, so it wins
+ // even over an already-selected global model rather than being dropped.
+ expect(normalized.vlm).toEqual({
+ llm_id: 'gpt-4o@OpenAI',
+ system_prompt: '',
+ });
+ expect(normalized.setups[0]).toEqual({
+ fileFormat: FileType.Image,
+ ocr_enabled: false,
+ });
+ });
+
+ it('drops a legacy image language: nothing reads it any more', () => {
+ const normalized = normalizeParserFormValues({
+ setups: [
+ {
+ fileFormat: FileType.Image,
+ ocr_enabled: true,
+ lang: 'French',
+ system_prompt: 'Describe the chart.',
+ },
+ ],
+ });
+ expect(normalized.vlm).toEqual({
+ llm_id: '',
+ system_prompt: 'Describe the chart.',
+ });
+ // Captions follow the knowledge base and the only engine that takes a
+ // language is MinerU on the pdf family, so an image language has no
+ // reader: drop it rather than let it masquerade as a user choice
+ // (issue #20727).
+ expect(normalized.setups[0]).toEqual({
+ fileFormat: FileType.Image,
+ ocr_enabled: true,
+ });
+ expect(normalizeParserFormValues(normalized)).toEqual(normalized);
+ });
+
+ it('materializes the switch so it matches what the backend will run', () => {
+ // An image setup with neither key runs local OCR in the backend (absent
+ // parse_method falls back to the legacy inference), so the toggle must
+ // read as on instead of showing an unchecked box.
+ const normalized = normalizeParserFormValues({
+ setups: [{ fileFormat: FileType.Image }],
+ });
+ expect(normalized.setups[0]).toEqual({
+ fileFormat: FileType.Image,
+ ocr_enabled: true,
+ });
+ });
+
+ it('drops a stale vlm.lang left by an earlier draft of the contract', () => {
+ const normalized = normalizeParserFormValues({
+ vlm: { llm_id: '', lang: 'German', system_prompt: '' },
+ setups: [
+ { fileFormat: FileType.Image, lang: 'French', system_prompt: 'old' },
+ ],
+ });
+ // Nothing renders that control any more, so it must not survive a save.
+ expect(normalized.vlm).toEqual({
+ llm_id: '',
+ system_prompt: 'old',
+ });
+ });
+
+ it('leaves an existing global model untouched when the image already uses the switch', () => {
+ // Migration only fires for a legacy parse_method. A normalized image
+ // setup (ocr_enabled present, no parse_method) never overrides a model
+ // the user chose at the top level.
+ const normalized = normalizeParserFormValues({
+ vlm: { llm_id: 'qwen-vl@DashScope' },
+ setups: [{ fileFormat: FileType.Image, ocr_enabled: false }],
+ });
+ expect(normalized.vlm).toEqual({
+ llm_id: 'qwen-vl@DashScope',
+ system_prompt: '',
+ });
+ expect(normalized.setups[0]).toEqual({
+ fileFormat: FileType.Image,
+ ocr_enabled: false,
+ });
+ });
+
+ it('is idempotent for the migrated image switch', () => {
+ const once = normalizeParserFormValues({
+ setups: [{ fileFormat: FileType.Image, parse_method: 'ocr' }],
+ });
+ expect(normalizeParserFormValues(once)).toEqual(once);
});
});
});
diff --git a/web/src/pages/agent/form/parser-form/utils.ts b/web/src/pages/agent/form/parser-form/utils.ts
index a06f50b6c9..e874ee6a91 100644
--- a/web/src/pages/agent/form/parser-form/utils.ts
+++ b/web/src/pages/agent/form/parser-form/utils.ts
@@ -25,18 +25,43 @@ export const VisionEnhancementFileTypes: FileType[] = [
];
// Lifts the legacy per-setup vision options (vlm.llm_id / flatten_media_to_text)
-// onto the top level. Idempotent, so every boundary (form defaults, canvas
-// save, dataset load) can apply it: values already carrying a top-level
-// enable_vision_enhancement keep it, and per-setup leftovers are always
-// stripped (except Audio's vlm, which holds the ASR model).
+// onto the top level and migrates the image family onto its own contract:
+// `ocr_enabled` replaces parse_method, and the prompt the image setup used to
+// own moves into the global vlm block. Idempotent, so every boundary (form
+// defaults, canvas save, dataset load) can apply it: values already carrying a
+// top-level enable_vision_enhancement keep it, and per-setup leftovers are always
+// stripped (except Audio's vlm, which holds the ASR model). A legacy image
+// parse_method that is neither "ocr" nor empty is a VLM model reference: it
+// lifts onto vlm.llm_id — deliberately beating an already-set global model, so
+// the choice a saved canvas encodes is not lost — and the switch turns off.
+// The family language is dropped outright: nothing reads it for images any
+// more, and a stored value cannot be told apart from the hardcoded default that
+// used to write it (issue #20727).
export function normalizeParserFormValues<T extends Record<string, any>>(
values: T,
-): T & { vlm: { llm_id: string }; enable_vision_enhancement: boolean } {
+): T & {
+ vlm: { llm_id: string; system_prompt: string };
+ enable_vision_enhancement: boolean;
+} {
const setups = Array.isArray(values?.setups) ? values.setups : [];
const visionSetups = setups.filter((x) =>
VisionEnhancementFileTypes.includes(x?.fileFormat),
);
+ const imageSetup = setups.find((x) => x?.fileFormat === FileType.Image);
+ // Mirrors the backend: a setup that already carries the ocr_enabled switch is
+ // on the new contract, so any leftover parse_method is dead data — not a model
+ // reference. Hoisting it anyway would overwrite the model the user picked.
+ const hasImageOcrSwitch = typeof imageSetup?.ocr_enabled === 'boolean';
+ const legacyImageMethod = imageSetup?.parse_method;
+ const legacyImageModel =
+ !hasImageOcrSwitch &&
+ typeof legacyImageMethod === 'string' &&
+ !isEmpty(legacyImageMethod) &&
+ legacyImageMethod.toLowerCase() !== 'ocr'
+ ? legacyImageMethod
+ : '';
+
const enableVisionEnhancement =
typeof values?.enable_vision_enhancement === 'boolean'
? values.enable_vision_enhancement
@@ -45,19 +70,43 @@ export function normalizeParserFormValues<T extends Record<string, any>>(
);
const llmId =
- values?.vlm?.llm_id ??
- visionSetups.find((x) => !isEmpty(x?.vlm?.llm_id))?.vlm?.llm_id ??
- '';
+ legacyImageModel ||
+ (values?.vlm?.llm_id ??
+ visionSetups.find((x) => !isEmpty(x?.vlm?.llm_id))?.vlm?.llm_id ??
+ '');
- const nextSetups = setups.map((x) =>
- x?.fileFormat === FileType.Audio
- ? x
- : omit(x, ['vlm', 'flatten_media_to_text']),
- );
+ const nextSetups = setups.map((x) => {
+ if (x?.fileFormat === FileType.Audio) return x;
+ const stripped = omit(x, ['vlm', 'flatten_media_to_text']);
+ if (x?.fileFormat !== FileType.Image) return stripped;
+ const { parse_method, ...rest } = omit(stripped, ['lang', 'system_prompt']);
+ // Materialize the switch either way. Absent or "ocr" both mean "run local
+ // OCR", matching the backend's legacy inference, so the toggle can never
+ // display the opposite of what will run. The family language is dropped
+ // because nothing reads it any more: captions follow the knowledge base and
+ // the only engine that takes a language is MinerU, on the pdf family.
+ const runsOCR =
+ parse_method === undefined ||
+ isEmpty(parse_method) ||
+ String(parse_method).toLowerCase() === 'ocr';
+ return { ...rest, ocr_enabled: rest.ocr_enabled ?? runsOCR };
+ });
return {
...values,
- vlm: { ...values?.vlm, llm_id: llmId },
+ vlm: {
+ // Drop any stored vlm.lang: the enhancement block no longer offers a
+ // language, so a stale one in the payload would resurrect a control that
+ // nothing renders any more.
+ ...omit(values?.vlm, ['lang']),
+ llm_id: llmId,
+ // Only the prompt lifts: a stored family prompt can only have come from a
+ // person, because its default is empty. Language is never lifted — the
+ // family value stays where it is as a legacy fallback, and lifting it would
+ // turn a machine-written default into an explicit choice. See issue #20727.
+ system_prompt:
+ values?.vlm?.system_prompt || imageSetup?.system_prompt || '',
+ },
enable_vision_enhancement: enableVisionEnhancement,
setups: nextSetups,
};
@@ -95,7 +144,10 @@ export function buildInitialParserValues(
) {
return {
...initialParserValues,
- vlm: { llm_id: defaultModelDictionary[ModelTypeToField.vision] ?? '' },
+ vlm: {
+ llm_id: defaultModelDictionary[ModelTypeToField.vision] ?? '',
+ system_prompt: '',
+ },
setups: initialParserValues.setups.map(
(setup) =>
buildInitialParserSetup(
diff --git a/web/src/pages/agent/form/parser-form/vision-form-fields.test.tsx b/web/src/pages/agent/form/parser-form/vision-form-fields.test.tsx
index 934340bfc9..9ab2c62c1e 100644
--- a/web/src/pages/agent/form/parser-form/vision-form-fields.test.tsx
+++ b/web/src/pages/agent/form/parser-form/vision-form-fields.test.tsx
@@ -23,7 +23,9 @@ jest.mock('@/components/ragflow-form', () => ({
<Controller
name={name}
control={control}
- render={({ field }) => children(field)}
+ render={({ field }) =>
+ typeof children === 'function' ? children(field) : children
+ }
/>
);
},
@@ -55,7 +57,10 @@ jest.mock('@/components/model-tree-select', () => ({
function ParserVisionForm({ model = '' }: { model?: string }) {
const form = useForm({
- defaultValues: { enable_vision_enhancement: false, vlm: { llm_id: model } },
+ defaultValues: {
+ enable_vision_enhancement: false,
+ vlm: { llm_id: model, system_prompt: '' },
+ },
});
return (
<FormProvider {...form}>
@@ -65,6 +70,22 @@ function ParserVisionForm({ model = '' }: { model?: string }) {
}
describe('global vision model selection', () => {
+ it('offers the shared prompt only while enhancement is on', () => {
+ render(<ParserVisionForm />);
+ expect(
+ screen.queryByPlaceholderText('flow.systemPromptPlaceholder'),
+ ).toBeNull();
+
+ fireEvent.click(screen.getByRole('switch'));
+
+ expect(
+ screen.getByPlaceholderText('flow.systemPromptPlaceholder'),
+ ).toBeInTheDocument();
+ // Language has no control here on purpose: captions follow the knowledge
+ // base language, which also drives tokenization.
+ expect(screen.queryByLabelText('Vision language')).toBeNull();
+ });
+
it('selects default A on first enable and preserves user-selected B across toggles', () => {
render(<ParserVisionForm />);
fireEvent.click(screen.getByRole('switch'));
diff --git a/web/src/pages/agent/form/parser-form/vision-form-fields.tsx b/web/src/pages/agent/form/parser-form/vision-form-fields.tsx
index 7dbf164962..0cb94bfac9 100644
--- a/web/src/pages/agent/form/parser-form/vision-form-fields.tsx
+++ b/web/src/pages/agent/form/parser-form/vision-form-fields.tsx
@@ -4,6 +4,7 @@ import {
} from '@/components/model-tree-select';
import { RAGFlowFormItem } from '@/components/ragflow-form';
import { Switch } from '@/components/ui/switch';
+import { Textarea } from '@/components/ui/textarea';
import { ModelTypeToField } from '@/constants/llm';
import { useFetchDefaultModelDictionary } from '@/hooks/use-llm-request';
import { useCallback } from 'react';
@@ -42,10 +43,11 @@ function VisionEnhancementSwitch({
return <Switch checked={!!enabled} onCheckedChange={handleCheckedChange} />;
}
-// Global vision enhancement: one switch + img2txt model shared by every
-// vision-capable file type. The fields live at the form's top level
-// (`enable_vision_enhancement` and `vlm.llm_id`) alongside `setups`, matching
-// the backend Parser component's params contract.
+// Global vision enhancement: one switch + img2txt model + description prompt.
+// The fields live at the form's top level (`enable_vision_enhancement` and
+// `vlm.{llm_id, system_prompt}`) alongside `setups`, matching the backend Parser
+// component's params contract. Response language is not offered here on purpose:
+// it belongs to the knowledge base, which also drives tokenization.
export function VisionEnhancementFormFields() {
const { t } = useTranslation();
const ownerTenantId = useOwnerTenantId();
@@ -77,6 +79,15 @@ export function VisionEnhancementFormFields() {
ownerTenantId={ownerTenantId}
/>
)}
+ {enabled && (
+ <RAGFlowFormItem
+ name="vlm.system_prompt"
+ label={t('flow.systemPrompt')}
+ tooltip={t('flow.visionEnhancementPromptTip')}
+ >
+ <Textarea placeholder={t('flow.systemPromptPlaceholder')} />
+ </RAGFlowFormItem>
+ )}
</>
);
}
diff --git a/web/src/pages/agent/utils.test.ts b/web/src/pages/agent/utils.test.ts
index c58c11828d..24dd6e6406 100644
--- a/web/src/pages/agent/utils.test.ts
+++ b/web/src/pages/agent/utils.test.ts
@@ -22,7 +22,10 @@ describe('transformParserParams vision settings', () => {
} as any);
expect(result.enable_vision_enhancement).toBe(true);
- expect(result.vlm).toEqual({ llm_id: 'model-B' });
+ expect(result.vlm).toEqual({
+ llm_id: 'model-B',
+ system_prompt: '',
+ });
expect(result).not.toHaveProperty(`${FileType.PDF}.vlm`);
expect(result).toHaveProperty(`${FileType.Audio}.vlm`, {
llm_id: 'asr-model',
@@ -37,7 +40,34 @@ describe('transformParserParams vision settings', () => {
} as any);
expect(result.enable_vision_enhancement).toBe(enabled);
- expect(result.vlm).toEqual({ llm_id: 'model-B' });
+ expect(result.vlm).toEqual({
+ llm_id: 'model-B',
+ system_prompt: '',
+ });
+ });
+
+ it('writes the image family as a switch and nothing else', () => {
+ // The saved shape is the contract the Go parser reads, so pin it: the
+ // switch out, and the retired per-family keys never written back.
+ const result = transformParserParams({
+ enable_vision_enhancement: false,
+ vlm: { llm_id: '', system_prompt: '' },
+ setups: [
+ {
+ fileFormat: FileType.Image,
+ ocr_enabled: false,
+ parse_method: 'stale-model@provider',
+ lang: 'French',
+ system_prompt: 'stale',
+ output_format: 'json',
+ },
+ ],
+ } as any);
+
+ expect(result.image).toMatchObject({ ocr_enabled: false });
+ expect(result.image).not.toHaveProperty('parse_method');
+ expect(result.image).not.toHaveProperty('lang');
+ expect(result.image).not.toHaveProperty('system_prompt');
});
});
diff --git a/web/src/pages/agent/utils.ts b/web/src/pages/agent/utils.ts
index df63ff27df..2a053badc1 100644
--- a/web/src/pages/agent/utils.ts
+++ b/web/src/pages/agent/utils.ts
@@ -270,9 +270,7 @@ export function transformParserParams(params: ParserFormSchemaType) {
case FileType.Image:
filteredSetup = {
...filteredSetup,
- parse_method: cur.parse_method,
- lang: cur.lang,
- system_prompt: cur.system_prompt,
+ ocr_enabled: cur.ocr_enabled,
};
break;
case FileType.Email:
diff --git a/web/src/pages/dataset/dataset/utils.ts b/web/src/pages/dataset/dataset/utils.ts
index 935169d583..6040256eaf 100644
--- a/web/src/pages/dataset/dataset/utils.ts
+++ b/web/src/pages/dataset/dataset/utils.ts
@@ -132,8 +132,9 @@ export const isDocumentProcessing = (
// no model (missingModel). The tenant default is deliberately not consulted:
// parsing reads the operator setup, so a global default does not make the
// file parsable. Image files only need their family declared: the image
-// parser always runs OCR and merely supplements it with the vision model
-// (picked as the image parse_method, with the tenant default as fallback).
+// OCR switch runs local extraction independently, and the vision model is a
+// global supplement picked from the top-level vlm.llm_id (falling back to the
+// tenant default).
// These helpers power the upload warning and the parse-click validation.
export type ParserModelGap = {