Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

fix(ingestion): honor the dataset language in the Go parser (#19966)

jay777218fef6cfb9e93
diff --git a/internal/ingestion/component/parser.go b/internal/ingestion/component/parser.go
index a336963a44..78ae0e24d6 100644
--- a/internal/ingestion/component/parser.go
+++ b/internal/ingestion/component/parser.go
@@ -449,6 +449,15 @@ func (c *ParserComponent) Invoke(ctx context.Context, db *gorm.DB, inputs map[st
inputs["tenant_id"] = tid
}
+ // Same pull-back for the run-level (dataset) language: File emits no
+ // lang, and the language consumers below — vision enhancement and the
+ // media dispatch branches — read the local inputs map, so without this
+ // the KB language never reaches them and the prompt language silently
+ // falls back to English.
+ if lang := globals.GlobalOrInput(ctx, inputs, "lang", ""); lang != "" {
+ inputs["lang"] = lang
+ }
+
// 2. Resolve the file family from the inputs. When the family
// is known, dispatchParse returns a typed parser payload.
// Otherwise the component stays in text-page mode.
diff --git a/internal/ingestion/component/parser_dispatch_test.go b/internal/ingestion/component/parser_dispatch_test.go
index 54158964e9..4e92430dba 100644
--- a/internal/ingestion/component/parser_dispatch_test.go
+++ b/internal/ingestion/component/parser_dispatch_test.go
@@ -1204,6 +1204,76 @@ func TestDispatch_PDFMinerU_BareModelUUID_InParseMethod(t *testing.T) {
requireJSONText(t, out, "Cloud MinerU Title")
}
+// TestDispatch_PDFMinerU_LanguageChain pins the lang_list chain
+// (mineru_lang → setup lang → "English"), mirroring Python's
+// mineru_parser.py:1181. The pdf setup's default lang ("Chinese") keeps the
+// unconfigured form value on Python's ch code.
+func TestDispatch_PDFMinerU_LanguageChain(t *testing.T) {
+ tests := []struct {
+ name string
+ overrides map[string]any
+ wantLang string
+ }{
+ {name: "default setup lang maps to ch", overrides: nil, wantLang: "ch"},
+ {name: "mineru_lang wins over setup lang", overrides: map[string]any{"mineru_lang": "English"}, wantLang: "en"},
+ {name: "setup lang used when mineru_lang unset", overrides: map[string]any{"lang": "Japanese"}, wantLang: "japan"},
+ }
+
+ for _, tc := range tests {
+ t.Run(tc.name, func(t *testing.T) {
+ withSSRFBypass(t)
+ var gotLang string
+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ if r.Method != http.MethodPost || r.URL.Path != "/file_parse" {
+ http.NotFound(w, r)
+ return
+ }
+ if err := r.ParseMultipartForm(1 << 20); err != nil {
+ t.Errorf("parse form: %v", err)
+ return
+ }
+ gotLang = r.FormValue("lang_list")
+ buf := new(bytes.Buffer)
+ zw := zip.NewWriter(buf)
+ f, _ := zw.Create("content_list.json")
+ _, _ = f.Write([]byte(`[{"type":"text","text":"# Title\n\nBody.\n"}]`))
+ _ = zw.Close()
+ w.Header().Set("Content-Type", "application/zip")
+ _, _ = w.Write(buf.Bytes())
+ }))
+ defer server.Close()
+
+ origResolve := resolveMinerUModelForDispatch
+ defer func() { resolveMinerUModelForDispatch = origResolve }()
+ baseURL := server.URL
+ apiKey := "mineru-secret"
+ resolveMinerUModelForDispatch = func(ctx context.Context, db *gorm.DB, tenantID, mid string) (models.ModelDriver, string, *models.APIConfig, error) {
+ return &mineruTestDriver{}, "vlm", &models.APIConfig{ApiKey: &apiKey, BaseURL: &baseURL}, nil
+ }
+
+ setups := defaultSetups()
+ setups["pdf"]["parse_method"] = "mineru"
+ setups["pdf"]["output_format"] = "markdown"
+ for k, v := range tc.overrides {
+ setups["pdf"][k] = v
+ }
+ c := &ParserComponent{setups: setups}
+
+ if _, err := c.Invoke(t.Context(), nil, map[string]any{
+ "binary": []byte("%PDF-1.4"),
+ "file_type": "pdf",
+ "name": "scansmpl.pdf",
+ "tenant_id": "test-tenant",
+ }); err != nil {
+ t.Fatalf("Invoke: %v", err)
+ }
+ if gotLang != tc.wantLang {
+ t.Errorf("lang_list = %q, want %q", gotLang, tc.wantLang)
+ }
+ })
+ }
+}
+
// TestDispatch_PDFMinerU_BareModelUUID_InLayoutRecognizer pins the same
// bare-UUID routing when the model id lands in layout_recognizer while
// parse_method stays a named method: the layout selector drives the probe and
diff --git a/internal/ingestion/component/parser_test.go b/internal/ingestion/component/parser_test.go
index 4318713dd3..593cb9c511 100644
--- a/internal/ingestion/component/parser_test.go
+++ b/internal/ingestion/component/parser_test.go
@@ -152,6 +152,34 @@ func TestParserComponentInvokeOutputsResolvedFileType(t *testing.T) {
}
}
+// TestParserComponentInvoke_LanguageFromGlobals covers the production run
+// shape: File emits no lang, the pipeline seeds the dataset language into
+// CanvasState.Globals, and the Parser must pull it into the local inputs so
+// the language consumers (vision enhancement, media dispatch) see it. The
+// inputs-map value stays the fallback when no CanvasState is attached.
+func TestParserComponentInvoke_LanguageFromGlobals(t *testing.T) {
+ component := &ParserComponent{setups: defaultSetups()}
+
+ ctx := runtime.WithState(t.Context(), &runtime.CanvasState{
+ Globals: map[string]any{"lang": "Chinese"},
+ })
+ out, err := component.Invoke(ctx, nil, map[string]any{"binary": "hello", "name": "notes.txt"})
+ if err != nil {
+ t.Fatalf("Invoke: %v", err)
+ }
+ if got := out["lang"]; got != "Chinese" {
+ t.Errorf("out[lang] = %v, want Chinese", got)
+ }
+
+ out, err = component.Invoke(t.Context(), nil, map[string]any{"binary": "hello", "name": "notes.txt", "lang": "Japanese"})
+ if err != nil {
+ t.Fatalf("Invoke: %v", err)
+ }
+ if got := out["lang"]; got != "Japanese" {
+ t.Errorf("out[lang] = %v, want Japanese", got)
+ }
+}
+
func TestNewParserComponentNormalizesOutputFormatToJSON(t *testing.T) {
component, err := NewParserComponent(map[string]any{
"pdf": map[string]any{"output_format": "markdown"},
diff --git a/internal/ingestion/component/pdf_vision_dispatch.go b/internal/ingestion/component/pdf_vision_dispatch.go
index 872860c9e3..76f85e833d 100644
--- a/internal/ingestion/component/pdf_vision_dispatch.go
+++ b/internal/ingestion/component/pdf_vision_dispatch.go
@@ -524,7 +524,14 @@ func dispatchMinerUPDF(
apiURL := strings.TrimRight(baseURL, "/") + "/file_parse"
parseMethod := mineruAPIParseMethod(getStringOr(setup, "mineru_parse_method", ""))
- lang := getStringOr(setup, "mineru_lang", "English")
+ // Language chain mirrors Python's mineru_parser.py:1181
+ // (mineru_lang → lang → "English"): the dedicated MinerU option wins,
+ // then the setup language, then English. The pdf setup's lang default
+ // ("Chinese") makes the unconfigured lang_list match Python's ch.
+ lang := getStringOr(setup, "mineru_lang", "")
+ if lang == "" {
+ lang = getStringOr(setup, "lang", "English")
+ }
mineruLang := mineruLangCode(lang)
backend := getStringOr(setup, "mineru_backend", "pipeline")
diff --git a/internal/ingestion/component/vision_enhancement.go b/internal/ingestion/component/vision_enhancement.go
index b1dbe9ed8b..7064e1c3d6 100644
--- a/internal/ingestion/component/vision_enhancement.go
+++ b/internal/ingestion/component/vision_enhancement.go
@@ -190,7 +190,13 @@ func maybeDispatchVisionEnhancement(
if tenantID == "" {
return dispatched, false, nil
}
- language := resolveVisionLanguage(inputs, "")
+ // Language priority mirrors Python's enhance_media_sections_with_vision
+ // (rag/flow/parser/parser.py:778): the run-level dataset language first
+ // (the Parser pulls it into inputs from Globals), then the family setup's
+ // lang, then English.
+ family := resolveParserFamily(fileType)
+ setup := setups[family]
+ language := resolveVisionLanguage(inputs, getStringOr(setup, "lang", ""))
// 1. Collect target items (images/tables that can be described). A target
// may carry an inlined image (docx/markdown, or any pre-inlined source) or,
@@ -229,8 +235,6 @@ func maybeDispatchVisionEnhancement(
// 2. Resolve the per-call IMAGE2TEXT model, then fall back to the tenant
// default. Mirror Python's vlm_conf["llm_id"] preference.
- family := resolveParserFamily(fileType)
- setup := setups[family]
modelRef := configuredMediaModelID(setup, family)
var driver modelModule.ModelDriver
var modelName string
diff --git a/internal/ingestion/component/vision_enhancement_test.go b/internal/ingestion/component/vision_enhancement_test.go
index 553e38df36..0b36fdb44a 100644
--- a/internal/ingestion/component/vision_enhancement_test.go
+++ b/internal/ingestion/component/vision_enhancement_test.go
@@ -171,6 +171,68 @@ func TestVisionEnhancement_EnhancesJSONImagesAndTables(t *testing.T) {
}
}
+// TestVisionEnhancement_LanguagePriority pins the prompt-language chain:
+// run-level dataset language (inputs) > family setup lang > English. The
+// setup fallback is what makes the DSL's pdf.lang effective, mirroring
+// Python's conf.get("lang") in parser.py:778.
+func TestVisionEnhancement_LanguagePriority(t *testing.T) {
+ tests := []struct {
+ name string
+ inputs map[string]any
+ setups map[string]schema.ParserSetup
+ want string
+ }{
+ {
+ name: "dataset language wins over setup lang",
+ inputs: map[string]any{"tenant_id": "t1", "lang": "Japanese"},
+ setups: map[string]schema.ParserSetup{"pdf": {"lang": "Chinese"}},
+ want: "Japanese",
+ },
+ {
+ name: "setup lang used when inputs carry no lang",
+ inputs: map[string]any{"tenant_id": "t1"},
+ setups: map[string]schema.ParserSetup{"pdf": {"lang": "Chinese"}},
+ want: "Chinese",
+ },
+ {
+ name: "english when neither is set",
+ inputs: map[string]any{"tenant_id": "t1"},
+ setups: nil,
+ want: "English",
+ },
+ }
+
+ for _, tc := range tests {
+ t.Run(tc.name, func(t *testing.T) {
+ var capturedLanguage string
+ swapVisionGlobals(t, fakeResolver, (&visionEnhanceCaptureInvoker{}).invoke,
+ func(language string) (string, error) {
+ capturedLanguage = language
+ return "describe the figure in " + language, nil
+ })
+
+ dispatched := parser.ParseResult{
+ OutputFormat: "json",
+ JSON: []map[string]any{
+ {"text": "", "image": "aGVsbG8taW1hZ2U=", "doc_type_kwd": "image"},
+ },
+ }
+
+ _, handled, err := maybeDispatchVisionEnhancement(
+ t.Context(), dao.DB, utility.FileTypePDF, dispatched, tc.inputs, tc.setups)
+ if err != nil {
+ t.Fatalf("unexpected error: %v", err)
+ }
+ if !handled {
+ t.Fatal("handled = false, want true")
+ }
+ if capturedLanguage != tc.want {
+ t.Errorf("figure prompt language = %q, want %q", capturedLanguage, tc.want)
+ }
+ })
+ }
+}
+
func TestVisionEnhancement_MarkdownOutputUntouched(t *testing.T) {
called := false
swapVisionGlobals(t,