RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
fix(ingestion): honor the dataset language in the Go parser (#19966)
jay777218fef6cfb9e93
diff --git a/internal/ingestion/component/parser.go b/internal/ingestion/component/parser.goindex a336963a44..78ae0e24d6 100644--- a/internal/ingestion/component/parser.go+++ b/internal/ingestion/component/parser.go@@ -449,6 +449,15 @@ func (c *ParserComponent) Invoke(ctx context.Context, db *gorm.DB, inputs map[stinputs["tenant_id"] = tid}+ // Same pull-back for the run-level (dataset) language: File emits no+ // lang, and the language consumers below — vision enhancement and the+ // media dispatch branches — read the local inputs map, so without this+ // the KB language never reaches them and the prompt language silently+ // falls back to English.+ if lang := globals.GlobalOrInput(ctx, inputs, "lang", ""); lang != "" {+ inputs["lang"] = lang+ }+// 2. Resolve the file family from the inputs. When the family// is known, dispatchParse returns a typed parser payload.// Otherwise the component stays in text-page mode.diff --git a/internal/ingestion/component/parser_dispatch_test.go b/internal/ingestion/component/parser_dispatch_test.goindex 54158964e9..4e92430dba 100644--- a/internal/ingestion/component/parser_dispatch_test.go+++ b/internal/ingestion/component/parser_dispatch_test.go@@ -1204,6 +1204,76 @@ func TestDispatch_PDFMinerU_BareModelUUID_InParseMethod(t *testing.T) {requireJSONText(t, out, "Cloud MinerU Title")}+// TestDispatch_PDFMinerU_LanguageChain pins the lang_list chain+// (mineru_lang → setup lang → "English"), mirroring Python's+// mineru_parser.py:1181. The pdf setup's default lang ("Chinese") keeps the+// unconfigured form value on Python's ch code.+func TestDispatch_PDFMinerU_LanguageChain(t *testing.T) {+ tests := []struct {+ name string+ overrides map[string]any+ wantLang string+ }{+ {name: "default setup lang maps to ch", overrides: nil, wantLang: "ch"},+ {name: "mineru_lang wins over setup lang", overrides: map[string]any{"mineru_lang": "English"}, wantLang: "en"},+ {name: "setup lang used when mineru_lang unset", overrides: map[string]any{"lang": "Japanese"}, wantLang: "japan"},+ }++ for _, tc := range tests {+ t.Run(tc.name, func(t *testing.T) {+ withSSRFBypass(t)+ var gotLang string+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ if r.Method != http.MethodPost || r.URL.Path != "/file_parse" {+ http.NotFound(w, r)+ return+ }+ if err := r.ParseMultipartForm(1 << 20); err != nil {+ t.Errorf("parse form: %v", err)+ return+ }+ gotLang = r.FormValue("lang_list")+ buf := new(bytes.Buffer)+ zw := zip.NewWriter(buf)+ f, _ := zw.Create("content_list.json")+ _, _ = f.Write([]byte(`[{"type":"text","text":"# Title\n\nBody.\n"}]`))+ _ = zw.Close()+ w.Header().Set("Content-Type", "application/zip")+ _, _ = w.Write(buf.Bytes())+ }))+ defer server.Close()++ origResolve := resolveMinerUModelForDispatch+ defer func() { resolveMinerUModelForDispatch = origResolve }()+ baseURL := server.URL+ apiKey := "mineru-secret"+ resolveMinerUModelForDispatch = func(ctx context.Context, db *gorm.DB, tenantID, mid string) (models.ModelDriver, string, *models.APIConfig, error) {+ return &mineruTestDriver{}, "vlm", &models.APIConfig{ApiKey: &apiKey, BaseURL: &baseURL}, nil+ }++ setups := defaultSetups()+ setups["pdf"]["parse_method"] = "mineru"+ setups["pdf"]["output_format"] = "markdown"+ for k, v := range tc.overrides {+ setups["pdf"][k] = v+ }+ c := &ParserComponent{setups: setups}++ if _, err := c.Invoke(t.Context(), nil, map[string]any{+ "binary": []byte("%PDF-1.4"),+ "file_type": "pdf",+ "name": "scansmpl.pdf",+ "tenant_id": "test-tenant",+ }); err != nil {+ t.Fatalf("Invoke: %v", err)+ }+ if gotLang != tc.wantLang {+ t.Errorf("lang_list = %q, want %q", gotLang, tc.wantLang)+ }+ })+ }+}+// TestDispatch_PDFMinerU_BareModelUUID_InLayoutRecognizer pins the same// bare-UUID routing when the model id lands in layout_recognizer while// parse_method stays a named method: the layout selector drives the probe anddiff --git a/internal/ingestion/component/parser_test.go b/internal/ingestion/component/parser_test.goindex 4318713dd3..593cb9c511 100644--- a/internal/ingestion/component/parser_test.go+++ b/internal/ingestion/component/parser_test.go@@ -152,6 +152,34 @@ func TestParserComponentInvokeOutputsResolvedFileType(t *testing.T) {}}+// TestParserComponentInvoke_LanguageFromGlobals covers the production run+// shape: File emits no lang, the pipeline seeds the dataset language into+// CanvasState.Globals, and the Parser must pull it into the local inputs so+// the language consumers (vision enhancement, media dispatch) see it. The+// inputs-map value stays the fallback when no CanvasState is attached.+func TestParserComponentInvoke_LanguageFromGlobals(t *testing.T) {+ component := &ParserComponent{setups: defaultSetups()}++ ctx := runtime.WithState(t.Context(), &runtime.CanvasState{+ Globals: map[string]any{"lang": "Chinese"},+ })+ out, err := component.Invoke(ctx, nil, map[string]any{"binary": "hello", "name": "notes.txt"})+ if err != nil {+ t.Fatalf("Invoke: %v", err)+ }+ if got := out["lang"]; got != "Chinese" {+ t.Errorf("out[lang] = %v, want Chinese", got)+ }++ out, err = component.Invoke(t.Context(), nil, map[string]any{"binary": "hello", "name": "notes.txt", "lang": "Japanese"})+ if err != nil {+ t.Fatalf("Invoke: %v", err)+ }+ if got := out["lang"]; got != "Japanese" {+ t.Errorf("out[lang] = %v, want Japanese", got)+ }+}+func TestNewParserComponentNormalizesOutputFormatToJSON(t *testing.T) {component, err := NewParserComponent(map[string]any{"pdf": map[string]any{"output_format": "markdown"},diff --git a/internal/ingestion/component/pdf_vision_dispatch.go b/internal/ingestion/component/pdf_vision_dispatch.goindex 872860c9e3..76f85e833d 100644--- a/internal/ingestion/component/pdf_vision_dispatch.go+++ b/internal/ingestion/component/pdf_vision_dispatch.go@@ -524,7 +524,14 @@ func dispatchMinerUPDF(apiURL := strings.TrimRight(baseURL, "/") + "/file_parse"parseMethod := mineruAPIParseMethod(getStringOr(setup, "mineru_parse_method", ""))- lang := getStringOr(setup, "mineru_lang", "English")+ // Language chain mirrors Python's mineru_parser.py:1181+ // (mineru_lang → lang → "English"): the dedicated MinerU option wins,+ // then the setup language, then English. The pdf setup's lang default+ // ("Chinese") makes the unconfigured lang_list match Python's ch.+ lang := getStringOr(setup, "mineru_lang", "")+ if lang == "" {+ lang = getStringOr(setup, "lang", "English")+ }mineruLang := mineruLangCode(lang)backend := getStringOr(setup, "mineru_backend", "pipeline")diff --git a/internal/ingestion/component/vision_enhancement.go b/internal/ingestion/component/vision_enhancement.goindex b1dbe9ed8b..7064e1c3d6 100644--- a/internal/ingestion/component/vision_enhancement.go+++ b/internal/ingestion/component/vision_enhancement.go@@ -190,7 +190,13 @@ func maybeDispatchVisionEnhancement(if tenantID == "" {return dispatched, false, nil}- language := resolveVisionLanguage(inputs, "")+ // Language priority mirrors Python's enhance_media_sections_with_vision+ // (rag/flow/parser/parser.py:778): the run-level dataset language first+ // (the Parser pulls it into inputs from Globals), then the family setup's+ // lang, then English.+ family := resolveParserFamily(fileType)+ setup := setups[family]+ language := resolveVisionLanguage(inputs, getStringOr(setup, "lang", ""))// 1. Collect target items (images/tables that can be described). A target// may carry an inlined image (docx/markdown, or any pre-inlined source) or,@@ -229,8 +235,6 @@ func maybeDispatchVisionEnhancement(// 2. Resolve the per-call IMAGE2TEXT model, then fall back to the tenant// default. Mirror Python's vlm_conf["llm_id"] preference.- family := resolveParserFamily(fileType)- setup := setups[family]modelRef := configuredMediaModelID(setup, family)var driver modelModule.ModelDrivervar modelName stringdiff --git a/internal/ingestion/component/vision_enhancement_test.go b/internal/ingestion/component/vision_enhancement_test.goindex 553e38df36..0b36fdb44a 100644--- a/internal/ingestion/component/vision_enhancement_test.go+++ b/internal/ingestion/component/vision_enhancement_test.go@@ -171,6 +171,68 @@ func TestVisionEnhancement_EnhancesJSONImagesAndTables(t *testing.T) {}}+// TestVisionEnhancement_LanguagePriority pins the prompt-language chain:+// run-level dataset language (inputs) > family setup lang > English. The+// setup fallback is what makes the DSL's pdf.lang effective, mirroring+// Python's conf.get("lang") in parser.py:778.+func TestVisionEnhancement_LanguagePriority(t *testing.T) {+ tests := []struct {+ name string+ inputs map[string]any+ setups map[string]schema.ParserSetup+ want string+ }{+ {+ name: "dataset language wins over setup lang",+ inputs: map[string]any{"tenant_id": "t1", "lang": "Japanese"},+ setups: map[string]schema.ParserSetup{"pdf": {"lang": "Chinese"}},+ want: "Japanese",+ },+ {+ name: "setup lang used when inputs carry no lang",+ inputs: map[string]any{"tenant_id": "t1"},+ setups: map[string]schema.ParserSetup{"pdf": {"lang": "Chinese"}},+ want: "Chinese",+ },+ {+ name: "english when neither is set",+ inputs: map[string]any{"tenant_id": "t1"},+ setups: nil,+ want: "English",+ },+ }++ for _, tc := range tests {+ t.Run(tc.name, func(t *testing.T) {+ var capturedLanguage string+ swapVisionGlobals(t, fakeResolver, (&visionEnhanceCaptureInvoker{}).invoke,+ func(language string) (string, error) {+ capturedLanguage = language+ return "describe the figure in " + language, nil+ })++ dispatched := parser.ParseResult{+ OutputFormat: "json",+ JSON: []map[string]any{+ {"text": "", "image": "aGVsbG8taW1hZ2U=", "doc_type_kwd": "image"},+ },+ }++ _, handled, err := maybeDispatchVisionEnhancement(+ t.Context(), dao.DB, utility.FileTypePDF, dispatched, tc.inputs, tc.setups)+ if err != nil {+ t.Fatalf("unexpected error: %v", err)+ }+ if !handled {+ t.Fatal("handled = false, want true")+ }+ if capturedLanguage != tc.want {+ t.Errorf("figure prompt language = %q, want %q", capturedLanguage, tc.want)+ }+ })+ }+}+func TestVisionEnhancement_MarkdownOutputUntouched(t *testing.T) {called := falseswapVisionGlobals(t,