RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
fix(parser): route UI "Plain Text" PDF method to the plain-text parser (#19979)
jay7772183c47d025182
diff --git a/internal/ingestion/component/parser.go b/internal/ingestion/component/parser.goindex 353ba66b6d..a99ceef5a3 100644--- a/internal/ingestion/component/parser.go+++ b/internal/ingestion/component/parser.go@@ -221,14 +221,10 @@ func (c *ParserComponent) Check() error {if pm == "" {return errors.New("parse method abnormal. does not support empty value")}- pmLower := strings.ToLower(pm)- pdfWhitelist := []string{- "deepdoc", "plain_text", "mineru", "monkeyocrv2", "docling",- "opendataloader", "tcadp parser", "paddleocr", "somark",- }- if !containsString(pdfWhitelist, pmLower) {- // Non-whitelist parse_method is treated as a VLM method,- // which requires lang (Python parser.py:257-258).+ if !parser.IsPDFParseMethod(pm) {+ // A parse_method outside the known vocabulary is treated as a+ // VLM model reference, which requires lang (Python+ // parser.py:257-258).if lang, _ := pdf["lang"].(string); lang == "" {return errors.New("PDF VLM language does not support empty value")}@@ -247,18 +243,6 @@ func (c *ParserComponent) Check() error {return nil}-// containsString reports whether s is in list. Used by Check() for-// whitelist membership tests; kept unexported and local to this file-// to avoid polluting the package namespace.-func containsString(list []string, s string) bool {- for _, v := range list {- if v == s {- return true- }- }- return false-}-func defaultSetups() map[string]schema.ParserSetup {return map[string]schema.ParserSetup{"pdf": {diff --git a/internal/ingestion/component/parser_check_test.go b/internal/ingestion/component/parser_check_test.goindex 63d526848f..f3f4b8c433 100644--- a/internal/ingestion/component/parser_check_test.go+++ b/internal/ingestion/component/parser_check_test.go@@ -54,6 +54,14 @@ func TestParserComponent_Check(t *testing.T) {name: "pdf: plain_text (whitelist, case-insensitive) without lang → pass",setups: map[string]schema.ParserSetup{"pdf": {"parse_method": "PLAIN_TEXT"}},},+ {+ name: "pdf: Plain Text (UI spelling of the plain-text option) without lang → pass",+ setups: map[string]schema.ParserSetup{"pdf": {"parse_method": "Plain Text"}},+ },+ {+ name: "pdf: plaintext (UI spelling, no space) without lang → pass",+ setups: map[string]schema.ParserSetup{"pdf": {"parse_method": "plaintext"}},+ },{name: "pdf: tcadp parser (whitelist with space) without lang → pass",setups: map[string]schema.ParserSetup{"pdf": {"parse_method": "tcadp parser"}},diff --git a/internal/ingestion/component/parser_dispatch_test.go b/internal/ingestion/component/parser_dispatch_test.goindex 4e92430dba..251fa1dc6c 100644--- a/internal/ingestion/component/parser_dispatch_test.go+++ b/internal/ingestion/component/parser_dispatch_test.go@@ -475,6 +475,11 @@ func TestDispatch_PDFLegacyMarkdownConfigurationEmitsJSON(t *testing.T) {}}+// TestDispatch_PDFPlainText_UsesConfiguredBackend pins the plain-text+// dispatch for every spelling the dataset configuration UI can persist.+// The UI option is labelled "Naive" and stores "Plain Text"; treating that+// spelling as a custom VLM model name made the run fail with+// `provider name missing in model name: Plain Text`.func TestDispatch_PDFPlainText_UsesConfiguredBackend(t *testing.T) {path := filepath.Join("..", "..", "..", "test", "benchmark", "test_docs", "Doc1.pdf")data, err := os.ReadFile(path)@@ -482,25 +487,29 @@ func TestDispatch_PDFPlainText_UsesConfiguredBackend(t *testing.T) {t.Fatalf("ReadFile(%s): %v", path, err)}- setups := defaultSetups()- setups["pdf"]["parse_method"] = "plain_text"- setups["pdf"]["output_format"] = "json"- c := &ParserComponent{setups: setups}+ for _, method := range []string{"plain_text", "plaintext", "Plain Text"} {+ t.Run(method, func(t *testing.T) {+ setups := defaultSetups()+ setups["pdf"]["parse_method"] = method+ setups["pdf"]["output_format"] = "json"+ c := &ParserComponent{setups: setups}- out, err := c.Invoke(t.Context(), nil, map[string]any{- "binary": data,- "file_type": "pdf",- "name": "Doc1.pdf",- })- if err != nil {- t.Fatalf("Invoke: %v", err)- }- jsonItems, ok := out["json"].([]map[string]any)- if !ok || len(jsonItems) == 0 {- t.Fatalf("json payload missing or empty: %T", out["json"])- }- if got, _ := jsonItems[0]["text"].(string); strings.TrimSpace(got) == "" {- t.Fatalf("json first item text = %q, want non-empty", got)+ out, err := c.Invoke(t.Context(), nil, map[string]any{+ "binary": data,+ "file_type": "pdf",+ "name": "Doc1.pdf",+ })+ if err != nil {+ t.Fatalf("Invoke: %v", err)+ }+ jsonItems, ok := out["json"].([]map[string]any)+ if !ok || len(jsonItems) == 0 {+ t.Fatalf("json payload missing or empty: %T", out["json"])+ }+ if got, _ := jsonItems[0]["text"].(string); strings.TrimSpace(got) == "" {+ t.Fatalf("json first item text = %q, want non-empty", got)+ }+ })}}diff --git a/internal/ingestion/component/pdf_vision_dispatch.go b/internal/ingestion/component/pdf_vision_dispatch.goindex 76f85e833d..0fc808f62e 100644--- a/internal/ingestion/component/pdf_vision_dispatch.go+++ b/internal/ingestion/component/pdf_vision_dispatch.go@@ -91,7 +91,7 @@ func maybeDispatchPDFVision(strings.HasPrefix(layoutLower, "monkeyocrv2") ||strings.Contains(layoutLower, "@monkeyocrv2")isMonkeyByUUID := false- if !isMonkeyMatch && tenantID != "" && strings.TrimSpace(monkeySelector) != "" && !isNamedPDFParseMethod(monkeySelector) {+ if !isMonkeyMatch && tenantID != "" && strings.TrimSpace(monkeySelector) != "" && !parser.IsPDFParseMethod(monkeySelector) {isMonkeyByUUID = isMonkeyOCRv2LayoutModelID(ctx, db, tenantID, monkeySelector)}if isMonkeyMatch || isMonkeyByUUID {@@ -120,7 +120,7 @@ func maybeDispatchPDFVision(}isMinerUByUUID := falseif !isMinerUMatch && strings.TrimSpace(minerUSelector) != "" &&- !isNamedPDFParseMethod(minerUSelector) {+ !parser.IsPDFParseMethod(minerUSelector) {isMinerUByUUID = isMinerULayoutModelID(ctx, db, tenantID, minerUSelector)}if isMinerUMatch || isMinerUByUUID {@@ -165,7 +165,7 @@ func maybeDispatchPDFVision(}isPaddleOCRByUUID := falseif !isPaddleOCRMatch && strings.TrimSpace(paddleOCRSelector) != "" &&- !isNamedPDFParseMethod(paddleOCRSelector) {+ !parser.IsPDFParseMethod(paddleOCRSelector) {isPaddleOCRByUUID = isPaddleOCRLayoutModelID(ctx, db, tenantID, paddleOCRSelector)}if isPaddleOCRMatch || isPaddleOCRByUUID {@@ -848,48 +848,19 @@ func resolvePDFVisionModelID(setup schema.ParserSetup) (string, bool) {}if raw, ok := setup["parse_method"].(string); ok {method := strings.TrimSpace(raw)- if method != "" && !isNamedPDFParseMethod(method) {+ if method != "" && !parser.IsPDFParseMethod(method) {return method, true}}if raw, ok := setup["layout_recognizer"].(string); ok {method := strings.TrimSpace(raw)- if method == "" || strings.EqualFold(method, "plain text") || strings.EqualFold(method, "plaintext") {- return "", false- }- if !isNamedPDFParseMethod(method) {+ if method != "" && !parser.IsPDFParseMethod(method) {return method, true}}return "", false}-// isNamedPDFParseMethod reports whether raw is a recognized named PDF-// parse method (as opposed to a CustomVLM model name). Its membership set-// MUST stay aligned with the PDF whitelist enforced by-// (*ParserComponent).Check() (parser.go:200-203):-//-// deepdoc, plain_text, mineru, docling,-// opendataloader, tcadp parser, paddleocr, somark-//-// A parse_method that Check() rejects must not be treated as a named method-// here, otherwise it silently falls through to the CustomVLM vision path-// instead of failing fast at construction.-//-// Note: "@"-suffixed spellings such as "foo@mineru" are layout_recognizer-// selectors, not parse_method values. Check() rejects them as parse_method,-// and the MinerU layout branch is resolved from the layout_recognizer field-// separately (pdf_vision_dispatch.go:62-68), so they must NOT be recognized-// here.-func isNamedPDFParseMethod(raw string) bool {- method := strings.ToLower(strings.TrimSpace(raw))- switch method {- case "deepdoc", "plain_text", "mineru", "monkeyocrv2", "docling", "opendataloader", "tcadp parser", "paddleocr", "somark":- return true- }- return false-}-func dispatchPDFVision(ctx context.Context,db *gorm.DB,diff --git a/internal/ingestion/component/pdf_vision_dispatch_test.go b/internal/ingestion/component/pdf_vision_dispatch_test.goindex 899b3c166b..80cd0853de 100644--- a/internal/ingestion/component/pdf_vision_dispatch_test.go+++ b/internal/ingestion/component/pdf_vision_dispatch_test.go@@ -262,72 +262,3 @@ func TestDispatchPaddleOCRPdfEmptyTextFails(t *testing.T) {t.Errorf("expected zero-value result on error, got %+v", res)}}--// TestIsNamedPDFParseMethodWhitelistAligned verifies that the runtime-// "named parse_method" classifier agrees with (*ParserComponent).Check()'s-// PDF whitelist (parser.go:200-203):-//-// deepdoc, plain_text, mineru, docling,-// opendataloader, tcadp parser, paddleocr, somark-//-// Diff 2.10: a parse_method that Check() rejects must NOT be treated as a-// recognized named method by isNamedPDFParseMethod — otherwise it silently-// falls through to the CustomVLM vision path instead of failing fast at-// construction (and Python would have rejected it outright).-func TestIsNamedPDFParseMethodWhitelistAligned(t *testing.T) {- // Values that MUST be recognized (subset of the Check() whitelist,- // case-insensitive).- named := []string{- "deepdoc", "plain_text", "mineru", "monkeyocrv2", "docling",- "opendataloader", "tcadp parser", "paddleocr", "somark",- "DeepDoc", "PLAIN_TEXT", "MinerU", "DocLing",- "OpenDataLoader", "TCADP Parser", "PaddleOCR", "SoMark",- }- for _, v := range named {- if !isNamedPDFParseMethod(v) {- t.Errorf("isNamedPDFParseMethod(%q) = false, want true (in Check() whitelist)", v)- }- }-- // Values that MUST NOT be recognized. These either duplicate the- // whitelist with non-canonical spelling ("plain text"/"plaintext")- // or are bare-family abbreviations ("tcadp") that Check() does not- // accept, so they should be funneled to the CustomVLM path (or fail- // construction) rather than masquerading as a named method.- notNamed := []string{- "plain text", "plaintext", "tcadp",- "CustomVLM", "some_vlm", "gpt-4o",- "", " ",- }- for _, v := range notNamed {- if isNamedPDFParseMethod(v) {- t.Errorf("isNamedPDFParseMethod(%q) = true, want false (not in Check() whitelist)", v)- }- }-}--// TestIsNamedPDFParseMethodLayoutSuffixes verifies that "@"-suffixed-// layout_recognizer spellings are NOT treated as named parse methods. They-// are layout_recognizer selectors (resolved separately at-// pdf_vision_dispatch.go:62-68), and Check() rejects them as parse_method,-// so they must fall through to the CustomVLM/VLM path — consistent with the-// (*ParserComponent).Check() whitelist (parser.go:200-203).-func TestIsNamedPDFParseMethodLayoutSuffixes(t *testing.T) {- suffixed := []string{- "foo@mineru", "@mineru",- "foo@monkeyocrv2", "@monkeyocrv2",- "foo@paddleocr", "@paddleocr",- "foo@somark", "@somark",- "foo@opendataloader", "@opendataloader",- }- for _, v := range suffixed {- if isNamedPDFParseMethod(v) {- t.Errorf("isNamedPDFParseMethod(%q) = true, want false (layout_recognizer selector, not a named parse_method)", v)- }- }-- // An unknown suffix is also not a named method.- if isNamedPDFParseMethod("foo@unknown") {- t.Errorf("isNamedPDFParseMethod(%q) = true, want false", "foo@unknown")- }-}diff --git a/internal/parser/parser/pdf_parser_common.go b/internal/parser/parser/pdf_parser_common.goindex fcde314cda..e4c50bba48 100644--- a/internal/parser/parser/pdf_parser_common.go+++ b/internal/parser/parser/pdf_parser_common.go@@ -42,6 +42,10 @@ import (// is compiled behind `//go:build cgo`.var ErrPDFEngineUnavailable = errors.New("parser: PDF backend unavailable in this build")+// supportedPDFParseMethods is the set of canonical tokens PDFParser can+// execute; see pdfParseMethodSpellings for the accepted spellings and+// TestPDFParseMethodTablesAgree for the two intentional divergences+// ("" sentinel, dispatcher-handled monkeyocrv2).var supportedPDFParseMethods = map[string]struct{}{"": {},"deepdoc": {},@@ -274,6 +278,37 @@ func (p *PDFParser) ConfigureFromSetup(setup map[string]any) {}}+// pdfParseMethodSpellings is the single vocabulary of PDF parse methods:+// every accepted spelling (lower-cased and trimmed) mapped to its canonical+// token. Consumers must not keep their own copies of this set — tell a parse+// method apart from a VLM model selector via IsPDFParseMethod, and resolve+// its canonical token via normalizePDFParseMethod.+//+// "plaintext" / "plain text" are the spellings the dataset configuration UI+// persists for its plain-text option (ParseDocumentType.PlainText), so they+// are parse methods rather than model names.+var pdfParseMethodSpellings = map[string]string{+ "deepdoc": "deepdoc",+ "plain_text": "plain_text",+ "plaintext": "plain_text",+ "plain text": "plain_text",+ "mineru": "mineru",+ "monkeyocrv2": "monkeyocrv2",+ "docling": "docling",+ "opendataloader": "opendataloader",+ "tcadp parser": "tcadp",+ "paddleocr": "paddleocr",+ "somark": "somark",+}++// IsPDFParseMethod reports whether raw names a PDF parse method rather than+// a VLM model selector. "@"-suffixed spellings such as "foo@mineru" are+// layout_recognizer selectors resolved separately, so they report false.+func IsPDFParseMethod(raw string) bool {+ _, ok := pdfParseMethodSpellings[strings.ToLower(strings.TrimSpace(raw))]+ return ok+}+func normalizePDFParseMethod(raw string) string {method := strings.ToLower(strings.TrimSpace(raw))switch {@@ -286,11 +321,8 @@ func normalizePDFParseMethod(raw string) string {case strings.HasSuffix(method, "@opendataloader"):return "opendataloader"}- switch method {- case "plaintext":- return "plain_text"- case "tcadp parser":- return "tcadp"+ if canonical, ok := pdfParseMethodSpellings[method]; ok {+ return canonical}return method}diff --git a/internal/parser/parser/pdf_parser_common_test.go b/internal/parser/parser/pdf_parser_common_test.goindex be57c5674b..46358f7285 100644--- a/internal/parser/parser/pdf_parser_common_test.go+++ b/internal/parser/parser/pdf_parser_common_test.go@@ -474,6 +474,130 @@ func TestPDFParser_ValidateParseMethod(t *testing.T) {}}+// TestNormalizePDFParseMethod_PlainTextSpellings pins the plain-text+// spellings the dataset configuration UI can persist. The UI option is+// labelled "Naive" and stores "Plain Text", so both the spaced and+// spaceless spellings must land on the canonical "plain_text" method+// instead of being rejected as unsupported (or, upstream, mistaken for a+// vision model name).+func TestNormalizePDFParseMethod_PlainTextSpellings(t *testing.T) {+ cases := []struct{ in, want string }{+ {"plain_text", "plain_text"},+ {"Plain_Text", "plain_text"},+ {"plaintext", "plain_text"},+ {"PlainText", "plain_text"},+ {"plain text", "plain_text"},+ {"Plain Text", "plain_text"},+ {" Plain Text ", "plain_text"},+ }+ for _, c := range cases {+ if got := normalizePDFParseMethod(c.in); got != c.want {+ t.Errorf("normalizePDFParseMethod(%q) = %q, want %q", c.in, got, c.want)+ }+ }+}++// TestIsPDFParseMethod pins the vocabulary shared by the construction-time+// check and the runtime vision dispatcher. "plain text"/"plaintext" are the+// spellings the dataset configuration UI writes for its plain-text option;+// treating them as model names breaks PDF parsing with "provider name+// missing in model name".+func TestIsPDFParseMethod(t *testing.T) {+ named := []string{+ "deepdoc", "plain_text", "plaintext", "plain text", "mineru",+ "monkeyocrv2", "docling", "opendataloader", "tcadp parser",+ "paddleocr", "somark",+ "DeepDoc", "PLAIN_TEXT", "MinerU", "DocLing",+ "OpenDataLoader", "TCADP Parser", "PaddleOCR", "SoMark",+ "Plain Text", "PlainText", "PLAIN TEXT",+ }+ for _, v := range named {+ if !IsPDFParseMethod(v) {+ t.Errorf("IsPDFParseMethod(%q) = false, want true", v)+ }+ }++ // Anything else is a VLM model name: the UI never writes the bare+ // "tcadp" abbreviation, and empty values are rejected separately.+ notNamed := []string{+ "tcadp",+ "CustomVLM", "some_vlm", "gpt-4o",+ "", " ",+ }+ for _, v := range notNamed {+ if IsPDFParseMethod(v) {+ t.Errorf("IsPDFParseMethod(%q) = true, want false", v)+ }+ }+}++// TestIsPDFParseMethodLayoutSuffixes pins that "@"-suffixed+// layout_recognizer selectors are not parse methods; they are resolved from+// the layout_recognizer field separately.+func TestIsPDFParseMethodLayoutSuffixes(t *testing.T) {+ suffixed := []string{+ "foo@mineru", "@mineru",+ "foo@monkeyocrv2", "@monkeyocrv2",+ "foo@paddleocr", "@paddleocr",+ "foo@somark", "@somark",+ "foo@opendataloader", "@opendataloader",+ "foo@unknown",+ }+ for _, v := range suffixed {+ if IsPDFParseMethod(v) {+ t.Errorf("IsPDFParseMethod(%q) = true, want false (layout_recognizer selector, not a parse_method)", v)+ }+ }+}++// TestPDFParseMethodTablesAgree pins the relationship between the two+// vocabulary tables in pdf_parser_common.go: every canonical token a+// spelling maps to must be executable by PDFParser, and every executable+// token must be reachable from some spelling. The only intended divergences+// are:+//+// - "" (the unset sentinel) is accepted by the parser as the default+// method but has no spelling, so IsPDFParseMethod("") stays false;+// - "monkeyocrv2" is dispatched by the ingestion vision dispatcher+// before PDFParser runs, so it is a recognized method without being+// executable by the parser switch itself.+//+// Any other difference means one table was updated without the other: a+// spelling whose canonical token is not executable dies in+// validateParseMethod, and an executable token without a spelling is+// misclassified as a VLM model name by Check and the dispatcher (the+// "Plain Text" bug class).+func TestPDFParseMethodTablesAgree(t *testing.T) {+ const (+ unsetSentinel = ""+ dispatcherHandled = "monkeyocrv2"+ )+ reachable := make(map[string]bool, len(pdfParseMethodSpellings))+ for spelling, canonical := range pdfParseMethodSpellings {+ if canonical == unsetSentinel {+ t.Errorf("spelling %q maps to the unset sentinel %q; no spelling may mean \"unset\"", spelling, unsetSentinel)+ continue+ }+ reachable[canonical] = true+ if canonical == dispatcherHandled {+ continue+ }+ if _, ok := supportedPDFParseMethods[canonical]; !ok {+ t.Errorf("spelling %q maps to canonical %q, which supportedPDFParseMethods cannot execute; add %q to that set, or handle it in the ingestion dispatcher like %q",+ spelling, canonical, canonical, dispatcherHandled)+ }+ }+ for supported := range supportedPDFParseMethods {+ if supported == unsetSentinel || reachable[supported] {+ continue+ }+ t.Errorf("supportedPDFParseMethods contains %q that no spelling maps to; without a spelling, Check() and the vision dispatcher route it to the VLM path", supported)+ }+ if !IsPDFParseMethod(dispatcherHandled) {+ t.Errorf("IsPDFParseMethod(%q) = false; the ingestion dispatcher relies on it being recognized (Check() must not demand lang for it)", dispatcherHandled)+ }+}+type mockPDFEngineForCommonTest struct {closed bool}