Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

fix(parser): route UI "Plain Text" PDF method to the plain-text parser (#19979)

jay7772183c47d025182
diff --git a/internal/ingestion/component/parser.go b/internal/ingestion/component/parser.go
index 353ba66b6d..a99ceef5a3 100644
--- a/internal/ingestion/component/parser.go
+++ b/internal/ingestion/component/parser.go
@@ -221,14 +221,10 @@ func (c *ParserComponent) Check() error {
if pm == "" {
return errors.New("parse method abnormal. does not support empty value")
}
- pmLower := strings.ToLower(pm)
- pdfWhitelist := []string{
- "deepdoc", "plain_text", "mineru", "monkeyocrv2", "docling",
- "opendataloader", "tcadp parser", "paddleocr", "somark",
- }
- if !containsString(pdfWhitelist, pmLower) {
- // Non-whitelist parse_method is treated as a VLM method,
- // which requires lang (Python parser.py:257-258).
+ if !parser.IsPDFParseMethod(pm) {
+ // A parse_method outside the known vocabulary is treated as a
+ // VLM model reference, which requires lang (Python
+ // parser.py:257-258).
if lang, _ := pdf["lang"].(string); lang == "" {
return errors.New("PDF VLM language does not support empty value")
}
@@ -247,18 +243,6 @@ func (c *ParserComponent) Check() error {
return nil
}
-// containsString reports whether s is in list. Used by Check() for
-// whitelist membership tests; kept unexported and local to this file
-// to avoid polluting the package namespace.
-func containsString(list []string, s string) bool {
- for _, v := range list {
- if v == s {
- return true
- }
- }
- return false
-}
-
func defaultSetups() map[string]schema.ParserSetup {
return map[string]schema.ParserSetup{
"pdf": {
diff --git a/internal/ingestion/component/parser_check_test.go b/internal/ingestion/component/parser_check_test.go
index 63d526848f..f3f4b8c433 100644
--- a/internal/ingestion/component/parser_check_test.go
+++ b/internal/ingestion/component/parser_check_test.go
@@ -54,6 +54,14 @@ func TestParserComponent_Check(t *testing.T) {
name: "pdf: plain_text (whitelist, case-insensitive) without lang → pass",
setups: map[string]schema.ParserSetup{"pdf": {"parse_method": "PLAIN_TEXT"}},
},
+ {
+ name: "pdf: Plain Text (UI spelling of the plain-text option) without lang → pass",
+ setups: map[string]schema.ParserSetup{"pdf": {"parse_method": "Plain Text"}},
+ },
+ {
+ name: "pdf: plaintext (UI spelling, no space) without lang → pass",
+ setups: map[string]schema.ParserSetup{"pdf": {"parse_method": "plaintext"}},
+ },
{
name: "pdf: tcadp parser (whitelist with space) without lang → pass",
setups: map[string]schema.ParserSetup{"pdf": {"parse_method": "tcadp parser"}},
diff --git a/internal/ingestion/component/parser_dispatch_test.go b/internal/ingestion/component/parser_dispatch_test.go
index 4e92430dba..251fa1dc6c 100644
--- a/internal/ingestion/component/parser_dispatch_test.go
+++ b/internal/ingestion/component/parser_dispatch_test.go
@@ -475,6 +475,11 @@ func TestDispatch_PDFLegacyMarkdownConfigurationEmitsJSON(t *testing.T) {
}
}
+// TestDispatch_PDFPlainText_UsesConfiguredBackend pins the plain-text
+// dispatch for every spelling the dataset configuration UI can persist.
+// The UI option is labelled "Naive" and stores "Plain Text"; treating that
+// spelling as a custom VLM model name made the run fail with
+// `provider name missing in model name: Plain Text`.
func TestDispatch_PDFPlainText_UsesConfiguredBackend(t *testing.T) {
path := filepath.Join("..", "..", "..", "test", "benchmark", "test_docs", "Doc1.pdf")
data, err := os.ReadFile(path)
@@ -482,25 +487,29 @@ func TestDispatch_PDFPlainText_UsesConfiguredBackend(t *testing.T) {
t.Fatalf("ReadFile(%s): %v", path, err)
}
- setups := defaultSetups()
- setups["pdf"]["parse_method"] = "plain_text"
- setups["pdf"]["output_format"] = "json"
- c := &ParserComponent{setups: setups}
+ for _, method := range []string{"plain_text", "plaintext", "Plain Text"} {
+ t.Run(method, func(t *testing.T) {
+ setups := defaultSetups()
+ setups["pdf"]["parse_method"] = method
+ setups["pdf"]["output_format"] = "json"
+ c := &ParserComponent{setups: setups}
- out, err := c.Invoke(t.Context(), nil, map[string]any{
- "binary": data,
- "file_type": "pdf",
- "name": "Doc1.pdf",
- })
- if err != nil {
- t.Fatalf("Invoke: %v", err)
- }
- jsonItems, ok := out["json"].([]map[string]any)
- if !ok || len(jsonItems) == 0 {
- t.Fatalf("json payload missing or empty: %T", out["json"])
- }
- if got, _ := jsonItems[0]["text"].(string); strings.TrimSpace(got) == "" {
- t.Fatalf("json first item text = %q, want non-empty", got)
+ out, err := c.Invoke(t.Context(), nil, map[string]any{
+ "binary": data,
+ "file_type": "pdf",
+ "name": "Doc1.pdf",
+ })
+ if err != nil {
+ t.Fatalf("Invoke: %v", err)
+ }
+ jsonItems, ok := out["json"].([]map[string]any)
+ if !ok || len(jsonItems) == 0 {
+ t.Fatalf("json payload missing or empty: %T", out["json"])
+ }
+ if got, _ := jsonItems[0]["text"].(string); strings.TrimSpace(got) == "" {
+ t.Fatalf("json first item text = %q, want non-empty", got)
+ }
+ })
}
}
diff --git a/internal/ingestion/component/pdf_vision_dispatch.go b/internal/ingestion/component/pdf_vision_dispatch.go
index 76f85e833d..0fc808f62e 100644
--- a/internal/ingestion/component/pdf_vision_dispatch.go
+++ b/internal/ingestion/component/pdf_vision_dispatch.go
@@ -91,7 +91,7 @@ func maybeDispatchPDFVision(
strings.HasPrefix(layoutLower, "monkeyocrv2") ||
strings.Contains(layoutLower, "@monkeyocrv2")
isMonkeyByUUID := false
- if !isMonkeyMatch && tenantID != "" && strings.TrimSpace(monkeySelector) != "" && !isNamedPDFParseMethod(monkeySelector) {
+ if !isMonkeyMatch && tenantID != "" && strings.TrimSpace(monkeySelector) != "" && !parser.IsPDFParseMethod(monkeySelector) {
isMonkeyByUUID = isMonkeyOCRv2LayoutModelID(ctx, db, tenantID, monkeySelector)
}
if isMonkeyMatch || isMonkeyByUUID {
@@ -120,7 +120,7 @@ func maybeDispatchPDFVision(
}
isMinerUByUUID := false
if !isMinerUMatch && strings.TrimSpace(minerUSelector) != "" &&
- !isNamedPDFParseMethod(minerUSelector) {
+ !parser.IsPDFParseMethod(minerUSelector) {
isMinerUByUUID = isMinerULayoutModelID(ctx, db, tenantID, minerUSelector)
}
if isMinerUMatch || isMinerUByUUID {
@@ -165,7 +165,7 @@ func maybeDispatchPDFVision(
}
isPaddleOCRByUUID := false
if !isPaddleOCRMatch && strings.TrimSpace(paddleOCRSelector) != "" &&
- !isNamedPDFParseMethod(paddleOCRSelector) {
+ !parser.IsPDFParseMethod(paddleOCRSelector) {
isPaddleOCRByUUID = isPaddleOCRLayoutModelID(ctx, db, tenantID, paddleOCRSelector)
}
if isPaddleOCRMatch || isPaddleOCRByUUID {
@@ -848,48 +848,19 @@ func resolvePDFVisionModelID(setup schema.ParserSetup) (string, bool) {
}
if raw, ok := setup["parse_method"].(string); ok {
method := strings.TrimSpace(raw)
- if method != "" && !isNamedPDFParseMethod(method) {
+ if method != "" && !parser.IsPDFParseMethod(method) {
return method, true
}
}
if raw, ok := setup["layout_recognizer"].(string); ok {
method := strings.TrimSpace(raw)
- if method == "" || strings.EqualFold(method, "plain text") || strings.EqualFold(method, "plaintext") {
- return "", false
- }
- if !isNamedPDFParseMethod(method) {
+ if method != "" && !parser.IsPDFParseMethod(method) {
return method, true
}
}
return "", false
}
-// isNamedPDFParseMethod reports whether raw is a recognized named PDF
-// parse method (as opposed to a CustomVLM model name). Its membership set
-// MUST stay aligned with the PDF whitelist enforced by
-// (*ParserComponent).Check() (parser.go:200-203):
-//
-// deepdoc, plain_text, mineru, docling,
-// opendataloader, tcadp parser, paddleocr, somark
-//
-// A parse_method that Check() rejects must not be treated as a named method
-// here, otherwise it silently falls through to the CustomVLM vision path
-// instead of failing fast at construction.
-//
-// Note: "@"-suffixed spellings such as "foo@mineru" are layout_recognizer
-// selectors, not parse_method values. Check() rejects them as parse_method,
-// and the MinerU layout branch is resolved from the layout_recognizer field
-// separately (pdf_vision_dispatch.go:62-68), so they must NOT be recognized
-// here.
-func isNamedPDFParseMethod(raw string) bool {
- method := strings.ToLower(strings.TrimSpace(raw))
- switch method {
- case "deepdoc", "plain_text", "mineru", "monkeyocrv2", "docling", "opendataloader", "tcadp parser", "paddleocr", "somark":
- return true
- }
- return false
-}
-
func dispatchPDFVision(
ctx context.Context,
db *gorm.DB,
diff --git a/internal/ingestion/component/pdf_vision_dispatch_test.go b/internal/ingestion/component/pdf_vision_dispatch_test.go
index 899b3c166b..80cd0853de 100644
--- a/internal/ingestion/component/pdf_vision_dispatch_test.go
+++ b/internal/ingestion/component/pdf_vision_dispatch_test.go
@@ -262,72 +262,3 @@ func TestDispatchPaddleOCRPdfEmptyTextFails(t *testing.T) {
t.Errorf("expected zero-value result on error, got %+v", res)
}
}
-
-// TestIsNamedPDFParseMethodWhitelistAligned verifies that the runtime
-// "named parse_method" classifier agrees with (*ParserComponent).Check()'s
-// PDF whitelist (parser.go:200-203):
-//
-// deepdoc, plain_text, mineru, docling,
-// opendataloader, tcadp parser, paddleocr, somark
-//
-// Diff 2.10: a parse_method that Check() rejects must NOT be treated as a
-// recognized named method by isNamedPDFParseMethod — otherwise it silently
-// falls through to the CustomVLM vision path instead of failing fast at
-// construction (and Python would have rejected it outright).
-func TestIsNamedPDFParseMethodWhitelistAligned(t *testing.T) {
- // Values that MUST be recognized (subset of the Check() whitelist,
- // case-insensitive).
- named := []string{
- "deepdoc", "plain_text", "mineru", "monkeyocrv2", "docling",
- "opendataloader", "tcadp parser", "paddleocr", "somark",
- "DeepDoc", "PLAIN_TEXT", "MinerU", "DocLing",
- "OpenDataLoader", "TCADP Parser", "PaddleOCR", "SoMark",
- }
- for _, v := range named {
- if !isNamedPDFParseMethod(v) {
- t.Errorf("isNamedPDFParseMethod(%q) = false, want true (in Check() whitelist)", v)
- }
- }
-
- // Values that MUST NOT be recognized. These either duplicate the
- // whitelist with non-canonical spelling ("plain text"/"plaintext")
- // or are bare-family abbreviations ("tcadp") that Check() does not
- // accept, so they should be funneled to the CustomVLM path (or fail
- // construction) rather than masquerading as a named method.
- notNamed := []string{
- "plain text", "plaintext", "tcadp",
- "CustomVLM", "some_vlm", "gpt-4o",
- "", " ",
- }
- for _, v := range notNamed {
- if isNamedPDFParseMethod(v) {
- t.Errorf("isNamedPDFParseMethod(%q) = true, want false (not in Check() whitelist)", v)
- }
- }
-}
-
-// TestIsNamedPDFParseMethodLayoutSuffixes verifies that "@"-suffixed
-// layout_recognizer spellings are NOT treated as named parse methods. They
-// are layout_recognizer selectors (resolved separately at
-// pdf_vision_dispatch.go:62-68), and Check() rejects them as parse_method,
-// so they must fall through to the CustomVLM/VLM path — consistent with the
-// (*ParserComponent).Check() whitelist (parser.go:200-203).
-func TestIsNamedPDFParseMethodLayoutSuffixes(t *testing.T) {
- suffixed := []string{
- "foo@mineru", "@mineru",
- "foo@monkeyocrv2", "@monkeyocrv2",
- "foo@paddleocr", "@paddleocr",
- "foo@somark", "@somark",
- "foo@opendataloader", "@opendataloader",
- }
- for _, v := range suffixed {
- if isNamedPDFParseMethod(v) {
- t.Errorf("isNamedPDFParseMethod(%q) = true, want false (layout_recognizer selector, not a named parse_method)", v)
- }
- }
-
- // An unknown suffix is also not a named method.
- if isNamedPDFParseMethod("foo@unknown") {
- t.Errorf("isNamedPDFParseMethod(%q) = true, want false", "foo@unknown")
- }
-}
diff --git a/internal/parser/parser/pdf_parser_common.go b/internal/parser/parser/pdf_parser_common.go
index fcde314cda..e4c50bba48 100644
--- a/internal/parser/parser/pdf_parser_common.go
+++ b/internal/parser/parser/pdf_parser_common.go
@@ -42,6 +42,10 @@ import (
// is compiled behind `//go:build cgo`.
var ErrPDFEngineUnavailable = errors.New("parser: PDF backend unavailable in this build")
+// supportedPDFParseMethods is the set of canonical tokens PDFParser can
+// execute; see pdfParseMethodSpellings for the accepted spellings and
+// TestPDFParseMethodTablesAgree for the two intentional divergences
+// ("" sentinel, dispatcher-handled monkeyocrv2).
var supportedPDFParseMethods = map[string]struct{}{
"": {},
"deepdoc": {},
@@ -274,6 +278,37 @@ func (p *PDFParser) ConfigureFromSetup(setup map[string]any) {
}
}
+// pdfParseMethodSpellings is the single vocabulary of PDF parse methods:
+// every accepted spelling (lower-cased and trimmed) mapped to its canonical
+// token. Consumers must not keep their own copies of this set — tell a parse
+// method apart from a VLM model selector via IsPDFParseMethod, and resolve
+// its canonical token via normalizePDFParseMethod.
+//
+// "plaintext" / "plain text" are the spellings the dataset configuration UI
+// persists for its plain-text option (ParseDocumentType.PlainText), so they
+// are parse methods rather than model names.
+var pdfParseMethodSpellings = map[string]string{
+ "deepdoc": "deepdoc",
+ "plain_text": "plain_text",
+ "plaintext": "plain_text",
+ "plain text": "plain_text",
+ "mineru": "mineru",
+ "monkeyocrv2": "monkeyocrv2",
+ "docling": "docling",
+ "opendataloader": "opendataloader",
+ "tcadp parser": "tcadp",
+ "paddleocr": "paddleocr",
+ "somark": "somark",
+}
+
+// IsPDFParseMethod reports whether raw names a PDF parse method rather than
+// a VLM model selector. "@"-suffixed spellings such as "foo@mineru" are
+// layout_recognizer selectors resolved separately, so they report false.
+func IsPDFParseMethod(raw string) bool {
+ _, ok := pdfParseMethodSpellings[strings.ToLower(strings.TrimSpace(raw))]
+ return ok
+}
+
func normalizePDFParseMethod(raw string) string {
method := strings.ToLower(strings.TrimSpace(raw))
switch {
@@ -286,11 +321,8 @@ func normalizePDFParseMethod(raw string) string {
case strings.HasSuffix(method, "@opendataloader"):
return "opendataloader"
}
- switch method {
- case "plaintext":
- return "plain_text"
- case "tcadp parser":
- return "tcadp"
+ if canonical, ok := pdfParseMethodSpellings[method]; ok {
+ return canonical
}
return method
}
diff --git a/internal/parser/parser/pdf_parser_common_test.go b/internal/parser/parser/pdf_parser_common_test.go
index be57c5674b..46358f7285 100644
--- a/internal/parser/parser/pdf_parser_common_test.go
+++ b/internal/parser/parser/pdf_parser_common_test.go
@@ -474,6 +474,130 @@ func TestPDFParser_ValidateParseMethod(t *testing.T) {
}
}
+// TestNormalizePDFParseMethod_PlainTextSpellings pins the plain-text
+// spellings the dataset configuration UI can persist. The UI option is
+// labelled "Naive" and stores "Plain Text", so both the spaced and
+// spaceless spellings must land on the canonical "plain_text" method
+// instead of being rejected as unsupported (or, upstream, mistaken for a
+// vision model name).
+func TestNormalizePDFParseMethod_PlainTextSpellings(t *testing.T) {
+ cases := []struct{ in, want string }{
+ {"plain_text", "plain_text"},
+ {"Plain_Text", "plain_text"},
+ {"plaintext", "plain_text"},
+ {"PlainText", "plain_text"},
+ {"plain text", "plain_text"},
+ {"Plain Text", "plain_text"},
+ {" Plain Text ", "plain_text"},
+ }
+ for _, c := range cases {
+ if got := normalizePDFParseMethod(c.in); got != c.want {
+ t.Errorf("normalizePDFParseMethod(%q) = %q, want %q", c.in, got, c.want)
+ }
+ }
+}
+
+// TestIsPDFParseMethod pins the vocabulary shared by the construction-time
+// check and the runtime vision dispatcher. "plain text"/"plaintext" are the
+// spellings the dataset configuration UI writes for its plain-text option;
+// treating them as model names breaks PDF parsing with "provider name
+// missing in model name".
+func TestIsPDFParseMethod(t *testing.T) {
+ named := []string{
+ "deepdoc", "plain_text", "plaintext", "plain text", "mineru",
+ "monkeyocrv2", "docling", "opendataloader", "tcadp parser",
+ "paddleocr", "somark",
+ "DeepDoc", "PLAIN_TEXT", "MinerU", "DocLing",
+ "OpenDataLoader", "TCADP Parser", "PaddleOCR", "SoMark",
+ "Plain Text", "PlainText", "PLAIN TEXT",
+ }
+ for _, v := range named {
+ if !IsPDFParseMethod(v) {
+ t.Errorf("IsPDFParseMethod(%q) = false, want true", v)
+ }
+ }
+
+ // Anything else is a VLM model name: the UI never writes the bare
+ // "tcadp" abbreviation, and empty values are rejected separately.
+ notNamed := []string{
+ "tcadp",
+ "CustomVLM", "some_vlm", "gpt-4o",
+ "", " ",
+ }
+ for _, v := range notNamed {
+ if IsPDFParseMethod(v) {
+ t.Errorf("IsPDFParseMethod(%q) = true, want false", v)
+ }
+ }
+}
+
+// TestIsPDFParseMethodLayoutSuffixes pins that "@"-suffixed
+// layout_recognizer selectors are not parse methods; they are resolved from
+// the layout_recognizer field separately.
+func TestIsPDFParseMethodLayoutSuffixes(t *testing.T) {
+ suffixed := []string{
+ "foo@mineru", "@mineru",
+ "foo@monkeyocrv2", "@monkeyocrv2",
+ "foo@paddleocr", "@paddleocr",
+ "foo@somark", "@somark",
+ "foo@opendataloader", "@opendataloader",
+ "foo@unknown",
+ }
+ for _, v := range suffixed {
+ if IsPDFParseMethod(v) {
+ t.Errorf("IsPDFParseMethod(%q) = true, want false (layout_recognizer selector, not a parse_method)", v)
+ }
+ }
+}
+
+// TestPDFParseMethodTablesAgree pins the relationship between the two
+// vocabulary tables in pdf_parser_common.go: every canonical token a
+// spelling maps to must be executable by PDFParser, and every executable
+// token must be reachable from some spelling. The only intended divergences
+// are:
+//
+// - "" (the unset sentinel) is accepted by the parser as the default
+// method but has no spelling, so IsPDFParseMethod("") stays false;
+// - "monkeyocrv2" is dispatched by the ingestion vision dispatcher
+// before PDFParser runs, so it is a recognized method without being
+// executable by the parser switch itself.
+//
+// Any other difference means one table was updated without the other: a
+// spelling whose canonical token is not executable dies in
+// validateParseMethod, and an executable token without a spelling is
+// misclassified as a VLM model name by Check and the dispatcher (the
+// "Plain Text" bug class).
+func TestPDFParseMethodTablesAgree(t *testing.T) {
+ const (
+ unsetSentinel = ""
+ dispatcherHandled = "monkeyocrv2"
+ )
+ reachable := make(map[string]bool, len(pdfParseMethodSpellings))
+ for spelling, canonical := range pdfParseMethodSpellings {
+ if canonical == unsetSentinel {
+ t.Errorf("spelling %q maps to the unset sentinel %q; no spelling may mean \"unset\"", spelling, unsetSentinel)
+ continue
+ }
+ reachable[canonical] = true
+ if canonical == dispatcherHandled {
+ continue
+ }
+ if _, ok := supportedPDFParseMethods[canonical]; !ok {
+ t.Errorf("spelling %q maps to canonical %q, which supportedPDFParseMethods cannot execute; add %q to that set, or handle it in the ingestion dispatcher like %q",
+ spelling, canonical, canonical, dispatcherHandled)
+ }
+ }
+ for supported := range supportedPDFParseMethods {
+ if supported == unsetSentinel || reachable[supported] {
+ continue
+ }
+ t.Errorf("supportedPDFParseMethods contains %q that no spelling maps to; without a spelling, Check() and the vision dispatcher route it to the VLM path", supported)
+ }
+ if !IsPDFParseMethod(dispatcherHandled) {
+ t.Errorf("IsPDFParseMethod(%q) = false; the ingestion dispatcher relies on it being recognized (Check() must not demand lang for it)", dispatcherHandled)
+ }
+}
+
type mockPDFEngineForCommonTest struct {
closed bool
}