RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
fix(parser): pass do_ocr and pdf_backend to Docling Serve (#20529)
Bilal Atique2b8a0fe19570
diff --git a/docs/faq.mdx b/docs/faq.mdxindex 2f87e52762..d7d342596b 100644--- a/docs/faq.mdx+++ b/docs/faq.mdx@@ -573,6 +573,13 @@ DOCLING_API_KEY=your-api-keyThe Go parser sends PDFs to Docling Serve using `/v1/convert/source` and also tries `/v1alpha/convert/source` for older servers. It sends `DOCLING_API_KEY` as a Bearer token. If neither the parser configuration nor the environment variable supplies a URL, Docling parsing fails with a configuration error. Leave the API key unset when the service does not require authentication.+To skip OCR on PDFs that already contain text, set `docling_do_ocr` to `false` in the parser configuration or provide `DOCLING_DO_OCR=false`. Choose the PDF backend with `docling_pdf_backend` or `DOCLING_PDF_BACKEND`. RAGFlow passes these to Docling Serve as `do_ocr` and `pdf_backend` in every convert request. Leave them unset to keep Docling Serve's defaults.++```bash+DOCLING_DO_OCR=false+DOCLING_PDF_BACKEND=pypdfium2+```+---### How do I use PaddleOCR for document parsing?diff --git a/internal/common/environments.go b/internal/common/environments.goindex f3ca53cc40..420897d828 100644--- a/internal/common/environments.go+++ b/internal/common/environments.go@@ -191,6 +191,8 @@ const (EnvInfinityURI = "INFINITY_URI"EnvDoclingServerURL = "DOCLING_SERVER_URL"EnvDoclingAPIKey = "DOCLING_API_KEY"+ EnvDoclingDoOCR = "DOCLING_DO_OCR"+ EnvDoclingPDFBackend = "DOCLING_PDF_BACKEND"EnvMineruAPIServer = "MINERU_APISERVER"EnvMineruAPIKey = "MINERU_API_KEY"EnvMineruBackend = "MINERU_BACKEND"diff --git a/internal/parser/parser/pdf_parser_common.go b/internal/parser/parser/pdf_parser_common.goindex 5174501285..bbc3d94221 100644--- a/internal/parser/parser/pdf_parser_common.go+++ b/internal/parser/parser/pdf_parser_common.go@@ -88,6 +88,8 @@ type PDFParser struct {PaddleOCRAlgorithm stringDoclingServerURL stringDoclingAPIKey string+ DoclingDoOCR *bool+ DoclingPDFBackend stringOpenDataLoaderAPIServer stringOpenDataLoaderAPIKey stringOpenDataLoaderTimeout int@@ -196,6 +198,12 @@ func (p *PDFParser) ConfigureFromSetup(setup map[string]any) {if v, ok := setup["docling_api_key"].(string); ok {p.DoclingAPIKey = v}+ if v, ok := setup["docling_do_ocr"].(bool); ok {+ p.DoclingDoOCR = &v+ }+ if v, ok := setup["docling_pdf_backend"].(string); ok && v != "" {+ p.DoclingPDFBackend = v+ }if v, ok := setup["opendataloader_apiserver"].(string); ok && v != "" {p.OpenDataLoaderAPIServer = v}diff --git a/internal/parser/parser/pdf_parser_docling.go b/internal/parser/parser/pdf_parser_docling.goindex e3fd1e50d6..ac49389613 100644--- a/internal/parser/parser/pdf_parser_docling.go+++ b/internal/parser/parser/pdf_parser_docling.go@@ -6,7 +6,9 @@ import ("encoding/json""fmt""io"+ "maps""ragflow/internal/common"+ "strconv""strings"models "ragflow/internal/entity/models"@@ -58,6 +60,7 @@ func parsePDFWithDocling(ctx context.Context, filename string, data []byte, parsauth = "Bearer " + apiKey}encoded := base64.StdEncoding.EncodeToString(data)+ convertOptions := doclingConvertOptions(parser)payloads := []struct {endpoint stringbody func() map[string]any@@ -66,22 +69,22 @@ func parsePDFWithDocling(ctx context.Context, filename string, data []byte, pars{endpoint: "/v1/convert/source",chunked: true,- body: func() map[string]any { return doclingChunkedPayload(filename, encoded, false) },+ body: func() map[string]any { return doclingChunkedPayload(filename, encoded, false, convertOptions) },},{endpoint: "/v1alpha/convert/source",chunked: true,- body: func() map[string]any { return doclingChunkedPayload(filename, encoded, true) },+ body: func() map[string]any { return doclingChunkedPayload(filename, encoded, true, convertOptions) },},{endpoint: "/v1/convert/source",chunked: false,- body: func() map[string]any { return doclingStandardPayload(filename, encoded, false) },+ body: func() map[string]any { return doclingStandardPayload(filename, encoded, false, convertOptions) },},{endpoint: "/v1alpha/convert/source",chunked: false,- body: func() map[string]any { return doclingStandardPayload(filename, encoded, true) },+ body: func() map[string]any { return doclingStandardPayload(filename, encoded, true, convertOptions) },},}@@ -127,9 +130,30 @@ func parsePDFWithDocling(ctx context.Context, filename string, data []byte, parsreturn ParseResult{Err: fmt.Errorf("parser: Docling convert: %w", lastErr)}}-func doclingStandardPayload(filename string, encoded string, alpha bool) map[string]any {+// doclingConvertOptions collects the Docling Serve conversion options set in the+// parser setup, falling back to the environment. Unset options are left out so+// Docling Serve keeps its own defaults.+func doclingConvertOptions(parser *PDFParser) map[string]any {+ options := map[string]any{}+ if parser.DoclingDoOCR != nil {+ options["do_ocr"] = *parser.DoclingDoOCR+ } else if v, err := strconv.ParseBool(strings.TrimSpace(common.GetEnv(common.EnvDoclingDoOCR))); err == nil {+ options["do_ocr"] = v+ }+ backend := strings.TrimSpace(parser.DoclingPDFBackend)+ if backend == "" {+ backend = strings.TrimSpace(common.GetEnv(common.EnvDoclingPDFBackend))+ }+ if backend != "" {+ options["pdf_backend"] = backend+ }+ return options+}++func doclingStandardPayload(filename string, encoded string, alpha bool, convertOptions map[string]any) map[string]any {source := map[string]any{"filename": filename, "base64_string": encoded}options := map[string]any{"from_formats": []string{"pdf"}, "to_formats": []string{"json", "md", "text"}}+ maps.Copy(options, convertOptions)if alpha {return map[string]any{"options": options,@@ -143,9 +167,9 @@ func doclingStandardPayload(filename string, encoded string, alpha bool) map[str}}-func doclingChunkedPayload(filename string, encoded string, alpha bool) map[string]any {- payload := doclingStandardPayload(filename, encoded, alpha)- payload["options"] = map[string]any{+func doclingChunkedPayload(filename string, encoded string, alpha bool, convertOptions map[string]any) map[string]any {+ payload := doclingStandardPayload(filename, encoded, alpha, convertOptions)+ options := map[string]any{"from_formats": []string{"pdf"},"to_formats": []string{"json", "md", "text"},"do_chunking": true,@@ -155,6 +179,8 @@ func doclingChunkedPayload(filename string, encoded string, alpha bool) map[stri"tokenizer": "sentencepiece",},}+ maps.Copy(options, convertOptions)+ payload["options"] = optionsreturn payload}diff --git a/internal/parser/parser/pdf_parser_docling_test.go b/internal/parser/parser/pdf_parser_docling_test.goindex a6fa3bdd1c..0115fe2a4d 100644--- a/internal/parser/parser/pdf_parser_docling_test.go+++ b/internal/parser/parser/pdf_parser_docling_test.go@@ -3,6 +3,7 @@ package parserimport ("encoding/base64""encoding/json"+ "maps""net/http""net/http/httptest""strings"@@ -164,3 +165,90 @@ func TestPDFParser_ParseWithResult_DoclingRequiresServerURL(t *testing.T) {t.Fatalf("error = %q, want docling_server_url context", res.Err.Error())}}++func TestPDFParser_ParseWithResult_DoclingConvertOptions(t *testing.T) {+ tests := []struct {+ name string+ setup map[string]any+ env map[string]string+ // A nil value means the option must be left out of the request.+ wantOptions map[string]any+ }{+ {+ name: "unset options keep the server defaults",+ wantOptions: map[string]any{"do_ocr": nil, "pdf_backend": nil},+ },+ {+ name: "parser setup",+ setup: map[string]any{"docling_do_ocr": false, "docling_pdf_backend": "pypdfium2"},+ wantOptions: map[string]any{"do_ocr": false, "pdf_backend": "pypdfium2"},+ },+ {+ name: "environment",+ env: map[string]string{"DOCLING_DO_OCR": "false", "DOCLING_PDF_BACKEND": "pypdfium2"},+ wantOptions: map[string]any{"do_ocr": false, "pdf_backend": "pypdfium2"},+ },+ {+ name: "parser setup wins over the environment",+ setup: map[string]any{"docling_do_ocr": true, "docling_pdf_backend": "pypdfium2"},+ env: map[string]string{"DOCLING_DO_OCR": "false", "DOCLING_PDF_BACKEND": "dlparse_v4"},+ wantOptions: map[string]any{"do_ocr": true, "pdf_backend": "pypdfium2"},+ },+ }++ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ withSSRFBypass(t)+ t.Setenv("DOCLING_DO_OCR", tt.env["DOCLING_DO_OCR"])+ t.Setenv("DOCLING_PDF_BACKEND", tt.env["DOCLING_PDF_BACKEND"])+ var requestCount atomic.Int32++ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ current := requestCount.Add(1)+ var body map[string]any+ if err := json.NewDecoder(r.Body).Decode(&body); err != nil {+ t.Errorf("request %d: Decode: %v", current, err)+ return+ }+ options, _ := body["options"].(map[string]any)+ for key, want := range tt.wantOptions {+ got, exists := options[key]+ if want == nil {+ if exists {+ t.Errorf("request %d: options[%q] = %#v, want it left out", current, key, got)+ }+ continue+ }+ if got != want {+ t.Errorf("request %d: options[%q] = %#v, want %#v", current, key, got, want)+ }+ }+ // Reject both chunked requests so the standard payload is checked too.+ if current < 3 {+ w.WriteHeader(http.StatusUnprocessableEntity)+ return+ }+ w.Header().Set("Content-Type", "application/json")+ _, _ = w.Write([]byte(`{"document":{"md_content":"# Docling Title\n\nDocling body.\n"}}`))+ }))+ defer server.Close()++ setup := map[string]any{+ "parse_method": "Docling",+ "output_format": "markdown",+ "docling_server_url": server.URL,+ }+ maps.Copy(setup, tt.setup)+ pdf := NewPDFParser()+ pdf.ConfigureFromSetup(setup)++ res := pdf.ParseWithResult(t.Context(), "sample.pdf", []byte("%PDF-1.4\nmock"))+ if res.Err != nil {+ t.Fatalf("ParseWithResult: %v", res.Err)+ }+ if got, want := requestCount.Load(), int32(3); got != want {+ t.Fatalf("requestCount = %d, want %d", got, want)+ }+ })+ }+}