Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

fix(parser): apply the OpenDataLoader request timeout (#21527)

CCLUCKYaea8be4777d6
diff --git a/internal/parser/parser/pdf_parser_common.go b/internal/parser/parser/pdf_parser_common.go
index f09a34d638..773ed4f38b 100644
--- a/internal/parser/parser/pdf_parser_common.go
+++ b/internal/parser/parser/pdf_parser_common.go
@@ -120,7 +120,7 @@ func NewPDFParser() *PDFParser {
ParseMethod: "deepdoc",
MinerUBackend: "",
MinerUPollTimeout: minerUPollTimeout,
- OpenDataLoaderTimeout: 600,
+ OpenDataLoaderTimeout: openDataLoaderTimeoutSeconds,
SoMarkBaseURL: "https://somark.cn/api/v1",
SoMarkImageFormat: "url",
SoMarkFormulaFormat: "latex",
diff --git a/internal/parser/parser/pdf_parser_opendataloader.go b/internal/parser/parser/pdf_parser_opendataloader.go
index daa1084b41..6e5708fdd4 100644
--- a/internal/parser/parser/pdf_parser_opendataloader.go
+++ b/internal/parser/parser/pdf_parser_opendataloader.go
@@ -12,8 +12,14 @@ import (
"ragflow/internal/deepdoc/parser/pdf/table"
"sort"
"strings"
+ "time"
)
+// openDataLoaderTimeoutSeconds bounds the single submit call. OpenDataLoader
+// parses synchronously, so a stalled server otherwise holds an ingestion worker
+// open until the transport's own 20-minute header timeout.
+const openDataLoaderTimeoutSeconds = 600
+
// parsePDFWithOpenDataLoader normalizes the selected server's JSON or Markdown
// output. Explicit tenant configuration never borrows a global endpoint or key.
func parsePDFWithOpenDataLoader(ctx context.Context, filename string, data []byte, parser *PDFParser) ParseResult {
@@ -36,7 +42,15 @@ func parsePDFWithOpenDataLoader(ctx context.Context, filename string, data []byt
if err != nil {
return ParseResult{Err: err}
}
- req, err := http.NewRequestWithContext(context.Background(), http.MethodPost, strings.TrimRight(baseURL, "/")+"/file_parse", bodyReader)
+ timeout := time.Duration(parser.OpenDataLoaderTimeout) * time.Second
+ if timeout <= 0 {
+ timeout = openDataLoaderTimeoutSeconds * time.Second
+ }
+ // Derive from the pipeline context so a cancelled or expiring run stops
+ // waiting on the server, and cap it by the configured timeout.
+ reqCtx, cancel := context.WithTimeout(ctx, timeout)
+ defer cancel()
+ req, err := http.NewRequestWithContext(reqCtx, http.MethodPost, strings.TrimRight(baseURL, "/")+"/file_parse", bodyReader)
if err != nil {
return ParseResult{Err: fmt.Errorf("parser: OpenDataLoader request: %w", err)}
}
diff --git a/internal/parser/parser/pdf_parser_opendataloader_test.go b/internal/parser/parser/pdf_parser_opendataloader_test.go
index e7c51693ce..51323dc633 100644
--- a/internal/parser/parser/pdf_parser_opendataloader_test.go
+++ b/internal/parser/parser/pdf_parser_opendataloader_test.go
@@ -1,12 +1,15 @@
package parser
import (
+ "context"
"encoding/json"
+ "errors"
"io"
"net/http"
"net/http/httptest"
"strings"
"testing"
+ "time"
"ragflow/internal/entity"
)
@@ -128,6 +131,73 @@ func TestPDFParser_ParseWithResult_OpenDataLoaderRequiresAPIServer(t *testing.T)
}
}
+// opendataloader_timeout is the only bound a synchronous OpenDataLoader parse
+// gets, so a stalled server has to be cut off at the configured value instead
+// of at the transport's 20-minute header timeout.
+func TestPDFParser_ParseWithResult_OpenDataLoaderAppliesConfiguredTimeout(t *testing.T) {
+ withSSRFBypass(t)
+ server := openDataLoaderStallServer(t, 3*time.Second)
+ defer server.Close()
+
+ pdf := NewPDFParser()
+ pdf.ConfigureFromSetup(map[string]any{
+ "parse_method": "OpenDataLoader",
+ "opendataloader_apiserver": server.URL,
+ "opendataloader_timeout": 1,
+ })
+
+ start := time.Now()
+ res := pdf.ParseWithResult(t.Context(), "sample.pdf", []byte("%PDF-1.4\nmock"))
+ elapsed := time.Since(start)
+ if !errors.Is(res.Err, context.DeadlineExceeded) {
+ t.Fatalf("error = %v, want context.DeadlineExceeded", res.Err)
+ }
+ // The configured value is in seconds: neither firing early (milliseconds)
+ // nor never firing (the setting ignored) is a pass.
+ if elapsed < 900*time.Millisecond || elapsed >= 2500*time.Millisecond {
+ t.Fatalf("elapsed = %s, want the request cut off at the configured 1s", elapsed)
+ }
+}
+
+// A pipeline cancellation must reach the in-flight request even when the
+// parser carries no configured timeout.
+func TestPDFParser_ParseWithResult_OpenDataLoaderAbortsOnCancelledContext(t *testing.T) {
+ withSSRFBypass(t)
+ server := openDataLoaderStallServer(t, 3*time.Second)
+ defer server.Close()
+
+ pdf := NewPDFParser()
+ pdf.ConfigureFromSetup(map[string]any{
+ "parse_method": "OpenDataLoader",
+ "opendataloader_apiserver": server.URL,
+ })
+ pdf.OpenDataLoaderTimeout = 0
+
+ ctx, cancel := context.WithCancel(t.Context())
+ cancel()
+ res := pdf.ParseWithResult(ctx, "sample.pdf", []byte("%PDF-1.4\nmock"))
+ if !errors.Is(res.Err, context.Canceled) {
+ t.Fatalf("error = %v, want context.Canceled", res.Err)
+ }
+}
+
+// openDataLoaderStallServer answers /file_parse only after stall has passed,
+// which stands in for an OpenDataLoader server that is still parsing.
+func openDataLoaderStallServer(t *testing.T, stall time.Duration) *httptest.Server {
+ t.Helper()
+ return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ if r.URL.Path != "/file_parse" {
+ http.NotFound(w, r)
+ return
+ }
+ select {
+ case <-time.After(stall):
+ _, _ = w.Write([]byte(`{"json_doc":null,"md_text":"late"}`))
+ case <-r.Context().Done():
+ }
+ }))
+}
+
func TestOpenDataLoaderItems_TableCellsFallback(t *testing.T) {
root := map[string]any{
"type": "table",