RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
fix(parser): apply the OpenDataLoader request timeout (#21527)
CCLUCKYaea8be4777d6
diff --git a/internal/parser/parser/pdf_parser_common.go b/internal/parser/parser/pdf_parser_common.goindex f09a34d638..773ed4f38b 100644--- a/internal/parser/parser/pdf_parser_common.go+++ b/internal/parser/parser/pdf_parser_common.go@@ -120,7 +120,7 @@ func NewPDFParser() *PDFParser {ParseMethod: "deepdoc",MinerUBackend: "",MinerUPollTimeout: minerUPollTimeout,- OpenDataLoaderTimeout: 600,+ OpenDataLoaderTimeout: openDataLoaderTimeoutSeconds,SoMarkBaseURL: "https://somark.cn/api/v1",SoMarkImageFormat: "url",SoMarkFormulaFormat: "latex",diff --git a/internal/parser/parser/pdf_parser_opendataloader.go b/internal/parser/parser/pdf_parser_opendataloader.goindex daa1084b41..6e5708fdd4 100644--- a/internal/parser/parser/pdf_parser_opendataloader.go+++ b/internal/parser/parser/pdf_parser_opendataloader.go@@ -12,8 +12,14 @@ import ("ragflow/internal/deepdoc/parser/pdf/table""sort""strings"+ "time")+// openDataLoaderTimeoutSeconds bounds the single submit call. OpenDataLoader+// parses synchronously, so a stalled server otherwise holds an ingestion worker+// open until the transport's own 20-minute header timeout.+const openDataLoaderTimeoutSeconds = 600+// parsePDFWithOpenDataLoader normalizes the selected server's JSON or Markdown// output. Explicit tenant configuration never borrows a global endpoint or key.func parsePDFWithOpenDataLoader(ctx context.Context, filename string, data []byte, parser *PDFParser) ParseResult {@@ -36,7 +42,15 @@ func parsePDFWithOpenDataLoader(ctx context.Context, filename string, data []bytif err != nil {return ParseResult{Err: err}}- req, err := http.NewRequestWithContext(context.Background(), http.MethodPost, strings.TrimRight(baseURL, "/")+"/file_parse", bodyReader)+ timeout := time.Duration(parser.OpenDataLoaderTimeout) * time.Second+ if timeout <= 0 {+ timeout = openDataLoaderTimeoutSeconds * time.Second+ }+ // Derive from the pipeline context so a cancelled or expiring run stops+ // waiting on the server, and cap it by the configured timeout.+ reqCtx, cancel := context.WithTimeout(ctx, timeout)+ defer cancel()+ req, err := http.NewRequestWithContext(reqCtx, http.MethodPost, strings.TrimRight(baseURL, "/")+"/file_parse", bodyReader)if err != nil {return ParseResult{Err: fmt.Errorf("parser: OpenDataLoader request: %w", err)}}diff --git a/internal/parser/parser/pdf_parser_opendataloader_test.go b/internal/parser/parser/pdf_parser_opendataloader_test.goindex e7c51693ce..51323dc633 100644--- a/internal/parser/parser/pdf_parser_opendataloader_test.go+++ b/internal/parser/parser/pdf_parser_opendataloader_test.go@@ -1,12 +1,15 @@package parserimport (+ "context""encoding/json"+ "errors""io""net/http""net/http/httptest""strings""testing"+ "time""ragflow/internal/entity")@@ -128,6 +131,73 @@ func TestPDFParser_ParseWithResult_OpenDataLoaderRequiresAPIServer(t *testing.T)}}+// opendataloader_timeout is the only bound a synchronous OpenDataLoader parse+// gets, so a stalled server has to be cut off at the configured value instead+// of at the transport's 20-minute header timeout.+func TestPDFParser_ParseWithResult_OpenDataLoaderAppliesConfiguredTimeout(t *testing.T) {+ withSSRFBypass(t)+ server := openDataLoaderStallServer(t, 3*time.Second)+ defer server.Close()++ pdf := NewPDFParser()+ pdf.ConfigureFromSetup(map[string]any{+ "parse_method": "OpenDataLoader",+ "opendataloader_apiserver": server.URL,+ "opendataloader_timeout": 1,+ })++ start := time.Now()+ res := pdf.ParseWithResult(t.Context(), "sample.pdf", []byte("%PDF-1.4\nmock"))+ elapsed := time.Since(start)+ if !errors.Is(res.Err, context.DeadlineExceeded) {+ t.Fatalf("error = %v, want context.DeadlineExceeded", res.Err)+ }+ // The configured value is in seconds: neither firing early (milliseconds)+ // nor never firing (the setting ignored) is a pass.+ if elapsed < 900*time.Millisecond || elapsed >= 2500*time.Millisecond {+ t.Fatalf("elapsed = %s, want the request cut off at the configured 1s", elapsed)+ }+}++// A pipeline cancellation must reach the in-flight request even when the+// parser carries no configured timeout.+func TestPDFParser_ParseWithResult_OpenDataLoaderAbortsOnCancelledContext(t *testing.T) {+ withSSRFBypass(t)+ server := openDataLoaderStallServer(t, 3*time.Second)+ defer server.Close()++ pdf := NewPDFParser()+ pdf.ConfigureFromSetup(map[string]any{+ "parse_method": "OpenDataLoader",+ "opendataloader_apiserver": server.URL,+ })+ pdf.OpenDataLoaderTimeout = 0++ ctx, cancel := context.WithCancel(t.Context())+ cancel()+ res := pdf.ParseWithResult(ctx, "sample.pdf", []byte("%PDF-1.4\nmock"))+ if !errors.Is(res.Err, context.Canceled) {+ t.Fatalf("error = %v, want context.Canceled", res.Err)+ }+}++// openDataLoaderStallServer answers /file_parse only after stall has passed,+// which stands in for an OpenDataLoader server that is still parsing.+func openDataLoaderStallServer(t *testing.T, stall time.Duration) *httptest.Server {+ t.Helper()+ return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ if r.URL.Path != "/file_parse" {+ http.NotFound(w, r)+ return+ }+ select {+ case <-time.After(stall):+ _, _ = w.Write([]byte(`{"json_doc":null,"md_text":"late"}`))+ case <-r.Context().Done():+ }+ }))+}+func TestOpenDataLoaderItems_TableCellsFallback(t *testing.T) {root := map[string]any{"type": "table",