RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
feat(ocr): make MinerU parser timeout configurable via environment and setup (#21832)
superwiboyf5d7ab789e69
diff --git a/docs/faq.mdx b/docs/faq.mdxindex e330f01c7d..f4847c25b1 100644--- a/docs/faq.mdx+++ b/docs/faq.mdx@@ -528,12 +528,14 @@ Use the following environment variables when MinerU is not configured directly i| `MINERU_APISERVER` | URL of the MinerU API service | _unset_ | `MINERU_APISERVER=http://your-mineru-server:8886` || `MINERU_API_KEY` | API key, when required by MinerU | _unset_ | `MINERU_API_KEY=your-key` || `MINERU_BACKEND` | MinerU 3.x backend or 4.0 tier | `pipeline` | `MINERU_BACKEND=pipeline\|standard\|vlm-http-client` |+| `MINERU_TIMEOUT` | MinerU request timeout | `1800` (30m) | `MINERU_TIMEOUT=3600` |1. Set `MINERU_APISERVER` to point RAGFlow to your MinerU API server.2. Set `MINERU_API_KEY` if your MinerU service requires authentication.3. Set `MINERU_BACKEND` to specify the backend sent with the parse request.+4. Optionally set `MINERU_TIMEOUT` (seconds or duration string like `1h`) to bound the parse request.-For a custom Parser component, the numeric `mineru_timeout_seconds` setup field controls how long RAGFlow polls for the result. Its default is 30 seconds; zero or a negative value also falls back to 30 seconds.+For native ingestion pipeline component parsing (`dispatchMinerUPDF`), `mineru_timeout` or `mineru_timeout_seconds` in the parser setup overrides this timeout; zero or negative values fall back to the default 30 minutes. Note that the legacy `PDFParser` path (`pdf_parser_mineru.go`) continues to use `mineru_timeout_seconds` with a 30s poll interval default.:::tip NOTEFor other environment variables supported by MinerU itself, see the [MinerU environment variable documentation](https://opendatalab.github.io/MinerU/usage/cli_tools/#environment-variables-description).diff --git a/internal/common/environments.go b/internal/common/environments.goindex 6f84cec141..a218907451 100644--- a/internal/common/environments.go+++ b/internal/common/environments.go@@ -229,6 +229,7 @@ const (EnvMineruAPIKey = "MINERU_API_KEY"EnvMineruBackend = "MINERU_BACKEND"EnvMineruServerURL = "MINERU_SERVER_URL"+ EnvMineruTimeout = "MINERU_TIMEOUT"EnvMonkeyOCRv2ServerURL = "MONKEYOCRV2_SERVER_URL"EnvMonkeyOCRv2Timeout = "MONKEYOCRV2_TIMEOUT"EnvMonkeyOCRAPIServer = "MONKEYOCR_APISERVER"diff --git a/internal/entity/models/mineru_cloud.go b/internal/entity/models/mineru_cloud.goindex 08000dd610..47b374cc92 100644--- a/internal/entity/models/mineru_cloud.go+++ b/internal/entity/models/mineru_cloud.go@@ -87,9 +87,11 @@ func ParseMinerUCloud(ctx context.Context, baseURL, apiKey, filename string, conif ctx == nil {ctx = context.Background()}- var cancel context.CancelFunc- ctx, cancel = context.WithTimeout(ctx, DefaultMinerUV1ParseTimeout)- defer cancel()+ if _, ok := ctx.Deadline(); !ok {+ var cancel context.CancelFunc+ ctx, cancel = context.WithTimeout(ctx, DefaultMinerUV1ParseTimeout)+ defer cancel()+ }token := minerUBearerFromAPIKey(apiKey)client := minerUHTTPClient()diff --git a/internal/entity/models/mineru_cloud_test.go b/internal/entity/models/mineru_cloud_test.goindex 59aad9841a..93760a745c 100644--- a/internal/entity/models/mineru_cloud_test.go+++ b/internal/entity/models/mineru_cloud_test.go@@ -6,6 +6,7 @@ package modelsimport ("context"+ "errors""io""net/http""net/http/httptest"@@ -153,3 +154,22 @@ func TestMinerUCloudReachable(t *testing.T) {t.Fatal(err)}}++func TestParseMinerUCloudRespectsContextDeadline(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), 50*time.Millisecond)+ defer cancel()++ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ time.Sleep(100 * time.Millisecond)+ http.Error(w, "too late", http.StatusOK)+ }))+ defer server.Close()++ _, err := ParseMinerUCloud(ctx, server.URL, "secret", "doc.pdf", []byte("%PDF"), "vlm-http-client", "ocr", "ch")+ if err == nil {+ t.Fatal("expected deadline exceeded error, got nil")+ }+ if !errors.Is(err, context.DeadlineExceeded) && !strings.Contains(err.Error(), "context deadline exceeded") {+ t.Fatalf("expected context deadline exceeded, got: %v", err)+ }+}diff --git a/internal/ingestion/component/pdf_vision_dispatch.go b/internal/ingestion/component/pdf_vision_dispatch.goindex 26ff401918..450b25b4eb 100644--- a/internal/ingestion/component/pdf_vision_dispatch.go+++ b/internal/ingestion/component/pdf_vision_dispatch.go@@ -628,6 +628,7 @@ func dispatchMinerUPDF(mineruLang := mineruLangCode(lang)backend := modelModule.ResolveMinerUBackend(getStringOr(setup, "mineru_backend", ""), apiKeyRaw)serverURL := modelModule.ResolveMinerUServerURL(getStringOr(setup, "mineru_server_url", ""), apiKeyRaw)+ timeout := mineruRequestTimeout(setup, apiConfig)if modelModule.IsMinerUCloudBase(baseURL) {if !extraction.formulaEnable || !extraction.tableEnable {@@ -636,7 +637,9 @@ func dispatchMinerUPDF(if err := modelModule.ValidateMinerUConfigForAPI(backend, serverURL, true); err != nil {return parser.ParseResult{}, err}- result, err := modelModule.ParseMinerUCloud(ctx, baseURL, apiKeyRaw, filename, binary, backend, parseMethod, mineruLang)+ cloudCtx, cloudCancel := context.WithTimeout(ctx, timeout)+ defer cloudCancel()+ result, err := modelModule.ParseMinerUCloud(cloudCtx, baseURL, apiKeyRaw, filename, binary, backend, parseMethod, mineruLang)if err != nil {return parser.ParseResult{}, fmt.Errorf("parser: MinerU cloud: %w", err)}@@ -657,7 +660,7 @@ func dispatchMinerUPDF(if !extraction.formulaEnable || !extraction.tableEnable {return parser.ParseResult{}, fmt.Errorf("parser: MinerU V1 does not support disabling formula or table extraction; use a /file_parse server to configure these switches")}- result, err := modelModule.ParseMinerUV1(ctx, baseURL, apiKeyRaw, filename, binary, backend, parseMethod, 0)+ result, err := modelModule.ParseMinerUV1(ctx, baseURL, apiKeyRaw, filename, binary, backend, parseMethod, timeout)if err != nil {return parser.ParseResult{}, fmt.Errorf("parser: MinerU V1: %w", err)}@@ -665,7 +668,7 @@ func dispatchMinerUPDF(}apiURL := strings.TrimRight(baseURL, "/") + "/file_parse"- zipBytes, err := mineruStreamParse(ctx, apiURL, apiKeyRaw, binary, parseMethod, mineruLang, backend, serverURL, extraction)+ zipBytes, err := mineruStreamParse(ctx, apiURL, apiKeyRaw, binary, parseMethod, mineruLang, backend, serverURL, extraction, timeout)if err != nil {return parser.ParseResult{}, fmt.Errorf("parser: MinerU stream: %w", err)}@@ -990,14 +993,61 @@ func mineruAPIParseMethod(raw string) string {return "auto"}-// mineruStreamTimeout bounds a single /file_parse upload. It matches the-// MinerU 4 V1 parse budget so both API generations give up at the same point.-const mineruStreamTimeout = 30 * time.Minute+// mineruDefaultTimeout bounds a single /file_parse upload or V1 parse cycle. It matches the+// MinerU 4 V1 parse budget so both API generations give up at the same point when unset.+const mineruDefaultTimeout = 30 * time.Minute++// mineruRequestTimeout resolves the MinerU parse timeout with precedence:+// 1. setup["mineru_timeout"] / setup["mineru_timeout_seconds"] (authoritative; non-empty invalid values+// do not fall through to env or API config and fall back to mineruDefaultTimeout)+// 2. apiConfig provider JSON (mineru_timeout, mineru_timeout_seconds, MINERU_TIMEOUT, MINERU_TIMEOUT_SECONDS)+// 3. Environment variables (MINERU_TIMEOUT, MINERU_TIMEOUT_SECONDS)+// 4. mineruDefaultTimeout (30m)+func mineruRequestTimeout(setup schema.ParserSetup, apiConfig *modelModule.APIConfig) time.Duration {+ value := ""+ if raw, ok := setup["mineru_timeout"]; ok && raw != nil {+ value = strings.TrimSpace(fmt.Sprint(raw))+ }+ if value == "" {+ if raw, ok := setup["mineru_timeout_seconds"]; ok && raw != nil {+ value = strings.TrimSpace(fmt.Sprint(raw))+ }+ }+ if value == "" {+ value = modelModule.ProviderJSONConfigValueFromAPIConfig(+ apiConfig,+ "mineru_timeout",+ "mineru_timeout_seconds",+ common.EnvMineruTimeout,+ "MINERU_TIMEOUT_SECONDS",+ )+ }+ if value == "" {+ value = os.Getenv(common.EnvMineruTimeout)+ }+ if value == "" {+ value = os.Getenv("MINERU_TIMEOUT_SECONDS")+ }+ value = strings.TrimSpace(value)+ if value == "" {+ return mineruDefaultTimeout+ }+ if seconds, err := strconv.Atoi(value); err == nil && seconds > 0 {+ return time.Duration(seconds) * time.Second+ }+ if seconds, err := strconv.ParseFloat(value, 64); err == nil && seconds > 0 {+ return time.Duration(seconds * float64(time.Second))+ }+ if d, err := time.ParseDuration(value); err == nil && d > 0 {+ return d+ }+ return mineruDefaultTimeout+}// mineruStreamParse POSTs the PDF binary to the MinerU /file_parse// endpoint with streaming and returns the zip response body.// Mirrors Python's mineru_parser.py._run_mineru_api with stream=True.-func mineruStreamParse(ctx context.Context, apiURL string, apiKey string, binary []byte, parseMethod, lang, backend, serverURL string, extraction mineruExtractionOptions) ([]byte, error) {+func mineruStreamParse(ctx context.Context, apiURL string, apiKey string, binary []byte, parseMethod, lang, backend, serverURL string, extraction mineruExtractionOptions, timeout time.Duration) ([]byte, error) {var body bytes.Bufferwriter := multipart.NewWriter(&body)@@ -1030,7 +1080,10 @@ func mineruStreamParse(ctx context.Context, apiURL string, apiKey string, binaryreturn nil, fmt.Errorf("finalize form: %w", err)}- ctx, cancel := context.WithTimeout(ctx, mineruStreamTimeout)+ if timeout <= 0 {+ timeout = mineruDefaultTimeout+ }+ ctx, cancel := context.WithTimeout(ctx, timeout)defer cancel()req, err := http.NewRequestWithContext(ctx, "POST", apiURL, &body)diff --git a/internal/ingestion/component/pdf_vision_dispatch_test.go b/internal/ingestion/component/pdf_vision_dispatch_test.goindex 53be7146c2..f3b0f6af2f 100644--- a/internal/ingestion/component/pdf_vision_dispatch_test.go+++ b/internal/ingestion/component/pdf_vision_dispatch_test.go@@ -89,6 +89,81 @@ func TestMonkeyOCRv2RequestTimeoutUsesSetupAndAPIConfig(t *testing.T) {}}+func TestMinerURequestTimeout(t *testing.T) {+ t.Setenv(common.EnvMineruTimeout, "")+ t.Setenv("MINERU_TIMEOUT_SECONDS", "")++ // 1. Default fallback+ if got := mineruRequestTimeout(nil, nil); got != 30*time.Minute {+ t.Fatalf("default timeout=%v, want 30m", got)+ }++ // 2. Setup mineru_timeout (int seconds)+ if got := mineruRequestTimeout(schema.ParserSetup{"mineru_timeout": 1200}, nil); got != 1200*time.Second {+ t.Fatalf("setup mineru_timeout=%v, want 1200s", got)+ }++ // 3. Setup mineru_timeout_seconds (int and float)+ if got := mineruRequestTimeout(schema.ParserSetup{"mineru_timeout_seconds": 3600}, nil); got != 3600*time.Second {+ t.Fatalf("setup mineru_timeout_seconds=%v, want 3600s", got)+ }+ if got := mineruRequestTimeout(schema.ParserSetup{"mineru_timeout_seconds": 1800.0}, nil); got != 1800*time.Second {+ t.Fatalf("setup mineru_timeout_seconds float=%v, want 1800s", got)+ }++ // 4. Setup duration strings+ if got := mineruRequestTimeout(schema.ParserSetup{"mineru_timeout": "45m"}, nil); got != 45*time.Minute {+ t.Fatalf("setup mineru_timeout 45m=%v, want 45m", got)+ }+ if got := mineruRequestTimeout(schema.ParserSetup{"mineru_timeout": "2h"}, nil); got != 2*time.Hour {+ t.Fatalf("setup mineru_timeout 2h=%v, want 2h", got)+ }++ // 5. APIConfig JSON+ apiKey := `{"mineru_timeout":"2400"}`+ if got := mineruRequestTimeout(nil, &modelModule.APIConfig{ApiKey: &apiKey}); got != 2400*time.Second {+ t.Fatalf("API config timeout=%v, want 2400s", got)+ }+ apiKeyEnv := `{"MINERU_TIMEOUT":"5000"}`+ if got := mineruRequestTimeout(nil, &modelModule.APIConfig{ApiKey: &apiKeyEnv}); got != 5000*time.Second {+ t.Fatalf("API config env key timeout=%v, want 5000s", got)+ }++ // 6. Environment variable+ t.Setenv(common.EnvMineruTimeout, "7200")+ if got := mineruRequestTimeout(nil, nil); got != 7200*time.Second {+ t.Fatalf("env MINERU_TIMEOUT=%v, want 7200s", got)+ }+ t.Setenv(common.EnvMineruTimeout, "")+ t.Setenv("MINERU_TIMEOUT_SECONDS", "3000")+ if got := mineruRequestTimeout(nil, nil); got != 3000*time.Second {+ t.Fatalf("env MINERU_TIMEOUT_SECONDS=%v, want 3000s", got)+ }++ // 7. Precedence: Setup > APIConfig > Env+ t.Setenv(common.EnvMineruTimeout, "9999")+ apiKeyPrecedence := `{"mineru_timeout":"600"}`+ if got := mineruRequestTimeout(schema.ParserSetup{"mineru_timeout": 300}, &modelModule.APIConfig{ApiKey: &apiKeyPrecedence}); got != 300*time.Second {+ t.Fatalf("precedence setup over apiConfig=%v, want 300s", got)+ }+ if got := mineruRequestTimeout(nil, &modelModule.APIConfig{ApiKey: &apiKeyPrecedence}); got != 600*time.Second {+ t.Fatalf("precedence apiConfig over env=%v, want 600s", got)+ }++ // 8. Invalid values fallback+ if got := mineruRequestTimeout(schema.ParserSetup{"mineru_timeout": -10}, nil); got != 30*time.Minute {+ t.Fatalf("negative timeout fallback=%v, want 30m", got)+ }+ if got := mineruRequestTimeout(schema.ParserSetup{"mineru_timeout": "invalid"}, nil); got != 30*time.Minute {+ t.Fatalf("invalid string timeout fallback=%v, want 30m", got)+ }+ // Present setup key is authoritative: invalid values do not fall through to MINERU_TIMEOUT env or apiConfig+ t.Setenv(common.EnvMineruTimeout, "7200")+ if got := mineruRequestTimeout(schema.ParserSetup{"mineru_timeout": "invalid"}, nil); got != 30*time.Minute {+ t.Fatalf("invalid setup with MINERU_TIMEOUT env fallback=%v, want 30m", got)+ }+}+func TestDispatchMonkeyOCRv2PDFPostsNativeParseRequest(t *testing.T) {server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, request *http.Request) {if request.URL.Path != "/parse" {@@ -803,7 +878,7 @@ func TestMineruStreamKeepsCancellationWhileReadingErrorResponse(t *testing.T) {ctx, cancel := context.WithCancel(t.Context())done := make(chan error, 1)go func() {- _, err := mineruStreamParse(ctx, server.URL+"/file_parse", "mineru-secret", []byte("%PDF-1.4"), "auto", "en", "", "", mineruExtractionOptions{})+ _, err := mineruStreamParse(ctx, server.URL+"/file_parse", "mineru-secret", []byte("%PDF-1.4"), "auto", "en", "", "", mineruExtractionOptions{}, 0)done <- err}()