Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

feat(ocr): make MinerU parser timeout configurable via environment and setup (#21832)

superwiboyf5d7ab789e69
diff --git a/docs/faq.mdx b/docs/faq.mdx
index e330f01c7d..f4847c25b1 100644
--- a/docs/faq.mdx
+++ b/docs/faq.mdx
@@ -528,12 +528,14 @@ Use the following environment variables when MinerU is not configured directly i
| `MINERU_APISERVER` | URL of the MinerU API service | _unset_ | `MINERU_APISERVER=http://your-mineru-server:8886` |
| `MINERU_API_KEY` | API key, when required by MinerU | _unset_ | `MINERU_API_KEY=your-key` |
| `MINERU_BACKEND` | MinerU 3.x backend or 4.0 tier | `pipeline` | `MINERU_BACKEND=pipeline\|standard\|vlm-http-client` |
+| `MINERU_TIMEOUT` | MinerU request timeout | `1800` (30m) | `MINERU_TIMEOUT=3600` |
1. Set `MINERU_APISERVER` to point RAGFlow to your MinerU API server.
2. Set `MINERU_API_KEY` if your MinerU service requires authentication.
3. Set `MINERU_BACKEND` to specify the backend sent with the parse request.
+4. Optionally set `MINERU_TIMEOUT` (seconds or duration string like `1h`) to bound the parse request.
-For a custom Parser component, the numeric `mineru_timeout_seconds` setup field controls how long RAGFlow polls for the result. Its default is 30 seconds; zero or a negative value also falls back to 30 seconds.
+For native ingestion pipeline component parsing (`dispatchMinerUPDF`), `mineru_timeout` or `mineru_timeout_seconds` in the parser setup overrides this timeout; zero or negative values fall back to the default 30 minutes. Note that the legacy `PDFParser` path (`pdf_parser_mineru.go`) continues to use `mineru_timeout_seconds` with a 30s poll interval default.
:::tip NOTE
For other environment variables supported by MinerU itself, see the [MinerU environment variable documentation](https://opendatalab.github.io/MinerU/usage/cli_tools/#environment-variables-description).
diff --git a/internal/common/environments.go b/internal/common/environments.go
index 6f84cec141..a218907451 100644
--- a/internal/common/environments.go
+++ b/internal/common/environments.go
@@ -229,6 +229,7 @@ const (
EnvMineruAPIKey = "MINERU_API_KEY"
EnvMineruBackend = "MINERU_BACKEND"
EnvMineruServerURL = "MINERU_SERVER_URL"
+ EnvMineruTimeout = "MINERU_TIMEOUT"
EnvMonkeyOCRv2ServerURL = "MONKEYOCRV2_SERVER_URL"
EnvMonkeyOCRv2Timeout = "MONKEYOCRV2_TIMEOUT"
EnvMonkeyOCRAPIServer = "MONKEYOCR_APISERVER"
diff --git a/internal/entity/models/mineru_cloud.go b/internal/entity/models/mineru_cloud.go
index 08000dd610..47b374cc92 100644
--- a/internal/entity/models/mineru_cloud.go
+++ b/internal/entity/models/mineru_cloud.go
@@ -87,9 +87,11 @@ func ParseMinerUCloud(ctx context.Context, baseURL, apiKey, filename string, con
if ctx == nil {
ctx = context.Background()
}
- var cancel context.CancelFunc
- ctx, cancel = context.WithTimeout(ctx, DefaultMinerUV1ParseTimeout)
- defer cancel()
+ if _, ok := ctx.Deadline(); !ok {
+ var cancel context.CancelFunc
+ ctx, cancel = context.WithTimeout(ctx, DefaultMinerUV1ParseTimeout)
+ defer cancel()
+ }
token := minerUBearerFromAPIKey(apiKey)
client := minerUHTTPClient()
diff --git a/internal/entity/models/mineru_cloud_test.go b/internal/entity/models/mineru_cloud_test.go
index 59aad9841a..93760a745c 100644
--- a/internal/entity/models/mineru_cloud_test.go
+++ b/internal/entity/models/mineru_cloud_test.go
@@ -6,6 +6,7 @@ package models
import (
"context"
+ "errors"
"io"
"net/http"
"net/http/httptest"
@@ -153,3 +154,22 @@ func TestMinerUCloudReachable(t *testing.T) {
t.Fatal(err)
}
}
+
+func TestParseMinerUCloudRespectsContextDeadline(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), 50*time.Millisecond)
+ defer cancel()
+
+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ time.Sleep(100 * time.Millisecond)
+ http.Error(w, "too late", http.StatusOK)
+ }))
+ defer server.Close()
+
+ _, err := ParseMinerUCloud(ctx, server.URL, "secret", "doc.pdf", []byte("%PDF"), "vlm-http-client", "ocr", "ch")
+ if err == nil {
+ t.Fatal("expected deadline exceeded error, got nil")
+ }
+ if !errors.Is(err, context.DeadlineExceeded) && !strings.Contains(err.Error(), "context deadline exceeded") {
+ t.Fatalf("expected context deadline exceeded, got: %v", err)
+ }
+}
diff --git a/internal/ingestion/component/pdf_vision_dispatch.go b/internal/ingestion/component/pdf_vision_dispatch.go
index 26ff401918..450b25b4eb 100644
--- a/internal/ingestion/component/pdf_vision_dispatch.go
+++ b/internal/ingestion/component/pdf_vision_dispatch.go
@@ -628,6 +628,7 @@ func dispatchMinerUPDF(
mineruLang := mineruLangCode(lang)
backend := modelModule.ResolveMinerUBackend(getStringOr(setup, "mineru_backend", ""), apiKeyRaw)
serverURL := modelModule.ResolveMinerUServerURL(getStringOr(setup, "mineru_server_url", ""), apiKeyRaw)
+ timeout := mineruRequestTimeout(setup, apiConfig)
if modelModule.IsMinerUCloudBase(baseURL) {
if !extraction.formulaEnable || !extraction.tableEnable {
@@ -636,7 +637,9 @@ func dispatchMinerUPDF(
if err := modelModule.ValidateMinerUConfigForAPI(backend, serverURL, true); err != nil {
return parser.ParseResult{}, err
}
- result, err := modelModule.ParseMinerUCloud(ctx, baseURL, apiKeyRaw, filename, binary, backend, parseMethod, mineruLang)
+ cloudCtx, cloudCancel := context.WithTimeout(ctx, timeout)
+ defer cloudCancel()
+ result, err := modelModule.ParseMinerUCloud(cloudCtx, baseURL, apiKeyRaw, filename, binary, backend, parseMethod, mineruLang)
if err != nil {
return parser.ParseResult{}, fmt.Errorf("parser: MinerU cloud: %w", err)
}
@@ -657,7 +660,7 @@ func dispatchMinerUPDF(
if !extraction.formulaEnable || !extraction.tableEnable {
return parser.ParseResult{}, fmt.Errorf("parser: MinerU V1 does not support disabling formula or table extraction; use a /file_parse server to configure these switches")
}
- result, err := modelModule.ParseMinerUV1(ctx, baseURL, apiKeyRaw, filename, binary, backend, parseMethod, 0)
+ result, err := modelModule.ParseMinerUV1(ctx, baseURL, apiKeyRaw, filename, binary, backend, parseMethod, timeout)
if err != nil {
return parser.ParseResult{}, fmt.Errorf("parser: MinerU V1: %w", err)
}
@@ -665,7 +668,7 @@ func dispatchMinerUPDF(
}
apiURL := strings.TrimRight(baseURL, "/") + "/file_parse"
- zipBytes, err := mineruStreamParse(ctx, apiURL, apiKeyRaw, binary, parseMethod, mineruLang, backend, serverURL, extraction)
+ zipBytes, err := mineruStreamParse(ctx, apiURL, apiKeyRaw, binary, parseMethod, mineruLang, backend, serverURL, extraction, timeout)
if err != nil {
return parser.ParseResult{}, fmt.Errorf("parser: MinerU stream: %w", err)
}
@@ -990,14 +993,61 @@ func mineruAPIParseMethod(raw string) string {
return "auto"
}
-// mineruStreamTimeout bounds a single /file_parse upload. It matches the
-// MinerU 4 V1 parse budget so both API generations give up at the same point.
-const mineruStreamTimeout = 30 * time.Minute
+// mineruDefaultTimeout bounds a single /file_parse upload or V1 parse cycle. It matches the
+// MinerU 4 V1 parse budget so both API generations give up at the same point when unset.
+const mineruDefaultTimeout = 30 * time.Minute
+
+// mineruRequestTimeout resolves the MinerU parse timeout with precedence:
+// 1. setup["mineru_timeout"] / setup["mineru_timeout_seconds"] (authoritative; non-empty invalid values
+// do not fall through to env or API config and fall back to mineruDefaultTimeout)
+// 2. apiConfig provider JSON (mineru_timeout, mineru_timeout_seconds, MINERU_TIMEOUT, MINERU_TIMEOUT_SECONDS)
+// 3. Environment variables (MINERU_TIMEOUT, MINERU_TIMEOUT_SECONDS)
+// 4. mineruDefaultTimeout (30m)
+func mineruRequestTimeout(setup schema.ParserSetup, apiConfig *modelModule.APIConfig) time.Duration {
+ value := ""
+ if raw, ok := setup["mineru_timeout"]; ok && raw != nil {
+ value = strings.TrimSpace(fmt.Sprint(raw))
+ }
+ if value == "" {
+ if raw, ok := setup["mineru_timeout_seconds"]; ok && raw != nil {
+ value = strings.TrimSpace(fmt.Sprint(raw))
+ }
+ }
+ if value == "" {
+ value = modelModule.ProviderJSONConfigValueFromAPIConfig(
+ apiConfig,
+ "mineru_timeout",
+ "mineru_timeout_seconds",
+ common.EnvMineruTimeout,
+ "MINERU_TIMEOUT_SECONDS",
+ )
+ }
+ if value == "" {
+ value = os.Getenv(common.EnvMineruTimeout)
+ }
+ if value == "" {
+ value = os.Getenv("MINERU_TIMEOUT_SECONDS")
+ }
+ value = strings.TrimSpace(value)
+ if value == "" {
+ return mineruDefaultTimeout
+ }
+ if seconds, err := strconv.Atoi(value); err == nil && seconds > 0 {
+ return time.Duration(seconds) * time.Second
+ }
+ if seconds, err := strconv.ParseFloat(value, 64); err == nil && seconds > 0 {
+ return time.Duration(seconds * float64(time.Second))
+ }
+ if d, err := time.ParseDuration(value); err == nil && d > 0 {
+ return d
+ }
+ return mineruDefaultTimeout
+}
// mineruStreamParse POSTs the PDF binary to the MinerU /file_parse
// endpoint with streaming and returns the zip response body.
// Mirrors Python's mineru_parser.py._run_mineru_api with stream=True.
-func mineruStreamParse(ctx context.Context, apiURL string, apiKey string, binary []byte, parseMethod, lang, backend, serverURL string, extraction mineruExtractionOptions) ([]byte, error) {
+func mineruStreamParse(ctx context.Context, apiURL string, apiKey string, binary []byte, parseMethod, lang, backend, serverURL string, extraction mineruExtractionOptions, timeout time.Duration) ([]byte, error) {
var body bytes.Buffer
writer := multipart.NewWriter(&body)
@@ -1030,7 +1080,10 @@ func mineruStreamParse(ctx context.Context, apiURL string, apiKey string, binary
return nil, fmt.Errorf("finalize form: %w", err)
}
- ctx, cancel := context.WithTimeout(ctx, mineruStreamTimeout)
+ if timeout <= 0 {
+ timeout = mineruDefaultTimeout
+ }
+ ctx, cancel := context.WithTimeout(ctx, timeout)
defer cancel()
req, err := http.NewRequestWithContext(ctx, "POST", apiURL, &body)
diff --git a/internal/ingestion/component/pdf_vision_dispatch_test.go b/internal/ingestion/component/pdf_vision_dispatch_test.go
index 53be7146c2..f3b0f6af2f 100644
--- a/internal/ingestion/component/pdf_vision_dispatch_test.go
+++ b/internal/ingestion/component/pdf_vision_dispatch_test.go
@@ -89,6 +89,81 @@ func TestMonkeyOCRv2RequestTimeoutUsesSetupAndAPIConfig(t *testing.T) {
}
}
+func TestMinerURequestTimeout(t *testing.T) {
+ t.Setenv(common.EnvMineruTimeout, "")
+ t.Setenv("MINERU_TIMEOUT_SECONDS", "")
+
+ // 1. Default fallback
+ if got := mineruRequestTimeout(nil, nil); got != 30*time.Minute {
+ t.Fatalf("default timeout=%v, want 30m", got)
+ }
+
+ // 2. Setup mineru_timeout (int seconds)
+ if got := mineruRequestTimeout(schema.ParserSetup{"mineru_timeout": 1200}, nil); got != 1200*time.Second {
+ t.Fatalf("setup mineru_timeout=%v, want 1200s", got)
+ }
+
+ // 3. Setup mineru_timeout_seconds (int and float)
+ if got := mineruRequestTimeout(schema.ParserSetup{"mineru_timeout_seconds": 3600}, nil); got != 3600*time.Second {
+ t.Fatalf("setup mineru_timeout_seconds=%v, want 3600s", got)
+ }
+ if got := mineruRequestTimeout(schema.ParserSetup{"mineru_timeout_seconds": 1800.0}, nil); got != 1800*time.Second {
+ t.Fatalf("setup mineru_timeout_seconds float=%v, want 1800s", got)
+ }
+
+ // 4. Setup duration strings
+ if got := mineruRequestTimeout(schema.ParserSetup{"mineru_timeout": "45m"}, nil); got != 45*time.Minute {
+ t.Fatalf("setup mineru_timeout 45m=%v, want 45m", got)
+ }
+ if got := mineruRequestTimeout(schema.ParserSetup{"mineru_timeout": "2h"}, nil); got != 2*time.Hour {
+ t.Fatalf("setup mineru_timeout 2h=%v, want 2h", got)
+ }
+
+ // 5. APIConfig JSON
+ apiKey := `{"mineru_timeout":"2400"}`
+ if got := mineruRequestTimeout(nil, &modelModule.APIConfig{ApiKey: &apiKey}); got != 2400*time.Second {
+ t.Fatalf("API config timeout=%v, want 2400s", got)
+ }
+ apiKeyEnv := `{"MINERU_TIMEOUT":"5000"}`
+ if got := mineruRequestTimeout(nil, &modelModule.APIConfig{ApiKey: &apiKeyEnv}); got != 5000*time.Second {
+ t.Fatalf("API config env key timeout=%v, want 5000s", got)
+ }
+
+ // 6. Environment variable
+ t.Setenv(common.EnvMineruTimeout, "7200")
+ if got := mineruRequestTimeout(nil, nil); got != 7200*time.Second {
+ t.Fatalf("env MINERU_TIMEOUT=%v, want 7200s", got)
+ }
+ t.Setenv(common.EnvMineruTimeout, "")
+ t.Setenv("MINERU_TIMEOUT_SECONDS", "3000")
+ if got := mineruRequestTimeout(nil, nil); got != 3000*time.Second {
+ t.Fatalf("env MINERU_TIMEOUT_SECONDS=%v, want 3000s", got)
+ }
+
+ // 7. Precedence: Setup > APIConfig > Env
+ t.Setenv(common.EnvMineruTimeout, "9999")
+ apiKeyPrecedence := `{"mineru_timeout":"600"}`
+ if got := mineruRequestTimeout(schema.ParserSetup{"mineru_timeout": 300}, &modelModule.APIConfig{ApiKey: &apiKeyPrecedence}); got != 300*time.Second {
+ t.Fatalf("precedence setup over apiConfig=%v, want 300s", got)
+ }
+ if got := mineruRequestTimeout(nil, &modelModule.APIConfig{ApiKey: &apiKeyPrecedence}); got != 600*time.Second {
+ t.Fatalf("precedence apiConfig over env=%v, want 600s", got)
+ }
+
+ // 8. Invalid values fallback
+ if got := mineruRequestTimeout(schema.ParserSetup{"mineru_timeout": -10}, nil); got != 30*time.Minute {
+ t.Fatalf("negative timeout fallback=%v, want 30m", got)
+ }
+ if got := mineruRequestTimeout(schema.ParserSetup{"mineru_timeout": "invalid"}, nil); got != 30*time.Minute {
+ t.Fatalf("invalid string timeout fallback=%v, want 30m", got)
+ }
+ // Present setup key is authoritative: invalid values do not fall through to MINERU_TIMEOUT env or apiConfig
+ t.Setenv(common.EnvMineruTimeout, "7200")
+ if got := mineruRequestTimeout(schema.ParserSetup{"mineru_timeout": "invalid"}, nil); got != 30*time.Minute {
+ t.Fatalf("invalid setup with MINERU_TIMEOUT env fallback=%v, want 30m", got)
+ }
+}
+
func TestDispatchMonkeyOCRv2PDFPostsNativeParseRequest(t *testing.T) {
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, request *http.Request) {
if request.URL.Path != "/parse" {
@@ -803,7 +878,7 @@ func TestMineruStreamKeepsCancellationWhileReadingErrorResponse(t *testing.T) {
ctx, cancel := context.WithCancel(t.Context())
done := make(chan error, 1)
go func() {
- _, err := mineruStreamParse(ctx, server.URL+"/file_parse", "mineru-secret", []byte("%PDF-1.4"), "auto", "en", "", "", mineruExtractionOptions{})
+ _, err := mineruStreamParse(ctx, server.URL+"/file_parse", "mineru-secret", []byte("%PDF-1.4"), "auto", "en", "", "", mineruExtractionOptions{}, 0)
done <- err
}()