Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

feat(parser): support MinerU 4.0 V1 HTTP API (#20712)

Sourav Rajvi68752fcddd39
diff --git a/docs/faq.mdx b/docs/faq.mdx
index d7d342596b..0eded1b533 100644
--- a/docs/faq.mdx
+++ b/docs/faq.mdx
@@ -503,16 +503,16 @@ To improve response speed:
RAGFlow sends PDF documents to a remote MinerU service and polls for the parsed result. To use it:
-1. Prepare a reachable MinerU API service that provides the `/file_parse` and `/tasks` endpoints used by RAGFlow.
+1. Prepare a reachable MinerU API service:
+ - **MinerU 4.0** (`mineru-kit api-server`): the V1 API (`/v1/health`, `/v1/uploads`, `/v1/parse/jobs`, `/v1/files`). RAGFlow probes `/v1/health` and uses this cycle automatically. Legacy `/file_parse` and `/tasks` are **not** provided by MinerU 4.
+ - **MinerU 3.x**: `/file_parse` and `/tasks` (RAGFlow falls back to this when `/v1/health` is missing).
2. In **docker/.env** or on the **Model providers** page in the UI, configure RAGFlow as a remote client to MinerU:
- - `MINERU_APISERVER`: The MinerU API endpoint (e.g., `http://mineru-host:8886`).
+ - `MINERU_APISERVER`: The MinerU API endpoint (e.g., `http://mineru-host:8000`).
- `MINERU_API_KEY`: An API key if the MinerU service requires one.
- - `MINERU_BACKEND`: The MinerU backend (matches current MinerU API / CLI `-b` values):
- - `"pipeline"` (default)
- - `"vlm-engine"`
- - `"hybrid-engine"`
- - `"vlm-http-client"`
- - `"hybrid-http-client"`.
+ - `MINERU_BACKEND`: On MinerU 3.x this is the `/file_parse` `backend` field. On MinerU 4.0 it is mapped to a V1 **tier** (`pipeline` → `basic`, `hybrid-*` → `standard`, `vlm-*` → `advanced`). You can also set a 4.0 tier directly:
+ - `"pipeline"` (default for 3.x)
+ - `"vlm-engine"` / `"hybrid-engine"` / `"vlm-http-client"` / `"hybrid-http-client"` (3.x)
+ - `"basic"` / `"standard"` / `"advanced"` / `"flash"` (MinerU 4).
3. In the web UI, navigate to your dataset's **Configuration** page and find the **Ingestion pipeline** section:
- If you decide to use a chunking method from the **Built-in** dropdown, ensure it supports PDF parsing, then select **MinerU** from the **PDF parser** dropdown.
- If you use a custom ingestion pipeline instead, select **MinerU** in the **PDF parser** section of the **Parser** component.
@@ -531,7 +531,7 @@ Use the following environment variables when MinerU is not configured directly i
| ---------------------- | ---------------------------------- | ----------------------------------- | ----------------------------------------------------------------------------------------------- |
| `MINERU_APISERVER` | URL of the MinerU API service | _unset_ | `MINERU_APISERVER=http://your-mineru-server:8886` |
| `MINERU_API_KEY` | API key, when required by MinerU | _unset_ | `MINERU_API_KEY=your-key` |
-| `MINERU_BACKEND` | MinerU parsing backend | `pipeline` | `MINERU_BACKEND=pipeline\|vlm-engine\|hybrid-engine\|vlm-http-client\|hybrid-http-client` |
+| `MINERU_BACKEND` | MinerU 3.x backend or 4.0 tier | `pipeline` | `MINERU_BACKEND=pipeline\|standard\|vlm-http-client` |
1. Set `MINERU_APISERVER` to point RAGFlow to your MinerU API server.
2. Set `MINERU_API_KEY` if your MinerU service requires authentication.
diff --git a/internal/development.md b/internal/development.md
index 5d2f3547f6..f12ea68cd9 100644
--- a/internal/development.md
+++ b/internal/development.md
@@ -221,7 +221,7 @@ missing from the model directory.
backend. Run it from the repo root so the weights above are auto-discovered
(see the CWD caveat); no environment variable is required there.
-- **Run the Go Docker image** — the container entrypoint only starts the server
+- **Run the Go Docker image** — the container entrypoint only starts the server
(`bin/ragflow_server --api/--ingestor/--admin`)
- ORT is resolved via `dlopen(NULL)` at runtime, so no `LD_LIBRARY_PATH` /
diff --git a/internal/entity/models/mineru_config.go b/internal/entity/models/mineru_config.go
index 154dcd41cd..08b8c8650e 100644
--- a/internal/entity/models/mineru_config.go
+++ b/internal/entity/models/mineru_config.go
@@ -10,22 +10,77 @@ import (
"strings"
)
+// MinerU 3.x /file_parse backend names. MinerU 4.0 dropped --backend in favor of
+// quality tiers (basic/standard/advanced/flash); see MinerUTierFromBackend.
var ValidMinerUBackends = map[string]struct{}{
"pipeline": {},
"vlm-engine": {},
"hybrid-engine": {},
"vlm-http-client": {},
"hybrid-http-client": {},
+ "vlm-auto-engine": {},
+ "hybrid-auto-engine": {},
+ "basic": {},
+ "standard": {},
+ "advanced": {},
+ "flash": {},
+}
+
+var validMinerUTiers = map[string]struct{}{
+ "basic": {},
+ "standard": {},
+ "advanced": {},
+ "flash": {},
}
func MinerUBackendRequiresServerURL(backend string) bool {
return backend == "vlm-http-client" || backend == "hybrid-http-client"
}
+func MinerUIsV1Tier(name string) bool {
+ _, ok := validMinerUTiers[strings.ToLower(strings.TrimSpace(name))]
+ return ok
+}
+
+// MinerUTierFromBackend maps a 3.x backend or a 4.0 tier onto a V1 `tier`.
+// MinerU 4.0 changelog: pipeline → basic; vlm-* → advanced; hybrid-* → standard.
+func MinerUTierFromBackend(backend string) string {
+ switch strings.ToLower(strings.TrimSpace(backend)) {
+ case "basic", "pipeline":
+ return "basic"
+ case "flash":
+ return "flash"
+ case "advanced", "vlm-engine", "vlm-auto-engine", "vlm-http-client":
+ return "advanced"
+ case "standard", "hybrid-engine", "hybrid-auto-engine", "hybrid-http-client", "":
+ return "standard"
+ default:
+ return "standard"
+ }
+}
+
func ValidateMinerUConfig(backend, serverURL string) error {
+ return validateMinerUConfig(backend, serverURL, false)
+}
+
+func ValidateMinerUConfigForAPI(backend, serverURL string, v1 bool) error {
+ return validateMinerUConfig(backend, serverURL, v1)
+}
+
+func validateMinerUConfig(backend, serverURL string, v1 bool) error {
if _, ok := ValidMinerUBackends[backend]; !ok {
return fmt.Errorf(
- "parser: MinerU invalid backend %q (valid: pipeline, vlm-engine, hybrid-engine, vlm-http-client, hybrid-http-client)",
+ "parser: MinerU invalid backend %q (valid: pipeline, vlm-engine, hybrid-engine, vlm-http-client, hybrid-http-client, vlm-auto-engine, hybrid-auto-engine, basic, standard, advanced, flash)",
+ backend,
+ )
+ }
+ if v1 {
+ // MinerU 4 configures the VLM on the service itself; server_url is not a V1 job field.
+ return nil
+ }
+ if MinerUIsV1Tier(backend) {
+ return fmt.Errorf(
+ "parser: MinerU backend %q is a 4.0 tier, but the connected service does not speak V1 (/v1/health). Use pipeline, vlm-engine, hybrid-engine, vlm-http-client, or hybrid-http-client, or upgrade MinerU to 4.0",
backend,
)
}
diff --git a/internal/entity/models/mineru_config_test.go b/internal/entity/models/mineru_config_test.go
index 8d50ce9ce5..445ce551c2 100644
--- a/internal/entity/models/mineru_config_test.go
+++ b/internal/entity/models/mineru_config_test.go
@@ -74,6 +74,40 @@ func TestResolveMinerUBackendAndServerURL(t *testing.T) {
}
}
+func TestMinerUTierFromBackend(t *testing.T) {
+ cases := map[string]string{
+ "pipeline": "basic",
+ "basic": "basic",
+ "vlm-engine": "advanced",
+ "vlm-http-client": "advanced",
+ "hybrid-engine": "standard",
+ "hybrid-http-client": "standard",
+ "flash": "flash",
+ "standard": "standard",
+ "": "standard",
+ }
+ for in, want := range cases {
+ if got := MinerUTierFromBackend(in); got != want {
+ t.Errorf("MinerUTierFromBackend(%q) = %q, want %q", in, got, want)
+ }
+ }
+}
+
+func TestValidateMinerUConfigAcceptsTiersOnV1(t *testing.T) {
+ if err := ValidateMinerUConfigForAPI("standard", "", true); err != nil {
+ t.Fatalf("V1 standard: %v", err)
+ }
+ if err := ValidateMinerUConfigForAPI("pipeline", "", true); err != nil {
+ t.Fatalf("V1 pipeline: %v", err)
+ }
+ if err := ValidateMinerUConfigForAPI("flash", "", false); err == nil {
+ t.Fatal("3.x flash tier should fail")
+ }
+ if err := ValidateMinerUConfig("vlm-http-client", ""); err == nil {
+ t.Fatal("3.x vlm-http-client without server_url should fail")
+ }
+}
+
func TestResolveMinerUAPIServer(t *testing.T) {
t.Setenv("MINERU_APISERVER", "http://env-mineru:8000")
diff --git a/internal/entity/models/mineru_local.go b/internal/entity/models/mineru_local.go
index bb94357b19..d433ae2e4c 100644
--- a/internal/entity/models/mineru_local.go
+++ b/internal/entity/models/mineru_local.go
@@ -123,34 +123,39 @@ func (m *MinerULocalModel) CheckConnection(ctx context.Context, apiConfig *APICo
if err != nil {
return fmt.Errorf("invalid base URL: %w", err)
}
- // MinerU's API service exposes GET /health; keep scheme/host and replace
- // only the path so a base URL with extra path segments or query stays sane.
- u.Path = "/health"
+ token := ""
+ if apiConfig != nil && apiConfig.ApiKey != nil {
+ token = MinerUBearerTokenFromAPIKey(*apiConfig.ApiKey)
+ }
ctx, cancel := context.WithTimeout(ctx, nonStreamCallTimeout)
defer cancel()
- req, err := http.NewRequestWithContext(ctx, http.MethodGet, u.String(), nil)
- if err != nil {
- return fmt.Errorf("failed to create request: %w", err)
- }
- // Same token extraction as ParseFile/ShowTask: a plain api_key is a bearer
- // token, while MinerU provider JSON config carries no bearer unless it
- // embeds mineru_api_key/access_token — sending the raw payload would be an
- // invalid header.
- if apiConfig != nil && apiConfig.ApiKey != nil {
- if token := MinerUBearerTokenFromAPIKey(*apiConfig.ApiKey); token != "" {
- req.Header.Set("Authorization", "Bearer "+token)
+
+ // MinerU 4.0 serves GET /v1/health; 3.x serves GET /health. Probe V1
+ // first so a 4.0-only service (no /health) still verifies, then fall
+ // back. 401/403 mean the credentials are wrong; any other HTTP answer
+ // (including 404 on a server without that path) still proves reachability
+ // once at least one probe got a response.
+ v1URL := *u
+ v1URL.Path = "/v1/health"
+ v1Code, v1Err := minerUProbeGET(ctx, m.baseModel.httpClient, v1URL.String(), token)
+ if v1Err == nil && v1Code != http.StatusNotFound && v1Code != http.StatusMethodNotAllowed {
+ if v1Code == http.StatusUnauthorized || v1Code == http.StatusForbidden {
+ return fmt.Errorf("authentication failed (HTTP %d)", v1Code)
}
+ return nil
}
- resp, err := m.baseModel.httpClient.Do(req)
+
+ legacy := *u
+ legacy.Path = "/health"
+ code, err := minerUProbeGET(ctx, m.baseModel.httpClient, legacy.String(), token)
if err != nil {
+ if v1Err != nil {
+ return fmt.Errorf("connection failed: %w", v1Err)
+ }
return fmt.Errorf("connection failed: %w", err)
}
- defer resp.Body.Close()
- // Match the hosted mineru.net driver's semantics (mineru.go): 401/403 mean
- // the credentials are wrong, while any other HTTP answer (e.g. 404 on a
- // service without /health) still proves the endpoint is reachable.
- if resp.StatusCode == http.StatusUnauthorized || resp.StatusCode == http.StatusForbidden {
- return fmt.Errorf("authentication failed (HTTP %d)", resp.StatusCode)
+ if code == http.StatusUnauthorized || code == http.StatusForbidden {
+ return fmt.Errorf("authentication failed (HTTP %d)", code)
}
return nil
}
diff --git a/internal/entity/models/mineru_v1.go b/internal/entity/models/mineru_v1.go
new file mode 100644
index 0000000000..2020255981
--- /dev/null
+++ b/internal/entity/models/mineru_v1.go
@@ -0,0 +1,565 @@
+//
+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+package models
+
+import (
+ "archive/zip"
+ "bytes"
+ "context"
+ "crypto/sha256"
+ "encoding/hex"
+ "encoding/json"
+ "fmt"
+ "io"
+ "net/http"
+ "net/url"
+ "path"
+ "strings"
+ "sync"
+ "time"
+
+ "ragflow/internal/common"
+)
+
+// minerUV1PollInitial and minerUV1PollMax bound GET /v1/parse/jobs/{id} polling.
+// Tests may shorten minerUV1PollInitial.
+var (
+ minerUV1PollInitial = 2 * time.Second
+ minerUV1PollMax = 30 * time.Second
+)
+
+// DefaultMinerUV1ParseTimeout is used when ParseMinerUV1 receives a non-positive timeout.
+const DefaultMinerUV1ParseTimeout = 30 * time.Minute
+
+type minerUV1SupportCacheEntry struct {
+ supported bool
+ expires time.Time
+}
+
+var minerUV1SupportCache sync.Map
+
+const minerUV1SupportCacheTTL = 5 * time.Minute
+
+type minerUV1UploadResponse struct {
+ ID string `json:"id"`
+ Status string `json:"status"`
+ UploadURL string `json:"upload_url"`
+ UploadMethod string `json:"upload_method"`
+ UploadHeaders map[string]string `json:"upload_headers"`
+ File *minerUV1FileRef `json:"file"`
+}
+
+type minerUV1FileRef struct {
+ ID string `json:"id"`
+}
+
+// minerUV1ArtifactRef maps parse-job output_files entries (file_id) and legacy test mocks (id).
+type minerUV1ArtifactRef struct {
+ ID string `json:"id"`
+ FileID string `json:"file_id"`
+}
+
+func (r minerUV1ArtifactRef) artifactID() string {
+ if id := strings.TrimSpace(r.FileID); id != "" {
+ return id
+ }
+ return strings.TrimSpace(r.ID)
+}
+
+type minerUV1JobResponse struct {
+ JobID string `json:"job_id"`
+ Status string `json:"status"`
+ Error *minerUV1Error `json:"error"`
+ Files []minerUV1JobFile `json:"files"`
+}
+
+type minerUV1Error struct {
+ Code string `json:"code"`
+ Message string `json:"message"`
+}
+
+type minerUV1JobFile struct {
+ Name string `json:"name"`
+ Status string `json:"status"`
+ Error *minerUV1Error `json:"error"`
+ OutputFiles map[string]minerUV1ArtifactRef `json:"output_files"`
+}
+
+// MinerUV1Result is the downloadable parse output from a MinerU 4 V1 job.
+type MinerUV1Result struct {
+ Zip []byte
+ Markdown string
+}
+
+func minerUHTTPClient() *http.Client {
+ return common.GetSchemeSafeHTTPClient()
+}
+
+func minerUBearerFromAPIKey(apiKey string) string {
+ return MinerUBearerTokenFromAPIKey(apiKey)
+}
+
+// MinerUV1OCRMode maps mineru_parse_method (auto, txt, ocr) onto the V1 job field ocr_mode.
+func MinerUV1OCRMode(raw string) string {
+ switch strings.ToLower(strings.TrimSpace(raw)) {
+ case "auto", "txt", "ocr":
+ return strings.ToLower(strings.TrimSpace(raw))
+ default:
+ return "auto"
+ }
+}
+
+func minerUV1SupportCacheKey(baseURL, apiKey string) string {
+ token := minerUBearerFromAPIKey(apiKey)
+ sum := sha256.Sum256([]byte(token))
+ return baseURL + "\x00" + hex.EncodeToString(sum[:])
+}
+
+func minerUProbeGET(ctx context.Context, client *http.Client, rawURL, token string) (int, error) {
+ req, err := http.NewRequestWithContext(ctx, http.MethodGet, rawURL, nil)
+ if err != nil {
+ return 0, err
+ }
+ if token != "" {
+ req.Header.Set("Authorization", "Bearer "+token)
+ }
+ resp, err := client.Do(req)
+ if err != nil {
+ return 0, err
+ }
+ defer resp.Body.Close()
+ _, _ = io.Copy(io.Discard, resp.Body)
+ return resp.StatusCode, nil
+}
+
+// MinerUSupportsV1 reports whether baseURL speaks MinerU 4's /v1 HTTP API.
+// 404/405 mean a 3.x /file_parse service. 401/403 still count as V1 (the
+// route exists; credentials may be wrong).
+func MinerUSupportsV1(ctx context.Context, baseURL, apiKey string) (bool, error) {
+ baseURL = strings.TrimRight(strings.TrimSpace(baseURL), "/")
+ if baseURL == "" {
+ return false, fmt.Errorf("MinerU base URL is empty")
+ }
+ cacheKey := minerUV1SupportCacheKey(baseURL, apiKey)
+ if cached, ok := minerUV1SupportCache.Load(cacheKey); ok {
+ entry := cached.(minerUV1SupportCacheEntry)
+ if time.Now().Before(entry.expires) {
+ return entry.supported, nil
+ }
+ }
+ token := minerUBearerFromAPIKey(apiKey)
+ code, err := minerUProbeGET(ctx, minerUHTTPClient(), baseURL+"/v1/health", token)
+ if err != nil {
+ return false, err
+ }
+ var supported bool
+ switch code {
+ case http.StatusNotFound, http.StatusMethodNotAllowed:
+ supported = false
+ default:
+ supported = true
+ }
+ minerUV1SupportCache.Store(cacheKey, minerUV1SupportCacheEntry{
+ supported: supported,
+ expires: time.Now().Add(minerUV1SupportCacheTTL),
+ })
+ return supported, nil
+}
+
+func applyMinerUAuth(req *http.Request, token string) {
+ if token != "" {
+ req.Header.Set("Authorization", "Bearer "+token)
+ }
+}
+
+func minerUJSON(ctx context.Context, client *http.Client, method, rawURL, token string, body any, dest any) error {
+ var reader io.Reader
+ if body != nil {
+ raw, err := json.Marshal(body)
+ if err != nil {
+ return err
+ }
+ reader = bytes.NewReader(raw)
+ }
+ req, err := http.NewRequestWithContext(ctx, method, rawURL, reader)
+ if err != nil {
+ return err
+ }
+ if body != nil {
+ req.Header.Set("Content-Type", "application/json")
+ }
+ applyMinerUAuth(req, token)
+ resp, err := client.Do(req)
+ if err != nil {
+ return err
+ }
+ defer resp.Body.Close()
+ payload, err := io.ReadAll(resp.Body)
+ if err != nil {
+ return fmt.Errorf("read %s %s: %w", method, rawURL, err)
+ }
+ if resp.StatusCode < 200 || resp.StatusCode >= 300 {
+ return fmt.Errorf("MinerU V1 %s %s: HTTP %d: %s", method, rawURL, resp.StatusCode, string(payload))
+ }
+ if dest == nil {
+ return nil
+ }
+ if err := json.Unmarshal(payload, dest); err != nil {
+ return fmt.Errorf("MinerU V1 decode %s: %w; body: %s", rawURL, err, string(payload))
+ }
+ return nil
+}
+
+func minerUSameOrigin(base, target string) bool {
+ bu, err := url.Parse(base)
+ if err != nil {
+ return false
+ }
+ tu, err := url.Parse(target)
+ if err != nil {
+ return false
+ }
+ port := func(u *url.URL) string {
+ if u.Port() != "" {
+ return u.Port()
+ }
+ if strings.EqualFold(u.Scheme, "https") {
+ return "443"
+ }
+ return "80"
+ }
+ return strings.EqualFold(bu.Scheme, tu.Scheme) &&
+ strings.EqualFold(bu.Hostname(), tu.Hostname()) &&
+ port(bu) == port(tu)
+}
+
+func minerUFilename(name string) string {
+ base := path.Base(strings.TrimSpace(name))
+ if base == "" || base == "." || base == "/" {
+ return "document.pdf"
+ }
+ return base
+}
+
+func minerUMimeType(filename string) string {
+ switch strings.ToLower(path.Ext(filename)) {
+ case ".pdf":
+ return "application/pdf"
+ case ".docx":
+ return "application/vnd.openxmlformats-officedocument.wordprocessingml.document"
+ case ".pptx":
+ return "application/vnd.openxmlformats-officedocument.presentationml.presentation"
+ case ".xlsx":
+ return "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"
+ case ".html", ".htm":
+ return "text/html"
+ default:
+ return "application/octet-stream"
+ }
+}
+
+func minerUV1Terminal(status string) bool {
+ switch status {
+ case "completed", "partial", "failed", "canceled":
+ return true
+ default:
+ return false
+ }
+}
+
+// ParseMinerUV1 runs the MinerU 4 V1 cycle: create upload → PUT bytes →
+// complete → parse job → poll → download zip and/or markdown.
+// ocrMode is the V1 ocr_mode (auto, txt, ocr); use MinerUV1OCRMode to normalize.
+func ParseMinerUV1(ctx context.Context, baseURL, apiKey, filename string, content []byte, backend, ocrMode string, timeout time.Duration) (*MinerUV1Result, error) {
+ baseURL = strings.TrimRight(strings.TrimSpace(baseURL), "/")
+ if baseURL == "" {
+ return nil, fmt.Errorf("MinerU base URL is empty")
+ }
+ if len(content) == 0 {
+ return nil, fmt.Errorf("MinerU V1 requires file content")
+ }
+ if timeout <= 0 {
+ timeout = DefaultMinerUV1ParseTimeout
+ }
+ ctx, cancel := context.WithTimeout(ctx, timeout)
+ defer cancel()
+
+ client := minerUHTTPClient()
+ token := minerUBearerFromAPIKey(apiKey)
+ name := minerUFilename(filename)
+ sum := sha256.Sum256(content)
+
+ var created minerUV1UploadResponse
+ if err := minerUJSON(ctx, client, http.MethodPost, baseURL+"/v1/uploads", token, map[string]any{
+ "filename": name,
+ "bytes": len(content),
+ "mime_type": minerUMimeType(name),
+ "purpose": "parse",
+ "sha256sum": hex.EncodeToString(sum[:]),
+ }, &created); err != nil {
+ return nil, fmt.Errorf("create upload: %w", err)
+ }
+ fileID := ""
+ if created.File != nil {
+ fileID = created.File.ID
+ }
+ switch created.Status {
+ case "completed":
+ if fileID == "" {
+ return nil, fmt.Errorf("MinerU V1 upload completed without file.id")
+ }
+ case "pending":
+ if created.UploadURL == "" {
+ return nil, fmt.Errorf("MinerU V1 upload missing upload_url")
+ }
+ uploadURL, err := url.Parse(created.UploadURL)
+ if err != nil {
+ return nil, fmt.Errorf("invalid upload_url: %w", err)
+ }
+ if !uploadURL.IsAbs() {
+ base, err := url.Parse(baseURL + "/")
+ if err != nil {
+ return nil, fmt.Errorf("invalid MinerU base URL: %w", err)
+ }
+ uploadURL = base.ResolveReference(uploadURL)
+ }
+ method := strings.ToUpper(strings.TrimSpace(created.UploadMethod))
+ if method == "" {
+ method = http.MethodPut
+ }
+ req, err := http.NewRequestWithContext(ctx, method, uploadURL.String(), bytes.NewReader(content))
+ if err != nil {
+ return nil, fmt.Errorf("create byte upload: %w", err)
+ }
+ for k, v := range created.UploadHeaders {
+ req.Header.Set(k, v)
+ }
+ if minerUSameOrigin(baseURL, uploadURL.String()) {
+ applyMinerUAuth(req, token)
+ }
+ resp, err := client.Do(req)
+ if err != nil {
+ return nil, fmt.Errorf("byte upload: %w", err)
+ }
+ _, _ = io.Copy(io.Discard, resp.Body)
+ resp.Body.Close()
+ if resp.StatusCode < 200 || resp.StatusCode >= 300 {
+ return nil, fmt.Errorf("byte upload: HTTP %d", resp.StatusCode)
+ }
+ var completed minerUV1UploadResponse
+ if err := minerUJSON(ctx, client, http.MethodPost, baseURL+"/v1/uploads/"+created.ID+"/complete", token, nil, &completed); err != nil {
+ return nil, fmt.Errorf("complete upload: %w", err)
+ }
+ if completed.File == nil || completed.File.ID == "" {
+ return nil, fmt.Errorf("MinerU V1 complete upload missing file.id")
+ }
+ fileID = completed.File.ID
+ default:
+ return nil, fmt.Errorf("MinerU V1 upload status %q", created.Status)
+ }
+
+ jobBody := map[string]any{
+ "files": []map[string]any{
+ {"source": map[string]any{"type": "file_id", "file_id": fileID}},
+ },
+ "tier": MinerUTierFromBackend(backend),
+ "output_formats": []string{"zip", "markdown"},
+ "ocr_mode": MinerUV1OCRMode(ocrMode),
+ }
+ var job minerUV1JobResponse
+ if err := minerUJSON(ctx, client, http.MethodPost, baseURL+"/v1/parse/jobs", token, jobBody, &job); err != nil {
+ return nil, fmt.Errorf("create parse job: %w", err)
+ }
+ if job.JobID == "" {
+ return nil, fmt.Errorf("MinerU V1 parse job missing job_id")
+ }
+
+ pollWait := minerUV1PollInitial
+ for !minerUV1Terminal(job.Status) {
+ select {
+ case <-ctx.Done():
+ return nil, ctx.Err()
+ case <-time.After(pollWait):
+ }
+ if pollWait < minerUV1PollMax {
+ pollWait *= 2
+ if pollWait > minerUV1PollMax {
+ pollWait = minerUV1PollMax
+ }
+ }
+ if err := minerUJSON(ctx, client, http.MethodGet, baseURL+"/v1/parse/jobs/"+job.JobID, token, nil, &job); err != nil {
+ return nil, fmt.Errorf("poll parse job: %w", err)
+ }
+ }
+ switch job.Status {
+ case "failed", "canceled":
+ msg := job.Status
+ if job.Error != nil && job.Error.Message != "" {
+ msg = job.Error.Message
+ }
+ return nil, fmt.Errorf("MinerU V1 job %s: %s", job.Status, msg)
+ }
+
+ result := &MinerUV1Result{}
+ var artifactErr error
+ for _, file := range job.Files {
+ if file.Status != "" && file.Status != "completed" {
+ continue
+ }
+ if ref, ok := file.OutputFiles["zip"]; ok && ref.artifactID() != "" && len(result.Zip) == 0 {
+ body, err := minerUV1DownloadFile(ctx, client, baseURL, token, ref.artifactID())
+ if err != nil {
+ if artifactErr == nil {
+ artifactErr = fmt.Errorf("download zip: %w", err)
+ }
+ } else {
+ result.Zip = body
+ }
+ }
+ if ref, ok := file.OutputFiles["markdown"]; ok && ref.artifactID() != "" && result.Markdown == "" {
+ body, err := minerUV1DownloadFile(ctx, client, baseURL, token, ref.artifactID())
+ if err != nil {
+ if artifactErr == nil {
+ artifactErr = fmt.Errorf("download markdown: %w", err)
+ }
+ } else {
+ result.Markdown = string(body)
+ }
+ }
+ }
+ if result.Markdown == "" && len(result.Zip) > 0 {
+ if md, err := minerUMarkdownFromZip(result.Zip); err == nil {
+ result.Markdown = md
+ }
+ }
+ if len(result.Zip) == 0 && strings.TrimSpace(result.Markdown) == "" {
+ if artifactErr != nil {
+ return nil, artifactErr
+ }
+ return nil, minerUV1NoArtifactsError(job)
+ }
+ return result, nil
+}
+
+func minerUV1NoArtifactsError(job minerUV1JobResponse) error {
+ msg := fmt.Sprintf("MinerU V1 job %s (%s) completed without zip or markdown artifacts", job.JobID, job.Status)
+ if fileMsg := minerUV1FirstFileErrorMessage(job.Files); fileMsg != "" {
+ msg += ": " + fileMsg
+ }
+ return fmt.Errorf("%s", msg)
+}
+
+func minerUV1FirstFileErrorMessage(files []minerUV1JobFile) string {
+ for _, file := range files {
+ if file.Error != nil && strings.TrimSpace(file.Error.Message) != "" {
+ return file.Error.Message
+ }
+ }
+ return ""
+}
+
+func minerUV1DownloadHTTPClient(apiOrigin string) *http.Client {
+ base := minerUHTTPClient()
+ return &http.Client{
+ Transport: base.Transport,
+ CheckRedirect: func(req *http.Request, via []*http.Request) error {
+ if len(via) >= 10 {
+ return fmt.Errorf("too many redirects")
+ }
+ if !minerUSameOrigin(apiOrigin, req.URL.String()) {
+ req.Header.Del("Authorization")
+ }
+ return nil
+ },
+ }
+}
+
+func minerUV1DownloadFile(ctx context.Context, client *http.Client, baseURL, token, fileID string) ([]byte, error) {
+ apiOrigin := strings.TrimRight(strings.TrimSpace(baseURL), "/")
+ dlClient := minerUV1DownloadHTTPClient(apiOrigin)
+ req, err := http.NewRequestWithContext(ctx, http.MethodGet, apiOrigin+"/v1/files/"+fileID+"/content", nil)
+ if err != nil {
+ return nil, err
+ }
+ applyMinerUAuth(req, token)
+ resp, err := dlClient.Do(req)
+ if err != nil {
+ return nil, err
+ }
+ defer resp.Body.Close()
+ body, err := io.ReadAll(resp.Body)
+ if err != nil {
+ return nil, err
+ }
+ if resp.StatusCode < 200 || resp.StatusCode >= 300 {
+ return nil, fmt.Errorf("HTTP %d: %s", resp.StatusCode, string(body))
+ }
+ return body, nil
+}
+
+// minerUMarkdownFromZip prefers a .md member, then concatenates content_list.json text.
+func minerUMarkdownFromZip(zipBytes []byte) (string, error) {
+ zipReader, err := zip.NewReader(bytes.NewReader(zipBytes), int64(len(zipBytes)))
+ if err != nil {
+ return "", fmt.Errorf("open zip: %w", err)
+ }
+ var md string
+ var contentList []byte
+ for _, f := range zipReader.File {
+ name := strings.ReplaceAll(f.Name, "\\", "/")
+ if strings.Contains(name, "__MACOSX") {
+ continue
+ }
+ rc, err := f.Open()
+ if err != nil {
+ continue
+ }
+ body, _ := io.ReadAll(rc)
+ rc.Close()
+ lower := strings.ToLower(name)
+ if md == "" && strings.HasSuffix(lower, ".md") {
+ md = string(body)
+ }
+ if strings.HasSuffix(lower, "content_list.json") {
+ contentList = body
+ }
+ }
+ if strings.TrimSpace(md) != "" {
+ return md, nil
+ }
+ if len(contentList) == 0 {
+ return "", fmt.Errorf("markdown and content_list.json not found in MinerU zip")
+ }
+ var items []map[string]any
+ if err := json.Unmarshal(contentList, &items); err != nil {
+ return "", fmt.Errorf("parse content_list.json: %w", err)
+ }
+ var parts []string
+ for _, item := range items {
+ if text, ok := item["text"].(string); ok && strings.TrimSpace(text) != "" {
+ parts = append(parts, text)
+ continue
+ }
+ if tb, ok := item["table_body"].(string); ok && strings.TrimSpace(tb) != "" {
+ parts = append(parts, tb)
+ }
+ }
+ if len(parts) == 0 {
+ return "", fmt.Errorf("content_list.json had no text")
+ }
+ return strings.Join(parts, "\n"), nil
+}
diff --git a/internal/entity/models/mineru_v1_test.go b/internal/entity/models/mineru_v1_test.go
new file mode 100644
index 0000000000..be2e632a42
--- /dev/null
+++ b/internal/entity/models/mineru_v1_test.go
@@ -0,0 +1,322 @@
+//
+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
+//
+
+package models
+
+import (
+ "archive/zip"
+ "bytes"
+ "context"
+ "encoding/json"
+ "io"
+ "net/http"
+ "net/http/httptest"
+ "strings"
+ "sync"
+ "sync/atomic"
+ "testing"
+ "time"
+)
+
+func TestMinerUSupportsV1(t *testing.T) {
+ t.Run("v1 health", func(t *testing.T) {
+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ if r.URL.Path != "/v1/health" {
+ http.NotFound(w, r)
+ return
+ }
+ w.WriteHeader(http.StatusOK)
+ _, _ = w.Write([]byte(`{"status":"ok"}`))
+ }))
+ defer server.Close()
+ ok, err := MinerUSupportsV1(context.Background(), server.URL, "")
+ if err != nil || !ok {
+ t.Fatalf("SupportsV1 = %v, %v, want true", ok, err)
+ }
+ })
+ t.Run("legacy file_parse only", func(t *testing.T) {
+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ http.NotFound(w, r)
+ }))
+ defer server.Close()
+ ok, err := MinerUSupportsV1(context.Background(), server.URL, "")
+ if err != nil {
+ t.Fatalf("SupportsV1: %v", err)
+ }
+ if ok {
+ t.Fatal("404 /v1/health should be treated as 3.x")
+ }
+ })
+}
+
+func TestParseMinerUV1UploadJobDownload(t *testing.T) {
+ orig := minerUV1PollInitial
+ minerUV1PollInitial = time.Millisecond
+ t.Cleanup(func() { minerUV1PollInitial = orig })
+
+ var (
+ mu sync.Mutex
+ gotBytes []byte
+ gotTier string
+ gotOCR string
+ polls int
+ )
+ zipBuf := minerUTestZip(t, "doc.md", "# Hello\n\nFrom V1.\n")
+
+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ switch {
+ case r.Method == http.MethodGet && r.URL.Path == "/v1/health":
+ w.WriteHeader(http.StatusOK)
+ _, _ = w.Write([]byte(`{"status":"ok"}`))
+ case r.Method == http.MethodPost && r.URL.Path == "/v1/uploads":
+ w.Header().Set("Content-Type", "application/json")
+ _, _ = w.Write([]byte(`{"id":"up-1","status":"pending","upload_url":"/v1/uploads/up-1/content","upload_method":"PUT"}`))
+ case r.Method == http.MethodPut && r.URL.Path == "/v1/uploads/up-1/content":
+ body, _ := io.ReadAll(r.Body)
+ mu.Lock()
+ gotBytes = body
+ mu.Unlock()
+ w.WriteHeader(http.StatusOK)
+ case r.Method == http.MethodPost && r.URL.Path == "/v1/uploads/up-1/complete":
+ w.Header().Set("Content-Type", "application/json")
+ _, _ = w.Write([]byte(`{"id":"up-1","status":"completed","file":{"id":"file-1"}}`))
+ case r.Method == http.MethodPost && r.URL.Path == "/v1/parse/jobs":
+ var payload map[string]any
+ _ = json.NewDecoder(r.Body).Decode(&payload)
+ if tier, ok := payload["tier"].(string); ok {
+ mu.Lock()
+ gotTier = tier
+ mu.Unlock()
+ }
+ if mode, ok := payload["ocr_mode"].(string); ok {
+ mu.Lock()
+ gotOCR = mode
+ mu.Unlock()
+ }
+ w.Header().Set("Content-Type", "application/json")
+ _, _ = w.Write([]byte(`{"job_id":"job-1","status":"queued"}`))
+ case r.Method == http.MethodGet && r.URL.Path == "/v1/parse/jobs/job-1":
+ mu.Lock()
+ polls++
+ n := polls
+ mu.Unlock()
+ w.Header().Set("Content-Type", "application/json")
+ if n < 2 {
+ _, _ = w.Write([]byte(`{"job_id":"job-1","status":"running"}`))
+ return
+ }
+ _, _ = w.Write([]byte(`{"job_id":"job-1","status":"completed","files":[{"name":"doc.pdf","status":"completed","output_files":{"zip":{"file_id":"out-zip"},"markdown":{"file_id":"out-md"}}}]}`))
+ case r.Method == http.MethodGet && r.URL.Path == "/v1/files/out-zip/content":
+ w.Header().Set("Content-Type", "application/zip")
+ _, _ = w.Write(zipBuf)
+ case r.Method == http.MethodGet && r.URL.Path == "/v1/files/out-md/content":
+ _, _ = w.Write([]byte("# Hello\n\nFrom V1.\n"))
+ default:
+ http.NotFound(w, r)
+ }
+ }))
+ defer server.Close()
+
+ result, err := ParseMinerUV1(context.Background(), server.URL, "secret", "sample.pdf", []byte("%PDF-1.4"), "pipeline", "ocr", 5*time.Second)
+ if err != nil {
+ t.Fatalf("ParseMinerUV1: %v", err)
+ }
+ if result.Markdown != "# Hello\n\nFrom V1.\n" {
+ t.Fatalf("markdown = %q", result.Markdown)
+ }
+ if len(result.Zip) == 0 {
+ t.Fatal("expected zip artifact")
+ }
+ mu.Lock()
+ defer mu.Unlock()
+ if !bytes.Equal(gotBytes, []byte("%PDF-1.4")) {
+ t.Fatalf("uploaded bytes = %q", gotBytes)
+ }
+ if gotTier != "basic" {
+ t.Fatalf("tier = %q, want basic (mapped from pipeline)", gotTier)
+ }
+ if gotOCR != "ocr" {
+ t.Fatalf("ocr_mode = %q, want ocr", gotOCR)
+ }
+}
+
+func TestMinerUV1DownloadStripsAuthOnCrossOriginRedirect(t *testing.T) {
+ var sawAuth atomic.Bool
+ redirect := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ if r.Header.Get("Authorization") != "" {
+ sawAuth.Store(true)
+ }
+ _, _ = w.Write([]byte("artifact"))
+ }))
+ defer redirect.Close()
+
+ api := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ if r.URL.Path == "/v1/files/out-1/content" {
+ http.Redirect(w, r, redirect.URL, http.StatusFound)
+ return
+ }
+ http.NotFound(w, r)
+ }))
+ defer api.Close()
+
+ body, err := minerUV1DownloadFile(context.Background(), minerUHTTPClient(), api.URL, "secret-token", "out-1")
+ if err != nil {
+ t.Fatalf("minerUV1DownloadFile: %v", err)
+ }
+ if body == nil || string(body) != "artifact" {
+ t.Fatalf("body = %q", body)
+ }
+ if sawAuth.Load() {
+ t.Fatal("Authorization must not be forwarded to a cross-origin redirect target")
+ }
+}
+
+func TestParseMinerUV1OutputFilesUseFileID(t *testing.T) {
+ orig := minerUV1PollInitial
+ minerUV1PollInitial = time.Millisecond
+ t.Cleanup(func() { minerUV1PollInitial = orig })
+
+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ switch {
+ case r.Method == http.MethodPost && r.URL.Path == "/v1/uploads":
+ _, _ = w.Write([]byte(`{"id":"up-1","status":"completed","file":{"id":"file-1"}}`))
+ case r.Method == http.MethodPost && r.URL.Path == "/v1/parse/jobs":
+ _, _ = w.Write([]byte(`{"job_id":"job-1","status":"completed","files":[{"name":"doc.pdf","status":"completed","output_files":{"markdown":{"file_id":"out-md"}}}]}`))
+ case r.Method == http.MethodGet && r.URL.Path == "/v1/files/out-md/content":
+ _, _ = w.Write([]byte("# From file_id\n"))
+ default:
+ http.NotFound(w, r)
+ }
+ }))
+ defer server.Close()
+
+ result, err := ParseMinerUV1(context.Background(), server.URL, "secret", "sample.pdf", []byte("%PDF-1.4"), "pipeline", "auto", 5*time.Second)
+ if err != nil {
+ t.Fatalf("ParseMinerUV1: %v", err)
+ }
+ if result.Markdown != "# From file_id\n" {
+ t.Fatalf("markdown = %q", result.Markdown)
+ }
+}
+
+func TestParseMinerUV1PartialJobIncludesFileError(t *testing.T) {
+ orig := minerUV1PollInitial
+ minerUV1PollInitial = time.Millisecond
+ t.Cleanup(func() { minerUV1PollInitial = orig })
+
+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ switch {
+ case r.Method == http.MethodPost && r.URL.Path == "/v1/uploads":
+ _, _ = w.Write([]byte(`{"id":"up-1","status":"completed","file":{"id":"file-1"}}`))
+ case r.Method == http.MethodPost && r.URL.Path == "/v1/parse/jobs":
+ _, _ = w.Write([]byte(`{"job_id":"job-1","status":"partial","files":[{"name":"doc.pdf","status":"failed","error":{"message":"page render failed"}}]}`))
+ default:
+ http.NotFound(w, r)
+ }
+ }))
+ defer server.Close()
+
+ _, err := ParseMinerUV1(context.Background(), server.URL, "secret", "sample.pdf", []byte("%PDF-1.4"), "pipeline", "auto", 5*time.Second)
+ if err == nil {
+ t.Fatal("expected error for partial job without artifacts")
+ }
+ if !strings.Contains(err.Error(), "partial") || !strings.Contains(err.Error(), "page render failed") {
+ t.Fatalf("error = %q, want partial status and file error", err.Error())
+ }
+}
+
+func TestParseMinerUV1ZipDownloadFailureFallsBackToMarkdown(t *testing.T) {
+ orig := minerUV1PollInitial
+ minerUV1PollInitial = time.Millisecond
+ t.Cleanup(func() { minerUV1PollInitial = orig })
+
+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ switch {
+ case r.Method == http.MethodPost && r.URL.Path == "/v1/uploads":
+ _, _ = w.Write([]byte(`{"id":"up-1","status":"completed","file":{"id":"file-1"}}`))
+ case r.Method == http.MethodPost && r.URL.Path == "/v1/parse/jobs":
+ _, _ = w.Write([]byte(`{"job_id":"job-1","status":"completed","files":[{"name":"doc.pdf","status":"completed","output_files":{"zip":{"file_id":"out-zip"},"markdown":{"file_id":"out-md"}}}]}`))
+ case r.Method == http.MethodGet && r.URL.Path == "/v1/files/out-zip/content":
+ http.Error(w, "zip gone", http.StatusInternalServerError)
+ case r.Method == http.MethodGet && r.URL.Path == "/v1/files/out-md/content":
+ _, _ = w.Write([]byte("# Fallback markdown\n"))
+ default:
+ http.NotFound(w, r)
+ }
+ }))
+ defer server.Close()
+
+ result, err := ParseMinerUV1(context.Background(), server.URL, "secret", "sample.pdf", []byte("%PDF-1.4"), "pipeline", "auto", 5*time.Second)
+ if err != nil {
+ t.Fatalf("ParseMinerUV1: %v", err)
+ }
+ if result.Markdown != "# Fallback markdown\n" {
+ t.Fatalf("markdown = %q, want fallback artifact", result.Markdown)
+ }
+ if len(result.Zip) != 0 {
+ t.Fatalf("zip = %d bytes, want empty after download failure", len(result.Zip))
+ }
+}
+
+func TestMinerUMarkdownFromZip(t *testing.T) {
+ buf := minerUTestZip(t, "out.md", "# Title\n")
+ md, err := minerUMarkdownFromZip(buf)
+ if err != nil {
+ t.Fatalf("minerUMarkdownFromZip: %v", err)
+ }
+ if md != "# Title\n" {
+ t.Fatalf("md = %q", md)
+ }
+}
+
+func TestMinerULocalCheckConnectionPrefersV1Health(t *testing.T) {
+ var paths []string
+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ paths = append(paths, r.URL.Path)
+ if r.URL.Path == "/v1/health" {
+ w.WriteHeader(http.StatusOK)
+ return
+ }
+ http.NotFound(w, r)
+ }))
+ defer server.Close()
+
+ baseURL := server.URL
+ driver := NewMinerLocalUModel(map[string]string{"default": baseURL}, URLSuffix{DocumentParse: "file_parse", Task: "tasks"})
+ if err := driver.CheckConnection(context.Background(), &APIConfig{BaseURL: &baseURL}); err != nil {
+ t.Fatalf("CheckConnection: %v", err)
+ }
+ if len(paths) != 1 || paths[0] != "/v1/health" {
+ t.Fatalf("paths = %v, want only /v1/health", paths)
+ }
+}
+
+func minerUTestZip(t *testing.T, name, body string) []byte {
+ t.Helper()
+ var buf bytes.Buffer
+ zw := zip.NewWriter(&buf)
+ w, err := zw.Create(name)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if _, err := io.WriteString(w, body); err != nil {
+ t.Fatal(err)
+ }
+ if err := zw.Close(); err != nil {
+ t.Fatal(err)
+ }
+ return buf.Bytes()
+}
+
+func TestMinerUSameOrigin(t *testing.T) {
+ if !minerUSameOrigin("http://127.0.0.1:8000", "http://127.0.0.1:8000/v1/uploads/1/content") {
+ t.Fatal("same host/port should be same-origin")
+ }
+ if minerUSameOrigin("http://127.0.0.1:8000", "http://127.0.0.1:9000/v1/uploads/1/content") {
+ t.Fatal("different port is not same-origin")
+ }
+ if strings.TrimSpace("") != "" {
+ t.Fatal("sanity")
+ }
+}
diff --git a/internal/ingestion/component/parser_dispatch_test.go b/internal/ingestion/component/parser_dispatch_test.go
index 5863baf638..61c1cc0b75 100644
--- a/internal/ingestion/component/parser_dispatch_test.go
+++ b/internal/ingestion/component/parser_dispatch_test.go
@@ -924,9 +924,13 @@ func TestDispatch_PDFMinerUMarkdown_SendsServerURLFromProviderConfig(t *testing.
}
func TestDispatch_PDFMinerUMarkdown_RequiresServerURLForHTTPClientBackend(t *testing.T) {
+ withSSRFBypass(t)
+ server := httptest.NewServer(http.NotFoundHandler())
+ defer server.Close()
+
origResolver := resolveMinerUModelForDispatch
defer func() { resolveMinerUModelForDispatch = origResolver }()
- baseURL := "http://mineru-api:8888"
+ baseURL := server.URL
apiKey := `{"mineru_backend":"hybrid-http-client"}`
resolveMinerUModelForDispatch = func(ctx context.Context, db *gorm.DB, tenantID, modelID string) (models.ModelDriver, string, *models.APIConfig, error) {
return &mineruTestDriver{}, "mineru-model", &models.APIConfig{ApiKey: &apiKey, BaseURL: &baseURL}, nil
@@ -950,6 +954,192 @@ func TestDispatch_PDFMinerUMarkdown_RequiresServerURLForHTTPClientBackend(t *tes
}
}
+func TestDispatch_PDFMinerUV1_WiresParseAndExtract(t *testing.T) {
+ withSSRFBypass(t)
+
+ invoke := func(t *testing.T, serverURL string) map[string]any {
+ t.Helper()
+ origResolver := resolveMinerUModelForDispatch
+ t.Cleanup(func() { resolveMinerUModelForDispatch = origResolver })
+ baseURL := serverURL
+ apiKey := ""
+ resolveMinerUModelForDispatch = func(ctx context.Context, db *gorm.DB, tenantID, modelID string) (models.ModelDriver, string, *models.APIConfig, error) {
+ return &mineruTestDriver{}, "mineru-model", &models.APIConfig{ApiKey: &apiKey, BaseURL: &baseURL}, nil
+ }
+ setups := defaultSetups()
+ setups["pdf"]["parse_method"] = "mineru"
+ setups["pdf"]["output_format"] = "markdown"
+ setups["pdf"]["mineru_backend"] = "standard"
+ return invokeMinerUDispatch(t, nil, setups)
+ }
+
+ t.Run("zip content_list uses mineruExtractJSONItems", func(t *testing.T) {
+ var zipBuf bytes.Buffer
+ zw := zip.NewWriter(&zipBuf)
+ f, _ := zw.Create("content_list.json")
+ _, _ = f.Write([]byte(`[{"type":"text","text":"V1 body"},{"type":"table","table_body":"<table></table>","table_caption":["Table caption"]}]`))
+ _ = zw.Close()
+ zipBytes := zipBuf.Bytes()
+
+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ switch {
+ case r.Method == http.MethodGet && r.URL.Path == "/v1/health":
+ w.WriteHeader(http.StatusOK)
+ case r.Method == http.MethodPost && r.URL.Path == "/v1/uploads":
+ _, _ = w.Write([]byte(`{"id":"up-1","status":"completed","file":{"id":"file-1"}}`))
+ case r.Method == http.MethodPost && r.URL.Path == "/v1/parse/jobs":
+ _, _ = w.Write([]byte(`{"job_id":"job-1","status":"completed","files":[{"name":"sample.pdf","status":"completed","output_files":{"zip":{"file_id":"out-zip"}}}]}`))
+ case r.Method == http.MethodGet && r.URL.Path == "/v1/files/out-zip/content":
+ w.Header().Set("Content-Type", "application/zip")
+ _, _ = w.Write(zipBytes)
+ default:
+ http.NotFound(w, r)
+ }
+ }))
+ defer server.Close()
+
+ out := invoke(t, server.URL)
+ requireJSONText(t, out, "V1 body")
+ requireJSONText(t, out, "Table caption")
+ })
+
+ t.Run("markdown fallback when zip has no content_list", func(t *testing.T) {
+ var zipBuf bytes.Buffer
+ zw := zip.NewWriter(&zipBuf)
+ f, _ := zw.Create("middle.json")
+ _, _ = f.Write([]byte(`{}`))
+ _ = zw.Close()
+ zipBytes := zipBuf.Bytes()
+
+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ switch {
+ case r.Method == http.MethodGet && r.URL.Path == "/v1/health":
+ w.WriteHeader(http.StatusOK)
+ case r.Method == http.MethodPost && r.URL.Path == "/v1/uploads":
+ _, _ = w.Write([]byte(`{"id":"up-1","status":"completed","file":{"id":"file-1"}}`))
+ case r.Method == http.MethodPost && r.URL.Path == "/v1/parse/jobs":
+ _, _ = w.Write([]byte(`{"job_id":"job-1","status":"completed","files":[{"name":"sample.pdf","status":"completed","output_files":{"zip":{"file_id":"out-zip"},"markdown":{"file_id":"out-md"}}}]}`))
+ case r.Method == http.MethodGet && r.URL.Path == "/v1/files/out-zip/content":
+ w.Header().Set("Content-Type", "application/zip")
+ _, _ = w.Write(zipBytes)
+ case r.Method == http.MethodGet && r.URL.Path == "/v1/files/out-md/content":
+ _, _ = w.Write([]byte("# Zip had no content_list\n"))
+ default:
+ http.NotFound(w, r)
+ }
+ }))
+ defer server.Close()
+
+ out := invoke(t, server.URL)
+ requireJSONText(t, out, "Zip had no content_list")
+ })
+
+ t.Run("markdown fallback when zip sections are blank", func(t *testing.T) {
+ var zipBuf bytes.Buffer
+ zw := zip.NewWriter(&zipBuf)
+ f, _ := zw.Create("content_list.json")
+ _, _ = f.Write([]byte(`[{"type":"text","text":""}]`))
+ _ = zw.Close()
+ zipBytes := zipBuf.Bytes()
+
+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ switch {
+ case r.Method == http.MethodGet && r.URL.Path == "/v1/health":
+ w.WriteHeader(http.StatusOK)
+ case r.Method == http.MethodPost && r.URL.Path == "/v1/uploads":
+ _, _ = w.Write([]byte(`{"id":"up-1","status":"completed","file":{"id":"file-1"}}`))
+ case r.Method == http.MethodPost && r.URL.Path == "/v1/parse/jobs":
+ _, _ = w.Write([]byte(`{"job_id":"job-1","status":"completed","files":[{"name":"sample.pdf","status":"completed","output_files":{"zip":{"file_id":"out-zip"},"markdown":{"file_id":"out-md"}}}]}`))
+ case r.Method == http.MethodGet && r.URL.Path == "/v1/files/out-zip/content":
+ w.Header().Set("Content-Type", "application/zip")
+ _, _ = w.Write(zipBytes)
+ case r.Method == http.MethodGet && r.URL.Path == "/v1/files/out-md/content":
+ _, _ = w.Write([]byte("# Blank zip sections\n"))
+ default:
+ http.NotFound(w, r)
+ }
+ }))
+ defer server.Close()
+
+ out := invoke(t, server.URL)
+ requireJSONText(t, out, "Blank zip sections")
+ })
+
+ t.Run("markdown fallback when zip download fails", func(t *testing.T) {
+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ switch {
+ case r.Method == http.MethodGet && r.URL.Path == "/v1/health":
+ w.WriteHeader(http.StatusOK)
+ case r.Method == http.MethodPost && r.URL.Path == "/v1/uploads":
+ _, _ = w.Write([]byte(`{"id":"up-1","status":"completed","file":{"id":"file-1"}}`))
+ case r.Method == http.MethodPost && r.URL.Path == "/v1/parse/jobs":
+ _, _ = w.Write([]byte(`{"job_id":"job-1","status":"completed","files":[{"name":"sample.pdf","status":"completed","output_files":{"zip":{"file_id":"out-zip"},"markdown":{"file_id":"out-md"}}}]}`))
+ case r.Method == http.MethodGet && r.URL.Path == "/v1/files/out-zip/content":
+ http.Error(w, "zip gone", http.StatusInternalServerError)
+ case r.Method == http.MethodGet && r.URL.Path == "/v1/files/out-md/content":
+ _, _ = w.Write([]byte("# Markdown fallback\n"))
+ default:
+ http.NotFound(w, r)
+ }
+ }))
+ defer server.Close()
+
+ out := invoke(t, server.URL)
+ requireJSONText(t, out, "Markdown fallback")
+ })
+
+ t.Run("empty content_list without markdown is an error", func(t *testing.T) {
+ var zipBuf bytes.Buffer
+ zw := zip.NewWriter(&zipBuf)
+ f, _ := zw.Create("content_list.json")
+ _, _ = f.Write([]byte(`[]`))
+ _ = zw.Close()
+ zipBytes := zipBuf.Bytes()
+
+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ switch {
+ case r.Method == http.MethodGet && r.URL.Path == "/v1/health":
+ w.WriteHeader(http.StatusOK)
+ case r.Method == http.MethodPost && r.URL.Path == "/v1/uploads":
+ _, _ = w.Write([]byte(`{"id":"up-1","status":"completed","file":{"id":"file-1"}}`))
+ case r.Method == http.MethodPost && r.URL.Path == "/v1/parse/jobs":
+ _, _ = w.Write([]byte(`{"job_id":"job-1","status":"completed","files":[{"name":"sample.pdf","status":"completed","output_files":{"zip":{"file_id":"out-zip"}}}]}`))
+ case r.Method == http.MethodGet && r.URL.Path == "/v1/files/out-zip/content":
+ w.Header().Set("Content-Type", "application/zip")
+ _, _ = w.Write(zipBytes)
+ default:
+ http.NotFound(w, r)
+ }
+ }))
+ defer server.Close()
+
+ origResolver := resolveMinerUModelForDispatch
+ t.Cleanup(func() { resolveMinerUModelForDispatch = origResolver })
+ baseURL := server.URL
+ apiKey := ""
+ resolveMinerUModelForDispatch = func(ctx context.Context, db *gorm.DB, tenantID, modelID string) (models.ModelDriver, string, *models.APIConfig, error) {
+ return &mineruTestDriver{}, "mineru-model", &models.APIConfig{ApiKey: &apiKey, BaseURL: &baseURL}, nil
+ }
+ setups := defaultSetups()
+ setups["pdf"]["parse_method"] = "mineru"
+ setups["pdf"]["output_format"] = "markdown"
+ setups["pdf"]["mineru_backend"] = "standard"
+ c := &ParserComponent{setups: setups}
+ _, err := c.Invoke(t.Context(), nil, map[string]any{
+ "binary": []byte("%PDF-1.4"),
+ "file_type": "pdf",
+ "name": "sample.pdf",
+ "tenant_id": "test-tenant",
+ })
+ if err == nil {
+ t.Fatal("Invoke: want error when V1 zip yields no sections")
+ }
+ if !strings.Contains(err.Error(), "no usable content_list") && !strings.Contains(err.Error(), "no sections") {
+ t.Fatalf("error = %q, want empty content_list extract error", err.Error())
+ }
+ })
+}
+
func TestDispatch_PDFMonkeyOCRv2Markdown_UsesNativeParseEndpoint(t *testing.T) {
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.Method != http.MethodPost || r.URL.Path != "/parse" {
diff --git a/internal/ingestion/component/pdf_vision_dispatch.go b/internal/ingestion/component/pdf_vision_dispatch.go
index d0b141a1c7..9cd60c1fc8 100644
--- a/internal/ingestion/component/pdf_vision_dispatch.go
+++ b/internal/ingestion/component/pdf_vision_dispatch.go
@@ -482,10 +482,9 @@ func getAnyString(object map[string]any, keys ...string) string {
return ""
}
-// dispatchMinerUPDF submits a PDF to the selected MinerU OCR model
-// via the streaming /file_parse endpoint and returns parsed sections.
-// Mirrors Python's mineru_parser.py:parse_PDF which POSTs with
-// stream=True and reads the zip response body directly (no polling).
+// dispatchMinerUPDF submits a PDF to the selected MinerU OCR model.
+// MinerU 4.0 uses the V1 upload/job API; 3.x still POSTs /file_parse with
+// stream=True and returns a zip body (Python mineru_parser.py:parse_PDF).
func dispatchMinerUPDF(
ctx context.Context,
db *gorm.DB,
@@ -516,7 +515,6 @@ func dispatchMinerUPDF(
if strings.TrimSpace(baseURL) == "" {
return parser.ParseResult{}, fmt.Errorf("parser: MinerU requires a base URL (instance base_url, mineru_apiserver, or MINERU_APISERVER)")
}
- apiURL := strings.TrimRight(baseURL, "/") + "/file_parse"
parseMethod := mineruAPIParseMethod(getStringOr(setup, "mineru_parse_method", ""))
// MinerU's language is an OCR engine parameter, not the caption language:
@@ -531,10 +529,40 @@ func dispatchMinerUPDF(
mineruLang := mineruLangCode(lang)
backend := modelModule.ResolveMinerUBackend(getStringOr(setup, "mineru_backend", ""), apiKeyRaw)
serverURL := modelModule.ResolveMinerUServerURL(getStringOr(setup, "mineru_server_url", ""), apiKeyRaw)
- if err := modelModule.ValidateMinerUConfig(backend, serverURL); err != nil {
+
+ v1, err := modelModule.MinerUSupportsV1(ctx, baseURL, apiKeyRaw)
+ if err != nil {
+ return parser.ParseResult{}, fmt.Errorf("parser: MinerU probe: %w", err)
+ }
+ if err := modelModule.ValidateMinerUConfigForAPI(backend, serverURL, v1); err != nil {
return parser.ParseResult{}, err
}
+ if v1 {
+ result, err := modelModule.ParseMinerUV1(ctx, baseURL, apiKeyRaw, filename, binary, backend, parseMethod, 0)
+ if err != nil {
+ return parser.ParseResult{}, fmt.Errorf("parser: MinerU V1: %w", err)
+ }
+ v1Markdown := ""
+ var zipBytes []byte
+ if result != nil {
+ v1Markdown = strings.TrimSpace(result.Markdown)
+ zipBytes = result.Zip
+ }
+ items, extractErr := mineruExtractJSONItems(zipBytes)
+ if extractErr == nil && len(items) > 0 {
+ return mineruJSONParseResult(filename, items), nil
+ }
+ if v1Markdown != "" {
+ return buildMarkdownOCRDispatchResult(result.Markdown), nil
+ }
+ if extractErr != nil {
+ return parser.ParseResult{}, fmt.Errorf("parser: MinerU extract: %w", extractErr)
+ }
+ return parser.ParseResult{}, fmt.Errorf("parser: MinerU extract: no sections")
+ }
+
+ apiURL := strings.TrimRight(baseURL, "/") + "/file_parse"
zipBytes, err := mineruStreamParse(apiURL, apiKeyRaw, binary, parseMethod, mineruLang, backend, serverURL)
if err != nil {
return parser.ParseResult{}, fmt.Errorf("parser: MinerU stream: %w", err)
@@ -545,16 +573,7 @@ func dispatchMinerUPDF(
return parser.ParseResult{}, fmt.Errorf("parser: MinerU extract: %w", err)
}
- return parser.ParseResult{
- OutputFormat: "json",
- File: map[string]any{
- "name": filename,
- "page_count": mineruItemPageCount(items),
- "outline": []map[string]any{},
- "parse_method": "mineru",
- },
- JSON: items,
- }, nil
+ return mineruJSONParseResult(filename, items), nil
}
// resolveMinerUModelForDispatch resolves the OCR model used by the MinerU PDF
@@ -1028,6 +1047,19 @@ func mineruZipImageDataURI(images map[string]*zip.File, imgPath string) string {
return fmt.Sprintf("data:%s;base64,%s", mediaType, base64.StdEncoding.EncodeToString(data))
}
+func mineruJSONParseResult(filename string, items []map[string]any) parser.ParseResult {
+ return parser.ParseResult{
+ OutputFormat: "json",
+ File: map[string]any{
+ "name": filename,
+ "page_count": mineruItemPageCount(items),
+ "outline": []map[string]any{},
+ "parse_method": "mineru",
+ },
+ JSON: items,
+ }
+}
+
func mineruItemPageCount(items []map[string]any) int {
maxPage := 0
for _, item := range items {
diff --git a/internal/parser/parser/pdf_parser_mineru_test.go b/internal/parser/parser/pdf_parser_mineru_test.go
index 0cdb2f5ec0..87304f7090 100644
--- a/internal/parser/parser/pdf_parser_mineru_test.go
+++ b/internal/parser/parser/pdf_parser_mineru_test.go
@@ -283,10 +283,14 @@ func TestPDFParser_ParseWithResult_MinerUReadsBackendFromProviderJSONApiKey(t *t
}
func TestPDFParser_ParseWithResult_MinerURequiresServerURLForHTTPClientBackend(t *testing.T) {
+ withSSRFBypass(t)
+ server := httptest.NewServer(http.NotFoundHandler())
+ defer server.Close()
+
pdf := NewPDFParser()
pdf.ConfigureFromSetup(map[string]any{
"parse_method": "MinerU",
- "mineru_apiserver": "http://mineru-api:8888",
+ "mineru_apiserver": server.URL,
"mineru_backend": "hybrid-http-client",
})
diff --git a/web/src/pages/user-setting/setting-model/provider-schema/field-config/provider-config-map.ts b/web/src/pages/user-setting/setting-model/provider-schema/field-config/provider-config-map.ts
index 228367bb8e..a7810bbb75 100644
--- a/web/src/pages/user-setting/setting-model/provider-schema/field-config/provider-config-map.ts
+++ b/web/src/pages/user-setting/setting-model/provider-schema/field-config/provider-config-map.ts
@@ -675,11 +675,21 @@ export const ProviderConfigMap: Record<string, ProviderConfig> = {
defaultValue: 'pipeline',
placeholder: 'mineruSelectBackend',
options: [
- { label: 'pipeline', value: 'pipeline' },
- { label: 'vlm-engine', value: 'vlm-engine' },
- { label: 'hybrid-engine', value: 'hybrid-engine' },
- { label: 'vlm-http-client', value: 'vlm-http-client' },
- { label: 'hybrid-http-client', value: 'hybrid-http-client' },
+ { label: 'pipeline (MinerU 3.x)', value: 'pipeline' },
+ { label: 'vlm-engine (MinerU 3.x)', value: 'vlm-engine' },
+ { label: 'hybrid-engine (MinerU 3.x)', value: 'hybrid-engine' },
+ {
+ label: 'vlm-http-client (MinerU 3.x)',
+ value: 'vlm-http-client',
+ },
+ {
+ label: 'hybrid-http-client (MinerU 3.x)',
+ value: 'hybrid-http-client',
+ },
+ { label: 'basic (MinerU 4)', value: 'basic' },
+ { label: 'standard (MinerU 4)', value: 'standard' },
+ { label: 'advanced (MinerU 4)', value: 'advanced' },
+ { label: 'flash (MinerU 4)', value: 'flash' },
],
validation: { message: 'mineruBackendMessage' },
},