RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
feat(parser): support MinerU 4.0 V1 HTTP API (#20712)
Sourav Rajvi68752fcddd39
diff --git a/docs/faq.mdx b/docs/faq.mdxindex d7d342596b..0eded1b533 100644--- a/docs/faq.mdx+++ b/docs/faq.mdx@@ -503,16 +503,16 @@ To improve response speed:RAGFlow sends PDF documents to a remote MinerU service and polls for the parsed result. To use it:-1. Prepare a reachable MinerU API service that provides the `/file_parse` and `/tasks` endpoints used by RAGFlow.+1. Prepare a reachable MinerU API service:+ - **MinerU 4.0** (`mineru-kit api-server`): the V1 API (`/v1/health`, `/v1/uploads`, `/v1/parse/jobs`, `/v1/files`). RAGFlow probes `/v1/health` and uses this cycle automatically. Legacy `/file_parse` and `/tasks` are **not** provided by MinerU 4.+ - **MinerU 3.x**: `/file_parse` and `/tasks` (RAGFlow falls back to this when `/v1/health` is missing).2. In **docker/.env** or on the **Model providers** page in the UI, configure RAGFlow as a remote client to MinerU:- - `MINERU_APISERVER`: The MinerU API endpoint (e.g., `http://mineru-host:8886`).+ - `MINERU_APISERVER`: The MinerU API endpoint (e.g., `http://mineru-host:8000`).- `MINERU_API_KEY`: An API key if the MinerU service requires one.- - `MINERU_BACKEND`: The MinerU backend (matches current MinerU API / CLI `-b` values):- - `"pipeline"` (default)- - `"vlm-engine"`- - `"hybrid-engine"`- - `"vlm-http-client"`- - `"hybrid-http-client"`.+ - `MINERU_BACKEND`: On MinerU 3.x this is the `/file_parse` `backend` field. On MinerU 4.0 it is mapped to a V1 **tier** (`pipeline` → `basic`, `hybrid-*` → `standard`, `vlm-*` → `advanced`). You can also set a 4.0 tier directly:+ - `"pipeline"` (default for 3.x)+ - `"vlm-engine"` / `"hybrid-engine"` / `"vlm-http-client"` / `"hybrid-http-client"` (3.x)+ - `"basic"` / `"standard"` / `"advanced"` / `"flash"` (MinerU 4).3. In the web UI, navigate to your dataset's **Configuration** page and find the **Ingestion pipeline** section:- If you decide to use a chunking method from the **Built-in** dropdown, ensure it supports PDF parsing, then select **MinerU** from the **PDF parser** dropdown.- If you use a custom ingestion pipeline instead, select **MinerU** in the **PDF parser** section of the **Parser** component.@@ -531,7 +531,7 @@ Use the following environment variables when MinerU is not configured directly i| ---------------------- | ---------------------------------- | ----------------------------------- | ----------------------------------------------------------------------------------------------- || `MINERU_APISERVER` | URL of the MinerU API service | _unset_ | `MINERU_APISERVER=http://your-mineru-server:8886` || `MINERU_API_KEY` | API key, when required by MinerU | _unset_ | `MINERU_API_KEY=your-key` |-| `MINERU_BACKEND` | MinerU parsing backend | `pipeline` | `MINERU_BACKEND=pipeline\|vlm-engine\|hybrid-engine\|vlm-http-client\|hybrid-http-client` |+| `MINERU_BACKEND` | MinerU 3.x backend or 4.0 tier | `pipeline` | `MINERU_BACKEND=pipeline\|standard\|vlm-http-client` |1. Set `MINERU_APISERVER` to point RAGFlow to your MinerU API server.2. Set `MINERU_API_KEY` if your MinerU service requires authentication.diff --git a/internal/development.md b/internal/development.mdindex 5d2f3547f6..f12ea68cd9 100644--- a/internal/development.md+++ b/internal/development.md@@ -221,7 +221,7 @@ missing from the model directory.backend. Run it from the repo root so the weights above are auto-discovered(see the CWD caveat); no environment variable is required there.-- **Run the Go Docker image** — the container entrypoint only starts the server+- **Run the Go Docker image** — the container entrypoint only starts the server(`bin/ragflow_server --api/--ingestor/--admin`)- ORT is resolved via `dlopen(NULL)` at runtime, so no `LD_LIBRARY_PATH` /diff --git a/internal/entity/models/mineru_config.go b/internal/entity/models/mineru_config.goindex 154dcd41cd..08b8c8650e 100644--- a/internal/entity/models/mineru_config.go+++ b/internal/entity/models/mineru_config.go@@ -10,22 +10,77 @@ import ("strings")+// MinerU 3.x /file_parse backend names. MinerU 4.0 dropped --backend in favor of+// quality tiers (basic/standard/advanced/flash); see MinerUTierFromBackend.var ValidMinerUBackends = map[string]struct{}{"pipeline": {},"vlm-engine": {},"hybrid-engine": {},"vlm-http-client": {},"hybrid-http-client": {},+ "vlm-auto-engine": {},+ "hybrid-auto-engine": {},+ "basic": {},+ "standard": {},+ "advanced": {},+ "flash": {},+}++var validMinerUTiers = map[string]struct{}{+ "basic": {},+ "standard": {},+ "advanced": {},+ "flash": {},}func MinerUBackendRequiresServerURL(backend string) bool {return backend == "vlm-http-client" || backend == "hybrid-http-client"}+func MinerUIsV1Tier(name string) bool {+ _, ok := validMinerUTiers[strings.ToLower(strings.TrimSpace(name))]+ return ok+}++// MinerUTierFromBackend maps a 3.x backend or a 4.0 tier onto a V1 `tier`.+// MinerU 4.0 changelog: pipeline → basic; vlm-* → advanced; hybrid-* → standard.+func MinerUTierFromBackend(backend string) string {+ switch strings.ToLower(strings.TrimSpace(backend)) {+ case "basic", "pipeline":+ return "basic"+ case "flash":+ return "flash"+ case "advanced", "vlm-engine", "vlm-auto-engine", "vlm-http-client":+ return "advanced"+ case "standard", "hybrid-engine", "hybrid-auto-engine", "hybrid-http-client", "":+ return "standard"+ default:+ return "standard"+ }+}+func ValidateMinerUConfig(backend, serverURL string) error {+ return validateMinerUConfig(backend, serverURL, false)+}++func ValidateMinerUConfigForAPI(backend, serverURL string, v1 bool) error {+ return validateMinerUConfig(backend, serverURL, v1)+}++func validateMinerUConfig(backend, serverURL string, v1 bool) error {if _, ok := ValidMinerUBackends[backend]; !ok {return fmt.Errorf(- "parser: MinerU invalid backend %q (valid: pipeline, vlm-engine, hybrid-engine, vlm-http-client, hybrid-http-client)",+ "parser: MinerU invalid backend %q (valid: pipeline, vlm-engine, hybrid-engine, vlm-http-client, hybrid-http-client, vlm-auto-engine, hybrid-auto-engine, basic, standard, advanced, flash)",+ backend,+ )+ }+ if v1 {+ // MinerU 4 configures the VLM on the service itself; server_url is not a V1 job field.+ return nil+ }+ if MinerUIsV1Tier(backend) {+ return fmt.Errorf(+ "parser: MinerU backend %q is a 4.0 tier, but the connected service does not speak V1 (/v1/health). Use pipeline, vlm-engine, hybrid-engine, vlm-http-client, or hybrid-http-client, or upgrade MinerU to 4.0",backend,)}diff --git a/internal/entity/models/mineru_config_test.go b/internal/entity/models/mineru_config_test.goindex 8d50ce9ce5..445ce551c2 100644--- a/internal/entity/models/mineru_config_test.go+++ b/internal/entity/models/mineru_config_test.go@@ -74,6 +74,40 @@ func TestResolveMinerUBackendAndServerURL(t *testing.T) {}}+func TestMinerUTierFromBackend(t *testing.T) {+ cases := map[string]string{+ "pipeline": "basic",+ "basic": "basic",+ "vlm-engine": "advanced",+ "vlm-http-client": "advanced",+ "hybrid-engine": "standard",+ "hybrid-http-client": "standard",+ "flash": "flash",+ "standard": "standard",+ "": "standard",+ }+ for in, want := range cases {+ if got := MinerUTierFromBackend(in); got != want {+ t.Errorf("MinerUTierFromBackend(%q) = %q, want %q", in, got, want)+ }+ }+}++func TestValidateMinerUConfigAcceptsTiersOnV1(t *testing.T) {+ if err := ValidateMinerUConfigForAPI("standard", "", true); err != nil {+ t.Fatalf("V1 standard: %v", err)+ }+ if err := ValidateMinerUConfigForAPI("pipeline", "", true); err != nil {+ t.Fatalf("V1 pipeline: %v", err)+ }+ if err := ValidateMinerUConfigForAPI("flash", "", false); err == nil {+ t.Fatal("3.x flash tier should fail")+ }+ if err := ValidateMinerUConfig("vlm-http-client", ""); err == nil {+ t.Fatal("3.x vlm-http-client without server_url should fail")+ }+}+func TestResolveMinerUAPIServer(t *testing.T) {t.Setenv("MINERU_APISERVER", "http://env-mineru:8000")diff --git a/internal/entity/models/mineru_local.go b/internal/entity/models/mineru_local.goindex bb94357b19..d433ae2e4c 100644--- a/internal/entity/models/mineru_local.go+++ b/internal/entity/models/mineru_local.go@@ -123,34 +123,39 @@ func (m *MinerULocalModel) CheckConnection(ctx context.Context, apiConfig *APICoif err != nil {return fmt.Errorf("invalid base URL: %w", err)}- // MinerU's API service exposes GET /health; keep scheme/host and replace- // only the path so a base URL with extra path segments or query stays sane.- u.Path = "/health"+ token := ""+ if apiConfig != nil && apiConfig.ApiKey != nil {+ token = MinerUBearerTokenFromAPIKey(*apiConfig.ApiKey)+ }ctx, cancel := context.WithTimeout(ctx, nonStreamCallTimeout)defer cancel()- req, err := http.NewRequestWithContext(ctx, http.MethodGet, u.String(), nil)- if err != nil {- return fmt.Errorf("failed to create request: %w", err)- }- // Same token extraction as ParseFile/ShowTask: a plain api_key is a bearer- // token, while MinerU provider JSON config carries no bearer unless it- // embeds mineru_api_key/access_token — sending the raw payload would be an- // invalid header.- if apiConfig != nil && apiConfig.ApiKey != nil {- if token := MinerUBearerTokenFromAPIKey(*apiConfig.ApiKey); token != "" {- req.Header.Set("Authorization", "Bearer "+token)++ // MinerU 4.0 serves GET /v1/health; 3.x serves GET /health. Probe V1+ // first so a 4.0-only service (no /health) still verifies, then fall+ // back. 401/403 mean the credentials are wrong; any other HTTP answer+ // (including 404 on a server without that path) still proves reachability+ // once at least one probe got a response.+ v1URL := *u+ v1URL.Path = "/v1/health"+ v1Code, v1Err := minerUProbeGET(ctx, m.baseModel.httpClient, v1URL.String(), token)+ if v1Err == nil && v1Code != http.StatusNotFound && v1Code != http.StatusMethodNotAllowed {+ if v1Code == http.StatusUnauthorized || v1Code == http.StatusForbidden {+ return fmt.Errorf("authentication failed (HTTP %d)", v1Code)}+ return nil}- resp, err := m.baseModel.httpClient.Do(req)++ legacy := *u+ legacy.Path = "/health"+ code, err := minerUProbeGET(ctx, m.baseModel.httpClient, legacy.String(), token)if err != nil {+ if v1Err != nil {+ return fmt.Errorf("connection failed: %w", v1Err)+ }return fmt.Errorf("connection failed: %w", err)}- defer resp.Body.Close()- // Match the hosted mineru.net driver's semantics (mineru.go): 401/403 mean- // the credentials are wrong, while any other HTTP answer (e.g. 404 on a- // service without /health) still proves the endpoint is reachable.- if resp.StatusCode == http.StatusUnauthorized || resp.StatusCode == http.StatusForbidden {- return fmt.Errorf("authentication failed (HTTP %d)", resp.StatusCode)+ if code == http.StatusUnauthorized || code == http.StatusForbidden {+ return fmt.Errorf("authentication failed (HTTP %d)", code)}return nil}diff --git a/internal/entity/models/mineru_v1.go b/internal/entity/models/mineru_v1.gonew file mode 100644index 0000000000..2020255981--- /dev/null+++ b/internal/entity/models/mineru_v1.go@@ -0,0 +1,565 @@+//+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.+//++package models++import (+ "archive/zip"+ "bytes"+ "context"+ "crypto/sha256"+ "encoding/hex"+ "encoding/json"+ "fmt"+ "io"+ "net/http"+ "net/url"+ "path"+ "strings"+ "sync"+ "time"++ "ragflow/internal/common"+)++// minerUV1PollInitial and minerUV1PollMax bound GET /v1/parse/jobs/{id} polling.+// Tests may shorten minerUV1PollInitial.+var (+ minerUV1PollInitial = 2 * time.Second+ minerUV1PollMax = 30 * time.Second+)++// DefaultMinerUV1ParseTimeout is used when ParseMinerUV1 receives a non-positive timeout.+const DefaultMinerUV1ParseTimeout = 30 * time.Minute++type minerUV1SupportCacheEntry struct {+ supported bool+ expires time.Time+}++var minerUV1SupportCache sync.Map++const minerUV1SupportCacheTTL = 5 * time.Minute++type minerUV1UploadResponse struct {+ ID string `json:"id"`+ Status string `json:"status"`+ UploadURL string `json:"upload_url"`+ UploadMethod string `json:"upload_method"`+ UploadHeaders map[string]string `json:"upload_headers"`+ File *minerUV1FileRef `json:"file"`+}++type minerUV1FileRef struct {+ ID string `json:"id"`+}++// minerUV1ArtifactRef maps parse-job output_files entries (file_id) and legacy test mocks (id).+type minerUV1ArtifactRef struct {+ ID string `json:"id"`+ FileID string `json:"file_id"`+}++func (r minerUV1ArtifactRef) artifactID() string {+ if id := strings.TrimSpace(r.FileID); id != "" {+ return id+ }+ return strings.TrimSpace(r.ID)+}++type minerUV1JobResponse struct {+ JobID string `json:"job_id"`+ Status string `json:"status"`+ Error *minerUV1Error `json:"error"`+ Files []minerUV1JobFile `json:"files"`+}++type minerUV1Error struct {+ Code string `json:"code"`+ Message string `json:"message"`+}++type minerUV1JobFile struct {+ Name string `json:"name"`+ Status string `json:"status"`+ Error *minerUV1Error `json:"error"`+ OutputFiles map[string]minerUV1ArtifactRef `json:"output_files"`+}++// MinerUV1Result is the downloadable parse output from a MinerU 4 V1 job.+type MinerUV1Result struct {+ Zip []byte+ Markdown string+}++func minerUHTTPClient() *http.Client {+ return common.GetSchemeSafeHTTPClient()+}++func minerUBearerFromAPIKey(apiKey string) string {+ return MinerUBearerTokenFromAPIKey(apiKey)+}++// MinerUV1OCRMode maps mineru_parse_method (auto, txt, ocr) onto the V1 job field ocr_mode.+func MinerUV1OCRMode(raw string) string {+ switch strings.ToLower(strings.TrimSpace(raw)) {+ case "auto", "txt", "ocr":+ return strings.ToLower(strings.TrimSpace(raw))+ default:+ return "auto"+ }+}++func minerUV1SupportCacheKey(baseURL, apiKey string) string {+ token := minerUBearerFromAPIKey(apiKey)+ sum := sha256.Sum256([]byte(token))+ return baseURL + "\x00" + hex.EncodeToString(sum[:])+}++func minerUProbeGET(ctx context.Context, client *http.Client, rawURL, token string) (int, error) {+ req, err := http.NewRequestWithContext(ctx, http.MethodGet, rawURL, nil)+ if err != nil {+ return 0, err+ }+ if token != "" {+ req.Header.Set("Authorization", "Bearer "+token)+ }+ resp, err := client.Do(req)+ if err != nil {+ return 0, err+ }+ defer resp.Body.Close()+ _, _ = io.Copy(io.Discard, resp.Body)+ return resp.StatusCode, nil+}++// MinerUSupportsV1 reports whether baseURL speaks MinerU 4's /v1 HTTP API.+// 404/405 mean a 3.x /file_parse service. 401/403 still count as V1 (the+// route exists; credentials may be wrong).+func MinerUSupportsV1(ctx context.Context, baseURL, apiKey string) (bool, error) {+ baseURL = strings.TrimRight(strings.TrimSpace(baseURL), "/")+ if baseURL == "" {+ return false, fmt.Errorf("MinerU base URL is empty")+ }+ cacheKey := minerUV1SupportCacheKey(baseURL, apiKey)+ if cached, ok := minerUV1SupportCache.Load(cacheKey); ok {+ entry := cached.(minerUV1SupportCacheEntry)+ if time.Now().Before(entry.expires) {+ return entry.supported, nil+ }+ }+ token := minerUBearerFromAPIKey(apiKey)+ code, err := minerUProbeGET(ctx, minerUHTTPClient(), baseURL+"/v1/health", token)+ if err != nil {+ return false, err+ }+ var supported bool+ switch code {+ case http.StatusNotFound, http.StatusMethodNotAllowed:+ supported = false+ default:+ supported = true+ }+ minerUV1SupportCache.Store(cacheKey, minerUV1SupportCacheEntry{+ supported: supported,+ expires: time.Now().Add(minerUV1SupportCacheTTL),+ })+ return supported, nil+}++func applyMinerUAuth(req *http.Request, token string) {+ if token != "" {+ req.Header.Set("Authorization", "Bearer "+token)+ }+}++func minerUJSON(ctx context.Context, client *http.Client, method, rawURL, token string, body any, dest any) error {+ var reader io.Reader+ if body != nil {+ raw, err := json.Marshal(body)+ if err != nil {+ return err+ }+ reader = bytes.NewReader(raw)+ }+ req, err := http.NewRequestWithContext(ctx, method, rawURL, reader)+ if err != nil {+ return err+ }+ if body != nil {+ req.Header.Set("Content-Type", "application/json")+ }+ applyMinerUAuth(req, token)+ resp, err := client.Do(req)+ if err != nil {+ return err+ }+ defer resp.Body.Close()+ payload, err := io.ReadAll(resp.Body)+ if err != nil {+ return fmt.Errorf("read %s %s: %w", method, rawURL, err)+ }+ if resp.StatusCode < 200 || resp.StatusCode >= 300 {+ return fmt.Errorf("MinerU V1 %s %s: HTTP %d: %s", method, rawURL, resp.StatusCode, string(payload))+ }+ if dest == nil {+ return nil+ }+ if err := json.Unmarshal(payload, dest); err != nil {+ return fmt.Errorf("MinerU V1 decode %s: %w; body: %s", rawURL, err, string(payload))+ }+ return nil+}++func minerUSameOrigin(base, target string) bool {+ bu, err := url.Parse(base)+ if err != nil {+ return false+ }+ tu, err := url.Parse(target)+ if err != nil {+ return false+ }+ port := func(u *url.URL) string {+ if u.Port() != "" {+ return u.Port()+ }+ if strings.EqualFold(u.Scheme, "https") {+ return "443"+ }+ return "80"+ }+ return strings.EqualFold(bu.Scheme, tu.Scheme) &&+ strings.EqualFold(bu.Hostname(), tu.Hostname()) &&+ port(bu) == port(tu)+}++func minerUFilename(name string) string {+ base := path.Base(strings.TrimSpace(name))+ if base == "" || base == "." || base == "/" {+ return "document.pdf"+ }+ return base+}++func minerUMimeType(filename string) string {+ switch strings.ToLower(path.Ext(filename)) {+ case ".pdf":+ return "application/pdf"+ case ".docx":+ return "application/vnd.openxmlformats-officedocument.wordprocessingml.document"+ case ".pptx":+ return "application/vnd.openxmlformats-officedocument.presentationml.presentation"+ case ".xlsx":+ return "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"+ case ".html", ".htm":+ return "text/html"+ default:+ return "application/octet-stream"+ }+}++func minerUV1Terminal(status string) bool {+ switch status {+ case "completed", "partial", "failed", "canceled":+ return true+ default:+ return false+ }+}++// ParseMinerUV1 runs the MinerU 4 V1 cycle: create upload → PUT bytes →+// complete → parse job → poll → download zip and/or markdown.+// ocrMode is the V1 ocr_mode (auto, txt, ocr); use MinerUV1OCRMode to normalize.+func ParseMinerUV1(ctx context.Context, baseURL, apiKey, filename string, content []byte, backend, ocrMode string, timeout time.Duration) (*MinerUV1Result, error) {+ baseURL = strings.TrimRight(strings.TrimSpace(baseURL), "/")+ if baseURL == "" {+ return nil, fmt.Errorf("MinerU base URL is empty")+ }+ if len(content) == 0 {+ return nil, fmt.Errorf("MinerU V1 requires file content")+ }+ if timeout <= 0 {+ timeout = DefaultMinerUV1ParseTimeout+ }+ ctx, cancel := context.WithTimeout(ctx, timeout)+ defer cancel()++ client := minerUHTTPClient()+ token := minerUBearerFromAPIKey(apiKey)+ name := minerUFilename(filename)+ sum := sha256.Sum256(content)++ var created minerUV1UploadResponse+ if err := minerUJSON(ctx, client, http.MethodPost, baseURL+"/v1/uploads", token, map[string]any{+ "filename": name,+ "bytes": len(content),+ "mime_type": minerUMimeType(name),+ "purpose": "parse",+ "sha256sum": hex.EncodeToString(sum[:]),+ }, &created); err != nil {+ return nil, fmt.Errorf("create upload: %w", err)+ }+ fileID := ""+ if created.File != nil {+ fileID = created.File.ID+ }+ switch created.Status {+ case "completed":+ if fileID == "" {+ return nil, fmt.Errorf("MinerU V1 upload completed without file.id")+ }+ case "pending":+ if created.UploadURL == "" {+ return nil, fmt.Errorf("MinerU V1 upload missing upload_url")+ }+ uploadURL, err := url.Parse(created.UploadURL)+ if err != nil {+ return nil, fmt.Errorf("invalid upload_url: %w", err)+ }+ if !uploadURL.IsAbs() {+ base, err := url.Parse(baseURL + "/")+ if err != nil {+ return nil, fmt.Errorf("invalid MinerU base URL: %w", err)+ }+ uploadURL = base.ResolveReference(uploadURL)+ }+ method := strings.ToUpper(strings.TrimSpace(created.UploadMethod))+ if method == "" {+ method = http.MethodPut+ }+ req, err := http.NewRequestWithContext(ctx, method, uploadURL.String(), bytes.NewReader(content))+ if err != nil {+ return nil, fmt.Errorf("create byte upload: %w", err)+ }+ for k, v := range created.UploadHeaders {+ req.Header.Set(k, v)+ }+ if minerUSameOrigin(baseURL, uploadURL.String()) {+ applyMinerUAuth(req, token)+ }+ resp, err := client.Do(req)+ if err != nil {+ return nil, fmt.Errorf("byte upload: %w", err)+ }+ _, _ = io.Copy(io.Discard, resp.Body)+ resp.Body.Close()+ if resp.StatusCode < 200 || resp.StatusCode >= 300 {+ return nil, fmt.Errorf("byte upload: HTTP %d", resp.StatusCode)+ }+ var completed minerUV1UploadResponse+ if err := minerUJSON(ctx, client, http.MethodPost, baseURL+"/v1/uploads/"+created.ID+"/complete", token, nil, &completed); err != nil {+ return nil, fmt.Errorf("complete upload: %w", err)+ }+ if completed.File == nil || completed.File.ID == "" {+ return nil, fmt.Errorf("MinerU V1 complete upload missing file.id")+ }+ fileID = completed.File.ID+ default:+ return nil, fmt.Errorf("MinerU V1 upload status %q", created.Status)+ }++ jobBody := map[string]any{+ "files": []map[string]any{+ {"source": map[string]any{"type": "file_id", "file_id": fileID}},+ },+ "tier": MinerUTierFromBackend(backend),+ "output_formats": []string{"zip", "markdown"},+ "ocr_mode": MinerUV1OCRMode(ocrMode),+ }+ var job minerUV1JobResponse+ if err := minerUJSON(ctx, client, http.MethodPost, baseURL+"/v1/parse/jobs", token, jobBody, &job); err != nil {+ return nil, fmt.Errorf("create parse job: %w", err)+ }+ if job.JobID == "" {+ return nil, fmt.Errorf("MinerU V1 parse job missing job_id")+ }++ pollWait := minerUV1PollInitial+ for !minerUV1Terminal(job.Status) {+ select {+ case <-ctx.Done():+ return nil, ctx.Err()+ case <-time.After(pollWait):+ }+ if pollWait < minerUV1PollMax {+ pollWait *= 2+ if pollWait > minerUV1PollMax {+ pollWait = minerUV1PollMax+ }+ }+ if err := minerUJSON(ctx, client, http.MethodGet, baseURL+"/v1/parse/jobs/"+job.JobID, token, nil, &job); err != nil {+ return nil, fmt.Errorf("poll parse job: %w", err)+ }+ }+ switch job.Status {+ case "failed", "canceled":+ msg := job.Status+ if job.Error != nil && job.Error.Message != "" {+ msg = job.Error.Message+ }+ return nil, fmt.Errorf("MinerU V1 job %s: %s", job.Status, msg)+ }++ result := &MinerUV1Result{}+ var artifactErr error+ for _, file := range job.Files {+ if file.Status != "" && file.Status != "completed" {+ continue+ }+ if ref, ok := file.OutputFiles["zip"]; ok && ref.artifactID() != "" && len(result.Zip) == 0 {+ body, err := minerUV1DownloadFile(ctx, client, baseURL, token, ref.artifactID())+ if err != nil {+ if artifactErr == nil {+ artifactErr = fmt.Errorf("download zip: %w", err)+ }+ } else {+ result.Zip = body+ }+ }+ if ref, ok := file.OutputFiles["markdown"]; ok && ref.artifactID() != "" && result.Markdown == "" {+ body, err := minerUV1DownloadFile(ctx, client, baseURL, token, ref.artifactID())+ if err != nil {+ if artifactErr == nil {+ artifactErr = fmt.Errorf("download markdown: %w", err)+ }+ } else {+ result.Markdown = string(body)+ }+ }+ }+ if result.Markdown == "" && len(result.Zip) > 0 {+ if md, err := minerUMarkdownFromZip(result.Zip); err == nil {+ result.Markdown = md+ }+ }+ if len(result.Zip) == 0 && strings.TrimSpace(result.Markdown) == "" {+ if artifactErr != nil {+ return nil, artifactErr+ }+ return nil, minerUV1NoArtifactsError(job)+ }+ return result, nil+}++func minerUV1NoArtifactsError(job minerUV1JobResponse) error {+ msg := fmt.Sprintf("MinerU V1 job %s (%s) completed without zip or markdown artifacts", job.JobID, job.Status)+ if fileMsg := minerUV1FirstFileErrorMessage(job.Files); fileMsg != "" {+ msg += ": " + fileMsg+ }+ return fmt.Errorf("%s", msg)+}++func minerUV1FirstFileErrorMessage(files []minerUV1JobFile) string {+ for _, file := range files {+ if file.Error != nil && strings.TrimSpace(file.Error.Message) != "" {+ return file.Error.Message+ }+ }+ return ""+}++func minerUV1DownloadHTTPClient(apiOrigin string) *http.Client {+ base := minerUHTTPClient()+ return &http.Client{+ Transport: base.Transport,+ CheckRedirect: func(req *http.Request, via []*http.Request) error {+ if len(via) >= 10 {+ return fmt.Errorf("too many redirects")+ }+ if !minerUSameOrigin(apiOrigin, req.URL.String()) {+ req.Header.Del("Authorization")+ }+ return nil+ },+ }+}++func minerUV1DownloadFile(ctx context.Context, client *http.Client, baseURL, token, fileID string) ([]byte, error) {+ apiOrigin := strings.TrimRight(strings.TrimSpace(baseURL), "/")+ dlClient := minerUV1DownloadHTTPClient(apiOrigin)+ req, err := http.NewRequestWithContext(ctx, http.MethodGet, apiOrigin+"/v1/files/"+fileID+"/content", nil)+ if err != nil {+ return nil, err+ }+ applyMinerUAuth(req, token)+ resp, err := dlClient.Do(req)+ if err != nil {+ return nil, err+ }+ defer resp.Body.Close()+ body, err := io.ReadAll(resp.Body)+ if err != nil {+ return nil, err+ }+ if resp.StatusCode < 200 || resp.StatusCode >= 300 {+ return nil, fmt.Errorf("HTTP %d: %s", resp.StatusCode, string(body))+ }+ return body, nil+}++// minerUMarkdownFromZip prefers a .md member, then concatenates content_list.json text.+func minerUMarkdownFromZip(zipBytes []byte) (string, error) {+ zipReader, err := zip.NewReader(bytes.NewReader(zipBytes), int64(len(zipBytes)))+ if err != nil {+ return "", fmt.Errorf("open zip: %w", err)+ }+ var md string+ var contentList []byte+ for _, f := range zipReader.File {+ name := strings.ReplaceAll(f.Name, "\\", "/")+ if strings.Contains(name, "__MACOSX") {+ continue+ }+ rc, err := f.Open()+ if err != nil {+ continue+ }+ body, _ := io.ReadAll(rc)+ rc.Close()+ lower := strings.ToLower(name)+ if md == "" && strings.HasSuffix(lower, ".md") {+ md = string(body)+ }+ if strings.HasSuffix(lower, "content_list.json") {+ contentList = body+ }+ }+ if strings.TrimSpace(md) != "" {+ return md, nil+ }+ if len(contentList) == 0 {+ return "", fmt.Errorf("markdown and content_list.json not found in MinerU zip")+ }+ var items []map[string]any+ if err := json.Unmarshal(contentList, &items); err != nil {+ return "", fmt.Errorf("parse content_list.json: %w", err)+ }+ var parts []string+ for _, item := range items {+ if text, ok := item["text"].(string); ok && strings.TrimSpace(text) != "" {+ parts = append(parts, text)+ continue+ }+ if tb, ok := item["table_body"].(string); ok && strings.TrimSpace(tb) != "" {+ parts = append(parts, tb)+ }+ }+ if len(parts) == 0 {+ return "", fmt.Errorf("content_list.json had no text")+ }+ return strings.Join(parts, "\n"), nil+}diff --git a/internal/entity/models/mineru_v1_test.go b/internal/entity/models/mineru_v1_test.gonew file mode 100644index 0000000000..be2e632a42--- /dev/null+++ b/internal/entity/models/mineru_v1_test.go@@ -0,0 +1,322 @@+//+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.+//++package models++import (+ "archive/zip"+ "bytes"+ "context"+ "encoding/json"+ "io"+ "net/http"+ "net/http/httptest"+ "strings"+ "sync"+ "sync/atomic"+ "testing"+ "time"+)++func TestMinerUSupportsV1(t *testing.T) {+ t.Run("v1 health", func(t *testing.T) {+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ if r.URL.Path != "/v1/health" {+ http.NotFound(w, r)+ return+ }+ w.WriteHeader(http.StatusOK)+ _, _ = w.Write([]byte(`{"status":"ok"}`))+ }))+ defer server.Close()+ ok, err := MinerUSupportsV1(context.Background(), server.URL, "")+ if err != nil || !ok {+ t.Fatalf("SupportsV1 = %v, %v, want true", ok, err)+ }+ })+ t.Run("legacy file_parse only", func(t *testing.T) {+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ http.NotFound(w, r)+ }))+ defer server.Close()+ ok, err := MinerUSupportsV1(context.Background(), server.URL, "")+ if err != nil {+ t.Fatalf("SupportsV1: %v", err)+ }+ if ok {+ t.Fatal("404 /v1/health should be treated as 3.x")+ }+ })+}++func TestParseMinerUV1UploadJobDownload(t *testing.T) {+ orig := minerUV1PollInitial+ minerUV1PollInitial = time.Millisecond+ t.Cleanup(func() { minerUV1PollInitial = orig })++ var (+ mu sync.Mutex+ gotBytes []byte+ gotTier string+ gotOCR string+ polls int+ )+ zipBuf := minerUTestZip(t, "doc.md", "# Hello\n\nFrom V1.\n")++ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ switch {+ case r.Method == http.MethodGet && r.URL.Path == "/v1/health":+ w.WriteHeader(http.StatusOK)+ _, _ = w.Write([]byte(`{"status":"ok"}`))+ case r.Method == http.MethodPost && r.URL.Path == "/v1/uploads":+ w.Header().Set("Content-Type", "application/json")+ _, _ = w.Write([]byte(`{"id":"up-1","status":"pending","upload_url":"/v1/uploads/up-1/content","upload_method":"PUT"}`))+ case r.Method == http.MethodPut && r.URL.Path == "/v1/uploads/up-1/content":+ body, _ := io.ReadAll(r.Body)+ mu.Lock()+ gotBytes = body+ mu.Unlock()+ w.WriteHeader(http.StatusOK)+ case r.Method == http.MethodPost && r.URL.Path == "/v1/uploads/up-1/complete":+ w.Header().Set("Content-Type", "application/json")+ _, _ = w.Write([]byte(`{"id":"up-1","status":"completed","file":{"id":"file-1"}}`))+ case r.Method == http.MethodPost && r.URL.Path == "/v1/parse/jobs":+ var payload map[string]any+ _ = json.NewDecoder(r.Body).Decode(&payload)+ if tier, ok := payload["tier"].(string); ok {+ mu.Lock()+ gotTier = tier+ mu.Unlock()+ }+ if mode, ok := payload["ocr_mode"].(string); ok {+ mu.Lock()+ gotOCR = mode+ mu.Unlock()+ }+ w.Header().Set("Content-Type", "application/json")+ _, _ = w.Write([]byte(`{"job_id":"job-1","status":"queued"}`))+ case r.Method == http.MethodGet && r.URL.Path == "/v1/parse/jobs/job-1":+ mu.Lock()+ polls+++ n := polls+ mu.Unlock()+ w.Header().Set("Content-Type", "application/json")+ if n < 2 {+ _, _ = w.Write([]byte(`{"job_id":"job-1","status":"running"}`))+ return+ }+ _, _ = w.Write([]byte(`{"job_id":"job-1","status":"completed","files":[{"name":"doc.pdf","status":"completed","output_files":{"zip":{"file_id":"out-zip"},"markdown":{"file_id":"out-md"}}}]}`))+ case r.Method == http.MethodGet && r.URL.Path == "/v1/files/out-zip/content":+ w.Header().Set("Content-Type", "application/zip")+ _, _ = w.Write(zipBuf)+ case r.Method == http.MethodGet && r.URL.Path == "/v1/files/out-md/content":+ _, _ = w.Write([]byte("# Hello\n\nFrom V1.\n"))+ default:+ http.NotFound(w, r)+ }+ }))+ defer server.Close()++ result, err := ParseMinerUV1(context.Background(), server.URL, "secret", "sample.pdf", []byte("%PDF-1.4"), "pipeline", "ocr", 5*time.Second)+ if err != nil {+ t.Fatalf("ParseMinerUV1: %v", err)+ }+ if result.Markdown != "# Hello\n\nFrom V1.\n" {+ t.Fatalf("markdown = %q", result.Markdown)+ }+ if len(result.Zip) == 0 {+ t.Fatal("expected zip artifact")+ }+ mu.Lock()+ defer mu.Unlock()+ if !bytes.Equal(gotBytes, []byte("%PDF-1.4")) {+ t.Fatalf("uploaded bytes = %q", gotBytes)+ }+ if gotTier != "basic" {+ t.Fatalf("tier = %q, want basic (mapped from pipeline)", gotTier)+ }+ if gotOCR != "ocr" {+ t.Fatalf("ocr_mode = %q, want ocr", gotOCR)+ }+}++func TestMinerUV1DownloadStripsAuthOnCrossOriginRedirect(t *testing.T) {+ var sawAuth atomic.Bool+ redirect := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ if r.Header.Get("Authorization") != "" {+ sawAuth.Store(true)+ }+ _, _ = w.Write([]byte("artifact"))+ }))+ defer redirect.Close()++ api := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ if r.URL.Path == "/v1/files/out-1/content" {+ http.Redirect(w, r, redirect.URL, http.StatusFound)+ return+ }+ http.NotFound(w, r)+ }))+ defer api.Close()++ body, err := minerUV1DownloadFile(context.Background(), minerUHTTPClient(), api.URL, "secret-token", "out-1")+ if err != nil {+ t.Fatalf("minerUV1DownloadFile: %v", err)+ }+ if body == nil || string(body) != "artifact" {+ t.Fatalf("body = %q", body)+ }+ if sawAuth.Load() {+ t.Fatal("Authorization must not be forwarded to a cross-origin redirect target")+ }+}++func TestParseMinerUV1OutputFilesUseFileID(t *testing.T) {+ orig := minerUV1PollInitial+ minerUV1PollInitial = time.Millisecond+ t.Cleanup(func() { minerUV1PollInitial = orig })++ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ switch {+ case r.Method == http.MethodPost && r.URL.Path == "/v1/uploads":+ _, _ = w.Write([]byte(`{"id":"up-1","status":"completed","file":{"id":"file-1"}}`))+ case r.Method == http.MethodPost && r.URL.Path == "/v1/parse/jobs":+ _, _ = w.Write([]byte(`{"job_id":"job-1","status":"completed","files":[{"name":"doc.pdf","status":"completed","output_files":{"markdown":{"file_id":"out-md"}}}]}`))+ case r.Method == http.MethodGet && r.URL.Path == "/v1/files/out-md/content":+ _, _ = w.Write([]byte("# From file_id\n"))+ default:+ http.NotFound(w, r)+ }+ }))+ defer server.Close()++ result, err := ParseMinerUV1(context.Background(), server.URL, "secret", "sample.pdf", []byte("%PDF-1.4"), "pipeline", "auto", 5*time.Second)+ if err != nil {+ t.Fatalf("ParseMinerUV1: %v", err)+ }+ if result.Markdown != "# From file_id\n" {+ t.Fatalf("markdown = %q", result.Markdown)+ }+}++func TestParseMinerUV1PartialJobIncludesFileError(t *testing.T) {+ orig := minerUV1PollInitial+ minerUV1PollInitial = time.Millisecond+ t.Cleanup(func() { minerUV1PollInitial = orig })++ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ switch {+ case r.Method == http.MethodPost && r.URL.Path == "/v1/uploads":+ _, _ = w.Write([]byte(`{"id":"up-1","status":"completed","file":{"id":"file-1"}}`))+ case r.Method == http.MethodPost && r.URL.Path == "/v1/parse/jobs":+ _, _ = w.Write([]byte(`{"job_id":"job-1","status":"partial","files":[{"name":"doc.pdf","status":"failed","error":{"message":"page render failed"}}]}`))+ default:+ http.NotFound(w, r)+ }+ }))+ defer server.Close()++ _, err := ParseMinerUV1(context.Background(), server.URL, "secret", "sample.pdf", []byte("%PDF-1.4"), "pipeline", "auto", 5*time.Second)+ if err == nil {+ t.Fatal("expected error for partial job without artifacts")+ }+ if !strings.Contains(err.Error(), "partial") || !strings.Contains(err.Error(), "page render failed") {+ t.Fatalf("error = %q, want partial status and file error", err.Error())+ }+}++func TestParseMinerUV1ZipDownloadFailureFallsBackToMarkdown(t *testing.T) {+ orig := minerUV1PollInitial+ minerUV1PollInitial = time.Millisecond+ t.Cleanup(func() { minerUV1PollInitial = orig })++ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ switch {+ case r.Method == http.MethodPost && r.URL.Path == "/v1/uploads":+ _, _ = w.Write([]byte(`{"id":"up-1","status":"completed","file":{"id":"file-1"}}`))+ case r.Method == http.MethodPost && r.URL.Path == "/v1/parse/jobs":+ _, _ = w.Write([]byte(`{"job_id":"job-1","status":"completed","files":[{"name":"doc.pdf","status":"completed","output_files":{"zip":{"file_id":"out-zip"},"markdown":{"file_id":"out-md"}}}]}`))+ case r.Method == http.MethodGet && r.URL.Path == "/v1/files/out-zip/content":+ http.Error(w, "zip gone", http.StatusInternalServerError)+ case r.Method == http.MethodGet && r.URL.Path == "/v1/files/out-md/content":+ _, _ = w.Write([]byte("# Fallback markdown\n"))+ default:+ http.NotFound(w, r)+ }+ }))+ defer server.Close()++ result, err := ParseMinerUV1(context.Background(), server.URL, "secret", "sample.pdf", []byte("%PDF-1.4"), "pipeline", "auto", 5*time.Second)+ if err != nil {+ t.Fatalf("ParseMinerUV1: %v", err)+ }+ if result.Markdown != "# Fallback markdown\n" {+ t.Fatalf("markdown = %q, want fallback artifact", result.Markdown)+ }+ if len(result.Zip) != 0 {+ t.Fatalf("zip = %d bytes, want empty after download failure", len(result.Zip))+ }+}++func TestMinerUMarkdownFromZip(t *testing.T) {+ buf := minerUTestZip(t, "out.md", "# Title\n")+ md, err := minerUMarkdownFromZip(buf)+ if err != nil {+ t.Fatalf("minerUMarkdownFromZip: %v", err)+ }+ if md != "# Title\n" {+ t.Fatalf("md = %q", md)+ }+}++func TestMinerULocalCheckConnectionPrefersV1Health(t *testing.T) {+ var paths []string+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ paths = append(paths, r.URL.Path)+ if r.URL.Path == "/v1/health" {+ w.WriteHeader(http.StatusOK)+ return+ }+ http.NotFound(w, r)+ }))+ defer server.Close()++ baseURL := server.URL+ driver := NewMinerLocalUModel(map[string]string{"default": baseURL}, URLSuffix{DocumentParse: "file_parse", Task: "tasks"})+ if err := driver.CheckConnection(context.Background(), &APIConfig{BaseURL: &baseURL}); err != nil {+ t.Fatalf("CheckConnection: %v", err)+ }+ if len(paths) != 1 || paths[0] != "/v1/health" {+ t.Fatalf("paths = %v, want only /v1/health", paths)+ }+}++func minerUTestZip(t *testing.T, name, body string) []byte {+ t.Helper()+ var buf bytes.Buffer+ zw := zip.NewWriter(&buf)+ w, err := zw.Create(name)+ if err != nil {+ t.Fatal(err)+ }+ if _, err := io.WriteString(w, body); err != nil {+ t.Fatal(err)+ }+ if err := zw.Close(); err != nil {+ t.Fatal(err)+ }+ return buf.Bytes()+}++func TestMinerUSameOrigin(t *testing.T) {+ if !minerUSameOrigin("http://127.0.0.1:8000", "http://127.0.0.1:8000/v1/uploads/1/content") {+ t.Fatal("same host/port should be same-origin")+ }+ if minerUSameOrigin("http://127.0.0.1:8000", "http://127.0.0.1:9000/v1/uploads/1/content") {+ t.Fatal("different port is not same-origin")+ }+ if strings.TrimSpace("") != "" {+ t.Fatal("sanity")+ }+}diff --git a/internal/ingestion/component/parser_dispatch_test.go b/internal/ingestion/component/parser_dispatch_test.goindex 5863baf638..61c1cc0b75 100644--- a/internal/ingestion/component/parser_dispatch_test.go+++ b/internal/ingestion/component/parser_dispatch_test.go@@ -924,9 +924,13 @@ func TestDispatch_PDFMinerUMarkdown_SendsServerURLFromProviderConfig(t *testing.}func TestDispatch_PDFMinerUMarkdown_RequiresServerURLForHTTPClientBackend(t *testing.T) {+ withSSRFBypass(t)+ server := httptest.NewServer(http.NotFoundHandler())+ defer server.Close()+origResolver := resolveMinerUModelForDispatchdefer func() { resolveMinerUModelForDispatch = origResolver }()- baseURL := "http://mineru-api:8888"+ baseURL := server.URLapiKey := `{"mineru_backend":"hybrid-http-client"}`resolveMinerUModelForDispatch = func(ctx context.Context, db *gorm.DB, tenantID, modelID string) (models.ModelDriver, string, *models.APIConfig, error) {return &mineruTestDriver{}, "mineru-model", &models.APIConfig{ApiKey: &apiKey, BaseURL: &baseURL}, nil@@ -950,6 +954,192 @@ func TestDispatch_PDFMinerUMarkdown_RequiresServerURLForHTTPClientBackend(t *tes}}+func TestDispatch_PDFMinerUV1_WiresParseAndExtract(t *testing.T) {+ withSSRFBypass(t)++ invoke := func(t *testing.T, serverURL string) map[string]any {+ t.Helper()+ origResolver := resolveMinerUModelForDispatch+ t.Cleanup(func() { resolveMinerUModelForDispatch = origResolver })+ baseURL := serverURL+ apiKey := ""+ resolveMinerUModelForDispatch = func(ctx context.Context, db *gorm.DB, tenantID, modelID string) (models.ModelDriver, string, *models.APIConfig, error) {+ return &mineruTestDriver{}, "mineru-model", &models.APIConfig{ApiKey: &apiKey, BaseURL: &baseURL}, nil+ }+ setups := defaultSetups()+ setups["pdf"]["parse_method"] = "mineru"+ setups["pdf"]["output_format"] = "markdown"+ setups["pdf"]["mineru_backend"] = "standard"+ return invokeMinerUDispatch(t, nil, setups)+ }++ t.Run("zip content_list uses mineruExtractJSONItems", func(t *testing.T) {+ var zipBuf bytes.Buffer+ zw := zip.NewWriter(&zipBuf)+ f, _ := zw.Create("content_list.json")+ _, _ = f.Write([]byte(`[{"type":"text","text":"V1 body"},{"type":"table","table_body":"<table></table>","table_caption":["Table caption"]}]`))+ _ = zw.Close()+ zipBytes := zipBuf.Bytes()++ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ switch {+ case r.Method == http.MethodGet && r.URL.Path == "/v1/health":+ w.WriteHeader(http.StatusOK)+ case r.Method == http.MethodPost && r.URL.Path == "/v1/uploads":+ _, _ = w.Write([]byte(`{"id":"up-1","status":"completed","file":{"id":"file-1"}}`))+ case r.Method == http.MethodPost && r.URL.Path == "/v1/parse/jobs":+ _, _ = w.Write([]byte(`{"job_id":"job-1","status":"completed","files":[{"name":"sample.pdf","status":"completed","output_files":{"zip":{"file_id":"out-zip"}}}]}`))+ case r.Method == http.MethodGet && r.URL.Path == "/v1/files/out-zip/content":+ w.Header().Set("Content-Type", "application/zip")+ _, _ = w.Write(zipBytes)+ default:+ http.NotFound(w, r)+ }+ }))+ defer server.Close()++ out := invoke(t, server.URL)+ requireJSONText(t, out, "V1 body")+ requireJSONText(t, out, "Table caption")+ })++ t.Run("markdown fallback when zip has no content_list", func(t *testing.T) {+ var zipBuf bytes.Buffer+ zw := zip.NewWriter(&zipBuf)+ f, _ := zw.Create("middle.json")+ _, _ = f.Write([]byte(`{}`))+ _ = zw.Close()+ zipBytes := zipBuf.Bytes()++ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ switch {+ case r.Method == http.MethodGet && r.URL.Path == "/v1/health":+ w.WriteHeader(http.StatusOK)+ case r.Method == http.MethodPost && r.URL.Path == "/v1/uploads":+ _, _ = w.Write([]byte(`{"id":"up-1","status":"completed","file":{"id":"file-1"}}`))+ case r.Method == http.MethodPost && r.URL.Path == "/v1/parse/jobs":+ _, _ = w.Write([]byte(`{"job_id":"job-1","status":"completed","files":[{"name":"sample.pdf","status":"completed","output_files":{"zip":{"file_id":"out-zip"},"markdown":{"file_id":"out-md"}}}]}`))+ case r.Method == http.MethodGet && r.URL.Path == "/v1/files/out-zip/content":+ w.Header().Set("Content-Type", "application/zip")+ _, _ = w.Write(zipBytes)+ case r.Method == http.MethodGet && r.URL.Path == "/v1/files/out-md/content":+ _, _ = w.Write([]byte("# Zip had no content_list\n"))+ default:+ http.NotFound(w, r)+ }+ }))+ defer server.Close()++ out := invoke(t, server.URL)+ requireJSONText(t, out, "Zip had no content_list")+ })++ t.Run("markdown fallback when zip sections are blank", func(t *testing.T) {+ var zipBuf bytes.Buffer+ zw := zip.NewWriter(&zipBuf)+ f, _ := zw.Create("content_list.json")+ _, _ = f.Write([]byte(`[{"type":"text","text":""}]`))+ _ = zw.Close()+ zipBytes := zipBuf.Bytes()++ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ switch {+ case r.Method == http.MethodGet && r.URL.Path == "/v1/health":+ w.WriteHeader(http.StatusOK)+ case r.Method == http.MethodPost && r.URL.Path == "/v1/uploads":+ _, _ = w.Write([]byte(`{"id":"up-1","status":"completed","file":{"id":"file-1"}}`))+ case r.Method == http.MethodPost && r.URL.Path == "/v1/parse/jobs":+ _, _ = w.Write([]byte(`{"job_id":"job-1","status":"completed","files":[{"name":"sample.pdf","status":"completed","output_files":{"zip":{"file_id":"out-zip"},"markdown":{"file_id":"out-md"}}}]}`))+ case r.Method == http.MethodGet && r.URL.Path == "/v1/files/out-zip/content":+ w.Header().Set("Content-Type", "application/zip")+ _, _ = w.Write(zipBytes)+ case r.Method == http.MethodGet && r.URL.Path == "/v1/files/out-md/content":+ _, _ = w.Write([]byte("# Blank zip sections\n"))+ default:+ http.NotFound(w, r)+ }+ }))+ defer server.Close()++ out := invoke(t, server.URL)+ requireJSONText(t, out, "Blank zip sections")+ })++ t.Run("markdown fallback when zip download fails", func(t *testing.T) {+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ switch {+ case r.Method == http.MethodGet && r.URL.Path == "/v1/health":+ w.WriteHeader(http.StatusOK)+ case r.Method == http.MethodPost && r.URL.Path == "/v1/uploads":+ _, _ = w.Write([]byte(`{"id":"up-1","status":"completed","file":{"id":"file-1"}}`))+ case r.Method == http.MethodPost && r.URL.Path == "/v1/parse/jobs":+ _, _ = w.Write([]byte(`{"job_id":"job-1","status":"completed","files":[{"name":"sample.pdf","status":"completed","output_files":{"zip":{"file_id":"out-zip"},"markdown":{"file_id":"out-md"}}}]}`))+ case r.Method == http.MethodGet && r.URL.Path == "/v1/files/out-zip/content":+ http.Error(w, "zip gone", http.StatusInternalServerError)+ case r.Method == http.MethodGet && r.URL.Path == "/v1/files/out-md/content":+ _, _ = w.Write([]byte("# Markdown fallback\n"))+ default:+ http.NotFound(w, r)+ }+ }))+ defer server.Close()++ out := invoke(t, server.URL)+ requireJSONText(t, out, "Markdown fallback")+ })++ t.Run("empty content_list without markdown is an error", func(t *testing.T) {+ var zipBuf bytes.Buffer+ zw := zip.NewWriter(&zipBuf)+ f, _ := zw.Create("content_list.json")+ _, _ = f.Write([]byte(`[]`))+ _ = zw.Close()+ zipBytes := zipBuf.Bytes()++ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ switch {+ case r.Method == http.MethodGet && r.URL.Path == "/v1/health":+ w.WriteHeader(http.StatusOK)+ case r.Method == http.MethodPost && r.URL.Path == "/v1/uploads":+ _, _ = w.Write([]byte(`{"id":"up-1","status":"completed","file":{"id":"file-1"}}`))+ case r.Method == http.MethodPost && r.URL.Path == "/v1/parse/jobs":+ _, _ = w.Write([]byte(`{"job_id":"job-1","status":"completed","files":[{"name":"sample.pdf","status":"completed","output_files":{"zip":{"file_id":"out-zip"}}}]}`))+ case r.Method == http.MethodGet && r.URL.Path == "/v1/files/out-zip/content":+ w.Header().Set("Content-Type", "application/zip")+ _, _ = w.Write(zipBytes)+ default:+ http.NotFound(w, r)+ }+ }))+ defer server.Close()++ origResolver := resolveMinerUModelForDispatch+ t.Cleanup(func() { resolveMinerUModelForDispatch = origResolver })+ baseURL := server.URL+ apiKey := ""+ resolveMinerUModelForDispatch = func(ctx context.Context, db *gorm.DB, tenantID, modelID string) (models.ModelDriver, string, *models.APIConfig, error) {+ return &mineruTestDriver{}, "mineru-model", &models.APIConfig{ApiKey: &apiKey, BaseURL: &baseURL}, nil+ }+ setups := defaultSetups()+ setups["pdf"]["parse_method"] = "mineru"+ setups["pdf"]["output_format"] = "markdown"+ setups["pdf"]["mineru_backend"] = "standard"+ c := &ParserComponent{setups: setups}+ _, err := c.Invoke(t.Context(), nil, map[string]any{+ "binary": []byte("%PDF-1.4"),+ "file_type": "pdf",+ "name": "sample.pdf",+ "tenant_id": "test-tenant",+ })+ if err == nil {+ t.Fatal("Invoke: want error when V1 zip yields no sections")+ }+ if !strings.Contains(err.Error(), "no usable content_list") && !strings.Contains(err.Error(), "no sections") {+ t.Fatalf("error = %q, want empty content_list extract error", err.Error())+ }+ })+}+func TestDispatch_PDFMonkeyOCRv2Markdown_UsesNativeParseEndpoint(t *testing.T) {server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {if r.Method != http.MethodPost || r.URL.Path != "/parse" {diff --git a/internal/ingestion/component/pdf_vision_dispatch.go b/internal/ingestion/component/pdf_vision_dispatch.goindex d0b141a1c7..9cd60c1fc8 100644--- a/internal/ingestion/component/pdf_vision_dispatch.go+++ b/internal/ingestion/component/pdf_vision_dispatch.go@@ -482,10 +482,9 @@ func getAnyString(object map[string]any, keys ...string) string {return ""}-// dispatchMinerUPDF submits a PDF to the selected MinerU OCR model-// via the streaming /file_parse endpoint and returns parsed sections.-// Mirrors Python's mineru_parser.py:parse_PDF which POSTs with-// stream=True and reads the zip response body directly (no polling).+// dispatchMinerUPDF submits a PDF to the selected MinerU OCR model.+// MinerU 4.0 uses the V1 upload/job API; 3.x still POSTs /file_parse with+// stream=True and returns a zip body (Python mineru_parser.py:parse_PDF).func dispatchMinerUPDF(ctx context.Context,db *gorm.DB,@@ -516,7 +515,6 @@ func dispatchMinerUPDF(if strings.TrimSpace(baseURL) == "" {return parser.ParseResult{}, fmt.Errorf("parser: MinerU requires a base URL (instance base_url, mineru_apiserver, or MINERU_APISERVER)")}- apiURL := strings.TrimRight(baseURL, "/") + "/file_parse"parseMethod := mineruAPIParseMethod(getStringOr(setup, "mineru_parse_method", ""))// MinerU's language is an OCR engine parameter, not the caption language:@@ -531,10 +529,40 @@ func dispatchMinerUPDF(mineruLang := mineruLangCode(lang)backend := modelModule.ResolveMinerUBackend(getStringOr(setup, "mineru_backend", ""), apiKeyRaw)serverURL := modelModule.ResolveMinerUServerURL(getStringOr(setup, "mineru_server_url", ""), apiKeyRaw)- if err := modelModule.ValidateMinerUConfig(backend, serverURL); err != nil {++ v1, err := modelModule.MinerUSupportsV1(ctx, baseURL, apiKeyRaw)+ if err != nil {+ return parser.ParseResult{}, fmt.Errorf("parser: MinerU probe: %w", err)+ }+ if err := modelModule.ValidateMinerUConfigForAPI(backend, serverURL, v1); err != nil {return parser.ParseResult{}, err}+ if v1 {+ result, err := modelModule.ParseMinerUV1(ctx, baseURL, apiKeyRaw, filename, binary, backend, parseMethod, 0)+ if err != nil {+ return parser.ParseResult{}, fmt.Errorf("parser: MinerU V1: %w", err)+ }+ v1Markdown := ""+ var zipBytes []byte+ if result != nil {+ v1Markdown = strings.TrimSpace(result.Markdown)+ zipBytes = result.Zip+ }+ items, extractErr := mineruExtractJSONItems(zipBytes)+ if extractErr == nil && len(items) > 0 {+ return mineruJSONParseResult(filename, items), nil+ }+ if v1Markdown != "" {+ return buildMarkdownOCRDispatchResult(result.Markdown), nil+ }+ if extractErr != nil {+ return parser.ParseResult{}, fmt.Errorf("parser: MinerU extract: %w", extractErr)+ }+ return parser.ParseResult{}, fmt.Errorf("parser: MinerU extract: no sections")+ }++ apiURL := strings.TrimRight(baseURL, "/") + "/file_parse"zipBytes, err := mineruStreamParse(apiURL, apiKeyRaw, binary, parseMethod, mineruLang, backend, serverURL)if err != nil {return parser.ParseResult{}, fmt.Errorf("parser: MinerU stream: %w", err)@@ -545,16 +573,7 @@ func dispatchMinerUPDF(return parser.ParseResult{}, fmt.Errorf("parser: MinerU extract: %w", err)}- return parser.ParseResult{- OutputFormat: "json",- File: map[string]any{- "name": filename,- "page_count": mineruItemPageCount(items),- "outline": []map[string]any{},- "parse_method": "mineru",- },- JSON: items,- }, nil+ return mineruJSONParseResult(filename, items), nil}// resolveMinerUModelForDispatch resolves the OCR model used by the MinerU PDF@@ -1028,6 +1047,19 @@ func mineruZipImageDataURI(images map[string]*zip.File, imgPath string) string {return fmt.Sprintf("data:%s;base64,%s", mediaType, base64.StdEncoding.EncodeToString(data))}+func mineruJSONParseResult(filename string, items []map[string]any) parser.ParseResult {+ return parser.ParseResult{+ OutputFormat: "json",+ File: map[string]any{+ "name": filename,+ "page_count": mineruItemPageCount(items),+ "outline": []map[string]any{},+ "parse_method": "mineru",+ },+ JSON: items,+ }+}+func mineruItemPageCount(items []map[string]any) int {maxPage := 0for _, item := range items {diff --git a/internal/parser/parser/pdf_parser_mineru_test.go b/internal/parser/parser/pdf_parser_mineru_test.goindex 0cdb2f5ec0..87304f7090 100644--- a/internal/parser/parser/pdf_parser_mineru_test.go+++ b/internal/parser/parser/pdf_parser_mineru_test.go@@ -283,10 +283,14 @@ func TestPDFParser_ParseWithResult_MinerUReadsBackendFromProviderJSONApiKey(t *t}func TestPDFParser_ParseWithResult_MinerURequiresServerURLForHTTPClientBackend(t *testing.T) {+ withSSRFBypass(t)+ server := httptest.NewServer(http.NotFoundHandler())+ defer server.Close()+pdf := NewPDFParser()pdf.ConfigureFromSetup(map[string]any{"parse_method": "MinerU",- "mineru_apiserver": "http://mineru-api:8888",+ "mineru_apiserver": server.URL,"mineru_backend": "hybrid-http-client",})diff --git a/web/src/pages/user-setting/setting-model/provider-schema/field-config/provider-config-map.ts b/web/src/pages/user-setting/setting-model/provider-schema/field-config/provider-config-map.tsindex 228367bb8e..a7810bbb75 100644--- a/web/src/pages/user-setting/setting-model/provider-schema/field-config/provider-config-map.ts+++ b/web/src/pages/user-setting/setting-model/provider-schema/field-config/provider-config-map.ts@@ -675,11 +675,21 @@ export const ProviderConfigMap: Record<string, ProviderConfig> = {defaultValue: 'pipeline',placeholder: 'mineruSelectBackend',options: [- { label: 'pipeline', value: 'pipeline' },- { label: 'vlm-engine', value: 'vlm-engine' },- { label: 'hybrid-engine', value: 'hybrid-engine' },- { label: 'vlm-http-client', value: 'vlm-http-client' },- { label: 'hybrid-http-client', value: 'hybrid-http-client' },+ { label: 'pipeline (MinerU 3.x)', value: 'pipeline' },+ { label: 'vlm-engine (MinerU 3.x)', value: 'vlm-engine' },+ { label: 'hybrid-engine (MinerU 3.x)', value: 'hybrid-engine' },+ {+ label: 'vlm-http-client (MinerU 3.x)',+ value: 'vlm-http-client',+ },+ {+ label: 'hybrid-http-client (MinerU 3.x)',+ value: 'hybrid-http-client',+ },+ { label: 'basic (MinerU 4)', value: 'basic' },+ { label: 'standard (MinerU 4)', value: 'standard' },+ { label: 'advanced (MinerU 4)', value: 'advanced' },+ { label: 'flash (MinerU 4)', value: 'flash' },],validation: { message: 'mineruBackendMessage' },},