Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

fix(parser): wire MinerU server_url through Go PDF parser (#19445)

Sourav Rajvibc8ee6a96770
diff --git a/build.sh b/build.sh
index 7de4e775a2..e3a89964a4 100755
--- a/build.sh
+++ b/build.sh
@@ -369,12 +369,28 @@ check_office_oxide_deps() {
if ! strings "$lib_path" 2>/dev/null | grep -Fxq "$OFFICE_OXIDE_VERSION"; then
local found_version
found_version=$(strings "$lib_path" 2>/dev/null | grep -E "^0\.[0-9]+\.[0-9]+$" | head -1)
- echo -e "${RED}Error: office_oxide native lib version mismatch${NC}"
- echo " Required: v${OFFICE_OXIDE_VERSION}; found: ${found_version:-unknown}"
- echo " A stale lib silently loses PPT97 (.ppt) slide content. Refresh:"
- echo " rm -rf ~/ragflow-native-libs/office_oxide ragflow_deps/office_oxide-linux-x86_64.tar.gz"
- echo " uv run python3 ragflow_deps/download_go_deps.py"
- exit 1
+ echo -e "${YELLOW}office_oxide native lib version mismatch (required v${OFFICE_OXIDE_VERSION}; found: ${found_version:-unknown}); refreshing from download${NC}"
+ rm -rf "${OFFICE_OXIDE_PREFIX}" "${PROJECT_ROOT}/office_oxide-linux-x86_64.tar.gz"
+ if ! (cd "${PROJECT_ROOT}" && uv run python3 ragflow_deps/download_go_deps.py); then
+ echo -e "${RED}Error: office_oxide native lib version mismatch${NC}"
+ echo " Required: v${OFFICE_OXIDE_VERSION}; found: ${found_version:-unknown}"
+ echo " A stale lib silently loses PPT97 (.ppt) slide content. Refresh:"
+ echo " rm -rf ~/ragflow-native-libs/office_oxide ragflow_deps/office_oxide-linux-x86_64.tar.gz"
+ echo " uv run python3 ragflow_deps/download_go_deps.py"
+ exit 1
+ fi
+ lib_path="${OFFICE_OXIDE_PREFIX}/lib/${lib_file}"
+ header_path="${OFFICE_OXIDE_PREFIX}/include/office_oxide_c/office_oxide.h"
+ if [ ! -f "$lib_path" ] || [ ! -f "$header_path" ]; then
+ echo -e "${RED}Error: office_oxide native library not found after refresh${NC}"
+ exit 1
+ fi
+ if ! strings "$lib_path" 2>/dev/null | grep -Fxq "$OFFICE_OXIDE_VERSION"; then
+ found_version=$(strings "$lib_path" 2>/dev/null | grep -E "^0\.[0-9]+\.[0-9]+$" | head -1)
+ echo -e "${RED}Error: office_oxide native lib version mismatch after refresh${NC}"
+ echo " Required: v${OFFICE_OXIDE_VERSION}; found: ${found_version:-unknown}"
+ exit 1
+ fi
fi
echo "✓ office_oxide v${OFFICE_OXIDE_VERSION} native library found at ${OFFICE_OXIDE_PREFIX}"
diff --git a/internal/common/environments.go b/internal/common/environments.go
index 522a236ea2..91b6583fe0 100644
--- a/internal/common/environments.go
+++ b/internal/common/environments.go
@@ -194,6 +194,7 @@ const (
EnvMineruAPIServer = "MINERU_APISERVER"
EnvMineruAPIKey = "MINERU_API_KEY"
EnvMineruBackend = "MINERU_BACKEND"
+ EnvMineruServerURL = "MINERU_SERVER_URL"
EnvMonkeyOCRv2ServerURL = "MONKEYOCRV2_SERVER_URL"
EnvMonkeyOCRv2Timeout = "MONKEYOCRV2_TIMEOUT"
EnvOpenDataLoaderAPIServer = "OPENDATALOADER_APISERVER"
diff --git a/internal/entity/models/mineru_config.go b/internal/entity/models/mineru_config.go
new file mode 100644
index 0000000000..621b4997d8
--- /dev/null
+++ b/internal/entity/models/mineru_config.go
@@ -0,0 +1,113 @@
+//
+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
+//
+
+package models
+
+import (
+ "fmt"
+ "ragflow/internal/common"
+ "strings"
+)
+
+var ValidMinerUBackends = map[string]struct{}{
+ "pipeline": {},
+ "vlm-engine": {},
+ "hybrid-engine": {},
+ "vlm-http-client": {},
+ "hybrid-http-client": {},
+}
+
+func MinerUBackendRequiresServerURL(backend string) bool {
+ return backend == "vlm-http-client" || backend == "hybrid-http-client"
+}
+
+func ValidateMinerUConfig(backend, serverURL string) error {
+ if _, ok := ValidMinerUBackends[backend]; !ok {
+ return fmt.Errorf(
+ "parser: MinerU invalid backend %q (valid: pipeline, vlm-engine, hybrid-engine, vlm-http-client, hybrid-http-client)",
+ backend,
+ )
+ }
+ if MinerUBackendRequiresServerURL(backend) && serverURL == "" {
+ return fmt.Errorf("parser: MinerU requires mineru_server_url or MINERU_SERVER_URL for backend %q", backend)
+ }
+ return nil
+}
+
+// MinerUProviderAPIKeyConfig holds fields stored in the tenant MinerU provider api_key JSON.
+type MinerUProviderAPIKeyConfig struct {
+ IsProviderJSON bool
+ APIServer string
+ Backend string
+ ServerURL string
+ AccessToken string
+}
+
+// MinerUProviderConfigFromAPIKey parses the tenant MinerU api_key payload the same way
+// the provider UI stores it: mineru_apiserver, mineru_backend, mineru_server_url, etc.
+// A non-JSON api_key is treated as a plain bearer token. JSON provider config does not
+// imply a bearer token unless mineru_api_key or access_token is present.
+func MinerUProviderConfigFromAPIKey(apiKey string) MinerUProviderAPIKeyConfig {
+ trimmed := strings.TrimSpace(apiKey)
+ if trimmed == "" {
+ return MinerUProviderAPIKeyConfig{}
+ }
+ raw := providerJSONConfigMap(trimmed)
+ if raw == nil {
+ return MinerUProviderAPIKeyConfig{AccessToken: trimmed}
+ }
+ get := func(key string) string {
+ if value, ok := raw[key]; ok && value != nil {
+ return strings.TrimSpace(fmt.Sprint(value))
+ }
+ return ""
+ }
+ token := get("mineru_api_key")
+ if token == "" {
+ token = get("access_token")
+ }
+ return MinerUProviderAPIKeyConfig{
+ IsProviderJSON: true,
+ APIServer: get("mineru_apiserver"),
+ Backend: get("mineru_backend"),
+ ServerURL: strings.TrimRight(get("mineru_server_url"), "/"),
+ AccessToken: token,
+ }
+}
+
+// MinerUBearerTokenFromAPIKey returns the Authorization bearer secret for MinerU HTTP calls.
+func MinerUBearerTokenFromAPIKey(apiKey string) string {
+ return MinerUProviderConfigFromAPIKey(apiKey).AccessToken
+}
+
+func ResolveMinerUBackend(setupBackend, apiKey string) string {
+ backend := strings.TrimSpace(setupBackend)
+ if backend == "" {
+ cfg := MinerUProviderConfigFromAPIKey(apiKey)
+ if cfg.Backend != "" {
+ backend = cfg.Backend
+ }
+ }
+ if backend == "" {
+ backend = strings.TrimSpace(common.GetEnv(common.EnvMineruBackend))
+ }
+ if backend == "" {
+ backend = "pipeline"
+ }
+ return backend
+}
+
+func ResolveMinerUServerURL(setupServerURL, apiKey string) string {
+ serverURL := strings.TrimSpace(setupServerURL)
+ if serverURL == "" {
+ cfg := MinerUProviderConfigFromAPIKey(apiKey)
+ if cfg.ServerURL != "" {
+ serverURL = cfg.ServerURL
+ }
+ }
+ if serverURL == "" {
+ serverURL = strings.TrimSpace(common.GetEnv(common.EnvMineruServerURL))
+ }
+ return strings.TrimRight(serverURL, "/")
+}
diff --git a/internal/entity/models/mineru_config_test.go b/internal/entity/models/mineru_config_test.go
new file mode 100644
index 0000000000..86e50f3539
--- /dev/null
+++ b/internal/entity/models/mineru_config_test.go
@@ -0,0 +1,73 @@
+//
+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
+//
+
+package models
+
+import "testing"
+
+func TestMinerUProviderConfigFromAPIKey(t *testing.T) {
+ t.Run("plain bearer token", func(t *testing.T) {
+ cfg := MinerUProviderConfigFromAPIKey("secret-token")
+ if cfg.IsProviderJSON || cfg.AccessToken != "secret-token" {
+ t.Fatalf("cfg = %#v, want plain token", cfg)
+ }
+ })
+
+ t.Run("provider JSON without bearer", func(t *testing.T) {
+ raw := `{"mineru_apiserver":"http://mineru:9987","mineru_backend":"vlm-http-client","mineru_server_url":"http://vllm:30000"}`
+ cfg := MinerUProviderConfigFromAPIKey(raw)
+ if !cfg.IsProviderJSON {
+ t.Fatal("expected provider JSON")
+ }
+ if cfg.AccessToken != "" {
+ t.Fatalf("AccessToken = %q, want empty", cfg.AccessToken)
+ }
+ if cfg.APIServer != "http://mineru:9987" || cfg.Backend != "vlm-http-client" || cfg.ServerURL != "http://vllm:30000" {
+ t.Fatalf("cfg = %#v", cfg)
+ }
+ })
+
+ t.Run("provider JSON with explicit token", func(t *testing.T) {
+ raw := `{"mineru_apiserver":"http://mineru:9987","mineru_api_key":"bearer-secret"}`
+ cfg := MinerUProviderConfigFromAPIKey(raw)
+ if cfg.AccessToken != "bearer-secret" {
+ t.Fatalf("AccessToken = %q", cfg.AccessToken)
+ }
+ })
+
+ t.Run("access_token only JSON object", func(t *testing.T) {
+ raw := `{"access_token":"secret"}`
+ cfg := MinerUProviderConfigFromAPIKey(raw)
+ if !cfg.IsProviderJSON {
+ t.Fatal("expected provider JSON")
+ }
+ if cfg.AccessToken != "secret" {
+ t.Fatalf("AccessToken = %q, want secret", cfg.AccessToken)
+ }
+ })
+
+ t.Run("JSON object without token fields", func(t *testing.T) {
+ raw := `{"other_field":"value"}`
+ cfg := MinerUProviderConfigFromAPIKey(raw)
+ if !cfg.IsProviderJSON {
+ t.Fatal("expected provider JSON")
+ }
+ if cfg.AccessToken != "" {
+ t.Fatalf("AccessToken = %q, want empty", cfg.AccessToken)
+ }
+ })
+}
+
+func TestResolveMinerUBackendAndServerURL(t *testing.T) {
+ apiKey := `{"mineru_backend":"hybrid-http-client","mineru_server_url":"http://vllm:30000"}`
+ if got := ResolveMinerUBackend("", apiKey); got != "hybrid-http-client" {
+ t.Fatalf("backend = %q", got)
+ }
+ if got := ResolveMinerUServerURL("", apiKey); got != "http://vllm:30000" {
+ t.Fatalf("server_url = %q", got)
+ }
+ if got := ResolveMinerUBackend("pipeline", apiKey); got != "pipeline" {
+ t.Fatalf("setup should win, backend = %q", got)
+ }
+}
diff --git a/internal/entity/models/mineru_local.go b/internal/entity/models/mineru_local.go
index 9613c804cb..c3b9813c1d 100644
--- a/internal/entity/models/mineru_local.go
+++ b/internal/entity/models/mineru_local.go
@@ -25,6 +25,7 @@ import (
"mime/multipart"
"net/http"
"ragflow/internal/common"
+ "strings"
)
type MinerULocalModel struct {
@@ -125,10 +126,28 @@ func (m *MinerULocalModel) ParseFile(ctx context.Context, modelName *string, con
return nil, fmt.Errorf("failed to write file content: %w", err)
}
- if modelName != nil && *modelName != "" {
- _ = writer.WriteField("backend", *modelName)
- } else {
- _ = writer.WriteField("backend", "pipeline")
+ apiKeyRaw := ""
+ if apiConfig != nil && apiConfig.ApiKey != nil {
+ apiKeyRaw = *apiConfig.ApiKey
+ }
+ setupBackend := ""
+ if modelName != nil && strings.TrimSpace(*modelName) != "" {
+ setupBackend = strings.TrimSpace(*modelName)
+ } else if parseFileConfig != nil {
+ setupBackend = strings.TrimSpace(parseFileConfig.Backend)
+ }
+ setupServerURL := ""
+ if parseFileConfig != nil {
+ setupServerURL = parseFileConfig.ServerURL
+ }
+ backend := ResolveMinerUBackend(setupBackend, apiKeyRaw)
+ serverURL := ResolveMinerUServerURL(setupServerURL, apiKeyRaw)
+ if err := ValidateMinerUConfig(backend, serverURL); err != nil {
+ return nil, err
+ }
+ _ = writer.WriteField("backend", backend)
+ if serverURL != "" {
+ _ = writer.WriteField("server_url", serverURL)
}
if err = writer.Close(); err != nil {
@@ -145,8 +164,10 @@ func (m *MinerULocalModel) ParseFile(ctx context.Context, modelName *string, con
req.Header.Set("Content-Type", writer.FormDataContentType())
- if auth := BearerAuth(apiConfig); auth != "" {
- req.Header.Set("Authorization", auth)
+ if apiConfig != nil && apiConfig.ApiKey != nil {
+ if token := MinerUBearerTokenFromAPIKey(*apiConfig.ApiKey); token != "" {
+ req.Header.Set("Authorization", "Bearer "+token)
+ }
}
resp, err := m.baseModel.httpClient.Do(req)
@@ -213,8 +234,10 @@ func (m *MinerULocalModel) ShowTask(ctx context.Context, taskID string, apiConfi
return nil, fmt.Errorf("failed to create status request: %w", err)
}
- if auth := BearerAuth(apiConfig); auth != "" {
- req.Header.Set("Authorization", auth)
+ if apiConfig != nil && apiConfig.ApiKey != nil {
+ if token := MinerUBearerTokenFromAPIKey(*apiConfig.ApiKey); token != "" {
+ req.Header.Set("Authorization", "Bearer "+token)
+ }
}
resp, err := m.baseModel.httpClient.Do(req)
diff --git a/internal/entity/models/mineru_local_test.go b/internal/entity/models/mineru_local_test.go
new file mode 100644
index 0000000000..bab9ab95e9
--- /dev/null
+++ b/internal/entity/models/mineru_local_test.go
@@ -0,0 +1,105 @@
+//
+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
+//
+
+package models
+
+import (
+ "context"
+ "io"
+ "net/http"
+ "net/http/httptest"
+ "testing"
+)
+
+func TestMinerULocalParseFileResolvesServerURLFromProviderAPIKey(t *testing.T) {
+ var gotServerURL string
+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ if err := r.ParseMultipartForm(10 << 20); err != nil {
+ http.Error(w, err.Error(), http.StatusBadRequest)
+ return
+ }
+ gotServerURL = r.FormValue("server_url")
+ w.WriteHeader(http.StatusOK)
+ _, _ = w.Write([]byte(`{"data":{"task_id":"task-1"}}`))
+ }))
+ defer server.Close()
+
+ baseURL := server.URL
+ driver := NewMinerLocalUModel(map[string]string{"default": baseURL}, URLSuffix{DocumentParse: "file_parse", Task: "tasks"})
+ apiKey := `{"mineru_backend":"vlm-http-client","mineru_server_url":"http://vllm:30000"}`
+ apiConfig := &APIConfig{
+ BaseURL: &baseURL,
+ ApiKey: &apiKey,
+ }
+ backend := "vlm-http-client"
+ _, err := driver.ParseFile(context.Background(), &backend, []byte("%PDF-1.4"), nil, apiConfig, &ParseFileConfig{}, nil)
+ if err != nil {
+ t.Fatalf("ParseFile: %v", err)
+ }
+ if gotServerURL != "http://vllm:30000" {
+ t.Fatalf("server_url = %q, want http://vllm:30000", gotServerURL)
+ }
+}
+
+func TestMinerULocalParseFileRequestOverridesProviderJSON(t *testing.T) {
+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ _ = r.ParseMultipartForm(10 << 20)
+ if r.FormValue("server_url") != "http://override:9" {
+ http.Error(w, "bad server_url", http.StatusBadRequest)
+ return
+ }
+ w.WriteHeader(http.StatusOK)
+ _, _ = w.Write([]byte(`{"task_id":"task-2"}`))
+ }))
+ defer server.Close()
+
+ baseURL := server.URL
+ driver := NewMinerLocalUModel(map[string]string{"default": baseURL}, URLSuffix{DocumentParse: "file_parse", Task: "tasks"})
+ apiKey := `{"mineru_server_url":"http://vllm:30000"}`
+ apiConfig := &APIConfig{BaseURL: &baseURL, ApiKey: &apiKey}
+ cfg := &ParseFileConfig{ServerURL: "http://override:9"}
+ _, err := driver.ParseFile(context.Background(), nil, []byte("x"), nil, apiConfig, cfg, nil)
+ if err != nil {
+ t.Fatalf("ParseFile: %v", err)
+ }
+}
+
+// Ensure multipart field names stay stable (regression guard).
+func TestMinerULocalParseFileMultipartFieldNames(t *testing.T) {
+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ reader, err := r.MultipartReader()
+ if err != nil {
+ http.Error(w, err.Error(), http.StatusBadRequest)
+ return
+ }
+ names := map[string]bool{}
+ for {
+ part, err := reader.NextPart()
+ if err == io.EOF {
+ break
+ }
+ if err != nil {
+ http.Error(w, err.Error(), http.StatusBadRequest)
+ return
+ }
+ names[part.FormName()] = true
+ _ = part.Close()
+ }
+ if !names["files"] || !names["backend"] {
+ http.Error(w, "missing fields", http.StatusBadRequest)
+ return
+ }
+ w.WriteHeader(http.StatusOK)
+ _, _ = w.Write([]byte(`{"data":{"task_id":"t"}}`))
+ }))
+ defer server.Close()
+
+ baseURL := server.URL
+ driver := NewMinerLocalUModel(map[string]string{"default": baseURL}, URLSuffix{DocumentParse: "file_parse", Task: "tasks"})
+ backend := "pipeline"
+ _, err := driver.ParseFile(context.Background(), &backend, []byte("data"), nil, &APIConfig{BaseURL: &baseURL}, nil, nil)
+ if err != nil {
+ t.Fatalf("ParseFile: %v", err)
+ }
+}
diff --git a/internal/entity/models/provider_json_config.go b/internal/entity/models/provider_json_config.go
new file mode 100644
index 0000000000..29b7a0aee4
--- /dev/null
+++ b/internal/entity/models/provider_json_config.go
@@ -0,0 +1,48 @@
+//
+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
+//
+
+package models
+
+import (
+ "encoding/json"
+ "fmt"
+ "strings"
+)
+
+func providerJSONConfigMap(apiKey string) map[string]any {
+ trimmed := strings.TrimSpace(apiKey)
+ if trimmed == "" {
+ return nil
+ }
+ var config map[string]any
+ if err := json.Unmarshal([]byte(trimmed), &config); err != nil {
+ return nil
+ }
+ if nested, ok := config["api_key"].(map[string]any); ok {
+ config = nested
+ }
+ return config
+}
+
+// ProviderJSONConfigValue reads string fields from a tenant provider api_key JSON blob.
+// keys are tried in order.
+func ProviderJSONConfigValue(apiKey string, keys ...string) string {
+ config := providerJSONConfigMap(apiKey)
+ if config == nil {
+ return ""
+ }
+ for _, key := range keys {
+ if value, ok := config[key]; ok && value != nil {
+ return strings.TrimSpace(fmt.Sprint(value))
+ }
+ }
+ return ""
+}
+
+func ProviderJSONConfigValueFromAPIConfig(apiConfig *APIConfig, keys ...string) string {
+ if apiConfig == nil || apiConfig.ApiKey == nil {
+ return ""
+ }
+ return ProviderJSONConfigValue(*apiConfig.ApiKey, keys...)
+}
diff --git a/internal/entity/models/types.go b/internal/entity/models/types.go
index c5f3622c85..b127ecd138 100644
--- a/internal/entity/models/types.go
+++ b/internal/entity/models/types.go
@@ -262,6 +262,8 @@ type OCRConfig struct {
type ParseFileConfig struct {
ParseMethod string `json:"parse_method"`
+ Backend string `json:"backend"`
+ ServerURL string `json:"server_url"`
}
// EmbeddingModel wraps a ModelDriver with embedding-specific configuration
diff --git a/internal/handler/providers.go b/internal/handler/providers.go
index 1ea79af4f9..f553e4d8a6 100644
--- a/internal/handler/providers.go
+++ b/internal/handler/providers.go
@@ -1569,6 +1569,9 @@ type ParseFileRequest struct {
ModelID *string `json:"model_id"`
Content []byte `json:"content"`
URL *string `json:"url"`
+ // Optional MinerU async /file/parse overrides (setup → provider api_key JSON → env).
+ Backend *string `json:"backend"`
+ ServerURL *string `json:"server_url"`
}
func (h *ProviderHandler) ParseFile(c *gin.Context) {
@@ -1612,6 +1615,12 @@ func (h *ProviderHandler) ParseFile(c *gin.Context) {
}
parseFileConfig := models.ParseFileConfig{}
+ if req.Backend != nil {
+ parseFileConfig.Backend = strings.TrimSpace(*req.Backend)
+ }
+ if req.ServerURL != nil {
+ parseFileConfig.ServerURL = strings.TrimRight(strings.TrimSpace(*req.ServerURL), "/")
+ }
// Non-stream response
var response *models.ParseFileResponse
diff --git a/internal/ingestion/component/parser_dispatch_test.go b/internal/ingestion/component/parser_dispatch_test.go
index 251fa1dc6c..22ef1f223f 100644
--- a/internal/ingestion/component/parser_dispatch_test.go
+++ b/internal/ingestion/component/parser_dispatch_test.go
@@ -771,6 +771,96 @@ func TestDispatch_PDFMinerUJSON_ParsesMarkdownToStructuredItems(t *testing.T) {
}
}
+func TestDispatch_PDFMinerUMarkdown_SendsServerURLFromProviderConfig(t *testing.T) {
+ withSSRFBypass(t)
+ var gotBackend, gotServerURL, gotAuth string
+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ if r.Method == http.MethodPost && r.URL.Path == "/file_parse" {
+ gotAuth = r.Header.Get("Authorization")
+ if err := r.ParseMultipartForm(1 << 20); err != nil {
+ t.Errorf("ParseMultipartForm: %v", err)
+ http.Error(w, err.Error(), http.StatusBadRequest)
+ return
+ }
+ gotBackend = r.FormValue("backend")
+ gotServerURL = r.FormValue("server_url")
+ buf := new(bytes.Buffer)
+ zw := zip.NewWriter(buf)
+ f, _ := zw.Create("content_list.json")
+ _, _ = f.Write([]byte(`[{"type":"text","text":"http-client"}]`))
+ _ = zw.Close()
+ w.Header().Set("Content-Type", "application/zip")
+ _, _ = w.Write(buf.Bytes())
+ return
+ }
+ http.NotFound(w, r)
+ }))
+ defer server.Close()
+
+ origResolver := resolveMinerUModelForDispatch
+ defer func() { resolveMinerUModelForDispatch = origResolver }()
+ baseURL := server.URL
+ apiKey := `{"mineru_backend":"vlm-http-client","mineru_server_url":"http://vllm-host:30000"}`
+ resolveMinerUModelForDispatch = func(ctx context.Context, db *gorm.DB, tenantID, modelID string) (models.ModelDriver, string, *models.APIConfig, error) {
+ return &mineruTestDriver{}, "mineru-model", &models.APIConfig{ApiKey: &apiKey, BaseURL: &baseURL}, nil
+ }
+
+ setups := defaultSetups()
+ setups["pdf"]["parse_method"] = "mineru"
+ setups["pdf"]["output_format"] = "markdown"
+ c := &ParserComponent{setups: setups}
+
+ out, err := c.Invoke(t.Context(), nil, map[string]any{
+ "binary": []byte("%PDF-1.4"),
+ "file_type": "pdf",
+ "name": "sample.pdf",
+ "tenant_id": "test-tenant",
+ })
+ if err != nil {
+ t.Fatalf("Invoke: %v", err)
+ }
+ if got, want := gotBackend, "vlm-http-client"; got != want {
+ t.Fatalf("backend = %q, want %q", got, want)
+ }
+ if got, want := gotServerURL, "http://vllm-host:30000"; got != want {
+ t.Fatalf("server_url = %q, want %q", got, want)
+ }
+ if gotAuth != "" {
+ t.Fatalf("Authorization = %q, want empty for provider JSON api_key", gotAuth)
+ }
+ if got := out["output_format"]; got != "json" {
+ t.Fatalf("output_format = %v, want json", got)
+ }
+ requireJSONText(t, out, "http-client")
+}
+
+func TestDispatch_PDFMinerUMarkdown_RequiresServerURLForHTTPClientBackend(t *testing.T) {
+ origResolver := resolveMinerUModelForDispatch
+ defer func() { resolveMinerUModelForDispatch = origResolver }()
+ baseURL := "http://mineru-api:8888"
+ apiKey := `{"mineru_backend":"hybrid-http-client"}`
+ resolveMinerUModelForDispatch = func(ctx context.Context, db *gorm.DB, tenantID, modelID string) (models.ModelDriver, string, *models.APIConfig, error) {
+ return &mineruTestDriver{}, "mineru-model", &models.APIConfig{ApiKey: &apiKey, BaseURL: &baseURL}, nil
+ }
+
+ setups := defaultSetups()
+ setups["pdf"]["parse_method"] = "mineru"
+ c := &ParserComponent{setups: setups}
+
+ _, err := c.Invoke(t.Context(), nil, map[string]any{
+ "binary": []byte("%PDF-1.4"),
+ "file_type": "pdf",
+ "name": "sample.pdf",
+ "tenant_id": "test-tenant",
+ })
+ if err == nil {
+ t.Fatal("Invoke: want error when mineru_server_url is missing, got nil")
+ }
+ if !strings.Contains(err.Error(), "mineru_server_url") {
+ t.Fatalf("error = %q, want mineru_server_url context", err.Error())
+ }
+}
+
func TestDispatch_PDFMonkeyOCRv2Markdown_UsesNativeParseEndpoint(t *testing.T) {
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.Method != http.MethodPost || r.URL.Path != "/parse" {
diff --git a/internal/ingestion/component/pdf_vision_dispatch.go b/internal/ingestion/component/pdf_vision_dispatch.go
index 0fc808f62e..65bec8a2f2 100644
--- a/internal/ingestion/component/pdf_vision_dispatch.go
+++ b/internal/ingestion/component/pdf_vision_dispatch.go
@@ -220,7 +220,7 @@ func dispatchMonkeyOCRv2PDF(ctx context.Context, db *gorm.DB, filename string, b
}
if apiConfig != nil && apiConfig.BaseURL != nil && strings.TrimSpace(*apiConfig.BaseURL) != "" {
baseURL = strings.TrimRight(*apiConfig.BaseURL, "/")
- } else if configuredURL := monkeyOCRv2APIConfigValue(apiConfig, "monkeyocrv2_server_url", common.EnvMonkeyOCRv2ServerURL); configuredURL != "" {
+ } else if configuredURL := modelModule.ProviderJSONConfigValueFromAPIConfig(apiConfig, "monkeyocrv2_server_url"); configuredURL != "" {
baseURL = strings.TrimRight(configuredURL, "/")
}
timeout = monkeyOCRv2RequestTimeout(setup, apiConfig)
@@ -324,32 +324,13 @@ func monkeyOCRv2ParseRequest(ctx context.Context, apiURL, filename string, binar
return data, nil
}
-func monkeyOCRv2APIConfigValue(apiConfig *modelModule.APIConfig, keys ...string) string {
- if apiConfig == nil || apiConfig.ApiKey == nil || strings.TrimSpace(*apiConfig.ApiKey) == "" {
- return ""
- }
- var config map[string]any
- if err := json.Unmarshal([]byte(*apiConfig.ApiKey), &config); err != nil {
- return ""
- }
- if nested, ok := config["api_key"].(map[string]any); ok {
- config = nested
- }
- for _, key := range keys {
- if value, ok := config[key]; ok && value != nil {
- return strings.TrimSpace(fmt.Sprint(value))
- }
- }
- return ""
-}
-
func monkeyOCRv2RequestTimeout(setup schema.ParserSetup, apiConfig *modelModule.APIConfig) time.Duration {
value := ""
if raw, ok := setup["monkeyocrv2_timeout"]; ok {
value = strings.TrimSpace(fmt.Sprint(raw))
}
if value == "" {
- value = monkeyOCRv2APIConfigValue(apiConfig, "monkeyocrv2_timeout", common.EnvMonkeyOCRv2Timeout)
+ value = modelModule.ProviderJSONConfigValueFromAPIConfig(apiConfig, "monkeyocrv2_timeout", common.EnvMonkeyOCRv2Timeout)
}
if value == "" {
value = os.Getenv(common.EnvMonkeyOCRv2Timeout)
@@ -514,10 +495,19 @@ func dispatchMinerUPDF(
"parser: MinerU requires a MinerU OCR model; found %q. Please add a MinerU OCR model to your tenant", driver.Name())
}
+ apiKeyRaw := ""
+ if apiConfig.ApiKey != nil {
+ apiKeyRaw = *apiConfig.ApiKey
+ }
+ providerCfg := modelModule.MinerUProviderConfigFromAPIKey(apiKeyRaw)
+
baseURL := ""
if apiConfig.BaseURL != nil {
baseURL = *apiConfig.BaseURL
}
+ if baseURL == "" {
+ baseURL = providerCfg.APIServer
+ }
if baseURL == "" {
baseURL, _ = resolveMinerUBaseURL(driver, apiConfig)
}
@@ -533,9 +523,13 @@ func dispatchMinerUPDF(
lang = getStringOr(setup, "lang", "English")
}
mineruLang := mineruLangCode(lang)
- backend := getStringOr(setup, "mineru_backend", "pipeline")
+ backend := modelModule.ResolveMinerUBackend(getStringOr(setup, "mineru_backend", ""), apiKeyRaw)
+ serverURL := modelModule.ResolveMinerUServerURL(getStringOr(setup, "mineru_server_url", ""), apiKeyRaw)
+ if err := modelModule.ValidateMinerUConfig(backend, serverURL); err != nil {
+ return parser.ParseResult{}, err
+ }
- zipBytes, err := mineruStreamParse(apiURL, apiConfig.ApiKey, binary, parseMethod, mineruLang, backend)
+ zipBytes, err := mineruStreamParse(apiURL, apiKeyRaw, binary, parseMethod, mineruLang, backend, serverURL)
if err != nil {
return parser.ParseResult{}, fmt.Errorf("parser: MinerU stream: %w", err)
}
@@ -694,7 +688,7 @@ func mineruAPIParseMethod(raw string) string {
// mineruStreamParse POSTs the PDF binary to the MinerU /file_parse
// endpoint with streaming and returns the zip response body.
// Mirrors Python's mineru_parser.py._run_mineru_api with stream=True.
-func mineruStreamParse(apiURL string, apiKey *string, binary []byte, parseMethod, lang, backend string) ([]byte, error) {
+func mineruStreamParse(apiURL string, apiKey string, binary []byte, parseMethod, lang, backend, serverURL string) ([]byte, error) {
var body bytes.Buffer
writer := multipart.NewWriter(&body)
@@ -719,6 +713,9 @@ func mineruStreamParse(apiURL string, apiKey *string, binary []byte, parseMethod
_ = writer.WriteField("return_model_output", "true")
_ = writer.WriteField("formula_enable", "true")
_ = writer.WriteField("table_enable", "true")
+ if strings.TrimSpace(serverURL) != "" {
+ _ = writer.WriteField("server_url", strings.TrimRight(strings.TrimSpace(serverURL), "/"))
+ }
if err := writer.Close(); err != nil {
return nil, fmt.Errorf("finalize form: %w", err)
@@ -732,8 +729,8 @@ func mineruStreamParse(apiURL string, apiKey *string, binary []byte, parseMethod
return nil, fmt.Errorf("create request: %w", err)
}
req.Header.Set("Content-Type", writer.FormDataContentType())
- if apiKey != nil && *apiKey != "" {
- req.Header.Set("Authorization", "Bearer "+*apiKey)
+ if token := modelModule.MinerUBearerTokenFromAPIKey(apiKey); token != "" {
+ req.Header.Set("Authorization", "Bearer "+token)
}
resp, err := http.DefaultClient.Do(req)
diff --git a/internal/parser/parser/pdf_parser_common.go b/internal/parser/parser/pdf_parser_common.go
index f5e4201c07..935e08f9e3 100644
--- a/internal/parser/parser/pdf_parser_common.go
+++ b/internal/parser/parser/pdf_parser_common.go
@@ -80,6 +80,7 @@ type PDFParser struct {
MinerUAPIServer string
MinerUAPIKey string
MinerUBackend string
+ MinerUServerURL string
MinerUPollTimeout time.Duration
PaddleOCRBaseURL string
PaddleOCRAPIKey string
@@ -118,7 +119,7 @@ func NewPDFParser() *PDFParser {
LibType: "pdf_oxide",
ParseMethod: "deepdoc",
OutputFormat: "json",
- MinerUBackend: "pipeline",
+ MinerUBackend: "",
MinerUPollTimeout: minerUPollTimeout,
PaddleOCRAlgorithm: "PaddleOCR-VL",
OpenDataLoaderTimeout: 600,
@@ -167,6 +168,9 @@ func (p *PDFParser) ConfigureFromSetup(setup map[string]any) {
if v, ok := setup["mineru_backend"].(string); ok && v != "" {
p.MinerUBackend = v
}
+ if v, ok := setup["mineru_server_url"].(string); ok && v != "" {
+ p.MinerUServerURL = v
+ }
if v, ok := setup["mineru_timeout_seconds"].(int); ok && v > 0 {
p.MinerUPollTimeout = time.Duration(v) * time.Second
}
diff --git a/internal/parser/parser/pdf_parser_mineru.go b/internal/parser/parser/pdf_parser_mineru.go
index fa9819f2b0..0b66f11d95 100644
--- a/internal/parser/parser/pdf_parser_mineru.go
+++ b/internal/parser/parser/pdf_parser_mineru.go
@@ -17,23 +17,31 @@ func parsePDFWithMinerU(ctx context.Context, filename string, data []byte, parse
if len(data) == 0 {
return emptyPDFResult(filename)
}
+ providerCfg := models.MinerUProviderConfigFromAPIKey(parser.MinerUAPIKey)
+
apiServer := strings.TrimSpace(parser.MinerUAPIServer)
+ if apiServer == "" {
+ apiServer = providerCfg.APIServer
+ }
if apiServer == "" {
apiServer = strings.TrimSpace(common.GetEnv(common.EnvMineruAPIServer))
}
if apiServer == "" {
return ParseResult{Err: fmt.Errorf("parser: MinerU requires mineru_apiserver or MINERU_APISERVER")}
}
- apiKey := parser.MinerUAPIKey
- if strings.TrimSpace(apiKey) == "" {
- apiKey = strings.TrimSpace(common.GetEnv(common.EnvMineruAPIKey))
+
+ apiKey := providerCfg.AccessToken
+ if apiKey == "" && !providerCfg.IsProviderJSON {
+ apiKey = strings.TrimSpace(parser.MinerUAPIKey)
}
- backend := strings.TrimSpace(parser.MinerUBackend)
- if backend == "" {
- backend = strings.TrimSpace(common.GetEnv(common.EnvMineruBackend))
+ if apiKey == "" {
+ apiKey = strings.TrimSpace(common.GetEnv(common.EnvMineruAPIKey))
}
- if backend == "" {
- backend = "pipeline"
+
+ backend := models.ResolveMinerUBackend(parser.MinerUBackend, parser.MinerUAPIKey)
+ serverURL := models.ResolveMinerUServerURL(parser.MinerUServerURL, parser.MinerUAPIKey)
+ if err := models.ValidateMinerUConfig(backend, serverURL); err != nil {
+ return ParseResult{Err: err}
}
timeout := parser.MinerUPollTimeout
if timeout <= 0 {
@@ -51,7 +59,8 @@ func parsePDFWithMinerU(ctx context.Context, filename string, data []byte, parse
apiConfig.ApiKey = &apiKey
}
- task, err := driver.ParseFile(ctx, &backend, data, nil, apiConfig, &models.ParseFileConfig{}, nil)
+ parseFileConfig := &models.ParseFileConfig{ServerURL: serverURL}
+ task, err := driver.ParseFile(ctx, &backend, data, nil, apiConfig, parseFileConfig, nil)
if err != nil {
return ParseResult{Err: fmt.Errorf("parser: MinerU submit: %w", err)}
}
diff --git a/internal/parser/parser/pdf_parser_mineru_test.go b/internal/parser/parser/pdf_parser_mineru_test.go
index b16af3b514..0cdb2f5ec0 100644
--- a/internal/parser/parser/pdf_parser_mineru_test.go
+++ b/internal/parser/parser/pdf_parser_mineru_test.go
@@ -1,6 +1,7 @@
package parser
import (
+ "fmt"
"io"
"mime/multipart"
"net/http"
@@ -164,3 +165,153 @@ func TestMinerUUploadShape(t *testing.T) {
t.Fatalf("multipart body = %q, want backend field", body.String())
}
}
+
+func TestPDFParser_ParseWithResult_MinerUSendsServerURLForHTTPClientBackend(t *testing.T) {
+ withSSRFBypass(t)
+ var gotServerURL string
+
+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ switch {
+ case r.Method == http.MethodPost && r.URL.Path == "/file_parse":
+ reader, err := r.MultipartReader()
+ if err != nil {
+ t.Errorf("MultipartReader: %v", err)
+ return
+ }
+ for {
+ part, err := reader.NextPart()
+ if err == io.EOF {
+ break
+ }
+ if err != nil {
+ t.Errorf("NextPart: %v", err)
+ return
+ }
+ if part.FormName() == "server_url" {
+ body, _ := io.ReadAll(part)
+ gotServerURL = string(body)
+ }
+ }
+ w.Header().Set("Content-Type", "application/json")
+ _, _ = w.Write([]byte(`{"data":{"task_id":"task-http"}}`))
+ case r.Method == http.MethodGet && r.URL.Path == "/tasks/task-http/result":
+ w.Header().Set("Content-Type", "application/json")
+ _, _ = w.Write([]byte(`{"results":{"doc":{"md_content":"# HTTP\n"}}}`))
+ default:
+ http.NotFound(w, r)
+ }
+ }))
+ defer server.Close()
+
+ pdf := NewPDFParser()
+ pdf.ConfigureFromSetup(map[string]any{
+ "parse_method": "MinerU",
+ "output_format": "markdown",
+ "mineru_apiserver": server.URL,
+ "mineru_backend": "vlm-http-client",
+ "mineru_server_url": "http://vllm-host:30000",
+ })
+
+ res := pdf.ParseWithResult(t.Context(), "sample.pdf", []byte("%PDF-1.4\nmock"))
+ if res.Err != nil {
+ t.Fatalf("ParseWithResult: %v", res.Err)
+ }
+ if got, want := gotServerURL, "http://vllm-host:30000"; got != want {
+ t.Fatalf("server_url = %q, want %q", got, want)
+ }
+}
+
+func TestPDFParser_ParseWithResult_MinerUReadsBackendFromProviderJSONApiKey(t *testing.T) {
+ withSSRFBypass(t)
+ var gotBackend, gotServerURL, gotAuth string
+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ switch {
+ case r.Method == http.MethodPost && r.URL.Path == "/file_parse":
+ gotAuth = r.Header.Get("Authorization")
+ reader, err := r.MultipartReader()
+ if err != nil {
+ t.Errorf("MultipartReader: %v", err)
+ http.Error(w, err.Error(), http.StatusBadRequest)
+ return
+ }
+ for {
+ part, err := reader.NextPart()
+ if err == io.EOF {
+ break
+ }
+ if err != nil {
+ t.Errorf("NextPart: %v", err)
+ return
+ }
+ switch part.FormName() {
+ case "backend":
+ body, _ := io.ReadAll(part)
+ gotBackend = string(body)
+ case "server_url":
+ body, _ := io.ReadAll(part)
+ gotServerURL = string(body)
+ }
+ }
+ w.Header().Set("Content-Type", "application/json")
+ _, _ = w.Write([]byte(`{"data":{"task_id":"task-json"}}`))
+ case r.Method == http.MethodGet && r.URL.Path == "/tasks/task-json/result":
+ w.Header().Set("Content-Type", "application/json")
+ _, _ = w.Write([]byte(`{"results":{"doc":{"md_content":"# JSON\n"}}}`))
+ default:
+ http.NotFound(w, r)
+ }
+ }))
+ defer server.Close()
+
+ apiKeyJSON := fmt.Sprintf(`{"mineru_apiserver":"%s","mineru_backend":"vlm-http-client","mineru_server_url":"http://vllm-host:30000"}`, server.URL)
+ pdf := NewPDFParser()
+ pdf.ConfigureFromSetup(map[string]any{
+ "parse_method": "MinerU",
+ "output_format": "markdown",
+ "mineru_api_key": apiKeyJSON,
+ })
+ res := pdf.ParseWithResult(t.Context(), "sample.pdf", []byte("%PDF-1.4\nmock"))
+ if res.Err != nil {
+ t.Fatalf("ParseWithResult: %v", res.Err)
+ }
+ if gotBackend != "vlm-http-client" || gotServerURL != "http://vllm-host:30000" {
+ t.Fatalf("backend=%q server_url=%q", gotBackend, gotServerURL)
+ }
+ if gotAuth != "" {
+ t.Fatalf("Authorization = %q, want empty for provider JSON api_key", gotAuth)
+ }
+}
+
+func TestPDFParser_ParseWithResult_MinerURequiresServerURLForHTTPClientBackend(t *testing.T) {
+ pdf := NewPDFParser()
+ pdf.ConfigureFromSetup(map[string]any{
+ "parse_method": "MinerU",
+ "mineru_apiserver": "http://mineru-api:8888",
+ "mineru_backend": "hybrid-http-client",
+ })
+
+ res := pdf.ParseWithResult(t.Context(), "sample.pdf", []byte("%PDF-1.4\nmock"))
+ if res.Err == nil {
+ t.Fatal("ParseWithResult: want error when mineru_server_url is missing, got nil")
+ }
+ if !strings.Contains(res.Err.Error(), "mineru_server_url") {
+ t.Fatalf("error = %q, want mineru_server_url context", res.Err.Error())
+ }
+}
+
+func TestPDFParser_ParseWithResult_MinerURejectsInvalidBackend(t *testing.T) {
+ pdf := NewPDFParser()
+ pdf.ConfigureFromSetup(map[string]any{
+ "parse_method": "MinerU",
+ "mineru_apiserver": "http://mineru-api:8888",
+ "mineru_backend": "vlm-transformers",
+ })
+
+ res := pdf.ParseWithResult(t.Context(), "sample.pdf", []byte("%PDF-1.4\nmock"))
+ if res.Err == nil {
+ t.Fatal("ParseWithResult: want error for invalid backend, got nil")
+ }
+ if !strings.Contains(res.Err.Error(), "invalid backend") {
+ t.Fatalf("error = %q, want invalid backend context", res.Err.Error())
+ }
+}