RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
fix(parser): wire MinerU server_url through Go PDF parser (#19445)
Sourav Rajvibc8ee6a96770
diff --git a/build.sh b/build.shindex 7de4e775a2..e3a89964a4 100755--- a/build.sh+++ b/build.sh@@ -369,12 +369,28 @@ check_office_oxide_deps() {if ! strings "$lib_path" 2>/dev/null | grep -Fxq "$OFFICE_OXIDE_VERSION"; thenlocal found_versionfound_version=$(strings "$lib_path" 2>/dev/null | grep -E "^0\.[0-9]+\.[0-9]+$" | head -1)- echo -e "${RED}Error: office_oxide native lib version mismatch${NC}"- echo " Required: v${OFFICE_OXIDE_VERSION}; found: ${found_version:-unknown}"- echo " A stale lib silently loses PPT97 (.ppt) slide content. Refresh:"- echo " rm -rf ~/ragflow-native-libs/office_oxide ragflow_deps/office_oxide-linux-x86_64.tar.gz"- echo " uv run python3 ragflow_deps/download_go_deps.py"- exit 1+ echo -e "${YELLOW}office_oxide native lib version mismatch (required v${OFFICE_OXIDE_VERSION}; found: ${found_version:-unknown}); refreshing from download${NC}"+ rm -rf "${OFFICE_OXIDE_PREFIX}" "${PROJECT_ROOT}/office_oxide-linux-x86_64.tar.gz"+ if ! (cd "${PROJECT_ROOT}" && uv run python3 ragflow_deps/download_go_deps.py); then+ echo -e "${RED}Error: office_oxide native lib version mismatch${NC}"+ echo " Required: v${OFFICE_OXIDE_VERSION}; found: ${found_version:-unknown}"+ echo " A stale lib silently loses PPT97 (.ppt) slide content. Refresh:"+ echo " rm -rf ~/ragflow-native-libs/office_oxide ragflow_deps/office_oxide-linux-x86_64.tar.gz"+ echo " uv run python3 ragflow_deps/download_go_deps.py"+ exit 1+ fi+ lib_path="${OFFICE_OXIDE_PREFIX}/lib/${lib_file}"+ header_path="${OFFICE_OXIDE_PREFIX}/include/office_oxide_c/office_oxide.h"+ if [ ! -f "$lib_path" ] || [ ! -f "$header_path" ]; then+ echo -e "${RED}Error: office_oxide native library not found after refresh${NC}"+ exit 1+ fi+ if ! strings "$lib_path" 2>/dev/null | grep -Fxq "$OFFICE_OXIDE_VERSION"; then+ found_version=$(strings "$lib_path" 2>/dev/null | grep -E "^0\.[0-9]+\.[0-9]+$" | head -1)+ echo -e "${RED}Error: office_oxide native lib version mismatch after refresh${NC}"+ echo " Required: v${OFFICE_OXIDE_VERSION}; found: ${found_version:-unknown}"+ exit 1+ fifiecho "✓ office_oxide v${OFFICE_OXIDE_VERSION} native library found at ${OFFICE_OXIDE_PREFIX}"diff --git a/internal/common/environments.go b/internal/common/environments.goindex 522a236ea2..91b6583fe0 100644--- a/internal/common/environments.go+++ b/internal/common/environments.go@@ -194,6 +194,7 @@ const (EnvMineruAPIServer = "MINERU_APISERVER"EnvMineruAPIKey = "MINERU_API_KEY"EnvMineruBackend = "MINERU_BACKEND"+ EnvMineruServerURL = "MINERU_SERVER_URL"EnvMonkeyOCRv2ServerURL = "MONKEYOCRV2_SERVER_URL"EnvMonkeyOCRv2Timeout = "MONKEYOCRV2_TIMEOUT"EnvOpenDataLoaderAPIServer = "OPENDATALOADER_APISERVER"diff --git a/internal/entity/models/mineru_config.go b/internal/entity/models/mineru_config.gonew file mode 100644index 0000000000..621b4997d8--- /dev/null+++ b/internal/entity/models/mineru_config.go@@ -0,0 +1,113 @@+//+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.+//++package models++import (+ "fmt"+ "ragflow/internal/common"+ "strings"+)++var ValidMinerUBackends = map[string]struct{}{+ "pipeline": {},+ "vlm-engine": {},+ "hybrid-engine": {},+ "vlm-http-client": {},+ "hybrid-http-client": {},+}++func MinerUBackendRequiresServerURL(backend string) bool {+ return backend == "vlm-http-client" || backend == "hybrid-http-client"+}++func ValidateMinerUConfig(backend, serverURL string) error {+ if _, ok := ValidMinerUBackends[backend]; !ok {+ return fmt.Errorf(+ "parser: MinerU invalid backend %q (valid: pipeline, vlm-engine, hybrid-engine, vlm-http-client, hybrid-http-client)",+ backend,+ )+ }+ if MinerUBackendRequiresServerURL(backend) && serverURL == "" {+ return fmt.Errorf("parser: MinerU requires mineru_server_url or MINERU_SERVER_URL for backend %q", backend)+ }+ return nil+}++// MinerUProviderAPIKeyConfig holds fields stored in the tenant MinerU provider api_key JSON.+type MinerUProviderAPIKeyConfig struct {+ IsProviderJSON bool+ APIServer string+ Backend string+ ServerURL string+ AccessToken string+}++// MinerUProviderConfigFromAPIKey parses the tenant MinerU api_key payload the same way+// the provider UI stores it: mineru_apiserver, mineru_backend, mineru_server_url, etc.+// A non-JSON api_key is treated as a plain bearer token. JSON provider config does not+// imply a bearer token unless mineru_api_key or access_token is present.+func MinerUProviderConfigFromAPIKey(apiKey string) MinerUProviderAPIKeyConfig {+ trimmed := strings.TrimSpace(apiKey)+ if trimmed == "" {+ return MinerUProviderAPIKeyConfig{}+ }+ raw := providerJSONConfigMap(trimmed)+ if raw == nil {+ return MinerUProviderAPIKeyConfig{AccessToken: trimmed}+ }+ get := func(key string) string {+ if value, ok := raw[key]; ok && value != nil {+ return strings.TrimSpace(fmt.Sprint(value))+ }+ return ""+ }+ token := get("mineru_api_key")+ if token == "" {+ token = get("access_token")+ }+ return MinerUProviderAPIKeyConfig{+ IsProviderJSON: true,+ APIServer: get("mineru_apiserver"),+ Backend: get("mineru_backend"),+ ServerURL: strings.TrimRight(get("mineru_server_url"), "/"),+ AccessToken: token,+ }+}++// MinerUBearerTokenFromAPIKey returns the Authorization bearer secret for MinerU HTTP calls.+func MinerUBearerTokenFromAPIKey(apiKey string) string {+ return MinerUProviderConfigFromAPIKey(apiKey).AccessToken+}++func ResolveMinerUBackend(setupBackend, apiKey string) string {+ backend := strings.TrimSpace(setupBackend)+ if backend == "" {+ cfg := MinerUProviderConfigFromAPIKey(apiKey)+ if cfg.Backend != "" {+ backend = cfg.Backend+ }+ }+ if backend == "" {+ backend = strings.TrimSpace(common.GetEnv(common.EnvMineruBackend))+ }+ if backend == "" {+ backend = "pipeline"+ }+ return backend+}++func ResolveMinerUServerURL(setupServerURL, apiKey string) string {+ serverURL := strings.TrimSpace(setupServerURL)+ if serverURL == "" {+ cfg := MinerUProviderConfigFromAPIKey(apiKey)+ if cfg.ServerURL != "" {+ serverURL = cfg.ServerURL+ }+ }+ if serverURL == "" {+ serverURL = strings.TrimSpace(common.GetEnv(common.EnvMineruServerURL))+ }+ return strings.TrimRight(serverURL, "/")+}diff --git a/internal/entity/models/mineru_config_test.go b/internal/entity/models/mineru_config_test.gonew file mode 100644index 0000000000..86e50f3539--- /dev/null+++ b/internal/entity/models/mineru_config_test.go@@ -0,0 +1,73 @@+//+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.+//++package models++import "testing"++func TestMinerUProviderConfigFromAPIKey(t *testing.T) {+ t.Run("plain bearer token", func(t *testing.T) {+ cfg := MinerUProviderConfigFromAPIKey("secret-token")+ if cfg.IsProviderJSON || cfg.AccessToken != "secret-token" {+ t.Fatalf("cfg = %#v, want plain token", cfg)+ }+ })++ t.Run("provider JSON without bearer", func(t *testing.T) {+ raw := `{"mineru_apiserver":"http://mineru:9987","mineru_backend":"vlm-http-client","mineru_server_url":"http://vllm:30000"}`+ cfg := MinerUProviderConfigFromAPIKey(raw)+ if !cfg.IsProviderJSON {+ t.Fatal("expected provider JSON")+ }+ if cfg.AccessToken != "" {+ t.Fatalf("AccessToken = %q, want empty", cfg.AccessToken)+ }+ if cfg.APIServer != "http://mineru:9987" || cfg.Backend != "vlm-http-client" || cfg.ServerURL != "http://vllm:30000" {+ t.Fatalf("cfg = %#v", cfg)+ }+ })++ t.Run("provider JSON with explicit token", func(t *testing.T) {+ raw := `{"mineru_apiserver":"http://mineru:9987","mineru_api_key":"bearer-secret"}`+ cfg := MinerUProviderConfigFromAPIKey(raw)+ if cfg.AccessToken != "bearer-secret" {+ t.Fatalf("AccessToken = %q", cfg.AccessToken)+ }+ })++ t.Run("access_token only JSON object", func(t *testing.T) {+ raw := `{"access_token":"secret"}`+ cfg := MinerUProviderConfigFromAPIKey(raw)+ if !cfg.IsProviderJSON {+ t.Fatal("expected provider JSON")+ }+ if cfg.AccessToken != "secret" {+ t.Fatalf("AccessToken = %q, want secret", cfg.AccessToken)+ }+ })++ t.Run("JSON object without token fields", func(t *testing.T) {+ raw := `{"other_field":"value"}`+ cfg := MinerUProviderConfigFromAPIKey(raw)+ if !cfg.IsProviderJSON {+ t.Fatal("expected provider JSON")+ }+ if cfg.AccessToken != "" {+ t.Fatalf("AccessToken = %q, want empty", cfg.AccessToken)+ }+ })+}++func TestResolveMinerUBackendAndServerURL(t *testing.T) {+ apiKey := `{"mineru_backend":"hybrid-http-client","mineru_server_url":"http://vllm:30000"}`+ if got := ResolveMinerUBackend("", apiKey); got != "hybrid-http-client" {+ t.Fatalf("backend = %q", got)+ }+ if got := ResolveMinerUServerURL("", apiKey); got != "http://vllm:30000" {+ t.Fatalf("server_url = %q", got)+ }+ if got := ResolveMinerUBackend("pipeline", apiKey); got != "pipeline" {+ t.Fatalf("setup should win, backend = %q", got)+ }+}diff --git a/internal/entity/models/mineru_local.go b/internal/entity/models/mineru_local.goindex 9613c804cb..c3b9813c1d 100644--- a/internal/entity/models/mineru_local.go+++ b/internal/entity/models/mineru_local.go@@ -25,6 +25,7 @@ import ("mime/multipart""net/http""ragflow/internal/common"+ "strings")type MinerULocalModel struct {@@ -125,10 +126,28 @@ func (m *MinerULocalModel) ParseFile(ctx context.Context, modelName *string, conreturn nil, fmt.Errorf("failed to write file content: %w", err)}- if modelName != nil && *modelName != "" {- _ = writer.WriteField("backend", *modelName)- } else {- _ = writer.WriteField("backend", "pipeline")+ apiKeyRaw := ""+ if apiConfig != nil && apiConfig.ApiKey != nil {+ apiKeyRaw = *apiConfig.ApiKey+ }+ setupBackend := ""+ if modelName != nil && strings.TrimSpace(*modelName) != "" {+ setupBackend = strings.TrimSpace(*modelName)+ } else if parseFileConfig != nil {+ setupBackend = strings.TrimSpace(parseFileConfig.Backend)+ }+ setupServerURL := ""+ if parseFileConfig != nil {+ setupServerURL = parseFileConfig.ServerURL+ }+ backend := ResolveMinerUBackend(setupBackend, apiKeyRaw)+ serverURL := ResolveMinerUServerURL(setupServerURL, apiKeyRaw)+ if err := ValidateMinerUConfig(backend, serverURL); err != nil {+ return nil, err+ }+ _ = writer.WriteField("backend", backend)+ if serverURL != "" {+ _ = writer.WriteField("server_url", serverURL)}if err = writer.Close(); err != nil {@@ -145,8 +164,10 @@ func (m *MinerULocalModel) ParseFile(ctx context.Context, modelName *string, conreq.Header.Set("Content-Type", writer.FormDataContentType())- if auth := BearerAuth(apiConfig); auth != "" {- req.Header.Set("Authorization", auth)+ if apiConfig != nil && apiConfig.ApiKey != nil {+ if token := MinerUBearerTokenFromAPIKey(*apiConfig.ApiKey); token != "" {+ req.Header.Set("Authorization", "Bearer "+token)+ }}resp, err := m.baseModel.httpClient.Do(req)@@ -213,8 +234,10 @@ func (m *MinerULocalModel) ShowTask(ctx context.Context, taskID string, apiConfireturn nil, fmt.Errorf("failed to create status request: %w", err)}- if auth := BearerAuth(apiConfig); auth != "" {- req.Header.Set("Authorization", auth)+ if apiConfig != nil && apiConfig.ApiKey != nil {+ if token := MinerUBearerTokenFromAPIKey(*apiConfig.ApiKey); token != "" {+ req.Header.Set("Authorization", "Bearer "+token)+ }}resp, err := m.baseModel.httpClient.Do(req)diff --git a/internal/entity/models/mineru_local_test.go b/internal/entity/models/mineru_local_test.gonew file mode 100644index 0000000000..bab9ab95e9--- /dev/null+++ b/internal/entity/models/mineru_local_test.go@@ -0,0 +1,105 @@+//+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.+//++package models++import (+ "context"+ "io"+ "net/http"+ "net/http/httptest"+ "testing"+)++func TestMinerULocalParseFileResolvesServerURLFromProviderAPIKey(t *testing.T) {+ var gotServerURL string+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ if err := r.ParseMultipartForm(10 << 20); err != nil {+ http.Error(w, err.Error(), http.StatusBadRequest)+ return+ }+ gotServerURL = r.FormValue("server_url")+ w.WriteHeader(http.StatusOK)+ _, _ = w.Write([]byte(`{"data":{"task_id":"task-1"}}`))+ }))+ defer server.Close()++ baseURL := server.URL+ driver := NewMinerLocalUModel(map[string]string{"default": baseURL}, URLSuffix{DocumentParse: "file_parse", Task: "tasks"})+ apiKey := `{"mineru_backend":"vlm-http-client","mineru_server_url":"http://vllm:30000"}`+ apiConfig := &APIConfig{+ BaseURL: &baseURL,+ ApiKey: &apiKey,+ }+ backend := "vlm-http-client"+ _, err := driver.ParseFile(context.Background(), &backend, []byte("%PDF-1.4"), nil, apiConfig, &ParseFileConfig{}, nil)+ if err != nil {+ t.Fatalf("ParseFile: %v", err)+ }+ if gotServerURL != "http://vllm:30000" {+ t.Fatalf("server_url = %q, want http://vllm:30000", gotServerURL)+ }+}++func TestMinerULocalParseFileRequestOverridesProviderJSON(t *testing.T) {+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ _ = r.ParseMultipartForm(10 << 20)+ if r.FormValue("server_url") != "http://override:9" {+ http.Error(w, "bad server_url", http.StatusBadRequest)+ return+ }+ w.WriteHeader(http.StatusOK)+ _, _ = w.Write([]byte(`{"task_id":"task-2"}`))+ }))+ defer server.Close()++ baseURL := server.URL+ driver := NewMinerLocalUModel(map[string]string{"default": baseURL}, URLSuffix{DocumentParse: "file_parse", Task: "tasks"})+ apiKey := `{"mineru_server_url":"http://vllm:30000"}`+ apiConfig := &APIConfig{BaseURL: &baseURL, ApiKey: &apiKey}+ cfg := &ParseFileConfig{ServerURL: "http://override:9"}+ _, err := driver.ParseFile(context.Background(), nil, []byte("x"), nil, apiConfig, cfg, nil)+ if err != nil {+ t.Fatalf("ParseFile: %v", err)+ }+}++// Ensure multipart field names stay stable (regression guard).+func TestMinerULocalParseFileMultipartFieldNames(t *testing.T) {+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ reader, err := r.MultipartReader()+ if err != nil {+ http.Error(w, err.Error(), http.StatusBadRequest)+ return+ }+ names := map[string]bool{}+ for {+ part, err := reader.NextPart()+ if err == io.EOF {+ break+ }+ if err != nil {+ http.Error(w, err.Error(), http.StatusBadRequest)+ return+ }+ names[part.FormName()] = true+ _ = part.Close()+ }+ if !names["files"] || !names["backend"] {+ http.Error(w, "missing fields", http.StatusBadRequest)+ return+ }+ w.WriteHeader(http.StatusOK)+ _, _ = w.Write([]byte(`{"data":{"task_id":"t"}}`))+ }))+ defer server.Close()++ baseURL := server.URL+ driver := NewMinerLocalUModel(map[string]string{"default": baseURL}, URLSuffix{DocumentParse: "file_parse", Task: "tasks"})+ backend := "pipeline"+ _, err := driver.ParseFile(context.Background(), &backend, []byte("data"), nil, &APIConfig{BaseURL: &baseURL}, nil, nil)+ if err != nil {+ t.Fatalf("ParseFile: %v", err)+ }+}diff --git a/internal/entity/models/provider_json_config.go b/internal/entity/models/provider_json_config.gonew file mode 100644index 0000000000..29b7a0aee4--- /dev/null+++ b/internal/entity/models/provider_json_config.go@@ -0,0 +1,48 @@+//+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.+//++package models++import (+ "encoding/json"+ "fmt"+ "strings"+)++func providerJSONConfigMap(apiKey string) map[string]any {+ trimmed := strings.TrimSpace(apiKey)+ if trimmed == "" {+ return nil+ }+ var config map[string]any+ if err := json.Unmarshal([]byte(trimmed), &config); err != nil {+ return nil+ }+ if nested, ok := config["api_key"].(map[string]any); ok {+ config = nested+ }+ return config+}++// ProviderJSONConfigValue reads string fields from a tenant provider api_key JSON blob.+// keys are tried in order.+func ProviderJSONConfigValue(apiKey string, keys ...string) string {+ config := providerJSONConfigMap(apiKey)+ if config == nil {+ return ""+ }+ for _, key := range keys {+ if value, ok := config[key]; ok && value != nil {+ return strings.TrimSpace(fmt.Sprint(value))+ }+ }+ return ""+}++func ProviderJSONConfigValueFromAPIConfig(apiConfig *APIConfig, keys ...string) string {+ if apiConfig == nil || apiConfig.ApiKey == nil {+ return ""+ }+ return ProviderJSONConfigValue(*apiConfig.ApiKey, keys...)+}diff --git a/internal/entity/models/types.go b/internal/entity/models/types.goindex c5f3622c85..b127ecd138 100644--- a/internal/entity/models/types.go+++ b/internal/entity/models/types.go@@ -262,6 +262,8 @@ type OCRConfig struct {type ParseFileConfig struct {ParseMethod string `json:"parse_method"`+ Backend string `json:"backend"`+ ServerURL string `json:"server_url"`}// EmbeddingModel wraps a ModelDriver with embedding-specific configurationdiff --git a/internal/handler/providers.go b/internal/handler/providers.goindex 1ea79af4f9..f553e4d8a6 100644--- a/internal/handler/providers.go+++ b/internal/handler/providers.go@@ -1569,6 +1569,9 @@ type ParseFileRequest struct {ModelID *string `json:"model_id"`Content []byte `json:"content"`URL *string `json:"url"`+ // Optional MinerU async /file/parse overrides (setup → provider api_key JSON → env).+ Backend *string `json:"backend"`+ ServerURL *string `json:"server_url"`}func (h *ProviderHandler) ParseFile(c *gin.Context) {@@ -1612,6 +1615,12 @@ func (h *ProviderHandler) ParseFile(c *gin.Context) {}parseFileConfig := models.ParseFileConfig{}+ if req.Backend != nil {+ parseFileConfig.Backend = strings.TrimSpace(*req.Backend)+ }+ if req.ServerURL != nil {+ parseFileConfig.ServerURL = strings.TrimRight(strings.TrimSpace(*req.ServerURL), "/")+ }// Non-stream responsevar response *models.ParseFileResponsediff --git a/internal/ingestion/component/parser_dispatch_test.go b/internal/ingestion/component/parser_dispatch_test.goindex 251fa1dc6c..22ef1f223f 100644--- a/internal/ingestion/component/parser_dispatch_test.go+++ b/internal/ingestion/component/parser_dispatch_test.go@@ -771,6 +771,96 @@ func TestDispatch_PDFMinerUJSON_ParsesMarkdownToStructuredItems(t *testing.T) {}}+func TestDispatch_PDFMinerUMarkdown_SendsServerURLFromProviderConfig(t *testing.T) {+ withSSRFBypass(t)+ var gotBackend, gotServerURL, gotAuth string+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ if r.Method == http.MethodPost && r.URL.Path == "/file_parse" {+ gotAuth = r.Header.Get("Authorization")+ if err := r.ParseMultipartForm(1 << 20); err != nil {+ t.Errorf("ParseMultipartForm: %v", err)+ http.Error(w, err.Error(), http.StatusBadRequest)+ return+ }+ gotBackend = r.FormValue("backend")+ gotServerURL = r.FormValue("server_url")+ buf := new(bytes.Buffer)+ zw := zip.NewWriter(buf)+ f, _ := zw.Create("content_list.json")+ _, _ = f.Write([]byte(`[{"type":"text","text":"http-client"}]`))+ _ = zw.Close()+ w.Header().Set("Content-Type", "application/zip")+ _, _ = w.Write(buf.Bytes())+ return+ }+ http.NotFound(w, r)+ }))+ defer server.Close()++ origResolver := resolveMinerUModelForDispatch+ defer func() { resolveMinerUModelForDispatch = origResolver }()+ baseURL := server.URL+ apiKey := `{"mineru_backend":"vlm-http-client","mineru_server_url":"http://vllm-host:30000"}`+ resolveMinerUModelForDispatch = func(ctx context.Context, db *gorm.DB, tenantID, modelID string) (models.ModelDriver, string, *models.APIConfig, error) {+ return &mineruTestDriver{}, "mineru-model", &models.APIConfig{ApiKey: &apiKey, BaseURL: &baseURL}, nil+ }++ setups := defaultSetups()+ setups["pdf"]["parse_method"] = "mineru"+ setups["pdf"]["output_format"] = "markdown"+ c := &ParserComponent{setups: setups}++ out, err := c.Invoke(t.Context(), nil, map[string]any{+ "binary": []byte("%PDF-1.4"),+ "file_type": "pdf",+ "name": "sample.pdf",+ "tenant_id": "test-tenant",+ })+ if err != nil {+ t.Fatalf("Invoke: %v", err)+ }+ if got, want := gotBackend, "vlm-http-client"; got != want {+ t.Fatalf("backend = %q, want %q", got, want)+ }+ if got, want := gotServerURL, "http://vllm-host:30000"; got != want {+ t.Fatalf("server_url = %q, want %q", got, want)+ }+ if gotAuth != "" {+ t.Fatalf("Authorization = %q, want empty for provider JSON api_key", gotAuth)+ }+ if got := out["output_format"]; got != "json" {+ t.Fatalf("output_format = %v, want json", got)+ }+ requireJSONText(t, out, "http-client")+}++func TestDispatch_PDFMinerUMarkdown_RequiresServerURLForHTTPClientBackend(t *testing.T) {+ origResolver := resolveMinerUModelForDispatch+ defer func() { resolveMinerUModelForDispatch = origResolver }()+ baseURL := "http://mineru-api:8888"+ apiKey := `{"mineru_backend":"hybrid-http-client"}`+ resolveMinerUModelForDispatch = func(ctx context.Context, db *gorm.DB, tenantID, modelID string) (models.ModelDriver, string, *models.APIConfig, error) {+ return &mineruTestDriver{}, "mineru-model", &models.APIConfig{ApiKey: &apiKey, BaseURL: &baseURL}, nil+ }++ setups := defaultSetups()+ setups["pdf"]["parse_method"] = "mineru"+ c := &ParserComponent{setups: setups}++ _, err := c.Invoke(t.Context(), nil, map[string]any{+ "binary": []byte("%PDF-1.4"),+ "file_type": "pdf",+ "name": "sample.pdf",+ "tenant_id": "test-tenant",+ })+ if err == nil {+ t.Fatal("Invoke: want error when mineru_server_url is missing, got nil")+ }+ if !strings.Contains(err.Error(), "mineru_server_url") {+ t.Fatalf("error = %q, want mineru_server_url context", err.Error())+ }+}+func TestDispatch_PDFMonkeyOCRv2Markdown_UsesNativeParseEndpoint(t *testing.T) {server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {if r.Method != http.MethodPost || r.URL.Path != "/parse" {diff --git a/internal/ingestion/component/pdf_vision_dispatch.go b/internal/ingestion/component/pdf_vision_dispatch.goindex 0fc808f62e..65bec8a2f2 100644--- a/internal/ingestion/component/pdf_vision_dispatch.go+++ b/internal/ingestion/component/pdf_vision_dispatch.go@@ -220,7 +220,7 @@ func dispatchMonkeyOCRv2PDF(ctx context.Context, db *gorm.DB, filename string, b}if apiConfig != nil && apiConfig.BaseURL != nil && strings.TrimSpace(*apiConfig.BaseURL) != "" {baseURL = strings.TrimRight(*apiConfig.BaseURL, "/")- } else if configuredURL := monkeyOCRv2APIConfigValue(apiConfig, "monkeyocrv2_server_url", common.EnvMonkeyOCRv2ServerURL); configuredURL != "" {+ } else if configuredURL := modelModule.ProviderJSONConfigValueFromAPIConfig(apiConfig, "monkeyocrv2_server_url"); configuredURL != "" {baseURL = strings.TrimRight(configuredURL, "/")}timeout = monkeyOCRv2RequestTimeout(setup, apiConfig)@@ -324,32 +324,13 @@ func monkeyOCRv2ParseRequest(ctx context.Context, apiURL, filename string, binarreturn data, nil}-func monkeyOCRv2APIConfigValue(apiConfig *modelModule.APIConfig, keys ...string) string {- if apiConfig == nil || apiConfig.ApiKey == nil || strings.TrimSpace(*apiConfig.ApiKey) == "" {- return ""- }- var config map[string]any- if err := json.Unmarshal([]byte(*apiConfig.ApiKey), &config); err != nil {- return ""- }- if nested, ok := config["api_key"].(map[string]any); ok {- config = nested- }- for _, key := range keys {- if value, ok := config[key]; ok && value != nil {- return strings.TrimSpace(fmt.Sprint(value))- }- }- return ""-}-func monkeyOCRv2RequestTimeout(setup schema.ParserSetup, apiConfig *modelModule.APIConfig) time.Duration {value := ""if raw, ok := setup["monkeyocrv2_timeout"]; ok {value = strings.TrimSpace(fmt.Sprint(raw))}if value == "" {- value = monkeyOCRv2APIConfigValue(apiConfig, "monkeyocrv2_timeout", common.EnvMonkeyOCRv2Timeout)+ value = modelModule.ProviderJSONConfigValueFromAPIConfig(apiConfig, "monkeyocrv2_timeout", common.EnvMonkeyOCRv2Timeout)}if value == "" {value = os.Getenv(common.EnvMonkeyOCRv2Timeout)@@ -514,10 +495,19 @@ func dispatchMinerUPDF("parser: MinerU requires a MinerU OCR model; found %q. Please add a MinerU OCR model to your tenant", driver.Name())}+ apiKeyRaw := ""+ if apiConfig.ApiKey != nil {+ apiKeyRaw = *apiConfig.ApiKey+ }+ providerCfg := modelModule.MinerUProviderConfigFromAPIKey(apiKeyRaw)+baseURL := ""if apiConfig.BaseURL != nil {baseURL = *apiConfig.BaseURL}+ if baseURL == "" {+ baseURL = providerCfg.APIServer+ }if baseURL == "" {baseURL, _ = resolveMinerUBaseURL(driver, apiConfig)}@@ -533,9 +523,13 @@ func dispatchMinerUPDF(lang = getStringOr(setup, "lang", "English")}mineruLang := mineruLangCode(lang)- backend := getStringOr(setup, "mineru_backend", "pipeline")+ backend := modelModule.ResolveMinerUBackend(getStringOr(setup, "mineru_backend", ""), apiKeyRaw)+ serverURL := modelModule.ResolveMinerUServerURL(getStringOr(setup, "mineru_server_url", ""), apiKeyRaw)+ if err := modelModule.ValidateMinerUConfig(backend, serverURL); err != nil {+ return parser.ParseResult{}, err+ }- zipBytes, err := mineruStreamParse(apiURL, apiConfig.ApiKey, binary, parseMethod, mineruLang, backend)+ zipBytes, err := mineruStreamParse(apiURL, apiKeyRaw, binary, parseMethod, mineruLang, backend, serverURL)if err != nil {return parser.ParseResult{}, fmt.Errorf("parser: MinerU stream: %w", err)}@@ -694,7 +688,7 @@ func mineruAPIParseMethod(raw string) string {// mineruStreamParse POSTs the PDF binary to the MinerU /file_parse// endpoint with streaming and returns the zip response body.// Mirrors Python's mineru_parser.py._run_mineru_api with stream=True.-func mineruStreamParse(apiURL string, apiKey *string, binary []byte, parseMethod, lang, backend string) ([]byte, error) {+func mineruStreamParse(apiURL string, apiKey string, binary []byte, parseMethod, lang, backend, serverURL string) ([]byte, error) {var body bytes.Bufferwriter := multipart.NewWriter(&body)@@ -719,6 +713,9 @@ func mineruStreamParse(apiURL string, apiKey *string, binary []byte, parseMethod_ = writer.WriteField("return_model_output", "true")_ = writer.WriteField("formula_enable", "true")_ = writer.WriteField("table_enable", "true")+ if strings.TrimSpace(serverURL) != "" {+ _ = writer.WriteField("server_url", strings.TrimRight(strings.TrimSpace(serverURL), "/"))+ }if err := writer.Close(); err != nil {return nil, fmt.Errorf("finalize form: %w", err)@@ -732,8 +729,8 @@ func mineruStreamParse(apiURL string, apiKey *string, binary []byte, parseMethodreturn nil, fmt.Errorf("create request: %w", err)}req.Header.Set("Content-Type", writer.FormDataContentType())- if apiKey != nil && *apiKey != "" {- req.Header.Set("Authorization", "Bearer "+*apiKey)+ if token := modelModule.MinerUBearerTokenFromAPIKey(apiKey); token != "" {+ req.Header.Set("Authorization", "Bearer "+token)}resp, err := http.DefaultClient.Do(req)diff --git a/internal/parser/parser/pdf_parser_common.go b/internal/parser/parser/pdf_parser_common.goindex f5e4201c07..935e08f9e3 100644--- a/internal/parser/parser/pdf_parser_common.go+++ b/internal/parser/parser/pdf_parser_common.go@@ -80,6 +80,7 @@ type PDFParser struct {MinerUAPIServer stringMinerUAPIKey stringMinerUBackend string+ MinerUServerURL stringMinerUPollTimeout time.DurationPaddleOCRBaseURL stringPaddleOCRAPIKey string@@ -118,7 +119,7 @@ func NewPDFParser() *PDFParser {LibType: "pdf_oxide",ParseMethod: "deepdoc",OutputFormat: "json",- MinerUBackend: "pipeline",+ MinerUBackend: "",MinerUPollTimeout: minerUPollTimeout,PaddleOCRAlgorithm: "PaddleOCR-VL",OpenDataLoaderTimeout: 600,@@ -167,6 +168,9 @@ func (p *PDFParser) ConfigureFromSetup(setup map[string]any) {if v, ok := setup["mineru_backend"].(string); ok && v != "" {p.MinerUBackend = v}+ if v, ok := setup["mineru_server_url"].(string); ok && v != "" {+ p.MinerUServerURL = v+ }if v, ok := setup["mineru_timeout_seconds"].(int); ok && v > 0 {p.MinerUPollTimeout = time.Duration(v) * time.Second}diff --git a/internal/parser/parser/pdf_parser_mineru.go b/internal/parser/parser/pdf_parser_mineru.goindex fa9819f2b0..0b66f11d95 100644--- a/internal/parser/parser/pdf_parser_mineru.go+++ b/internal/parser/parser/pdf_parser_mineru.go@@ -17,23 +17,31 @@ func parsePDFWithMinerU(ctx context.Context, filename string, data []byte, parseif len(data) == 0 {return emptyPDFResult(filename)}+ providerCfg := models.MinerUProviderConfigFromAPIKey(parser.MinerUAPIKey)+apiServer := strings.TrimSpace(parser.MinerUAPIServer)+ if apiServer == "" {+ apiServer = providerCfg.APIServer+ }if apiServer == "" {apiServer = strings.TrimSpace(common.GetEnv(common.EnvMineruAPIServer))}if apiServer == "" {return ParseResult{Err: fmt.Errorf("parser: MinerU requires mineru_apiserver or MINERU_APISERVER")}}- apiKey := parser.MinerUAPIKey- if strings.TrimSpace(apiKey) == "" {- apiKey = strings.TrimSpace(common.GetEnv(common.EnvMineruAPIKey))++ apiKey := providerCfg.AccessToken+ if apiKey == "" && !providerCfg.IsProviderJSON {+ apiKey = strings.TrimSpace(parser.MinerUAPIKey)}- backend := strings.TrimSpace(parser.MinerUBackend)- if backend == "" {- backend = strings.TrimSpace(common.GetEnv(common.EnvMineruBackend))+ if apiKey == "" {+ apiKey = strings.TrimSpace(common.GetEnv(common.EnvMineruAPIKey))}- if backend == "" {- backend = "pipeline"++ backend := models.ResolveMinerUBackend(parser.MinerUBackend, parser.MinerUAPIKey)+ serverURL := models.ResolveMinerUServerURL(parser.MinerUServerURL, parser.MinerUAPIKey)+ if err := models.ValidateMinerUConfig(backend, serverURL); err != nil {+ return ParseResult{Err: err}}timeout := parser.MinerUPollTimeoutif timeout <= 0 {@@ -51,7 +59,8 @@ func parsePDFWithMinerU(ctx context.Context, filename string, data []byte, parseapiConfig.ApiKey = &apiKey}- task, err := driver.ParseFile(ctx, &backend, data, nil, apiConfig, &models.ParseFileConfig{}, nil)+ parseFileConfig := &models.ParseFileConfig{ServerURL: serverURL}+ task, err := driver.ParseFile(ctx, &backend, data, nil, apiConfig, parseFileConfig, nil)if err != nil {return ParseResult{Err: fmt.Errorf("parser: MinerU submit: %w", err)}}diff --git a/internal/parser/parser/pdf_parser_mineru_test.go b/internal/parser/parser/pdf_parser_mineru_test.goindex b16af3b514..0cdb2f5ec0 100644--- a/internal/parser/parser/pdf_parser_mineru_test.go+++ b/internal/parser/parser/pdf_parser_mineru_test.go@@ -1,6 +1,7 @@package parserimport (+ "fmt""io""mime/multipart""net/http"@@ -164,3 +165,153 @@ func TestMinerUUploadShape(t *testing.T) {t.Fatalf("multipart body = %q, want backend field", body.String())}}++func TestPDFParser_ParseWithResult_MinerUSendsServerURLForHTTPClientBackend(t *testing.T) {+ withSSRFBypass(t)+ var gotServerURL string++ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ switch {+ case r.Method == http.MethodPost && r.URL.Path == "/file_parse":+ reader, err := r.MultipartReader()+ if err != nil {+ t.Errorf("MultipartReader: %v", err)+ return+ }+ for {+ part, err := reader.NextPart()+ if err == io.EOF {+ break+ }+ if err != nil {+ t.Errorf("NextPart: %v", err)+ return+ }+ if part.FormName() == "server_url" {+ body, _ := io.ReadAll(part)+ gotServerURL = string(body)+ }+ }+ w.Header().Set("Content-Type", "application/json")+ _, _ = w.Write([]byte(`{"data":{"task_id":"task-http"}}`))+ case r.Method == http.MethodGet && r.URL.Path == "/tasks/task-http/result":+ w.Header().Set("Content-Type", "application/json")+ _, _ = w.Write([]byte(`{"results":{"doc":{"md_content":"# HTTP\n"}}}`))+ default:+ http.NotFound(w, r)+ }+ }))+ defer server.Close()++ pdf := NewPDFParser()+ pdf.ConfigureFromSetup(map[string]any{+ "parse_method": "MinerU",+ "output_format": "markdown",+ "mineru_apiserver": server.URL,+ "mineru_backend": "vlm-http-client",+ "mineru_server_url": "http://vllm-host:30000",+ })++ res := pdf.ParseWithResult(t.Context(), "sample.pdf", []byte("%PDF-1.4\nmock"))+ if res.Err != nil {+ t.Fatalf("ParseWithResult: %v", res.Err)+ }+ if got, want := gotServerURL, "http://vllm-host:30000"; got != want {+ t.Fatalf("server_url = %q, want %q", got, want)+ }+}++func TestPDFParser_ParseWithResult_MinerUReadsBackendFromProviderJSONApiKey(t *testing.T) {+ withSSRFBypass(t)+ var gotBackend, gotServerURL, gotAuth string+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ switch {+ case r.Method == http.MethodPost && r.URL.Path == "/file_parse":+ gotAuth = r.Header.Get("Authorization")+ reader, err := r.MultipartReader()+ if err != nil {+ t.Errorf("MultipartReader: %v", err)+ http.Error(w, err.Error(), http.StatusBadRequest)+ return+ }+ for {+ part, err := reader.NextPart()+ if err == io.EOF {+ break+ }+ if err != nil {+ t.Errorf("NextPart: %v", err)+ return+ }+ switch part.FormName() {+ case "backend":+ body, _ := io.ReadAll(part)+ gotBackend = string(body)+ case "server_url":+ body, _ := io.ReadAll(part)+ gotServerURL = string(body)+ }+ }+ w.Header().Set("Content-Type", "application/json")+ _, _ = w.Write([]byte(`{"data":{"task_id":"task-json"}}`))+ case r.Method == http.MethodGet && r.URL.Path == "/tasks/task-json/result":+ w.Header().Set("Content-Type", "application/json")+ _, _ = w.Write([]byte(`{"results":{"doc":{"md_content":"# JSON\n"}}}`))+ default:+ http.NotFound(w, r)+ }+ }))+ defer server.Close()++ apiKeyJSON := fmt.Sprintf(`{"mineru_apiserver":"%s","mineru_backend":"vlm-http-client","mineru_server_url":"http://vllm-host:30000"}`, server.URL)+ pdf := NewPDFParser()+ pdf.ConfigureFromSetup(map[string]any{+ "parse_method": "MinerU",+ "output_format": "markdown",+ "mineru_api_key": apiKeyJSON,+ })+ res := pdf.ParseWithResult(t.Context(), "sample.pdf", []byte("%PDF-1.4\nmock"))+ if res.Err != nil {+ t.Fatalf("ParseWithResult: %v", res.Err)+ }+ if gotBackend != "vlm-http-client" || gotServerURL != "http://vllm-host:30000" {+ t.Fatalf("backend=%q server_url=%q", gotBackend, gotServerURL)+ }+ if gotAuth != "" {+ t.Fatalf("Authorization = %q, want empty for provider JSON api_key", gotAuth)+ }+}++func TestPDFParser_ParseWithResult_MinerURequiresServerURLForHTTPClientBackend(t *testing.T) {+ pdf := NewPDFParser()+ pdf.ConfigureFromSetup(map[string]any{+ "parse_method": "MinerU",+ "mineru_apiserver": "http://mineru-api:8888",+ "mineru_backend": "hybrid-http-client",+ })++ res := pdf.ParseWithResult(t.Context(), "sample.pdf", []byte("%PDF-1.4\nmock"))+ if res.Err == nil {+ t.Fatal("ParseWithResult: want error when mineru_server_url is missing, got nil")+ }+ if !strings.Contains(res.Err.Error(), "mineru_server_url") {+ t.Fatalf("error = %q, want mineru_server_url context", res.Err.Error())+ }+}++func TestPDFParser_ParseWithResult_MinerURejectsInvalidBackend(t *testing.T) {+ pdf := NewPDFParser()+ pdf.ConfigureFromSetup(map[string]any{+ "parse_method": "MinerU",+ "mineru_apiserver": "http://mineru-api:8888",+ "mineru_backend": "vlm-transformers",+ })++ res := pdf.ParseWithResult(t.Context(), "sample.pdf", []byte("%PDF-1.4\nmock"))+ if res.Err == nil {+ t.Fatal("ParseWithResult: want error for invalid backend, got nil")+ }+ if !strings.Contains(res.Err.Error(), "invalid backend") {+ t.Fatalf("error = %q, want invalid backend context", res.Err.Error())+ }+}