Open-source LLM knowledge platform: turn raw documents into a queryable RAG, an autonomous reasoning agent, and a self-maintaining Wiki.
Go agentagenticaichatbotdsh-pluginembeddingsevaluationgenerative-aigolangknowledge-basellmmulti-tenantollamaopenaiquestion-answeringragrerankingsemantic-searchvector-searchwiki
fix(docparser): attach api key in MinerU protocol detection (#4470)
JSJ189e0f7f7ea0d5d
diff --git a/internal/infrastructure/docparser/mineru_converter.go b/internal/infrastructure/docparser/mineru_converter.goindex e08b744ac..ad16699c0 100644--- a/internal/infrastructure/docparser/mineru_converter.go+++ b/internal/infrastructure/docparser/mineru_converter.go@@ -104,7 +104,7 @@ func (c *MinerUReader) Read(ctx context.Context, req *types.ReadRequest) (*typesprotocol, err := detectMinerUProtocol(ctx, utils.NewSSRFSafeHTTPClient(utils.SSRFSafeHTTPClientConfig{Timeout: mineruV1APITimeout,MaxRedirects: 5,- }), c.endpoint)+ }), c.endpoint, c.apiKey)if err != nil {return nil, fmt.Errorf("MinerU protocol detection: %w", err)}@@ -434,7 +434,7 @@ func validateMinerUOutboundURL(rawURL string) error {// PingMinerU checks if the self-hosted MinerU service is reachable. For// MinerU 4.0 it also verifies the configured API key, since /v1/health itself-// is served without authentication.+// is served without authentication unless sitting behind an authenticating gateway.func PingMinerU(endpoint, apiKey string) (bool, string) {endpoint = strings.TrimRight(endpoint, "/")if endpoint == "" {@@ -449,7 +449,7 @@ func PingMinerU(endpoint, apiKey string) (bool, string) {})ctx := context.Background()- protocol, err := detectMinerUProtocol(ctx, client, endpoint)+ protocol, err := detectMinerUProtocol(ctx, client, endpoint, apiKey)if err != nil {return false, fmt.Sprintf("MinerU 服务不可用: %v", err)}diff --git a/internal/infrastructure/docparser/mineru_v1_client.go b/internal/infrastructure/docparser/mineru_v1_client.goindex 232d4c501..5c430763a 100644--- a/internal/infrastructure/docparser/mineru_v1_client.go+++ b/internal/infrastructure/docparser/mineru_v1_client.go@@ -152,12 +152,19 @@ func minerUV1JobTerminal(status string) bool {// detectMinerUProtocol probes GET /v1/health. MinerU 4.0 answers 200 (or 503// while models are still preloading); older servers have no such route and-// answer 404/405.-func detectMinerUProtocol(ctx context.Context, client *http.Client, endpoint string) (minerUProtocol, error) {+// answer 404/405. When the service sits behind an authenticating API gateway+// or reverse proxy, the configured apiKey is attached as Authorization: Bearer.+func detectMinerUProtocol(+ ctx context.Context, client *http.Client, endpoint, apiKey string,+) (minerUProtocol, error) {+ endpoint = strings.TrimRight(endpoint, "/")req, err := http.NewRequestWithContext(ctx, http.MethodGet, endpoint+"/v1/health", nil)if err != nil {return minerUProtocolLegacy, fmt.Errorf("create health request: %w", err)}+ if apiKey = strings.TrimSpace(apiKey); apiKey != "" {+ req.Header.Set("Authorization", "Bearer "+apiKey)+ }resp, err := client.Do(req)if err != nil {return minerUProtocolLegacy, fmt.Errorf("MinerU service unreachable: %w", err)@@ -175,6 +182,15 @@ func detectMinerUProtocol(ctx context.Context, client *http.Client, endpoint strtruncateForLog(string(body), 300))}return minerUProtocolV1, nil+ case http.StatusUnauthorized:+ if apiKey == "" {+ return minerUProtocolV1, fmt.Errorf("MinerU service requires authentication, please configure API key")+ }+ if msg := minerUV1ErrorMessage(body); msg != "" {+ return minerUProtocolV1, fmt.Errorf("MinerU authentication failed (status 401): %s", msg)+ }+ return minerUProtocolV1, fmt.Errorf("MinerU /v1/health status 401: %s",+ truncateForLog(string(body), 300))case http.StatusNotFound, http.StatusMethodNotAllowed:return minerUProtocolLegacy, nildefault:diff --git a/internal/infrastructure/docparser/mineru_v1_client_test.go b/internal/infrastructure/docparser/mineru_v1_client_test.goindex 4f8c0b915..f1be88784 100644--- a/internal/infrastructure/docparser/mineru_v1_client_test.go+++ b/internal/infrastructure/docparser/mineru_v1_client_test.go@@ -21,8 +21,9 @@ import (// fakeMinerUV1 is a minimal in-memory MinerU 4.0 api-server.type fakeMinerUV1 struct {- t *testing.T- apiKey string+ t *testing.T+ apiKey string+ authHealth boolmu sync.MutexuploadURL string // absolute or relative; defaults to same-origin relative path@@ -58,7 +59,13 @@ func (f *fakeMinerUV1) handler() http.Handler {}}- mux.HandleFunc("GET /v1/health", func(w http.ResponseWriter, _ *http.Request) {+ mux.HandleFunc("GET /v1/health", func(w http.ResponseWriter, r *http.Request) {+ if f.authHealth && f.apiKey != "" && r.Header.Get("Authorization") != "Bearer "+f.apiKey {+ writeJSON(w, http.StatusUnauthorized, map[string]any{+ "error": map[string]string{"message": "bad key", "type": "auth_error", "code": "invalid_api_key"},+ })+ return+ }writeJSON(w, http.StatusOK, map[string]any{"status": "ok", "version": "1.0.0"})})mux.HandleFunc("POST /v1/uploads", authed(func(w http.ResponseWriter, r *http.Request) {@@ -318,13 +325,41 @@ func TestMinerUV1ClientAPIErrorMessage(t *testing.T) {func TestDetectMinerUProtocol(t *testing.T) {allowLoopbackSSRF(t)tests := []struct {- name string- status int- body string- want minerUProtocol- wantErr string+ name string+ apiKey string+ requireKey string+ status int+ body string+ want minerUProtocol+ wantErr string}{{name: "v1 healthy", status: 200, body: `{"status":"ok","version":"1.0.0"}`, want: minerUProtocolV1},+ {+ name: "v1 healthy behind gateway with api key",+ apiKey: "test-key",+ requireKey: "test-key",+ status: 200,+ body: `{"status":"ok"}`,+ want: minerUProtocolV1,+ },+ {+ name: "v1 behind gateway missing api key",+ requireKey: "test-key",+ status: 401,+ body: `{"error":{"message":"Authentication Error, No api key passed in.",` ++ `"type":"auth_error","code":"401"}}`,+ want: minerUProtocolV1,+ wantErr: "requires authentication, please configure API key",+ },+ {+ name: "v1 behind gateway invalid api key",+ apiKey: "bad-key",+ requireKey: "test-key",+ status: 401,+ body: `{"error":{"message":"Invalid API key","type":"auth_error","code":"401"}}`,+ want: minerUProtocolV1,+ wantErr: "401: Invalid API key",+ },{name: "legacy 404", status: 404, body: `{"detail":"Not Found"}`, want: minerUProtocolLegacy},{name: "legacy 405", status: 405, body: ``, want: minerUProtocolLegacy},{@@ -341,12 +376,23 @@ func TestDetectMinerUProtocol(t *testing.T) {t.Run(tt.name, func(t *testing.T) {server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {assert.Equal(t, "/v1/health", r.URL.Path)+ if tt.requireKey != "" {+ auth := r.Header.Get("Authorization")+ if auth != "Bearer "+tt.requireKey {+ w.WriteHeader(http.StatusUnauthorized)+ if tt.body != "" {+ _, _ = w.Write([]byte(tt.body))+ }+ return+ }+ }w.WriteHeader(tt.status)_, _ = w.Write([]byte(tt.body))}))defer server.Close()- got, err := detectMinerUProtocol(context.Background(), http.DefaultClient, server.URL)+ got, err := detectMinerUProtocol(+ context.Background(), http.DefaultClient, server.URL, tt.apiKey)if tt.wantErr != "" {require.Error(t, err)assert.Contains(t, err.Error(), tt.wantErr)@@ -411,6 +457,18 @@ func TestPingMinerUV1(t *testing.T) {assert.False(t, ok)assert.Contains(t, msg, "配置 API Key")+ // Gateway requiring authentication on /v1/health+ gwFake := &fakeMinerUV1{t: t, apiKey: "server-key", authHealth: true}+ gwServer := httptest.NewServer(gwFake.handler())+ defer gwServer.Close()++ ok, msg = PingMinerU(gwServer.URL, "server-key")+ assert.True(t, ok, msg)++ ok, msg = PingMinerU(gwServer.URL, "wrong")+ assert.False(t, ok)+ assert.Contains(t, msg, "401")+open := &fakeMinerUV1{t: t}openServer := httptest.NewServer(open.handler())defer openServer.Close()