Skip to content

Open-source LLM knowledge platform: turn raw documents into a queryable RAG, an autonomous reasoning agent, and a self-maintaining Wiki.

fix(docparser): attach api key in MinerU protocol detection (#4470)

JSJ189e0f7f7ea0d5d
diff --git a/internal/infrastructure/docparser/mineru_converter.go b/internal/infrastructure/docparser/mineru_converter.go
index e08b744ac..ad16699c0 100644
--- a/internal/infrastructure/docparser/mineru_converter.go
+++ b/internal/infrastructure/docparser/mineru_converter.go
@@ -104,7 +104,7 @@ func (c *MinerUReader) Read(ctx context.Context, req *types.ReadRequest) (*types
protocol, err := detectMinerUProtocol(ctx, utils.NewSSRFSafeHTTPClient(utils.SSRFSafeHTTPClientConfig{
Timeout: mineruV1APITimeout,
MaxRedirects: 5,
- }), c.endpoint)
+ }), c.endpoint, c.apiKey)
if err != nil {
return nil, fmt.Errorf("MinerU protocol detection: %w", err)
}
@@ -434,7 +434,7 @@ func validateMinerUOutboundURL(rawURL string) error {
// PingMinerU checks if the self-hosted MinerU service is reachable. For
// MinerU 4.0 it also verifies the configured API key, since /v1/health itself
-// is served without authentication.
+// is served without authentication unless sitting behind an authenticating gateway.
func PingMinerU(endpoint, apiKey string) (bool, string) {
endpoint = strings.TrimRight(endpoint, "/")
if endpoint == "" {
@@ -449,7 +449,7 @@ func PingMinerU(endpoint, apiKey string) (bool, string) {
})
ctx := context.Background()
- protocol, err := detectMinerUProtocol(ctx, client, endpoint)
+ protocol, err := detectMinerUProtocol(ctx, client, endpoint, apiKey)
if err != nil {
return false, fmt.Sprintf("MinerU 服务不可用: %v", err)
}
diff --git a/internal/infrastructure/docparser/mineru_v1_client.go b/internal/infrastructure/docparser/mineru_v1_client.go
index 232d4c501..5c430763a 100644
--- a/internal/infrastructure/docparser/mineru_v1_client.go
+++ b/internal/infrastructure/docparser/mineru_v1_client.go
@@ -152,12 +152,19 @@ func minerUV1JobTerminal(status string) bool {
// detectMinerUProtocol probes GET /v1/health. MinerU 4.0 answers 200 (or 503
// while models are still preloading); older servers have no such route and
-// answer 404/405.
-func detectMinerUProtocol(ctx context.Context, client *http.Client, endpoint string) (minerUProtocol, error) {
+// answer 404/405. When the service sits behind an authenticating API gateway
+// or reverse proxy, the configured apiKey is attached as Authorization: Bearer.
+func detectMinerUProtocol(
+ ctx context.Context, client *http.Client, endpoint, apiKey string,
+) (minerUProtocol, error) {
+ endpoint = strings.TrimRight(endpoint, "/")
req, err := http.NewRequestWithContext(ctx, http.MethodGet, endpoint+"/v1/health", nil)
if err != nil {
return minerUProtocolLegacy, fmt.Errorf("create health request: %w", err)
}
+ if apiKey = strings.TrimSpace(apiKey); apiKey != "" {
+ req.Header.Set("Authorization", "Bearer "+apiKey)
+ }
resp, err := client.Do(req)
if err != nil {
return minerUProtocolLegacy, fmt.Errorf("MinerU service unreachable: %w", err)
@@ -175,6 +182,15 @@ func detectMinerUProtocol(ctx context.Context, client *http.Client, endpoint str
truncateForLog(string(body), 300))
}
return minerUProtocolV1, nil
+ case http.StatusUnauthorized:
+ if apiKey == "" {
+ return minerUProtocolV1, fmt.Errorf("MinerU service requires authentication, please configure API key")
+ }
+ if msg := minerUV1ErrorMessage(body); msg != "" {
+ return minerUProtocolV1, fmt.Errorf("MinerU authentication failed (status 401): %s", msg)
+ }
+ return minerUProtocolV1, fmt.Errorf("MinerU /v1/health status 401: %s",
+ truncateForLog(string(body), 300))
case http.StatusNotFound, http.StatusMethodNotAllowed:
return minerUProtocolLegacy, nil
default:
diff --git a/internal/infrastructure/docparser/mineru_v1_client_test.go b/internal/infrastructure/docparser/mineru_v1_client_test.go
index 4f8c0b915..f1be88784 100644
--- a/internal/infrastructure/docparser/mineru_v1_client_test.go
+++ b/internal/infrastructure/docparser/mineru_v1_client_test.go
@@ -21,8 +21,9 @@ import (
// fakeMinerUV1 is a minimal in-memory MinerU 4.0 api-server.
type fakeMinerUV1 struct {
- t *testing.T
- apiKey string
+ t *testing.T
+ apiKey string
+ authHealth bool
mu sync.Mutex
uploadURL string // absolute or relative; defaults to same-origin relative path
@@ -58,7 +59,13 @@ func (f *fakeMinerUV1) handler() http.Handler {
}
}
- mux.HandleFunc("GET /v1/health", func(w http.ResponseWriter, _ *http.Request) {
+ mux.HandleFunc("GET /v1/health", func(w http.ResponseWriter, r *http.Request) {
+ if f.authHealth && f.apiKey != "" && r.Header.Get("Authorization") != "Bearer "+f.apiKey {
+ writeJSON(w, http.StatusUnauthorized, map[string]any{
+ "error": map[string]string{"message": "bad key", "type": "auth_error", "code": "invalid_api_key"},
+ })
+ return
+ }
writeJSON(w, http.StatusOK, map[string]any{"status": "ok", "version": "1.0.0"})
})
mux.HandleFunc("POST /v1/uploads", authed(func(w http.ResponseWriter, r *http.Request) {
@@ -318,13 +325,41 @@ func TestMinerUV1ClientAPIErrorMessage(t *testing.T) {
func TestDetectMinerUProtocol(t *testing.T) {
allowLoopbackSSRF(t)
tests := []struct {
- name string
- status int
- body string
- want minerUProtocol
- wantErr string
+ name string
+ apiKey string
+ requireKey string
+ status int
+ body string
+ want minerUProtocol
+ wantErr string
}{
{name: "v1 healthy", status: 200, body: `{"status":"ok","version":"1.0.0"}`, want: minerUProtocolV1},
+ {
+ name: "v1 healthy behind gateway with api key",
+ apiKey: "test-key",
+ requireKey: "test-key",
+ status: 200,
+ body: `{"status":"ok"}`,
+ want: minerUProtocolV1,
+ },
+ {
+ name: "v1 behind gateway missing api key",
+ requireKey: "test-key",
+ status: 401,
+ body: `{"error":{"message":"Authentication Error, No api key passed in.",` +
+ `"type":"auth_error","code":"401"}}`,
+ want: minerUProtocolV1,
+ wantErr: "requires authentication, please configure API key",
+ },
+ {
+ name: "v1 behind gateway invalid api key",
+ apiKey: "bad-key",
+ requireKey: "test-key",
+ status: 401,
+ body: `{"error":{"message":"Invalid API key","type":"auth_error","code":"401"}}`,
+ want: minerUProtocolV1,
+ wantErr: "401: Invalid API key",
+ },
{name: "legacy 404", status: 404, body: `{"detail":"Not Found"}`, want: minerUProtocolLegacy},
{name: "legacy 405", status: 405, body: ``, want: minerUProtocolLegacy},
{
@@ -341,12 +376,23 @@ func TestDetectMinerUProtocol(t *testing.T) {
t.Run(tt.name, func(t *testing.T) {
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
assert.Equal(t, "/v1/health", r.URL.Path)
+ if tt.requireKey != "" {
+ auth := r.Header.Get("Authorization")
+ if auth != "Bearer "+tt.requireKey {
+ w.WriteHeader(http.StatusUnauthorized)
+ if tt.body != "" {
+ _, _ = w.Write([]byte(tt.body))
+ }
+ return
+ }
+ }
w.WriteHeader(tt.status)
_, _ = w.Write([]byte(tt.body))
}))
defer server.Close()
- got, err := detectMinerUProtocol(context.Background(), http.DefaultClient, server.URL)
+ got, err := detectMinerUProtocol(
+ context.Background(), http.DefaultClient, server.URL, tt.apiKey)
if tt.wantErr != "" {
require.Error(t, err)
assert.Contains(t, err.Error(), tt.wantErr)
@@ -411,6 +457,18 @@ func TestPingMinerUV1(t *testing.T) {
assert.False(t, ok)
assert.Contains(t, msg, "配置 API Key")
+ // Gateway requiring authentication on /v1/health
+ gwFake := &fakeMinerUV1{t: t, apiKey: "server-key", authHealth: true}
+ gwServer := httptest.NewServer(gwFake.handler())
+ defer gwServer.Close()
+
+ ok, msg = PingMinerU(gwServer.URL, "server-key")
+ assert.True(t, ok, msg)
+
+ ok, msg = PingMinerU(gwServer.URL, "wrong")
+ assert.False(t, ok)
+ assert.Contains(t, msg, "401")
+
open := &fakeMinerUV1{t: t}
openServer := httptest.NewServer(open.handler())
defer openServer.Close()