Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

Go: fix other models http clients (#19920)

Jin Hai9169c9c05fda
diff --git a/internal/common/http_test.go b/internal/common/http_test.go
new file mode 100644
index 0000000000..142539e669
--- /dev/null
+++ b/internal/common/http_test.go
@@ -0,0 +1,187 @@
+package common
+
+import (
+ "fmt"
+ "io"
+ "net/http"
+ "net/http/httptest"
+ "strings"
+ "testing"
+ "time"
+
+ "go.uber.org/zap"
+ "go.uber.org/zap/zapcore"
+ "go.uber.org/zap/zaptest/observer"
+)
+
+func TestDriverHTTPClientLogsProviderRequestAndResponseWhenEnabled(t *testing.T) {
+ t.Setenv(EnvLLMDebug, "true")
+ core, logs := observer.New(zapcore.InfoLevel)
+ previousLogger := Logger
+ Logger = zap.New(core)
+ t.Cleanup(func() {
+ Logger = previousLogger
+ })
+
+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ if r.URL.Path != "/v1/rerank" {
+ t.Fatalf("path = %q, want /v1/rerank", r.URL.Path)
+ }
+ fmt.Fprint(w, `{"answer":"ok","access_token":"response-secret"}`)
+ }))
+ defer server.Close()
+
+ client := GetSchemeSafeHTTPClient()
+ req, err := http.NewRequestWithContext(t.Context(), http.MethodPost, server.URL+"/v1/rerank?key=request-secret", strings.NewReader(`{"query":"hello","api_key":"payload-secret"}`))
+ if err != nil {
+ t.Fatalf("NewRequestWithContext() error = %v", err)
+ }
+ resp, err := client.Do(req)
+ if err != nil {
+ t.Fatalf("Do() error = %v", err)
+ }
+ if _, err = io.ReadAll(resp.Body); err != nil {
+ t.Fatalf("ReadAll() error = %v", err)
+ }
+ if err = resp.Body.Close(); err != nil {
+ t.Fatalf("Close() error = %v", err)
+ }
+
+ entries := logs.All()
+ if len(entries) != 1 {
+ t.Fatalf("provider log count = %d, want 1", len(entries))
+ }
+ message := entries[0].Message
+ if strings.Contains(message, `\"`) {
+ t.Errorf("provider log contains escaped JSON: %q", message)
+ }
+ if !strings.Contains(message, `payload={"api_key":"[REDACTED]","query":"hello"}`) {
+ t.Errorf("provider log payload is not raw redacted JSON: %q", message)
+ }
+ if !strings.Contains(message, `response_code=200 took=`) || !strings.Contains(message, ` first-token=`) || !strings.Contains(message, ` response_body={"access_token":"[REDACTED]","answer":"ok"}`) {
+ t.Errorf("provider log response is not raw redacted JSON: %q", message)
+ }
+ if strings.Contains(message, "request-secret") || strings.Contains(message, "payload-secret") || strings.Contains(message, "response-secret") {
+ t.Errorf("provider log contains an unredacted secret: %q", message)
+ }
+}
+
+type roundTripFunc func(*http.Request) (*http.Response, error)
+
+func (f roundTripFunc) RoundTrip(req *http.Request) (*http.Response, error) {
+ return f(req)
+}
+
+func TestProviderLoggingRecordsTimingsAndTruncatesVectors(t *testing.T) {
+ core, logs := observer.New(zapcore.InfoLevel)
+ previousLogger := Logger
+ Logger = zap.New(core)
+ t.Cleanup(func() {
+ Logger = previousLogger
+ })
+
+ startedAt := time.Unix(100, 0)
+ times := []time.Time{
+ startedAt,
+ startedAt.Add(time.Second),
+ startedAt.Add(3 * time.Second),
+ }
+ timeIndex := 0
+ transport := &providerLoggingTransport{
+ base: roundTripFunc(func(req *http.Request) (*http.Response, error) {
+ return &http.Response{
+ StatusCode: http.StatusOK,
+ Header: make(http.Header),
+ Body: io.NopCloser(strings.NewReader(`{"data":[{"embedding":[0.1,0.2,0.3,0.4,0.5]}],"vectors":[[1,2,3,4],[5,6]]}`)),
+ Request: req,
+ }, nil
+ }),
+ now: func() time.Time {
+ current := times[timeIndex]
+ timeIndex++
+ return current
+ },
+ }
+ req, err := http.NewRequestWithContext(t.Context(), http.MethodPost, "https://provider.example/v1/embeddings", nil)
+ if err != nil {
+ t.Fatalf("NewRequestWithContext() error = %v", err)
+ }
+ resp, err := transport.RoundTrip(req)
+ if err != nil {
+ t.Fatalf("RoundTrip() error = %v", err)
+ }
+ if _, err = io.ReadAll(resp.Body); err != nil {
+ t.Fatalf("ReadAll() error = %v", err)
+ }
+ if err = resp.Body.Close(); err != nil {
+ t.Fatalf("Close() error = %v", err)
+ }
+
+ entries := logs.All()
+ if len(entries) != 1 {
+ t.Fatalf("provider log count = %d, want 1", len(entries))
+ }
+ message := entries[0].Message
+ if !strings.Contains(message, "took=3s first-token=1s") {
+ t.Errorf("provider log timings = %q, want took=3s first-token=1s", message)
+ }
+ if !strings.Contains(message, `response_body={"data":[{"embedding":[0.1,0.2,0.3]}],"vectors":[[1,2,3],[5,6]]}`) {
+ t.Errorf("provider log did not truncate vectors: %q", message)
+ }
+ if strings.Contains(message, "0.4") || strings.Contains(message, "0.5") || strings.Contains(message, "[1,2,3,4]") {
+ t.Errorf("provider log contains vector values after the first three: %q", message)
+ }
+}
+
+type trackingReadCloser struct {
+ reader io.Reader
+ reads int
+}
+
+func (r *trackingReadCloser) Read(p []byte) (int, error) {
+ r.reads++
+ return r.reader.Read(p)
+}
+
+func (r *trackingReadCloser) Close() error {
+ return nil
+}
+
+func TestProviderLoggingDisabledAvoidsPayloadWork(t *testing.T) {
+ t.Setenv(EnvLLMDebug, "")
+ core, logs := observer.New(zapcore.InfoLevel)
+ previousLogger := Logger
+ Logger = zap.New(core)
+ t.Cleanup(func() {
+ Logger = previousLogger
+ })
+
+ payload := &trackingReadCloser{reader: strings.NewReader(`{"query":"hello"}`)}
+ client := &http.Client{Transport: newProviderLoggingTransport(roundTripFunc(func(req *http.Request) (*http.Response, error) {
+ return &http.Response{
+ StatusCode: http.StatusNoContent,
+ Header: make(http.Header),
+ Body: http.NoBody,
+ Request: req,
+ }, nil
+ }))}
+ req, err := http.NewRequestWithContext(t.Context(), http.MethodPost, "https://provider.example/v1/rerank", payload)
+ if err != nil {
+ t.Fatalf("NewRequestWithContext() error = %v", err)
+ }
+ resp, err := client.Do(req)
+ if err != nil {
+ t.Fatalf("Do() error = %v", err)
+ }
+ if resp.Body != http.NoBody {
+ t.Fatalf("response body was wrapped while LLM_DEBUG is disabled")
+ }
+ _ = resp.Body.Close()
+
+ if payload.reads != 0 {
+ t.Fatalf("request body reads = %d, want 0 while LLM_DEBUG is disabled", payload.reads)
+ }
+ if count := logs.Len(); count != 0 {
+ t.Fatalf("provider log count = %d, want 0", count)
+ }
+}
diff --git a/internal/entity/models/302ai.go b/internal/entity/models/302ai.go
index b0d999a3be..7ead25b986 100644
--- a/internal/entity/models/302ai.go
+++ b/internal/entity/models/302ai.go
@@ -42,7 +42,7 @@ func NewAI302Model(baseURL map[string]string, urlSuffix URLSuffix) *AI302Model {
baseModel: BaseModel{
BaseURL: baseURL,
URLSuffix: urlSuffix,
- httpClient: NewDriverHTTPClient(false),
+ httpClient: common.GetSSRFHTTPClient(),
},
}
}
diff --git a/internal/entity/models/aliyun.go b/internal/entity/models/aliyun.go
index 3abe2344e3..7af5f893c2 100644
--- a/internal/entity/models/aliyun.go
+++ b/internal/entity/models/aliyun.go
@@ -42,7 +42,7 @@ func NewAliyunModel(baseURL map[string]string, urlSuffix URLSuffix) *AliyunModel
baseModel: BaseModel{
BaseURL: baseURL,
URLSuffix: urlSuffix,
- httpClient: NewDriverHTTPClient(false),
+ httpClient: common.GetSSRFHTTPClient(),
},
}
}
diff --git a/internal/entity/models/anthropic.go b/internal/entity/models/anthropic.go
index 50d26eb2f3..225a36a5bc 100644
--- a/internal/entity/models/anthropic.go
+++ b/internal/entity/models/anthropic.go
@@ -41,7 +41,7 @@ func NewAnthropicModel(baseURL map[string]string, urlSuffix URLSuffix) *Anthropi
baseModel: BaseModel{
BaseURL: baseURL,
URLSuffix: urlSuffix,
- httpClient: NewDriverHTTPClient(false),
+ httpClient: common.GetSSRFHTTPClient(),
// Anthropic authenticates with the "x-api-key" header instead
// of the default "Authorization: Bearer".
authHeader: func(cfg *APIConfig) (string, string) {
diff --git a/internal/entity/models/astraflow.go b/internal/entity/models/astraflow.go
index b913450d06..d50c6e4468 100644
--- a/internal/entity/models/astraflow.go
+++ b/internal/entity/models/astraflow.go
@@ -54,7 +54,7 @@ func NewAstraflowModel(baseURL map[string]string, urlSuffix URLSuffix) *Astraflo
baseModel: BaseModel{
BaseURL: baseURL,
URLSuffix: urlSuffix,
- httpClient: NewDriverHTTPClient(false),
+ httpClient: common.GetSSRFHTTPClient(),
},
}
}
diff --git a/internal/entity/models/avian.go b/internal/entity/models/avian.go
index bae804df52..d1f8f03b8c 100644
--- a/internal/entity/models/avian.go
+++ b/internal/entity/models/avian.go
@@ -39,7 +39,7 @@ func NewAvianModel(baseURL map[string]string, urlSuffix URLSuffix) *AvianModel {
baseModel: BaseModel{
BaseURL: baseURL,
URLSuffix: urlSuffix,
- httpClient: NewDriverHTTPClient(false),
+ httpClient: common.GetSSRFHTTPClient(),
},
}
}
diff --git a/internal/entity/models/azure_openai.go b/internal/entity/models/azure_openai.go
index baa4fed4ac..c2d4d5658f 100644
--- a/internal/entity/models/azure_openai.go
+++ b/internal/entity/models/azure_openai.go
@@ -39,7 +39,7 @@ func NewAzureOpenAIModel(baseURL map[string]string, urlSuffix URLSuffix) *AzureO
baseModel: BaseModel{
BaseURL: baseURL,
URLSuffix: urlSuffix,
- httpClient: NewDriverHTTPClient(false),
+ httpClient: common.GetSSRFHTTPClient(),
// Azure OpenAI authenticates with the non-standard "api-key"
// header instead of "Authorization: Bearer".
authHeader: func(cfg *APIConfig) (string, string) {
diff --git a/internal/entity/models/baichuan.go b/internal/entity/models/baichuan.go
index a7ebe6785f..82675253e8 100644
--- a/internal/entity/models/baichuan.go
+++ b/internal/entity/models/baichuan.go
@@ -36,7 +36,7 @@ func NewBaichuanModel(baseURL map[string]string, urlSuffix URLSuffix) *BaichuanM
baseModel: BaseModel{
BaseURL: baseURL,
URLSuffix: urlSuffix,
- httpClient: NewDriverHTTPClient(false),
+ httpClient: common.GetSSRFHTTPClient(),
},
}
}
diff --git a/internal/entity/models/baidu.go b/internal/entity/models/baidu.go
index 3187125b13..556766c1e2 100644
--- a/internal/entity/models/baidu.go
+++ b/internal/entity/models/baidu.go
@@ -41,7 +41,7 @@ func NewBaiduModel(baseURL map[string]string, urlSuffix URLSuffix) *BaiduModel {
baseModel: BaseModel{
BaseURL: baseURL,
URLSuffix: urlSuffix,
- httpClient: NewDriverHTTPClient(false),
+ httpClient: common.GetSSRFHTTPClient(),
},
}
}
diff --git a/internal/entity/models/base_model.go b/internal/entity/models/base_model.go
index ef56635bd6..21a9c553ca 100644
--- a/internal/entity/models/base_model.go
+++ b/internal/entity/models/base_model.go
@@ -24,12 +24,10 @@ import (
"fmt"
"io"
"net/http"
- "net/url"
"ragflow/internal/common"
"ragflow/internal/engine/clickhouse"
"sort"
"strings"
- "sync"
"time"
)
@@ -463,305 +461,6 @@ func ParseListModel(modelList ModelList) []ListModelResponse {
return FillMissingModelTypes(models)
}
-// NewDriverHTTPClient returns an *http.Client with the standard connection-pool
-// settings, an SSRF guard, and opt-in provider request/response logging wired
-// into its Transport. Logging is disabled unless LLM_DEBUG is true when the
-// client is created, normally during process startup.
-//
-// allowPrivate selects the guard strictness:
-// - false (cloud-hosted drivers): every request is validated with
-// common.AssertURLSafe — scheme + host must be present and every resolved
-// IP must be globally routable (private/loopback/link-local/metadata are
-// rejected). This is the default and closes the go/request-forgery sink.
-// - true (local-inference drivers): requests are validated with
-// utility.AssertURLSchemeSafe — only the scheme and a non-empty host are
-// enforced, so self-hosted backends on private networks or loopback work.
-func NewDriverHTTPClient(allowPrivate bool) *http.Client {
- var t *http.Transport
- if dt, ok := http.DefaultTransport.(*http.Transport); ok {
- t = dt.Clone()
- } else {
- t = &http.Transport{Proxy: http.ProxyFromEnvironment}
- }
- t.MaxIdleConns = 100
- t.MaxIdleConnsPerHost = 10
- t.IdleConnTimeout = 90 * time.Second
- t.DisableCompression = false
- t.ResponseHeaderTimeout = 20 * time.Minute
- t.TLSHandshakeTimeout = 30 * time.Second
-
- var rt http.RoundTripper = t
- if allowPrivate {
- rt = &schemeSafeTransport{base: rt}
- } else {
- rt = &strictSSRFTransport{base: rt}
- }
- rt = newProviderLoggingTransport(rt)
- return &http.Client{Transport: rt}
-}
-
-func newProviderLoggingTransport(base http.RoundTripper) http.RoundTripper {
- if base == nil {
- base = http.DefaultTransport
- }
- if !common.IsLLMDebugEnabled() {
- return base
- }
- return &providerLoggingTransport{base: base, now: time.Now}
-}
-
-type providerLoggingTransport struct {
- base http.RoundTripper
- now func() time.Time
-}
-
-func (t *providerLoggingTransport) RoundTrip(req *http.Request) (*http.Response, error) {
- payload, err := readAndRestoreRequestBody(req)
- if err != nil {
- return nil, err
- }
- providerURL := redactProviderURL(req.URL)
- logPayload := redactProviderBody(payload)
-
- startedAt := t.now()
- resp, err := t.base.RoundTrip(req)
- if err != nil {
- logProviderCall(providerURL, logPayload, 0, "", t.now().Sub(startedAt), 0, err)
- return nil, err
- }
- if resp.Body == nil {
- logProviderCall(providerURL, logPayload, resp.StatusCode, "", t.now().Sub(startedAt), 0, nil)
- return resp, nil
- }
-
- resp.Body = &providerResponseBody{
- ReadCloser: resp.Body,
- startedAt: startedAt,
- now: t.now,
- log: func(body []byte, took, firstToken time.Duration) {
- logProviderCall(providerURL, logPayload, resp.StatusCode, redactProviderBody(body), took, firstToken, nil)
- },
- }
- return resp, nil
-}
-
-// providerResponseBody captures bytes while callers consume them, preserving
-// streaming delivery instead of eagerly reading the entire provider response.
-type providerResponseBody struct {
- io.ReadCloser
- body bytes.Buffer
- startedAt time.Time
- now func() time.Time
- firstToken time.Duration
- firstOnce sync.Once
- logOnce sync.Once
- log func([]byte, time.Duration, time.Duration)
-}
-
-func (b *providerResponseBody) Read(p []byte) (int, error) {
- n, err := b.ReadCloser.Read(p)
- if n > 0 {
- b.firstOnce.Do(func() {
- b.firstToken = b.now().Sub(b.startedAt)
- })
- _, _ = b.body.Write(p[:n])
- }
- if err == io.EOF {
- b.writeLogOnce()
- }
- return n, err
-}
-
-func (b *providerResponseBody) Close() error {
- err := b.ReadCloser.Close()
- b.writeLogOnce()
- return err
-}
-
-func (b *providerResponseBody) writeLogOnce() {
- b.logOnce.Do(func() {
- b.log(b.body.Bytes(), b.now().Sub(b.startedAt), b.firstToken)
- })
-}
-
-func readAndRestoreRequestBody(req *http.Request) ([]byte, error) {
- if req.Body == nil {
- return nil, nil
- }
- body, err := io.ReadAll(req.Body)
- if err != nil {
- return nil, fmt.Errorf("failed to read provider request body for logging: %w", err)
- }
- _ = req.Body.Close()
- req.Body = io.NopCloser(bytes.NewReader(body))
- req.ContentLength = int64(len(body))
- return body, nil
-}
-
-func redactProviderURL(requestURL *url.URL) string {
- if requestURL == nil {
- return ""
- }
- redacted := *requestURL
- if redacted.User != nil {
- redacted.User = url.User(redacted.User.Username())
- }
- query := redacted.Query()
- for key := range query {
- if isSensitiveLogKey(key) {
- query.Set(key, redactedLogValue)
- }
- }
- redacted.RawQuery = query.Encode()
- return redacted.String()
-}
-
-func redactProviderBody(body []byte) string {
- if len(body) == 0 {
- return ""
- }
-
- var value any
- if err := json.Unmarshal(body, &value); err == nil {
- redactProviderValue(value)
- if redacted, err := json.Marshal(value); err == nil {
- return string(redacted)
- }
- }
-
- lines := strings.Split(string(body), "\n")
- redactedAny := false
- for i, line := range lines {
- prefix, data, ok := strings.Cut(line, "data:")
- if !ok || strings.TrimSpace(data) == "[DONE]" {
- continue
- }
- var event any
- if err := json.Unmarshal([]byte(strings.TrimSpace(data)), &event); err != nil {
- continue
- }
- redactProviderValue(event)
- redacted, err := json.Marshal(event)
- if err != nil {
- continue
- }
- lines[i] = prefix + "data: " + string(redacted)
- redactedAny = true
- }
- if redactedAny {
- return strings.Join(lines, "\n")
- }
- return string(body)
-}
-
-func redactProviderValue(value any) {
- switch value := value.(type) {
- case map[string]any:
- for key, child := range value {
- if isSensitiveLogKey(key) {
- value[key] = redactedLogValue
- continue
- }
- if isVectorLogKey(key) {
- child = truncateLoggedVectors(child)
- value[key] = child
- }
- redactProviderValue(child)
- }
- case []any:
- for _, child := range value {
- redactProviderValue(child)
- }
- }
-}
-
-func isVectorLogKey(key string) bool {
- switch strings.ToLower(key) {
- case "embedding", "embeddings", "vector", "vectors":
- return true
- default:
- return false
- }
-}
-
-func truncateLoggedVectors(value any) any {
- switch value := value.(type) {
- case []any:
- if isNumericVector(value) {
- if len(value) > maxLoggedVectorFloats {
- return value[:maxLoggedVectorFloats]
- }
- return value
- }
- for i, child := range value {
- value[i] = truncateLoggedVectors(child)
- }
- case map[string]any:
- for key, child := range value {
- value[key] = truncateLoggedVectors(child)
- }
- }
- return value
-}
-
-func isNumericVector(value []any) bool {
- if len(value) == 0 {
- return false
- }
- for _, item := range value {
- if _, ok := item.(float64); !ok {
- return false
- }
- }
- return true
-}
-
-func isSensitiveLogKey(key string) bool {
- normalized := strings.NewReplacer("-", "", "_", "", ".", "").Replace(strings.ToLower(key))
- switch normalized {
- case "apikey", "authorization", "accesstoken", "refreshtoken", "password", "secret", "token", "key":
- return true
- default:
- return false
- }
-}
-
-func logProviderCall(providerURL, payload string, statusCode int, responseBody string, took, firstToken time.Duration, err error) {
- request := fmt.Sprintf("url=%s payload=%s", providerURL, payload)
- response := fmt.Sprintf("response_code=%d took=%s first-token=%s response_body=%s", statusCode, took, firstToken, responseBody)
- if err != nil {
- response += " error=" + err.Error()
- }
- common.LogRequestResponseInfo(request, response, err == nil && statusCode >= 200 && statusCode < 300)
-}
-
-// schemeSafeTransport wraps an http.RoundTripper so every outgoing request is
-// validated by the lenient SSRF guard (http/https scheme + non-empty host).
-// Private and loopback hosts are permitted. Used only by local-inference
-// drivers that may target a user's own network.
-type schemeSafeTransport struct{ base http.RoundTripper }
-
-func (t *schemeSafeTransport) RoundTrip(req *http.Request) (*http.Response, error) {
- if err := common.AssertURLSchemeSafe(req.URL.String()); err != nil {
- return nil, err
- }
- return t.base.RoundTrip(req)
-}
-
-// strictSSRFTransport wraps an http.RoundTripper so every outgoing request is
-// validated by the strict SSRF guard (scheme + host + globally routable IP).
-// This is the default for cloud-hosted model drivers and closes the
-// go/request-forgery data flow: the user-controllable BaseURL cannot be made to
-// point at private hosts, loopback, link-local, or cloud metadata endpoints.
-type strictSSRFTransport struct{ base http.RoundTripper }
-
-func (t *strictSSRFTransport) RoundTrip(req *http.Request) (*http.Response, error) {
- if _, _, err := common.AssertURLSafe(req.URL.String()); err != nil {
- return nil, err
- }
- return t.base.RoundTrip(req)
-}
-
// PostJSONRequest marshals body to JSON, creates a POST request to url
func PostJSONRequest(ctx context.Context, client *http.Client, url, auth string, body map[string]interface{}) (*http.Response, error) {
data, err := json.Marshal(body)
@@ -779,12 +478,6 @@ func PostJSONRequest(ctx context.Context, client *http.Client, url, auth string,
return client.Do(req)
}
-// ReadErrorBody reads all bytes from r and returns them as a string suitable
-func ReadErrorBody(r io.Reader) string {
- b, _ := io.ReadAll(r)
- return string(b)
-}
-
func buildRequestBody(cfg *ChatConfig, modelName string, messages []Message, stream bool) map[string]any {
reqBody := map[string]any{
"model": modelName,
diff --git a/internal/entity/models/bedrock.go b/internal/entity/models/bedrock.go
index 382bd07a45..f04c070675 100644
--- a/internal/entity/models/bedrock.go
+++ b/internal/entity/models/bedrock.go
@@ -117,7 +117,7 @@ func NewBedrockModel(baseURL map[string]string, urlSuffix URLSuffix) *BedrockMod
baseModel: BaseModel{
BaseURL: baseURL,
URLSuffix: urlSuffix,
- httpClient: NewDriverHTTPClient(false),
+ httpClient: common.GetSSRFHTTPClient(),
},
}
}
diff --git a/internal/entity/models/builtin.go b/internal/entity/models/builtin.go
index e49b90a04c..9872662fe2 100644
--- a/internal/entity/models/builtin.go
+++ b/internal/entity/models/builtin.go
@@ -9,31 +9,21 @@ import (
"net/http"
"ragflow/internal/common"
"strings"
- "time"
)
-// builtinHTTPClient is a shared client with timeouts for all Builtin model
-// requests. http.DefaultClient has no timeout, so a hung or slow TEI server
-// would otherwise block goroutines indefinitely.
-var builtinHTTPClient = &http.Client{
- Timeout: 30 * time.Second,
- Transport: newProviderLoggingTransport(&http.Transport{
- MaxIdleConnsPerHost: 10,
- ResponseHeaderTimeout: 10 * time.Second,
- }),
-}
-
// BuiltinModel implements ModelDriver for Builtin (local embedding models via TEI)
type BuiltinModel struct {
- baseURL string
- model string
+ baseURL string
+ model string
+ httpClient *http.Client
}
// NewBuiltinModel creates a new Builtin model instance
func NewBuiltinModel(baseURL, model string) *BuiltinModel {
return &BuiltinModel{
- baseURL: baseURL,
- model: model,
+ baseURL: baseURL,
+ model: model,
+ httpClient: common.GetSchemeSafeHTTPClient(),
}
}
@@ -91,7 +81,7 @@ func (b *BuiltinModel) Embed(ctx context.Context, modelName *string, request Emb
req.Header.Set("Content-Type", "application/json")
// Note: TEI server typically doesn't require auth for local deployments
- resp, err := builtinHTTPClient.Do(req)
+ resp, err := b.httpClient.Do(req)
if err != nil {
return nil, err
}
diff --git a/internal/entity/models/cohere.go b/internal/entity/models/cohere.go
index f08194b04c..88497eb863 100644
--- a/internal/entity/models/cohere.go
+++ b/internal/entity/models/cohere.go
@@ -44,7 +44,7 @@ func NewCoHereModel(baseURL map[string]string, urlSuffix URLSuffix) *CoHereModel
baseModel: BaseModel{
BaseURL: baseURL,
URLSuffix: urlSuffix,
- httpClient: NewDriverHTTPClient(false),
+ httpClient: common.GetSSRFHTTPClient(),
},
}
}
diff --git a/internal/entity/models/cometapi.go b/internal/entity/models/cometapi.go
index f8842fae4d..e6fc94c587 100644
--- a/internal/entity/models/cometapi.go
+++ b/internal/entity/models/cometapi.go
@@ -43,7 +43,7 @@ func NewCometAPIModel(baseURL map[string]string, urlSuffix URLSuffix) *CometAPIM
baseModel: BaseModel{
BaseURL: baseURL,
URLSuffix: urlSuffix,
- httpClient: NewDriverHTTPClient(false),
+ httpClient: common.GetSSRFHTTPClient(),
},
}
}
diff --git a/internal/entity/models/common_test.go b/internal/entity/models/common_test.go
index db8608a087..8a08adfb5b 100644
--- a/internal/entity/models/common_test.go
+++ b/internal/entity/models/common_test.go
@@ -7,167 +7,10 @@ import (
"net/http"
"net/http/httptest"
"ragflow/internal/common"
- "strings"
"testing"
"time"
-
- "go.uber.org/zap"
- "go.uber.org/zap/zapcore"
- "go.uber.org/zap/zaptest/observer"
)
-func TestDriverHTTPClientLogsProviderRequestAndResponseWhenEnabled(t *testing.T) {
- t.Setenv(common.EnvLLMDebug, "true")
- core, logs := observer.New(zapcore.InfoLevel)
- previousLogger := common.Logger
- common.Logger = zap.New(core)
- t.Cleanup(func() {
- common.Logger = previousLogger
- })
-
- server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
- if r.URL.Path != "/v1/rerank" {
- t.Fatalf("path = %q, want /v1/rerank", r.URL.Path)
- }
- fmt.Fprint(w, `{"answer":"ok","access_token":"response-secret"}`)
- }))
- defer server.Close()
-
- client := NewDriverHTTPClient(true)
- req, err := http.NewRequestWithContext(t.Context(), http.MethodPost, server.URL+"/v1/rerank?key=request-secret", strings.NewReader(`{"query":"hello","api_key":"payload-secret"}`))
- if err != nil {
- t.Fatalf("NewRequestWithContext() error = %v", err)
- }
- resp, err := client.Do(req)
- if err != nil {
- t.Fatalf("Do() error = %v", err)
- }
- if _, err = io.ReadAll(resp.Body); err != nil {
- t.Fatalf("ReadAll() error = %v", err)
- }
- if err = resp.Body.Close(); err != nil {
- t.Fatalf("Close() error = %v", err)
- }
-
- entries := logs.All()
- if len(entries) != 1 {
- t.Fatalf("provider log count = %d, want 1", len(entries))
- }
- message := entries[0].Message
- if strings.Contains(message, `\"`) {
- t.Errorf("provider log contains escaped JSON: %q", message)
- }
- if !strings.Contains(message, `payload={"api_key":"[REDACTED]","query":"hello"}`) {
- t.Errorf("provider log payload is not raw redacted JSON: %q", message)
- }
- if !strings.Contains(message, `response_code=200 took=`) || !strings.Contains(message, ` first-token=`) || !strings.Contains(message, ` response_body={"access_token":"[REDACTED]","answer":"ok"}`) {
- t.Errorf("provider log response is not raw redacted JSON: %q", message)
- }
- if strings.Contains(message, "request-secret") || strings.Contains(message, "payload-secret") || strings.Contains(message, "response-secret") {
- t.Errorf("provider log contains an unredacted secret: %q", message)
- }
-}
-
-func TestProviderLoggingRecordsTimingsAndTruncatesVectors(t *testing.T) {
- core, logs := observer.New(zapcore.InfoLevel)
- previousLogger := common.Logger
- common.Logger = zap.New(core)
- t.Cleanup(func() {
- common.Logger = previousLogger
- })
-
- startedAt := time.Unix(100, 0)
- times := []time.Time{
- startedAt,
- startedAt.Add(time.Second),
- startedAt.Add(3 * time.Second),
- }
- timeIndex := 0
- transport := &providerLoggingTransport{
- base: roundTripFunc(func(req *http.Request) (*http.Response, error) {
- return &http.Response{
- StatusCode: http.StatusOK,
- Header: make(http.Header),
- Body: io.NopCloser(strings.NewReader(`{"data":[{"embedding":[0.1,0.2,0.3,0.4,0.5]}],"vectors":[[1,2,3,4],[5,6]]}`)),
- Request: req,
- }, nil
- }),
- now: func() time.Time {
- current := times[timeIndex]
- timeIndex++
- return current
- },
- }
- req, err := http.NewRequestWithContext(t.Context(), http.MethodPost, "https://provider.example/v1/embeddings", nil)
- if err != nil {
- t.Fatalf("NewRequestWithContext() error = %v", err)
- }
- resp, err := transport.RoundTrip(req)
- if err != nil {
- t.Fatalf("RoundTrip() error = %v", err)
- }
- if _, err = io.ReadAll(resp.Body); err != nil {
- t.Fatalf("ReadAll() error = %v", err)
- }
- if err = resp.Body.Close(); err != nil {
- t.Fatalf("Close() error = %v", err)
- }
-
- entries := logs.All()
- if len(entries) != 1 {
- t.Fatalf("provider log count = %d, want 1", len(entries))
- }
- message := entries[0].Message
- if !strings.Contains(message, "took=3s first-token=1s") {
- t.Errorf("provider log timings = %q, want took=3s first-token=1s", message)
- }
- if !strings.Contains(message, `response_body={"data":[{"embedding":[0.1,0.2,0.3]}],"vectors":[[1,2,3],[5,6]]}`) {
- t.Errorf("provider log did not truncate vectors: %q", message)
- }
- if strings.Contains(message, "0.4") || strings.Contains(message, "0.5") || strings.Contains(message, "[1,2,3,4]") {
- t.Errorf("provider log contains vector values after the first three: %q", message)
- }
-}
-
-func TestProviderLoggingDisabledAvoidsPayloadWork(t *testing.T) {
- t.Setenv(common.EnvLLMDebug, "")
- core, logs := observer.New(zapcore.InfoLevel)
- previousLogger := common.Logger
- common.Logger = zap.New(core)
- t.Cleanup(func() {
- common.Logger = previousLogger
- })
-
- payload := &trackingReadCloser{reader: strings.NewReader(`{"query":"hello"}`)}
- client := &http.Client{Transport: newProviderLoggingTransport(roundTripFunc(func(req *http.Request) (*http.Response, error) {
- return &http.Response{
- StatusCode: http.StatusNoContent,
- Header: make(http.Header),
- Body: http.NoBody,
- Request: req,
- }, nil
- }))}
- req, err := http.NewRequestWithContext(t.Context(), http.MethodPost, "https://provider.example/v1/rerank", payload)
- if err != nil {
- t.Fatalf("NewRequestWithContext() error = %v", err)
- }
- resp, err := client.Do(req)
- if err != nil {
- t.Fatalf("Do() error = %v", err)
- }
- if resp.Body != http.NoBody {
- t.Fatalf("response body was wrapped while LLM_DEBUG is disabled")
- }
- _ = resp.Body.Close()
-
- if payload.reads != 0 {
- t.Fatalf("request body reads = %d, want 0 while LLM_DEBUG is disabled", payload.reads)
- }
- if count := logs.Len(); count != 0 {
- t.Fatalf("provider log count = %d, want 0", count)
- }
-}
-
func TestIsLLMDebugEnabled(t *testing.T) {
tests := []struct {
value string
@@ -190,26 +33,6 @@ func TestIsLLMDebugEnabled(t *testing.T) {
}
}
-type roundTripFunc func(*http.Request) (*http.Response, error)
-
-func (f roundTripFunc) RoundTrip(req *http.Request) (*http.Response, error) {
- return f(req)
-}
-
-type trackingReadCloser struct {
- reader io.Reader
- reads int
-}
-
-func (r *trackingReadCloser) Read(p []byte) (int, error) {
- r.reads++
- return r.reader.Read(p)
-}
-
-func (r *trackingReadCloser) Close() error {
- return nil
-}
-
func TestBaseModelDoRequestAuthorizationHeader(t *testing.T) {
tests := []struct {
name string
diff --git a/internal/entity/models/deepinfra.go b/internal/entity/models/deepinfra.go
index 8177dca5c6..fd24a29345 100644
--- a/internal/entity/models/deepinfra.go
+++ b/internal/entity/models/deepinfra.go
@@ -42,7 +42,7 @@ func NewDeepInfraModel(baseURL map[string]string, urlSuffix URLSuffix) *DeepInfr
baseModel: BaseModel{
BaseURL: baseURL,
URLSuffix: urlSuffix,
- httpClient: NewDriverHTTPClient(false),
+ httpClient: common.GetSSRFHTTPClient(),
},
}
}
diff --git a/internal/entity/models/deepseek.go b/internal/entity/models/deepseek.go
index 5bf8808975..a642f0b1a8 100644
--- a/internal/entity/models/deepseek.go
+++ b/internal/entity/models/deepseek.go
@@ -39,7 +39,7 @@ func NewDeepSeekModel(baseURL map[string]string, urlSuffix URLSuffix) *DeepSeekM
baseModel: BaseModel{
BaseURL: baseURL,
URLSuffix: urlSuffix,
- httpClient: NewDriverHTTPClient(false),
+ httpClient: common.GetSSRFHTTPClient(),
},
}
}
diff --git a/internal/entity/models/fishaudio.go b/internal/entity/models/fishaudio.go
index c642520a1f..de198093e2 100644
--- a/internal/entity/models/fishaudio.go
+++ b/internal/entity/models/fishaudio.go
@@ -41,7 +41,7 @@ func NewFishAudioModel(baseURL map[string]string, urlSuffix URLSuffix) *FishAudi
baseModel: BaseModel{
BaseURL: baseURL,
URLSuffix: urlSuffix,
- httpClient: NewDriverHTTPClient(false),
+ httpClient: common.GetSSRFHTTPClient(),
},
}
}
diff --git a/internal/entity/models/funasr.go b/internal/entity/models/funasr.go
index 15989bd694..63b4568043 100644
--- a/internal/entity/models/funasr.go
+++ b/internal/entity/models/funasr.go
@@ -41,7 +41,7 @@ func NewFunASRModel(baseURL map[string]string, urlSuffix URLSuffix) *FunASR {
BaseURL: baseURL,
URLSuffix: urlSuffix,
AllowEmptyAPIKey: true,
- httpClient: NewDriverHTTPClient(true),
+ httpClient: common.GetSchemeSafeHTTPClient(),
},
}
}
diff --git a/internal/entity/models/futurmix.go b/internal/entity/models/futurmix.go
index a6d62732e5..691f0e7be8 100644
--- a/internal/entity/models/futurmix.go
+++ b/internal/entity/models/futurmix.go
@@ -35,7 +35,7 @@ func NewFuturMixModel(baseURL map[string]string, urlSuffix URLSuffix) *FuturMixM
baseModel: BaseModel{
BaseURL: baseURL,
URLSuffix: urlSuffix,
- httpClient: NewDriverHTTPClient(false),
+ httpClient: common.GetSSRFHTTPClient(),
},
}
}
diff --git a/internal/entity/models/gitee.go b/internal/entity/models/gitee.go
index 09ff023765..5a900bc28c 100644
--- a/internal/entity/models/gitee.go
+++ b/internal/entity/models/gitee.go
@@ -40,7 +40,7 @@ func NewGiteeModel(baseURL map[string]string, urlSuffix URLSuffix) *GiteeModel {
baseModel: BaseModel{
BaseURL: baseURL,
URLSuffix: urlSuffix,
- httpClient: NewDriverHTTPClient(false),
+ httpClient: common.GetSSRFHTTPClient(),
},
}
}
diff --git a/internal/entity/models/google.go b/internal/entity/models/google.go
index b13195400c..cbd2b928c9 100644
--- a/internal/entity/models/google.go
+++ b/internal/entity/models/google.go
@@ -21,7 +21,6 @@ import (
"encoding/json"
"fmt"
"math"
- "net/http"
"ragflow/internal/common"
"ragflow/internal/entity"
"strings"
@@ -153,7 +152,7 @@ func NewGoogleModel(baseURL map[string]string, urlSuffix URLSuffix) *GoogleModel
baseModel: BaseModel{
BaseURL: baseURL,
URLSuffix: urlSuffix,
- httpClient: &http.Client{Transport: newProviderLoggingTransport(http.DefaultTransport)},
+ httpClient: common.GetSSRFHTTPClient(),
},
}
}
diff --git a/internal/entity/models/gpustack.go b/internal/entity/models/gpustack.go
index 610670dd2b..62e7c9c8f3 100644
--- a/internal/entity/models/gpustack.go
+++ b/internal/entity/models/gpustack.go
@@ -38,7 +38,7 @@ func NewGPUStackModel(baseURL map[string]string, urlSuffix URLSuffix) *GPUStackM
BaseURL: baseURL,
URLSuffix: urlSuffix,
AllowEmptyAPIKey: true,
- httpClient: NewDriverHTTPClient(true),
+ httpClient: common.GetSchemeSafeHTTPClient(),
},
}
}
diff --git a/internal/entity/models/groq.go b/internal/entity/models/groq.go
index 27e0d4701b..ac0db3ca47 100644
--- a/internal/entity/models/groq.go
+++ b/internal/entity/models/groq.go
@@ -41,7 +41,7 @@ func NewGroqModel(baseURL map[string]string, urlSuffix URLSuffix) *GroqModel {
baseModel: BaseModel{
BaseURL: baseURL,
URLSuffix: urlSuffix,
- httpClient: NewDriverHTTPClient(false),
+ httpClient: common.GetSSRFHTTPClient(),
},
}
}
diff --git a/internal/entity/models/huaweicloud.go b/internal/entity/models/huaweicloud.go
index 8346e782e2..3d663110b4 100644
--- a/internal/entity/models/huaweicloud.go
+++ b/internal/entity/models/huaweicloud.go
@@ -36,7 +36,7 @@ func NewHuaweiCloudModel(baseURL map[string]string, urlSuffix URLSuffix) *Huawei
baseModel: BaseModel{
BaseURL: baseURL,
URLSuffix: urlSuffix,
- httpClient: NewDriverHTTPClient(false),
+ httpClient: common.GetSSRFHTTPClient(),
},
}
}
diff --git a/internal/entity/models/huggingface.go b/internal/entity/models/huggingface.go
index 041e1f7515..aff9704f75 100644
--- a/internal/entity/models/huggingface.go
+++ b/internal/entity/models/huggingface.go
@@ -37,7 +37,7 @@ func NewHuggingFaceModel(baseURL map[string]string, urlSuffix URLSuffix) *Huggin
baseModel: BaseModel{
BaseURL: baseURL,
URLSuffix: urlSuffix,
- httpClient: NewDriverHTTPClient(false),
+ httpClient: common.GetSSRFHTTPClient(),
},
}
}
diff --git a/internal/entity/models/hunyuan.go b/internal/entity/models/hunyuan.go
index 951830d9a7..9cf3ad913f 100644
--- a/internal/entity/models/hunyuan.go
+++ b/internal/entity/models/hunyuan.go
@@ -38,7 +38,7 @@ func NewHunyuanModel(baseURL map[string]string, urlSuffix URLSuffix) *HunyuanMod
baseModel: BaseModel{
BaseURL: baseURL,
URLSuffix: urlSuffix,
- httpClient: NewDriverHTTPClient(false),
+ httpClient: common.GetSSRFHTTPClient(),
},
}
}
diff --git a/internal/entity/models/jiekouai.go b/internal/entity/models/jiekouai.go
index 5207da35b7..926f26776a 100644
--- a/internal/entity/models/jiekouai.go
+++ b/internal/entity/models/jiekouai.go
@@ -36,7 +36,7 @@ func NewJieKouAIModel(baseURL map[string]string, urlSuffix URLSuffix) *JieKouAIM
// JieKouAI's methods issue requests without a per-call context deadline, so
// keep an explicit 120s client-level timeout to bound them. Built on the
// shared transport via NewDriverHTTPClient.
- client := NewDriverHTTPClient(false)
+ client := common.GetSSRFHTTPClient()
client.Timeout = 120 * time.Second
return &JieKouAIModel{
baseModel: BaseModel{
diff --git a/internal/entity/models/jina.go b/internal/entity/models/jina.go
index 2181804f1f..35022dc363 100644
--- a/internal/entity/models/jina.go
+++ b/internal/entity/models/jina.go
@@ -41,7 +41,7 @@ func NewJinaModel(baseURL map[string]string, urlSuffix URLSuffix) *JinaModel {
// Embed/Rerank/ListModels issue requests without a per-call context
// deadline, so keep an explicit 90s client-level timeout to bound them.
// Built on the shared transport via NewDriverHTTPClient.
- client := NewDriverHTTPClient(false)
+ client := common.GetSSRFHTTPClient()
client.Timeout = 90 * time.Second
return &JinaModel{
baseModel: BaseModel{
diff --git a/internal/entity/models/lmstudio.go b/internal/entity/models/lmstudio.go
index eb1fdef3a0..b2780e1312 100644
--- a/internal/entity/models/lmstudio.go
+++ b/internal/entity/models/lmstudio.go
@@ -38,7 +38,7 @@ func NewLmStudioModel(baseURL map[string]string, urlSuffix URLSuffix) *LmStudioM
BaseURL: baseURL,
URLSuffix: urlSuffix,
AllowEmptyAPIKey: true,
- httpClient: NewDriverHTTPClient(true),
+ httpClient: common.GetSchemeSafeHTTPClient(),
},
}
}
diff --git a/internal/entity/models/localai.go b/internal/entity/models/localai.go
index cffb4a7289..dcdead9775 100644
--- a/internal/entity/models/localai.go
+++ b/internal/entity/models/localai.go
@@ -39,7 +39,7 @@ func NewLocalAIModel(baseURL map[string]string, urlSuffix URLSuffix) *LocalAIMod
BaseURL: baseURL,
URLSuffix: urlSuffix,
AllowEmptyAPIKey: true,
- httpClient: NewDriverHTTPClient(true),
+ httpClient: common.GetSchemeSafeHTTPClient(),
},
}
}
diff --git a/internal/entity/models/longcat.go b/internal/entity/models/longcat.go
index d381b1deba..a5416d3c0f 100644
--- a/internal/entity/models/longcat.go
+++ b/internal/entity/models/longcat.go
@@ -37,7 +37,7 @@ func NewLongCatModel(baseURL map[string]string, urlSuffix URLSuffix) *LongCatMod
baseModel: BaseModel{
BaseURL: baseURL,
URLSuffix: urlSuffix,
- httpClient: NewDriverHTTPClient(false),
+ httpClient: common.GetSSRFHTTPClient(),
},
}
}
diff --git a/internal/entity/models/mineru.go b/internal/entity/models/mineru.go
index a643d64043..a4bd547423 100644
--- a/internal/entity/models/mineru.go
+++ b/internal/entity/models/mineru.go
@@ -35,7 +35,7 @@ func NewMinerUModel(baseURL map[string]string, urlSuffix URLSuffix) *MinerUModel
baseModel: BaseModel{
BaseURL: baseURL,
URLSuffix: urlSuffix,
- httpClient: NewDriverHTTPClient(false),
+ httpClient: common.GetSSRFHTTPClient(),
},
}
}
diff --git a/internal/entity/models/mineru_local.go b/internal/entity/models/mineru_local.go
index 8a5fb042cd..9613c804cb 100644
--- a/internal/entity/models/mineru_local.go
+++ b/internal/entity/models/mineru_local.go
@@ -37,7 +37,7 @@ func NewMinerLocalUModel(baseURL map[string]string, urlSuffix URLSuffix) *MinerU
BaseURL: baseURL,
URLSuffix: urlSuffix,
AllowEmptyAPIKey: true,
- httpClient: NewDriverHTTPClient(true),
+ httpClient: common.GetSchemeSafeHTTPClient(),
},
}
}
diff --git a/internal/entity/models/minimax.go b/internal/entity/models/minimax.go
index 15b77646e0..54ca7ea6fa 100644
--- a/internal/entity/models/minimax.go
+++ b/internal/entity/models/minimax.go
@@ -40,7 +40,7 @@ func NewMinimaxModel(baseURL map[string]string, urlSuffix URLSuffix) *MinimaxMod
baseModel: BaseModel{
BaseURL: baseURL,
URLSuffix: urlSuffix,
- httpClient: NewDriverHTTPClient(false),
+ httpClient: common.GetSSRFHTTPClient(),
},
}
}
diff --git a/internal/entity/models/mistral.go b/internal/entity/models/mistral.go
index c6fa8bb9e6..7a9256911c 100644
--- a/internal/entity/models/mistral.go
+++ b/internal/entity/models/mistral.go
@@ -114,7 +114,7 @@ func NewMistralModel(baseURL map[string]string, urlSuffix URLSuffix) *MistralMod
baseModel: BaseModel{
BaseURL: baseURL,
URLSuffix: urlSuffix,
- httpClient: NewDriverHTTPClient(false),
+ httpClient: common.GetSSRFHTTPClient(),
},
}
}
diff --git a/internal/entity/models/modelscope.go b/internal/entity/models/modelscope.go
index 344f561a0c..71a4b18d86 100644
--- a/internal/entity/models/modelscope.go
+++ b/internal/entity/models/modelscope.go
@@ -42,7 +42,7 @@ func NewModelScopeModel(baseURL map[string]string, urlSuffix URLSuffix) *ModelSc
BaseURL: baseURL,
URLSuffix: urlSuffix,
AllowEmptyAPIKey: true,
- httpClient: NewDriverHTTPClient(true),
+ httpClient: common.GetSchemeSafeHTTPClient(),
},
}
}
diff --git a/internal/entity/models/moonshot.go b/internal/entity/models/moonshot.go
index 933d3e687e..5b57329813 100644
--- a/internal/entity/models/moonshot.go
+++ b/internal/entity/models/moonshot.go
@@ -37,7 +37,7 @@ func NewMoonshotModel(baseURL map[string]string, urlSuffix URLSuffix) *MoonshotM
baseModel: BaseModel{
BaseURL: baseURL,
URLSuffix: urlSuffix,
- httpClient: NewDriverHTTPClient(false),
+ httpClient: common.GetSSRFHTTPClient(),
},
}
}
diff --git a/internal/entity/models/mws.go b/internal/entity/models/mws.go
index ab5f26677d..5ef18ed3d2 100644
--- a/internal/entity/models/mws.go
+++ b/internal/entity/models/mws.go
@@ -35,7 +35,7 @@ type MWSModel struct {
// NewMWSModel creates an MWS model driver.
func NewMWSModel(baseURL map[string]string, urlSuffix URLSuffix) *MWSModel {
driver := NewDummyModel(baseURL, urlSuffix)
- driver.baseModel.httpClient = NewDriverHTTPClient(true)
+ driver.baseModel.httpClient = common.GetSchemeSafeHTTPClient()
return &MWSModel{DummyModel: driver}
}
diff --git a/internal/entity/models/n1n.go b/internal/entity/models/n1n.go
index 766fb939f1..74e82e9c26 100644
--- a/internal/entity/models/n1n.go
+++ b/internal/entity/models/n1n.go
@@ -38,7 +38,7 @@ func NewN1NModel(baseURL map[string]string, urlSuffix URLSuffix) *N1NModel {
baseModel: BaseModel{
BaseURL: baseURL,
URLSuffix: urlSuffix,
- httpClient: NewDriverHTTPClient(false),
+ httpClient: common.GetSSRFHTTPClient(),
},
}
}
diff --git a/internal/entity/models/novita.go b/internal/entity/models/novita.go
index f3a83636d5..f81715268e 100644
--- a/internal/entity/models/novita.go
+++ b/internal/entity/models/novita.go
@@ -39,7 +39,7 @@ func NewNovitaModel(baseURL map[string]string, urlSuffix URLSuffix) *NovitaModel
baseModel: BaseModel{
BaseURL: baseURL,
URLSuffix: urlSuffix,
- httpClient: NewDriverHTTPClient(false),
+ httpClient: common.GetSSRFHTTPClient(),
},
}
}
diff --git a/internal/entity/models/nvidia.go b/internal/entity/models/nvidia.go
index 99fbc6079a..9081166cc2 100644
--- a/internal/entity/models/nvidia.go
+++ b/internal/entity/models/nvidia.go
@@ -49,7 +49,7 @@ func NewNvidiaModel(baseURL map[string]string, urlSuffix URLSuffix) *NvidiaModel
baseModel: BaseModel{
BaseURL: baseURL,
URLSuffix: urlSuffix,
- httpClient: NewDriverHTTPClient(false),
+ httpClient: common.GetSSRFHTTPClient(),
},
catalogURL: nvidiaCatalogURL,
hostedAPIHost: nvidiaHostedAPIHost,
diff --git a/internal/entity/models/openai.go b/internal/entity/models/openai.go
index 3f5ffc9813..1e6ab5fa08 100644
--- a/internal/entity/models/openai.go
+++ b/internal/entity/models/openai.go
@@ -44,7 +44,7 @@ func NewOpenAIModel(baseURL map[string]string, urlSuffix URLSuffix) *OpenAIModel
baseModel: BaseModel{
BaseURL: baseURL,
URLSuffix: urlSuffix,
- httpClient: NewDriverHTTPClient(false),
+ httpClient: common.GetSSRFHTTPClient(),
},
}
}
diff --git a/internal/entity/models/openrouter.go b/internal/entity/models/openrouter.go
index 3e9330dd81..86c9b75074 100644
--- a/internal/entity/models/openrouter.go
+++ b/internal/entity/models/openrouter.go
@@ -41,7 +41,7 @@ func NewOpenRouterModel(baseURL map[string]string, urlSuffix URLSuffix) *OpenRou
baseModel: BaseModel{
BaseURL: baseURL,
URLSuffix: urlSuffix,
- httpClient: NewDriverHTTPClient(false),
+ httpClient: common.GetSSRFHTTPClient(),
},
}
}
diff --git a/internal/entity/models/orcarouter.go b/internal/entity/models/orcarouter.go
index cecd2f9153..d36c137460 100644
--- a/internal/entity/models/orcarouter.go
+++ b/internal/entity/models/orcarouter.go
@@ -35,7 +35,7 @@ func NewOrcaRouterModel(baseURL map[string]string, urlSuffix URLSuffix) *OrcaRou
baseModel: BaseModel{
BaseURL: baseURL,
URLSuffix: urlSuffix,
- httpClient: NewDriverHTTPClient(false),
+ httpClient: common.GetSSRFHTTPClient(),
},
}
}
diff --git a/internal/entity/models/paddleocr.go b/internal/entity/models/paddleocr.go
index 2c9751fa36..5f4cd5b798 100644
--- a/internal/entity/models/paddleocr.go
+++ b/internal/entity/models/paddleocr.go
@@ -43,7 +43,7 @@ func NewPaddleOCRModel(baseURL map[string]string, urlSuffix URLSuffix) *PaddleOC
BaseURL: baseURL,
URLSuffix: urlSuffix,
AllowEmptyAPIKey: true,
- httpClient: NewDriverHTTPClient(false),
+ httpClient: common.GetSSRFHTTPClient(),
},
}
}
diff --git a/internal/entity/models/paddleocr_local.go b/internal/entity/models/paddleocr_local.go
index b90f2b12e3..6e0b224ebc 100644
--- a/internal/entity/models/paddleocr_local.go
+++ b/internal/entity/models/paddleocr_local.go
@@ -40,7 +40,7 @@ func NewPaddleOCRLocalModel(baseURL map[string]string, urlSuffix URLSuffix) *Pad
baseModel: BaseModel{
BaseURL: baseURL,
URLSuffix: urlSuffix,
- httpClient: NewDriverHTTPClient(true),
+ httpClient: common.GetSchemeSafeHTTPClient(),
},
}
}
diff --git a/internal/entity/models/perplexity.go b/internal/entity/models/perplexity.go
index 848ddf492f..566e7beb8c 100644
--- a/internal/entity/models/perplexity.go
+++ b/internal/entity/models/perplexity.go
@@ -36,7 +36,7 @@ func NewPerplexityModel(baseURL map[string]string, urlSuffix URLSuffix) *Perplex
baseModel: BaseModel{
BaseURL: baseURL,
URLSuffix: urlSuffix,
- httpClient: NewDriverHTTPClient(false),
+ httpClient: common.GetSSRFHTTPClient(),
},
}
}
diff --git a/internal/entity/models/ppio.go b/internal/entity/models/ppio.go
index f0b3548b6c..a48d2bd4a7 100644
--- a/internal/entity/models/ppio.go
+++ b/internal/entity/models/ppio.go
@@ -39,7 +39,7 @@ func NewPPIOModel(baseURL map[string]string, urlSuffix URLSuffix) *PPIOModel {
baseModel: BaseModel{
BaseURL: baseURL,
URLSuffix: urlSuffix,
- httpClient: NewDriverHTTPClient(false),
+ httpClient: common.GetSSRFHTTPClient(),
},
}
}
diff --git a/internal/entity/models/qiniu.go b/internal/entity/models/qiniu.go
index 029486d75f..0c39bfc38b 100644
--- a/internal/entity/models/qiniu.go
+++ b/internal/entity/models/qiniu.go
@@ -35,7 +35,7 @@ func NewQiniuModel(baseURL map[string]string, urlSuffix URLSuffix) *QiniuModel {
baseModel: BaseModel{
BaseURL: baseURL,
URLSuffix: urlSuffix,
- httpClient: NewDriverHTTPClient(false),
+ httpClient: common.GetSSRFHTTPClient(),
},
}
}
diff --git a/internal/entity/models/ragcon.go b/internal/entity/models/ragcon.go
index 5cf32b8bd9..4d0ba4261f 100644
--- a/internal/entity/models/ragcon.go
+++ b/internal/entity/models/ragcon.go
@@ -44,7 +44,7 @@ func NewRAGconModel(baseURL map[string]string, urlSuffix URLSuffix) *RAGconModel
baseModel: BaseModel{
BaseURL: baseURL,
URLSuffix: urlSuffix,
- httpClient: NewDriverHTTPClient(false),
+ httpClient: common.GetSSRFHTTPClient(),
},
}
}
diff --git a/internal/entity/models/replicate.go b/internal/entity/models/replicate.go
index 976277e634..ca4ffb3580 100644
--- a/internal/entity/models/replicate.go
+++ b/internal/entity/models/replicate.go
@@ -42,7 +42,7 @@ func NewReplicateModel(baseURL map[string]string, urlSuffix URLSuffix) *Replicat
baseModel: BaseModel{
BaseURL: baseURL,
URLSuffix: urlSuffix,
- httpClient: NewDriverHTTPClient(false),
+ httpClient: common.GetSSRFHTTPClient(),
},
}
}
diff --git a/internal/entity/models/siliconflow.go b/internal/entity/models/siliconflow.go
index 7cf2d4ecb0..fd44dbb91c 100644
--- a/internal/entity/models/siliconflow.go
+++ b/internal/entity/models/siliconflow.go
@@ -42,7 +42,7 @@ func NewSiliconflowModel(baseURL map[string]string, urlSuffix URLSuffix) *Silico
baseModel: BaseModel{
BaseURL: baseURL,
URLSuffix: urlSuffix,
- httpClient: NewDriverHTTPClient(false),
+ httpClient: common.GetSSRFHTTPClient(),
},
}
}
diff --git a/internal/entity/models/stepfun.go b/internal/entity/models/stepfun.go
index 16485de0e1..6882673c2b 100644
--- a/internal/entity/models/stepfun.go
+++ b/internal/entity/models/stepfun.go
@@ -39,7 +39,7 @@ func NewStepFunModel(baseURL map[string]string, urlSuffix URLSuffix) *StepFunMod
baseModel: BaseModel{
BaseURL: baseURL,
URLSuffix: urlSuffix,
- httpClient: NewDriverHTTPClient(false),
+ httpClient: common.GetSSRFHTTPClient(),
},
}
}
diff --git a/internal/entity/models/togetherai.go b/internal/entity/models/togetherai.go
index cdddb68a97..450a865e68 100644
--- a/internal/entity/models/togetherai.go
+++ b/internal/entity/models/togetherai.go
@@ -41,7 +41,7 @@ func NewTogetherAIModel(baseURL map[string]string, urlSuffix URLSuffix) *Togethe
baseModel: BaseModel{
BaseURL: baseURL,
URLSuffix: urlSuffix,
- httpClient: NewDriverHTTPClient(false),
+ httpClient: common.GetSSRFHTTPClient(),
},
}
}
diff --git a/internal/entity/models/tokenhub.go b/internal/entity/models/tokenhub.go
index c9223c3cea..89f7202ac6 100644
--- a/internal/entity/models/tokenhub.go
+++ b/internal/entity/models/tokenhub.go
@@ -36,7 +36,7 @@ func NewTokenHubModel(baseURL map[string]string, urlSuffix URLSuffix) *TokenHubM
baseModel: BaseModel{
BaseURL: baseURL,
URLSuffix: urlSuffix,
- httpClient: NewDriverHTTPClient(false),
+ httpClient: common.GetSSRFHTTPClient(),
},
}
}
diff --git a/internal/entity/models/tokenpony.go b/internal/entity/models/tokenpony.go
index 1f9f305de8..2dad871c5e 100644
--- a/internal/entity/models/tokenpony.go
+++ b/internal/entity/models/tokenpony.go
@@ -37,7 +37,7 @@ func NewTokenPonyModel(baseURL map[string]string, urlSuffix URLSuffix) *TokenPon
baseModel: BaseModel{
BaseURL: baseURL,
URLSuffix: urlSuffix,
- httpClient: NewDriverHTTPClient(false),
+ httpClient: common.GetSSRFHTTPClient(),
},
}
}
diff --git a/internal/entity/models/upstage.go b/internal/entity/models/upstage.go
index 20b7818832..6665f58af6 100644
--- a/internal/entity/models/upstage.go
+++ b/internal/entity/models/upstage.go
@@ -38,7 +38,7 @@ func NewUpstageModel(baseURL map[string]string, urlSuffix URLSuffix) *UpstageMod
baseModel: BaseModel{
BaseURL: baseURL,
URLSuffix: urlSuffix,
- httpClient: NewDriverHTTPClient(false),
+ httpClient: common.GetSSRFHTTPClient(),
},
}
}
diff --git a/internal/entity/models/vllm.go b/internal/entity/models/vllm.go
index 391310f278..0d40701005 100644
--- a/internal/entity/models/vllm.go
+++ b/internal/entity/models/vllm.go
@@ -39,7 +39,7 @@ func NewVllmModel(baseURL map[string]string, urlSuffix URLSuffix) *VllmModel {
BaseURL: baseURL,
URLSuffix: urlSuffix,
AllowEmptyAPIKey: true,
- httpClient: NewDriverHTTPClient(true),
+ httpClient: common.GetSchemeSafeHTTPClient(),
},
}
}
diff --git a/internal/entity/models/volcengine.go b/internal/entity/models/volcengine.go
index a98a267de0..6161685dc8 100644
--- a/internal/entity/models/volcengine.go
+++ b/internal/entity/models/volcengine.go
@@ -38,7 +38,7 @@ func NewVolcEngine(baseURL map[string]string, urlSuffix URLSuffix) *VolcEngine {
baseModel: BaseModel{
BaseURL: baseURL,
URLSuffix: urlSuffix,
- httpClient: NewDriverHTTPClient(false),
+ httpClient: common.GetSSRFHTTPClient(),
},
}
}
diff --git a/internal/entity/models/voyage.go b/internal/entity/models/voyage.go
index 0e524906ee..20d070544f 100644
--- a/internal/entity/models/voyage.go
+++ b/internal/entity/models/voyage.go
@@ -38,7 +38,7 @@ func NewVoyageModel(baseURL map[string]string, urlSuffix URLSuffix) *VoyageModel
baseModel: BaseModel{
BaseURL: baseURL,
URLSuffix: urlSuffix,
- httpClient: NewDriverHTTPClient(false),
+ httpClient: common.GetSSRFHTTPClient(),
},
}
}
diff --git a/internal/entity/models/xai.go b/internal/entity/models/xai.go
index 9757dabae0..9637617747 100644
--- a/internal/entity/models/xai.go
+++ b/internal/entity/models/xai.go
@@ -49,7 +49,7 @@ func NewXAIModel(baseURL map[string]string, urlSuffix URLSuffix) *XAIModel {
baseModel: BaseModel{
BaseURL: baseURL,
URLSuffix: urlSuffix,
- httpClient: NewDriverHTTPClient(false),
+ httpClient: common.GetSSRFHTTPClient(),
},
}
}
diff --git a/internal/entity/models/xiaomi.go b/internal/entity/models/xiaomi.go
index 3b2e6c4ecd..a9a1f9d78d 100644
--- a/internal/entity/models/xiaomi.go
+++ b/internal/entity/models/xiaomi.go
@@ -40,7 +40,7 @@ func NewXiaomiModel(baseURL map[string]string, urlSuffix URLSuffix) *XiaomiModel
baseModel: BaseModel{
BaseURL: baseURL,
URLSuffix: urlSuffix,
- httpClient: NewDriverHTTPClient(false),
+ httpClient: common.GetSSRFHTTPClient(),
// Xiaomi authenticates with the non-standard "api-key" header
// instead of "Authorization: Bearer".
authHeader: func(cfg *APIConfig) (string, string) {
diff --git a/internal/entity/models/xinference.go b/internal/entity/models/xinference.go
index b072e44f7e..6e7b30e599 100644
--- a/internal/entity/models/xinference.go
+++ b/internal/entity/models/xinference.go
@@ -46,7 +46,7 @@ func NewXinferenceModel(baseURL map[string]string, urlSuffix URLSuffix) *Xinfere
BaseURL: baseURL,
URLSuffix: urlSuffix,
AllowEmptyAPIKey: true,
- httpClient: NewDriverHTTPClient(true),
+ httpClient: common.GetSchemeSafeHTTPClient(),
},
}
}
diff --git a/internal/entity/models/xunfei.go b/internal/entity/models/xunfei.go
index 5cd35cb409..13bf0c4752 100644
--- a/internal/entity/models/xunfei.go
+++ b/internal/entity/models/xunfei.go
@@ -73,7 +73,7 @@ func NewXunFeiModel(baseURL map[string]string, urlSuffix URLSuffix) *XunFeiModel
baseModel: BaseModel{
BaseURL: baseURL,
URLSuffix: urlSuffix,
- httpClient: NewDriverHTTPClient(false),
+ httpClient: common.GetSSRFHTTPClient(),
// The Spark HTTP API authenticates with the credential bundle's
// spark_api_password, not the raw stored key.
authHeader: func(cfg *APIConfig) (string, string) {
diff --git a/internal/entity/models/zhipu-ai.go b/internal/entity/models/zhipu-ai.go
index a99ad0fad5..53715a3747 100644
--- a/internal/entity/models/zhipu-ai.go
+++ b/internal/entity/models/zhipu-ai.go
@@ -42,7 +42,7 @@ func NewZhipuAIModel(baseURL map[string]string, urlSuffix URLSuffix) *ZhipuAIMod
baseModel: BaseModel{
BaseURL: baseURL,
URLSuffix: urlSuffix,
- httpClient: NewDriverHTTPClient(false),
+ httpClient: common.GetSSRFHTTPClient(),
},
}
}
diff --git a/internal/parser/parser/pdf_parser_docling.go b/internal/parser/parser/pdf_parser_docling.go
index 0dd5f75148..e3fd1e50d6 100644
--- a/internal/parser/parser/pdf_parser_docling.go
+++ b/internal/parser/parser/pdf_parser_docling.go
@@ -88,7 +88,7 @@ func parsePDFWithDocling(ctx context.Context, filename string, data []byte, pars
var lastErr error
for _, candidate := range payloads {
url := baseURL + candidate.endpoint
- resp, err := models.PostJSONRequest(context.Background(), models.NewDriverHTTPClient(false), url, auth, candidate.body())
+ resp, err := models.PostJSONRequest(context.Background(), common.GetSSRFHTTPClient(), url, auth, candidate.body())
if err != nil {
lastErr = fmt.Errorf("%s: %w", candidate.endpoint, err)
continue
diff --git a/internal/parser/parser/pdf_parser_opendataloader.go b/internal/parser/parser/pdf_parser_opendataloader.go
index 8920efec07..dd191f2431 100644
--- a/internal/parser/parser/pdf_parser_opendataloader.go
+++ b/internal/parser/parser/pdf_parser_opendataloader.go
@@ -9,8 +9,6 @@ import (
"net/http"
"ragflow/internal/common"
"strings"
-
- models "ragflow/internal/entity/models"
)
func parsePDFWithOpenDataLoader(ctx context.Context, filename string, data []byte, parser *PDFParser) ParseResult {
@@ -41,7 +39,7 @@ func parsePDFWithOpenDataLoader(ctx context.Context, filename string, data []byt
if apiKey != "" {
req.Header.Set("Authorization", "Bearer "+apiKey)
}
- resp, err := models.NewDriverHTTPClient(false).Do(req)
+ resp, err := common.GetSSRFHTTPClient().Do(req)
if err != nil {
return ParseResult{Err: fmt.Errorf("parser: OpenDataLoader submit: %w", err)}
}
diff --git a/internal/parser/parser/pdf_parser_somark.go b/internal/parser/parser/pdf_parser_somark.go
index b2639cfa4c..d9b0c88931 100644
--- a/internal/parser/parser/pdf_parser_somark.go
+++ b/internal/parser/parser/pdf_parser_somark.go
@@ -11,8 +11,6 @@ import (
"net/url"
"ragflow/internal/common"
"strings"
-
- models "ragflow/internal/entity/models"
)
func parsePDFWithSoMark(filename string, data []byte, parser *PDFParser) ParseResult {
@@ -84,7 +82,7 @@ func soMarkSubmit(baseURL, filename string, data []byte, parser *PDFParser, apiK
return "", fmt.Errorf("parser: SoMark request: %w", err)
}
req.Header.Set("Content-Type", writer.FormDataContentType())
- resp, err := models.NewDriverHTTPClient(false).Do(req)
+ resp, err := common.GetSSRFHTTPClient().Do(req)
if err != nil {
return "", fmt.Errorf("parser: SoMark submit: %w", err)
}
@@ -122,7 +120,7 @@ func soMarkPoll(baseURL, taskID, apiKey string) (map[string]any, error) {
return nil, fmt.Errorf("parser: SoMark poll request: %w", err)
}
req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
- resp, err := models.NewDriverHTTPClient(false).Do(req)
+ resp, err := common.GetSSRFHTTPClient().Do(req)
if err != nil {
return nil, fmt.Errorf("parser: SoMark poll: %w", err)
}
diff --git a/internal/parser/parser/tcadp_common.go b/internal/parser/parser/tcadp_common.go
index 9b2dad9bab..a17f5a77ea 100644
--- a/internal/parser/parser/tcadp_common.go
+++ b/internal/parser/parser/tcadp_common.go
@@ -8,6 +8,7 @@ import (
"io"
"net/http"
"os"
+ "ragflow/internal/common"
"strings"
models "ragflow/internal/entity/models"
@@ -70,7 +71,7 @@ func parseWithTCADP(
"MarkdownImageResponseType": markdownImageResponseType,
},
}
- resp, err := models.PostJSONRequest(ctx, models.NewDriverHTTPClient(false),
+ resp, err := models.PostJSONRequest(ctx, common.GetSSRFHTTPClient(),
strings.TrimRight(baseURL, "/")+"/reconstruct_document", bearer(apiKey), requestBody)
if err != nil {
return ParseResult{Err: fmt.Errorf("parser: TCADP submit: %w", err)}
@@ -86,7 +87,7 @@ func parseWithTCADP(
var payload struct {
DocumentRecognizeResultURL string `json:"DocumentRecognizeResultUrl"`
}
- if err := json.Unmarshal(raw, &payload); err != nil {
+ if err = json.Unmarshal(raw, &payload); err != nil {
return ParseResult{Err: fmt.Errorf("parser: TCADP decode submit: %w", err)}
}
if payload.DocumentRecognizeResultURL == "" {
@@ -100,7 +101,7 @@ func parseWithTCADP(
if auth := bearer(apiKey); auth != "" {
downloadReq.Header.Set("Authorization", auth)
}
- downloadResp, err := models.NewDriverHTTPClient(false).Do(downloadReq)
+ downloadResp, err := common.GetSSRFHTTPClient().Do(downloadReq)
if err != nil {
return ParseResult{Err: fmt.Errorf("parser: TCADP download: %w", err)}
}