RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
fix: secure MWS model requests (#21505)
buua436de93dd2d46c5
diff --git a/internal/entity/models/base_model.go b/internal/entity/models/base_model.goindex 82c9c0e516..640b5ec8f9 100644--- a/internal/entity/models/base_model.go+++ b/internal/entity/models/base_model.go@@ -33,14 +33,18 @@ import (// APIStatusError is a provider HTTP failure with its status code preserved, so// callers can act on the status (failover cooldown, retry) instead of matching-// the error text. The shared request helpers return it; the message is byte-for--// byte what plain fmt.Errorf produced before, so existing assertions still hold.+// the error text. hideBody prevents untrusted MWS responses from echoing+// credentials through user-visible error messages.type APIStatusError struct {- Status int- Body string+ Status int+ Body string+ hideBody bool}func (e *APIStatusError) Error() string {+ if e.hideBody {+ return fmt.Sprintf("API request failed with status %d", e.Status)+ }return fmt.Sprintf("API request failed with status %d: %s", e.Status, e.Body)}@@ -49,6 +53,7 @@ type BaseModel struct {URLSuffix URLSuffixhttpClient *http.ClientAllowEmptyAPIKey bool+ hideAPIErrorBody bool// authHeader, when non-nil, supplies the (name, value) pair used for// authentication instead of the default "Authorization: Bearer <key>".// Drivers with non-standard auth (e.g. Xiaomi's api-key header, Xunfei's@@ -235,7 +240,7 @@ func (b *BaseModel) doRequest(ctx context.Context, url string, apiConfig *APIConif err != nil {return nil, fmt.Errorf("API request failed with status %d; failed to read error response: %w", resp.StatusCode, err)}- return nil, &APIStatusError{Status: resp.StatusCode, Body: string(body)}+ return nil, &APIStatusError{Status: resp.StatusCode, Body: string(body), hideBody: b.hideAPIErrorBody}}body, err = readModelResponseBody(resp.Body)@@ -270,7 +275,7 @@ func (b *BaseModel) doGetRequest(ctx context.Context, url string, apiConfig *APIif err != nil {return nil, fmt.Errorf("API request failed with status %d; failed to read error response: %w", resp.StatusCode, err)}- return nil, &APIStatusError{Status: resp.StatusCode, Body: string(body)}+ return nil, &APIStatusError{Status: resp.StatusCode, Body: string(body), hideBody: b.hideAPIErrorBody}}body, err := readModelResponseBody(resp.Body)@@ -303,7 +308,7 @@ func (b *BaseModel) doStreamRequest(ctx context.Context, url string, apiConfig *if err != nil {return fmt.Errorf("API request failed with status %d; failed to read error response: %w", resp.StatusCode, err)}- return &APIStatusError{Status: resp.StatusCode, Body: string(body)}+ return &APIStatusError{Status: resp.StatusCode, Body: string(body), hideBody: b.hideAPIErrorBody}}return handler(resp.Body)diff --git a/internal/entity/models/mws.go b/internal/entity/models/mws.goindex 0d1cf2bea8..8aa7104f57 100644--- a/internal/entity/models/mws.go+++ b/internal/entity/models/mws.go@@ -35,7 +35,8 @@ type MWSModel struct {// NewMWSModel creates an MWS model driver.func NewMWSModel(baseURL map[string]string, urlSuffix URLSuffix) *MWSModel {driver := NewDummyModel(baseURL, urlSuffix)- driver.baseModel.httpClient = common.GetSchemeSafeHTTPClient()+ driver.baseModel.httpClient = newMWSHTTPClient()+ driver.baseModel.hideAPIErrorBody = truereturn &MWSModel{DummyModel: driver}}@@ -52,16 +53,18 @@ func (m *MWSModel) NewInstance(baseURL map[string]string) ModelDriver {func normalizeMWSProjectURL(rawURL string) (string, error) {value := strings.TrimSuffix(strings.TrimSpace(rawURL), "/")parsed, err := url.Parse(value)- if err != nil || (parsed.Scheme != "http" && parsed.Scheme != "https") || parsed.Hostname() == "" {+ if err != nil || !strings.EqualFold(parsed.Scheme, "https") || !strings.EqualFold(parsed.Hostname(), mwsAPIHost) || (parsed.Port() != "" && parsed.Port() != "443") {return "", fmt.Errorf("MWS API URL must be a project root in the form https://gpt.mwsapis.ru/projects/<project>")}parts := strings.Split(strings.Trim(parsed.Path, "/"), "/")- if len(parts) != 2 || parts[0] != "projects" || parts[1] == "" {+ if len(parts) != 2 || parts[0] != "projects" || parts[1] == "" || parts[1] == "." || parts[1] == ".." {return "", fmt.Errorf("MWS API URL must be a project root in the form https://gpt.mwsapis.ru/projects/<project>")}- if parsed.RawQuery != "" || parsed.ForceQuery || parsed.Fragment != "" || parsed.User != nil {+ if parsed.RawQuery != "" || parsed.ForceQuery || parsed.Fragment != "" || parsed.User != nil || parsed.Opaque != "" {return "", fmt.Errorf("MWS API URL must not contain credentials, a query string, or a fragment")}+ parsed.Scheme = "https"+ parsed.Host = mwsAPIHostparsed.Path = "/projects/" + parts[1]parsed.RawPath = ""return parsed.String(), nildiff --git a/internal/entity/models/mws_http.go b/internal/entity/models/mws_http.gonew file mode 100644index 0000000000..20991d5863--- /dev/null+++ b/internal/entity/models/mws_http.go@@ -0,0 +1,116 @@+//+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.+//++package models++import (+ "context"+ "fmt"+ "net"+ "net/http"+ "net/url"+ "strings"+ "time"++ "ragflow/internal/common"+)++const mwsAPIHost = "gpt.mwsapis.ru"++type mwsDialTargetKey struct{}++type mwsDialTarget struct {+ hostname string+ resolvedIP string+}++type mwsDialer interface {+ DialContext(context.Context, string, string) (net.Conn, error)+}++type mwsSSRFTransport struct {+ base http.RoundTripper+}++func (t *mwsSSRFTransport) RoundTrip(request *http.Request) (*http.Response, error) {+ if err := validateMWSRequestURL(request.URL); err != nil {+ return nil, err+ }+ hostname, resolvedIP, err := common.AssertURLSafe(request.URL.String())+ if err != nil {+ return nil, fmt.Errorf("MWS request destination is not allowed: %w", err)+ }+ if !strings.EqualFold(hostname, mwsAPIHost) || resolvedIP == "" {+ return nil, fmt.Errorf("MWS request destination is not allowed")+ }++ ctx := context.WithValue(request.Context(), mwsDialTargetKey{}, mwsDialTarget{+ hostname: hostname,+ resolvedIP: resolvedIP,+ })+ return t.base.RoundTrip(request.Clone(ctx))+}++func validateMWSRequestURL(requestURL *url.URL) error {+ if requestURL == nil || !strings.EqualFold(requestURL.Scheme, "https") || !strings.EqualFold(requestURL.Hostname(), mwsAPIHost) {+ return fmt.Errorf("MWS requests must use https://%s", mwsAPIHost)+ }+ if port := requestURL.Port(); port != "" && port != "443" {+ return fmt.Errorf("MWS requests must use the default HTTPS port")+ }+ if requestURL.User != nil || requestURL.Opaque != "" || requestURL.RawQuery != "" || requestURL.ForceQuery || requestURL.Fragment != "" {+ return fmt.Errorf("MWS requests must not contain credentials, a query string, or a fragment")+ }+ return nil+}++func newMWSHTTPClient() *http.Client {+ var transport *http.Transport+ if defaultTransport, ok := http.DefaultTransport.(*http.Transport); ok {+ transport = defaultTransport.Clone()+ } else {+ transport = &http.Transport{}+ }+ transport.Proxy = nil+ transport.MaxIdleConns = 100+ transport.MaxIdleConnsPerHost = 10+ transport.IdleConnTimeout = 90 * time.Second+ transport.ResponseHeaderTimeout = 20 * time.Minute+ transport.TLSHandshakeTimeout = 30 * time.Second++ dialer := &net.Dialer{Timeout: 30 * time.Second, KeepAlive: 30 * time.Second}+ transport.DialContext = newMWSDialContext(dialer)++ return &http.Client{+ Transport: common.NewProviderLoggingTransport(&mwsSSRFTransport{base: transport}),+ CheckRedirect: rejectMWSRedirect,+ }+}++func newMWSDialContext(dialer mwsDialer) func(context.Context, string, string) (net.Conn, error) {+ return func(ctx context.Context, network, address string) (net.Conn, error) {+ target, ok := ctx.Value(mwsDialTargetKey{}).(mwsDialTarget)+ hostname, port, err := net.SplitHostPort(address)+ if !ok || err != nil || target.resolvedIP == "" || !strings.EqualFold(hostname, target.hostname) || !strings.EqualFold(hostname, mwsAPIHost) {+ return nil, fmt.Errorf("unvalidated MWS HTTP destination %q", address)+ }+ return dialer.DialContext(ctx, network, net.JoinHostPort(target.resolvedIP, port))+ }+}++func rejectMWSRedirect(_ *http.Request, _ []*http.Request) error {+ return http.ErrUseLastResponse+}diff --git a/internal/entity/models/mws_test.go b/internal/entity/models/mws_test.goindex ccea47fafc..65cdc33a75 100644--- a/internal/entity/models/mws_test.go+++ b/internal/entity/models/mws_test.go@@ -17,23 +17,54 @@package modelsimport (+ "context""encoding/json"+ "errors""io"+ "net""net/http""net/http/httptest"+ "net/url""reflect""strings""sync/atomic""testing"++ "ragflow/internal/common")func newMWSTestDriver(serverURL string) (*MWSModel, *APIConfig) {token := "token"- baseURL := serverURL + "/projects/test-project"+ baseURL := "https://gpt.mwsapis.ru/projects/test-project"driver := NewMWSModel(map[string]string{"default": baseURL}, URLSuffix{})+ driver.baseModel.httpClient = &http.Client{+ Transport: mwsTestRoundTripper{serverURL: serverURL},+ CheckRedirect: rejectMWSRedirect,+ }return driver, &APIConfig{ApiKey: &token, BaseURL: &baseURL}}+type mwsTestRoundTripper struct {+ serverURL string+}++func (t mwsTestRoundTripper) RoundTrip(request *http.Request) (*http.Response, error) {+ if err := validateMWSRequestURL(request.URL); err != nil {+ return nil, err+ }+ targetURL, err := url.Parse(t.serverURL)+ if err != nil {+ return nil, err+ }+ request = request.Clone(request.Context())+ target := *request.URL+ target.Scheme = targetURL.Scheme+ target.Host = targetURL.Host+ request.URL = &target+ request.Host = targetURL.Host+ return http.DefaultTransport.RoundTrip(request)+}+func decodeMWSRequest(t *testing.T, request *http.Request) map[string]any {t.Helper()if request.Method != http.MethodPost {@@ -67,6 +98,98 @@ func TestNormalizeMWSProjectURL(t *testing.T) {if _, err = normalizeMWSProjectURL("https://gpt.mwsapis.ru/projects/demo/openai/v1"); err == nil {t.Fatal("expected a non-root URL to be rejected")}+ for _, unsafeURL := range []string{+ "http://gpt.mwsapis.ru/projects/demo",+ "https://attacker.example/projects/demo",+ "https://127.0.0.1/projects/demo",+ "https://gpt.mwsapis.ru:8443/projects/demo",+ "https://user:[email protected]/projects/demo",+ "https://gpt.mwsapis.ru/projects/../demo",+ } {+ if _, err = normalizeMWSProjectURL(unsafeURL); err == nil {+ t.Errorf("expected unsafe URL %q to be rejected", unsafeURL)+ }+ }+}++func TestMWSHTTPClientRejectsPrivateDNSResult(t *testing.T) {+ previousLookup := common.LookupHost+ previousAllowAny := common.AllowAnyHostForTest+ common.LookupHost = func(string) ([]string, error) { return []string{"127.0.0.1"}, nil }+ common.AllowAnyHostForTest = false+ t.Cleanup(func() {+ common.LookupHost = previousLookup+ common.AllowAnyHostForTest = previousAllowAny+ })++ request, err := http.NewRequest(http.MethodGet, "https://gpt.mwsapis.ru/projects/demo/openai/v1/models", nil)+ if err != nil {+ t.Fatal(err)+ }+ _, err = newMWSHTTPClient().Do(request)+ if err == nil || !strings.Contains(err.Error(), "non-public address") {+ t.Fatalf("expected a private DNS result to be rejected, got %v", err)+ }+}++type recordingMWSDialer struct {+ network string+ address string+}++func (d *recordingMWSDialer) DialContext(_ context.Context, network, address string) (net.Conn, error) {+ d.network = network+ d.address = address+ return nil, errors.New("recorded dial")+}++func TestMWSDialContextUsesValidatedIP(t *testing.T) {+ dialer := &recordingMWSDialer{}+ ctx := context.WithValue(context.Background(), mwsDialTargetKey{}, mwsDialTarget{+ hostname: mwsAPIHost,+ resolvedIP: "203.0.113.10",+ })+ _, err := newMWSDialContext(dialer)(ctx, "tcp", mwsAPIHost+":443")+ if err == nil || err.Error() != "recorded dial" {+ t.Fatalf("expected the dialer to be called, got %v", err)+ }+ if dialer.network != "tcp" || dialer.address != "203.0.113.10:443" {+ t.Fatalf("dial target = %s %s, want tcp 203.0.113.10:443", dialer.network, dialer.address)+ }++ dialer.address = ""+ if _, err = newMWSDialContext(dialer)(ctx, "tcp", "attacker.example:443"); err == nil {+ t.Fatal("expected an unvalidated host to be rejected")+ }+ if dialer.address != "" {+ t.Fatalf("unvalidated host reached dialer: %s", dialer.address)+ }+}++func TestMWSHTTPClientDoesNotFollowRedirects(t *testing.T) {+ var requests atomic.Int32+ server := httptest.NewServer(http.HandlerFunc(func(response http.ResponseWriter, request *http.Request) {+ requests.Add(1)+ http.Redirect(response, request, "https://attacker.example/collect", http.StatusFound)+ }))+ defer server.Close()++ driver, _ := newMWSTestDriver(server.URL)+ request, err := http.NewRequest(http.MethodGet, "https://gpt.mwsapis.ru/projects/demo/openai/v1/models", nil)+ if err != nil {+ t.Fatal(err)+ }+ response, err := driver.baseModel.httpClient.Do(request)+ if err != nil {+ t.Fatalf("request should return the redirect response without following it: %v", err)+ }+ defer response.Body.Close()+ if response.StatusCode != http.StatusFound {+ t.Fatalf("status = %d, want %d", response.StatusCode, http.StatusFound)+ }+ if got := requests.Load(); got != 1 {+ t.Fatalf("server request count = %d, want 1", got)+ }}func TestMWSListModelsUsesOpenAIEndpointAndFiltersTypes(t *testing.T) {@@ -309,9 +432,16 @@ func TestMWSErrorResponseIsReturned(t *testing.T) {driver, apiConfig := newMWSTestDriver(server.URL)modelName := "bge-m3"_, err := driver.Embed(t.Context(), &modelName, EmbedRequest{Texts: []string{"text"}}, apiConfig, nil, nil)- if err == nil || !strings.Contains(err.Error(), "status 503") || !strings.Contains(err.Error(), "MWS unavailable") {+ if err == nil || !strings.Contains(err.Error(), "status 503") || strings.Contains(err.Error(), "MWS unavailable") {t.Fatalf("unexpected MWS error: %v", err)}+ var statusErr *APIStatusError+ if !errors.As(err, &statusErr) || statusErr.Status != http.StatusServiceUnavailable {+ t.Fatalf("expected status error, got %T: %v", err, err)+ }+ if !strings.Contains(statusErr.Body, "MWS unavailable") {+ t.Fatalf("error body was not retained for internal diagnostics: %q", statusErr.Body)+ }}func TestMWSFactoryRegistration(t *testing.T) {