RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
fix(agent): normalize Invoke proxy URLs and avoid panics (#21855)
adws3269b6a88fd8
diff --git a/internal/agent/component/invoke.go b/internal/agent/component/invoke.goindex ad23743a2b..590577efe9 100644--- a/internal/agent/component/invoke.go+++ b/internal/agent/component/invoke.go@@ -150,11 +150,12 @@ func (i *InvokeComponent) Invoke(ctx context.Context, db *gorm.DB, inputs map[st// Step 2: SSRF guard for the proxy URL (if configured).// Mirrors the Python assert_url_is_safe(proxy_url) check.var (- proxyHost string- proxyIP string+ proxyURL *url.URL+ proxyIP string)proxyStr, _ := inputs["proxy"].(string)if proxyStr != "" {+ proxyStr = strings.TrimSpace(proxyStr)// Fail-closed target check (PR #15426 round-2 review).//// When a proxy is configured, Go dials the proxy host and then@@ -181,11 +182,15 @@ func (i *InvokeComponent) Invoke(ctx context.Context, db *gorm.DB, inputs map[streturn invokeSSRFError("url", rawURL,fmt.Errorf("Invoke: proxy mode requires a literal-IP target URL (hostnames are unsafe because the proxy re-resolves them)")), nil}- ph, pip, perr := validateInvokeURL(proxyStr)- if perr != nil {- return invokeSSRFError("proxy", proxyStr, perr), nil+ proxyURL, err = url.Parse(proxyStr)+ if err != nil {+ // Parse errors include the raw URL, which may contain credentials.+ return invokeSSRFError("proxy", proxyStr, errors.New("invalid proxy URL")), nil+ }+ _, proxyIP, err = validateInvokeURL(proxyStr)+ if err != nil {+ return invokeSSRFError("proxy", proxyStr, err), nil}- proxyHost, proxyIP = ph, pip}timeout := defaultInvokeTimeout@@ -281,8 +286,7 @@ func (i *InvokeComponent) Invoke(ctx context.Context, db *gorm.DB, inputs map[st// different answer (TOCTOU). The validated IP is captured// from the proxy SSRF check above (proxyIP).var client *http.Client- if proxyStr != "" {- proxyURL := mustParseProxy(proxyStr)+ if proxyURL != nil {pinnedProxyDialer := &net.Dialer{Timeout: timeout,KeepAlive: 30 * time.Second,@@ -322,9 +326,6 @@ func (i *InvokeComponent) Invoke(ctx context.Context, db *gorm.DB, inputs map[stpinned.CheckRedirect = noRedirectsclient = pinned}- _ = proxyHost- _ = proxyIP-// a generic HTTP client node in the canvas DSL — operators wire it// to arbitrary endpoints. SSRF surface is limited to operators// (not end users), and outbound traffic is rate-limited by the@@ -517,24 +518,6 @@ func (i *InvokeComponent) Outputs() map[string]string {}}-// mustParseProxy parses a proxy URL string. We keep this helper here-// (rather than calling url.Parse inline) so the panic-on-bad-input-// behavior is uniform across the package — proxy strings are operator--// configured, a malformed one is a deployment error worth crashing-// loud on.-func mustParseProxy(raw string) *url.URL {- u, err := url.Parse(raw)- if err != nil {- panic(fmt.Sprintf("Invoke: invalid proxy URL %q: %v", raw, err))- }- // Defensive check: net/http.ProxyURL will silently no-op on a- // URL with no Host. Surface a clear panic instead.- if u.Host == "" {- panic(fmt.Sprintf("Invoke: proxy URL %q has no host", raw))- }- return u-}-// stripHTMLTags removes HTML tags from the input string. This is a// best-effort implementation — it uses a simple regexp to remove// everything between < and >. It is NOT a full HTML sanitizer anddiff --git a/internal/agent/component/invoke_test.go b/internal/agent/component/invoke_test.goindex f759ff7c6f..ee0c58fefb 100644--- a/internal/agent/component/invoke_test.go+++ b/internal/agent/component/invoke_test.go@@ -27,6 +27,7 @@ import ("ragflow/internal/common""reflect""strings"+ "sync/atomic""testing""time"@@ -521,22 +522,158 @@ func TestInvoke_ProxyDNSPin(t *testing.T) {}t.Cleanup(func() { common.LookupHost = originalLookup })- c, _ := NewInvokeComponent(nil)- ctx, cancel := context.WithTimeout(t.Context(), 2*time.Second)- defer cancel()- _, err = c.Invoke(ctx, nil, map[string]any{- "method": "GET",- "url": "http://8.8.8.8/api",- "proxy": "http://proxy.test.invalid:" + proxyPort,- "timeout": 2,- })- if err != nil {- t.Fatalf("Invoke: %v", err)+ for _, tc := range []struct {+ name string+ prefix string+ suffix string+ }{+ {name: "plain"},+ {name: "spaces", prefix: " ", suffix: " "},+ {name: "tabs and newlines", prefix: "\t", suffix: "\r\n"},+ {name: "unicode whitespace", prefix: "\u3000", suffix: "\u00a0"},+ } {+ t.Run(tc.name, func(t *testing.T) {+ c, err := NewInvokeComponent(map[string]any{+ "method": "GET",+ "url": "http://8.8.8.8/api",+ "proxy": tc.prefix + "http://proxy.test.invalid:" + proxyPort + tc.suffix,+ "timeout": 2,+ })+ if err != nil {+ t.Fatal(err)+ }+ ctx, cancel := context.WithTimeout(t.Context(), 2*time.Second)+ defer cancel()+ out, err := c.Invoke(ctx, nil, nil)+ if err != nil {+ t.Fatalf("Invoke: %v", err)+ }+ if out["_ERROR"] != nil || out["result"] != "" {+ t.Fatalf("output = %#v, want empty proxy response", out)+ }+ select {+ case <-proxyHit:+ case <-time.After(time.Second):+ t.Fatal("fake proxy was not hit; proxy dial was not pinned to validated IP")+ }+ })+ }+}++func TestInvokeBlankProxy(t *testing.T) {+ setupAllowAnyHost(t, true)+ var requests atomic.Int32+ srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ requests.Add(1)+ _, _ = io.WriteString(w, "direct response")+ }))+ defer srv.Close()++ for _, proxy := range []string{"", " ", "\t\r\n", "\u3000\u00a0"} {+ t.Run(fmt.Sprintf("%q", proxy), func(t *testing.T) {+ before := requests.Load()+ c, err := NewInvokeComponent(map[string]any{+ "method": "GET",+ "url": srv.URL,+ "proxy": proxy,+ })+ if err != nil {+ t.Fatal(err)+ }+ out, err := c.Invoke(t.Context(), nil, nil)+ if err != nil {+ t.Fatalf("Invoke: %v", err)+ }+ if proxy == "" {+ if out["result"] != "direct response" || out["_ERROR"] != nil {+ t.Fatalf("output = %#v, want direct response", out)+ }+ return+ }+ if out["_ERROR"] != "URL not valid" || out["result"] != nil {+ t.Errorf("output = %#v, want rejected proxy", out)+ }+ if requests.Load() != before {+ t.Error("whitespace-only proxy sent a direct request")+ }+ })+ }+}++func TestInvokeInvalidProxy(t *testing.T) {+ setupAllowAnyHost(t, false)+ for _, proxy := range []string{+ "http://%",+ "http://8.8.4.4:invalid",+ "http://8.8.4.4:\t3128",+ "http:///proxy",+ "file://8.8.4.4/proxy",+ " http://127.0.0.1:3128 ",+ } {+ t.Run(fmt.Sprintf("%q", proxy), func(t *testing.T) {+ c, err := NewInvokeComponent(map[string]any{+ "method": "GET",+ "url": "http://8.8.8.8/api",+ "proxy": proxy,+ })+ if err != nil {+ t.Fatal(err)+ }+ out, err := c.Invoke(t.Context(), nil, nil)+ if err != nil {+ t.Fatalf("Invoke: want _ERROR output, got Go error: %v", err)+ }+ if out["_ERROR"] != "URL not valid" || out["result"] != nil {+ t.Fatalf("output = %#v, want rejected proxy", out)+ }+ })}- select {- case <-proxyHit:- case <-time.After(time.Second):- t.Fatal("fake proxy was not hit; proxy dial was not pinned to validated IP")+}++func TestInvokeInvalidProxyDoesNotLogCredentials(t *testing.T) {+ setupAllowAnyHost(t, false)+ for _, tc := range []struct {+ name string+ proxy string+ }{+ {+ name: "invalid port",+ proxy: "http://proxy-user:[email protected]:invalid/private-path?token=proxy-token",+ },+ {+ name: "control character",+ proxy: "http://proxy-user:[email protected]/private-path\t?token=proxy-token",+ },+ } {+ t.Run(tc.name, func(t *testing.T) {+ previous := zap.L()+ core, logs := observer.New(zap.WarnLevel)+ zap.ReplaceGlobals(zap.New(core))+ t.Cleanup(func() { zap.ReplaceGlobals(previous) })++ c, err := NewInvokeComponent(map[string]any{+ "method": "GET",+ "url": "http://8.8.8.8/api",+ "proxy": tc.proxy,+ })+ if err != nil {+ t.Fatal(err)+ }+ out, err := c.Invoke(t.Context(), nil, nil)+ if err != nil || out["_ERROR"] != "URL not valid" || out["result"] != nil {+ t.Fatalf("Invoke = %#v, %v; want rejected proxy", out, err)+ }+ if logs.Len() != 1 {+ t.Fatalf("log count = %d, want one proxy validation warning", logs.Len())+ }+ for _, entry := range logs.All() {+ for _, secret := range []string{"proxy-user", "proxy-password", "private-path", "proxy-token"} {+ if strings.Contains(entry.Message, secret) || strings.Contains(fmt.Sprint(entry.ContextMap()), secret) {+ t.Errorf("log contains proxy credentials or path/query: %#v", entry)+ }+ }+ }+ })}}