RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
fix(go): harden connector SSRF (#19694)
mkaaad59a103073e60
diff --git a/internal/syncer/connector/bitbucket.go b/internal/syncer/connector/bitbucket.goindex 4412c3c997..eeb5a41d2d 100644--- a/internal/syncer/connector/bitbucket.go+++ b/internal/syncer/connector/bitbucket.go@@ -29,8 +29,6 @@ import ("strconv""strings""time"-- "ragflow/internal/utility")const (@@ -402,18 +400,16 @@ func (c *BitbucketConnector) getJSONWithRetry(ctx context.Context, apiURL string// get performs one SSRF-safe authenticated GET request.func (c *BitbucketConnector) get(ctx context.Context, apiURL string) (*http.Response, error) {- hostname, resolvedIP, err := utility.AssertURLSafe(apiURL)- if err != nil {- return nil, err- }- req, err := http.NewRequestWithContext(ctx, http.MethodGet, apiURL, nil)+ resp, err := connectorRequest(ctx, connectorRequestOptions{+ Method: http.MethodGet,+ RawURL: apiURL,+ Headers: map[string]string{"Accept": "application/json", "Authorization": "Basic " + basicAuthHeader(c.email, c.apiToken)},+ Timeout: bitbucketRequestTimeout,+ })if err != nil {- return nil, err+ return nil, connectorUnsafeErr(err)}- req.Header.Set("Accept", "application/json")- req.SetBasicAuth(c.email, c.apiToken)- client := utility.PinnedHTTPClient(hostname, resolvedIP, bitbucketRequestTimeout)- return client.Do(req)+ return resp, nil}func sleepFor(ctx context.Context, duration time.Duration) error {diff --git a/internal/syncer/connector/confluence.go b/internal/syncer/connector/confluence.goindex a8c025ee3d..9d665b04b7 100644--- a/internal/syncer/connector/confluence.go+++ b/internal/syncer/connector/confluence.go@@ -113,6 +113,9 @@ func (c *ConfluenceConnector) Validate(ctx context.Context) error {if parsed.Scheme != "http" && parsed.Scheme != "https" {return fmt.Errorf("Confluence wiki_base must use HTTP or HTTPS")}+ if err := validateConnectorURL(c.wikiBase); err != nil {+ return err+ }if c.accessToken == "" {return fmt.Errorf("Confluence access token is required")}@@ -419,17 +422,19 @@ func (c *ConfluenceConnector) do(ctx context.Context, method, rawURL string) ([]if err != nil {return nil, err}- req, err := http.NewRequestWithContext(ctx, method, resolved, nil)- if err != nil {- return nil, err- }- req.Header.Set("Accept", "application/json")+ headers := map[string]string{"Accept": "application/json"}if c.isCloud || c.username != "" {- req.SetBasicAuth(c.username, c.accessToken)+ headers["Authorization"] = "Basic " + basicAuthHeader(c.username, c.accessToken)} else {- req.Header.Set("Authorization", "Bearer "+c.accessToken)- }- res, err := c.client.Do(req)+ headers["Authorization"] = "Bearer " + c.accessToken+ }+ res, err := connectorRequest(ctx, connectorRequestOptions{+ Method: method,+ RawURL: resolved,+ Headers: headers,+ Timeout: confluenceRequestTimeout,+ Base: c.client,+ })if err != nil {return nil, err}diff --git a/internal/syncer/connector/confluence_test.go b/internal/syncer/connector/confluence_test.goindex 5765981077..a394bbbb3b 100644--- a/internal/syncer/connector/confluence_test.go+++ b/internal/syncer/connector/confluence_test.go@@ -199,6 +199,7 @@ func newTestConfluenceConnector(t *testing.T, wikiBase string, overrides map[strfunc newConfluenceFixtureServer(t *testing.T, validateStatus int) *httptest.Server {t.Helper()+ withConnectorLoopbackTestHook(t)server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {if validateStatus != http.StatusOK && strings.HasPrefix(r.URL.Path, "/wiki/rest/api/space") {http.Error(w, "unauthorized", validateStatus)diff --git a/internal/syncer/connector/github.go b/internal/syncer/connector/github.goindex 3bdc29e181..47397033da 100644--- a/internal/syncer/connector/github.go+++ b/internal/syncer/connector/github.go@@ -29,8 +29,6 @@ import ("strconv""strings""time"-- "ragflow/internal/utility")const (@@ -385,20 +383,22 @@ func (c *GitHubConnector) getJSON(ctx context.Context, apiURL string, out any) (if c.doJSON != nil {return c.doJSON(ctx, apiURL, out)}- hostname, resolvedIP, err := utility.AssertURLSafe(apiURL)- if err != nil {- return nil, err- }- client := utility.PinnedHTTPClient(hostname, resolvedIP, githubRequestTimeout)- req, err := http.NewRequestWithContext(ctx, http.MethodGet, apiURL, nil)- if err != nil {- return nil, err- }- req.Header.Set("Accept", "application/vnd.github+json")- req.Header.Set("X-GitHub-Api-Version", "2022-11-28")- req.Header.Set("Authorization", "Bearer "+c.token)- resp, err := client.Do(req)+ resp, err := connectorRequest(ctx, connectorRequestOptions{+ Method: http.MethodGet,+ RawURL: apiURL,+ Validate: assertConnectorURLSafeHTTPS,+ Headers: map[string]string{+ "Accept": "application/vnd.github+json",+ "X-GitHub-Api-Version": "2022-11-28",+ "Authorization": "Bearer " + c.token,+ },+ Timeout: githubRequestTimeout,+ })if err != nil {+ var unsafe *connectorUnsafeURLError+ if errors.As(err, &unsafe) {+ return nil, unsafe.Err+ }return nil, fmt.Errorf("failed to fetch GitHub API: %w", err)}defer resp.Body.Close()diff --git a/internal/syncer/connector/gitlab.go b/internal/syncer/connector/gitlab.goindex 44eb79ce09..edc04ed76b 100644--- a/internal/syncer/connector/gitlab.go+++ b/internal/syncer/connector/gitlab.go@@ -19,6 +19,7 @@ package connectorimport ("context""encoding/json"+ "errors""fmt""io""net/http"@@ -27,8 +28,6 @@ import ("strconv""strings""time"-- "ragflow/internal/utility")const (@@ -298,19 +297,18 @@ func (c *GitlabConnector) getJSON(ctx context.Context, apiURL string, out any) (if c.doJSON != nil {return c.doJSON(ctx, apiURL, out)}- hostname, resolvedIP, err := utility.AssertURLSafe(apiURL)- if err != nil {- return nil, err- }- client := utility.PinnedHTTPClient(hostname, resolvedIP, gitlabRequestTimeout)- req, err := http.NewRequestWithContext(ctx, http.MethodGet, apiURL, nil)- if err != nil {- return nil, err- }- req.Header.Set("Accept", "application/json")- req.Header.Set("PRIVATE-TOKEN", c.token)- resp, err := client.Do(req)+ resp, err := connectorRequest(ctx, connectorRequestOptions{+ Method: http.MethodGet,+ RawURL: apiURL,+ Validate: assertConnectorURLSafeHTTPS,+ Headers: map[string]string{"Accept": "application/json", "PRIVATE-TOKEN": c.token},+ Timeout: gitlabRequestTimeout,+ })if err != nil {+ var unsafe *connectorUnsafeURLError+ if errors.As(err, &unsafe) {+ return nil, unsafe.Err+ }return nil, fmt.Errorf("failed to fetch GitLab API: %w", err)}defer resp.Body.Close()@@ -329,18 +327,18 @@ func (c *GitlabConnector) getRaw(ctx context.Context, apiURL string) ([]byte, erif c.doRaw != nil {return c.doRaw(ctx, apiURL)}- hostname, resolvedIP, err := utility.AssertURLSafe(apiURL)- if err != nil {- return nil, err- }- client := utility.PinnedHTTPClient(hostname, resolvedIP, gitlabRequestTimeout)- req, err := http.NewRequestWithContext(ctx, http.MethodGet, apiURL, nil)- if err != nil {- return nil, err- }- req.Header.Set("PRIVATE-TOKEN", c.token)- resp, err := client.Do(req)+ resp, err := connectorRequest(ctx, connectorRequestOptions{+ Method: http.MethodGet,+ RawURL: apiURL,+ Validate: assertConnectorURLSafeHTTPS,+ Headers: map[string]string{"PRIVATE-TOKEN": c.token},+ Timeout: gitlabRequestTimeout,+ })if err != nil {+ var unsafe *connectorUnsafeURLError+ if errors.As(err, &unsafe) {+ return nil, unsafe.Err+ }return nil, fmt.Errorf("failed to fetch GitLab raw file: %w", err)}defer resp.Body.Close()diff --git a/internal/syncer/connector/imap.go b/internal/syncer/connector/imap.goindex d19d430b08..92d2bbd770 100644--- a/internal/syncer/connector/imap.go+++ b/internal/syncer/connector/imap.go@@ -813,7 +813,15 @@ func runIMAPCommand[T any](ctx context.Context, client *imapclient.Client, run f}func dialRealIMAPClient(ctx context.Context, host string, port int, username, password string) (imapClient, error) {- address := net.JoinHostPort(host, strconv.Itoa(port))+ // The configured host is server-side controlled, so it is guarded by the+ // shared host-type SSRF check and the TCP dial is pinned to the validated+ // public IP. The TLS layer still presents the original hostname as SNI so+ // certificate verification is unchanged.+ pinIP, err := assertConnectorHostSafe(host)+ if err != nil {+ return nil, err+ }+ address := net.JoinHostPort(pinIP.String(), strconv.Itoa(port))dialCtx, cancelDial := context.WithTimeout(ctx, imapDialTimeout)defer cancelDial()rawConn, err := (&net.Dialer{}).DialContext(dialCtx, "tcp", address)diff --git a/internal/syncer/connector/jira.go b/internal/syncer/connector/jira.goindex 73caf58a6f..71a23407db 100644--- a/internal/syncer/connector/jira.go+++ b/internal/syncer/connector/jira.go@@ -17,7 +17,6 @@package connectorimport (- "bytes""context""encoding/base64""encoding/json"@@ -112,7 +111,6 @@ func NewJiraConnector(config map[string]any) (*JiraConnector, error) {commentEmailBlacklist: jiraStringSet(jiraStringSlice(config["comment_email_blacklist"])),client: &http.Client{Timeout: jiraRequestTimeout},}- c.client.CheckRedirect = c.checkRedirectif len(c.labelsToSkip) == 0 {c.labelsToSkip = jiraStringSet(strings.Split(os.Getenv("JIRA_CONNECTOR_LABELS_TO_SKIP"), ","))}@@ -133,6 +131,9 @@ func (c *JiraConnector) Validate(ctx context.Context) error {if err != nil || parsed.Scheme == "" || parsed.Host == "" {return &ConnectorValidationError{Message: "invalid Jira base URL"}}+ if err := validateConnectorURL(c.baseURL); err != nil {+ return &ConnectorValidationError{Message: err.Error()}+ }if c.apiToken == "" && (c.userEmail == "" || c.password == "") {return &ConnectorMissingCredentialError{Message: "Jira credentials must include either an API token or username/password."}}@@ -484,29 +485,33 @@ func (c *JiraConnector) syncTimezoneFromServer(ctx context.Context) error {func (c *JiraConnector) doJiraJSON(ctx context.Context, method, apiPath string, query url.Values, body any, out any) (http.Header, error) {apiURL := c.apiURL(apiPath, query)- var reader io.Reader+ var data []byteif body != nil {- data, err := json.Marshal(body)+ var err error+ data, err = json.Marshal(body)if err != nil {return nil, err}- reader = bytes.NewReader(data)}- req, err := http.NewRequestWithContext(ctx, method, apiURL, reader)- if err != nil {- return nil, err- }- c.authorize(req)+ headers := map[string]string{"Accept": "application/json"}if body != nil {- req.Header.Set("Content-Type", "application/json")- }- req.Header.Set("Accept", "application/json")- resp, err := c.client.Do(req)+ headers["Content-Type"] = "application/json"+ }+ resp, err := connectorRequest(ctx, connectorRequestOptions{+ Method: method,+ RawURL: apiURL,+ Body: data,+ Headers: headers,+ Timeout: jiraRequestTimeout,+ Base: c.client,+ Prepare: func(req *http.Request, _ connectorRequestHop) error { c.authorize(req); return nil },+ CheckRedirect: c.checkRedirect,+ })if err != nil {return nil, err}defer resp.Body.Close()- data, err := io.ReadAll(io.LimitReader(resp.Body, 32*1024*1024))+ data, err = io.ReadAll(io.LimitReader(resp.Body, 32*1024*1024))if err != nil {return resp.Header, err}@@ -534,12 +539,14 @@ func (c *JiraConnector) downloadURL(ctx context.Context, rawURL string) ([]byte,if !sameJiraOrigin(parsed, base) {return nil, fmt.Errorf("Jira attachment origin %q does not match the configured base URL", parsed.Scheme+"://"+parsed.Host)}- req, err := http.NewRequestWithContext(ctx, http.MethodGet, rawURL, nil)- if err != nil {- return nil, err- }- c.authorize(req)- resp, err := c.client.Do(req)+ resp, err := connectorRequest(ctx, connectorRequestOptions{+ Method: http.MethodGet,+ RawURL: rawURL,+ Timeout: jiraRequestTimeout,+ Base: c.client,+ Prepare: func(req *http.Request, _ connectorRequestHop) error { c.authorize(req); return nil },+ CheckRedirect: c.checkRedirect,+ })if err != nil {return nil, err}diff --git a/internal/syncer/connector/jira_test.go b/internal/syncer/connector/jira_test.goindex 8f160f0995..3a2da7b4fa 100644--- a/internal/syncer/connector/jira_test.go+++ b/internal/syncer/connector/jira_test.go@@ -260,6 +260,7 @@ func TestJiraConnectorBuildJQLIgnoresQuotedOrderBy(t *testing.T) {}func TestJiraConnectorDownloadRejectsSchemeMismatch(t *testing.T) {+ withConnectorLoopbackTestHook(t)server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {t.Fatalf("request should not be sent for mismatched scheme")}))@@ -283,6 +284,7 @@ func TestJiraConnectorDownloadRejectsSchemeMismatch(t *testing.T) {}func TestJiraConnectorDownloadRejectsCrossOriginRedirect(t *testing.T) {+ withConnectorLoopbackTestHook(t)destinationHit := falsedestination := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {destinationHit = true@@ -313,6 +315,7 @@ func TestJiraConnectorDownloadRejectsCrossOriginRedirect(t *testing.T) {}func TestJiraConnectorDownloadKeepsAuthorizationOnSameOriginRedirect(t *testing.T) {+ withConnectorLoopbackTestHook(t)var server *httptest.Servermux := http.NewServeMux()mux.HandleFunc("/redirect", func(w http.ResponseWriter, r *http.Request) {@@ -349,6 +352,7 @@ func TestJiraConnectorDownloadKeepsAuthorizationOnSameOriginRedirect(t *testing.func jiraFixtureServer(t *testing.T) *httptest.Server {t.Helper()+ withConnectorLoopbackTestHook(t)mux := http.NewServeMux()mux.HandleFunc("/rest/api/3/project/RAG", func(w http.ResponseWriter, r *http.Request) {jiraWriteJSON(t, w, map[string]any{"key": "RAG"})@@ -375,6 +379,7 @@ func jiraFixtureServer(t *testing.T) *httptest.Server {func jiraResumeFixtureServer(t *testing.T, keys []map[string]any) *httptest.Server {t.Helper()+ withConnectorLoopbackTestHook(t)var server *httptest.Servermux := http.NewServeMux()mux.HandleFunc("/rest/api/3/project/RAG", func(w http.ResponseWriter, r *http.Request) {diff --git a/internal/syncer/connector/moodle.go b/internal/syncer/connector/moodle.goindex 4ccc81aec0..adcc3bf5f0 100644--- a/internal/syncer/connector/moodle.go+++ b/internal/syncer/connector/moodle.go@@ -17,7 +17,6 @@package connectorimport (- "bytes""context""encoding/json""fmt"@@ -888,166 +887,44 @@ func capitalizeMoodleType(mtype string) string {// ---------------------------------------------------------------------------func validateMoodleURLForSSRF(rawURL string) error {- parsed, err := url.Parse(rawURL)- if err != nil {- return &ConnectorValidationError{Message: "Moodle URL must include a hostname."}- }- if parsed.Scheme != "http" && parsed.Scheme != "https" {- return &ConnectorValidationError{Message: fmt.Sprintf("Unsupported URL scheme for Moodle connector: %q. Only http/https are allowed.", parsed.Scheme)}- }- hostname := parsed.Hostname()- if hostname == "" {- return &ConnectorValidationError{Message: "Moodle URL must include a hostname."}- }- if strings.EqualFold(hostname, "localhost") {- return &ConnectorValidationError{Message: fmt.Sprintf("Moodle URL hostname %q is not allowed (localhost is blocked).", hostname)}- }- addrs, err := net.LookupIP(hostname)- if err != nil {- // Resolution failure is not an SSRF condition by itself; the- // per-request check surfaces it if it matters.- return nil- }- if restAPISSRFAllowLoopback {- allLoopback := true- for _, addr := range addrs {- if !addr.IsLoopback() {- allLoopback = false- break- }- }- if allLoopback {- return nil- }- // Not all loopback — fall through to normal validation.- }- for _, addr := range addrs {- if !restAPIIPIsGlobal(restAPIEffectiveIP(addr)) {- return &ConnectorValidationError{Message: fmt.Sprintf(- "Moodle URL %q resolves to disallowed address %s (localhost, private, link-local, reserved, or multicast addresses are blocked).",- rawURL, addr)}- }- }- return nil+ return validateConnectorURL(rawURL)}// moodleAssertURLSafe validates a per-request URL for SSRF, HTTPS, and// same-origin policy. It returns the hostname and first validated IP so the// caller can pin DNS for the actual dial.func moodleAssertURLSafe(ctx context.Context, rawURL, originURL string) (string, net.IP, error) {- parsed, err := url.Parse(rawURL)- if err != nil {- return "", nil, fmt.Errorf("Moodle URL is missing a host.")- }- if parsed.Scheme != "http" && parsed.Scheme != "https" {- return "", nil, fmt.Errorf("Disallowed URL scheme: %q. Only [http https] are allowed.", parsed.Scheme)- }- hostname := parsed.Hostname()- if hostname == "" {- return "", nil, fmt.Errorf("Moodle URL is missing a host.")- }// Reject cross-origin targets: every request must go to the configured// Moodle host.if originURL != "" {if origin, err := url.Parse(originURL); err == nil && origin.Hostname() != "" {- if !strings.EqualFold(hostname, origin.Hostname()) {- return "", nil, fmt.Errorf("Moodle URL host %q does not match configured origin %q.", hostname, origin.Hostname())+ parsed, err := url.Parse(rawURL)+ if err != nil {+ return "", nil, fmt.Errorf("Moodle URL is missing a host.")}- }- }- addrs, err := net.DefaultResolver.LookupIPAddr(ctx, hostname)- if err != nil {- return "", nil, fmt.Errorf("Could not resolve hostname %q: %w", hostname, err)- }- if len(addrs) == 0 {- return "", nil, fmt.Errorf("Hostname %q resolved to no addresses.", hostname)- }- if restAPISSRFAllowLoopback {- allLoopback := true- for _, addr := range addrs {- if !addr.IP.IsLoopback() {- allLoopback = false- break+ if !strings.EqualFold(parsed.Hostname(), origin.Hostname()) {+ return "", nil, fmt.Errorf("Moodle URL host %q does not match configured origin %q.", parsed.Hostname(), origin.Hostname())}}- if allLoopback {- return hostname, addrs[0].IP, nil- }- // Not all loopback — fall through to normal validation.}- var first net.IP- for _, addr := range addrs {- if !restAPIIPIsGlobal(restAPIEffectiveIP(addr.IP)) {- return "", nil, fmt.Errorf("Moodle URL resolves to a non-public address (%s), which is not allowed.", addr.IP)- }- if first == nil {- first = addr.IP- }- }- if first == nil {- return "", nil, fmt.Errorf("Hostname %q resolved to no addresses.", hostname)- }- return hostname, first, nil+ return assertConnectorURLSafe(rawURL)}// moodleHTTPDo sends an HTTP request with DNS-pinned transport and manual// redirect handling. Each hop is independently validated for SSRF, HTTPS, and// same-origin policy, preventing DNS rebinding and redirect-based bypasses.func (c *MoodleConnector) moodleHTTPDo(ctx context.Context, method, rawURL string, body []byte, headers map[string]string) (*http.Response, error) {- currentURL := rawURL- currentMethod := method- currentBody := body- for hop := 0; hop <= moodleMaxRedirects; hop++ {- hostname, pinIP, err := moodleAssertURLSafe(ctx, currentURL, c.moodleURL)- if err != nil {- return nil, err- }- transport := newRestAPIPinnedTransport(hostname, pinIP)- client := &http.Client{- Transport: transport,- Timeout: moodleRequestTimeout,- CheckRedirect: func(req *http.Request, via []*http.Request) error {- return http.ErrUseLastResponse- },- }- var bodyReader io.Reader- if currentBody != nil {- bodyReader = bytes.NewReader(currentBody)- }- req, err := http.NewRequestWithContext(ctx, currentMethod, currentURL, bodyReader)- if err != nil {- transport.CloseIdleConnections()- return nil, err- }- for k, v := range headers {- req.Header.Set(k, v)- }- resp, err := client.Do(req)- if err != nil {- transport.CloseIdleConnections()- return nil, err- }- if !restAPIIsRedirect(resp.StatusCode) {- resp.Body = &restAPICloseIdleBody{body: resp.Body, transport: transport}- return resp, nil- }- location := resp.Header.Get("Location")- resp.Body.Close()- transport.CloseIdleConnections()- if location == "" {- return nil, fmt.Errorf("Moodle redirect with empty Location header")- }- nextURL, err := restAPIResolveURL(currentURL, location)- if err != nil {- return nil, err- }- currentURL = nextURL- if resp.StatusCode == http.StatusMovedPermanently || resp.StatusCode == http.StatusFound || resp.StatusCode == http.StatusSeeOther {- currentMethod = http.MethodGet- currentBody = nil- }- }- return nil, fmt.Errorf("Moodle request stopped after %d redirects", moodleMaxRedirects)+ return connectorRequest(ctx, connectorRequestOptions{+ Method: method,+ RawURL: rawURL,+ Body: body,+ Headers: headers,+ Timeout: moodleRequestTimeout,+ MaxRedirects: moodleMaxRedirects,+ Validate: func(raw string) (string, net.IP, error) {+ return moodleAssertURLSafe(context.Background(), raw, c.moodleURL)+ },+ })}// moodleRedactedURL strips query and fragment from a URL so that tokens ordiff --git a/internal/syncer/connector/moodle_test.go b/internal/syncer/connector/moodle_test.goindex f0432ae206..a3b993e900 100644--- a/internal/syncer/connector/moodle_test.go+++ b/internal/syncer/connector/moodle_test.go@@ -14,16 +14,16 @@ import (func withMoodleTestHooks(t *testing.T) {t.Helper()- origLoopback := restAPISSRFAllowLoopback+ origLoopback := connectorAllowLoopbackForTestorigTries := moodleRetryTriesorigBaseDelay := moodleRetryBaseDelayorigBackoff := moodleRetryBackoff- restAPISSRFAllowLoopback = true+ connectorAllowLoopbackForTest = truemoodleRetryTries = 2moodleRetryBaseDelay = time.MillisecondmoodleRetryBackoff = 2t.Cleanup(func() {- restAPISSRFAllowLoopback = origLoopback+ connectorAllowLoopbackForTest = origLoopbackmoodleRetryTries = origTriesmoodleRetryBaseDelay = origBaseDelaymoodleRetryBackoff = origBackoff@@ -525,9 +525,9 @@ func TestAddMoodleToken(t *testing.T) {func TestValidateMoodleURLForSSRF(t *testing.T) {withMoodleTestHooks(t)- origLoopback := restAPISSRFAllowLoopback- restAPISSRFAllowLoopback = false- defer func() { restAPISSRFAllowLoopback = origLoopback }()+ origLoopback := connectorAllowLoopbackForTest+ connectorAllowLoopbackForTest = false+ defer func() { connectorAllowLoopbackForTest = origLoopback }()if err := validateMoodleURLForSSRF("ftp://example.com"); err == nil {t.Fatalf("expected scheme rejection")@@ -640,9 +640,9 @@ func TestMoodleConnectorMetadataRedactsFileURL(t *testing.T) {}func TestMoodleAssertURLSafeRejectsCrossOrigin(t *testing.T) {- origLoopback := restAPISSRFAllowLoopback- restAPISSRFAllowLoopback = false- defer func() { restAPISSRFAllowLoopback = origLoopback }()+ origLoopback := connectorAllowLoopbackForTest+ connectorAllowLoopbackForTest = false+ defer func() { connectorAllowLoopbackForTest = origLoopback }()_, _, err := moodleAssertURLSafe(context.Background(), "https://evil.com/api", "https://example.com")if err == nil {@@ -651,9 +651,9 @@ func TestMoodleAssertURLSafeRejectsCrossOrigin(t *testing.T) {}func TestMoodleAssertURLSafeLoopbackAllAddresses(t *testing.T) {- origLoopback := restAPISSRFAllowLoopback- restAPISSRFAllowLoopback = true- defer func() { restAPISSRFAllowLoopback = origLoopback }()+ origLoopback := connectorAllowLoopbackForTest+ connectorAllowLoopbackForTest = true+ defer func() { connectorAllowLoopbackForTest = origLoopback }()// All loopback → allowed._, _, err := moodleAssertURLSafe(context.Background(), "http://127.0.0.1/path", "http://127.0.0.1")diff --git a/internal/syncer/connector/mysql.go b/internal/syncer/connector/mysql.goindex 6e621183cd..098823cfdf 100644--- a/internal/syncer/connector/mysql.go+++ b/internal/syncer/connector/mysql.go@@ -24,7 +24,9 @@ import ("encoding/hex""fmt""io"+ "net""regexp"+ "strconv""strings""time"@@ -70,9 +72,11 @@ func NewMySQLConnector(config map[string]any) (*MySQLConnector, error) {batchSize: configInt(config["batch_size"], defaultMySQLBatchSize),username: strings.TrimSpace(stringConfig(credentials["username"])),password: stringConfig(credentials["password"]),- openDB: func(dsn string) (*sql.DB, error) {- return sql.Open("mysql", dsn)- },+ }+ // Production dials through the SSRF-guarded, DNS-pinned openDB. Tests+ // replace it with an injected openDB that avoids the real network.+ connector.openDB = func(dsn string) (*sql.DB, error) {+ return connector.openPinned(dsn)}connector.query = connector.sanitizeQuery(stringConfig(config["query"]))connector.contentColumns = connector.splitColumns(config["content_columns"])@@ -148,7 +152,10 @@ func (c *MySQLConnector) OpenPrune(ctx context.Context, request PruneRequest) (Preturn &mysqlPruneSession{connector: c, db: db, queries: queries, batchSize: c.batchSize}, nil}-// open builds a MySQL connection with Python-compatible settings.+// open builds a MySQL connection with Python-compatible settings. The default+// openDB (wired in NewMySQLConnector) validates the host against the shared+// SSRF guard and pins the dial; tests inject an openDB that bypasses the real+// network.func (c *MySQLConnector) open() (*sql.DB, error) {cfg := mysql.NewConfig()cfg.User = c.username@@ -162,6 +169,49 @@ func (c *MySQLConnector) open() (*sql.DB, error) {return c.openDB(cfg.FormatDSN())}+// openPinned is the production openDB: it validates the configured host with+// the shared host-type SSRF guard and routes the connection through a custom+// network whose dialer is pinned to the validated IP, closing the+// DNS-rebinding window between validation and the TCP connect. The DSN keeps+// the original hostname (TLS ServerName / host-based routing), while+// go-sql-driver resolves the custom network through mysql.RegisterDialContext.+func (c *MySQLConnector) openPinned(_ string) (*sql.DB, error) {+ pinIP, err := assertConnectorHostSafe(c.host)+ if err != nil {+ return nil, err+ }+ network := mysqlPinnedNetwork(c.host, c.port, pinIP)+ mysql.RegisterDialContext(network, mysqlPinnedDial(pinIP, c.port))+ cfg := mysql.NewConfig()+ cfg.User = c.username+ cfg.Passwd = c.password+ cfg.Net = network+ cfg.Addr = net.JoinHostPort(c.host, strconv.Itoa(c.port))+ cfg.DBName = c.database+ cfg.Params = map[string]string{"charset": "utf8mb4"}+ cfg.ParseTime = true+ cfg.Loc = time.UTC+ return sql.Open("mysql", cfg.FormatDSN())+}++// mysqlPinnedNetwork returns a deterministic custom network name for a+// host/port/IP pin. Each distinct pin gets its own registered network, so a DNS+// change can never make one connector reuse another connector's dialer. The+// name is hex-only so it round-trips through go-sql-driver's DSN parser.+func mysqlPinnedNetwork(host string, port int, pinIP net.IP) string {+ sum := md5.Sum([]byte(fmt.Sprintf("%s:%d:%s", host, port, pinIP.String())))+ return "pinned-" + hex.EncodeToString(sum[:8])+}++// mysqlPinnedDial returns a go-sql-driver dialer that connects every dial on+// its registered network to pinIP:port, ignoring the host the driver parsed+// from the DSN.+func mysqlPinnedDial(pinIP net.IP, port int) mysql.DialContextFunc {+ return func(ctx context.Context, _ string) (net.Conn, error) {+ return (&net.Dialer{}).DialContext(ctx, "tcp", net.JoinHostPort(pinIP.String(), strconv.Itoa(port)))+ }+}+// baseQueries returns the configured query or a SELECT per table.func (c *MySQLConnector) baseQueries(ctx context.Context, db *sql.DB) ([]string, error) {if c.query != "" {diff --git a/internal/syncer/connector/postgresql.go b/internal/syncer/connector/postgresql.goindex 70d6dcdf29..a581ee9268 100644--- a/internal/syncer/connector/postgresql.go+++ b/internal/syncer/connector/postgresql.go@@ -24,13 +24,15 @@ import ("encoding/hex""fmt""io"+ "net""net/url""regexp""strconv""strings""time"- _ "github.com/jackc/pgx/v5/stdlib"+ "github.com/jackc/pgx/v5"+ "github.com/jackc/pgx/v5/stdlib")const (@@ -80,9 +82,11 @@ func NewPostgreSQLConnector(config map[string]any) (*PostgreSQLConnector, error)password: stringConfig(credentials["password"]),sslmode: strings.TrimSpace(stringConfig(config["sslmode"])),connectTimeout: configInt(config["connect_timeout"], defaultPostgresConnectTimeout),- openDB: func(dsn string) (*sql.DB, error) {- return sql.Open("pgx", dsn)- },+ }+ // Production dials through the SSRF-guarded, DNS-pinned openDB. Tests+ // replace it with an injected openDB that avoids the real network.+ connector.openDB = func(dsn string) (*sql.DB, error) {+ return connector.openPinned(dsn)}if connector.sslmode == "" {connector.sslmode = "prefer"@@ -179,6 +183,35 @@ func (c *PostgreSQLConnector) open() (*sql.DB, error) {return c.openDB(dsn.String())}+// openPinned is the production openDB: it validates the configured host with+// the shared host-type SSRF guard and installs a pgx DialFunc pinned to the+// validated IP, closing the DNS-rebinding window between validation and the+// TCP connect. The DSN keeps the original hostname so TLS ServerName /+// host-based authentication are unchanged; only the underlying TCP dial is+// rewritten.+func (c *PostgreSQLConnector) openPinned(dsn string) (*sql.DB, error) {+ pinIP, err := assertConnectorHostSafe(c.host)+ if err != nil {+ return nil, err+ }+ config, err := pgx.ParseConfig(dsn)+ if err != nil {+ return nil, err+ }+ timeout := time.Duration(c.connectTimeout) * time.Second+ port := strconv.Itoa(int(config.Port))+ config.DialFunc = postgresPinnedDial(pinIP, port, timeout)+ return stdlib.OpenDB(*config), nil+}++// postgresPinnedDial returns a pgx DialFunc that connects every dial to+// pinIP:port, ignoring the host parsed from the DSN.+func postgresPinnedDial(pinIP net.IP, port string, timeout time.Duration) func(ctx context.Context, network, addr string) (net.Conn, error) {+ return func(ctx context.Context, network, _ string) (net.Conn, error) {+ return (&net.Dialer{Timeout: timeout}).DialContext(ctx, network, net.JoinHostPort(pinIP.String(), port))+ }+}+// baseQueries returns the configured query or a SELECT per table.func (c *PostgreSQLConnector) baseQueries(ctx context.Context, db *sql.DB) ([]string, error) {if c.query != "" {diff --git a/internal/syncer/connector/rest_api.go b/internal/syncer/connector/rest_api.goindex 6b3002c7de..77dfb3db42 100644--- a/internal/syncer/connector/rest_api.go+++ b/internal/syncer/connector/rest_api.go@@ -18,7 +18,6 @@ package connectorimport ("context"- "crypto/tls""encoding/json""errors""fmt"@@ -134,14 +133,10 @@ var (restAPI429DefaultWait = 30 * time.Second)-// restAPISSRFAllowLoopback is a test hook that lets unit tests exercise the-// real HTTP path against httptest servers bound to loopback. Production code-// keeps it false so loopback/private endpoints stay blocked.-var restAPISSRFAllowLoopback bool-// NewRestAPIConnector parses a connector config and returns a connector. It-// performs schema validation and the base-URL SSRF check but performs no-// network I/O. Credentials are read from config["credentials"].+// performs schema validation and the base-URL SSRF check (which resolves the+// hostname to enforce the shared guard) but performs no HTTP request I/O.+// Credentials are read from config["credentials"].func NewRestAPIConnector(config map[string]any) (*RestAPIConnector, error) {cfg, err := parseRestAPIConfig(config)if err != nil {@@ -627,160 +622,90 @@ func (c *RestAPIConnector) fetchPageOnce(ctx context.Context, params map[string]currentHeaders[k] = v}currentBody := c.cfg.RequestBody- currentAuth := c.basicAuth- previousNetloc := restAPINetloc(currentURL)-- for hop := 0; hop <= restAPIMaxRedirects; hop++ {- hostname, pinIP, err := assertRestAPIURLSafe(ctx, currentURL)- if err != nil {- return nil, &ConnectorValidationError{Message: "Unsafe REST API URL: " + err.Error()}+ var bodyBytes []byte+ if currentMethod == http.MethodPost {+ if currentBody == nil {+ bodyBytes = []byte("{}")+ } else {+ var err error+ bodyBytes, err = json.Marshal(currentBody)+ if err != nil {+ return nil, &ConnectorValidationError{Message: "REST API request body is not valid JSON"}+ }}-- var resp *http.Response- var respErr error- for wait := 0; wait < restAPI429MaxWaits; wait++ {- resp, respErr = c.doRequest(ctx, currentMethod, currentURL, currentHeaders, queryParams, currentBody, currentAuth, hostname, pinIP)- if respErr != nil {- return nil, respErr+ }+ currentAuth := c.basicAuth+ previousNetloc := connectorNetloc(currentURL)++ resp, err := connectorRequest(ctx, connectorRequestOptions{+ Method: currentMethod,+ RawURL: currentURL,+ Body: bodyBytes,+ Headers: currentHeaders,+ Timeout: restAPIRequestTimeout,+ MaxRedirects: restAPIMaxRedirects,+ Prepare: func(req *http.Request, hop connectorRequestHop) error {+ if hop.Method == http.MethodPost && req.Header.Get("Content-Type") == "" {+ req.Header.Set("Content-Type", "application/json")}- if resp.StatusCode != http.StatusTooManyRequests {- break+ if currentAuth != nil {+ req.SetBasicAuth(currentAuth.username, currentAuth.password)}- retryAfter := restAPI429DefaultWait- if raw := resp.Header.Get("Retry-After"); raw != "" {- if seconds, err := strconv.Atoi(strings.TrimSpace(raw)); err == nil && seconds >= 0 {- retryAfter = time.Duration(seconds) * time.Second+ if encoded := restAPIEncodeParams(queryParams); encoded != "" {+ sep := "?"+ if strings.Contains(hop.URL, "?") {+ sep = "&"}+ fullURL, err := url.Parse(hop.URL + sep + encoded)+ if err != nil {+ return err+ }+ req.URL = fullURL}- resp.Body.Close()- select {- case <-ctx.Done():- return nil, ctx.Err()- case <-time.After(retryAfter):+ return nil+ },+ RetryStatus: func(resp *http.Response, attempt int) (time.Duration, bool) {+ if resp.StatusCode != http.StatusTooManyRequests {+ return 0, false}- }- if respErr != nil {- return nil, respErr- }- if resp.StatusCode == http.StatusTooManyRequests {- resp.Body.Close()- return nil, &RateLimitTriedTooManyTimesError{Message: fmt.Sprintf("REST API rate limited: exceeded '%d' retries (too many requests)", restAPI429MaxWaits)}- }-- if restAPIIsRedirect(resp.StatusCode) {- location := resp.Header.Get("Location")- if location == "" {- return c.handleResponse(resp)+ if attempt >= restAPI429MaxWaits {+ return 0, false}- nextURL, err := restAPIResolveURL(currentURL, location)- if err != nil {- resp.Body.Close()- return nil, &ConnectorValidationError{Message: "Unsafe REST API URL: " + err.Error()}+ wait := restAPI429DefaultWait+ if raw := resp.Header.Get("Retry-After"); raw != "" {+ if seconds, err := strconv.Atoi(strings.TrimSpace(raw)); err == nil && seconds >= 0 {+ wait = time.Duration(seconds) * time.Second+ }}- nextNetloc := restAPINetloc(nextURL)+ return wait, true+ },+ NextHop: func(nextURL string, status int, hop connectorRequestHop) connectorRequestHop {+ nextNetloc := connectorNetloc(nextURL)if nextNetloc != "" && nextNetloc != previousNetloc {- currentHeaders = restAPIStripAuthHeaders(currentHeaders)currentAuth = nil}previousNetloc = nextNetloc- if resp.StatusCode == http.StatusMovedPermanently || resp.StatusCode == http.StatusFound || resp.StatusCode == http.StatusSeeOther {- currentMethod = "GET"- currentBody = nil+ if status == http.StatusMovedPermanently || status == http.StatusFound || status == http.StatusSeeOther {+ hop.Method = http.MethodGet+ hop.Body = nilqueryParams = nil}- resp.Body.Close()- currentURL = nextURL- continue- }- return c.handleResponse(resp)- }- return nil, &ConnectorValidationError{Message: fmt.Sprintf("Exceeded %d redirects fetching %q", restAPIMaxRedirects, currentURL)}-}--func (c *RestAPIConnector) doRequest(- ctx context.Context,- method, rawURL string,- headers map[string]string,- params map[string]any,- body map[string]any,- auth *restAPIBasicAuth,- hostname string,- pinIP net.IP,-) (*http.Response, error) {- transport := newRestAPIPinnedTransport(hostname, pinIP)- client := &http.Client{- Transport: transport,- Timeout: restAPIRequestTimeout,- // Redirects are handled manually so auth headers can be stripped on- // cross-origin hops and each hop is re-validated against SSRF.- CheckRedirect: func(req *http.Request, via []*http.Request) error {- return http.ErrUseLastResponse+ hop.URL = nextURL+ return hop},- }-- fullURL := rawURL- if encoded := restAPIEncodeParams(params); encoded != "" {- sep := "?"- if strings.Contains(rawURL, "?") {- sep = "&"- }- fullURL = rawURL + sep + encoded- }-- var req *http.Request- var err error- switch method {- case "GET":- req, err = http.NewRequestWithContext(ctx, http.MethodGet, fullURL, nil)- case "POST":- var payload []byte- if body == nil {- payload = []byte("{}")- } else {- payload, err = json.Marshal(body)- if err != nil {- return nil, &ConnectorValidationError{Message: "REST API request body is not valid JSON"}- }- }- req, err = http.NewRequestWithContext(ctx, http.MethodPost, fullURL, strings.NewReader(string(payload)))- if err == nil {- req.Header.Set("Content-Type", "application/json")- }- default:- return nil, &ConnectorValidationError{Message: fmt.Sprintf("Unsupported HTTP method: %s", method)}- }+ })if err != nil {+ var unsafe *connectorUnsafeURLError+ if errors.As(err, &unsafe) {+ return nil, &ConnectorValidationError{Message: "Unsafe REST API URL: " + unsafe.Err.Error()}+ }return nil, err}- for k, v := range headers {- req.Header.Set(k, v)- }- if auth != nil {- req.SetBasicAuth(auth.username, auth.password)- }- resp, err := client.Do(req)- if err != nil {- transport.CloseIdleConnections()- return nil, err+ if resp.StatusCode == http.StatusTooManyRequests {+ resp.Body.Close()+ return nil, &RateLimitTriedTooManyTimesError{Message: fmt.Sprintf("REST API rate limited: exceeded '%d' retries (too many requests)", restAPI429MaxWaits)}}- resp.Body = &restAPICloseIdleBody{body: resp.Body, transport: transport}- return resp, nil-}--// restAPICloseIdleBody closes the underlying response body and then releases-// the pinned transport's idle connections, so the per-request transports do-// not leak keep-alive sockets during long paginated syncs.-type restAPICloseIdleBody struct {- body io.ReadCloser- transport *http.Transport-}--func (b *restAPICloseIdleBody) Read(p []byte) (int, error) { return b.body.Read(p) }--func (b *restAPICloseIdleBody) Close() error {- err := b.body.Close()- b.transport.CloseIdleConnections()- return err+ return c.handleResponse(resp)}func (c *RestAPIConnector) handleResponse(resp *http.Response) (any, error) {@@ -810,26 +735,6 @@ func (c *RestAPIConnector) handleResponse(resp *http.Response) (any, error) {return value, nil}-// newRestAPIPinnedTransport resolves the target once (in assertRestAPIURLSafe)-// and pins that address for the actual connection, preventing DNS rebinding.-func newRestAPIPinnedTransport(hostname string, pinIP net.IP) *http.Transport {- dialer := &net.Dialer{Timeout: restAPIRequestTimeout, KeepAlive: 30 * time.Second}- return &http.Transport{- DialContext: func(ctx context.Context, network, addr string) (net.Conn, error) {- _, port, err := net.SplitHostPort(addr)- if err != nil {- port = "443"- }- return dialer.DialContext(ctx, network, net.JoinHostPort(pinIP.String(), port))- },- TLSClientConfig: &tls.Config{- ServerName: hostname,- MinVersion: tls.VersionTLS12,- },- ForceAttemptHTTP2: true,- }-}-// buildURLWithTemplates substitutes {key} placeholders in the URL and returns// the remaining query parameters.func (c *RestAPIConnector) buildURLWithTemplates(params map[string]any) (string, map[string]any) {@@ -877,176 +782,23 @@ func restAPIEncodeParams(params map[string]any) string {return values.Encode()}-func restAPINetloc(rawURL string) string {- parsed, err := url.Parse(rawURL)- if err != nil {- return ""- }- return parsed.Host-}--func restAPIResolveURL(base, location string) (string, error) {- baseParsed, err := url.Parse(base)- if err != nil {- return "", err- }- ref, err := url.Parse(location)- if err != nil {- return "", err- }- return baseParsed.ResolveReference(ref).String(), nil-}--func restAPIIsRedirect(status int) bool {- switch status {- case http.StatusMovedPermanently, http.StatusFound, http.StatusSeeOther,- http.StatusTemporaryRedirect, http.StatusPermanentRedirect:- return true- }- return false-}--var restAPIAuthSensitiveHeaders = map[string]struct{}{- "authorization": {},- "proxy-authorization": {},- "apikey": {},- "api-key": {},- "x-api-key": {},- "x-auth-token": {},-}--func restAPIStripAuthHeaders(headers map[string]string) map[string]string {- out := make(map[string]string, len(headers))- for k, v := range headers {- if _, sensitive := restAPIAuthSensitiveHeaders[strings.ToLower(k)]; sensitive {- continue- }- out[k] = v- }- return out-}-// ---------------------------------------------------------------------------// SSRF protection// ----------------------------------------------------------------------------// validateRestAPIURLForSSRF performs quick deny-list checks plus DNS-// resolution. Resolution failure is logged and tolerated because the-// per-request check re-validates.+// validateRestAPIURLForSSRF is the config-time SSRF check for the REST API+// connector. It delegates to the shared connector guard, which validates the+// scheme, rejects localhost and non-public literal addresses, and resolves the+// hostname. The per-request path re-validates and DNS-pins every hop.func validateRestAPIURLForSSRF(rawURL string) error {- parsed, err := url.Parse(rawURL)- if err != nil {- return &ConnectorValidationError{Message: "REST API connector URL must include a hostname."}- }- if parsed.Scheme != "http" && parsed.Scheme != "https" {- return &ConnectorValidationError{Message: fmt.Sprintf("Unsupported URL scheme for REST API connector: %q. Only http/https are allowed.", parsed.Scheme)}- }- hostname := parsed.Hostname()- if hostname == "" {- return &ConnectorValidationError{Message: "REST API connector URL must include a hostname."}- }- if strings.EqualFold(hostname, "localhost") {- return &ConnectorValidationError{Message: fmt.Sprintf("REST API connector URL hostname %q is not allowed (localhost is blocked).", hostname)}- }- if restAPISSRFAllowLoopback {- return nil- }- addrs, err := net.LookupIP(hostname)- if err != nil {- // DNS failure is not an SSRF condition by itself; the per-request- // check will surface it if it matters.- return nil- }- for _, addr := range addrs {- if !restAPIIPIsGlobal(restAPIEffectiveIP(addr)) {- return &ConnectorValidationError{Message: fmt.Sprintf(- "REST API connector URL %q resolves to disallowed address %s (localhost, private, link-local, reserved, or multicast addresses are blocked).",- rawURL, addr)}- }- }- return nil+ return validateConnectorURL(rawURL)}// assertRestAPIURLSafe mirrors ssrf_guard.assert_url_is_safe: every resolved// address must be globally routable. It returns the hostname and the first// validated IP so the caller can pin DNS.-func assertRestAPIURLSafe(ctx context.Context, rawURL string) (string, net.IP, error) {- parsed, err := url.Parse(rawURL)- if err != nil {- return "", nil, fmt.Errorf("URL is missing a host.")- }- if parsed.Scheme != "http" && parsed.Scheme != "https" {- return "", nil, fmt.Errorf("Disallowed URL scheme: %q. Only [http https] are allowed.", parsed.Scheme)- }- hostname := parsed.Hostname()- if hostname == "" {- return "", nil, fmt.Errorf("URL is missing a host.")- }- if restAPISSRFAllowLoopback {- addrs, err := net.DefaultResolver.LookupIPAddr(ctx, hostname)- if err != nil {- return "", nil, fmt.Errorf("Could not resolve hostname %q: %w", hostname, err)- }- if len(addrs) == 0 {- return "", nil, fmt.Errorf("Hostname %q resolved to no addresses.", hostname)- }- return hostname, addrs[0].IP, nil- }-- addrs, err := net.DefaultResolver.LookupIPAddr(ctx, hostname)- if err != nil {- return "", nil, fmt.Errorf("Could not resolve hostname %q: %w", hostname, err)- }- var first net.IP- for _, addr := range addrs {- eff := restAPIEffectiveIP(addr.IP)- if !restAPIIPIsGlobal(eff) {- return "", nil, fmt.Errorf("URL resolves to a non-public address (%s), which is not allowed.", addr.IP)- }- if first == nil {- first = addr.IP- }- }- if first == nil {- return "", nil, fmt.Errorf("Hostname %q resolved to no addresses.", hostname)- }- return hostname, first, nil-}--// restAPIEffectiveIP returns the IPv4 equivalent for IPv4-mapped IPv6-// addresses, mirroring ssrf_guard._effective_ip.-func restAPIEffectiveIP(ip net.IP) net.IP {- if v4 := ip.To4(); v4 != nil && len(ip) == net.IPv6len {- return v4- }- return ip-}--// restAPIIPIsGlobal mirrors ipaddress.is_global for the address classes that-// matter in practice.-func restAPIIPIsGlobal(ip net.IP) bool {- if ip == nil {- return false- }- if ip.IsLoopback() || ip.IsPrivate() || ip.IsUnspecified() || ip.IsMulticast() ||- ip.IsLinkLocalUnicast() || ip.IsLinkLocalMulticast() || ip.IsInterfaceLocalMulticast() {- return false- }- if v4 := ip.To4(); v4 != nil {- first := v4[0]- // 0.0.0.0/8, 100.64.0.0/10 (CGNAT), 192.0.0.0/24 (IETF protocol- // assignments), 198.18.0.0/15 (benchmarking), 240.0.0.0/4, broadcast.- if first == 0 || first == 100 || (first == 192 && v4[1] == 0) ||- (first == 198 && v4[1]&0xfe == 18) || first >= 240 {- return false- }- return true- }- // IPv6 documentation range (2001:db8::/32) is not globally routable.- if len(ip) == net.IPv6len && ip[0] == 0x20 && ip[1] == 0x01 && ip[2] == 0x0d && ip[3] == 0xb8 {- return false- }- return true+func assertRestAPIURLSafe(_ context.Context, rawURL string) (string, net.IP, error) {+ return assertConnectorURLSafe(rawURL)}// ---------------------------------------------------------------------------diff --git a/internal/syncer/connector/rest_api_test.go b/internal/syncer/connector/rest_api_test.goindex d978823256..a1d37ac08e 100644--- a/internal/syncer/connector/rest_api_test.go+++ b/internal/syncer/connector/rest_api_test.go@@ -21,6 +21,7 @@ import ("encoding/json""errors""io"+ "net""net/http""net/http/httptest""strings"@@ -28,11 +29,29 @@ import ("sync/atomic""testing""time"++ "ragflow/internal/utility")+// restAPITestPublicIP is the fixed public address used to satisfy the+// config-time SSRF check for hostname-based test URLs (e.g. example.com)+// without touching the real resolver, keeping unit tests hermetic.+const restAPITestPublicIP = "93.184.216.34"++// restAPITestLookupHost fakes DNS resolution for the REST API unit tests:+// literal IPs resolve to themselves (so the guard's private-address rejection+// still works), any other hostname resolves to a fixed public address.+func restAPITestLookupHost(host string) ([]string, error) {+ if ip := net.ParseIP(host); ip != nil {+ return []string{ip.String()}, nil+ }+ return []string{restAPITestPublicIP}, nil+}+func withRestAPITestHooks(t *testing.T) {t.Helper()- origLoopback := restAPISSRFAllowLoopback+ origLoopback := connectorAllowLoopbackForTest+ origLookup := utility.LookupHostorigTries := restAPIRetryTriesorigBaseDelay := restAPIRetryBaseDelayorigMaxDelay := restAPIRetryMaxDelay@@ -40,7 +59,8 @@ func withRestAPITestHooks(t *testing.T) {origJitter := restAPIRetryJitterorig429Waits := restAPI429MaxWaitsorig429Wait := restAPI429DefaultWait- restAPISSRFAllowLoopback = true+ connectorAllowLoopbackForTest = true+ utility.LookupHost = restAPITestLookupHostrestAPIRetryTries = 3restAPIRetryBaseDelay = time.MillisecondrestAPIRetryMaxDelay = 10 * time.Millisecond@@ -49,7 +69,8 @@ func withRestAPITestHooks(t *testing.T) {restAPI429MaxWaits = 3restAPI429DefaultWait = time.Millisecondt.Cleanup(func() {- restAPISSRFAllowLoopback = origLoopback+ connectorAllowLoopbackForTest = origLoopback+ utility.LookupHost = origLookuprestAPIRetryTries = origTriesrestAPIRetryBaseDelay = origBaseDelayrestAPIRetryMaxDelay = origMaxDelay@@ -120,7 +141,7 @@ func (b *restAPITestReadCloser) Close() error { return b.closeErrfunc TestRestAPICloseIdleBodyPreservesBodyAndCloseError(t *testing.T) {closeErr := errors.New("close boom")body := &restAPITestReadCloser{reader: strings.NewReader("hello"), closeErr: closeErr}- wrapped := &restAPICloseIdleBody{body: body, transport: &http.Transport{}}+ wrapped := &connectorCloseIdleBody{body: body, transport: &http.Transport{}}got, err := io.ReadAll(wrapped)if err != nil || string(got) != "hello" {t.Fatalf("read data=%q err=%v", got, err)@@ -144,7 +165,7 @@ func TestNewRestAPIConnectorValidationErrors(t *testing.T) {{name: "missing content fields", config: map[string]any{"url": "https://example.com"}, want: "At least one content field must be configured (content_fields)."},{name: "zero max_pages", config: map[string]any{"url": "https://example.com", "max_pages": 0, "content_fields": "title"}, want: "max_pages must be a positive integer"},{name: "negative max_pages", config: map[string]any{"url": "https://example.com", "max_pages": -1, "content_fields": "title"}, want: "max_pages must be a positive integer"},- {name: "bad scheme", config: map[string]any{"url": "ftp://example.com/x", "content_fields": "title"}, want: "Unsupported URL scheme"},+ {name: "bad scheme", config: map[string]any{"url": "ftp://example.com/x", "content_fields": "title"}, want: "disallowed URL scheme"},{name: "localhost", config: map[string]any{"url": "http://localhost/x", "content_fields": "title"}, want: "localhost is blocked"},}for _, tt := range tests {@@ -162,13 +183,13 @@ func TestNewRestAPIConnectorValidationErrors(t *testing.T) {}func TestNewRestAPIConnectorBlocksPrivateAddress(t *testing.T) {- restAPISSRFAllowLoopback = false- defer func() { restAPISSRFAllowLoopback = false }()+ connectorAllowLoopbackForTest = false+ defer func() { connectorAllowLoopbackForTest = false }()_, err := NewRestAPIConnector(map[string]any{"url": "http://127.0.0.1:8080/api","content_fields": "title",})- if err == nil || !strings.Contains(err.Error(), "resolves to disallowed address") {+ if err == nil || !strings.Contains(err.Error(), "resolves to a non-public address") {t.Fatalf("err=%v want private address rejection", err)}}diff --git a/internal/syncer/connector/s3_compatible.go b/internal/syncer/connector/s3_compatible.goindex 63dd769311..0056437827 100644--- a/internal/syncer/connector/s3_compatible.go+++ b/internal/syncer/connector/s3_compatible.go@@ -20,6 +20,8 @@ import ("context""encoding/json""fmt"+ "net"+ "net/http""net/url""strings"@@ -174,10 +176,56 @@ func (c *S3CompatibleConnector) ensureClient(ctx context.Context) (*s3.Client, ec.client = s3.NewFromConfig(cfg, func(options *s3.Options) {options.BaseEndpoint = aws.String(c.endpointURL)options.UsePathStyle = c.addressingStyle == "path"+ // The SDK resolves the endpoint host itself (bucket.virtual-host style+ // included), so pin every connection at dial time against the shared+ // SSRF guard instead of only validating the configured endpoint once.+ options.HTTPClient = s3PinnedHTTPClient()})return c.client, nil}+// s3PinnedHTTPClient returns an HTTP client whose transport validates and+// DNS-pins every outbound dial against the shared SSRF guard. The request URL+// keeps the original endpoint hostname (correct TLS SNI / certificate+// verification) while the TCP connect is pinned to the validated IP, closing+// the DNS-rebinding window between validation and the actual connection.+func s3PinnedHTTPClient() *http.Client {+ return &http.Client{+ // Mirror the AWS SDK's own default client policy (BuildableClient's+ // limitedRedirect): never auto-follow 301/302 — S3 region redirects+ // must be returned to the SDK so it can re-sign and retry against the+ // correct endpoint. Only method-preserving 307/308 are followed, same+ // as the SDK default. Without this, Go's default client would follow a+ // 301/302 before the SDK sees it, replaying a request signed for one+ // endpoint against another host.+ CheckRedirect: func(req *http.Request, via []*http.Request) error {+ if req.Response != nil {+ switch req.Response.StatusCode {+ case http.StatusTemporaryRedirect, http.StatusPermanentRedirect:+ return nil+ }+ }+ return http.ErrUseLastResponse+ },+ Transport: &http.Transport{+ // Ignore environment proxies: HTTP_PROXY / HTTPS_PROXY would route+ // the connection through a proxy host instead of the pinned IP.+ Proxy: nil,+ DialContext: func(ctx context.Context, network, addr string) (net.Conn, error) {+ host, port, err := net.SplitHostPort(addr)+ if err != nil {+ return nil, err+ }+ _, pinIP, err := assertConnectorURLSafe("https://" + host)+ if err != nil {+ return nil, err+ }+ return (&net.Dialer{}).DialContext(ctx, network, net.JoinHostPort(pinIP.String(), port))+ },+ },+ }+}+func (c *S3CompatibleConnector) listObjectPage(ctx context.Context, startAfter string, maxKeys int32) ([]s3Object, string, bool, error) {return listS3ObjectPage(ctx, c.listObjects, c.ensureClient, c.bucketName, c.prefix, s3CompatibleSource, startAfter, maxKeys)}@@ -229,5 +277,12 @@ func validateS3CompatibleEndpoint(raw string) error {if parsed.Fragment != "" {return fmt.Errorf("invalid S3-compatible endpoint_url %q: fragment is not allowed", raw)}+ // The endpoint host is user-controlled and connected to server-side, so it+ // must resolve only to public addresses (blocks loopback, RFC1918 and+ // cloud metadata). The per-connection transport in s3PinnedHTTPClient+ // re-validates and pins at dial time.+ if _, _, err := assertConnectorURLSafe(raw); err != nil {+ return fmt.Errorf("invalid S3-compatible endpoint_url %q: %v", raw, err)+ }return nil}diff --git a/internal/syncer/connector/s3_compatible_test.go b/internal/syncer/connector/s3_compatible_test.goindex a73939313e..6cd720df87 100644--- a/internal/syncer/connector/s3_compatible_test.go+++ b/internal/syncer/connector/s3_compatible_test.go@@ -7,6 +7,8 @@ import ("io""strings""testing"++ "ragflow/internal/utility")func TestS3CompatibleConnectorOpenSyncUsesFingerprintAndFetch(t *testing.T) {@@ -240,6 +242,7 @@ func TestS3CompatibleConnectorOpenPruneReturnsSlimSnapshot(t *testing.T) {}func TestS3CompatibleConnectorOpenPruneStreamsAcrossPages(t *testing.T) {+ stubS3EndpointResolver(t)connector, err := NewS3CompatibleConnector(map[string]any{"bucket_name": "bucket","batch_size": 2,@@ -301,6 +304,7 @@ func TestS3CompatibleConnectorOpenPruneStreamsAcrossPages(t *testing.T) {func TestS3CompatibleConnectorValidate(t *testing.T) {ctx := context.Background()+ stubS3EndpointResolver(t)missingBucket := &S3CompatibleConnector{endpointURL: "https://s3.example.com", accessKeyID: "access", secretKey: "secret", addressingStyle: "virtual", batchSize: 2}if err := missingBucket.Validate(ctx); err == nil {t.Fatalf("expected bucket name validation error")@@ -416,8 +420,25 @@ func TestS3CompatibleHelpers(t *testing.T) {}}+// stubS3EndpointResolver makes the shared SSRF guard resolve the+// s3.example.com test endpoint to a stable public IP without touching the+// network. The production guard resolves real endpoints; unit tests must not+// depend on DNS.+func stubS3EndpointResolver(t *testing.T) {+ t.Helper()+ orig := utility.LookupHost+ utility.LookupHost = func(host string) ([]string, error) {+ if host == "s3.example.com" {+ return []string{"93.184.216.34"}, nil+ }+ return orig(host)+ }+ t.Cleanup(func() { utility.LookupHost = orig })+}+func newTestS3CompatibleConnector(t *testing.T, objects []s3Object) *S3CompatibleConnector {t.Helper()+ stubS3EndpointResolver(t)connector, err := NewS3CompatibleConnector(map[string]any{"bucket_name": "bucket","prefix": "docs",diff --git a/internal/syncer/connector/salesforce.go b/internal/syncer/connector/salesforce.goindex 2db18483dc..6684e88dc5 100644--- a/internal/syncer/connector/salesforce.go+++ b/internal/syncer/connector/salesforce.go@@ -22,6 +22,7 @@ import ("errors""fmt""io"+ "net""net/http""net/url""sort"@@ -29,8 +30,6 @@ import ("strings""sync""time"-- "ragflow/internal/utility")const (@@ -364,22 +363,24 @@ func salesforceHostAllowed(host string) bool {strings.HasSuffix(host, ".lightning.force.com")}+// salesforceAssertURLSafe validates a Salesforce request URL for SSRF and the+// approved-host policy, returning the hostname and the IP to pin. Every hop of+// a Salesforce request (including redirects) must stay on an approved+// Salesforce host over HTTPS so credentials are only ever transmitted to the+// intended provider.+func salesforceAssertURLSafe(rawURL string) (string, net.IP, error) {+ parsed, err := url.Parse(rawURL)+ if err != nil || !strings.EqualFold(parsed.Scheme, "https") || !salesforceHostAllowed(parsed.Hostname()) {+ return "", nil, fmt.Errorf("Salesforce request URL must use HTTPS on an approved Salesforce host")+ }+ return assertConnectorURLSafe(rawURL)+}+// requestAccessToken performs the OAuth2 client-credentials exchange, validating// the token endpoint for SSRF, HTTPS, and the approved Salesforce host policy// before any credentials are transmitted.func (c *SalesforceConnector) requestAccessToken(ctx context.Context) (salesforceToken, error) {tokenURL := c.instanceBaseURL() + "/services/oauth2/token"- hostname, resolvedIP, err := utility.AssertURLSafe(tokenURL)- if err != nil {- return salesforceToken{}, &ConnectorMissingCredentialError{Message: fmt.Sprintf("Salesforce token request failed: %v", err)}- }- if !salesforceHostAllowed(hostname) {- return salesforceToken{}, &ConnectorMissingCredentialError{Message: "Salesforce instance_url is not an approved Salesforce host"}- }- parsedURL, err := url.Parse(tokenURL)- if err != nil || !strings.EqualFold(parsedURL.Scheme, "https") {- return salesforceToken{}, &ConnectorMissingCredentialError{Message: "Salesforce OAuth token endpoint must use HTTPS"}- }form := url.Values{"grant_type": {"client_credentials"},"client_id": {c.clientID},@@ -387,13 +388,14 @@ func (c *SalesforceConnector) requestAccessToken(ctx context.Context) (salesforc}requestCtx, cancel := context.WithTimeout(ctx, salesforceRequestTimeout)defer cancel()- req, err := http.NewRequestWithContext(requestCtx, http.MethodPost, tokenURL, strings.NewReader(form.Encode()))- if err != nil {- return salesforceToken{}, &ConnectorMissingCredentialError{Message: fmt.Sprintf("Salesforce token request failed: %v", err)}- }- req.Header.Set("Content-Type", "application/x-www-form-urlencoded")- client := utility.PinnedHTTPClient(hostname, resolvedIP, salesforceRequestTimeout)- resp, err := client.Do(req)+ resp, err := connectorRequest(requestCtx, connectorRequestOptions{+ Method: http.MethodPost,+ RawURL: tokenURL,+ Body: []byte(form.Encode()),+ Headers: map[string]string{"Content-Type": "application/x-www-form-urlencoded"},+ Timeout: salesforceRequestTimeout,+ Validate: salesforceAssertURLSafe,+ })if err != nil {return salesforceToken{}, &ConnectorMissingCredentialError{Message: fmt.Sprintf("Salesforce token request failed: %v", err)}}@@ -510,26 +512,15 @@ func (c *SalesforceConnector) getJSON(ctx context.Context, path string, out any)// doGet performs one authenticated GET with SSRF protection.func (c *SalesforceConnector) doGet(ctx context.Context, apiURL, token string) (int, []byte, error) {- parsed, err := url.Parse(apiURL)- if err != nil || !strings.EqualFold(parsed.Scheme, "https") || !salesforceHostAllowed(parsed.Hostname()) {- return 0, nil, fmt.Errorf("Salesforce request URL must use HTTPS on an approved Salesforce host")- }- hostname, resolvedIP, err := utility.AssertURLSafe(apiURL)- if err != nil {- return 0, nil, err- }- client := utility.PinnedHTTPClient(hostname, resolvedIP, salesforceRequestTimeout)- requestCtx, cancel := context.WithTimeout(ctx, salesforceRequestTimeout)- defer cancel()- req, err := http.NewRequestWithContext(requestCtx, http.MethodGet, apiURL, nil)- if err != nil {- return 0, nil, err- }- req.Header.Set("Accept", "application/json")- req.Header.Set("Authorization", "Bearer "+token)- resp, err := client.Do(req)+ resp, err := connectorRequest(ctx, connectorRequestOptions{+ Method: http.MethodGet,+ RawURL: apiURL,+ Headers: map[string]string{"Accept": "application/json", "Authorization": "Bearer " + token},+ Timeout: salesforceRequestTimeout,+ Validate: salesforceAssertURLSafe,+ })if err != nil {- return 0, nil, err+ return 0, nil, connectorUnsafeErr(err)}defer resp.Body.Close()body, err := io.ReadAll(io.LimitReader(resp.Body, 32*1024*1024))diff --git a/internal/syncer/connector/seafile.go b/internal/syncer/connector/seafile.goindex 81a2889e10..401896649c 100644--- a/internal/syncer/connector/seafile.go+++ b/internal/syncer/connector/seafile.go@@ -380,12 +380,15 @@ func (c *SeaFileConnector) defaultAuthenticate(ctx context.Context, username, paform.Set("password", password)reqCtx, cancel := context.WithTimeout(ctx, seafileRequestTimeout)defer cancel()- req, err := http.NewRequestWithContext(reqCtx, http.MethodPost, c.seafileURL+"/api2/auth-token/", strings.NewReader(form.Encode()))- if err != nil {- return "", err- }- req.Header.Set("Content-Type", "application/x-www-form-urlencoded")- resp, err := c.httpClient.Do(req)+ resp, err := connectorRequest(reqCtx, connectorRequestOptions{+ Method: http.MethodPost,+ RawURL: c.seafileURL + "/api2/auth-token/",+ Body: []byte(form.Encode()),+ Headers: map[string]string{"Content-Type": "application/x-www-form-urlencoded"},+ Timeout: seafileRequestTimeout,+ MaxRedirects: seafileMaxRedirects,+ Base: c.httpClient,+ })if err != nil {return "", &ConnectorMissingCredentialError{Message: fmt.Sprintf("Failed to authenticate with SeaFile: %v", err)}}@@ -539,66 +542,29 @@ func (c *SeaFileConnector) getBody(ctx context.Context, endpoint string, useRepo}reqCtx, cancel := context.WithTimeout(ctx, seafileRequestTimeout)defer cancel()- currentURL := c.apiURL(endpoint, useRepoToken, query)- previousNetloc := restAPINetloc(currentURL)- for hop := 0; hop <= seafileMaxRedirects; hop++ {- hostname, pinIP, err := seafileAssertURLSafe(reqCtx, currentURL)- if err != nil {- return nil, &ConnectorValidationError{Message: "Unsafe SeaFile URL: " + err.Error()}- }- req, err := http.NewRequestWithContext(reqCtx, http.MethodGet, currentURL, nil)- if err != nil {- return nil, err- }- for key, value := range headers {- req.Header.Set(key, value)- }- transport := newRestAPIPinnedTransport(hostname, pinIP)- client := &http.Client{- Transport: transport,- Timeout: seafileRequestTimeout,- // Redirects are handled manually so every hop is re-validated- // for SSRF and DNS-pinned before a connection is made.- CheckRedirect: func(req *http.Request, via []*http.Request) error {- return http.ErrUseLastResponse- },- }- resp, err := client.Do(req)- if err != nil {- transport.CloseIdleConnections()- return nil, err- }- if !restAPIIsRedirect(resp.StatusCode) {- resp.Body = &restAPICloseIdleBody{body: resp.Body, transport: transport}- body, err := readSeaFileBody(resp)- if err != nil {- return nil, err- }- if resp.StatusCode >= 400 {- return nil, &seafileHTTPError{Status: resp.StatusCode, Body: string(body), URL: req.URL.String()}- }- return body, nil- }- location := resp.Header.Get("Location")- resp.Body.Close()- transport.CloseIdleConnections()- if location == "" {- return nil, fmt.Errorf("SeaFile API redirect with empty Location header")- }- nextURL, err := restAPIResolveURL(currentURL, location)- if err != nil {- return nil, err- }- // Never forward credentials to a different host, matching Go's default- // redirect handling.- nextNetloc := restAPINetloc(nextURL)- if nextNetloc != "" && nextNetloc != previousNetloc {- headers = restAPIStripAuthHeaders(headers)+ resp, err := connectorRequest(reqCtx, connectorRequestOptions{+ Method: http.MethodGet,+ RawURL: c.apiURL(endpoint, useRepoToken, query),+ Headers: headers,+ Timeout: seafileRequestTimeout,+ MaxRedirects: seafileMaxRedirects,+ Base: c.httpClient,+ })+ if err != nil {+ var unsafe *connectorUnsafeURLError+ if errors.As(err, &unsafe) {+ return nil, &ConnectorValidationError{Message: "Unsafe SeaFile URL: " + unsafe.Err.Error()}}- previousNetloc = nextNetloc- currentURL = nextURL+ return nil, err}- return nil, fmt.Errorf("SeaFile API request exceeded %d redirects", seafileMaxRedirects)+ body, err := readSeaFileBody(resp)+ if err != nil {+ return nil, err+ }+ if resp.StatusCode >= 400 {+ return nil, &seafileHTTPError{Status: resp.StatusCode, Body: string(body), URL: resp.Request.URL.String()}+ }+ return body, nil}func (c *SeaFileConnector) requestHeaders(useRepoToken bool) (map[string]string, error) {@@ -837,98 +803,15 @@ func classifySeaFileError(err error) error {}func validateSeaFileURLForSSRF(rawURL string) error {- parsed, err := url.Parse(rawURL)- if err != nil {- return &ConnectorValidationError{Message: "Invalid SeaFile server URL."}- }- if parsed.Scheme != "http" && parsed.Scheme != "https" {- return &ConnectorValidationError{Message: fmt.Sprintf("Unsupported SeaFile server URL scheme %q. Only http/https are allowed.", parsed.Scheme)}- }- hostname := parsed.Hostname()- if hostname == "" {- return &ConnectorValidationError{Message: "SeaFile server URL must include a hostname."}- }- if strings.EqualFold(hostname, "localhost") && !restAPISSRFAllowLoopback {- return &ConnectorValidationError{Message: "SeaFile server URL hostname \"localhost\" is not allowed."}- }- addrs, err := net.LookupIP(hostname)- if err != nil {- return nil- }- if restAPISSRFAllowLoopback {- allLoopback := true- for _, addr := range addrs {- if !addr.IsLoopback() {- allLoopback = false- break- }- }- if allLoopback {- return nil- }- }- for _, addr := range addrs {- if !restAPIIPIsGlobal(restAPIEffectiveIP(addr)) {- return &ConnectorValidationError{Message: fmt.Sprintf(- "SeaFile server URL %q resolves to disallowed address %s (localhost, private, link-local, reserved, or multicast addresses are blocked).",- rawURL, addr)}- }- }- return nil+ return validateConnectorURL(rawURL)}// seafileAssertURLSafe validates a per-request SeaFile URL for SSRF and// returns the hostname plus the first validated IP so the caller can pin DNS// for the actual dial, preventing DNS rebinding between validation and the// connection.-func seafileAssertURLSafe(ctx context.Context, rawURL string) (string, net.IP, error) {- parsed, err := url.Parse(rawURL)- if err != nil {- return "", nil, fmt.Errorf("SeaFile URL is missing a host.")- }- if parsed.Scheme != "http" && parsed.Scheme != "https" {- return "", nil, fmt.Errorf("Disallowed SeaFile URL scheme: %q. Only [http https] are allowed.", parsed.Scheme)- }- hostname := parsed.Hostname()- if hostname == "" {- return "", nil, fmt.Errorf("SeaFile URL is missing a host.")- }- if strings.EqualFold(hostname, "localhost") && !restAPISSRFAllowLoopback {- return "", nil, fmt.Errorf("SeaFile URL hostname %q is not allowed (localhost is blocked).", hostname)- }- addrs, err := net.DefaultResolver.LookupIPAddr(ctx, hostname)- if err != nil {- return "", nil, fmt.Errorf("Could not resolve hostname %q: %w", hostname, err)- }- if len(addrs) == 0 {- return "", nil, fmt.Errorf("Hostname %q resolved to no addresses.", hostname)- }- if restAPISSRFAllowLoopback {- allLoopback := true- for _, addr := range addrs {- if !addr.IP.IsLoopback() {- allLoopback = false- break- }- }- if allLoopback {- return hostname, addrs[0].IP, nil- }- // Not all loopback — fall through to normal validation.- }- var first net.IP- for _, addr := range addrs {- if !restAPIIPIsGlobal(restAPIEffectiveIP(addr.IP)) {- return "", nil, fmt.Errorf("SeaFile URL resolves to a non-public address (%s), which is not allowed.", addr.IP)- }- if first == nil {- first = addr.IP- }- }- if first == nil {- return "", nil, fmt.Errorf("Hostname %q resolved to no addresses.", hostname)- }- return hostname, first, nil+func seafileAssertURLSafe(_ context.Context, rawURL string) (string, net.IP, error) {+ return assertConnectorURLSafe(rawURL)}type seafileSyncSession struct {diff --git a/internal/syncer/connector/seafile_test.go b/internal/syncer/connector/seafile_test.goindex 34df555404..052b34a9db 100644--- a/internal/syncer/connector/seafile_test.go+++ b/internal/syncer/connector/seafile_test.go@@ -231,9 +231,9 @@ func seafileValidationServer(t *testing.T) *httptest.Server {func TestSeaFileValidateAccountToken(t *testing.T) {server := seafileValidationServer(t)defer server.Close()- orig := restAPISSRFAllowLoopback- restAPISSRFAllowLoopback = true- defer func() { restAPISSRFAllowLoopback = orig }()+ orig := connectorAllowLoopbackForTest+ connectorAllowLoopbackForTest = true+ defer func() { connectorAllowLoopbackForTest = orig }()connector := seafileTestConnector(t, map[string]any{"seafile_url": server.URL,@@ -251,9 +251,9 @@ func TestSeaFileValidateAccountToken(t *testing.T) {func TestSeaFileValidateConnectorSettingUsesRequestUsernamePassword(t *testing.T) {server := seafileValidationServer(t)defer server.Close()- orig := restAPISSRFAllowLoopback- restAPISSRFAllowLoopback = true- defer func() { restAPISSRFAllowLoopback = orig }()+ orig := connectorAllowLoopbackForTest+ connectorAllowLoopbackForTest = true+ defer func() { connectorAllowLoopbackForTest = orig }()receiver := seafileTestConnector(t, map[string]any{"seafile_url": server.URL,@@ -275,9 +275,9 @@ func TestSeaFileValidateConnectorSettingUsesRequestUsernamePassword(t *testing.Tfunc TestSeaFileValidateRepoToken(t *testing.T) {server := seafileValidationServer(t)defer server.Close()- orig := restAPISSRFAllowLoopback- restAPISSRFAllowLoopback = true- defer func() { restAPISSRFAllowLoopback = orig }()+ orig := connectorAllowLoopbackForTest+ connectorAllowLoopbackForTest = true+ defer func() { connectorAllowLoopbackForTest = orig }()connector := seafileTestConnector(t, map[string]any{"seafile_url": server.URL,@@ -292,9 +292,9 @@ func TestSeaFileValidateRepoToken(t *testing.T) {}func TestSeaFileDefaultListLibrariesFiltersShared(t *testing.T) {- orig := restAPISSRFAllowLoopback- restAPISSRFAllowLoopback = true- defer func() { restAPISSRFAllowLoopback = orig }()+ orig := connectorAllowLoopbackForTest+ connectorAllowLoopbackForTest = true+ defer func() { connectorAllowLoopbackForTest = orig }()server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {if r.URL.Path != "/api2/repos/" || r.Header.Get("Authorization") != "Token token" {http.Error(w, "unauthorized", http.StatusUnauthorized)@@ -687,9 +687,9 @@ func TestSeaFileRegistryOpen(t *testing.T) {}func TestSeaFileDirentWrapperAndDownloadLinkDecode(t *testing.T) {- orig := restAPISSRFAllowLoopback- restAPISSRFAllowLoopback = true- defer func() { restAPISSRFAllowLoopback = orig }()+ orig := connectorAllowLoopbackForTest+ connectorAllowLoopbackForTest = true+ defer func() { connectorAllowLoopbackForTest = orig }()server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {w.Header().Set("Content-Type", "application/json")switch r.URL.Path {@@ -741,10 +741,10 @@ func TestSeaFileDirentWrapperAndDownloadLinkDecode(t *testing.T) {}func TestSeaFileAssertURLSafe(t *testing.T) {- orig := restAPISSRFAllowLoopback- defer func() { restAPISSRFAllowLoopback = orig }()+ orig := connectorAllowLoopbackForTest+ defer func() { connectorAllowLoopbackForTest = orig }()- restAPISSRFAllowLoopback = false+ connectorAllowLoopbackForTest = falseif _, _, err := seafileAssertURLSafe(context.Background(), "ftp://example.com"); err == nil {t.Fatalf("expected scheme rejection")}@@ -758,7 +758,7 @@ func TestSeaFileAssertURLSafe(t *testing.T) {t.Fatalf("expected loopback rejection")}- restAPISSRFAllowLoopback = true+ connectorAllowLoopbackForTest = truehostname, pinIP, err := seafileAssertURLSafe(context.Background(), "http://127.0.0.1:8080/seafile")if err != nil {t.Fatalf("loopback should be allowed: %v", err)diff --git a/internal/syncer/connector/ssrf.go b/internal/syncer/connector/ssrf.gonew file mode 100644index 0000000000..c07498797b--- /dev/null+++ b/internal/syncer/connector/ssrf.go@@ -0,0 +1,489 @@+//+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.+//++package connector++import (+ "bytes"+ "context"+ "crypto/tls"+ "encoding/base64"+ "errors"+ "fmt"+ "io"+ "net"+ "net/http"+ "net/url"+ "strings"+ "time"++ "ragflow/internal/utility"+)++// connectorAssertURLSafe is the SSRF guard shared by all connectors. It is an+// indirection over utility.AssertURLSafe so unit tests can substitute a stub+// without touching the shared utility guard (same pattern as sitemap.go and+// azure_blob.go).+var connectorAssertURLSafe = utility.AssertURLSafe++// connectorAssertHostSafe is the host-type SSRF guard shared by the host-based+// connectors (IMAP/MySQL/PostgreSQL). It is an indirection over+// utility.AssertHostSafe so unit tests can substitute a stub without touching+// the shared utility guard (same pattern as connectorAssertURLSafe).+var connectorAssertHostSafe = utility.AssertHostSafe++// assertConnectorHostSafe validates a bare host (hostname or literal IP, no+// scheme or port) with the shared strict SSRF guard and returns the first+// validated public IP so callers can pin the dial, preventing DNS rebinding+// between validation and the connection.+//+// When connectorAllowLoopbackForTest is set, loopback-only hosts are allowed+// (unit tests run against local listeners); anything else falls through to the+// strict guard, so a mixed private address is still rejected.+func assertConnectorHostSafe(host string) (net.IP, error) {+ host = strings.TrimSpace(host)+ if connectorAllowLoopbackForTest {+ if ip := net.ParseIP(host); ip != nil {+ if ip.IsLoopback() {+ return ip, nil+ }+ } else {+ lower := strings.ToLower(host)+ if lower == "localhost" || strings.HasSuffix(lower, ".localhost") {+ if _, ip, ok := loopbackTestAllow(lower); ok {+ return ip, nil+ }+ }+ }+ // Not allowed by the test hook — fall through to the strict guard.+ }+ ipStr, err := connectorAssertHostSafe(host)+ if err != nil {+ return nil, err+ }+ ip := net.ParseIP(ipStr)+ if ip == nil {+ return nil, fmt.Errorf("Could not parse validated address %q for host %q", ipStr, host)+ }+ return ip, nil+}++// connectorAllowLoopbackForTest lets unit tests exercise the real HTTP path+// against httptest servers bound to loopback. Production code keeps it false so+// loopback/private endpoints stay blocked.+var connectorAllowLoopbackForTest bool++// assertConnectorURLSafe validates rawURL with the shared strict SSRF guard and+// returns the hostname plus the first validated IP so callers can pin DNS for+// the actual dial, preventing DNS rebinding between validation and the+// connection.+//+// When connectorAllowLoopbackForTest is set, URLs whose hostname resolves only+// to loopback addresses are allowed; anything else falls through to the strict+// guard, so a mixed private address is still rejected.+func assertConnectorURLSafe(rawURL string) (string, net.IP, error) {+ if connectorAllowLoopbackForTest {+ parsed, err := url.Parse(strings.TrimSpace(rawURL))+ if err != nil || parsed.Hostname() == "" {+ return "", nil, fmt.Errorf("URL is missing a host.")+ }+ if scheme := strings.ToLower(parsed.Scheme); scheme == "http" || scheme == "https" {+ if host, ip, ok := loopbackTestAllow(parsed.Hostname()); ok {+ return host, ip, nil+ }+ }+ // Not allowed by the test hook — fall through to the strict guard.+ }+ hostname, ipStr, err := connectorAssertURLSafe(rawURL)+ if err != nil {+ return "", nil, err+ }+ ip := net.ParseIP(ipStr)+ if ip == nil {+ return "", nil, fmt.Errorf("Could not parse validated address %q for hostname %q", ipStr, hostname)+ }+ return hostname, ip, nil+}++// loopbackTestAllow reports whether hostname is a loopback-only target that the+// test hook may allow. Literal loopback IPs are allowed directly; localhost+// names must resolve (via the shared resolver seam) to loopback addresses only.+func loopbackTestAllow(hostname string) (string, net.IP, bool) {+ if ip := net.ParseIP(hostname); ip != nil {+ if ip.IsLoopback() {+ return hostname, ip, true+ }+ return "", nil, false+ }+ lower := strings.ToLower(hostname)+ if lower != "localhost" && !strings.HasSuffix(lower, ".localhost") {+ return "", nil, false+ }+ addrs, err := utility.LookupHost(hostname)+ if err != nil {+ return "", nil, false+ }+ allLoopback := true+ var first net.IP+ for _, addr := range addrs {+ ip := net.ParseIP(addr)+ if ip == nil || !ip.IsLoopback() {+ allLoopback = false+ break+ }+ if first == nil {+ first = ip+ }+ }+ if allLoopback && first != nil {+ return hostname, first, true+ }+ return "", nil, false+}++// assertConnectorURLSafeHTTPS validates rawURL with the strict SSRF guard and+// additionally requires HTTPS. Plain-HTTP loopback targets are still permitted+// under the test hook (httptest servers are plain HTTP); anything else must use+// TLS so credentials in request headers are never sent in the clear.+func assertConnectorURLSafeHTTPS(rawURL string) (string, net.IP, error) {+ if parsed, err := url.Parse(strings.TrimSpace(rawURL)); err == nil {+ if scheme := strings.ToLower(parsed.Scheme); scheme != "https" {+ if !connectorAllowLoopbackForTest {+ return "", nil, fmt.Errorf("URL must use HTTPS (got %q)", scheme)+ }+ if _, _, ok := loopbackTestAllow(parsed.Hostname()); !ok {+ return "", nil, fmt.Errorf("URL must use HTTPS (got %q)", scheme)+ }+ }+ }+ return assertConnectorURLSafe(rawURL)+}++// validateConnectorURL is the config-time SSRF check used by connector+// Validate/New paths.+func validateConnectorURL(rawURL string) error {+ // localhost is never a legitimate connector base URL, even under the test+ // hook (which only relaxes loopback IPs such as httptest's 127.0.0.1).+ if parsed, err := url.Parse(strings.TrimSpace(rawURL)); err == nil {+ host := strings.ToLower(parsed.Hostname())+ if host == "localhost" || strings.HasSuffix(host, ".localhost") {+ return &ConnectorValidationError{Message: fmt.Sprintf("URL hostname %q is not allowed (localhost is blocked).", parsed.Hostname())}+ }+ }+ if _, _, err := assertConnectorURLSafe(rawURL); err != nil {+ return &ConnectorValidationError{Message: err.Error()}+ }+ return nil+}++// basicAuthHeader returns the value for an HTTP Basic Authorization header.+func basicAuthHeader(username, password string) string {+ return base64.StdEncoding.EncodeToString([]byte(username + ":" + password))+}++// ---------------------------------------------------------------------------+// Shared request path+// ---------------------------------------------------------------------------++// connectorRequestHop is the per-hop request state carried by connectorRequest.+type connectorRequestHop struct {+ Method string+ URL string+ Body []byte+ Headers map[string]string+}++// connectorRequestOptions configures connectorRequest. Zero values use safe+// defaults: Validate falls back to assertConnectorURLSafe and MaxRedirects to 10.+type connectorRequestOptions struct {+ Method string+ RawURL string+ Body []byte+ Headers map[string]string+ Timeout time.Duration+ MaxRedirects int+ // Base, when non-nil, provides the starting transport so custom TLS/CA+ // settings are preserved; otherwise a default pinned transport is used.+ Base *http.Client+ // Validate checks a hop URL for SSRF and returns the hostname plus the IP+ // to pin for that hop.+ Validate func(rawURL string) (string, net.IP, error)+ // Prepare customizes each hop's request before it is sent (basic auth,+ // query parameters, ...).+ Prepare func(req *http.Request, hop connectorRequestHop) error+ // CheckRedirect, when non-nil, is invoked before following to the next hop+ // (after the next hop's SSRF validation) and may enforce a same-origin+ // policy or re-apply credentials.+ CheckRedirect func(req *http.Request, via []*http.Request) error+ // RetryStatus, when non-nil, decides whether a non-redirect response should+ // be retried on the same hop URL; it returns the wait duration and whether+ // to retry. Used for REST API 429 handling.+ RetryStatus func(resp *http.Response, attempt int) (time.Duration, bool)+ // NextHop adapts the hop state (method/body/headers/URL) for the hop after+ // a redirect; it runs after connectorRequest's default GET downgrade and+ // cross-origin credential stripping.+ NextHop func(nextURL string, status int, hop connectorRequestHop) connectorRequestHop+}++// connectorUnsafeErr returns the underlying error when err was produced by the+// SSRF guard (connectorUnsafeURLError), so callers can surface the original+// message; otherwise it returns err unchanged.+func connectorUnsafeErr(err error) error {+ var unsafe *connectorUnsafeURLError+ if errors.As(err, &unsafe) {+ return unsafe.Err+ }+ return err+}++// connectorUnsafeURLError marks a hop URL rejected by the SSRF guard so+// callers can tell validation failures apart from transport errors.+type connectorUnsafeURLError struct{ Err error }++func (e *connectorUnsafeURLError) Error() string { return e.Err.Error() }+func (e *connectorUnsafeURLError) Unwrap() error { return e.Err }++// connectorRequest is the single HTTP send path shared by every connector. It+// follows redirects manually so that each hop is SSRF-validated and DNS-pinned+// before a connection is made, closing the TOCTOU window that an automatic+// redirect policy would leave open (the redirect target would otherwise be+// validated once and resolved again at dial time).+//+// Per-hop behavior:+// - Validate checks the hop URL and returns the hostname + IP to pin.+// - The hop is sent through a transport that dials exactly the validated IP,+// so a DNS change after validation cannot rebind the connection.+// - Sensitive headers (Authorization etc.) are dropped when a redirect+// crosses to a different netloc, mirroring http.Client's default.+// - 301/302/303 downgrade the next hop to GET with an empty body.+//+// Connector-specific behavior is expressed through the Prepare, CheckRedirect,+// RetryStatus, and NextHop hooks.+func connectorRequest(ctx context.Context, opts connectorRequestOptions) (*http.Response, error) {+ if opts.Validate == nil {+ opts.Validate = assertConnectorURLSafe+ }+ if opts.MaxRedirects <= 0 {+ opts.MaxRedirects = 10+ }+ hop := connectorRequestHop{Method: opts.Method, URL: opts.RawURL, Body: opts.Body, Headers: opts.Headers}+ previousNetloc := connectorNetloc(hop.URL)+ var via []*http.Request+ for redirects := 0; ; redirects++ {+ hostname, pinIP, err := opts.Validate(hop.URL)+ if err != nil {+ return nil, &connectorUnsafeURLError{Err: err}+ }+ var reader io.Reader+ if hop.Body != nil {+ reader = bytes.NewReader(hop.Body)+ }+ req, err := http.NewRequestWithContext(ctx, hop.Method, hop.URL, reader)+ if err != nil {+ return nil, err+ }+ for k, v := range hop.Headers {+ req.Header.Set(k, v)+ }+ if opts.Prepare != nil {+ if err := opts.Prepare(req, hop); err != nil {+ return nil, err+ }+ }+ if redirects > 0 && opts.CheckRedirect != nil {+ if err := opts.CheckRedirect(req, via); err != nil {+ return nil, err+ }+ }+ transport := newConnectorPinnedTransport(opts.Base, hostname, pinIP, opts.Timeout)+ client := &http.Client{+ Transport: transport,+ Timeout: opts.Timeout,+ // Redirects are followed manually by this loop; each hop gets its+ // own validated, pinned transport.+ CheckRedirect: func(*http.Request, []*http.Request) error {+ return http.ErrUseLastResponse+ },+ }+ var resp *http.Response+ for attempt := 1; ; attempt++ {+ resp, err = client.Do(req)+ if err != nil {+ transport.CloseIdleConnections()+ return nil, err+ }+ if !connectorIsRedirect(resp.StatusCode) && opts.RetryStatus != nil {+ if wait, retry := opts.RetryStatus(resp, attempt); retry {+ resp.Body.Close()+ select {+ case <-ctx.Done():+ transport.CloseIdleConnections()+ return nil, ctx.Err()+ case <-time.After(wait):+ }+ continue+ }+ }+ break+ }+ if !connectorIsRedirect(resp.StatusCode) {+ resp.Body = &connectorCloseIdleBody{body: resp.Body, transport: transport}+ return resp, nil+ }+ location := resp.Header.Get("Location")+ resp.Body.Close()+ transport.CloseIdleConnections()+ if location == "" {+ return nil, fmt.Errorf("redirect with empty Location header")+ }+ if redirects >= opts.MaxRedirects {+ return nil, fmt.Errorf("request stopped after %d redirects", opts.MaxRedirects)+ }+ nextURL, err := connectorResolveURL(hop.URL, location)+ if err != nil {+ return nil, err+ }+ nextNetloc := connectorNetloc(nextURL)+ if nextNetloc != "" && nextNetloc != previousNetloc {+ hop.Headers = connectorStripAuthHeaders(hop.Headers)+ // 307/308 preserve the request method and body; never forward a+ // request body (which may carry credentials, e.g. a Seafile or+ // Moodle token exchange) to a different origin.+ if len(hop.Body) > 0 && (resp.StatusCode == http.StatusTemporaryRedirect || resp.StatusCode == http.StatusPermanentRedirect) {+ return nil, fmt.Errorf("redirect to a different origin is not allowed for a request with a body")+ }+ }+ previousNetloc = nextNetloc+ if resp.StatusCode == http.StatusMovedPermanently || resp.StatusCode == http.StatusFound || resp.StatusCode == http.StatusSeeOther {+ hop.Method = http.MethodGet+ hop.Body = nil+ }+ if opts.NextHop != nil {+ hop = opts.NextHop(nextURL, resp.StatusCode, hop)+ } else {+ hop.URL = nextURL+ }+ via = append(via, req)+ }+}++// newConnectorPinnedTransport builds a transport that dials exactly pinIP for+// every connection, so the hop is connected to the address validated by the+// SSRF guard regardless of later DNS changes. When base provides an+// *http.Transport (custom TLS/CA), it is cloned and reused; otherwise a default+// transport is used. Environment proxies are deliberately ignored: HTTP_PROXY /+// HTTPS_PROXY would route the connection through a proxy host instead of pinIP.+func newConnectorPinnedTransport(base *http.Client, hostname string, pinIP net.IP, timeout time.Duration) *http.Transport {+ dialer := &net.Dialer{Timeout: timeout, KeepAlive: 30 * time.Second}+ dial := func(ctx context.Context, network, addr string) (net.Conn, error) {+ _, port, err := net.SplitHostPort(addr)+ if err != nil {+ port = "443"+ }+ return dialer.DialContext(ctx, network, net.JoinHostPort(pinIP.String(), port))+ }+ if base != nil {+ if t, ok := base.Transport.(*http.Transport); ok {+ clone := t.Clone()+ clone.DialContext = dial+ if clone.TLSClientConfig == nil {+ clone.TLSClientConfig = &tls.Config{ServerName: hostname, MinVersion: tls.VersionTLS12}+ } else {+ clone.TLSClientConfig = clone.TLSClientConfig.Clone()+ if clone.TLSClientConfig.ServerName == "" {+ clone.TLSClientConfig.ServerName = hostname+ }+ }+ return clone+ }+ }+ return &http.Transport{+ Proxy: nil,+ DialContext: dial,+ TLSHandshakeTimeout: timeout,+ ResponseHeaderTimeout: timeout,+ ExpectContinueTimeout: 1 * time.Second,+ ForceAttemptHTTP2: false,+ TLSClientConfig: &tls.Config{ServerName: hostname, MinVersion: tls.VersionTLS12},+ }+}++// connectorCloseIdleBody closes the underlying response body and then releases+// the per-hop pinned transport's idle connections, so per-hop transports do not+// leak keep-alive sockets during long syncs.+type connectorCloseIdleBody struct {+ body io.ReadCloser+ transport *http.Transport+}++func (b *connectorCloseIdleBody) Read(p []byte) (int, error) { return b.body.Read(p) }+func (b *connectorCloseIdleBody) Close() error {+ err := b.body.Close()+ b.transport.CloseIdleConnections()+ return err+}++// connectorAuthSensitiveHeaders lists headers that must not be forwarded to a+// different origin on redirect.+var connectorAuthSensitiveHeaders = map[string]struct{}{+ "authorization": {},+ "proxy-authorization": {},+ "apikey": {},+ "api-key": {},+ "x-api-key": {},+ "x-auth-token": {},+ "private-token": {}, // GitLab credential header+}++func connectorStripAuthHeaders(headers map[string]string) map[string]string {+ out := make(map[string]string, len(headers))+ for k, v := range headers {+ if _, sensitive := connectorAuthSensitiveHeaders[strings.ToLower(k)]; sensitive {+ continue+ }+ out[k] = v+ }+ return out+}++func connectorNetloc(rawURL string) string {+ parsed, err := url.Parse(rawURL)+ if err != nil {+ return ""+ }+ return parsed.Host+}++func connectorResolveURL(base, location string) (string, error) {+ baseParsed, err := url.Parse(base)+ if err != nil {+ return "", err+ }+ ref, err := url.Parse(location)+ if err != nil {+ return "", err+ }+ return baseParsed.ResolveReference(ref).String(), nil+}++func connectorIsRedirect(status int) bool {+ switch status {+ case http.StatusMovedPermanently, http.StatusFound, http.StatusSeeOther,+ http.StatusTemporaryRedirect, http.StatusPermanentRedirect:+ return true+ }+ return false+}diff --git a/internal/syncer/connector/ssrf_test.go b/internal/syncer/connector/ssrf_test.gonew file mode 100644index 0000000000..1dae568656--- /dev/null+++ b/internal/syncer/connector/ssrf_test.go@@ -0,0 +1,636 @@+//+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.+//++package connector++import (+ "context"+ "io"+ "net"+ "net/http"+ "net/http/httptest"+ "strconv"+ "strings"+ "sync/atomic"+ "testing"+ "time"++ "ragflow/internal/utility"+)++// withConnectorLoopbackTestHook enables the loopback test seam for the duration+// of a test so connectors can exercise the real HTTP path against httptest+// servers bound to loopback.+func withConnectorLoopbackTestHook(t *testing.T) {+ t.Helper()+ prev := connectorAllowLoopbackForTest+ connectorAllowLoopbackForTest = true+ t.Cleanup(func() { connectorAllowLoopbackForTest = prev })+}++func TestValidateConnectorURL(t *testing.T) {+ tests := []struct {+ name string+ url string+ want bool // true = accepted, false = rejected+ }{+ {name: "public literal ip", url: "http://8.8.8.8/x", want: true},+ {name: "loopback", url: "http://127.0.0.1/x", want: false},+ {name: "localhost", url: "http://localhost/x", want: false},+ {name: "cloud metadata", url: "http://169.254.169.254/latest/meta-data/", want: false},+ {name: "private 10/8", url: "http://10.0.0.5/x", want: false},+ {name: "private 192.168/16", url: "http://192.168.1.10/x", want: false},+ {name: "bad scheme", url: "ftp://8.8.8.8/x", want: false},+ {name: "missing host", url: "http://", want: false},+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ connectorAllowLoopbackForTest = false+ err := validateConnectorURL(tt.url)+ if tt.want && err != nil {+ t.Fatalf("validateConnectorURL(%q) = %v, want nil", tt.url, err)+ }+ if !tt.want && err == nil {+ t.Fatalf("validateConnectorURL(%q) = nil, want rejection", tt.url)+ }+ })+ }+}++func TestValidateConnectorURLLoopbackHookAllowsOnlyAllLoopback(t *testing.T) {+ withConnectorLoopbackTestHook(t)+ if err := validateConnectorURL("http://127.0.0.1/x"); err != nil {+ t.Fatalf("loopback should be allowed with hook on: %v", err)+ }+ // A private address is still rejected even with the hook on.+ if err := validateConnectorURL("http://10.0.0.1/x"); err == nil {+ t.Fatalf("private address should be rejected even with hook on")+ }+}++func TestWebDAVConnectorRejectsInternalBaseURL(t *testing.T) {+ connectorAllowLoopbackForTest = false+ t.Cleanup(func() { connectorAllowLoopbackForTest = false })++ for _, base := range []string{"http://127.0.0.1/dav", "http://169.254.169.254/dav", "http://10.0.0.5/dav"} {+ c, err := NewWebDAVConnector(map[string]any{+ "base_url": base,+ "credentials": map[string]any{+ "username": "user",+ "password": "pass",+ },+ })+ if err != nil {+ t.Fatalf("NewWebDAVConnector(%q) failed: %v", base, err)+ }+ if err := c.Validate(context.Background()); err == nil || !strings.Contains(err.Error(), "non-public address") {+ t.Fatalf("Validate(%q) = %v, want non-public rejection", base, err)+ }+ }+}++func TestConfluenceConnectorRejectsInternalWikiBase(t *testing.T) {+ connectorAllowLoopbackForTest = false+ t.Cleanup(func() { connectorAllowLoopbackForTest = false })++ for _, base := range []string{"http://127.0.0.1", "http://169.254.169.254", "http://10.0.0.5"} {+ c, err := NewConfluenceConnector(map[string]any{+ "wiki_base": base,+ "is_cloud": true,+ "credentials": map[string]any{+ "confluence_username": "[email protected]",+ "confluence_access_token": "token",+ },+ })+ if err != nil {+ t.Fatalf("NewConfluenceConnector(%q) failed: %v", base, err)+ }+ if err := c.Validate(context.Background()); err == nil || !strings.Contains(err.Error(), "non-public address") {+ t.Fatalf("Validate(%q) = %v, want non-public rejection", base, err)+ }+ }+}++func TestJiraConnectorRejectsInternalBaseURL(t *testing.T) {+ connectorAllowLoopbackForTest = false+ t.Cleanup(func() { connectorAllowLoopbackForTest = false })++ for _, base := range []string{"http://127.0.0.1", "http://169.254.169.254", "http://10.0.0.5"} {+ c, err := NewJiraConnector(map[string]any{+ "base_url": base,+ "project_key": "RAG",+ "credentials": map[string]any{+ "jira_api_token": "token",+ },+ })+ if err != nil {+ t.Fatalf("NewJiraConnector(%q) failed: %v", base, err)+ }+ if err := c.Validate(context.Background()); err == nil || !strings.Contains(err.Error(), "non-public address") {+ t.Fatalf("Validate(%q) = %v, want non-public rejection", base, err)+ }+ }+}++func TestConnectorRequestPinsRedirectHopAgainstDNSRebinding(t *testing.T) {+ withConnectorLoopbackTestHook(t)+ origLookup := utility.LookupHost+ lookups := 0+ utility.LookupHost = func(host string) ([]string, error) {+ if host == "localhost" {+ lookups+++ if lookups == 1 {+ return []string{"127.0.0.1"}, nil+ }+ // A second resolution simulates DNS rebinding to a private address+ // after validation; the dial must never consult it.+ return []string{"10.0.0.5"}, nil+ }+ return origLookup(host)+ }+ t.Cleanup(func() { utility.LookupHost = origLookup })++ var targetHit atomic.Bool+ target := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ targetHit.Store(true)+ }))+ defer target.Close()+ targetPort := strings.TrimPrefix(target.URL, "http://127.0.0.1:")+ source := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ http.Redirect(w, r, "http://localhost:"+targetPort+"/final", http.StatusFound)+ }))+ defer source.Close()++ resp, err := connectorRequest(context.Background(), connectorRequestOptions{+ Method: http.MethodGet,+ RawURL: source.URL,+ Timeout: webdavRequestTimeout,+ MaxRedirects: 5,+ })+ if err != nil {+ t.Fatalf("connectorRequest: %v", err)+ }+ resp.Body.Close()+ if !targetHit.Load() {+ t.Fatalf("redirect target was not reached via its pinned address")+ }+ if lookups != 1 {+ t.Fatalf("LookupHost(localhost) called %d times, want 1 (the redirect hop dial must reuse the validated pin, not re-resolve)", lookups)+ }+}++func TestConnectorRequestRejectsCrossOriginBodyRedirect(t *testing.T) {+ withConnectorLoopbackTestHook(t)++ var targetHit atomic.Bool+ target := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ targetHit.Store(true)+ }))+ defer target.Close()+ targetPort := strings.TrimPrefix(target.URL, "http://127.0.0.1:")++ source := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ switch r.URL.Path {+ case "/cross":+ http.Redirect(w, r, "http://127.0.0.1:"+targetPort+"/final", http.StatusTemporaryRedirect)+ case "/same":+ http.Redirect(w, r, "/same-final", http.StatusTemporaryRedirect)+ case "/same-final":+ got, _ := io.ReadAll(r.Body)+ if string(got) != "secret=abc" {+ t.Errorf("same-origin 307 body = %q, want forwarded body", got)+ }+ w.WriteHeader(http.StatusOK)+ }+ }))+ defer source.Close()++ opts := func(path string) connectorRequestOptions {+ return connectorRequestOptions{+ Method: http.MethodPost,+ RawURL: source.URL + path,+ Body: []byte("secret=abc"),+ Headers: map[string]string{"Content-Type": "application/x-www-form-urlencoded"},+ Timeout: webdavRequestTimeout,+ MaxRedirects: 5,+ }+ }++ // A 307 to a different origin must not forward the credential-bearing body.+ crossResp, err := connectorRequest(context.Background(), opts("/cross"))+ if crossResp != nil {+ crossResp.Body.Close()+ }+ if err == nil || !strings.Contains(err.Error(), "different origin") {+ t.Fatalf("cross-origin body redirect err = %v, want rejection", err)+ }+ if targetHit.Load() {+ t.Fatalf("cross-origin 307 target must not be reached")+ }++ // A same-origin 307 still preserves the request body.+ resp, err := connectorRequest(context.Background(), opts("/same"))+ if err != nil {+ t.Fatalf("same-origin 307: %v", err)+ }+ resp.Body.Close()+}++func TestAssertConnectorURLSafeHTTPS(t *testing.T) {+ orig := utility.LookupHost+ utility.LookupHost = func(host string) ([]string, error) {+ return []string{"93.184.216.34"}, nil+ }+ t.Cleanup(func() { utility.LookupHost = orig })++ connectorAllowLoopbackForTest = false+ t.Cleanup(func() { connectorAllowLoopbackForTest = false })++ // HTTPS public URLs pass.+ if _, _, err := assertConnectorURLSafeHTTPS("https://api.example.com/v1"); err != nil {+ t.Fatalf("https should be allowed: %v", err)+ }+ // Plain-HTTP (non-loopback) is rejected so credentials stay off the wire.+ if _, _, err := assertConnectorURLSafeHTTPS("http://api.example.com/v1"); err == nil {+ t.Fatalf("plain http should be rejected")+ }+ // Loopback HTTP is rejected outside the test hook.+ if _, _, err := assertConnectorURLSafeHTTPS("http://127.0.0.1:8080/v1"); err == nil {+ t.Fatalf("loopback http should be rejected outside the test hook")+ }++ // Loopback HTTP is permitted under the test hook (httptest servers).+ withConnectorLoopbackTestHook(t)+ if _, _, err := assertConnectorURLSafeHTTPS("http://127.0.0.1:8080/v1"); err != nil {+ t.Fatalf("loopback http should be allowed under the test hook: %v", err)+ }+ // But a plain-http non-loopback target is still rejected even under the hook.+ if _, _, err := assertConnectorURLSafeHTTPS("http://api.example.com/v1"); err == nil {+ t.Fatalf("plain http should still be rejected under the test hook")+ }+}++func TestConnectorStripAuthHeadersRemovesCredentials(t *testing.T) {+ headers := map[string]string{+ "Authorization": "Bearer tok",+ "PRIVATE-TOKEN": "gitlab-tok",+ "X-API-Key": "key",+ "X-Auth-Token": "auth",+ "Content-Type": "application/json",+ "X-Custom-Header": "keep",+ }+ stripped := connectorStripAuthHeaders(headers)+ for _, k := range []string{"Authorization", "PRIVATE-TOKEN", "X-API-Key", "X-Auth-Token"} {+ if _, ok := stripped[k]; ok {+ t.Fatalf("sensitive header %q was not stripped", k)+ }+ }+ for _, k := range []string{"Content-Type", "X-Custom-Header"} {+ if stripped[k] == "" {+ t.Fatalf("non-sensitive header %q was stripped", k)+ }+ }+}++// ---------------------------------------------------------------------------+// Host-type SSRF guards (IMAP / MySQL / PostgreSQL / S3-compatible)+// ---------------------------------------------------------------------------++func TestAssertConnectorHostSafe(t *testing.T) {+ connectorAllowLoopbackForTest = false+ t.Cleanup(func() { connectorAllowLoopbackForTest = false })++ tests := []struct {+ name string+ host string+ want bool // true = accepted, false = rejected+ }{+ {name: "public literal ip", host: "8.8.8.8", want: true},+ {name: "loopback", host: "127.0.0.1", want: false},+ {name: "localhost", host: "localhost", want: false},+ {name: "cloud metadata", host: "169.254.169.254", want: false},+ {name: "private 10/8", host: "10.0.0.5", want: false},+ {name: "private 192.168/16", host: "192.168.1.10", want: false},+ {name: "unspecified", host: "0.0.0.0", want: false},+ {name: "empty", host: "", want: false},+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ ip, err := assertConnectorHostSafe(tt.host)+ if tt.want && err != nil {+ t.Fatalf("assertConnectorHostSafe(%q) = %v, want nil", tt.host, err)+ }+ if tt.want && (ip == nil || ip.String() != "8.8.8.8") {+ t.Fatalf("assertConnectorHostSafe(%q) = %v, want 8.8.8.8", tt.host, ip)+ }+ if !tt.want && err == nil {+ t.Fatalf("assertConnectorHostSafe(%q) = nil, want rejection", tt.host)+ }+ })+ }+}++func TestAssertConnectorHostSafeLoopbackHookAllowsOnlyLoopback(t *testing.T) {+ withConnectorLoopbackTestHook(t)+ if _, err := assertConnectorHostSafe("127.0.0.1"); err != nil {+ t.Fatalf("loopback should be allowed with hook on: %v", err)+ }+ if _, err := assertConnectorHostSafe("localhost"); err != nil {+ t.Fatalf("localhost should be allowed with hook on: %v", err)+ }+ // Private / metadata addresses are still rejected even with the hook on.+ if _, err := assertConnectorHostSafe("10.0.0.1"); err == nil {+ t.Fatalf("private address should be rejected even with hook on")+ }+ if _, err := assertConnectorHostSafe("169.254.169.254"); err == nil {+ t.Fatalf("metadata address should be rejected even with hook on")+ }+}++// TestConnectorHostSSRFDNSRebinding closes the host-type TOCTOU window: after+// the guard validates a public host, the dial must be pinned to the validated+// IP and never re-resolve the hostname. The guard seam is stubbed to simulate+// "validates to a public IP" while the dial helper is pointed at a loopback+// listener.+func TestMySQLPinnedDialConnectsToPinnedIP(t *testing.T) {+ ln, err := net.Listen("tcp4", "127.0.0.1:0")+ if err != nil {+ t.Fatalf("listen: %v", err)+ }+ defer ln.Close()+ go func() {+ for {+ conn, acceptErr := ln.Accept()+ if acceptErr != nil {+ return+ }+ _ = conn.Close()+ }+ }()+ _, portStr, err := net.SplitHostPort(ln.Addr().String())+ if err != nil {+ t.Fatal(err)+ }+ port, err := strconv.Atoi(portStr)+ if err != nil {+ t.Fatal(err)+ }+ dial := mysqlPinnedDial(net.ParseIP("127.0.0.1"), port)+ // The driver passes the DSN host (attacker-controlled); the pinned dialer+ // must ignore it and connect to pinIP:port.+ conn, err := dial(context.Background(), "attacker.example:9999")+ if err != nil {+ t.Fatalf("pinned dial did not reach the pinned listener: %v", err)+ }+ _ = conn.Close()+}++func TestPostgresPinnedDialConnectsToPinnedIP(t *testing.T) {+ ln, err := net.Listen("tcp4", "127.0.0.1:0")+ if err != nil {+ t.Fatalf("listen: %v", err)+ }+ defer ln.Close()+ go func() {+ for {+ conn, acceptErr := ln.Accept()+ if acceptErr != nil {+ return+ }+ _ = conn.Close()+ }+ }()+ _, portStr, err := net.SplitHostPort(ln.Addr().String())+ if err != nil {+ t.Fatal(err)+ }+ dial := postgresPinnedDial(net.ParseIP("127.0.0.1"), portStr, time.Second)+ conn, err := dial(context.Background(), "tcp", "attacker.example:9999")+ if err != nil {+ t.Fatalf("pinned dial did not reach the pinned listener: %v", err)+ }+ _ = conn.Close()+}++// TestDialRealIMAPClientPinsToLoopback verifies the IMAP production dial is+// SSRF-guarded and pinned: under the test hook it connects to the loopback+// listener (failing only at the TLS handshake, never at the TCP dial), proving+// the dial target is the validated IP rather than a re-resolved hostname.+func TestDialRealIMAPClientPinsToLoopback(t *testing.T) {+ withConnectorLoopbackTestHook(t)+ ln, err := net.Listen("tcp4", "127.0.0.1:0")+ if err != nil {+ t.Fatalf("listen: %v", err)+ }+ defer ln.Close()+ go func() {+ for {+ conn, acceptErr := ln.Accept()+ if acceptErr != nil {+ return+ }+ _ = conn.Close()+ }+ }()+ _, portStr, err := net.SplitHostPort(ln.Addr().String())+ if err != nil {+ t.Fatal(err)+ }+ port, err := strconv.Atoi(portStr)+ if err != nil {+ t.Fatal(err)+ }+ _, err = dialRealIMAPClient(context.Background(), "127.0.0.1", port, "user", "pass")+ if err == nil {+ t.Fatalf("expected a TLS handshake failure against a plain TCP listener")+ }+ if strings.Contains(err.Error(), "connection refused") {+ t.Fatalf("dial was not pinned to the loopback listener: %v", err)+ }+}++// TestDialRealIMAPClientRejectsPrivateHost verifies IMAP config-time validation+// rejects a non-public host before any dial.+func TestDialRealIMAPClientRejectsPrivateHost(t *testing.T) {+ connectorAllowLoopbackForTest = false+ t.Cleanup(func() { connectorAllowLoopbackForTest = false })+ _, err := dialRealIMAPClient(context.Background(), "10.0.0.5", 993, "user", "pass")+ if err == nil || !strings.Contains(err.Error(), "public address") {+ t.Fatalf("dialRealIMAPClient(10.0.0.5) = %v, want public-address rejection", err)+ }+}++func TestMySQLConnectorValidateRejectsPrivateHost(t *testing.T) {+ connectorAllowLoopbackForTest = false+ t.Cleanup(func() { connectorAllowLoopbackForTest = false })+ c, err := NewMySQLConnector(map[string]any{+ "host": "10.0.0.5",+ "port": "3306",+ "database": "db",+ "credentials": map[string]any{+ "username": "root",+ "password": "secret",+ },+ })+ if err != nil {+ t.Fatalf("NewMySQLConnector: %v", err)+ }+ if err := c.Validate(context.Background()); err == nil || !strings.Contains(err.Error(), "public address") {+ t.Fatalf("Validate(10.0.0.5) = %v, want public-address rejection", err)+ }+}++func TestPostgreSQLConnectorValidateRejectsPrivateHost(t *testing.T) {+ connectorAllowLoopbackForTest = false+ t.Cleanup(func() { connectorAllowLoopbackForTest = false })+ c, err := NewPostgreSQLConnector(map[string]any{+ "host": "10.0.0.5",+ "port": "5432",+ "database": "db",+ "credentials": map[string]any{+ "username": "postgres",+ "password": "secret",+ },+ })+ if err != nil {+ t.Fatalf("NewPostgreSQLConnector: %v", err)+ }+ if err := c.Validate(context.Background()); err == nil || !strings.Contains(err.Error(), "public address") {+ t.Fatalf("Validate(10.0.0.5) = %v, want public-address rejection", err)+ }+}++// TestMySQLConnectorOpenAllowsLoopbackUnderHook verifies the loopback test hook+// lets host-based connectors build a client against a local listener; sql.Open+// is lazy so no server is required.+func TestMySQLConnectorOpenAllowsLoopbackUnderHook(t *testing.T) {+ withConnectorLoopbackTestHook(t)+ c, err := NewMySQLConnector(map[string]any{+ "host": "127.0.0.1",+ "port": "3306",+ "database": "db",+ "credentials": map[string]any{+ "username": "root",+ "password": "secret",+ },+ })+ if err != nil {+ t.Fatalf("NewMySQLConnector: %v", err)+ }+ db, err := c.open()+ if err != nil {+ t.Fatalf("open with loopback host under hook: %v", err)+ }+ _ = db.Close()+}++func TestPostgreSQLConnectorOpenAllowsLoopbackUnderHook(t *testing.T) {+ withConnectorLoopbackTestHook(t)+ c, err := NewPostgreSQLConnector(map[string]any{+ "host": "127.0.0.1",+ "port": "5432",+ "database": "db",+ "credentials": map[string]any{+ "username": "postgres",+ "password": "secret",+ },+ })+ if err != nil {+ t.Fatalf("NewPostgreSQLConnector: %v", err)+ }+ db, err := c.open()+ if err != nil {+ t.Fatalf("open with loopback host under hook: %v", err)+ }+ _ = db.Close()+}++// TestValidateS3CompatibleEndpointSSRF verifies the user-controlled endpoint+// is rejected when it resolves to non-public addresses and allowed for+// loopback under the test hook.+func TestValidateS3CompatibleEndpointSSRF(t *testing.T) {+ connectorAllowLoopbackForTest = false+ t.Cleanup(func() { connectorAllowLoopbackForTest = false })+ for _, endpoint := range []string{"http://10.0.0.5", "https://169.254.169.254", "http://127.0.0.1"} {+ if err := validateS3CompatibleEndpoint(endpoint); err == nil {+ t.Fatalf("endpoint %q should be rejected", endpoint)+ }+ }+ withConnectorLoopbackTestHook(t)+ if err := validateS3CompatibleEndpoint("http://127.0.0.1"); err != nil {+ t.Fatalf("loopback endpoint should be allowed under the test hook: %v", err)+ }+}++// TestS3PinnedTransportPinsDial verifies the S3-compatible transport validates+// and pins every outbound dial against the SSRF guard.+func TestS3PinnedTransportPinsDial(t *testing.T) {+ transport := s3PinnedHTTPClient().Transport.(*http.Transport)++ connectorAllowLoopbackForTest = false+ t.Cleanup(func() { connectorAllowLoopbackForTest = false })+ if _, err := transport.DialContext(context.Background(), "tcp", "127.0.0.1:9999"); err == nil {+ t.Fatalf("loopback dial should be rejected without the test hook")+ }++ withConnectorLoopbackTestHook(t)+ ln, err := net.Listen("tcp4", "127.0.0.1:0")+ if err != nil {+ t.Fatalf("listen: %v", err)+ }+ defer ln.Close()+ go func() {+ for {+ conn, acceptErr := ln.Accept()+ if acceptErr != nil {+ return+ }+ _ = conn.Close()+ }+ }()+ conn, err := transport.DialContext(context.Background(), "tcp", ln.Addr().String())+ if err != nil {+ t.Fatalf("pinned dial did not reach the listener: %v", err)+ }+ _ = conn.Close()+}++// TestS3PinnedHTTPClientRedirectPolicy locks in the SDK-faithful redirect+// policy: 301/302/303 are returned to the AWS SDK (which re-signs and retries+// S3 region redirects) instead of being auto-followed by http.Client, while+// method-preserving 307/308 are followed like the SDK default.+func TestS3PinnedHTTPClientRedirectPolicy(t *testing.T) {+ client := s3PinnedHTTPClient()+ if client.CheckRedirect == nil {+ t.Fatalf("s3PinnedHTTPClient must set CheckRedirect so the AWS SDK processes 3xx itself")+ }+ req, err := http.NewRequest(http.MethodGet, "https://bucket.s3.example.com/obj", nil)+ if err != nil {+ t.Fatal(err)+ }+ for _, code := range []int{http.StatusMovedPermanently, http.StatusFound, http.StatusSeeOther} {+ req.Response = &http.Response{StatusCode: code, Request: req}+ if err := client.CheckRedirect(req, nil); err != http.ErrUseLastResponse {+ t.Fatalf("CheckRedirect(%d) = %v, want http.ErrUseLastResponse", code, err)+ }+ }+ for _, code := range []int{http.StatusTemporaryRedirect, http.StatusPermanentRedirect} {+ req.Response = &http.Response{StatusCode: code, Request: req}+ if err := client.CheckRedirect(req, nil); err != nil {+ t.Fatalf("CheckRedirect(%d) = %v, want nil (follow)", code, err)+ }+ }+}diff --git a/internal/syncer/connector/teams.go b/internal/syncer/connector/teams.goindex c33d30ce98..040871ffff 100644--- a/internal/syncer/connector/teams.go+++ b/internal/syncer/connector/teams.go@@ -28,8 +28,6 @@ import ("strings""sync""time"-- "ragflow/internal/utility")const (@@ -330,27 +328,22 @@ func (c *TeamsConnector) getJSON(ctx context.Context, apiURL string, out any) erif err != nil {return err}- hostname, resolvedIP, err := utility.AssertURLSafe(apiURL)- if err != nil {- return err- }-var lastErr errorretriedUnauthorized := falsefor attempt := 1; attempt <= teamsRetryCount; attempt++ {requestCtx, cancel := context.WithTimeout(ctx, teamsRequestTimeout)- req, err := http.NewRequestWithContext(requestCtx, http.MethodGet, apiURL, nil)- if err != nil {- cancel()- return err- }- req.Header.Set("Authorization", "Bearer "+token)- req.Header.Set("Accept", "application/json")-- client := utility.PinnedHTTPClient(hostname, resolvedIP, teamsRequestTimeout)- resp, err := client.Do(req)+ resp, err := connectorRequest(requestCtx, connectorRequestOptions{+ Method: http.MethodGet,+ RawURL: apiURL,+ Headers: map[string]string{"Authorization": "Bearer " + token, "Accept": "application/json"},+ Timeout: teamsRequestTimeout,+ })if err != nil {cancel()+ var unsafe *connectorUnsafeURLError+ if errors.As(err, &unsafe) {+ return unsafe.Err+ }lastErr = err} else {body, readErr := io.ReadAll(io.LimitReader(resp.Body, 32*1024*1024))diff --git a/internal/syncer/connector/webdav.go b/internal/syncer/connector/webdav.goindex 012b41f77c..0f58d1c2db 100644--- a/internal/syncer/connector/webdav.go+++ b/internal/syncer/connector/webdav.go@@ -17,7 +17,6 @@package connectorimport (- "bytes""context""crypto/tls""crypto/x509"@@ -142,6 +141,9 @@ func (c *WebDAVConnector) Validate(ctx context.Context) error {if c.baseURL == "" {return fmt.Errorf("WebDAV base URL is required")}+ if err := validateConnectorURL(c.baseURL); err != nil {+ return err+ }if c.username == "" || c.password == "" {return fmt.Errorf("WebDAV requires username and password credentials")}@@ -406,16 +408,18 @@ func (c *webdavClient) propfind(ctx context.Context, target string) ([]webdavFilif err != nil {return nil, err}- req, err := http.NewRequestWithContext(ctx, "PROPFIND", resolved, bytes.NewReader([]byte(webdavPropfindBody)))- if err != nil {- return nil, err- }- req.Header.Set("Depth", "1")- req.Header.Set("Content-Type", "application/xml")+ headers := map[string]string{"Depth": "1", "Content-Type": "application/xml"}if c.username != "" {- req.SetBasicAuth(c.username, c.password)- }- resp, err := c.httpClient.Do(req)+ headers["Authorization"] = "Basic " + basicAuthHeader(c.username, c.password)+ }+ resp, err := connectorRequest(ctx, connectorRequestOptions{+ Method: "PROPFIND",+ RawURL: resolved,+ Body: []byte(webdavPropfindBody),+ Headers: headers,+ Timeout: webdavRequestTimeout,+ Base: c.httpClient,+ })if err != nil {return nil, err}@@ -432,14 +436,17 @@ func (c *webdavClient) propfind(ctx context.Context, target string) ([]webdavFil// download fetches a file body over GET.func (c *webdavClient) download(ctx context.Context, fileURL string) ([]byte, error) {- req, err := http.NewRequestWithContext(ctx, http.MethodGet, fileURL, nil)- if err != nil {- return nil, err- }+ headers := map[string]string{}if c.username != "" {- req.SetBasicAuth(c.username, c.password)- }- resp, err := c.httpClient.Do(req)+ headers["Authorization"] = "Basic " + basicAuthHeader(c.username, c.password)+ }+ resp, err := connectorRequest(ctx, connectorRequestOptions{+ Method: http.MethodGet,+ RawURL: fileURL,+ Headers: headers,+ Timeout: webdavRequestTimeout,+ Base: c.httpClient,+ })if err != nil {return nil, err}diff --git a/internal/syncer/connector/webdav_test.go b/internal/syncer/connector/webdav_test.goindex 392b252afa..ac195b18f7 100644--- a/internal/syncer/connector/webdav_test.go+++ b/internal/syncer/connector/webdav_test.go@@ -54,6 +54,7 @@ func newWebDAVMountedTestServer(t *testing.T, mountPath string, files map[stringfunc newWebDAVTestServerAtMount(t *testing.T, mountPath string, files map[string]webdavTestFile) (*httptest.Server, *[]string, *[]string) {t.Helper()+ withConnectorLoopbackTestHook(t)var authHeaders []stringvar getRequests []stringhandler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {@@ -668,6 +669,7 @@ func TestWebDAVConnectorFiltersAndImages(t *testing.T) {// containing absolute or protocol-relative hrefs to a different origin are rejected// at URL resolution time, preventing any GET to the external host.func TestWebDAVConnectorOpenSyncRejectsExternalHref(t *testing.T) {+ withConnectorLoopbackTestHook(t)t.Setenv("BLOB_STORAGE_SIZE_THRESHOLD", "20")var externalRequests intdiff --git a/internal/syncer/connector/xquik_test.go b/internal/syncer/connector/xquik_test.goindex 46f1923139..d2dc7584a5 100644--- a/internal/syncer/connector/xquik_test.go+++ b/internal/syncer/connector/xquik_test.go@@ -56,11 +56,11 @@ func newXquikTestConnector(t *testing.T, config map[string]any) *XquikConnectorfunc withXquikTestServer(t *testing.T, connector *XquikConnector, handler http.Handler) *httptest.Server {t.Helper()server := httptest.NewServer(handler)- previous := restAPISSRFAllowLoopback- restAPISSRFAllowLoopback = true+ previous := connectorAllowLoopbackForTest+ connectorAllowLoopbackForTest = trueconnector.baseURL = server.URLt.Cleanup(func() {- restAPISSRFAllowLoopback = previous+ connectorAllowLoopbackForTest = previousserver.Close()})return serverdiff --git a/internal/utility/ssrf.go b/internal/utility/ssrf.goindex 44735de857..e20a36d9f0 100644--- a/internal/utility/ssrf.go+++ b/internal/utility/ssrf.go@@ -34,11 +34,14 @@ var AllowedURLSchemes = []string{"http", "https"}// LookupHost is the indirection used to resolve hostnames. Tests override it.var LookupHost = net.LookupHost-// AllowAnyHostForTest is a test-only override that bypasses the-// SSRF guard (no public-IP check, no DNS resolution, no DNS-// pinning). Production code MUST leave this at its zero value-// (false). Tests that need to talk to a local httptest server-// flip it on and reset it in t.Cleanup.+// AllowAnyHostForTest is a test-only override that skips the+// public-IP routability check in AssertURLSafe and AssertHostSafe.+// Scheme, host, and DNS resolution checks are unchanged: hostnames+// still resolve, unresolvable hostnames still fail, and callers+// still pin connections to the returned resolved address. Production+// code MUST leave this at its zero value (false). Tests that need to+// talk to a local httptest server flip it on and reset it in+// t.Cleanup.//// The previous form (env-var ALLOW_ANY_HOST) was a live runtime// toggle that any operator could flip to disable the SSRF guard@@ -105,6 +108,54 @@ var AssertURLSafe = func(rawURL string) (hostname, resolvedIP string, err error)return hostname, resolvedIP, nil}+// AssertHostSafe validates a bare host (a hostname or a literal IP, with no+// scheme or port) and returns the first resolved public IP. It is the+// host-type counterpart of AssertURLSafe: every resolved address must be+// globally routable (private, loopback, link-local, metadata, multicast and+// reserved ranges are rejected). Callers dial the returned IP directly so DNS+// cannot rebind the connection to an internal address between validation and+// the TCP connect.+//+// Used by host-based data sources (IMAP/MySQL/PostgreSQL) and by the ExeSQL /+// test_db_connection host guards, mirroring common/ssrf_guard.py:+// assert_host_is_safe.+var AssertHostSafe = func(host string) (resolvedIP string, err error) {+ host = strings.TrimSpace(host)+ if host == "" {+ return "", fmt.Errorf("host is missing")+ }++ allowAny := allowAnyHost()+ if ip := net.ParseIP(host); ip != nil {+ if !allowAny && !isGlobalIP(effectiveIP(ip)) {+ return "", fmt.Errorf("host is not a public address (%s), which is not allowed", ip.String())+ }+ return ip.String(), nil+ }++ addresses, err := LookupHost(host)+ if err != nil {+ return "", fmt.Errorf("could not resolve hostname '%s': %w", host, err)+ }+ if len(addresses) == 0 {+ return "", fmt.Errorf("hostname '%s' resolved to no addresses", host)+ }++ for _, addr := range addresses {+ ip := net.ParseIP(addr)+ if ip == nil {+ return "", fmt.Errorf("could not parse resolved address '%s' for hostname '%s'", addr, host)+ }+ if !allowAny && !isGlobalIP(effectiveIP(ip)) {+ return "", fmt.Errorf("hostname '%s' resolves to a non-public address (%s), which is not allowed", host, ip.String())+ }+ if resolvedIP == "" {+ resolvedIP = ip.String()+ }+ }+ return resolvedIP, nil+}+// effectiveIP unwraps IPv4-mapped IPv6 addresses (e.g. ::ffff:127.0.0.1) so// the routability check sees the IPv4 form. Without this, an attacker could// bypass the guard with an IPv4-mapped IPv6 representation of a private host.