Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

fix(go): harden connector SSRF (#19694)

mkaaad59a103073e60
diff --git a/internal/syncer/connector/bitbucket.go b/internal/syncer/connector/bitbucket.go
index 4412c3c997..eeb5a41d2d 100644
--- a/internal/syncer/connector/bitbucket.go
+++ b/internal/syncer/connector/bitbucket.go
@@ -29,8 +29,6 @@ import (
"strconv"
"strings"
"time"
-
- "ragflow/internal/utility"
)
const (
@@ -402,18 +400,16 @@ func (c *BitbucketConnector) getJSONWithRetry(ctx context.Context, apiURL string
// get performs one SSRF-safe authenticated GET request.
func (c *BitbucketConnector) get(ctx context.Context, apiURL string) (*http.Response, error) {
- hostname, resolvedIP, err := utility.AssertURLSafe(apiURL)
- if err != nil {
- return nil, err
- }
- req, err := http.NewRequestWithContext(ctx, http.MethodGet, apiURL, nil)
+ resp, err := connectorRequest(ctx, connectorRequestOptions{
+ Method: http.MethodGet,
+ RawURL: apiURL,
+ Headers: map[string]string{"Accept": "application/json", "Authorization": "Basic " + basicAuthHeader(c.email, c.apiToken)},
+ Timeout: bitbucketRequestTimeout,
+ })
if err != nil {
- return nil, err
+ return nil, connectorUnsafeErr(err)
}
- req.Header.Set("Accept", "application/json")
- req.SetBasicAuth(c.email, c.apiToken)
- client := utility.PinnedHTTPClient(hostname, resolvedIP, bitbucketRequestTimeout)
- return client.Do(req)
+ return resp, nil
}
func sleepFor(ctx context.Context, duration time.Duration) error {
diff --git a/internal/syncer/connector/confluence.go b/internal/syncer/connector/confluence.go
index a8c025ee3d..9d665b04b7 100644
--- a/internal/syncer/connector/confluence.go
+++ b/internal/syncer/connector/confluence.go
@@ -113,6 +113,9 @@ func (c *ConfluenceConnector) Validate(ctx context.Context) error {
if parsed.Scheme != "http" && parsed.Scheme != "https" {
return fmt.Errorf("Confluence wiki_base must use HTTP or HTTPS")
}
+ if err := validateConnectorURL(c.wikiBase); err != nil {
+ return err
+ }
if c.accessToken == "" {
return fmt.Errorf("Confluence access token is required")
}
@@ -419,17 +422,19 @@ func (c *ConfluenceConnector) do(ctx context.Context, method, rawURL string) ([]
if err != nil {
return nil, err
}
- req, err := http.NewRequestWithContext(ctx, method, resolved, nil)
- if err != nil {
- return nil, err
- }
- req.Header.Set("Accept", "application/json")
+ headers := map[string]string{"Accept": "application/json"}
if c.isCloud || c.username != "" {
- req.SetBasicAuth(c.username, c.accessToken)
+ headers["Authorization"] = "Basic " + basicAuthHeader(c.username, c.accessToken)
} else {
- req.Header.Set("Authorization", "Bearer "+c.accessToken)
- }
- res, err := c.client.Do(req)
+ headers["Authorization"] = "Bearer " + c.accessToken
+ }
+ res, err := connectorRequest(ctx, connectorRequestOptions{
+ Method: method,
+ RawURL: resolved,
+ Headers: headers,
+ Timeout: confluenceRequestTimeout,
+ Base: c.client,
+ })
if err != nil {
return nil, err
}
diff --git a/internal/syncer/connector/confluence_test.go b/internal/syncer/connector/confluence_test.go
index 5765981077..a394bbbb3b 100644
--- a/internal/syncer/connector/confluence_test.go
+++ b/internal/syncer/connector/confluence_test.go
@@ -199,6 +199,7 @@ func newTestConfluenceConnector(t *testing.T, wikiBase string, overrides map[str
func newConfluenceFixtureServer(t *testing.T, validateStatus int) *httptest.Server {
t.Helper()
+ withConnectorLoopbackTestHook(t)
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if validateStatus != http.StatusOK && strings.HasPrefix(r.URL.Path, "/wiki/rest/api/space") {
http.Error(w, "unauthorized", validateStatus)
diff --git a/internal/syncer/connector/github.go b/internal/syncer/connector/github.go
index 3bdc29e181..47397033da 100644
--- a/internal/syncer/connector/github.go
+++ b/internal/syncer/connector/github.go
@@ -29,8 +29,6 @@ import (
"strconv"
"strings"
"time"
-
- "ragflow/internal/utility"
)
const (
@@ -385,20 +383,22 @@ func (c *GitHubConnector) getJSON(ctx context.Context, apiURL string, out any) (
if c.doJSON != nil {
return c.doJSON(ctx, apiURL, out)
}
- hostname, resolvedIP, err := utility.AssertURLSafe(apiURL)
- if err != nil {
- return nil, err
- }
- client := utility.PinnedHTTPClient(hostname, resolvedIP, githubRequestTimeout)
- req, err := http.NewRequestWithContext(ctx, http.MethodGet, apiURL, nil)
- if err != nil {
- return nil, err
- }
- req.Header.Set("Accept", "application/vnd.github+json")
- req.Header.Set("X-GitHub-Api-Version", "2022-11-28")
- req.Header.Set("Authorization", "Bearer "+c.token)
- resp, err := client.Do(req)
+ resp, err := connectorRequest(ctx, connectorRequestOptions{
+ Method: http.MethodGet,
+ RawURL: apiURL,
+ Validate: assertConnectorURLSafeHTTPS,
+ Headers: map[string]string{
+ "Accept": "application/vnd.github+json",
+ "X-GitHub-Api-Version": "2022-11-28",
+ "Authorization": "Bearer " + c.token,
+ },
+ Timeout: githubRequestTimeout,
+ })
if err != nil {
+ var unsafe *connectorUnsafeURLError
+ if errors.As(err, &unsafe) {
+ return nil, unsafe.Err
+ }
return nil, fmt.Errorf("failed to fetch GitHub API: %w", err)
}
defer resp.Body.Close()
diff --git a/internal/syncer/connector/gitlab.go b/internal/syncer/connector/gitlab.go
index 44eb79ce09..edc04ed76b 100644
--- a/internal/syncer/connector/gitlab.go
+++ b/internal/syncer/connector/gitlab.go
@@ -19,6 +19,7 @@ package connector
import (
"context"
"encoding/json"
+ "errors"
"fmt"
"io"
"net/http"
@@ -27,8 +28,6 @@ import (
"strconv"
"strings"
"time"
-
- "ragflow/internal/utility"
)
const (
@@ -298,19 +297,18 @@ func (c *GitlabConnector) getJSON(ctx context.Context, apiURL string, out any) (
if c.doJSON != nil {
return c.doJSON(ctx, apiURL, out)
}
- hostname, resolvedIP, err := utility.AssertURLSafe(apiURL)
- if err != nil {
- return nil, err
- }
- client := utility.PinnedHTTPClient(hostname, resolvedIP, gitlabRequestTimeout)
- req, err := http.NewRequestWithContext(ctx, http.MethodGet, apiURL, nil)
- if err != nil {
- return nil, err
- }
- req.Header.Set("Accept", "application/json")
- req.Header.Set("PRIVATE-TOKEN", c.token)
- resp, err := client.Do(req)
+ resp, err := connectorRequest(ctx, connectorRequestOptions{
+ Method: http.MethodGet,
+ RawURL: apiURL,
+ Validate: assertConnectorURLSafeHTTPS,
+ Headers: map[string]string{"Accept": "application/json", "PRIVATE-TOKEN": c.token},
+ Timeout: gitlabRequestTimeout,
+ })
if err != nil {
+ var unsafe *connectorUnsafeURLError
+ if errors.As(err, &unsafe) {
+ return nil, unsafe.Err
+ }
return nil, fmt.Errorf("failed to fetch GitLab API: %w", err)
}
defer resp.Body.Close()
@@ -329,18 +327,18 @@ func (c *GitlabConnector) getRaw(ctx context.Context, apiURL string) ([]byte, er
if c.doRaw != nil {
return c.doRaw(ctx, apiURL)
}
- hostname, resolvedIP, err := utility.AssertURLSafe(apiURL)
- if err != nil {
- return nil, err
- }
- client := utility.PinnedHTTPClient(hostname, resolvedIP, gitlabRequestTimeout)
- req, err := http.NewRequestWithContext(ctx, http.MethodGet, apiURL, nil)
- if err != nil {
- return nil, err
- }
- req.Header.Set("PRIVATE-TOKEN", c.token)
- resp, err := client.Do(req)
+ resp, err := connectorRequest(ctx, connectorRequestOptions{
+ Method: http.MethodGet,
+ RawURL: apiURL,
+ Validate: assertConnectorURLSafeHTTPS,
+ Headers: map[string]string{"PRIVATE-TOKEN": c.token},
+ Timeout: gitlabRequestTimeout,
+ })
if err != nil {
+ var unsafe *connectorUnsafeURLError
+ if errors.As(err, &unsafe) {
+ return nil, unsafe.Err
+ }
return nil, fmt.Errorf("failed to fetch GitLab raw file: %w", err)
}
defer resp.Body.Close()
diff --git a/internal/syncer/connector/imap.go b/internal/syncer/connector/imap.go
index d19d430b08..92d2bbd770 100644
--- a/internal/syncer/connector/imap.go
+++ b/internal/syncer/connector/imap.go
@@ -813,7 +813,15 @@ func runIMAPCommand[T any](ctx context.Context, client *imapclient.Client, run f
}
func dialRealIMAPClient(ctx context.Context, host string, port int, username, password string) (imapClient, error) {
- address := net.JoinHostPort(host, strconv.Itoa(port))
+ // The configured host is server-side controlled, so it is guarded by the
+ // shared host-type SSRF check and the TCP dial is pinned to the validated
+ // public IP. The TLS layer still presents the original hostname as SNI so
+ // certificate verification is unchanged.
+ pinIP, err := assertConnectorHostSafe(host)
+ if err != nil {
+ return nil, err
+ }
+ address := net.JoinHostPort(pinIP.String(), strconv.Itoa(port))
dialCtx, cancelDial := context.WithTimeout(ctx, imapDialTimeout)
defer cancelDial()
rawConn, err := (&net.Dialer{}).DialContext(dialCtx, "tcp", address)
diff --git a/internal/syncer/connector/jira.go b/internal/syncer/connector/jira.go
index 73caf58a6f..71a23407db 100644
--- a/internal/syncer/connector/jira.go
+++ b/internal/syncer/connector/jira.go
@@ -17,7 +17,6 @@
package connector
import (
- "bytes"
"context"
"encoding/base64"
"encoding/json"
@@ -112,7 +111,6 @@ func NewJiraConnector(config map[string]any) (*JiraConnector, error) {
commentEmailBlacklist: jiraStringSet(jiraStringSlice(config["comment_email_blacklist"])),
client: &http.Client{Timeout: jiraRequestTimeout},
}
- c.client.CheckRedirect = c.checkRedirect
if len(c.labelsToSkip) == 0 {
c.labelsToSkip = jiraStringSet(strings.Split(os.Getenv("JIRA_CONNECTOR_LABELS_TO_SKIP"), ","))
}
@@ -133,6 +131,9 @@ func (c *JiraConnector) Validate(ctx context.Context) error {
if err != nil || parsed.Scheme == "" || parsed.Host == "" {
return &ConnectorValidationError{Message: "invalid Jira base URL"}
}
+ if err := validateConnectorURL(c.baseURL); err != nil {
+ return &ConnectorValidationError{Message: err.Error()}
+ }
if c.apiToken == "" && (c.userEmail == "" || c.password == "") {
return &ConnectorMissingCredentialError{Message: "Jira credentials must include either an API token or username/password."}
}
@@ -484,29 +485,33 @@ func (c *JiraConnector) syncTimezoneFromServer(ctx context.Context) error {
func (c *JiraConnector) doJiraJSON(ctx context.Context, method, apiPath string, query url.Values, body any, out any) (http.Header, error) {
apiURL := c.apiURL(apiPath, query)
- var reader io.Reader
+ var data []byte
if body != nil {
- data, err := json.Marshal(body)
+ var err error
+ data, err = json.Marshal(body)
if err != nil {
return nil, err
}
- reader = bytes.NewReader(data)
}
- req, err := http.NewRequestWithContext(ctx, method, apiURL, reader)
- if err != nil {
- return nil, err
- }
- c.authorize(req)
+ headers := map[string]string{"Accept": "application/json"}
if body != nil {
- req.Header.Set("Content-Type", "application/json")
- }
- req.Header.Set("Accept", "application/json")
- resp, err := c.client.Do(req)
+ headers["Content-Type"] = "application/json"
+ }
+ resp, err := connectorRequest(ctx, connectorRequestOptions{
+ Method: method,
+ RawURL: apiURL,
+ Body: data,
+ Headers: headers,
+ Timeout: jiraRequestTimeout,
+ Base: c.client,
+ Prepare: func(req *http.Request, _ connectorRequestHop) error { c.authorize(req); return nil },
+ CheckRedirect: c.checkRedirect,
+ })
if err != nil {
return nil, err
}
defer resp.Body.Close()
- data, err := io.ReadAll(io.LimitReader(resp.Body, 32*1024*1024))
+ data, err = io.ReadAll(io.LimitReader(resp.Body, 32*1024*1024))
if err != nil {
return resp.Header, err
}
@@ -534,12 +539,14 @@ func (c *JiraConnector) downloadURL(ctx context.Context, rawURL string) ([]byte,
if !sameJiraOrigin(parsed, base) {
return nil, fmt.Errorf("Jira attachment origin %q does not match the configured base URL", parsed.Scheme+"://"+parsed.Host)
}
- req, err := http.NewRequestWithContext(ctx, http.MethodGet, rawURL, nil)
- if err != nil {
- return nil, err
- }
- c.authorize(req)
- resp, err := c.client.Do(req)
+ resp, err := connectorRequest(ctx, connectorRequestOptions{
+ Method: http.MethodGet,
+ RawURL: rawURL,
+ Timeout: jiraRequestTimeout,
+ Base: c.client,
+ Prepare: func(req *http.Request, _ connectorRequestHop) error { c.authorize(req); return nil },
+ CheckRedirect: c.checkRedirect,
+ })
if err != nil {
return nil, err
}
diff --git a/internal/syncer/connector/jira_test.go b/internal/syncer/connector/jira_test.go
index 8f160f0995..3a2da7b4fa 100644
--- a/internal/syncer/connector/jira_test.go
+++ b/internal/syncer/connector/jira_test.go
@@ -260,6 +260,7 @@ func TestJiraConnectorBuildJQLIgnoresQuotedOrderBy(t *testing.T) {
}
func TestJiraConnectorDownloadRejectsSchemeMismatch(t *testing.T) {
+ withConnectorLoopbackTestHook(t)
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
t.Fatalf("request should not be sent for mismatched scheme")
}))
@@ -283,6 +284,7 @@ func TestJiraConnectorDownloadRejectsSchemeMismatch(t *testing.T) {
}
func TestJiraConnectorDownloadRejectsCrossOriginRedirect(t *testing.T) {
+ withConnectorLoopbackTestHook(t)
destinationHit := false
destination := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
destinationHit = true
@@ -313,6 +315,7 @@ func TestJiraConnectorDownloadRejectsCrossOriginRedirect(t *testing.T) {
}
func TestJiraConnectorDownloadKeepsAuthorizationOnSameOriginRedirect(t *testing.T) {
+ withConnectorLoopbackTestHook(t)
var server *httptest.Server
mux := http.NewServeMux()
mux.HandleFunc("/redirect", func(w http.ResponseWriter, r *http.Request) {
@@ -349,6 +352,7 @@ func TestJiraConnectorDownloadKeepsAuthorizationOnSameOriginRedirect(t *testing.
func jiraFixtureServer(t *testing.T) *httptest.Server {
t.Helper()
+ withConnectorLoopbackTestHook(t)
mux := http.NewServeMux()
mux.HandleFunc("/rest/api/3/project/RAG", func(w http.ResponseWriter, r *http.Request) {
jiraWriteJSON(t, w, map[string]any{"key": "RAG"})
@@ -375,6 +379,7 @@ func jiraFixtureServer(t *testing.T) *httptest.Server {
func jiraResumeFixtureServer(t *testing.T, keys []map[string]any) *httptest.Server {
t.Helper()
+ withConnectorLoopbackTestHook(t)
var server *httptest.Server
mux := http.NewServeMux()
mux.HandleFunc("/rest/api/3/project/RAG", func(w http.ResponseWriter, r *http.Request) {
diff --git a/internal/syncer/connector/moodle.go b/internal/syncer/connector/moodle.go
index 4ccc81aec0..adcc3bf5f0 100644
--- a/internal/syncer/connector/moodle.go
+++ b/internal/syncer/connector/moodle.go
@@ -17,7 +17,6 @@
package connector
import (
- "bytes"
"context"
"encoding/json"
"fmt"
@@ -888,166 +887,44 @@ func capitalizeMoodleType(mtype string) string {
// ---------------------------------------------------------------------------
func validateMoodleURLForSSRF(rawURL string) error {
- parsed, err := url.Parse(rawURL)
- if err != nil {
- return &ConnectorValidationError{Message: "Moodle URL must include a hostname."}
- }
- if parsed.Scheme != "http" && parsed.Scheme != "https" {
- return &ConnectorValidationError{Message: fmt.Sprintf("Unsupported URL scheme for Moodle connector: %q. Only http/https are allowed.", parsed.Scheme)}
- }
- hostname := parsed.Hostname()
- if hostname == "" {
- return &ConnectorValidationError{Message: "Moodle URL must include a hostname."}
- }
- if strings.EqualFold(hostname, "localhost") {
- return &ConnectorValidationError{Message: fmt.Sprintf("Moodle URL hostname %q is not allowed (localhost is blocked).", hostname)}
- }
- addrs, err := net.LookupIP(hostname)
- if err != nil {
- // Resolution failure is not an SSRF condition by itself; the
- // per-request check surfaces it if it matters.
- return nil
- }
- if restAPISSRFAllowLoopback {
- allLoopback := true
- for _, addr := range addrs {
- if !addr.IsLoopback() {
- allLoopback = false
- break
- }
- }
- if allLoopback {
- return nil
- }
- // Not all loopback — fall through to normal validation.
- }
- for _, addr := range addrs {
- if !restAPIIPIsGlobal(restAPIEffectiveIP(addr)) {
- return &ConnectorValidationError{Message: fmt.Sprintf(
- "Moodle URL %q resolves to disallowed address %s (localhost, private, link-local, reserved, or multicast addresses are blocked).",
- rawURL, addr)}
- }
- }
- return nil
+ return validateConnectorURL(rawURL)
}
// moodleAssertURLSafe validates a per-request URL for SSRF, HTTPS, and
// same-origin policy. It returns the hostname and first validated IP so the
// caller can pin DNS for the actual dial.
func moodleAssertURLSafe(ctx context.Context, rawURL, originURL string) (string, net.IP, error) {
- parsed, err := url.Parse(rawURL)
- if err != nil {
- return "", nil, fmt.Errorf("Moodle URL is missing a host.")
- }
- if parsed.Scheme != "http" && parsed.Scheme != "https" {
- return "", nil, fmt.Errorf("Disallowed URL scheme: %q. Only [http https] are allowed.", parsed.Scheme)
- }
- hostname := parsed.Hostname()
- if hostname == "" {
- return "", nil, fmt.Errorf("Moodle URL is missing a host.")
- }
// Reject cross-origin targets: every request must go to the configured
// Moodle host.
if originURL != "" {
if origin, err := url.Parse(originURL); err == nil && origin.Hostname() != "" {
- if !strings.EqualFold(hostname, origin.Hostname()) {
- return "", nil, fmt.Errorf("Moodle URL host %q does not match configured origin %q.", hostname, origin.Hostname())
+ parsed, err := url.Parse(rawURL)
+ if err != nil {
+ return "", nil, fmt.Errorf("Moodle URL is missing a host.")
}
- }
- }
- addrs, err := net.DefaultResolver.LookupIPAddr(ctx, hostname)
- if err != nil {
- return "", nil, fmt.Errorf("Could not resolve hostname %q: %w", hostname, err)
- }
- if len(addrs) == 0 {
- return "", nil, fmt.Errorf("Hostname %q resolved to no addresses.", hostname)
- }
- if restAPISSRFAllowLoopback {
- allLoopback := true
- for _, addr := range addrs {
- if !addr.IP.IsLoopback() {
- allLoopback = false
- break
+ if !strings.EqualFold(parsed.Hostname(), origin.Hostname()) {
+ return "", nil, fmt.Errorf("Moodle URL host %q does not match configured origin %q.", parsed.Hostname(), origin.Hostname())
}
}
- if allLoopback {
- return hostname, addrs[0].IP, nil
- }
- // Not all loopback — fall through to normal validation.
}
- var first net.IP
- for _, addr := range addrs {
- if !restAPIIPIsGlobal(restAPIEffectiveIP(addr.IP)) {
- return "", nil, fmt.Errorf("Moodle URL resolves to a non-public address (%s), which is not allowed.", addr.IP)
- }
- if first == nil {
- first = addr.IP
- }
- }
- if first == nil {
- return "", nil, fmt.Errorf("Hostname %q resolved to no addresses.", hostname)
- }
- return hostname, first, nil
+ return assertConnectorURLSafe(rawURL)
}
// moodleHTTPDo sends an HTTP request with DNS-pinned transport and manual
// redirect handling. Each hop is independently validated for SSRF, HTTPS, and
// same-origin policy, preventing DNS rebinding and redirect-based bypasses.
func (c *MoodleConnector) moodleHTTPDo(ctx context.Context, method, rawURL string, body []byte, headers map[string]string) (*http.Response, error) {
- currentURL := rawURL
- currentMethod := method
- currentBody := body
- for hop := 0; hop <= moodleMaxRedirects; hop++ {
- hostname, pinIP, err := moodleAssertURLSafe(ctx, currentURL, c.moodleURL)
- if err != nil {
- return nil, err
- }
- transport := newRestAPIPinnedTransport(hostname, pinIP)
- client := &http.Client{
- Transport: transport,
- Timeout: moodleRequestTimeout,
- CheckRedirect: func(req *http.Request, via []*http.Request) error {
- return http.ErrUseLastResponse
- },
- }
- var bodyReader io.Reader
- if currentBody != nil {
- bodyReader = bytes.NewReader(currentBody)
- }
- req, err := http.NewRequestWithContext(ctx, currentMethod, currentURL, bodyReader)
- if err != nil {
- transport.CloseIdleConnections()
- return nil, err
- }
- for k, v := range headers {
- req.Header.Set(k, v)
- }
- resp, err := client.Do(req)
- if err != nil {
- transport.CloseIdleConnections()
- return nil, err
- }
- if !restAPIIsRedirect(resp.StatusCode) {
- resp.Body = &restAPICloseIdleBody{body: resp.Body, transport: transport}
- return resp, nil
- }
- location := resp.Header.Get("Location")
- resp.Body.Close()
- transport.CloseIdleConnections()
- if location == "" {
- return nil, fmt.Errorf("Moodle redirect with empty Location header")
- }
- nextURL, err := restAPIResolveURL(currentURL, location)
- if err != nil {
- return nil, err
- }
- currentURL = nextURL
- if resp.StatusCode == http.StatusMovedPermanently || resp.StatusCode == http.StatusFound || resp.StatusCode == http.StatusSeeOther {
- currentMethod = http.MethodGet
- currentBody = nil
- }
- }
- return nil, fmt.Errorf("Moodle request stopped after %d redirects", moodleMaxRedirects)
+ return connectorRequest(ctx, connectorRequestOptions{
+ Method: method,
+ RawURL: rawURL,
+ Body: body,
+ Headers: headers,
+ Timeout: moodleRequestTimeout,
+ MaxRedirects: moodleMaxRedirects,
+ Validate: func(raw string) (string, net.IP, error) {
+ return moodleAssertURLSafe(context.Background(), raw, c.moodleURL)
+ },
+ })
}
// moodleRedactedURL strips query and fragment from a URL so that tokens or
diff --git a/internal/syncer/connector/moodle_test.go b/internal/syncer/connector/moodle_test.go
index f0432ae206..a3b993e900 100644
--- a/internal/syncer/connector/moodle_test.go
+++ b/internal/syncer/connector/moodle_test.go
@@ -14,16 +14,16 @@ import (
func withMoodleTestHooks(t *testing.T) {
t.Helper()
- origLoopback := restAPISSRFAllowLoopback
+ origLoopback := connectorAllowLoopbackForTest
origTries := moodleRetryTries
origBaseDelay := moodleRetryBaseDelay
origBackoff := moodleRetryBackoff
- restAPISSRFAllowLoopback = true
+ connectorAllowLoopbackForTest = true
moodleRetryTries = 2
moodleRetryBaseDelay = time.Millisecond
moodleRetryBackoff = 2
t.Cleanup(func() {
- restAPISSRFAllowLoopback = origLoopback
+ connectorAllowLoopbackForTest = origLoopback
moodleRetryTries = origTries
moodleRetryBaseDelay = origBaseDelay
moodleRetryBackoff = origBackoff
@@ -525,9 +525,9 @@ func TestAddMoodleToken(t *testing.T) {
func TestValidateMoodleURLForSSRF(t *testing.T) {
withMoodleTestHooks(t)
- origLoopback := restAPISSRFAllowLoopback
- restAPISSRFAllowLoopback = false
- defer func() { restAPISSRFAllowLoopback = origLoopback }()
+ origLoopback := connectorAllowLoopbackForTest
+ connectorAllowLoopbackForTest = false
+ defer func() { connectorAllowLoopbackForTest = origLoopback }()
if err := validateMoodleURLForSSRF("ftp://example.com"); err == nil {
t.Fatalf("expected scheme rejection")
@@ -640,9 +640,9 @@ func TestMoodleConnectorMetadataRedactsFileURL(t *testing.T) {
}
func TestMoodleAssertURLSafeRejectsCrossOrigin(t *testing.T) {
- origLoopback := restAPISSRFAllowLoopback
- restAPISSRFAllowLoopback = false
- defer func() { restAPISSRFAllowLoopback = origLoopback }()
+ origLoopback := connectorAllowLoopbackForTest
+ connectorAllowLoopbackForTest = false
+ defer func() { connectorAllowLoopbackForTest = origLoopback }()
_, _, err := moodleAssertURLSafe(context.Background(), "https://evil.com/api", "https://example.com")
if err == nil {
@@ -651,9 +651,9 @@ func TestMoodleAssertURLSafeRejectsCrossOrigin(t *testing.T) {
}
func TestMoodleAssertURLSafeLoopbackAllAddresses(t *testing.T) {
- origLoopback := restAPISSRFAllowLoopback
- restAPISSRFAllowLoopback = true
- defer func() { restAPISSRFAllowLoopback = origLoopback }()
+ origLoopback := connectorAllowLoopbackForTest
+ connectorAllowLoopbackForTest = true
+ defer func() { connectorAllowLoopbackForTest = origLoopback }()
// All loopback → allowed.
_, _, err := moodleAssertURLSafe(context.Background(), "http://127.0.0.1/path", "http://127.0.0.1")
diff --git a/internal/syncer/connector/mysql.go b/internal/syncer/connector/mysql.go
index 6e621183cd..098823cfdf 100644
--- a/internal/syncer/connector/mysql.go
+++ b/internal/syncer/connector/mysql.go
@@ -24,7 +24,9 @@ import (
"encoding/hex"
"fmt"
"io"
+ "net"
"regexp"
+ "strconv"
"strings"
"time"
@@ -70,9 +72,11 @@ func NewMySQLConnector(config map[string]any) (*MySQLConnector, error) {
batchSize: configInt(config["batch_size"], defaultMySQLBatchSize),
username: strings.TrimSpace(stringConfig(credentials["username"])),
password: stringConfig(credentials["password"]),
- openDB: func(dsn string) (*sql.DB, error) {
- return sql.Open("mysql", dsn)
- },
+ }
+ // Production dials through the SSRF-guarded, DNS-pinned openDB. Tests
+ // replace it with an injected openDB that avoids the real network.
+ connector.openDB = func(dsn string) (*sql.DB, error) {
+ return connector.openPinned(dsn)
}
connector.query = connector.sanitizeQuery(stringConfig(config["query"]))
connector.contentColumns = connector.splitColumns(config["content_columns"])
@@ -148,7 +152,10 @@ func (c *MySQLConnector) OpenPrune(ctx context.Context, request PruneRequest) (P
return &mysqlPruneSession{connector: c, db: db, queries: queries, batchSize: c.batchSize}, nil
}
-// open builds a MySQL connection with Python-compatible settings.
+// open builds a MySQL connection with Python-compatible settings. The default
+// openDB (wired in NewMySQLConnector) validates the host against the shared
+// SSRF guard and pins the dial; tests inject an openDB that bypasses the real
+// network.
func (c *MySQLConnector) open() (*sql.DB, error) {
cfg := mysql.NewConfig()
cfg.User = c.username
@@ -162,6 +169,49 @@ func (c *MySQLConnector) open() (*sql.DB, error) {
return c.openDB(cfg.FormatDSN())
}
+// openPinned is the production openDB: it validates the configured host with
+// the shared host-type SSRF guard and routes the connection through a custom
+// network whose dialer is pinned to the validated IP, closing the
+// DNS-rebinding window between validation and the TCP connect. The DSN keeps
+// the original hostname (TLS ServerName / host-based routing), while
+// go-sql-driver resolves the custom network through mysql.RegisterDialContext.
+func (c *MySQLConnector) openPinned(_ string) (*sql.DB, error) {
+ pinIP, err := assertConnectorHostSafe(c.host)
+ if err != nil {
+ return nil, err
+ }
+ network := mysqlPinnedNetwork(c.host, c.port, pinIP)
+ mysql.RegisterDialContext(network, mysqlPinnedDial(pinIP, c.port))
+ cfg := mysql.NewConfig()
+ cfg.User = c.username
+ cfg.Passwd = c.password
+ cfg.Net = network
+ cfg.Addr = net.JoinHostPort(c.host, strconv.Itoa(c.port))
+ cfg.DBName = c.database
+ cfg.Params = map[string]string{"charset": "utf8mb4"}
+ cfg.ParseTime = true
+ cfg.Loc = time.UTC
+ return sql.Open("mysql", cfg.FormatDSN())
+}
+
+// mysqlPinnedNetwork returns a deterministic custom network name for a
+// host/port/IP pin. Each distinct pin gets its own registered network, so a DNS
+// change can never make one connector reuse another connector's dialer. The
+// name is hex-only so it round-trips through go-sql-driver's DSN parser.
+func mysqlPinnedNetwork(host string, port int, pinIP net.IP) string {
+ sum := md5.Sum([]byte(fmt.Sprintf("%s:%d:%s", host, port, pinIP.String())))
+ return "pinned-" + hex.EncodeToString(sum[:8])
+}
+
+// mysqlPinnedDial returns a go-sql-driver dialer that connects every dial on
+// its registered network to pinIP:port, ignoring the host the driver parsed
+// from the DSN.
+func mysqlPinnedDial(pinIP net.IP, port int) mysql.DialContextFunc {
+ return func(ctx context.Context, _ string) (net.Conn, error) {
+ return (&net.Dialer{}).DialContext(ctx, "tcp", net.JoinHostPort(pinIP.String(), strconv.Itoa(port)))
+ }
+}
+
// baseQueries returns the configured query or a SELECT per table.
func (c *MySQLConnector) baseQueries(ctx context.Context, db *sql.DB) ([]string, error) {
if c.query != "" {
diff --git a/internal/syncer/connector/postgresql.go b/internal/syncer/connector/postgresql.go
index 70d6dcdf29..a581ee9268 100644
--- a/internal/syncer/connector/postgresql.go
+++ b/internal/syncer/connector/postgresql.go
@@ -24,13 +24,15 @@ import (
"encoding/hex"
"fmt"
"io"
+ "net"
"net/url"
"regexp"
"strconv"
"strings"
"time"
- _ "github.com/jackc/pgx/v5/stdlib"
+ "github.com/jackc/pgx/v5"
+ "github.com/jackc/pgx/v5/stdlib"
)
const (
@@ -80,9 +82,11 @@ func NewPostgreSQLConnector(config map[string]any) (*PostgreSQLConnector, error)
password: stringConfig(credentials["password"]),
sslmode: strings.TrimSpace(stringConfig(config["sslmode"])),
connectTimeout: configInt(config["connect_timeout"], defaultPostgresConnectTimeout),
- openDB: func(dsn string) (*sql.DB, error) {
- return sql.Open("pgx", dsn)
- },
+ }
+ // Production dials through the SSRF-guarded, DNS-pinned openDB. Tests
+ // replace it with an injected openDB that avoids the real network.
+ connector.openDB = func(dsn string) (*sql.DB, error) {
+ return connector.openPinned(dsn)
}
if connector.sslmode == "" {
connector.sslmode = "prefer"
@@ -179,6 +183,35 @@ func (c *PostgreSQLConnector) open() (*sql.DB, error) {
return c.openDB(dsn.String())
}
+// openPinned is the production openDB: it validates the configured host with
+// the shared host-type SSRF guard and installs a pgx DialFunc pinned to the
+// validated IP, closing the DNS-rebinding window between validation and the
+// TCP connect. The DSN keeps the original hostname so TLS ServerName /
+// host-based authentication are unchanged; only the underlying TCP dial is
+// rewritten.
+func (c *PostgreSQLConnector) openPinned(dsn string) (*sql.DB, error) {
+ pinIP, err := assertConnectorHostSafe(c.host)
+ if err != nil {
+ return nil, err
+ }
+ config, err := pgx.ParseConfig(dsn)
+ if err != nil {
+ return nil, err
+ }
+ timeout := time.Duration(c.connectTimeout) * time.Second
+ port := strconv.Itoa(int(config.Port))
+ config.DialFunc = postgresPinnedDial(pinIP, port, timeout)
+ return stdlib.OpenDB(*config), nil
+}
+
+// postgresPinnedDial returns a pgx DialFunc that connects every dial to
+// pinIP:port, ignoring the host parsed from the DSN.
+func postgresPinnedDial(pinIP net.IP, port string, timeout time.Duration) func(ctx context.Context, network, addr string) (net.Conn, error) {
+ return func(ctx context.Context, network, _ string) (net.Conn, error) {
+ return (&net.Dialer{Timeout: timeout}).DialContext(ctx, network, net.JoinHostPort(pinIP.String(), port))
+ }
+}
+
// baseQueries returns the configured query or a SELECT per table.
func (c *PostgreSQLConnector) baseQueries(ctx context.Context, db *sql.DB) ([]string, error) {
if c.query != "" {
diff --git a/internal/syncer/connector/rest_api.go b/internal/syncer/connector/rest_api.go
index 6b3002c7de..77dfb3db42 100644
--- a/internal/syncer/connector/rest_api.go
+++ b/internal/syncer/connector/rest_api.go
@@ -18,7 +18,6 @@ package connector
import (
"context"
- "crypto/tls"
"encoding/json"
"errors"
"fmt"
@@ -134,14 +133,10 @@ var (
restAPI429DefaultWait = 30 * time.Second
)
-// restAPISSRFAllowLoopback is a test hook that lets unit tests exercise the
-// real HTTP path against httptest servers bound to loopback. Production code
-// keeps it false so loopback/private endpoints stay blocked.
-var restAPISSRFAllowLoopback bool
-
// NewRestAPIConnector parses a connector config and returns a connector. It
-// performs schema validation and the base-URL SSRF check but performs no
-// network I/O. Credentials are read from config["credentials"].
+// performs schema validation and the base-URL SSRF check (which resolves the
+// hostname to enforce the shared guard) but performs no HTTP request I/O.
+// Credentials are read from config["credentials"].
func NewRestAPIConnector(config map[string]any) (*RestAPIConnector, error) {
cfg, err := parseRestAPIConfig(config)
if err != nil {
@@ -627,160 +622,90 @@ func (c *RestAPIConnector) fetchPageOnce(ctx context.Context, params map[string]
currentHeaders[k] = v
}
currentBody := c.cfg.RequestBody
- currentAuth := c.basicAuth
- previousNetloc := restAPINetloc(currentURL)
-
- for hop := 0; hop <= restAPIMaxRedirects; hop++ {
- hostname, pinIP, err := assertRestAPIURLSafe(ctx, currentURL)
- if err != nil {
- return nil, &ConnectorValidationError{Message: "Unsafe REST API URL: " + err.Error()}
+ var bodyBytes []byte
+ if currentMethod == http.MethodPost {
+ if currentBody == nil {
+ bodyBytes = []byte("{}")
+ } else {
+ var err error
+ bodyBytes, err = json.Marshal(currentBody)
+ if err != nil {
+ return nil, &ConnectorValidationError{Message: "REST API request body is not valid JSON"}
+ }
}
-
- var resp *http.Response
- var respErr error
- for wait := 0; wait < restAPI429MaxWaits; wait++ {
- resp, respErr = c.doRequest(ctx, currentMethod, currentURL, currentHeaders, queryParams, currentBody, currentAuth, hostname, pinIP)
- if respErr != nil {
- return nil, respErr
+ }
+ currentAuth := c.basicAuth
+ previousNetloc := connectorNetloc(currentURL)
+
+ resp, err := connectorRequest(ctx, connectorRequestOptions{
+ Method: currentMethod,
+ RawURL: currentURL,
+ Body: bodyBytes,
+ Headers: currentHeaders,
+ Timeout: restAPIRequestTimeout,
+ MaxRedirects: restAPIMaxRedirects,
+ Prepare: func(req *http.Request, hop connectorRequestHop) error {
+ if hop.Method == http.MethodPost && req.Header.Get("Content-Type") == "" {
+ req.Header.Set("Content-Type", "application/json")
}
- if resp.StatusCode != http.StatusTooManyRequests {
- break
+ if currentAuth != nil {
+ req.SetBasicAuth(currentAuth.username, currentAuth.password)
}
- retryAfter := restAPI429DefaultWait
- if raw := resp.Header.Get("Retry-After"); raw != "" {
- if seconds, err := strconv.Atoi(strings.TrimSpace(raw)); err == nil && seconds >= 0 {
- retryAfter = time.Duration(seconds) * time.Second
+ if encoded := restAPIEncodeParams(queryParams); encoded != "" {
+ sep := "?"
+ if strings.Contains(hop.URL, "?") {
+ sep = "&"
}
+ fullURL, err := url.Parse(hop.URL + sep + encoded)
+ if err != nil {
+ return err
+ }
+ req.URL = fullURL
}
- resp.Body.Close()
- select {
- case <-ctx.Done():
- return nil, ctx.Err()
- case <-time.After(retryAfter):
+ return nil
+ },
+ RetryStatus: func(resp *http.Response, attempt int) (time.Duration, bool) {
+ if resp.StatusCode != http.StatusTooManyRequests {
+ return 0, false
}
- }
- if respErr != nil {
- return nil, respErr
- }
- if resp.StatusCode == http.StatusTooManyRequests {
- resp.Body.Close()
- return nil, &RateLimitTriedTooManyTimesError{Message: fmt.Sprintf("REST API rate limited: exceeded '%d' retries (too many requests)", restAPI429MaxWaits)}
- }
-
- if restAPIIsRedirect(resp.StatusCode) {
- location := resp.Header.Get("Location")
- if location == "" {
- return c.handleResponse(resp)
+ if attempt >= restAPI429MaxWaits {
+ return 0, false
}
- nextURL, err := restAPIResolveURL(currentURL, location)
- if err != nil {
- resp.Body.Close()
- return nil, &ConnectorValidationError{Message: "Unsafe REST API URL: " + err.Error()}
+ wait := restAPI429DefaultWait
+ if raw := resp.Header.Get("Retry-After"); raw != "" {
+ if seconds, err := strconv.Atoi(strings.TrimSpace(raw)); err == nil && seconds >= 0 {
+ wait = time.Duration(seconds) * time.Second
+ }
}
- nextNetloc := restAPINetloc(nextURL)
+ return wait, true
+ },
+ NextHop: func(nextURL string, status int, hop connectorRequestHop) connectorRequestHop {
+ nextNetloc := connectorNetloc(nextURL)
if nextNetloc != "" && nextNetloc != previousNetloc {
- currentHeaders = restAPIStripAuthHeaders(currentHeaders)
currentAuth = nil
}
previousNetloc = nextNetloc
- if resp.StatusCode == http.StatusMovedPermanently || resp.StatusCode == http.StatusFound || resp.StatusCode == http.StatusSeeOther {
- currentMethod = "GET"
- currentBody = nil
+ if status == http.StatusMovedPermanently || status == http.StatusFound || status == http.StatusSeeOther {
+ hop.Method = http.MethodGet
+ hop.Body = nil
queryParams = nil
}
- resp.Body.Close()
- currentURL = nextURL
- continue
- }
- return c.handleResponse(resp)
- }
- return nil, &ConnectorValidationError{Message: fmt.Sprintf("Exceeded %d redirects fetching %q", restAPIMaxRedirects, currentURL)}
-}
-
-func (c *RestAPIConnector) doRequest(
- ctx context.Context,
- method, rawURL string,
- headers map[string]string,
- params map[string]any,
- body map[string]any,
- auth *restAPIBasicAuth,
- hostname string,
- pinIP net.IP,
-) (*http.Response, error) {
- transport := newRestAPIPinnedTransport(hostname, pinIP)
- client := &http.Client{
- Transport: transport,
- Timeout: restAPIRequestTimeout,
- // Redirects are handled manually so auth headers can be stripped on
- // cross-origin hops and each hop is re-validated against SSRF.
- CheckRedirect: func(req *http.Request, via []*http.Request) error {
- return http.ErrUseLastResponse
+ hop.URL = nextURL
+ return hop
},
- }
-
- fullURL := rawURL
- if encoded := restAPIEncodeParams(params); encoded != "" {
- sep := "?"
- if strings.Contains(rawURL, "?") {
- sep = "&"
- }
- fullURL = rawURL + sep + encoded
- }
-
- var req *http.Request
- var err error
- switch method {
- case "GET":
- req, err = http.NewRequestWithContext(ctx, http.MethodGet, fullURL, nil)
- case "POST":
- var payload []byte
- if body == nil {
- payload = []byte("{}")
- } else {
- payload, err = json.Marshal(body)
- if err != nil {
- return nil, &ConnectorValidationError{Message: "REST API request body is not valid JSON"}
- }
- }
- req, err = http.NewRequestWithContext(ctx, http.MethodPost, fullURL, strings.NewReader(string(payload)))
- if err == nil {
- req.Header.Set("Content-Type", "application/json")
- }
- default:
- return nil, &ConnectorValidationError{Message: fmt.Sprintf("Unsupported HTTP method: %s", method)}
- }
+ })
if err != nil {
+ var unsafe *connectorUnsafeURLError
+ if errors.As(err, &unsafe) {
+ return nil, &ConnectorValidationError{Message: "Unsafe REST API URL: " + unsafe.Err.Error()}
+ }
return nil, err
}
- for k, v := range headers {
- req.Header.Set(k, v)
- }
- if auth != nil {
- req.SetBasicAuth(auth.username, auth.password)
- }
- resp, err := client.Do(req)
- if err != nil {
- transport.CloseIdleConnections()
- return nil, err
+ if resp.StatusCode == http.StatusTooManyRequests {
+ resp.Body.Close()
+ return nil, &RateLimitTriedTooManyTimesError{Message: fmt.Sprintf("REST API rate limited: exceeded '%d' retries (too many requests)", restAPI429MaxWaits)}
}
- resp.Body = &restAPICloseIdleBody{body: resp.Body, transport: transport}
- return resp, nil
-}
-
-// restAPICloseIdleBody closes the underlying response body and then releases
-// the pinned transport's idle connections, so the per-request transports do
-// not leak keep-alive sockets during long paginated syncs.
-type restAPICloseIdleBody struct {
- body io.ReadCloser
- transport *http.Transport
-}
-
-func (b *restAPICloseIdleBody) Read(p []byte) (int, error) { return b.body.Read(p) }
-
-func (b *restAPICloseIdleBody) Close() error {
- err := b.body.Close()
- b.transport.CloseIdleConnections()
- return err
+ return c.handleResponse(resp)
}
func (c *RestAPIConnector) handleResponse(resp *http.Response) (any, error) {
@@ -810,26 +735,6 @@ func (c *RestAPIConnector) handleResponse(resp *http.Response) (any, error) {
return value, nil
}
-// newRestAPIPinnedTransport resolves the target once (in assertRestAPIURLSafe)
-// and pins that address for the actual connection, preventing DNS rebinding.
-func newRestAPIPinnedTransport(hostname string, pinIP net.IP) *http.Transport {
- dialer := &net.Dialer{Timeout: restAPIRequestTimeout, KeepAlive: 30 * time.Second}
- return &http.Transport{
- DialContext: func(ctx context.Context, network, addr string) (net.Conn, error) {
- _, port, err := net.SplitHostPort(addr)
- if err != nil {
- port = "443"
- }
- return dialer.DialContext(ctx, network, net.JoinHostPort(pinIP.String(), port))
- },
- TLSClientConfig: &tls.Config{
- ServerName: hostname,
- MinVersion: tls.VersionTLS12,
- },
- ForceAttemptHTTP2: true,
- }
-}
-
// buildURLWithTemplates substitutes {key} placeholders in the URL and returns
// the remaining query parameters.
func (c *RestAPIConnector) buildURLWithTemplates(params map[string]any) (string, map[string]any) {
@@ -877,176 +782,23 @@ func restAPIEncodeParams(params map[string]any) string {
return values.Encode()
}
-func restAPINetloc(rawURL string) string {
- parsed, err := url.Parse(rawURL)
- if err != nil {
- return ""
- }
- return parsed.Host
-}
-
-func restAPIResolveURL(base, location string) (string, error) {
- baseParsed, err := url.Parse(base)
- if err != nil {
- return "", err
- }
- ref, err := url.Parse(location)
- if err != nil {
- return "", err
- }
- return baseParsed.ResolveReference(ref).String(), nil
-}
-
-func restAPIIsRedirect(status int) bool {
- switch status {
- case http.StatusMovedPermanently, http.StatusFound, http.StatusSeeOther,
- http.StatusTemporaryRedirect, http.StatusPermanentRedirect:
- return true
- }
- return false
-}
-
-var restAPIAuthSensitiveHeaders = map[string]struct{}{
- "authorization": {},
- "proxy-authorization": {},
- "apikey": {},
- "api-key": {},
- "x-api-key": {},
- "x-auth-token": {},
-}
-
-func restAPIStripAuthHeaders(headers map[string]string) map[string]string {
- out := make(map[string]string, len(headers))
- for k, v := range headers {
- if _, sensitive := restAPIAuthSensitiveHeaders[strings.ToLower(k)]; sensitive {
- continue
- }
- out[k] = v
- }
- return out
-}
-
// ---------------------------------------------------------------------------
// SSRF protection
// ---------------------------------------------------------------------------
-// validateRestAPIURLForSSRF performs quick deny-list checks plus DNS
-// resolution. Resolution failure is logged and tolerated because the
-// per-request check re-validates.
+// validateRestAPIURLForSSRF is the config-time SSRF check for the REST API
+// connector. It delegates to the shared connector guard, which validates the
+// scheme, rejects localhost and non-public literal addresses, and resolves the
+// hostname. The per-request path re-validates and DNS-pins every hop.
func validateRestAPIURLForSSRF(rawURL string) error {
- parsed, err := url.Parse(rawURL)
- if err != nil {
- return &ConnectorValidationError{Message: "REST API connector URL must include a hostname."}
- }
- if parsed.Scheme != "http" && parsed.Scheme != "https" {
- return &ConnectorValidationError{Message: fmt.Sprintf("Unsupported URL scheme for REST API connector: %q. Only http/https are allowed.", parsed.Scheme)}
- }
- hostname := parsed.Hostname()
- if hostname == "" {
- return &ConnectorValidationError{Message: "REST API connector URL must include a hostname."}
- }
- if strings.EqualFold(hostname, "localhost") {
- return &ConnectorValidationError{Message: fmt.Sprintf("REST API connector URL hostname %q is not allowed (localhost is blocked).", hostname)}
- }
- if restAPISSRFAllowLoopback {
- return nil
- }
- addrs, err := net.LookupIP(hostname)
- if err != nil {
- // DNS failure is not an SSRF condition by itself; the per-request
- // check will surface it if it matters.
- return nil
- }
- for _, addr := range addrs {
- if !restAPIIPIsGlobal(restAPIEffectiveIP(addr)) {
- return &ConnectorValidationError{Message: fmt.Sprintf(
- "REST API connector URL %q resolves to disallowed address %s (localhost, private, link-local, reserved, or multicast addresses are blocked).",
- rawURL, addr)}
- }
- }
- return nil
+ return validateConnectorURL(rawURL)
}
// assertRestAPIURLSafe mirrors ssrf_guard.assert_url_is_safe: every resolved
// address must be globally routable. It returns the hostname and the first
// validated IP so the caller can pin DNS.
-func assertRestAPIURLSafe(ctx context.Context, rawURL string) (string, net.IP, error) {
- parsed, err := url.Parse(rawURL)
- if err != nil {
- return "", nil, fmt.Errorf("URL is missing a host.")
- }
- if parsed.Scheme != "http" && parsed.Scheme != "https" {
- return "", nil, fmt.Errorf("Disallowed URL scheme: %q. Only [http https] are allowed.", parsed.Scheme)
- }
- hostname := parsed.Hostname()
- if hostname == "" {
- return "", nil, fmt.Errorf("URL is missing a host.")
- }
- if restAPISSRFAllowLoopback {
- addrs, err := net.DefaultResolver.LookupIPAddr(ctx, hostname)
- if err != nil {
- return "", nil, fmt.Errorf("Could not resolve hostname %q: %w", hostname, err)
- }
- if len(addrs) == 0 {
- return "", nil, fmt.Errorf("Hostname %q resolved to no addresses.", hostname)
- }
- return hostname, addrs[0].IP, nil
- }
-
- addrs, err := net.DefaultResolver.LookupIPAddr(ctx, hostname)
- if err != nil {
- return "", nil, fmt.Errorf("Could not resolve hostname %q: %w", hostname, err)
- }
- var first net.IP
- for _, addr := range addrs {
- eff := restAPIEffectiveIP(addr.IP)
- if !restAPIIPIsGlobal(eff) {
- return "", nil, fmt.Errorf("URL resolves to a non-public address (%s), which is not allowed.", addr.IP)
- }
- if first == nil {
- first = addr.IP
- }
- }
- if first == nil {
- return "", nil, fmt.Errorf("Hostname %q resolved to no addresses.", hostname)
- }
- return hostname, first, nil
-}
-
-// restAPIEffectiveIP returns the IPv4 equivalent for IPv4-mapped IPv6
-// addresses, mirroring ssrf_guard._effective_ip.
-func restAPIEffectiveIP(ip net.IP) net.IP {
- if v4 := ip.To4(); v4 != nil && len(ip) == net.IPv6len {
- return v4
- }
- return ip
-}
-
-// restAPIIPIsGlobal mirrors ipaddress.is_global for the address classes that
-// matter in practice.
-func restAPIIPIsGlobal(ip net.IP) bool {
- if ip == nil {
- return false
- }
- if ip.IsLoopback() || ip.IsPrivate() || ip.IsUnspecified() || ip.IsMulticast() ||
- ip.IsLinkLocalUnicast() || ip.IsLinkLocalMulticast() || ip.IsInterfaceLocalMulticast() {
- return false
- }
- if v4 := ip.To4(); v4 != nil {
- first := v4[0]
- // 0.0.0.0/8, 100.64.0.0/10 (CGNAT), 192.0.0.0/24 (IETF protocol
- // assignments), 198.18.0.0/15 (benchmarking), 240.0.0.0/4, broadcast.
- if first == 0 || first == 100 || (first == 192 && v4[1] == 0) ||
- (first == 198 && v4[1]&0xfe == 18) || first >= 240 {
- return false
- }
- return true
- }
- // IPv6 documentation range (2001:db8::/32) is not globally routable.
- if len(ip) == net.IPv6len && ip[0] == 0x20 && ip[1] == 0x01 && ip[2] == 0x0d && ip[3] == 0xb8 {
- return false
- }
- return true
+func assertRestAPIURLSafe(_ context.Context, rawURL string) (string, net.IP, error) {
+ return assertConnectorURLSafe(rawURL)
}
// ---------------------------------------------------------------------------
diff --git a/internal/syncer/connector/rest_api_test.go b/internal/syncer/connector/rest_api_test.go
index d978823256..a1d37ac08e 100644
--- a/internal/syncer/connector/rest_api_test.go
+++ b/internal/syncer/connector/rest_api_test.go
@@ -21,6 +21,7 @@ import (
"encoding/json"
"errors"
"io"
+ "net"
"net/http"
"net/http/httptest"
"strings"
@@ -28,11 +29,29 @@ import (
"sync/atomic"
"testing"
"time"
+
+ "ragflow/internal/utility"
)
+// restAPITestPublicIP is the fixed public address used to satisfy the
+// config-time SSRF check for hostname-based test URLs (e.g. example.com)
+// without touching the real resolver, keeping unit tests hermetic.
+const restAPITestPublicIP = "93.184.216.34"
+
+// restAPITestLookupHost fakes DNS resolution for the REST API unit tests:
+// literal IPs resolve to themselves (so the guard's private-address rejection
+// still works), any other hostname resolves to a fixed public address.
+func restAPITestLookupHost(host string) ([]string, error) {
+ if ip := net.ParseIP(host); ip != nil {
+ return []string{ip.String()}, nil
+ }
+ return []string{restAPITestPublicIP}, nil
+}
+
func withRestAPITestHooks(t *testing.T) {
t.Helper()
- origLoopback := restAPISSRFAllowLoopback
+ origLoopback := connectorAllowLoopbackForTest
+ origLookup := utility.LookupHost
origTries := restAPIRetryTries
origBaseDelay := restAPIRetryBaseDelay
origMaxDelay := restAPIRetryMaxDelay
@@ -40,7 +59,8 @@ func withRestAPITestHooks(t *testing.T) {
origJitter := restAPIRetryJitter
orig429Waits := restAPI429MaxWaits
orig429Wait := restAPI429DefaultWait
- restAPISSRFAllowLoopback = true
+ connectorAllowLoopbackForTest = true
+ utility.LookupHost = restAPITestLookupHost
restAPIRetryTries = 3
restAPIRetryBaseDelay = time.Millisecond
restAPIRetryMaxDelay = 10 * time.Millisecond
@@ -49,7 +69,8 @@ func withRestAPITestHooks(t *testing.T) {
restAPI429MaxWaits = 3
restAPI429DefaultWait = time.Millisecond
t.Cleanup(func() {
- restAPISSRFAllowLoopback = origLoopback
+ connectorAllowLoopbackForTest = origLoopback
+ utility.LookupHost = origLookup
restAPIRetryTries = origTries
restAPIRetryBaseDelay = origBaseDelay
restAPIRetryMaxDelay = origMaxDelay
@@ -120,7 +141,7 @@ func (b *restAPITestReadCloser) Close() error { return b.closeErr
func TestRestAPICloseIdleBodyPreservesBodyAndCloseError(t *testing.T) {
closeErr := errors.New("close boom")
body := &restAPITestReadCloser{reader: strings.NewReader("hello"), closeErr: closeErr}
- wrapped := &restAPICloseIdleBody{body: body, transport: &http.Transport{}}
+ wrapped := &connectorCloseIdleBody{body: body, transport: &http.Transport{}}
got, err := io.ReadAll(wrapped)
if err != nil || string(got) != "hello" {
t.Fatalf("read data=%q err=%v", got, err)
@@ -144,7 +165,7 @@ func TestNewRestAPIConnectorValidationErrors(t *testing.T) {
{name: "missing content fields", config: map[string]any{"url": "https://example.com"}, want: "At least one content field must be configured (content_fields)."},
{name: "zero max_pages", config: map[string]any{"url": "https://example.com", "max_pages": 0, "content_fields": "title"}, want: "max_pages must be a positive integer"},
{name: "negative max_pages", config: map[string]any{"url": "https://example.com", "max_pages": -1, "content_fields": "title"}, want: "max_pages must be a positive integer"},
- {name: "bad scheme", config: map[string]any{"url": "ftp://example.com/x", "content_fields": "title"}, want: "Unsupported URL scheme"},
+ {name: "bad scheme", config: map[string]any{"url": "ftp://example.com/x", "content_fields": "title"}, want: "disallowed URL scheme"},
{name: "localhost", config: map[string]any{"url": "http://localhost/x", "content_fields": "title"}, want: "localhost is blocked"},
}
for _, tt := range tests {
@@ -162,13 +183,13 @@ func TestNewRestAPIConnectorValidationErrors(t *testing.T) {
}
func TestNewRestAPIConnectorBlocksPrivateAddress(t *testing.T) {
- restAPISSRFAllowLoopback = false
- defer func() { restAPISSRFAllowLoopback = false }()
+ connectorAllowLoopbackForTest = false
+ defer func() { connectorAllowLoopbackForTest = false }()
_, err := NewRestAPIConnector(map[string]any{
"url": "http://127.0.0.1:8080/api",
"content_fields": "title",
})
- if err == nil || !strings.Contains(err.Error(), "resolves to disallowed address") {
+ if err == nil || !strings.Contains(err.Error(), "resolves to a non-public address") {
t.Fatalf("err=%v want private address rejection", err)
}
}
diff --git a/internal/syncer/connector/s3_compatible.go b/internal/syncer/connector/s3_compatible.go
index 63dd769311..0056437827 100644
--- a/internal/syncer/connector/s3_compatible.go
+++ b/internal/syncer/connector/s3_compatible.go
@@ -20,6 +20,8 @@ import (
"context"
"encoding/json"
"fmt"
+ "net"
+ "net/http"
"net/url"
"strings"
@@ -174,10 +176,56 @@ func (c *S3CompatibleConnector) ensureClient(ctx context.Context) (*s3.Client, e
c.client = s3.NewFromConfig(cfg, func(options *s3.Options) {
options.BaseEndpoint = aws.String(c.endpointURL)
options.UsePathStyle = c.addressingStyle == "path"
+ // The SDK resolves the endpoint host itself (bucket.virtual-host style
+ // included), so pin every connection at dial time against the shared
+ // SSRF guard instead of only validating the configured endpoint once.
+ options.HTTPClient = s3PinnedHTTPClient()
})
return c.client, nil
}
+// s3PinnedHTTPClient returns an HTTP client whose transport validates and
+// DNS-pins every outbound dial against the shared SSRF guard. The request URL
+// keeps the original endpoint hostname (correct TLS SNI / certificate
+// verification) while the TCP connect is pinned to the validated IP, closing
+// the DNS-rebinding window between validation and the actual connection.
+func s3PinnedHTTPClient() *http.Client {
+ return &http.Client{
+ // Mirror the AWS SDK's own default client policy (BuildableClient's
+ // limitedRedirect): never auto-follow 301/302 — S3 region redirects
+ // must be returned to the SDK so it can re-sign and retry against the
+ // correct endpoint. Only method-preserving 307/308 are followed, same
+ // as the SDK default. Without this, Go's default client would follow a
+ // 301/302 before the SDK sees it, replaying a request signed for one
+ // endpoint against another host.
+ CheckRedirect: func(req *http.Request, via []*http.Request) error {
+ if req.Response != nil {
+ switch req.Response.StatusCode {
+ case http.StatusTemporaryRedirect, http.StatusPermanentRedirect:
+ return nil
+ }
+ }
+ return http.ErrUseLastResponse
+ },
+ Transport: &http.Transport{
+ // Ignore environment proxies: HTTP_PROXY / HTTPS_PROXY would route
+ // the connection through a proxy host instead of the pinned IP.
+ Proxy: nil,
+ DialContext: func(ctx context.Context, network, addr string) (net.Conn, error) {
+ host, port, err := net.SplitHostPort(addr)
+ if err != nil {
+ return nil, err
+ }
+ _, pinIP, err := assertConnectorURLSafe("https://" + host)
+ if err != nil {
+ return nil, err
+ }
+ return (&net.Dialer{}).DialContext(ctx, network, net.JoinHostPort(pinIP.String(), port))
+ },
+ },
+ }
+}
+
func (c *S3CompatibleConnector) listObjectPage(ctx context.Context, startAfter string, maxKeys int32) ([]s3Object, string, bool, error) {
return listS3ObjectPage(ctx, c.listObjects, c.ensureClient, c.bucketName, c.prefix, s3CompatibleSource, startAfter, maxKeys)
}
@@ -229,5 +277,12 @@ func validateS3CompatibleEndpoint(raw string) error {
if parsed.Fragment != "" {
return fmt.Errorf("invalid S3-compatible endpoint_url %q: fragment is not allowed", raw)
}
+ // The endpoint host is user-controlled and connected to server-side, so it
+ // must resolve only to public addresses (blocks loopback, RFC1918 and
+ // cloud metadata). The per-connection transport in s3PinnedHTTPClient
+ // re-validates and pins at dial time.
+ if _, _, err := assertConnectorURLSafe(raw); err != nil {
+ return fmt.Errorf("invalid S3-compatible endpoint_url %q: %v", raw, err)
+ }
return nil
}
diff --git a/internal/syncer/connector/s3_compatible_test.go b/internal/syncer/connector/s3_compatible_test.go
index a73939313e..6cd720df87 100644
--- a/internal/syncer/connector/s3_compatible_test.go
+++ b/internal/syncer/connector/s3_compatible_test.go
@@ -7,6 +7,8 @@ import (
"io"
"strings"
"testing"
+
+ "ragflow/internal/utility"
)
func TestS3CompatibleConnectorOpenSyncUsesFingerprintAndFetch(t *testing.T) {
@@ -240,6 +242,7 @@ func TestS3CompatibleConnectorOpenPruneReturnsSlimSnapshot(t *testing.T) {
}
func TestS3CompatibleConnectorOpenPruneStreamsAcrossPages(t *testing.T) {
+ stubS3EndpointResolver(t)
connector, err := NewS3CompatibleConnector(map[string]any{
"bucket_name": "bucket",
"batch_size": 2,
@@ -301,6 +304,7 @@ func TestS3CompatibleConnectorOpenPruneStreamsAcrossPages(t *testing.T) {
func TestS3CompatibleConnectorValidate(t *testing.T) {
ctx := context.Background()
+ stubS3EndpointResolver(t)
missingBucket := &S3CompatibleConnector{endpointURL: "https://s3.example.com", accessKeyID: "access", secretKey: "secret", addressingStyle: "virtual", batchSize: 2}
if err := missingBucket.Validate(ctx); err == nil {
t.Fatalf("expected bucket name validation error")
@@ -416,8 +420,25 @@ func TestS3CompatibleHelpers(t *testing.T) {
}
}
+// stubS3EndpointResolver makes the shared SSRF guard resolve the
+// s3.example.com test endpoint to a stable public IP without touching the
+// network. The production guard resolves real endpoints; unit tests must not
+// depend on DNS.
+func stubS3EndpointResolver(t *testing.T) {
+ t.Helper()
+ orig := utility.LookupHost
+ utility.LookupHost = func(host string) ([]string, error) {
+ if host == "s3.example.com" {
+ return []string{"93.184.216.34"}, nil
+ }
+ return orig(host)
+ }
+ t.Cleanup(func() { utility.LookupHost = orig })
+}
+
func newTestS3CompatibleConnector(t *testing.T, objects []s3Object) *S3CompatibleConnector {
t.Helper()
+ stubS3EndpointResolver(t)
connector, err := NewS3CompatibleConnector(map[string]any{
"bucket_name": "bucket",
"prefix": "docs",
diff --git a/internal/syncer/connector/salesforce.go b/internal/syncer/connector/salesforce.go
index 2db18483dc..6684e88dc5 100644
--- a/internal/syncer/connector/salesforce.go
+++ b/internal/syncer/connector/salesforce.go
@@ -22,6 +22,7 @@ import (
"errors"
"fmt"
"io"
+ "net"
"net/http"
"net/url"
"sort"
@@ -29,8 +30,6 @@ import (
"strings"
"sync"
"time"
-
- "ragflow/internal/utility"
)
const (
@@ -364,22 +363,24 @@ func salesforceHostAllowed(host string) bool {
strings.HasSuffix(host, ".lightning.force.com")
}
+// salesforceAssertURLSafe validates a Salesforce request URL for SSRF and the
+// approved-host policy, returning the hostname and the IP to pin. Every hop of
+// a Salesforce request (including redirects) must stay on an approved
+// Salesforce host over HTTPS so credentials are only ever transmitted to the
+// intended provider.
+func salesforceAssertURLSafe(rawURL string) (string, net.IP, error) {
+ parsed, err := url.Parse(rawURL)
+ if err != nil || !strings.EqualFold(parsed.Scheme, "https") || !salesforceHostAllowed(parsed.Hostname()) {
+ return "", nil, fmt.Errorf("Salesforce request URL must use HTTPS on an approved Salesforce host")
+ }
+ return assertConnectorURLSafe(rawURL)
+}
+
// requestAccessToken performs the OAuth2 client-credentials exchange, validating
// the token endpoint for SSRF, HTTPS, and the approved Salesforce host policy
// before any credentials are transmitted.
func (c *SalesforceConnector) requestAccessToken(ctx context.Context) (salesforceToken, error) {
tokenURL := c.instanceBaseURL() + "/services/oauth2/token"
- hostname, resolvedIP, err := utility.AssertURLSafe(tokenURL)
- if err != nil {
- return salesforceToken{}, &ConnectorMissingCredentialError{Message: fmt.Sprintf("Salesforce token request failed: %v", err)}
- }
- if !salesforceHostAllowed(hostname) {
- return salesforceToken{}, &ConnectorMissingCredentialError{Message: "Salesforce instance_url is not an approved Salesforce host"}
- }
- parsedURL, err := url.Parse(tokenURL)
- if err != nil || !strings.EqualFold(parsedURL.Scheme, "https") {
- return salesforceToken{}, &ConnectorMissingCredentialError{Message: "Salesforce OAuth token endpoint must use HTTPS"}
- }
form := url.Values{
"grant_type": {"client_credentials"},
"client_id": {c.clientID},
@@ -387,13 +388,14 @@ func (c *SalesforceConnector) requestAccessToken(ctx context.Context) (salesforc
}
requestCtx, cancel := context.WithTimeout(ctx, salesforceRequestTimeout)
defer cancel()
- req, err := http.NewRequestWithContext(requestCtx, http.MethodPost, tokenURL, strings.NewReader(form.Encode()))
- if err != nil {
- return salesforceToken{}, &ConnectorMissingCredentialError{Message: fmt.Sprintf("Salesforce token request failed: %v", err)}
- }
- req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
- client := utility.PinnedHTTPClient(hostname, resolvedIP, salesforceRequestTimeout)
- resp, err := client.Do(req)
+ resp, err := connectorRequest(requestCtx, connectorRequestOptions{
+ Method: http.MethodPost,
+ RawURL: tokenURL,
+ Body: []byte(form.Encode()),
+ Headers: map[string]string{"Content-Type": "application/x-www-form-urlencoded"},
+ Timeout: salesforceRequestTimeout,
+ Validate: salesforceAssertURLSafe,
+ })
if err != nil {
return salesforceToken{}, &ConnectorMissingCredentialError{Message: fmt.Sprintf("Salesforce token request failed: %v", err)}
}
@@ -510,26 +512,15 @@ func (c *SalesforceConnector) getJSON(ctx context.Context, path string, out any)
// doGet performs one authenticated GET with SSRF protection.
func (c *SalesforceConnector) doGet(ctx context.Context, apiURL, token string) (int, []byte, error) {
- parsed, err := url.Parse(apiURL)
- if err != nil || !strings.EqualFold(parsed.Scheme, "https") || !salesforceHostAllowed(parsed.Hostname()) {
- return 0, nil, fmt.Errorf("Salesforce request URL must use HTTPS on an approved Salesforce host")
- }
- hostname, resolvedIP, err := utility.AssertURLSafe(apiURL)
- if err != nil {
- return 0, nil, err
- }
- client := utility.PinnedHTTPClient(hostname, resolvedIP, salesforceRequestTimeout)
- requestCtx, cancel := context.WithTimeout(ctx, salesforceRequestTimeout)
- defer cancel()
- req, err := http.NewRequestWithContext(requestCtx, http.MethodGet, apiURL, nil)
- if err != nil {
- return 0, nil, err
- }
- req.Header.Set("Accept", "application/json")
- req.Header.Set("Authorization", "Bearer "+token)
- resp, err := client.Do(req)
+ resp, err := connectorRequest(ctx, connectorRequestOptions{
+ Method: http.MethodGet,
+ RawURL: apiURL,
+ Headers: map[string]string{"Accept": "application/json", "Authorization": "Bearer " + token},
+ Timeout: salesforceRequestTimeout,
+ Validate: salesforceAssertURLSafe,
+ })
if err != nil {
- return 0, nil, err
+ return 0, nil, connectorUnsafeErr(err)
}
defer resp.Body.Close()
body, err := io.ReadAll(io.LimitReader(resp.Body, 32*1024*1024))
diff --git a/internal/syncer/connector/seafile.go b/internal/syncer/connector/seafile.go
index 81a2889e10..401896649c 100644
--- a/internal/syncer/connector/seafile.go
+++ b/internal/syncer/connector/seafile.go
@@ -380,12 +380,15 @@ func (c *SeaFileConnector) defaultAuthenticate(ctx context.Context, username, pa
form.Set("password", password)
reqCtx, cancel := context.WithTimeout(ctx, seafileRequestTimeout)
defer cancel()
- req, err := http.NewRequestWithContext(reqCtx, http.MethodPost, c.seafileURL+"/api2/auth-token/", strings.NewReader(form.Encode()))
- if err != nil {
- return "", err
- }
- req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
- resp, err := c.httpClient.Do(req)
+ resp, err := connectorRequest(reqCtx, connectorRequestOptions{
+ Method: http.MethodPost,
+ RawURL: c.seafileURL + "/api2/auth-token/",
+ Body: []byte(form.Encode()),
+ Headers: map[string]string{"Content-Type": "application/x-www-form-urlencoded"},
+ Timeout: seafileRequestTimeout,
+ MaxRedirects: seafileMaxRedirects,
+ Base: c.httpClient,
+ })
if err != nil {
return "", &ConnectorMissingCredentialError{Message: fmt.Sprintf("Failed to authenticate with SeaFile: %v", err)}
}
@@ -539,66 +542,29 @@ func (c *SeaFileConnector) getBody(ctx context.Context, endpoint string, useRepo
}
reqCtx, cancel := context.WithTimeout(ctx, seafileRequestTimeout)
defer cancel()
- currentURL := c.apiURL(endpoint, useRepoToken, query)
- previousNetloc := restAPINetloc(currentURL)
- for hop := 0; hop <= seafileMaxRedirects; hop++ {
- hostname, pinIP, err := seafileAssertURLSafe(reqCtx, currentURL)
- if err != nil {
- return nil, &ConnectorValidationError{Message: "Unsafe SeaFile URL: " + err.Error()}
- }
- req, err := http.NewRequestWithContext(reqCtx, http.MethodGet, currentURL, nil)
- if err != nil {
- return nil, err
- }
- for key, value := range headers {
- req.Header.Set(key, value)
- }
- transport := newRestAPIPinnedTransport(hostname, pinIP)
- client := &http.Client{
- Transport: transport,
- Timeout: seafileRequestTimeout,
- // Redirects are handled manually so every hop is re-validated
- // for SSRF and DNS-pinned before a connection is made.
- CheckRedirect: func(req *http.Request, via []*http.Request) error {
- return http.ErrUseLastResponse
- },
- }
- resp, err := client.Do(req)
- if err != nil {
- transport.CloseIdleConnections()
- return nil, err
- }
- if !restAPIIsRedirect(resp.StatusCode) {
- resp.Body = &restAPICloseIdleBody{body: resp.Body, transport: transport}
- body, err := readSeaFileBody(resp)
- if err != nil {
- return nil, err
- }
- if resp.StatusCode >= 400 {
- return nil, &seafileHTTPError{Status: resp.StatusCode, Body: string(body), URL: req.URL.String()}
- }
- return body, nil
- }
- location := resp.Header.Get("Location")
- resp.Body.Close()
- transport.CloseIdleConnections()
- if location == "" {
- return nil, fmt.Errorf("SeaFile API redirect with empty Location header")
- }
- nextURL, err := restAPIResolveURL(currentURL, location)
- if err != nil {
- return nil, err
- }
- // Never forward credentials to a different host, matching Go's default
- // redirect handling.
- nextNetloc := restAPINetloc(nextURL)
- if nextNetloc != "" && nextNetloc != previousNetloc {
- headers = restAPIStripAuthHeaders(headers)
+ resp, err := connectorRequest(reqCtx, connectorRequestOptions{
+ Method: http.MethodGet,
+ RawURL: c.apiURL(endpoint, useRepoToken, query),
+ Headers: headers,
+ Timeout: seafileRequestTimeout,
+ MaxRedirects: seafileMaxRedirects,
+ Base: c.httpClient,
+ })
+ if err != nil {
+ var unsafe *connectorUnsafeURLError
+ if errors.As(err, &unsafe) {
+ return nil, &ConnectorValidationError{Message: "Unsafe SeaFile URL: " + unsafe.Err.Error()}
}
- previousNetloc = nextNetloc
- currentURL = nextURL
+ return nil, err
}
- return nil, fmt.Errorf("SeaFile API request exceeded %d redirects", seafileMaxRedirects)
+ body, err := readSeaFileBody(resp)
+ if err != nil {
+ return nil, err
+ }
+ if resp.StatusCode >= 400 {
+ return nil, &seafileHTTPError{Status: resp.StatusCode, Body: string(body), URL: resp.Request.URL.String()}
+ }
+ return body, nil
}
func (c *SeaFileConnector) requestHeaders(useRepoToken bool) (map[string]string, error) {
@@ -837,98 +803,15 @@ func classifySeaFileError(err error) error {
}
func validateSeaFileURLForSSRF(rawURL string) error {
- parsed, err := url.Parse(rawURL)
- if err != nil {
- return &ConnectorValidationError{Message: "Invalid SeaFile server URL."}
- }
- if parsed.Scheme != "http" && parsed.Scheme != "https" {
- return &ConnectorValidationError{Message: fmt.Sprintf("Unsupported SeaFile server URL scheme %q. Only http/https are allowed.", parsed.Scheme)}
- }
- hostname := parsed.Hostname()
- if hostname == "" {
- return &ConnectorValidationError{Message: "SeaFile server URL must include a hostname."}
- }
- if strings.EqualFold(hostname, "localhost") && !restAPISSRFAllowLoopback {
- return &ConnectorValidationError{Message: "SeaFile server URL hostname \"localhost\" is not allowed."}
- }
- addrs, err := net.LookupIP(hostname)
- if err != nil {
- return nil
- }
- if restAPISSRFAllowLoopback {
- allLoopback := true
- for _, addr := range addrs {
- if !addr.IsLoopback() {
- allLoopback = false
- break
- }
- }
- if allLoopback {
- return nil
- }
- }
- for _, addr := range addrs {
- if !restAPIIPIsGlobal(restAPIEffectiveIP(addr)) {
- return &ConnectorValidationError{Message: fmt.Sprintf(
- "SeaFile server URL %q resolves to disallowed address %s (localhost, private, link-local, reserved, or multicast addresses are blocked).",
- rawURL, addr)}
- }
- }
- return nil
+ return validateConnectorURL(rawURL)
}
// seafileAssertURLSafe validates a per-request SeaFile URL for SSRF and
// returns the hostname plus the first validated IP so the caller can pin DNS
// for the actual dial, preventing DNS rebinding between validation and the
// connection.
-func seafileAssertURLSafe(ctx context.Context, rawURL string) (string, net.IP, error) {
- parsed, err := url.Parse(rawURL)
- if err != nil {
- return "", nil, fmt.Errorf("SeaFile URL is missing a host.")
- }
- if parsed.Scheme != "http" && parsed.Scheme != "https" {
- return "", nil, fmt.Errorf("Disallowed SeaFile URL scheme: %q. Only [http https] are allowed.", parsed.Scheme)
- }
- hostname := parsed.Hostname()
- if hostname == "" {
- return "", nil, fmt.Errorf("SeaFile URL is missing a host.")
- }
- if strings.EqualFold(hostname, "localhost") && !restAPISSRFAllowLoopback {
- return "", nil, fmt.Errorf("SeaFile URL hostname %q is not allowed (localhost is blocked).", hostname)
- }
- addrs, err := net.DefaultResolver.LookupIPAddr(ctx, hostname)
- if err != nil {
- return "", nil, fmt.Errorf("Could not resolve hostname %q: %w", hostname, err)
- }
- if len(addrs) == 0 {
- return "", nil, fmt.Errorf("Hostname %q resolved to no addresses.", hostname)
- }
- if restAPISSRFAllowLoopback {
- allLoopback := true
- for _, addr := range addrs {
- if !addr.IP.IsLoopback() {
- allLoopback = false
- break
- }
- }
- if allLoopback {
- return hostname, addrs[0].IP, nil
- }
- // Not all loopback — fall through to normal validation.
- }
- var first net.IP
- for _, addr := range addrs {
- if !restAPIIPIsGlobal(restAPIEffectiveIP(addr.IP)) {
- return "", nil, fmt.Errorf("SeaFile URL resolves to a non-public address (%s), which is not allowed.", addr.IP)
- }
- if first == nil {
- first = addr.IP
- }
- }
- if first == nil {
- return "", nil, fmt.Errorf("Hostname %q resolved to no addresses.", hostname)
- }
- return hostname, first, nil
+func seafileAssertURLSafe(_ context.Context, rawURL string) (string, net.IP, error) {
+ return assertConnectorURLSafe(rawURL)
}
type seafileSyncSession struct {
diff --git a/internal/syncer/connector/seafile_test.go b/internal/syncer/connector/seafile_test.go
index 34df555404..052b34a9db 100644
--- a/internal/syncer/connector/seafile_test.go
+++ b/internal/syncer/connector/seafile_test.go
@@ -231,9 +231,9 @@ func seafileValidationServer(t *testing.T) *httptest.Server {
func TestSeaFileValidateAccountToken(t *testing.T) {
server := seafileValidationServer(t)
defer server.Close()
- orig := restAPISSRFAllowLoopback
- restAPISSRFAllowLoopback = true
- defer func() { restAPISSRFAllowLoopback = orig }()
+ orig := connectorAllowLoopbackForTest
+ connectorAllowLoopbackForTest = true
+ defer func() { connectorAllowLoopbackForTest = orig }()
connector := seafileTestConnector(t, map[string]any{
"seafile_url": server.URL,
@@ -251,9 +251,9 @@ func TestSeaFileValidateAccountToken(t *testing.T) {
func TestSeaFileValidateConnectorSettingUsesRequestUsernamePassword(t *testing.T) {
server := seafileValidationServer(t)
defer server.Close()
- orig := restAPISSRFAllowLoopback
- restAPISSRFAllowLoopback = true
- defer func() { restAPISSRFAllowLoopback = orig }()
+ orig := connectorAllowLoopbackForTest
+ connectorAllowLoopbackForTest = true
+ defer func() { connectorAllowLoopbackForTest = orig }()
receiver := seafileTestConnector(t, map[string]any{
"seafile_url": server.URL,
@@ -275,9 +275,9 @@ func TestSeaFileValidateConnectorSettingUsesRequestUsernamePassword(t *testing.T
func TestSeaFileValidateRepoToken(t *testing.T) {
server := seafileValidationServer(t)
defer server.Close()
- orig := restAPISSRFAllowLoopback
- restAPISSRFAllowLoopback = true
- defer func() { restAPISSRFAllowLoopback = orig }()
+ orig := connectorAllowLoopbackForTest
+ connectorAllowLoopbackForTest = true
+ defer func() { connectorAllowLoopbackForTest = orig }()
connector := seafileTestConnector(t, map[string]any{
"seafile_url": server.URL,
@@ -292,9 +292,9 @@ func TestSeaFileValidateRepoToken(t *testing.T) {
}
func TestSeaFileDefaultListLibrariesFiltersShared(t *testing.T) {
- orig := restAPISSRFAllowLoopback
- restAPISSRFAllowLoopback = true
- defer func() { restAPISSRFAllowLoopback = orig }()
+ orig := connectorAllowLoopbackForTest
+ connectorAllowLoopbackForTest = true
+ defer func() { connectorAllowLoopbackForTest = orig }()
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.URL.Path != "/api2/repos/" || r.Header.Get("Authorization") != "Token token" {
http.Error(w, "unauthorized", http.StatusUnauthorized)
@@ -687,9 +687,9 @@ func TestSeaFileRegistryOpen(t *testing.T) {
}
func TestSeaFileDirentWrapperAndDownloadLinkDecode(t *testing.T) {
- orig := restAPISSRFAllowLoopback
- restAPISSRFAllowLoopback = true
- defer func() { restAPISSRFAllowLoopback = orig }()
+ orig := connectorAllowLoopbackForTest
+ connectorAllowLoopbackForTest = true
+ defer func() { connectorAllowLoopbackForTest = orig }()
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/json")
switch r.URL.Path {
@@ -741,10 +741,10 @@ func TestSeaFileDirentWrapperAndDownloadLinkDecode(t *testing.T) {
}
func TestSeaFileAssertURLSafe(t *testing.T) {
- orig := restAPISSRFAllowLoopback
- defer func() { restAPISSRFAllowLoopback = orig }()
+ orig := connectorAllowLoopbackForTest
+ defer func() { connectorAllowLoopbackForTest = orig }()
- restAPISSRFAllowLoopback = false
+ connectorAllowLoopbackForTest = false
if _, _, err := seafileAssertURLSafe(context.Background(), "ftp://example.com"); err == nil {
t.Fatalf("expected scheme rejection")
}
@@ -758,7 +758,7 @@ func TestSeaFileAssertURLSafe(t *testing.T) {
t.Fatalf("expected loopback rejection")
}
- restAPISSRFAllowLoopback = true
+ connectorAllowLoopbackForTest = true
hostname, pinIP, err := seafileAssertURLSafe(context.Background(), "http://127.0.0.1:8080/seafile")
if err != nil {
t.Fatalf("loopback should be allowed: %v", err)
diff --git a/internal/syncer/connector/ssrf.go b/internal/syncer/connector/ssrf.go
new file mode 100644
index 0000000000..c07498797b
--- /dev/null
+++ b/internal/syncer/connector/ssrf.go
@@ -0,0 +1,489 @@
+//
+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+package connector
+
+import (
+ "bytes"
+ "context"
+ "crypto/tls"
+ "encoding/base64"
+ "errors"
+ "fmt"
+ "io"
+ "net"
+ "net/http"
+ "net/url"
+ "strings"
+ "time"
+
+ "ragflow/internal/utility"
+)
+
+// connectorAssertURLSafe is the SSRF guard shared by all connectors. It is an
+// indirection over utility.AssertURLSafe so unit tests can substitute a stub
+// without touching the shared utility guard (same pattern as sitemap.go and
+// azure_blob.go).
+var connectorAssertURLSafe = utility.AssertURLSafe
+
+// connectorAssertHostSafe is the host-type SSRF guard shared by the host-based
+// connectors (IMAP/MySQL/PostgreSQL). It is an indirection over
+// utility.AssertHostSafe so unit tests can substitute a stub without touching
+// the shared utility guard (same pattern as connectorAssertURLSafe).
+var connectorAssertHostSafe = utility.AssertHostSafe
+
+// assertConnectorHostSafe validates a bare host (hostname or literal IP, no
+// scheme or port) with the shared strict SSRF guard and returns the first
+// validated public IP so callers can pin the dial, preventing DNS rebinding
+// between validation and the connection.
+//
+// When connectorAllowLoopbackForTest is set, loopback-only hosts are allowed
+// (unit tests run against local listeners); anything else falls through to the
+// strict guard, so a mixed private address is still rejected.
+func assertConnectorHostSafe(host string) (net.IP, error) {
+ host = strings.TrimSpace(host)
+ if connectorAllowLoopbackForTest {
+ if ip := net.ParseIP(host); ip != nil {
+ if ip.IsLoopback() {
+ return ip, nil
+ }
+ } else {
+ lower := strings.ToLower(host)
+ if lower == "localhost" || strings.HasSuffix(lower, ".localhost") {
+ if _, ip, ok := loopbackTestAllow(lower); ok {
+ return ip, nil
+ }
+ }
+ }
+ // Not allowed by the test hook — fall through to the strict guard.
+ }
+ ipStr, err := connectorAssertHostSafe(host)
+ if err != nil {
+ return nil, err
+ }
+ ip := net.ParseIP(ipStr)
+ if ip == nil {
+ return nil, fmt.Errorf("Could not parse validated address %q for host %q", ipStr, host)
+ }
+ return ip, nil
+}
+
+// connectorAllowLoopbackForTest lets unit tests exercise the real HTTP path
+// against httptest servers bound to loopback. Production code keeps it false so
+// loopback/private endpoints stay blocked.
+var connectorAllowLoopbackForTest bool
+
+// assertConnectorURLSafe validates rawURL with the shared strict SSRF guard and
+// returns the hostname plus the first validated IP so callers can pin DNS for
+// the actual dial, preventing DNS rebinding between validation and the
+// connection.
+//
+// When connectorAllowLoopbackForTest is set, URLs whose hostname resolves only
+// to loopback addresses are allowed; anything else falls through to the strict
+// guard, so a mixed private address is still rejected.
+func assertConnectorURLSafe(rawURL string) (string, net.IP, error) {
+ if connectorAllowLoopbackForTest {
+ parsed, err := url.Parse(strings.TrimSpace(rawURL))
+ if err != nil || parsed.Hostname() == "" {
+ return "", nil, fmt.Errorf("URL is missing a host.")
+ }
+ if scheme := strings.ToLower(parsed.Scheme); scheme == "http" || scheme == "https" {
+ if host, ip, ok := loopbackTestAllow(parsed.Hostname()); ok {
+ return host, ip, nil
+ }
+ }
+ // Not allowed by the test hook — fall through to the strict guard.
+ }
+ hostname, ipStr, err := connectorAssertURLSafe(rawURL)
+ if err != nil {
+ return "", nil, err
+ }
+ ip := net.ParseIP(ipStr)
+ if ip == nil {
+ return "", nil, fmt.Errorf("Could not parse validated address %q for hostname %q", ipStr, hostname)
+ }
+ return hostname, ip, nil
+}
+
+// loopbackTestAllow reports whether hostname is a loopback-only target that the
+// test hook may allow. Literal loopback IPs are allowed directly; localhost
+// names must resolve (via the shared resolver seam) to loopback addresses only.
+func loopbackTestAllow(hostname string) (string, net.IP, bool) {
+ if ip := net.ParseIP(hostname); ip != nil {
+ if ip.IsLoopback() {
+ return hostname, ip, true
+ }
+ return "", nil, false
+ }
+ lower := strings.ToLower(hostname)
+ if lower != "localhost" && !strings.HasSuffix(lower, ".localhost") {
+ return "", nil, false
+ }
+ addrs, err := utility.LookupHost(hostname)
+ if err != nil {
+ return "", nil, false
+ }
+ allLoopback := true
+ var first net.IP
+ for _, addr := range addrs {
+ ip := net.ParseIP(addr)
+ if ip == nil || !ip.IsLoopback() {
+ allLoopback = false
+ break
+ }
+ if first == nil {
+ first = ip
+ }
+ }
+ if allLoopback && first != nil {
+ return hostname, first, true
+ }
+ return "", nil, false
+}
+
+// assertConnectorURLSafeHTTPS validates rawURL with the strict SSRF guard and
+// additionally requires HTTPS. Plain-HTTP loopback targets are still permitted
+// under the test hook (httptest servers are plain HTTP); anything else must use
+// TLS so credentials in request headers are never sent in the clear.
+func assertConnectorURLSafeHTTPS(rawURL string) (string, net.IP, error) {
+ if parsed, err := url.Parse(strings.TrimSpace(rawURL)); err == nil {
+ if scheme := strings.ToLower(parsed.Scheme); scheme != "https" {
+ if !connectorAllowLoopbackForTest {
+ return "", nil, fmt.Errorf("URL must use HTTPS (got %q)", scheme)
+ }
+ if _, _, ok := loopbackTestAllow(parsed.Hostname()); !ok {
+ return "", nil, fmt.Errorf("URL must use HTTPS (got %q)", scheme)
+ }
+ }
+ }
+ return assertConnectorURLSafe(rawURL)
+}
+
+// validateConnectorURL is the config-time SSRF check used by connector
+// Validate/New paths.
+func validateConnectorURL(rawURL string) error {
+ // localhost is never a legitimate connector base URL, even under the test
+ // hook (which only relaxes loopback IPs such as httptest's 127.0.0.1).
+ if parsed, err := url.Parse(strings.TrimSpace(rawURL)); err == nil {
+ host := strings.ToLower(parsed.Hostname())
+ if host == "localhost" || strings.HasSuffix(host, ".localhost") {
+ return &ConnectorValidationError{Message: fmt.Sprintf("URL hostname %q is not allowed (localhost is blocked).", parsed.Hostname())}
+ }
+ }
+ if _, _, err := assertConnectorURLSafe(rawURL); err != nil {
+ return &ConnectorValidationError{Message: err.Error()}
+ }
+ return nil
+}
+
+// basicAuthHeader returns the value for an HTTP Basic Authorization header.
+func basicAuthHeader(username, password string) string {
+ return base64.StdEncoding.EncodeToString([]byte(username + ":" + password))
+}
+
+// ---------------------------------------------------------------------------
+// Shared request path
+// ---------------------------------------------------------------------------
+
+// connectorRequestHop is the per-hop request state carried by connectorRequest.
+type connectorRequestHop struct {
+ Method string
+ URL string
+ Body []byte
+ Headers map[string]string
+}
+
+// connectorRequestOptions configures connectorRequest. Zero values use safe
+// defaults: Validate falls back to assertConnectorURLSafe and MaxRedirects to 10.
+type connectorRequestOptions struct {
+ Method string
+ RawURL string
+ Body []byte
+ Headers map[string]string
+ Timeout time.Duration
+ MaxRedirects int
+ // Base, when non-nil, provides the starting transport so custom TLS/CA
+ // settings are preserved; otherwise a default pinned transport is used.
+ Base *http.Client
+ // Validate checks a hop URL for SSRF and returns the hostname plus the IP
+ // to pin for that hop.
+ Validate func(rawURL string) (string, net.IP, error)
+ // Prepare customizes each hop's request before it is sent (basic auth,
+ // query parameters, ...).
+ Prepare func(req *http.Request, hop connectorRequestHop) error
+ // CheckRedirect, when non-nil, is invoked before following to the next hop
+ // (after the next hop's SSRF validation) and may enforce a same-origin
+ // policy or re-apply credentials.
+ CheckRedirect func(req *http.Request, via []*http.Request) error
+ // RetryStatus, when non-nil, decides whether a non-redirect response should
+ // be retried on the same hop URL; it returns the wait duration and whether
+ // to retry. Used for REST API 429 handling.
+ RetryStatus func(resp *http.Response, attempt int) (time.Duration, bool)
+ // NextHop adapts the hop state (method/body/headers/URL) for the hop after
+ // a redirect; it runs after connectorRequest's default GET downgrade and
+ // cross-origin credential stripping.
+ NextHop func(nextURL string, status int, hop connectorRequestHop) connectorRequestHop
+}
+
+// connectorUnsafeErr returns the underlying error when err was produced by the
+// SSRF guard (connectorUnsafeURLError), so callers can surface the original
+// message; otherwise it returns err unchanged.
+func connectorUnsafeErr(err error) error {
+ var unsafe *connectorUnsafeURLError
+ if errors.As(err, &unsafe) {
+ return unsafe.Err
+ }
+ return err
+}
+
+// connectorUnsafeURLError marks a hop URL rejected by the SSRF guard so
+// callers can tell validation failures apart from transport errors.
+type connectorUnsafeURLError struct{ Err error }
+
+func (e *connectorUnsafeURLError) Error() string { return e.Err.Error() }
+func (e *connectorUnsafeURLError) Unwrap() error { return e.Err }
+
+// connectorRequest is the single HTTP send path shared by every connector. It
+// follows redirects manually so that each hop is SSRF-validated and DNS-pinned
+// before a connection is made, closing the TOCTOU window that an automatic
+// redirect policy would leave open (the redirect target would otherwise be
+// validated once and resolved again at dial time).
+//
+// Per-hop behavior:
+// - Validate checks the hop URL and returns the hostname + IP to pin.
+// - The hop is sent through a transport that dials exactly the validated IP,
+// so a DNS change after validation cannot rebind the connection.
+// - Sensitive headers (Authorization etc.) are dropped when a redirect
+// crosses to a different netloc, mirroring http.Client's default.
+// - 301/302/303 downgrade the next hop to GET with an empty body.
+//
+// Connector-specific behavior is expressed through the Prepare, CheckRedirect,
+// RetryStatus, and NextHop hooks.
+func connectorRequest(ctx context.Context, opts connectorRequestOptions) (*http.Response, error) {
+ if opts.Validate == nil {
+ opts.Validate = assertConnectorURLSafe
+ }
+ if opts.MaxRedirects <= 0 {
+ opts.MaxRedirects = 10
+ }
+ hop := connectorRequestHop{Method: opts.Method, URL: opts.RawURL, Body: opts.Body, Headers: opts.Headers}
+ previousNetloc := connectorNetloc(hop.URL)
+ var via []*http.Request
+ for redirects := 0; ; redirects++ {
+ hostname, pinIP, err := opts.Validate(hop.URL)
+ if err != nil {
+ return nil, &connectorUnsafeURLError{Err: err}
+ }
+ var reader io.Reader
+ if hop.Body != nil {
+ reader = bytes.NewReader(hop.Body)
+ }
+ req, err := http.NewRequestWithContext(ctx, hop.Method, hop.URL, reader)
+ if err != nil {
+ return nil, err
+ }
+ for k, v := range hop.Headers {
+ req.Header.Set(k, v)
+ }
+ if opts.Prepare != nil {
+ if err := opts.Prepare(req, hop); err != nil {
+ return nil, err
+ }
+ }
+ if redirects > 0 && opts.CheckRedirect != nil {
+ if err := opts.CheckRedirect(req, via); err != nil {
+ return nil, err
+ }
+ }
+ transport := newConnectorPinnedTransport(opts.Base, hostname, pinIP, opts.Timeout)
+ client := &http.Client{
+ Transport: transport,
+ Timeout: opts.Timeout,
+ // Redirects are followed manually by this loop; each hop gets its
+ // own validated, pinned transport.
+ CheckRedirect: func(*http.Request, []*http.Request) error {
+ return http.ErrUseLastResponse
+ },
+ }
+ var resp *http.Response
+ for attempt := 1; ; attempt++ {
+ resp, err = client.Do(req)
+ if err != nil {
+ transport.CloseIdleConnections()
+ return nil, err
+ }
+ if !connectorIsRedirect(resp.StatusCode) && opts.RetryStatus != nil {
+ if wait, retry := opts.RetryStatus(resp, attempt); retry {
+ resp.Body.Close()
+ select {
+ case <-ctx.Done():
+ transport.CloseIdleConnections()
+ return nil, ctx.Err()
+ case <-time.After(wait):
+ }
+ continue
+ }
+ }
+ break
+ }
+ if !connectorIsRedirect(resp.StatusCode) {
+ resp.Body = &connectorCloseIdleBody{body: resp.Body, transport: transport}
+ return resp, nil
+ }
+ location := resp.Header.Get("Location")
+ resp.Body.Close()
+ transport.CloseIdleConnections()
+ if location == "" {
+ return nil, fmt.Errorf("redirect with empty Location header")
+ }
+ if redirects >= opts.MaxRedirects {
+ return nil, fmt.Errorf("request stopped after %d redirects", opts.MaxRedirects)
+ }
+ nextURL, err := connectorResolveURL(hop.URL, location)
+ if err != nil {
+ return nil, err
+ }
+ nextNetloc := connectorNetloc(nextURL)
+ if nextNetloc != "" && nextNetloc != previousNetloc {
+ hop.Headers = connectorStripAuthHeaders(hop.Headers)
+ // 307/308 preserve the request method and body; never forward a
+ // request body (which may carry credentials, e.g. a Seafile or
+ // Moodle token exchange) to a different origin.
+ if len(hop.Body) > 0 && (resp.StatusCode == http.StatusTemporaryRedirect || resp.StatusCode == http.StatusPermanentRedirect) {
+ return nil, fmt.Errorf("redirect to a different origin is not allowed for a request with a body")
+ }
+ }
+ previousNetloc = nextNetloc
+ if resp.StatusCode == http.StatusMovedPermanently || resp.StatusCode == http.StatusFound || resp.StatusCode == http.StatusSeeOther {
+ hop.Method = http.MethodGet
+ hop.Body = nil
+ }
+ if opts.NextHop != nil {
+ hop = opts.NextHop(nextURL, resp.StatusCode, hop)
+ } else {
+ hop.URL = nextURL
+ }
+ via = append(via, req)
+ }
+}
+
+// newConnectorPinnedTransport builds a transport that dials exactly pinIP for
+// every connection, so the hop is connected to the address validated by the
+// SSRF guard regardless of later DNS changes. When base provides an
+// *http.Transport (custom TLS/CA), it is cloned and reused; otherwise a default
+// transport is used. Environment proxies are deliberately ignored: HTTP_PROXY /
+// HTTPS_PROXY would route the connection through a proxy host instead of pinIP.
+func newConnectorPinnedTransport(base *http.Client, hostname string, pinIP net.IP, timeout time.Duration) *http.Transport {
+ dialer := &net.Dialer{Timeout: timeout, KeepAlive: 30 * time.Second}
+ dial := func(ctx context.Context, network, addr string) (net.Conn, error) {
+ _, port, err := net.SplitHostPort(addr)
+ if err != nil {
+ port = "443"
+ }
+ return dialer.DialContext(ctx, network, net.JoinHostPort(pinIP.String(), port))
+ }
+ if base != nil {
+ if t, ok := base.Transport.(*http.Transport); ok {
+ clone := t.Clone()
+ clone.DialContext = dial
+ if clone.TLSClientConfig == nil {
+ clone.TLSClientConfig = &tls.Config{ServerName: hostname, MinVersion: tls.VersionTLS12}
+ } else {
+ clone.TLSClientConfig = clone.TLSClientConfig.Clone()
+ if clone.TLSClientConfig.ServerName == "" {
+ clone.TLSClientConfig.ServerName = hostname
+ }
+ }
+ return clone
+ }
+ }
+ return &http.Transport{
+ Proxy: nil,
+ DialContext: dial,
+ TLSHandshakeTimeout: timeout,
+ ResponseHeaderTimeout: timeout,
+ ExpectContinueTimeout: 1 * time.Second,
+ ForceAttemptHTTP2: false,
+ TLSClientConfig: &tls.Config{ServerName: hostname, MinVersion: tls.VersionTLS12},
+ }
+}
+
+// connectorCloseIdleBody closes the underlying response body and then releases
+// the per-hop pinned transport's idle connections, so per-hop transports do not
+// leak keep-alive sockets during long syncs.
+type connectorCloseIdleBody struct {
+ body io.ReadCloser
+ transport *http.Transport
+}
+
+func (b *connectorCloseIdleBody) Read(p []byte) (int, error) { return b.body.Read(p) }
+func (b *connectorCloseIdleBody) Close() error {
+ err := b.body.Close()
+ b.transport.CloseIdleConnections()
+ return err
+}
+
+// connectorAuthSensitiveHeaders lists headers that must not be forwarded to a
+// different origin on redirect.
+var connectorAuthSensitiveHeaders = map[string]struct{}{
+ "authorization": {},
+ "proxy-authorization": {},
+ "apikey": {},
+ "api-key": {},
+ "x-api-key": {},
+ "x-auth-token": {},
+ "private-token": {}, // GitLab credential header
+}
+
+func connectorStripAuthHeaders(headers map[string]string) map[string]string {
+ out := make(map[string]string, len(headers))
+ for k, v := range headers {
+ if _, sensitive := connectorAuthSensitiveHeaders[strings.ToLower(k)]; sensitive {
+ continue
+ }
+ out[k] = v
+ }
+ return out
+}
+
+func connectorNetloc(rawURL string) string {
+ parsed, err := url.Parse(rawURL)
+ if err != nil {
+ return ""
+ }
+ return parsed.Host
+}
+
+func connectorResolveURL(base, location string) (string, error) {
+ baseParsed, err := url.Parse(base)
+ if err != nil {
+ return "", err
+ }
+ ref, err := url.Parse(location)
+ if err != nil {
+ return "", err
+ }
+ return baseParsed.ResolveReference(ref).String(), nil
+}
+
+func connectorIsRedirect(status int) bool {
+ switch status {
+ case http.StatusMovedPermanently, http.StatusFound, http.StatusSeeOther,
+ http.StatusTemporaryRedirect, http.StatusPermanentRedirect:
+ return true
+ }
+ return false
+}
diff --git a/internal/syncer/connector/ssrf_test.go b/internal/syncer/connector/ssrf_test.go
new file mode 100644
index 0000000000..1dae568656
--- /dev/null
+++ b/internal/syncer/connector/ssrf_test.go
@@ -0,0 +1,636 @@
+//
+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+package connector
+
+import (
+ "context"
+ "io"
+ "net"
+ "net/http"
+ "net/http/httptest"
+ "strconv"
+ "strings"
+ "sync/atomic"
+ "testing"
+ "time"
+
+ "ragflow/internal/utility"
+)
+
+// withConnectorLoopbackTestHook enables the loopback test seam for the duration
+// of a test so connectors can exercise the real HTTP path against httptest
+// servers bound to loopback.
+func withConnectorLoopbackTestHook(t *testing.T) {
+ t.Helper()
+ prev := connectorAllowLoopbackForTest
+ connectorAllowLoopbackForTest = true
+ t.Cleanup(func() { connectorAllowLoopbackForTest = prev })
+}
+
+func TestValidateConnectorURL(t *testing.T) {
+ tests := []struct {
+ name string
+ url string
+ want bool // true = accepted, false = rejected
+ }{
+ {name: "public literal ip", url: "http://8.8.8.8/x", want: true},
+ {name: "loopback", url: "http://127.0.0.1/x", want: false},
+ {name: "localhost", url: "http://localhost/x", want: false},
+ {name: "cloud metadata", url: "http://169.254.169.254/latest/meta-data/", want: false},
+ {name: "private 10/8", url: "http://10.0.0.5/x", want: false},
+ {name: "private 192.168/16", url: "http://192.168.1.10/x", want: false},
+ {name: "bad scheme", url: "ftp://8.8.8.8/x", want: false},
+ {name: "missing host", url: "http://", want: false},
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ connectorAllowLoopbackForTest = false
+ err := validateConnectorURL(tt.url)
+ if tt.want && err != nil {
+ t.Fatalf("validateConnectorURL(%q) = %v, want nil", tt.url, err)
+ }
+ if !tt.want && err == nil {
+ t.Fatalf("validateConnectorURL(%q) = nil, want rejection", tt.url)
+ }
+ })
+ }
+}
+
+func TestValidateConnectorURLLoopbackHookAllowsOnlyAllLoopback(t *testing.T) {
+ withConnectorLoopbackTestHook(t)
+ if err := validateConnectorURL("http://127.0.0.1/x"); err != nil {
+ t.Fatalf("loopback should be allowed with hook on: %v", err)
+ }
+ // A private address is still rejected even with the hook on.
+ if err := validateConnectorURL("http://10.0.0.1/x"); err == nil {
+ t.Fatalf("private address should be rejected even with hook on")
+ }
+}
+
+func TestWebDAVConnectorRejectsInternalBaseURL(t *testing.T) {
+ connectorAllowLoopbackForTest = false
+ t.Cleanup(func() { connectorAllowLoopbackForTest = false })
+
+ for _, base := range []string{"http://127.0.0.1/dav", "http://169.254.169.254/dav", "http://10.0.0.5/dav"} {
+ c, err := NewWebDAVConnector(map[string]any{
+ "base_url": base,
+ "credentials": map[string]any{
+ "username": "user",
+ "password": "pass",
+ },
+ })
+ if err != nil {
+ t.Fatalf("NewWebDAVConnector(%q) failed: %v", base, err)
+ }
+ if err := c.Validate(context.Background()); err == nil || !strings.Contains(err.Error(), "non-public address") {
+ t.Fatalf("Validate(%q) = %v, want non-public rejection", base, err)
+ }
+ }
+}
+
+func TestConfluenceConnectorRejectsInternalWikiBase(t *testing.T) {
+ connectorAllowLoopbackForTest = false
+ t.Cleanup(func() { connectorAllowLoopbackForTest = false })
+
+ for _, base := range []string{"http://127.0.0.1", "http://169.254.169.254", "http://10.0.0.5"} {
+ c, err := NewConfluenceConnector(map[string]any{
+ "wiki_base": base,
+ "is_cloud": true,
+ "credentials": map[string]any{
+ "confluence_username": "[email protected]",
+ "confluence_access_token": "token",
+ },
+ })
+ if err != nil {
+ t.Fatalf("NewConfluenceConnector(%q) failed: %v", base, err)
+ }
+ if err := c.Validate(context.Background()); err == nil || !strings.Contains(err.Error(), "non-public address") {
+ t.Fatalf("Validate(%q) = %v, want non-public rejection", base, err)
+ }
+ }
+}
+
+func TestJiraConnectorRejectsInternalBaseURL(t *testing.T) {
+ connectorAllowLoopbackForTest = false
+ t.Cleanup(func() { connectorAllowLoopbackForTest = false })
+
+ for _, base := range []string{"http://127.0.0.1", "http://169.254.169.254", "http://10.0.0.5"} {
+ c, err := NewJiraConnector(map[string]any{
+ "base_url": base,
+ "project_key": "RAG",
+ "credentials": map[string]any{
+ "jira_api_token": "token",
+ },
+ })
+ if err != nil {
+ t.Fatalf("NewJiraConnector(%q) failed: %v", base, err)
+ }
+ if err := c.Validate(context.Background()); err == nil || !strings.Contains(err.Error(), "non-public address") {
+ t.Fatalf("Validate(%q) = %v, want non-public rejection", base, err)
+ }
+ }
+}
+
+func TestConnectorRequestPinsRedirectHopAgainstDNSRebinding(t *testing.T) {
+ withConnectorLoopbackTestHook(t)
+ origLookup := utility.LookupHost
+ lookups := 0
+ utility.LookupHost = func(host string) ([]string, error) {
+ if host == "localhost" {
+ lookups++
+ if lookups == 1 {
+ return []string{"127.0.0.1"}, nil
+ }
+ // A second resolution simulates DNS rebinding to a private address
+ // after validation; the dial must never consult it.
+ return []string{"10.0.0.5"}, nil
+ }
+ return origLookup(host)
+ }
+ t.Cleanup(func() { utility.LookupHost = origLookup })
+
+ var targetHit atomic.Bool
+ target := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ targetHit.Store(true)
+ }))
+ defer target.Close()
+ targetPort := strings.TrimPrefix(target.URL, "http://127.0.0.1:")
+ source := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ http.Redirect(w, r, "http://localhost:"+targetPort+"/final", http.StatusFound)
+ }))
+ defer source.Close()
+
+ resp, err := connectorRequest(context.Background(), connectorRequestOptions{
+ Method: http.MethodGet,
+ RawURL: source.URL,
+ Timeout: webdavRequestTimeout,
+ MaxRedirects: 5,
+ })
+ if err != nil {
+ t.Fatalf("connectorRequest: %v", err)
+ }
+ resp.Body.Close()
+ if !targetHit.Load() {
+ t.Fatalf("redirect target was not reached via its pinned address")
+ }
+ if lookups != 1 {
+ t.Fatalf("LookupHost(localhost) called %d times, want 1 (the redirect hop dial must reuse the validated pin, not re-resolve)", lookups)
+ }
+}
+
+func TestConnectorRequestRejectsCrossOriginBodyRedirect(t *testing.T) {
+ withConnectorLoopbackTestHook(t)
+
+ var targetHit atomic.Bool
+ target := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ targetHit.Store(true)
+ }))
+ defer target.Close()
+ targetPort := strings.TrimPrefix(target.URL, "http://127.0.0.1:")
+
+ source := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ switch r.URL.Path {
+ case "/cross":
+ http.Redirect(w, r, "http://127.0.0.1:"+targetPort+"/final", http.StatusTemporaryRedirect)
+ case "/same":
+ http.Redirect(w, r, "/same-final", http.StatusTemporaryRedirect)
+ case "/same-final":
+ got, _ := io.ReadAll(r.Body)
+ if string(got) != "secret=abc" {
+ t.Errorf("same-origin 307 body = %q, want forwarded body", got)
+ }
+ w.WriteHeader(http.StatusOK)
+ }
+ }))
+ defer source.Close()
+
+ opts := func(path string) connectorRequestOptions {
+ return connectorRequestOptions{
+ Method: http.MethodPost,
+ RawURL: source.URL + path,
+ Body: []byte("secret=abc"),
+ Headers: map[string]string{"Content-Type": "application/x-www-form-urlencoded"},
+ Timeout: webdavRequestTimeout,
+ MaxRedirects: 5,
+ }
+ }
+
+ // A 307 to a different origin must not forward the credential-bearing body.
+ crossResp, err := connectorRequest(context.Background(), opts("/cross"))
+ if crossResp != nil {
+ crossResp.Body.Close()
+ }
+ if err == nil || !strings.Contains(err.Error(), "different origin") {
+ t.Fatalf("cross-origin body redirect err = %v, want rejection", err)
+ }
+ if targetHit.Load() {
+ t.Fatalf("cross-origin 307 target must not be reached")
+ }
+
+ // A same-origin 307 still preserves the request body.
+ resp, err := connectorRequest(context.Background(), opts("/same"))
+ if err != nil {
+ t.Fatalf("same-origin 307: %v", err)
+ }
+ resp.Body.Close()
+}
+
+func TestAssertConnectorURLSafeHTTPS(t *testing.T) {
+ orig := utility.LookupHost
+ utility.LookupHost = func(host string) ([]string, error) {
+ return []string{"93.184.216.34"}, nil
+ }
+ t.Cleanup(func() { utility.LookupHost = orig })
+
+ connectorAllowLoopbackForTest = false
+ t.Cleanup(func() { connectorAllowLoopbackForTest = false })
+
+ // HTTPS public URLs pass.
+ if _, _, err := assertConnectorURLSafeHTTPS("https://api.example.com/v1"); err != nil {
+ t.Fatalf("https should be allowed: %v", err)
+ }
+ // Plain-HTTP (non-loopback) is rejected so credentials stay off the wire.
+ if _, _, err := assertConnectorURLSafeHTTPS("http://api.example.com/v1"); err == nil {
+ t.Fatalf("plain http should be rejected")
+ }
+ // Loopback HTTP is rejected outside the test hook.
+ if _, _, err := assertConnectorURLSafeHTTPS("http://127.0.0.1:8080/v1"); err == nil {
+ t.Fatalf("loopback http should be rejected outside the test hook")
+ }
+
+ // Loopback HTTP is permitted under the test hook (httptest servers).
+ withConnectorLoopbackTestHook(t)
+ if _, _, err := assertConnectorURLSafeHTTPS("http://127.0.0.1:8080/v1"); err != nil {
+ t.Fatalf("loopback http should be allowed under the test hook: %v", err)
+ }
+ // But a plain-http non-loopback target is still rejected even under the hook.
+ if _, _, err := assertConnectorURLSafeHTTPS("http://api.example.com/v1"); err == nil {
+ t.Fatalf("plain http should still be rejected under the test hook")
+ }
+}
+
+func TestConnectorStripAuthHeadersRemovesCredentials(t *testing.T) {
+ headers := map[string]string{
+ "Authorization": "Bearer tok",
+ "PRIVATE-TOKEN": "gitlab-tok",
+ "X-API-Key": "key",
+ "X-Auth-Token": "auth",
+ "Content-Type": "application/json",
+ "X-Custom-Header": "keep",
+ }
+ stripped := connectorStripAuthHeaders(headers)
+ for _, k := range []string{"Authorization", "PRIVATE-TOKEN", "X-API-Key", "X-Auth-Token"} {
+ if _, ok := stripped[k]; ok {
+ t.Fatalf("sensitive header %q was not stripped", k)
+ }
+ }
+ for _, k := range []string{"Content-Type", "X-Custom-Header"} {
+ if stripped[k] == "" {
+ t.Fatalf("non-sensitive header %q was stripped", k)
+ }
+ }
+}
+
+// ---------------------------------------------------------------------------
+// Host-type SSRF guards (IMAP / MySQL / PostgreSQL / S3-compatible)
+// ---------------------------------------------------------------------------
+
+func TestAssertConnectorHostSafe(t *testing.T) {
+ connectorAllowLoopbackForTest = false
+ t.Cleanup(func() { connectorAllowLoopbackForTest = false })
+
+ tests := []struct {
+ name string
+ host string
+ want bool // true = accepted, false = rejected
+ }{
+ {name: "public literal ip", host: "8.8.8.8", want: true},
+ {name: "loopback", host: "127.0.0.1", want: false},
+ {name: "localhost", host: "localhost", want: false},
+ {name: "cloud metadata", host: "169.254.169.254", want: false},
+ {name: "private 10/8", host: "10.0.0.5", want: false},
+ {name: "private 192.168/16", host: "192.168.1.10", want: false},
+ {name: "unspecified", host: "0.0.0.0", want: false},
+ {name: "empty", host: "", want: false},
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ ip, err := assertConnectorHostSafe(tt.host)
+ if tt.want && err != nil {
+ t.Fatalf("assertConnectorHostSafe(%q) = %v, want nil", tt.host, err)
+ }
+ if tt.want && (ip == nil || ip.String() != "8.8.8.8") {
+ t.Fatalf("assertConnectorHostSafe(%q) = %v, want 8.8.8.8", tt.host, ip)
+ }
+ if !tt.want && err == nil {
+ t.Fatalf("assertConnectorHostSafe(%q) = nil, want rejection", tt.host)
+ }
+ })
+ }
+}
+
+func TestAssertConnectorHostSafeLoopbackHookAllowsOnlyLoopback(t *testing.T) {
+ withConnectorLoopbackTestHook(t)
+ if _, err := assertConnectorHostSafe("127.0.0.1"); err != nil {
+ t.Fatalf("loopback should be allowed with hook on: %v", err)
+ }
+ if _, err := assertConnectorHostSafe("localhost"); err != nil {
+ t.Fatalf("localhost should be allowed with hook on: %v", err)
+ }
+ // Private / metadata addresses are still rejected even with the hook on.
+ if _, err := assertConnectorHostSafe("10.0.0.1"); err == nil {
+ t.Fatalf("private address should be rejected even with hook on")
+ }
+ if _, err := assertConnectorHostSafe("169.254.169.254"); err == nil {
+ t.Fatalf("metadata address should be rejected even with hook on")
+ }
+}
+
+// TestConnectorHostSSRFDNSRebinding closes the host-type TOCTOU window: after
+// the guard validates a public host, the dial must be pinned to the validated
+// IP and never re-resolve the hostname. The guard seam is stubbed to simulate
+// "validates to a public IP" while the dial helper is pointed at a loopback
+// listener.
+func TestMySQLPinnedDialConnectsToPinnedIP(t *testing.T) {
+ ln, err := net.Listen("tcp4", "127.0.0.1:0")
+ if err != nil {
+ t.Fatalf("listen: %v", err)
+ }
+ defer ln.Close()
+ go func() {
+ for {
+ conn, acceptErr := ln.Accept()
+ if acceptErr != nil {
+ return
+ }
+ _ = conn.Close()
+ }
+ }()
+ _, portStr, err := net.SplitHostPort(ln.Addr().String())
+ if err != nil {
+ t.Fatal(err)
+ }
+ port, err := strconv.Atoi(portStr)
+ if err != nil {
+ t.Fatal(err)
+ }
+ dial := mysqlPinnedDial(net.ParseIP("127.0.0.1"), port)
+ // The driver passes the DSN host (attacker-controlled); the pinned dialer
+ // must ignore it and connect to pinIP:port.
+ conn, err := dial(context.Background(), "attacker.example:9999")
+ if err != nil {
+ t.Fatalf("pinned dial did not reach the pinned listener: %v", err)
+ }
+ _ = conn.Close()
+}
+
+func TestPostgresPinnedDialConnectsToPinnedIP(t *testing.T) {
+ ln, err := net.Listen("tcp4", "127.0.0.1:0")
+ if err != nil {
+ t.Fatalf("listen: %v", err)
+ }
+ defer ln.Close()
+ go func() {
+ for {
+ conn, acceptErr := ln.Accept()
+ if acceptErr != nil {
+ return
+ }
+ _ = conn.Close()
+ }
+ }()
+ _, portStr, err := net.SplitHostPort(ln.Addr().String())
+ if err != nil {
+ t.Fatal(err)
+ }
+ dial := postgresPinnedDial(net.ParseIP("127.0.0.1"), portStr, time.Second)
+ conn, err := dial(context.Background(), "tcp", "attacker.example:9999")
+ if err != nil {
+ t.Fatalf("pinned dial did not reach the pinned listener: %v", err)
+ }
+ _ = conn.Close()
+}
+
+// TestDialRealIMAPClientPinsToLoopback verifies the IMAP production dial is
+// SSRF-guarded and pinned: under the test hook it connects to the loopback
+// listener (failing only at the TLS handshake, never at the TCP dial), proving
+// the dial target is the validated IP rather than a re-resolved hostname.
+func TestDialRealIMAPClientPinsToLoopback(t *testing.T) {
+ withConnectorLoopbackTestHook(t)
+ ln, err := net.Listen("tcp4", "127.0.0.1:0")
+ if err != nil {
+ t.Fatalf("listen: %v", err)
+ }
+ defer ln.Close()
+ go func() {
+ for {
+ conn, acceptErr := ln.Accept()
+ if acceptErr != nil {
+ return
+ }
+ _ = conn.Close()
+ }
+ }()
+ _, portStr, err := net.SplitHostPort(ln.Addr().String())
+ if err != nil {
+ t.Fatal(err)
+ }
+ port, err := strconv.Atoi(portStr)
+ if err != nil {
+ t.Fatal(err)
+ }
+ _, err = dialRealIMAPClient(context.Background(), "127.0.0.1", port, "user", "pass")
+ if err == nil {
+ t.Fatalf("expected a TLS handshake failure against a plain TCP listener")
+ }
+ if strings.Contains(err.Error(), "connection refused") {
+ t.Fatalf("dial was not pinned to the loopback listener: %v", err)
+ }
+}
+
+// TestDialRealIMAPClientRejectsPrivateHost verifies IMAP config-time validation
+// rejects a non-public host before any dial.
+func TestDialRealIMAPClientRejectsPrivateHost(t *testing.T) {
+ connectorAllowLoopbackForTest = false
+ t.Cleanup(func() { connectorAllowLoopbackForTest = false })
+ _, err := dialRealIMAPClient(context.Background(), "10.0.0.5", 993, "user", "pass")
+ if err == nil || !strings.Contains(err.Error(), "public address") {
+ t.Fatalf("dialRealIMAPClient(10.0.0.5) = %v, want public-address rejection", err)
+ }
+}
+
+func TestMySQLConnectorValidateRejectsPrivateHost(t *testing.T) {
+ connectorAllowLoopbackForTest = false
+ t.Cleanup(func() { connectorAllowLoopbackForTest = false })
+ c, err := NewMySQLConnector(map[string]any{
+ "host": "10.0.0.5",
+ "port": "3306",
+ "database": "db",
+ "credentials": map[string]any{
+ "username": "root",
+ "password": "secret",
+ },
+ })
+ if err != nil {
+ t.Fatalf("NewMySQLConnector: %v", err)
+ }
+ if err := c.Validate(context.Background()); err == nil || !strings.Contains(err.Error(), "public address") {
+ t.Fatalf("Validate(10.0.0.5) = %v, want public-address rejection", err)
+ }
+}
+
+func TestPostgreSQLConnectorValidateRejectsPrivateHost(t *testing.T) {
+ connectorAllowLoopbackForTest = false
+ t.Cleanup(func() { connectorAllowLoopbackForTest = false })
+ c, err := NewPostgreSQLConnector(map[string]any{
+ "host": "10.0.0.5",
+ "port": "5432",
+ "database": "db",
+ "credentials": map[string]any{
+ "username": "postgres",
+ "password": "secret",
+ },
+ })
+ if err != nil {
+ t.Fatalf("NewPostgreSQLConnector: %v", err)
+ }
+ if err := c.Validate(context.Background()); err == nil || !strings.Contains(err.Error(), "public address") {
+ t.Fatalf("Validate(10.0.0.5) = %v, want public-address rejection", err)
+ }
+}
+
+// TestMySQLConnectorOpenAllowsLoopbackUnderHook verifies the loopback test hook
+// lets host-based connectors build a client against a local listener; sql.Open
+// is lazy so no server is required.
+func TestMySQLConnectorOpenAllowsLoopbackUnderHook(t *testing.T) {
+ withConnectorLoopbackTestHook(t)
+ c, err := NewMySQLConnector(map[string]any{
+ "host": "127.0.0.1",
+ "port": "3306",
+ "database": "db",
+ "credentials": map[string]any{
+ "username": "root",
+ "password": "secret",
+ },
+ })
+ if err != nil {
+ t.Fatalf("NewMySQLConnector: %v", err)
+ }
+ db, err := c.open()
+ if err != nil {
+ t.Fatalf("open with loopback host under hook: %v", err)
+ }
+ _ = db.Close()
+}
+
+func TestPostgreSQLConnectorOpenAllowsLoopbackUnderHook(t *testing.T) {
+ withConnectorLoopbackTestHook(t)
+ c, err := NewPostgreSQLConnector(map[string]any{
+ "host": "127.0.0.1",
+ "port": "5432",
+ "database": "db",
+ "credentials": map[string]any{
+ "username": "postgres",
+ "password": "secret",
+ },
+ })
+ if err != nil {
+ t.Fatalf("NewPostgreSQLConnector: %v", err)
+ }
+ db, err := c.open()
+ if err != nil {
+ t.Fatalf("open with loopback host under hook: %v", err)
+ }
+ _ = db.Close()
+}
+
+// TestValidateS3CompatibleEndpointSSRF verifies the user-controlled endpoint
+// is rejected when it resolves to non-public addresses and allowed for
+// loopback under the test hook.
+func TestValidateS3CompatibleEndpointSSRF(t *testing.T) {
+ connectorAllowLoopbackForTest = false
+ t.Cleanup(func() { connectorAllowLoopbackForTest = false })
+ for _, endpoint := range []string{"http://10.0.0.5", "https://169.254.169.254", "http://127.0.0.1"} {
+ if err := validateS3CompatibleEndpoint(endpoint); err == nil {
+ t.Fatalf("endpoint %q should be rejected", endpoint)
+ }
+ }
+ withConnectorLoopbackTestHook(t)
+ if err := validateS3CompatibleEndpoint("http://127.0.0.1"); err != nil {
+ t.Fatalf("loopback endpoint should be allowed under the test hook: %v", err)
+ }
+}
+
+// TestS3PinnedTransportPinsDial verifies the S3-compatible transport validates
+// and pins every outbound dial against the SSRF guard.
+func TestS3PinnedTransportPinsDial(t *testing.T) {
+ transport := s3PinnedHTTPClient().Transport.(*http.Transport)
+
+ connectorAllowLoopbackForTest = false
+ t.Cleanup(func() { connectorAllowLoopbackForTest = false })
+ if _, err := transport.DialContext(context.Background(), "tcp", "127.0.0.1:9999"); err == nil {
+ t.Fatalf("loopback dial should be rejected without the test hook")
+ }
+
+ withConnectorLoopbackTestHook(t)
+ ln, err := net.Listen("tcp4", "127.0.0.1:0")
+ if err != nil {
+ t.Fatalf("listen: %v", err)
+ }
+ defer ln.Close()
+ go func() {
+ for {
+ conn, acceptErr := ln.Accept()
+ if acceptErr != nil {
+ return
+ }
+ _ = conn.Close()
+ }
+ }()
+ conn, err := transport.DialContext(context.Background(), "tcp", ln.Addr().String())
+ if err != nil {
+ t.Fatalf("pinned dial did not reach the listener: %v", err)
+ }
+ _ = conn.Close()
+}
+
+// TestS3PinnedHTTPClientRedirectPolicy locks in the SDK-faithful redirect
+// policy: 301/302/303 are returned to the AWS SDK (which re-signs and retries
+// S3 region redirects) instead of being auto-followed by http.Client, while
+// method-preserving 307/308 are followed like the SDK default.
+func TestS3PinnedHTTPClientRedirectPolicy(t *testing.T) {
+ client := s3PinnedHTTPClient()
+ if client.CheckRedirect == nil {
+ t.Fatalf("s3PinnedHTTPClient must set CheckRedirect so the AWS SDK processes 3xx itself")
+ }
+ req, err := http.NewRequest(http.MethodGet, "https://bucket.s3.example.com/obj", nil)
+ if err != nil {
+ t.Fatal(err)
+ }
+ for _, code := range []int{http.StatusMovedPermanently, http.StatusFound, http.StatusSeeOther} {
+ req.Response = &http.Response{StatusCode: code, Request: req}
+ if err := client.CheckRedirect(req, nil); err != http.ErrUseLastResponse {
+ t.Fatalf("CheckRedirect(%d) = %v, want http.ErrUseLastResponse", code, err)
+ }
+ }
+ for _, code := range []int{http.StatusTemporaryRedirect, http.StatusPermanentRedirect} {
+ req.Response = &http.Response{StatusCode: code, Request: req}
+ if err := client.CheckRedirect(req, nil); err != nil {
+ t.Fatalf("CheckRedirect(%d) = %v, want nil (follow)", code, err)
+ }
+ }
+}
diff --git a/internal/syncer/connector/teams.go b/internal/syncer/connector/teams.go
index c33d30ce98..040871ffff 100644
--- a/internal/syncer/connector/teams.go
+++ b/internal/syncer/connector/teams.go
@@ -28,8 +28,6 @@ import (
"strings"
"sync"
"time"
-
- "ragflow/internal/utility"
)
const (
@@ -330,27 +328,22 @@ func (c *TeamsConnector) getJSON(ctx context.Context, apiURL string, out any) er
if err != nil {
return err
}
- hostname, resolvedIP, err := utility.AssertURLSafe(apiURL)
- if err != nil {
- return err
- }
-
var lastErr error
retriedUnauthorized := false
for attempt := 1; attempt <= teamsRetryCount; attempt++ {
requestCtx, cancel := context.WithTimeout(ctx, teamsRequestTimeout)
- req, err := http.NewRequestWithContext(requestCtx, http.MethodGet, apiURL, nil)
- if err != nil {
- cancel()
- return err
- }
- req.Header.Set("Authorization", "Bearer "+token)
- req.Header.Set("Accept", "application/json")
-
- client := utility.PinnedHTTPClient(hostname, resolvedIP, teamsRequestTimeout)
- resp, err := client.Do(req)
+ resp, err := connectorRequest(requestCtx, connectorRequestOptions{
+ Method: http.MethodGet,
+ RawURL: apiURL,
+ Headers: map[string]string{"Authorization": "Bearer " + token, "Accept": "application/json"},
+ Timeout: teamsRequestTimeout,
+ })
if err != nil {
cancel()
+ var unsafe *connectorUnsafeURLError
+ if errors.As(err, &unsafe) {
+ return unsafe.Err
+ }
lastErr = err
} else {
body, readErr := io.ReadAll(io.LimitReader(resp.Body, 32*1024*1024))
diff --git a/internal/syncer/connector/webdav.go b/internal/syncer/connector/webdav.go
index 012b41f77c..0f58d1c2db 100644
--- a/internal/syncer/connector/webdav.go
+++ b/internal/syncer/connector/webdav.go
@@ -17,7 +17,6 @@
package connector
import (
- "bytes"
"context"
"crypto/tls"
"crypto/x509"
@@ -142,6 +141,9 @@ func (c *WebDAVConnector) Validate(ctx context.Context) error {
if c.baseURL == "" {
return fmt.Errorf("WebDAV base URL is required")
}
+ if err := validateConnectorURL(c.baseURL); err != nil {
+ return err
+ }
if c.username == "" || c.password == "" {
return fmt.Errorf("WebDAV requires username and password credentials")
}
@@ -406,16 +408,18 @@ func (c *webdavClient) propfind(ctx context.Context, target string) ([]webdavFil
if err != nil {
return nil, err
}
- req, err := http.NewRequestWithContext(ctx, "PROPFIND", resolved, bytes.NewReader([]byte(webdavPropfindBody)))
- if err != nil {
- return nil, err
- }
- req.Header.Set("Depth", "1")
- req.Header.Set("Content-Type", "application/xml")
+ headers := map[string]string{"Depth": "1", "Content-Type": "application/xml"}
if c.username != "" {
- req.SetBasicAuth(c.username, c.password)
- }
- resp, err := c.httpClient.Do(req)
+ headers["Authorization"] = "Basic " + basicAuthHeader(c.username, c.password)
+ }
+ resp, err := connectorRequest(ctx, connectorRequestOptions{
+ Method: "PROPFIND",
+ RawURL: resolved,
+ Body: []byte(webdavPropfindBody),
+ Headers: headers,
+ Timeout: webdavRequestTimeout,
+ Base: c.httpClient,
+ })
if err != nil {
return nil, err
}
@@ -432,14 +436,17 @@ func (c *webdavClient) propfind(ctx context.Context, target string) ([]webdavFil
// download fetches a file body over GET.
func (c *webdavClient) download(ctx context.Context, fileURL string) ([]byte, error) {
- req, err := http.NewRequestWithContext(ctx, http.MethodGet, fileURL, nil)
- if err != nil {
- return nil, err
- }
+ headers := map[string]string{}
if c.username != "" {
- req.SetBasicAuth(c.username, c.password)
- }
- resp, err := c.httpClient.Do(req)
+ headers["Authorization"] = "Basic " + basicAuthHeader(c.username, c.password)
+ }
+ resp, err := connectorRequest(ctx, connectorRequestOptions{
+ Method: http.MethodGet,
+ RawURL: fileURL,
+ Headers: headers,
+ Timeout: webdavRequestTimeout,
+ Base: c.httpClient,
+ })
if err != nil {
return nil, err
}
diff --git a/internal/syncer/connector/webdav_test.go b/internal/syncer/connector/webdav_test.go
index 392b252afa..ac195b18f7 100644
--- a/internal/syncer/connector/webdav_test.go
+++ b/internal/syncer/connector/webdav_test.go
@@ -54,6 +54,7 @@ func newWebDAVMountedTestServer(t *testing.T, mountPath string, files map[string
func newWebDAVTestServerAtMount(t *testing.T, mountPath string, files map[string]webdavTestFile) (*httptest.Server, *[]string, *[]string) {
t.Helper()
+ withConnectorLoopbackTestHook(t)
var authHeaders []string
var getRequests []string
handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
@@ -668,6 +669,7 @@ func TestWebDAVConnectorFiltersAndImages(t *testing.T) {
// containing absolute or protocol-relative hrefs to a different origin are rejected
// at URL resolution time, preventing any GET to the external host.
func TestWebDAVConnectorOpenSyncRejectsExternalHref(t *testing.T) {
+ withConnectorLoopbackTestHook(t)
t.Setenv("BLOB_STORAGE_SIZE_THRESHOLD", "20")
var externalRequests int
diff --git a/internal/syncer/connector/xquik_test.go b/internal/syncer/connector/xquik_test.go
index 46f1923139..d2dc7584a5 100644
--- a/internal/syncer/connector/xquik_test.go
+++ b/internal/syncer/connector/xquik_test.go
@@ -56,11 +56,11 @@ func newXquikTestConnector(t *testing.T, config map[string]any) *XquikConnector
func withXquikTestServer(t *testing.T, connector *XquikConnector, handler http.Handler) *httptest.Server {
t.Helper()
server := httptest.NewServer(handler)
- previous := restAPISSRFAllowLoopback
- restAPISSRFAllowLoopback = true
+ previous := connectorAllowLoopbackForTest
+ connectorAllowLoopbackForTest = true
connector.baseURL = server.URL
t.Cleanup(func() {
- restAPISSRFAllowLoopback = previous
+ connectorAllowLoopbackForTest = previous
server.Close()
})
return server
diff --git a/internal/utility/ssrf.go b/internal/utility/ssrf.go
index 44735de857..e20a36d9f0 100644
--- a/internal/utility/ssrf.go
+++ b/internal/utility/ssrf.go
@@ -34,11 +34,14 @@ var AllowedURLSchemes = []string{"http", "https"}
// LookupHost is the indirection used to resolve hostnames. Tests override it.
var LookupHost = net.LookupHost
-// AllowAnyHostForTest is a test-only override that bypasses the
-// SSRF guard (no public-IP check, no DNS resolution, no DNS
-// pinning). Production code MUST leave this at its zero value
-// (false). Tests that need to talk to a local httptest server
-// flip it on and reset it in t.Cleanup.
+// AllowAnyHostForTest is a test-only override that skips the
+// public-IP routability check in AssertURLSafe and AssertHostSafe.
+// Scheme, host, and DNS resolution checks are unchanged: hostnames
+// still resolve, unresolvable hostnames still fail, and callers
+// still pin connections to the returned resolved address. Production
+// code MUST leave this at its zero value (false). Tests that need to
+// talk to a local httptest server flip it on and reset it in
+// t.Cleanup.
//
// The previous form (env-var ALLOW_ANY_HOST) was a live runtime
// toggle that any operator could flip to disable the SSRF guard
@@ -105,6 +108,54 @@ var AssertURLSafe = func(rawURL string) (hostname, resolvedIP string, err error)
return hostname, resolvedIP, nil
}
+// AssertHostSafe validates a bare host (a hostname or a literal IP, with no
+// scheme or port) and returns the first resolved public IP. It is the
+// host-type counterpart of AssertURLSafe: every resolved address must be
+// globally routable (private, loopback, link-local, metadata, multicast and
+// reserved ranges are rejected). Callers dial the returned IP directly so DNS
+// cannot rebind the connection to an internal address between validation and
+// the TCP connect.
+//
+// Used by host-based data sources (IMAP/MySQL/PostgreSQL) and by the ExeSQL /
+// test_db_connection host guards, mirroring common/ssrf_guard.py:
+// assert_host_is_safe.
+var AssertHostSafe = func(host string) (resolvedIP string, err error) {
+ host = strings.TrimSpace(host)
+ if host == "" {
+ return "", fmt.Errorf("host is missing")
+ }
+
+ allowAny := allowAnyHost()
+ if ip := net.ParseIP(host); ip != nil {
+ if !allowAny && !isGlobalIP(effectiveIP(ip)) {
+ return "", fmt.Errorf("host is not a public address (%s), which is not allowed", ip.String())
+ }
+ return ip.String(), nil
+ }
+
+ addresses, err := LookupHost(host)
+ if err != nil {
+ return "", fmt.Errorf("could not resolve hostname '%s': %w", host, err)
+ }
+ if len(addresses) == 0 {
+ return "", fmt.Errorf("hostname '%s' resolved to no addresses", host)
+ }
+
+ for _, addr := range addresses {
+ ip := net.ParseIP(addr)
+ if ip == nil {
+ return "", fmt.Errorf("could not parse resolved address '%s' for hostname '%s'", addr, host)
+ }
+ if !allowAny && !isGlobalIP(effectiveIP(ip)) {
+ return "", fmt.Errorf("hostname '%s' resolves to a non-public address (%s), which is not allowed", host, ip.String())
+ }
+ if resolvedIP == "" {
+ resolvedIP = ip.String()
+ }
+ }
+ return resolvedIP, nil
+}
+
// effectiveIP unwraps IPv4-mapped IPv6 addresses (e.g. ::ffff:127.0.0.1) so
// the routability check sees the IPv4 form. Without this, an attacker could
// bypass the guard with an IPv4-mapped IPv6 representation of a private host.