Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

fix(syncer): use the S3-compatible client for S3 sources set to compatible mode (#20342)

Marcel Jiron · MarMar Labsd05c4ce5ae7b
diff --git a/internal/syncer/connector/builtin.go b/internal/syncer/connector/builtin.go
index 74952a85a4..d6d76713ea 100644
--- a/internal/syncer/connector/builtin.go
+++ b/internal/syncer/connector/builtin.go
@@ -18,6 +18,7 @@ package connector
import (
"context"
+ "strings"
"ragflow/internal/dao"
)
@@ -47,7 +48,7 @@ func RegisterBuiltIns(registry *Registry) {
registerBuiltIn(registry, "azure_blob", NewAzureBlobStorageConnector)
registerBuiltIn(registry, "airtable", NewAirtableConnector)
registerBuiltIn(registry, "r2", NewR2Connector)
- registerBuiltIn(registry, "s3", NewS3Connector)
+ registerBuiltIn(registry, "s3", newS3SourceConnector)
registerBuiltIn(registry, "s3_compatible", NewS3CompatibleConnector)
registerBuiltIn(registry, "dingtalk_ai_table", NewDingTalkAITableConnector)
registerBuiltIn(registry, "imap", NewIMAPConnector)
@@ -70,6 +71,16 @@ func RegisterBuiltIns(registry *Registry) {
registerBuiltIn(registry, "zotero", NewZoteroConnector)
}
+// newS3SourceConnector selects the client for the bucket type stored under an S3 source.
+// The data source form keeps an S3-compatible bucket under source "s3" with bucket_type "s3_compatible".
+// Compatible mode keeps the "s3" ID namespace, so documents already indexed under this source keep their IDs.
+func newS3SourceConnector(config map[string]any) (Connector, error) {
+ if strings.TrimSpace(stringConfig(config["bucket_type"])) == s3CompatibleSource {
+ return newS3CompatibleConnector(config, s3Source)
+ }
+ return NewS3Connector(config)
+}
+
func registerBuiltIn[T Connector](registry *Registry, source string, factory func(map[string]any) (T, error)) {
registry.RegisterConfigFactory(source, func(config map[string]any) (Connector, error) {
return factory(config)
diff --git a/internal/syncer/connector/registry_test.go b/internal/syncer/connector/registry_test.go
index 090a9a9eae..56c5659964 100644
--- a/internal/syncer/connector/registry_test.go
+++ b/internal/syncer/connector/registry_test.go
@@ -3,8 +3,13 @@ package connector
import (
"context"
"errors"
+ "fmt"
+ "io"
+ "net/http"
+ "net/http/httptest"
"ragflow/internal/dao"
"ragflow/internal/entity"
+ "strconv"
"strings"
"testing"
)
@@ -45,3 +50,130 @@ func TestRegistryOpenUsesTaskFactory(t *testing.T) {
t.Fatalf("connector type = %T, want *RSSConnector", connector)
}
}
+
+func TestRegistryS3SourceHonorsBucketType(t *testing.T) {
+ registry := NewRegistry()
+ RegisterBuiltIns(registry)
+ credentials := map[string]any{
+ "endpoint_url": "https://objects.example.com",
+ "aws_access_key_id": "key",
+ "aws_secret_access_key": "secret",
+ "addressing_style": "path",
+ }
+ open := func(t *testing.T, config map[string]any) []Connector {
+ t.Helper()
+ fromConfig, err := registry.OpenFromConfig("s3", config)
+ if err != nil {
+ t.Fatalf("OpenFromConfig failed: %v", err)
+ }
+ fromTask, err := registry.Open(context.Background(), dao.SyncTaskContext{
+ Connector: entity.Connector{Source: "s3", Config: entity.JSONMap(config)},
+ })
+ if err != nil {
+ t.Fatalf("Open failed: %v", err)
+ }
+ return []Connector{fromConfig, fromTask}
+ }
+
+ for _, c := range open(t, map[string]any{"bucket_name": "docs", "bucket_type": "s3_compatible", "credentials": credentials}) {
+ compatible, ok := c.(*S3CompatibleConnector)
+ if !ok {
+ t.Fatalf("s3_compatible connector type = %T, want *S3CompatibleConnector", c)
+ }
+ if compatible.endpointURL != "https://objects.example.com" || compatible.addressingStyle != "path" {
+ t.Fatalf("endpoint_url = %q, addressing_style = %q", compatible.endpointURL, compatible.addressingStyle)
+ }
+ }
+ for _, config := range []map[string]any{
+ {"bucket_name": "docs", "bucket_type": "s3", "credentials": credentials},
+ {"bucket_name": "docs", "credentials": credentials},
+ } {
+ for _, c := range open(t, config) {
+ if _, ok := c.(*S3Connector); !ok {
+ t.Fatalf("bucket_type %v connector type = %T, want *S3Connector", config["bucket_type"], c)
+ }
+ }
+ }
+}
+
+// TestRegistryS3SourceCompatibleModeKeepsS3IDs verifies the S3 source keeps the
+// "s3" ID namespace in compatible mode, so a checkpoint and documents written
+// in AWS mode still match, while the s3_compatible source keeps its own.
+func TestRegistryS3SourceCompatibleModeKeepsS3IDs(t *testing.T) {
+ keys := []string{"a.txt", "b.txt", "c.txt", "d.txt"}
+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ startAfter := r.URL.Query().Get("start-after")
+ maxKeys, _ := strconv.Atoi(r.URL.Query().Get("max-keys"))
+ var page []string
+ for _, key := range keys {
+ if key > startAfter && (maxKeys <= 0 || len(page) < maxKeys) {
+ page = append(page, key)
+ }
+ }
+ truncated := len(page) > 0 && page[len(page)-1] != keys[len(keys)-1]
+ var body strings.Builder
+ fmt.Fprintf(&body, `<ListBucketResult xmlns="http://s3.amazonaws.com/doc/2006-03-01/"><Name>bucket</Name><IsTruncated>%t</IsTruncated>`, truncated)
+ for _, key := range page {
+ fmt.Fprintf(&body, `<Contents><Key>%s</Key><LastModified>2024-01-01T00:00:00.000Z</LastModified><ETag>"%s"</ETag><Size>1</Size></Contents>`, key, key)
+ }
+ body.WriteString(`</ListBucketResult>`)
+ w.Header().Set("Content-Type", "application/xml")
+ _, _ = io.WriteString(w, body.String())
+ }))
+ defer server.Close()
+ withConnectorLoopbackTestHook(t)
+
+ registry := NewRegistry()
+ RegisterBuiltIns(registry)
+ config := map[string]any{
+ "bucket_name": "bucket",
+ "bucket_type": "s3_compatible",
+ "batch_size": 2,
+ "credentials": map[string]any{
+ "endpoint_url": server.URL,
+ "aws_access_key_id": "key",
+ "aws_secret_access_key": "secret",
+ "addressing_style": "path",
+ },
+ }
+ awsMode, err := NewS3Connector(config)
+ if err != nil {
+ t.Fatalf("NewS3Connector: %v", err)
+ }
+ compatibleSource, err := registry.OpenFromConfig("s3_compatible", config)
+ if err != nil {
+ t.Fatalf("OpenFromConfig(s3_compatible): %v", err)
+ }
+ if got, want := compatibleSource.(*S3CompatibleConnector).sourceID("a.txt"), s3SourceID(s3CompatibleSource, "bucket", "a.txt"); got != want {
+ t.Fatalf("s3_compatible sourceID = %q, want %q", got, want)
+ }
+
+ connector, err := registry.OpenFromConfig("s3", config)
+ if err != nil {
+ t.Fatalf("OpenFromConfig(s3): %v", err)
+ }
+ session, err := connector.OpenSync(context.Background(), SyncRequest{
+ FromBeginning: true,
+ Resume: &SyncCheckpoint{Cursor: awsMode.sourceID("a.txt")},
+ })
+ if err != nil {
+ t.Fatalf("OpenSync from an AWS mode checkpoint: %v", err)
+ }
+ var got []string
+ for {
+ batch, err := session.NextBatch(context.Background())
+ if err == io.EOF {
+ break
+ }
+ if err != nil {
+ t.Fatalf("NextBatch: %v", err)
+ }
+ for _, document := range batch.Documents {
+ got = append(got, document.SourceID)
+ }
+ }
+ want := []string{awsMode.sourceID("b.txt"), awsMode.sourceID("c.txt"), awsMode.sourceID("d.txt")}
+ if strings.Join(got, ",") != strings.Join(want, ",") {
+ t.Fatalf("source ids = %v, want %v", got, want)
+ }
+}
diff --git a/internal/syncer/connector/s3_compatible.go b/internal/syncer/connector/s3_compatible.go
index 0056437827..a7a90075cd 100644
--- a/internal/syncer/connector/s3_compatible.go
+++ b/internal/syncer/connector/s3_compatible.go
@@ -38,6 +38,9 @@ const (
// S3CompatibleConnector reads objects from any S3-compatible object store.
type S3CompatibleConnector struct {
+ // source is the registered source name; it prefixes document IDs and
+ // resume anchors.
+ source string
bucketName string
prefix string
endpointURL string
@@ -57,6 +60,12 @@ type S3CompatibleConnector struct {
// NewS3CompatibleConnector creates an S3-compatible connector from
// Python-compatible config.
func NewS3CompatibleConnector(config map[string]any) (*S3CompatibleConnector, error) {
+ return newS3CompatibleConnector(config, s3CompatibleSource)
+}
+
+// newS3CompatibleConnector creates the connector under the source name it is
+// registered as, which names the ID namespace of its documents.
+func newS3CompatibleConnector(config map[string]any, source string) (*S3CompatibleConnector, error) {
credentials := configAnyMap(config["credentials"])
batchSize := configInt(firstNonEmpty(stringConfig(config["sync_batch_size"]), stringConfig(config["batch_size"])), defaultS3BatchSize)
sizeThreshold := int64(configInt(config["size_threshold"], defaultS3SizeThreshold))
@@ -65,6 +74,7 @@ func NewS3CompatibleConnector(config map[string]any) (*S3CompatibleConnector, er
}
addressingStyle := firstNonEmpty(stringConfig(credentials["addressing_style"]), "virtual")
return &S3CompatibleConnector{
+ source: source,
bucketName: strings.TrimSpace(stringConfig(config["bucket_name"])),
prefix: normalizeS3Prefix(stringConfig(config["prefix"])),
endpointURL: strings.TrimSpace(stringConfig(credentials["endpoint_url"])),
@@ -197,11 +207,16 @@ func s3PinnedHTTPClient() *http.Client {
// correct endpoint. Only method-preserving 307/308 are followed, same
// as the SDK default. Without this, Go's default client would follow a
// 301/302 before the SDK sees it, replaying a request signed for one
- // endpoint against another host.
+ // endpoint against another host. A followed hop must still be HTTPS:
+ // the dial-time pin cannot see the scheme, and Go keeps the
+ // Authorization header on a same-host or subdomain downgrade.
CheckRedirect: func(req *http.Request, via []*http.Request) error {
if req.Response != nil {
switch req.Response.StatusCode {
case http.StatusTemporaryRedirect, http.StatusPermanentRedirect:
+ if err := assertConnectorSchemeHTTPS(req.URL); err != nil {
+ return fmt.Errorf("refusing redirect: %w", err)
+ }
return nil
}
}
@@ -227,7 +242,7 @@ func s3PinnedHTTPClient() *http.Client {
}
func (c *S3CompatibleConnector) listObjectPage(ctx context.Context, startAfter string, maxKeys int32) ([]s3Object, string, bool, error) {
- return listS3ObjectPage(ctx, c.listObjects, c.ensureClient, c.bucketName, c.prefix, s3CompatibleSource, startAfter, maxKeys)
+ return listS3ObjectPage(ctx, c.listObjects, c.ensureClient, c.bucketName, c.prefix, c.source, startAfter, maxKeys)
}
func (c *S3CompatibleConnector) download(ctx context.Context, key string) ([]byte, error) {
@@ -239,7 +254,7 @@ func (c *S3CompatibleConnector) sourceName() string {
}
func (c *S3CompatibleConnector) sourceID(key string) string {
- return s3SourceID(s3CompatibleSource, c.bucketName, key)
+ return s3SourceID(c.source, c.bucketName, key)
}
func (c *S3CompatibleConnector) sourceDocument(sourceID string, object s3Object) (SourceDocument, bool) {
@@ -279,9 +294,11 @@ func validateS3CompatibleEndpoint(raw string) error {
}
// The endpoint host is user-controlled and connected to server-side, so it
// must resolve only to public addresses (blocks loopback, RFC1918 and
- // cloud metadata). The per-connection transport in s3PinnedHTTPClient
- // re-validates and pins at dial time.
- if _, _, err := assertConnectorURLSafe(raw); err != nil {
+ // cloud metadata). Every request is signed with the configured credentials
+ // and carries object contents, so the endpoint must also use HTTPS. The
+ // per-connection transport in s3PinnedHTTPClient re-validates and pins at
+ // dial time.
+ if _, _, err := assertConnectorURLSafeHTTPS(raw); err != nil {
return fmt.Errorf("invalid S3-compatible endpoint_url %q: %v", raw, err)
}
return nil
diff --git a/internal/syncer/connector/s3_compatible_test.go b/internal/syncer/connector/s3_compatible_test.go
index 3ce4c94baf..a5616cdcbd 100644
--- a/internal/syncer/connector/s3_compatible_test.go
+++ b/internal/syncer/connector/s3_compatible_test.go
@@ -331,6 +331,7 @@ func TestS3CompatibleConnectorRejectsInvalidEndpointBeforeClientCreation(t *test
"https://evil.example#",
"evil.example#",
"ftp://evil.example",
+ "http://8.8.8.8",
"https://user:[email protected]",
"https://evil.example#fragment",
"https://",
diff --git a/internal/syncer/connector/ssrf.go b/internal/syncer/connector/ssrf.go
index b5085042d3..38e64a8fec 100644
--- a/internal/syncer/connector/ssrf.go
+++ b/internal/syncer/connector/ssrf.go
@@ -173,18 +173,29 @@ func loopbackTestAllow(hostname string) (string, net.IP, bool) {
// TLS so credentials in request headers are never sent in the clear.
func assertConnectorURLSafeHTTPS(rawURL string) (string, net.IP, error) {
if parsed, err := url.Parse(strings.TrimSpace(rawURL)); err == nil {
- if scheme := strings.ToLower(parsed.Scheme); scheme != "https" {
- if !connectorAllowLoopbackForTest {
- return "", nil, fmt.Errorf("URL must use HTTPS (got %q)", scheme)
- }
- if _, _, ok := loopbackTestAllow(parsed.Hostname()); !ok {
- return "", nil, fmt.Errorf("URL must use HTTPS (got %q)", scheme)
- }
+ if err := assertConnectorSchemeHTTPS(parsed); err != nil {
+ return "", nil, err
}
}
return assertConnectorURLSafe(rawURL)
}
+// assertConnectorSchemeHTTPS is the scheme half of assertConnectorURLSafeHTTPS
+// for callers that already hold a parsed URL and whose host is validated
+// elsewhere (the S3 transport pins every dial).
+func assertConnectorSchemeHTTPS(parsed *url.URL) error {
+ scheme := strings.ToLower(parsed.Scheme)
+ if scheme == "https" {
+ return nil
+ }
+ if connectorAllowLoopbackForTest {
+ if _, _, ok := loopbackTestAllow(parsed.Hostname()); ok {
+ return nil
+ }
+ }
+ return fmt.Errorf("URL must use HTTPS (got %q)", scheme)
+}
+
// validateConnectorURL is the config-time SSRF check used by connector
// Validate/New paths.
func validateConnectorURL(rawURL string) error {
diff --git a/internal/syncer/connector/ssrf_test.go b/internal/syncer/connector/ssrf_test.go
index 921379949d..739ac05f48 100644
--- a/internal/syncer/connector/ssrf_test.go
+++ b/internal/syncer/connector/ssrf_test.go
@@ -602,20 +602,26 @@ func TestPostgreSQLConnectorOpenAllowsLoopbackUnderHook(t *testing.T) {
}
// TestValidateS3CompatibleEndpointSSRF verifies the user-controlled endpoint
-// is rejected when it resolves to non-public addresses and allowed for
-// loopback under the test hook.
+// is rejected when it resolves to non-public addresses or uses plain HTTP, and
+// that plain-HTTP loopback is allowed only under the test hook.
func TestValidateS3CompatibleEndpointSSRF(t *testing.T) {
connectorAllowLoopbackForTest = false
t.Cleanup(func() { connectorAllowLoopbackForTest = false })
- for _, endpoint := range []string{"http://10.0.0.5", "https://169.254.169.254", "http://127.0.0.1"} {
+ for _, endpoint := range []string{"https://10.0.0.5", "https://169.254.169.254", "https://127.0.0.1", "http://8.8.8.8"} {
if err := validateS3CompatibleEndpoint(endpoint); err == nil {
t.Fatalf("endpoint %q should be rejected", endpoint)
}
}
+ if err := validateS3CompatibleEndpoint("https://8.8.8.8"); err != nil {
+ t.Fatalf("public HTTPS endpoint should be allowed: %v", err)
+ }
withConnectorLoopbackTestHook(t)
if err := validateS3CompatibleEndpoint("http://127.0.0.1"); err != nil {
t.Fatalf("loopback endpoint should be allowed under the test hook: %v", err)
}
+ if err := validateS3CompatibleEndpoint("http://8.8.8.8"); err == nil {
+ t.Fatalf("public plain-HTTP endpoint should be rejected under the test hook")
+ }
}
// TestS3PinnedTransportPinsDial verifies the S3-compatible transport validates
@@ -677,3 +683,52 @@ func TestS3PinnedHTTPClientRedirectPolicy(t *testing.T) {
}
}
}
+
+// TestS3PinnedHTTPClientRedirectRequiresHTTPS verifies a followed 307/308 hop
+// cannot downgrade the signed request to plain HTTP: the dial-time pin only
+// sees the host, so CheckRedirect must reject a non-HTTPS destination, with
+// loopback excepted under the test hook.
+func TestS3PinnedHTTPClientRedirectRequiresHTTPS(t *testing.T) {
+ client := s3PinnedHTTPClient()
+ prev, err := http.NewRequest(http.MethodGet, "https://bucket.s3.example.com/obj", nil)
+ if err != nil {
+ t.Fatal(err)
+ }
+ check := func(t *testing.T, dest string, wantFollow bool) {
+ t.Helper()
+ for _, code := range []int{http.StatusTemporaryRedirect, http.StatusPermanentRedirect} {
+ next, err := http.NewRequest(http.MethodGet, dest, nil)
+ if err != nil {
+ t.Fatal(err)
+ }
+ next.Response = &http.Response{StatusCode: code, Request: prev}
+ err = client.CheckRedirect(next, []*http.Request{prev})
+ if wantFollow && err != nil {
+ t.Fatalf("CheckRedirect(%d -> %s) = %v, want nil (follow)", code, dest, err)
+ }
+ if !wantFollow && (err == nil || err == http.ErrUseLastResponse) {
+ t.Fatalf("CheckRedirect(%d -> %s) = %v, want a scheme error", code, dest, err)
+ }
+ }
+ }
+
+ connectorAllowLoopbackForTest = false
+ t.Cleanup(func() { connectorAllowLoopbackForTest = false })
+ check(t, "http://8.8.8.8/x", false)
+ check(t, "https://8.8.8.8/x", true)
+ // Other redirects still go back to the SDK whatever their scheme.
+ for _, code := range []int{http.StatusMovedPermanently, http.StatusFound, http.StatusSeeOther} {
+ next, err := http.NewRequest(http.MethodGet, "http://8.8.8.8/x", nil)
+ if err != nil {
+ t.Fatal(err)
+ }
+ next.Response = &http.Response{StatusCode: code, Request: prev}
+ if err := client.CheckRedirect(next, []*http.Request{prev}); err != http.ErrUseLastResponse {
+ t.Fatalf("CheckRedirect(%d -> http://8.8.8.8/x) = %v, want http.ErrUseLastResponse", code, err)
+ }
+ }
+
+ withConnectorLoopbackTestHook(t)
+ check(t, "http://8.8.8.8/x", false)
+ check(t, "http://127.0.0.1/x", true)
+}