RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
fix(syncer): use the S3-compatible client for S3 sources set to compatible mode (#20342)
Marcel Jiron · MarMar Labsd05c4ce5ae7b
diff --git a/internal/syncer/connector/builtin.go b/internal/syncer/connector/builtin.goindex 74952a85a4..d6d76713ea 100644--- a/internal/syncer/connector/builtin.go+++ b/internal/syncer/connector/builtin.go@@ -18,6 +18,7 @@ package connectorimport ("context"+ "strings""ragflow/internal/dao")@@ -47,7 +48,7 @@ func RegisterBuiltIns(registry *Registry) {registerBuiltIn(registry, "azure_blob", NewAzureBlobStorageConnector)registerBuiltIn(registry, "airtable", NewAirtableConnector)registerBuiltIn(registry, "r2", NewR2Connector)- registerBuiltIn(registry, "s3", NewS3Connector)+ registerBuiltIn(registry, "s3", newS3SourceConnector)registerBuiltIn(registry, "s3_compatible", NewS3CompatibleConnector)registerBuiltIn(registry, "dingtalk_ai_table", NewDingTalkAITableConnector)registerBuiltIn(registry, "imap", NewIMAPConnector)@@ -70,6 +71,16 @@ func RegisterBuiltIns(registry *Registry) {registerBuiltIn(registry, "zotero", NewZoteroConnector)}+// newS3SourceConnector selects the client for the bucket type stored under an S3 source.+// The data source form keeps an S3-compatible bucket under source "s3" with bucket_type "s3_compatible".+// Compatible mode keeps the "s3" ID namespace, so documents already indexed under this source keep their IDs.+func newS3SourceConnector(config map[string]any) (Connector, error) {+ if strings.TrimSpace(stringConfig(config["bucket_type"])) == s3CompatibleSource {+ return newS3CompatibleConnector(config, s3Source)+ }+ return NewS3Connector(config)+}+func registerBuiltIn[T Connector](registry *Registry, source string, factory func(map[string]any) (T, error)) {registry.RegisterConfigFactory(source, func(config map[string]any) (Connector, error) {return factory(config)diff --git a/internal/syncer/connector/registry_test.go b/internal/syncer/connector/registry_test.goindex 090a9a9eae..56c5659964 100644--- a/internal/syncer/connector/registry_test.go+++ b/internal/syncer/connector/registry_test.go@@ -3,8 +3,13 @@ package connectorimport ("context""errors"+ "fmt"+ "io"+ "net/http"+ "net/http/httptest""ragflow/internal/dao""ragflow/internal/entity"+ "strconv""strings""testing")@@ -45,3 +50,130 @@ func TestRegistryOpenUsesTaskFactory(t *testing.T) {t.Fatalf("connector type = %T, want *RSSConnector", connector)}}++func TestRegistryS3SourceHonorsBucketType(t *testing.T) {+ registry := NewRegistry()+ RegisterBuiltIns(registry)+ credentials := map[string]any{+ "endpoint_url": "https://objects.example.com",+ "aws_access_key_id": "key",+ "aws_secret_access_key": "secret",+ "addressing_style": "path",+ }+ open := func(t *testing.T, config map[string]any) []Connector {+ t.Helper()+ fromConfig, err := registry.OpenFromConfig("s3", config)+ if err != nil {+ t.Fatalf("OpenFromConfig failed: %v", err)+ }+ fromTask, err := registry.Open(context.Background(), dao.SyncTaskContext{+ Connector: entity.Connector{Source: "s3", Config: entity.JSONMap(config)},+ })+ if err != nil {+ t.Fatalf("Open failed: %v", err)+ }+ return []Connector{fromConfig, fromTask}+ }++ for _, c := range open(t, map[string]any{"bucket_name": "docs", "bucket_type": "s3_compatible", "credentials": credentials}) {+ compatible, ok := c.(*S3CompatibleConnector)+ if !ok {+ t.Fatalf("s3_compatible connector type = %T, want *S3CompatibleConnector", c)+ }+ if compatible.endpointURL != "https://objects.example.com" || compatible.addressingStyle != "path" {+ t.Fatalf("endpoint_url = %q, addressing_style = %q", compatible.endpointURL, compatible.addressingStyle)+ }+ }+ for _, config := range []map[string]any{+ {"bucket_name": "docs", "bucket_type": "s3", "credentials": credentials},+ {"bucket_name": "docs", "credentials": credentials},+ } {+ for _, c := range open(t, config) {+ if _, ok := c.(*S3Connector); !ok {+ t.Fatalf("bucket_type %v connector type = %T, want *S3Connector", config["bucket_type"], c)+ }+ }+ }+}++// TestRegistryS3SourceCompatibleModeKeepsS3IDs verifies the S3 source keeps the+// "s3" ID namespace in compatible mode, so a checkpoint and documents written+// in AWS mode still match, while the s3_compatible source keeps its own.+func TestRegistryS3SourceCompatibleModeKeepsS3IDs(t *testing.T) {+ keys := []string{"a.txt", "b.txt", "c.txt", "d.txt"}+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ startAfter := r.URL.Query().Get("start-after")+ maxKeys, _ := strconv.Atoi(r.URL.Query().Get("max-keys"))+ var page []string+ for _, key := range keys {+ if key > startAfter && (maxKeys <= 0 || len(page) < maxKeys) {+ page = append(page, key)+ }+ }+ truncated := len(page) > 0 && page[len(page)-1] != keys[len(keys)-1]+ var body strings.Builder+ fmt.Fprintf(&body, `<ListBucketResult xmlns="http://s3.amazonaws.com/doc/2006-03-01/"><Name>bucket</Name><IsTruncated>%t</IsTruncated>`, truncated)+ for _, key := range page {+ fmt.Fprintf(&body, `<Contents><Key>%s</Key><LastModified>2024-01-01T00:00:00.000Z</LastModified><ETag>"%s"</ETag><Size>1</Size></Contents>`, key, key)+ }+ body.WriteString(`</ListBucketResult>`)+ w.Header().Set("Content-Type", "application/xml")+ _, _ = io.WriteString(w, body.String())+ }))+ defer server.Close()+ withConnectorLoopbackTestHook(t)++ registry := NewRegistry()+ RegisterBuiltIns(registry)+ config := map[string]any{+ "bucket_name": "bucket",+ "bucket_type": "s3_compatible",+ "batch_size": 2,+ "credentials": map[string]any{+ "endpoint_url": server.URL,+ "aws_access_key_id": "key",+ "aws_secret_access_key": "secret",+ "addressing_style": "path",+ },+ }+ awsMode, err := NewS3Connector(config)+ if err != nil {+ t.Fatalf("NewS3Connector: %v", err)+ }+ compatibleSource, err := registry.OpenFromConfig("s3_compatible", config)+ if err != nil {+ t.Fatalf("OpenFromConfig(s3_compatible): %v", err)+ }+ if got, want := compatibleSource.(*S3CompatibleConnector).sourceID("a.txt"), s3SourceID(s3CompatibleSource, "bucket", "a.txt"); got != want {+ t.Fatalf("s3_compatible sourceID = %q, want %q", got, want)+ }++ connector, err := registry.OpenFromConfig("s3", config)+ if err != nil {+ t.Fatalf("OpenFromConfig(s3): %v", err)+ }+ session, err := connector.OpenSync(context.Background(), SyncRequest{+ FromBeginning: true,+ Resume: &SyncCheckpoint{Cursor: awsMode.sourceID("a.txt")},+ })+ if err != nil {+ t.Fatalf("OpenSync from an AWS mode checkpoint: %v", err)+ }+ var got []string+ for {+ batch, err := session.NextBatch(context.Background())+ if err == io.EOF {+ break+ }+ if err != nil {+ t.Fatalf("NextBatch: %v", err)+ }+ for _, document := range batch.Documents {+ got = append(got, document.SourceID)+ }+ }+ want := []string{awsMode.sourceID("b.txt"), awsMode.sourceID("c.txt"), awsMode.sourceID("d.txt")}+ if strings.Join(got, ",") != strings.Join(want, ",") {+ t.Fatalf("source ids = %v, want %v", got, want)+ }+}diff --git a/internal/syncer/connector/s3_compatible.go b/internal/syncer/connector/s3_compatible.goindex 0056437827..a7a90075cd 100644--- a/internal/syncer/connector/s3_compatible.go+++ b/internal/syncer/connector/s3_compatible.go@@ -38,6 +38,9 @@ const (// S3CompatibleConnector reads objects from any S3-compatible object store.type S3CompatibleConnector struct {+ // source is the registered source name; it prefixes document IDs and+ // resume anchors.+ source stringbucketName stringprefix stringendpointURL string@@ -57,6 +60,12 @@ type S3CompatibleConnector struct {// NewS3CompatibleConnector creates an S3-compatible connector from// Python-compatible config.func NewS3CompatibleConnector(config map[string]any) (*S3CompatibleConnector, error) {+ return newS3CompatibleConnector(config, s3CompatibleSource)+}++// newS3CompatibleConnector creates the connector under the source name it is+// registered as, which names the ID namespace of its documents.+func newS3CompatibleConnector(config map[string]any, source string) (*S3CompatibleConnector, error) {credentials := configAnyMap(config["credentials"])batchSize := configInt(firstNonEmpty(stringConfig(config["sync_batch_size"]), stringConfig(config["batch_size"])), defaultS3BatchSize)sizeThreshold := int64(configInt(config["size_threshold"], defaultS3SizeThreshold))@@ -65,6 +74,7 @@ func NewS3CompatibleConnector(config map[string]any) (*S3CompatibleConnector, er}addressingStyle := firstNonEmpty(stringConfig(credentials["addressing_style"]), "virtual")return &S3CompatibleConnector{+ source: source,bucketName: strings.TrimSpace(stringConfig(config["bucket_name"])),prefix: normalizeS3Prefix(stringConfig(config["prefix"])),endpointURL: strings.TrimSpace(stringConfig(credentials["endpoint_url"])),@@ -197,11 +207,16 @@ func s3PinnedHTTPClient() *http.Client {// correct endpoint. Only method-preserving 307/308 are followed, same// as the SDK default. Without this, Go's default client would follow a// 301/302 before the SDK sees it, replaying a request signed for one- // endpoint against another host.+ // endpoint against another host. A followed hop must still be HTTPS:+ // the dial-time pin cannot see the scheme, and Go keeps the+ // Authorization header on a same-host or subdomain downgrade.CheckRedirect: func(req *http.Request, via []*http.Request) error {if req.Response != nil {switch req.Response.StatusCode {case http.StatusTemporaryRedirect, http.StatusPermanentRedirect:+ if err := assertConnectorSchemeHTTPS(req.URL); err != nil {+ return fmt.Errorf("refusing redirect: %w", err)+ }return nil}}@@ -227,7 +242,7 @@ func s3PinnedHTTPClient() *http.Client {}func (c *S3CompatibleConnector) listObjectPage(ctx context.Context, startAfter string, maxKeys int32) ([]s3Object, string, bool, error) {- return listS3ObjectPage(ctx, c.listObjects, c.ensureClient, c.bucketName, c.prefix, s3CompatibleSource, startAfter, maxKeys)+ return listS3ObjectPage(ctx, c.listObjects, c.ensureClient, c.bucketName, c.prefix, c.source, startAfter, maxKeys)}func (c *S3CompatibleConnector) download(ctx context.Context, key string) ([]byte, error) {@@ -239,7 +254,7 @@ func (c *S3CompatibleConnector) sourceName() string {}func (c *S3CompatibleConnector) sourceID(key string) string {- return s3SourceID(s3CompatibleSource, c.bucketName, key)+ return s3SourceID(c.source, c.bucketName, key)}func (c *S3CompatibleConnector) sourceDocument(sourceID string, object s3Object) (SourceDocument, bool) {@@ -279,9 +294,11 @@ func validateS3CompatibleEndpoint(raw string) error {}// The endpoint host is user-controlled and connected to server-side, so it// must resolve only to public addresses (blocks loopback, RFC1918 and- // cloud metadata). The per-connection transport in s3PinnedHTTPClient- // re-validates and pins at dial time.- if _, _, err := assertConnectorURLSafe(raw); err != nil {+ // cloud metadata). Every request is signed with the configured credentials+ // and carries object contents, so the endpoint must also use HTTPS. The+ // per-connection transport in s3PinnedHTTPClient re-validates and pins at+ // dial time.+ if _, _, err := assertConnectorURLSafeHTTPS(raw); err != nil {return fmt.Errorf("invalid S3-compatible endpoint_url %q: %v", raw, err)}return nildiff --git a/internal/syncer/connector/s3_compatible_test.go b/internal/syncer/connector/s3_compatible_test.goindex 3ce4c94baf..a5616cdcbd 100644--- a/internal/syncer/connector/s3_compatible_test.go+++ b/internal/syncer/connector/s3_compatible_test.go@@ -331,6 +331,7 @@ func TestS3CompatibleConnectorRejectsInvalidEndpointBeforeClientCreation(t *test"https://evil.example#","evil.example#","ftp://evil.example",+ "http://8.8.8.8","https://user:[email protected]","https://evil.example#fragment","https://",diff --git a/internal/syncer/connector/ssrf.go b/internal/syncer/connector/ssrf.goindex b5085042d3..38e64a8fec 100644--- a/internal/syncer/connector/ssrf.go+++ b/internal/syncer/connector/ssrf.go@@ -173,18 +173,29 @@ func loopbackTestAllow(hostname string) (string, net.IP, bool) {// TLS so credentials in request headers are never sent in the clear.func assertConnectorURLSafeHTTPS(rawURL string) (string, net.IP, error) {if parsed, err := url.Parse(strings.TrimSpace(rawURL)); err == nil {- if scheme := strings.ToLower(parsed.Scheme); scheme != "https" {- if !connectorAllowLoopbackForTest {- return "", nil, fmt.Errorf("URL must use HTTPS (got %q)", scheme)- }- if _, _, ok := loopbackTestAllow(parsed.Hostname()); !ok {- return "", nil, fmt.Errorf("URL must use HTTPS (got %q)", scheme)- }+ if err := assertConnectorSchemeHTTPS(parsed); err != nil {+ return "", nil, err}}return assertConnectorURLSafe(rawURL)}+// assertConnectorSchemeHTTPS is the scheme half of assertConnectorURLSafeHTTPS+// for callers that already hold a parsed URL and whose host is validated+// elsewhere (the S3 transport pins every dial).+func assertConnectorSchemeHTTPS(parsed *url.URL) error {+ scheme := strings.ToLower(parsed.Scheme)+ if scheme == "https" {+ return nil+ }+ if connectorAllowLoopbackForTest {+ if _, _, ok := loopbackTestAllow(parsed.Hostname()); ok {+ return nil+ }+ }+ return fmt.Errorf("URL must use HTTPS (got %q)", scheme)+}+// validateConnectorURL is the config-time SSRF check used by connector// Validate/New paths.func validateConnectorURL(rawURL string) error {diff --git a/internal/syncer/connector/ssrf_test.go b/internal/syncer/connector/ssrf_test.goindex 921379949d..739ac05f48 100644--- a/internal/syncer/connector/ssrf_test.go+++ b/internal/syncer/connector/ssrf_test.go@@ -602,20 +602,26 @@ func TestPostgreSQLConnectorOpenAllowsLoopbackUnderHook(t *testing.T) {}// TestValidateS3CompatibleEndpointSSRF verifies the user-controlled endpoint-// is rejected when it resolves to non-public addresses and allowed for-// loopback under the test hook.+// is rejected when it resolves to non-public addresses or uses plain HTTP, and+// that plain-HTTP loopback is allowed only under the test hook.func TestValidateS3CompatibleEndpointSSRF(t *testing.T) {connectorAllowLoopbackForTest = falset.Cleanup(func() { connectorAllowLoopbackForTest = false })- for _, endpoint := range []string{"http://10.0.0.5", "https://169.254.169.254", "http://127.0.0.1"} {+ for _, endpoint := range []string{"https://10.0.0.5", "https://169.254.169.254", "https://127.0.0.1", "http://8.8.8.8"} {if err := validateS3CompatibleEndpoint(endpoint); err == nil {t.Fatalf("endpoint %q should be rejected", endpoint)}}+ if err := validateS3CompatibleEndpoint("https://8.8.8.8"); err != nil {+ t.Fatalf("public HTTPS endpoint should be allowed: %v", err)+ }withConnectorLoopbackTestHook(t)if err := validateS3CompatibleEndpoint("http://127.0.0.1"); err != nil {t.Fatalf("loopback endpoint should be allowed under the test hook: %v", err)}+ if err := validateS3CompatibleEndpoint("http://8.8.8.8"); err == nil {+ t.Fatalf("public plain-HTTP endpoint should be rejected under the test hook")+ }}// TestS3PinnedTransportPinsDial verifies the S3-compatible transport validates@@ -677,3 +683,52 @@ func TestS3PinnedHTTPClientRedirectPolicy(t *testing.T) {}}}++// TestS3PinnedHTTPClientRedirectRequiresHTTPS verifies a followed 307/308 hop+// cannot downgrade the signed request to plain HTTP: the dial-time pin only+// sees the host, so CheckRedirect must reject a non-HTTPS destination, with+// loopback excepted under the test hook.+func TestS3PinnedHTTPClientRedirectRequiresHTTPS(t *testing.T) {+ client := s3PinnedHTTPClient()+ prev, err := http.NewRequest(http.MethodGet, "https://bucket.s3.example.com/obj", nil)+ if err != nil {+ t.Fatal(err)+ }+ check := func(t *testing.T, dest string, wantFollow bool) {+ t.Helper()+ for _, code := range []int{http.StatusTemporaryRedirect, http.StatusPermanentRedirect} {+ next, err := http.NewRequest(http.MethodGet, dest, nil)+ if err != nil {+ t.Fatal(err)+ }+ next.Response = &http.Response{StatusCode: code, Request: prev}+ err = client.CheckRedirect(next, []*http.Request{prev})+ if wantFollow && err != nil {+ t.Fatalf("CheckRedirect(%d -> %s) = %v, want nil (follow)", code, dest, err)+ }+ if !wantFollow && (err == nil || err == http.ErrUseLastResponse) {+ t.Fatalf("CheckRedirect(%d -> %s) = %v, want a scheme error", code, dest, err)+ }+ }+ }++ connectorAllowLoopbackForTest = false+ t.Cleanup(func() { connectorAllowLoopbackForTest = false })+ check(t, "http://8.8.8.8/x", false)+ check(t, "https://8.8.8.8/x", true)+ // Other redirects still go back to the SDK whatever their scheme.+ for _, code := range []int{http.StatusMovedPermanently, http.StatusFound, http.StatusSeeOther} {+ next, err := http.NewRequest(http.MethodGet, "http://8.8.8.8/x", nil)+ if err != nil {+ t.Fatal(err)+ }+ next.Response = &http.Response{StatusCode: code, Request: prev}+ if err := client.CheckRedirect(next, []*http.Request{prev}); err != http.ErrUseLastResponse {+ t.Fatalf("CheckRedirect(%d -> http://8.8.8.8/x) = %v, want http.ErrUseLastResponse", code, err)+ }+ }++ withConnectorLoopbackTestHook(t)+ check(t, "http://8.8.8.8/x", false)+ check(t, "http://127.0.0.1/x", true)+}