Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

feat(go-syncer): add S3 and S3 compatible connectors (#18861)

mkaaadb11b13f8fac6
diff --git a/internal/syncer/connector/builtin.go b/internal/syncer/connector/builtin.go
index 4e4efea62a..dcff5b9246 100644
--- a/internal/syncer/connector/builtin.go
+++ b/internal/syncer/connector/builtin.go
@@ -41,6 +41,8 @@ func RegisterBuiltIns(registry *Registry) {
registerBuiltIn(registry, "oci_storage", NewOCIStorageConnector)
registerBuiltIn(registry, "azure_blob", NewAzureBlobStorageConnector)
registerBuiltIn(registry, "r2", NewR2Connector)
+ registerBuiltIn(registry, "s3", NewS3Connector)
+ registerBuiltIn(registry, "s3_compatible", NewS3CompatibleConnector)
registerBuiltIn(registry, "dingtalk_ai_table", NewDingTalkAITableConnector)
registerBuiltIn(registry, "imap", NewIMAPConnector)
registerBuiltIn(registry, "jira", NewJiraConnector)
diff --git a/internal/syncer/connector/s3.go b/internal/syncer/connector/s3.go
new file mode 100644
index 0000000000..cc22e5c64a
--- /dev/null
+++ b/internal/syncer/connector/s3.go
@@ -0,0 +1,588 @@
+//
+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+package connector
+
+import (
+ "context"
+ "encoding/json"
+ "fmt"
+ "io"
+ "net/url"
+ "path"
+ "path/filepath"
+ "regexp"
+ "strings"
+ "time"
+
+ "github.com/aws/aws-sdk-go-v2/aws"
+ awssdkconfig "github.com/aws/aws-sdk-go-v2/config"
+ "github.com/aws/aws-sdk-go-v2/credentials"
+ "github.com/aws/aws-sdk-go-v2/credentials/stscreds"
+ "github.com/aws/aws-sdk-go-v2/service/s3"
+ "github.com/aws/aws-sdk-go-v2/service/s3/types"
+ "github.com/aws/aws-sdk-go-v2/service/sts"
+)
+
+const (
+ defaultS3BatchSize = 2
+ defaultS3SizeThreshold = 20 * 1024 * 1024
+ s3ListPageSize = 1000
+ s3Source = "s3"
+)
+
+var awsRegionLabelPattern = regexp.MustCompile(`^[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?$`)
+
+// S3Connector reads objects from AWS S3.
+type S3Connector struct {
+ bucketName string
+ prefix string
+ region string
+ authMethod string
+ accessKeyID string
+ secretKey string
+ roleARN string
+ allowImages bool
+ batchSize int
+
+ sizeThreshold int64
+ client *s3.Client
+ listObjects func(ctx context.Context, startAfter string, maxKeys int32) ([]s3Object, string, bool, error)
+ downloadObject func(ctx context.Context, key string, sizeThreshold int64) ([]byte, error)
+}
+
+// NewS3Connector creates an AWS S3 connector from Python-compatible config.
+func NewS3Connector(config map[string]any) (*S3Connector, error) {
+ credentials := configAnyMap(config["credentials"])
+ batchSize := configInt(firstNonEmpty(stringConfig(config["sync_batch_size"]), stringConfig(config["batch_size"])), defaultS3BatchSize)
+ sizeThreshold := int64(configInt(config["size_threshold"], defaultS3SizeThreshold))
+ if sizeThreshold <= 0 {
+ sizeThreshold = defaultS3SizeThreshold
+ }
+ return &S3Connector{
+ bucketName: strings.TrimSpace(stringConfig(config["bucket_name"])),
+ prefix: normalizeS3Prefix(stringConfig(config["prefix"])),
+ region: strings.TrimSpace(stringConfig(credentials["region"])),
+ authMethod: firstNonEmpty(stringConfig(credentials["authentication_method"]), "access_key"),
+ accessKeyID: strings.TrimSpace(stringConfig(credentials["aws_access_key_id"])),
+ secretKey: stringConfig(credentials["aws_secret_access_key"]),
+ roleARN: strings.TrimSpace(stringConfig(credentials["aws_role_arn"])),
+ allowImages: configBoolDefault(config["allow_images"], false),
+ batchSize: batchSize,
+ sizeThreshold: sizeThreshold,
+ }, nil
+}
+
+// Validate validates AWS S3 connector settings and credentials.
+func (c *S3Connector) Validate(ctx context.Context) error {
+ if c == nil {
+ return fmt.Errorf("S3 connector is nil")
+ }
+ if c.bucketName == "" {
+ return fmt.Errorf("No bucket name was provided in connector settings.")
+ }
+ if err := validateS3AuthMethod(c.authMethod, c.accessKeyID, c.secretKey, c.roleARN); err != nil {
+ return err
+ }
+ if c.region != "" {
+ if err := validateAWSRegion(c.region); err != nil {
+ return err
+ }
+ }
+ if c.batchSize <= 0 {
+ return fmt.Errorf("batch_size must be a positive integer")
+ }
+ if _, err := c.ensureClient(ctx); err != nil {
+ return err
+ }
+ return nil
+}
+
+// ValidateConnectorSetting validates AWS S3 settings from an unsaved config.
+func (c *S3Connector) ValidateConnectorSetting(ctx context.Context, request map[string]any) error {
+ ctx, cancel := context.WithTimeout(ctx, connectorSettingValidationTimeout)
+ defer cancel()
+ if err := c.Validate(ctx); err != nil {
+ return err
+ }
+ _, _, _, err := c.listObjectPage(ctx, "", 1)
+ if err != nil {
+ return fmt.Errorf("access S3 bucket %q: %w", c.bucketName, err)
+ }
+ return nil
+}
+
+// OpenSync opens one AWS S3 sync session.
+func (c *S3Connector) OpenSync(ctx context.Context, request SyncRequest) (SyncSession, error) {
+ if err := c.Validate(ctx); err != nil {
+ return nil, err
+ }
+ session := &s3SyncSession{
+ connector: c,
+ request: request,
+ batchSize: c.batchSize,
+ }
+ if request.Resume != nil {
+ if err := session.applyResume(ctx, request.Resume); err != nil {
+ return nil, err
+ }
+ }
+ return session, nil
+}
+
+// OpenPrune opens one complete AWS S3 prune snapshot session.
+func (c *S3Connector) OpenPrune(ctx context.Context, request PruneRequest) (PruneSession, error) {
+ if err := c.Validate(ctx); err != nil {
+ return nil, err
+ }
+ return &s3PruneSession{connector: c, batchSize: c.batchSize}, nil
+}
+
+// Fetch downloads an AWS S3 object body.
+func (c *S3Connector) Fetch(ctx context.Context, ref FetchReference) ([]byte, error) {
+ var fetch s3FetchReference
+ if err := json.Unmarshal([]byte(ref.Key), &fetch); err != nil {
+ return nil, err
+ }
+ return c.download(ctx, fetch.Key)
+}
+
+func (c *S3Connector) ensureClient(ctx context.Context) (*s3.Client, error) {
+ if c.client != nil {
+ return c.client, nil
+ }
+ if c.region != "" {
+ if err := validateAWSRegion(c.region); err != nil {
+ return nil, err
+ }
+ }
+ authMethod := firstNonEmpty(c.authMethod, "access_key")
+ options := make([]func(*awssdkconfig.LoadOptions) error, 0, 2)
+ if c.region != "" {
+ options = append(options, awssdkconfig.WithRegion(c.region))
+ }
+ switch authMethod {
+ case "access_key":
+ options = append(options, awssdkconfig.WithCredentialsProvider(credentials.NewStaticCredentialsProvider(c.accessKeyID, c.secretKey, "")))
+ case "iam_role":
+ if c.roleARN == "" {
+ return nil, fmt.Errorf("S3 aws_role_arn is required for iam_role authentication")
+ }
+ case "assume_role":
+ default:
+ return nil, fmt.Errorf("unsupported S3 authentication_method %q", authMethod)
+ }
+ cfg, err := awssdkconfig.LoadDefaultConfig(ctx, options...)
+ if err != nil {
+ return nil, fmt.Errorf("load AWS S3 config: %w", err)
+ }
+ if authMethod == "iam_role" {
+ cfg.Credentials = aws.NewCredentialsCache(stscreds.NewAssumeRoleProvider(sts.NewFromConfig(cfg), c.roleARN))
+ }
+ c.client = s3.NewFromConfig(cfg)
+ return c.client, nil
+}
+
+func (c *S3Connector) listObjectPage(ctx context.Context, startAfter string, maxKeys int32) ([]s3Object, string, bool, error) {
+ return listS3ObjectPage(ctx, c.listObjects, c.ensureClient, c.bucketName, c.prefix, s3Source, startAfter, maxKeys)
+}
+
+func (c *S3Connector) download(ctx context.Context, key string) ([]byte, error) {
+ return downloadS3Object(ctx, c.downloadObject, c.ensureClient, c.bucketName, key, c.sizeThreshold)
+}
+
+func listS3ObjectPage(ctx context.Context, listObjects func(ctx context.Context, startAfter string, maxKeys int32) ([]s3Object, string, bool, error), ensureClient func(context.Context) (*s3.Client, error), bucketName, prefix, source, startAfter string, maxKeys int32) ([]s3Object, string, bool, error) {
+ if listObjects != nil {
+ return listObjects(ctx, startAfter, maxKeys)
+ }
+ client, err := ensureClient(ctx)
+ if err != nil {
+ return nil, "", false, err
+ }
+ input := &s3.ListObjectsV2Input{
+ Bucket: aws.String(bucketName),
+ Prefix: aws.String(prefix),
+ StartAfter: aws.String(startAfter),
+ }
+ if maxKeys > 0 {
+ input.MaxKeys = aws.Int32(maxKeys)
+ }
+ output, err := client.ListObjectsV2(ctx, input)
+ if err != nil {
+ return nil, "", false, err
+ }
+ objects := make([]s3Object, 0, len(output.Contents))
+ for _, object := range output.Contents {
+ objects = append(objects, s3ObjectFromAWS(object))
+ }
+ nextStartAfter := ""
+ if len(objects) > 0 {
+ nextStartAfter = s3SourceID(source, bucketName, objects[len(objects)-1].Key)
+ }
+ return objects, nextStartAfter, aws.ToBool(output.IsTruncated), nil
+}
+
+func downloadS3Object(ctx context.Context, downloadObject func(ctx context.Context, key string, sizeThreshold int64) ([]byte, error), ensureClient func(context.Context) (*s3.Client, error), bucketName, key string, sizeThreshold int64) ([]byte, error) {
+ if downloadObject != nil {
+ return downloadObject(ctx, key, sizeThreshold)
+ }
+ client, err := ensureClient(ctx)
+ if err != nil {
+ return nil, err
+ }
+ output, err := client.GetObject(ctx, &s3.GetObjectInput{
+ Bucket: aws.String(bucketName),
+ Key: aws.String(key),
+ })
+ if err != nil {
+ return nil, err
+ }
+ defer output.Body.Close()
+ return readS3Body(output.Body, key, sizeThreshold)
+}
+
+type s3Object struct {
+ Key string
+ LastModified time.Time
+ Size int64
+ ETag string
+}
+
+func s3ObjectFromAWS(object types.Object) s3Object {
+ updatedAt := time.Time{}
+ if object.LastModified != nil {
+ updatedAt = object.LastModified.UTC()
+ }
+ return s3Object{
+ Key: aws.ToString(object.Key),
+ LastModified: updatedAt,
+ Size: aws.ToInt64(object.Size),
+ ETag: aws.ToString(object.ETag),
+ }
+}
+
+type s3ObjectStore interface {
+ sourceName() string
+ listObjectPage(ctx context.Context, startAfter string, maxKeys int32) ([]s3Object, string, bool, error)
+ sourceID(key string) string
+ sourceDocument(sourceID string, object s3Object) (SourceDocument, bool)
+ Fetch(ctx context.Context, ref FetchReference) ([]byte, error)
+}
+
+type s3SyncSession struct {
+ connector s3ObjectStore
+ request SyncRequest
+ batchSize int
+ startAfter string
+ done bool
+}
+
+// NextBatch returns the next S3 or S3-compatible document batch.
+func (s *s3SyncSession) NextBatch(ctx context.Context) (SyncBatch, error) {
+ for {
+ if s.done {
+ return SyncBatch{}, io.EOF
+ }
+ previousStartAfter := s.startAfter
+ objects, nextStartAfter, hasMore, err := s.connector.listObjectPage(ctx, s.startAfter, int32(s.batchSize))
+ if err != nil {
+ return SyncBatch{}, err
+ }
+ if !hasMore {
+ s.done = true
+ }
+ if nextStartAfter != "" {
+ s.startAfter = strings.TrimPrefix(nextStartAfter, s.connector.sourceID(""))
+ }
+ if hasMore && s.startAfter == previousStartAfter {
+ return SyncBatch{}, fmt.Errorf("%s listing did not advance from %q", s.connector.sourceName(), previousStartAfter)
+ }
+
+ documents := make([]SourceDocument, 0, len(objects))
+ for _, object := range objects {
+ sourceID := s.connector.sourceID(object.Key)
+ if !includeS3Object(s.request, sourceID, object) {
+ continue
+ }
+ document, ok := s.connector.sourceDocument(sourceID, object)
+ if ok {
+ documents = append(documents, document)
+ }
+ }
+ if len(documents) == 0 {
+ continue
+ }
+ last := documents[len(documents)-1]
+ updatedAt := last.UpdatedAt
+ return SyncBatch{
+ Documents: documents,
+ Checkpoint: &SyncCheckpoint{
+ Cursor: last.SourceID,
+ SourceID: last.SourceID,
+ UpdatedAt: &updatedAt,
+ },
+ }, nil
+ }
+}
+
+// Close closes the S3 or S3-compatible sync session.
+func (s *s3SyncSession) Close() error {
+ return nil
+}
+
+// Fetch downloads a delayed S3 or S3-compatible object body.
+func (s *s3SyncSession) Fetch(ctx context.Context, ref FetchReference) ([]byte, error) {
+ return s.connector.Fetch(ctx, ref)
+}
+
+func (s *s3SyncSession) applyResume(ctx context.Context, checkpoint *SyncCheckpoint) error {
+ if checkpoint == nil {
+ return nil
+ }
+ sourceID := firstNonEmpty(checkpoint.SourceID, checkpoint.Cursor)
+ prefix := s.connector.sourceID("")
+ if sourceID == "" || !strings.HasPrefix(sourceID, prefix) {
+ return fmt.Errorf("%s sync checkpoint has no source anchor: %w", s.connector.sourceName(), ErrSyncResumeInvalid)
+ }
+ anchor := strings.TrimPrefix(sourceID, prefix)
+ if anchor == "" {
+ return fmt.Errorf("%s sync checkpoint has no source anchor: %w", s.connector.sourceName(), ErrSyncResumeInvalid)
+ }
+
+ startAfter := ""
+ for {
+ objects, nextStartAfter, hasMore, err := s.connector.listObjectPage(ctx, startAfter, int32(s.batchSize))
+ if err != nil {
+ return err
+ }
+ for _, object := range objects {
+ if object.Key == anchor {
+ s.startAfter = anchor
+ return nil
+ }
+ if object.Key > anchor {
+ return fmt.Errorf("%s resume anchor %q was not found in the current listing: %w", s.connector.sourceName(), anchor, ErrSyncResumeInvalid)
+ }
+ }
+ if !hasMore {
+ break
+ }
+ startAfter = strings.TrimPrefix(nextStartAfter, prefix)
+ }
+ return fmt.Errorf("%s resume anchor %q was not found in the current listing: %w", s.connector.sourceName(), anchor, ErrSyncResumeInvalid)
+}
+
+func (c *S3Connector) sourceName() string {
+ return "S3"
+}
+
+func (c *S3Connector) sourceID(key string) string {
+ return s3SourceID(s3Source, c.bucketName, key)
+}
+
+func (c *S3Connector) sourceDocument(sourceID string, object s3Object) (SourceDocument, bool) {
+ return newS3SourceDocument(sourceID, object, c.prefix, c.allowImages, map[string]any{
+ "url": s3ConsoleURL(c.region, c.bucketName, object.Key),
+ })
+}
+
+func validateS3AuthMethod(authMethod, accessKeyID, secretKey, roleARN string) error {
+ switch authMethod {
+ case "access_key":
+ if accessKeyID == "" || secretKey == "" {
+ return fmt.Errorf("S3 access key credentials are required")
+ }
+ case "iam_role":
+ if roleARN == "" {
+ return fmt.Errorf("S3 aws_role_arn is required for iam_role authentication")
+ }
+ case "assume_role":
+ default:
+ return fmt.Errorf("unsupported S3 authentication_method %q", authMethod)
+ }
+ return nil
+}
+
+func validateAWSRegion(region string) error {
+ if !awsRegionLabelPattern.MatchString(region) {
+ return fmt.Errorf("invalid AWS region %q: must be a lowercase DNS label containing only letters, numbers, and hyphens", region)
+ }
+ return nil
+}
+
+type s3PruneSession struct {
+ connector s3ObjectStore
+ batchSize int
+ startAfter string
+ done bool
+ buffer []SlimDocument
+}
+
+// NextBatch returns the next S3 or S3-compatible prune snapshot batch.
+func (s *s3PruneSession) NextBatch(ctx context.Context) (PruneBatch, error) {
+ documents := make([]SlimDocument, 0, s.batchSize)
+ if len(s.buffer) > 0 {
+ n := s.batchSize
+ if n > len(s.buffer) {
+ n = len(s.buffer)
+ }
+ documents = append(documents, s.buffer[:n]...)
+ s.buffer = s.buffer[n:]
+ }
+ for len(documents) < s.batchSize && !s.done {
+ page, nextStartAfter, hasMore, err := s.connector.listObjectPage(ctx, s.startAfter, s3ListPageSize)
+ if err != nil {
+ return PruneBatch{}, err
+ }
+ previousStartAfter := s.startAfter
+ if hasMore {
+ next := strings.TrimPrefix(nextStartAfter, s.connector.sourceID(""))
+ if next == "" || next == previousStartAfter {
+ return PruneBatch{}, fmt.Errorf("%s listing did not advance from %q", s.connector.sourceName(), previousStartAfter)
+ }
+ s.startAfter = next
+ } else {
+ s.done = true
+ }
+ remaining := s.batchSize - len(documents)
+ for _, object := range page {
+ if object.Key == "" || strings.HasSuffix(object.Key, "/") {
+ continue
+ }
+ doc := SlimDocument{SourceID: s.connector.sourceID(object.Key)}
+ if remaining > 0 {
+ documents = append(documents, doc)
+ remaining--
+ } else {
+ s.buffer = append(s.buffer, doc)
+ }
+ }
+ if remaining <= 0 {
+ break
+ }
+ }
+ if len(documents) == 0 && len(s.buffer) == 0 && s.done {
+ return PruneBatch{}, io.EOF
+ }
+ return PruneBatch{Documents: documents}, nil
+}
+
+// Close closes the S3 or S3-compatible prune session.
+func (s *s3PruneSession) Close() error {
+ return nil
+}
+
+type s3FetchReference struct {
+ Key string `json:"key"`
+}
+
+func includeS3Object(request SyncRequest, sourceID string, object s3Object) bool {
+ if request.FromBeginning {
+ return true
+ }
+ if object.LastModified.IsZero() {
+ return true
+ }
+ if len(request.Fingerprints) > 0 {
+ fingerprint := normalizedS3ETag(object.ETag)
+ stored, ok := request.Fingerprints[sourceID]
+ return fingerprint == "" || !ok || stored == "" || stored != fingerprint
+ }
+ return !beforeOrAtWindowStart(object.LastModified, request.WindowStart) && !afterWindowEnd(object.LastModified, request.WindowEnd)
+}
+
+func normalizeS3Prefix(prefix string) string {
+ prefix = strings.TrimSpace(prefix)
+ if prefix != "" && !strings.HasSuffix(prefix, "/") {
+ prefix += "/"
+ }
+ return prefix
+}
+
+func s3SourceID(source, bucketName, key string) string {
+ return fmt.Sprintf("%s:%s:%s", source, bucketName, key)
+}
+
+func normalizedS3ETag(rawETag string) string {
+ rawETag = strings.Trim(strings.TrimSpace(rawETag), `"`)
+ if rawETag == "" {
+ return ""
+ }
+ return contentFingerprint([]byte(rawETag))
+}
+
+func s3ConsoleURL(region, bucketName, key string) string {
+ return fmt.Sprintf("https://s3.console.aws.amazon.com/s3/object/%s?region=%s&prefix=%s", bucketName, region, pathEscapeS3Key(key))
+}
+
+func pathEscapeS3Key(key string) string {
+ parts := strings.Split(key, "/")
+ for index, part := range parts {
+ parts[index] = url.PathEscape(part)
+ }
+ return strings.Join(parts, "/")
+}
+
+func readS3Body(body io.Reader, key string, sizeThreshold int64) ([]byte, error) {
+ limited := io.LimitReader(body, sizeThreshold+1)
+ data, err := io.ReadAll(limited)
+ if err != nil {
+ return nil, err
+ }
+ if int64(len(data)) > sizeThreshold {
+ return nil, fmt.Errorf("%s exceeds size threshold of %d", key, sizeThreshold)
+ }
+ return data, nil
+}
+
+func newS3SourceDocument(sourceID string, object s3Object, prefix string, allowImages bool, metadata map[string]any) (SourceDocument, bool) {
+ if object.Key == "" || strings.HasSuffix(object.Key, "/") || (!allowImages && object.isImage()) {
+ return SourceDocument{}, false
+ }
+ fileName := path.Base(object.Key)
+ fetch := s3FetchReference{Key: object.Key}
+ fetchKey, _ := json.Marshal(fetch)
+ return SourceDocument{
+ SourceID: sourceID,
+ SemanticIdentifier: s3SemanticIdentifier(prefix, object.Key, fileName),
+ Extension: strings.ToLower(filepath.Ext(fileName)),
+ FetchRef: &FetchReference{Key: string(fetchKey), SizeHint: object.Size},
+ UpdatedAt: object.LastModified,
+ SizeBytes: object.Size,
+ Metadata: metadata,
+ Fingerprint: normalizedS3ETag(object.ETag),
+ }, true
+}
+
+func s3SemanticIdentifier(prefix, key, fileName string) string {
+ relativePath := key
+ if prefix != "" {
+ relativePath = strings.TrimPrefix(key, prefix)
+ }
+ if relativePath == "" {
+ return fileName
+ }
+ return strings.ReplaceAll(relativePath, "/", " / ")
+}
+
+func (o s3Object) isImage() bool {
+ switch strings.ToLower(filepath.Ext(o.Key)) {
+ case ".jpg", ".jpeg", ".png", ".webp", ".heic", ".heif":
+ return true
+ default:
+ return false
+ }
+}
diff --git a/internal/syncer/connector/s3_compatible.go b/internal/syncer/connector/s3_compatible.go
new file mode 100644
index 0000000000..63dd769311
--- /dev/null
+++ b/internal/syncer/connector/s3_compatible.go
@@ -0,0 +1,233 @@
+//
+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+package connector
+
+import (
+ "context"
+ "encoding/json"
+ "fmt"
+ "net/url"
+ "strings"
+
+ "github.com/aws/aws-sdk-go-v2/aws"
+ awssdkconfig "github.com/aws/aws-sdk-go-v2/config"
+ "github.com/aws/aws-sdk-go-v2/credentials"
+ "github.com/aws/aws-sdk-go-v2/service/s3"
+)
+
+const (
+ defaultS3CompatibleRegion = "us-east-1"
+ s3CompatibleSource = "s3_compatible"
+)
+
+// S3CompatibleConnector reads objects from any S3-compatible object store.
+type S3CompatibleConnector struct {
+ bucketName string
+ prefix string
+ endpointURL string
+ region string
+ accessKeyID string
+ secretKey string
+ addressingStyle string
+ allowImages bool
+ batchSize int
+
+ sizeThreshold int64
+ client *s3.Client
+ listObjects func(ctx context.Context, startAfter string, maxKeys int32) ([]s3Object, string, bool, error)
+ downloadObject func(ctx context.Context, key string, sizeThreshold int64) ([]byte, error)
+}
+
+// NewS3CompatibleConnector creates an S3-compatible connector from
+// Python-compatible config.
+func NewS3CompatibleConnector(config map[string]any) (*S3CompatibleConnector, error) {
+ credentials := configAnyMap(config["credentials"])
+ batchSize := configInt(firstNonEmpty(stringConfig(config["sync_batch_size"]), stringConfig(config["batch_size"])), defaultS3BatchSize)
+ sizeThreshold := int64(configInt(config["size_threshold"], defaultS3SizeThreshold))
+ if sizeThreshold <= 0 {
+ sizeThreshold = defaultS3SizeThreshold
+ }
+ addressingStyle := firstNonEmpty(stringConfig(credentials["addressing_style"]), "virtual")
+ return &S3CompatibleConnector{
+ bucketName: strings.TrimSpace(stringConfig(config["bucket_name"])),
+ prefix: normalizeS3Prefix(stringConfig(config["prefix"])),
+ endpointURL: strings.TrimSpace(stringConfig(credentials["endpoint_url"])),
+ region: firstNonEmpty(stringConfig(credentials["region"]), defaultS3CompatibleRegion),
+ accessKeyID: strings.TrimSpace(stringConfig(credentials["aws_access_key_id"])),
+ secretKey: stringConfig(credentials["aws_secret_access_key"]),
+ addressingStyle: addressingStyle,
+ allowImages: configBoolDefault(config["allow_images"], false),
+ batchSize: batchSize,
+ sizeThreshold: sizeThreshold,
+ }, nil
+}
+
+// Validate validates S3-compatible connector settings and credentials.
+func (c *S3CompatibleConnector) Validate(ctx context.Context) error {
+ if c == nil {
+ return fmt.Errorf("S3-compatible connector is nil")
+ }
+ if c.bucketName == "" {
+ return fmt.Errorf("No bucket name was provided in connector settings.")
+ }
+ if err := validateS3CompatibleEndpoint(c.endpointURL); err != nil {
+ return err
+ }
+ if c.accessKeyID == "" || c.secretKey == "" {
+ return fmt.Errorf("S3-compatible access key credentials are required")
+ }
+ if err := validateS3CompatibleAddressingStyle(c.addressingStyle); err != nil {
+ return err
+ }
+ if c.batchSize <= 0 {
+ return fmt.Errorf("batch_size must be a positive integer")
+ }
+ if _, err := c.ensureClient(ctx); err != nil {
+ return err
+ }
+ return nil
+}
+
+// ValidateConnectorSetting validates S3-compatible settings from an unsaved
+// config.
+func (c *S3CompatibleConnector) ValidateConnectorSetting(ctx context.Context, request map[string]any) error {
+ ctx, cancel := context.WithTimeout(ctx, connectorSettingValidationTimeout)
+ defer cancel()
+ if err := c.Validate(ctx); err != nil {
+ return err
+ }
+ _, _, _, err := c.listObjectPage(ctx, "", 1)
+ if err != nil {
+ return fmt.Errorf("access S3-compatible bucket %q: %w", c.bucketName, err)
+ }
+ return nil
+}
+
+// OpenSync opens one S3-compatible sync session.
+func (c *S3CompatibleConnector) OpenSync(ctx context.Context, request SyncRequest) (SyncSession, error) {
+ if err := c.Validate(ctx); err != nil {
+ return nil, err
+ }
+ session := &s3SyncSession{
+ connector: c,
+ request: request,
+ batchSize: c.batchSize,
+ }
+ if request.Resume != nil {
+ if err := session.applyResume(ctx, request.Resume); err != nil {
+ return nil, err
+ }
+ }
+ return session, nil
+}
+
+// OpenPrune opens one complete S3-compatible prune snapshot session.
+func (c *S3CompatibleConnector) OpenPrune(ctx context.Context, request PruneRequest) (PruneSession, error) {
+ if err := c.Validate(ctx); err != nil {
+ return nil, err
+ }
+ return &s3PruneSession{connector: c, batchSize: c.batchSize}, nil
+}
+
+// Fetch downloads an S3-compatible object body.
+func (c *S3CompatibleConnector) Fetch(ctx context.Context, ref FetchReference) ([]byte, error) {
+ var fetch s3FetchReference
+ if err := json.Unmarshal([]byte(ref.Key), &fetch); err != nil {
+ return nil, err
+ }
+ return c.download(ctx, fetch.Key)
+}
+
+func (c *S3CompatibleConnector) ensureClient(ctx context.Context) (*s3.Client, error) {
+ if c.client != nil {
+ return c.client, nil
+ }
+ if err := validateS3CompatibleEndpoint(c.endpointURL); err != nil {
+ return nil, err
+ }
+ if err := validateS3CompatibleAddressingStyle(c.addressingStyle); err != nil {
+ return nil, err
+ }
+ region := firstNonEmpty(c.region, defaultS3CompatibleRegion)
+ cfg, err := awssdkconfig.LoadDefaultConfig(
+ ctx,
+ awssdkconfig.WithRegion(region),
+ awssdkconfig.WithCredentialsProvider(credentials.NewStaticCredentialsProvider(c.accessKeyID, c.secretKey, "")),
+ )
+ if err != nil {
+ return nil, fmt.Errorf("load S3-compatible config: %w", err)
+ }
+ c.client = s3.NewFromConfig(cfg, func(options *s3.Options) {
+ options.BaseEndpoint = aws.String(c.endpointURL)
+ options.UsePathStyle = c.addressingStyle == "path"
+ })
+ return c.client, nil
+}
+
+func (c *S3CompatibleConnector) listObjectPage(ctx context.Context, startAfter string, maxKeys int32) ([]s3Object, string, bool, error) {
+ return listS3ObjectPage(ctx, c.listObjects, c.ensureClient, c.bucketName, c.prefix, s3CompatibleSource, startAfter, maxKeys)
+}
+
+func (c *S3CompatibleConnector) download(ctx context.Context, key string) ([]byte, error) {
+ return downloadS3Object(ctx, c.downloadObject, c.ensureClient, c.bucketName, key, c.sizeThreshold)
+}
+
+func (c *S3CompatibleConnector) sourceName() string {
+ return "S3-compatible"
+}
+
+func (c *S3CompatibleConnector) sourceID(key string) string {
+ return s3SourceID(s3CompatibleSource, c.bucketName, key)
+}
+
+func (c *S3CompatibleConnector) sourceDocument(sourceID string, object s3Object) (SourceDocument, bool) {
+ return newS3SourceDocument(sourceID, object, c.prefix, c.allowImages, nil)
+}
+
+func validateS3CompatibleAddressingStyle(addressingStyle string) error {
+ if addressingStyle != "virtual" && addressingStyle != "path" {
+ return fmt.Errorf("unsupported S3-compatible addressing_style %q", addressingStyle)
+ }
+ return nil
+}
+
+func validateS3CompatibleEndpoint(raw string) error {
+ raw = strings.TrimSpace(raw)
+ if raw == "" {
+ return fmt.Errorf("S3-compatible endpoint_url is required")
+ }
+ if strings.ContainsAny(raw, " \t\r\n") || strings.Contains(raw, "#") {
+ return fmt.Errorf("invalid S3-compatible endpoint_url %q", raw)
+ }
+ parsed, err := url.Parse(raw)
+ if err != nil {
+ return fmt.Errorf("invalid S3-compatible endpoint_url %q: %w", raw, err)
+ }
+ if parsed.Scheme != "http" && parsed.Scheme != "https" {
+ return fmt.Errorf("invalid S3-compatible endpoint_url %q: scheme must be http or https", raw)
+ }
+ if parsed.Host == "" || parsed.Hostname() == "" {
+ return fmt.Errorf("invalid S3-compatible endpoint_url %q: host is required", raw)
+ }
+ if parsed.User != nil {
+ return fmt.Errorf("invalid S3-compatible endpoint_url %q: userinfo is not allowed", raw)
+ }
+ if parsed.Fragment != "" {
+ return fmt.Errorf("invalid S3-compatible endpoint_url %q: fragment is not allowed", raw)
+ }
+ return nil
+}
diff --git a/internal/syncer/connector/s3_compatible_test.go b/internal/syncer/connector/s3_compatible_test.go
new file mode 100644
index 0000000000..a73939313e
--- /dev/null
+++ b/internal/syncer/connector/s3_compatible_test.go
@@ -0,0 +1,466 @@
+package connector
+
+import (
+ "bytes"
+ "context"
+ "errors"
+ "io"
+ "strings"
+ "testing"
+)
+
+func TestS3CompatibleConnectorOpenSyncUsesFingerprintAndFetch(t *testing.T) {
+ old := mustTime(t, "2026-01-01T00:00:00Z")
+ updated := mustTime(t, "2026-01-03T00:00:00Z")
+ connector := newTestS3CompatibleConnector(t, []s3Object{
+ {Key: "docs/old.txt", LastModified: old, Size: 8, ETag: `"old-etag"`},
+ {Key: "docs/new.txt", LastModified: updated, Size: 8, ETag: `"new-etag"`},
+ })
+
+ start := mustTime(t, "2026-01-02T00:00:00Z")
+ session, err := connector.OpenSync(context.Background(), SyncRequest{
+ WindowStart: &start,
+ WindowEnd: mustTime(t, "2026-01-04T00:00:00Z"),
+ Fingerprints: map[string]string{
+ s3SourceID(s3CompatibleSource, "bucket", "docs/old.txt"): normalizedS3ETag(`"old-etag"`),
+ },
+ })
+ if err != nil {
+ t.Fatalf("OpenSync: %v", err)
+ }
+
+ batch, err := session.NextBatch(context.Background())
+ if err != nil {
+ t.Fatalf("NextBatch: %v", err)
+ }
+ if len(batch.Documents) != 1 {
+ t.Fatalf("documents len = %d, want 1", len(batch.Documents))
+ }
+ doc := batch.Documents[0]
+ if doc.SourceID != s3SourceID(s3CompatibleSource, "bucket", "docs/new.txt") {
+ t.Fatalf("source id = %q", doc.SourceID)
+ }
+ if doc.SemanticIdentifier != "new.txt" {
+ t.Fatalf("semantic identifier = %q", doc.SemanticIdentifier)
+ }
+ if doc.Extension != ".txt" {
+ t.Fatalf("extension = %q", doc.Extension)
+ }
+ if doc.Fingerprint != normalizedS3ETag(`"new-etag"`) {
+ t.Fatalf("fingerprint = %q", doc.Fingerprint)
+ }
+ if doc.FetchRef == nil {
+ t.Fatalf("fetch ref is nil")
+ }
+ fetcher, ok := session.(Fetcher)
+ if !ok {
+ t.Fatalf("session does not implement Fetcher")
+ }
+ blob, err := fetcher.Fetch(context.Background(), *doc.FetchRef)
+ if err != nil {
+ t.Fatalf("Fetch: %v", err)
+ }
+ if string(blob) != "body:new" {
+ t.Fatalf("blob = %q", blob)
+ }
+ if _, err = session.NextBatch(context.Background()); !errors.Is(err, io.EOF) {
+ t.Fatalf("NextBatch EOF = %v", err)
+ }
+}
+
+func TestS3CompatibleConnectorOpenSyncDefersListingUntilNextBatch(t *testing.T) {
+ connector := newTestS3CompatibleConnector(t, []s3Object{
+ {Key: "docs/a.txt", LastModified: mustTime(t, "2026-01-01T00:00:00Z"), Size: 1, ETag: "a"},
+ })
+ var listCalls int
+ baseListObjects := connector.listObjects
+ connector.listObjects = func(ctx context.Context, startAfter string, maxKeys int32) ([]s3Object, string, bool, error) {
+ listCalls++
+ return baseListObjects(ctx, startAfter, maxKeys)
+ }
+
+ session, err := connector.OpenSync(context.Background(), SyncRequest{FromBeginning: true})
+ if err != nil {
+ t.Fatalf("OpenSync: %v", err)
+ }
+ if listCalls != 0 {
+ t.Fatalf("OpenSync list calls = %d, want 0", listCalls)
+ }
+ if _, err := session.NextBatch(context.Background()); err != nil {
+ t.Fatalf("NextBatch: %v", err)
+ }
+ if listCalls != 1 {
+ t.Fatalf("NextBatch list calls = %d, want 1", listCalls)
+ }
+}
+
+func TestS3CompatibleConnectorOpenSyncFiltersImagesUnlessAllowed(t *testing.T) {
+ objects := []s3Object{
+ {Key: "docs/a.png", LastModified: mustTime(t, "2026-01-01T00:00:00Z"), Size: 1, ETag: "a"},
+ {Key: "docs/b.txt", LastModified: mustTime(t, "2026-01-02T00:00:00Z"), Size: 1, ETag: "b"},
+ }
+ connector := newTestS3CompatibleConnector(t, objects)
+ session, err := connector.OpenSync(context.Background(), SyncRequest{FromBeginning: true})
+ if err != nil {
+ t.Fatalf("OpenSync: %v", err)
+ }
+ batch, err := session.NextBatch(context.Background())
+ if err != nil {
+ t.Fatalf("NextBatch: %v", err)
+ }
+ if len(batch.Documents) != 1 || batch.Documents[0].SourceID != s3SourceID(s3CompatibleSource, "bucket", "docs/b.txt") {
+ t.Fatalf("documents = %+v", batch.Documents)
+ }
+
+ allowed := newTestS3CompatibleConnector(t, objects)
+ allowed.allowImages = true
+ session, err = allowed.OpenSync(context.Background(), SyncRequest{FromBeginning: true})
+ if err != nil {
+ t.Fatalf("allow images OpenSync: %v", err)
+ }
+ batch, err = session.NextBatch(context.Background())
+ if err != nil {
+ t.Fatalf("allow images NextBatch: %v", err)
+ }
+ if len(batch.Documents) != 2 {
+ t.Fatalf("allow images documents len = %d, want 2", len(batch.Documents))
+ }
+}
+
+func TestS3CompatibleConnectorOpenSyncResumesFromCheckpoint(t *testing.T) {
+ connector := newTestS3CompatibleConnector(t, []s3Object{
+ {Key: "docs/a.txt", LastModified: mustTime(t, "2026-01-01T00:00:00Z"), Size: 1, ETag: "a"},
+ {Key: "docs/b.txt", LastModified: mustTime(t, "2026-01-02T00:00:00Z"), Size: 1, ETag: "b"},
+ {Key: "docs/c.txt", LastModified: mustTime(t, "2026-01-03T00:00:00Z"), Size: 1, ETag: "c"},
+ })
+ checkpointUpdatedAt := mustTime(t, "2026-01-02T00:00:00Z")
+ session, err := connector.OpenSync(context.Background(), SyncRequest{
+ FromBeginning: true,
+ Resume: &SyncCheckpoint{
+ Cursor: s3SourceID(s3CompatibleSource, "bucket", "docs/b.txt"),
+ UpdatedAt: &checkpointUpdatedAt,
+ },
+ })
+ if err != nil {
+ t.Fatalf("OpenSync: %v", err)
+ }
+ batch, err := session.NextBatch(context.Background())
+ if err != nil {
+ t.Fatalf("NextBatch: %v", err)
+ }
+ if len(batch.Documents) != 1 {
+ t.Fatalf("documents len = %d, want 1", len(batch.Documents))
+ }
+ if batch.Documents[0].SourceID != s3SourceID(s3CompatibleSource, "bucket", "docs/c.txt") {
+ t.Fatalf("resumed source id = %q, want docs/c.txt", batch.Documents[0].SourceID)
+ }
+ if batch.Checkpoint == nil || batch.Checkpoint.SourceID != s3SourceID(s3CompatibleSource, "bucket", "docs/c.txt") {
+ t.Fatalf("checkpoint = %+v", batch.Checkpoint)
+ }
+}
+
+func TestS3CompatibleConnectorOpenSyncRejectsInvalidResumeAnchors(t *testing.T) {
+ connector := newTestS3CompatibleConnector(t, []s3Object{
+ {Key: "docs/a.txt", LastModified: mustTime(t, "2026-01-01T00:00:00Z"), Size: 1, ETag: "a"},
+ {Key: "docs/b.txt", LastModified: mustTime(t, "2026-01-02T00:00:00Z"), Size: 1, ETag: "b"},
+ })
+ checkpoints := []*SyncCheckpoint{
+ {Cursor: "other:bucket:docs/a.txt"},
+ {Cursor: s3SourceID(s3CompatibleSource, "bucket", "")},
+ {Cursor: s3SourceID(s3CompatibleSource, "bucket", "docs/missing.txt")},
+ }
+ for _, checkpoint := range checkpoints {
+ _, err := connector.OpenSync(context.Background(), SyncRequest{FromBeginning: true, Resume: checkpoint})
+ if !errors.Is(err, ErrSyncResumeInvalid) {
+ t.Fatalf("OpenSync error = %v for checkpoint %+v, want ErrSyncResumeInvalid", err, checkpoint)
+ }
+ }
+}
+
+func TestS3CompatibleConnectorOpenSyncIncludesMissingFingerprint(t *testing.T) {
+ connector := newTestS3CompatibleConnector(t, []s3Object{
+ {Key: "docs/no-etag.txt", LastModified: mustTime(t, "2026-01-01T00:00:00Z"), Size: 1},
+ })
+ session, err := connector.OpenSync(context.Background(), SyncRequest{
+ Fingerprints: map[string]string{"other": "fingerprint"},
+ })
+ if err != nil {
+ t.Fatalf("OpenSync: %v", err)
+ }
+ batch, err := session.NextBatch(context.Background())
+ if err != nil {
+ t.Fatalf("NextBatch: %v", err)
+ }
+ if len(batch.Documents) != 1 {
+ t.Fatalf("documents len = %d, want 1", len(batch.Documents))
+ }
+}
+
+func TestS3CompatibleConnectorOpenSyncFiltersByWindow(t *testing.T) {
+ start := mustTime(t, "2026-01-02T00:00:00Z")
+ end := mustTime(t, "2026-01-04T00:00:00Z")
+ connector := newTestS3CompatibleConnector(t, []s3Object{
+ {Key: "docs/before.txt", LastModified: mustTime(t, "2026-01-01T00:00:00Z"), Size: 1, ETag: "before"},
+ {Key: "docs/within.txt", LastModified: mustTime(t, "2026-01-03T00:00:00Z"), Size: 1, ETag: "within"},
+ {Key: "docs/after.txt", LastModified: mustTime(t, "2026-01-05T00:00:00Z"), Size: 1, ETag: "after"},
+ })
+ session, err := connector.OpenSync(context.Background(), SyncRequest{WindowStart: &start, WindowEnd: end})
+ if err != nil {
+ t.Fatalf("OpenSync: %v", err)
+ }
+ batch, err := session.NextBatch(context.Background())
+ if err != nil {
+ t.Fatalf("NextBatch: %v", err)
+ }
+ if len(batch.Documents) != 1 || batch.Documents[0].SourceID != s3SourceID(s3CompatibleSource, "bucket", "docs/within.txt") {
+ t.Fatalf("documents = %+v", batch.Documents)
+ }
+}
+
+func TestS3CompatibleConnectorOpenPruneReturnsSlimSnapshot(t *testing.T) {
+ connector := newTestS3CompatibleConnector(t, []s3Object{
+ {Key: "docs/a.txt", LastModified: mustTime(t, "2026-01-01T00:00:00Z"), Size: 1, ETag: "a"},
+ {Key: "docs/b.txt", LastModified: mustTime(t, "2026-01-02T00:00:00Z"), Size: 1, ETag: "b"},
+ {Key: "docs/folder/", LastModified: mustTime(t, "2026-01-02T00:00:00Z"), Size: 0, ETag: "folder"},
+ })
+ session, err := connector.OpenPrune(context.Background(), PruneRequest{})
+ if err != nil {
+ t.Fatalf("OpenPrune: %v", err)
+ }
+ batch, err := session.NextBatch(context.Background())
+ if err != nil {
+ t.Fatalf("NextBatch: %v", err)
+ }
+ if len(batch.Documents) != 2 {
+ t.Fatalf("documents len = %d, want 2", len(batch.Documents))
+ }
+ if batch.Documents[0].SourceID != s3SourceID(s3CompatibleSource, "bucket", "docs/a.txt") || batch.Documents[1].SourceID != s3SourceID(s3CompatibleSource, "bucket", "docs/b.txt") {
+ t.Fatalf("documents = %+v", batch.Documents)
+ }
+}
+
+func TestS3CompatibleConnectorOpenPruneStreamsAcrossPages(t *testing.T) {
+ connector, err := NewS3CompatibleConnector(map[string]any{
+ "bucket_name": "bucket",
+ "batch_size": 2,
+ "credentials": map[string]any{
+ "endpoint_url": "https://s3.example.com",
+ "aws_access_key_id": "access",
+ "aws_secret_access_key": "secret",
+ "addressing_style": "path",
+ },
+ })
+ if err != nil {
+ t.Fatalf("NewS3CompatibleConnector: %v", err)
+ }
+ pages := [][]s3Object{
+ {
+ {Key: "docs/a.txt", Size: 1},
+ {Key: "docs/b.txt", Size: 1},
+ },
+ {
+ {Key: "docs/c.txt", Size: 1},
+ {Key: "docs/d.txt", Size: 1},
+ },
+ }
+ call := 0
+ connector.listObjects = func(ctx context.Context, startAfter string, maxKeys int32) ([]s3Object, string, bool, error) {
+ if call >= len(pages) {
+ return nil, "", false, nil
+ }
+ page := pages[call]
+ call++
+ nextStartAfter := ""
+ if len(page) > 0 {
+ nextStartAfter = s3SourceID(s3CompatibleSource, connector.bucketName, page[len(page)-1].Key)
+ }
+ return page, nextStartAfter, call < len(pages), nil
+ }
+ session, err := connector.OpenPrune(context.Background(), PruneRequest{})
+ if err != nil {
+ t.Fatalf("OpenPrune: %v", err)
+ }
+ first, err := session.NextBatch(context.Background())
+ if err != nil {
+ t.Fatalf("first NextBatch: %v", err)
+ }
+ if len(first.Documents) != 2 || first.Documents[0].SourceID != s3SourceID(s3CompatibleSource, "bucket", "docs/a.txt") || first.Documents[1].SourceID != s3SourceID(s3CompatibleSource, "bucket", "docs/b.txt") {
+ t.Fatalf("first batch = %+v", first.Documents)
+ }
+ second, err := session.NextBatch(context.Background())
+ if err != nil {
+ t.Fatalf("second NextBatch: %v", err)
+ }
+ if len(second.Documents) != 2 || second.Documents[0].SourceID != s3SourceID(s3CompatibleSource, "bucket", "docs/c.txt") || second.Documents[1].SourceID != s3SourceID(s3CompatibleSource, "bucket", "docs/d.txt") {
+ t.Fatalf("second batch = %+v", second.Documents)
+ }
+ if _, err := session.NextBatch(context.Background()); !errors.Is(err, io.EOF) {
+ t.Fatalf("NextBatch EOF = %v", err)
+ }
+}
+
+func TestS3CompatibleConnectorValidate(t *testing.T) {
+ ctx := context.Background()
+ missingBucket := &S3CompatibleConnector{endpointURL: "https://s3.example.com", accessKeyID: "access", secretKey: "secret", addressingStyle: "virtual", batchSize: 2}
+ if err := missingBucket.Validate(ctx); err == nil {
+ t.Fatalf("expected bucket name validation error")
+ }
+ missingEndpoint := &S3CompatibleConnector{bucketName: "bucket", accessKeyID: "access", secretKey: "secret", addressingStyle: "virtual", batchSize: 2}
+ if err := missingEndpoint.Validate(ctx); err == nil {
+ t.Fatalf("expected endpoint validation error")
+ }
+ missingCreds := &S3CompatibleConnector{bucketName: "bucket", endpointURL: "https://s3.example.com", addressingStyle: "virtual", batchSize: 2}
+ if err := missingCreds.Validate(ctx); err == nil {
+ t.Fatalf("expected credentials validation error")
+ }
+ badStyle := &S3CompatibleConnector{bucketName: "bucket", endpointURL: "https://s3.example.com", accessKeyID: "access", secretKey: "secret", addressingStyle: "invalid", batchSize: 2}
+ if err := badStyle.Validate(ctx); err == nil {
+ t.Fatalf("expected addressing style validation error")
+ }
+ badBatch := &S3CompatibleConnector{bucketName: "bucket", endpointURL: "https://s3.example.com", accessKeyID: "access", secretKey: "secret", addressingStyle: "virtual"}
+ if err := badBatch.Validate(ctx); err == nil {
+ t.Fatalf("expected batch size validation error")
+ }
+}
+
+func TestS3CompatibleConnectorRejectsInvalidEndpointBeforeClientCreation(t *testing.T) {
+ for _, endpoint := range []string{
+ "https://evil.example#",
+ "evil.example#",
+ "ftp://evil.example",
+ "https://user:[email protected]",
+ "https://evil.example#fragment",
+ "https://",
+ "https://evil.example/pa th",
+ } {
+ connector := newTestS3CompatibleConnector(t, nil)
+ connector.endpointURL = endpoint
+ connector.client = nil
+ if err := connector.Validate(context.Background()); err == nil {
+ t.Fatalf("expected invalid endpoint %q to be rejected", endpoint)
+ }
+ if connector.client != nil {
+ t.Fatalf("client was created for invalid endpoint %q", endpoint)
+ }
+ }
+}
+
+func TestS3CompatibleConnectorValidateSetting(t *testing.T) {
+ connector := newTestS3CompatibleConnector(t, []s3Object{
+ {Key: "docs/a.txt", Size: 1},
+ })
+ var listCalls int
+ connector.listObjects = func(ctx context.Context, startAfter string, maxKeys int32) ([]s3Object, string, bool, error) {
+ listCalls++
+ return nil, "", false, nil
+ }
+ if err := connector.ValidateConnectorSetting(context.Background(), nil); err != nil {
+ t.Fatalf("ValidateConnectorSetting: %v", err)
+ }
+ if listCalls != 1 {
+ t.Fatalf("list calls = %d, want 1", listCalls)
+ }
+}
+
+func TestNewS3CompatibleConnectorDefaultsAndConfig(t *testing.T) {
+ connector, err := NewS3CompatibleConnector(map[string]any{
+ "bucket_name": " bucket ",
+ "prefix": "docs",
+ "batch_size": 4,
+ "size_threshold": 42,
+ "allow_images": true,
+ "credentials": map[string]any{
+ "endpoint_url": " https://s3.example.com ",
+ "aws_access_key_id": "access",
+ "aws_secret_access_key": "secret",
+ "addressing_style": "path",
+ },
+ })
+ if err != nil {
+ t.Fatalf("NewS3CompatibleConnector: %v", err)
+ }
+ if connector.bucketName != "bucket" || connector.prefix != "docs/" {
+ t.Fatalf("bucket/prefix = %q/%q", connector.bucketName, connector.prefix)
+ }
+ if connector.batchSize != 4 || connector.sizeThreshold != 42 || !connector.allowImages {
+ t.Fatalf("batch/threshold/allow_images = %d/%d/%v", connector.batchSize, connector.sizeThreshold, connector.allowImages)
+ }
+ if connector.endpointURL != "https://s3.example.com" || connector.addressingStyle != "path" || connector.region != defaultS3CompatibleRegion {
+ t.Fatalf("endpoint/style/region = %q/%q/%q", connector.endpointURL, connector.addressingStyle, connector.region)
+ }
+
+ defaults, err := NewS3CompatibleConnector(map[string]any{"bucket_name": "bucket", "credentials": map[string]any{}})
+ if err != nil {
+ t.Fatalf("NewS3CompatibleConnector defaults: %v", err)
+ }
+ if defaults.batchSize != defaultS3BatchSize || defaults.sizeThreshold != defaultS3SizeThreshold || defaults.region != defaultS3CompatibleRegion || defaults.addressingStyle != "virtual" {
+ t.Fatalf("defaults = batch %d threshold %d region %q style %q", defaults.batchSize, defaults.sizeThreshold, defaults.region, defaults.addressingStyle)
+ }
+}
+
+func TestS3CompatibleHelpers(t *testing.T) {
+ if got := s3SourceID(s3CompatibleSource, "bucket", "docs/a.txt"); got != "s3_compatible:bucket:docs/a.txt" {
+ t.Fatalf("source id = %q", got)
+ }
+ if got := normalizeS3Prefix("docs"); got != "docs/" {
+ t.Fatalf("prefix = %q", got)
+ }
+ if got := pathEscapeS3Key("docs/a b.txt"); got != "docs/a%20b.txt" {
+ t.Fatalf("escaped key = %q", got)
+ }
+ if got := strings.TrimSpace(s3ConsoleURL("us-east-1", "bucket", "docs/a b.txt")); !strings.HasPrefix(got, "https://s3.console.aws.amazon.com/s3/object/bucket?region=us-east-1&prefix=docs/a%20b.txt") {
+ t.Fatalf("console url = %q", got)
+ }
+ if _, err := readS3Body(bytes.NewBufferString("12345"), "large.txt", 4); err == nil {
+ t.Fatalf("expected size threshold error")
+ }
+}
+
+func newTestS3CompatibleConnector(t *testing.T, objects []s3Object) *S3CompatibleConnector {
+ t.Helper()
+ connector, err := NewS3CompatibleConnector(map[string]any{
+ "bucket_name": "bucket",
+ "prefix": "docs",
+ "batch_size": 2,
+ "credentials": map[string]any{
+ "endpoint_url": "https://s3.example.com",
+ "aws_access_key_id": "access",
+ "aws_secret_access_key": "secret",
+ "addressing_style": "virtual",
+ },
+ })
+ if err != nil {
+ t.Fatalf("NewS3CompatibleConnector: %v", err)
+ }
+ connector.listObjects = func(ctx context.Context, startAfter string, maxKeys int32) ([]s3Object, string, bool, error) {
+ var out []s3Object
+ for _, object := range objects {
+ if startAfter != "" && object.Key <= startAfter {
+ continue
+ }
+ out = append(out, object)
+ if maxKeys > 0 && len(out) >= int(maxKeys) {
+ break
+ }
+ }
+ hasMore := false
+ nextStartAfter := ""
+ if len(out) > 0 {
+ nextStartAfter = s3SourceID(s3CompatibleSource, connector.bucketName, out[len(out)-1].Key)
+ for _, object := range objects {
+ if object.Key > out[len(out)-1].Key {
+ hasMore = true
+ break
+ }
+ }
+ }
+ return out, nextStartAfter, hasMore, nil
+ }
+ connector.downloadObject = func(ctx context.Context, key string, sizeThreshold int64) ([]byte, error) {
+ if key == "docs/new.txt" {
+ return []byte("body:new"), nil
+ }
+ return []byte("body:" + key), nil
+ }
+ return connector
+}
diff --git a/internal/syncer/connector/s3_test.go b/internal/syncer/connector/s3_test.go
new file mode 100644
index 0000000000..23f27e60ce
--- /dev/null
+++ b/internal/syncer/connector/s3_test.go
@@ -0,0 +1,496 @@
+package connector
+
+import (
+ "bytes"
+ "context"
+ "errors"
+ "io"
+ "strings"
+ "testing"
+)
+
+func TestS3ConnectorOpenSyncUsesFingerprintAndFetch(t *testing.T) {
+ old := mustTime(t, "2026-01-01T00:00:00Z")
+ updated := mustTime(t, "2026-01-03T00:00:00Z")
+ connector := newTestS3Connector(t, []s3Object{
+ {Key: "docs/old.txt", LastModified: old, Size: 8, ETag: `"old-etag"`},
+ {Key: "docs/new.txt", LastModified: updated, Size: 8, ETag: `"new-etag"`},
+ })
+
+ start := mustTime(t, "2026-01-02T00:00:00Z")
+ session, err := connector.OpenSync(context.Background(), SyncRequest{
+ WindowStart: &start,
+ WindowEnd: mustTime(t, "2026-01-04T00:00:00Z"),
+ Fingerprints: map[string]string{
+ s3SourceID(s3Source, "bucket", "docs/old.txt"): normalizedS3ETag(`"old-etag"`),
+ },
+ })
+ if err != nil {
+ t.Fatalf("OpenSync: %v", err)
+ }
+
+ batch, err := session.NextBatch(context.Background())
+ if err != nil {
+ t.Fatalf("NextBatch: %v", err)
+ }
+ if len(batch.Documents) != 1 {
+ t.Fatalf("documents len = %d, want 1", len(batch.Documents))
+ }
+ doc := batch.Documents[0]
+ if doc.SourceID != s3SourceID(s3Source, "bucket", "docs/new.txt") {
+ t.Fatalf("source id = %q", doc.SourceID)
+ }
+ if doc.SemanticIdentifier != "new.txt" {
+ t.Fatalf("semantic identifier = %q", doc.SemanticIdentifier)
+ }
+ if doc.Extension != ".txt" {
+ t.Fatalf("extension = %q", doc.Extension)
+ }
+ if doc.Fingerprint != normalizedS3ETag(`"new-etag"`) {
+ t.Fatalf("fingerprint = %q", doc.Fingerprint)
+ }
+ if doc.FetchRef == nil {
+ t.Fatalf("fetch ref is nil")
+ }
+ fetcher, ok := session.(Fetcher)
+ if !ok {
+ t.Fatalf("session does not implement Fetcher")
+ }
+ blob, err := fetcher.Fetch(context.Background(), *doc.FetchRef)
+ if err != nil {
+ t.Fatalf("Fetch: %v", err)
+ }
+ if string(blob) != "body:new" {
+ t.Fatalf("blob = %q", blob)
+ }
+ if _, err = session.NextBatch(context.Background()); !errors.Is(err, io.EOF) {
+ t.Fatalf("NextBatch EOF = %v", err)
+ }
+}
+
+func TestS3ConnectorOpenSyncDefersListingUntilNextBatch(t *testing.T) {
+ connector := newTestS3Connector(t, []s3Object{
+ {Key: "docs/a.txt", LastModified: mustTime(t, "2026-01-01T00:00:00Z"), Size: 1, ETag: "a"},
+ })
+ var listCalls int
+ baseListObjects := connector.listObjects
+ connector.listObjects = func(ctx context.Context, startAfter string, maxKeys int32) ([]s3Object, string, bool, error) {
+ listCalls++
+ return baseListObjects(ctx, startAfter, maxKeys)
+ }
+
+ session, err := connector.OpenSync(context.Background(), SyncRequest{FromBeginning: true})
+ if err != nil {
+ t.Fatalf("OpenSync: %v", err)
+ }
+ if listCalls != 0 {
+ t.Fatalf("OpenSync list calls = %d, want 0", listCalls)
+ }
+ if _, err := session.NextBatch(context.Background()); err != nil {
+ t.Fatalf("NextBatch: %v", err)
+ }
+ if listCalls != 1 {
+ t.Fatalf("NextBatch list calls = %d, want 1", listCalls)
+ }
+}
+
+func TestS3ConnectorOpenSyncFiltersImagesUnlessAllowed(t *testing.T) {
+ objects := []s3Object{
+ {Key: "docs/a.png", LastModified: mustTime(t, "2026-01-01T00:00:00Z"), Size: 1, ETag: "a"},
+ {Key: "docs/b.txt", LastModified: mustTime(t, "2026-01-02T00:00:00Z"), Size: 1, ETag: "b"},
+ }
+ connector := newTestS3Connector(t, objects)
+ session, err := connector.OpenSync(context.Background(), SyncRequest{FromBeginning: true})
+ if err != nil {
+ t.Fatalf("OpenSync: %v", err)
+ }
+ batch, err := session.NextBatch(context.Background())
+ if err != nil {
+ t.Fatalf("NextBatch: %v", err)
+ }
+ if len(batch.Documents) != 1 || batch.Documents[0].SourceID != s3SourceID(s3Source, "bucket", "docs/b.txt") {
+ t.Fatalf("documents = %+v", batch.Documents)
+ }
+
+ allowed := newTestS3Connector(t, objects)
+ allowed.allowImages = true
+ session, err = allowed.OpenSync(context.Background(), SyncRequest{FromBeginning: true})
+ if err != nil {
+ t.Fatalf("allow images OpenSync: %v", err)
+ }
+ batch, err = session.NextBatch(context.Background())
+ if err != nil {
+ t.Fatalf("allow images NextBatch: %v", err)
+ }
+ if len(batch.Documents) != 2 {
+ t.Fatalf("allow images documents len = %d, want 2", len(batch.Documents))
+ }
+}
+
+func TestS3ConnectorOpenSyncResumesFromCheckpoint(t *testing.T) {
+ connector := newTestS3Connector(t, []s3Object{
+ {Key: "docs/a.txt", LastModified: mustTime(t, "2026-01-01T00:00:00Z"), Size: 1, ETag: "a"},
+ {Key: "docs/b.txt", LastModified: mustTime(t, "2026-01-02T00:00:00Z"), Size: 1, ETag: "b"},
+ {Key: "docs/c.txt", LastModified: mustTime(t, "2026-01-03T00:00:00Z"), Size: 1, ETag: "c"},
+ })
+ checkpointUpdatedAt := mustTime(t, "2026-01-02T00:00:00Z")
+ session, err := connector.OpenSync(context.Background(), SyncRequest{
+ FromBeginning: true,
+ Resume: &SyncCheckpoint{
+ Cursor: s3SourceID(s3Source, "bucket", "docs/b.txt"),
+ SourceID: s3SourceID(s3Source, "bucket", "docs/b.txt"),
+ UpdatedAt: &checkpointUpdatedAt,
+ },
+ })
+ if err != nil {
+ t.Fatalf("OpenSync: %v", err)
+ }
+ batch, err := session.NextBatch(context.Background())
+ if err != nil {
+ t.Fatalf("NextBatch: %v", err)
+ }
+ if len(batch.Documents) != 1 {
+ t.Fatalf("documents len = %d, want 1", len(batch.Documents))
+ }
+ if batch.Documents[0].SourceID != s3SourceID(s3Source, "bucket", "docs/c.txt") {
+ t.Fatalf("resumed source id = %q, want docs/c.txt", batch.Documents[0].SourceID)
+ }
+ if batch.Checkpoint == nil || batch.Checkpoint.SourceID != s3SourceID(s3Source, "bucket", "docs/c.txt") {
+ t.Fatalf("checkpoint = %+v", batch.Checkpoint)
+ }
+}
+
+func TestS3ConnectorOpenSyncRejectsInvalidResumeAnchors(t *testing.T) {
+ connector := newTestS3Connector(t, []s3Object{
+ {Key: "docs/a.txt", LastModified: mustTime(t, "2026-01-01T00:00:00Z"), Size: 1, ETag: "a"},
+ {Key: "docs/b.txt", LastModified: mustTime(t, "2026-01-02T00:00:00Z"), Size: 1, ETag: "b"},
+ })
+ checkpoints := []*SyncCheckpoint{
+ {SourceID: "other:bucket:docs/a.txt"},
+ {SourceID: s3SourceID(s3Source, "bucket", "")},
+ {SourceID: s3SourceID(s3Source, "bucket", "docs/missing.txt")},
+ }
+ for _, checkpoint := range checkpoints {
+ _, err := connector.OpenSync(context.Background(), SyncRequest{FromBeginning: true, Resume: checkpoint})
+ if !errors.Is(err, ErrSyncResumeInvalid) {
+ t.Fatalf("OpenSync error = %v for checkpoint %+v, want ErrSyncResumeInvalid", err, checkpoint)
+ }
+ }
+}
+
+func TestS3ConnectorOpenSyncIncludesMissingFingerprint(t *testing.T) {
+ connector := newTestS3Connector(t, []s3Object{
+ {Key: "docs/no-etag.txt", LastModified: mustTime(t, "2026-01-01T00:00:00Z"), Size: 1},
+ })
+ session, err := connector.OpenSync(context.Background(), SyncRequest{
+ Fingerprints: map[string]string{"other": "fingerprint"},
+ })
+ if err != nil {
+ t.Fatalf("OpenSync: %v", err)
+ }
+ batch, err := session.NextBatch(context.Background())
+ if err != nil {
+ t.Fatalf("NextBatch: %v", err)
+ }
+ if len(batch.Documents) != 1 {
+ t.Fatalf("documents len = %d, want 1", len(batch.Documents))
+ }
+}
+
+func TestS3ConnectorOpenSyncFiltersByWindow(t *testing.T) {
+ start := mustTime(t, "2026-01-02T00:00:00Z")
+ end := mustTime(t, "2026-01-04T00:00:00Z")
+ connector := newTestS3Connector(t, []s3Object{
+ {Key: "docs/before.txt", LastModified: mustTime(t, "2026-01-01T00:00:00Z"), Size: 1, ETag: "before"},
+ {Key: "docs/within.txt", LastModified: mustTime(t, "2026-01-03T00:00:00Z"), Size: 1, ETag: "within"},
+ {Key: "docs/after.txt", LastModified: mustTime(t, "2026-01-05T00:00:00Z"), Size: 1, ETag: "after"},
+ })
+ session, err := connector.OpenSync(context.Background(), SyncRequest{WindowStart: &start, WindowEnd: end})
+ if err != nil {
+ t.Fatalf("OpenSync: %v", err)
+ }
+ batch, err := session.NextBatch(context.Background())
+ if err != nil {
+ t.Fatalf("NextBatch: %v", err)
+ }
+ if len(batch.Documents) != 1 || batch.Documents[0].SourceID != s3SourceID(s3Source, "bucket", "docs/within.txt") {
+ t.Fatalf("documents = %+v", batch.Documents)
+ }
+}
+
+func TestS3ConnectorOpenPruneReturnsSlimSnapshot(t *testing.T) {
+ connector := newTestS3Connector(t, []s3Object{
+ {Key: "docs/a.txt", LastModified: mustTime(t, "2026-01-01T00:00:00Z"), Size: 1, ETag: "a"},
+ {Key: "docs/b.txt", LastModified: mustTime(t, "2026-01-02T00:00:00Z"), Size: 1, ETag: "b"},
+ {Key: "docs/folder/", LastModified: mustTime(t, "2026-01-02T00:00:00Z"), Size: 0, ETag: "folder"},
+ })
+ session, err := connector.OpenPrune(context.Background(), PruneRequest{})
+ if err != nil {
+ t.Fatalf("OpenPrune: %v", err)
+ }
+ batch, err := session.NextBatch(context.Background())
+ if err != nil {
+ t.Fatalf("NextBatch: %v", err)
+ }
+ if len(batch.Documents) != 2 {
+ t.Fatalf("documents len = %d, want 2", len(batch.Documents))
+ }
+ if batch.Documents[0].SourceID != s3SourceID(s3Source, "bucket", "docs/a.txt") || batch.Documents[1].SourceID != s3SourceID(s3Source, "bucket", "docs/b.txt") {
+ t.Fatalf("documents = %+v", batch.Documents)
+ }
+}
+
+func TestS3ConnectorOpenPruneStreamsAcrossPages(t *testing.T) {
+ connector, err := NewS3Connector(map[string]any{
+ "bucket_name": "bucket",
+ "batch_size": 2,
+ "credentials": map[string]any{
+ "aws_access_key_id": "access",
+ "aws_secret_access_key": "secret",
+ },
+ })
+ if err != nil {
+ t.Fatalf("NewS3Connector: %v", err)
+ }
+ pages := [][]s3Object{
+ {
+ {Key: "docs/a.txt", Size: 1},
+ {Key: "docs/b.txt", Size: 1},
+ },
+ {
+ {Key: "docs/c.txt", Size: 1},
+ {Key: "docs/d.txt", Size: 1},
+ },
+ }
+ call := 0
+ connector.listObjects = func(ctx context.Context, startAfter string, maxKeys int32) ([]s3Object, string, bool, error) {
+ if call >= len(pages) {
+ return nil, "", false, nil
+ }
+ page := pages[call]
+ call++
+ nextStartAfter := ""
+ if len(page) > 0 {
+ nextStartAfter = s3SourceID(s3Source, connector.bucketName, page[len(page)-1].Key)
+ }
+ return page, nextStartAfter, call < len(pages), nil
+ }
+ session, err := connector.OpenPrune(context.Background(), PruneRequest{})
+ if err != nil {
+ t.Fatalf("OpenPrune: %v", err)
+ }
+ first, err := session.NextBatch(context.Background())
+ if err != nil {
+ t.Fatalf("first NextBatch: %v", err)
+ }
+ if len(first.Documents) != 2 || first.Documents[0].SourceID != s3SourceID(s3Source, "bucket", "docs/a.txt") || first.Documents[1].SourceID != s3SourceID(s3Source, "bucket", "docs/b.txt") {
+ t.Fatalf("first batch = %+v", first.Documents)
+ }
+ second, err := session.NextBatch(context.Background())
+ if err != nil {
+ t.Fatalf("second NextBatch: %v", err)
+ }
+ if len(second.Documents) != 2 || second.Documents[0].SourceID != s3SourceID(s3Source, "bucket", "docs/c.txt") || second.Documents[1].SourceID != s3SourceID(s3Source, "bucket", "docs/d.txt") {
+ t.Fatalf("second batch = %+v", second.Documents)
+ }
+ if _, err := session.NextBatch(context.Background()); !errors.Is(err, io.EOF) {
+ t.Fatalf("NextBatch EOF = %v", err)
+ }
+}
+
+func TestS3ConnectorOpenPruneErrorsWhenPaginationDoesNotAdvance(t *testing.T) {
+ tests := []struct {
+ name string
+ nextStartAfter string
+ }{
+ {name: "empty cursor", nextStartAfter: ""},
+ {name: "unchanged cursor", nextStartAfter: s3SourceID(s3Source, "bucket", "docs/a.txt")},
+ }
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ connector := newTestS3Connector(t, []s3Object{{Key: "docs/a.txt", Size: 1}})
+ connector.listObjects = func(ctx context.Context, startAfter string, maxKeys int32) ([]s3Object, string, bool, error) {
+ return []s3Object{{Key: "docs/a.txt", Size: 1}}, test.nextStartAfter, true, nil
+ }
+ session, err := connector.OpenPrune(context.Background(), PruneRequest{})
+ if err != nil {
+ t.Fatalf("OpenPrune: %v", err)
+ }
+ batch, err := session.NextBatch(context.Background())
+ if err == nil || errors.Is(err, io.EOF) {
+ t.Fatalf("NextBatch error = %v, want non-EOF pagination error", err)
+ }
+ if !strings.Contains(err.Error(), "listing did not advance") {
+ t.Fatalf("NextBatch error = %q, want listing did not advance", err)
+ }
+ if len(batch.Documents) != 0 {
+ t.Fatalf("NextBatch returned partial documents = %+v", batch.Documents)
+ }
+ })
+ }
+}
+
+func TestS3ConnectorValidate(t *testing.T) {
+ ctx := context.Background()
+ missingBucket := &S3Connector{authMethod: "access_key", accessKeyID: "access", secretKey: "secret", batchSize: 2}
+ if err := missingBucket.Validate(ctx); err == nil {
+ t.Fatalf("expected bucket name validation error")
+ }
+ missingCreds := &S3Connector{bucketName: "bucket", authMethod: "access_key", batchSize: 2}
+ if err := missingCreds.Validate(ctx); err == nil {
+ t.Fatalf("expected credentials validation error")
+ }
+ missingRole := &S3Connector{bucketName: "bucket", authMethod: "iam_role", batchSize: 2}
+ if err := missingRole.Validate(ctx); err == nil {
+ t.Fatalf("expected IAM role validation error")
+ }
+ badAuth := &S3Connector{bucketName: "bucket", authMethod: "invalid", batchSize: 2}
+ if err := badAuth.Validate(ctx); err == nil {
+ t.Fatalf("expected unsupported auth method error")
+ }
+ badBatch := &S3Connector{bucketName: "bucket", authMethod: "access_key", accessKeyID: "access", secretKey: "secret"}
+ if err := badBatch.Validate(ctx); err == nil {
+ t.Fatalf("expected batch size validation error")
+ }
+}
+
+func TestS3ConnectorRejectsInvalidRegionBeforeClientCreation(t *testing.T) {
+ for _, region := range []string{
+ "evil.example#",
+ "evil.example.com",
+ "https://evil.example.org",
+ "region space",
+ "-region",
+ "region-",
+ "Region",
+ } {
+ connector := newTestS3Connector(t, nil)
+ connector.region = region
+ connector.client = nil
+ if err := connector.Validate(context.Background()); err == nil {
+ t.Fatalf("expected invalid region %q to be rejected", region)
+ }
+ if connector.client != nil {
+ t.Fatalf("client was created for invalid region %q", region)
+ }
+ }
+}
+
+func TestS3ConnectorValidateSetting(t *testing.T) {
+ connector := newTestS3Connector(t, []s3Object{
+ {Key: "docs/a.txt", Size: 1},
+ })
+ var listCalls int
+ connector.listObjects = func(ctx context.Context, startAfter string, maxKeys int32) ([]s3Object, string, bool, error) {
+ listCalls++
+ return nil, "", false, nil
+ }
+ if err := connector.ValidateConnectorSetting(context.Background(), nil); err != nil {
+ t.Fatalf("ValidateConnectorSetting: %v", err)
+ }
+ if listCalls != 1 {
+ t.Fatalf("list calls = %d, want 1", listCalls)
+ }
+}
+
+func TestNewS3ConnectorDefaultsAndConfig(t *testing.T) {
+ connector, err := NewS3Connector(map[string]any{
+ "bucket_name": " bucket ",
+ "prefix": "docs",
+ "batch_size": 4,
+ "size_threshold": 42,
+ "allow_images": true,
+ "credentials": map[string]any{
+ "authentication_method": "access_key",
+ "region": "us-east-1",
+ "aws_access_key_id": "access",
+ "aws_secret_access_key": "secret",
+ },
+ })
+ if err != nil {
+ t.Fatalf("NewS3Connector: %v", err)
+ }
+ if connector.bucketName != "bucket" || connector.prefix != "docs/" {
+ t.Fatalf("bucket/prefix = %q/%q", connector.bucketName, connector.prefix)
+ }
+ if connector.batchSize != 4 || connector.sizeThreshold != 42 || !connector.allowImages {
+ t.Fatalf("batch/threshold/allow_images = %d/%d/%v", connector.batchSize, connector.sizeThreshold, connector.allowImages)
+ }
+ if connector.region != "us-east-1" || connector.accessKeyID != "access" || connector.secretKey != "secret" {
+ t.Fatalf("credentials = %+v", connector)
+ }
+
+ defaults, err := NewS3Connector(map[string]any{"bucket_name": "bucket", "credentials": map[string]any{}})
+ if err != nil {
+ t.Fatalf("NewS3Connector defaults: %v", err)
+ }
+ if defaults.batchSize != defaultS3BatchSize || defaults.sizeThreshold != defaultS3SizeThreshold || defaults.authMethod != "access_key" {
+ t.Fatalf("defaults = batch %d threshold %d auth %q", defaults.batchSize, defaults.sizeThreshold, defaults.authMethod)
+ }
+}
+
+func TestS3Helpers(t *testing.T) {
+ if got := s3SourceID(s3Source, "bucket", "docs/a.txt"); got != "s3:bucket:docs/a.txt" {
+ t.Fatalf("source id = %q", got)
+ }
+ if got := normalizeS3Prefix("docs"); got != "docs/" {
+ t.Fatalf("prefix = %q", got)
+ }
+ if got := pathEscapeS3Key("docs/a b.txt"); got != "docs/a%20b.txt" {
+ t.Fatalf("escaped key = %q", got)
+ }
+ if got := s3ConsoleURL("us-east-1", "bucket", "docs/a b.txt"); !strings.Contains(got, "https://s3.console.aws.amazon.com/s3/object/bucket?region=us-east-1&prefix=docs/a%20b.txt") {
+ t.Fatalf("console url = %q", got)
+ }
+ if _, err := readS3Body(bytes.NewBufferString("12345"), "large.txt", 4); err == nil {
+ t.Fatalf("expected size threshold error")
+ }
+}
+
+func newTestS3Connector(t *testing.T, objects []s3Object) *S3Connector {
+ t.Helper()
+ connector, err := NewS3Connector(map[string]any{
+ "bucket_name": "bucket",
+ "prefix": "docs",
+ "batch_size": 2,
+ "credentials": map[string]any{
+ "aws_access_key_id": "access",
+ "aws_secret_access_key": "secret",
+ "region": "us-east-1",
+ },
+ })
+ if err != nil {
+ t.Fatalf("NewS3Connector: %v", err)
+ }
+ connector.listObjects = func(ctx context.Context, startAfter string, maxKeys int32) ([]s3Object, string, bool, error) {
+ var out []s3Object
+ for _, object := range objects {
+ if startAfter != "" && object.Key <= startAfter {
+ continue
+ }
+ out = append(out, object)
+ if maxKeys > 0 && len(out) >= int(maxKeys) {
+ break
+ }
+ }
+ hasMore := false
+ nextStartAfter := ""
+ if len(out) > 0 {
+ nextStartAfter = s3SourceID(s3Source, connector.bucketName, out[len(out)-1].Key)
+ for _, object := range objects {
+ if object.Key > out[len(out)-1].Key {
+ hasMore = true
+ break
+ }
+ }
+ }
+ return out, nextStartAfter, hasMore, nil
+ }
+ connector.downloadObject = func(ctx context.Context, key string, sizeThreshold int64) ([]byte, error) {
+ if key == "docs/new.txt" {
+ return []byte("body:new"), nil
+ }
+ return []byte("body:" + key), nil
+ }
+ return connector
+}