RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
feat(go-syncer): add S3 and S3 compatible connectors (#18861)
mkaaadb11b13f8fac6
diff --git a/internal/syncer/connector/builtin.go b/internal/syncer/connector/builtin.goindex 4e4efea62a..dcff5b9246 100644--- a/internal/syncer/connector/builtin.go+++ b/internal/syncer/connector/builtin.go@@ -41,6 +41,8 @@ func RegisterBuiltIns(registry *Registry) {registerBuiltIn(registry, "oci_storage", NewOCIStorageConnector)registerBuiltIn(registry, "azure_blob", NewAzureBlobStorageConnector)registerBuiltIn(registry, "r2", NewR2Connector)+ registerBuiltIn(registry, "s3", NewS3Connector)+ registerBuiltIn(registry, "s3_compatible", NewS3CompatibleConnector)registerBuiltIn(registry, "dingtalk_ai_table", NewDingTalkAITableConnector)registerBuiltIn(registry, "imap", NewIMAPConnector)registerBuiltIn(registry, "jira", NewJiraConnector)diff --git a/internal/syncer/connector/s3.go b/internal/syncer/connector/s3.gonew file mode 100644index 0000000000..cc22e5c64a--- /dev/null+++ b/internal/syncer/connector/s3.go@@ -0,0 +1,588 @@+//+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.+//++package connector++import (+ "context"+ "encoding/json"+ "fmt"+ "io"+ "net/url"+ "path"+ "path/filepath"+ "regexp"+ "strings"+ "time"++ "github.com/aws/aws-sdk-go-v2/aws"+ awssdkconfig "github.com/aws/aws-sdk-go-v2/config"+ "github.com/aws/aws-sdk-go-v2/credentials"+ "github.com/aws/aws-sdk-go-v2/credentials/stscreds"+ "github.com/aws/aws-sdk-go-v2/service/s3"+ "github.com/aws/aws-sdk-go-v2/service/s3/types"+ "github.com/aws/aws-sdk-go-v2/service/sts"+)++const (+ defaultS3BatchSize = 2+ defaultS3SizeThreshold = 20 * 1024 * 1024+ s3ListPageSize = 1000+ s3Source = "s3"+)++var awsRegionLabelPattern = regexp.MustCompile(`^[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?$`)++// S3Connector reads objects from AWS S3.+type S3Connector struct {+ bucketName string+ prefix string+ region string+ authMethod string+ accessKeyID string+ secretKey string+ roleARN string+ allowImages bool+ batchSize int++ sizeThreshold int64+ client *s3.Client+ listObjects func(ctx context.Context, startAfter string, maxKeys int32) ([]s3Object, string, bool, error)+ downloadObject func(ctx context.Context, key string, sizeThreshold int64) ([]byte, error)+}++// NewS3Connector creates an AWS S3 connector from Python-compatible config.+func NewS3Connector(config map[string]any) (*S3Connector, error) {+ credentials := configAnyMap(config["credentials"])+ batchSize := configInt(firstNonEmpty(stringConfig(config["sync_batch_size"]), stringConfig(config["batch_size"])), defaultS3BatchSize)+ sizeThreshold := int64(configInt(config["size_threshold"], defaultS3SizeThreshold))+ if sizeThreshold <= 0 {+ sizeThreshold = defaultS3SizeThreshold+ }+ return &S3Connector{+ bucketName: strings.TrimSpace(stringConfig(config["bucket_name"])),+ prefix: normalizeS3Prefix(stringConfig(config["prefix"])),+ region: strings.TrimSpace(stringConfig(credentials["region"])),+ authMethod: firstNonEmpty(stringConfig(credentials["authentication_method"]), "access_key"),+ accessKeyID: strings.TrimSpace(stringConfig(credentials["aws_access_key_id"])),+ secretKey: stringConfig(credentials["aws_secret_access_key"]),+ roleARN: strings.TrimSpace(stringConfig(credentials["aws_role_arn"])),+ allowImages: configBoolDefault(config["allow_images"], false),+ batchSize: batchSize,+ sizeThreshold: sizeThreshold,+ }, nil+}++// Validate validates AWS S3 connector settings and credentials.+func (c *S3Connector) Validate(ctx context.Context) error {+ if c == nil {+ return fmt.Errorf("S3 connector is nil")+ }+ if c.bucketName == "" {+ return fmt.Errorf("No bucket name was provided in connector settings.")+ }+ if err := validateS3AuthMethod(c.authMethod, c.accessKeyID, c.secretKey, c.roleARN); err != nil {+ return err+ }+ if c.region != "" {+ if err := validateAWSRegion(c.region); err != nil {+ return err+ }+ }+ if c.batchSize <= 0 {+ return fmt.Errorf("batch_size must be a positive integer")+ }+ if _, err := c.ensureClient(ctx); err != nil {+ return err+ }+ return nil+}++// ValidateConnectorSetting validates AWS S3 settings from an unsaved config.+func (c *S3Connector) ValidateConnectorSetting(ctx context.Context, request map[string]any) error {+ ctx, cancel := context.WithTimeout(ctx, connectorSettingValidationTimeout)+ defer cancel()+ if err := c.Validate(ctx); err != nil {+ return err+ }+ _, _, _, err := c.listObjectPage(ctx, "", 1)+ if err != nil {+ return fmt.Errorf("access S3 bucket %q: %w", c.bucketName, err)+ }+ return nil+}++// OpenSync opens one AWS S3 sync session.+func (c *S3Connector) OpenSync(ctx context.Context, request SyncRequest) (SyncSession, error) {+ if err := c.Validate(ctx); err != nil {+ return nil, err+ }+ session := &s3SyncSession{+ connector: c,+ request: request,+ batchSize: c.batchSize,+ }+ if request.Resume != nil {+ if err := session.applyResume(ctx, request.Resume); err != nil {+ return nil, err+ }+ }+ return session, nil+}++// OpenPrune opens one complete AWS S3 prune snapshot session.+func (c *S3Connector) OpenPrune(ctx context.Context, request PruneRequest) (PruneSession, error) {+ if err := c.Validate(ctx); err != nil {+ return nil, err+ }+ return &s3PruneSession{connector: c, batchSize: c.batchSize}, nil+}++// Fetch downloads an AWS S3 object body.+func (c *S3Connector) Fetch(ctx context.Context, ref FetchReference) ([]byte, error) {+ var fetch s3FetchReference+ if err := json.Unmarshal([]byte(ref.Key), &fetch); err != nil {+ return nil, err+ }+ return c.download(ctx, fetch.Key)+}++func (c *S3Connector) ensureClient(ctx context.Context) (*s3.Client, error) {+ if c.client != nil {+ return c.client, nil+ }+ if c.region != "" {+ if err := validateAWSRegion(c.region); err != nil {+ return nil, err+ }+ }+ authMethod := firstNonEmpty(c.authMethod, "access_key")+ options := make([]func(*awssdkconfig.LoadOptions) error, 0, 2)+ if c.region != "" {+ options = append(options, awssdkconfig.WithRegion(c.region))+ }+ switch authMethod {+ case "access_key":+ options = append(options, awssdkconfig.WithCredentialsProvider(credentials.NewStaticCredentialsProvider(c.accessKeyID, c.secretKey, "")))+ case "iam_role":+ if c.roleARN == "" {+ return nil, fmt.Errorf("S3 aws_role_arn is required for iam_role authentication")+ }+ case "assume_role":+ default:+ return nil, fmt.Errorf("unsupported S3 authentication_method %q", authMethod)+ }+ cfg, err := awssdkconfig.LoadDefaultConfig(ctx, options...)+ if err != nil {+ return nil, fmt.Errorf("load AWS S3 config: %w", err)+ }+ if authMethod == "iam_role" {+ cfg.Credentials = aws.NewCredentialsCache(stscreds.NewAssumeRoleProvider(sts.NewFromConfig(cfg), c.roleARN))+ }+ c.client = s3.NewFromConfig(cfg)+ return c.client, nil+}++func (c *S3Connector) listObjectPage(ctx context.Context, startAfter string, maxKeys int32) ([]s3Object, string, bool, error) {+ return listS3ObjectPage(ctx, c.listObjects, c.ensureClient, c.bucketName, c.prefix, s3Source, startAfter, maxKeys)+}++func (c *S3Connector) download(ctx context.Context, key string) ([]byte, error) {+ return downloadS3Object(ctx, c.downloadObject, c.ensureClient, c.bucketName, key, c.sizeThreshold)+}++func listS3ObjectPage(ctx context.Context, listObjects func(ctx context.Context, startAfter string, maxKeys int32) ([]s3Object, string, bool, error), ensureClient func(context.Context) (*s3.Client, error), bucketName, prefix, source, startAfter string, maxKeys int32) ([]s3Object, string, bool, error) {+ if listObjects != nil {+ return listObjects(ctx, startAfter, maxKeys)+ }+ client, err := ensureClient(ctx)+ if err != nil {+ return nil, "", false, err+ }+ input := &s3.ListObjectsV2Input{+ Bucket: aws.String(bucketName),+ Prefix: aws.String(prefix),+ StartAfter: aws.String(startAfter),+ }+ if maxKeys > 0 {+ input.MaxKeys = aws.Int32(maxKeys)+ }+ output, err := client.ListObjectsV2(ctx, input)+ if err != nil {+ return nil, "", false, err+ }+ objects := make([]s3Object, 0, len(output.Contents))+ for _, object := range output.Contents {+ objects = append(objects, s3ObjectFromAWS(object))+ }+ nextStartAfter := ""+ if len(objects) > 0 {+ nextStartAfter = s3SourceID(source, bucketName, objects[len(objects)-1].Key)+ }+ return objects, nextStartAfter, aws.ToBool(output.IsTruncated), nil+}++func downloadS3Object(ctx context.Context, downloadObject func(ctx context.Context, key string, sizeThreshold int64) ([]byte, error), ensureClient func(context.Context) (*s3.Client, error), bucketName, key string, sizeThreshold int64) ([]byte, error) {+ if downloadObject != nil {+ return downloadObject(ctx, key, sizeThreshold)+ }+ client, err := ensureClient(ctx)+ if err != nil {+ return nil, err+ }+ output, err := client.GetObject(ctx, &s3.GetObjectInput{+ Bucket: aws.String(bucketName),+ Key: aws.String(key),+ })+ if err != nil {+ return nil, err+ }+ defer output.Body.Close()+ return readS3Body(output.Body, key, sizeThreshold)+}++type s3Object struct {+ Key string+ LastModified time.Time+ Size int64+ ETag string+}++func s3ObjectFromAWS(object types.Object) s3Object {+ updatedAt := time.Time{}+ if object.LastModified != nil {+ updatedAt = object.LastModified.UTC()+ }+ return s3Object{+ Key: aws.ToString(object.Key),+ LastModified: updatedAt,+ Size: aws.ToInt64(object.Size),+ ETag: aws.ToString(object.ETag),+ }+}++type s3ObjectStore interface {+ sourceName() string+ listObjectPage(ctx context.Context, startAfter string, maxKeys int32) ([]s3Object, string, bool, error)+ sourceID(key string) string+ sourceDocument(sourceID string, object s3Object) (SourceDocument, bool)+ Fetch(ctx context.Context, ref FetchReference) ([]byte, error)+}++type s3SyncSession struct {+ connector s3ObjectStore+ request SyncRequest+ batchSize int+ startAfter string+ done bool+}++// NextBatch returns the next S3 or S3-compatible document batch.+func (s *s3SyncSession) NextBatch(ctx context.Context) (SyncBatch, error) {+ for {+ if s.done {+ return SyncBatch{}, io.EOF+ }+ previousStartAfter := s.startAfter+ objects, nextStartAfter, hasMore, err := s.connector.listObjectPage(ctx, s.startAfter, int32(s.batchSize))+ if err != nil {+ return SyncBatch{}, err+ }+ if !hasMore {+ s.done = true+ }+ if nextStartAfter != "" {+ s.startAfter = strings.TrimPrefix(nextStartAfter, s.connector.sourceID(""))+ }+ if hasMore && s.startAfter == previousStartAfter {+ return SyncBatch{}, fmt.Errorf("%s listing did not advance from %q", s.connector.sourceName(), previousStartAfter)+ }++ documents := make([]SourceDocument, 0, len(objects))+ for _, object := range objects {+ sourceID := s.connector.sourceID(object.Key)+ if !includeS3Object(s.request, sourceID, object) {+ continue+ }+ document, ok := s.connector.sourceDocument(sourceID, object)+ if ok {+ documents = append(documents, document)+ }+ }+ if len(documents) == 0 {+ continue+ }+ last := documents[len(documents)-1]+ updatedAt := last.UpdatedAt+ return SyncBatch{+ Documents: documents,+ Checkpoint: &SyncCheckpoint{+ Cursor: last.SourceID,+ SourceID: last.SourceID,+ UpdatedAt: &updatedAt,+ },+ }, nil+ }+}++// Close closes the S3 or S3-compatible sync session.+func (s *s3SyncSession) Close() error {+ return nil+}++// Fetch downloads a delayed S3 or S3-compatible object body.+func (s *s3SyncSession) Fetch(ctx context.Context, ref FetchReference) ([]byte, error) {+ return s.connector.Fetch(ctx, ref)+}++func (s *s3SyncSession) applyResume(ctx context.Context, checkpoint *SyncCheckpoint) error {+ if checkpoint == nil {+ return nil+ }+ sourceID := firstNonEmpty(checkpoint.SourceID, checkpoint.Cursor)+ prefix := s.connector.sourceID("")+ if sourceID == "" || !strings.HasPrefix(sourceID, prefix) {+ return fmt.Errorf("%s sync checkpoint has no source anchor: %w", s.connector.sourceName(), ErrSyncResumeInvalid)+ }+ anchor := strings.TrimPrefix(sourceID, prefix)+ if anchor == "" {+ return fmt.Errorf("%s sync checkpoint has no source anchor: %w", s.connector.sourceName(), ErrSyncResumeInvalid)+ }++ startAfter := ""+ for {+ objects, nextStartAfter, hasMore, err := s.connector.listObjectPage(ctx, startAfter, int32(s.batchSize))+ if err != nil {+ return err+ }+ for _, object := range objects {+ if object.Key == anchor {+ s.startAfter = anchor+ return nil+ }+ if object.Key > anchor {+ return fmt.Errorf("%s resume anchor %q was not found in the current listing: %w", s.connector.sourceName(), anchor, ErrSyncResumeInvalid)+ }+ }+ if !hasMore {+ break+ }+ startAfter = strings.TrimPrefix(nextStartAfter, prefix)+ }+ return fmt.Errorf("%s resume anchor %q was not found in the current listing: %w", s.connector.sourceName(), anchor, ErrSyncResumeInvalid)+}++func (c *S3Connector) sourceName() string {+ return "S3"+}++func (c *S3Connector) sourceID(key string) string {+ return s3SourceID(s3Source, c.bucketName, key)+}++func (c *S3Connector) sourceDocument(sourceID string, object s3Object) (SourceDocument, bool) {+ return newS3SourceDocument(sourceID, object, c.prefix, c.allowImages, map[string]any{+ "url": s3ConsoleURL(c.region, c.bucketName, object.Key),+ })+}++func validateS3AuthMethod(authMethod, accessKeyID, secretKey, roleARN string) error {+ switch authMethod {+ case "access_key":+ if accessKeyID == "" || secretKey == "" {+ return fmt.Errorf("S3 access key credentials are required")+ }+ case "iam_role":+ if roleARN == "" {+ return fmt.Errorf("S3 aws_role_arn is required for iam_role authentication")+ }+ case "assume_role":+ default:+ return fmt.Errorf("unsupported S3 authentication_method %q", authMethod)+ }+ return nil+}++func validateAWSRegion(region string) error {+ if !awsRegionLabelPattern.MatchString(region) {+ return fmt.Errorf("invalid AWS region %q: must be a lowercase DNS label containing only letters, numbers, and hyphens", region)+ }+ return nil+}++type s3PruneSession struct {+ connector s3ObjectStore+ batchSize int+ startAfter string+ done bool+ buffer []SlimDocument+}++// NextBatch returns the next S3 or S3-compatible prune snapshot batch.+func (s *s3PruneSession) NextBatch(ctx context.Context) (PruneBatch, error) {+ documents := make([]SlimDocument, 0, s.batchSize)+ if len(s.buffer) > 0 {+ n := s.batchSize+ if n > len(s.buffer) {+ n = len(s.buffer)+ }+ documents = append(documents, s.buffer[:n]...)+ s.buffer = s.buffer[n:]+ }+ for len(documents) < s.batchSize && !s.done {+ page, nextStartAfter, hasMore, err := s.connector.listObjectPage(ctx, s.startAfter, s3ListPageSize)+ if err != nil {+ return PruneBatch{}, err+ }+ previousStartAfter := s.startAfter+ if hasMore {+ next := strings.TrimPrefix(nextStartAfter, s.connector.sourceID(""))+ if next == "" || next == previousStartAfter {+ return PruneBatch{}, fmt.Errorf("%s listing did not advance from %q", s.connector.sourceName(), previousStartAfter)+ }+ s.startAfter = next+ } else {+ s.done = true+ }+ remaining := s.batchSize - len(documents)+ for _, object := range page {+ if object.Key == "" || strings.HasSuffix(object.Key, "/") {+ continue+ }+ doc := SlimDocument{SourceID: s.connector.sourceID(object.Key)}+ if remaining > 0 {+ documents = append(documents, doc)+ remaining--+ } else {+ s.buffer = append(s.buffer, doc)+ }+ }+ if remaining <= 0 {+ break+ }+ }+ if len(documents) == 0 && len(s.buffer) == 0 && s.done {+ return PruneBatch{}, io.EOF+ }+ return PruneBatch{Documents: documents}, nil+}++// Close closes the S3 or S3-compatible prune session.+func (s *s3PruneSession) Close() error {+ return nil+}++type s3FetchReference struct {+ Key string `json:"key"`+}++func includeS3Object(request SyncRequest, sourceID string, object s3Object) bool {+ if request.FromBeginning {+ return true+ }+ if object.LastModified.IsZero() {+ return true+ }+ if len(request.Fingerprints) > 0 {+ fingerprint := normalizedS3ETag(object.ETag)+ stored, ok := request.Fingerprints[sourceID]+ return fingerprint == "" || !ok || stored == "" || stored != fingerprint+ }+ return !beforeOrAtWindowStart(object.LastModified, request.WindowStart) && !afterWindowEnd(object.LastModified, request.WindowEnd)+}++func normalizeS3Prefix(prefix string) string {+ prefix = strings.TrimSpace(prefix)+ if prefix != "" && !strings.HasSuffix(prefix, "/") {+ prefix += "/"+ }+ return prefix+}++func s3SourceID(source, bucketName, key string) string {+ return fmt.Sprintf("%s:%s:%s", source, bucketName, key)+}++func normalizedS3ETag(rawETag string) string {+ rawETag = strings.Trim(strings.TrimSpace(rawETag), `"`)+ if rawETag == "" {+ return ""+ }+ return contentFingerprint([]byte(rawETag))+}++func s3ConsoleURL(region, bucketName, key string) string {+ return fmt.Sprintf("https://s3.console.aws.amazon.com/s3/object/%s?region=%s&prefix=%s", bucketName, region, pathEscapeS3Key(key))+}++func pathEscapeS3Key(key string) string {+ parts := strings.Split(key, "/")+ for index, part := range parts {+ parts[index] = url.PathEscape(part)+ }+ return strings.Join(parts, "/")+}++func readS3Body(body io.Reader, key string, sizeThreshold int64) ([]byte, error) {+ limited := io.LimitReader(body, sizeThreshold+1)+ data, err := io.ReadAll(limited)+ if err != nil {+ return nil, err+ }+ if int64(len(data)) > sizeThreshold {+ return nil, fmt.Errorf("%s exceeds size threshold of %d", key, sizeThreshold)+ }+ return data, nil+}++func newS3SourceDocument(sourceID string, object s3Object, prefix string, allowImages bool, metadata map[string]any) (SourceDocument, bool) {+ if object.Key == "" || strings.HasSuffix(object.Key, "/") || (!allowImages && object.isImage()) {+ return SourceDocument{}, false+ }+ fileName := path.Base(object.Key)+ fetch := s3FetchReference{Key: object.Key}+ fetchKey, _ := json.Marshal(fetch)+ return SourceDocument{+ SourceID: sourceID,+ SemanticIdentifier: s3SemanticIdentifier(prefix, object.Key, fileName),+ Extension: strings.ToLower(filepath.Ext(fileName)),+ FetchRef: &FetchReference{Key: string(fetchKey), SizeHint: object.Size},+ UpdatedAt: object.LastModified,+ SizeBytes: object.Size,+ Metadata: metadata,+ Fingerprint: normalizedS3ETag(object.ETag),+ }, true+}++func s3SemanticIdentifier(prefix, key, fileName string) string {+ relativePath := key+ if prefix != "" {+ relativePath = strings.TrimPrefix(key, prefix)+ }+ if relativePath == "" {+ return fileName+ }+ return strings.ReplaceAll(relativePath, "/", " / ")+}++func (o s3Object) isImage() bool {+ switch strings.ToLower(filepath.Ext(o.Key)) {+ case ".jpg", ".jpeg", ".png", ".webp", ".heic", ".heif":+ return true+ default:+ return false+ }+}diff --git a/internal/syncer/connector/s3_compatible.go b/internal/syncer/connector/s3_compatible.gonew file mode 100644index 0000000000..63dd769311--- /dev/null+++ b/internal/syncer/connector/s3_compatible.go@@ -0,0 +1,233 @@+//+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.+//++package connector++import (+ "context"+ "encoding/json"+ "fmt"+ "net/url"+ "strings"++ "github.com/aws/aws-sdk-go-v2/aws"+ awssdkconfig "github.com/aws/aws-sdk-go-v2/config"+ "github.com/aws/aws-sdk-go-v2/credentials"+ "github.com/aws/aws-sdk-go-v2/service/s3"+)++const (+ defaultS3CompatibleRegion = "us-east-1"+ s3CompatibleSource = "s3_compatible"+)++// S3CompatibleConnector reads objects from any S3-compatible object store.+type S3CompatibleConnector struct {+ bucketName string+ prefix string+ endpointURL string+ region string+ accessKeyID string+ secretKey string+ addressingStyle string+ allowImages bool+ batchSize int++ sizeThreshold int64+ client *s3.Client+ listObjects func(ctx context.Context, startAfter string, maxKeys int32) ([]s3Object, string, bool, error)+ downloadObject func(ctx context.Context, key string, sizeThreshold int64) ([]byte, error)+}++// NewS3CompatibleConnector creates an S3-compatible connector from+// Python-compatible config.+func NewS3CompatibleConnector(config map[string]any) (*S3CompatibleConnector, error) {+ credentials := configAnyMap(config["credentials"])+ batchSize := configInt(firstNonEmpty(stringConfig(config["sync_batch_size"]), stringConfig(config["batch_size"])), defaultS3BatchSize)+ sizeThreshold := int64(configInt(config["size_threshold"], defaultS3SizeThreshold))+ if sizeThreshold <= 0 {+ sizeThreshold = defaultS3SizeThreshold+ }+ addressingStyle := firstNonEmpty(stringConfig(credentials["addressing_style"]), "virtual")+ return &S3CompatibleConnector{+ bucketName: strings.TrimSpace(stringConfig(config["bucket_name"])),+ prefix: normalizeS3Prefix(stringConfig(config["prefix"])),+ endpointURL: strings.TrimSpace(stringConfig(credentials["endpoint_url"])),+ region: firstNonEmpty(stringConfig(credentials["region"]), defaultS3CompatibleRegion),+ accessKeyID: strings.TrimSpace(stringConfig(credentials["aws_access_key_id"])),+ secretKey: stringConfig(credentials["aws_secret_access_key"]),+ addressingStyle: addressingStyle,+ allowImages: configBoolDefault(config["allow_images"], false),+ batchSize: batchSize,+ sizeThreshold: sizeThreshold,+ }, nil+}++// Validate validates S3-compatible connector settings and credentials.+func (c *S3CompatibleConnector) Validate(ctx context.Context) error {+ if c == nil {+ return fmt.Errorf("S3-compatible connector is nil")+ }+ if c.bucketName == "" {+ return fmt.Errorf("No bucket name was provided in connector settings.")+ }+ if err := validateS3CompatibleEndpoint(c.endpointURL); err != nil {+ return err+ }+ if c.accessKeyID == "" || c.secretKey == "" {+ return fmt.Errorf("S3-compatible access key credentials are required")+ }+ if err := validateS3CompatibleAddressingStyle(c.addressingStyle); err != nil {+ return err+ }+ if c.batchSize <= 0 {+ return fmt.Errorf("batch_size must be a positive integer")+ }+ if _, err := c.ensureClient(ctx); err != nil {+ return err+ }+ return nil+}++// ValidateConnectorSetting validates S3-compatible settings from an unsaved+// config.+func (c *S3CompatibleConnector) ValidateConnectorSetting(ctx context.Context, request map[string]any) error {+ ctx, cancel := context.WithTimeout(ctx, connectorSettingValidationTimeout)+ defer cancel()+ if err := c.Validate(ctx); err != nil {+ return err+ }+ _, _, _, err := c.listObjectPage(ctx, "", 1)+ if err != nil {+ return fmt.Errorf("access S3-compatible bucket %q: %w", c.bucketName, err)+ }+ return nil+}++// OpenSync opens one S3-compatible sync session.+func (c *S3CompatibleConnector) OpenSync(ctx context.Context, request SyncRequest) (SyncSession, error) {+ if err := c.Validate(ctx); err != nil {+ return nil, err+ }+ session := &s3SyncSession{+ connector: c,+ request: request,+ batchSize: c.batchSize,+ }+ if request.Resume != nil {+ if err := session.applyResume(ctx, request.Resume); err != nil {+ return nil, err+ }+ }+ return session, nil+}++// OpenPrune opens one complete S3-compatible prune snapshot session.+func (c *S3CompatibleConnector) OpenPrune(ctx context.Context, request PruneRequest) (PruneSession, error) {+ if err := c.Validate(ctx); err != nil {+ return nil, err+ }+ return &s3PruneSession{connector: c, batchSize: c.batchSize}, nil+}++// Fetch downloads an S3-compatible object body.+func (c *S3CompatibleConnector) Fetch(ctx context.Context, ref FetchReference) ([]byte, error) {+ var fetch s3FetchReference+ if err := json.Unmarshal([]byte(ref.Key), &fetch); err != nil {+ return nil, err+ }+ return c.download(ctx, fetch.Key)+}++func (c *S3CompatibleConnector) ensureClient(ctx context.Context) (*s3.Client, error) {+ if c.client != nil {+ return c.client, nil+ }+ if err := validateS3CompatibleEndpoint(c.endpointURL); err != nil {+ return nil, err+ }+ if err := validateS3CompatibleAddressingStyle(c.addressingStyle); err != nil {+ return nil, err+ }+ region := firstNonEmpty(c.region, defaultS3CompatibleRegion)+ cfg, err := awssdkconfig.LoadDefaultConfig(+ ctx,+ awssdkconfig.WithRegion(region),+ awssdkconfig.WithCredentialsProvider(credentials.NewStaticCredentialsProvider(c.accessKeyID, c.secretKey, "")),+ )+ if err != nil {+ return nil, fmt.Errorf("load S3-compatible config: %w", err)+ }+ c.client = s3.NewFromConfig(cfg, func(options *s3.Options) {+ options.BaseEndpoint = aws.String(c.endpointURL)+ options.UsePathStyle = c.addressingStyle == "path"+ })+ return c.client, nil+}++func (c *S3CompatibleConnector) listObjectPage(ctx context.Context, startAfter string, maxKeys int32) ([]s3Object, string, bool, error) {+ return listS3ObjectPage(ctx, c.listObjects, c.ensureClient, c.bucketName, c.prefix, s3CompatibleSource, startAfter, maxKeys)+}++func (c *S3CompatibleConnector) download(ctx context.Context, key string) ([]byte, error) {+ return downloadS3Object(ctx, c.downloadObject, c.ensureClient, c.bucketName, key, c.sizeThreshold)+}++func (c *S3CompatibleConnector) sourceName() string {+ return "S3-compatible"+}++func (c *S3CompatibleConnector) sourceID(key string) string {+ return s3SourceID(s3CompatibleSource, c.bucketName, key)+}++func (c *S3CompatibleConnector) sourceDocument(sourceID string, object s3Object) (SourceDocument, bool) {+ return newS3SourceDocument(sourceID, object, c.prefix, c.allowImages, nil)+}++func validateS3CompatibleAddressingStyle(addressingStyle string) error {+ if addressingStyle != "virtual" && addressingStyle != "path" {+ return fmt.Errorf("unsupported S3-compatible addressing_style %q", addressingStyle)+ }+ return nil+}++func validateS3CompatibleEndpoint(raw string) error {+ raw = strings.TrimSpace(raw)+ if raw == "" {+ return fmt.Errorf("S3-compatible endpoint_url is required")+ }+ if strings.ContainsAny(raw, " \t\r\n") || strings.Contains(raw, "#") {+ return fmt.Errorf("invalid S3-compatible endpoint_url %q", raw)+ }+ parsed, err := url.Parse(raw)+ if err != nil {+ return fmt.Errorf("invalid S3-compatible endpoint_url %q: %w", raw, err)+ }+ if parsed.Scheme != "http" && parsed.Scheme != "https" {+ return fmt.Errorf("invalid S3-compatible endpoint_url %q: scheme must be http or https", raw)+ }+ if parsed.Host == "" || parsed.Hostname() == "" {+ return fmt.Errorf("invalid S3-compatible endpoint_url %q: host is required", raw)+ }+ if parsed.User != nil {+ return fmt.Errorf("invalid S3-compatible endpoint_url %q: userinfo is not allowed", raw)+ }+ if parsed.Fragment != "" {+ return fmt.Errorf("invalid S3-compatible endpoint_url %q: fragment is not allowed", raw)+ }+ return nil+}diff --git a/internal/syncer/connector/s3_compatible_test.go b/internal/syncer/connector/s3_compatible_test.gonew file mode 100644index 0000000000..a73939313e--- /dev/null+++ b/internal/syncer/connector/s3_compatible_test.go@@ -0,0 +1,466 @@+package connector++import (+ "bytes"+ "context"+ "errors"+ "io"+ "strings"+ "testing"+)++func TestS3CompatibleConnectorOpenSyncUsesFingerprintAndFetch(t *testing.T) {+ old := mustTime(t, "2026-01-01T00:00:00Z")+ updated := mustTime(t, "2026-01-03T00:00:00Z")+ connector := newTestS3CompatibleConnector(t, []s3Object{+ {Key: "docs/old.txt", LastModified: old, Size: 8, ETag: `"old-etag"`},+ {Key: "docs/new.txt", LastModified: updated, Size: 8, ETag: `"new-etag"`},+ })++ start := mustTime(t, "2026-01-02T00:00:00Z")+ session, err := connector.OpenSync(context.Background(), SyncRequest{+ WindowStart: &start,+ WindowEnd: mustTime(t, "2026-01-04T00:00:00Z"),+ Fingerprints: map[string]string{+ s3SourceID(s3CompatibleSource, "bucket", "docs/old.txt"): normalizedS3ETag(`"old-etag"`),+ },+ })+ if err != nil {+ t.Fatalf("OpenSync: %v", err)+ }++ batch, err := session.NextBatch(context.Background())+ if err != nil {+ t.Fatalf("NextBatch: %v", err)+ }+ if len(batch.Documents) != 1 {+ t.Fatalf("documents len = %d, want 1", len(batch.Documents))+ }+ doc := batch.Documents[0]+ if doc.SourceID != s3SourceID(s3CompatibleSource, "bucket", "docs/new.txt") {+ t.Fatalf("source id = %q", doc.SourceID)+ }+ if doc.SemanticIdentifier != "new.txt" {+ t.Fatalf("semantic identifier = %q", doc.SemanticIdentifier)+ }+ if doc.Extension != ".txt" {+ t.Fatalf("extension = %q", doc.Extension)+ }+ if doc.Fingerprint != normalizedS3ETag(`"new-etag"`) {+ t.Fatalf("fingerprint = %q", doc.Fingerprint)+ }+ if doc.FetchRef == nil {+ t.Fatalf("fetch ref is nil")+ }+ fetcher, ok := session.(Fetcher)+ if !ok {+ t.Fatalf("session does not implement Fetcher")+ }+ blob, err := fetcher.Fetch(context.Background(), *doc.FetchRef)+ if err != nil {+ t.Fatalf("Fetch: %v", err)+ }+ if string(blob) != "body:new" {+ t.Fatalf("blob = %q", blob)+ }+ if _, err = session.NextBatch(context.Background()); !errors.Is(err, io.EOF) {+ t.Fatalf("NextBatch EOF = %v", err)+ }+}++func TestS3CompatibleConnectorOpenSyncDefersListingUntilNextBatch(t *testing.T) {+ connector := newTestS3CompatibleConnector(t, []s3Object{+ {Key: "docs/a.txt", LastModified: mustTime(t, "2026-01-01T00:00:00Z"), Size: 1, ETag: "a"},+ })+ var listCalls int+ baseListObjects := connector.listObjects+ connector.listObjects = func(ctx context.Context, startAfter string, maxKeys int32) ([]s3Object, string, bool, error) {+ listCalls+++ return baseListObjects(ctx, startAfter, maxKeys)+ }++ session, err := connector.OpenSync(context.Background(), SyncRequest{FromBeginning: true})+ if err != nil {+ t.Fatalf("OpenSync: %v", err)+ }+ if listCalls != 0 {+ t.Fatalf("OpenSync list calls = %d, want 0", listCalls)+ }+ if _, err := session.NextBatch(context.Background()); err != nil {+ t.Fatalf("NextBatch: %v", err)+ }+ if listCalls != 1 {+ t.Fatalf("NextBatch list calls = %d, want 1", listCalls)+ }+}++func TestS3CompatibleConnectorOpenSyncFiltersImagesUnlessAllowed(t *testing.T) {+ objects := []s3Object{+ {Key: "docs/a.png", LastModified: mustTime(t, "2026-01-01T00:00:00Z"), Size: 1, ETag: "a"},+ {Key: "docs/b.txt", LastModified: mustTime(t, "2026-01-02T00:00:00Z"), Size: 1, ETag: "b"},+ }+ connector := newTestS3CompatibleConnector(t, objects)+ session, err := connector.OpenSync(context.Background(), SyncRequest{FromBeginning: true})+ if err != nil {+ t.Fatalf("OpenSync: %v", err)+ }+ batch, err := session.NextBatch(context.Background())+ if err != nil {+ t.Fatalf("NextBatch: %v", err)+ }+ if len(batch.Documents) != 1 || batch.Documents[0].SourceID != s3SourceID(s3CompatibleSource, "bucket", "docs/b.txt") {+ t.Fatalf("documents = %+v", batch.Documents)+ }++ allowed := newTestS3CompatibleConnector(t, objects)+ allowed.allowImages = true+ session, err = allowed.OpenSync(context.Background(), SyncRequest{FromBeginning: true})+ if err != nil {+ t.Fatalf("allow images OpenSync: %v", err)+ }+ batch, err = session.NextBatch(context.Background())+ if err != nil {+ t.Fatalf("allow images NextBatch: %v", err)+ }+ if len(batch.Documents) != 2 {+ t.Fatalf("allow images documents len = %d, want 2", len(batch.Documents))+ }+}++func TestS3CompatibleConnectorOpenSyncResumesFromCheckpoint(t *testing.T) {+ connector := newTestS3CompatibleConnector(t, []s3Object{+ {Key: "docs/a.txt", LastModified: mustTime(t, "2026-01-01T00:00:00Z"), Size: 1, ETag: "a"},+ {Key: "docs/b.txt", LastModified: mustTime(t, "2026-01-02T00:00:00Z"), Size: 1, ETag: "b"},+ {Key: "docs/c.txt", LastModified: mustTime(t, "2026-01-03T00:00:00Z"), Size: 1, ETag: "c"},+ })+ checkpointUpdatedAt := mustTime(t, "2026-01-02T00:00:00Z")+ session, err := connector.OpenSync(context.Background(), SyncRequest{+ FromBeginning: true,+ Resume: &SyncCheckpoint{+ Cursor: s3SourceID(s3CompatibleSource, "bucket", "docs/b.txt"),+ UpdatedAt: &checkpointUpdatedAt,+ },+ })+ if err != nil {+ t.Fatalf("OpenSync: %v", err)+ }+ batch, err := session.NextBatch(context.Background())+ if err != nil {+ t.Fatalf("NextBatch: %v", err)+ }+ if len(batch.Documents) != 1 {+ t.Fatalf("documents len = %d, want 1", len(batch.Documents))+ }+ if batch.Documents[0].SourceID != s3SourceID(s3CompatibleSource, "bucket", "docs/c.txt") {+ t.Fatalf("resumed source id = %q, want docs/c.txt", batch.Documents[0].SourceID)+ }+ if batch.Checkpoint == nil || batch.Checkpoint.SourceID != s3SourceID(s3CompatibleSource, "bucket", "docs/c.txt") {+ t.Fatalf("checkpoint = %+v", batch.Checkpoint)+ }+}++func TestS3CompatibleConnectorOpenSyncRejectsInvalidResumeAnchors(t *testing.T) {+ connector := newTestS3CompatibleConnector(t, []s3Object{+ {Key: "docs/a.txt", LastModified: mustTime(t, "2026-01-01T00:00:00Z"), Size: 1, ETag: "a"},+ {Key: "docs/b.txt", LastModified: mustTime(t, "2026-01-02T00:00:00Z"), Size: 1, ETag: "b"},+ })+ checkpoints := []*SyncCheckpoint{+ {Cursor: "other:bucket:docs/a.txt"},+ {Cursor: s3SourceID(s3CompatibleSource, "bucket", "")},+ {Cursor: s3SourceID(s3CompatibleSource, "bucket", "docs/missing.txt")},+ }+ for _, checkpoint := range checkpoints {+ _, err := connector.OpenSync(context.Background(), SyncRequest{FromBeginning: true, Resume: checkpoint})+ if !errors.Is(err, ErrSyncResumeInvalid) {+ t.Fatalf("OpenSync error = %v for checkpoint %+v, want ErrSyncResumeInvalid", err, checkpoint)+ }+ }+}++func TestS3CompatibleConnectorOpenSyncIncludesMissingFingerprint(t *testing.T) {+ connector := newTestS3CompatibleConnector(t, []s3Object{+ {Key: "docs/no-etag.txt", LastModified: mustTime(t, "2026-01-01T00:00:00Z"), Size: 1},+ })+ session, err := connector.OpenSync(context.Background(), SyncRequest{+ Fingerprints: map[string]string{"other": "fingerprint"},+ })+ if err != nil {+ t.Fatalf("OpenSync: %v", err)+ }+ batch, err := session.NextBatch(context.Background())+ if err != nil {+ t.Fatalf("NextBatch: %v", err)+ }+ if len(batch.Documents) != 1 {+ t.Fatalf("documents len = %d, want 1", len(batch.Documents))+ }+}++func TestS3CompatibleConnectorOpenSyncFiltersByWindow(t *testing.T) {+ start := mustTime(t, "2026-01-02T00:00:00Z")+ end := mustTime(t, "2026-01-04T00:00:00Z")+ connector := newTestS3CompatibleConnector(t, []s3Object{+ {Key: "docs/before.txt", LastModified: mustTime(t, "2026-01-01T00:00:00Z"), Size: 1, ETag: "before"},+ {Key: "docs/within.txt", LastModified: mustTime(t, "2026-01-03T00:00:00Z"), Size: 1, ETag: "within"},+ {Key: "docs/after.txt", LastModified: mustTime(t, "2026-01-05T00:00:00Z"), Size: 1, ETag: "after"},+ })+ session, err := connector.OpenSync(context.Background(), SyncRequest{WindowStart: &start, WindowEnd: end})+ if err != nil {+ t.Fatalf("OpenSync: %v", err)+ }+ batch, err := session.NextBatch(context.Background())+ if err != nil {+ t.Fatalf("NextBatch: %v", err)+ }+ if len(batch.Documents) != 1 || batch.Documents[0].SourceID != s3SourceID(s3CompatibleSource, "bucket", "docs/within.txt") {+ t.Fatalf("documents = %+v", batch.Documents)+ }+}++func TestS3CompatibleConnectorOpenPruneReturnsSlimSnapshot(t *testing.T) {+ connector := newTestS3CompatibleConnector(t, []s3Object{+ {Key: "docs/a.txt", LastModified: mustTime(t, "2026-01-01T00:00:00Z"), Size: 1, ETag: "a"},+ {Key: "docs/b.txt", LastModified: mustTime(t, "2026-01-02T00:00:00Z"), Size: 1, ETag: "b"},+ {Key: "docs/folder/", LastModified: mustTime(t, "2026-01-02T00:00:00Z"), Size: 0, ETag: "folder"},+ })+ session, err := connector.OpenPrune(context.Background(), PruneRequest{})+ if err != nil {+ t.Fatalf("OpenPrune: %v", err)+ }+ batch, err := session.NextBatch(context.Background())+ if err != nil {+ t.Fatalf("NextBatch: %v", err)+ }+ if len(batch.Documents) != 2 {+ t.Fatalf("documents len = %d, want 2", len(batch.Documents))+ }+ if batch.Documents[0].SourceID != s3SourceID(s3CompatibleSource, "bucket", "docs/a.txt") || batch.Documents[1].SourceID != s3SourceID(s3CompatibleSource, "bucket", "docs/b.txt") {+ t.Fatalf("documents = %+v", batch.Documents)+ }+}++func TestS3CompatibleConnectorOpenPruneStreamsAcrossPages(t *testing.T) {+ connector, err := NewS3CompatibleConnector(map[string]any{+ "bucket_name": "bucket",+ "batch_size": 2,+ "credentials": map[string]any{+ "endpoint_url": "https://s3.example.com",+ "aws_access_key_id": "access",+ "aws_secret_access_key": "secret",+ "addressing_style": "path",+ },+ })+ if err != nil {+ t.Fatalf("NewS3CompatibleConnector: %v", err)+ }+ pages := [][]s3Object{+ {+ {Key: "docs/a.txt", Size: 1},+ {Key: "docs/b.txt", Size: 1},+ },+ {+ {Key: "docs/c.txt", Size: 1},+ {Key: "docs/d.txt", Size: 1},+ },+ }+ call := 0+ connector.listObjects = func(ctx context.Context, startAfter string, maxKeys int32) ([]s3Object, string, bool, error) {+ if call >= len(pages) {+ return nil, "", false, nil+ }+ page := pages[call]+ call+++ nextStartAfter := ""+ if len(page) > 0 {+ nextStartAfter = s3SourceID(s3CompatibleSource, connector.bucketName, page[len(page)-1].Key)+ }+ return page, nextStartAfter, call < len(pages), nil+ }+ session, err := connector.OpenPrune(context.Background(), PruneRequest{})+ if err != nil {+ t.Fatalf("OpenPrune: %v", err)+ }+ first, err := session.NextBatch(context.Background())+ if err != nil {+ t.Fatalf("first NextBatch: %v", err)+ }+ if len(first.Documents) != 2 || first.Documents[0].SourceID != s3SourceID(s3CompatibleSource, "bucket", "docs/a.txt") || first.Documents[1].SourceID != s3SourceID(s3CompatibleSource, "bucket", "docs/b.txt") {+ t.Fatalf("first batch = %+v", first.Documents)+ }+ second, err := session.NextBatch(context.Background())+ if err != nil {+ t.Fatalf("second NextBatch: %v", err)+ }+ if len(second.Documents) != 2 || second.Documents[0].SourceID != s3SourceID(s3CompatibleSource, "bucket", "docs/c.txt") || second.Documents[1].SourceID != s3SourceID(s3CompatibleSource, "bucket", "docs/d.txt") {+ t.Fatalf("second batch = %+v", second.Documents)+ }+ if _, err := session.NextBatch(context.Background()); !errors.Is(err, io.EOF) {+ t.Fatalf("NextBatch EOF = %v", err)+ }+}++func TestS3CompatibleConnectorValidate(t *testing.T) {+ ctx := context.Background()+ missingBucket := &S3CompatibleConnector{endpointURL: "https://s3.example.com", accessKeyID: "access", secretKey: "secret", addressingStyle: "virtual", batchSize: 2}+ if err := missingBucket.Validate(ctx); err == nil {+ t.Fatalf("expected bucket name validation error")+ }+ missingEndpoint := &S3CompatibleConnector{bucketName: "bucket", accessKeyID: "access", secretKey: "secret", addressingStyle: "virtual", batchSize: 2}+ if err := missingEndpoint.Validate(ctx); err == nil {+ t.Fatalf("expected endpoint validation error")+ }+ missingCreds := &S3CompatibleConnector{bucketName: "bucket", endpointURL: "https://s3.example.com", addressingStyle: "virtual", batchSize: 2}+ if err := missingCreds.Validate(ctx); err == nil {+ t.Fatalf("expected credentials validation error")+ }+ badStyle := &S3CompatibleConnector{bucketName: "bucket", endpointURL: "https://s3.example.com", accessKeyID: "access", secretKey: "secret", addressingStyle: "invalid", batchSize: 2}+ if err := badStyle.Validate(ctx); err == nil {+ t.Fatalf("expected addressing style validation error")+ }+ badBatch := &S3CompatibleConnector{bucketName: "bucket", endpointURL: "https://s3.example.com", accessKeyID: "access", secretKey: "secret", addressingStyle: "virtual"}+ if err := badBatch.Validate(ctx); err == nil {+ t.Fatalf("expected batch size validation error")+ }+}++func TestS3CompatibleConnectorRejectsInvalidEndpointBeforeClientCreation(t *testing.T) {+ for _, endpoint := range []string{+ "https://evil.example#",+ "evil.example#",+ "ftp://evil.example",+ "https://user:[email protected]",+ "https://evil.example#fragment",+ "https://",+ "https://evil.example/pa th",+ } {+ connector := newTestS3CompatibleConnector(t, nil)+ connector.endpointURL = endpoint+ connector.client = nil+ if err := connector.Validate(context.Background()); err == nil {+ t.Fatalf("expected invalid endpoint %q to be rejected", endpoint)+ }+ if connector.client != nil {+ t.Fatalf("client was created for invalid endpoint %q", endpoint)+ }+ }+}++func TestS3CompatibleConnectorValidateSetting(t *testing.T) {+ connector := newTestS3CompatibleConnector(t, []s3Object{+ {Key: "docs/a.txt", Size: 1},+ })+ var listCalls int+ connector.listObjects = func(ctx context.Context, startAfter string, maxKeys int32) ([]s3Object, string, bool, error) {+ listCalls+++ return nil, "", false, nil+ }+ if err := connector.ValidateConnectorSetting(context.Background(), nil); err != nil {+ t.Fatalf("ValidateConnectorSetting: %v", err)+ }+ if listCalls != 1 {+ t.Fatalf("list calls = %d, want 1", listCalls)+ }+}++func TestNewS3CompatibleConnectorDefaultsAndConfig(t *testing.T) {+ connector, err := NewS3CompatibleConnector(map[string]any{+ "bucket_name": " bucket ",+ "prefix": "docs",+ "batch_size": 4,+ "size_threshold": 42,+ "allow_images": true,+ "credentials": map[string]any{+ "endpoint_url": " https://s3.example.com ",+ "aws_access_key_id": "access",+ "aws_secret_access_key": "secret",+ "addressing_style": "path",+ },+ })+ if err != nil {+ t.Fatalf("NewS3CompatibleConnector: %v", err)+ }+ if connector.bucketName != "bucket" || connector.prefix != "docs/" {+ t.Fatalf("bucket/prefix = %q/%q", connector.bucketName, connector.prefix)+ }+ if connector.batchSize != 4 || connector.sizeThreshold != 42 || !connector.allowImages {+ t.Fatalf("batch/threshold/allow_images = %d/%d/%v", connector.batchSize, connector.sizeThreshold, connector.allowImages)+ }+ if connector.endpointURL != "https://s3.example.com" || connector.addressingStyle != "path" || connector.region != defaultS3CompatibleRegion {+ t.Fatalf("endpoint/style/region = %q/%q/%q", connector.endpointURL, connector.addressingStyle, connector.region)+ }++ defaults, err := NewS3CompatibleConnector(map[string]any{"bucket_name": "bucket", "credentials": map[string]any{}})+ if err != nil {+ t.Fatalf("NewS3CompatibleConnector defaults: %v", err)+ }+ if defaults.batchSize != defaultS3BatchSize || defaults.sizeThreshold != defaultS3SizeThreshold || defaults.region != defaultS3CompatibleRegion || defaults.addressingStyle != "virtual" {+ t.Fatalf("defaults = batch %d threshold %d region %q style %q", defaults.batchSize, defaults.sizeThreshold, defaults.region, defaults.addressingStyle)+ }+}++func TestS3CompatibleHelpers(t *testing.T) {+ if got := s3SourceID(s3CompatibleSource, "bucket", "docs/a.txt"); got != "s3_compatible:bucket:docs/a.txt" {+ t.Fatalf("source id = %q", got)+ }+ if got := normalizeS3Prefix("docs"); got != "docs/" {+ t.Fatalf("prefix = %q", got)+ }+ if got := pathEscapeS3Key("docs/a b.txt"); got != "docs/a%20b.txt" {+ t.Fatalf("escaped key = %q", got)+ }+ if got := strings.TrimSpace(s3ConsoleURL("us-east-1", "bucket", "docs/a b.txt")); !strings.HasPrefix(got, "https://s3.console.aws.amazon.com/s3/object/bucket?region=us-east-1&prefix=docs/a%20b.txt") {+ t.Fatalf("console url = %q", got)+ }+ if _, err := readS3Body(bytes.NewBufferString("12345"), "large.txt", 4); err == nil {+ t.Fatalf("expected size threshold error")+ }+}++func newTestS3CompatibleConnector(t *testing.T, objects []s3Object) *S3CompatibleConnector {+ t.Helper()+ connector, err := NewS3CompatibleConnector(map[string]any{+ "bucket_name": "bucket",+ "prefix": "docs",+ "batch_size": 2,+ "credentials": map[string]any{+ "endpoint_url": "https://s3.example.com",+ "aws_access_key_id": "access",+ "aws_secret_access_key": "secret",+ "addressing_style": "virtual",+ },+ })+ if err != nil {+ t.Fatalf("NewS3CompatibleConnector: %v", err)+ }+ connector.listObjects = func(ctx context.Context, startAfter string, maxKeys int32) ([]s3Object, string, bool, error) {+ var out []s3Object+ for _, object := range objects {+ if startAfter != "" && object.Key <= startAfter {+ continue+ }+ out = append(out, object)+ if maxKeys > 0 && len(out) >= int(maxKeys) {+ break+ }+ }+ hasMore := false+ nextStartAfter := ""+ if len(out) > 0 {+ nextStartAfter = s3SourceID(s3CompatibleSource, connector.bucketName, out[len(out)-1].Key)+ for _, object := range objects {+ if object.Key > out[len(out)-1].Key {+ hasMore = true+ break+ }+ }+ }+ return out, nextStartAfter, hasMore, nil+ }+ connector.downloadObject = func(ctx context.Context, key string, sizeThreshold int64) ([]byte, error) {+ if key == "docs/new.txt" {+ return []byte("body:new"), nil+ }+ return []byte("body:" + key), nil+ }+ return connector+}diff --git a/internal/syncer/connector/s3_test.go b/internal/syncer/connector/s3_test.gonew file mode 100644index 0000000000..23f27e60ce--- /dev/null+++ b/internal/syncer/connector/s3_test.go@@ -0,0 +1,496 @@+package connector++import (+ "bytes"+ "context"+ "errors"+ "io"+ "strings"+ "testing"+)++func TestS3ConnectorOpenSyncUsesFingerprintAndFetch(t *testing.T) {+ old := mustTime(t, "2026-01-01T00:00:00Z")+ updated := mustTime(t, "2026-01-03T00:00:00Z")+ connector := newTestS3Connector(t, []s3Object{+ {Key: "docs/old.txt", LastModified: old, Size: 8, ETag: `"old-etag"`},+ {Key: "docs/new.txt", LastModified: updated, Size: 8, ETag: `"new-etag"`},+ })++ start := mustTime(t, "2026-01-02T00:00:00Z")+ session, err := connector.OpenSync(context.Background(), SyncRequest{+ WindowStart: &start,+ WindowEnd: mustTime(t, "2026-01-04T00:00:00Z"),+ Fingerprints: map[string]string{+ s3SourceID(s3Source, "bucket", "docs/old.txt"): normalizedS3ETag(`"old-etag"`),+ },+ })+ if err != nil {+ t.Fatalf("OpenSync: %v", err)+ }++ batch, err := session.NextBatch(context.Background())+ if err != nil {+ t.Fatalf("NextBatch: %v", err)+ }+ if len(batch.Documents) != 1 {+ t.Fatalf("documents len = %d, want 1", len(batch.Documents))+ }+ doc := batch.Documents[0]+ if doc.SourceID != s3SourceID(s3Source, "bucket", "docs/new.txt") {+ t.Fatalf("source id = %q", doc.SourceID)+ }+ if doc.SemanticIdentifier != "new.txt" {+ t.Fatalf("semantic identifier = %q", doc.SemanticIdentifier)+ }+ if doc.Extension != ".txt" {+ t.Fatalf("extension = %q", doc.Extension)+ }+ if doc.Fingerprint != normalizedS3ETag(`"new-etag"`) {+ t.Fatalf("fingerprint = %q", doc.Fingerprint)+ }+ if doc.FetchRef == nil {+ t.Fatalf("fetch ref is nil")+ }+ fetcher, ok := session.(Fetcher)+ if !ok {+ t.Fatalf("session does not implement Fetcher")+ }+ blob, err := fetcher.Fetch(context.Background(), *doc.FetchRef)+ if err != nil {+ t.Fatalf("Fetch: %v", err)+ }+ if string(blob) != "body:new" {+ t.Fatalf("blob = %q", blob)+ }+ if _, err = session.NextBatch(context.Background()); !errors.Is(err, io.EOF) {+ t.Fatalf("NextBatch EOF = %v", err)+ }+}++func TestS3ConnectorOpenSyncDefersListingUntilNextBatch(t *testing.T) {+ connector := newTestS3Connector(t, []s3Object{+ {Key: "docs/a.txt", LastModified: mustTime(t, "2026-01-01T00:00:00Z"), Size: 1, ETag: "a"},+ })+ var listCalls int+ baseListObjects := connector.listObjects+ connector.listObjects = func(ctx context.Context, startAfter string, maxKeys int32) ([]s3Object, string, bool, error) {+ listCalls+++ return baseListObjects(ctx, startAfter, maxKeys)+ }++ session, err := connector.OpenSync(context.Background(), SyncRequest{FromBeginning: true})+ if err != nil {+ t.Fatalf("OpenSync: %v", err)+ }+ if listCalls != 0 {+ t.Fatalf("OpenSync list calls = %d, want 0", listCalls)+ }+ if _, err := session.NextBatch(context.Background()); err != nil {+ t.Fatalf("NextBatch: %v", err)+ }+ if listCalls != 1 {+ t.Fatalf("NextBatch list calls = %d, want 1", listCalls)+ }+}++func TestS3ConnectorOpenSyncFiltersImagesUnlessAllowed(t *testing.T) {+ objects := []s3Object{+ {Key: "docs/a.png", LastModified: mustTime(t, "2026-01-01T00:00:00Z"), Size: 1, ETag: "a"},+ {Key: "docs/b.txt", LastModified: mustTime(t, "2026-01-02T00:00:00Z"), Size: 1, ETag: "b"},+ }+ connector := newTestS3Connector(t, objects)+ session, err := connector.OpenSync(context.Background(), SyncRequest{FromBeginning: true})+ if err != nil {+ t.Fatalf("OpenSync: %v", err)+ }+ batch, err := session.NextBatch(context.Background())+ if err != nil {+ t.Fatalf("NextBatch: %v", err)+ }+ if len(batch.Documents) != 1 || batch.Documents[0].SourceID != s3SourceID(s3Source, "bucket", "docs/b.txt") {+ t.Fatalf("documents = %+v", batch.Documents)+ }++ allowed := newTestS3Connector(t, objects)+ allowed.allowImages = true+ session, err = allowed.OpenSync(context.Background(), SyncRequest{FromBeginning: true})+ if err != nil {+ t.Fatalf("allow images OpenSync: %v", err)+ }+ batch, err = session.NextBatch(context.Background())+ if err != nil {+ t.Fatalf("allow images NextBatch: %v", err)+ }+ if len(batch.Documents) != 2 {+ t.Fatalf("allow images documents len = %d, want 2", len(batch.Documents))+ }+}++func TestS3ConnectorOpenSyncResumesFromCheckpoint(t *testing.T) {+ connector := newTestS3Connector(t, []s3Object{+ {Key: "docs/a.txt", LastModified: mustTime(t, "2026-01-01T00:00:00Z"), Size: 1, ETag: "a"},+ {Key: "docs/b.txt", LastModified: mustTime(t, "2026-01-02T00:00:00Z"), Size: 1, ETag: "b"},+ {Key: "docs/c.txt", LastModified: mustTime(t, "2026-01-03T00:00:00Z"), Size: 1, ETag: "c"},+ })+ checkpointUpdatedAt := mustTime(t, "2026-01-02T00:00:00Z")+ session, err := connector.OpenSync(context.Background(), SyncRequest{+ FromBeginning: true,+ Resume: &SyncCheckpoint{+ Cursor: s3SourceID(s3Source, "bucket", "docs/b.txt"),+ SourceID: s3SourceID(s3Source, "bucket", "docs/b.txt"),+ UpdatedAt: &checkpointUpdatedAt,+ },+ })+ if err != nil {+ t.Fatalf("OpenSync: %v", err)+ }+ batch, err := session.NextBatch(context.Background())+ if err != nil {+ t.Fatalf("NextBatch: %v", err)+ }+ if len(batch.Documents) != 1 {+ t.Fatalf("documents len = %d, want 1", len(batch.Documents))+ }+ if batch.Documents[0].SourceID != s3SourceID(s3Source, "bucket", "docs/c.txt") {+ t.Fatalf("resumed source id = %q, want docs/c.txt", batch.Documents[0].SourceID)+ }+ if batch.Checkpoint == nil || batch.Checkpoint.SourceID != s3SourceID(s3Source, "bucket", "docs/c.txt") {+ t.Fatalf("checkpoint = %+v", batch.Checkpoint)+ }+}++func TestS3ConnectorOpenSyncRejectsInvalidResumeAnchors(t *testing.T) {+ connector := newTestS3Connector(t, []s3Object{+ {Key: "docs/a.txt", LastModified: mustTime(t, "2026-01-01T00:00:00Z"), Size: 1, ETag: "a"},+ {Key: "docs/b.txt", LastModified: mustTime(t, "2026-01-02T00:00:00Z"), Size: 1, ETag: "b"},+ })+ checkpoints := []*SyncCheckpoint{+ {SourceID: "other:bucket:docs/a.txt"},+ {SourceID: s3SourceID(s3Source, "bucket", "")},+ {SourceID: s3SourceID(s3Source, "bucket", "docs/missing.txt")},+ }+ for _, checkpoint := range checkpoints {+ _, err := connector.OpenSync(context.Background(), SyncRequest{FromBeginning: true, Resume: checkpoint})+ if !errors.Is(err, ErrSyncResumeInvalid) {+ t.Fatalf("OpenSync error = %v for checkpoint %+v, want ErrSyncResumeInvalid", err, checkpoint)+ }+ }+}++func TestS3ConnectorOpenSyncIncludesMissingFingerprint(t *testing.T) {+ connector := newTestS3Connector(t, []s3Object{+ {Key: "docs/no-etag.txt", LastModified: mustTime(t, "2026-01-01T00:00:00Z"), Size: 1},+ })+ session, err := connector.OpenSync(context.Background(), SyncRequest{+ Fingerprints: map[string]string{"other": "fingerprint"},+ })+ if err != nil {+ t.Fatalf("OpenSync: %v", err)+ }+ batch, err := session.NextBatch(context.Background())+ if err != nil {+ t.Fatalf("NextBatch: %v", err)+ }+ if len(batch.Documents) != 1 {+ t.Fatalf("documents len = %d, want 1", len(batch.Documents))+ }+}++func TestS3ConnectorOpenSyncFiltersByWindow(t *testing.T) {+ start := mustTime(t, "2026-01-02T00:00:00Z")+ end := mustTime(t, "2026-01-04T00:00:00Z")+ connector := newTestS3Connector(t, []s3Object{+ {Key: "docs/before.txt", LastModified: mustTime(t, "2026-01-01T00:00:00Z"), Size: 1, ETag: "before"},+ {Key: "docs/within.txt", LastModified: mustTime(t, "2026-01-03T00:00:00Z"), Size: 1, ETag: "within"},+ {Key: "docs/after.txt", LastModified: mustTime(t, "2026-01-05T00:00:00Z"), Size: 1, ETag: "after"},+ })+ session, err := connector.OpenSync(context.Background(), SyncRequest{WindowStart: &start, WindowEnd: end})+ if err != nil {+ t.Fatalf("OpenSync: %v", err)+ }+ batch, err := session.NextBatch(context.Background())+ if err != nil {+ t.Fatalf("NextBatch: %v", err)+ }+ if len(batch.Documents) != 1 || batch.Documents[0].SourceID != s3SourceID(s3Source, "bucket", "docs/within.txt") {+ t.Fatalf("documents = %+v", batch.Documents)+ }+}++func TestS3ConnectorOpenPruneReturnsSlimSnapshot(t *testing.T) {+ connector := newTestS3Connector(t, []s3Object{+ {Key: "docs/a.txt", LastModified: mustTime(t, "2026-01-01T00:00:00Z"), Size: 1, ETag: "a"},+ {Key: "docs/b.txt", LastModified: mustTime(t, "2026-01-02T00:00:00Z"), Size: 1, ETag: "b"},+ {Key: "docs/folder/", LastModified: mustTime(t, "2026-01-02T00:00:00Z"), Size: 0, ETag: "folder"},+ })+ session, err := connector.OpenPrune(context.Background(), PruneRequest{})+ if err != nil {+ t.Fatalf("OpenPrune: %v", err)+ }+ batch, err := session.NextBatch(context.Background())+ if err != nil {+ t.Fatalf("NextBatch: %v", err)+ }+ if len(batch.Documents) != 2 {+ t.Fatalf("documents len = %d, want 2", len(batch.Documents))+ }+ if batch.Documents[0].SourceID != s3SourceID(s3Source, "bucket", "docs/a.txt") || batch.Documents[1].SourceID != s3SourceID(s3Source, "bucket", "docs/b.txt") {+ t.Fatalf("documents = %+v", batch.Documents)+ }+}++func TestS3ConnectorOpenPruneStreamsAcrossPages(t *testing.T) {+ connector, err := NewS3Connector(map[string]any{+ "bucket_name": "bucket",+ "batch_size": 2,+ "credentials": map[string]any{+ "aws_access_key_id": "access",+ "aws_secret_access_key": "secret",+ },+ })+ if err != nil {+ t.Fatalf("NewS3Connector: %v", err)+ }+ pages := [][]s3Object{+ {+ {Key: "docs/a.txt", Size: 1},+ {Key: "docs/b.txt", Size: 1},+ },+ {+ {Key: "docs/c.txt", Size: 1},+ {Key: "docs/d.txt", Size: 1},+ },+ }+ call := 0+ connector.listObjects = func(ctx context.Context, startAfter string, maxKeys int32) ([]s3Object, string, bool, error) {+ if call >= len(pages) {+ return nil, "", false, nil+ }+ page := pages[call]+ call+++ nextStartAfter := ""+ if len(page) > 0 {+ nextStartAfter = s3SourceID(s3Source, connector.bucketName, page[len(page)-1].Key)+ }+ return page, nextStartAfter, call < len(pages), nil+ }+ session, err := connector.OpenPrune(context.Background(), PruneRequest{})+ if err != nil {+ t.Fatalf("OpenPrune: %v", err)+ }+ first, err := session.NextBatch(context.Background())+ if err != nil {+ t.Fatalf("first NextBatch: %v", err)+ }+ if len(first.Documents) != 2 || first.Documents[0].SourceID != s3SourceID(s3Source, "bucket", "docs/a.txt") || first.Documents[1].SourceID != s3SourceID(s3Source, "bucket", "docs/b.txt") {+ t.Fatalf("first batch = %+v", first.Documents)+ }+ second, err := session.NextBatch(context.Background())+ if err != nil {+ t.Fatalf("second NextBatch: %v", err)+ }+ if len(second.Documents) != 2 || second.Documents[0].SourceID != s3SourceID(s3Source, "bucket", "docs/c.txt") || second.Documents[1].SourceID != s3SourceID(s3Source, "bucket", "docs/d.txt") {+ t.Fatalf("second batch = %+v", second.Documents)+ }+ if _, err := session.NextBatch(context.Background()); !errors.Is(err, io.EOF) {+ t.Fatalf("NextBatch EOF = %v", err)+ }+}++func TestS3ConnectorOpenPruneErrorsWhenPaginationDoesNotAdvance(t *testing.T) {+ tests := []struct {+ name string+ nextStartAfter string+ }{+ {name: "empty cursor", nextStartAfter: ""},+ {name: "unchanged cursor", nextStartAfter: s3SourceID(s3Source, "bucket", "docs/a.txt")},+ }+ for _, test := range tests {+ t.Run(test.name, func(t *testing.T) {+ connector := newTestS3Connector(t, []s3Object{{Key: "docs/a.txt", Size: 1}})+ connector.listObjects = func(ctx context.Context, startAfter string, maxKeys int32) ([]s3Object, string, bool, error) {+ return []s3Object{{Key: "docs/a.txt", Size: 1}}, test.nextStartAfter, true, nil+ }+ session, err := connector.OpenPrune(context.Background(), PruneRequest{})+ if err != nil {+ t.Fatalf("OpenPrune: %v", err)+ }+ batch, err := session.NextBatch(context.Background())+ if err == nil || errors.Is(err, io.EOF) {+ t.Fatalf("NextBatch error = %v, want non-EOF pagination error", err)+ }+ if !strings.Contains(err.Error(), "listing did not advance") {+ t.Fatalf("NextBatch error = %q, want listing did not advance", err)+ }+ if len(batch.Documents) != 0 {+ t.Fatalf("NextBatch returned partial documents = %+v", batch.Documents)+ }+ })+ }+}++func TestS3ConnectorValidate(t *testing.T) {+ ctx := context.Background()+ missingBucket := &S3Connector{authMethod: "access_key", accessKeyID: "access", secretKey: "secret", batchSize: 2}+ if err := missingBucket.Validate(ctx); err == nil {+ t.Fatalf("expected bucket name validation error")+ }+ missingCreds := &S3Connector{bucketName: "bucket", authMethod: "access_key", batchSize: 2}+ if err := missingCreds.Validate(ctx); err == nil {+ t.Fatalf("expected credentials validation error")+ }+ missingRole := &S3Connector{bucketName: "bucket", authMethod: "iam_role", batchSize: 2}+ if err := missingRole.Validate(ctx); err == nil {+ t.Fatalf("expected IAM role validation error")+ }+ badAuth := &S3Connector{bucketName: "bucket", authMethod: "invalid", batchSize: 2}+ if err := badAuth.Validate(ctx); err == nil {+ t.Fatalf("expected unsupported auth method error")+ }+ badBatch := &S3Connector{bucketName: "bucket", authMethod: "access_key", accessKeyID: "access", secretKey: "secret"}+ if err := badBatch.Validate(ctx); err == nil {+ t.Fatalf("expected batch size validation error")+ }+}++func TestS3ConnectorRejectsInvalidRegionBeforeClientCreation(t *testing.T) {+ for _, region := range []string{+ "evil.example#",+ "evil.example.com",+ "https://evil.example.org",+ "region space",+ "-region",+ "region-",+ "Region",+ } {+ connector := newTestS3Connector(t, nil)+ connector.region = region+ connector.client = nil+ if err := connector.Validate(context.Background()); err == nil {+ t.Fatalf("expected invalid region %q to be rejected", region)+ }+ if connector.client != nil {+ t.Fatalf("client was created for invalid region %q", region)+ }+ }+}++func TestS3ConnectorValidateSetting(t *testing.T) {+ connector := newTestS3Connector(t, []s3Object{+ {Key: "docs/a.txt", Size: 1},+ })+ var listCalls int+ connector.listObjects = func(ctx context.Context, startAfter string, maxKeys int32) ([]s3Object, string, bool, error) {+ listCalls+++ return nil, "", false, nil+ }+ if err := connector.ValidateConnectorSetting(context.Background(), nil); err != nil {+ t.Fatalf("ValidateConnectorSetting: %v", err)+ }+ if listCalls != 1 {+ t.Fatalf("list calls = %d, want 1", listCalls)+ }+}++func TestNewS3ConnectorDefaultsAndConfig(t *testing.T) {+ connector, err := NewS3Connector(map[string]any{+ "bucket_name": " bucket ",+ "prefix": "docs",+ "batch_size": 4,+ "size_threshold": 42,+ "allow_images": true,+ "credentials": map[string]any{+ "authentication_method": "access_key",+ "region": "us-east-1",+ "aws_access_key_id": "access",+ "aws_secret_access_key": "secret",+ },+ })+ if err != nil {+ t.Fatalf("NewS3Connector: %v", err)+ }+ if connector.bucketName != "bucket" || connector.prefix != "docs/" {+ t.Fatalf("bucket/prefix = %q/%q", connector.bucketName, connector.prefix)+ }+ if connector.batchSize != 4 || connector.sizeThreshold != 42 || !connector.allowImages {+ t.Fatalf("batch/threshold/allow_images = %d/%d/%v", connector.batchSize, connector.sizeThreshold, connector.allowImages)+ }+ if connector.region != "us-east-1" || connector.accessKeyID != "access" || connector.secretKey != "secret" {+ t.Fatalf("credentials = %+v", connector)+ }++ defaults, err := NewS3Connector(map[string]any{"bucket_name": "bucket", "credentials": map[string]any{}})+ if err != nil {+ t.Fatalf("NewS3Connector defaults: %v", err)+ }+ if defaults.batchSize != defaultS3BatchSize || defaults.sizeThreshold != defaultS3SizeThreshold || defaults.authMethod != "access_key" {+ t.Fatalf("defaults = batch %d threshold %d auth %q", defaults.batchSize, defaults.sizeThreshold, defaults.authMethod)+ }+}++func TestS3Helpers(t *testing.T) {+ if got := s3SourceID(s3Source, "bucket", "docs/a.txt"); got != "s3:bucket:docs/a.txt" {+ t.Fatalf("source id = %q", got)+ }+ if got := normalizeS3Prefix("docs"); got != "docs/" {+ t.Fatalf("prefix = %q", got)+ }+ if got := pathEscapeS3Key("docs/a b.txt"); got != "docs/a%20b.txt" {+ t.Fatalf("escaped key = %q", got)+ }+ if got := s3ConsoleURL("us-east-1", "bucket", "docs/a b.txt"); !strings.Contains(got, "https://s3.console.aws.amazon.com/s3/object/bucket?region=us-east-1&prefix=docs/a%20b.txt") {+ t.Fatalf("console url = %q", got)+ }+ if _, err := readS3Body(bytes.NewBufferString("12345"), "large.txt", 4); err == nil {+ t.Fatalf("expected size threshold error")+ }+}++func newTestS3Connector(t *testing.T, objects []s3Object) *S3Connector {+ t.Helper()+ connector, err := NewS3Connector(map[string]any{+ "bucket_name": "bucket",+ "prefix": "docs",+ "batch_size": 2,+ "credentials": map[string]any{+ "aws_access_key_id": "access",+ "aws_secret_access_key": "secret",+ "region": "us-east-1",+ },+ })+ if err != nil {+ t.Fatalf("NewS3Connector: %v", err)+ }+ connector.listObjects = func(ctx context.Context, startAfter string, maxKeys int32) ([]s3Object, string, bool, error) {+ var out []s3Object+ for _, object := range objects {+ if startAfter != "" && object.Key <= startAfter {+ continue+ }+ out = append(out, object)+ if maxKeys > 0 && len(out) >= int(maxKeys) {+ break+ }+ }+ hasMore := false+ nextStartAfter := ""+ if len(out) > 0 {+ nextStartAfter = s3SourceID(s3Source, connector.bucketName, out[len(out)-1].Key)+ for _, object := range objects {+ if object.Key > out[len(out)-1].Key {+ hasMore = true+ break+ }+ }+ }+ return out, nextStartAfter, hasMore, nil+ }+ connector.downloadObject = func(ctx context.Context, key string, sizeThreshold int64) ([]byte, error) {+ if key == "docs/new.txt" {+ return []byte("body:new"), nil+ }+ return []byte("body:" + key), nil+ }+ return connector+}