RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
feat(syncer): add R2 connector with checkpoint resume (#18548)
mkaaad0b7ec8a9166c
diff --git a/internal/syncer/connector/builtin.go b/internal/syncer/connector/builtin.goindex 57efd1f03e..1d5e9f2278 100644--- a/internal/syncer/connector/builtin.go+++ b/internal/syncer/connector/builtin.go@@ -36,6 +36,7 @@ func RegisterBuiltIns(registry *Registry) {registerBuiltIn(registry, "google-drive", NewGoogleDriveConnector)registerBuiltIn(registry, "google_drive", NewGoogleDriveConnector)registerBuiltIn(registry, "google_cloud_storage", NewGoogleCloudStorageConnector)+ registerBuiltIn(registry, "r2", NewR2Connector)registerBuiltIn(registry, "dingtalk_ai_table", NewDingTalkAITableConnector)registerBuiltIn(registry, "imap", NewIMAPConnector)registerBuiltIn(registry, "outlook", NewOutlookConnector)diff --git a/internal/syncer/connector/r2.go b/internal/syncer/connector/r2.gonew file mode 100644index 0000000000..3705f7f03c--- /dev/null+++ b/internal/syncer/connector/r2.go@@ -0,0 +1,495 @@+//+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.+//++package connector++import (+ "context"+ "encoding/json"+ "fmt"+ "io"+ "net/url"+ "path"+ "path/filepath"+ "strings"+ "time"++ "github.com/aws/aws-sdk-go-v2/aws"+ awssdkconfig "github.com/aws/aws-sdk-go-v2/config"+ "github.com/aws/aws-sdk-go-v2/credentials"+ "github.com/aws/aws-sdk-go-v2/service/s3"+ "github.com/aws/aws-sdk-go-v2/service/s3/types"+)++const (+ defaultR2BatchSize = 2+ defaultR2SizeThreshold = 20 * 1024 * 1024+ r2ListPageSize = 1000+ r2Source = "r2"+)++// R2Connector reads objects from Cloudflare R2 through its S3-compatible API.+type R2Connector struct {+ bucketName string+ prefix string+ accountID string+ accessKeyID string+ secretKey string+ allowImages bool+ batchSize int+ sizeThreshold int64+ europeanResidency bool++ client *s3.Client+ listObjects func(ctx context.Context, startAfter string, maxKeys int32) ([]r2Object, string, bool, error)+ downloadObject func(ctx context.Context, key string, sizeThreshold int64) ([]byte, error)+}++// NewR2Connector creates a Cloudflare R2 connector from connector config.+func NewR2Connector(config map[string]any) (*R2Connector, error) {+ credentials := configAnyMap(config["credentials"])+ batchSize := configInt(firstNonEmpty(stringConfig(config["sync_batch_size"]), stringConfig(config["batch_size"])), defaultR2BatchSize)+ sizeThreshold := int64(configInt(config["size_threshold"], defaultR2SizeThreshold))+ if sizeThreshold <= 0 {+ sizeThreshold = defaultR2SizeThreshold+ }+ return &R2Connector{+ bucketName: strings.TrimSpace(stringConfig(config["bucket_name"])),+ prefix: normalizeR2Prefix(stringConfig(config["prefix"])),+ accountID: strings.TrimSpace(stringConfig(credentials["account_id"])),+ accessKeyID: strings.TrimSpace(stringConfig(credentials["r2_access_key_id"])),+ secretKey: stringConfig(credentials["r2_secret_access_key"]),+ allowImages: configBoolDefault(config["allow_images"], false),+ europeanResidency: configBoolDefault(config["european_residency"], false),+ batchSize: batchSize,+ sizeThreshold: sizeThreshold,+ }, nil+}++// Validate validates Cloudflare R2 connector settings and credentials.+func (c *R2Connector) Validate(ctx context.Context) error {+ if c == nil {+ return fmt.Errorf("Cloudflare R2 connector is nil")+ }+ if c.bucketName == "" {+ return fmt.Errorf("No bucket name was provided in connector settings.")+ }+ if c.accountID == "" || c.accessKeyID == "" || c.secretKey == "" {+ return fmt.Errorf("Cloudflare R2 credentials are required")+ }+ if c.batchSize <= 0 {+ return fmt.Errorf("batch_size must be a positive integer")+ }+ if _, err := c.ensureClient(ctx); err != nil {+ return err+ }+ return nil+}++// ValidateConnectorSetting validates Cloudflare R2 settings from an unsaved config.+func (c *R2Connector) ValidateConnectorSetting(ctx context.Context, request map[string]any) error {+ ctx, cancel := context.WithTimeout(ctx, connectorSettingValidationTimeout)+ defer cancel()+ if err := c.Validate(ctx); err != nil {+ return err+ }+ client, err := c.ensureClient(ctx)+ if err != nil {+ return err+ }+ _, err = client.HeadBucket(ctx, &s3.HeadBucketInput{Bucket: aws.String(c.bucketName)})+ if err != nil {+ return fmt.Errorf("access Cloudflare R2 bucket %q: %w", c.bucketName, err)+ }+ return nil+}++// OpenSync opens one Cloudflare R2 sync session.+func (c *R2Connector) OpenSync(ctx context.Context, request SyncRequest) (SyncSession, error) {+ if err := c.Validate(ctx); err != nil {+ return nil, err+ }+ session := &r2SyncSession{+ connector: c,+ request: request,+ batchSize: c.batchSize,+ }+ if request.Resume != nil {+ session.applyResume(request.Resume)+ }+ return session, nil+}++// OpenPrune opens one complete Cloudflare R2 prune snapshot session.+func (c *R2Connector) OpenPrune(ctx context.Context, request PruneRequest) (PruneSession, error) {+ if err := c.Validate(ctx); err != nil {+ return nil, err+ }+ return &r2PruneSession{connector: c, batchSize: c.batchSize}, nil+}++// Fetch downloads a Cloudflare R2 object body.+func (c *R2Connector) Fetch(ctx context.Context, ref FetchReference) ([]byte, error) {+ var fetch r2FetchReference+ if err := json.Unmarshal([]byte(ref.Key), &fetch); err != nil {+ return nil, err+ }+ return c.download(ctx, fetch.Key)+}++func (c *R2Connector) ensureClient(ctx context.Context) (*s3.Client, error) {+ if c.client != nil {+ return c.client, nil+ }+ cfg, err := awssdkconfig.LoadDefaultConfig(+ ctx,+ awssdkconfig.WithRegion("auto"),+ awssdkconfig.WithCredentialsProvider(credentials.NewStaticCredentialsProvider(c.accessKeyID, c.secretKey, "")),+ )+ if err != nil {+ return nil, fmt.Errorf("load Cloudflare R2 config: %w", err)+ }+ c.client = s3.NewFromConfig(cfg, func(options *s3.Options) {+ options.BaseEndpoint = aws.String(r2Endpoint(c.accountID, c.europeanResidency))+ options.UsePathStyle = true+ })+ return c.client, nil+}++func (c *R2Connector) listObjectPage(ctx context.Context, startAfter string, maxKeys int32) ([]r2Object, string, bool, error) {+ if c.listObjects != nil {+ return c.listObjects(ctx, startAfter, maxKeys)+ }+ client, err := c.ensureClient(ctx)+ if err != nil {+ return nil, "", false, err+ }+ input := &s3.ListObjectsV2Input{+ Bucket: aws.String(c.bucketName),+ Prefix: aws.String(c.prefix),+ StartAfter: aws.String(startAfter),+ }+ if maxKeys > 0 {+ input.MaxKeys = aws.Int32(maxKeys)+ }+ output, err := client.ListObjectsV2(ctx, input)+ if err != nil {+ return nil, "", false, err+ }+ objects := make([]r2Object, 0, len(output.Contents))+ for _, object := range output.Contents {+ objects = append(objects, r2ObjectFromS3(object))+ }+ nextStartAfter := ""+ if len(objects) > 0 {+ nextStartAfter = r2SourceID(c.bucketName, objects[len(objects)-1].Key)+ }+ return objects, nextStartAfter, aws.ToBool(output.IsTruncated), nil+}++func (c *R2Connector) download(ctx context.Context, key string) ([]byte, error) {+ if c.downloadObject != nil {+ return c.downloadObject(ctx, key, c.sizeThreshold)+ }+ client, err := c.ensureClient(ctx)+ if err != nil {+ return nil, err+ }+ output, err := client.GetObject(ctx, &s3.GetObjectInput{+ Bucket: aws.String(c.bucketName),+ Key: aws.String(key),+ })+ if err != nil {+ return nil, err+ }+ defer output.Body.Close()+ return readR2Body(output.Body, key, c.sizeThreshold)+}++type r2Object struct {+ Key string+ LastModified time.Time+ Size int64+ ETag string+}++func r2ObjectFromS3(object types.Object) r2Object {+ updatedAt := time.Time{}+ if object.LastModified != nil {+ updatedAt = object.LastModified.UTC()+ }+ return r2Object{+ Key: aws.ToString(object.Key),+ LastModified: updatedAt,+ Size: aws.ToInt64(object.Size),+ ETag: aws.ToString(object.ETag),+ }+}++type r2SyncSession struct {+ connector *R2Connector+ request SyncRequest+ batchSize int+ startAfter string+ done bool+}++// NextBatch returns the next Cloudflare R2 document batch.+func (s *r2SyncSession) NextBatch(ctx context.Context) (SyncBatch, error) {+ for {+ if s.done {+ return SyncBatch{}, io.EOF+ }+ previousStartAfter := s.startAfter+ objects, nextStartAfter, hasMore, err := s.connector.listObjectPage(ctx, s.startAfter, int32(s.batchSize))+ if err != nil {+ return SyncBatch{}, err+ }+ if !hasMore {+ s.done = true+ }+ if nextStartAfter != "" {+ s.startAfter = strings.TrimPrefix(nextStartAfter, r2SourceID(s.connector.bucketName, ""))+ }+ if hasMore && s.startAfter == previousStartAfter {+ return SyncBatch{}, fmt.Errorf("Cloudflare R2 listing did not advance from %q", previousStartAfter)+ }++ documents := make([]SourceDocument, 0, len(objects))+ for _, object := range objects {+ sourceID := r2SourceID(s.connector.bucketName, object.Key)+ if !includeR2Object(s.request, sourceID, object) {+ continue+ }+ document, ok := s.connector.sourceDocument(sourceID, object)+ if ok {+ documents = append(documents, document)+ }+ }+ if len(documents) == 0 {+ continue+ }+ last := documents[len(documents)-1]+ updatedAt := last.UpdatedAt+ cursor := r2SourceID(s.connector.bucketName, s.startAfter)+ return SyncBatch{+ Documents: documents,+ Checkpoint: &SyncCheckpoint{+ Cursor: cursor,+ SourceID: cursor,+ UpdatedAt: &updatedAt,+ },+ }, nil+ }+}++// Close closes the Cloudflare R2 sync session.+func (s *r2SyncSession) Close() error {+ return nil+}++// Fetch downloads a delayed Cloudflare R2 document body.+func (s *r2SyncSession) Fetch(ctx context.Context, ref FetchReference) ([]byte, error) {+ return s.connector.Fetch(ctx, ref)+}++func (s *r2SyncSession) applyResume(checkpoint *SyncCheckpoint) {+ sourceID := firstNonEmpty(checkpoint.SourceID, checkpoint.Cursor)+ prefix := r2SourceID(s.connector.bucketName, "")+ if sourceID == "" || !strings.HasPrefix(sourceID, prefix) {+ return+ }+ s.startAfter = strings.TrimPrefix(sourceID, prefix)+}++func (c *R2Connector) sourceDocument(sourceID string, object r2Object) (SourceDocument, bool) {+ if object.Key == "" || strings.HasSuffix(object.Key, "/") || (!c.allowImages && object.isImage()) {+ return SourceDocument{}, false+ }+ fileName := path.Base(object.Key)+ fetch := r2FetchReference{Key: object.Key}+ fetchKey, _ := json.Marshal(fetch)+ return SourceDocument{+ SourceID: sourceID,+ SemanticIdentifier: c.semanticIdentifier(object.Key, fileName),+ Extension: strings.ToLower(filepath.Ext(fileName)),+ FetchRef: &FetchReference{Key: string(fetchKey), SizeHint: object.Size},+ UpdatedAt: object.LastModified,+ SizeBytes: object.Size,+ Metadata: map[string]any{+ "url": r2ConsoleURL(c.accountID, c.europeanResidency, c.bucketName, object.Key),+ },+ Fingerprint: normalizedR2ETag(object.ETag),+ }, true+}++func (c *R2Connector) semanticIdentifier(key, fileName string) string {+ relativePath := key+ if c.prefix != "" {+ relativePath = strings.TrimPrefix(key, c.prefix)+ }+ if relativePath == "" {+ return fileName+ }+ return strings.ReplaceAll(relativePath, "/", " / ")+}++func (o r2Object) isImage() bool {+ switch strings.ToLower(filepath.Ext(o.Key)) {+ case ".jpg", ".jpeg", ".png", ".webp", ".heic", ".heif":+ return true+ default:+ return false+ }+}++type r2PruneSession struct {+ connector *R2Connector+ batchSize int+ startAfter string+ done bool+ buffer []SlimDocument+}++// NextBatch returns the next Cloudflare R2 prune snapshot batch.+func (s *r2PruneSession) NextBatch(ctx context.Context) (PruneBatch, error) {+ documents := make([]SlimDocument, 0, s.batchSize)+ if len(s.buffer) > 0 {+ n := s.batchSize+ if n > len(s.buffer) {+ n = len(s.buffer)+ }+ documents = append(documents, s.buffer[:n]...)+ s.buffer = s.buffer[n:]+ }+ for len(documents) < s.batchSize && !s.done {+ page, nextStartAfter, hasMore, err := s.connector.listObjectPage(ctx, s.startAfter, r2ListPageSize)+ if err != nil {+ return PruneBatch{}, err+ }+ previousStartAfter := s.startAfter+ if hasMore {+ next := strings.TrimPrefix(nextStartAfter, r2SourceID(s.connector.bucketName, ""))+ if next == "" || next == previousStartAfter {+ s.done = true+ } else {+ s.startAfter = next+ }+ } else {+ s.done = true+ }+ remaining := s.batchSize - len(documents)+ for _, object := range page {+ if object.Key == "" || strings.HasSuffix(object.Key, "/") {+ continue+ }+ doc := SlimDocument{SourceID: r2SourceID(s.connector.bucketName, object.Key)}+ if remaining > 0 {+ documents = append(documents, doc)+ remaining--+ } else {+ s.buffer = append(s.buffer, doc)+ }+ }+ if remaining <= 0 {+ break+ }+ }+ if len(documents) == 0 && len(s.buffer) == 0 && s.done {+ return PruneBatch{}, io.EOF+ }+ return PruneBatch{Documents: documents}, nil+}++// Close closes the Cloudflare R2 prune session.+func (s *r2PruneSession) Close() error {+ return nil+}++type r2FetchReference struct {+ Key string `json:"key"`+}++func includeR2Object(request SyncRequest, sourceID string, object r2Object) bool {+ if request.FromBeginning {+ return true+ }+ if object.LastModified.IsZero() {+ return true+ }+ if len(request.Fingerprints) > 0 {+ fingerprint := normalizedR2ETag(object.ETag)+ stored, ok := request.Fingerprints[sourceID]+ return fingerprint == "" || !ok || stored == "" || stored != fingerprint+ }+ return !beforeOrAtWindowStart(object.LastModified, request.WindowStart) && !afterWindowEnd(object.LastModified, request.WindowEnd)+}++func normalizeR2Prefix(prefix string) string {+ prefix = strings.TrimSpace(prefix)+ if prefix != "" && !strings.HasSuffix(prefix, "/") {+ prefix += "/"+ }+ return prefix+}++func r2Endpoint(accountID string, europeanResidency bool) string {+ subdomain := ""+ if europeanResidency {+ subdomain = "eu."+ }+ return fmt.Sprintf("https://%s.%sr2.cloudflarestorage.com", accountID, subdomain)+}++func r2SourceID(bucketName, key string) string {+ return fmt.Sprintf("%s:%s:%s", r2Source, bucketName, key)+}++func normalizedR2ETag(rawETag string) string {+ rawETag = strings.Trim(strings.TrimSpace(rawETag), `"`)+ if rawETag == "" {+ return ""+ }+ return contentFingerprint([]byte(rawETag))+}++func r2ConsoleURL(accountID string, europeanResidency bool, bucketName, key string) string {+ region := "default"+ if europeanResidency {+ region = "eu"+ }+ return fmt.Sprintf("https://dash.cloudflare.com/%s/r2/%s/buckets/%s/objects/%s/details", accountID, region, bucketName, pathEscapeR2Key(key))+}++func pathEscapeR2Key(key string) string {+ parts := strings.Split(key, "/")+ for index, part := range parts {+ parts[index] = url.PathEscape(part)+ }+ return strings.Join(parts, "/")+}++func readR2Body(body io.Reader, key string, sizeThreshold int64) ([]byte, error) {+ limited := io.LimitReader(body, sizeThreshold+1)+ data, err := io.ReadAll(limited)+ if err != nil {+ return nil, err+ }+ if int64(len(data)) > sizeThreshold {+ return nil, fmt.Errorf("%s exceeds size threshold of %d", key, sizeThreshold)+ }+ return data, nil+}diff --git a/internal/syncer/connector/r2_test.go b/internal/syncer/connector/r2_test.gonew file mode 100644index 0000000000..21bfbfec8e--- /dev/null+++ b/internal/syncer/connector/r2_test.go@@ -0,0 +1,346 @@+package connector++import (+ "bytes"+ "context"+ "errors"+ "io"+ "strings"+ "testing"+)++func TestR2ConnectorOpenSyncUsesFingerprintAndFetch(t *testing.T) {+ old := mustTime(t, "2026-01-01T00:00:00Z")+ updated := mustTime(t, "2026-01-03T00:00:00Z")+ connector := newTestR2Connector(t, []r2Object{+ {Key: "docs/old.txt", LastModified: old, Size: 8, ETag: `"old-etag"`},+ {Key: "docs/new.txt", LastModified: updated, Size: 8, ETag: `"new-etag"`},+ })++ start := mustTime(t, "2026-01-02T00:00:00Z")+ session, err := connector.OpenSync(context.Background(), SyncRequest{+ WindowStart: &start,+ WindowEnd: mustTime(t, "2026-01-04T00:00:00Z"),+ Fingerprints: map[string]string{+ r2SourceID("bucket", "docs/old.txt"): normalizedR2ETag(`"old-etag"`),+ },+ })+ if err != nil {+ t.Fatalf("OpenSync: %v", err)+ }++ batch, err := session.NextBatch(context.Background())+ if err != nil {+ t.Fatalf("NextBatch: %v", err)+ }+ if len(batch.Documents) != 1 {+ t.Fatalf("documents len = %d, want 1", len(batch.Documents))+ }+ doc := batch.Documents[0]+ if doc.SourceID != r2SourceID("bucket", "docs/new.txt") {+ t.Fatalf("source id = %q", doc.SourceID)+ }+ if doc.SemanticIdentifier != "new.txt" {+ t.Fatalf("semantic identifier = %q", doc.SemanticIdentifier)+ }+ if doc.Extension != ".txt" {+ t.Fatalf("extension = %q", doc.Extension)+ }+ if doc.Fingerprint != normalizedR2ETag(`"new-etag"`) {+ t.Fatalf("fingerprint = %q", doc.Fingerprint)+ }+ if doc.FetchRef == nil {+ t.Fatalf("fetch ref is nil")+ }+ fetcher, ok := session.(Fetcher)+ if !ok {+ t.Fatalf("session does not implement Fetcher")+ }+ blob, err := fetcher.Fetch(context.Background(), *doc.FetchRef)+ if err != nil {+ t.Fatalf("Fetch: %v", err)+ }+ if string(blob) != "body:new" {+ t.Fatalf("blob = %q", blob)+ }+ if _, err = session.NextBatch(context.Background()); !errors.Is(err, io.EOF) {+ t.Fatalf("NextBatch EOF = %v", err)+ }+}++func TestR2ConnectorOpenSyncDefersListingUntilNextBatch(t *testing.T) {+ connector := newTestR2Connector(t, []r2Object{+ {Key: "docs/a.txt", LastModified: mustTime(t, "2026-01-01T00:00:00Z"), Size: 1, ETag: "a"},+ })+ var listCalls int+ baseListObjects := connector.listObjects+ connector.listObjects = func(ctx context.Context, startAfter string, maxKeys int32) ([]r2Object, string, bool, error) {+ listCalls+++ return baseListObjects(ctx, startAfter, maxKeys)+ }++ session, err := connector.OpenSync(context.Background(), SyncRequest{FromBeginning: true})+ if err != nil {+ t.Fatalf("OpenSync: %v", err)+ }+ if listCalls != 0 {+ t.Fatalf("OpenSync list calls = %d, want 0", listCalls)+ }+ if _, err := session.NextBatch(context.Background()); err != nil {+ t.Fatalf("NextBatch: %v", err)+ }+ if listCalls != 1 {+ t.Fatalf("NextBatch list calls = %d, want 1", listCalls)+ }+}++func TestR2ConnectorOpenSyncFiltersImagesUnlessAllowed(t *testing.T) {+ objects := []r2Object{+ {Key: "docs/a.png", LastModified: mustTime(t, "2026-01-01T00:00:00Z"), Size: 1, ETag: "a"},+ {Key: "docs/b.txt", LastModified: mustTime(t, "2026-01-02T00:00:00Z"), Size: 1, ETag: "b"},+ }+ connector := newTestR2Connector(t, objects)+ session, err := connector.OpenSync(context.Background(), SyncRequest{FromBeginning: true})+ if err != nil {+ t.Fatalf("OpenSync: %v", err)+ }+ batch, err := session.NextBatch(context.Background())+ if err != nil {+ t.Fatalf("NextBatch: %v", err)+ }+ if len(batch.Documents) != 1 || batch.Documents[0].SourceID != r2SourceID("bucket", "docs/b.txt") {+ t.Fatalf("documents = %+v", batch.Documents)+ }++ allowed := newTestR2Connector(t, objects)+ allowed.allowImages = true+ session, err = allowed.OpenSync(context.Background(), SyncRequest{FromBeginning: true})+ if err != nil {+ t.Fatalf("allow images OpenSync: %v", err)+ }+ batch, err = session.NextBatch(context.Background())+ if err != nil {+ t.Fatalf("allow images NextBatch: %v", err)+ }+ if len(batch.Documents) != 2 {+ t.Fatalf("allow images documents len = %d, want 2", len(batch.Documents))+ }+}++func TestR2ConnectorOpenSyncResumesFromCheckpoint(t *testing.T) {+ objects := []r2Object{+ {Key: "docs/a.txt", LastModified: mustTime(t, "2026-01-01T00:00:00Z"), Size: 1, ETag: "a"},+ {Key: "docs/b.txt", LastModified: mustTime(t, "2026-01-02T00:00:00Z"), Size: 1, ETag: "b"},+ {Key: "docs/c.txt", LastModified: mustTime(t, "2026-01-03T00:00:00Z"), Size: 1, ETag: "c"},+ }+ connector := newTestR2Connector(t, objects)++ checkpointUpdatedAt := mustTime(t, "2026-01-02T00:00:00Z")+ session, err := connector.OpenSync(context.Background(), SyncRequest{+ FromBeginning: true,+ Resume: &SyncCheckpoint{+ Cursor: r2SourceID("bucket", "docs/b.txt"),+ SourceID: r2SourceID("bucket", "docs/b.txt"),+ UpdatedAt: &checkpointUpdatedAt,+ },+ })+ if err != nil {+ t.Fatalf("OpenSync: %v", err)+ }+ batch, err := session.NextBatch(context.Background())+ if err != nil {+ t.Fatalf("NextBatch: %v", err)+ }+ if len(batch.Documents) != 1 {+ t.Fatalf("documents len = %d, want 1", len(batch.Documents))+ }+ if batch.Documents[0].SourceID != r2SourceID("bucket", "docs/c.txt") {+ t.Fatalf("resumed source id = %q, want docs/c.txt", batch.Documents[0].SourceID)+ }+ if batch.Checkpoint == nil || batch.Checkpoint.Cursor != r2SourceID("bucket", "docs/c.txt") {+ t.Fatalf("checkpoint = %+v", batch.Checkpoint)+ }+}++func TestR2ConnectorOpenSyncResumeIgnoresForeignCheckpoint(t *testing.T) {+ connector := newTestR2Connector(t, []r2Object{+ {Key: "docs/a.txt", LastModified: mustTime(t, "2026-01-01T00:00:00Z"), Size: 1, ETag: "a"},+ })+ session, err := connector.OpenSync(context.Background(), SyncRequest{+ FromBeginning: true,+ Resume: &SyncCheckpoint{+ Cursor: "other:bucket:docs/a.txt",+ },+ })+ if err != nil {+ t.Fatalf("OpenSync: %v", err)+ }+ batch, err := session.NextBatch(context.Background())+ if err != nil {+ t.Fatalf("NextBatch: %v", err)+ }+ if len(batch.Documents) != 1 || batch.Documents[0].SourceID != r2SourceID("bucket", "docs/a.txt") {+ t.Fatalf("documents = %+v", batch.Documents)+ }+}++func TestR2ConnectorOpenPruneReturnsSlimSnapshot(t *testing.T) {+ connector := newTestR2Connector(t, []r2Object{+ {Key: "docs/a.txt", LastModified: mustTime(t, "2026-01-01T00:00:00Z"), Size: 1, ETag: "a"},+ {Key: "docs/b.txt", LastModified: mustTime(t, "2026-01-02T00:00:00Z"), Size: 1, ETag: "b"},+ {Key: "docs/folder/", LastModified: mustTime(t, "2026-01-02T00:00:00Z"), Size: 0, ETag: "folder"},+ })+ session, err := connector.OpenPrune(context.Background(), PruneRequest{})+ if err != nil {+ t.Fatalf("OpenPrune: %v", err)+ }+ batch, err := session.NextBatch(context.Background())+ if err != nil {+ t.Fatalf("NextBatch: %v", err)+ }+ if len(batch.Documents) != 2 {+ t.Fatalf("documents len = %d, want 2", len(batch.Documents))+ }+ if batch.Documents[0].SourceID != r2SourceID("bucket", "docs/a.txt") || batch.Documents[1].SourceID != r2SourceID("bucket", "docs/b.txt") {+ t.Fatalf("documents = %+v", batch.Documents)+ }+}++func TestR2ConnectorOpenSyncIncludesMissingFingerprint(t *testing.T) {+ connector := newTestR2Connector(t, []r2Object{+ {Key: "docs/no-etag.txt", LastModified: mustTime(t, "2026-01-01T00:00:00Z"), Size: 1},+ })+ session, err := connector.OpenSync(context.Background(), SyncRequest{+ Fingerprints: map[string]string{"other": "fingerprint"},+ })+ if err != nil {+ t.Fatalf("OpenSync: %v", err)+ }+ batch, err := session.NextBatch(context.Background())+ if err != nil {+ t.Fatalf("NextBatch: %v", err)+ }+ if len(batch.Documents) != 1 {+ t.Fatalf("documents len = %d, want 1", len(batch.Documents))+ }+}++func TestR2ConnectorValidate(t *testing.T) {+ ctx := context.Background()+ _, err := NewR2Connector(map[string]any{"credentials": map[string]any{}})+ if err != nil {+ t.Fatalf("NewR2Connector: %v", err)+ }++ missingBucket := &R2Connector{accountID: "account", accessKeyID: "access", secretKey: "secret", batchSize: 2}+ if err := missingBucket.Validate(ctx); err == nil {+ t.Fatalf("expected bucket name validation error")+ }++ missingCreds := &R2Connector{bucketName: "bucket", batchSize: 2}+ if err := missingCreds.Validate(ctx); err == nil {+ t.Fatalf("expected credentials validation error")+ }++ badBatch := &R2Connector{bucketName: "bucket", accountID: "account", accessKeyID: "access", secretKey: "secret", batchSize: 0}+ if err := badBatch.Validate(ctx); err == nil {+ t.Fatalf("expected batch size validation error")+ }+}++func TestR2Endpoint(t *testing.T) {+ if got := r2Endpoint("myaccount", false); got != "https://myaccount.r2.cloudflarestorage.com" {+ t.Fatalf("endpoint = %q", got)+ }+ if got := r2Endpoint("myaccount", true); got != "https://myaccount.eu.r2.cloudflarestorage.com" {+ t.Fatalf("eu endpoint = %q", got)+ }+}++func TestNewR2ConnectorDefaultsAndConfig(t *testing.T) {+ connector, err := NewR2Connector(map[string]any{+ "bucket_name": " bucket ",+ "prefix": "docs",+ "credentials": map[string]any{+ "account_id": "myaccount",+ "r2_access_key_id": "access",+ "r2_secret_access_key": "secret",+ },+ })+ if err != nil {+ t.Fatalf("NewR2Connector: %v", err)+ }+ if connector.bucketName != "bucket" {+ t.Fatalf("bucket name = %q", connector.bucketName)+ }+ if connector.prefix != "docs/" {+ t.Fatalf("prefix = %q", connector.prefix)+ }+ if connector.batchSize != defaultR2BatchSize {+ t.Fatalf("batch size = %d, want %d", connector.batchSize, defaultR2BatchSize)+ }+ if connector.sizeThreshold != defaultR2SizeThreshold {+ t.Fatalf("size threshold = %d, want %d", connector.sizeThreshold, defaultR2SizeThreshold)+ }+ if connector.accountID != "myaccount" || connector.accessKeyID != "access" || connector.secretKey != "secret" {+ t.Fatalf("credentials = %+v", connector)+ }+ if got := r2ConsoleURL(connector.accountID, connector.europeanResidency, connector.bucketName, "docs/a b.txt"); !strings.Contains(got, "https://dash.cloudflare.com/myaccount/r2/default/buckets/bucket/objects/docs/a%20b.txt/details") {+ t.Fatalf("console url = %q", got)+ }+}++func TestReadR2BodySizeThreshold(t *testing.T) {+ _, err := readR2Body(bytes.NewBufferString("12345"), "large.txt", 4)+ if err == nil {+ t.Fatalf("expected size threshold error")+ }+}++func newTestR2Connector(t *testing.T, objects []r2Object) *R2Connector {+ t.Helper()+ connector, err := NewR2Connector(map[string]any{+ "bucket_name": "bucket",+ "prefix": "docs",+ "batch_size": 2,+ "credentials": map[string]any{+ "account_id": "myaccount",+ "r2_access_key_id": "access",+ "r2_secret_access_key": "secret",+ },+ })+ if err != nil {+ t.Fatalf("NewR2Connector: %v", err)+ }+ connector.listObjects = func(ctx context.Context, startAfter string, maxKeys int32) ([]r2Object, string, bool, error) {+ var out []r2Object+ for _, object := range objects {+ if startAfter != "" && object.Key <= startAfter {+ continue+ }+ out = append(out, object)+ if maxKeys > 0 && len(out) >= int(maxKeys) {+ break+ }+ }+ hasMore := false+ nextStartAfter := ""+ if len(out) > 0 {+ nextStartAfter = r2SourceID(connector.bucketName, out[len(out)-1].Key)+ for _, object := range objects {+ if object.Key > out[len(out)-1].Key {+ hasMore = true+ break+ }+ }+ }+ return out, nextStartAfter, hasMore, nil+ }+ connector.downloadObject = func(ctx context.Context, key string, sizeThreshold int64) ([]byte, error) {+ if key == "docs/new.txt" {+ return []byte("body:new"), nil+ }+ return []byte("body:" + key), nil+ }+ return connector+}