Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

feat(syncer): add R2 connector with checkpoint resume (#18548)

mkaaad0b7ec8a9166c
diff --git a/internal/syncer/connector/builtin.go b/internal/syncer/connector/builtin.go
index 57efd1f03e..1d5e9f2278 100644
--- a/internal/syncer/connector/builtin.go
+++ b/internal/syncer/connector/builtin.go
@@ -36,6 +36,7 @@ func RegisterBuiltIns(registry *Registry) {
registerBuiltIn(registry, "google-drive", NewGoogleDriveConnector)
registerBuiltIn(registry, "google_drive", NewGoogleDriveConnector)
registerBuiltIn(registry, "google_cloud_storage", NewGoogleCloudStorageConnector)
+ registerBuiltIn(registry, "r2", NewR2Connector)
registerBuiltIn(registry, "dingtalk_ai_table", NewDingTalkAITableConnector)
registerBuiltIn(registry, "imap", NewIMAPConnector)
registerBuiltIn(registry, "outlook", NewOutlookConnector)
diff --git a/internal/syncer/connector/r2.go b/internal/syncer/connector/r2.go
new file mode 100644
index 0000000000..3705f7f03c
--- /dev/null
+++ b/internal/syncer/connector/r2.go
@@ -0,0 +1,495 @@
+//
+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+package connector
+
+import (
+ "context"
+ "encoding/json"
+ "fmt"
+ "io"
+ "net/url"
+ "path"
+ "path/filepath"
+ "strings"
+ "time"
+
+ "github.com/aws/aws-sdk-go-v2/aws"
+ awssdkconfig "github.com/aws/aws-sdk-go-v2/config"
+ "github.com/aws/aws-sdk-go-v2/credentials"
+ "github.com/aws/aws-sdk-go-v2/service/s3"
+ "github.com/aws/aws-sdk-go-v2/service/s3/types"
+)
+
+const (
+ defaultR2BatchSize = 2
+ defaultR2SizeThreshold = 20 * 1024 * 1024
+ r2ListPageSize = 1000
+ r2Source = "r2"
+)
+
+// R2Connector reads objects from Cloudflare R2 through its S3-compatible API.
+type R2Connector struct {
+ bucketName string
+ prefix string
+ accountID string
+ accessKeyID string
+ secretKey string
+ allowImages bool
+ batchSize int
+ sizeThreshold int64
+ europeanResidency bool
+
+ client *s3.Client
+ listObjects func(ctx context.Context, startAfter string, maxKeys int32) ([]r2Object, string, bool, error)
+ downloadObject func(ctx context.Context, key string, sizeThreshold int64) ([]byte, error)
+}
+
+// NewR2Connector creates a Cloudflare R2 connector from connector config.
+func NewR2Connector(config map[string]any) (*R2Connector, error) {
+ credentials := configAnyMap(config["credentials"])
+ batchSize := configInt(firstNonEmpty(stringConfig(config["sync_batch_size"]), stringConfig(config["batch_size"])), defaultR2BatchSize)
+ sizeThreshold := int64(configInt(config["size_threshold"], defaultR2SizeThreshold))
+ if sizeThreshold <= 0 {
+ sizeThreshold = defaultR2SizeThreshold
+ }
+ return &R2Connector{
+ bucketName: strings.TrimSpace(stringConfig(config["bucket_name"])),
+ prefix: normalizeR2Prefix(stringConfig(config["prefix"])),
+ accountID: strings.TrimSpace(stringConfig(credentials["account_id"])),
+ accessKeyID: strings.TrimSpace(stringConfig(credentials["r2_access_key_id"])),
+ secretKey: stringConfig(credentials["r2_secret_access_key"]),
+ allowImages: configBoolDefault(config["allow_images"], false),
+ europeanResidency: configBoolDefault(config["european_residency"], false),
+ batchSize: batchSize,
+ sizeThreshold: sizeThreshold,
+ }, nil
+}
+
+// Validate validates Cloudflare R2 connector settings and credentials.
+func (c *R2Connector) Validate(ctx context.Context) error {
+ if c == nil {
+ return fmt.Errorf("Cloudflare R2 connector is nil")
+ }
+ if c.bucketName == "" {
+ return fmt.Errorf("No bucket name was provided in connector settings.")
+ }
+ if c.accountID == "" || c.accessKeyID == "" || c.secretKey == "" {
+ return fmt.Errorf("Cloudflare R2 credentials are required")
+ }
+ if c.batchSize <= 0 {
+ return fmt.Errorf("batch_size must be a positive integer")
+ }
+ if _, err := c.ensureClient(ctx); err != nil {
+ return err
+ }
+ return nil
+}
+
+// ValidateConnectorSetting validates Cloudflare R2 settings from an unsaved config.
+func (c *R2Connector) ValidateConnectorSetting(ctx context.Context, request map[string]any) error {
+ ctx, cancel := context.WithTimeout(ctx, connectorSettingValidationTimeout)
+ defer cancel()
+ if err := c.Validate(ctx); err != nil {
+ return err
+ }
+ client, err := c.ensureClient(ctx)
+ if err != nil {
+ return err
+ }
+ _, err = client.HeadBucket(ctx, &s3.HeadBucketInput{Bucket: aws.String(c.bucketName)})
+ if err != nil {
+ return fmt.Errorf("access Cloudflare R2 bucket %q: %w", c.bucketName, err)
+ }
+ return nil
+}
+
+// OpenSync opens one Cloudflare R2 sync session.
+func (c *R2Connector) OpenSync(ctx context.Context, request SyncRequest) (SyncSession, error) {
+ if err := c.Validate(ctx); err != nil {
+ return nil, err
+ }
+ session := &r2SyncSession{
+ connector: c,
+ request: request,
+ batchSize: c.batchSize,
+ }
+ if request.Resume != nil {
+ session.applyResume(request.Resume)
+ }
+ return session, nil
+}
+
+// OpenPrune opens one complete Cloudflare R2 prune snapshot session.
+func (c *R2Connector) OpenPrune(ctx context.Context, request PruneRequest) (PruneSession, error) {
+ if err := c.Validate(ctx); err != nil {
+ return nil, err
+ }
+ return &r2PruneSession{connector: c, batchSize: c.batchSize}, nil
+}
+
+// Fetch downloads a Cloudflare R2 object body.
+func (c *R2Connector) Fetch(ctx context.Context, ref FetchReference) ([]byte, error) {
+ var fetch r2FetchReference
+ if err := json.Unmarshal([]byte(ref.Key), &fetch); err != nil {
+ return nil, err
+ }
+ return c.download(ctx, fetch.Key)
+}
+
+func (c *R2Connector) ensureClient(ctx context.Context) (*s3.Client, error) {
+ if c.client != nil {
+ return c.client, nil
+ }
+ cfg, err := awssdkconfig.LoadDefaultConfig(
+ ctx,
+ awssdkconfig.WithRegion("auto"),
+ awssdkconfig.WithCredentialsProvider(credentials.NewStaticCredentialsProvider(c.accessKeyID, c.secretKey, "")),
+ )
+ if err != nil {
+ return nil, fmt.Errorf("load Cloudflare R2 config: %w", err)
+ }
+ c.client = s3.NewFromConfig(cfg, func(options *s3.Options) {
+ options.BaseEndpoint = aws.String(r2Endpoint(c.accountID, c.europeanResidency))
+ options.UsePathStyle = true
+ })
+ return c.client, nil
+}
+
+func (c *R2Connector) listObjectPage(ctx context.Context, startAfter string, maxKeys int32) ([]r2Object, string, bool, error) {
+ if c.listObjects != nil {
+ return c.listObjects(ctx, startAfter, maxKeys)
+ }
+ client, err := c.ensureClient(ctx)
+ if err != nil {
+ return nil, "", false, err
+ }
+ input := &s3.ListObjectsV2Input{
+ Bucket: aws.String(c.bucketName),
+ Prefix: aws.String(c.prefix),
+ StartAfter: aws.String(startAfter),
+ }
+ if maxKeys > 0 {
+ input.MaxKeys = aws.Int32(maxKeys)
+ }
+ output, err := client.ListObjectsV2(ctx, input)
+ if err != nil {
+ return nil, "", false, err
+ }
+ objects := make([]r2Object, 0, len(output.Contents))
+ for _, object := range output.Contents {
+ objects = append(objects, r2ObjectFromS3(object))
+ }
+ nextStartAfter := ""
+ if len(objects) > 0 {
+ nextStartAfter = r2SourceID(c.bucketName, objects[len(objects)-1].Key)
+ }
+ return objects, nextStartAfter, aws.ToBool(output.IsTruncated), nil
+}
+
+func (c *R2Connector) download(ctx context.Context, key string) ([]byte, error) {
+ if c.downloadObject != nil {
+ return c.downloadObject(ctx, key, c.sizeThreshold)
+ }
+ client, err := c.ensureClient(ctx)
+ if err != nil {
+ return nil, err
+ }
+ output, err := client.GetObject(ctx, &s3.GetObjectInput{
+ Bucket: aws.String(c.bucketName),
+ Key: aws.String(key),
+ })
+ if err != nil {
+ return nil, err
+ }
+ defer output.Body.Close()
+ return readR2Body(output.Body, key, c.sizeThreshold)
+}
+
+type r2Object struct {
+ Key string
+ LastModified time.Time
+ Size int64
+ ETag string
+}
+
+func r2ObjectFromS3(object types.Object) r2Object {
+ updatedAt := time.Time{}
+ if object.LastModified != nil {
+ updatedAt = object.LastModified.UTC()
+ }
+ return r2Object{
+ Key: aws.ToString(object.Key),
+ LastModified: updatedAt,
+ Size: aws.ToInt64(object.Size),
+ ETag: aws.ToString(object.ETag),
+ }
+}
+
+type r2SyncSession struct {
+ connector *R2Connector
+ request SyncRequest
+ batchSize int
+ startAfter string
+ done bool
+}
+
+// NextBatch returns the next Cloudflare R2 document batch.
+func (s *r2SyncSession) NextBatch(ctx context.Context) (SyncBatch, error) {
+ for {
+ if s.done {
+ return SyncBatch{}, io.EOF
+ }
+ previousStartAfter := s.startAfter
+ objects, nextStartAfter, hasMore, err := s.connector.listObjectPage(ctx, s.startAfter, int32(s.batchSize))
+ if err != nil {
+ return SyncBatch{}, err
+ }
+ if !hasMore {
+ s.done = true
+ }
+ if nextStartAfter != "" {
+ s.startAfter = strings.TrimPrefix(nextStartAfter, r2SourceID(s.connector.bucketName, ""))
+ }
+ if hasMore && s.startAfter == previousStartAfter {
+ return SyncBatch{}, fmt.Errorf("Cloudflare R2 listing did not advance from %q", previousStartAfter)
+ }
+
+ documents := make([]SourceDocument, 0, len(objects))
+ for _, object := range objects {
+ sourceID := r2SourceID(s.connector.bucketName, object.Key)
+ if !includeR2Object(s.request, sourceID, object) {
+ continue
+ }
+ document, ok := s.connector.sourceDocument(sourceID, object)
+ if ok {
+ documents = append(documents, document)
+ }
+ }
+ if len(documents) == 0 {
+ continue
+ }
+ last := documents[len(documents)-1]
+ updatedAt := last.UpdatedAt
+ cursor := r2SourceID(s.connector.bucketName, s.startAfter)
+ return SyncBatch{
+ Documents: documents,
+ Checkpoint: &SyncCheckpoint{
+ Cursor: cursor,
+ SourceID: cursor,
+ UpdatedAt: &updatedAt,
+ },
+ }, nil
+ }
+}
+
+// Close closes the Cloudflare R2 sync session.
+func (s *r2SyncSession) Close() error {
+ return nil
+}
+
+// Fetch downloads a delayed Cloudflare R2 document body.
+func (s *r2SyncSession) Fetch(ctx context.Context, ref FetchReference) ([]byte, error) {
+ return s.connector.Fetch(ctx, ref)
+}
+
+func (s *r2SyncSession) applyResume(checkpoint *SyncCheckpoint) {
+ sourceID := firstNonEmpty(checkpoint.SourceID, checkpoint.Cursor)
+ prefix := r2SourceID(s.connector.bucketName, "")
+ if sourceID == "" || !strings.HasPrefix(sourceID, prefix) {
+ return
+ }
+ s.startAfter = strings.TrimPrefix(sourceID, prefix)
+}
+
+func (c *R2Connector) sourceDocument(sourceID string, object r2Object) (SourceDocument, bool) {
+ if object.Key == "" || strings.HasSuffix(object.Key, "/") || (!c.allowImages && object.isImage()) {
+ return SourceDocument{}, false
+ }
+ fileName := path.Base(object.Key)
+ fetch := r2FetchReference{Key: object.Key}
+ fetchKey, _ := json.Marshal(fetch)
+ return SourceDocument{
+ SourceID: sourceID,
+ SemanticIdentifier: c.semanticIdentifier(object.Key, fileName),
+ Extension: strings.ToLower(filepath.Ext(fileName)),
+ FetchRef: &FetchReference{Key: string(fetchKey), SizeHint: object.Size},
+ UpdatedAt: object.LastModified,
+ SizeBytes: object.Size,
+ Metadata: map[string]any{
+ "url": r2ConsoleURL(c.accountID, c.europeanResidency, c.bucketName, object.Key),
+ },
+ Fingerprint: normalizedR2ETag(object.ETag),
+ }, true
+}
+
+func (c *R2Connector) semanticIdentifier(key, fileName string) string {
+ relativePath := key
+ if c.prefix != "" {
+ relativePath = strings.TrimPrefix(key, c.prefix)
+ }
+ if relativePath == "" {
+ return fileName
+ }
+ return strings.ReplaceAll(relativePath, "/", " / ")
+}
+
+func (o r2Object) isImage() bool {
+ switch strings.ToLower(filepath.Ext(o.Key)) {
+ case ".jpg", ".jpeg", ".png", ".webp", ".heic", ".heif":
+ return true
+ default:
+ return false
+ }
+}
+
+type r2PruneSession struct {
+ connector *R2Connector
+ batchSize int
+ startAfter string
+ done bool
+ buffer []SlimDocument
+}
+
+// NextBatch returns the next Cloudflare R2 prune snapshot batch.
+func (s *r2PruneSession) NextBatch(ctx context.Context) (PruneBatch, error) {
+ documents := make([]SlimDocument, 0, s.batchSize)
+ if len(s.buffer) > 0 {
+ n := s.batchSize
+ if n > len(s.buffer) {
+ n = len(s.buffer)
+ }
+ documents = append(documents, s.buffer[:n]...)
+ s.buffer = s.buffer[n:]
+ }
+ for len(documents) < s.batchSize && !s.done {
+ page, nextStartAfter, hasMore, err := s.connector.listObjectPage(ctx, s.startAfter, r2ListPageSize)
+ if err != nil {
+ return PruneBatch{}, err
+ }
+ previousStartAfter := s.startAfter
+ if hasMore {
+ next := strings.TrimPrefix(nextStartAfter, r2SourceID(s.connector.bucketName, ""))
+ if next == "" || next == previousStartAfter {
+ s.done = true
+ } else {
+ s.startAfter = next
+ }
+ } else {
+ s.done = true
+ }
+ remaining := s.batchSize - len(documents)
+ for _, object := range page {
+ if object.Key == "" || strings.HasSuffix(object.Key, "/") {
+ continue
+ }
+ doc := SlimDocument{SourceID: r2SourceID(s.connector.bucketName, object.Key)}
+ if remaining > 0 {
+ documents = append(documents, doc)
+ remaining--
+ } else {
+ s.buffer = append(s.buffer, doc)
+ }
+ }
+ if remaining <= 0 {
+ break
+ }
+ }
+ if len(documents) == 0 && len(s.buffer) == 0 && s.done {
+ return PruneBatch{}, io.EOF
+ }
+ return PruneBatch{Documents: documents}, nil
+}
+
+// Close closes the Cloudflare R2 prune session.
+func (s *r2PruneSession) Close() error {
+ return nil
+}
+
+type r2FetchReference struct {
+ Key string `json:"key"`
+}
+
+func includeR2Object(request SyncRequest, sourceID string, object r2Object) bool {
+ if request.FromBeginning {
+ return true
+ }
+ if object.LastModified.IsZero() {
+ return true
+ }
+ if len(request.Fingerprints) > 0 {
+ fingerprint := normalizedR2ETag(object.ETag)
+ stored, ok := request.Fingerprints[sourceID]
+ return fingerprint == "" || !ok || stored == "" || stored != fingerprint
+ }
+ return !beforeOrAtWindowStart(object.LastModified, request.WindowStart) && !afterWindowEnd(object.LastModified, request.WindowEnd)
+}
+
+func normalizeR2Prefix(prefix string) string {
+ prefix = strings.TrimSpace(prefix)
+ if prefix != "" && !strings.HasSuffix(prefix, "/") {
+ prefix += "/"
+ }
+ return prefix
+}
+
+func r2Endpoint(accountID string, europeanResidency bool) string {
+ subdomain := ""
+ if europeanResidency {
+ subdomain = "eu."
+ }
+ return fmt.Sprintf("https://%s.%sr2.cloudflarestorage.com", accountID, subdomain)
+}
+
+func r2SourceID(bucketName, key string) string {
+ return fmt.Sprintf("%s:%s:%s", r2Source, bucketName, key)
+}
+
+func normalizedR2ETag(rawETag string) string {
+ rawETag = strings.Trim(strings.TrimSpace(rawETag), `"`)
+ if rawETag == "" {
+ return ""
+ }
+ return contentFingerprint([]byte(rawETag))
+}
+
+func r2ConsoleURL(accountID string, europeanResidency bool, bucketName, key string) string {
+ region := "default"
+ if europeanResidency {
+ region = "eu"
+ }
+ return fmt.Sprintf("https://dash.cloudflare.com/%s/r2/%s/buckets/%s/objects/%s/details", accountID, region, bucketName, pathEscapeR2Key(key))
+}
+
+func pathEscapeR2Key(key string) string {
+ parts := strings.Split(key, "/")
+ for index, part := range parts {
+ parts[index] = url.PathEscape(part)
+ }
+ return strings.Join(parts, "/")
+}
+
+func readR2Body(body io.Reader, key string, sizeThreshold int64) ([]byte, error) {
+ limited := io.LimitReader(body, sizeThreshold+1)
+ data, err := io.ReadAll(limited)
+ if err != nil {
+ return nil, err
+ }
+ if int64(len(data)) > sizeThreshold {
+ return nil, fmt.Errorf("%s exceeds size threshold of %d", key, sizeThreshold)
+ }
+ return data, nil
+}
diff --git a/internal/syncer/connector/r2_test.go b/internal/syncer/connector/r2_test.go
new file mode 100644
index 0000000000..21bfbfec8e
--- /dev/null
+++ b/internal/syncer/connector/r2_test.go
@@ -0,0 +1,346 @@
+package connector
+
+import (
+ "bytes"
+ "context"
+ "errors"
+ "io"
+ "strings"
+ "testing"
+)
+
+func TestR2ConnectorOpenSyncUsesFingerprintAndFetch(t *testing.T) {
+ old := mustTime(t, "2026-01-01T00:00:00Z")
+ updated := mustTime(t, "2026-01-03T00:00:00Z")
+ connector := newTestR2Connector(t, []r2Object{
+ {Key: "docs/old.txt", LastModified: old, Size: 8, ETag: `"old-etag"`},
+ {Key: "docs/new.txt", LastModified: updated, Size: 8, ETag: `"new-etag"`},
+ })
+
+ start := mustTime(t, "2026-01-02T00:00:00Z")
+ session, err := connector.OpenSync(context.Background(), SyncRequest{
+ WindowStart: &start,
+ WindowEnd: mustTime(t, "2026-01-04T00:00:00Z"),
+ Fingerprints: map[string]string{
+ r2SourceID("bucket", "docs/old.txt"): normalizedR2ETag(`"old-etag"`),
+ },
+ })
+ if err != nil {
+ t.Fatalf("OpenSync: %v", err)
+ }
+
+ batch, err := session.NextBatch(context.Background())
+ if err != nil {
+ t.Fatalf("NextBatch: %v", err)
+ }
+ if len(batch.Documents) != 1 {
+ t.Fatalf("documents len = %d, want 1", len(batch.Documents))
+ }
+ doc := batch.Documents[0]
+ if doc.SourceID != r2SourceID("bucket", "docs/new.txt") {
+ t.Fatalf("source id = %q", doc.SourceID)
+ }
+ if doc.SemanticIdentifier != "new.txt" {
+ t.Fatalf("semantic identifier = %q", doc.SemanticIdentifier)
+ }
+ if doc.Extension != ".txt" {
+ t.Fatalf("extension = %q", doc.Extension)
+ }
+ if doc.Fingerprint != normalizedR2ETag(`"new-etag"`) {
+ t.Fatalf("fingerprint = %q", doc.Fingerprint)
+ }
+ if doc.FetchRef == nil {
+ t.Fatalf("fetch ref is nil")
+ }
+ fetcher, ok := session.(Fetcher)
+ if !ok {
+ t.Fatalf("session does not implement Fetcher")
+ }
+ blob, err := fetcher.Fetch(context.Background(), *doc.FetchRef)
+ if err != nil {
+ t.Fatalf("Fetch: %v", err)
+ }
+ if string(blob) != "body:new" {
+ t.Fatalf("blob = %q", blob)
+ }
+ if _, err = session.NextBatch(context.Background()); !errors.Is(err, io.EOF) {
+ t.Fatalf("NextBatch EOF = %v", err)
+ }
+}
+
+func TestR2ConnectorOpenSyncDefersListingUntilNextBatch(t *testing.T) {
+ connector := newTestR2Connector(t, []r2Object{
+ {Key: "docs/a.txt", LastModified: mustTime(t, "2026-01-01T00:00:00Z"), Size: 1, ETag: "a"},
+ })
+ var listCalls int
+ baseListObjects := connector.listObjects
+ connector.listObjects = func(ctx context.Context, startAfter string, maxKeys int32) ([]r2Object, string, bool, error) {
+ listCalls++
+ return baseListObjects(ctx, startAfter, maxKeys)
+ }
+
+ session, err := connector.OpenSync(context.Background(), SyncRequest{FromBeginning: true})
+ if err != nil {
+ t.Fatalf("OpenSync: %v", err)
+ }
+ if listCalls != 0 {
+ t.Fatalf("OpenSync list calls = %d, want 0", listCalls)
+ }
+ if _, err := session.NextBatch(context.Background()); err != nil {
+ t.Fatalf("NextBatch: %v", err)
+ }
+ if listCalls != 1 {
+ t.Fatalf("NextBatch list calls = %d, want 1", listCalls)
+ }
+}
+
+func TestR2ConnectorOpenSyncFiltersImagesUnlessAllowed(t *testing.T) {
+ objects := []r2Object{
+ {Key: "docs/a.png", LastModified: mustTime(t, "2026-01-01T00:00:00Z"), Size: 1, ETag: "a"},
+ {Key: "docs/b.txt", LastModified: mustTime(t, "2026-01-02T00:00:00Z"), Size: 1, ETag: "b"},
+ }
+ connector := newTestR2Connector(t, objects)
+ session, err := connector.OpenSync(context.Background(), SyncRequest{FromBeginning: true})
+ if err != nil {
+ t.Fatalf("OpenSync: %v", err)
+ }
+ batch, err := session.NextBatch(context.Background())
+ if err != nil {
+ t.Fatalf("NextBatch: %v", err)
+ }
+ if len(batch.Documents) != 1 || batch.Documents[0].SourceID != r2SourceID("bucket", "docs/b.txt") {
+ t.Fatalf("documents = %+v", batch.Documents)
+ }
+
+ allowed := newTestR2Connector(t, objects)
+ allowed.allowImages = true
+ session, err = allowed.OpenSync(context.Background(), SyncRequest{FromBeginning: true})
+ if err != nil {
+ t.Fatalf("allow images OpenSync: %v", err)
+ }
+ batch, err = session.NextBatch(context.Background())
+ if err != nil {
+ t.Fatalf("allow images NextBatch: %v", err)
+ }
+ if len(batch.Documents) != 2 {
+ t.Fatalf("allow images documents len = %d, want 2", len(batch.Documents))
+ }
+}
+
+func TestR2ConnectorOpenSyncResumesFromCheckpoint(t *testing.T) {
+ objects := []r2Object{
+ {Key: "docs/a.txt", LastModified: mustTime(t, "2026-01-01T00:00:00Z"), Size: 1, ETag: "a"},
+ {Key: "docs/b.txt", LastModified: mustTime(t, "2026-01-02T00:00:00Z"), Size: 1, ETag: "b"},
+ {Key: "docs/c.txt", LastModified: mustTime(t, "2026-01-03T00:00:00Z"), Size: 1, ETag: "c"},
+ }
+ connector := newTestR2Connector(t, objects)
+
+ checkpointUpdatedAt := mustTime(t, "2026-01-02T00:00:00Z")
+ session, err := connector.OpenSync(context.Background(), SyncRequest{
+ FromBeginning: true,
+ Resume: &SyncCheckpoint{
+ Cursor: r2SourceID("bucket", "docs/b.txt"),
+ SourceID: r2SourceID("bucket", "docs/b.txt"),
+ UpdatedAt: &checkpointUpdatedAt,
+ },
+ })
+ if err != nil {
+ t.Fatalf("OpenSync: %v", err)
+ }
+ batch, err := session.NextBatch(context.Background())
+ if err != nil {
+ t.Fatalf("NextBatch: %v", err)
+ }
+ if len(batch.Documents) != 1 {
+ t.Fatalf("documents len = %d, want 1", len(batch.Documents))
+ }
+ if batch.Documents[0].SourceID != r2SourceID("bucket", "docs/c.txt") {
+ t.Fatalf("resumed source id = %q, want docs/c.txt", batch.Documents[0].SourceID)
+ }
+ if batch.Checkpoint == nil || batch.Checkpoint.Cursor != r2SourceID("bucket", "docs/c.txt") {
+ t.Fatalf("checkpoint = %+v", batch.Checkpoint)
+ }
+}
+
+func TestR2ConnectorOpenSyncResumeIgnoresForeignCheckpoint(t *testing.T) {
+ connector := newTestR2Connector(t, []r2Object{
+ {Key: "docs/a.txt", LastModified: mustTime(t, "2026-01-01T00:00:00Z"), Size: 1, ETag: "a"},
+ })
+ session, err := connector.OpenSync(context.Background(), SyncRequest{
+ FromBeginning: true,
+ Resume: &SyncCheckpoint{
+ Cursor: "other:bucket:docs/a.txt",
+ },
+ })
+ if err != nil {
+ t.Fatalf("OpenSync: %v", err)
+ }
+ batch, err := session.NextBatch(context.Background())
+ if err != nil {
+ t.Fatalf("NextBatch: %v", err)
+ }
+ if len(batch.Documents) != 1 || batch.Documents[0].SourceID != r2SourceID("bucket", "docs/a.txt") {
+ t.Fatalf("documents = %+v", batch.Documents)
+ }
+}
+
+func TestR2ConnectorOpenPruneReturnsSlimSnapshot(t *testing.T) {
+ connector := newTestR2Connector(t, []r2Object{
+ {Key: "docs/a.txt", LastModified: mustTime(t, "2026-01-01T00:00:00Z"), Size: 1, ETag: "a"},
+ {Key: "docs/b.txt", LastModified: mustTime(t, "2026-01-02T00:00:00Z"), Size: 1, ETag: "b"},
+ {Key: "docs/folder/", LastModified: mustTime(t, "2026-01-02T00:00:00Z"), Size: 0, ETag: "folder"},
+ })
+ session, err := connector.OpenPrune(context.Background(), PruneRequest{})
+ if err != nil {
+ t.Fatalf("OpenPrune: %v", err)
+ }
+ batch, err := session.NextBatch(context.Background())
+ if err != nil {
+ t.Fatalf("NextBatch: %v", err)
+ }
+ if len(batch.Documents) != 2 {
+ t.Fatalf("documents len = %d, want 2", len(batch.Documents))
+ }
+ if batch.Documents[0].SourceID != r2SourceID("bucket", "docs/a.txt") || batch.Documents[1].SourceID != r2SourceID("bucket", "docs/b.txt") {
+ t.Fatalf("documents = %+v", batch.Documents)
+ }
+}
+
+func TestR2ConnectorOpenSyncIncludesMissingFingerprint(t *testing.T) {
+ connector := newTestR2Connector(t, []r2Object{
+ {Key: "docs/no-etag.txt", LastModified: mustTime(t, "2026-01-01T00:00:00Z"), Size: 1},
+ })
+ session, err := connector.OpenSync(context.Background(), SyncRequest{
+ Fingerprints: map[string]string{"other": "fingerprint"},
+ })
+ if err != nil {
+ t.Fatalf("OpenSync: %v", err)
+ }
+ batch, err := session.NextBatch(context.Background())
+ if err != nil {
+ t.Fatalf("NextBatch: %v", err)
+ }
+ if len(batch.Documents) != 1 {
+ t.Fatalf("documents len = %d, want 1", len(batch.Documents))
+ }
+}
+
+func TestR2ConnectorValidate(t *testing.T) {
+ ctx := context.Background()
+ _, err := NewR2Connector(map[string]any{"credentials": map[string]any{}})
+ if err != nil {
+ t.Fatalf("NewR2Connector: %v", err)
+ }
+
+ missingBucket := &R2Connector{accountID: "account", accessKeyID: "access", secretKey: "secret", batchSize: 2}
+ if err := missingBucket.Validate(ctx); err == nil {
+ t.Fatalf("expected bucket name validation error")
+ }
+
+ missingCreds := &R2Connector{bucketName: "bucket", batchSize: 2}
+ if err := missingCreds.Validate(ctx); err == nil {
+ t.Fatalf("expected credentials validation error")
+ }
+
+ badBatch := &R2Connector{bucketName: "bucket", accountID: "account", accessKeyID: "access", secretKey: "secret", batchSize: 0}
+ if err := badBatch.Validate(ctx); err == nil {
+ t.Fatalf("expected batch size validation error")
+ }
+}
+
+func TestR2Endpoint(t *testing.T) {
+ if got := r2Endpoint("myaccount", false); got != "https://myaccount.r2.cloudflarestorage.com" {
+ t.Fatalf("endpoint = %q", got)
+ }
+ if got := r2Endpoint("myaccount", true); got != "https://myaccount.eu.r2.cloudflarestorage.com" {
+ t.Fatalf("eu endpoint = %q", got)
+ }
+}
+
+func TestNewR2ConnectorDefaultsAndConfig(t *testing.T) {
+ connector, err := NewR2Connector(map[string]any{
+ "bucket_name": " bucket ",
+ "prefix": "docs",
+ "credentials": map[string]any{
+ "account_id": "myaccount",
+ "r2_access_key_id": "access",
+ "r2_secret_access_key": "secret",
+ },
+ })
+ if err != nil {
+ t.Fatalf("NewR2Connector: %v", err)
+ }
+ if connector.bucketName != "bucket" {
+ t.Fatalf("bucket name = %q", connector.bucketName)
+ }
+ if connector.prefix != "docs/" {
+ t.Fatalf("prefix = %q", connector.prefix)
+ }
+ if connector.batchSize != defaultR2BatchSize {
+ t.Fatalf("batch size = %d, want %d", connector.batchSize, defaultR2BatchSize)
+ }
+ if connector.sizeThreshold != defaultR2SizeThreshold {
+ t.Fatalf("size threshold = %d, want %d", connector.sizeThreshold, defaultR2SizeThreshold)
+ }
+ if connector.accountID != "myaccount" || connector.accessKeyID != "access" || connector.secretKey != "secret" {
+ t.Fatalf("credentials = %+v", connector)
+ }
+ if got := r2ConsoleURL(connector.accountID, connector.europeanResidency, connector.bucketName, "docs/a b.txt"); !strings.Contains(got, "https://dash.cloudflare.com/myaccount/r2/default/buckets/bucket/objects/docs/a%20b.txt/details") {
+ t.Fatalf("console url = %q", got)
+ }
+}
+
+func TestReadR2BodySizeThreshold(t *testing.T) {
+ _, err := readR2Body(bytes.NewBufferString("12345"), "large.txt", 4)
+ if err == nil {
+ t.Fatalf("expected size threshold error")
+ }
+}
+
+func newTestR2Connector(t *testing.T, objects []r2Object) *R2Connector {
+ t.Helper()
+ connector, err := NewR2Connector(map[string]any{
+ "bucket_name": "bucket",
+ "prefix": "docs",
+ "batch_size": 2,
+ "credentials": map[string]any{
+ "account_id": "myaccount",
+ "r2_access_key_id": "access",
+ "r2_secret_access_key": "secret",
+ },
+ })
+ if err != nil {
+ t.Fatalf("NewR2Connector: %v", err)
+ }
+ connector.listObjects = func(ctx context.Context, startAfter string, maxKeys int32) ([]r2Object, string, bool, error) {
+ var out []r2Object
+ for _, object := range objects {
+ if startAfter != "" && object.Key <= startAfter {
+ continue
+ }
+ out = append(out, object)
+ if maxKeys > 0 && len(out) >= int(maxKeys) {
+ break
+ }
+ }
+ hasMore := false
+ nextStartAfter := ""
+ if len(out) > 0 {
+ nextStartAfter = r2SourceID(connector.bucketName, out[len(out)-1].Key)
+ for _, object := range objects {
+ if object.Key > out[len(out)-1].Key {
+ hasMore = true
+ break
+ }
+ }
+ }
+ return out, nextStartAfter, hasMore, nil
+ }
+ connector.downloadObject = func(ctx context.Context, key string, sizeThreshold int64) ([]byte, error) {
+ if key == "docs/new.txt" {
+ return []byte("body:new"), nil
+ }
+ return []byte("body:" + key), nil
+ }
+ return connector
+}