Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

feat(go-syncer): add OCI Storage connector with resume (#18850)

mkaaad41bc87243302
diff --git a/internal/syncer/connector/builtin.go b/internal/syncer/connector/builtin.go
index 9290d2ebcc..4e4efea62a 100644
--- a/internal/syncer/connector/builtin.go
+++ b/internal/syncer/connector/builtin.go
@@ -38,6 +38,7 @@ func RegisterBuiltIns(registry *Registry) {
registerBuiltIn(registry, "google-drive", NewGoogleDriveConnector)
registerBuiltIn(registry, "google_drive", NewGoogleDriveConnector)
registerBuiltIn(registry, "google_cloud_storage", NewGoogleCloudStorageConnector)
+ registerBuiltIn(registry, "oci_storage", NewOCIStorageConnector)
registerBuiltIn(registry, "azure_blob", NewAzureBlobStorageConnector)
registerBuiltIn(registry, "r2", NewR2Connector)
registerBuiltIn(registry, "dingtalk_ai_table", NewDingTalkAITableConnector)
diff --git a/internal/syncer/connector/oci_storage.go b/internal/syncer/connector/oci_storage.go
new file mode 100644
index 0000000000..b37288d8ee
--- /dev/null
+++ b/internal/syncer/connector/oci_storage.go
@@ -0,0 +1,544 @@
+//
+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+package connector
+
+import (
+ "context"
+ "encoding/json"
+ "fmt"
+ "io"
+ "net/url"
+ "path"
+ "path/filepath"
+ "regexp"
+ "strings"
+ "time"
+
+ "github.com/aws/aws-sdk-go-v2/aws"
+ awssdkconfig "github.com/aws/aws-sdk-go-v2/config"
+ "github.com/aws/aws-sdk-go-v2/credentials"
+ "github.com/aws/aws-sdk-go-v2/service/s3"
+ "github.com/aws/aws-sdk-go-v2/service/s3/types"
+)
+
+const (
+ defaultOCIStorageBatchSize = 2
+ defaultOCIStorageSizeThreshold = 20 * 1024 * 1024
+ ociStorageListPageSize = 1000
+ ociStorageSource = "oci_storage"
+)
+
+var ociStorageHostLabelPattern = regexp.MustCompile(`^[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?$`)
+
+// OCIStorageConnector reads objects from Oracle Cloud Infrastructure Object
+// Storage through its S3-compatible API.
+type OCIStorageConnector struct {
+ bucketName string
+ prefix string
+ namespace string
+ region string
+ accessKeyID string
+ secretKey string
+ allowImages bool
+ batchSize int
+ sizeThreshold int64
+
+ client *s3.Client
+ listObjects func(ctx context.Context, startAfter string, maxKeys int32) ([]ociStorageObject, string, bool, error)
+ downloadObject func(ctx context.Context, key string, sizeThreshold int64) ([]byte, error)
+}
+
+// NewOCIStorageConnector creates an OCI Object Storage connector from
+// Python-compatible config.
+func NewOCIStorageConnector(config map[string]any) (*OCIStorageConnector, error) {
+ credentials := configAnyMap(config["credentials"])
+ batchSize := configInt(firstNonEmpty(stringConfig(config["sync_batch_size"]), stringConfig(config["batch_size"])), defaultOCIStorageBatchSize)
+ sizeThreshold := int64(configInt(config["size_threshold"], defaultOCIStorageSizeThreshold))
+ if sizeThreshold <= 0 {
+ sizeThreshold = defaultOCIStorageSizeThreshold
+ }
+ return &OCIStorageConnector{
+ bucketName: strings.TrimSpace(stringConfig(config["bucket_name"])),
+ prefix: normalizeOCIStoragePrefix(stringConfig(config["prefix"])),
+ namespace: strings.TrimSpace(stringConfig(credentials["namespace"])),
+ region: strings.TrimSpace(stringConfig(credentials["region"])),
+ accessKeyID: strings.TrimSpace(stringConfig(credentials["access_key_id"])),
+ secretKey: stringConfig(credentials["secret_access_key"]),
+ allowImages: configBoolDefault(config["allow_images"], false),
+ batchSize: batchSize,
+ sizeThreshold: sizeThreshold,
+ }, nil
+}
+
+// Validate validates OCI Object Storage connector settings and credentials.
+func (c *OCIStorageConnector) Validate(ctx context.Context) error {
+ if c == nil {
+ return fmt.Errorf("OCI Storage connector is nil")
+ }
+ if c.bucketName == "" {
+ return fmt.Errorf("No bucket name was provided in connector settings.")
+ }
+ if c.namespace == "" || c.region == "" || c.accessKeyID == "" || c.secretKey == "" {
+ return fmt.Errorf("OCI Storage credentials are required")
+ }
+ if err := validateOCIStorageNamespace(c.namespace); err != nil {
+ return err
+ }
+ if err := validateOCIStorageRegion(c.region); err != nil {
+ return err
+ }
+ if c.batchSize <= 0 {
+ return fmt.Errorf("batch_size must be a positive integer")
+ }
+ if _, err := c.ensureClient(ctx); err != nil {
+ return err
+ }
+ return nil
+}
+
+// ValidateConnectorSetting validates OCI Object Storage settings from an
+// unsaved config.
+func (c *OCIStorageConnector) ValidateConnectorSetting(ctx context.Context, request map[string]any) error {
+ ctx, cancel := context.WithTimeout(ctx, connectorSettingValidationTimeout)
+ defer cancel()
+ if err := c.Validate(ctx); err != nil {
+ return err
+ }
+ _, _, _, err := c.listObjectPage(ctx, "", 1)
+ if err != nil {
+ return fmt.Errorf("access OCI bucket %q: %w", c.bucketName, err)
+ }
+ return nil
+}
+
+// OpenSync opens one OCI Object Storage sync session.
+func (c *OCIStorageConnector) OpenSync(ctx context.Context, request SyncRequest) (SyncSession, error) {
+ if err := c.Validate(ctx); err != nil {
+ return nil, err
+ }
+ session := &ociStorageSyncSession{
+ connector: c,
+ request: request,
+ batchSize: c.batchSize,
+ }
+ if request.Resume != nil {
+ if err := session.applyResume(ctx, request.Resume); err != nil {
+ return nil, err
+ }
+ }
+ return session, nil
+}
+
+// OpenPrune opens one complete OCI Object Storage prune snapshot session.
+func (c *OCIStorageConnector) OpenPrune(ctx context.Context, request PruneRequest) (PruneSession, error) {
+ if err := c.Validate(ctx); err != nil {
+ return nil, err
+ }
+ return &ociStoragePruneSession{connector: c, batchSize: c.batchSize}, nil
+}
+
+// Fetch downloads an OCI Object Storage object body.
+func (c *OCIStorageConnector) Fetch(ctx context.Context, ref FetchReference) ([]byte, error) {
+ var fetch ociStorageFetchReference
+ if err := json.Unmarshal([]byte(ref.Key), &fetch); err != nil {
+ return nil, err
+ }
+ return c.download(ctx, fetch.Key)
+}
+
+func (c *OCIStorageConnector) ensureClient(ctx context.Context) (*s3.Client, error) {
+ if err := validateOCIStorageNamespace(c.namespace); err != nil {
+ return nil, err
+ }
+ if err := validateOCIStorageRegion(c.region); err != nil {
+ return nil, err
+ }
+ if c.client != nil {
+ return c.client, nil
+ }
+ cfg, err := awssdkconfig.LoadDefaultConfig(
+ ctx,
+ awssdkconfig.WithRegion(c.region),
+ awssdkconfig.WithCredentialsProvider(credentials.NewStaticCredentialsProvider(c.accessKeyID, c.secretKey, "")),
+ )
+ if err != nil {
+ return nil, fmt.Errorf("load OCI Storage config: %w", err)
+ }
+ c.client = s3.NewFromConfig(cfg, func(options *s3.Options) {
+ options.BaseEndpoint = aws.String(ociStorageEndpoint(c.namespace, c.region))
+ options.UsePathStyle = true
+ })
+ return c.client, nil
+}
+
+func (c *OCIStorageConnector) listObjectPage(ctx context.Context, startAfter string, maxKeys int32) ([]ociStorageObject, string, bool, error) {
+ if c.listObjects != nil {
+ return c.listObjects(ctx, startAfter, maxKeys)
+ }
+ client, err := c.ensureClient(ctx)
+ if err != nil {
+ return nil, "", false, err
+ }
+ input := &s3.ListObjectsV2Input{
+ Bucket: aws.String(c.bucketName),
+ Prefix: aws.String(c.prefix),
+ StartAfter: aws.String(startAfter),
+ }
+ if maxKeys > 0 {
+ input.MaxKeys = aws.Int32(maxKeys)
+ }
+ output, err := client.ListObjectsV2(ctx, input)
+ if err != nil {
+ return nil, "", false, err
+ }
+ objects := make([]ociStorageObject, 0, len(output.Contents))
+ for _, object := range output.Contents {
+ objects = append(objects, ociStorageObjectFromS3(object))
+ }
+ nextStartAfter := ""
+ if len(objects) > 0 {
+ nextStartAfter = ociStorageSourceID(c.bucketName, objects[len(objects)-1].Key)
+ }
+ return objects, nextStartAfter, aws.ToBool(output.IsTruncated), nil
+}
+
+func (c *OCIStorageConnector) download(ctx context.Context, key string) ([]byte, error) {
+ if c.downloadObject != nil {
+ return c.downloadObject(ctx, key, c.sizeThreshold)
+ }
+ client, err := c.ensureClient(ctx)
+ if err != nil {
+ return nil, err
+ }
+ output, err := client.GetObject(ctx, &s3.GetObjectInput{
+ Bucket: aws.String(c.bucketName),
+ Key: aws.String(key),
+ })
+ if err != nil {
+ return nil, err
+ }
+ defer output.Body.Close()
+ return readOCIBody(output.Body, key, c.sizeThreshold)
+}
+
+type ociStorageObject struct {
+ Key string
+ LastModified time.Time
+ Size int64
+ ETag string
+}
+
+func ociStorageObjectFromS3(object types.Object) ociStorageObject {
+ updatedAt := time.Time{}
+ if object.LastModified != nil {
+ updatedAt = object.LastModified.UTC()
+ }
+ return ociStorageObject{
+ Key: aws.ToString(object.Key),
+ LastModified: updatedAt,
+ Size: aws.ToInt64(object.Size),
+ ETag: aws.ToString(object.ETag),
+ }
+}
+
+type ociStorageSyncSession struct {
+ connector *OCIStorageConnector
+ request SyncRequest
+ batchSize int
+ startAfter string
+ done bool
+}
+
+// NextBatch returns the next OCI Object Storage document batch.
+func (s *ociStorageSyncSession) NextBatch(ctx context.Context) (SyncBatch, error) {
+ for {
+ if s.done {
+ return SyncBatch{}, io.EOF
+ }
+ previousStartAfter := s.startAfter
+ objects, nextStartAfter, hasMore, err := s.connector.listObjectPage(ctx, s.startAfter, int32(s.batchSize))
+ if err != nil {
+ return SyncBatch{}, err
+ }
+ if !hasMore {
+ s.done = true
+ }
+ if nextStartAfter != "" {
+ s.startAfter = strings.TrimPrefix(nextStartAfter, ociStorageSourceID(s.connector.bucketName, ""))
+ }
+ if hasMore && s.startAfter == previousStartAfter {
+ return SyncBatch{}, fmt.Errorf("OCI Storage listing did not advance from %q", previousStartAfter)
+ }
+
+ documents := make([]SourceDocument, 0, len(objects))
+ for _, object := range objects {
+ sourceID := ociStorageSourceID(s.connector.bucketName, object.Key)
+ if !includeOCIStorageObject(s.request, sourceID, object) {
+ continue
+ }
+ document, ok := s.connector.sourceDocument(sourceID, object)
+ if ok {
+ documents = append(documents, document)
+ }
+ }
+ if len(documents) == 0 {
+ continue
+ }
+ last := documents[len(documents)-1]
+ updatedAt := last.UpdatedAt
+ return SyncBatch{
+ Documents: documents,
+ Checkpoint: &SyncCheckpoint{
+ Cursor: last.SourceID,
+ SourceID: last.SourceID,
+ UpdatedAt: &updatedAt,
+ },
+ }, nil
+ }
+}
+
+// Close closes the OCI Object Storage sync session.
+func (s *ociStorageSyncSession) Close() error {
+ return nil
+}
+
+// Fetch downloads a delayed OCI Object Storage document body.
+func (s *ociStorageSyncSession) Fetch(ctx context.Context, ref FetchReference) ([]byte, error) {
+ return s.connector.Fetch(ctx, ref)
+}
+
+func (s *ociStorageSyncSession) applyResume(ctx context.Context, checkpoint *SyncCheckpoint) error {
+ if checkpoint == nil {
+ return nil
+ }
+ sourceID := firstNonEmpty(checkpoint.SourceID, checkpoint.Cursor)
+ prefix := ociStorageSourceID(s.connector.bucketName, "")
+ if sourceID == "" || !strings.HasPrefix(sourceID, prefix) {
+ return fmt.Errorf("oci storage sync checkpoint has no source anchor: %w", ErrSyncResumeInvalid)
+ }
+ anchor := strings.TrimPrefix(sourceID, prefix)
+ if anchor == "" {
+ return fmt.Errorf("oci storage sync checkpoint has no source anchor: %w", ErrSyncResumeInvalid)
+ }
+
+ startAfter := ""
+ for {
+ objects, nextStartAfter, hasMore, err := s.connector.listObjectPage(ctx, startAfter, int32(s.batchSize))
+ if err != nil {
+ return err
+ }
+ for _, object := range objects {
+ if object.Key == anchor {
+ s.startAfter = anchor
+ return nil
+ }
+ if object.Key > anchor {
+ return fmt.Errorf("oci storage resume anchor %q was not found in the current listing: %w", anchor, ErrSyncResumeInvalid)
+ }
+ }
+ if !hasMore {
+ break
+ }
+ startAfter = strings.TrimPrefix(nextStartAfter, prefix)
+ }
+ return fmt.Errorf("oci storage resume anchor %q was not found in the current listing: %w", anchor, ErrSyncResumeInvalid)
+}
+
+func (c *OCIStorageConnector) sourceDocument(sourceID string, object ociStorageObject) (SourceDocument, bool) {
+ if object.Key == "" || strings.HasSuffix(object.Key, "/") || (!c.allowImages && object.isImage()) {
+ return SourceDocument{}, false
+ }
+ fileName := path.Base(object.Key)
+ fetch := ociStorageFetchReference{Key: object.Key}
+ fetchKey, _ := json.Marshal(fetch)
+ return SourceDocument{
+ SourceID: sourceID,
+ SemanticIdentifier: c.semanticIdentifier(object.Key, fileName),
+ Extension: strings.ToLower(filepath.Ext(fileName)),
+ FetchRef: &FetchReference{Key: string(fetchKey), SizeHint: object.Size},
+ UpdatedAt: object.LastModified,
+ SizeBytes: object.Size,
+ Metadata: map[string]any{
+ "url": ociStorageConsoleURL(c.namespace, c.region, c.bucketName, object.Key),
+ },
+ Fingerprint: normalizedOCIStorageETag(object.ETag),
+ }, true
+}
+
+func (c *OCIStorageConnector) semanticIdentifier(key, fileName string) string {
+ relativePath := key
+ if c.prefix != "" {
+ relativePath = strings.TrimPrefix(key, c.prefix)
+ }
+ if relativePath == "" {
+ return fileName
+ }
+ return strings.ReplaceAll(relativePath, "/", " / ")
+}
+
+func (o ociStorageObject) isImage() bool {
+ switch strings.ToLower(filepath.Ext(o.Key)) {
+ case ".jpg", ".jpeg", ".png", ".webp", ".heic", ".heif":
+ return true
+ default:
+ return false
+ }
+}
+
+type ociStoragePruneSession struct {
+ connector *OCIStorageConnector
+ batchSize int
+ startAfter string
+ done bool
+ buffer []SlimDocument
+}
+
+// NextBatch returns the next OCI Object Storage prune snapshot batch.
+func (s *ociStoragePruneSession) NextBatch(ctx context.Context) (PruneBatch, error) {
+ documents := make([]SlimDocument, 0, s.batchSize)
+ if len(s.buffer) > 0 {
+ n := s.batchSize
+ if n > len(s.buffer) {
+ n = len(s.buffer)
+ }
+ documents = append(documents, s.buffer[:n]...)
+ s.buffer = s.buffer[n:]
+ }
+ for len(documents) < s.batchSize && !s.done {
+ page, nextStartAfter, hasMore, err := s.connector.listObjectPage(ctx, s.startAfter, ociStorageListPageSize)
+ if err != nil {
+ return PruneBatch{}, err
+ }
+ previousStartAfter := s.startAfter
+ if hasMore {
+ next := strings.TrimPrefix(nextStartAfter, ociStorageSourceID(s.connector.bucketName, ""))
+ if next == "" || next == previousStartAfter {
+ s.done = true
+ } else {
+ s.startAfter = next
+ }
+ } else {
+ s.done = true
+ }
+ remaining := s.batchSize - len(documents)
+ for _, object := range page {
+ if object.Key == "" || strings.HasSuffix(object.Key, "/") {
+ continue
+ }
+ doc := SlimDocument{SourceID: ociStorageSourceID(s.connector.bucketName, object.Key)}
+ if remaining > 0 {
+ documents = append(documents, doc)
+ remaining--
+ } else {
+ s.buffer = append(s.buffer, doc)
+ }
+ }
+ if remaining <= 0 {
+ break
+ }
+ }
+ if len(documents) == 0 && len(s.buffer) == 0 && s.done {
+ return PruneBatch{}, io.EOF
+ }
+ return PruneBatch{Documents: documents}, nil
+}
+
+// Close closes the OCI Object Storage prune session.
+func (s *ociStoragePruneSession) Close() error {
+ return nil
+}
+
+type ociStorageFetchReference struct {
+ Key string `json:"key"`
+}
+
+func includeOCIStorageObject(request SyncRequest, sourceID string, object ociStorageObject) bool {
+ if request.FromBeginning {
+ return true
+ }
+ if object.LastModified.IsZero() {
+ return true
+ }
+ if len(request.Fingerprints) > 0 {
+ fingerprint := normalizedOCIStorageETag(object.ETag)
+ stored, ok := request.Fingerprints[sourceID]
+ return fingerprint == "" || !ok || stored == "" || stored != fingerprint
+ }
+ return !beforeOrAtWindowStart(object.LastModified, request.WindowStart) && !afterWindowEnd(object.LastModified, request.WindowEnd)
+}
+
+func normalizeOCIStoragePrefix(prefix string) string {
+ prefix = strings.TrimSpace(prefix)
+ if prefix != "" && !strings.HasSuffix(prefix, "/") {
+ prefix += "/"
+ }
+ return prefix
+}
+
+func validateOCIStorageNamespace(namespace string) error {
+ if !ociStorageHostLabelPattern.MatchString(namespace) {
+ return fmt.Errorf("invalid OCI Storage namespace %q: must be a lowercase DNS label containing only letters, numbers, and hyphens", namespace)
+ }
+ return nil
+}
+
+func validateOCIStorageRegion(region string) error {
+ if !ociStorageHostLabelPattern.MatchString(region) {
+ return fmt.Errorf("invalid OCI Storage region %q: must be a lowercase DNS label containing only letters, numbers, and hyphens", region)
+ }
+ return nil
+}
+
+func ociStorageEndpoint(namespace, region string) string {
+ return fmt.Sprintf("https://%s.compat.objectstorage.%s.oraclecloud.com", namespace, region)
+}
+
+func ociStorageSourceID(bucketName, key string) string {
+ return fmt.Sprintf("%s:%s:%s", ociStorageSource, bucketName, key)
+}
+
+func normalizedOCIStorageETag(rawETag string) string {
+ rawETag = strings.Trim(strings.TrimSpace(rawETag), `"`)
+ if rawETag == "" {
+ return ""
+ }
+ return contentFingerprint([]byte(rawETag))
+}
+
+func ociStorageConsoleURL(namespace, region, bucketName, key string) string {
+ return fmt.Sprintf("https://objectstorage.%s.oraclecloud.com/n/%s/b/%s/o/%s", region, namespace, bucketName, pathEscapeOCIStorageKey(key))
+}
+
+func pathEscapeOCIStorageKey(key string) string {
+ parts := strings.Split(key, "/")
+ for index, part := range parts {
+ parts[index] = url.PathEscape(part)
+ }
+ return strings.Join(parts, "/")
+}
+
+func readOCIBody(body io.Reader, key string, sizeThreshold int64) ([]byte, error) {
+ limited := io.LimitReader(body, sizeThreshold+1)
+ data, err := io.ReadAll(limited)
+ if err != nil {
+ return nil, err
+ }
+ if int64(len(data)) > sizeThreshold {
+ return nil, fmt.Errorf("%s exceeds size threshold of %d", key, sizeThreshold)
+ }
+ return data, nil
+}
diff --git a/internal/syncer/connector/oci_storage_test.go b/internal/syncer/connector/oci_storage_test.go
new file mode 100644
index 0000000000..99aed91553
--- /dev/null
+++ b/internal/syncer/connector/oci_storage_test.go
@@ -0,0 +1,487 @@
+package connector
+
+import (
+ "bytes"
+ "context"
+ "errors"
+ "io"
+ "strings"
+ "testing"
+ "time"
+)
+
+func TestOCIStorageConnectorOpenSyncUsesFingerprintAndFetch(t *testing.T) {
+ old := mustTime(t, "2026-01-01T00:00:00Z")
+ updated := mustTime(t, "2026-01-03T00:00:00Z")
+ connector := newTestOCIStorageConnector(t, []ociStorageObject{
+ {Key: "docs/old.txt", LastModified: old, Size: 8, ETag: `"old-etag"`},
+ {Key: "docs/new.txt", LastModified: updated, Size: 8, ETag: `"new-etag"`},
+ })
+
+ start := mustTime(t, "2026-01-02T00:00:00Z")
+ session, err := connector.OpenSync(context.Background(), SyncRequest{
+ WindowStart: &start,
+ WindowEnd: mustTime(t, "2026-01-04T00:00:00Z"),
+ Fingerprints: map[string]string{
+ ociStorageSourceID("bucket", "docs/old.txt"): normalizedOCIStorageETag(`"old-etag"`),
+ },
+ })
+ if err != nil {
+ t.Fatalf("OpenSync: %v", err)
+ }
+
+ batch, err := session.NextBatch(context.Background())
+ if err != nil {
+ t.Fatalf("NextBatch: %v", err)
+ }
+ if len(batch.Documents) != 1 {
+ t.Fatalf("documents len = %d, want 1", len(batch.Documents))
+ }
+ doc := batch.Documents[0]
+ if doc.SourceID != ociStorageSourceID("bucket", "docs/new.txt") {
+ t.Fatalf("source id = %q", doc.SourceID)
+ }
+ if doc.SemanticIdentifier != "new.txt" {
+ t.Fatalf("semantic identifier = %q", doc.SemanticIdentifier)
+ }
+ if doc.Extension != ".txt" {
+ t.Fatalf("extension = %q", doc.Extension)
+ }
+ if doc.Fingerprint != normalizedOCIStorageETag(`"new-etag"`) {
+ t.Fatalf("fingerprint = %q", doc.Fingerprint)
+ }
+ if doc.FetchRef == nil {
+ t.Fatalf("fetch ref is nil")
+ }
+ fetcher, ok := session.(Fetcher)
+ if !ok {
+ t.Fatalf("session does not implement Fetcher")
+ }
+ blob, err := fetcher.Fetch(context.Background(), *doc.FetchRef)
+ if err != nil {
+ t.Fatalf("Fetch: %v", err)
+ }
+ if string(blob) != "body:new" {
+ t.Fatalf("blob = %q", blob)
+ }
+ if _, err = session.NextBatch(context.Background()); !errors.Is(err, io.EOF) {
+ t.Fatalf("NextBatch EOF = %v", err)
+ }
+}
+
+func TestOCIStorageConnectorOpenSyncDefersListingUntilNextBatch(t *testing.T) {
+ connector := newTestOCIStorageConnector(t, []ociStorageObject{
+ {Key: "docs/a.txt", LastModified: mustTime(t, "2026-01-01T00:00:00Z"), Size: 1, ETag: "a"},
+ })
+ var listCalls int
+ baseListObjects := connector.listObjects
+ connector.listObjects = func(ctx context.Context, startAfter string, maxKeys int32) ([]ociStorageObject, string, bool, error) {
+ listCalls++
+ return baseListObjects(ctx, startAfter, maxKeys)
+ }
+
+ session, err := connector.OpenSync(context.Background(), SyncRequest{FromBeginning: true})
+ if err != nil {
+ t.Fatalf("OpenSync: %v", err)
+ }
+ if listCalls != 0 {
+ t.Fatalf("OpenSync list calls = %d, want 0", listCalls)
+ }
+ if _, err := session.NextBatch(context.Background()); err != nil {
+ t.Fatalf("NextBatch: %v", err)
+ }
+ if listCalls != 1 {
+ t.Fatalf("NextBatch list calls = %d, want 1", listCalls)
+ }
+}
+
+func TestOCIStorageConnectorOpenSyncFiltersImagesUnlessAllowed(t *testing.T) {
+ objects := []ociStorageObject{
+ {Key: "docs/a.png", LastModified: mustTime(t, "2026-01-01T00:00:00Z"), Size: 1, ETag: "a"},
+ {Key: "docs/b.txt", LastModified: mustTime(t, "2026-01-02T00:00:00Z"), Size: 1, ETag: "b"},
+ }
+ connector := newTestOCIStorageConnector(t, objects)
+ session, err := connector.OpenSync(context.Background(), SyncRequest{FromBeginning: true})
+ if err != nil {
+ t.Fatalf("OpenSync: %v", err)
+ }
+ batch, err := session.NextBatch(context.Background())
+ if err != nil {
+ t.Fatalf("NextBatch: %v", err)
+ }
+ if len(batch.Documents) != 1 || batch.Documents[0].SourceID != ociStorageSourceID("bucket", "docs/b.txt") {
+ t.Fatalf("documents = %+v", batch.Documents)
+ }
+
+ allowed := newTestOCIStorageConnector(t, objects)
+ allowed.allowImages = true
+ session, err = allowed.OpenSync(context.Background(), SyncRequest{FromBeginning: true})
+ if err != nil {
+ t.Fatalf("allow images OpenSync: %v", err)
+ }
+ batch, err = session.NextBatch(context.Background())
+ if err != nil {
+ t.Fatalf("allow images NextBatch: %v", err)
+ }
+ if len(batch.Documents) != 2 {
+ t.Fatalf("allow images documents len = %d, want 2", len(batch.Documents))
+ }
+}
+
+func TestOCIStorageConnectorOpenSyncResumesFromCheckpoint(t *testing.T) {
+ connector := newTestOCIStorageConnector(t, []ociStorageObject{
+ {Key: "docs/a.txt", LastModified: mustTime(t, "2026-01-01T00:00:00Z"), Size: 1, ETag: "a"},
+ {Key: "docs/b.txt", LastModified: mustTime(t, "2026-01-02T00:00:00Z"), Size: 1, ETag: "b"},
+ {Key: "docs/c.txt", LastModified: mustTime(t, "2026-01-03T00:00:00Z"), Size: 1, ETag: "c"},
+ })
+
+ checkpointUpdatedAt := mustTime(t, "2026-01-02T00:00:00Z")
+ session, err := connector.OpenSync(context.Background(), SyncRequest{
+ FromBeginning: true,
+ Resume: &SyncCheckpoint{
+ Cursor: ociStorageSourceID("bucket", "docs/b.txt"),
+ SourceID: ociStorageSourceID("bucket", "docs/b.txt"),
+ UpdatedAt: &checkpointUpdatedAt,
+ },
+ })
+ if err != nil {
+ t.Fatalf("OpenSync: %v", err)
+ }
+ batch, err := session.NextBatch(context.Background())
+ if err != nil {
+ t.Fatalf("NextBatch: %v", err)
+ }
+ if len(batch.Documents) != 1 || batch.Documents[0].SourceID != ociStorageSourceID("bucket", "docs/c.txt") {
+ t.Fatalf("documents = %+v", batch.Documents)
+ }
+}
+
+func TestOCIStorageConnectorOpenSyncResumeRejectsInvalidCheckpoints(t *testing.T) {
+ connector := newTestOCIStorageConnector(t, []ociStorageObject{
+ {Key: "docs/a.txt", LastModified: mustTime(t, "2026-01-01T00:00:00Z"), Size: 1, ETag: "a"},
+ {Key: "docs/c.txt", LastModified: mustTime(t, "2026-01-03T00:00:00Z"), Size: 1, ETag: "c"},
+ })
+
+ cases := []*SyncCheckpoint{
+ {
+ Cursor: "other:bucket:docs/b.txt",
+ },
+ {
+ Cursor: ociStorageSourceID("bucket", "docs/b.txt"),
+ },
+ {},
+ }
+ for _, checkpoint := range cases {
+ session, err := connector.OpenSync(context.Background(), SyncRequest{
+ FromBeginning: true,
+ Resume: checkpoint,
+ })
+ if session != nil || err == nil || !errors.Is(err, ErrSyncResumeInvalid) {
+ t.Fatalf("resume OpenSync = session %v, err %v, want ErrSyncResumeInvalid", session, err)
+ }
+ }
+}
+
+func TestOCIStorageConnectorOpenSyncIncludesMissingFingerprint(t *testing.T) {
+ connector := newTestOCIStorageConnector(t, []ociStorageObject{
+ {Key: "docs/no-etag.txt", LastModified: mustTime(t, "2026-01-01T00:00:00Z"), Size: 1},
+ })
+ session, err := connector.OpenSync(context.Background(), SyncRequest{
+ Fingerprints: map[string]string{"other": "fingerprint"},
+ })
+ if err != nil {
+ t.Fatalf("OpenSync: %v", err)
+ }
+ batch, err := session.NextBatch(context.Background())
+ if err != nil {
+ t.Fatalf("NextBatch: %v", err)
+ }
+ if len(batch.Documents) != 1 {
+ t.Fatalf("documents len = %d, want 1", len(batch.Documents))
+ }
+}
+
+func TestOCIStorageConnectorOpenSyncFiltersByWindow(t *testing.T) {
+ connector := newTestOCIStorageConnector(t, []ociStorageObject{
+ {Key: "docs/a.txt", LastModified: mustTime(t, "2026-01-01T00:00:00Z"), Size: 1, ETag: "a"},
+ {Key: "docs/b.txt", LastModified: mustTime(t, "2026-01-03T00:00:00Z"), Size: 1, ETag: "b"},
+ {Key: "docs/c.txt", LastModified: mustTime(t, "2026-01-05T00:00:00Z"), Size: 1, ETag: "c"},
+ })
+ session, err := connector.OpenSync(context.Background(), SyncRequest{
+ WindowStart: mustTimePointer(t, "2026-01-02T00:00:00Z"),
+ WindowEnd: mustTime(t, "2026-01-04T00:00:00Z"),
+ })
+ if err != nil {
+ t.Fatalf("OpenSync: %v", err)
+ }
+ batch, err := session.NextBatch(context.Background())
+ if err != nil {
+ t.Fatalf("NextBatch: %v", err)
+ }
+ if len(batch.Documents) != 1 || batch.Documents[0].SourceID != ociStorageSourceID("bucket", "docs/b.txt") {
+ t.Fatalf("documents = %+v", batch.Documents)
+ }
+}
+
+func TestOCIStorageConnectorOpenPruneReturnsSlimSnapshot(t *testing.T) {
+ connector := newTestOCIStorageConnector(t, []ociStorageObject{
+ {Key: "docs/a.txt", LastModified: mustTime(t, "2026-01-01T00:00:00Z"), Size: 1, ETag: "a"},
+ {Key: "docs/b.txt", LastModified: mustTime(t, "2026-01-02T00:00:00Z"), Size: 1, ETag: "b"},
+ {Key: "docs/folder/", LastModified: mustTime(t, "2026-01-02T00:00:00Z"), Size: 0, ETag: "folder"},
+ })
+ session, err := connector.OpenPrune(context.Background(), PruneRequest{})
+ if err != nil {
+ t.Fatalf("OpenPrune: %v", err)
+ }
+ batch, err := session.NextBatch(context.Background())
+ if err != nil {
+ t.Fatalf("NextBatch: %v", err)
+ }
+ if len(batch.Documents) != 2 {
+ t.Fatalf("documents len = %d, want 2", len(batch.Documents))
+ }
+ if batch.Documents[0].SourceID != ociStorageSourceID("bucket", "docs/a.txt") || batch.Documents[1].SourceID != ociStorageSourceID("bucket", "docs/b.txt") {
+ t.Fatalf("documents = %+v", batch.Documents)
+ }
+}
+
+func TestOCIStorageConnectorOpenPruneStreamsAcrossPages(t *testing.T) {
+ connector := newTestOCIStorageConnector(t, nil)
+ connector.batchSize = 2
+ connector.listObjects = func(ctx context.Context, startAfter string, maxKeys int32) ([]ociStorageObject, string, bool, error) {
+ if startAfter == "" {
+ return []ociStorageObject{
+ {Key: "docs/a.txt", LastModified: mustTime(t, "2026-01-01T00:00:00Z"), Size: 1, ETag: "a"},
+ {Key: "docs/b.txt", LastModified: mustTime(t, "2026-01-02T00:00:00Z"), Size: 1, ETag: "b"},
+ }, ociStorageSourceID("bucket", "docs/b.txt"), true, nil
+ }
+ return []ociStorageObject{
+ {Key: "docs/c.txt", LastModified: mustTime(t, "2026-01-03T00:00:00Z"), Size: 1, ETag: "c"},
+ {Key: "docs/d.txt", LastModified: mustTime(t, "2026-01-04T00:00:00Z"), Size: 1, ETag: "d"},
+ }, "", false, nil
+ }
+
+ session, err := connector.OpenPrune(context.Background(), PruneRequest{})
+ if err != nil {
+ t.Fatalf("OpenPrune: %v", err)
+ }
+ batch, err := session.NextBatch(context.Background())
+ if err != nil {
+ t.Fatalf("NextBatch: %v", err)
+ }
+ if len(batch.Documents) != 2 || batch.Documents[0].SourceID != ociStorageSourceID("bucket", "docs/a.txt") || batch.Documents[1].SourceID != ociStorageSourceID("bucket", "docs/b.txt") {
+ t.Fatalf("first documents = %+v", batch.Documents)
+ }
+ batch, err = session.NextBatch(context.Background())
+ if err != nil {
+ t.Fatalf("second NextBatch: %v", err)
+ }
+ if len(batch.Documents) != 2 || batch.Documents[0].SourceID != ociStorageSourceID("bucket", "docs/c.txt") || batch.Documents[1].SourceID != ociStorageSourceID("bucket", "docs/d.txt") {
+ t.Fatalf("second documents = %+v", batch.Documents)
+ }
+ if _, err = session.NextBatch(context.Background()); !errors.Is(err, io.EOF) {
+ t.Fatalf("NextBatch EOF = %v", err)
+ }
+}
+
+func TestOCIStorageConnectorValidate(t *testing.T) {
+ ctx := context.Background()
+
+ missingBucket := &OCIStorageConnector{namespace: "namespace", region: "region", accessKeyID: "access", secretKey: "secret", batchSize: 2}
+ if err := missingBucket.Validate(ctx); err == nil {
+ t.Fatalf("expected bucket name validation error")
+ }
+
+ missingCredentials := &OCIStorageConnector{bucketName: "bucket", batchSize: 2}
+ if err := missingCredentials.Validate(ctx); err == nil {
+ t.Fatalf("expected credentials validation error")
+ }
+
+ badBatch := &OCIStorageConnector{bucketName: "bucket", namespace: "namespace", region: "region", accessKeyID: "access", secretKey: "secret", batchSize: 0}
+ if err := badBatch.Validate(ctx); err == nil {
+ t.Fatalf("expected batch size validation error")
+ }
+
+ badRegion := &OCIStorageConnector{bucketName: "bucket", namespace: "namespace", region: "evil.example#", accessKeyID: "access", secretKey: "secret", batchSize: 2}
+ if err := badRegion.Validate(ctx); err == nil {
+ t.Fatalf("expected region validation error")
+ }
+}
+
+func TestOCIStorageConnectorRejectsInvalidNamespaceBeforeClientCreation(t *testing.T) {
+ for _, namespace := range []string{
+ "namespace.example.com",
+ "https://evil.example.org",
+ "evil.example.org:443",
+ "namespace/tenant",
+ "namespace space",
+ "-namespace",
+ "namespace-",
+ "Namespace",
+ } {
+ connector := newTestOCIStorageConnector(t, nil)
+ connector.namespace = namespace
+ connector.client = nil
+ if _, err := connector.ensureClient(context.Background()); err == nil {
+ t.Fatalf("expected invalid namespace %q to be rejected", namespace)
+ }
+ if connector.client != nil {
+ t.Fatalf("client was created for invalid namespace %q", namespace)
+ }
+ }
+}
+
+func TestOCIStorageConnectorRejectsInvalidRegionBeforeClientCreation(t *testing.T) {
+ for _, region := range []string{
+ "evil.example#",
+ "evil.example.com",
+ "https://evil.example.org",
+ "region/tenant",
+ "region space",
+ "-region",
+ "region-",
+ "Region",
+ } {
+ connector := newTestOCIStorageConnector(t, nil)
+ connector.region = region
+ connector.client = nil
+ if _, err := connector.ensureClient(context.Background()); err == nil {
+ t.Fatalf("expected invalid region %q to be rejected", region)
+ }
+ if connector.client != nil {
+ t.Fatalf("client was created for invalid region %q", region)
+ }
+ }
+}
+
+func TestOCIStorageConnectorValidateSetting(t *testing.T) {
+ connector := newTestOCIStorageConnector(t, []ociStorageObject{
+ {Key: "docs/a.txt", LastModified: mustTime(t, "2026-01-01T00:00:00Z"), Size: 1, ETag: "a"},
+ })
+ var listCalls int
+ connector.listObjects = func(ctx context.Context, startAfter string, maxKeys int32) ([]ociStorageObject, string, bool, error) {
+ listCalls++
+ return nil, "", false, nil
+ }
+ if err := connector.ValidateConnectorSetting(context.Background(), nil); err != nil {
+ t.Fatalf("ValidateConnectorSetting: %v", err)
+ }
+ if listCalls != 1 {
+ t.Fatalf("list calls = %d, want 1", listCalls)
+ }
+}
+
+func TestNewOCIStorageConnectorDefaultsAndConfig(t *testing.T) {
+ connector, err := NewOCIStorageConnector(map[string]any{
+ "bucket_name": " bucket ",
+ "prefix": "docs",
+ "batch_size": 4,
+ "size_threshold": 42,
+ "allow_images": true,
+ "credentials": map[string]any{
+ "namespace": "mynamespace",
+ "region": "us-ashburn-1",
+ "access_key_id": "access",
+ "secret_access_key": "secret",
+ },
+ })
+ if err != nil {
+ t.Fatalf("NewOCIStorageConnector: %v", err)
+ }
+ if connector.bucketName != "bucket" {
+ t.Fatalf("bucket name = %q", connector.bucketName)
+ }
+ if connector.prefix != "docs/" {
+ t.Fatalf("prefix = %q", connector.prefix)
+ }
+ if connector.batchSize != 4 {
+ t.Fatalf("batch size = %d, want 4", connector.batchSize)
+ }
+ if connector.sizeThreshold != 42 {
+ t.Fatalf("size threshold = %d, want 42", connector.sizeThreshold)
+ }
+ if !connector.allowImages {
+ t.Fatalf("allow images = false, want true")
+ }
+ if connector.namespace != "mynamespace" || connector.region != "us-ashburn-1" || connector.accessKeyID != "access" || connector.secretKey != "secret" {
+ t.Fatalf("credentials = %+v", connector)
+ }
+
+ defaults, err := NewOCIStorageConnector(map[string]any{
+ "bucket_name": "bucket",
+ "credentials": map[string]any{},
+ })
+ if err != nil {
+ t.Fatalf("NewOCIStorageConnector defaults: %v", err)
+ }
+ if defaults.batchSize != defaultOCIStorageBatchSize || defaults.sizeThreshold != defaultOCIStorageSizeThreshold {
+ t.Fatalf("defaults = batch %d threshold %d", defaults.batchSize, defaults.sizeThreshold)
+ }
+}
+
+func TestOCIStorageHelpers(t *testing.T) {
+ if got := ociStorageEndpoint("mynamespace", "us-ashburn-1"); got != "https://mynamespace.compat.objectstorage.us-ashburn-1.oraclecloud.com" {
+ t.Fatalf("endpoint = %q", got)
+ }
+ if got := ociStorageConsoleURL("mynamespace", "us-ashburn-1", "bucket", "docs/a b.txt"); !strings.Contains(got, "https://objectstorage.us-ashburn-1.oraclecloud.com/n/mynamespace/b/bucket/o/docs/a%20b.txt") {
+ t.Fatalf("console url = %q", got)
+ }
+ if _, err := readOCIBody(bytes.NewBufferString("12345"), "large.txt", 4); err == nil {
+ t.Fatalf("expected size threshold error")
+ }
+}
+
+func newTestOCIStorageConnector(t *testing.T, objects []ociStorageObject) *OCIStorageConnector {
+ t.Helper()
+ connector, err := NewOCIStorageConnector(map[string]any{
+ "bucket_name": "bucket",
+ "prefix": "docs",
+ "batch_size": 2,
+ "credentials": map[string]any{
+ "namespace": "mynamespace",
+ "region": "us-ashburn-1",
+ "access_key_id": "access",
+ "secret_access_key": "secret",
+ },
+ })
+ if err != nil {
+ t.Fatalf("NewOCIStorageConnector: %v", err)
+ }
+ connector.listObjects = func(ctx context.Context, startAfter string, maxKeys int32) ([]ociStorageObject, string, bool, error) {
+ var out []ociStorageObject
+ for _, object := range objects {
+ if startAfter != "" && object.Key <= startAfter {
+ continue
+ }
+ out = append(out, object)
+ if maxKeys > 0 && len(out) >= int(maxKeys) {
+ break
+ }
+ }
+ hasMore := false
+ nextStartAfter := ""
+ if len(out) > 0 {
+ nextStartAfter = ociStorageSourceID(connector.bucketName, out[len(out)-1].Key)
+ for _, object := range objects {
+ if object.Key > out[len(out)-1].Key {
+ hasMore = true
+ break
+ }
+ }
+ }
+ return out, nextStartAfter, hasMore, nil
+ }
+ connector.downloadObject = func(ctx context.Context, key string, sizeThreshold int64) ([]byte, error) {
+ if key == "docs/new.txt" {
+ return []byte("body:new"), nil
+ }
+ return []byte("body:" + key), nil
+ }
+ return connector
+}
+
+func mustTimePointer(t *testing.T, value string) *time.Time {
+ t.Helper()
+ parsed := mustTime(t, value)
+ return &parsed
+}