RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
feat(go-syncer): add OCI Storage connector with resume (#18850)
mkaaad41bc87243302
diff --git a/internal/syncer/connector/builtin.go b/internal/syncer/connector/builtin.goindex 9290d2ebcc..4e4efea62a 100644--- a/internal/syncer/connector/builtin.go+++ b/internal/syncer/connector/builtin.go@@ -38,6 +38,7 @@ func RegisterBuiltIns(registry *Registry) {registerBuiltIn(registry, "google-drive", NewGoogleDriveConnector)registerBuiltIn(registry, "google_drive", NewGoogleDriveConnector)registerBuiltIn(registry, "google_cloud_storage", NewGoogleCloudStorageConnector)+ registerBuiltIn(registry, "oci_storage", NewOCIStorageConnector)registerBuiltIn(registry, "azure_blob", NewAzureBlobStorageConnector)registerBuiltIn(registry, "r2", NewR2Connector)registerBuiltIn(registry, "dingtalk_ai_table", NewDingTalkAITableConnector)diff --git a/internal/syncer/connector/oci_storage.go b/internal/syncer/connector/oci_storage.gonew file mode 100644index 0000000000..b37288d8ee--- /dev/null+++ b/internal/syncer/connector/oci_storage.go@@ -0,0 +1,544 @@+//+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.+//++package connector++import (+ "context"+ "encoding/json"+ "fmt"+ "io"+ "net/url"+ "path"+ "path/filepath"+ "regexp"+ "strings"+ "time"++ "github.com/aws/aws-sdk-go-v2/aws"+ awssdkconfig "github.com/aws/aws-sdk-go-v2/config"+ "github.com/aws/aws-sdk-go-v2/credentials"+ "github.com/aws/aws-sdk-go-v2/service/s3"+ "github.com/aws/aws-sdk-go-v2/service/s3/types"+)++const (+ defaultOCIStorageBatchSize = 2+ defaultOCIStorageSizeThreshold = 20 * 1024 * 1024+ ociStorageListPageSize = 1000+ ociStorageSource = "oci_storage"+)++var ociStorageHostLabelPattern = regexp.MustCompile(`^[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?$`)++// OCIStorageConnector reads objects from Oracle Cloud Infrastructure Object+// Storage through its S3-compatible API.+type OCIStorageConnector struct {+ bucketName string+ prefix string+ namespace string+ region string+ accessKeyID string+ secretKey string+ allowImages bool+ batchSize int+ sizeThreshold int64++ client *s3.Client+ listObjects func(ctx context.Context, startAfter string, maxKeys int32) ([]ociStorageObject, string, bool, error)+ downloadObject func(ctx context.Context, key string, sizeThreshold int64) ([]byte, error)+}++// NewOCIStorageConnector creates an OCI Object Storage connector from+// Python-compatible config.+func NewOCIStorageConnector(config map[string]any) (*OCIStorageConnector, error) {+ credentials := configAnyMap(config["credentials"])+ batchSize := configInt(firstNonEmpty(stringConfig(config["sync_batch_size"]), stringConfig(config["batch_size"])), defaultOCIStorageBatchSize)+ sizeThreshold := int64(configInt(config["size_threshold"], defaultOCIStorageSizeThreshold))+ if sizeThreshold <= 0 {+ sizeThreshold = defaultOCIStorageSizeThreshold+ }+ return &OCIStorageConnector{+ bucketName: strings.TrimSpace(stringConfig(config["bucket_name"])),+ prefix: normalizeOCIStoragePrefix(stringConfig(config["prefix"])),+ namespace: strings.TrimSpace(stringConfig(credentials["namespace"])),+ region: strings.TrimSpace(stringConfig(credentials["region"])),+ accessKeyID: strings.TrimSpace(stringConfig(credentials["access_key_id"])),+ secretKey: stringConfig(credentials["secret_access_key"]),+ allowImages: configBoolDefault(config["allow_images"], false),+ batchSize: batchSize,+ sizeThreshold: sizeThreshold,+ }, nil+}++// Validate validates OCI Object Storage connector settings and credentials.+func (c *OCIStorageConnector) Validate(ctx context.Context) error {+ if c == nil {+ return fmt.Errorf("OCI Storage connector is nil")+ }+ if c.bucketName == "" {+ return fmt.Errorf("No bucket name was provided in connector settings.")+ }+ if c.namespace == "" || c.region == "" || c.accessKeyID == "" || c.secretKey == "" {+ return fmt.Errorf("OCI Storage credentials are required")+ }+ if err := validateOCIStorageNamespace(c.namespace); err != nil {+ return err+ }+ if err := validateOCIStorageRegion(c.region); err != nil {+ return err+ }+ if c.batchSize <= 0 {+ return fmt.Errorf("batch_size must be a positive integer")+ }+ if _, err := c.ensureClient(ctx); err != nil {+ return err+ }+ return nil+}++// ValidateConnectorSetting validates OCI Object Storage settings from an+// unsaved config.+func (c *OCIStorageConnector) ValidateConnectorSetting(ctx context.Context, request map[string]any) error {+ ctx, cancel := context.WithTimeout(ctx, connectorSettingValidationTimeout)+ defer cancel()+ if err := c.Validate(ctx); err != nil {+ return err+ }+ _, _, _, err := c.listObjectPage(ctx, "", 1)+ if err != nil {+ return fmt.Errorf("access OCI bucket %q: %w", c.bucketName, err)+ }+ return nil+}++// OpenSync opens one OCI Object Storage sync session.+func (c *OCIStorageConnector) OpenSync(ctx context.Context, request SyncRequest) (SyncSession, error) {+ if err := c.Validate(ctx); err != nil {+ return nil, err+ }+ session := &ociStorageSyncSession{+ connector: c,+ request: request,+ batchSize: c.batchSize,+ }+ if request.Resume != nil {+ if err := session.applyResume(ctx, request.Resume); err != nil {+ return nil, err+ }+ }+ return session, nil+}++// OpenPrune opens one complete OCI Object Storage prune snapshot session.+func (c *OCIStorageConnector) OpenPrune(ctx context.Context, request PruneRequest) (PruneSession, error) {+ if err := c.Validate(ctx); err != nil {+ return nil, err+ }+ return &ociStoragePruneSession{connector: c, batchSize: c.batchSize}, nil+}++// Fetch downloads an OCI Object Storage object body.+func (c *OCIStorageConnector) Fetch(ctx context.Context, ref FetchReference) ([]byte, error) {+ var fetch ociStorageFetchReference+ if err := json.Unmarshal([]byte(ref.Key), &fetch); err != nil {+ return nil, err+ }+ return c.download(ctx, fetch.Key)+}++func (c *OCIStorageConnector) ensureClient(ctx context.Context) (*s3.Client, error) {+ if err := validateOCIStorageNamespace(c.namespace); err != nil {+ return nil, err+ }+ if err := validateOCIStorageRegion(c.region); err != nil {+ return nil, err+ }+ if c.client != nil {+ return c.client, nil+ }+ cfg, err := awssdkconfig.LoadDefaultConfig(+ ctx,+ awssdkconfig.WithRegion(c.region),+ awssdkconfig.WithCredentialsProvider(credentials.NewStaticCredentialsProvider(c.accessKeyID, c.secretKey, "")),+ )+ if err != nil {+ return nil, fmt.Errorf("load OCI Storage config: %w", err)+ }+ c.client = s3.NewFromConfig(cfg, func(options *s3.Options) {+ options.BaseEndpoint = aws.String(ociStorageEndpoint(c.namespace, c.region))+ options.UsePathStyle = true+ })+ return c.client, nil+}++func (c *OCIStorageConnector) listObjectPage(ctx context.Context, startAfter string, maxKeys int32) ([]ociStorageObject, string, bool, error) {+ if c.listObjects != nil {+ return c.listObjects(ctx, startAfter, maxKeys)+ }+ client, err := c.ensureClient(ctx)+ if err != nil {+ return nil, "", false, err+ }+ input := &s3.ListObjectsV2Input{+ Bucket: aws.String(c.bucketName),+ Prefix: aws.String(c.prefix),+ StartAfter: aws.String(startAfter),+ }+ if maxKeys > 0 {+ input.MaxKeys = aws.Int32(maxKeys)+ }+ output, err := client.ListObjectsV2(ctx, input)+ if err != nil {+ return nil, "", false, err+ }+ objects := make([]ociStorageObject, 0, len(output.Contents))+ for _, object := range output.Contents {+ objects = append(objects, ociStorageObjectFromS3(object))+ }+ nextStartAfter := ""+ if len(objects) > 0 {+ nextStartAfter = ociStorageSourceID(c.bucketName, objects[len(objects)-1].Key)+ }+ return objects, nextStartAfter, aws.ToBool(output.IsTruncated), nil+}++func (c *OCIStorageConnector) download(ctx context.Context, key string) ([]byte, error) {+ if c.downloadObject != nil {+ return c.downloadObject(ctx, key, c.sizeThreshold)+ }+ client, err := c.ensureClient(ctx)+ if err != nil {+ return nil, err+ }+ output, err := client.GetObject(ctx, &s3.GetObjectInput{+ Bucket: aws.String(c.bucketName),+ Key: aws.String(key),+ })+ if err != nil {+ return nil, err+ }+ defer output.Body.Close()+ return readOCIBody(output.Body, key, c.sizeThreshold)+}++type ociStorageObject struct {+ Key string+ LastModified time.Time+ Size int64+ ETag string+}++func ociStorageObjectFromS3(object types.Object) ociStorageObject {+ updatedAt := time.Time{}+ if object.LastModified != nil {+ updatedAt = object.LastModified.UTC()+ }+ return ociStorageObject{+ Key: aws.ToString(object.Key),+ LastModified: updatedAt,+ Size: aws.ToInt64(object.Size),+ ETag: aws.ToString(object.ETag),+ }+}++type ociStorageSyncSession struct {+ connector *OCIStorageConnector+ request SyncRequest+ batchSize int+ startAfter string+ done bool+}++// NextBatch returns the next OCI Object Storage document batch.+func (s *ociStorageSyncSession) NextBatch(ctx context.Context) (SyncBatch, error) {+ for {+ if s.done {+ return SyncBatch{}, io.EOF+ }+ previousStartAfter := s.startAfter+ objects, nextStartAfter, hasMore, err := s.connector.listObjectPage(ctx, s.startAfter, int32(s.batchSize))+ if err != nil {+ return SyncBatch{}, err+ }+ if !hasMore {+ s.done = true+ }+ if nextStartAfter != "" {+ s.startAfter = strings.TrimPrefix(nextStartAfter, ociStorageSourceID(s.connector.bucketName, ""))+ }+ if hasMore && s.startAfter == previousStartAfter {+ return SyncBatch{}, fmt.Errorf("OCI Storage listing did not advance from %q", previousStartAfter)+ }++ documents := make([]SourceDocument, 0, len(objects))+ for _, object := range objects {+ sourceID := ociStorageSourceID(s.connector.bucketName, object.Key)+ if !includeOCIStorageObject(s.request, sourceID, object) {+ continue+ }+ document, ok := s.connector.sourceDocument(sourceID, object)+ if ok {+ documents = append(documents, document)+ }+ }+ if len(documents) == 0 {+ continue+ }+ last := documents[len(documents)-1]+ updatedAt := last.UpdatedAt+ return SyncBatch{+ Documents: documents,+ Checkpoint: &SyncCheckpoint{+ Cursor: last.SourceID,+ SourceID: last.SourceID,+ UpdatedAt: &updatedAt,+ },+ }, nil+ }+}++// Close closes the OCI Object Storage sync session.+func (s *ociStorageSyncSession) Close() error {+ return nil+}++// Fetch downloads a delayed OCI Object Storage document body.+func (s *ociStorageSyncSession) Fetch(ctx context.Context, ref FetchReference) ([]byte, error) {+ return s.connector.Fetch(ctx, ref)+}++func (s *ociStorageSyncSession) applyResume(ctx context.Context, checkpoint *SyncCheckpoint) error {+ if checkpoint == nil {+ return nil+ }+ sourceID := firstNonEmpty(checkpoint.SourceID, checkpoint.Cursor)+ prefix := ociStorageSourceID(s.connector.bucketName, "")+ if sourceID == "" || !strings.HasPrefix(sourceID, prefix) {+ return fmt.Errorf("oci storage sync checkpoint has no source anchor: %w", ErrSyncResumeInvalid)+ }+ anchor := strings.TrimPrefix(sourceID, prefix)+ if anchor == "" {+ return fmt.Errorf("oci storage sync checkpoint has no source anchor: %w", ErrSyncResumeInvalid)+ }++ startAfter := ""+ for {+ objects, nextStartAfter, hasMore, err := s.connector.listObjectPage(ctx, startAfter, int32(s.batchSize))+ if err != nil {+ return err+ }+ for _, object := range objects {+ if object.Key == anchor {+ s.startAfter = anchor+ return nil+ }+ if object.Key > anchor {+ return fmt.Errorf("oci storage resume anchor %q was not found in the current listing: %w", anchor, ErrSyncResumeInvalid)+ }+ }+ if !hasMore {+ break+ }+ startAfter = strings.TrimPrefix(nextStartAfter, prefix)+ }+ return fmt.Errorf("oci storage resume anchor %q was not found in the current listing: %w", anchor, ErrSyncResumeInvalid)+}++func (c *OCIStorageConnector) sourceDocument(sourceID string, object ociStorageObject) (SourceDocument, bool) {+ if object.Key == "" || strings.HasSuffix(object.Key, "/") || (!c.allowImages && object.isImage()) {+ return SourceDocument{}, false+ }+ fileName := path.Base(object.Key)+ fetch := ociStorageFetchReference{Key: object.Key}+ fetchKey, _ := json.Marshal(fetch)+ return SourceDocument{+ SourceID: sourceID,+ SemanticIdentifier: c.semanticIdentifier(object.Key, fileName),+ Extension: strings.ToLower(filepath.Ext(fileName)),+ FetchRef: &FetchReference{Key: string(fetchKey), SizeHint: object.Size},+ UpdatedAt: object.LastModified,+ SizeBytes: object.Size,+ Metadata: map[string]any{+ "url": ociStorageConsoleURL(c.namespace, c.region, c.bucketName, object.Key),+ },+ Fingerprint: normalizedOCIStorageETag(object.ETag),+ }, true+}++func (c *OCIStorageConnector) semanticIdentifier(key, fileName string) string {+ relativePath := key+ if c.prefix != "" {+ relativePath = strings.TrimPrefix(key, c.prefix)+ }+ if relativePath == "" {+ return fileName+ }+ return strings.ReplaceAll(relativePath, "/", " / ")+}++func (o ociStorageObject) isImage() bool {+ switch strings.ToLower(filepath.Ext(o.Key)) {+ case ".jpg", ".jpeg", ".png", ".webp", ".heic", ".heif":+ return true+ default:+ return false+ }+}++type ociStoragePruneSession struct {+ connector *OCIStorageConnector+ batchSize int+ startAfter string+ done bool+ buffer []SlimDocument+}++// NextBatch returns the next OCI Object Storage prune snapshot batch.+func (s *ociStoragePruneSession) NextBatch(ctx context.Context) (PruneBatch, error) {+ documents := make([]SlimDocument, 0, s.batchSize)+ if len(s.buffer) > 0 {+ n := s.batchSize+ if n > len(s.buffer) {+ n = len(s.buffer)+ }+ documents = append(documents, s.buffer[:n]...)+ s.buffer = s.buffer[n:]+ }+ for len(documents) < s.batchSize && !s.done {+ page, nextStartAfter, hasMore, err := s.connector.listObjectPage(ctx, s.startAfter, ociStorageListPageSize)+ if err != nil {+ return PruneBatch{}, err+ }+ previousStartAfter := s.startAfter+ if hasMore {+ next := strings.TrimPrefix(nextStartAfter, ociStorageSourceID(s.connector.bucketName, ""))+ if next == "" || next == previousStartAfter {+ s.done = true+ } else {+ s.startAfter = next+ }+ } else {+ s.done = true+ }+ remaining := s.batchSize - len(documents)+ for _, object := range page {+ if object.Key == "" || strings.HasSuffix(object.Key, "/") {+ continue+ }+ doc := SlimDocument{SourceID: ociStorageSourceID(s.connector.bucketName, object.Key)}+ if remaining > 0 {+ documents = append(documents, doc)+ remaining--+ } else {+ s.buffer = append(s.buffer, doc)+ }+ }+ if remaining <= 0 {+ break+ }+ }+ if len(documents) == 0 && len(s.buffer) == 0 && s.done {+ return PruneBatch{}, io.EOF+ }+ return PruneBatch{Documents: documents}, nil+}++// Close closes the OCI Object Storage prune session.+func (s *ociStoragePruneSession) Close() error {+ return nil+}++type ociStorageFetchReference struct {+ Key string `json:"key"`+}++func includeOCIStorageObject(request SyncRequest, sourceID string, object ociStorageObject) bool {+ if request.FromBeginning {+ return true+ }+ if object.LastModified.IsZero() {+ return true+ }+ if len(request.Fingerprints) > 0 {+ fingerprint := normalizedOCIStorageETag(object.ETag)+ stored, ok := request.Fingerprints[sourceID]+ return fingerprint == "" || !ok || stored == "" || stored != fingerprint+ }+ return !beforeOrAtWindowStart(object.LastModified, request.WindowStart) && !afterWindowEnd(object.LastModified, request.WindowEnd)+}++func normalizeOCIStoragePrefix(prefix string) string {+ prefix = strings.TrimSpace(prefix)+ if prefix != "" && !strings.HasSuffix(prefix, "/") {+ prefix += "/"+ }+ return prefix+}++func validateOCIStorageNamespace(namespace string) error {+ if !ociStorageHostLabelPattern.MatchString(namespace) {+ return fmt.Errorf("invalid OCI Storage namespace %q: must be a lowercase DNS label containing only letters, numbers, and hyphens", namespace)+ }+ return nil+}++func validateOCIStorageRegion(region string) error {+ if !ociStorageHostLabelPattern.MatchString(region) {+ return fmt.Errorf("invalid OCI Storage region %q: must be a lowercase DNS label containing only letters, numbers, and hyphens", region)+ }+ return nil+}++func ociStorageEndpoint(namespace, region string) string {+ return fmt.Sprintf("https://%s.compat.objectstorage.%s.oraclecloud.com", namespace, region)+}++func ociStorageSourceID(bucketName, key string) string {+ return fmt.Sprintf("%s:%s:%s", ociStorageSource, bucketName, key)+}++func normalizedOCIStorageETag(rawETag string) string {+ rawETag = strings.Trim(strings.TrimSpace(rawETag), `"`)+ if rawETag == "" {+ return ""+ }+ return contentFingerprint([]byte(rawETag))+}++func ociStorageConsoleURL(namespace, region, bucketName, key string) string {+ return fmt.Sprintf("https://objectstorage.%s.oraclecloud.com/n/%s/b/%s/o/%s", region, namespace, bucketName, pathEscapeOCIStorageKey(key))+}++func pathEscapeOCIStorageKey(key string) string {+ parts := strings.Split(key, "/")+ for index, part := range parts {+ parts[index] = url.PathEscape(part)+ }+ return strings.Join(parts, "/")+}++func readOCIBody(body io.Reader, key string, sizeThreshold int64) ([]byte, error) {+ limited := io.LimitReader(body, sizeThreshold+1)+ data, err := io.ReadAll(limited)+ if err != nil {+ return nil, err+ }+ if int64(len(data)) > sizeThreshold {+ return nil, fmt.Errorf("%s exceeds size threshold of %d", key, sizeThreshold)+ }+ return data, nil+}diff --git a/internal/syncer/connector/oci_storage_test.go b/internal/syncer/connector/oci_storage_test.gonew file mode 100644index 0000000000..99aed91553--- /dev/null+++ b/internal/syncer/connector/oci_storage_test.go@@ -0,0 +1,487 @@+package connector++import (+ "bytes"+ "context"+ "errors"+ "io"+ "strings"+ "testing"+ "time"+)++func TestOCIStorageConnectorOpenSyncUsesFingerprintAndFetch(t *testing.T) {+ old := mustTime(t, "2026-01-01T00:00:00Z")+ updated := mustTime(t, "2026-01-03T00:00:00Z")+ connector := newTestOCIStorageConnector(t, []ociStorageObject{+ {Key: "docs/old.txt", LastModified: old, Size: 8, ETag: `"old-etag"`},+ {Key: "docs/new.txt", LastModified: updated, Size: 8, ETag: `"new-etag"`},+ })++ start := mustTime(t, "2026-01-02T00:00:00Z")+ session, err := connector.OpenSync(context.Background(), SyncRequest{+ WindowStart: &start,+ WindowEnd: mustTime(t, "2026-01-04T00:00:00Z"),+ Fingerprints: map[string]string{+ ociStorageSourceID("bucket", "docs/old.txt"): normalizedOCIStorageETag(`"old-etag"`),+ },+ })+ if err != nil {+ t.Fatalf("OpenSync: %v", err)+ }++ batch, err := session.NextBatch(context.Background())+ if err != nil {+ t.Fatalf("NextBatch: %v", err)+ }+ if len(batch.Documents) != 1 {+ t.Fatalf("documents len = %d, want 1", len(batch.Documents))+ }+ doc := batch.Documents[0]+ if doc.SourceID != ociStorageSourceID("bucket", "docs/new.txt") {+ t.Fatalf("source id = %q", doc.SourceID)+ }+ if doc.SemanticIdentifier != "new.txt" {+ t.Fatalf("semantic identifier = %q", doc.SemanticIdentifier)+ }+ if doc.Extension != ".txt" {+ t.Fatalf("extension = %q", doc.Extension)+ }+ if doc.Fingerprint != normalizedOCIStorageETag(`"new-etag"`) {+ t.Fatalf("fingerprint = %q", doc.Fingerprint)+ }+ if doc.FetchRef == nil {+ t.Fatalf("fetch ref is nil")+ }+ fetcher, ok := session.(Fetcher)+ if !ok {+ t.Fatalf("session does not implement Fetcher")+ }+ blob, err := fetcher.Fetch(context.Background(), *doc.FetchRef)+ if err != nil {+ t.Fatalf("Fetch: %v", err)+ }+ if string(blob) != "body:new" {+ t.Fatalf("blob = %q", blob)+ }+ if _, err = session.NextBatch(context.Background()); !errors.Is(err, io.EOF) {+ t.Fatalf("NextBatch EOF = %v", err)+ }+}++func TestOCIStorageConnectorOpenSyncDefersListingUntilNextBatch(t *testing.T) {+ connector := newTestOCIStorageConnector(t, []ociStorageObject{+ {Key: "docs/a.txt", LastModified: mustTime(t, "2026-01-01T00:00:00Z"), Size: 1, ETag: "a"},+ })+ var listCalls int+ baseListObjects := connector.listObjects+ connector.listObjects = func(ctx context.Context, startAfter string, maxKeys int32) ([]ociStorageObject, string, bool, error) {+ listCalls+++ return baseListObjects(ctx, startAfter, maxKeys)+ }++ session, err := connector.OpenSync(context.Background(), SyncRequest{FromBeginning: true})+ if err != nil {+ t.Fatalf("OpenSync: %v", err)+ }+ if listCalls != 0 {+ t.Fatalf("OpenSync list calls = %d, want 0", listCalls)+ }+ if _, err := session.NextBatch(context.Background()); err != nil {+ t.Fatalf("NextBatch: %v", err)+ }+ if listCalls != 1 {+ t.Fatalf("NextBatch list calls = %d, want 1", listCalls)+ }+}++func TestOCIStorageConnectorOpenSyncFiltersImagesUnlessAllowed(t *testing.T) {+ objects := []ociStorageObject{+ {Key: "docs/a.png", LastModified: mustTime(t, "2026-01-01T00:00:00Z"), Size: 1, ETag: "a"},+ {Key: "docs/b.txt", LastModified: mustTime(t, "2026-01-02T00:00:00Z"), Size: 1, ETag: "b"},+ }+ connector := newTestOCIStorageConnector(t, objects)+ session, err := connector.OpenSync(context.Background(), SyncRequest{FromBeginning: true})+ if err != nil {+ t.Fatalf("OpenSync: %v", err)+ }+ batch, err := session.NextBatch(context.Background())+ if err != nil {+ t.Fatalf("NextBatch: %v", err)+ }+ if len(batch.Documents) != 1 || batch.Documents[0].SourceID != ociStorageSourceID("bucket", "docs/b.txt") {+ t.Fatalf("documents = %+v", batch.Documents)+ }++ allowed := newTestOCIStorageConnector(t, objects)+ allowed.allowImages = true+ session, err = allowed.OpenSync(context.Background(), SyncRequest{FromBeginning: true})+ if err != nil {+ t.Fatalf("allow images OpenSync: %v", err)+ }+ batch, err = session.NextBatch(context.Background())+ if err != nil {+ t.Fatalf("allow images NextBatch: %v", err)+ }+ if len(batch.Documents) != 2 {+ t.Fatalf("allow images documents len = %d, want 2", len(batch.Documents))+ }+}++func TestOCIStorageConnectorOpenSyncResumesFromCheckpoint(t *testing.T) {+ connector := newTestOCIStorageConnector(t, []ociStorageObject{+ {Key: "docs/a.txt", LastModified: mustTime(t, "2026-01-01T00:00:00Z"), Size: 1, ETag: "a"},+ {Key: "docs/b.txt", LastModified: mustTime(t, "2026-01-02T00:00:00Z"), Size: 1, ETag: "b"},+ {Key: "docs/c.txt", LastModified: mustTime(t, "2026-01-03T00:00:00Z"), Size: 1, ETag: "c"},+ })++ checkpointUpdatedAt := mustTime(t, "2026-01-02T00:00:00Z")+ session, err := connector.OpenSync(context.Background(), SyncRequest{+ FromBeginning: true,+ Resume: &SyncCheckpoint{+ Cursor: ociStorageSourceID("bucket", "docs/b.txt"),+ SourceID: ociStorageSourceID("bucket", "docs/b.txt"),+ UpdatedAt: &checkpointUpdatedAt,+ },+ })+ if err != nil {+ t.Fatalf("OpenSync: %v", err)+ }+ batch, err := session.NextBatch(context.Background())+ if err != nil {+ t.Fatalf("NextBatch: %v", err)+ }+ if len(batch.Documents) != 1 || batch.Documents[0].SourceID != ociStorageSourceID("bucket", "docs/c.txt") {+ t.Fatalf("documents = %+v", batch.Documents)+ }+}++func TestOCIStorageConnectorOpenSyncResumeRejectsInvalidCheckpoints(t *testing.T) {+ connector := newTestOCIStorageConnector(t, []ociStorageObject{+ {Key: "docs/a.txt", LastModified: mustTime(t, "2026-01-01T00:00:00Z"), Size: 1, ETag: "a"},+ {Key: "docs/c.txt", LastModified: mustTime(t, "2026-01-03T00:00:00Z"), Size: 1, ETag: "c"},+ })++ cases := []*SyncCheckpoint{+ {+ Cursor: "other:bucket:docs/b.txt",+ },+ {+ Cursor: ociStorageSourceID("bucket", "docs/b.txt"),+ },+ {},+ }+ for _, checkpoint := range cases {+ session, err := connector.OpenSync(context.Background(), SyncRequest{+ FromBeginning: true,+ Resume: checkpoint,+ })+ if session != nil || err == nil || !errors.Is(err, ErrSyncResumeInvalid) {+ t.Fatalf("resume OpenSync = session %v, err %v, want ErrSyncResumeInvalid", session, err)+ }+ }+}++func TestOCIStorageConnectorOpenSyncIncludesMissingFingerprint(t *testing.T) {+ connector := newTestOCIStorageConnector(t, []ociStorageObject{+ {Key: "docs/no-etag.txt", LastModified: mustTime(t, "2026-01-01T00:00:00Z"), Size: 1},+ })+ session, err := connector.OpenSync(context.Background(), SyncRequest{+ Fingerprints: map[string]string{"other": "fingerprint"},+ })+ if err != nil {+ t.Fatalf("OpenSync: %v", err)+ }+ batch, err := session.NextBatch(context.Background())+ if err != nil {+ t.Fatalf("NextBatch: %v", err)+ }+ if len(batch.Documents) != 1 {+ t.Fatalf("documents len = %d, want 1", len(batch.Documents))+ }+}++func TestOCIStorageConnectorOpenSyncFiltersByWindow(t *testing.T) {+ connector := newTestOCIStorageConnector(t, []ociStorageObject{+ {Key: "docs/a.txt", LastModified: mustTime(t, "2026-01-01T00:00:00Z"), Size: 1, ETag: "a"},+ {Key: "docs/b.txt", LastModified: mustTime(t, "2026-01-03T00:00:00Z"), Size: 1, ETag: "b"},+ {Key: "docs/c.txt", LastModified: mustTime(t, "2026-01-05T00:00:00Z"), Size: 1, ETag: "c"},+ })+ session, err := connector.OpenSync(context.Background(), SyncRequest{+ WindowStart: mustTimePointer(t, "2026-01-02T00:00:00Z"),+ WindowEnd: mustTime(t, "2026-01-04T00:00:00Z"),+ })+ if err != nil {+ t.Fatalf("OpenSync: %v", err)+ }+ batch, err := session.NextBatch(context.Background())+ if err != nil {+ t.Fatalf("NextBatch: %v", err)+ }+ if len(batch.Documents) != 1 || batch.Documents[0].SourceID != ociStorageSourceID("bucket", "docs/b.txt") {+ t.Fatalf("documents = %+v", batch.Documents)+ }+}++func TestOCIStorageConnectorOpenPruneReturnsSlimSnapshot(t *testing.T) {+ connector := newTestOCIStorageConnector(t, []ociStorageObject{+ {Key: "docs/a.txt", LastModified: mustTime(t, "2026-01-01T00:00:00Z"), Size: 1, ETag: "a"},+ {Key: "docs/b.txt", LastModified: mustTime(t, "2026-01-02T00:00:00Z"), Size: 1, ETag: "b"},+ {Key: "docs/folder/", LastModified: mustTime(t, "2026-01-02T00:00:00Z"), Size: 0, ETag: "folder"},+ })+ session, err := connector.OpenPrune(context.Background(), PruneRequest{})+ if err != nil {+ t.Fatalf("OpenPrune: %v", err)+ }+ batch, err := session.NextBatch(context.Background())+ if err != nil {+ t.Fatalf("NextBatch: %v", err)+ }+ if len(batch.Documents) != 2 {+ t.Fatalf("documents len = %d, want 2", len(batch.Documents))+ }+ if batch.Documents[0].SourceID != ociStorageSourceID("bucket", "docs/a.txt") || batch.Documents[1].SourceID != ociStorageSourceID("bucket", "docs/b.txt") {+ t.Fatalf("documents = %+v", batch.Documents)+ }+}++func TestOCIStorageConnectorOpenPruneStreamsAcrossPages(t *testing.T) {+ connector := newTestOCIStorageConnector(t, nil)+ connector.batchSize = 2+ connector.listObjects = func(ctx context.Context, startAfter string, maxKeys int32) ([]ociStorageObject, string, bool, error) {+ if startAfter == "" {+ return []ociStorageObject{+ {Key: "docs/a.txt", LastModified: mustTime(t, "2026-01-01T00:00:00Z"), Size: 1, ETag: "a"},+ {Key: "docs/b.txt", LastModified: mustTime(t, "2026-01-02T00:00:00Z"), Size: 1, ETag: "b"},+ }, ociStorageSourceID("bucket", "docs/b.txt"), true, nil+ }+ return []ociStorageObject{+ {Key: "docs/c.txt", LastModified: mustTime(t, "2026-01-03T00:00:00Z"), Size: 1, ETag: "c"},+ {Key: "docs/d.txt", LastModified: mustTime(t, "2026-01-04T00:00:00Z"), Size: 1, ETag: "d"},+ }, "", false, nil+ }++ session, err := connector.OpenPrune(context.Background(), PruneRequest{})+ if err != nil {+ t.Fatalf("OpenPrune: %v", err)+ }+ batch, err := session.NextBatch(context.Background())+ if err != nil {+ t.Fatalf("NextBatch: %v", err)+ }+ if len(batch.Documents) != 2 || batch.Documents[0].SourceID != ociStorageSourceID("bucket", "docs/a.txt") || batch.Documents[1].SourceID != ociStorageSourceID("bucket", "docs/b.txt") {+ t.Fatalf("first documents = %+v", batch.Documents)+ }+ batch, err = session.NextBatch(context.Background())+ if err != nil {+ t.Fatalf("second NextBatch: %v", err)+ }+ if len(batch.Documents) != 2 || batch.Documents[0].SourceID != ociStorageSourceID("bucket", "docs/c.txt") || batch.Documents[1].SourceID != ociStorageSourceID("bucket", "docs/d.txt") {+ t.Fatalf("second documents = %+v", batch.Documents)+ }+ if _, err = session.NextBatch(context.Background()); !errors.Is(err, io.EOF) {+ t.Fatalf("NextBatch EOF = %v", err)+ }+}++func TestOCIStorageConnectorValidate(t *testing.T) {+ ctx := context.Background()++ missingBucket := &OCIStorageConnector{namespace: "namespace", region: "region", accessKeyID: "access", secretKey: "secret", batchSize: 2}+ if err := missingBucket.Validate(ctx); err == nil {+ t.Fatalf("expected bucket name validation error")+ }++ missingCredentials := &OCIStorageConnector{bucketName: "bucket", batchSize: 2}+ if err := missingCredentials.Validate(ctx); err == nil {+ t.Fatalf("expected credentials validation error")+ }++ badBatch := &OCIStorageConnector{bucketName: "bucket", namespace: "namespace", region: "region", accessKeyID: "access", secretKey: "secret", batchSize: 0}+ if err := badBatch.Validate(ctx); err == nil {+ t.Fatalf("expected batch size validation error")+ }++ badRegion := &OCIStorageConnector{bucketName: "bucket", namespace: "namespace", region: "evil.example#", accessKeyID: "access", secretKey: "secret", batchSize: 2}+ if err := badRegion.Validate(ctx); err == nil {+ t.Fatalf("expected region validation error")+ }+}++func TestOCIStorageConnectorRejectsInvalidNamespaceBeforeClientCreation(t *testing.T) {+ for _, namespace := range []string{+ "namespace.example.com",+ "https://evil.example.org",+ "evil.example.org:443",+ "namespace/tenant",+ "namespace space",+ "-namespace",+ "namespace-",+ "Namespace",+ } {+ connector := newTestOCIStorageConnector(t, nil)+ connector.namespace = namespace+ connector.client = nil+ if _, err := connector.ensureClient(context.Background()); err == nil {+ t.Fatalf("expected invalid namespace %q to be rejected", namespace)+ }+ if connector.client != nil {+ t.Fatalf("client was created for invalid namespace %q", namespace)+ }+ }+}++func TestOCIStorageConnectorRejectsInvalidRegionBeforeClientCreation(t *testing.T) {+ for _, region := range []string{+ "evil.example#",+ "evil.example.com",+ "https://evil.example.org",+ "region/tenant",+ "region space",+ "-region",+ "region-",+ "Region",+ } {+ connector := newTestOCIStorageConnector(t, nil)+ connector.region = region+ connector.client = nil+ if _, err := connector.ensureClient(context.Background()); err == nil {+ t.Fatalf("expected invalid region %q to be rejected", region)+ }+ if connector.client != nil {+ t.Fatalf("client was created for invalid region %q", region)+ }+ }+}++func TestOCIStorageConnectorValidateSetting(t *testing.T) {+ connector := newTestOCIStorageConnector(t, []ociStorageObject{+ {Key: "docs/a.txt", LastModified: mustTime(t, "2026-01-01T00:00:00Z"), Size: 1, ETag: "a"},+ })+ var listCalls int+ connector.listObjects = func(ctx context.Context, startAfter string, maxKeys int32) ([]ociStorageObject, string, bool, error) {+ listCalls+++ return nil, "", false, nil+ }+ if err := connector.ValidateConnectorSetting(context.Background(), nil); err != nil {+ t.Fatalf("ValidateConnectorSetting: %v", err)+ }+ if listCalls != 1 {+ t.Fatalf("list calls = %d, want 1", listCalls)+ }+}++func TestNewOCIStorageConnectorDefaultsAndConfig(t *testing.T) {+ connector, err := NewOCIStorageConnector(map[string]any{+ "bucket_name": " bucket ",+ "prefix": "docs",+ "batch_size": 4,+ "size_threshold": 42,+ "allow_images": true,+ "credentials": map[string]any{+ "namespace": "mynamespace",+ "region": "us-ashburn-1",+ "access_key_id": "access",+ "secret_access_key": "secret",+ },+ })+ if err != nil {+ t.Fatalf("NewOCIStorageConnector: %v", err)+ }+ if connector.bucketName != "bucket" {+ t.Fatalf("bucket name = %q", connector.bucketName)+ }+ if connector.prefix != "docs/" {+ t.Fatalf("prefix = %q", connector.prefix)+ }+ if connector.batchSize != 4 {+ t.Fatalf("batch size = %d, want 4", connector.batchSize)+ }+ if connector.sizeThreshold != 42 {+ t.Fatalf("size threshold = %d, want 42", connector.sizeThreshold)+ }+ if !connector.allowImages {+ t.Fatalf("allow images = false, want true")+ }+ if connector.namespace != "mynamespace" || connector.region != "us-ashburn-1" || connector.accessKeyID != "access" || connector.secretKey != "secret" {+ t.Fatalf("credentials = %+v", connector)+ }++ defaults, err := NewOCIStorageConnector(map[string]any{+ "bucket_name": "bucket",+ "credentials": map[string]any{},+ })+ if err != nil {+ t.Fatalf("NewOCIStorageConnector defaults: %v", err)+ }+ if defaults.batchSize != defaultOCIStorageBatchSize || defaults.sizeThreshold != defaultOCIStorageSizeThreshold {+ t.Fatalf("defaults = batch %d threshold %d", defaults.batchSize, defaults.sizeThreshold)+ }+}++func TestOCIStorageHelpers(t *testing.T) {+ if got := ociStorageEndpoint("mynamespace", "us-ashburn-1"); got != "https://mynamespace.compat.objectstorage.us-ashburn-1.oraclecloud.com" {+ t.Fatalf("endpoint = %q", got)+ }+ if got := ociStorageConsoleURL("mynamespace", "us-ashburn-1", "bucket", "docs/a b.txt"); !strings.Contains(got, "https://objectstorage.us-ashburn-1.oraclecloud.com/n/mynamespace/b/bucket/o/docs/a%20b.txt") {+ t.Fatalf("console url = %q", got)+ }+ if _, err := readOCIBody(bytes.NewBufferString("12345"), "large.txt", 4); err == nil {+ t.Fatalf("expected size threshold error")+ }+}++func newTestOCIStorageConnector(t *testing.T, objects []ociStorageObject) *OCIStorageConnector {+ t.Helper()+ connector, err := NewOCIStorageConnector(map[string]any{+ "bucket_name": "bucket",+ "prefix": "docs",+ "batch_size": 2,+ "credentials": map[string]any{+ "namespace": "mynamespace",+ "region": "us-ashburn-1",+ "access_key_id": "access",+ "secret_access_key": "secret",+ },+ })+ if err != nil {+ t.Fatalf("NewOCIStorageConnector: %v", err)+ }+ connector.listObjects = func(ctx context.Context, startAfter string, maxKeys int32) ([]ociStorageObject, string, bool, error) {+ var out []ociStorageObject+ for _, object := range objects {+ if startAfter != "" && object.Key <= startAfter {+ continue+ }+ out = append(out, object)+ if maxKeys > 0 && len(out) >= int(maxKeys) {+ break+ }+ }+ hasMore := false+ nextStartAfter := ""+ if len(out) > 0 {+ nextStartAfter = ociStorageSourceID(connector.bucketName, out[len(out)-1].Key)+ for _, object := range objects {+ if object.Key > out[len(out)-1].Key {+ hasMore = true+ break+ }+ }+ }+ return out, nextStartAfter, hasMore, nil+ }+ connector.downloadObject = func(ctx context.Context, key string, sizeThreshold int64) ([]byte, error) {+ if key == "docs/new.txt" {+ return []byte("body:new"), nil+ }+ return []byte("body:" + key), nil+ }+ return connector+}++func mustTimePointer(t *testing.T, value string) *time.Time {+ t.Helper()+ parsed := mustTime(t, value)+ return &parsed+}