RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
Extract shared storage layer: s3Base + retry/move helpers (#21158)
xugangqiang322b72682517
diff --git a/internal/storage/copy_source_test.go b/internal/storage/copy_source_test.goindex 52df2bf9e9..eaf2636c79 100644--- a/internal/storage/copy_source_test.go+++ b/internal/storage/copy_source_test.go@@ -36,9 +36,29 @@ func newCopySourceTestStorage(backend, endpoint, bucket, prefix string) copySouroptions.RetryMaxAttempts = 1})if backend == "oss" {- return &OSSStorage{client: client, bucket: bucket, prefixPath: prefix}+ o := &OSSStorage{client: client}+ o.s3Base = s3Base{+ client: func() *s3.Client { return o.client },+ bucket: bucket,+ prefixPath: prefix,+ typeName: "oss",+ resolve: o.resolveBucketAndPath,+ makeBucket: o.createBucket,+ reconnect: func(context.Context) {},+ }+ return o+ }+ s := &S3Storage{client: client}+ s.s3Base = s3Base{+ client: func() *s3.Client { return s.client },+ bucket: bucket,+ prefixPath: prefix,+ typeName: "s3",+ resolve: s.resolveBucketAndPath,+ makeBucket: s.createBucket,+ reconnect: func(context.Context) {},}- return &S3Storage{client: client, bucket: bucket, prefixPath: prefix}+ return s}func TestStorageCopyEncodesSource(t *testing.T) {diff --git a/internal/storage/doc.go b/internal/storage/doc.gonew file mode 100644index 0000000000..a490d13f37--- /dev/null+++ b/internal/storage/doc.go@@ -0,0 +1,61 @@+//+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.+//++// Package storage abstracts object storage behind the Storage interface.+//+// Layout: s3base.go holds the shared implementation for aws-sdk-go-v2+// backends (s3.go = AWS S3, oss.go = Aliyun OSS); storage_common.go holds+// the backend-agnostic retry/move helpers used by every backend including+// minio.go (minio-go SDK), gcs.go (Google SDK) and memory.go (test double).+//+// # Deliberately non-uniform path folding+//+// resolveBucketAndPath intentionally differs per backend. The pre-migration+// Python connectors (rag/utils/minio_conn.py and rag/utils/s3_conn.py, both+// removed in #20289) folded object keys differently — minio only inserted+// the logical bucket as a path segment in single-bucket mode, while s3+// inserted it whenever a prefix was configured — and the Go ports preserved+// each vendor's semantics so that keys written before the migration keep+// resolving after it. The table below is the contract; do not "unify" it.+//+// | minio | s3 | oss+// ----------------------+--------------------+---------------------+---------------------+// bucket=="" prefix=="" | fnm | fnm | fnm+// bucket=="" prefix!="" | prefix/fnm | prefix/bucket/fnm | prefix/fnm+// bucket!="" prefix=="" | bucket/fnm | bucket/fnm | fnm+// bucket!="" prefix!="" | prefix/bucket/fnm | prefix/bucket/fnm | prefix/fnm+//+// GCS and the in-memory backend perform no folding at all.+//+// Consequences to be aware of:+//+// - Switching STORAGE_ENGINE does not migrate existing object keys; the+// same (bucket, fnm) argument resolves to different keys on different+// backends. Treat the engine as fixed per deployment.+// - In shared-bucket mode OSS cannot scope a removal to a logical bucket,+// so RemoveBucket/RemoveEmptyBucket refuse rather than guess.+//+// # Deliberately non-uniform health checks+//+// MinioStorage.Health is a lightweight liveness probe (client HealthCheck ++// IsOnline, no writes). The s3Base health check uploads a probe object to+// <bucket or "health-check-bucket">/<prefix>/txtxtxtxt1, matching the+// pre-migration Python S3 connector. The difference is intentional: a+// write-based probe also verifies write permissions, but it requires them,+// so read-only credentials make MinIO-style and S3-style deployments report+// health differently. Callers must not treat a false from one engine as+// comparable to a false from the other.+package storagediff --git a/internal/storage/memory.go b/internal/storage/memory.goindex 10d3edc8aa..cb30d519f8 100644--- a/internal/storage/memory.go+++ b/internal/storage/memory.go@@ -20,11 +20,8 @@ import ("context""errors""fmt"- "ragflow/internal/common""sync""time"-- "go.uber.org/zap")// ErrMemoryNotFound is returned when a key does not exist in the in-memory backend.@@ -265,24 +262,15 @@ func (m *MemoryStorage) Copy(ctx context.Context, srcBucket, srcPath, destBucket// Move transfers an object to a new location, deleting the source on success.// Returns false if the source does not exist or the copy step fails.func (m *MemoryStorage) Move(ctx context.Context, srcBucket, srcPath, destBucket, destPath string) bool {-if err := ctx.Err(); err != nil {return false}- if !m.Copy(ctx, srcBucket, srcPath, destBucket, destPath) {- return false- }- if err := m.Remove(ctx, srcBucket, srcPath); err != nil {- rollbackCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 30*time.Second)- defer cancel()- err = m.Remove(rollbackCtx, destBucket, destPath)- if err != nil {- common.Warn("Failed to roll back copied destination object", zap.String("bucket", destBucket), zap.String("key", destPath), zap.Error(err))- }- return false- }- return true+ return moveWithRollback(ctx, srcBucket, srcPath, destBucket, destPath,+ func() bool { return m.Copy(ctx, srcBucket, srcPath, destBucket, destPath) },+ func(rctx context.Context) error { return m.Remove(rctx, srcBucket, srcPath) },+ func(rctx context.Context) error { return m.Remove(rctx, destBucket, destPath) },+ )}func (m *MemoryStorage) Close() error { return nil }diff --git a/internal/storage/minio.go b/internal/storage/minio.goindex b48085b617..104ad2e339 100644--- a/internal/storage/minio.go+++ b/internal/storage/minio.go@@ -122,76 +122,43 @@ func (m *MinioStorage) Health(ctx context.Context) bool {func (m *MinioStorage) Put(ctx context.Context, bucket, fnm string, binary []byte, tenantID ...string) error {bucket, fnm = m.resolveBucketAndPath(bucket, fnm)- var lastErr error-- for i := 0; i < 3; i++ {+ return retryObjectOp(ctx, 3, nil, func() error {// Ensure bucket existsif m.bucket == "" {exists, err := m.client.BucketExists(ctx, bucket)if err != nil {- lastErr = err- if ctxErr := ctx.Err(); ctxErr != nil {- return ctxErr- }common.Warn("Failed to check bucket existence", zap.String("bucket", bucket), zap.Error(err))- if sleepErr := sleepOrAbort(ctx, time.Second); sleepErr != nil {- return sleepErr- }- continue+ return err}if !exists {if err := m.client.MakeBucket(ctx, bucket, minio.MakeBucketOptions{}); err != nil {- lastErr = err- if ctxErr := ctx.Err(); ctxErr != nil {- return ctxErr- }common.Warn("Failed to create bucket", zap.String("bucket", bucket), zap.Error(err))- if sleepErr := sleepOrAbort(ctx, time.Second); sleepErr != nil {- return sleepErr- }- continue+ return err}}}reader := bytes.NewReader(binary)- _, err := m.client.PutObject(ctx, bucket, fnm, reader, int64(len(binary)), minio.PutObjectOptions{})- if err != nil {+ if _, err := m.client.PutObject(ctx, bucket, fnm, reader, int64(len(binary)), minio.PutObjectOptions{}); err != nil {const warnMessage = "Failed to put object"- lastErr = fmt.Errorf("%s: %w", warnMessage, err)- if ctxErr := ctx.Err(); ctxErr != nil {- return ctxErr- }common.Warn(warnMessage, zap.String("bucket", bucket), zap.String("key", fnm), zap.Error(err))- if sleepErr := sleepOrAbort(ctx, time.Second); sleepErr != nil {- return sleepErr- }- continue+ return fmt.Errorf("%s: %w", warnMessage, err)}return nil- }-- return lastErr+ })}// Get retrieves an object from MinIOfunc (m *MinioStorage) Get(ctx context.Context, bucket, fnm string, tenantID ...string) ([]byte, error) {bucket, fnm = m.resolveBucketAndPath(bucket, fnm)- var lastErr error- for i := 0; i < 2; i++ {+ var data []byte+ err := retryObjectOp(ctx, 2, nil, func() error {obj, err := m.client.GetObject(ctx, bucket, fnm, minio.GetObjectOptions{})if err != nil {- lastErr = err- if ctxErr := ctx.Err(); ctxErr != nil {- return nil, ctxErr- }common.Warn("failed to get object", zap.String("bucket", bucket), zap.String("key", fnm), zap.Error(err))- if sleepErr := sleepOrAbort(ctx, time.Second); sleepErr != nil {- return nil, sleepErr- }- continue+ return err}buf := new(bytes.Buffer)@@ -201,21 +168,18 @@ func (m *MinioStorage) Get(ctx context.Context, bucket, fnm string, tenantID ...return err}()if readErr != nil {- lastErr = readErr- if ctxErr := ctx.Err(); ctxErr != nil {- return nil, ctxErr- }common.Error("failed to read object data", err, zap.String("bucket", bucket), zap.String("key", fnm))- if sleepErr := sleepOrAbort(ctx, time.Second); sleepErr != nil {- return nil, sleepErr- }- continue+ return readErr}+ data = buf.Bytes()- return buf.Bytes(), nil+ return nil+ })+ if err != nil {+ return nil, err}- return nil, lastErr+ return data, nil}// Remove removes an object from MinIO@@ -273,23 +237,22 @@ func (m *MinioStorage) ObjectExists(ctx context.Context, bucket, fnm string) (bofunc (m *MinioStorage) GetPresignedURL(ctx context.Context, bucket, fnm string, expires time.Duration, tenantID ...string) (string, error) {bucket, fnm = m.resolveBucketAndPath(bucket, fnm)- for i := 0; i < 10; i++ {- url, err := m.client.PresignedGetObject(ctx, bucket, fnm, expires, nil)+ var url string+ err := retryObjectOp(ctx, 10, nil, func() error {+ u, err := m.client.PresignedGetObject(ctx, bucket, fnm, expires, nil)if err != nil {- if ctxErr := ctx.Err(); ctxErr != nil {- return "", ctxErr- }common.Warn("Failed to get presigned URL", zap.String("bucket", bucket), zap.String("key", fnm), zap.Error(err))- if err = sleepOrAbort(ctx, time.Second); err != nil {- return "", err- }- continue+ return err}+ url = u.String()- return url.String(), nil+ return nil+ })+ if err != nil {+ return "", fmt.Errorf("failed to get presigned URL after 10 retries: %w", err)}- return "", fmt.Errorf("failed to get presigned URL after 10 retries")+ return url, nil}// BucketExists checks if a bucket exists@@ -496,20 +459,11 @@ func (m *MinioStorage) Copy(ctx context.Context, srcBucket, srcPath, destBucket,// Move moves an object from source to destinationfunc (m *MinioStorage) Move(ctx context.Context, srcBucket, srcPath, destBucket, destPath string) bool {- if m.Copy(ctx, srcBucket, srcPath, destBucket, destPath) {- if err := m.Remove(ctx, srcBucket, srcPath); err != nil {- common.Warn("Failed to remove source object after copy", zap.String("bucket", srcBucket), zap.String("key", srcPath), zap.Error(err))- rollbackCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 30*time.Second)- defer cancel()- err = m.Remove(rollbackCtx, destBucket, destPath)- if err != nil {- common.Warn("Failed to roll back copied destination object", zap.String("bucket", destBucket), zap.String("key", destPath), zap.Error(err))- }- return false- }- return true- }- return false+ return moveWithRollback(ctx, srcBucket, srcPath, destBucket, destPath,+ func() bool { return m.Copy(ctx, srcBucket, srcPath, destBucket, destPath) },+ func(rctx context.Context) error { return m.Remove(rctx, srcBucket, srcPath) },+ func(rctx context.Context) error { return m.Remove(rctx, destBucket, destPath) },+ )}func (m *MinioStorage) Close() error { return nil }diff --git a/internal/storage/oss.go b/internal/storage/oss.goindex 6a9209d5e3..d940cb1367 100644--- a/internal/storage/oss.go+++ b/internal/storage/oss.go@@ -17,44 +17,49 @@package storageimport (- "bytes""context"- "errors""fmt""ragflow/internal/common""ragflow/internal/server/config"- "time""github.com/aws/aws-sdk-go-v2/aws"s3Config "github.com/aws/aws-sdk-go-v2/config""github.com/aws/aws-sdk-go-v2/credentials""github.com/aws/aws-sdk-go-v2/service/s3"- "github.com/aws/smithy-go"- "github.com/aws/smithy-go/encoding/httpbinding""go.uber.org/zap")-// OSSStorage implements Storage interface for Aliyun OSS-// OSS uses S3-compatible API+// OSSStorage implements Storage for Aliyun OSS. The shared S3-compatible+// surface lives in s3Base; this file holds only what differs for OSS: the+// connection setup, the path-folding semantics (no logical-bucket segment —+// see doc.go), and the refusal to remove logical buckets in shared-bucket+// mode.type OSSStorage struct {- client *s3.Client- bucket string- prefixPath string- config config.OSSConfig+ s3Base+ client *s3.Client // owned by connect/reconnect; the base reads it via its getter+ config config.OSSConfig}// NewOSSStorage creates a new OSS storage instancefunc NewOSSStorage(ctx context.Context, config config.OSSConfig) (*OSSStorage, error) {storage := &OSSStorage{- bucket: config.Bucket,- prefixPath: config.PrefixPath,- config: config,+ config: config,}if err := storage.connect(ctx); err != nil {return nil, err}+ storage.s3Base = s3Base{+ client: func() *s3.Client { return storage.client },+ bucket: config.Bucket,+ prefixPath: config.PrefixPath,+ typeName: "oss",+ resolve: storage.resolveBucketAndPath,+ makeBucket: storage.createBucket,+ reconnect: storage.reconnect,+ }+return storage, nil}@@ -104,277 +109,16 @@ func (o *OSSStorage) resolveBucketAndPath(bucket, fnm string) (string, string) {return actualBucket, actualPath}-func (o *OSSStorage) Type() string { return "oss" }--// Health checks OSS service availability-func (o *OSSStorage) Health(ctx context.Context) bool {- bucket := o.bucket- if bucket == "" {- bucket = "health-check-bucket"- }-- fnm := "txtxtxtxt1"- if o.prefixPath != "" {- fnm = fmt.Sprintf("%s/%s", o.prefixPath, fnm)- }- binary := []byte("_t@@@1")-- // Ensure bucket exists- if !o.BucketExists(ctx, bucket) {- _, err := o.client.CreateBucket(ctx, &s3.CreateBucketInput{- Bucket: aws.String(bucket),- })- if err != nil {- common.Error("Failed to create bucket for health check", err, zap.String("bucket", bucket))- return false- }- }-- // Try to upload a test object- reader := bytes.NewReader(binary)- _, err := o.client.PutObject(ctx, &s3.PutObjectInput{- Bucket: aws.String(bucket),- Key: aws.String(fnm),- Body: reader,- })-- if err != nil {- common.Error("Health check failed", err)- return false- }-- return true-}--// Put uploads an object to OSS-func (o *OSSStorage) Put(ctx context.Context, bucket, fnm string, binary []byte, tenantID ...string) error {- bucket, fnm = o.resolveBucketAndPath(bucket, fnm)- var lastErr error-- for i := 0; i < 2; i++ {- // Ensure bucket exists- if !o.BucketExists(ctx, bucket) {- _, err := o.client.CreateBucket(ctx, &s3.CreateBucketInput{- Bucket: aws.String(bucket),- })- if err != nil {- lastErr = err- if ctxErr := ctx.Err(); ctxErr != nil {- return ctxErr- }- common.Error("Failed to create bucket", err, zap.String("bucket", bucket))- o.reconnect(ctx)- if sleepErr := sleepOrAbort(ctx, time.Second); sleepErr != nil {- return sleepErr- }- continue- }- common.Info("Created bucket", zap.String("bucket", bucket))- }-- reader := bytes.NewReader(binary)- _, err := o.client.PutObject(ctx, &s3.PutObjectInput{- Bucket: aws.String(bucket),- Key: aws.String(fnm),- Body: reader,- })- if err != nil {- lastErr = err- if ctxErr := ctx.Err(); ctxErr != nil {- return ctxErr- }- common.Error("Failed to put object", err, zap.String("bucket", bucket), zap.String("key", fnm))- o.reconnect(ctx)- if sleepErr := sleepOrAbort(ctx, time.Second); sleepErr != nil {- return sleepErr- }- continue- }-- return nil- }-- return lastErr-}--// Get retrieves an object from OSS-func (o *OSSStorage) Get(ctx context.Context, bucket, fnm string, tenantID ...string) ([]byte, error) {- bucket, fnm = o.resolveBucketAndPath(bucket, fnm)- var lastErr error-- for i := 0; i < 2; i++ {- result, err := o.client.GetObject(ctx, &s3.GetObjectInput{- Bucket: aws.String(bucket),- Key: aws.String(fnm),- })- if err != nil {- lastErr = err- if ctxErr := ctx.Err(); ctxErr != nil {- return nil, ctxErr- }- common.Error("Failed to get object", err, zap.String("bucket", bucket), zap.String("key", fnm))- o.reconnect(ctx)- if sleepErr := sleepOrAbort(ctx, time.Second); sleepErr != nil {- return nil, sleepErr- }- continue- }- buf := new(bytes.Buffer)-- readErr := func() error {- defer result.Body.Close()- _, err = buf.ReadFrom(result.Body)- return err- }()- if readErr != nil {- lastErr = readErr- if ctxErr := ctx.Err(); ctxErr != nil {- return nil, ctxErr- }- common.Error("Failed to read object data", readErr, zap.String("bucket", bucket), zap.String("key", fnm))- o.reconnect(ctx)- if sleepErr := sleepOrAbort(ctx, time.Second); sleepErr != nil {- return nil, sleepErr- }- continue- }-- return buf.Bytes(), nil- }-- return nil, lastErr-}--// Remove removes an object from OSS-func (o *OSSStorage) Remove(ctx context.Context, bucket, fnm string, tenantID ...string) error {- bucket, fnm = o.resolveBucketAndPath(bucket, fnm)-- _, err := o.client.DeleteObject(ctx, &s3.DeleteObjectInput{- Bucket: aws.String(bucket),- Key: aws.String(fnm),- })- if err != nil {- if isS3NotFound(err) {- return nil- }- common.Error("Failed to remove object", err, zap.String("bucket", bucket), zap.String("key", fnm))- return err- }-- return nil-}--// ObjExist checks if an object exists in OSS-func (o *OSSStorage) ObjExist(ctx context.Context, bucket, fnm string, tenantID ...string) bool {- bucket, fnm = o.resolveBucketAndPath(bucket, fnm)-- _, err := o.client.HeadObject(ctx, &s3.HeadObjectInput{- Bucket: aws.String(bucket),- Key: aws.String(fnm),- })- if err != nil {- if isOSSNotFound(err) {- return false- }- return false- }-- return true-}--func (o *OSSStorage) ObjectExists(ctx context.Context, bucket, fnm string) (bool, error) {- bucket, fnm = o.resolveBucketAndPath(bucket, fnm)- _, err := o.client.HeadObject(ctx, &s3.HeadObjectInput{Bucket: aws.String(bucket), Key: aws.String(fnm)})- if err == nil {- return true, nil- }- if isS3NotFound(err) {- return false, nil- }- return false, err-}--func (o *OSSStorage) ListObjects(ctx context.Context, bucket string, tenantID ...string) ([]string, error) {-- listInput := &s3.ListObjectsV2Input{+func (o *OSSStorage) createBucket(ctx context.Context, bucket string) error {+ _, err := o.client.CreateBucket(ctx, &s3.CreateBucketInput{Bucket: aws.String(bucket),- }-- result, err := o.client.ListObjectsV2(ctx, listInput)- if err != nil {- return nil, err- }-- var objects []string-- for _, obj := range result.Contents {- objects = append(objects, *obj.Key)- }-- return objects, nil-}--func (o *OSSStorage) GetPresignedURL(ctx context.Context, bucket, fnm string, expires time.Duration, tenantID ...string) (string, error) {- bucket, fnm = o.resolveBucketAndPath(bucket, fnm)-- presignClient := s3.NewPresignClient(o.client)-- for i := 0; i < 10; i++ {- req, err := presignClient.PresignGetObject(ctx, &s3.GetObjectInput{- Bucket: aws.String(bucket),- Key: aws.String(fnm),- }, s3.WithPresignExpires(expires))- if err != nil {- if ctxErr := ctx.Err(); ctxErr != nil {- return "", ctxErr- }- common.Error("Failed to generate presigned URL", err, zap.String("bucket", bucket), zap.String("key", fnm))- o.reconnect(ctx)- if err = sleepOrAbort(ctx, time.Second); err != nil {- return "", err- }- continue- }-- return req.URL, nil- }-- return "", fmt.Errorf("failed to generate presigned URL after 10 retries")-}--// BucketExists checks if a bucket exists-func (o *OSSStorage) BucketExists(ctx context.Context, bucket string) bool {- actualBucket := bucket- if o.bucket != "" {- actualBucket = o.bucket- }-- _, err := o.client.HeadBucket(ctx, &s3.HeadBucketInput{- Bucket: aws.String(actualBucket),})- if err != nil {- common.Error("Bucket does not exist or error", err, zap.String("bucket", actualBucket))- return false- }-- return true-}--func (o *OSSStorage) BucketExistsWithError(ctx context.Context, bucket string) (bool, error) {- if o.bucket != "" {- bucket = o.bucket- }- _, err := o.client.HeadBucket(ctx, &s3.HeadBucketInput{Bucket: aws.String(bucket)})- if err == nil {- return true, nil- }- if isS3BucketNotFound(err) {- return false, nil- }- return false, err+ return err}-// RemoveBucket removes a bucket and all its objects+// RemoveBucket removes a bucket and all its objects. In shared-bucket mode a+// logical bucket has no physical isolation, so removal is refused rather than+// silently clearing the shared prefix.func (o *OSSStorage) RemoveBucket(ctx context.Context, bucket string) error {if o.bucket != "" {return fmt.Errorf("cannot remove logical OSS bucket %s: shared bucket mode does not isolate objects by logical bucket", bucket)@@ -400,11 +144,10 @@ func (o *OSSStorage) RemoveBucket(ctx context.Context, bucket string) error {}for _, obj := range result.Contents {- _, err = o.client.DeleteObject(ctx, &s3.DeleteObjectInput{+ if _, err = o.client.DeleteObject(ctx, &s3.DeleteObjectInput{Bucket: aws.String(actualBucket),Key: obj.Key,- })- if err != nil {+ }); err != nil {common.Error("Failed to delete object", err, zap.String("bucket", actualBucket))}}@@ -416,10 +159,9 @@ func (o *OSSStorage) RemoveBucket(ctx context.Context, bucket string) error {}// Delete bucket- _, err := o.client.DeleteBucket(ctx, &s3.DeleteBucketInput{+ if _, err := o.client.DeleteBucket(ctx, &s3.DeleteBucketInput{Bucket: aws.String(actualBucket),- })- if err != nil {+ }); err != nil {common.Error("Failed to delete bucket", err, zap.String("bucket", actualBucket))return err}@@ -427,7 +169,8 @@ func (o *OSSStorage) RemoveBucket(ctx context.Context, bucket string) error {return nil}-// RemoveEmptyBucket removes a physical bucket only when it is empty.+// RemoveEmptyBucket removes a physical bucket only when it is empty. In+// shared-bucket mode a logical bucket cannot be removed at all.func (o *OSSStorage) RemoveEmptyBucket(ctx context.Context, bucket string) error {if o.bucket != "" {return fmt.Errorf("cannot remove logical OSS bucket %s: shared bucket mode does not isolate objects by logical bucket", bucket)@@ -438,55 +181,3 @@ func (o *OSSStorage) RemoveEmptyBucket(ctx context.Context, bucket string) error}return err}--// Copy copies an object from source to destination-func (o *OSSStorage) Copy(ctx context.Context, srcBucket, srcPath, destBucket, destPath string) bool {- srcBucket, srcPath = o.resolveBucketAndPath(srcBucket, srcPath)- destBucket, destPath = o.resolveBucketAndPath(destBucket, destPath)-- copySource := httpbinding.EscapePath(fmt.Sprintf("%s/%s", srcBucket, srcPath), false)-- _, err := o.client.CopyObject(ctx, &s3.CopyObjectInput{- Bucket: aws.String(destBucket),- Key: aws.String(destPath),- CopySource: aws.String(copySource),- })- if err != nil {- common.Error("Failed to copy object", err, zap.String("src", copySource), zap.String("dest", fmt.Sprintf("%s/%s", destBucket, destPath)))- return false- }-- return true-}--// Move moves an object from source to destination-func (o *OSSStorage) Move(ctx context.Context, srcBucket, srcPath, destBucket, destPath string) bool {- if o.Copy(ctx, srcBucket, srcPath, destBucket, destPath) {- if err := o.Remove(ctx, srcBucket, srcPath); err != nil {- common.Error("Failed to remove source object after copy", err, zap.String("bucket", srcBucket), zap.String("key", srcPath))- rollbackCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 30*time.Second)- defer cancel()- err = o.Remove(rollbackCtx, destBucket, destPath)- if err != nil {- common.Warn("Failed to roll back copied destination object", zap.String("bucket", destBucket), zap.String("key", destPath), zap.Error(err))- }- return false- }- return true- }- return false-}--func (o *OSSStorage) Close() error { return nil }--// Helper functions-func isOSSNotFound(err error) bool {- if err == nil {- return false- }- var apiErr smithy.APIError- if errors.As(err, &apiErr) {- return apiErr.ErrorCode() == "NotFound" || apiErr.ErrorCode() == "404" || apiErr.ErrorCode() == "NoSuchKey"- }- return false-}diff --git a/internal/storage/oss_test.go b/internal/storage/oss_test.gonew file mode 100644index 0000000000..9c25cdadf4--- /dev/null+++ b/internal/storage/oss_test.go@@ -0,0 +1,235 @@+//+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.+//++package storage++import (+ "context"+ "errors"+ "fmt"+ "net/http"+ "strings"+ "testing"+ "time"++ "github.com/aws/aws-sdk-go-v2/aws"+ "github.com/aws/aws-sdk-go-v2/credentials"+ "github.com/aws/aws-sdk-go-v2/service/s3"+)++// newOSSTestStorage builds an OSS backend over a fake transport. These tests+// were written BEFORE oss.go was slimmed onto s3Base and passed unchanged+// against the old standalone implementation — they are the behavior contract+// the refactor must preserve (characterization tests).+func newOSSTestStorage(handler s3RoundTripper, bucket, prefix string) *OSSStorage {+ client := s3.NewFromConfig(aws.Config{+ Region: "us-east-1",+ Credentials: aws.NewCredentialsCache(credentials.NewStaticCredentialsProvider("access", "secret", "")),+ HTTPClient: &http.Client{Transport: handler},+ }, func(options *s3.Options) {+ options.BaseEndpoint = aws.String("https://oss.test")+ options.UsePathStyle = true+ options.RetryMaxAttempts = 1+ })+ o := &OSSStorage{client: client}+ o.s3Base = s3Base{+ client: func() *s3.Client { return o.client },+ bucket: bucket,+ prefixPath: prefix,+ typeName: "oss",+ resolve: o.resolveBucketAndPath,+ makeBucket: o.createBucket,+ reconnect: func(context.Context) {},+ }+ return o+}++func TestOSSStoragePutAndGet(t *testing.T) {+ o := newOSSTestStorage(func(req *http.Request) (*http.Response, error) {+ if req.Method == http.MethodPut {+ return s3Response(200, ""), nil+ }+ return s3Response(200, "oss-bytes"), nil+ }, "", "")++ if err := o.Put(context.Background(), "bucket", "key", []byte("data")); err != nil {+ t.Fatalf("Put: %v", err)+ }+ got, err := o.Get(context.Background(), "bucket", "key")+ if err != nil {+ t.Fatalf("Get: %v", err)+ }+ if string(got) != "oss-bytes" {+ t.Fatalf("Get returned %q, want oss-bytes", got)+ }+}++func TestOSSStoragePutSurfacesServiceError(t *testing.T) {+ // A failing bucket creation must surface an error after the retry loop+ // gives up — Put must never silently succeed. The assertion checks+ // non-nil only: with this aws-sdk version, any failing response's error+ // chain re-resolves the auth scheme and masks the real code behind an+ // endpoint-rule error (verified experimentally). Existing s3_test.go+ // failure cases follow the same err != nil convention for this reason.+ o := newOSSTestStorage(func(req *http.Request) (*http.Response, error) {+ return s3Response(403, `<Error><Code>AccessDenied</Code></Error>`), nil+ }, "", "")++ if err := o.Put(context.Background(), "bucket", "key", []byte("data")); err == nil {+ t.Fatal("Put returned nil after the bucket creation was denied")+ }+}++func TestOSSStorageGetSurfacesContextCancellation(t *testing.T) {+ o := newOSSTestStorage(func(req *http.Request) (*http.Response, error) {+ return nil, fmt.Errorf("simulated transport error")+ }, "", "")++ ctx, cancel := context.WithCancel(context.Background())+ cancel()++ start := time.Now()+ _, err := o.Get(ctx, "bucket", "key")+ elapsed := time.Since(start)++ if !errors.Is(err, context.Canceled) {+ t.Fatalf("Get err = %v, want context.Canceled", err)+ }+ if elapsed > 500*time.Millisecond {+ t.Fatalf("Get took %v on an already-canceled ctx; want < 500ms", elapsed)+ }+}++func TestOSSStorageHealthUploadsProbeObject(t *testing.T) {+ var sawCreateBucket, sawProbePut bool+ o := newOSSTestStorage(func(req *http.Request) (*http.Response, error) {+ switch {+ case req.Method == http.MethodHead:+ return s3Response(404, ""), nil // bucket missing+ case req.Method == http.MethodPut && strings.HasSuffix(req.URL.Path, "/txtxtxtxt1"):+ sawProbePut = true+ return s3Response(200, ""), nil+ case req.Method == http.MethodPut:+ sawCreateBucket = true+ return s3Response(200, ""), nil+ default:+ return s3Response(200, ""), nil+ }+ }, "", "")++ if !o.Health(context.Background()) {+ t.Fatal("Health = false, want true")+ }+ if !sawCreateBucket {+ t.Error("Health did not create the missing probe bucket")+ }+ if !sawProbePut {+ t.Error("Health did not upload the probe object")+ }+}++func TestOSSStorageRemoveMissingObjectIsNoOp(t *testing.T) {+ o := newOSSTestStorage(func(req *http.Request) (*http.Response, error) {+ return s3Response(404, `<Error><Code>NoSuchKey</Code></Error>`), nil+ }, "", "")++ if err := o.Remove(context.Background(), "bucket", "missing"); err != nil {+ t.Fatalf("Remove of a missing object: %v, want nil", err)+ }+}++func TestOSSStorageObjExistDistinguishesPresent(t *testing.T) {+ o := newOSSTestStorage(func(req *http.Request) (*http.Response, error) {+ if strings.HasSuffix(req.URL.Path, "/there") {+ return s3Response(200, ""), nil+ }+ return s3Response(404, `<Error><Code>NoSuchKey</Code></Error>`), nil+ }, "", "")++ if !o.ObjExist(context.Background(), "bucket", "there") {+ t.Error("ObjExist(there) = false, want true")+ }+ if o.ObjExist(context.Background(), "bucket", "absent") {+ t.Error("ObjExist(absent) = true, want false")+ }+}++func TestOSSStorageBucketExistsWithErrorSeparatesFailure(t *testing.T) {+ o := newOSSTestStorage(func(req *http.Request) (*http.Response, error) {+ if strings.Contains(req.URL.Path, "real") {+ return s3Response(200, ""), nil+ }+ return s3Response(404, `<Error><Code>NoSuchBucket</Code></Error>`), nil+ }, "", "")++ exists, err := o.BucketExistsWithError(context.Background(), "real")+ if err != nil || !exists {+ t.Errorf("BucketExistsWithError(real) = %v, %v; want true, nil", exists, err)+ }+ exists, err = o.BucketExistsWithError(context.Background(), "ghost")+ if err != nil || exists {+ t.Errorf("BucketExistsWithError(ghost) = %v, %v; want false, nil", exists, err)+ }+}++func TestOSSStorageGetPresignedURL(t *testing.T) {+ o := newOSSTestStorage(func(req *http.Request) (*http.Response, error) {+ return s3Response(200, ""), nil+ }, "", "")++ url, err := o.GetPresignedURL(context.Background(), "bucket", "key", 0)+ if err != nil {+ t.Fatalf("GetPresignedURL: %v", err)+ }+ if !strings.Contains(url, "oss.test") || !strings.Contains(url, "bucket/key") {+ t.Errorf("presigned URL = %q, want the oss.test endpoint and bucket/key path", url)+ }+}++func TestOSSStorageRemoveBucketRejectsLogicalBucketInSharedMode(t *testing.T) {+ o := newOSSTestStorage(func(req *http.Request) (*http.Response, error) {+ return s3Response(200, ""), nil+ }, "physical", "data")++ err := o.RemoveBucket(context.Background(), "logical")+ if err == nil || !strings.Contains(err.Error(), "cannot remove logical OSS bucket") {+ t.Fatalf("RemoveBucket err = %v, want the shared-mode rejection", err)+ }+}++func TestOSSStorageRemoveEmptyBucketRejectsLogicalBucketInSharedMode(t *testing.T) {+ o := newOSSTestStorage(func(req *http.Request) (*http.Response, error) {+ return s3Response(200, ""), nil+ }, "physical", "data")++ err := o.RemoveEmptyBucket(context.Background(), "logical")+ if err == nil || !strings.Contains(err.Error(), "cannot remove logical OSS bucket") {+ t.Fatalf("RemoveEmptyBucket err = %v, want the shared-mode rejection", err)+ }+}++func TestOSSStorageResolveOmitsBucketSegment(t *testing.T) {+ // OSS folds <prefix>/<fnm> — unlike S3, the logical bucket is NOT part of+ // the key in shared-bucket mode. See doc.go for the per-vendor table.+ o := newOSSTestStorage(func(req *http.Request) (*http.Response, error) {+ return s3Response(200, ""), nil+ }, "physical", "data")++ bucket, fnm := o.resolveBucketAndPath("logical", "report.txt")+ if bucket != "physical" || fnm != "data/report.txt" {+ t.Fatalf("resolve = (%q, %q), want (physical, data/report.txt)", bucket, fnm)+ }+}diff --git a/internal/storage/presign_errwrap_test.go b/internal/storage/presign_errwrap_test.gonew file mode 100644index 0000000000..87518cce4a--- /dev/null+++ b/internal/storage/presign_errwrap_test.go@@ -0,0 +1,120 @@+//+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.+//++package storage++import (+ "context"+ "errors"+ "net/http"+ "strings"+ "testing"+ "time"++ "github.com/aws/aws-sdk-go-v2/aws"+ awsCreds "github.com/aws/aws-sdk-go-v2/credentials"+ "github.com/aws/aws-sdk-go-v2/service/s3"+ "github.com/aws/smithy-go/middleware"+ "github.com/minio/minio-go/v7"+ minioCreds "github.com/minio/minio-go/v7/pkg/credentials"+)++// errSentinelPresign is a controlled error injected into the S3 presign path to+// verify that GetPresignedURL preserves (wraps) the underlying error instead of+// replacing it with a generic "after N retries" string.+var errSentinelPresign = errors.New("injected presign failure")++// failBuild is a build-stage middleware that forces every operation to fail with+// errSentinelPresign. Presigning runs the build/serialize middleware stack, so+// this deterministically fails the shared s3Base.GetPresignedURL path.+type failBuild struct{ err error }++func (f failBuild) ID() string { return "failBuild" }++func (f failBuild) HandleBuild(ctx context.Context, in middleware.BuildInput, next middleware.BuildHandler) (middleware.BuildOutput, middleware.Metadata, error) {+ return middleware.BuildOutput{}, middleware.Metadata{}, f.err+}++// newFailingPresignS3Base builds an s3Base whose s3 client fails every presign+// attempt through a build-stage middleware, so the retry helper returns the+// sentinel error. This exercises the shared s3Base.GetPresignedURL path used by+// both S3 and OSS.+func newFailingPresignS3Base() *s3Base {+ client := s3.NewFromConfig(aws.Config{+ Region: "us-east-1",+ Credentials: aws.NewCredentialsCache(awsCreds.NewStaticCredentialsProvider("access", "secret", "")),+ HTTPClient: &http.Client{Transport: s3RoundTripper(func(req *http.Request) (*http.Response, error) { return s3Response(200, ""), nil })},+ }, func(o *s3.Options) {+ o.BaseEndpoint = aws.String("https://s3.test")+ o.UsePathStyle = true+ o.RetryMaxAttempts = 1+ o.APIOptions = append(o.APIOptions, func(stack *middleware.Stack) error {+ return stack.Build.Add(failBuild{err: errSentinelPresign}, middleware.Before)+ })+ })+ return &s3Base{+ client: func() *s3.Client { return client },+ typeName: "s3",+ resolve: func(bucket, fnm string) (string, string) { return bucket, fnm },+ makeBucket: func(ctx context.Context, bucket string) error { return nil },+ reconnect: func(context.Context) {},+ }+}++// TestS3BaseGetPresignedURLWrapsUnderlyingError pins that the shared+// GetPresignedURL surfaces the underlying presign error via errors.Is, so+// callers can still detect cancellations/deadlines and specific failures.+func TestS3BaseGetPresignedURLWrapsUnderlyingError(t *testing.T) {+ base := newFailingPresignS3Base()++ _, err := base.GetPresignedURL(context.Background(), "bucket", "key", time.Minute)+ if err == nil {+ t.Fatalf("GetPresignedURL err = nil, want the underlying presign error")+ }+ if !errors.Is(err, errSentinelPresign) {+ t.Fatalf("GetPresignedURL err = %v, want it to wrap %v (errors.Is must hold)", err, errSentinelPresign)+ }+}++// TestMinioGetPresignedURLWrapsUnderlyingError pins the same contract for the+// MinIO backend: a failing presign attempt must preserve the underlying error+// identity rather than replacing it with a generic string.+func TestMinioGetPresignedURLWrapsUnderlyingError(t *testing.T) {+ // A negative expiry makes minio-go's PresignedGetObject fail locally and+ // deterministically on every retry attempt, with no server required.+ storage := &MinioStorage{client: mustMinioClient(t)}+ _, err := storage.GetPresignedURL(context.Background(), "bucket", "key", -1*time.Second)+ if err == nil {+ t.Fatalf("GetPresignedURL err = nil, want a non-nil underlying error")+ }+ if !strings.Contains(strings.ToLower(err.Error()), "expires") {+ t.Fatalf("GetPresignedURL err = %v, want it to preserve the underlying expiry error text", err)+ }+}++// mustMinioClient builds a MinIO client pointed at a dummy endpoint. Presigning+// is computed locally and never contacts the server, so no service is needed.+func mustMinioClient(t *testing.T) *minio.Client {+ t.Helper()+ c, err := minio.New("localhost:9000", &minio.Options{+ Creds: minioCreds.NewStaticV4("access", "secret", ""),+ Secure: false,+ })+ if err != nil {+ t.Fatalf("minio.New: %v", err)+ }+ return c+}diff --git a/internal/storage/s3.go b/internal/storage/s3.goindex 5313cdd175..f95f503eee 100644--- a/internal/storage/s3.go+++ b/internal/storage/s3.go@@ -17,13 +17,11 @@package storageimport (- "bytes""context""errors""fmt""ragflow/internal/common""ragflow/internal/server/config"- "time""github.com/aws/aws-sdk-go-v2/aws""github.com/aws/aws-sdk-go-v2/aws/signer/v4"@@ -32,30 +30,39 @@ import ("github.com/aws/aws-sdk-go-v2/service/s3""github.com/aws/aws-sdk-go-v2/service/s3/types""github.com/aws/smithy-go"- "github.com/aws/smithy-go/encoding/httpbinding""go.uber.org/zap")-// S3Storage implements Storage interface for AWS S3+// S3Storage implements Storage for AWS S3. The shared S3-compatible surface+// lives in s3Base; this file holds only what differs for AWS: connection+// options (signature version, addressing style), the region-aware bucket+// factory, the path-folding semantics, and the versioned bucket removal.type S3Storage struct {- client *s3.Client- bucket string- prefixPath string- config config.S3Config+ s3Base+ client *s3.Client // owned by connect/reconnect; the base reads it via its getter+ config config.S3Config}// NewS3Storage creates a new S3 storage instancefunc NewS3Storage(ctx context.Context, config config.S3Config) (*S3Storage, error) {storage := &S3Storage{- bucket: config.Bucket,- prefixPath: config.PrefixPath,- config: config,+ config: config,}if err := storage.connect(ctx); err != nil {return nil, err}+ storage.s3Base = s3Base{+ client: func() *s3.Client { return storage.client },+ bucket: config.Bucket,+ prefixPath: config.PrefixPath,+ typeName: "s3",+ resolve: storage.resolveBucketAndPath,+ makeBucket: storage.createBucket,+ reconnect: storage.reconnect,+ }+return storage, nil}@@ -135,256 +142,6 @@ func (s *S3Storage) createBucket(ctx context.Context, bucket string) error {return err}-func (s *S3Storage) Type() string { return "s3" }--// Health checks S3 service availability-func (s *S3Storage) Health(ctx context.Context) bool {- bucket := s.bucket- if bucket == "" {- bucket = "health-check-bucket"- }-- fnm := "txtxtxtxt1"- if s.prefixPath != "" {- fnm = fmt.Sprintf("%s/%s", s.prefixPath, fnm)- }- binary := []byte("_t@@@1")-- // Ensure bucket exists- if !s.BucketExists(ctx, bucket) {- err := s.createBucket(ctx, bucket)- if err != nil {- common.Error("Failed to create bucket for health check", err, zap.String("bucket", bucket), zap.Error(err))- return false- }- }-- // Try to upload a test object- reader := bytes.NewReader(binary)- _, err := s.client.PutObject(ctx, &s3.PutObjectInput{- Bucket: aws.String(bucket),- Key: aws.String(fnm),- Body: reader,- })-- if err != nil {- common.Error("Health check failed", err, zap.Error(err))- return false- }-- return true-}--// Put uploads an object to S3-func (s *S3Storage) Put(ctx context.Context, bucket, fnm string, binary []byte, tenantID ...string) error {- bucket, fnm = s.resolveBucketAndPath(bucket, fnm)-- for i := 0; i < 2; i++ {- // Ensure bucket exists- if !s.BucketExists(ctx, bucket) {- err := s.createBucket(ctx, bucket)- if err != nil {- if ctxErr := ctx.Err(); ctxErr != nil {- return ctxErr- }- common.Error("Failed to create bucket", err, zap.String("bucket", bucket), zap.Error(err))- s.reconnect(ctx)- if err = sleepOrAbort(ctx, time.Second); err != nil {- return err- }- continue- }- common.Info("Created bucket", zap.String("bucket", bucket))- }-- reader := bytes.NewReader(binary)- _, err := s.client.PutObject(ctx, &s3.PutObjectInput{- Bucket: aws.String(bucket),- Key: aws.String(fnm),- Body: reader,- })- if err != nil {- if ctxErr := ctx.Err(); ctxErr != nil {- return ctxErr- }- common.Error("Failed to put object", err, zap.String("bucket", bucket), zap.String("key", fnm), zap.Error(err))- s.reconnect(ctx)- if err = sleepOrAbort(ctx, time.Second); err != nil {- return err- }- continue- }-- return nil- }-- return fmt.Errorf("failed to put object after retries")-}--// Get retrieves an object from S3-func (s *S3Storage) Get(ctx context.Context, bucket, fnm string, tenantID ...string) ([]byte, error) {- bucket, fnm = s.resolveBucketAndPath(bucket, fnm)-- for i := 0; i < 2; i++ {- // Honor cancellation up front so an already-canceled context returns- // promptly instead of running the reconnect/backoff loop.- if err := ctx.Err(); err != nil {- return nil, err- }- result, err := s.client.GetObject(ctx, &s3.GetObjectInput{- Bucket: aws.String(bucket),- Key: aws.String(fnm),- })- if err != nil {- // The request may have failed because ctx was canceled mid-flight;- // surface that instead of sleeping and retrying with a dead context.- if cerr := ctx.Err(); cerr != nil {- return nil, cerr- }- common.Error("Failed to get object", err, zap.String("bucket", bucket), zap.String("key", fnm), zap.Error(err))- s.reconnect(ctx)- time.Sleep(time.Second)- continue- }- buf := new(bytes.Buffer)-- readErr := func() error {- defer result.Body.Close()- _, err = buf.ReadFrom(result.Body)- return err- }()- if readErr != nil {- if cerr := ctx.Err(); cerr != nil {- return nil, cerr- }- common.Error("Failed to read object data", readErr, zap.String("bucket", bucket), zap.String("key", fnm), zap.Error(readErr))- s.reconnect(ctx)- time.Sleep(time.Second)- continue- }-- return buf.Bytes(), nil- }-- return nil, fmt.Errorf("failed to get object after retries")-}--// Remove removes an object from S3-func (s *S3Storage) Remove(ctx context.Context, bucket, fnm string, tenantID ...string) error {- bucket, fnm = s.resolveBucketAndPath(bucket, fnm)-- _, err := s.client.DeleteObject(ctx, &s3.DeleteObjectInput{- Bucket: aws.String(bucket),- Key: aws.String(fnm),- })- if err != nil {- if isS3NotFound(err) {- return nil- }- common.Error("Failed to remove object", err, zap.String("bucket", bucket), zap.String("key", fnm), zap.Error(err))- return err- }-- return nil-}--// ObjExist checks if an object exists in S3-func (s *S3Storage) ObjExist(ctx context.Context, bucket, fnm string, tenantID ...string) bool {- bucket, fnm = s.resolveBucketAndPath(bucket, fnm)-- _, err := s.client.HeadObject(ctx, &s3.HeadObjectInput{- Bucket: aws.String(bucket),- Key: aws.String(fnm),- })- if err != nil {- if isS3NotFound(err) {- return false- }- return false- }-- return true-}--func (s *S3Storage) ObjectExists(ctx context.Context, bucket, fnm string) (bool, error) {- bucket, fnm = s.resolveBucketAndPath(bucket, fnm)- _, err := s.client.HeadObject(ctx, &s3.HeadObjectInput{Bucket: aws.String(bucket), Key: aws.String(fnm)})- if err == nil {- return true, nil- }- if isS3NotFound(err) {- return false, nil- }- return false, err-}--func (s *S3Storage) ListObjects(ctx context.Context, bucket string, tenantID ...string) ([]string, error) {-- listInput := &s3.ListObjectsV2Input{- Bucket: aws.String(bucket),- }-- result, err := s.client.ListObjectsV2(ctx, listInput)- if err != nil {- return nil, err- }-- var objects []string-- for _, obj := range result.Contents {- objects = append(objects, *obj.Key)- }-- return objects, nil-}--// GetPresignedURL generates a presigned URL for accessing an object-func (s *S3Storage) GetPresignedURL(ctx context.Context, bucket, fnm string, expires time.Duration, tenantID ...string) (string, error) {- bucket, fnm = s.resolveBucketAndPath(bucket, fnm)-- presignClient := s3.NewPresignClient(s.client)-- for i := 0; i < 10; i++ {- req, err := presignClient.PresignGetObject(ctx, &s3.GetObjectInput{- Bucket: aws.String(bucket),- Key: aws.String(fnm),- }, s3.WithPresignExpires(expires))- if err != nil {- common.Error("Failed to generate presigned URL", err, zap.String("bucket", bucket), zap.String("key", fnm), zap.Error(err))- s.reconnect(ctx)- time.Sleep(time.Second)- continue- }-- return req.URL, nil- }-- return "", fmt.Errorf("failed to generate presigned URL after 10 retries")-}--// BucketExists checks if a bucket exists-func (s *S3Storage) BucketExists(ctx context.Context, bucket string) bool {- actualBucket := bucket- if s.bucket != "" {- actualBucket = s.bucket- }-- _, err := s.client.HeadBucket(ctx, &s3.HeadBucketInput{- Bucket: aws.String(actualBucket),- })- if err != nil {- common.Debug("Bucket does not exist or error", zap.String("bucket", actualBucket), zap.Error(err))- return false- }-- return true-}--func (s *S3Storage) BucketExistsWithError(ctx context.Context, bucket string) (bool, error) {- actualBucket, _ := s.resolveBucketAndPrefix(bucket)- return s.bucketExistsForRemoval(ctx, actualBucket)-}-// RemoveBucket removes a bucket and all its objectsfunc (s *S3Storage) RemoveBucket(ctx context.Context, bucket string) error {actualBucket, prefix := s.resolveBucketAndPrefix(bucket)@@ -502,46 +259,6 @@ func (s *S3Storage) deleteObjects(ctx context.Context, bucket string, objects []return nil}-// Copy copies an object from source to destination-func (s *S3Storage) Copy(ctx context.Context, srcBucket, srcPath, destBucket, destPath string) bool {- srcBucket, srcPath = s.resolveBucketAndPath(srcBucket, srcPath)- destBucket, destPath = s.resolveBucketAndPath(destBucket, destPath)-- copySource := httpbinding.EscapePath(fmt.Sprintf("%s/%s", srcBucket, srcPath), false)-- _, err := s.client.CopyObject(ctx, &s3.CopyObjectInput{- Bucket: aws.String(destBucket),- Key: aws.String(destPath),- CopySource: aws.String(copySource),- })- if err != nil {- common.Error("Failed to copy object", err, zap.String("src", copySource), zap.String("dest", fmt.Sprintf("%s/%s", destBucket, destPath)), zap.Error(err))- return false- }-- return true-}--// Move moves an object from source to destination-func (s *S3Storage) Move(ctx context.Context, srcBucket, srcPath, destBucket, destPath string) bool {- if s.Copy(ctx, srcBucket, srcPath, destBucket, destPath) {- if err := s.Remove(ctx, srcBucket, srcPath); err != nil {- common.Error("Failed to remove source object after copy", err, zap.String("bucket", srcBucket), zap.String("key", srcPath), zap.Error(err))- rollbackCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 30*time.Second)- defer cancel()- err = s.Remove(rollbackCtx, destBucket, destPath)- if err != nil {- common.Warn("Failed to roll back copied destination object", zap.String("bucket", destBucket), zap.String("key", destPath), zap.Error(err))- }- return false- }- return true- }- return false-}--func (s *S3Storage) Close() error { return nil }-// isNotFound checks if the error is a not found errorfunc isS3NotFound(err error) bool {if err == nil {diff --git a/internal/storage/s3_test.go b/internal/storage/s3_test.goindex 75662525e3..d05f71b855 100644--- a/internal/storage/s3_test.go+++ b/internal/storage/s3_test.go@@ -2,6 +2,7 @@ package storageimport ("bytes"+ "context""crypto/sha256""fmt""io"@@ -31,7 +32,19 @@ func newS3TestStorage(region string, handler s3RoundTripper) *S3Storage {options.UsePathStyle = trueoptions.RetryMaxAttempts = 1})- return &S3Storage{client: client}+ s := &S3Storage{client: client}+ s.s3Base = s3Base{+ client: func() *s3.Client { return s.client },+ bucket: "",+ prefixPath: "",+ typeName: "s3",+ // Identity folding: these tests exercise the shared surface with no+ // default bucket / prefix configured.+ resolve: func(bucket, fnm string) (string, string) { return bucket, fnm },+ makeBucket: s.createBucket,+ reconnect: func(context.Context) {},+ }+ return s}func s3Response(status int, body string) *http.Response {diff --git a/internal/storage/s3base.go b/internal/storage/s3base.gonew file mode 100644index 0000000000..585851f951--- /dev/null+++ b/internal/storage/s3base.go@@ -0,0 +1,319 @@+//+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.+//++package storage++import (+ "bytes"+ "context"+ "fmt"+ "time"++ "ragflow/internal/common"++ "github.com/aws/aws-sdk-go-v2/aws"+ "github.com/aws/aws-sdk-go-v2/service/s3"+ "github.com/aws/smithy-go/encoding/httpbinding"+ "go.uber.org/zap"+)++// s3Base is the shared implementation for S3-compatible backends built on+// aws-sdk-go-v2 (AWS S3 and Aliyun OSS). A backend embeds it and injects the+// few pieces that genuinely differ between vendors:+//+// - client: a getter returning the CURRENT client, so reconnect swaps+// are visible to shared methods without stale handles;+// - resolve: the bucket/key folding semantics (see doc.go for the+// deliberately non-uniform per-vendor rules);+// - makeBucket: bucket creation (AWS pins a LocationConstraint per region,+// OSS does not);+// - reconnect: rebuild the client after a failed attempt;+// - typeName: the Type() identifier.+//+// Methods that differ structurally between vendors — connect, RemoveBucket,+// RemoveEmptyBucket and the versioned-object removal helpers — stay on the+// concrete backend and override anything the base would otherwise provide.+type s3Base struct {+ client func() *s3.Client+ bucket string+ prefixPath string+ typeName string+ resolve func(bucket, fnm string) (string, string)+ makeBucket func(ctx context.Context, bucket string) error+ reconnect func(ctx context.Context)+}++func (b *s3Base) Type() string { return b.typeName }++// Close is a no-op: the aws-sdk client holds no resources to release.+func (b *s3Base) Close() error { return nil }++// Health checks service availability by uploading a probe object.+func (b *s3Base) Health(ctx context.Context) bool {+ bucket := b.bucket+ if bucket == "" {+ bucket = "health-check-bucket"+ }++ fnm := "txtxtxtxt1"+ if b.prefixPath != "" {+ fnm = fmt.Sprintf("%s/%s", b.prefixPath, fnm)+ }+ binary := []byte("_t@@@1")++ // Ensure bucket exists+ if !b.BucketExists(ctx, bucket) {+ if err := b.makeBucket(ctx, bucket); err != nil {+ common.Error("Failed to create bucket for health check", err, zap.String("bucket", bucket), zap.Error(err))+ return false+ }+ }++ // Try to upload a test object+ reader := bytes.NewReader(binary)+ if _, err := b.client().PutObject(ctx, &s3.PutObjectInput{+ Bucket: aws.String(bucket),+ Key: aws.String(fnm),+ Body: reader,+ }); err != nil {+ common.Error("Health check failed", err, zap.Error(err))+ return false+ }++ return true+}++// Put uploads an object, creating the bucket on first write.+func (b *s3Base) Put(ctx context.Context, bucket, fnm string, binary []byte, tenantID ...string) error {+ bucket, fnm = b.resolve(bucket, fnm)++ return retryObjectOp(ctx, 2, func() { b.reconnect(ctx) }, func() error {+ // Ensure bucket exists+ if !b.BucketExists(ctx, bucket) {+ if err := b.makeBucket(ctx, bucket); err != nil {+ common.Error("Failed to create bucket", err, zap.String("bucket", bucket), zap.Error(err))+ return err+ }+ common.Info("Created bucket", zap.String("bucket", bucket))+ }++ reader := bytes.NewReader(binary)+ if _, err := b.client().PutObject(ctx, &s3.PutObjectInput{+ Bucket: aws.String(bucket),+ Key: aws.String(fnm),+ Body: reader,+ }); err != nil {+ common.Error("Failed to put object", err, zap.String("bucket", bucket), zap.String("key", fnm), zap.Error(err))+ return err+ }++ return nil+ })+}++// Get retrieves an object. Cancellation surfaces promptly per the contract+// pinned by s3_cancel_test.go (retryObjectOp checks ctx before each attempt+// and after each op error, winning over the backoff sleep).+func (b *s3Base) Get(ctx context.Context, bucket, fnm string, tenantID ...string) ([]byte, error) {+ bucket, fnm = b.resolve(bucket, fnm)++ var data []byte+ err := retryObjectOp(ctx, 2, func() { b.reconnect(ctx) }, func() error {+ result, err := b.client().GetObject(ctx, &s3.GetObjectInput{+ Bucket: aws.String(bucket),+ Key: aws.String(fnm),+ })+ if err != nil {+ common.Error("Failed to get object", err, zap.String("bucket", bucket), zap.String("key", fnm), zap.Error(err))+ return err+ }+ buf := new(bytes.Buffer)++ readErr := func() error {+ defer result.Body.Close()+ _, err = buf.ReadFrom(result.Body)+ return err+ }()+ if readErr != nil {+ common.Error("Failed to read object data", readErr, zap.String("bucket", bucket), zap.String("key", fnm), zap.Error(readErr))+ return readErr+ }+ data = buf.Bytes()++ return nil+ })+ if err != nil {+ return nil, err+ }++ return data, nil+}++// Remove removes an object; a missing object/bucket is not an error.+func (b *s3Base) Remove(ctx context.Context, bucket, fnm string, tenantID ...string) error {+ bucket, fnm = b.resolve(bucket, fnm)++ if _, err := b.client().DeleteObject(ctx, &s3.DeleteObjectInput{+ Bucket: aws.String(bucket),+ Key: aws.String(fnm),+ }); err != nil {+ if isS3NotFound(err) {+ return nil+ }+ common.Error("Failed to remove object", err, zap.String("bucket", bucket), zap.String("key", fnm), zap.Error(err))+ return err+ }++ return nil+}++// ObjExist checks if an object exists.+func (b *s3Base) ObjExist(ctx context.Context, bucket, fnm string, tenantID ...string) bool {+ bucket, fnm = b.resolve(bucket, fnm)++ if _, err := b.client().HeadObject(ctx, &s3.HeadObjectInput{+ Bucket: aws.String(bucket),+ Key: aws.String(fnm),+ }); err != nil {+ return false+ }++ return true+}++// ObjectExists distinguishes a missing object from a storage error.+func (b *s3Base) ObjectExists(ctx context.Context, bucket, fnm string) (bool, error) {+ bucket, fnm = b.resolve(bucket, fnm)++ if _, err := b.client().HeadObject(ctx, &s3.HeadObjectInput{+ Bucket: aws.String(bucket),+ Key: aws.String(fnm),+ }); err != nil {+ if isS3NotFound(err) {+ return false, nil+ }+ return false, err+ }++ return true, nil+}++// ListObjects lists the keys in a bucket.+func (b *s3Base) ListObjects(ctx context.Context, bucket string, tenantID ...string) ([]string, error) {+ result, err := b.client().ListObjectsV2(ctx, &s3.ListObjectsV2Input{+ Bucket: aws.String(bucket),+ })+ if err != nil {+ return nil, err+ }++ var objects []string+ for _, obj := range result.Contents {+ objects = append(objects, *obj.Key)+ }++ return objects, nil+}++// GetPresignedURL generates a presigned URL for accessing an object.+func (b *s3Base) GetPresignedURL(ctx context.Context, bucket, fnm string, expires time.Duration, tenantID ...string) (string, error) {+ bucket, fnm = b.resolve(bucket, fnm)++ presignClient := s3.NewPresignClient(b.client())++ var url string+ err := retryObjectOp(ctx, 10, func() { b.reconnect(ctx) }, func() error {+ req, err := presignClient.PresignGetObject(ctx, &s3.GetObjectInput{+ Bucket: aws.String(bucket),+ Key: aws.String(fnm),+ }, s3.WithPresignExpires(expires))+ if err != nil {+ common.Error("Failed to generate presigned URL", err, zap.String("bucket", bucket), zap.String("key", fnm), zap.Error(err))+ return err+ }+ url = req.URL++ return nil+ })+ if err != nil {+ return "", fmt.Errorf("failed to generate presigned URL after 10 retries: %w", err)+ }++ return url, nil+}++// BucketExists checks if a bucket exists.+func (b *s3Base) BucketExists(ctx context.Context, bucket string) bool {+ if b.bucket != "" {+ bucket = b.bucket+ }++ if _, err := b.client().HeadBucket(ctx, &s3.HeadBucketInput{+ Bucket: aws.String(bucket),+ }); err != nil {+ common.Debug("Bucket does not exist or error", zap.String("bucket", bucket), zap.Error(err))+ return false+ }++ return true+}++// BucketExistsWithError distinguishes a missing bucket from a storage error.+func (b *s3Base) BucketExistsWithError(ctx context.Context, bucket string) (bool, error) {+ if b.bucket != "" {+ bucket = b.bucket+ }++ if _, err := b.client().HeadBucket(ctx, &s3.HeadBucketInput{+ Bucket: aws.String(bucket),+ }); err != nil {+ if isS3BucketNotFound(err) {+ return false, nil+ }+ return false, fmt.Errorf("head S3 bucket %q: %w", bucket, err)+ }++ return true, nil+}++// Copy copies an object from source to destination.+func (b *s3Base) Copy(ctx context.Context, srcBucket, srcPath, destBucket, destPath string) bool {+ srcBucket, srcPath = b.resolve(srcBucket, srcPath)+ destBucket, destPath = b.resolve(destBucket, destPath)++ copySource := httpbinding.EscapePath(fmt.Sprintf("%s/%s", srcBucket, srcPath), false)++ if _, err := b.client().CopyObject(ctx, &s3.CopyObjectInput{+ Bucket: aws.String(destBucket),+ Key: aws.String(destPath),+ CopySource: aws.String(copySource),+ }); err != nil {+ common.Error("Failed to copy object", err, zap.String("src", copySource), zap.String("dest", fmt.Sprintf("%s/%s", destBucket, destPath)), zap.Error(err))+ return false+ }++ return true+}++// Move moves an object from source to destination, rolling back the+// destination if the source removal fails.+func (b *s3Base) Move(ctx context.Context, srcBucket, srcPath, destBucket, destPath string) bool {+ return moveWithRollback(ctx, srcBucket, srcPath, destBucket, destPath,+ func() bool { return b.Copy(ctx, srcBucket, srcPath, destBucket, destPath) },+ func(rctx context.Context) error { return b.Remove(rctx, srcBucket, srcPath) },+ func(rctx context.Context) error { return b.Remove(rctx, destBucket, destPath) },+ )+}diff --git a/internal/storage/s3base_test.go b/internal/storage/s3base_test.gonew file mode 100644index 0000000000..b882cefeb4--- /dev/null+++ b/internal/storage/s3base_test.go@@ -0,0 +1,144 @@+//+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.+//++package storage++import (+ "context"+ "fmt"+ "net/http"+ "strings"+ "testing"++ "github.com/aws/aws-sdk-go-v2/aws"+ "github.com/aws/aws-sdk-go-v2/credentials"+ "github.com/aws/aws-sdk-go-v2/service/s3"+)++// Compile-time contract: every backend must satisfy the full Storage+// interface. These pin the embedding-based decomposition — if a future+// refactor drops a method from s3Base or a thin shell, compilation breaks.+var (+ _ Storage = (*S3Storage)(nil)+ _ Storage = (*OSSStorage)(nil)+ _ Storage = (*MinioStorage)(nil)+ _ Storage = (*MemoryStorage)(nil)+ _ Storage = (*GCSStorage)(nil)+)++// newBareS3Base builds an s3Base directly — no S3Storage wrapper — proving+// the base class is self-sufficient: a backend only needs to inject its+// client getter, path resolver, bucket factory, reconnect and type name.+func newBareS3Base(handler s3RoundTripper, typeName string) *s3Base {+ client := s3.NewFromConfig(aws.Config{+ Region: "us-east-1",+ Credentials: aws.NewCredentialsCache(credentials.NewStaticCredentialsProvider("access", "secret", "")),+ HTTPClient: &http.Client{Transport: handler},+ }, func(options *s3.Options) {+ options.BaseEndpoint = aws.String("https://s3.test")+ options.UsePathStyle = true+ options.RetryMaxAttempts = 1+ })+ return &s3Base{+ client: func() *s3.Client { return client },+ typeName: typeName,+ resolve: func(bucket, fnm string) (string, string) { return bucket, fnm },+ makeBucket: func(ctx context.Context, bucket string) error { return nil },+ reconnect: func(context.Context) {},+ }+}++func TestS3BaseStandalonePutAndGet(t *testing.T) {+ base := newBareS3Base(func(req *http.Request) (*http.Response, error) {+ if req.Method == http.MethodPut {+ return s3Response(200, ""), nil+ }+ return s3Response(200, "hello-bucket"), nil+ }, "s3")++ if err := base.Put(context.Background(), "bucket", "key", []byte("data")); err != nil {+ t.Fatalf("Put: %v", err)+ }+ got, err := base.Get(context.Background(), "bucket", "key")+ if err != nil {+ t.Fatalf("Get: %v", err)+ }+ if string(got) != "hello-bucket" {+ t.Fatalf("Get returned %q, want hello-bucket", got)+ }+}++func TestS3BaseTypeReturnsInjectedName(t *testing.T) {+ base := newBareS3Base(func(req *http.Request) (*http.Response, error) {+ return s3Response(200, ""), nil+ }, "oss")+ if got := base.Type(); got != "oss" {+ t.Errorf("Type() = %q, want oss (injected name, not a hardcoded one)", got)+ }+}++func TestS3BasePutSurfacesUnderlyingErrorAfterRetries(t *testing.T) {+ // Put must surface the last underlying error (not a generic string) once+ // attempts are exhausted — the behavior unified in the helper extraction.+ base := newBareS3Base(func(req *http.Request) (*http.Response, error) {+ return nil, fmt.Errorf("simulated transport error")+ }, "s3")++ err := base.Put(context.Background(), "bucket", "key", []byte("data"))+ if err == nil || !strings.Contains(err.Error(), "simulated transport error") {+ t.Fatalf("Put err = %v, want the underlying transport error", err)+ }+}++func TestS3BaseClientGetterSeesReconnect(t *testing.T) {+ // The client getter must return the CURRENT client: after reconnect swaps+ // it, the next attempt must use the replacement, not a stale handle.+ oldClient := s3.NewFromConfig(aws.Config{+ Region: "us-east-1",+ Credentials: aws.NewCredentialsCache(credentials.NewStaticCredentialsProvider("access", "secret", "")),+ HTTPClient: &http.Client{Transport: s3RoundTripper(func(req *http.Request) (*http.Response, error) {+ return nil, fmt.Errorf("old client: offline")+ })},+ }, func(options *s3.Options) {+ options.BaseEndpoint = aws.String("https://s3.test")+ options.UsePathStyle = true+ options.RetryMaxAttempts = 1+ })+ newClient := s3.NewFromConfig(aws.Config{+ Region: "us-east-1",+ Credentials: aws.NewCredentialsCache(credentials.NewStaticCredentialsProvider("access", "secret", "")),+ HTTPClient: &http.Client{Transport: s3RoundTripper(func(req *http.Request) (*http.Response, error) {+ return s3Response(200, ""), nil+ })},+ }, func(options *s3.Options) {+ options.BaseEndpoint = aws.String("https://s3.test")+ options.UsePathStyle = true+ options.RetryMaxAttempts = 1+ })++ var current = oldClient+ base := &s3Base{+ client: func() *s3.Client { return current },+ typeName: "s3",+ resolve: func(bucket, fnm string) (string, string) { return bucket, fnm },+ makeBucket: func(ctx context.Context, bucket string) error { return nil },+ reconnect: func(context.Context) { current = newClient },+ }++ if err := base.Put(context.Background(), "bucket", "key", []byte("data")); err != nil {+ t.Fatalf("Put after reconnect: %v, want nil (second attempt must use the new client)", err)+ }+}diff --git a/internal/storage/storage_common.go b/internal/storage/storage_common.gonew file mode 100644index 0000000000..ca114cefb0--- /dev/null+++ b/internal/storage/storage_common.go@@ -0,0 +1,79 @@+//+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.+//++package storage++import (+ "context"+ "time"++ "ragflow/internal/common"++ "go.uber.org/zap"+)++// retryObjectOp runs op up to attempts times, sleeping one second between+// attempts and invoking onRetry (e.g. a client reconnect) before each sleep.+// It returns nil on the first success and the last error once attempts are+// exhausted. A canceled context short-circuits before an attempt and wins+// over the backoff sleep, so cancellation always surfaces promptly instead+// of costing a sleep window — the contract pinned by s3_cancel_test.go.+func retryObjectOp(ctx context.Context, attempts int, onRetry func(), op func() error) error {+ var lastErr error+ for i := 0; i < attempts; i++ {+ if err := ctx.Err(); err != nil {+ return err+ }+ if err := op(); err != nil {+ lastErr = err+ if cerr := ctx.Err(); cerr != nil {+ return cerr+ }+ if onRetry != nil {+ onRetry()+ }+ if serr := sleepOrAbort(ctx, time.Second); serr != nil {+ return serr+ }+ continue+ }+ return nil+ }+ return lastErr+}++// moveWithRollback implements the shared Move contract across backends: copy+// the object, then remove the source. If the source removal fails, the+// destination is removed best-effort under a detached 30s context derived via+// context.WithoutCancel — so cleanup survives a parent canceled during the+// source removal — and the move reports failure.+func moveWithRollback(ctx context.Context, srcBucket, srcPath, destBucket, destPath string, copyFn func() bool, removeSrc, removeDest func(context.Context) error) bool {+ if !copyFn() {+ return false+ }+ if err := removeSrc(ctx); err != nil {+ common.Warn("Failed to remove source object after copy",+ zap.String("bucket", srcBucket), zap.String("key", srcPath), zap.Error(err))+ rollbackCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 30*time.Second)+ defer cancel()+ if rbErr := removeDest(rollbackCtx); rbErr != nil {+ common.Warn("Failed to roll back copied destination object",+ zap.String("bucket", destBucket), zap.String("key", destPath), zap.Error(rbErr))+ }+ return false+ }+ return true+}diff --git a/internal/storage/storage_common_test.go b/internal/storage/storage_common_test.gonew file mode 100644index 0000000000..e58fb28616--- /dev/null+++ b/internal/storage/storage_common_test.go@@ -0,0 +1,275 @@+//+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.+//++package storage++import (+ "context"+ "errors"+ "fmt"+ "testing"+ "time"+)++// --- retryObjectOp ---++func TestRetryObjectOp_SucceedsFirstAttempt(t *testing.T) {+ var calls, retries int+ err := retryObjectOp(context.Background(), 3, func() { retries++ }, func() error {+ calls+++ return nil+ })+ if err != nil {+ t.Fatalf("err = %v, want nil", err)+ }+ if calls != 1 {+ t.Errorf("op calls = %d, want 1", calls)+ }+ if retries != 0 {+ t.Errorf("onRetry calls = %d, want 0", retries)+ }+}++func TestRetryObjectOp_SucceedsOnLastAttempt(t *testing.T) {+ // Off-by-one guard: with attempts=3 the third attempt must still run.+ var calls int+ err := retryObjectOp(context.Background(), 3, nil, func() error {+ calls+++ if calls < 3 {+ return fmt.Errorf("attempt %d failed", calls)+ }+ return nil+ })+ if err != nil {+ t.Fatalf("err = %v, want nil", err)+ }+ if calls != 3 {+ t.Errorf("op calls = %d, want 3", calls)+ }+}++func TestRetryObjectOp_ReturnsLastErrorAfterAllAttempts(t *testing.T) {+ var calls, retries int+ err := retryObjectOp(context.Background(), 3, func() { retries++ }, func() error {+ calls+++ return fmt.Errorf("boom-%d", calls)+ })+ if err == nil || err.Error() != "boom-3" {+ t.Fatalf("err = %v, want boom-3 (the LAST error, not the first)", err)+ }+ if calls != 3 {+ t.Errorf("op calls = %d, want 3", calls)+ }+ // Legacy loops reconnect after every failed attempt, including the last+ // one before giving up: the refreshed client benefits the NEXT request.+ // The helper preserves that (zero behavior change), so onRetry runs 3x.+ if retries != 3 {+ t.Errorf("onRetry calls = %d, want 3 (after every failed attempt)", retries)+ }+}++func TestRetryObjectOp_CanceledContextNeverRunsOp(t *testing.T) {+ ctx, cancel := context.WithCancel(context.Background())+ cancel()++ var calls int+ err := retryObjectOp(ctx, 3, nil, func() error {+ calls+++ return nil+ })+ if !errors.Is(err, context.Canceled) {+ t.Fatalf("err = %v, want context.Canceled", err)+ }+ if calls != 0 {+ t.Errorf("op calls = %d, want 0 (canceled ctx must short-circuit before any attempt)", calls)+ }+}++func TestRetryObjectOp_CancelDuringBackoffSurfacesPromptly(t *testing.T) {+ // The op itself cancels the context while failing: the next loop's ctx+ // check must surface context.Canceled without sleeping the backoff window.+ ctx, cancel := context.WithCancel(context.Background())+ var calls int+ start := time.Now()+ err := retryObjectOp(ctx, 3, nil, func() error {+ calls+++ cancel()+ return fmt.Errorf("fail after cancel")+ })+ elapsed := time.Since(start)+ if !errors.Is(err, context.Canceled) {+ t.Fatalf("err = %v, want context.Canceled", err)+ }+ if calls != 1 {+ t.Errorf("op calls = %d, want 1", calls)+ }+ if elapsed > 500*time.Millisecond {+ t.Errorf("took %v; want < 500ms (no backoff sleep on a dead ctx)", elapsed)+ }+}++func TestRetryObjectOp_ContextCanceledAfterOpErrorSkipsSleep(t *testing.T) {+ // ctx is canceled between the op failure and the backoff sleep: the+ // post-error ctx check must win over sleepOrAbort.+ ctx, cancel := context.WithCancel(context.Background())+ start := time.Now()+ err := retryObjectOp(ctx, 3, nil, func() error {+ cancel() // cancel as a side effect of the failing op+ return fmt.Errorf("transport error")+ })+ elapsed := time.Since(start)+ if !errors.Is(err, context.Canceled) {+ t.Fatalf("err = %v, want context.Canceled", err)+ }+ if elapsed > 500*time.Millisecond {+ t.Errorf("took %v; want < 500ms", elapsed)+ }+}++func TestRetryObjectOp_SingleAttemptNoRetry(t *testing.T) {+ var calls, retries int+ err := retryObjectOp(context.Background(), 1, func() { retries++ }, func() error {+ calls+++ return fmt.Errorf("once")+ })+ if err == nil || err.Error() != "once" {+ t.Fatalf("err = %v, want once", err)+ }+ if calls != 1 || retries != 1 {+ t.Errorf("calls=%d retries=%d, want 1/1", calls, retries)+ }+}++func TestRetryObjectOp_ZeroAttemptsIsNoOp(t *testing.T) {+ // Boundary guard: matches the old `for i := 0; i < 0` loops, which never+ // ran and returned the nil lastErr.+ var calls int+ err := retryObjectOp(context.Background(), 0, nil, func() error {+ calls+++ return nil+ })+ if err != nil {+ t.Fatalf("err = %v, want nil", err)+ }+ if calls != 0 {+ t.Errorf("op calls = %d, want 0", calls)+ }+}++// --- moveWithRollback ---++func TestMoveWithRollback_CopyThenRemoveSucceeds(t *testing.T) {+ var copyCalls, removeSrc, removeDest int+ ok := moveWithRollback(context.Background(),+ "srcb", "srck", "dstb", "dstk",+ func() bool { copyCalls++; return true },+ func(context.Context) error { removeSrc++; return nil },+ func(context.Context) error { removeDest++; return nil },+ )+ if !ok {+ t.Fatal("ok = false, want true")+ }+ if copyCalls != 1 || removeSrc != 1 || removeDest != 0 {+ t.Errorf("copy=%d removeSrc=%d removeDest=%d, want 1/1/0", copyCalls, removeSrc, removeDest)+ }+}++func TestMoveWithRollback_CopyFailureLeavesNoSideEffects(t *testing.T) {+ var removeSrc, removeDest int+ ok := moveWithRollback(context.Background(),+ "srcb", "srck", "dstb", "dstk",+ func() bool { return false },+ func(context.Context) error { removeSrc++; return nil },+ func(context.Context) error { removeDest++; return nil },+ )+ if ok {+ t.Fatal("ok = true, want false")+ }+ if removeSrc != 0 || removeDest != 0 {+ t.Errorf("removeSrc=%d removeDest=%d, want 0/0 (no removes when copy fails)", removeSrc, removeDest)+ }+}++func TestMoveWithRollback_SourceRemoveFailureRollsBackDestination(t *testing.T) {+ var removeDest int+ var rollbackCtxErr error+ var hasDeadline bool+ ok := moveWithRollback(context.Background(),+ "srcb", "srck", "dstb", "dstk",+ func() bool { return true },+ func(context.Context) error { return fmt.Errorf("source delete failed") },+ func(ctx context.Context) error {+ removeDest+++ rollbackCtxErr = ctx.Err()+ _, hasDeadline = ctx.Deadline()+ return nil+ },+ )+ if ok {+ t.Fatal("ok = true, want false")+ }+ if removeDest != 1 {+ t.Fatalf("removeDest calls = %d, want 1 (rollback must run)", removeDest)+ }+ if rollbackCtxErr != nil {+ t.Errorf("rollback ctx err = %v, want nil", rollbackCtxErr)+ }+ if !hasDeadline {+ t.Error("rollback ctx has no deadline, want a 30s timeout")+ }+}++func TestMoveWithRollback_RollbackRunsEvenWhenParentCanceled(t *testing.T) {+ // The rollback path derives from context.WithoutCancel: a parent canceled+ // during the source removal must not skip the destination cleanup.+ ctx, cancel := context.WithCancel(context.Background())+ var removeDest int+ var rollbackCtxErr error+ ok := moveWithRollback(ctx,+ "srcb", "srck", "dstb", "dstk",+ func() bool { return true },+ func(context.Context) error {+ cancel() // parent dies while removing the source+ return fmt.Errorf("source delete failed")+ },+ func(rctx context.Context) error {+ removeDest+++ rollbackCtxErr = rctx.Err()+ return nil+ },+ )+ if ok {+ t.Fatal("ok = true, want false")+ }+ if removeDest != 1 {+ t.Fatalf("removeDest calls = %d, want 1 (rollback must survive parent cancellation)", removeDest)+ }+ if rollbackCtxErr != nil {+ t.Errorf("rollback ctx err = %v, want nil (WithoutCancel)", rollbackCtxErr)+ }+}++func TestMoveWithRollback_RollbackFailureStillReportsFailure(t *testing.T) {+ ok := moveWithRollback(context.Background(),+ "srcb", "srck", "dstb", "dstk",+ func() bool { return true },+ func(context.Context) error { return fmt.Errorf("src") },+ func(context.Context) error { return fmt.Errorf("dest") },+ )+ if ok {+ t.Fatal("ok = true, want false (a failed rollback must not report success)")+ }+}