Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

Extract shared storage layer: s3Base + retry/move helpers (#21158)

xugangqiang322b72682517
diff --git a/internal/storage/copy_source_test.go b/internal/storage/copy_source_test.go
index 52df2bf9e9..eaf2636c79 100644
--- a/internal/storage/copy_source_test.go
+++ b/internal/storage/copy_source_test.go
@@ -36,9 +36,29 @@ func newCopySourceTestStorage(backend, endpoint, bucket, prefix string) copySour
options.RetryMaxAttempts = 1
})
if backend == "oss" {
- return &OSSStorage{client: client, bucket: bucket, prefixPath: prefix}
+ o := &OSSStorage{client: client}
+ o.s3Base = s3Base{
+ client: func() *s3.Client { return o.client },
+ bucket: bucket,
+ prefixPath: prefix,
+ typeName: "oss",
+ resolve: o.resolveBucketAndPath,
+ makeBucket: o.createBucket,
+ reconnect: func(context.Context) {},
+ }
+ return o
+ }
+ s := &S3Storage{client: client}
+ s.s3Base = s3Base{
+ client: func() *s3.Client { return s.client },
+ bucket: bucket,
+ prefixPath: prefix,
+ typeName: "s3",
+ resolve: s.resolveBucketAndPath,
+ makeBucket: s.createBucket,
+ reconnect: func(context.Context) {},
}
- return &S3Storage{client: client, bucket: bucket, prefixPath: prefix}
+ return s
}
func TestStorageCopyEncodesSource(t *testing.T) {
diff --git a/internal/storage/doc.go b/internal/storage/doc.go
new file mode 100644
index 0000000000..a490d13f37
--- /dev/null
+++ b/internal/storage/doc.go
@@ -0,0 +1,61 @@
+//
+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+// Package storage abstracts object storage behind the Storage interface.
+//
+// Layout: s3base.go holds the shared implementation for aws-sdk-go-v2
+// backends (s3.go = AWS S3, oss.go = Aliyun OSS); storage_common.go holds
+// the backend-agnostic retry/move helpers used by every backend including
+// minio.go (minio-go SDK), gcs.go (Google SDK) and memory.go (test double).
+//
+// # Deliberately non-uniform path folding
+//
+// resolveBucketAndPath intentionally differs per backend. The pre-migration
+// Python connectors (rag/utils/minio_conn.py and rag/utils/s3_conn.py, both
+// removed in #20289) folded object keys differently — minio only inserted
+// the logical bucket as a path segment in single-bucket mode, while s3
+// inserted it whenever a prefix was configured — and the Go ports preserved
+// each vendor's semantics so that keys written before the migration keep
+// resolving after it. The table below is the contract; do not "unify" it.
+//
+// | minio | s3 | oss
+// ----------------------+--------------------+---------------------+---------------------
+// bucket=="" prefix=="" | fnm | fnm | fnm
+// bucket=="" prefix!="" | prefix/fnm | prefix/bucket/fnm | prefix/fnm
+// bucket!="" prefix=="" | bucket/fnm | bucket/fnm | fnm
+// bucket!="" prefix!="" | prefix/bucket/fnm | prefix/bucket/fnm | prefix/fnm
+//
+// GCS and the in-memory backend perform no folding at all.
+//
+// Consequences to be aware of:
+//
+// - Switching STORAGE_ENGINE does not migrate existing object keys; the
+// same (bucket, fnm) argument resolves to different keys on different
+// backends. Treat the engine as fixed per deployment.
+// - In shared-bucket mode OSS cannot scope a removal to a logical bucket,
+// so RemoveBucket/RemoveEmptyBucket refuse rather than guess.
+//
+// # Deliberately non-uniform health checks
+//
+// MinioStorage.Health is a lightweight liveness probe (client HealthCheck +
+// IsOnline, no writes). The s3Base health check uploads a probe object to
+// <bucket or "health-check-bucket">/<prefix>/txtxtxtxt1, matching the
+// pre-migration Python S3 connector. The difference is intentional: a
+// write-based probe also verifies write permissions, but it requires them,
+// so read-only credentials make MinIO-style and S3-style deployments report
+// health differently. Callers must not treat a false from one engine as
+// comparable to a false from the other.
+package storage
diff --git a/internal/storage/memory.go b/internal/storage/memory.go
index 10d3edc8aa..cb30d519f8 100644
--- a/internal/storage/memory.go
+++ b/internal/storage/memory.go
@@ -20,11 +20,8 @@ import (
"context"
"errors"
"fmt"
- "ragflow/internal/common"
"sync"
"time"
-
- "go.uber.org/zap"
)
// ErrMemoryNotFound is returned when a key does not exist in the in-memory backend.
@@ -265,24 +262,15 @@ func (m *MemoryStorage) Copy(ctx context.Context, srcBucket, srcPath, destBucket
// Move transfers an object to a new location, deleting the source on success.
// Returns false if the source does not exist or the copy step fails.
func (m *MemoryStorage) Move(ctx context.Context, srcBucket, srcPath, destBucket, destPath string) bool {
-
if err := ctx.Err(); err != nil {
return false
}
- if !m.Copy(ctx, srcBucket, srcPath, destBucket, destPath) {
- return false
- }
- if err := m.Remove(ctx, srcBucket, srcPath); err != nil {
- rollbackCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 30*time.Second)
- defer cancel()
- err = m.Remove(rollbackCtx, destBucket, destPath)
- if err != nil {
- common.Warn("Failed to roll back copied destination object", zap.String("bucket", destBucket), zap.String("key", destPath), zap.Error(err))
- }
- return false
- }
- return true
+ return moveWithRollback(ctx, srcBucket, srcPath, destBucket, destPath,
+ func() bool { return m.Copy(ctx, srcBucket, srcPath, destBucket, destPath) },
+ func(rctx context.Context) error { return m.Remove(rctx, srcBucket, srcPath) },
+ func(rctx context.Context) error { return m.Remove(rctx, destBucket, destPath) },
+ )
}
func (m *MemoryStorage) Close() error { return nil }
diff --git a/internal/storage/minio.go b/internal/storage/minio.go
index b48085b617..104ad2e339 100644
--- a/internal/storage/minio.go
+++ b/internal/storage/minio.go
@@ -122,76 +122,43 @@ func (m *MinioStorage) Health(ctx context.Context) bool {
func (m *MinioStorage) Put(ctx context.Context, bucket, fnm string, binary []byte, tenantID ...string) error {
bucket, fnm = m.resolveBucketAndPath(bucket, fnm)
- var lastErr error
-
- for i := 0; i < 3; i++ {
+ return retryObjectOp(ctx, 3, nil, func() error {
// Ensure bucket exists
if m.bucket == "" {
exists, err := m.client.BucketExists(ctx, bucket)
if err != nil {
- lastErr = err
- if ctxErr := ctx.Err(); ctxErr != nil {
- return ctxErr
- }
common.Warn("Failed to check bucket existence", zap.String("bucket", bucket), zap.Error(err))
- if sleepErr := sleepOrAbort(ctx, time.Second); sleepErr != nil {
- return sleepErr
- }
- continue
+ return err
}
if !exists {
if err := m.client.MakeBucket(ctx, bucket, minio.MakeBucketOptions{}); err != nil {
- lastErr = err
- if ctxErr := ctx.Err(); ctxErr != nil {
- return ctxErr
- }
common.Warn("Failed to create bucket", zap.String("bucket", bucket), zap.Error(err))
- if sleepErr := sleepOrAbort(ctx, time.Second); sleepErr != nil {
- return sleepErr
- }
- continue
+ return err
}
}
}
reader := bytes.NewReader(binary)
- _, err := m.client.PutObject(ctx, bucket, fnm, reader, int64(len(binary)), minio.PutObjectOptions{})
- if err != nil {
+ if _, err := m.client.PutObject(ctx, bucket, fnm, reader, int64(len(binary)), minio.PutObjectOptions{}); err != nil {
const warnMessage = "Failed to put object"
- lastErr = fmt.Errorf("%s: %w", warnMessage, err)
- if ctxErr := ctx.Err(); ctxErr != nil {
- return ctxErr
- }
common.Warn(warnMessage, zap.String("bucket", bucket), zap.String("key", fnm), zap.Error(err))
- if sleepErr := sleepOrAbort(ctx, time.Second); sleepErr != nil {
- return sleepErr
- }
- continue
+ return fmt.Errorf("%s: %w", warnMessage, err)
}
return nil
- }
-
- return lastErr
+ })
}
// Get retrieves an object from MinIO
func (m *MinioStorage) Get(ctx context.Context, bucket, fnm string, tenantID ...string) ([]byte, error) {
bucket, fnm = m.resolveBucketAndPath(bucket, fnm)
- var lastErr error
- for i := 0; i < 2; i++ {
+ var data []byte
+ err := retryObjectOp(ctx, 2, nil, func() error {
obj, err := m.client.GetObject(ctx, bucket, fnm, minio.GetObjectOptions{})
if err != nil {
- lastErr = err
- if ctxErr := ctx.Err(); ctxErr != nil {
- return nil, ctxErr
- }
common.Warn("failed to get object", zap.String("bucket", bucket), zap.String("key", fnm), zap.Error(err))
- if sleepErr := sleepOrAbort(ctx, time.Second); sleepErr != nil {
- return nil, sleepErr
- }
- continue
+ return err
}
buf := new(bytes.Buffer)
@@ -201,21 +168,18 @@ func (m *MinioStorage) Get(ctx context.Context, bucket, fnm string, tenantID ...
return err
}()
if readErr != nil {
- lastErr = readErr
- if ctxErr := ctx.Err(); ctxErr != nil {
- return nil, ctxErr
- }
common.Error("failed to read object data", err, zap.String("bucket", bucket), zap.String("key", fnm))
- if sleepErr := sleepOrAbort(ctx, time.Second); sleepErr != nil {
- return nil, sleepErr
- }
- continue
+ return readErr
}
+ data = buf.Bytes()
- return buf.Bytes(), nil
+ return nil
+ })
+ if err != nil {
+ return nil, err
}
- return nil, lastErr
+ return data, nil
}
// Remove removes an object from MinIO
@@ -273,23 +237,22 @@ func (m *MinioStorage) ObjectExists(ctx context.Context, bucket, fnm string) (bo
func (m *MinioStorage) GetPresignedURL(ctx context.Context, bucket, fnm string, expires time.Duration, tenantID ...string) (string, error) {
bucket, fnm = m.resolveBucketAndPath(bucket, fnm)
- for i := 0; i < 10; i++ {
- url, err := m.client.PresignedGetObject(ctx, bucket, fnm, expires, nil)
+ var url string
+ err := retryObjectOp(ctx, 10, nil, func() error {
+ u, err := m.client.PresignedGetObject(ctx, bucket, fnm, expires, nil)
if err != nil {
- if ctxErr := ctx.Err(); ctxErr != nil {
- return "", ctxErr
- }
common.Warn("Failed to get presigned URL", zap.String("bucket", bucket), zap.String("key", fnm), zap.Error(err))
- if err = sleepOrAbort(ctx, time.Second); err != nil {
- return "", err
- }
- continue
+ return err
}
+ url = u.String()
- return url.String(), nil
+ return nil
+ })
+ if err != nil {
+ return "", fmt.Errorf("failed to get presigned URL after 10 retries: %w", err)
}
- return "", fmt.Errorf("failed to get presigned URL after 10 retries")
+ return url, nil
}
// BucketExists checks if a bucket exists
@@ -496,20 +459,11 @@ func (m *MinioStorage) Copy(ctx context.Context, srcBucket, srcPath, destBucket,
// Move moves an object from source to destination
func (m *MinioStorage) Move(ctx context.Context, srcBucket, srcPath, destBucket, destPath string) bool {
- if m.Copy(ctx, srcBucket, srcPath, destBucket, destPath) {
- if err := m.Remove(ctx, srcBucket, srcPath); err != nil {
- common.Warn("Failed to remove source object after copy", zap.String("bucket", srcBucket), zap.String("key", srcPath), zap.Error(err))
- rollbackCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 30*time.Second)
- defer cancel()
- err = m.Remove(rollbackCtx, destBucket, destPath)
- if err != nil {
- common.Warn("Failed to roll back copied destination object", zap.String("bucket", destBucket), zap.String("key", destPath), zap.Error(err))
- }
- return false
- }
- return true
- }
- return false
+ return moveWithRollback(ctx, srcBucket, srcPath, destBucket, destPath,
+ func() bool { return m.Copy(ctx, srcBucket, srcPath, destBucket, destPath) },
+ func(rctx context.Context) error { return m.Remove(rctx, srcBucket, srcPath) },
+ func(rctx context.Context) error { return m.Remove(rctx, destBucket, destPath) },
+ )
}
func (m *MinioStorage) Close() error { return nil }
diff --git a/internal/storage/oss.go b/internal/storage/oss.go
index 6a9209d5e3..d940cb1367 100644
--- a/internal/storage/oss.go
+++ b/internal/storage/oss.go
@@ -17,44 +17,49 @@
package storage
import (
- "bytes"
"context"
- "errors"
"fmt"
"ragflow/internal/common"
"ragflow/internal/server/config"
- "time"
"github.com/aws/aws-sdk-go-v2/aws"
s3Config "github.com/aws/aws-sdk-go-v2/config"
"github.com/aws/aws-sdk-go-v2/credentials"
"github.com/aws/aws-sdk-go-v2/service/s3"
- "github.com/aws/smithy-go"
- "github.com/aws/smithy-go/encoding/httpbinding"
"go.uber.org/zap"
)
-// OSSStorage implements Storage interface for Aliyun OSS
-// OSS uses S3-compatible API
+// OSSStorage implements Storage for Aliyun OSS. The shared S3-compatible
+// surface lives in s3Base; this file holds only what differs for OSS: the
+// connection setup, the path-folding semantics (no logical-bucket segment —
+// see doc.go), and the refusal to remove logical buckets in shared-bucket
+// mode.
type OSSStorage struct {
- client *s3.Client
- bucket string
- prefixPath string
- config config.OSSConfig
+ s3Base
+ client *s3.Client // owned by connect/reconnect; the base reads it via its getter
+ config config.OSSConfig
}
// NewOSSStorage creates a new OSS storage instance
func NewOSSStorage(ctx context.Context, config config.OSSConfig) (*OSSStorage, error) {
storage := &OSSStorage{
- bucket: config.Bucket,
- prefixPath: config.PrefixPath,
- config: config,
+ config: config,
}
if err := storage.connect(ctx); err != nil {
return nil, err
}
+ storage.s3Base = s3Base{
+ client: func() *s3.Client { return storage.client },
+ bucket: config.Bucket,
+ prefixPath: config.PrefixPath,
+ typeName: "oss",
+ resolve: storage.resolveBucketAndPath,
+ makeBucket: storage.createBucket,
+ reconnect: storage.reconnect,
+ }
+
return storage, nil
}
@@ -104,277 +109,16 @@ func (o *OSSStorage) resolveBucketAndPath(bucket, fnm string) (string, string) {
return actualBucket, actualPath
}
-func (o *OSSStorage) Type() string { return "oss" }
-
-// Health checks OSS service availability
-func (o *OSSStorage) Health(ctx context.Context) bool {
- bucket := o.bucket
- if bucket == "" {
- bucket = "health-check-bucket"
- }
-
- fnm := "txtxtxtxt1"
- if o.prefixPath != "" {
- fnm = fmt.Sprintf("%s/%s", o.prefixPath, fnm)
- }
- binary := []byte("_t@@@1")
-
- // Ensure bucket exists
- if !o.BucketExists(ctx, bucket) {
- _, err := o.client.CreateBucket(ctx, &s3.CreateBucketInput{
- Bucket: aws.String(bucket),
- })
- if err != nil {
- common.Error("Failed to create bucket for health check", err, zap.String("bucket", bucket))
- return false
- }
- }
-
- // Try to upload a test object
- reader := bytes.NewReader(binary)
- _, err := o.client.PutObject(ctx, &s3.PutObjectInput{
- Bucket: aws.String(bucket),
- Key: aws.String(fnm),
- Body: reader,
- })
-
- if err != nil {
- common.Error("Health check failed", err)
- return false
- }
-
- return true
-}
-
-// Put uploads an object to OSS
-func (o *OSSStorage) Put(ctx context.Context, bucket, fnm string, binary []byte, tenantID ...string) error {
- bucket, fnm = o.resolveBucketAndPath(bucket, fnm)
- var lastErr error
-
- for i := 0; i < 2; i++ {
- // Ensure bucket exists
- if !o.BucketExists(ctx, bucket) {
- _, err := o.client.CreateBucket(ctx, &s3.CreateBucketInput{
- Bucket: aws.String(bucket),
- })
- if err != nil {
- lastErr = err
- if ctxErr := ctx.Err(); ctxErr != nil {
- return ctxErr
- }
- common.Error("Failed to create bucket", err, zap.String("bucket", bucket))
- o.reconnect(ctx)
- if sleepErr := sleepOrAbort(ctx, time.Second); sleepErr != nil {
- return sleepErr
- }
- continue
- }
- common.Info("Created bucket", zap.String("bucket", bucket))
- }
-
- reader := bytes.NewReader(binary)
- _, err := o.client.PutObject(ctx, &s3.PutObjectInput{
- Bucket: aws.String(bucket),
- Key: aws.String(fnm),
- Body: reader,
- })
- if err != nil {
- lastErr = err
- if ctxErr := ctx.Err(); ctxErr != nil {
- return ctxErr
- }
- common.Error("Failed to put object", err, zap.String("bucket", bucket), zap.String("key", fnm))
- o.reconnect(ctx)
- if sleepErr := sleepOrAbort(ctx, time.Second); sleepErr != nil {
- return sleepErr
- }
- continue
- }
-
- return nil
- }
-
- return lastErr
-}
-
-// Get retrieves an object from OSS
-func (o *OSSStorage) Get(ctx context.Context, bucket, fnm string, tenantID ...string) ([]byte, error) {
- bucket, fnm = o.resolveBucketAndPath(bucket, fnm)
- var lastErr error
-
- for i := 0; i < 2; i++ {
- result, err := o.client.GetObject(ctx, &s3.GetObjectInput{
- Bucket: aws.String(bucket),
- Key: aws.String(fnm),
- })
- if err != nil {
- lastErr = err
- if ctxErr := ctx.Err(); ctxErr != nil {
- return nil, ctxErr
- }
- common.Error("Failed to get object", err, zap.String("bucket", bucket), zap.String("key", fnm))
- o.reconnect(ctx)
- if sleepErr := sleepOrAbort(ctx, time.Second); sleepErr != nil {
- return nil, sleepErr
- }
- continue
- }
- buf := new(bytes.Buffer)
-
- readErr := func() error {
- defer result.Body.Close()
- _, err = buf.ReadFrom(result.Body)
- return err
- }()
- if readErr != nil {
- lastErr = readErr
- if ctxErr := ctx.Err(); ctxErr != nil {
- return nil, ctxErr
- }
- common.Error("Failed to read object data", readErr, zap.String("bucket", bucket), zap.String("key", fnm))
- o.reconnect(ctx)
- if sleepErr := sleepOrAbort(ctx, time.Second); sleepErr != nil {
- return nil, sleepErr
- }
- continue
- }
-
- return buf.Bytes(), nil
- }
-
- return nil, lastErr
-}
-
-// Remove removes an object from OSS
-func (o *OSSStorage) Remove(ctx context.Context, bucket, fnm string, tenantID ...string) error {
- bucket, fnm = o.resolveBucketAndPath(bucket, fnm)
-
- _, err := o.client.DeleteObject(ctx, &s3.DeleteObjectInput{
- Bucket: aws.String(bucket),
- Key: aws.String(fnm),
- })
- if err != nil {
- if isS3NotFound(err) {
- return nil
- }
- common.Error("Failed to remove object", err, zap.String("bucket", bucket), zap.String("key", fnm))
- return err
- }
-
- return nil
-}
-
-// ObjExist checks if an object exists in OSS
-func (o *OSSStorage) ObjExist(ctx context.Context, bucket, fnm string, tenantID ...string) bool {
- bucket, fnm = o.resolveBucketAndPath(bucket, fnm)
-
- _, err := o.client.HeadObject(ctx, &s3.HeadObjectInput{
- Bucket: aws.String(bucket),
- Key: aws.String(fnm),
- })
- if err != nil {
- if isOSSNotFound(err) {
- return false
- }
- return false
- }
-
- return true
-}
-
-func (o *OSSStorage) ObjectExists(ctx context.Context, bucket, fnm string) (bool, error) {
- bucket, fnm = o.resolveBucketAndPath(bucket, fnm)
- _, err := o.client.HeadObject(ctx, &s3.HeadObjectInput{Bucket: aws.String(bucket), Key: aws.String(fnm)})
- if err == nil {
- return true, nil
- }
- if isS3NotFound(err) {
- return false, nil
- }
- return false, err
-}
-
-func (o *OSSStorage) ListObjects(ctx context.Context, bucket string, tenantID ...string) ([]string, error) {
-
- listInput := &s3.ListObjectsV2Input{
+func (o *OSSStorage) createBucket(ctx context.Context, bucket string) error {
+ _, err := o.client.CreateBucket(ctx, &s3.CreateBucketInput{
Bucket: aws.String(bucket),
- }
-
- result, err := o.client.ListObjectsV2(ctx, listInput)
- if err != nil {
- return nil, err
- }
-
- var objects []string
-
- for _, obj := range result.Contents {
- objects = append(objects, *obj.Key)
- }
-
- return objects, nil
-}
-
-func (o *OSSStorage) GetPresignedURL(ctx context.Context, bucket, fnm string, expires time.Duration, tenantID ...string) (string, error) {
- bucket, fnm = o.resolveBucketAndPath(bucket, fnm)
-
- presignClient := s3.NewPresignClient(o.client)
-
- for i := 0; i < 10; i++ {
- req, err := presignClient.PresignGetObject(ctx, &s3.GetObjectInput{
- Bucket: aws.String(bucket),
- Key: aws.String(fnm),
- }, s3.WithPresignExpires(expires))
- if err != nil {
- if ctxErr := ctx.Err(); ctxErr != nil {
- return "", ctxErr
- }
- common.Error("Failed to generate presigned URL", err, zap.String("bucket", bucket), zap.String("key", fnm))
- o.reconnect(ctx)
- if err = sleepOrAbort(ctx, time.Second); err != nil {
- return "", err
- }
- continue
- }
-
- return req.URL, nil
- }
-
- return "", fmt.Errorf("failed to generate presigned URL after 10 retries")
-}
-
-// BucketExists checks if a bucket exists
-func (o *OSSStorage) BucketExists(ctx context.Context, bucket string) bool {
- actualBucket := bucket
- if o.bucket != "" {
- actualBucket = o.bucket
- }
-
- _, err := o.client.HeadBucket(ctx, &s3.HeadBucketInput{
- Bucket: aws.String(actualBucket),
})
- if err != nil {
- common.Error("Bucket does not exist or error", err, zap.String("bucket", actualBucket))
- return false
- }
-
- return true
-}
-
-func (o *OSSStorage) BucketExistsWithError(ctx context.Context, bucket string) (bool, error) {
- if o.bucket != "" {
- bucket = o.bucket
- }
- _, err := o.client.HeadBucket(ctx, &s3.HeadBucketInput{Bucket: aws.String(bucket)})
- if err == nil {
- return true, nil
- }
- if isS3BucketNotFound(err) {
- return false, nil
- }
- return false, err
+ return err
}
-// RemoveBucket removes a bucket and all its objects
+// RemoveBucket removes a bucket and all its objects. In shared-bucket mode a
+// logical bucket has no physical isolation, so removal is refused rather than
+// silently clearing the shared prefix.
func (o *OSSStorage) RemoveBucket(ctx context.Context, bucket string) error {
if o.bucket != "" {
return fmt.Errorf("cannot remove logical OSS bucket %s: shared bucket mode does not isolate objects by logical bucket", bucket)
@@ -400,11 +144,10 @@ func (o *OSSStorage) RemoveBucket(ctx context.Context, bucket string) error {
}
for _, obj := range result.Contents {
- _, err = o.client.DeleteObject(ctx, &s3.DeleteObjectInput{
+ if _, err = o.client.DeleteObject(ctx, &s3.DeleteObjectInput{
Bucket: aws.String(actualBucket),
Key: obj.Key,
- })
- if err != nil {
+ }); err != nil {
common.Error("Failed to delete object", err, zap.String("bucket", actualBucket))
}
}
@@ -416,10 +159,9 @@ func (o *OSSStorage) RemoveBucket(ctx context.Context, bucket string) error {
}
// Delete bucket
- _, err := o.client.DeleteBucket(ctx, &s3.DeleteBucketInput{
+ if _, err := o.client.DeleteBucket(ctx, &s3.DeleteBucketInput{
Bucket: aws.String(actualBucket),
- })
- if err != nil {
+ }); err != nil {
common.Error("Failed to delete bucket", err, zap.String("bucket", actualBucket))
return err
}
@@ -427,7 +169,8 @@ func (o *OSSStorage) RemoveBucket(ctx context.Context, bucket string) error {
return nil
}
-// RemoveEmptyBucket removes a physical bucket only when it is empty.
+// RemoveEmptyBucket removes a physical bucket only when it is empty. In
+// shared-bucket mode a logical bucket cannot be removed at all.
func (o *OSSStorage) RemoveEmptyBucket(ctx context.Context, bucket string) error {
if o.bucket != "" {
return fmt.Errorf("cannot remove logical OSS bucket %s: shared bucket mode does not isolate objects by logical bucket", bucket)
@@ -438,55 +181,3 @@ func (o *OSSStorage) RemoveEmptyBucket(ctx context.Context, bucket string) error
}
return err
}
-
-// Copy copies an object from source to destination
-func (o *OSSStorage) Copy(ctx context.Context, srcBucket, srcPath, destBucket, destPath string) bool {
- srcBucket, srcPath = o.resolveBucketAndPath(srcBucket, srcPath)
- destBucket, destPath = o.resolveBucketAndPath(destBucket, destPath)
-
- copySource := httpbinding.EscapePath(fmt.Sprintf("%s/%s", srcBucket, srcPath), false)
-
- _, err := o.client.CopyObject(ctx, &s3.CopyObjectInput{
- Bucket: aws.String(destBucket),
- Key: aws.String(destPath),
- CopySource: aws.String(copySource),
- })
- if err != nil {
- common.Error("Failed to copy object", err, zap.String("src", copySource), zap.String("dest", fmt.Sprintf("%s/%s", destBucket, destPath)))
- return false
- }
-
- return true
-}
-
-// Move moves an object from source to destination
-func (o *OSSStorage) Move(ctx context.Context, srcBucket, srcPath, destBucket, destPath string) bool {
- if o.Copy(ctx, srcBucket, srcPath, destBucket, destPath) {
- if err := o.Remove(ctx, srcBucket, srcPath); err != nil {
- common.Error("Failed to remove source object after copy", err, zap.String("bucket", srcBucket), zap.String("key", srcPath))
- rollbackCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 30*time.Second)
- defer cancel()
- err = o.Remove(rollbackCtx, destBucket, destPath)
- if err != nil {
- common.Warn("Failed to roll back copied destination object", zap.String("bucket", destBucket), zap.String("key", destPath), zap.Error(err))
- }
- return false
- }
- return true
- }
- return false
-}
-
-func (o *OSSStorage) Close() error { return nil }
-
-// Helper functions
-func isOSSNotFound(err error) bool {
- if err == nil {
- return false
- }
- var apiErr smithy.APIError
- if errors.As(err, &apiErr) {
- return apiErr.ErrorCode() == "NotFound" || apiErr.ErrorCode() == "404" || apiErr.ErrorCode() == "NoSuchKey"
- }
- return false
-}
diff --git a/internal/storage/oss_test.go b/internal/storage/oss_test.go
new file mode 100644
index 0000000000..9c25cdadf4
--- /dev/null
+++ b/internal/storage/oss_test.go
@@ -0,0 +1,235 @@
+//
+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+package storage
+
+import (
+ "context"
+ "errors"
+ "fmt"
+ "net/http"
+ "strings"
+ "testing"
+ "time"
+
+ "github.com/aws/aws-sdk-go-v2/aws"
+ "github.com/aws/aws-sdk-go-v2/credentials"
+ "github.com/aws/aws-sdk-go-v2/service/s3"
+)
+
+// newOSSTestStorage builds an OSS backend over a fake transport. These tests
+// were written BEFORE oss.go was slimmed onto s3Base and passed unchanged
+// against the old standalone implementation — they are the behavior contract
+// the refactor must preserve (characterization tests).
+func newOSSTestStorage(handler s3RoundTripper, bucket, prefix string) *OSSStorage {
+ client := s3.NewFromConfig(aws.Config{
+ Region: "us-east-1",
+ Credentials: aws.NewCredentialsCache(credentials.NewStaticCredentialsProvider("access", "secret", "")),
+ HTTPClient: &http.Client{Transport: handler},
+ }, func(options *s3.Options) {
+ options.BaseEndpoint = aws.String("https://oss.test")
+ options.UsePathStyle = true
+ options.RetryMaxAttempts = 1
+ })
+ o := &OSSStorage{client: client}
+ o.s3Base = s3Base{
+ client: func() *s3.Client { return o.client },
+ bucket: bucket,
+ prefixPath: prefix,
+ typeName: "oss",
+ resolve: o.resolveBucketAndPath,
+ makeBucket: o.createBucket,
+ reconnect: func(context.Context) {},
+ }
+ return o
+}
+
+func TestOSSStoragePutAndGet(t *testing.T) {
+ o := newOSSTestStorage(func(req *http.Request) (*http.Response, error) {
+ if req.Method == http.MethodPut {
+ return s3Response(200, ""), nil
+ }
+ return s3Response(200, "oss-bytes"), nil
+ }, "", "")
+
+ if err := o.Put(context.Background(), "bucket", "key", []byte("data")); err != nil {
+ t.Fatalf("Put: %v", err)
+ }
+ got, err := o.Get(context.Background(), "bucket", "key")
+ if err != nil {
+ t.Fatalf("Get: %v", err)
+ }
+ if string(got) != "oss-bytes" {
+ t.Fatalf("Get returned %q, want oss-bytes", got)
+ }
+}
+
+func TestOSSStoragePutSurfacesServiceError(t *testing.T) {
+ // A failing bucket creation must surface an error after the retry loop
+ // gives up — Put must never silently succeed. The assertion checks
+ // non-nil only: with this aws-sdk version, any failing response's error
+ // chain re-resolves the auth scheme and masks the real code behind an
+ // endpoint-rule error (verified experimentally). Existing s3_test.go
+ // failure cases follow the same err != nil convention for this reason.
+ o := newOSSTestStorage(func(req *http.Request) (*http.Response, error) {
+ return s3Response(403, `<Error><Code>AccessDenied</Code></Error>`), nil
+ }, "", "")
+
+ if err := o.Put(context.Background(), "bucket", "key", []byte("data")); err == nil {
+ t.Fatal("Put returned nil after the bucket creation was denied")
+ }
+}
+
+func TestOSSStorageGetSurfacesContextCancellation(t *testing.T) {
+ o := newOSSTestStorage(func(req *http.Request) (*http.Response, error) {
+ return nil, fmt.Errorf("simulated transport error")
+ }, "", "")
+
+ ctx, cancel := context.WithCancel(context.Background())
+ cancel()
+
+ start := time.Now()
+ _, err := o.Get(ctx, "bucket", "key")
+ elapsed := time.Since(start)
+
+ if !errors.Is(err, context.Canceled) {
+ t.Fatalf("Get err = %v, want context.Canceled", err)
+ }
+ if elapsed > 500*time.Millisecond {
+ t.Fatalf("Get took %v on an already-canceled ctx; want < 500ms", elapsed)
+ }
+}
+
+func TestOSSStorageHealthUploadsProbeObject(t *testing.T) {
+ var sawCreateBucket, sawProbePut bool
+ o := newOSSTestStorage(func(req *http.Request) (*http.Response, error) {
+ switch {
+ case req.Method == http.MethodHead:
+ return s3Response(404, ""), nil // bucket missing
+ case req.Method == http.MethodPut && strings.HasSuffix(req.URL.Path, "/txtxtxtxt1"):
+ sawProbePut = true
+ return s3Response(200, ""), nil
+ case req.Method == http.MethodPut:
+ sawCreateBucket = true
+ return s3Response(200, ""), nil
+ default:
+ return s3Response(200, ""), nil
+ }
+ }, "", "")
+
+ if !o.Health(context.Background()) {
+ t.Fatal("Health = false, want true")
+ }
+ if !sawCreateBucket {
+ t.Error("Health did not create the missing probe bucket")
+ }
+ if !sawProbePut {
+ t.Error("Health did not upload the probe object")
+ }
+}
+
+func TestOSSStorageRemoveMissingObjectIsNoOp(t *testing.T) {
+ o := newOSSTestStorage(func(req *http.Request) (*http.Response, error) {
+ return s3Response(404, `<Error><Code>NoSuchKey</Code></Error>`), nil
+ }, "", "")
+
+ if err := o.Remove(context.Background(), "bucket", "missing"); err != nil {
+ t.Fatalf("Remove of a missing object: %v, want nil", err)
+ }
+}
+
+func TestOSSStorageObjExistDistinguishesPresent(t *testing.T) {
+ o := newOSSTestStorage(func(req *http.Request) (*http.Response, error) {
+ if strings.HasSuffix(req.URL.Path, "/there") {
+ return s3Response(200, ""), nil
+ }
+ return s3Response(404, `<Error><Code>NoSuchKey</Code></Error>`), nil
+ }, "", "")
+
+ if !o.ObjExist(context.Background(), "bucket", "there") {
+ t.Error("ObjExist(there) = false, want true")
+ }
+ if o.ObjExist(context.Background(), "bucket", "absent") {
+ t.Error("ObjExist(absent) = true, want false")
+ }
+}
+
+func TestOSSStorageBucketExistsWithErrorSeparatesFailure(t *testing.T) {
+ o := newOSSTestStorage(func(req *http.Request) (*http.Response, error) {
+ if strings.Contains(req.URL.Path, "real") {
+ return s3Response(200, ""), nil
+ }
+ return s3Response(404, `<Error><Code>NoSuchBucket</Code></Error>`), nil
+ }, "", "")
+
+ exists, err := o.BucketExistsWithError(context.Background(), "real")
+ if err != nil || !exists {
+ t.Errorf("BucketExistsWithError(real) = %v, %v; want true, nil", exists, err)
+ }
+ exists, err = o.BucketExistsWithError(context.Background(), "ghost")
+ if err != nil || exists {
+ t.Errorf("BucketExistsWithError(ghost) = %v, %v; want false, nil", exists, err)
+ }
+}
+
+func TestOSSStorageGetPresignedURL(t *testing.T) {
+ o := newOSSTestStorage(func(req *http.Request) (*http.Response, error) {
+ return s3Response(200, ""), nil
+ }, "", "")
+
+ url, err := o.GetPresignedURL(context.Background(), "bucket", "key", 0)
+ if err != nil {
+ t.Fatalf("GetPresignedURL: %v", err)
+ }
+ if !strings.Contains(url, "oss.test") || !strings.Contains(url, "bucket/key") {
+ t.Errorf("presigned URL = %q, want the oss.test endpoint and bucket/key path", url)
+ }
+}
+
+func TestOSSStorageRemoveBucketRejectsLogicalBucketInSharedMode(t *testing.T) {
+ o := newOSSTestStorage(func(req *http.Request) (*http.Response, error) {
+ return s3Response(200, ""), nil
+ }, "physical", "data")
+
+ err := o.RemoveBucket(context.Background(), "logical")
+ if err == nil || !strings.Contains(err.Error(), "cannot remove logical OSS bucket") {
+ t.Fatalf("RemoveBucket err = %v, want the shared-mode rejection", err)
+ }
+}
+
+func TestOSSStorageRemoveEmptyBucketRejectsLogicalBucketInSharedMode(t *testing.T) {
+ o := newOSSTestStorage(func(req *http.Request) (*http.Response, error) {
+ return s3Response(200, ""), nil
+ }, "physical", "data")
+
+ err := o.RemoveEmptyBucket(context.Background(), "logical")
+ if err == nil || !strings.Contains(err.Error(), "cannot remove logical OSS bucket") {
+ t.Fatalf("RemoveEmptyBucket err = %v, want the shared-mode rejection", err)
+ }
+}
+
+func TestOSSStorageResolveOmitsBucketSegment(t *testing.T) {
+ // OSS folds <prefix>/<fnm> — unlike S3, the logical bucket is NOT part of
+ // the key in shared-bucket mode. See doc.go for the per-vendor table.
+ o := newOSSTestStorage(func(req *http.Request) (*http.Response, error) {
+ return s3Response(200, ""), nil
+ }, "physical", "data")
+
+ bucket, fnm := o.resolveBucketAndPath("logical", "report.txt")
+ if bucket != "physical" || fnm != "data/report.txt" {
+ t.Fatalf("resolve = (%q, %q), want (physical, data/report.txt)", bucket, fnm)
+ }
+}
diff --git a/internal/storage/presign_errwrap_test.go b/internal/storage/presign_errwrap_test.go
new file mode 100644
index 0000000000..87518cce4a
--- /dev/null
+++ b/internal/storage/presign_errwrap_test.go
@@ -0,0 +1,120 @@
+//
+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+package storage
+
+import (
+ "context"
+ "errors"
+ "net/http"
+ "strings"
+ "testing"
+ "time"
+
+ "github.com/aws/aws-sdk-go-v2/aws"
+ awsCreds "github.com/aws/aws-sdk-go-v2/credentials"
+ "github.com/aws/aws-sdk-go-v2/service/s3"
+ "github.com/aws/smithy-go/middleware"
+ "github.com/minio/minio-go/v7"
+ minioCreds "github.com/minio/minio-go/v7/pkg/credentials"
+)
+
+// errSentinelPresign is a controlled error injected into the S3 presign path to
+// verify that GetPresignedURL preserves (wraps) the underlying error instead of
+// replacing it with a generic "after N retries" string.
+var errSentinelPresign = errors.New("injected presign failure")
+
+// failBuild is a build-stage middleware that forces every operation to fail with
+// errSentinelPresign. Presigning runs the build/serialize middleware stack, so
+// this deterministically fails the shared s3Base.GetPresignedURL path.
+type failBuild struct{ err error }
+
+func (f failBuild) ID() string { return "failBuild" }
+
+func (f failBuild) HandleBuild(ctx context.Context, in middleware.BuildInput, next middleware.BuildHandler) (middleware.BuildOutput, middleware.Metadata, error) {
+ return middleware.BuildOutput{}, middleware.Metadata{}, f.err
+}
+
+// newFailingPresignS3Base builds an s3Base whose s3 client fails every presign
+// attempt through a build-stage middleware, so the retry helper returns the
+// sentinel error. This exercises the shared s3Base.GetPresignedURL path used by
+// both S3 and OSS.
+func newFailingPresignS3Base() *s3Base {
+ client := s3.NewFromConfig(aws.Config{
+ Region: "us-east-1",
+ Credentials: aws.NewCredentialsCache(awsCreds.NewStaticCredentialsProvider("access", "secret", "")),
+ HTTPClient: &http.Client{Transport: s3RoundTripper(func(req *http.Request) (*http.Response, error) { return s3Response(200, ""), nil })},
+ }, func(o *s3.Options) {
+ o.BaseEndpoint = aws.String("https://s3.test")
+ o.UsePathStyle = true
+ o.RetryMaxAttempts = 1
+ o.APIOptions = append(o.APIOptions, func(stack *middleware.Stack) error {
+ return stack.Build.Add(failBuild{err: errSentinelPresign}, middleware.Before)
+ })
+ })
+ return &s3Base{
+ client: func() *s3.Client { return client },
+ typeName: "s3",
+ resolve: func(bucket, fnm string) (string, string) { return bucket, fnm },
+ makeBucket: func(ctx context.Context, bucket string) error { return nil },
+ reconnect: func(context.Context) {},
+ }
+}
+
+// TestS3BaseGetPresignedURLWrapsUnderlyingError pins that the shared
+// GetPresignedURL surfaces the underlying presign error via errors.Is, so
+// callers can still detect cancellations/deadlines and specific failures.
+func TestS3BaseGetPresignedURLWrapsUnderlyingError(t *testing.T) {
+ base := newFailingPresignS3Base()
+
+ _, err := base.GetPresignedURL(context.Background(), "bucket", "key", time.Minute)
+ if err == nil {
+ t.Fatalf("GetPresignedURL err = nil, want the underlying presign error")
+ }
+ if !errors.Is(err, errSentinelPresign) {
+ t.Fatalf("GetPresignedURL err = %v, want it to wrap %v (errors.Is must hold)", err, errSentinelPresign)
+ }
+}
+
+// TestMinioGetPresignedURLWrapsUnderlyingError pins the same contract for the
+// MinIO backend: a failing presign attempt must preserve the underlying error
+// identity rather than replacing it with a generic string.
+func TestMinioGetPresignedURLWrapsUnderlyingError(t *testing.T) {
+ // A negative expiry makes minio-go's PresignedGetObject fail locally and
+ // deterministically on every retry attempt, with no server required.
+ storage := &MinioStorage{client: mustMinioClient(t)}
+ _, err := storage.GetPresignedURL(context.Background(), "bucket", "key", -1*time.Second)
+ if err == nil {
+ t.Fatalf("GetPresignedURL err = nil, want a non-nil underlying error")
+ }
+ if !strings.Contains(strings.ToLower(err.Error()), "expires") {
+ t.Fatalf("GetPresignedURL err = %v, want it to preserve the underlying expiry error text", err)
+ }
+}
+
+// mustMinioClient builds a MinIO client pointed at a dummy endpoint. Presigning
+// is computed locally and never contacts the server, so no service is needed.
+func mustMinioClient(t *testing.T) *minio.Client {
+ t.Helper()
+ c, err := minio.New("localhost:9000", &minio.Options{
+ Creds: minioCreds.NewStaticV4("access", "secret", ""),
+ Secure: false,
+ })
+ if err != nil {
+ t.Fatalf("minio.New: %v", err)
+ }
+ return c
+}
diff --git a/internal/storage/s3.go b/internal/storage/s3.go
index 5313cdd175..f95f503eee 100644
--- a/internal/storage/s3.go
+++ b/internal/storage/s3.go
@@ -17,13 +17,11 @@
package storage
import (
- "bytes"
"context"
"errors"
"fmt"
"ragflow/internal/common"
"ragflow/internal/server/config"
- "time"
"github.com/aws/aws-sdk-go-v2/aws"
"github.com/aws/aws-sdk-go-v2/aws/signer/v4"
@@ -32,30 +30,39 @@ import (
"github.com/aws/aws-sdk-go-v2/service/s3"
"github.com/aws/aws-sdk-go-v2/service/s3/types"
"github.com/aws/smithy-go"
- "github.com/aws/smithy-go/encoding/httpbinding"
"go.uber.org/zap"
)
-// S3Storage implements Storage interface for AWS S3
+// S3Storage implements Storage for AWS S3. The shared S3-compatible surface
+// lives in s3Base; this file holds only what differs for AWS: connection
+// options (signature version, addressing style), the region-aware bucket
+// factory, the path-folding semantics, and the versioned bucket removal.
type S3Storage struct {
- client *s3.Client
- bucket string
- prefixPath string
- config config.S3Config
+ s3Base
+ client *s3.Client // owned by connect/reconnect; the base reads it via its getter
+ config config.S3Config
}
// NewS3Storage creates a new S3 storage instance
func NewS3Storage(ctx context.Context, config config.S3Config) (*S3Storage, error) {
storage := &S3Storage{
- bucket: config.Bucket,
- prefixPath: config.PrefixPath,
- config: config,
+ config: config,
}
if err := storage.connect(ctx); err != nil {
return nil, err
}
+ storage.s3Base = s3Base{
+ client: func() *s3.Client { return storage.client },
+ bucket: config.Bucket,
+ prefixPath: config.PrefixPath,
+ typeName: "s3",
+ resolve: storage.resolveBucketAndPath,
+ makeBucket: storage.createBucket,
+ reconnect: storage.reconnect,
+ }
+
return storage, nil
}
@@ -135,256 +142,6 @@ func (s *S3Storage) createBucket(ctx context.Context, bucket string) error {
return err
}
-func (s *S3Storage) Type() string { return "s3" }
-
-// Health checks S3 service availability
-func (s *S3Storage) Health(ctx context.Context) bool {
- bucket := s.bucket
- if bucket == "" {
- bucket = "health-check-bucket"
- }
-
- fnm := "txtxtxtxt1"
- if s.prefixPath != "" {
- fnm = fmt.Sprintf("%s/%s", s.prefixPath, fnm)
- }
- binary := []byte("_t@@@1")
-
- // Ensure bucket exists
- if !s.BucketExists(ctx, bucket) {
- err := s.createBucket(ctx, bucket)
- if err != nil {
- common.Error("Failed to create bucket for health check", err, zap.String("bucket", bucket), zap.Error(err))
- return false
- }
- }
-
- // Try to upload a test object
- reader := bytes.NewReader(binary)
- _, err := s.client.PutObject(ctx, &s3.PutObjectInput{
- Bucket: aws.String(bucket),
- Key: aws.String(fnm),
- Body: reader,
- })
-
- if err != nil {
- common.Error("Health check failed", err, zap.Error(err))
- return false
- }
-
- return true
-}
-
-// Put uploads an object to S3
-func (s *S3Storage) Put(ctx context.Context, bucket, fnm string, binary []byte, tenantID ...string) error {
- bucket, fnm = s.resolveBucketAndPath(bucket, fnm)
-
- for i := 0; i < 2; i++ {
- // Ensure bucket exists
- if !s.BucketExists(ctx, bucket) {
- err := s.createBucket(ctx, bucket)
- if err != nil {
- if ctxErr := ctx.Err(); ctxErr != nil {
- return ctxErr
- }
- common.Error("Failed to create bucket", err, zap.String("bucket", bucket), zap.Error(err))
- s.reconnect(ctx)
- if err = sleepOrAbort(ctx, time.Second); err != nil {
- return err
- }
- continue
- }
- common.Info("Created bucket", zap.String("bucket", bucket))
- }
-
- reader := bytes.NewReader(binary)
- _, err := s.client.PutObject(ctx, &s3.PutObjectInput{
- Bucket: aws.String(bucket),
- Key: aws.String(fnm),
- Body: reader,
- })
- if err != nil {
- if ctxErr := ctx.Err(); ctxErr != nil {
- return ctxErr
- }
- common.Error("Failed to put object", err, zap.String("bucket", bucket), zap.String("key", fnm), zap.Error(err))
- s.reconnect(ctx)
- if err = sleepOrAbort(ctx, time.Second); err != nil {
- return err
- }
- continue
- }
-
- return nil
- }
-
- return fmt.Errorf("failed to put object after retries")
-}
-
-// Get retrieves an object from S3
-func (s *S3Storage) Get(ctx context.Context, bucket, fnm string, tenantID ...string) ([]byte, error) {
- bucket, fnm = s.resolveBucketAndPath(bucket, fnm)
-
- for i := 0; i < 2; i++ {
- // Honor cancellation up front so an already-canceled context returns
- // promptly instead of running the reconnect/backoff loop.
- if err := ctx.Err(); err != nil {
- return nil, err
- }
- result, err := s.client.GetObject(ctx, &s3.GetObjectInput{
- Bucket: aws.String(bucket),
- Key: aws.String(fnm),
- })
- if err != nil {
- // The request may have failed because ctx was canceled mid-flight;
- // surface that instead of sleeping and retrying with a dead context.
- if cerr := ctx.Err(); cerr != nil {
- return nil, cerr
- }
- common.Error("Failed to get object", err, zap.String("bucket", bucket), zap.String("key", fnm), zap.Error(err))
- s.reconnect(ctx)
- time.Sleep(time.Second)
- continue
- }
- buf := new(bytes.Buffer)
-
- readErr := func() error {
- defer result.Body.Close()
- _, err = buf.ReadFrom(result.Body)
- return err
- }()
- if readErr != nil {
- if cerr := ctx.Err(); cerr != nil {
- return nil, cerr
- }
- common.Error("Failed to read object data", readErr, zap.String("bucket", bucket), zap.String("key", fnm), zap.Error(readErr))
- s.reconnect(ctx)
- time.Sleep(time.Second)
- continue
- }
-
- return buf.Bytes(), nil
- }
-
- return nil, fmt.Errorf("failed to get object after retries")
-}
-
-// Remove removes an object from S3
-func (s *S3Storage) Remove(ctx context.Context, bucket, fnm string, tenantID ...string) error {
- bucket, fnm = s.resolveBucketAndPath(bucket, fnm)
-
- _, err := s.client.DeleteObject(ctx, &s3.DeleteObjectInput{
- Bucket: aws.String(bucket),
- Key: aws.String(fnm),
- })
- if err != nil {
- if isS3NotFound(err) {
- return nil
- }
- common.Error("Failed to remove object", err, zap.String("bucket", bucket), zap.String("key", fnm), zap.Error(err))
- return err
- }
-
- return nil
-}
-
-// ObjExist checks if an object exists in S3
-func (s *S3Storage) ObjExist(ctx context.Context, bucket, fnm string, tenantID ...string) bool {
- bucket, fnm = s.resolveBucketAndPath(bucket, fnm)
-
- _, err := s.client.HeadObject(ctx, &s3.HeadObjectInput{
- Bucket: aws.String(bucket),
- Key: aws.String(fnm),
- })
- if err != nil {
- if isS3NotFound(err) {
- return false
- }
- return false
- }
-
- return true
-}
-
-func (s *S3Storage) ObjectExists(ctx context.Context, bucket, fnm string) (bool, error) {
- bucket, fnm = s.resolveBucketAndPath(bucket, fnm)
- _, err := s.client.HeadObject(ctx, &s3.HeadObjectInput{Bucket: aws.String(bucket), Key: aws.String(fnm)})
- if err == nil {
- return true, nil
- }
- if isS3NotFound(err) {
- return false, nil
- }
- return false, err
-}
-
-func (s *S3Storage) ListObjects(ctx context.Context, bucket string, tenantID ...string) ([]string, error) {
-
- listInput := &s3.ListObjectsV2Input{
- Bucket: aws.String(bucket),
- }
-
- result, err := s.client.ListObjectsV2(ctx, listInput)
- if err != nil {
- return nil, err
- }
-
- var objects []string
-
- for _, obj := range result.Contents {
- objects = append(objects, *obj.Key)
- }
-
- return objects, nil
-}
-
-// GetPresignedURL generates a presigned URL for accessing an object
-func (s *S3Storage) GetPresignedURL(ctx context.Context, bucket, fnm string, expires time.Duration, tenantID ...string) (string, error) {
- bucket, fnm = s.resolveBucketAndPath(bucket, fnm)
-
- presignClient := s3.NewPresignClient(s.client)
-
- for i := 0; i < 10; i++ {
- req, err := presignClient.PresignGetObject(ctx, &s3.GetObjectInput{
- Bucket: aws.String(bucket),
- Key: aws.String(fnm),
- }, s3.WithPresignExpires(expires))
- if err != nil {
- common.Error("Failed to generate presigned URL", err, zap.String("bucket", bucket), zap.String("key", fnm), zap.Error(err))
- s.reconnect(ctx)
- time.Sleep(time.Second)
- continue
- }
-
- return req.URL, nil
- }
-
- return "", fmt.Errorf("failed to generate presigned URL after 10 retries")
-}
-
-// BucketExists checks if a bucket exists
-func (s *S3Storage) BucketExists(ctx context.Context, bucket string) bool {
- actualBucket := bucket
- if s.bucket != "" {
- actualBucket = s.bucket
- }
-
- _, err := s.client.HeadBucket(ctx, &s3.HeadBucketInput{
- Bucket: aws.String(actualBucket),
- })
- if err != nil {
- common.Debug("Bucket does not exist or error", zap.String("bucket", actualBucket), zap.Error(err))
- return false
- }
-
- return true
-}
-
-func (s *S3Storage) BucketExistsWithError(ctx context.Context, bucket string) (bool, error) {
- actualBucket, _ := s.resolveBucketAndPrefix(bucket)
- return s.bucketExistsForRemoval(ctx, actualBucket)
-}
-
// RemoveBucket removes a bucket and all its objects
func (s *S3Storage) RemoveBucket(ctx context.Context, bucket string) error {
actualBucket, prefix := s.resolveBucketAndPrefix(bucket)
@@ -502,46 +259,6 @@ func (s *S3Storage) deleteObjects(ctx context.Context, bucket string, objects []
return nil
}
-// Copy copies an object from source to destination
-func (s *S3Storage) Copy(ctx context.Context, srcBucket, srcPath, destBucket, destPath string) bool {
- srcBucket, srcPath = s.resolveBucketAndPath(srcBucket, srcPath)
- destBucket, destPath = s.resolveBucketAndPath(destBucket, destPath)
-
- copySource := httpbinding.EscapePath(fmt.Sprintf("%s/%s", srcBucket, srcPath), false)
-
- _, err := s.client.CopyObject(ctx, &s3.CopyObjectInput{
- Bucket: aws.String(destBucket),
- Key: aws.String(destPath),
- CopySource: aws.String(copySource),
- })
- if err != nil {
- common.Error("Failed to copy object", err, zap.String("src", copySource), zap.String("dest", fmt.Sprintf("%s/%s", destBucket, destPath)), zap.Error(err))
- return false
- }
-
- return true
-}
-
-// Move moves an object from source to destination
-func (s *S3Storage) Move(ctx context.Context, srcBucket, srcPath, destBucket, destPath string) bool {
- if s.Copy(ctx, srcBucket, srcPath, destBucket, destPath) {
- if err := s.Remove(ctx, srcBucket, srcPath); err != nil {
- common.Error("Failed to remove source object after copy", err, zap.String("bucket", srcBucket), zap.String("key", srcPath), zap.Error(err))
- rollbackCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 30*time.Second)
- defer cancel()
- err = s.Remove(rollbackCtx, destBucket, destPath)
- if err != nil {
- common.Warn("Failed to roll back copied destination object", zap.String("bucket", destBucket), zap.String("key", destPath), zap.Error(err))
- }
- return false
- }
- return true
- }
- return false
-}
-
-func (s *S3Storage) Close() error { return nil }
-
// isNotFound checks if the error is a not found error
func isS3NotFound(err error) bool {
if err == nil {
diff --git a/internal/storage/s3_test.go b/internal/storage/s3_test.go
index 75662525e3..d05f71b855 100644
--- a/internal/storage/s3_test.go
+++ b/internal/storage/s3_test.go
@@ -2,6 +2,7 @@ package storage
import (
"bytes"
+ "context"
"crypto/sha256"
"fmt"
"io"
@@ -31,7 +32,19 @@ func newS3TestStorage(region string, handler s3RoundTripper) *S3Storage {
options.UsePathStyle = true
options.RetryMaxAttempts = 1
})
- return &S3Storage{client: client}
+ s := &S3Storage{client: client}
+ s.s3Base = s3Base{
+ client: func() *s3.Client { return s.client },
+ bucket: "",
+ prefixPath: "",
+ typeName: "s3",
+ // Identity folding: these tests exercise the shared surface with no
+ // default bucket / prefix configured.
+ resolve: func(bucket, fnm string) (string, string) { return bucket, fnm },
+ makeBucket: s.createBucket,
+ reconnect: func(context.Context) {},
+ }
+ return s
}
func s3Response(status int, body string) *http.Response {
diff --git a/internal/storage/s3base.go b/internal/storage/s3base.go
new file mode 100644
index 0000000000..585851f951
--- /dev/null
+++ b/internal/storage/s3base.go
@@ -0,0 +1,319 @@
+//
+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+package storage
+
+import (
+ "bytes"
+ "context"
+ "fmt"
+ "time"
+
+ "ragflow/internal/common"
+
+ "github.com/aws/aws-sdk-go-v2/aws"
+ "github.com/aws/aws-sdk-go-v2/service/s3"
+ "github.com/aws/smithy-go/encoding/httpbinding"
+ "go.uber.org/zap"
+)
+
+// s3Base is the shared implementation for S3-compatible backends built on
+// aws-sdk-go-v2 (AWS S3 and Aliyun OSS). A backend embeds it and injects the
+// few pieces that genuinely differ between vendors:
+//
+// - client: a getter returning the CURRENT client, so reconnect swaps
+// are visible to shared methods without stale handles;
+// - resolve: the bucket/key folding semantics (see doc.go for the
+// deliberately non-uniform per-vendor rules);
+// - makeBucket: bucket creation (AWS pins a LocationConstraint per region,
+// OSS does not);
+// - reconnect: rebuild the client after a failed attempt;
+// - typeName: the Type() identifier.
+//
+// Methods that differ structurally between vendors — connect, RemoveBucket,
+// RemoveEmptyBucket and the versioned-object removal helpers — stay on the
+// concrete backend and override anything the base would otherwise provide.
+type s3Base struct {
+ client func() *s3.Client
+ bucket string
+ prefixPath string
+ typeName string
+ resolve func(bucket, fnm string) (string, string)
+ makeBucket func(ctx context.Context, bucket string) error
+ reconnect func(ctx context.Context)
+}
+
+func (b *s3Base) Type() string { return b.typeName }
+
+// Close is a no-op: the aws-sdk client holds no resources to release.
+func (b *s3Base) Close() error { return nil }
+
+// Health checks service availability by uploading a probe object.
+func (b *s3Base) Health(ctx context.Context) bool {
+ bucket := b.bucket
+ if bucket == "" {
+ bucket = "health-check-bucket"
+ }
+
+ fnm := "txtxtxtxt1"
+ if b.prefixPath != "" {
+ fnm = fmt.Sprintf("%s/%s", b.prefixPath, fnm)
+ }
+ binary := []byte("_t@@@1")
+
+ // Ensure bucket exists
+ if !b.BucketExists(ctx, bucket) {
+ if err := b.makeBucket(ctx, bucket); err != nil {
+ common.Error("Failed to create bucket for health check", err, zap.String("bucket", bucket), zap.Error(err))
+ return false
+ }
+ }
+
+ // Try to upload a test object
+ reader := bytes.NewReader(binary)
+ if _, err := b.client().PutObject(ctx, &s3.PutObjectInput{
+ Bucket: aws.String(bucket),
+ Key: aws.String(fnm),
+ Body: reader,
+ }); err != nil {
+ common.Error("Health check failed", err, zap.Error(err))
+ return false
+ }
+
+ return true
+}
+
+// Put uploads an object, creating the bucket on first write.
+func (b *s3Base) Put(ctx context.Context, bucket, fnm string, binary []byte, tenantID ...string) error {
+ bucket, fnm = b.resolve(bucket, fnm)
+
+ return retryObjectOp(ctx, 2, func() { b.reconnect(ctx) }, func() error {
+ // Ensure bucket exists
+ if !b.BucketExists(ctx, bucket) {
+ if err := b.makeBucket(ctx, bucket); err != nil {
+ common.Error("Failed to create bucket", err, zap.String("bucket", bucket), zap.Error(err))
+ return err
+ }
+ common.Info("Created bucket", zap.String("bucket", bucket))
+ }
+
+ reader := bytes.NewReader(binary)
+ if _, err := b.client().PutObject(ctx, &s3.PutObjectInput{
+ Bucket: aws.String(bucket),
+ Key: aws.String(fnm),
+ Body: reader,
+ }); err != nil {
+ common.Error("Failed to put object", err, zap.String("bucket", bucket), zap.String("key", fnm), zap.Error(err))
+ return err
+ }
+
+ return nil
+ })
+}
+
+// Get retrieves an object. Cancellation surfaces promptly per the contract
+// pinned by s3_cancel_test.go (retryObjectOp checks ctx before each attempt
+// and after each op error, winning over the backoff sleep).
+func (b *s3Base) Get(ctx context.Context, bucket, fnm string, tenantID ...string) ([]byte, error) {
+ bucket, fnm = b.resolve(bucket, fnm)
+
+ var data []byte
+ err := retryObjectOp(ctx, 2, func() { b.reconnect(ctx) }, func() error {
+ result, err := b.client().GetObject(ctx, &s3.GetObjectInput{
+ Bucket: aws.String(bucket),
+ Key: aws.String(fnm),
+ })
+ if err != nil {
+ common.Error("Failed to get object", err, zap.String("bucket", bucket), zap.String("key", fnm), zap.Error(err))
+ return err
+ }
+ buf := new(bytes.Buffer)
+
+ readErr := func() error {
+ defer result.Body.Close()
+ _, err = buf.ReadFrom(result.Body)
+ return err
+ }()
+ if readErr != nil {
+ common.Error("Failed to read object data", readErr, zap.String("bucket", bucket), zap.String("key", fnm), zap.Error(readErr))
+ return readErr
+ }
+ data = buf.Bytes()
+
+ return nil
+ })
+ if err != nil {
+ return nil, err
+ }
+
+ return data, nil
+}
+
+// Remove removes an object; a missing object/bucket is not an error.
+func (b *s3Base) Remove(ctx context.Context, bucket, fnm string, tenantID ...string) error {
+ bucket, fnm = b.resolve(bucket, fnm)
+
+ if _, err := b.client().DeleteObject(ctx, &s3.DeleteObjectInput{
+ Bucket: aws.String(bucket),
+ Key: aws.String(fnm),
+ }); err != nil {
+ if isS3NotFound(err) {
+ return nil
+ }
+ common.Error("Failed to remove object", err, zap.String("bucket", bucket), zap.String("key", fnm), zap.Error(err))
+ return err
+ }
+
+ return nil
+}
+
+// ObjExist checks if an object exists.
+func (b *s3Base) ObjExist(ctx context.Context, bucket, fnm string, tenantID ...string) bool {
+ bucket, fnm = b.resolve(bucket, fnm)
+
+ if _, err := b.client().HeadObject(ctx, &s3.HeadObjectInput{
+ Bucket: aws.String(bucket),
+ Key: aws.String(fnm),
+ }); err != nil {
+ return false
+ }
+
+ return true
+}
+
+// ObjectExists distinguishes a missing object from a storage error.
+func (b *s3Base) ObjectExists(ctx context.Context, bucket, fnm string) (bool, error) {
+ bucket, fnm = b.resolve(bucket, fnm)
+
+ if _, err := b.client().HeadObject(ctx, &s3.HeadObjectInput{
+ Bucket: aws.String(bucket),
+ Key: aws.String(fnm),
+ }); err != nil {
+ if isS3NotFound(err) {
+ return false, nil
+ }
+ return false, err
+ }
+
+ return true, nil
+}
+
+// ListObjects lists the keys in a bucket.
+func (b *s3Base) ListObjects(ctx context.Context, bucket string, tenantID ...string) ([]string, error) {
+ result, err := b.client().ListObjectsV2(ctx, &s3.ListObjectsV2Input{
+ Bucket: aws.String(bucket),
+ })
+ if err != nil {
+ return nil, err
+ }
+
+ var objects []string
+ for _, obj := range result.Contents {
+ objects = append(objects, *obj.Key)
+ }
+
+ return objects, nil
+}
+
+// GetPresignedURL generates a presigned URL for accessing an object.
+func (b *s3Base) GetPresignedURL(ctx context.Context, bucket, fnm string, expires time.Duration, tenantID ...string) (string, error) {
+ bucket, fnm = b.resolve(bucket, fnm)
+
+ presignClient := s3.NewPresignClient(b.client())
+
+ var url string
+ err := retryObjectOp(ctx, 10, func() { b.reconnect(ctx) }, func() error {
+ req, err := presignClient.PresignGetObject(ctx, &s3.GetObjectInput{
+ Bucket: aws.String(bucket),
+ Key: aws.String(fnm),
+ }, s3.WithPresignExpires(expires))
+ if err != nil {
+ common.Error("Failed to generate presigned URL", err, zap.String("bucket", bucket), zap.String("key", fnm), zap.Error(err))
+ return err
+ }
+ url = req.URL
+
+ return nil
+ })
+ if err != nil {
+ return "", fmt.Errorf("failed to generate presigned URL after 10 retries: %w", err)
+ }
+
+ return url, nil
+}
+
+// BucketExists checks if a bucket exists.
+func (b *s3Base) BucketExists(ctx context.Context, bucket string) bool {
+ if b.bucket != "" {
+ bucket = b.bucket
+ }
+
+ if _, err := b.client().HeadBucket(ctx, &s3.HeadBucketInput{
+ Bucket: aws.String(bucket),
+ }); err != nil {
+ common.Debug("Bucket does not exist or error", zap.String("bucket", bucket), zap.Error(err))
+ return false
+ }
+
+ return true
+}
+
+// BucketExistsWithError distinguishes a missing bucket from a storage error.
+func (b *s3Base) BucketExistsWithError(ctx context.Context, bucket string) (bool, error) {
+ if b.bucket != "" {
+ bucket = b.bucket
+ }
+
+ if _, err := b.client().HeadBucket(ctx, &s3.HeadBucketInput{
+ Bucket: aws.String(bucket),
+ }); err != nil {
+ if isS3BucketNotFound(err) {
+ return false, nil
+ }
+ return false, fmt.Errorf("head S3 bucket %q: %w", bucket, err)
+ }
+
+ return true, nil
+}
+
+// Copy copies an object from source to destination.
+func (b *s3Base) Copy(ctx context.Context, srcBucket, srcPath, destBucket, destPath string) bool {
+ srcBucket, srcPath = b.resolve(srcBucket, srcPath)
+ destBucket, destPath = b.resolve(destBucket, destPath)
+
+ copySource := httpbinding.EscapePath(fmt.Sprintf("%s/%s", srcBucket, srcPath), false)
+
+ if _, err := b.client().CopyObject(ctx, &s3.CopyObjectInput{
+ Bucket: aws.String(destBucket),
+ Key: aws.String(destPath),
+ CopySource: aws.String(copySource),
+ }); err != nil {
+ common.Error("Failed to copy object", err, zap.String("src", copySource), zap.String("dest", fmt.Sprintf("%s/%s", destBucket, destPath)), zap.Error(err))
+ return false
+ }
+
+ return true
+}
+
+// Move moves an object from source to destination, rolling back the
+// destination if the source removal fails.
+func (b *s3Base) Move(ctx context.Context, srcBucket, srcPath, destBucket, destPath string) bool {
+ return moveWithRollback(ctx, srcBucket, srcPath, destBucket, destPath,
+ func() bool { return b.Copy(ctx, srcBucket, srcPath, destBucket, destPath) },
+ func(rctx context.Context) error { return b.Remove(rctx, srcBucket, srcPath) },
+ func(rctx context.Context) error { return b.Remove(rctx, destBucket, destPath) },
+ )
+}
diff --git a/internal/storage/s3base_test.go b/internal/storage/s3base_test.go
new file mode 100644
index 0000000000..b882cefeb4
--- /dev/null
+++ b/internal/storage/s3base_test.go
@@ -0,0 +1,144 @@
+//
+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+package storage
+
+import (
+ "context"
+ "fmt"
+ "net/http"
+ "strings"
+ "testing"
+
+ "github.com/aws/aws-sdk-go-v2/aws"
+ "github.com/aws/aws-sdk-go-v2/credentials"
+ "github.com/aws/aws-sdk-go-v2/service/s3"
+)
+
+// Compile-time contract: every backend must satisfy the full Storage
+// interface. These pin the embedding-based decomposition — if a future
+// refactor drops a method from s3Base or a thin shell, compilation breaks.
+var (
+ _ Storage = (*S3Storage)(nil)
+ _ Storage = (*OSSStorage)(nil)
+ _ Storage = (*MinioStorage)(nil)
+ _ Storage = (*MemoryStorage)(nil)
+ _ Storage = (*GCSStorage)(nil)
+)
+
+// newBareS3Base builds an s3Base directly — no S3Storage wrapper — proving
+// the base class is self-sufficient: a backend only needs to inject its
+// client getter, path resolver, bucket factory, reconnect and type name.
+func newBareS3Base(handler s3RoundTripper, typeName string) *s3Base {
+ client := s3.NewFromConfig(aws.Config{
+ Region: "us-east-1",
+ Credentials: aws.NewCredentialsCache(credentials.NewStaticCredentialsProvider("access", "secret", "")),
+ HTTPClient: &http.Client{Transport: handler},
+ }, func(options *s3.Options) {
+ options.BaseEndpoint = aws.String("https://s3.test")
+ options.UsePathStyle = true
+ options.RetryMaxAttempts = 1
+ })
+ return &s3Base{
+ client: func() *s3.Client { return client },
+ typeName: typeName,
+ resolve: func(bucket, fnm string) (string, string) { return bucket, fnm },
+ makeBucket: func(ctx context.Context, bucket string) error { return nil },
+ reconnect: func(context.Context) {},
+ }
+}
+
+func TestS3BaseStandalonePutAndGet(t *testing.T) {
+ base := newBareS3Base(func(req *http.Request) (*http.Response, error) {
+ if req.Method == http.MethodPut {
+ return s3Response(200, ""), nil
+ }
+ return s3Response(200, "hello-bucket"), nil
+ }, "s3")
+
+ if err := base.Put(context.Background(), "bucket", "key", []byte("data")); err != nil {
+ t.Fatalf("Put: %v", err)
+ }
+ got, err := base.Get(context.Background(), "bucket", "key")
+ if err != nil {
+ t.Fatalf("Get: %v", err)
+ }
+ if string(got) != "hello-bucket" {
+ t.Fatalf("Get returned %q, want hello-bucket", got)
+ }
+}
+
+func TestS3BaseTypeReturnsInjectedName(t *testing.T) {
+ base := newBareS3Base(func(req *http.Request) (*http.Response, error) {
+ return s3Response(200, ""), nil
+ }, "oss")
+ if got := base.Type(); got != "oss" {
+ t.Errorf("Type() = %q, want oss (injected name, not a hardcoded one)", got)
+ }
+}
+
+func TestS3BasePutSurfacesUnderlyingErrorAfterRetries(t *testing.T) {
+ // Put must surface the last underlying error (not a generic string) once
+ // attempts are exhausted — the behavior unified in the helper extraction.
+ base := newBareS3Base(func(req *http.Request) (*http.Response, error) {
+ return nil, fmt.Errorf("simulated transport error")
+ }, "s3")
+
+ err := base.Put(context.Background(), "bucket", "key", []byte("data"))
+ if err == nil || !strings.Contains(err.Error(), "simulated transport error") {
+ t.Fatalf("Put err = %v, want the underlying transport error", err)
+ }
+}
+
+func TestS3BaseClientGetterSeesReconnect(t *testing.T) {
+ // The client getter must return the CURRENT client: after reconnect swaps
+ // it, the next attempt must use the replacement, not a stale handle.
+ oldClient := s3.NewFromConfig(aws.Config{
+ Region: "us-east-1",
+ Credentials: aws.NewCredentialsCache(credentials.NewStaticCredentialsProvider("access", "secret", "")),
+ HTTPClient: &http.Client{Transport: s3RoundTripper(func(req *http.Request) (*http.Response, error) {
+ return nil, fmt.Errorf("old client: offline")
+ })},
+ }, func(options *s3.Options) {
+ options.BaseEndpoint = aws.String("https://s3.test")
+ options.UsePathStyle = true
+ options.RetryMaxAttempts = 1
+ })
+ newClient := s3.NewFromConfig(aws.Config{
+ Region: "us-east-1",
+ Credentials: aws.NewCredentialsCache(credentials.NewStaticCredentialsProvider("access", "secret", "")),
+ HTTPClient: &http.Client{Transport: s3RoundTripper(func(req *http.Request) (*http.Response, error) {
+ return s3Response(200, ""), nil
+ })},
+ }, func(options *s3.Options) {
+ options.BaseEndpoint = aws.String("https://s3.test")
+ options.UsePathStyle = true
+ options.RetryMaxAttempts = 1
+ })
+
+ var current = oldClient
+ base := &s3Base{
+ client: func() *s3.Client { return current },
+ typeName: "s3",
+ resolve: func(bucket, fnm string) (string, string) { return bucket, fnm },
+ makeBucket: func(ctx context.Context, bucket string) error { return nil },
+ reconnect: func(context.Context) { current = newClient },
+ }
+
+ if err := base.Put(context.Background(), "bucket", "key", []byte("data")); err != nil {
+ t.Fatalf("Put after reconnect: %v, want nil (second attempt must use the new client)", err)
+ }
+}
diff --git a/internal/storage/storage_common.go b/internal/storage/storage_common.go
new file mode 100644
index 0000000000..ca114cefb0
--- /dev/null
+++ b/internal/storage/storage_common.go
@@ -0,0 +1,79 @@
+//
+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+package storage
+
+import (
+ "context"
+ "time"
+
+ "ragflow/internal/common"
+
+ "go.uber.org/zap"
+)
+
+// retryObjectOp runs op up to attempts times, sleeping one second between
+// attempts and invoking onRetry (e.g. a client reconnect) before each sleep.
+// It returns nil on the first success and the last error once attempts are
+// exhausted. A canceled context short-circuits before an attempt and wins
+// over the backoff sleep, so cancellation always surfaces promptly instead
+// of costing a sleep window — the contract pinned by s3_cancel_test.go.
+func retryObjectOp(ctx context.Context, attempts int, onRetry func(), op func() error) error {
+ var lastErr error
+ for i := 0; i < attempts; i++ {
+ if err := ctx.Err(); err != nil {
+ return err
+ }
+ if err := op(); err != nil {
+ lastErr = err
+ if cerr := ctx.Err(); cerr != nil {
+ return cerr
+ }
+ if onRetry != nil {
+ onRetry()
+ }
+ if serr := sleepOrAbort(ctx, time.Second); serr != nil {
+ return serr
+ }
+ continue
+ }
+ return nil
+ }
+ return lastErr
+}
+
+// moveWithRollback implements the shared Move contract across backends: copy
+// the object, then remove the source. If the source removal fails, the
+// destination is removed best-effort under a detached 30s context derived via
+// context.WithoutCancel — so cleanup survives a parent canceled during the
+// source removal — and the move reports failure.
+func moveWithRollback(ctx context.Context, srcBucket, srcPath, destBucket, destPath string, copyFn func() bool, removeSrc, removeDest func(context.Context) error) bool {
+ if !copyFn() {
+ return false
+ }
+ if err := removeSrc(ctx); err != nil {
+ common.Warn("Failed to remove source object after copy",
+ zap.String("bucket", srcBucket), zap.String("key", srcPath), zap.Error(err))
+ rollbackCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 30*time.Second)
+ defer cancel()
+ if rbErr := removeDest(rollbackCtx); rbErr != nil {
+ common.Warn("Failed to roll back copied destination object",
+ zap.String("bucket", destBucket), zap.String("key", destPath), zap.Error(rbErr))
+ }
+ return false
+ }
+ return true
+}
diff --git a/internal/storage/storage_common_test.go b/internal/storage/storage_common_test.go
new file mode 100644
index 0000000000..e58fb28616
--- /dev/null
+++ b/internal/storage/storage_common_test.go
@@ -0,0 +1,275 @@
+//
+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+package storage
+
+import (
+ "context"
+ "errors"
+ "fmt"
+ "testing"
+ "time"
+)
+
+// --- retryObjectOp ---
+
+func TestRetryObjectOp_SucceedsFirstAttempt(t *testing.T) {
+ var calls, retries int
+ err := retryObjectOp(context.Background(), 3, func() { retries++ }, func() error {
+ calls++
+ return nil
+ })
+ if err != nil {
+ t.Fatalf("err = %v, want nil", err)
+ }
+ if calls != 1 {
+ t.Errorf("op calls = %d, want 1", calls)
+ }
+ if retries != 0 {
+ t.Errorf("onRetry calls = %d, want 0", retries)
+ }
+}
+
+func TestRetryObjectOp_SucceedsOnLastAttempt(t *testing.T) {
+ // Off-by-one guard: with attempts=3 the third attempt must still run.
+ var calls int
+ err := retryObjectOp(context.Background(), 3, nil, func() error {
+ calls++
+ if calls < 3 {
+ return fmt.Errorf("attempt %d failed", calls)
+ }
+ return nil
+ })
+ if err != nil {
+ t.Fatalf("err = %v, want nil", err)
+ }
+ if calls != 3 {
+ t.Errorf("op calls = %d, want 3", calls)
+ }
+}
+
+func TestRetryObjectOp_ReturnsLastErrorAfterAllAttempts(t *testing.T) {
+ var calls, retries int
+ err := retryObjectOp(context.Background(), 3, func() { retries++ }, func() error {
+ calls++
+ return fmt.Errorf("boom-%d", calls)
+ })
+ if err == nil || err.Error() != "boom-3" {
+ t.Fatalf("err = %v, want boom-3 (the LAST error, not the first)", err)
+ }
+ if calls != 3 {
+ t.Errorf("op calls = %d, want 3", calls)
+ }
+ // Legacy loops reconnect after every failed attempt, including the last
+ // one before giving up: the refreshed client benefits the NEXT request.
+ // The helper preserves that (zero behavior change), so onRetry runs 3x.
+ if retries != 3 {
+ t.Errorf("onRetry calls = %d, want 3 (after every failed attempt)", retries)
+ }
+}
+
+func TestRetryObjectOp_CanceledContextNeverRunsOp(t *testing.T) {
+ ctx, cancel := context.WithCancel(context.Background())
+ cancel()
+
+ var calls int
+ err := retryObjectOp(ctx, 3, nil, func() error {
+ calls++
+ return nil
+ })
+ if !errors.Is(err, context.Canceled) {
+ t.Fatalf("err = %v, want context.Canceled", err)
+ }
+ if calls != 0 {
+ t.Errorf("op calls = %d, want 0 (canceled ctx must short-circuit before any attempt)", calls)
+ }
+}
+
+func TestRetryObjectOp_CancelDuringBackoffSurfacesPromptly(t *testing.T) {
+ // The op itself cancels the context while failing: the next loop's ctx
+ // check must surface context.Canceled without sleeping the backoff window.
+ ctx, cancel := context.WithCancel(context.Background())
+ var calls int
+ start := time.Now()
+ err := retryObjectOp(ctx, 3, nil, func() error {
+ calls++
+ cancel()
+ return fmt.Errorf("fail after cancel")
+ })
+ elapsed := time.Since(start)
+ if !errors.Is(err, context.Canceled) {
+ t.Fatalf("err = %v, want context.Canceled", err)
+ }
+ if calls != 1 {
+ t.Errorf("op calls = %d, want 1", calls)
+ }
+ if elapsed > 500*time.Millisecond {
+ t.Errorf("took %v; want < 500ms (no backoff sleep on a dead ctx)", elapsed)
+ }
+}
+
+func TestRetryObjectOp_ContextCanceledAfterOpErrorSkipsSleep(t *testing.T) {
+ // ctx is canceled between the op failure and the backoff sleep: the
+ // post-error ctx check must win over sleepOrAbort.
+ ctx, cancel := context.WithCancel(context.Background())
+ start := time.Now()
+ err := retryObjectOp(ctx, 3, nil, func() error {
+ cancel() // cancel as a side effect of the failing op
+ return fmt.Errorf("transport error")
+ })
+ elapsed := time.Since(start)
+ if !errors.Is(err, context.Canceled) {
+ t.Fatalf("err = %v, want context.Canceled", err)
+ }
+ if elapsed > 500*time.Millisecond {
+ t.Errorf("took %v; want < 500ms", elapsed)
+ }
+}
+
+func TestRetryObjectOp_SingleAttemptNoRetry(t *testing.T) {
+ var calls, retries int
+ err := retryObjectOp(context.Background(), 1, func() { retries++ }, func() error {
+ calls++
+ return fmt.Errorf("once")
+ })
+ if err == nil || err.Error() != "once" {
+ t.Fatalf("err = %v, want once", err)
+ }
+ if calls != 1 || retries != 1 {
+ t.Errorf("calls=%d retries=%d, want 1/1", calls, retries)
+ }
+}
+
+func TestRetryObjectOp_ZeroAttemptsIsNoOp(t *testing.T) {
+ // Boundary guard: matches the old `for i := 0; i < 0` loops, which never
+ // ran and returned the nil lastErr.
+ var calls int
+ err := retryObjectOp(context.Background(), 0, nil, func() error {
+ calls++
+ return nil
+ })
+ if err != nil {
+ t.Fatalf("err = %v, want nil", err)
+ }
+ if calls != 0 {
+ t.Errorf("op calls = %d, want 0", calls)
+ }
+}
+
+// --- moveWithRollback ---
+
+func TestMoveWithRollback_CopyThenRemoveSucceeds(t *testing.T) {
+ var copyCalls, removeSrc, removeDest int
+ ok := moveWithRollback(context.Background(),
+ "srcb", "srck", "dstb", "dstk",
+ func() bool { copyCalls++; return true },
+ func(context.Context) error { removeSrc++; return nil },
+ func(context.Context) error { removeDest++; return nil },
+ )
+ if !ok {
+ t.Fatal("ok = false, want true")
+ }
+ if copyCalls != 1 || removeSrc != 1 || removeDest != 0 {
+ t.Errorf("copy=%d removeSrc=%d removeDest=%d, want 1/1/0", copyCalls, removeSrc, removeDest)
+ }
+}
+
+func TestMoveWithRollback_CopyFailureLeavesNoSideEffects(t *testing.T) {
+ var removeSrc, removeDest int
+ ok := moveWithRollback(context.Background(),
+ "srcb", "srck", "dstb", "dstk",
+ func() bool { return false },
+ func(context.Context) error { removeSrc++; return nil },
+ func(context.Context) error { removeDest++; return nil },
+ )
+ if ok {
+ t.Fatal("ok = true, want false")
+ }
+ if removeSrc != 0 || removeDest != 0 {
+ t.Errorf("removeSrc=%d removeDest=%d, want 0/0 (no removes when copy fails)", removeSrc, removeDest)
+ }
+}
+
+func TestMoveWithRollback_SourceRemoveFailureRollsBackDestination(t *testing.T) {
+ var removeDest int
+ var rollbackCtxErr error
+ var hasDeadline bool
+ ok := moveWithRollback(context.Background(),
+ "srcb", "srck", "dstb", "dstk",
+ func() bool { return true },
+ func(context.Context) error { return fmt.Errorf("source delete failed") },
+ func(ctx context.Context) error {
+ removeDest++
+ rollbackCtxErr = ctx.Err()
+ _, hasDeadline = ctx.Deadline()
+ return nil
+ },
+ )
+ if ok {
+ t.Fatal("ok = true, want false")
+ }
+ if removeDest != 1 {
+ t.Fatalf("removeDest calls = %d, want 1 (rollback must run)", removeDest)
+ }
+ if rollbackCtxErr != nil {
+ t.Errorf("rollback ctx err = %v, want nil", rollbackCtxErr)
+ }
+ if !hasDeadline {
+ t.Error("rollback ctx has no deadline, want a 30s timeout")
+ }
+}
+
+func TestMoveWithRollback_RollbackRunsEvenWhenParentCanceled(t *testing.T) {
+ // The rollback path derives from context.WithoutCancel: a parent canceled
+ // during the source removal must not skip the destination cleanup.
+ ctx, cancel := context.WithCancel(context.Background())
+ var removeDest int
+ var rollbackCtxErr error
+ ok := moveWithRollback(ctx,
+ "srcb", "srck", "dstb", "dstk",
+ func() bool { return true },
+ func(context.Context) error {
+ cancel() // parent dies while removing the source
+ return fmt.Errorf("source delete failed")
+ },
+ func(rctx context.Context) error {
+ removeDest++
+ rollbackCtxErr = rctx.Err()
+ return nil
+ },
+ )
+ if ok {
+ t.Fatal("ok = true, want false")
+ }
+ if removeDest != 1 {
+ t.Fatalf("removeDest calls = %d, want 1 (rollback must survive parent cancellation)", removeDest)
+ }
+ if rollbackCtxErr != nil {
+ t.Errorf("rollback ctx err = %v, want nil (WithoutCancel)", rollbackCtxErr)
+ }
+}
+
+func TestMoveWithRollback_RollbackFailureStillReportsFailure(t *testing.T) {
+ ok := moveWithRollback(context.Background(),
+ "srcb", "srck", "dstb", "dstk",
+ func() bool { return true },
+ func(context.Context) error { return fmt.Errorf("src") },
+ func(context.Context) error { return fmt.Errorf("dest") },
+ )
+ if ok {
+ t.Fatal("ok = true, want false (a failed rollback must not report success)")
+ }
+}