"rsync for cloud storage" - Google Drive, S3, Dropbox, Backblaze B2, One Drive, Swift, Hubic, Wasabi, Google Cloud Storage, Azure Blob, Azure Files, Yandex Files
Go azure-blobazure-blob-storageazure-filesbackblaze-b2cloud-storagedropboxencryptionftpfuse-filesystemgogolanggoogle-cloud-storagegoogle-driveonedriveopenstack-swiftrclones3sftpsyncwebdav
oracleobjectstorage: purge objects in batches
tomaszni934c21de2680
diff --git a/backend/oracleobjectstorage/oracleobjectstorage.go b/backend/oracleobjectstorage/oracleobjectstorage.goindex 305397b17..06605f5d9 100644--- a/backend/oracleobjectstorage/oracleobjectstorage.go+++ b/backend/oracleobjectstorage/oracleobjectstorage.go@@ -5,6 +5,7 @@ package oracleobjectstorageimport ("context"+ "errors""fmt""io""net/http"@@ -302,17 +303,28 @@ func (f *Fs) ListP(ctx context.Context, dir string, callback fs.ListRCallback) e// listFn is called from list to handle an object.type listFn func(remote string, object *objectstorage.ObjectSummary, isDirectory bool) error-// list the objects into the function supplied from-// the bucket and root supplied-// (bucket, directory) is the starting directory+// list calls fn for objects in bucket and directory.+//+// ctx controls the API requests.+// bucket is the bucket to list.+// directory is the path to list within bucket.// If prefix is set then it is removed from all file names// If addBucket is set then it adds the bucket to the start of the remotes generated// If recurse is set the function will recursively list// If limit is > 0 then it limits to that many files (must be less than 1000)-// If hidden is set then it will list the hidden (deleted) files too.-// if findFile is set it will look for files called (bucket, directory)-func (f *Fs) list(ctx context.Context, bucket, directory, prefix string, addBucket bool, recurse bool, limit int,- fn listFn) (err error) {+// includeDirectoryMarkers includes zero-length objects with names ending in a slash.+// fn receives each listed object.+func (f *Fs) list(+ ctx context.Context,+ bucket string,+ directory string,+ prefix string,+ addBucket bool,+ recurse bool,+ limit int,+ includeDirectoryMarkers bool,+ fn listFn,+) (err error) {if prefix != "" {prefix += "/"}@@ -407,7 +419,7 @@ func (f *Fs) list(ctx context.Context, bucket, directory, prefix string, addBuckremote = path.Join(bucket, remote)}// is this a directory marker?- if isDirectory && object.Size != nil && *object.Size == 0 {+ if isDirectory && object.Size != nil && *object.Size == 0 && !includeDirectoryMarkers {continue // skip directory marker}if isDirectory && len(remote) > 1 {@@ -456,7 +468,7 @@ func (f *Fs) listDir(ctx context.Context, bucket, directory, prefix string, addB}return nil}- err = f.list(ctx, bucket, directory, prefix, addBucket, false, 0, fn)+ err = f.list(ctx, bucket, directory, prefix, addBucket, false, 0, false, fn)if err != nil {return err}@@ -641,6 +653,66 @@ func (f *Fs) Rmdir(ctx context.Context, dir string) error {})}+// ------------------------------------------------------------+// Implement Purger is an optional interfaces for Fs+//------------------------------------------------------------++// Purge deletes all the files in dir.+//+// It implements fs.Purger, using OCI batch requests rather than removing each+// object returned by List. It also removes directory markers and, when dir is+// a bucket root, removes the empty bucket.+func (f *Fs) Purge(ctx context.Context, dir string) error {+ bucketName, directory := f.split(dir)+ if bucketName == "" {+ return fs.ErrorListBucketRequired+ }+ const batchSize = 1000+ objects := make([]objectstorage.BatchDeleteObjectIdentifier, 0, batchSize)+ deleteObjects := func() error {+ if len(objects) == 0 {+ return nil+ }+ req := objectstorage.BatchDeleteObjectsRequest{+ NamespaceName: new(f.opt.Namespace),+ BucketName: new(bucketName),+ BatchDeleteObjectsDetails: objectstorage.BatchDeleteObjectsDetails{+ Objects: objects,+ IsSkipDeletedResult: new(true),+ },+ }+ var resp objectstorage.BatchDeleteObjectsResponse+ err := f.pacer.Call(func() (bool, error) {+ var err error+ resp, err = f.srv.BatchDeleteObjects(ctx, req)+ return shouldRetry(ctx, resp.HTTPResponse(), err)+ })+ if err != nil {+ return err+ }+ var errs []error+ for _, failed := range resp.Failed {+ errs = append(errs, fmt.Errorf("failed to delete object %q: %s", *failed.ObjectName, *failed.ErrorMessage))+ }+ objects = objects[:0]+ return errors.Join(errs...)+ }+ err := f.list(ctx, bucketName, directory, f.rootDirectory, false, true, 0, true, func(_ string, object *objectstorage.ObjectSummary, _ bool) error {+ objects = append(objects, objectstorage.BatchDeleteObjectIdentifier{ObjectName: object.Name})+ if len(objects) == batchSize {+ return deleteObjects()+ }+ return nil+ })+ if err != nil {+ return err+ }+ if err = deleteObjects(); err != nil {+ return err+ }+ return f.Rmdir(ctx, dir)+}+func (f *Fs) abortMultiPartUpload(ctx context.Context, bucketName, bucketPath, uploadID *string) (err error) {if uploadID == nil || *uploadID == "" {return nil@@ -728,7 +800,7 @@ func (f *Fs) ListR(ctx context.Context, dir string, callback fs.ListRCallback) (bucketName, directory := f.split(dir)list := list.NewHelper(callback)listR := func(bucket, directory, prefix string, addBucket bool) error {- return f.list(ctx, bucket, directory, prefix, addBucket, true, 0, func(remote string, object *objectstorage.ObjectSummary, isDirectory bool) error {+ return f.list(ctx, bucket, directory, prefix, addBucket, true, 0, false, func(remote string, object *objectstorage.ObjectSummary, isDirectory bool) error {entry, err := f.itemToDirEntry(ctx, remote, object, isDirectory)if err != nil {return err@@ -807,6 +879,7 @@ var (_ fs.ListPer = &Fs{}_ fs.Commander = &Fs{}_ fs.CleanUpper = &Fs{}+ _ fs.Purger = &Fs{}_ fs.OpenChunkWriter = &Fs{}_ fs.Object = &Object{}diff --git a/backend/oracleobjectstorage/oracleobjectstorage_test.go b/backend/oracleobjectstorage/oracleobjectstorage_test.goindex feb215db5..899fd3a17 100644--- a/backend/oracleobjectstorage/oracleobjectstorage_test.go+++ b/backend/oracleobjectstorage/oracleobjectstorage_test.go@@ -8,10 +8,15 @@ import ("context""crypto/md5""fmt"+ "io"+ "sync""testing"+ "time"+ "github.com/oracle/oci-go-sdk/v65/objectstorage""github.com/rclone/rclone/fs""github.com/rclone/rclone/fs/hash"+ "github.com/rclone/rclone/fs/object""github.com/rclone/rclone/fstest""github.com/rclone/rclone/fstest/fstests""github.com/rclone/rclone/lib/random"@@ -81,9 +86,99 @@ func (f *Fs) InternalTestGzipEncoding(t *testing.T) {})}+// InternalTestPurgeBatches tests purging more objects than fit in one batch.+func (f *Fs) InternalTestPurgeBatches(t *testing.T) {+ ctx := context.Background()+ const (+ dir = "purge-batches"+ objectCount = 1001+ uploadConcurrency = 16+ )+ defer func() { _ = f.Purge(ctx, dir) }()+ jobs := make(chan int)+ errs := make(chan error, objectCount)+ var wg sync.WaitGroup+ for range uploadConcurrency {+ wg.Go(func() {+ for i := range jobs {+ remote := fmt.Sprintf("%s/%04d", dir, i)+ src := object.NewStaticObjectInfo(remote, time.Time{}, 0, true, nil, f)+ _, err := f.Put(ctx, bytes.NewReader(nil), src)+ if err != nil {+ errs <- err+ }+ }+ })+ }+ for i := range objectCount {+ jobs <- i+ }+ close(jobs)+ wg.Wait()+ close(errs)+ for err := range errs {+ require.NoError(t, err)+ }+ require.NoError(t, f.Purge(ctx, dir))+ entries, err := f.List(ctx, dir)+ require.NoError(t, err)+ assert.Empty(t, entries)+}++// InternalTestPurgeDirectoryMarkers tests purging directory marker objects.+func (f *Fs) InternalTestPurgeDirectoryMarkers(t *testing.T) {+ ctx := context.Background()+ const dir = "purge-directory-markers"+ bucketName, directory := f.split(dir)+ objectNames := []string{+ directory + "/",+ directory + "/child/",+ directory + "/child/object",+ }+ defer func() { _ = f.Purge(ctx, dir) }()+ for _, objectName := range objectNames {+ req := objectstorage.PutObjectRequest{+ NamespaceName: new(f.opt.Namespace),+ BucketName: new(bucketName),+ ObjectName: new(objectName),+ PutObjectBody: io.NopCloser(bytes.NewReader(nil)),+ }+ err := f.pacer.Call(func() (bool, error) {+ resp, err := f.srv.PutObject(ctx, req)+ return shouldRetry(ctx, resp.HTTPResponse(), err)+ })+ require.NoError(t, err)+ }+ // Use the SDK directly because normal OOS listings hide directory markers.+ listObjectNames := func() []string {+ req := objectstorage.ListObjectsRequest{+ NamespaceName: new(f.opt.Namespace),+ BucketName: new(bucketName),+ Prefix: new(directory + "/"),+ }+ var resp objectstorage.ListObjectsResponse+ err := f.pacer.Call(func() (bool, error) {+ var err error+ resp, err = f.srv.ListObjects(ctx, req)+ return shouldRetry(ctx, resp.HTTPResponse(), err)+ })+ require.NoError(t, err)+ names := make([]string, 0, len(resp.Objects))+ for _, object := range resp.Objects {+ names = append(names, *object.Name)+ }+ return names+ }+ assert.ElementsMatch(t, objectNames, listObjectNames())+ require.NoError(t, f.Purge(ctx, dir))+ assert.Empty(t, listObjectNames())+}+// InternalTest is called by fstests.Run to extra testsfunc (f *Fs) InternalTest(t *testing.T) {t.Run("GzipEncoding", f.InternalTestGzipEncoding)+ t.Run("PurgeBatches", f.InternalTestPurgeBatches)+ t.Run("PurgeDirectoryMarkers", f.InternalTestPurgeDirectoryMarkers)}func (f *Fs) SetUploadChunkSize(cs fs.SizeSuffix) (fs.SizeSuffix, error) {diff --git a/docs/data/backends/oracleobjectstorage.yaml b/docs/data/backends/oracleobjectstorage.yamlindex b038bba18..112dc6449 100644--- a/docs/data/backends/oracleobjectstorage.yaml+++ b/docs/data/backends/oracleobjectstorage.yaml@@ -22,6 +22,7 @@ features:- ListR- Move- OpenChunkWriter+- Purge- PutStream- ReadMetadata- ReadMimeType