Skip to content

"rsync for cloud storage" - Google Drive, S3, Dropbox, Backblaze B2, One Drive, Swift, Hubic, Wasabi, Google Cloud Storage, Azure Blob, Azure Files, Yandex Files

oracleobjectstorage: purge objects in batches

tomaszni934c21de2680
diff --git a/backend/oracleobjectstorage/oracleobjectstorage.go b/backend/oracleobjectstorage/oracleobjectstorage.go
index 305397b17..06605f5d9 100644
--- a/backend/oracleobjectstorage/oracleobjectstorage.go
+++ b/backend/oracleobjectstorage/oracleobjectstorage.go
@@ -5,6 +5,7 @@ package oracleobjectstorage
import (
"context"
+ "errors"
"fmt"
"io"
"net/http"
@@ -302,17 +303,28 @@ func (f *Fs) ListP(ctx context.Context, dir string, callback fs.ListRCallback) e
// listFn is called from list to handle an object.
type listFn func(remote string, object *objectstorage.ObjectSummary, isDirectory bool) error
-// list the objects into the function supplied from
-// the bucket and root supplied
-// (bucket, directory) is the starting directory
+// list calls fn for objects in bucket and directory.
+//
+// ctx controls the API requests.
+// bucket is the bucket to list.
+// directory is the path to list within bucket.
// If prefix is set then it is removed from all file names
// If addBucket is set then it adds the bucket to the start of the remotes generated
// If recurse is set the function will recursively list
// If limit is > 0 then it limits to that many files (must be less than 1000)
-// If hidden is set then it will list the hidden (deleted) files too.
-// if findFile is set it will look for files called (bucket, directory)
-func (f *Fs) list(ctx context.Context, bucket, directory, prefix string, addBucket bool, recurse bool, limit int,
- fn listFn) (err error) {
+// includeDirectoryMarkers includes zero-length objects with names ending in a slash.
+// fn receives each listed object.
+func (f *Fs) list(
+ ctx context.Context,
+ bucket string,
+ directory string,
+ prefix string,
+ addBucket bool,
+ recurse bool,
+ limit int,
+ includeDirectoryMarkers bool,
+ fn listFn,
+) (err error) {
if prefix != "" {
prefix += "/"
}
@@ -407,7 +419,7 @@ func (f *Fs) list(ctx context.Context, bucket, directory, prefix string, addBuck
remote = path.Join(bucket, remote)
}
// is this a directory marker?
- if isDirectory && object.Size != nil && *object.Size == 0 {
+ if isDirectory && object.Size != nil && *object.Size == 0 && !includeDirectoryMarkers {
continue // skip directory marker
}
if isDirectory && len(remote) > 1 {
@@ -456,7 +468,7 @@ func (f *Fs) listDir(ctx context.Context, bucket, directory, prefix string, addB
}
return nil
}
- err = f.list(ctx, bucket, directory, prefix, addBucket, false, 0, fn)
+ err = f.list(ctx, bucket, directory, prefix, addBucket, false, 0, false, fn)
if err != nil {
return err
}
@@ -641,6 +653,66 @@ func (f *Fs) Rmdir(ctx context.Context, dir string) error {
})
}
+// ------------------------------------------------------------
+// Implement Purger is an optional interfaces for Fs
+//------------------------------------------------------------
+
+// Purge deletes all the files in dir.
+//
+// It implements fs.Purger, using OCI batch requests rather than removing each
+// object returned by List. It also removes directory markers and, when dir is
+// a bucket root, removes the empty bucket.
+func (f *Fs) Purge(ctx context.Context, dir string) error {
+ bucketName, directory := f.split(dir)
+ if bucketName == "" {
+ return fs.ErrorListBucketRequired
+ }
+ const batchSize = 1000
+ objects := make([]objectstorage.BatchDeleteObjectIdentifier, 0, batchSize)
+ deleteObjects := func() error {
+ if len(objects) == 0 {
+ return nil
+ }
+ req := objectstorage.BatchDeleteObjectsRequest{
+ NamespaceName: new(f.opt.Namespace),
+ BucketName: new(bucketName),
+ BatchDeleteObjectsDetails: objectstorage.BatchDeleteObjectsDetails{
+ Objects: objects,
+ IsSkipDeletedResult: new(true),
+ },
+ }
+ var resp objectstorage.BatchDeleteObjectsResponse
+ err := f.pacer.Call(func() (bool, error) {
+ var err error
+ resp, err = f.srv.BatchDeleteObjects(ctx, req)
+ return shouldRetry(ctx, resp.HTTPResponse(), err)
+ })
+ if err != nil {
+ return err
+ }
+ var errs []error
+ for _, failed := range resp.Failed {
+ errs = append(errs, fmt.Errorf("failed to delete object %q: %s", *failed.ObjectName, *failed.ErrorMessage))
+ }
+ objects = objects[:0]
+ return errors.Join(errs...)
+ }
+ err := f.list(ctx, bucketName, directory, f.rootDirectory, false, true, 0, true, func(_ string, object *objectstorage.ObjectSummary, _ bool) error {
+ objects = append(objects, objectstorage.BatchDeleteObjectIdentifier{ObjectName: object.Name})
+ if len(objects) == batchSize {
+ return deleteObjects()
+ }
+ return nil
+ })
+ if err != nil {
+ return err
+ }
+ if err = deleteObjects(); err != nil {
+ return err
+ }
+ return f.Rmdir(ctx, dir)
+}
+
func (f *Fs) abortMultiPartUpload(ctx context.Context, bucketName, bucketPath, uploadID *string) (err error) {
if uploadID == nil || *uploadID == "" {
return nil
@@ -728,7 +800,7 @@ func (f *Fs) ListR(ctx context.Context, dir string, callback fs.ListRCallback) (
bucketName, directory := f.split(dir)
list := list.NewHelper(callback)
listR := func(bucket, directory, prefix string, addBucket bool) error {
- return f.list(ctx, bucket, directory, prefix, addBucket, true, 0, func(remote string, object *objectstorage.ObjectSummary, isDirectory bool) error {
+ return f.list(ctx, bucket, directory, prefix, addBucket, true, 0, false, func(remote string, object *objectstorage.ObjectSummary, isDirectory bool) error {
entry, err := f.itemToDirEntry(ctx, remote, object, isDirectory)
if err != nil {
return err
@@ -807,6 +879,7 @@ var (
_ fs.ListPer = &Fs{}
_ fs.Commander = &Fs{}
_ fs.CleanUpper = &Fs{}
+ _ fs.Purger = &Fs{}
_ fs.OpenChunkWriter = &Fs{}
_ fs.Object = &Object{}
diff --git a/backend/oracleobjectstorage/oracleobjectstorage_test.go b/backend/oracleobjectstorage/oracleobjectstorage_test.go
index feb215db5..899fd3a17 100644
--- a/backend/oracleobjectstorage/oracleobjectstorage_test.go
+++ b/backend/oracleobjectstorage/oracleobjectstorage_test.go
@@ -8,10 +8,15 @@ import (
"context"
"crypto/md5"
"fmt"
+ "io"
+ "sync"
"testing"
+ "time"
+ "github.com/oracle/oci-go-sdk/v65/objectstorage"
"github.com/rclone/rclone/fs"
"github.com/rclone/rclone/fs/hash"
+ "github.com/rclone/rclone/fs/object"
"github.com/rclone/rclone/fstest"
"github.com/rclone/rclone/fstest/fstests"
"github.com/rclone/rclone/lib/random"
@@ -81,9 +86,99 @@ func (f *Fs) InternalTestGzipEncoding(t *testing.T) {
})
}
+// InternalTestPurgeBatches tests purging more objects than fit in one batch.
+func (f *Fs) InternalTestPurgeBatches(t *testing.T) {
+ ctx := context.Background()
+ const (
+ dir = "purge-batches"
+ objectCount = 1001
+ uploadConcurrency = 16
+ )
+ defer func() { _ = f.Purge(ctx, dir) }()
+ jobs := make(chan int)
+ errs := make(chan error, objectCount)
+ var wg sync.WaitGroup
+ for range uploadConcurrency {
+ wg.Go(func() {
+ for i := range jobs {
+ remote := fmt.Sprintf("%s/%04d", dir, i)
+ src := object.NewStaticObjectInfo(remote, time.Time{}, 0, true, nil, f)
+ _, err := f.Put(ctx, bytes.NewReader(nil), src)
+ if err != nil {
+ errs <- err
+ }
+ }
+ })
+ }
+ for i := range objectCount {
+ jobs <- i
+ }
+ close(jobs)
+ wg.Wait()
+ close(errs)
+ for err := range errs {
+ require.NoError(t, err)
+ }
+ require.NoError(t, f.Purge(ctx, dir))
+ entries, err := f.List(ctx, dir)
+ require.NoError(t, err)
+ assert.Empty(t, entries)
+}
+
+// InternalTestPurgeDirectoryMarkers tests purging directory marker objects.
+func (f *Fs) InternalTestPurgeDirectoryMarkers(t *testing.T) {
+ ctx := context.Background()
+ const dir = "purge-directory-markers"
+ bucketName, directory := f.split(dir)
+ objectNames := []string{
+ directory + "/",
+ directory + "/child/",
+ directory + "/child/object",
+ }
+ defer func() { _ = f.Purge(ctx, dir) }()
+ for _, objectName := range objectNames {
+ req := objectstorage.PutObjectRequest{
+ NamespaceName: new(f.opt.Namespace),
+ BucketName: new(bucketName),
+ ObjectName: new(objectName),
+ PutObjectBody: io.NopCloser(bytes.NewReader(nil)),
+ }
+ err := f.pacer.Call(func() (bool, error) {
+ resp, err := f.srv.PutObject(ctx, req)
+ return shouldRetry(ctx, resp.HTTPResponse(), err)
+ })
+ require.NoError(t, err)
+ }
+ // Use the SDK directly because normal OOS listings hide directory markers.
+ listObjectNames := func() []string {
+ req := objectstorage.ListObjectsRequest{
+ NamespaceName: new(f.opt.Namespace),
+ BucketName: new(bucketName),
+ Prefix: new(directory + "/"),
+ }
+ var resp objectstorage.ListObjectsResponse
+ err := f.pacer.Call(func() (bool, error) {
+ var err error
+ resp, err = f.srv.ListObjects(ctx, req)
+ return shouldRetry(ctx, resp.HTTPResponse(), err)
+ })
+ require.NoError(t, err)
+ names := make([]string, 0, len(resp.Objects))
+ for _, object := range resp.Objects {
+ names = append(names, *object.Name)
+ }
+ return names
+ }
+ assert.ElementsMatch(t, objectNames, listObjectNames())
+ require.NoError(t, f.Purge(ctx, dir))
+ assert.Empty(t, listObjectNames())
+}
+
// InternalTest is called by fstests.Run to extra tests
func (f *Fs) InternalTest(t *testing.T) {
t.Run("GzipEncoding", f.InternalTestGzipEncoding)
+ t.Run("PurgeBatches", f.InternalTestPurgeBatches)
+ t.Run("PurgeDirectoryMarkers", f.InternalTestPurgeDirectoryMarkers)
}
func (f *Fs) SetUploadChunkSize(cs fs.SizeSuffix) (fs.SizeSuffix, error) {
diff --git a/docs/data/backends/oracleobjectstorage.yaml b/docs/data/backends/oracleobjectstorage.yaml
index b038bba18..112dc6449 100644
--- a/docs/data/backends/oracleobjectstorage.yaml
+++ b/docs/data/backends/oracleobjectstorage.yaml
@@ -22,6 +22,7 @@ features:
- ListR
- Move
- OpenChunkWriter
+- Purge
- PutStream
- ReadMetadata
- ReadMimeType