"rsync for cloud storage" - Google Drive, S3, Dropbox, Backblaze B2, One Drive, Swift, Hubic, Wasabi, Google Cloud Storage, Azure Blob, Azure Files, Yandex Files
Go azure-blobazure-blob-storageazure-filesbackblaze-b2cloud-storagedropboxencryptionftpfuse-filesystemgogolanggoogle-cloud-storagegoogle-driveonedriveopenstack-swiftrclones3sftpsyncwebdav
serve s3: reserve the .rclone_temp_ prefix for temporary objects
Nick Craig-Wood0aa90200bd66
diff --git a/cmd/serve/s3/backend.go b/cmd/serve/s3/backend.goindex 1ac2d2cba..afa4e2cae 100644--- a/cmd/serve/s3/backend.go+++ b/cmd/serve/s3/backend.go@@ -25,9 +25,13 @@ var (emptyPrefix = &gofakes3.Prefix{})+// tempObjectPrefix is reserved for serve s3's temporary objects: an object+// whose leaf name starts with it is hidden from S3 listings.+const tempObjectPrefix = ".rclone_temp_"+// putObjectPrefix is prepended to the leaf name of the temporary object a// PutObject upload is written to before it is renamed into place.-const putObjectPrefix = ".rclone_put_object_"+const putObjectPrefix = tempObjectPrefix + "put_"// s3Backend implements the gofacess3.Backend interface to make an S3// backend for gofakes3. It also implements gofakes3.MultipartBackend so thatdiff --git a/cmd/serve/s3/list.go b/cmd/serve/s3/list.goindex a95bb8dee..67e9f21df 100644--- a/cmd/serve/s3/list.go+++ b/cmd/serve/s3/list.go@@ -8,6 +8,11 @@ import ("github.com/rclone/rclone/vfs")+// legacyMultipartUploadPrefix marked the temporary objects of in-progress+// multipart uploads before the tempObjectPrefix namespace was reserved+// (rclone v1.75); leftovers from an older server are still hidden.+const legacyMultipartUploadPrefix = ".rclone_multipart_upload_"+func (b *s3Backend) entryListR(_vfs *vfs.VFS, bucketName, fdPath, name string, addPrefix bool, response *gofakes3.ObjectList) error {fp, err := bucketDirPath(bucketName, fdPath)if err != nil {@@ -24,7 +29,7 @@ func (b *s3Backend) entryListR(_vfs *vfs.VFS, bucketName, fdPath, name string, aobject := entry.Name()// Hide the temporary objects of in-progress uploads- if strings.HasPrefix(object, multipartUploadPrefix) || strings.HasPrefix(object, putObjectPrefix) {+ if strings.HasPrefix(object, tempObjectPrefix) || strings.HasPrefix(object, legacyMultipartUploadPrefix) {continue}diff --git a/cmd/serve/s3/multipart.go b/cmd/serve/s3/multipart.goindex aa17a71ef..dd38e3c3d 100644--- a/cmd/serve/s3/multipart.go+++ b/cmd/serve/s3/multipart.go@@ -37,7 +37,7 @@ import (// multipartUploadPrefix is prepended to the leaf name of the temporary object// a streamed multipart upload is written to before it is moved into place.-const multipartUploadPrefix = ".rclone_multipart_upload_"+const multipartUploadPrefix = tempObjectPrefix + "multipart_"// multipartUpload tracks one in-flight S3 multipart upload that is being// streamed, in part order, into a single PutStream upload to the underlying Fs.diff --git a/cmd/serve/s3/multipart_test.go b/cmd/serve/s3/multipart_test.goindex f54605d11..67c3a69bb 100644--- a/cmd/serve/s3/multipart_test.go+++ b/cmd/serve/s3/multipart_test.go@@ -21,6 +21,7 @@ import (_ "github.com/rclone/rclone/backend/memory""github.com/rclone/rclone/cmd/serve/proxy""github.com/rclone/rclone/fs"+ "github.com/rclone/rclone/fs/object""github.com/rclone/rclone/fstest""github.com/rclone/rclone/lib/multipart""github.com/rclone/rclone/lib/random"@@ -492,6 +493,39 @@ func TestMultipartAbortDuringUploadPart(t *testing.T) {up.mu.Unlock()}+// TestTempObjectsHiddenFromListings checks that the reserved .rclone_temp_+// prefix, and the multipart prefix used before it was reserved, are hidden+// from S3 listings while remaining visible to rclone itself for cleanup.+func TestTempObjectsHiddenFromListings(t *testing.T) {+ core, f, bucket := newMultipartTestServer(t, false)+ ctx := context.Background()++ names := []string{+ "visible.bin",+ tempObjectPrefix + "anything",+ multipartUploadPrefix + "leftover",+ putObjectPrefix + "leftover",+ legacyMultipartUploadPrefix + "leftover",+ }+ for _, name := range names {+ data := []byte("x")+ src := object.NewStaticObjectInfo(path.Join(bucket, name), time.Now(), int64(len(data)), true, nil, f)+ _, err := f.Put(ctx, bytes.NewReader(data), src)+ require.NoError(t, err)+ }+ // All the objects really exist on the backing remote...+ requireOnly(t, f, bucket, names...)++ // ...but only the visible one appears in an S3 listing.+ result, err := core.ListObjects(bucket, "", "", "", 1000)+ require.NoError(t, err)+ var keys []string+ for _, o := range result.Contents {+ keys = append(keys, o.Key)+ }+ assert.Equal(t, []string{"visible.bin"}, keys)+}+// TestMultipartOverwrite checks that a completed multipart upload atomically// replaces an existing object of the same name.func TestMultipartOverwrite(t *testing.T) {diff --git a/cmd/serve/s3/serve_s3.md b/cmd/serve/s3/serve_s3.mdindex 4d214a0f0..2c9152007 100644--- a/cmd/serve/s3/serve_s3.md+++ b/cmd/serve/s3/serve_s3.md@@ -103,7 +103,7 @@ renamed into place on success; these remotes need to support a server-sidemove or copy for this (nearly all do - without move or copy the upload iswritten directly and a failed PUT may leave a partial object at the key). If`serve s3` is killed part-way through an upload the temporary object (named-with a leading `.rclone_put_object_`) may be left behind; it is hidden from+with a leading `.rclone_temp_put_`) may be left behind; it is hidden fromS3 listings but must be removed manually.### Multipart uploads@@ -173,9 +173,33 @@ also need to support a server-side move or copy.completed object is moved into place with a server-side operation.This is a cheap rename on most such remotes. On these remotes, if`serve s3` is killed part-way through an upload the temporary object- (named with a leading `.rclone_multipart_upload_`) may be left behind;+ (named with a leading `.rclone_temp_multipart_`) may be left behind;it is hidden from S3 listings but must be removed manually.+#### Cleaning up temporary objects++If `serve s3` is killed part-way through an upload it can leave a+temporary object behind, named with a leading `.rclone_temp_`. This+whole prefix is reserved: any object whose name (the last+`/`-separated segment of its key) starts with `.rclone_temp_` is+hidden from S3 listings, so don't give real objects such names - an+existing object with such a name disappears from listings (though it+stays accessible directly by its key: only listings hide reserved+names, `GET`, `HEAD` and `DELETE` of the exact key still work). A+temporary object never holds acknowledged data - uploads whose+temporary object survived were never confirmed to the client - so old+ones are safe to delete:++ rclone delete --min-age 24h --include ".rclone_temp_*" remote:path++The `--min-age` protects uploads which are still in progress: make sure+it is longer than your longest upload, especially if several `serve s3`+instances share the same remote.++rclone v1.75 named its temporary multipart objects+`.rclone_multipart_upload_*`; leftovers from an older server are also+hidden from listings and can be cleaned up the same way.+#### Disabling streamingIf you pass `--disable-multipart-streaming`, or the remote doesn't