Skip to content

"rsync for cloud storage" - Google Drive, S3, Dropbox, Backblaze B2, One Drive, Swift, Hubic, Wasabi, Google Cloud Storage, Azure Blob, Azure Files, Yandex Files

serve s3: reserve the .rclone_temp_ prefix for temporary objects

Nick Craig-Wood0aa90200bd66
diff --git a/cmd/serve/s3/backend.go b/cmd/serve/s3/backend.go
index 1ac2d2cba..afa4e2cae 100644
--- a/cmd/serve/s3/backend.go
+++ b/cmd/serve/s3/backend.go
@@ -25,9 +25,13 @@ var (
emptyPrefix = &gofakes3.Prefix{}
)
+// tempObjectPrefix is reserved for serve s3's temporary objects: an object
+// whose leaf name starts with it is hidden from S3 listings.
+const tempObjectPrefix = ".rclone_temp_"
+
// putObjectPrefix is prepended to the leaf name of the temporary object a
// PutObject upload is written to before it is renamed into place.
-const putObjectPrefix = ".rclone_put_object_"
+const putObjectPrefix = tempObjectPrefix + "put_"
// s3Backend implements the gofacess3.Backend interface to make an S3
// backend for gofakes3. It also implements gofakes3.MultipartBackend so that
diff --git a/cmd/serve/s3/list.go b/cmd/serve/s3/list.go
index a95bb8dee..67e9f21df 100644
--- a/cmd/serve/s3/list.go
+++ b/cmd/serve/s3/list.go
@@ -8,6 +8,11 @@ import (
"github.com/rclone/rclone/vfs"
)
+// legacyMultipartUploadPrefix marked the temporary objects of in-progress
+// multipart uploads before the tempObjectPrefix namespace was reserved
+// (rclone v1.75); leftovers from an older server are still hidden.
+const legacyMultipartUploadPrefix = ".rclone_multipart_upload_"
+
func (b *s3Backend) entryListR(_vfs *vfs.VFS, bucketName, fdPath, name string, addPrefix bool, response *gofakes3.ObjectList) error {
fp, err := bucketDirPath(bucketName, fdPath)
if err != nil {
@@ -24,7 +29,7 @@ func (b *s3Backend) entryListR(_vfs *vfs.VFS, bucketName, fdPath, name string, a
object := entry.Name()
// Hide the temporary objects of in-progress uploads
- if strings.HasPrefix(object, multipartUploadPrefix) || strings.HasPrefix(object, putObjectPrefix) {
+ if strings.HasPrefix(object, tempObjectPrefix) || strings.HasPrefix(object, legacyMultipartUploadPrefix) {
continue
}
diff --git a/cmd/serve/s3/multipart.go b/cmd/serve/s3/multipart.go
index aa17a71ef..dd38e3c3d 100644
--- a/cmd/serve/s3/multipart.go
+++ b/cmd/serve/s3/multipart.go
@@ -37,7 +37,7 @@ import (
// multipartUploadPrefix is prepended to the leaf name of the temporary object
// a streamed multipart upload is written to before it is moved into place.
-const multipartUploadPrefix = ".rclone_multipart_upload_"
+const multipartUploadPrefix = tempObjectPrefix + "multipart_"
// multipartUpload tracks one in-flight S3 multipart upload that is being
// streamed, in part order, into a single PutStream upload to the underlying Fs.
diff --git a/cmd/serve/s3/multipart_test.go b/cmd/serve/s3/multipart_test.go
index f54605d11..67c3a69bb 100644
--- a/cmd/serve/s3/multipart_test.go
+++ b/cmd/serve/s3/multipart_test.go
@@ -21,6 +21,7 @@ import (
_ "github.com/rclone/rclone/backend/memory"
"github.com/rclone/rclone/cmd/serve/proxy"
"github.com/rclone/rclone/fs"
+ "github.com/rclone/rclone/fs/object"
"github.com/rclone/rclone/fstest"
"github.com/rclone/rclone/lib/multipart"
"github.com/rclone/rclone/lib/random"
@@ -492,6 +493,39 @@ func TestMultipartAbortDuringUploadPart(t *testing.T) {
up.mu.Unlock()
}
+// TestTempObjectsHiddenFromListings checks that the reserved .rclone_temp_
+// prefix, and the multipart prefix used before it was reserved, are hidden
+// from S3 listings while remaining visible to rclone itself for cleanup.
+func TestTempObjectsHiddenFromListings(t *testing.T) {
+ core, f, bucket := newMultipartTestServer(t, false)
+ ctx := context.Background()
+
+ names := []string{
+ "visible.bin",
+ tempObjectPrefix + "anything",
+ multipartUploadPrefix + "leftover",
+ putObjectPrefix + "leftover",
+ legacyMultipartUploadPrefix + "leftover",
+ }
+ for _, name := range names {
+ data := []byte("x")
+ src := object.NewStaticObjectInfo(path.Join(bucket, name), time.Now(), int64(len(data)), true, nil, f)
+ _, err := f.Put(ctx, bytes.NewReader(data), src)
+ require.NoError(t, err)
+ }
+ // All the objects really exist on the backing remote...
+ requireOnly(t, f, bucket, names...)
+
+ // ...but only the visible one appears in an S3 listing.
+ result, err := core.ListObjects(bucket, "", "", "", 1000)
+ require.NoError(t, err)
+ var keys []string
+ for _, o := range result.Contents {
+ keys = append(keys, o.Key)
+ }
+ assert.Equal(t, []string{"visible.bin"}, keys)
+}
+
// TestMultipartOverwrite checks that a completed multipart upload atomically
// replaces an existing object of the same name.
func TestMultipartOverwrite(t *testing.T) {
diff --git a/cmd/serve/s3/serve_s3.md b/cmd/serve/s3/serve_s3.md
index 4d214a0f0..2c9152007 100644
--- a/cmd/serve/s3/serve_s3.md
+++ b/cmd/serve/s3/serve_s3.md
@@ -103,7 +103,7 @@ renamed into place on success; these remotes need to support a server-side
move or copy for this (nearly all do - without move or copy the upload is
written directly and a failed PUT may leave a partial object at the key). If
`serve s3` is killed part-way through an upload the temporary object (named
-with a leading `.rclone_put_object_`) may be left behind; it is hidden from
+with a leading `.rclone_temp_put_`) may be left behind; it is hidden from
S3 listings but must be removed manually.
### Multipart uploads
@@ -173,9 +173,33 @@ also need to support a server-side move or copy.
completed object is moved into place with a server-side operation.
This is a cheap rename on most such remotes. On these remotes, if
`serve s3` is killed part-way through an upload the temporary object
- (named with a leading `.rclone_multipart_upload_`) may be left behind;
+ (named with a leading `.rclone_temp_multipart_`) may be left behind;
it is hidden from S3 listings but must be removed manually.
+#### Cleaning up temporary objects
+
+If `serve s3` is killed part-way through an upload it can leave a
+temporary object behind, named with a leading `.rclone_temp_`. This
+whole prefix is reserved: any object whose name (the last
+`/`-separated segment of its key) starts with `.rclone_temp_` is
+hidden from S3 listings, so don't give real objects such names - an
+existing object with such a name disappears from listings (though it
+stays accessible directly by its key: only listings hide reserved
+names, `GET`, `HEAD` and `DELETE` of the exact key still work). A
+temporary object never holds acknowledged data - uploads whose
+temporary object survived were never confirmed to the client - so old
+ones are safe to delete:
+
+ rclone delete --min-age 24h --include ".rclone_temp_*" remote:path
+
+The `--min-age` protects uploads which are still in progress: make sure
+it is longer than your longest upload, especially if several `serve s3`
+instances share the same remote.
+
+rclone v1.75 named its temporary multipart objects
+`.rclone_multipart_upload_*`; leftovers from an older server are also
+hidden from listings and can be cleaned up the same way.
+
#### Disabling streaming
If you pass `--disable-multipart-streaming`, or the remote doesn't