"rsync for cloud storage" - Google Drive, S3, Dropbox, Backblaze B2, One Drive, Swift, Hubic, Wasabi, Google Cloud Storage, Azure Blob, Azure Files, Yandex Files
Go azure-blobazure-blob-storageazure-filesbackblaze-b2cloud-storagedropboxencryptionftpfuse-filesystemgogolanggoogle-cloud-storagegoogle-driveonedriveopenstack-swiftrclones3sftpsyncwebdav
serve s3: remove the directories made for a failed upload
Nick Craig-Woodf0e9e4c70ef6
diff --git a/cmd/serve/s3/backend.go b/cmd/serve/s3/backend.goindex 55c4fa4b1..9fc9a0fc1 100644--- a/cmd/serve/s3/backend.go+++ b/cmd/serve/s3/backend.go@@ -580,8 +580,11 @@ func (b *s3Backend) PutObject(// return result, gofakes3.KeyNotFound(objectName)// }+ var madeDir stringif objectDir != "." {- if _, err := mkdirRecursive(objectDir, _vfs); err != nil {+ madeDir, err = mkdirRecursive(objectDir, _vfs)+ if err != nil {+ removeMadeDirs(objectDir, madeDir, _vfs)return result, err}}@@ -597,7 +600,9 @@ func (b *s3Backend) PutObject(}// cleanup discards a failed upload, removing the temporary object- // (never the object at fp) and any stale VFS state for fp.+ // (never the object at fp), any stale VFS state for fp and the+ // directories made for it, which would be listed as directory+ // markers.cleanup := func() {if tmpFp != fp {b.forgetPath(_vfs, tmpFp)@@ -605,10 +610,12 @@ func (b *s3Backend) PutObject(} else {b.forgetPath(_vfs, fp)}+ removeMadeDirs(objectDir, madeDir, _vfs)}f, err := _vfs.Create(tmpFp)if err != nil {+ cleanup()return result, err}diff --git a/cmd/serve/s3/multipart.go b/cmd/serve/s3/multipart.goindex bb14846b5..518b0afc8 100644--- a/cmd/serve/s3/multipart.go+++ b/cmd/serve/s3/multipart.go@@ -304,6 +304,7 @@ func (b *s3Backend) CreateMultipartUpload(ctx context.Context, bucketName, objecif objectDir != "." {madeDir, err = mkdirRecursive(objectDir, _vfs)if err != nil {+ removeMadeDirs(objectDir, madeDir, _vfs)b.releaseUpload(t)return "", err}@@ -327,6 +328,7 @@ func (b *s3Backend) CreateMultipartUpload(ctx context.Context, bucketName, objecup.madeDir = madeDirif err := b.addUpload(t, uploadID, up); err != nil {+ removeMadeDirs(objectDir, madeDir, _vfs)b.releaseUpload(t)return "", err}@@ -1079,13 +1081,7 @@ func (b *s3Backend) discardUpload(up *multipartUpload) {// Remove the directories made for the upload if they are still// empty, as an empty directory left behind would stop the bucket// being deleted.- if up.madeDir != "" {- for dir := path.Dir(up.fp); isEmptyDir(dir, up.vfs); dir = path.Dir(dir) {- if up.vfs.Remove(dir) != nil || dir == up.madeDir {- break- }- }- }+ removeMadeDirs(path.Dir(up.fp), up.madeDir, up.vfs)}// forgetPath invalidates the parent directory's cached VFS listing so thatdiff --git a/cmd/serve/s3/put_test.go b/cmd/serve/s3/put_test.goindex 1b0c9653b..ed8d8c6bd 100644--- a/cmd/serve/s3/put_test.go+++ b/cmd/serve/s3/put_test.go@@ -156,6 +156,30 @@ func TestPutObjectFailureNewKey(t *testing.T) {}}+// TestPutObjectFailureNewDir checks that a failed PUT to a key in+// directories which don't exist removes the directories it made, which+// would otherwise be listed as directory markers.+func TestPutObjectFailureNewDir(t *testing.T) {+ for _, tc := range testRemotes {+ t.Run(tc.name, func(t *testing.T) {+ b, _, bucket := newPutTestBackend(t, tc.backing, nil)+ ctx := context.Background()+ _, err := b.PutObject(ctx, bucket, "existing/file.txt", map[string]string{}, bytes.NewReader([]byte("hello")), 5)+ require.NoError(t, err)++ failPut(t, b, bucket, "existing/new/dirs/new.txt", errBoom, errBoom)++ list, err := b.ListBucket(ctx, bucket, nil, gofakes3.ListBucketPage{})+ require.NoError(t, err)+ var keys []string+ for _, content := range list.Contents {+ keys = append(keys, content.Key)+ }+ assert.Equal(t, []string{"existing/file.txt"}, keys)+ })+ }+}+// TestPutObjectMtime checks that the object's modtime is set from the// "X-Amz-Meta-Mtime" or "mtime" metadata supplied with the PUT.func TestPutObjectMtime(t *testing.T) {diff --git a/cmd/serve/s3/utils.go b/cmd/serve/s3/utils.goindex 37de2636f..790755f3b 100644--- a/cmd/serve/s3/utils.go+++ b/cmd/serve/s3/utils.go@@ -213,6 +213,20 @@ func isEmptyDir(dir string, VFS *vfs.VFS) bool {return err == nil && len(entries) == 0}+// removeMadeDirs removes the directory dir, which mkdirRecursive+// returned made for, and its parents while they are empty, up to and+// including made. It does nothing if made is "".+func removeMadeDirs(dir, made string, VFS *vfs.VFS) {+ if made == "" {+ return+ }+ for ; isEmptyDir(dir, VFS); dir = path.Dir(dir) {+ if VFS.Remove(dir) != nil || dir == made {+ return+ }+ }+}+// rmdirRecursive removes the parent directories of p which are empty,// stopping at the bucket.func rmdirRecursive(p string, VFS *vfs.VFS) {