Skip to content

"rsync for cloud storage" - Google Drive, S3, Dropbox, Backblaze B2, One Drive, Swift, Hubic, Wasabi, Google Cloud Storage, Azure Blob, Azure Files, Yandex Files

serve s3: remove the directories made for a failed upload

Nick Craig-Woodf0e9e4c70ef6
diff --git a/cmd/serve/s3/backend.go b/cmd/serve/s3/backend.go
index 55c4fa4b1..9fc9a0fc1 100644
--- a/cmd/serve/s3/backend.go
+++ b/cmd/serve/s3/backend.go
@@ -580,8 +580,11 @@ func (b *s3Backend) PutObject(
// return result, gofakes3.KeyNotFound(objectName)
// }
+ var madeDir string
if objectDir != "." {
- if _, err := mkdirRecursive(objectDir, _vfs); err != nil {
+ madeDir, err = mkdirRecursive(objectDir, _vfs)
+ if err != nil {
+ removeMadeDirs(objectDir, madeDir, _vfs)
return result, err
}
}
@@ -597,7 +600,9 @@ func (b *s3Backend) PutObject(
}
// cleanup discards a failed upload, removing the temporary object
- // (never the object at fp) and any stale VFS state for fp.
+ // (never the object at fp), any stale VFS state for fp and the
+ // directories made for it, which would be listed as directory
+ // markers.
cleanup := func() {
if tmpFp != fp {
b.forgetPath(_vfs, tmpFp)
@@ -605,10 +610,12 @@ func (b *s3Backend) PutObject(
} else {
b.forgetPath(_vfs, fp)
}
+ removeMadeDirs(objectDir, madeDir, _vfs)
}
f, err := _vfs.Create(tmpFp)
if err != nil {
+ cleanup()
return result, err
}
diff --git a/cmd/serve/s3/multipart.go b/cmd/serve/s3/multipart.go
index bb14846b5..518b0afc8 100644
--- a/cmd/serve/s3/multipart.go
+++ b/cmd/serve/s3/multipart.go
@@ -304,6 +304,7 @@ func (b *s3Backend) CreateMultipartUpload(ctx context.Context, bucketName, objec
if objectDir != "." {
madeDir, err = mkdirRecursive(objectDir, _vfs)
if err != nil {
+ removeMadeDirs(objectDir, madeDir, _vfs)
b.releaseUpload(t)
return "", err
}
@@ -327,6 +328,7 @@ func (b *s3Backend) CreateMultipartUpload(ctx context.Context, bucketName, objec
up.madeDir = madeDir
if err := b.addUpload(t, uploadID, up); err != nil {
+ removeMadeDirs(objectDir, madeDir, _vfs)
b.releaseUpload(t)
return "", err
}
@@ -1079,13 +1081,7 @@ func (b *s3Backend) discardUpload(up *multipartUpload) {
// Remove the directories made for the upload if they are still
// empty, as an empty directory left behind would stop the bucket
// being deleted.
- if up.madeDir != "" {
- for dir := path.Dir(up.fp); isEmptyDir(dir, up.vfs); dir = path.Dir(dir) {
- if up.vfs.Remove(dir) != nil || dir == up.madeDir {
- break
- }
- }
- }
+ removeMadeDirs(path.Dir(up.fp), up.madeDir, up.vfs)
}
// forgetPath invalidates the parent directory's cached VFS listing so that
diff --git a/cmd/serve/s3/put_test.go b/cmd/serve/s3/put_test.go
index 1b0c9653b..ed8d8c6bd 100644
--- a/cmd/serve/s3/put_test.go
+++ b/cmd/serve/s3/put_test.go
@@ -156,6 +156,30 @@ func TestPutObjectFailureNewKey(t *testing.T) {
}
}
+// TestPutObjectFailureNewDir checks that a failed PUT to a key in
+// directories which don't exist removes the directories it made, which
+// would otherwise be listed as directory markers.
+func TestPutObjectFailureNewDir(t *testing.T) {
+ for _, tc := range testRemotes {
+ t.Run(tc.name, func(t *testing.T) {
+ b, _, bucket := newPutTestBackend(t, tc.backing, nil)
+ ctx := context.Background()
+ _, err := b.PutObject(ctx, bucket, "existing/file.txt", map[string]string{}, bytes.NewReader([]byte("hello")), 5)
+ require.NoError(t, err)
+
+ failPut(t, b, bucket, "existing/new/dirs/new.txt", errBoom, errBoom)
+
+ list, err := b.ListBucket(ctx, bucket, nil, gofakes3.ListBucketPage{})
+ require.NoError(t, err)
+ var keys []string
+ for _, content := range list.Contents {
+ keys = append(keys, content.Key)
+ }
+ assert.Equal(t, []string{"existing/file.txt"}, keys)
+ })
+ }
+}
+
// TestPutObjectMtime checks that the object's modtime is set from the
// "X-Amz-Meta-Mtime" or "mtime" metadata supplied with the PUT.
func TestPutObjectMtime(t *testing.T) {
diff --git a/cmd/serve/s3/utils.go b/cmd/serve/s3/utils.go
index 37de2636f..790755f3b 100644
--- a/cmd/serve/s3/utils.go
+++ b/cmd/serve/s3/utils.go
@@ -213,6 +213,20 @@ func isEmptyDir(dir string, VFS *vfs.VFS) bool {
return err == nil && len(entries) == 0
}
+// removeMadeDirs removes the directory dir, which mkdirRecursive
+// returned made for, and its parents while they are empty, up to and
+// including made. It does nothing if made is "".
+func removeMadeDirs(dir, made string, VFS *vfs.VFS) {
+ if made == "" {
+ return
+ }
+ for ; isEmptyDir(dir, VFS); dir = path.Dir(dir) {
+ if VFS.Remove(dir) != nil || dir == made {
+ return
+ }
+ }
+}
+
// rmdirRecursive removes the parent directories of p which are empty,
// stopping at the bucket.
func rmdirRecursive(p string, VFS *vfs.VFS) {