"rsync for cloud storage" - Google Drive, S3, Dropbox, Backblaze B2, One Drive, Swift, Hubic, Wasabi, Google Cloud Storage, Azure Blob, Azure Files, Yandex Files
Go azure-blobazure-blob-storageazure-filesbackblaze-b2cloud-storagedropboxencryptionftpfuse-filesystemgogolanggoogle-cloud-storagegoogle-driveonedriveopenstack-swiftrclones3sftpsyncwebdav
serve s3: support directory markers and refuse file/directory conflicts
Nick Craig-Wood63bd68739b86
diff --git a/cmd/serve/s3/backend.go b/cmd/serve/s3/backend.goindex 8da2e5830..cc16e4728 100644--- a/cmd/serve/s3/backend.go+++ b/cmd/serve/s3/backend.go@@ -309,6 +309,12 @@ func (b *s3Backend) HeadObject(ctx context.Context, bucketName, objectName strinreturn nil, gofakes3.BucketNotFound(bucketName)}+ if fp, isMarker, err := bucketDirMarkerPath(bucketName, objectName); isMarker {+ if err != nil {+ return nil, err+ }+ return b.dirMarkerObject(_vfs, fp, objectName)+ }fp, err := bucketObjectPath(bucketName, objectName)if err != nil {return nil, err@@ -366,6 +372,15 @@ func (b *s3Backend) GetObject(ctx context.Context, bucketName, objectName stringreturn nil, gofakes3.BucketNotFound(bucketName)}+ if fp, isMarker, err := bucketDirMarkerPath(bucketName, objectName); isMarker {+ if err != nil {+ return nil, err+ }+ if _, err := rangeRequest.Range(0); err != nil {+ return nil, err+ }+ return b.dirMarkerObject(_vfs, fp, objectName)+ }fp, err := bucketObjectPath(bucketName, objectName)if err != nil {return nil, err@@ -436,6 +451,44 @@ func (b *s3Backend) GetObject(ctx context.Context, bucketName, objectName string}, nil}+// dirMarkerObject returns the empty object the directory marker key+// objectName stands for if its directory fp exists.+func (b *s3Backend) dirMarkerObject(_vfs *vfs.VFS, fp, objectName string) (*gofakes3.Object, error) {+ node, err := _vfs.Stat(fp)+ if err != nil || !node.IsDir() {+ return nil, gofakes3.KeyNotFound(objectName)+ }+ return &gofakes3.Object{+ Name: objectName,+ Hash: emptyHash(b.s.etagHash(_vfs)),+ Metadata: map[string]string{+ "Last-Modified": formatHeaderTime(node.ModTime()),+ "Content-Type": "application/x-directory",+ },+ Size: 0,+ Contents: noOpReadCloser{},+ }, nil+}++// putDirMarker makes the directory fp the directory marker key+// objectName stands for. A directory marker can't hold data.+func putDirMarker(_vfs *vfs.VFS, fp, objectName string, input io.Reader) error {+ // Read to the end of an empty input so any checksum of it is+ // checked, but no further.+ n, err := io.Copy(io.Discard, io.LimitReader(input, 1))+ if err != nil {+ return err+ }+ if n != 0 {+ return gofakes3.ErrorMessagef(gofakes3.ErrInvalidArgument, "Directory marker %q must be empty", objectName)+ }+ if node, err := _vfs.Stat(fp); err == nil && !node.IsDir() {+ return errParentIsObject(strings.TrimSuffix(objectName, "/"))+ }+ _, err = mkdirRecursive(fp, _vfs)+ return err+}+// TouchObject creates or updates meta on specified object.func (b *s3Backend) TouchObject(ctx context.Context, fp string, meta map[string]string) (result gofakes3.PutObjectResult, err error) {_vfs, err := b.s.getVFS(ctx)@@ -481,10 +534,19 @@ func (b *s3Backend) PutObject(return result, gofakes3.BucketNotFound(bucketName)}+ if fp, isMarker, err := bucketDirMarkerPath(bucketName, objectName); isMarker {+ if err != nil {+ return result, err+ }+ return result, putDirMarker(_vfs, fp, objectName, input)+ }fp, err := bucketObjectPath(bucketName, objectName)if err != nil {return result, err}+ if node, err := _vfs.Stat(fp); err == nil && node.IsDir() {+ return result, errIsDirectory(objectName)+ }objectDir := path.Dir(fp)// _, err = db.fs.Stat(objectDir)// if err == vfs.ENOENT {@@ -599,6 +661,20 @@ func (b *s3Backend) deleteObject(ctx context.Context, bucketName, objectName strreturn gofakes3.BucketNotFound(bucketName)}+ if fp, isMarker, err := bucketDirMarkerPath(bucketName, objectName); isMarker {+ if err != nil {+ return err+ }+ // Deleting a directory marker leaves the objects in its+ // directory, so the directory is only removed if empty.+ if isEmptyDir(fp, _vfs) {+ if err := _vfs.Remove(fp); err != nil {+ return err+ }+ rmdirRecursive(fp, _vfs)+ }+ return nil+ }fp, err := bucketObjectPath(bucketName, objectName)if err != nil {return errdiff --git a/cmd/serve/s3/backend_test.go b/cmd/serve/s3/backend_test.goindex 60ab12640..5dd12fb3e 100644--- a/cmd/serve/s3/backend_test.go+++ b/cmd/serve/s3/backend_test.go@@ -3,6 +3,7 @@ package s3import ("context""crypto/md5"+ "encoding/hex""io""os""path/filepath"@@ -367,3 +368,94 @@ func TestSetObjectTagging(t *testing.T) {err = b.SetObjectTagging(ctx, "bucket", "missing.txt", "a=1")assert.True(t, gofakes3.HasErrorCode(err, gofakes3.ErrNoSuchKey), "%v", err)}++// TestDirMarkers checks directory marker keys, which end in "/", stand+// for directories.+func TestDirMarkers(t *testing.T) {+ b, root := newTestBackend(t)+ ctx := context.Background()+ put := func(key, data string) error {+ _, err := b.PutObject(ctx, "bucket", key, nil, strings.NewReader(data), int64(len(data)))+ return err+ }+ isDir := func(dir string) bool {+ info, err := os.Stat(filepath.Join(root, "bucket", dir))+ return err == nil && info.IsDir()+ }++ // Putting an empty marker makes the directory+ require.NoError(t, put("dir/sub/", ""))+ assert.True(t, isDir("dir/sub"))+ obj, err := b.HeadObject(ctx, "bucket", "dir/sub/")+ require.NoError(t, err)+ assert.Equal(t, int64(0), obj.Size)+ assert.Equal(t, "d41d8cd98f00b204e9800998ecf8427e", hex.EncodeToString(obj.Hash)) // md5("")+ obj, err = b.GetObject(ctx, "bucket", "dir/", nil)+ require.NoError(t, err)+ data, err := io.ReadAll(obj.Contents)+ require.NoError(t, err)+ assert.Empty(t, data)++ // A marker with data is refused+ err = put("data/", "x")+ assert.True(t, gofakes3.HasErrorCode(err, gofakes3.ErrInvalidArgument), "%v", err)+ assert.False(t, isDir("data"))++ // A marker for a directory which doesn't exist isn't found, nor is+ // a directory without the "/"+ for _, key := range []string{"missing/", "dir"} {+ _, err = b.HeadObject(ctx, "bucket", key)+ assert.True(t, gofakes3.HasErrorCode(err, gofakes3.ErrNoSuchKey), "%s: %v", key, err)+ }++ // Deleting a marker removes its directory only if empty+ require.NoError(t, put("dir/sub/file.txt", "data"))+ require.NoError(t, b.deleteObject(ctx, "bucket", "dir/sub/"))+ assert.True(t, isDir("dir/sub"))+ require.NoError(t, b.deleteObject(ctx, "bucket", "dir/sub/file.txt"))+ require.NoError(t, put("dir/sub/", ""))+ require.NoError(t, b.deleteObject(ctx, "bucket", "dir/sub/"))+ assert.False(t, isDir("dir/sub"))++ // Markers must be in canonical form apart from the "/"+ for _, key := range []string{"/", "a//", "a/./"} {+ err = put(key, "")+ assert.True(t, gofakes3.HasErrorCode(err, gofakes3.ErrInvalidArgument), "%s: %v", key, err)+ }+}++// TestFileDirConflicts checks keys which conflict with directories or+// files are refused with a client error rather than an internal one,+// which clients would retry.+func TestFileDirConflicts(t *testing.T) {+ b, _ := newTestBackend(t)+ ctx := context.Background()+ put := func(key string) error {+ _, err := b.PutObject(ctx, "bucket", key, nil, strings.NewReader("data"), 4)+ return err+ }+ require.NoError(t, put("dir/file.txt"))++ for _, test := range []struct {+ key string+ call func(key string) error+ }{+ {"object.txt/below", put},+ {"object.txt/", func(key string) error {+ _, err := b.PutObject(ctx, "bucket", key, nil, strings.NewReader(""), 0)+ return err+ }},+ {"dir", put},+ {"dir", func(key string) error {+ _, err := b.CreateMultipartUpload(ctx, "bucket", key, nil)+ return err+ }},+ {"object.txt/below", func(key string) error {+ _, err := b.CreateMultipartUpload(ctx, "bucket", key, nil)+ return err+ }},+ } {+ err := test.call(test.key)+ assert.True(t, gofakes3.HasErrorCode(err, gofakes3.ErrInvalidRequest), "%s: %v", test.key, err)+ }+}diff --git a/cmd/serve/s3/multipart.go b/cmd/serve/s3/multipart.goindex 32eb0ff50..f915e7fb3 100644--- a/cmd/serve/s3/multipart.go+++ b/cmd/serve/s3/multipart.go@@ -286,6 +286,9 @@ func (b *s3Backend) CreateMultipartUpload(ctx context.Context, bucketName, objecif err != nil {return "", err}+ if node, err := _vfs.Stat(fp); err == nil && node.IsDir() {+ return "", errIsDirectory(objectName)+ }// Reserve a place before doing anything with the remote, so a refused// upload leaves nothing behind.t := b.tenant(ctx)diff --git a/cmd/serve/s3/serve_s3.md b/cmd/serve/s3/serve_s3.mdindex fcc91a1cc..d9bb1139a 100644--- a/cmd/serve/s3/serve_s3.md+++ b/cmd/serve/s3/serve_s3.md@@ -465,11 +465,23 @@ are kept with the metadata in the same way.`serve s3` stores objects as files in the backend, so object keys aremapped to file paths rather than treated as the opaque strings AWS S3allows. Keys must be in canonical path form: keys that contain `..` or-`.` path segments, repeated slashes (`//`), or a leading or trailing-slash are rejected with a `400 Bad Request` (`InvalidArgument`)-instead of being normalised, since normalising them could alias two-distinct keys to the same file or resolve a key outside its bucket.-This matches the behaviour of other S3 servers such as MinIO.+`.` path segments, repeated slashes (`//`), or a leading slash are+rejected with a `400 Bad Request` (`InvalidArgument`) instead of being+normalised, since normalising them could alias two distinct keys to+the same file or resolve a key outside its bucket. This matches the+behaviour of other S3 servers such as MinIO.++A key ending in `/`, such as `dir/`, is a directory marker, standing+for the directory `dir`. Putting an empty directory marker creates the+directory (one with data is rejected with `400 Bad Request`), and+deleting it removes the directory if it is empty. Directory markers+are found by `HeadObject` and `GetObject` for any directory, but+aren't listed as objects.++As keys are file paths, a key can't be both an object and a directory+holding other objects, for example `a/b` and `a/b/c`. Writing an+object which would need this fails with `400 Bad Request`+(`InvalidRequest`).### Supported operationsdiff --git a/cmd/serve/s3/utils.go b/cmd/serve/s3/utils.goindex e6498874d..c00930181 100644--- a/cmd/serve/s3/utils.go+++ b/cmd/serve/s3/utils.go@@ -45,6 +45,22 @@ func getFileHashByte(node any, hashType hash.Type) []byte {return b}+// emptyHash returns the hash of no data.+func emptyHash(hashType hash.Type) []byte {+ if hashType == hash.None {+ return nil+ }+ hasher, err := hash.NewMultiHasherTypes(hash.NewHashSet(hashType))+ if err != nil {+ return nil+ }+ b, err := hex.DecodeString(hasher.Sums()[hashType])+ if err != nil {+ return nil+ }+ return b+}+func getFileHash(node any, hashType hash.Type) string {if hashType == hash.None {return ""@@ -118,6 +134,34 @@ func bucketObjectPath(bucketName, objectName string) (string, error) {return path.Join(bucketName, objectName), nil}+// bucketDirMarkerPath returns the backend path of the directory a+// directory marker key, one ending in "/" such as "dir/", stands for.+// isMarker is false if objectName isn't a directory marker key.+func bucketDirMarkerPath(bucketName, objectName string) (fp string, isMarker bool, err error) {+ dir, isMarker := strings.CutSuffix(objectName, "/")+ if !isMarker {+ return "", false, nil+ }+ fp, err = bucketObjectPath(bucketName, dir)+ if err != nil {+ return "", true, errInvalidObjectName(objectName)+ }+ return fp, true, nil+}++// errIsDirectory is returned for an object key which is the name of a+// directory, which can't be an object too.+func errIsDirectory(key string) error {+ return gofakes3.ErrorMessagef(gofakes3.ErrInvalidRequest, "Object name %q is the name of a directory", key)+}++// errParentIsObject is returned for an object key whose directory, or+// one of its parents, is the name of an object, so can't be a directory+// too.+func errParentIsObject(dir string) error {+ return gofakes3.ErrorMessagef(gofakes3.ErrInvalidRequest, "Directory %q is the name of an object", dir)+}+// bucketDirPath joins the bucket name and a directory path (a listing prefix)// into a backend path. It is like bucketObjectPath except that the empty path// is allowed and addresses the bucket root. Every other non-canonical path,@@ -134,6 +178,8 @@ func bucketDirPath(bucketName, dirName string) (string, error) {// don't exist, returning the top-most directory it made, or "" if path// existed already.//+// It returns errParentIsObject if path or one of its parents is a file.+//// FIXME this could be implemented by VFS.MkdirAll()func mkdirRecursive(path string, VFS *vfs.VFS) (made string, err error) {path = strings.Trim(path, "/")@@ -141,7 +187,13 @@ func mkdirRecursive(path string, VFS *vfs.VFS) (made string, err error) {var dir strings.Builderfor _, d := range dirs {dir.WriteString("/" + d)- if _, err := VFS.Stat(dir.String()); err != nil {+ if node, err := VFS.Stat(dir.String()); err == nil {+ if !node.IsDir() {+ // Report the directory without its bucket+ _, inBucket, _ := strings.Cut(strings.TrimPrefix(dir.String(), "/"), "/")+ return made, errParentIsObject(inBucket)+ }+ } else {err := VFS.Mkdir(dir.String(), 0777)if err != nil {return made, err