Skip to content

"rsync for cloud storage" - Google Drive, S3, Dropbox, Backblaze B2, One Drive, Swift, Hubic, Wasabi, Google Cloud Storage, Azure Blob, Azure Files, Yandex Files

serve s3: support directory markers and refuse file/directory conflicts

Nick Craig-Wood63bd68739b86
diff --git a/cmd/serve/s3/backend.go b/cmd/serve/s3/backend.go
index 8da2e5830..cc16e4728 100644
--- a/cmd/serve/s3/backend.go
+++ b/cmd/serve/s3/backend.go
@@ -309,6 +309,12 @@ func (b *s3Backend) HeadObject(ctx context.Context, bucketName, objectName strin
return nil, gofakes3.BucketNotFound(bucketName)
}
+ if fp, isMarker, err := bucketDirMarkerPath(bucketName, objectName); isMarker {
+ if err != nil {
+ return nil, err
+ }
+ return b.dirMarkerObject(_vfs, fp, objectName)
+ }
fp, err := bucketObjectPath(bucketName, objectName)
if err != nil {
return nil, err
@@ -366,6 +372,15 @@ func (b *s3Backend) GetObject(ctx context.Context, bucketName, objectName string
return nil, gofakes3.BucketNotFound(bucketName)
}
+ if fp, isMarker, err := bucketDirMarkerPath(bucketName, objectName); isMarker {
+ if err != nil {
+ return nil, err
+ }
+ if _, err := rangeRequest.Range(0); err != nil {
+ return nil, err
+ }
+ return b.dirMarkerObject(_vfs, fp, objectName)
+ }
fp, err := bucketObjectPath(bucketName, objectName)
if err != nil {
return nil, err
@@ -436,6 +451,44 @@ func (b *s3Backend) GetObject(ctx context.Context, bucketName, objectName string
}, nil
}
+// dirMarkerObject returns the empty object the directory marker key
+// objectName stands for if its directory fp exists.
+func (b *s3Backend) dirMarkerObject(_vfs *vfs.VFS, fp, objectName string) (*gofakes3.Object, error) {
+ node, err := _vfs.Stat(fp)
+ if err != nil || !node.IsDir() {
+ return nil, gofakes3.KeyNotFound(objectName)
+ }
+ return &gofakes3.Object{
+ Name: objectName,
+ Hash: emptyHash(b.s.etagHash(_vfs)),
+ Metadata: map[string]string{
+ "Last-Modified": formatHeaderTime(node.ModTime()),
+ "Content-Type": "application/x-directory",
+ },
+ Size: 0,
+ Contents: noOpReadCloser{},
+ }, nil
+}
+
+// putDirMarker makes the directory fp the directory marker key
+// objectName stands for. A directory marker can't hold data.
+func putDirMarker(_vfs *vfs.VFS, fp, objectName string, input io.Reader) error {
+ // Read to the end of an empty input so any checksum of it is
+ // checked, but no further.
+ n, err := io.Copy(io.Discard, io.LimitReader(input, 1))
+ if err != nil {
+ return err
+ }
+ if n != 0 {
+ return gofakes3.ErrorMessagef(gofakes3.ErrInvalidArgument, "Directory marker %q must be empty", objectName)
+ }
+ if node, err := _vfs.Stat(fp); err == nil && !node.IsDir() {
+ return errParentIsObject(strings.TrimSuffix(objectName, "/"))
+ }
+ _, err = mkdirRecursive(fp, _vfs)
+ return err
+}
+
// TouchObject creates or updates meta on specified object.
func (b *s3Backend) TouchObject(ctx context.Context, fp string, meta map[string]string) (result gofakes3.PutObjectResult, err error) {
_vfs, err := b.s.getVFS(ctx)
@@ -481,10 +534,19 @@ func (b *s3Backend) PutObject(
return result, gofakes3.BucketNotFound(bucketName)
}
+ if fp, isMarker, err := bucketDirMarkerPath(bucketName, objectName); isMarker {
+ if err != nil {
+ return result, err
+ }
+ return result, putDirMarker(_vfs, fp, objectName, input)
+ }
fp, err := bucketObjectPath(bucketName, objectName)
if err != nil {
return result, err
}
+ if node, err := _vfs.Stat(fp); err == nil && node.IsDir() {
+ return result, errIsDirectory(objectName)
+ }
objectDir := path.Dir(fp)
// _, err = db.fs.Stat(objectDir)
// if err == vfs.ENOENT {
@@ -599,6 +661,20 @@ func (b *s3Backend) deleteObject(ctx context.Context, bucketName, objectName str
return gofakes3.BucketNotFound(bucketName)
}
+ if fp, isMarker, err := bucketDirMarkerPath(bucketName, objectName); isMarker {
+ if err != nil {
+ return err
+ }
+ // Deleting a directory marker leaves the objects in its
+ // directory, so the directory is only removed if empty.
+ if isEmptyDir(fp, _vfs) {
+ if err := _vfs.Remove(fp); err != nil {
+ return err
+ }
+ rmdirRecursive(fp, _vfs)
+ }
+ return nil
+ }
fp, err := bucketObjectPath(bucketName, objectName)
if err != nil {
return err
diff --git a/cmd/serve/s3/backend_test.go b/cmd/serve/s3/backend_test.go
index 60ab12640..5dd12fb3e 100644
--- a/cmd/serve/s3/backend_test.go
+++ b/cmd/serve/s3/backend_test.go
@@ -3,6 +3,7 @@ package s3
import (
"context"
"crypto/md5"
+ "encoding/hex"
"io"
"os"
"path/filepath"
@@ -367,3 +368,94 @@ func TestSetObjectTagging(t *testing.T) {
err = b.SetObjectTagging(ctx, "bucket", "missing.txt", "a=1")
assert.True(t, gofakes3.HasErrorCode(err, gofakes3.ErrNoSuchKey), "%v", err)
}
+
+// TestDirMarkers checks directory marker keys, which end in "/", stand
+// for directories.
+func TestDirMarkers(t *testing.T) {
+ b, root := newTestBackend(t)
+ ctx := context.Background()
+ put := func(key, data string) error {
+ _, err := b.PutObject(ctx, "bucket", key, nil, strings.NewReader(data), int64(len(data)))
+ return err
+ }
+ isDir := func(dir string) bool {
+ info, err := os.Stat(filepath.Join(root, "bucket", dir))
+ return err == nil && info.IsDir()
+ }
+
+ // Putting an empty marker makes the directory
+ require.NoError(t, put("dir/sub/", ""))
+ assert.True(t, isDir("dir/sub"))
+ obj, err := b.HeadObject(ctx, "bucket", "dir/sub/")
+ require.NoError(t, err)
+ assert.Equal(t, int64(0), obj.Size)
+ assert.Equal(t, "d41d8cd98f00b204e9800998ecf8427e", hex.EncodeToString(obj.Hash)) // md5("")
+ obj, err = b.GetObject(ctx, "bucket", "dir/", nil)
+ require.NoError(t, err)
+ data, err := io.ReadAll(obj.Contents)
+ require.NoError(t, err)
+ assert.Empty(t, data)
+
+ // A marker with data is refused
+ err = put("data/", "x")
+ assert.True(t, gofakes3.HasErrorCode(err, gofakes3.ErrInvalidArgument), "%v", err)
+ assert.False(t, isDir("data"))
+
+ // A marker for a directory which doesn't exist isn't found, nor is
+ // a directory without the "/"
+ for _, key := range []string{"missing/", "dir"} {
+ _, err = b.HeadObject(ctx, "bucket", key)
+ assert.True(t, gofakes3.HasErrorCode(err, gofakes3.ErrNoSuchKey), "%s: %v", key, err)
+ }
+
+ // Deleting a marker removes its directory only if empty
+ require.NoError(t, put("dir/sub/file.txt", "data"))
+ require.NoError(t, b.deleteObject(ctx, "bucket", "dir/sub/"))
+ assert.True(t, isDir("dir/sub"))
+ require.NoError(t, b.deleteObject(ctx, "bucket", "dir/sub/file.txt"))
+ require.NoError(t, put("dir/sub/", ""))
+ require.NoError(t, b.deleteObject(ctx, "bucket", "dir/sub/"))
+ assert.False(t, isDir("dir/sub"))
+
+ // Markers must be in canonical form apart from the "/"
+ for _, key := range []string{"/", "a//", "a/./"} {
+ err = put(key, "")
+ assert.True(t, gofakes3.HasErrorCode(err, gofakes3.ErrInvalidArgument), "%s: %v", key, err)
+ }
+}
+
+// TestFileDirConflicts checks keys which conflict with directories or
+// files are refused with a client error rather than an internal one,
+// which clients would retry.
+func TestFileDirConflicts(t *testing.T) {
+ b, _ := newTestBackend(t)
+ ctx := context.Background()
+ put := func(key string) error {
+ _, err := b.PutObject(ctx, "bucket", key, nil, strings.NewReader("data"), 4)
+ return err
+ }
+ require.NoError(t, put("dir/file.txt"))
+
+ for _, test := range []struct {
+ key string
+ call func(key string) error
+ }{
+ {"object.txt/below", put},
+ {"object.txt/", func(key string) error {
+ _, err := b.PutObject(ctx, "bucket", key, nil, strings.NewReader(""), 0)
+ return err
+ }},
+ {"dir", put},
+ {"dir", func(key string) error {
+ _, err := b.CreateMultipartUpload(ctx, "bucket", key, nil)
+ return err
+ }},
+ {"object.txt/below", func(key string) error {
+ _, err := b.CreateMultipartUpload(ctx, "bucket", key, nil)
+ return err
+ }},
+ } {
+ err := test.call(test.key)
+ assert.True(t, gofakes3.HasErrorCode(err, gofakes3.ErrInvalidRequest), "%s: %v", test.key, err)
+ }
+}
diff --git a/cmd/serve/s3/multipart.go b/cmd/serve/s3/multipart.go
index 32eb0ff50..f915e7fb3 100644
--- a/cmd/serve/s3/multipart.go
+++ b/cmd/serve/s3/multipart.go
@@ -286,6 +286,9 @@ func (b *s3Backend) CreateMultipartUpload(ctx context.Context, bucketName, objec
if err != nil {
return "", err
}
+ if node, err := _vfs.Stat(fp); err == nil && node.IsDir() {
+ return "", errIsDirectory(objectName)
+ }
// Reserve a place before doing anything with the remote, so a refused
// upload leaves nothing behind.
t := b.tenant(ctx)
diff --git a/cmd/serve/s3/serve_s3.md b/cmd/serve/s3/serve_s3.md
index fcc91a1cc..d9bb1139a 100644
--- a/cmd/serve/s3/serve_s3.md
+++ b/cmd/serve/s3/serve_s3.md
@@ -465,11 +465,23 @@ are kept with the metadata in the same way.
`serve s3` stores objects as files in the backend, so object keys are
mapped to file paths rather than treated as the opaque strings AWS S3
allows. Keys must be in canonical path form: keys that contain `..` or
-`.` path segments, repeated slashes (`//`), or a leading or trailing
-slash are rejected with a `400 Bad Request` (`InvalidArgument`)
-instead of being normalised, since normalising them could alias two
-distinct keys to the same file or resolve a key outside its bucket.
-This matches the behaviour of other S3 servers such as MinIO.
+`.` path segments, repeated slashes (`//`), or a leading slash are
+rejected with a `400 Bad Request` (`InvalidArgument`) instead of being
+normalised, since normalising them could alias two distinct keys to
+the same file or resolve a key outside its bucket. This matches the
+behaviour of other S3 servers such as MinIO.
+
+A key ending in `/`, such as `dir/`, is a directory marker, standing
+for the directory `dir`. Putting an empty directory marker creates the
+directory (one with data is rejected with `400 Bad Request`), and
+deleting it removes the directory if it is empty. Directory markers
+are found by `HeadObject` and `GetObject` for any directory, but
+aren't listed as objects.
+
+As keys are file paths, a key can't be both an object and a directory
+holding other objects, for example `a/b` and `a/b/c`. Writing an
+object which would need this fails with `400 Bad Request`
+(`InvalidRequest`).
### Supported operations
diff --git a/cmd/serve/s3/utils.go b/cmd/serve/s3/utils.go
index e6498874d..c00930181 100644
--- a/cmd/serve/s3/utils.go
+++ b/cmd/serve/s3/utils.go
@@ -45,6 +45,22 @@ func getFileHashByte(node any, hashType hash.Type) []byte {
return b
}
+// emptyHash returns the hash of no data.
+func emptyHash(hashType hash.Type) []byte {
+ if hashType == hash.None {
+ return nil
+ }
+ hasher, err := hash.NewMultiHasherTypes(hash.NewHashSet(hashType))
+ if err != nil {
+ return nil
+ }
+ b, err := hex.DecodeString(hasher.Sums()[hashType])
+ if err != nil {
+ return nil
+ }
+ return b
+}
+
func getFileHash(node any, hashType hash.Type) string {
if hashType == hash.None {
return ""
@@ -118,6 +134,34 @@ func bucketObjectPath(bucketName, objectName string) (string, error) {
return path.Join(bucketName, objectName), nil
}
+// bucketDirMarkerPath returns the backend path of the directory a
+// directory marker key, one ending in "/" such as "dir/", stands for.
+// isMarker is false if objectName isn't a directory marker key.
+func bucketDirMarkerPath(bucketName, objectName string) (fp string, isMarker bool, err error) {
+ dir, isMarker := strings.CutSuffix(objectName, "/")
+ if !isMarker {
+ return "", false, nil
+ }
+ fp, err = bucketObjectPath(bucketName, dir)
+ if err != nil {
+ return "", true, errInvalidObjectName(objectName)
+ }
+ return fp, true, nil
+}
+
+// errIsDirectory is returned for an object key which is the name of a
+// directory, which can't be an object too.
+func errIsDirectory(key string) error {
+ return gofakes3.ErrorMessagef(gofakes3.ErrInvalidRequest, "Object name %q is the name of a directory", key)
+}
+
+// errParentIsObject is returned for an object key whose directory, or
+// one of its parents, is the name of an object, so can't be a directory
+// too.
+func errParentIsObject(dir string) error {
+ return gofakes3.ErrorMessagef(gofakes3.ErrInvalidRequest, "Directory %q is the name of an object", dir)
+}
+
// bucketDirPath joins the bucket name and a directory path (a listing prefix)
// into a backend path. It is like bucketObjectPath except that the empty path
// is allowed and addresses the bucket root. Every other non-canonical path,
@@ -134,6 +178,8 @@ func bucketDirPath(bucketName, dirName string) (string, error) {
// don't exist, returning the top-most directory it made, or "" if path
// existed already.
//
+// It returns errParentIsObject if path or one of its parents is a file.
+//
// FIXME this could be implemented by VFS.MkdirAll()
func mkdirRecursive(path string, VFS *vfs.VFS) (made string, err error) {
path = strings.Trim(path, "/")
@@ -141,7 +187,13 @@ func mkdirRecursive(path string, VFS *vfs.VFS) (made string, err error) {
var dir strings.Builder
for _, d := range dirs {
dir.WriteString("/" + d)
- if _, err := VFS.Stat(dir.String()); err != nil {
+ if node, err := VFS.Stat(dir.String()); err == nil {
+ if !node.IsDir() {
+ // Report the directory without its bucket
+ _, inBucket, _ := strings.Cut(strings.TrimPrefix(dir.String(), "/"), "/")
+ return made, errParentIsObject(inBucket)
+ }
+ } else {
err := VFS.Mkdir(dir.String(), 0777)
if err != nil {
return made, err