"rsync for cloud storage" - Google Drive, S3, Dropbox, Backblaze B2, One Drive, Swift, Hubic, Wasabi, Google Cloud Storage, Azure Blob, Azure Files, Yandex Files
Go azure-blobazure-blob-storageazure-filesbackblaze-b2cloud-storagedropboxencryptionftpfuse-filesystemgogolanggoogle-cloud-storagegoogle-driveonedriveopenstack-swiftrclones3sftpsyncwebdav
serve s3: list objects with the storage class they were put with
Nick Craig-Woodcb150c014d58
diff --git a/cmd/serve/s3/list.go b/cmd/serve/s3/list.goindex 96f7b2eb4..ab3a36d50 100644--- a/cmd/serve/s3/list.go+++ b/cmd/serve/s3/list.go@@ -30,9 +30,10 @@ var errPageFull = errors.New("listing page full")// the size of the subtree below the prefix.type lister struct {vfs *vfs.VFS- t *tenant // the user listing- hashType hash.Type // hash used for ETags- etag func(fp string, entry vfs.Node) []byte // returns the ETag of the object at fp+ t *tenant // the user listing+ hashType hash.Type // hash used for ETags+ etag func(fp string, entry vfs.Node) []byte // returns the ETag of the object at fp+ meta func(fp string, entry vfs.Node) (objectMeta, bool) // returns the stored metadata of the object at fpbucket stringmarker string // keys at or before this one have already been returnedhasMarker bool@@ -59,9 +60,10 @@ func newLister(ctx context.Context, b *s3Backend, _vfs *vfs.VFS, bucket string,// particular page size, so use the S3 maximum.max = 1000}+ t := b.tenant(ctx)return &lister{vfs: _vfs,- t: b.tenant(ctx),+ t: t,hashType: b.s.etagHash(_vfs),bucket: bucket,marker: page.Marker,@@ -71,6 +73,9 @@ func newLister(ctx context.Context, b *s3Backend, _vfs *vfs.VFS, bucket string,etag: func(fp string, entry vfs.Node) []byte {return b.objectHash(ctx, _vfs, fp, entry)},+ meta: func(fp string, entry vfs.Node) (objectMeta, bool) {+ return t.getMeta(ctx, fp, entry)+ },}}@@ -126,12 +131,17 @@ func (l *lister) addObject(key string, entry vfs.Node) error {if l.keys >= l.max {return l.truncate()}+ fp := path.Join(l.bucket, key)+ storageClass := gofakes3.StorageStandard+ if om, ok := l.meta(fp, entry); ok && om.meta["X-Amz-Storage-Class"] != "" {+ storageClass = gofakes3.StorageClass(om.meta["X-Amz-Storage-Class"])+ }l.response.Add(&gofakes3.Content{Key: key,LastModified: gofakes3.NewContentTime(entry.ModTime()),- ETag: quoteETag(l.etag(path.Join(l.bucket, key), entry)),+ ETag: quoteETag(l.etag(fp, entry)),Size: entry.Size(),- StorageClass: gofakes3.StorageStandard,+ StorageClass: storageClass,})l.keys++l.lastKey = keydiff --git a/cmd/serve/s3/list_test.go b/cmd/serve/s3/list_test.goindex 0f370ba72..3d0b25c9b 100644--- a/cmd/serve/s3/list_test.go+++ b/cmd/serve/s3/list_test.go@@ -400,3 +400,25 @@ func TestListPutDirMarkers(t *testing.T) {require.Len(t, list.Contents, 1)assert.Equal(t, "d/f", list.Contents[0].Key)}++// TestListStorageClass checks the storage class an object was put with+// is listed.+func TestListStorageClass(t *testing.T) {+ b := newListBackend(t, "bucket/existing")+ ctx := context.Background()+ _, err := b.PutObject(ctx, "bucket", "ia", map[string]string{"X-Amz-Storage-Class": "STANDARD_IA"}, strings.NewReader("ia"), 2)+ require.NoError(t, err)+ _, err = b.PutObject(ctx, "bucket", "plain", nil, strings.NewReader("plain"), 5)+ require.NoError(t, err)++ _, list := listOnce(t, b, listPrefix("", ""), gofakes3.ListBucketPage{})+ got := map[string]gofakes3.StorageClass{}+ for _, c := range list.Contents {+ got[c.Key] = c.StorageClass+ }+ assert.Equal(t, map[string]gofakes3.StorageClass{+ "existing": gofakes3.StorageStandard,+ "ia": "STANDARD_IA",+ "plain": gofakes3.StorageStandard,+ }, got)+}diff --git a/cmd/serve/s3/serve_s3.md b/cmd/serve/s3/serve_s3.mdindex 2882c70fa..6711745b0 100644--- a/cmd/serve/s3/serve_s3.md+++ b/cmd/serve/s3/serve_s3.md@@ -527,7 +527,7 @@ ignored.| Conditional writes | Partial | `If-Match` and `If-None-Match` on `PutObject` and `CompleteMultipartUpload` are checked, but not atomically against changes made to the remote some other way (see [Limitations](#limitations)). || Authentication | Partial | AWS Signature Version 4, in headers, presigned URLs and browser form (`POST`) uploads. Signature Version 2 isn't supported. || Anonymous access | Partial | All requests are allowed without `--auth-key` or `--auth-proxy`, and none are with them, as buckets and objects can't be made public. |-| Metadata | Partial | `x-amz-meta-*` and the standard headers such as `Content-Type` are kept in memory only (see [Limitations](#limitations)), except the modification time. |+| Metadata | Partial | `x-amz-meta-*`, the storage class and the standard headers such as `Content-Type` are kept in memory only (see [Limitations](#limitations)), except the modification time. The storage class is only recorded, so doesn't change how the object is stored. || Directory markers | Partial | Keys ending in `/` are stored as directories, so can't hold data. A marker put for a directory holding other keys is only listed while it is remembered in memory (see [Object names](#object-names)). || Checksums | Partial | The `x-amz-checksum-*` of a `PutObject` (CRC32, CRC32C, CRC64NVME, SHA1 or SHA256) is checked and returned with the metadata. The checksums of multipart uploads aren't supported. || `GetObjectAttributes` | Partial | Everything but the checksums of multipart uploads. |