Skip to content

"rsync for cloud storage" - Google Drive, S3, Dropbox, Backblaze B2, One Drive, Swift, Hubic, Wasabi, Google Cloud Storage, Azure Blob, Azure Files, Yandex Files

serve s3: list objects with the storage class they were put with

Nick Craig-Woodcb150c014d58
diff --git a/cmd/serve/s3/list.go b/cmd/serve/s3/list.go
index 96f7b2eb4..ab3a36d50 100644
--- a/cmd/serve/s3/list.go
+++ b/cmd/serve/s3/list.go
@@ -30,9 +30,10 @@ var errPageFull = errors.New("listing page full")
// the size of the subtree below the prefix.
type lister struct {
vfs *vfs.VFS
- t *tenant // the user listing
- hashType hash.Type // hash used for ETags
- etag func(fp string, entry vfs.Node) []byte // returns the ETag of the object at fp
+ t *tenant // the user listing
+ hashType hash.Type // hash used for ETags
+ etag func(fp string, entry vfs.Node) []byte // returns the ETag of the object at fp
+ meta func(fp string, entry vfs.Node) (objectMeta, bool) // returns the stored metadata of the object at fp
bucket string
marker string // keys at or before this one have already been returned
hasMarker bool
@@ -59,9 +60,10 @@ func newLister(ctx context.Context, b *s3Backend, _vfs *vfs.VFS, bucket string,
// particular page size, so use the S3 maximum.
max = 1000
}
+ t := b.tenant(ctx)
return &lister{
vfs: _vfs,
- t: b.tenant(ctx),
+ t: t,
hashType: b.s.etagHash(_vfs),
bucket: bucket,
marker: page.Marker,
@@ -71,6 +73,9 @@ func newLister(ctx context.Context, b *s3Backend, _vfs *vfs.VFS, bucket string,
etag: func(fp string, entry vfs.Node) []byte {
return b.objectHash(ctx, _vfs, fp, entry)
},
+ meta: func(fp string, entry vfs.Node) (objectMeta, bool) {
+ return t.getMeta(ctx, fp, entry)
+ },
}
}
@@ -126,12 +131,17 @@ func (l *lister) addObject(key string, entry vfs.Node) error {
if l.keys >= l.max {
return l.truncate()
}
+ fp := path.Join(l.bucket, key)
+ storageClass := gofakes3.StorageStandard
+ if om, ok := l.meta(fp, entry); ok && om.meta["X-Amz-Storage-Class"] != "" {
+ storageClass = gofakes3.StorageClass(om.meta["X-Amz-Storage-Class"])
+ }
l.response.Add(&gofakes3.Content{
Key: key,
LastModified: gofakes3.NewContentTime(entry.ModTime()),
- ETag: quoteETag(l.etag(path.Join(l.bucket, key), entry)),
+ ETag: quoteETag(l.etag(fp, entry)),
Size: entry.Size(),
- StorageClass: gofakes3.StorageStandard,
+ StorageClass: storageClass,
})
l.keys++
l.lastKey = key
diff --git a/cmd/serve/s3/list_test.go b/cmd/serve/s3/list_test.go
index 0f370ba72..3d0b25c9b 100644
--- a/cmd/serve/s3/list_test.go
+++ b/cmd/serve/s3/list_test.go
@@ -400,3 +400,25 @@ func TestListPutDirMarkers(t *testing.T) {
require.Len(t, list.Contents, 1)
assert.Equal(t, "d/f", list.Contents[0].Key)
}
+
+// TestListStorageClass checks the storage class an object was put with
+// is listed.
+func TestListStorageClass(t *testing.T) {
+ b := newListBackend(t, "bucket/existing")
+ ctx := context.Background()
+ _, err := b.PutObject(ctx, "bucket", "ia", map[string]string{"X-Amz-Storage-Class": "STANDARD_IA"}, strings.NewReader("ia"), 2)
+ require.NoError(t, err)
+ _, err = b.PutObject(ctx, "bucket", "plain", nil, strings.NewReader("plain"), 5)
+ require.NoError(t, err)
+
+ _, list := listOnce(t, b, listPrefix("", ""), gofakes3.ListBucketPage{})
+ got := map[string]gofakes3.StorageClass{}
+ for _, c := range list.Contents {
+ got[c.Key] = c.StorageClass
+ }
+ assert.Equal(t, map[string]gofakes3.StorageClass{
+ "existing": gofakes3.StorageStandard,
+ "ia": "STANDARD_IA",
+ "plain": gofakes3.StorageStandard,
+ }, got)
+}
diff --git a/cmd/serve/s3/serve_s3.md b/cmd/serve/s3/serve_s3.md
index 2882c70fa..6711745b0 100644
--- a/cmd/serve/s3/serve_s3.md
+++ b/cmd/serve/s3/serve_s3.md
@@ -527,7 +527,7 @@ ignored.
| Conditional writes | Partial | `If-Match` and `If-None-Match` on `PutObject` and `CompleteMultipartUpload` are checked, but not atomically against changes made to the remote some other way (see [Limitations](#limitations)). |
| Authentication | Partial | AWS Signature Version 4, in headers, presigned URLs and browser form (`POST`) uploads. Signature Version 2 isn't supported. |
| Anonymous access | Partial | All requests are allowed without `--auth-key` or `--auth-proxy`, and none are with them, as buckets and objects can't be made public. |
-| Metadata | Partial | `x-amz-meta-*` and the standard headers such as `Content-Type` are kept in memory only (see [Limitations](#limitations)), except the modification time. |
+| Metadata | Partial | `x-amz-meta-*`, the storage class and the standard headers such as `Content-Type` are kept in memory only (see [Limitations](#limitations)), except the modification time. The storage class is only recorded, so doesn't change how the object is stored. |
| Directory markers | Partial | Keys ending in `/` are stored as directories, so can't hold data. A marker put for a directory holding other keys is only listed while it is remembered in memory (see [Object names](#object-names)). |
| Checksums | Partial | The `x-amz-checksum-*` of a `PutObject` (CRC32, CRC32C, CRC64NVME, SHA1 or SHA256) is checked and returned with the metadata. The checksums of multipart uploads aren't supported. |
| `GetObjectAttributes` | Partial | Everything but the checksums of multipart uploads. |