SeaweedFS is a distributed storage system for object storage (S3), file systems, Iceberg table buckets, and Lance vector buckets, designed to handle billions of files with O(1) disk access and effortless horizontal scaling.
Go blob-storagecloud-drivedistributed-file-systemdistributed-storageerasure-codingfusehadoop-hdfshdfskuberneteslancelancedbobject-storageposixs3s3-storageseaweedfstiered-file-systemvector-search
package s3api
import (
"context"
"crypto/md5"
"encoding/json"
"fmt"
"net/http"
"net/http/httptest"
"path"
"strings"
"sync"
"testing"
"time"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/service/s3"
"github.com/gorilla/mux"
"github.com/seaweedfs/seaweedfs/weed/cluster"
"github.com/seaweedfs/seaweedfs/weed/pb"
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
"github.com/seaweedfs/seaweedfs/weed/s3api/policy_engine"
"github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
"github.com/seaweedfs/seaweedfs/weed/s3api/s3err"
"github.com/stretchr/testify/require"
"google.golang.org/grpc/codes"
"google.golang.org/grpc/status"
"google.golang.org/protobuf/proto"
)
// Read through the real gRPC lookup and HTTP handlers; replacing an entry does
// not mutate an earlier snapshot, just as a filer metadata update would not.
type anonymousReadFiler struct {
filer_pb.UnimplementedSeaweedFilerServer
mu sync.Mutex
entries map[string]*filer_pb.Entry
lookupErr error
onLookup func()
}
func (f *anonymousReadFiler) LookupDirectoryEntry(_ context.Context, req *filer_pb.LookupDirectoryEntryRequest) (*filer_pb.LookupDirectoryEntryResponse, error) {
f.mu.Lock()
defer f.mu.Unlock()
if f.onLookup != nil {
f.onLookup()
}
if f.lookupErr != nil {
return nil, f.lookupErr
}
entry := f.entries[path.Join(req.Directory, req.Name)]
if entry == nil {
return &filer_pb.LookupDirectoryEntryResponse{}, nil
}
return &filer_pb.LookupDirectoryEntryResponse{Entry: proto.Clone(entry).(*filer_pb.Entry)}, nil
}
func newAnonymousReadTestServer(t *testing.T) (*S3ApiServer, *anonymousReadFiler) {
t.Helper()
f := &anonymousReadFiler{entries: make(map[string]*filer_pb.Entry)}
s3a := newPutTestServer(t, startFakeFiler(t, f))
s3a.iam = &IdentityAccessManagement{isAuthEnabled: true}
s3a.bucketConfigCache = NewBucketConfigCache(time.Minute)
s3a.bucketConfigCache.Set("b", &BucketConfig{Name: "b", Ownership: s3_constants.OwnershipObjectWriter})
s3a.policyEngine = NewBucketPolicyEngine()
s3a.iam.policyEngine = s3a.policyEngine
return s3a, f
}
func anonymousReadGrant(permission string) []byte {
grants := []*s3.Grant{{Grantee: &s3.Grantee{Type: aws.String("Group"), URI: aws.String(s3_constants.GranteeGroupAllUsers)}, Permission: aws.String(permission)}}
data, _ := json.Marshal(grants)
return data
}
func anonymousReadEntry(acl []byte) *filer_pb.Entry {
sum := md5.Sum([]byte("hello world"))
entry := &filer_pb.Entry{Name: "o", Attributes: &filer_pb.FuseAttributes{FileSize: 11, Md5: sum[:], Mtime: 1700000000, Mime: "text/plain"}, Content: []byte("hello world"), Extended: make(map[string][]byte)}
if acl != nil {
entry.Extended[s3_constants.ExtAmzAclKey] = acl
}
return entry
}
func serveAnonymousRead(s3a *S3ApiServer, method, object, query string, headers map[string]string) *httptest.ResponseRecorder {
r := httptest.NewRequest(method, "/b/"+object+"?"+query, nil)
r = mux.SetURLVars(r, map[string]string{"bucket": "b", "object": object})
for name, value := range headers {
r.Header.Set(name, value)
}
handler := s3a.GetObjectHandler
if method == http.MethodHead {
handler = s3a.HeadObjectHandler
}
rr := httptest.NewRecorder()
s3a.AuthWithPublicRead(handler, s3_constants.ACTION_READ)(rr, r)
return rr
}
func TestAnonymousObjectACLRead(t *testing.T) {
for _, tc := range []struct {
name string
acl []byte
publicBucket bool
enforced bool
want int
}{
{"public read", anonymousReadGrant("READ"), false, false, 200},
{"public full control", anonymousReadGrant("FULL_CONTROL"), false, false, 200},
{"write is not read", anonymousReadGrant("WRITE"), false, false, 403},
{"private", []byte(`[]`), false, false, 403},
{"missing ACL private bucket", nil, false, false, 403},
{"legacy public bucket", nil, true, false, 200},
{"explicit private public bucket", []byte(`[]`), true, false, 403},
{"malformed public bucket", []byte(`{broken`), true, false, 403},
{"null grant", []byte(`[null]`), false, false, 403},
{"wrong grantee type", []byte(`[{"Grantee":{"Type":"CanonicalUser","URI":"http://acs.amazonaws.com/groups/global/AllUsers"},"Permission":"READ"}]`), false, false, 403},
{"authenticated users are not anonymous", []byte(`[ {"Grantee":{"Type":"Group","URI":"http://acs.amazonaws.com/groups/global/AuthenticatedUsers"},"Permission":"READ"}]`), false, false, 403},
{"ACL disabled", anonymousReadGrant("READ"), false, true, 403},
} {
for _, method := range []string{http.MethodGet, http.MethodHead} {
t.Run(tc.name+"/"+method, func(t *testing.T) {
s3a, f := newAnonymousReadTestServer(t)
config, _ := s3a.getBucketConfig("b")
config.IsPublicRead = tc.publicBucket
if tc.enforced {
config.Ownership = s3_constants.OwnershipBucketOwnerEnforced
}
f.entries["/buckets/b/o"] = anonymousReadEntry(tc.acl)
rr := serveAnonymousRead(s3a, method, "o", "", nil)
require.Equal(t, tc.want, rr.Code, rr.Body.String())
if tc.want == 200 && method == http.MethodGet {
require.Equal(t, "hello world", rr.Body.String())
}
if tc.want == 403 {
require.NotContains(t, rr.Body.String(), "hello world")
require.Empty(t, rr.Header().Get("ETag"))
}
})
}
}
}
func TestAnonymousObjectACLRangeConditionalAndRevocation(t *testing.T) {
s3a, f := newAnonymousReadTestServer(t)
f.entries["/buckets/b/o"] = anonymousReadEntry(anonymousReadGrant("READ"))
rr := serveAnonymousRead(s3a, http.MethodGet, "o", "", map[string]string{"Range": "bytes=1-4"})
require.Equal(t, 206, rr.Code, rr.Body.String())
require.Equal(t, "ello", rr.Body.String())
require.Equal(t, "bytes 1-4/11", rr.Header().Get("Content-Range"))
etag := rr.Header().Get("ETag")
require.NotEmpty(t, etag)
rr = serveAnonymousRead(s3a, http.MethodGet, "o", "", map[string]string{"If-None-Match": etag})
require.Equal(t, 304, rr.Code)
f.mu.Lock()
f.entries["/buckets/b/o"] = anonymousReadEntry([]byte(`[]`))
f.mu.Unlock()
for _, method := range []string{http.MethodGet, http.MethodHead} {
for _, headers := range []map[string]string{nil, {"Range": "bytes=1-4"}, {"If-None-Match": etag}, {"If-Match": "\"other\""}} {
rr = serveAnonymousRead(s3a, method, "o", "", headers)
require.Equal(t, 403, rr.Code, rr.Body.String())
require.Empty(t, rr.Header().Get("ETag"))
}
}
}
func TestAnonymousObjectACLPolicy(t *testing.T) {
for _, tc := range []struct {
name, statement string
acl []byte
tag string
want int
}{
{"explicit deny", `{"Effect":"Deny","Principal":"*","Action":"s3:GetObject","Resource":"arn:aws:s3:::b/*"}`, anonymousReadGrant("READ"), "", 403},
{"unrelated deny", `{"Effect":"Deny","Principal":"*","Action":"s3:PutObject","Resource":"arn:aws:s3:::b/*"}`, anonymousReadGrant("READ"), "", 200},
{"tag deny matching", `{"Effect":"Deny","Principal":"*","Action":"s3:GetObject","Resource":"arn:aws:s3:::b/*","Condition":{"StringNotEquals":{"s3:ExistingObjectTag/classification":"public"}}}`, anonymousReadGrant("READ"), "private", 403},
{"tag deny nonmatching", `{"Effect":"Deny","Principal":"*","Action":"s3:GetObject","Resource":"arn:aws:s3:::b/*","Condition":{"StringNotEquals":{"s3:ExistingObjectTag/classification":"public"}}}`, anonymousReadGrant("READ"), "public", 200},
{"tag allow private ACL", `{"Effect":"Allow","Principal":"*","Action":"s3:GetObject","Resource":"arn:aws:s3:::b/*","Condition":{"StringEquals":{"s3:ExistingObjectTag/classification":"public"}}}`, []byte(`[]`), "public", 200},
{"tag allow nonmatching", `{"Effect":"Allow","Principal":"*","Action":"s3:GetObject","Resource":"arn:aws:s3:::b/*","Condition":{"StringEquals":{"s3:ExistingObjectTag/classification":"public"}}}`, []byte(`[]`), "private", 403},
} {
t.Run(tc.name, func(t *testing.T) {
s3a, f := newAnonymousReadTestServer(t)
entry := anonymousReadEntry(tc.acl)
entry.Extended[s3_constants.AmzObjectTaggingPrefix+"classification"] = []byte(tc.tag)
f.entries["/buckets/b/o"] = entry
require.NoError(t, s3a.policyEngine.engine.SetBucketPolicy("b", `{"Version":"2012-10-17","Statement":[`+tc.statement+`]}`))
for _, method := range []string{http.MethodGet, http.MethodHead} {
rr := serveAnonymousRead(s3a, method, "o", "", nil)
require.Equal(t, tc.want, rr.Code, rr.Body.String())
}
})
}
}
func TestAnonymousObjectACLVersionAndDirectory(t *testing.T) {
s3a, f := newAnonymousReadTestServer(t)
config, _ := s3a.getBucketConfig("b")
config.Versioning = "Enabled"
public := anonymousReadEntry(anonymousReadGrant("READ"))
public.Extended[s3_constants.ExtVersionIdKey] = []byte("public-version")
public.Name = "v_public-version"
private := anonymousReadEntry([]byte(`[]`))
private.Extended[s3_constants.ExtVersionIdKey] = []byte("private-version")
f.entries["/buckets/b/o.versions/v_public-version"] = public
f.entries["/buckets/b/o.versions/v_private-version"] = private
f.entries["/buckets/b/o"] = anonymousReadEntry([]byte(`[]`))
for _, method := range []string{http.MethodGet, http.MethodHead} {
for _, tc := range []struct {
query string
want int
}{{"versionId=public-version", 200}, {"versionId=private-version", 403}, {"versionId=null", 403}, {"", 403}} {
rr := serveAnonymousRead(s3a, method, "o", tc.query, nil)
require.Equal(t, tc.want, rr.Code, rr.Body.String())
if tc.want == 403 {
require.Empty(t, rr.Header().Get("x-amz-version-id"))
}
}
}
require.NoError(t, s3a.policyEngine.engine.SetBucketPolicy("b", `{"Version":"2012-10-17","Statement":[{"Effect":"Deny","Principal":"*","Action":"s3:GetObjectVersion","Resource":"arn:aws:s3:::b/o"}]}`))
rr := serveAnonymousRead(s3a, http.MethodGet, "o", "versionId=public-version", nil)
require.Equal(t, 403, rr.Code)
// The backing file is not an alternative key for bypassing GetObjectVersion policy.
rr = serveAnonymousRead(s3a, http.MethodGet, "o.versions/v_public-version", "", nil)
require.Equal(t, 403, rr.Code)
// A versioned directory key must not return its public null-version marker.
dir := anonymousReadEntry(anonymousReadGrant("READ"))
dir.IsDirectory = true
f.entries["/buckets/b/folder"] = dir
f.entries["/buckets/b/folder/.versions/v_private-version"] = private
rr = serveAnonymousRead(s3a, http.MethodGet, "folder/", "versionId=private-version", nil)
require.Equal(t, 403, rr.Code)
}
func TestSignedReadOfVersionFileKey(t *testing.T) {
s3a, f := newAnonymousReadTestServer(t)
config, _ := s3a.getBucketConfig("b")
config.Versioning = "Enabled"
version := anonymousReadEntry(nil)
version.Name = "v_v1"
version.Extended[s3_constants.ExtVersionIdKey] = []byte("v1")
f.entries["/buckets/b/o.versions/v_v1"] = version
f.entries["/buckets/b/u.versions/v_literal"] = anonymousReadEntry(nil)
identity := &Identity{Name: "reader", Actions: []Action{s3_constants.ACTION_READ}}
serve := func(method, object, query string, headers map[string]string) int {
r := httptest.NewRequest(method, "/b/"+object+"?"+query, nil)
r = mux.SetURLVars(r, map[string]string{"bucket": "b", "object": object})
r = r.WithContext(s3_constants.SetIdentityInContext(r.Context(), identity))
for name, value := range headers {
r.Header.Set(name, value)
}
handler := s3a.GetObjectHandler
if method == http.MethodHead {
handler = s3a.HeadObjectHandler
}
rr := httptest.NewRecorder()
handler(rr, r)
return rr.Code
}
for _, method := range []string{http.MethodGet, http.MethodHead} {
require.Equal(t, 200, serve(method, "o", "versionId=v1", nil))
require.Equal(t, 403, serve(method, "o.versions/v_v1", "", nil))
require.NotEqual(t, 200, serve(method, "o.versions/v_v1/", "", nil))
require.Equal(t, 403, serve(method, "o.versions/v_v1", "", map[string]string{"If-None-Match": "*"}))
require.Equal(t, 200, serve(method, "u.versions/v_literal", "", nil))
}
r := httptest.NewRequest(http.MethodPut, "/b/dst", nil)
r = r.WithContext(s3_constants.SetIdentityInContext(r.Context(), identity))
require.Equal(t, s3err.ErrAccessDenied, s3a.recheckCopySourcePolicy(r, "b", "o.versions/v_v1", "", version.Extended, "CopyObjectHandler"))
require.Equal(t, s3err.ErrNone, s3a.recheckCopySourcePolicy(r, "b", "o", "v1", version.Extended, "CopyObjectHandler"))
for _, action := range []string{s3_constants.ACTION_GET_OBJECT_RETENTION, s3_constants.ACTION_PUT_OBJECT_LEGAL_HOLD, string(s3_constants.ACTION_WRITE)} {
require.Equal(t, s3err.ErrAccessDenied, s3a.recheckTagConditionedPolicyWithLookup(r, "b", "o.versions/v_v1", "", action, "t"))
require.Equal(t, s3err.ErrNone, s3a.recheckTagConditionedPolicyWithLookup(r, "b", "o", "v1", action, "t"))
require.Equal(t, s3err.ErrNone, s3a.recheckTagConditionedPolicyWithLookup(r, "b", "u.versions/v_literal", "", action, "t"))
}
require.Equal(t, s3err.ErrAccessDenied, s3a.deleteDirectoryMarker(r, "b", "o.versions/v_v1/"))
require.Contains(t, f.entries, "/buckets/b/o.versions/v_v1")
}
func TestWriteToVersionFileKey(t *testing.T) {
for _, state := range []string{"Suspended", "Enabled"} {
s3a, f := newAnonymousReadTestServer(t)
config, _ := s3a.getBucketConfig("b")
config.Versioning = state
version := anonymousReadEntry(nil)
version.Name = "v_v1"
version.Extended[s3_constants.ExtVersionIdKey] = []byte("v1")
f.entries["/buckets/b/o.versions/v_v1"] = version
f.entries["/buckets/b/u.versions/v_literal"] = anonymousReadEntry(nil)
nullVersion := anonymousReadEntry(nil)
nullVersion.Name = "v_null"
nullVersion.Extended[s3_constants.ExtVersionIdKey] = []byte("null")
f.entries["/buckets/b/o.versions/v_null"] = nullVersion
f.entries["/buckets/b/o.versions/v_null.versions"] = &filer_pb.Entry{Name: "v_null.versions", IsDirectory: true, Extended: map[string][]byte{
s3_constants.ExtLatestVersionIdKey: []byte("v9"),
s3_constants.ExtLatestVersionFileNameKey: []byte("v_v9"),
}}
nested := anonymousReadEntry(nil)
nested.Name = "v_v9"
nested.Extended[s3_constants.ExtVersionIdKey] = []byte("v9")
f.entries["/buckets/b/o.versions/v_null.versions/v_v9"] = nested
require.Equal(t, s3err.ErrAccessDenied, s3a.checkVersionFileWrite("b", "o.versions/v_v1", "t"), state)
require.Equal(t, s3err.ErrAccessDenied, s3a.checkVersionFileWrite("b", "o.versions/v_null", "t"), state)
require.Equal(t, s3err.ErrAccessDenied, s3a.checkVersionFileWrite("b", "/o.versions/v_v1/", "t"), state)
require.Equal(t, s3err.ErrNone, s3a.checkVersionFileWrite("b", "o.versions/v_v2", "t"), state)
require.Equal(t, s3err.ErrNone, s3a.checkVersionFileWrite("b", "u.versions/v_literal", "t"), state)
require.Equal(t, s3err.ErrNone, s3a.checkVersionFileWrite("b", "o", "t"), state)
}
}
func TestWriteToVersionFileKeyReadsLogicalOwner(t *testing.T) {
gateway := &anonymousReadFiler{entries: make(map[string]*filer_pb.Entry)}
owner := &anonymousReadFiler{entries: make(map[string]*filer_pb.Entry)}
gatewayAddr, ownerAddr := startFakeFiler(t, gateway), startFakeFiler(t, owner)
s3a := newPutTestServer(t, gatewayAddr)
s3a.bucketConfigCache = NewBucketConfigCache(time.Minute)
s3a.bucketConfigCache.Set("b", &BucketConfig{Name: "b", Versioning: s3_constants.VersioningSuspended})
s3a.objectWriteLockClient = cluster.NewLockClient(s3a.option.GrpcDialOption, gatewayAddr)
s3a.objectWriteLockClient.SetRing([]pb.ServerAddress{gatewayAddr, ownerAddr}, 1)
object := ""
for i := 0; i < 1000 && object == ""; i++ {
candidate := fmt.Sprintf("o%d", i)
if s3a.routableWriteOwner("b", candidate) == ownerAddr && s3a.routableWriteOwner("b", candidate+".versions/v_v1") == gatewayAddr {
object = candidate
}
}
require.NotEmpty(t, object)
version := anonymousReadEntry(nil)
version.Name = "v_v1"
version.Extended[s3_constants.ExtVersionIdKey] = []byte("v1")
owner.entries["/buckets/b/"+object+".versions/v_v1"] = version
require.Equal(t, s3err.ErrAccessDenied, s3a.checkVersionFileWrite("b", object+".versions/v_v1", "t"))
s3a.markOwnerUnreachable(ownerAddr)
require.Equal(t, s3err.ErrAccessDenied, s3a.checkVersionFileWrite("b", object+".versions/v_v1", "t"))
gateway.mu.Lock()
gateway.onLookup = func() { s3a.unreachableOwners.Delete(ownerAddr) }
gateway.mu.Unlock()
s3a.markOwnerUnreachable(ownerAddr)
require.Equal(t, s3err.ErrAccessDenied, s3a.checkVersionFileWrite("b", object+".versions/v_v1", "t"))
gateway.mu.Lock()
gateway.onLookup = nil
gateway.mu.Unlock()
s3a.markOwnerUnreachable(ownerAddr)
require.Equal(t, s3err.ErrNone, s3a.checkVersionFileWrite("b", object+".versions/v_v2", "t"))
owner.mu.Lock()
owner.lookupErr = status.Error(codes.Unavailable, "owner down")
owner.mu.Unlock()
s3a.markOwnerUnreachable(ownerAddr)
require.Equal(t, s3err.ErrInternalError, s3a.checkVersionFileWrite("b", object+".versions/v_v2", "t"))
}
func TestAnonymousObjectACLDoesNotReplaceAuthentication(t *testing.T) {
s3a, f := newAnonymousReadTestServer(t)
f.entries["/buckets/b/o"] = anonymousReadEntry(anonymousReadGrant("READ"))
for _, header := range []string{"", "invalid", "AWS4-HMAC-SHA256 broken", "AWS invalid:signature"} {
rr := serveAnonymousRead(s3a, http.MethodGet, "o", "", map[string]string{"Authorization": header})
require.NotEqual(t, 200, rr.Code, rr.Body.String())
}
for _, query := range []string{"X-Amz-Credential=unknown", "AWSAccessKeyId=unknown"} {
rr := serveAnonymousRead(s3a, http.MethodGet, "o", query, nil)
require.NotEqual(t, 200, rr.Code, rr.Body.String())
}
// The metadata ACL must never provide a grant for another API operation.
for _, query := range []string{"acl", "tagging", "attributes", "uploadId=upload", "uploads", "retention", "legal-hold"} {
called := false
r := mux.SetURLVars(httptest.NewRequest(http.MethodGet, "/b/o?"+query, nil), map[string]string{"bucket": "b", "object": "o"})
rr := httptest.NewRecorder()
s3a.AuthWithPublicRead(func(http.ResponseWriter, *http.Request) { called = true }, s3_constants.ACTION_READ)(rr, r)
require.False(t, called, query)
}
// Keep the existing no-auth development mode, including objects without ACLs.
s3a.iam.isAuthEnabled = false
f.entries["/buckets/b/o"] = anonymousReadEntry(nil)
rr := serveAnonymousRead(s3a, http.MethodGet, "o", "", nil)
require.Equal(t, 200, rr.Code, rr.Body.String())
}
func TestAnonymousObjectACLMetadataFailure(t *testing.T) {
s3a, f := newAnonymousReadTestServer(t)
f.lookupErr = status.Error(codes.Internal, "metadata unavailable")
rr := serveAnonymousRead(s3a, http.MethodGet, "o", "", nil)
require.NotEqual(t, 200, rr.Code)
require.NotContains(t, rr.Body.String(), "hello world")
// Injected internal headers cannot select an authenticated principal.
f.lookupErr = nil
f.entries["/buckets/b/o"] = anonymousReadEntry([]byte(`[]`))
require.NoError(t, s3a.policyEngine.engine.SetBucketPolicy("b", `{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Principal":{"AWS":"arn:aws:iam::admin:user/admin"},"Action":"s3:GetObject","Resource":"arn:aws:s3:::b/*"}]}`))
rr = serveAnonymousRead(s3a, http.MethodGet, "o", "", map[string]string{s3_constants.SeaweedFSPrincipalHeader: "arn:aws:iam::admin:user/admin", s3_constants.AmzAccountId: "admin"})
require.Equal(t, 403, rr.Code)
require.False(t, strings.Contains(rr.Body.String(), "hello world"))
}
func TestAnonymousObjectACLLegacyOwnership(t *testing.T) {
s3a, f := newAnonymousReadTestServer(t)
config, _ := s3a.getBucketConfig("b")
f.entries["/buckets/b/o"] = anonymousReadEntry(anonymousReadGrant("READ"))
for _, tc := range []struct {
ownership string
want int
}{
{"", 200}, {s3_constants.OwnershipBucketOwnerPreferred, 200},
{s3_constants.OwnershipBucketOwnerEnforced, 403}, {"invalid", 403},
} {
config.Ownership = tc.ownership
for _, method := range []string{http.MethodGet, http.MethodHead} {
rr := serveAnonymousRead(s3a, method, "o", "", nil)
require.Equal(t, tc.want, rr.Code, tc.ownership)
}
}
// A literal user key containing .versions is allowed without internal version metadata.
config.Ownership = ""
f.entries["/buckets/b/o.versions/v_literal"] = anonymousReadEntry(anonymousReadGrant("READ"))
rr := serveAnonymousRead(s3a, http.MethodGet, "o.versions/v_literal", "", nil)
require.Equal(t, 200, rr.Code, rr.Body.String())
}
func TestAnonymousObjectACLIgnorePublicAcls(t *testing.T) {
publicBucket := &filer_pb.Entry{Name: "b", IsDirectory: true, Extended: map[string][]byte{
s3_constants.ExtAmzAclKey: anonymousReadGrant("READ"),
}}
ignoring := proto.Clone(publicBucket).(*filer_pb.Entry)
ignoring.Extended[s3_constants.ExtPublicAccessBlockKey] = []byte(`<PublicAccessBlockConfiguration><IgnorePublicAcls>true</IgnorePublicAcls></PublicAccessBlockConfiguration>`)
for _, tc := range []struct {
name string
bucket *filer_pb.Entry
acl []byte
want int
}{
{"public object", publicBucket, anonymousReadGrant("READ"), 200},
{"public object ignored", ignoring, anonymousReadGrant("READ"), 403},
{"public bucket", publicBucket, nil, 200},
{"public bucket ignored", ignoring, nil, 403},
} {
for _, method := range []string{http.MethodGet, http.MethodHead} {
t.Run(tc.name+"/"+method, func(t *testing.T) {
s3a, f := newAnonymousReadTestServer(t)
s3a.bucketConfigCache.Set("b", s3a.newBucketConfigFromEntry("b", tc.bucket))
f.entries["/buckets/b/o"] = anonymousReadEntry(tc.acl)
rr := serveAnonymousRead(s3a, method, "o", "", nil)
require.Equal(t, tc.want, rr.Code, rr.Body.String())
if tc.want == 403 {
require.NotContains(t, rr.Body.String(), "hello world")
}
})
}
}
}
func TestAnonymousObjectACLConfiguredIdentity(t *testing.T) {
for _, tc := range []struct {
name string
public bool
actions []Action
policy string
disabled bool
want int
}{
{"legacy anonymous permission", false, []Action{s3_constants.ACTION_READ}, "", false, 200},
{"identity policy allow", false, nil, `{"Effect":"Allow","Action":"s3:GetObject","Resource":"arn:aws:s3:::b/*"}`, false, 200},
// The fixture bucket "b" is shorter than S3's three-character minimum,
// so a denial for a resolved identity answers InvalidBucketName.
{"identity deny beats object ACL", true, nil, `{"Effect":"Deny","Action":"s3:GetObject","Resource":"arn:aws:s3:::b/*"}`, false, 400},
{"disabled identity cannot grant", false, []Action{s3_constants.ACTION_READ}, "", true, 403},
{"public ACL does not require identity", true, nil, "", true, 200},
} {
t.Run(tc.name, func(t *testing.T) {
s3a, f := newAnonymousReadTestServer(t)
acl := []byte(`[]`)
if tc.public {
acl = anonymousReadGrant("READ")
}
f.entries["/buckets/b/o"] = anonymousReadEntry(acl)
identity := &Identity{Name: s3_constants.AccountAnonymousId, Actions: tc.actions, Disabled: tc.disabled}
if tc.policy != "" {
identity.PolicyNames = []string{"anonymous-policy"}
s3a.iam.iamPolicyEngine = policy_engine.NewPolicyEngine()
require.NoError(t, s3a.iam.iamPolicyEngine.SetBucketPolicy("anonymous-policy", `{"Version":"2012-10-17","Statement":[`+tc.policy+`]}`))
}
s3a.iam.identityAnonymous = identity
rr := serveAnonymousRead(s3a, http.MethodGet, "o", "", nil)
require.Equal(t, tc.want, rr.Code, rr.Body.String())
})
}
}
func TestAnonymousObjectACLRemoteCacheReauthorization(t *testing.T) {
for _, revokeACL := range []bool{true, false} {
t.Run(map[bool]string{true: "ACL revoked", false: "policy tag changed"}[revokeACL], func(t *testing.T) {
cached := anonymousReadEntry(anonymousReadGrant("READ"))
cached.Content = nil
cached.Chunks = []*filer_pb.FileChunk{{FileId: "1,abc", Size: 11}}
if revokeACL {
cached.Extended[s3_constants.ExtAmzAclKey] = []byte(`[]`)
} else {
cached.Extended[s3_constants.AmzObjectTaggingPrefix+"classification"] = []byte("private")
}
address := startFakeCacheFiler(t, &fakeCacheFiler{cache: func(context.Context, *filer_pb.CacheRemoteObjectToLocalClusterRequest) (*filer_pb.CacheRemoteObjectToLocalClusterResponse, error) {
return &filer_pb.CacheRemoteObjectToLocalClusterResponse{Entry: cached}, nil
}})
s3a := newPutTestServer(t, address)
s3a.iam = &IdentityAccessManagement{isAuthEnabled: true}
s3a.bucketConfigCache = NewBucketConfigCache(time.Minute)
s3a.bucketConfigCache.Set("b", &BucketConfig{Name: "b", Ownership: s3_constants.OwnershipObjectWriter})
s3a.policyEngine = NewBucketPolicyEngine()
if !revokeACL {
require.NoError(t, s3a.policyEngine.engine.SetBucketPolicy("b", `{"Version":"2012-10-17","Statement":[{"Effect":"Deny","Principal":"*","Action":"s3:GetObject","Resource":"arn:aws:s3:::b/*","Condition":{"StringNotEquals":{"s3:ExistingObjectTag/classification":"public"}}}]}`))
}
r := mux.SetURLVars(httptest.NewRequest(http.MethodGet, "/b/o", nil), map[string]string{"bucket": "b", "object": "o"})
r, deferred := s3a.deferAnonymousObjectRead(r, s3_constants.ACTION_READ, "b", "o")
require.True(t, deferred)
entry := anonymousReadEntry(anonymousReadGrant("READ"))
entry.Content = nil
entry.RemoteEntry = &filer_pb.RemoteEntry{RemoteSize: 11}
entry.Extended[s3_constants.AmzObjectTaggingPrefix+"classification"] = []byte("public")
require.Equal(t, s3err.ErrNone, s3a.authorizeAnonymousObjectRead(r, "b", "o", entry.Extended))
w := httptest.NewRecorder()
err := s3a.streamFromVolumeServers(w, r, entry, "", "b", "o", "")
require.Error(t, err)
require.Equal(t, http.StatusForbidden, w.Code, w.Body.String())
require.NotContains(t, w.Body.String(), "hello world")
})
}
}