Skip to content

"rsync for cloud storage" - Google Drive, S3, Dropbox, Backblaze B2, One Drive, Swift, Hubic, Wasabi, Google Cloud Storage, Azure Blob, Azure Files, Yandex Files

27.1 KB
Raw Blame
// Serve s3 tests set up a server and run the integration tests
// for the s3 remote against it.

package s3

import (
	"bytes"
	"context"
	"fmt"
	"io"
	"maps"
	"net/http"
	"net/http/httptest"
	"net/url"
	"os"
	"path"
	"path/filepath"
	"slices"
	"strings"
	"testing"
	"time"

	"github.com/aws/aws-sdk-go-v2/aws"
	v4 "github.com/aws/aws-sdk-go-v2/aws/signer/v4"
	"github.com/minio/minio-go/v7"
	"github.com/minio/minio-go/v7/pkg/credentials"
	"github.com/rclone/gofakes3/signature"
	_ "github.com/rclone/rclone/backend/local"
	_ "github.com/rclone/rclone/backend/s3" // for TestS3Minio backing remote
	"github.com/rclone/rclone/cmd/serve/proxy"
	"github.com/rclone/rclone/cmd/serve/servetest"
	"github.com/rclone/rclone/fs"
	"github.com/rclone/rclone/fs/config/configmap"
	"github.com/rclone/rclone/fs/hash"
	"github.com/rclone/rclone/fs/object"
	"github.com/rclone/rclone/fs/rc"
	"github.com/rclone/rclone/fstest"
	"github.com/rclone/rclone/fstest/fstests"
	"github.com/rclone/rclone/fstest/testy"
	"github.com/rclone/rclone/lib/random"
	"github.com/rclone/rclone/vfs"
	"github.com/rclone/rclone/vfs/vfscommon"
	"github.com/stretchr/testify/assert"
	"github.com/stretchr/testify/require"
)

const (
	endpoint = "localhost:0"
)

// Configure and serve the server
//
// If f is nil the server is expected to be using an auth proxy, in
// which case the credentials are handed to the test proxy via the
// environment rather than --auth-key.
func serveS3(t *testing.T, f fs.Fs) (testURL string, keyid string, keysec string, w *Server) {
	return serveS3Opt(t, f, Opt)
}

// serveS3Opt is serveS3 with the options opt.
func serveS3Opt(t *testing.T, f fs.Fs, opt Options) (testURL string, keyid string, keysec string, w *Server) {
	keyid = random.String(16)
	keysec = random.String(16)
	if f == nil {
		t.Setenv("RCLONE_TEST_PROXY_AUTH_KEY", fmt.Sprintf("%s,%s", keyid, keysec))
	} else {
		opt.AuthKey = []string{fmt.Sprintf("%s,%s", keyid, keysec)}
	}
	opt.HTTP.ListenAddr = []string{endpoint}
	w, _ = newServer(context.Background(), f, &opt, &vfscommon.Opt, &proxy.Opt)
	go func() {
		require.NoError(t, w.Serve())
	}()
	testURL = w.server.URLs()[0]

	return
}

// startS3 builds the start callback that brings up a serve s3 server
// wrapping f and returns the client config for connecting to it.
func startS3(t *testing.T) servetest.StartFn {
	return startS3Opt(t, Opt, nil)
}

// startS3Opt is startS3 with the server options opt and the extra
// backend config extra.
func startS3Opt(t *testing.T, opt Options, extra configmap.Simple) servetest.StartFn {
	return func(f fs.Fs) (configmap.Simple, func()) {
		testURL, keyid, keysec, _ := serveS3Opt(t, f, opt)
		// Config for the backend we'll use to connect to the server
		config := configmap.Simple{
			"type":              "s3",
			"provider":          "Rclone",
			"endpoint":          testURL,
			"access_key_id":     keyid,
			"secret_access_key": keysec,
		}
		maps.Copy(config, extra)
		return config, func() {}
	}
}

// TestS3 runs the s3 server backed by a local directory then runs the
// s3 backend integration tests against it. The local backend only
// supports OpenWriterAt, so this exercises that streaming path in
// serve s3.
func TestS3(t *testing.T) {
	servetest.Run(t, "s3", startS3(t))
}

// TestS3DirectoryMarkers runs the s3 backend integration tests with
// directory markers against the s3 server with --directory-markers.
func TestS3DirectoryMarkers(t *testing.T) {
	opt := Opt
	opt.DirectoryMarkers = true
	servetest.Run(t, "s3", startS3Opt(t, opt, configmap.Simple{"directory_markers": "true"}))
}

// TestS3Minio runs the s3 server backed by a minio docker container
// (via fstest/testserver/init.d/TestS3Minio) then runs the s3 backend
// integration tests against it. Minio supports OpenChunkWriter, so
// this exercises that streaming path in serve s3 - the path the local
// backing in TestS3 cannot cover.
func TestS3Minio(t *testing.T) {
	testy.SkipUnlessDocker(t)
	servetest.RunWithBackend(t, "s3", startS3(t), "TestS3Minio:")
}

// tests using the minio client
func TestEncodingWithMinioClient(t *testing.T) {
	cases := []struct {
		description string
		bucket      string
		path        string
		filename    string
		expected    string
	}{
		{
			description: "weird file in bucket root",
			bucket:      "mybucket",
			path:        "",
			filename:    " file with w€r^d ch@r \\#~+§4%&'. txt ",
		},
		{
			description: "weird file inside a weird folder",
			bucket:      "mybucket",
			path:        "ä#/नेपाल&/?/",
			filename:    " file with w€r^d ch@r \\#~+§4%&'. txt ",
		},
	}

	for _, tt := range cases {
		t.Run(tt.description, func(t *testing.T) {
			fstest.Initialise()
			f, _, clean, err := fstest.RandomRemote()
			assert.NoError(t, err)
			defer clean()
			err = f.Mkdir(context.Background(), path.Join(tt.bucket, tt.path))
			assert.NoError(t, err)

			buf := bytes.NewBufferString("contents")
			uploadHash := hash.NewMultiHasher()
			in := io.TeeReader(buf, uploadHash)

			obji := object.NewStaticObjectInfo(
				path.Join(tt.bucket, tt.path, tt.filename),
				time.Now(),
				int64(buf.Len()),
				true,
				nil,
				nil,
			)
			_, err = f.Put(context.Background(), in, obji)
			assert.NoError(t, err)

			endpoint, keyid, keysec, _ := serveS3(t, f)
			testURL, _ := url.Parse(endpoint)
			minioClient, err := minio.New(testURL.Host, &minio.Options{
				Creds:  credentials.NewStaticV4(keyid, keysec, ""),
				Secure: false,
			})
			assert.NoError(t, err)

			buckets, err := minioClient.ListBuckets(context.Background())
			assert.NoError(t, err)
			assert.Equal(t, buckets[0].Name, tt.bucket)
			objects := minioClient.ListObjects(context.Background(), tt.bucket, minio.ListObjectsOptions{
				Recursive: true,
			})
			for object := range objects {
				assert.Equal(t, path.Join(tt.path, tt.filename), object.Key)
			}
		})
	}
}

type FileStuct struct {
	path     string
	filename string
}

type TestCase struct {
	description string
	bucket      string
	files       []FileStuct
	keyID       string
	keySec      string
	shouldFail  bool
}

func testListBuckets(t *testing.T, cases []TestCase, useProxy bool) {
	fstest.Initialise()

	var f fs.Fs
	if useProxy {
		// the backend config will be made by the proxy
		prog, err := filepath.Abs("../servetest/proxy_code.go")
		require.NoError(t, err)
		files, err := filepath.Abs("testdata")
		require.NoError(t, err)
		cmd := "go run " + prog + " " + files

		// FIXME: this is untidy setting a global variable!
		proxy.Opt.AuthProxy = cmd
		defer func() {
			proxy.Opt.AuthProxy = ""
		}()

		f = nil
	} else {
		// create a test Fs
		var err error
		f, err = fs.NewFs(context.Background(), "testdata")
		require.NoError(t, err)
	}

	for _, tt := range cases {
		t.Run(tt.description, func(t *testing.T) {
			endpoint, keyid, keysec, s := serveS3(t, f)
			defer func() {
				assert.NoError(t, s.server.Shutdown())
			}()

			if tt.keyID != "" {
				keyid = tt.keyID
			}
			if tt.keySec != "" {
				keysec = tt.keySec
			}

			testURL, _ := url.Parse(endpoint)
			minioClient, err := minio.New(testURL.Host, &minio.Options{
				Creds:  credentials.NewStaticV4(keyid, keysec, ""),
				Secure: false,
			})
			assert.NoError(t, err)

			buckets, err := minioClient.ListBuckets(context.Background())
			if tt.shouldFail {
				require.Error(t, err)
			} else {
				require.NoError(t, err)
				require.NotEmpty(t, buckets)
				assert.Equal(t, buckets[0].Name, tt.bucket)

				o := minioClient.ListObjects(context.Background(), tt.bucket, minio.ListObjectsOptions{
					Recursive: true,
				})
				// save files after reading from channel
				objects := []string{}
				for object := range o {
					objects = append(objects, object.Key)
				}

				for _, tt := range tt.files {
					file := path.Join(tt.path, tt.filename)
					found := slices.Contains(objects, file)
					require.Equal(t, true, found, "Object not found: "+file)
				}
			}
		})
	}
}

func TestListBuckets(t *testing.T) {
	var cases = []TestCase{
		{
			description: "list buckets",
			bucket:      "mybucket",
			files: []FileStuct{
				{
					path:     "",
					filename: "lorem.txt",
				},
				{
					path:     "foo",
					filename: "bar.txt",
				},
			},
		},
		{
			description: "list buckets: wrong s3 key",
			bucket:      "mybucket",
			keyID:       "invalid",
			shouldFail:  true,
		},
		{
			description: "list buckets: wrong s3 secret",
			bucket:      "mybucket",
			keySec:      "invalid",
			shouldFail:  true,
		},
	}

	testListBuckets(t, cases, false)
}

func TestListBucketsAuthProxy(t *testing.T) {
	var cases = []TestCase{
		{
			description: "list buckets",
			bucket:      "mybucket",
			files: []FileStuct{
				{
					path:     "",
					filename: "lorem.txt",
				},
				{
					path:     "foo",
					filename: "bar.txt",
				},
			},
		},
		{
			description: "list buckets: unknown s3 key",
			bucket:      "mybucket",
			keyID:       random.String(16),
			shouldFail:  true,
		},
		{
			description: "list buckets: wrong s3 secret",
			bucket:      "mybucket",
			keySec:      "invalid",
			shouldFail:  true,
		},
	}

	testListBuckets(t, cases, true)
}

// TestNewServerPerServerAuthProxy checks that a per-server proxyOpt.AuthProxy
// enables proxy mode even when the process-global proxy.Opt.AuthProxy is empty,
// which is the normal case when the server is configured via serve/start.
func TestNewServerPerServerAuthProxy(t *testing.T) {
	fstest.Initialise()

	// Ensure the global is empty so we only test the per-server option.
	assert.Equal(t, "", proxy.Opt.AuthProxy)

	f, err := fs.NewFs(context.Background(), "testdata")
	require.NoError(t, err)

	opt := Opt
	opt.AuthKey = []string{"access-key,secret-key"}
	opt.HTTP.ListenAddr = []string{endpoint}

	proxyOpt := proxy.Opt
	proxyOpt.AuthProxy = "/path/to/auth/proxy"

	w, err := newServer(context.Background(), f, &opt, &vfscommon.Opt, &proxyOpt)
	require.NoError(t, err)
	defer func() {
		assert.NoError(t, w.Shutdown())
	}()
	assert.True(t, w.provider.IsProxy(), "expected auth proxy to be enabled by per-server option")
	assert.Nil(t, w.provider.VFS(), "expected no fixed VFS when auth proxy is in use")
}

// TestAuthProxyEmptySecret checks that a request for an arbitrary
// access key ID signed with an empty secret is refused when using an
// auth proxy without --auth-key.
func TestAuthProxyEmptySecret(t *testing.T) {
	fstest.Initialise()

	prog, err := filepath.Abs("../servetest/proxy_code.go")
	require.NoError(t, err)
	files, err := filepath.Abs("testdata")
	require.NoError(t, err)
	proxy.Opt.AuthProxy = "go run " + prog + " " + files
	defer func() {
		proxy.Opt.AuthProxy = ""
	}()

	endpoint, keyid, _, s := serveS3(t, nil)
	defer func() {
		assert.NoError(t, s.server.Shutdown())
	}()

	for _, accessKeyID := range []string{keyid, random.String(16)} {
		req, err := http.NewRequest("GET", endpoint+"/", nil)
		require.NoError(t, err)
		req.Header.Set("X-Amz-Content-Sha256", "UNSIGNED-PAYLOAD")
		signer := v4.NewSigner()
		err = signer.SignHTTP(context.Background(), aws.Credentials{AccessKeyID: accessKeyID, SecretAccessKey: ""}, req, "UNSIGNED-PAYLOAD", "s3", "us-east-1", time.Now())
		require.NoError(t, err)

		resp, err := http.DefaultClient.Do(req)
		require.NoError(t, err)
		body, _ := io.ReadAll(resp.Body)
		_ = resp.Body.Close()
		assert.Equal(t, http.StatusForbidden, resp.StatusCode, string(body))
		assert.NotContains(t, string(body), "ListAllMyBucketsResult")
	}
}

// TestAuthProxyKeysNotRegistered checks that secrets supplied by the
// auth proxy are never registered in gofakes3's process wide key
// store where any other serve s3 instance in the process would honour
// them.
func TestAuthProxyKeysNotRegistered(t *testing.T) {
	fstest.Initialise()

	prog, err := filepath.Abs("../servetest/proxy_code.go")
	require.NoError(t, err)
	files, err := filepath.Abs("testdata")
	require.NoError(t, err)
	proxy.Opt.AuthProxy = "go run " + prog + " " + files
	defer func() {
		proxy.Opt.AuthProxy = ""
	}()

	endpoint, keyid, keysec, s := serveS3(t, nil)
	defer func() {
		assert.NoError(t, s.server.Shutdown())
	}()

	sign := func() *http.Request {
		req, err := http.NewRequest("GET", endpoint+"/", nil)
		require.NoError(t, err)
		req.Header.Set("X-Amz-Content-Sha256", "UNSIGNED-PAYLOAD")
		signer := v4.NewSigner()
		err = signer.SignHTTP(context.Background(), aws.Credentials{AccessKeyID: keyid, SecretAccessKey: keysec}, req, "UNSIGNED-PAYLOAD", "s3", "us-east-1", time.Now())
		require.NoError(t, err)
		return req
	}

	// A correctly signed request is accepted by the server
	resp, err := http.DefaultClient.Do(sign())
	require.NoError(t, err)
	body, _ := io.ReadAll(resp.Body)
	_ = resp.Body.Close()
	assert.Equal(t, http.StatusOK, resp.StatusCode, string(body))

	// But the key it used must not be in the global key store
	assert.NotEqual(t, signature.ErrNone, signature.V4SignVerify(sign()), "proxy secret was registered in the gofakes3 key store")
}

// serveS3AuthProxy serves root over s3 with opt and an auth proxy which
// maps every one of users (access key IDs) to root, returning a client
// for each user.
func serveS3AuthProxy(t *testing.T, root string, opt Options, users ...string) map[string]*minio.Core {
	fstest.Initialise()
	prog, err := filepath.Abs("../servetest/proxy_code.go")
	require.NoError(t, err)
	secrets := map[string]string{}
	var pairs []string
	for _, user := range users {
		secrets[user] = random.String(16)
		pairs = append(pairs, user+","+secrets[user])
	}
	t.Setenv("RCLONE_TEST_PROXY_AUTH_KEY", strings.Join(pairs, ";"))

	opt.HTTP.ListenAddr = []string{endpoint}
	proxyOpt := proxy.Opt
	proxyOpt.AuthProxy = "go run " + prog + " " + root
	w, err := newServer(context.Background(), nil, &opt, &vfscommon.Opt, &proxyOpt)
	require.NoError(t, err)
	go func() {
		require.NoError(t, w.Serve())
	}()
	t.Cleanup(func() { _ = w.Shutdown() })
	testURL, err := url.Parse(w.server.URLs()[0])
	require.NoError(t, err)

	clients := map[string]*minio.Core{}
	for _, user := range users {
		clients[user], err = minio.NewCore(testURL.Host, &minio.Options{
			Creds: credentials.NewStaticV4(user, secrets[user], ""),
		})
		require.NoError(t, err)
	}
	return clients
}

// TestAuthProxyUsersKeptApart checks that one auth proxy user can't use
// another user's multipart upload or see the metadata of their objects,
// even when both are mapped to the same backend.
func TestAuthProxyUsersKeptApart(t *testing.T) {
	for _, disableStreaming := range []bool{false, true} {
		t.Run(fmt.Sprintf("DisableMultipartStreaming=%v", disableStreaming), func(t *testing.T) {
			opt := Opt
			opt.DisableMultipartStreaming = disableStreaming
			testAuthProxyUsersKeptApart(t, opt)
		})
	}
}

func testAuthProxyUsersKeptApart(t *testing.T, opt Options) {
	ctx := context.Background()
	root := t.TempDir()
	const bucket = "bucket"
	require.NoError(t, os.Mkdir(filepath.Join(root, bucket), 0777))
	clients := serveS3AuthProxy(t, root, opt, "alice", "bob")
	alice, bob := clients["alice"], clients["bob"]

	uploadID, err := alice.NewMultipartUpload(ctx, bucket, "upload", minio.PutObjectOptions{})
	require.NoError(t, err)
	listed := func(client *minio.Core) (uploadIDs []string) {
		result, err := client.ListMultipartUploads(ctx, bucket, "", "", "", "/", 1000)
		if minio.ToErrorResponse(err).Code == "NoSuchUpload" {
			return nil
		}
		require.NoError(t, err)
		for _, upload := range result.Uploads {
			uploadIDs = append(uploadIDs, upload.UploadID)
		}
		return uploadIDs
	}
	assert.Equal(t, []string{uploadID}, listed(alice))
	assert.Empty(t, listed(bob), "bob listing alice's upload")
	_, err = bob.ListObjectParts(ctx, bucket, "upload", uploadID, 0, 1000)
	assert.Equal(t, "NoSuchUpload", minio.ToErrorResponse(err).Code, "bob listing the parts of alice's upload")
	data := []byte("alice's data")
	_, err = bob.PutObjectPart(ctx, bucket, "upload", uploadID, 1, bytes.NewReader(data), int64(len(data)), minio.PutObjectPartOptions{})
	assert.Equal(t, "NoSuchUpload", minio.ToErrorResponse(err).Code, "bob uploading a part to alice's upload")
	err = bob.AbortMultipartUpload(ctx, bucket, "upload", uploadID)
	assert.Equal(t, "NoSuchUpload", minio.ToErrorResponse(err).Code, "bob aborting alice's upload")
	part, err := alice.PutObjectPart(ctx, bucket, "upload", uploadID, 1, bytes.NewReader(data), int64(len(data)), minio.PutObjectPartOptions{})
	require.NoError(t, err)
	_, err = alice.CompleteMultipartUpload(ctx, bucket, "upload", uploadID, []minio.CompletePart{{PartNumber: 1, ETag: part.ETag}}, minio.PutObjectOptions{})
	require.NoError(t, err)

	_, err = alice.PutObject(ctx, bucket, "meta", bytes.NewReader(data), int64(len(data)), "", "", minio.PutObjectOptions{UserMetadata: map[string]string{"Secret": "alice's"}})
	require.NoError(t, err)
	info, err := alice.StatObject(ctx, bucket, "meta", minio.StatObjectOptions{})
	require.NoError(t, err)
	assert.Equal(t, "alice's", info.UserMetadata["Secret"])
	info, err = bob.StatObject(ctx, bucket, "meta", minio.StatObjectOptions{})
	require.NoError(t, err)
	assert.NotContains(t, info.UserMetadata, "Secret", "bob seeing alice's metadata")
}

// TestAuthProxyHoldsVFS checks the VFS the auth proxy gives a request is
// held until the request finishes, so it isn't shut down under a long
// upload or download if it expires from the proxy's cache.
func TestAuthProxyHoldsVFS(t *testing.T) {
	fstest.Initialise()
	prog, err := filepath.Abs("../servetest/proxy_code.go")
	require.NoError(t, err)
	t.Setenv("RCLONE_TEST_PROXY_AUTH_KEY", "alice,secret")
	opt := Opt
	opt.HTTP.ListenAddr = []string{endpoint}
	proxyOpt := proxy.Opt
	proxyOpt.AuthProxy = "go run " + prog + " " + t.TempDir()
	w, err := newServer(context.Background(), nil, &opt, &vfscommon.Opt, &proxyOpt)
	require.NoError(t, err)
	t.Cleanup(func() { _ = w.Shutdown() })

	var VFS *vfs.VFS
	var inUse any
	handler := proxyAuthMiddleware(http.HandlerFunc(func(rw http.ResponseWriter, r *http.Request) {
		VFS, err = w.getVFS(r.Context())
		require.NoError(t, err)
		inUse = VFS.Stats()["inUse"]
	}), w)

	req, err := http.NewRequest("GET", "http://localhost/", nil)
	require.NoError(t, err)
	req.Header.Set("X-Amz-Content-Sha256", "UNSIGNED-PAYLOAD")
	err = v4.NewSigner().SignHTTP(context.Background(), aws.Credentials{AccessKeyID: "alice", SecretAccessKey: "secret"}, req, "UNSIGNED-PAYLOAD", "s3", "us-east-1", time.Now())
	require.NoError(t, err)
	handler.ServeHTTP(httptest.NewRecorder(), req)

	require.NotNil(t, VFS, "request not authenticated")
	assert.Equal(t, int32(2), inUse, "VFS not held by the request")
	assert.Equal(t, int32(1), VFS.Stats()["inUse"], "VFS still held after the request")
}

// TestAuthKeyPerServer checks that two servers in the same process
// with different --auth-key pairs only accept their own credentials.
func TestAuthKeyPerServer(t *testing.T) {
	fstest.Initialise()
	f, err := fs.NewFs(context.Background(), "testdata")
	require.NoError(t, err)

	urlA, keyA, secA, sA := serveS3(t, f)
	defer func() { assert.NoError(t, sA.server.Shutdown()) }()
	urlB, keyB, secB, sB := serveS3(t, f)
	defer func() { assert.NoError(t, sB.server.Shutdown()) }()

	signedGet := func(url, accessKeyID, secret string) int {
		req, err := http.NewRequest("GET", url+"/", nil)
		require.NoError(t, err)
		req.Header.Set("X-Amz-Content-Sha256", "UNSIGNED-PAYLOAD")
		err = v4.NewSigner().SignHTTP(context.Background(), aws.Credentials{AccessKeyID: accessKeyID, SecretAccessKey: secret}, req, "UNSIGNED-PAYLOAD", "s3", "us-east-1", time.Now())
		require.NoError(t, err)
		resp, err := http.DefaultClient.Do(req)
		require.NoError(t, err)
		_, _ = io.ReadAll(resp.Body)
		_ = resp.Body.Close()
		return resp.StatusCode
	}

	assert.Equal(t, http.StatusOK, signedGet(urlA, keyA, secA), "A with A's key")
	assert.Equal(t, http.StatusOK, signedGet(urlB, keyB, secB), "B with B's key")
	assert.Equal(t, http.StatusForbidden, signedGet(urlA, keyB, secB), "A with B's key")
	assert.Equal(t, http.StatusForbidden, signedGet(urlB, keyA, secA), "B with A's key")
}

// newMinioClient serves f and returns a minio client connected to it.
func newMinioClient(t *testing.T, f fs.Fs) *minio.Client {
	endpoint, keyid, keysec, _ := serveS3(t, f)
	testURL, err := url.Parse(endpoint)
	require.NoError(t, err)
	minioClient, err := minio.New(testURL.Host, &minio.Options{
		Creds:  credentials.NewStaticV4(keyid, keysec, ""),
		Secure: false,
	})
	require.NoError(t, err)
	return minioClient
}

// putTestObject puts an object with the given key into bucket on f.
func putTestObject(t *testing.T, f fs.Fs, bucket, key string) {
	contents := "contents"
	obji := object.NewStaticObjectInfo(path.Join(bucket, key), time.Now(), int64(len(contents)), true, nil, nil)
	_, err := f.Put(context.Background(), bytes.NewBufferString(contents), obji)
	require.NoError(t, err)
}

// listKeysWithMinioClient pages through a listing with the given options and
// returns the keys it yields, common prefixes included.
func listKeysWithMinioClient(t *testing.T, minioClient *minio.Client, bucket string, opts minio.ListObjectsOptions) (keys []string) {
	for object := range minioClient.ListObjects(context.Background(), bucket, opts) {
		require.NoError(t, object.Err)
		keys = append(keys, object.Key)
	}
	return keys
}

// TestListObjectsPagingWithMinioClient checks that a client paging through a
// deep hierarchy with a small max-keys sees every key exactly once, in order,
// exercising the continuation tokens end to end.
func TestListObjectsPagingWithMinioClient(t *testing.T) {
	fstest.Initialise()
	f, _, clean, err := fstest.RandomRemote()
	require.NoError(t, err)
	defer clean()

	// A chunk store shaped like the one Proxmox Backup Server lists:
	// many directories, each holding a few objects, listed with a prefix
	// and no delimiter.
	var want []string
	for dir := range 20 {
		for file := range 5 {
			key := fmt.Sprintf(".chunks/%04x/chunk%d", dir, file)
			want = append(want, key)
			putTestObject(t, f, "mybucket", key)
		}
	}
	slices.Sort(want)

	minioClient := newMinioClient(t, f)
	for _, useV1 := range []bool{false, true} {
		t.Run(fmt.Sprintf("UseV1=%v", useV1), func(t *testing.T) {
			got := listKeysWithMinioClient(t, minioClient, "mybucket", minio.ListObjectsOptions{
				Prefix:    ".chunks",
				Recursive: true,
				MaxKeys:   7,
				UseV1:     useV1,
			})
			assert.Equal(t, want, got)
		})
	}
}

// TestListObjectsDelimitedPagingWithMinioClient checks that a client paging
// through a delimited listing sees every common prefix and every object key
// exactly once, whichever side of a page boundary they fall.
func TestListObjectsDelimitedPagingWithMinioClient(t *testing.T) {
	fstest.Initialise()
	f, _, clean, err := fstest.RandomRemote()
	require.NoError(t, err)
	defer clean()

	// Directories and objects which interleave in key order, so that a page
	// boundary has to fall between a common prefix and an object key.
	var want []string
	for i := range 20 {
		if i%2 == 0 {
			putTestObject(t, f, "mybucket", fmt.Sprintf("a%02d/object", i))
			want = append(want, fmt.Sprintf("a%02d/", i))
		} else {
			key := fmt.Sprintf("a%02d.txt", i)
			putTestObject(t, f, "mybucket", key)
			want = append(want, key)
		}
	}
	slices.Sort(want)

	minioClient := newMinioClient(t, f)
	for _, maxKeys := range []int{1, 3, 7, 20} {
		for _, useV1 := range []bool{false, true} {
			t.Run(fmt.Sprintf("MaxKeys%d/UseV1=%v", maxKeys, useV1), func(t *testing.T) {
				got := listKeysWithMinioClient(t, minioClient, "mybucket", minio.ListObjectsOptions{
					MaxKeys: maxKeys,
					UseV1:   useV1,
				})
				// The client yields the objects of a page before its common
				// prefixes, so only the keys of the whole listing are sorted.
				slices.Sort(got)
				assert.Equal(t, want, got)
			})
		}
	}
}

// testUnsignedAmzHeader checks that a request carrying an x-amz-*
// header which is not in its signed headers list is refused, as AWS
// does. Otherwise a presigned PUT URL for one object, which signs only
// the host header, could be turned into a copy from any object the
// server can reach by adding an unsigned x-amz-copy-source header.
func testUnsignedAmzHeader(t *testing.T, useProxy bool) {
	fstest.Initialise()
	root := t.TempDir()
	f, err := fs.NewFs(context.Background(), root)
	require.NoError(t, err)

	const (
		secret   = "THIS-IS-THE-VICTIM-FILE"
		original = "PLACEHOLDER-ORIGINAL-CONTENT"
		uploaded = "UPLOADED-WITH-PRESIGNED-URL"
	)
	ctx := context.Background()
	fstests.PutTestContents(ctx, t, f, &fstest.Item{Path: "src/secret.txt", ModTime: time.Now()}, secret, true)
	fstests.PutTestContents(ctx, t, f, &fstest.Item{Path: "dst/target.txt", ModTime: time.Now()}, original, true)
	readFile := func(name string) string {
		return fstests.ReadObject(ctx, t, fstest.NewObject(ctx, t, f, name), -1)
	}

	serveFs := f
	if useProxy {
		prog, err := filepath.Abs("../servetest/proxy_code.go")
		require.NoError(t, err)
		proxy.Opt.AuthProxy = "go run " + prog + " " + root
		defer func() {
			proxy.Opt.AuthProxy = ""
		}()
		serveFs = nil
	}
	endpoint, keyid, keysec, s := serveS3(t, serveFs)
	defer func() {
		assert.NoError(t, s.server.Shutdown())
	}()

	creds := aws.Credentials{AccessKeyID: keyid, SecretAccessKey: keysec}
	do := func(req *http.Request) (int, string) {
		resp, err := http.DefaultClient.Do(req)
		require.NoError(t, err)
		body, _ := io.ReadAll(resp.Body)
		_ = resp.Body.Close()
		return resp.StatusCode, string(body)
	}
	sign := func(req *http.Request) {
		req.Header.Set("X-Amz-Content-Sha256", "UNSIGNED-PAYLOAD")
		err := v4.NewSigner().SignHTTP(context.Background(), creds, req, "UNSIGNED-PAYLOAD", "s3", "us-east-1", time.Now())
		require.NoError(t, err)
	}

	// A presigned PUT URL for dst/target.txt which signs only the host header
	req, err := http.NewRequest("PUT", endpoint+"/dst/target.txt", nil)
	require.NoError(t, err)
	presigned, _, err := v4.NewSigner().PresignHTTP(context.Background(), creds, req, "UNSIGNED-PAYLOAD", "s3", "us-east-1", time.Now())
	require.NoError(t, err)
	presignedURL, err := url.Parse(presigned)
	require.NoError(t, err)
	require.Equal(t, "host", presignedURL.Query().Get("X-Amz-SignedHeaders"))

	// It works as intended
	req, err = http.NewRequest("PUT", presigned, strings.NewReader(uploaded))
	require.NoError(t, err)
	status, body := do(req)
	assert.Equal(t, http.StatusOK, status, body)
	assert.Equal(t, uploaded, readFile("dst/target.txt"))

	// But must not become a copy from an object the URL does not name
	req, err = http.NewRequest("PUT", presigned, nil)
	require.NoError(t, err)
	req.Header.Set("x-amz-copy-source", "/src/secret.txt")
	status, body = do(req)
	assert.Equal(t, http.StatusForbidden, status, body)
	assert.Contains(t, body, "AccessDenied")
	assert.Equal(t, uploaded, readFile("dst/target.txt"))

	// Nor may an unsigned header be added to a request signed with an
	// Authorization header
	req, err = http.NewRequest("PUT", endpoint+"/dst/target.txt", nil)
	require.NoError(t, err)
	sign(req)
	req.Header.Set("x-amz-copy-source", "/src/secret.txt")
	status, body = do(req)
	assert.Equal(t, http.StatusForbidden, status, body)
	assert.Contains(t, body, "AccessDenied")
	assert.Equal(t, uploaded, readFile("dst/target.txt"))

	// Whereas a copy whose x-amz-copy-source header is signed is allowed
	req, err = http.NewRequest("PUT", endpoint+"/dst/target.txt", nil)
	require.NoError(t, err)
	req.Header.Set("x-amz-copy-source", "/src/secret.txt")
	sign(req)
	status, body = do(req)
	assert.Equal(t, http.StatusOK, status, body)
	assert.Equal(t, secret, readFile("dst/target.txt"))
}

func TestUnsignedAmzHeader(t *testing.T) {
	testUnsignedAmzHeader(t, false)
}

func TestUnsignedAmzHeaderAuthProxy(t *testing.T) {
	testUnsignedAmzHeader(t, true)
}

func TestRc(t *testing.T) {
	servetest.TestRc(t, rc.Params{
		"type":           "s3",
		"vfs_cache_mode": "off",
	})
}