"rsync for cloud storage" - Google Drive, S3, Dropbox, Backblaze B2, One Drive, Swift, Hubic, Wasabi, Google Cloud Storage, Azure Blob, Azure Files, Yandex Files
Go azure-blobazure-blob-storageazure-filesbackblaze-b2cloud-storagedropboxencryptionftpfuse-filesystemgogolanggoogle-cloud-storagegoogle-driveonedriveopenstack-swiftrclones3sftpsyncwebdav
s3: add backend link command with signed response header overrides
eliotee39b487d7f758
diff --git a/backend/s3/s3.go b/backend/s3/s3.goindex af365c00c..a444eff35 100644--- a/backend/s3/s3.go+++ b/backend/s3/s3.go@@ -3303,6 +3303,10 @@ func (f *Fs) Hashes() hash.Set {// PublicLink generates a public link to the remote path (usually readable by anyone)func (f *Fs) PublicLink(ctx context.Context, remote string, expire fs.Duration, unlink bool) (link string, err error) {+ return f.publicLink(ctx, remote, expire, &s3.GetObjectInput{})+}++func (f *Fs) publicLink(ctx context.Context, remote string, expire fs.Duration, req *s3.GetObjectInput) (link string, err error) {if strings.HasSuffix(remote, "/") {return "", fs.ErrorCantShareDirectories}@@ -3316,11 +3320,10 @@ func (f *Fs) PublicLink(ctx context.Context, remote string, expire fs.Duration,expire = maxExpireDuration}bucket, bucketPath := f.split(remote)- httpReq, err := s3.NewPresignClient(f.c).PresignGetObject(ctx, &s3.GetObjectInput{- Bucket: &bucket,- Key: &bucketPath,- VersionId: o.versionID,- }, s3.WithPresignExpires(time.Duration(expire)))+ req.Bucket = &bucket+ req.Key = &bucketPath+ req.VersionId = o.versionID+ httpReq, err := s3.NewPresignClient(f.c).PresignGetObject(ctx, req, s3.WithPresignExpires(time.Duration(expire)))if err != nil {return "", err}@@ -3328,6 +3331,40 @@ func (f *Fs) PublicLink(ctx context.Context, remote string, expire fs.Duration,}var commandHelp = []fs.CommandHelp{{+ Name: "link",+ Short: "Generate a signed link with response header overrides.",+ Long: `This command generates a signed download link for one file, with optional+overrides for the HTTP response headers. Pass the file path as a separate+argument, relative to the remote path.++Usage examples:++` + "```console" + `+rclone backend link s3:bucket path/to/file -o expire=1h+rclone backend link s3:bucket path/to/file -o response-expires="Thu, 01 Jan 1970 00:00:00 GMT"+rclone backend link s3:bucket path/to/file -o response-content-disposition='attachment; filename="download.txt"'+` + "```" + `++The link expires after 7 days by default. Use ` + "`-o expire=1h`" + ` to change+its lifetime. Durations must be at least 1 second; values above 7 days are+limited to 7 days, as with ` + "`rclone link`" + `.++The ` + "`response-expires`" + ` option sets the HTTP Expires response header,+not the lifetime of the signed link. It must be an HTTP date, such as+` + "`Thu, 01 Jan 1970 00:00:00 GMT`" + `.++The overrides are included in the signature and must not be changed in the+returned URL. They do not modify the object's stored metadata.`,+ Opts: map[string]string{+ "expire": "How long the link will be valid (default 7d, maximum 7d).",+ "response-cache-control": "Set the Cache-Control response header.",+ "response-content-disposition": "Set the Content-Disposition response header.",+ "response-content-encoding": "Set the Content-Encoding response header.",+ "response-content-language": "Set the Content-Language response header.",+ "response-content-type": "Set the Content-Type response header.",+ "response-expires": "Set the Expires response header to an HTTP date.",+ },+}, {Name: "restore",Short: "Restore objects from GLACIER or INTELLIGENT-TIERING archive tier.",Long: `This command can be used to restore one or more objects from GLACIER to normal@@ -3555,6 +3592,44 @@ It doesn't return anything.`,// otherwise it will be JSON encoded and shown to the user like thatfunc (f *Fs) Command(ctx context.Context, name string, arg []string, opt map[string]string) (out any, err error) {switch name {+ case "link":+ if len(arg) != 1 || arg[0] == "" {+ return nil, errors.New("link requires exactly one file path argument")+ }+ req := s3.GetObjectInput{}+ expire := maxExpireDuration+ for key, value := range opt {+ switch key {+ case "expire":+ duration, err := fs.ParseDuration(value)+ if err != nil {+ return nil, fmt.Errorf("invalid expire: %w", err)+ }+ if duration < time.Second {+ return nil, errors.New("expire must be at least 1 second")+ }+ expire = fs.Duration(duration)+ case "response-cache-control":+ req.ResponseCacheControl = aws.String(value)+ case "response-content-disposition":+ req.ResponseContentDisposition = aws.String(value)+ case "response-content-encoding":+ req.ResponseContentEncoding = aws.String(value)+ case "response-content-language":+ req.ResponseContentLanguage = aws.String(value)+ case "response-content-type":+ req.ResponseContentType = aws.String(value)+ case "response-expires":+ date, err := http.ParseTime(value)+ if err != nil {+ return nil, fmt.Errorf("invalid response-expires: %w", err)+ }+ req.ResponseExpires = &date+ default:+ return nil, fmt.Errorf("unknown link option %q", key)+ }+ }+ return f.publicLink(ctx, arg[0], expire, &req)case "restore":req := s3.RestoreObjectInput{//Bucket: &f.rootBucket,diff --git a/backend/s3/s3_link_test.go b/backend/s3/s3_link_test.gonew file mode 100644index 000000000..7a8b01708--- /dev/null+++ b/backend/s3/s3_link_test.go@@ -0,0 +1,227 @@+package s3++import (+ "context"+ "io"+ "net/http"+ "net/http/httptest"+ "net/url"+ "strings"+ "testing"+ "time"++ "github.com/aws/aws-sdk-go-v2/aws"+ v4 "github.com/aws/aws-sdk-go-v2/aws/signer/v4"+ "github.com/rclone/rclone/fs"+ "github.com/rclone/rclone/lib/bucket"+ "github.com/rclone/rclone/lib/pacer"+ "github.com/stretchr/testify/assert"+ "github.com/stretchr/testify/require"+)++func newLinkTestFs(t *testing.T) *Fs {+ t.Helper()+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ if r.URL.Query().Has("versions") {+ assert.Equal(t, http.MethodGet, r.Method)+ assert.Equal(t, "prefix/file", r.URL.Query().Get("prefix"))+ _, err := io.WriteString(w, `<ListVersionsResult xmlns="http://s3.amazonaws.com/doc/2006-03-01/">+<IsTruncated>false</IsTruncated>+<Version><Key>prefix/file</Key><VersionId>version+id/1</VersionId><IsLatest>false</IsLatest>+<LastModified>2024-01-01T00:00:00Z</LastModified><Size>1</Size></Version>+</ListVersionsResult>`)+ assert.NoError(t, err)+ return+ }+ assert.Equal(t, http.MethodHead, r.Method)+ if strings.HasSuffix(r.URL.Path, "/missing") {+ w.WriteHeader(http.StatusNotFound)+ return+ }+ w.Header().Set("Content-Length", "1")+ w.Header().Set("Last-Modified", "Wed, 01 Jan 2025 00:00:00 GMT")+ }))+ t.Cleanup(server.Close)++ ctx, opt, client := SetupS3Test(t)+ opt.Endpoint = server.URL+ opt.ForcePathStyle = true+ opt.Region = "us-east-1"+ opt.AccessKeyID = "test-access-key"+ opt.SecretAccessKey = "test-secret-key"+ opt.SessionToken = "test-token+/="+ c, _, err := s3Connection(ctx, opt, client)+ require.NoError(t, err)+ f := &Fs{+ name: "s3test",+ opt: *opt,+ ctx: ctx,+ c: c,+ pacer: fs.NewPacer(ctx, pacer.NewS3(pacer.MinSleep(minSleep))),+ cache: bucket.NewCache(),+ }+ f.setRoot("bucket/prefix")+ return f+}++func linkSignature(t *testing.T, u *url.URL) string {+ t.Helper()+ q := u.Query()+ signingTime, err := time.Parse("20060102T150405Z", q.Get("X-Amz-Date"))+ require.NoError(t, err)+ q.Del("X-Amz-Signature")+ unsigned := *u+ unsigned.RawQuery = q.Encode()+ req, err := http.NewRequestWithContext(context.Background(), http.MethodGet, unsigned.String(), nil)+ require.NoError(t, err)+ signed, _, err := v4.NewSigner().PresignHTTP(context.Background(), aws.Credentials{+ AccessKeyID: "test-access-key",+ SecretAccessKey: "test-secret-key",+ SessionToken: "test-token+/=",+ }, req, "UNSIGNED-PAYLOAD", "s3", "us-east-1", signingTime, func(opt *v4.SignerOptions) {+ opt.DisableURIPathEscaping = true+ })+ require.NoError(t, err)+ result, err := url.Parse(signed)+ require.NoError(t, err)+ return result.Query().Get("X-Amz-Signature")+}++func TestCommandLink(t *testing.T) {+ f := newLinkTestFs(t)+ remote := "dir/a file +&?%\u2603.txt"+ overrides := map[string]string{+ "response-cache-control": "private, max-age=0",+ "response-content-disposition": `attachment; filename="a +&?.txt"`,+ "response-content-encoding": "identity",+ "response-content-language": "en-US",+ "response-content-type": "text/plain; charset=utf-8",+ "response-expires": "Thu, 01 Jan 1970 00:00:00 GMT",+ }+ opts := map[string]string{"expire": "1h"}+ for k, v := range overrides {+ opts[k] = v+ }+ out, err := f.Command(context.Background(), "link", []string{remote}, opts)+ require.NoError(t, err)+ link, ok := out.(string)+ require.True(t, ok)+ u, err := url.Parse(link)+ require.NoError(t, err)+ assert.Equal(t, "/bucket/prefix/"+remote, u.Path)+ assert.NotContains(t, u.RawQuery, " ")+ q := u.Query()+ assert.Equal(t, "3600", q.Get("X-Amz-Expires"))+ assert.Equal(t, "test-token+/=", q.Get("X-Amz-Security-Token"))+ assert.Equal(t, "AWS4-HMAC-SHA256", q.Get("X-Amz-Algorithm"))+ assert.Equal(t, "host", q.Get("X-Amz-SignedHeaders"))+ for k, v := range overrides {+ assert.Equal(t, v, q.Get(k), k)+ }+ signature := q.Get("X-Amz-Signature")+ require.NotEmpty(t, signature)+ assert.Equal(t, signature, linkSignature(t, u))+ for k := range overrides {+ t.Run("Signed/"+k, func(t *testing.T) {+ tampered := *u+ changed := u.Query()+ changed.Set(k, "changed")+ tampered.RawQuery = changed.Encode()+ assert.NotEqual(t, signature, linkSignature(t, &tampered))+ })+ }+}++func TestCommandLinkExpire(t *testing.T) {+ f := newLinkTestFs(t)+ for _, test := range []struct {+ name string+ opts map[string]string+ expire string+ }{+ {"Default", nil, "604800"},+ {"Minimum", map[string]string{"expire": "1s"}, "1"},+ {"Maximum", map[string]string{"expire": "7d"}, "604800"},+ {"Clamped", map[string]string{"expire": "8d"}, "604800"},+ {"Off", map[string]string{"expire": "off"}, "604800"},+ } {+ t.Run(test.name, func(t *testing.T) {+ out, err := f.Command(context.Background(), "link", []string{"file"}, test.opts)+ require.NoError(t, err)+ u, err := url.Parse(out.(string))+ require.NoError(t, err)+ assert.Equal(t, test.expire, u.Query().Get("X-Amz-Expires"))+ for key := range u.Query() {+ assert.False(t, strings.HasPrefix(key, "response-"))+ }+ assert.Equal(t, u.Query().Get("X-Amz-Signature"), linkSignature(t, u))+ })+ }+}++func TestCommandLinkErrors(t *testing.T) {+ f := newLinkTestFs(t)+ for _, test := range []struct {+ name string+ args []string+ opts map[string]string+ }{+ {"NoPath", nil, nil},+ {"TooManyPaths", []string{"one", "two"}, nil},+ {"EmptyPath", []string{""}, nil},+ {"Directory", []string{"dir/"}, nil},+ {"Missing", []string{"missing"}, nil},+ {"UnknownOption", []string{"file"}, map[string]string{"response-unknown": "value"}},+ {"InvalidDate", []string{"file"}, map[string]string{"response-expires": "tomorrow"}},+ {"InvalidExpire", []string{"file"}, map[string]string{"expire": "invalid"}},+ {"EmptyExpire", []string{"file"}, map[string]string{"expire": ""}},+ {"ZeroExpire", []string{"file"}, map[string]string{"expire": "0"}},+ {"NegativeExpire", []string{"file"}, map[string]string{"expire": "-1s"}},+ {"SubsecondExpire", []string{"file"}, map[string]string{"expire": "500ms"}},+ } {+ t.Run(test.name, func(t *testing.T) {+ _, err := f.Command(context.Background(), "link", test.args, test.opts)+ require.Error(t, err)+ assert.NotErrorIs(t, err, fs.ErrorCommandNotFound)+ })+ }+}++func TestCommandLinkVersionAt(t *testing.T) {+ f := newLinkTestFs(t)+ f.opt.VersionAt = fs.Time(time.Date(2025, time.January, 1, 0, 0, 0, 0, time.UTC))+ for _, command := range []bool{false, true} {+ var link string+ if command {+ out, err := f.Command(context.Background(), "link", []string{"file"}, map[string]string{+ "response-content-type": "text/plain",+ })+ require.NoError(t, err)+ link = out.(string)+ } else {+ var err error+ link, err = f.PublicLink(context.Background(), "file", fs.DurationOff, false)+ require.NoError(t, err)+ }+ u, err := url.Parse(link)+ require.NoError(t, err)+ assert.Equal(t, "/bucket/prefix/file", u.Path)+ assert.Equal(t, "version+id/1", u.Query().Get("versionId"))+ assert.Equal(t, u.Query().Get("X-Amz-Signature"), linkSignature(t, u))+ }+}++func TestPublicLink(t *testing.T) {+ f := newLinkTestFs(t)+ link, err := f.PublicLink(context.Background(), "file", fs.DurationOff, false)+ require.NoError(t, err)+ u, err := url.Parse(link)+ require.NoError(t, err)+ assert.Equal(t, "/bucket/prefix/file", u.Path)+ assert.Equal(t, "604800", u.Query().Get("X-Amz-Expires"))+ assert.Equal(t, u.Query().Get("X-Amz-Signature"), linkSignature(t, u))+ _, err = f.PublicLink(context.Background(), "dir/", fs.DurationOff, false)+ assert.ErrorIs(t, err, fs.ErrorCantShareDirectories)+ _, err = f.PublicLink(context.Background(), "missing", fs.DurationOff, false)+ assert.ErrorIs(t, err, fs.ErrorObjectNotFound)+}