"rsync for cloud storage" - Google Drive, S3, Dropbox, Backblaze B2, One Drive, Swift, Hubic, Wasabi, Google Cloud Storage, Azure Blob, Azure Files, Yandex Files
Go azure-blobazure-blob-storageazure-filesbackblaze-b2cloud-storagedropboxencryptionftpfuse-filesystemgogolanggoogle-cloud-storagegoogle-driveonedriveopenstack-swiftrclones3sftpsyncwebdav
s3: fix GCS provider after SDKv2 update #4989
Nick Craig-Wood66ae050a8b53
diff --git a/backend/s3/s3.go b/backend/s3/s3.goindex 07f7e6d3c..968b65944 100644--- a/backend/s3/s3.go+++ b/backend/s3/s3.go@@ -2931,6 +2931,76 @@ func getClient(ctx context.Context, opt *Options) *http.Client {}}+// Google Cloud Storage alters the Accept-Encoding header, which+// breaks the v2 request signature+//+// It also doesn't like the x-id URL parameter SDKv2 puts in so we+// remove that too.+//+// See https://github.com/aws/aws-sdk-go-v2/issues/1816.+// Adapted from: https://github.com/aws/aws-sdk-go-v2/issues/1816#issuecomment-1927281540+func fixupGCS(o *s3.Options) {+ type ignoredHeadersKey struct{}+ headers := []string{"Accept-Encoding"}++ fixup := middleware.FinalizeMiddlewareFunc(+ "FixupGCS",+ func(ctx context.Context, in middleware.FinalizeInput, next middleware.FinalizeHandler) (out middleware.FinalizeOutput, metadata middleware.Metadata, err error) {+ req, ok := in.Request.(*smithyhttp.Request)+ if !ok {+ return out, metadata, fmt.Errorf("fixupGCS: unexpected request middleware type %T", in.Request)+ }++ // Delete headers from being signed - will restore later+ ignored := make(map[string]string, len(headers))+ for _, h := range headers {+ ignored[h] = req.Header.Get(h)+ req.Header.Del(h)+ }++ // Remove x-id because Google doesn't like them+ if query := req.URL.Query(); query.Has("x-id") {+ query.Del("x-id")+ req.URL.RawQuery = query.Encode()+ }++ // Store ignored on context+ ctx = middleware.WithStackValue(ctx, ignoredHeadersKey{}, ignored)++ return next.HandleFinalize(ctx, in)+ },+ )++ // Restore headers if necessary+ restore := middleware.FinalizeMiddlewareFunc(+ "FixupGCSRestoreHeaders",+ func(ctx context.Context, in middleware.FinalizeInput, next middleware.FinalizeHandler) (out middleware.FinalizeOutput, metadata middleware.Metadata, err error) {+ req, ok := in.Request.(*smithyhttp.Request)+ if !ok {+ return out, metadata, fmt.Errorf("fixupGCS: unexpected request middleware type %T", in.Request)+ }++ // Restore ignored from ctx+ ignored, _ := middleware.GetStackValue(ctx, ignoredHeadersKey{}).(map[string]string)+ for k, v := range ignored {+ req.Header.Set(k, v)+ }++ return next.HandleFinalize(ctx, in)+ },+ )++ o.APIOptions = append(o.APIOptions, func(stack *middleware.Stack) error {+ if err := stack.Finalize.Insert(fixup, "Signing", middleware.Before); err != nil {+ return err+ }+ if err := stack.Finalize.Insert(restore, "Signing", middleware.After); err != nil {+ return err+ }+ return nil+ })+}+// s3Connection makes a connection to s3func s3Connection(ctx context.Context, opt *Options, client *http.Client) (s3Client *s3.Client, err error) {ci := fs.GetConfig(ctx)@@ -3017,6 +3087,12 @@ func s3Connection(ctx context.Context, opt *Options, client *http.Client) (s3Cli})}+ if opt.Provider == "GCS" {+ options = append(options, func(o *s3.Options) {+ fixupGCS(o)+ })+ }+c := s3.NewFromConfig(awsConfig, options...)return c, nil}diff --git a/fstest/test_all/config.yaml b/fstest/test_all/config.yamlindex 10279dd66..37a561b81 100644--- a/fstest/test_all/config.yaml+++ b/fstest/test_all/config.yaml@@ -215,6 +215,23 @@ backends:- TestIntegration/FsMkdir/FsEncoding/leading_HT- TestIntegration/FsMkdir/FsEncoding/leading_VT- TestIntegration/FsMkdir/FsPutFiles/FsPutStream/0+ - backend: "s3"+ remote: "TestS3GCS:"+ fastlist: true+ ignore:+ - TestIntegration/FsMkdir/FsEncoding/control_chars+ - TestIntegration/FsMkdir/FsEncoding/leading_CR+ - TestIntegration/FsMkdir/FsEncoding/leading_LF+ - TestIntegration/FsMkdir/FsEncoding/trailing_CR+ - TestIntegration/FsMkdir/FsEncoding/trailing_LF+ - TestIntegration/FsMkdir/FsPutFiles/PublicLink+ - TestIntegration/FsMkdir/FsPutFiles/SetTier+ - TestIntegration/FsMkdir/FsPutFiles/Internal/Metadata/GzipEncoding+ - TestIntegration/FsMkdir/FsPutFiles/Internal/Versions/VersionAt/AfterDelete/List+ - TestIntegration/FsMkdir/FsPutFiles/Internal/Versions/VersionAt/AfterDelete/NewObject+ - TestIntegration/FsMkdir/FsPutFiles/Internal/Versions/VersionAt/AfterTwo/List+ - TestIntegration/FsMkdir/FsPutFiles/Internal/Versions/VersionAt/AfterTwo/NewObject+ - TestBisyncRemoteRemote/extended_filenames# Disabled due to excessive rate limiting at DO which cause the tests never to pass# This hits the rate limit as documented here: https://www.digitalocean.com/docs/spaces/#limits# 2 COPYs per 5 minutes on any individual object in a Space