"rsync for cloud storage" - Google Drive, S3, Dropbox, Backblaze B2, One Drive, Swift, Hubic, Wasabi, Google Cloud Storage, Azure Blob, Azure Files, Yandex Files
Go azure-blobazure-blob-storageazure-filesbackblaze-b2cloud-storagedropboxencryptionftpfuse-filesystemgogolanggoogle-cloud-storagegoogle-driveonedriveopenstack-swiftrclones3sftpsyncwebdav
s3: split the GCS quirks into -s3-use-x-id and -s3-sign-accept-encoding #8373
Nick Craig-Woode175c863aa99
diff --git a/backend/s3/s3.go b/backend/s3/s3.goindex d698da4be..a3b1d770c 100644--- a/backend/s3/s3.go+++ b/backend/s3/s3.go@@ -2680,6 +2680,34 @@ knows about - please make a bug report if not.You can change this if you want to disable the use of multipart uploads.This shouldn't be necessary in normal operation.+This should be automatically set correctly for all providers rclone+knows about - please make a bug report if not.+`,+ Default: fs.Tristate{},+ Advanced: true,+ }, {+ Name: "use_x_id",+ Help: `Set if rclone should add x-id URL parameters.++You can change this if you want to disable the AWS SDK from+adding x-id URL parameters.++This shouldn't be necessary in normal operation.++This should be automatically set correctly for all providers rclone+knows about - please make a bug report if not.+`,+ Default: fs.Tristate{},+ Advanced: true,+ }, {+ Name: "sign_accept_encoding",+ Help: `Set if rclone should include Accept-Encoding as part of the signature.++You can change this if you want to stop rclone including+Accept-Encoding as part of the signature.++This shouldn't be necessary in normal operation.+This should be automatically set correctly for all providers rcloneknows about - please make a bug report if not.`,@@ -2901,6 +2929,8 @@ type Options struct {DirectoryBucket bool `config:"directory_bucket"`IBMAPIKey string `config:"ibm_api_key"`IBMInstanceID string `config:"ibm_resource_instance_id"`+ UseXID fs.Tristate `config:"use_x_id"`+ SignAcceptEncoding fs.Tristate `config:"sign_accept_encoding"`}// Fs represents a remote s3 server@@ -3085,41 +3115,47 @@ func getClient(ctx context.Context, opt *Options) *http.Client {}}+// Fixup the request if needed.+//// Google Cloud Storage alters the Accept-Encoding header, which-// breaks the v2 request signature+// breaks the v2 request signature. This is set with opt.SignAcceptEncoding.//// It also doesn't like the x-id URL parameter SDKv2 puts in so we-// remove that too.+// remove that too. This is set with opt.UseXID.Value.//// See https://github.com/aws/aws-sdk-go-v2/issues/1816.// Adapted from: https://github.com/aws/aws-sdk-go-v2/issues/1816#issuecomment-1927281540-func fixupGCS(o *s3.Options) {+func fixupRequest(o *s3.Options, opt *Options) {type ignoredHeadersKey struct{}headers := []string{"Accept-Encoding"}fixup := middleware.FinalizeMiddlewareFunc(- "FixupGCS",+ "FixupRequest",func(ctx context.Context, in middleware.FinalizeInput, next middleware.FinalizeHandler) (out middleware.FinalizeOutput, metadata middleware.Metadata, err error) {req, ok := in.Request.(*smithyhttp.Request)if !ok {- return out, metadata, fmt.Errorf("fixupGCS: unexpected request middleware type %T", in.Request)+ return out, metadata, fmt.Errorf("fixupRequest: unexpected request middleware type %T", in.Request)}- // Delete headers from being signed - will restore later- ignored := make(map[string]string, len(headers))- for _, h := range headers {- ignored[h] = req.Header.Get(h)- req.Header.Del(h)- }+ if !opt.SignAcceptEncoding.Value {+ // Delete headers from being signed - will restore later+ ignored := make(map[string]string, len(headers))+ for _, h := range headers {+ ignored[h] = req.Header.Get(h)+ req.Header.Del(h)+ }- // Remove x-id because Google doesn't like them- if query := req.URL.Query(); query.Has("x-id") {- query.Del("x-id")- req.URL.RawQuery = query.Encode()+ // Store ignored on context+ ctx = middleware.WithStackValue(ctx, ignoredHeadersKey{}, ignored)}- // Store ignored on context- ctx = middleware.WithStackValue(ctx, ignoredHeadersKey{}, ignored)+ if !opt.UseXID.Value {+ // Remove x-id+ if query := req.URL.Query(); query.Has("x-id") {+ query.Del("x-id")+ req.URL.RawQuery = query.Encode()+ }+ }return next.HandleFinalize(ctx, in)},@@ -3127,17 +3163,19 @@ func fixupGCS(o *s3.Options) {// Restore headers if necessaryrestore := middleware.FinalizeMiddlewareFunc(- "FixupGCSRestoreHeaders",+ "FixupRequestRestoreHeaders",func(ctx context.Context, in middleware.FinalizeInput, next middleware.FinalizeHandler) (out middleware.FinalizeOutput, metadata middleware.Metadata, err error) {req, ok := in.Request.(*smithyhttp.Request)if !ok {- return out, metadata, fmt.Errorf("fixupGCS: unexpected request middleware type %T", in.Request)+ return out, metadata, fmt.Errorf("fixupRequest: unexpected request middleware type %T", in.Request)}- // Restore ignored from ctx- ignored, _ := middleware.GetStackValue(ctx, ignoredHeadersKey{}).(map[string]string)- for k, v := range ignored {- req.Header.Set(k, v)+ if !opt.SignAcceptEncoding.Value {+ // Restore ignored from ctx+ ignored, _ := middleware.GetStackValue(ctx, ignoredHeadersKey{}).(map[string]string)+ for k, v := range ignored {+ req.Header.Set(k, v)+ }}return next.HandleFinalize(ctx, in)@@ -3267,9 +3305,10 @@ func s3Connection(ctx context.Context, opt *Options, client *http.Client) (s3Cli}}- if opt.Provider == "GCS" {+ // Fixup the request if needed+ if !opt.UseXID.Value || !opt.SignAcceptEncoding.Value {options = append(options, func(o *s3.Options) {- fixupGCS(o)+ fixupRequest(o, opt)})}@@ -3384,6 +3423,8 @@ func setQuirks(opt *Options) {useAlreadyExists = true // Set if provider returns AlreadyOwnedByYou or no error if you try to remake your own bucketuseMultipartUploads = true // Set if provider supports multipart uploadsuseUnsignedPayload = true // Do we need to use unsigned payloads to avoid seeking in PutObject+ useXID = true // Add x-id URL parameter into requests+ signAcceptEncoding = true // If we should include AcceptEncoding in the signature)switch opt.Provider {case "AWS":@@ -3528,11 +3569,14 @@ func setQuirks(opt *Options) {// Google break request Signature by mutating accept-encoding HTTP header// https://github.com/rclone/rclone/issues/6670useAcceptEncodingGzip = false+ signAcceptEncoding = falseuseAlreadyExists = true // returns BucketNameUnavailable instead of BucketAlreadyExists but good enough!// GCS S3 doesn't support multi-part server side copy:// See: https://issuetracker.google.com/issues/323465186// So make cutoff very large which it does seem to supportopt.CopyCutoff = math.MaxInt64+ // GCS doesn't like the x-id URL parameter the SDKv2 inserts+ useXID = falsedefault: //nolint:gocritic // Don't include gocritic when running golangci-lint to avoid defaultCaseOrder: consider to make `default` case as first or as last casefs.Logf("s3", "s3 provider %q not known - please set correctly", opt.Provider)fallthrough@@ -3602,6 +3646,18 @@ func setQuirks(opt *Options) {opt.UseUnsignedPayload.Valid = trueopt.UseUnsignedPayload.Value = useUnsignedPayload}++ // Set the correct use UseXID if not manually set+ if !opt.UseXID.Valid {+ opt.UseXID.Valid = true+ opt.UseXID.Value = useXID+ }++ // Set the correct SignAcceptEncoding if not manually set+ if !opt.SignAcceptEncoding.Valid {+ opt.SignAcceptEncoding.Valid = true+ opt.SignAcceptEncoding.Value = signAcceptEncoding+ }}// setRoot changes the root of the Fs