Skip to content

"rsync for cloud storage" - Google Drive, S3, Dropbox, Backblaze B2, One Drive, Swift, Hubic, Wasabi, Google Cloud Storage, Azure Blob, Azure Files, Yandex Files

s3: split the GCS quirks into -s3-use-x-id and -s3-sign-accept-encoding #8373

Nick Craig-Woode175c863aa99
diff --git a/backend/s3/s3.go b/backend/s3/s3.go
index d698da4be..a3b1d770c 100644
--- a/backend/s3/s3.go
+++ b/backend/s3/s3.go
@@ -2680,6 +2680,34 @@ knows about - please make a bug report if not.
You can change this if you want to disable the use of multipart uploads.
This shouldn't be necessary in normal operation.
+This should be automatically set correctly for all providers rclone
+knows about - please make a bug report if not.
+`,
+ Default: fs.Tristate{},
+ Advanced: true,
+ }, {
+ Name: "use_x_id",
+ Help: `Set if rclone should add x-id URL parameters.
+
+You can change this if you want to disable the AWS SDK from
+adding x-id URL parameters.
+
+This shouldn't be necessary in normal operation.
+
+This should be automatically set correctly for all providers rclone
+knows about - please make a bug report if not.
+`,
+ Default: fs.Tristate{},
+ Advanced: true,
+ }, {
+ Name: "sign_accept_encoding",
+ Help: `Set if rclone should include Accept-Encoding as part of the signature.
+
+You can change this if you want to stop rclone including
+Accept-Encoding as part of the signature.
+
+This shouldn't be necessary in normal operation.
+
This should be automatically set correctly for all providers rclone
knows about - please make a bug report if not.
`,
@@ -2901,6 +2929,8 @@ type Options struct {
DirectoryBucket bool `config:"directory_bucket"`
IBMAPIKey string `config:"ibm_api_key"`
IBMInstanceID string `config:"ibm_resource_instance_id"`
+ UseXID fs.Tristate `config:"use_x_id"`
+ SignAcceptEncoding fs.Tristate `config:"sign_accept_encoding"`
}
// Fs represents a remote s3 server
@@ -3085,41 +3115,47 @@ func getClient(ctx context.Context, opt *Options) *http.Client {
}
}
+// Fixup the request if needed.
+//
// Google Cloud Storage alters the Accept-Encoding header, which
-// breaks the v2 request signature
+// breaks the v2 request signature. This is set with opt.SignAcceptEncoding.
//
// It also doesn't like the x-id URL parameter SDKv2 puts in so we
-// remove that too.
+// remove that too. This is set with opt.UseXID.Value.
//
// See https://github.com/aws/aws-sdk-go-v2/issues/1816.
// Adapted from: https://github.com/aws/aws-sdk-go-v2/issues/1816#issuecomment-1927281540
-func fixupGCS(o *s3.Options) {
+func fixupRequest(o *s3.Options, opt *Options) {
type ignoredHeadersKey struct{}
headers := []string{"Accept-Encoding"}
fixup := middleware.FinalizeMiddlewareFunc(
- "FixupGCS",
+ "FixupRequest",
func(ctx context.Context, in middleware.FinalizeInput, next middleware.FinalizeHandler) (out middleware.FinalizeOutput, metadata middleware.Metadata, err error) {
req, ok := in.Request.(*smithyhttp.Request)
if !ok {
- return out, metadata, fmt.Errorf("fixupGCS: unexpected request middleware type %T", in.Request)
+ return out, metadata, fmt.Errorf("fixupRequest: unexpected request middleware type %T", in.Request)
}
- // Delete headers from being signed - will restore later
- ignored := make(map[string]string, len(headers))
- for _, h := range headers {
- ignored[h] = req.Header.Get(h)
- req.Header.Del(h)
- }
+ if !opt.SignAcceptEncoding.Value {
+ // Delete headers from being signed - will restore later
+ ignored := make(map[string]string, len(headers))
+ for _, h := range headers {
+ ignored[h] = req.Header.Get(h)
+ req.Header.Del(h)
+ }
- // Remove x-id because Google doesn't like them
- if query := req.URL.Query(); query.Has("x-id") {
- query.Del("x-id")
- req.URL.RawQuery = query.Encode()
+ // Store ignored on context
+ ctx = middleware.WithStackValue(ctx, ignoredHeadersKey{}, ignored)
}
- // Store ignored on context
- ctx = middleware.WithStackValue(ctx, ignoredHeadersKey{}, ignored)
+ if !opt.UseXID.Value {
+ // Remove x-id
+ if query := req.URL.Query(); query.Has("x-id") {
+ query.Del("x-id")
+ req.URL.RawQuery = query.Encode()
+ }
+ }
return next.HandleFinalize(ctx, in)
},
@@ -3127,17 +3163,19 @@ func fixupGCS(o *s3.Options) {
// Restore headers if necessary
restore := middleware.FinalizeMiddlewareFunc(
- "FixupGCSRestoreHeaders",
+ "FixupRequestRestoreHeaders",
func(ctx context.Context, in middleware.FinalizeInput, next middleware.FinalizeHandler) (out middleware.FinalizeOutput, metadata middleware.Metadata, err error) {
req, ok := in.Request.(*smithyhttp.Request)
if !ok {
- return out, metadata, fmt.Errorf("fixupGCS: unexpected request middleware type %T", in.Request)
+ return out, metadata, fmt.Errorf("fixupRequest: unexpected request middleware type %T", in.Request)
}
- // Restore ignored from ctx
- ignored, _ := middleware.GetStackValue(ctx, ignoredHeadersKey{}).(map[string]string)
- for k, v := range ignored {
- req.Header.Set(k, v)
+ if !opt.SignAcceptEncoding.Value {
+ // Restore ignored from ctx
+ ignored, _ := middleware.GetStackValue(ctx, ignoredHeadersKey{}).(map[string]string)
+ for k, v := range ignored {
+ req.Header.Set(k, v)
+ }
}
return next.HandleFinalize(ctx, in)
@@ -3267,9 +3305,10 @@ func s3Connection(ctx context.Context, opt *Options, client *http.Client) (s3Cli
}
}
- if opt.Provider == "GCS" {
+ // Fixup the request if needed
+ if !opt.UseXID.Value || !opt.SignAcceptEncoding.Value {
options = append(options, func(o *s3.Options) {
- fixupGCS(o)
+ fixupRequest(o, opt)
})
}
@@ -3384,6 +3423,8 @@ func setQuirks(opt *Options) {
useAlreadyExists = true // Set if provider returns AlreadyOwnedByYou or no error if you try to remake your own bucket
useMultipartUploads = true // Set if provider supports multipart uploads
useUnsignedPayload = true // Do we need to use unsigned payloads to avoid seeking in PutObject
+ useXID = true // Add x-id URL parameter into requests
+ signAcceptEncoding = true // If we should include AcceptEncoding in the signature
)
switch opt.Provider {
case "AWS":
@@ -3528,11 +3569,14 @@ func setQuirks(opt *Options) {
// Google break request Signature by mutating accept-encoding HTTP header
// https://github.com/rclone/rclone/issues/6670
useAcceptEncodingGzip = false
+ signAcceptEncoding = false
useAlreadyExists = true // returns BucketNameUnavailable instead of BucketAlreadyExists but good enough!
// GCS S3 doesn't support multi-part server side copy:
// See: https://issuetracker.google.com/issues/323465186
// So make cutoff very large which it does seem to support
opt.CopyCutoff = math.MaxInt64
+ // GCS doesn't like the x-id URL parameter the SDKv2 inserts
+ useXID = false
default: //nolint:gocritic // Don't include gocritic when running golangci-lint to avoid defaultCaseOrder: consider to make `default` case as first or as last case
fs.Logf("s3", "s3 provider %q not known - please set correctly", opt.Provider)
fallthrough
@@ -3602,6 +3646,18 @@ func setQuirks(opt *Options) {
opt.UseUnsignedPayload.Valid = true
opt.UseUnsignedPayload.Value = useUnsignedPayload
}
+
+ // Set the correct use UseXID if not manually set
+ if !opt.UseXID.Valid {
+ opt.UseXID.Valid = true
+ opt.UseXID.Value = useXID
+ }
+
+ // Set the correct SignAcceptEncoding if not manually set
+ if !opt.SignAcceptEncoding.Valid {
+ opt.SignAcceptEncoding.Valid = true
+ opt.SignAcceptEncoding.Value = signAcceptEncoding
+ }
}
// setRoot changes the root of the Fs