Skip to content

"rsync for cloud storage" - Google Drive, S3, Dropbox, Backblaze B2, One Drive, Swift, Hubic, Wasabi, Google Cloud Storage, Azure Blob, Azure Files, Yandex Files

s3: add --s3-use-data-integrity-protections to fix BadDigest error in Alibaba, Tencent

hunshcn16971ab6b9c3
diff --git a/backend/s3/provider/AWS.yaml b/backend/s3/provider/AWS.yaml
index 39c12ae08..989462d32 100644
--- a/backend/s3/provider/AWS.yaml
+++ b/backend/s3/provider/AWS.yaml
@@ -137,3 +137,4 @@ use_accelerate_endpoint: true
quirks:
might_gzip: false # Never auto gzips objects
use_unsigned_payload: false # AWS has trailer support which means it adds checksums in the trailer without seeking
+ use_data_integrity_protections: true
diff --git a/backend/s3/providers.go b/backend/s3/providers.go
index 6c9eb8a17..858aaea6e 100644
--- a/backend/s3/providers.go
+++ b/backend/s3/providers.go
@@ -20,20 +20,21 @@ var NewYamlMap = orderedmap.New[string, string]
// Quirks defines all the S3 provider quirks
type Quirks struct {
- ListVersion *int `yaml:"list_version,omitempty"` // 1 or 2
- ForcePathStyle *bool `yaml:"force_path_style,omitempty"` // true = path-style
- ListURLEncode *bool `yaml:"list_url_encode,omitempty"`
- UseMultipartEtag *bool `yaml:"use_multipart_etag,omitempty"`
- UseAlreadyExists *bool `yaml:"use_already_exists,omitempty"`
- UseAcceptEncodingGzip *bool `yaml:"use_accept_encoding_gzip,omitempty"`
- MightGzip *bool `yaml:"might_gzip,omitempty"`
- UseMultipartUploads *bool `yaml:"use_multipart_uploads,omitempty"`
- UseUnsignedPayload *bool `yaml:"use_unsigned_payload,omitempty"`
- UseXID *bool `yaml:"use_x_id,omitempty"`
- SignAcceptEncoding *bool `yaml:"sign_accept_encoding,omitempty"`
- CopyCutoff *int64 `yaml:"copy_cutoff,omitempty"`
- MaxUploadParts *int `yaml:"max_upload_parts,omitempty"`
- MinChunkSize *int64 `yaml:"min_chunk_size,omitempty"`
+ ListVersion *int `yaml:"list_version,omitempty"` // 1 or 2
+ ForcePathStyle *bool `yaml:"force_path_style,omitempty"` // true = path-style
+ ListURLEncode *bool `yaml:"list_url_encode,omitempty"`
+ UseMultipartEtag *bool `yaml:"use_multipart_etag,omitempty"`
+ UseAlreadyExists *bool `yaml:"use_already_exists,omitempty"`
+ UseAcceptEncodingGzip *bool `yaml:"use_accept_encoding_gzip,omitempty"`
+ UseDataIntegrityProtections *bool `yaml:"use_data_integrity_protections,omitempty"`
+ MightGzip *bool `yaml:"might_gzip,omitempty"`
+ UseMultipartUploads *bool `yaml:"use_multipart_uploads,omitempty"`
+ UseUnsignedPayload *bool `yaml:"use_unsigned_payload,omitempty"`
+ UseXID *bool `yaml:"use_x_id,omitempty"`
+ SignAcceptEncoding *bool `yaml:"sign_accept_encoding,omitempty"`
+ CopyCutoff *int64 `yaml:"copy_cutoff,omitempty"`
+ MaxUploadParts *int `yaml:"max_upload_parts,omitempty"`
+ MinChunkSize *int64 `yaml:"min_chunk_size,omitempty"`
}
// Provider defines the configurable data in each provider.yaml
diff --git a/backend/s3/s3.go b/backend/s3/s3.go
index 137b31adb..c12a97619 100644
--- a/backend/s3/s3.go
+++ b/backend/s3/s3.go
@@ -39,6 +39,9 @@ import (
smithyhttp "github.com/aws/smithy-go/transport/http"
"github.com/ncw/swift/v2"
+ "golang.org/x/net/http/httpguts"
+ "golang.org/x/sync/errgroup"
+
"github.com/rclone/rclone/fs"
"github.com/rclone/rclone/fs/accounting"
"github.com/rclone/rclone/fs/chunksize"
@@ -59,8 +62,6 @@ import (
"github.com/rclone/rclone/lib/readers"
"github.com/rclone/rclone/lib/rest"
"github.com/rclone/rclone/lib/version"
- "golang.org/x/net/http/httpguts"
- "golang.org/x/sync/errgroup"
)
// Register with Fs
@@ -574,6 +575,13 @@ circumstances or for testing.
`,
Default: false,
Advanced: true,
+ }, {
+ Name: "use_data_integrity_protections",
+ Help: `If true use AWS S3 data integrity protections.
+
+See [AWS Docs on Data Integrity Protections](https://docs.aws.amazon.com/sdkref/latest/guide/feature-dataintegrity.html)`,
+ Default: fs.Tristate{},
+ Advanced: true,
}, {
Name: "versions",
Help: "Include old versions in directory listings.",
@@ -892,67 +900,68 @@ var systemMetadataInfo = map[string]fs.MetadataHelp{
// Options defines the configuration for this backend
type Options struct {
- Provider string `config:"provider"`
- EnvAuth bool `config:"env_auth"`
- AccessKeyID string `config:"access_key_id"`
- SecretAccessKey string `config:"secret_access_key"`
- Region string `config:"region"`
- Endpoint string `config:"endpoint"`
- STSEndpoint string `config:"sts_endpoint"`
- UseDualStack bool `config:"use_dual_stack"`
- LocationConstraint string `config:"location_constraint"`
- ACL string `config:"acl"`
- BucketACL string `config:"bucket_acl"`
- RequesterPays bool `config:"requester_pays"`
- ServerSideEncryption string `config:"server_side_encryption"`
- SSEKMSKeyID string `config:"sse_kms_key_id"`
- SSECustomerAlgorithm string `config:"sse_customer_algorithm"`
- SSECustomerKey string `config:"sse_customer_key"`
- SSECustomerKeyBase64 string `config:"sse_customer_key_base64"`
- SSECustomerKeyMD5 string `config:"sse_customer_key_md5"`
- StorageClass string `config:"storage_class"`
- UploadCutoff fs.SizeSuffix `config:"upload_cutoff"`
- CopyCutoff fs.SizeSuffix `config:"copy_cutoff"`
- ChunkSize fs.SizeSuffix `config:"chunk_size"`
- MaxUploadParts int `config:"max_upload_parts"`
- DisableChecksum bool `config:"disable_checksum"`
- SharedCredentialsFile string `config:"shared_credentials_file"`
- Profile string `config:"profile"`
- SessionToken string `config:"session_token"`
- UploadConcurrency int `config:"upload_concurrency"`
- ForcePathStyle bool `config:"force_path_style"`
- V2Auth bool `config:"v2_auth"`
- UseAccelerateEndpoint bool `config:"use_accelerate_endpoint"`
- UseARNRegion bool `config:"use_arn_region"`
- LeavePartsOnError bool `config:"leave_parts_on_error"`
- ListChunk int32 `config:"list_chunk"`
- ListVersion int `config:"list_version"`
- ListURLEncode fs.Tristate `config:"list_url_encode"`
- NoCheckBucket bool `config:"no_check_bucket"`
- NoHead bool `config:"no_head"`
- NoHeadObject bool `config:"no_head_object"`
- Enc encoder.MultiEncoder `config:"encoding"`
- DisableHTTP2 bool `config:"disable_http2"`
- DownloadURL string `config:"download_url"`
- DirectoryMarkers bool `config:"directory_markers"`
- UseMultipartEtag fs.Tristate `config:"use_multipart_etag"`
- UsePresignedRequest bool `config:"use_presigned_request"`
- Versions bool `config:"versions"`
- VersionAt fs.Time `config:"version_at"`
- VersionDeleted bool `config:"version_deleted"`
- Decompress bool `config:"decompress"`
- MightGzip fs.Tristate `config:"might_gzip"`
- UseAcceptEncodingGzip fs.Tristate `config:"use_accept_encoding_gzip"`
- NoSystemMetadata bool `config:"no_system_metadata"`
- UseAlreadyExists fs.Tristate `config:"use_already_exists"`
- UseMultipartUploads fs.Tristate `config:"use_multipart_uploads"`
- UseUnsignedPayload fs.Tristate `config:"use_unsigned_payload"`
- SDKLogMode sdkLogMode `config:"sdk_log_mode"`
- DirectoryBucket bool `config:"directory_bucket"`
- IBMAPIKey string `config:"ibm_api_key"`
- IBMInstanceID string `config:"ibm_resource_instance_id"`
- UseXID fs.Tristate `config:"use_x_id"`
- SignAcceptEncoding fs.Tristate `config:"sign_accept_encoding"`
+ Provider string `config:"provider"`
+ EnvAuth bool `config:"env_auth"`
+ AccessKeyID string `config:"access_key_id"`
+ SecretAccessKey string `config:"secret_access_key"`
+ Region string `config:"region"`
+ Endpoint string `config:"endpoint"`
+ STSEndpoint string `config:"sts_endpoint"`
+ UseDualStack bool `config:"use_dual_stack"`
+ LocationConstraint string `config:"location_constraint"`
+ ACL string `config:"acl"`
+ BucketACL string `config:"bucket_acl"`
+ RequesterPays bool `config:"requester_pays"`
+ ServerSideEncryption string `config:"server_side_encryption"`
+ SSEKMSKeyID string `config:"sse_kms_key_id"`
+ SSECustomerAlgorithm string `config:"sse_customer_algorithm"`
+ SSECustomerKey string `config:"sse_customer_key"`
+ SSECustomerKeyBase64 string `config:"sse_customer_key_base64"`
+ SSECustomerKeyMD5 string `config:"sse_customer_key_md5"`
+ StorageClass string `config:"storage_class"`
+ UploadCutoff fs.SizeSuffix `config:"upload_cutoff"`
+ CopyCutoff fs.SizeSuffix `config:"copy_cutoff"`
+ ChunkSize fs.SizeSuffix `config:"chunk_size"`
+ MaxUploadParts int `config:"max_upload_parts"`
+ DisableChecksum bool `config:"disable_checksum"`
+ SharedCredentialsFile string `config:"shared_credentials_file"`
+ Profile string `config:"profile"`
+ SessionToken string `config:"session_token"`
+ UploadConcurrency int `config:"upload_concurrency"`
+ ForcePathStyle bool `config:"force_path_style"`
+ V2Auth bool `config:"v2_auth"`
+ UseAccelerateEndpoint bool `config:"use_accelerate_endpoint"`
+ UseARNRegion bool `config:"use_arn_region"`
+ LeavePartsOnError bool `config:"leave_parts_on_error"`
+ ListChunk int32 `config:"list_chunk"`
+ ListVersion int `config:"list_version"`
+ ListURLEncode fs.Tristate `config:"list_url_encode"`
+ NoCheckBucket bool `config:"no_check_bucket"`
+ NoHead bool `config:"no_head"`
+ NoHeadObject bool `config:"no_head_object"`
+ Enc encoder.MultiEncoder `config:"encoding"`
+ DisableHTTP2 bool `config:"disable_http2"`
+ DownloadURL string `config:"download_url"`
+ DirectoryMarkers bool `config:"directory_markers"`
+ UseMultipartEtag fs.Tristate `config:"use_multipart_etag"`
+ UsePresignedRequest bool `config:"use_presigned_request"`
+ UseDataIntegrityProtections fs.Tristate `config:"use_data_integrity_protections"`
+ Versions bool `config:"versions"`
+ VersionAt fs.Time `config:"version_at"`
+ VersionDeleted bool `config:"version_deleted"`
+ Decompress bool `config:"decompress"`
+ MightGzip fs.Tristate `config:"might_gzip"`
+ UseAcceptEncodingGzip fs.Tristate `config:"use_accept_encoding_gzip"`
+ NoSystemMetadata bool `config:"no_system_metadata"`
+ UseAlreadyExists fs.Tristate `config:"use_already_exists"`
+ UseMultipartUploads fs.Tristate `config:"use_multipart_uploads"`
+ UseUnsignedPayload fs.Tristate `config:"use_unsigned_payload"`
+ SDKLogMode sdkLogMode `config:"sdk_log_mode"`
+ DirectoryBucket bool `config:"directory_bucket"`
+ IBMAPIKey string `config:"ibm_api_key"`
+ IBMInstanceID string `config:"ibm_resource_instance_id"`
+ UseXID fs.Tristate `config:"use_x_id"`
+ SignAcceptEncoding fs.Tristate `config:"sign_accept_encoding"`
}
// Fs represents a remote s3 server
@@ -1302,6 +1311,10 @@ func s3Connection(ctx context.Context, opt *Options, client *http.Client) (s3Cli
} else {
s3Opt.EndpointOptions.UseDualStackEndpoint = aws.DualStackEndpointStateDisabled
}
+ if !opt.UseDataIntegrityProtections.Value {
+ s3Opt.RequestChecksumCalculation = aws.RequestChecksumCalculationWhenRequired
+ s3Opt.ResponseChecksumValidation = aws.ResponseChecksumValidationWhenRequired
+ }
// FIXME not ported from SDK v1 - not sure what this does
// s3Opt.UsEast1RegionalEndpoint = endpoints.RegionalS3UsEast1Endpoint
})
@@ -1497,6 +1510,7 @@ func setQuirks(opt *Options, provider *Provider) {
set(&opt.ListURLEncode, true, provider.Quirks.ListURLEncode)
set(&opt.UseMultipartEtag, true, provider.Quirks.UseMultipartEtag)
set(&opt.UseAcceptEncodingGzip, true, provider.Quirks.UseAcceptEncodingGzip)
+ set(&opt.UseDataIntegrityProtections, false, provider.Quirks.UseDataIntegrityProtections)
set(&opt.MightGzip, true, provider.Quirks.MightGzip)
set(&opt.UseAlreadyExists, true, provider.Quirks.UseAlreadyExists)
set(&opt.UseMultipartUploads, true, provider.Quirks.UseMultipartUploads)