"rsync for cloud storage" - Google Drive, S3, Dropbox, Backblaze B2, One Drive, Swift, Hubic, Wasabi, Google Cloud Storage, Azure Blob, Azure Files, Yandex Files
Go azure-blobazure-blob-storageazure-filesbackblaze-b2cloud-storagedropboxencryptionftpfuse-filesystemgogolanggoogle-cloud-storagegoogle-driveonedriveopenstack-swiftrclones3sftpsyncwebdav
s3: add --s3-use-data-integrity-protections to fix BadDigest error in Alibaba, Tencent
hunshcn16971ab6b9c3
diff --git a/backend/s3/provider/AWS.yaml b/backend/s3/provider/AWS.yamlindex 39c12ae08..989462d32 100644--- a/backend/s3/provider/AWS.yaml+++ b/backend/s3/provider/AWS.yaml@@ -137,3 +137,4 @@ use_accelerate_endpoint: truequirks:might_gzip: false # Never auto gzips objectsuse_unsigned_payload: false # AWS has trailer support which means it adds checksums in the trailer without seeking+ use_data_integrity_protections: truediff --git a/backend/s3/providers.go b/backend/s3/providers.goindex 6c9eb8a17..858aaea6e 100644--- a/backend/s3/providers.go+++ b/backend/s3/providers.go@@ -20,20 +20,21 @@ var NewYamlMap = orderedmap.New[string, string]// Quirks defines all the S3 provider quirkstype Quirks struct {- ListVersion *int `yaml:"list_version,omitempty"` // 1 or 2- ForcePathStyle *bool `yaml:"force_path_style,omitempty"` // true = path-style- ListURLEncode *bool `yaml:"list_url_encode,omitempty"`- UseMultipartEtag *bool `yaml:"use_multipart_etag,omitempty"`- UseAlreadyExists *bool `yaml:"use_already_exists,omitempty"`- UseAcceptEncodingGzip *bool `yaml:"use_accept_encoding_gzip,omitempty"`- MightGzip *bool `yaml:"might_gzip,omitempty"`- UseMultipartUploads *bool `yaml:"use_multipart_uploads,omitempty"`- UseUnsignedPayload *bool `yaml:"use_unsigned_payload,omitempty"`- UseXID *bool `yaml:"use_x_id,omitempty"`- SignAcceptEncoding *bool `yaml:"sign_accept_encoding,omitempty"`- CopyCutoff *int64 `yaml:"copy_cutoff,omitempty"`- MaxUploadParts *int `yaml:"max_upload_parts,omitempty"`- MinChunkSize *int64 `yaml:"min_chunk_size,omitempty"`+ ListVersion *int `yaml:"list_version,omitempty"` // 1 or 2+ ForcePathStyle *bool `yaml:"force_path_style,omitempty"` // true = path-style+ ListURLEncode *bool `yaml:"list_url_encode,omitempty"`+ UseMultipartEtag *bool `yaml:"use_multipart_etag,omitempty"`+ UseAlreadyExists *bool `yaml:"use_already_exists,omitempty"`+ UseAcceptEncodingGzip *bool `yaml:"use_accept_encoding_gzip,omitempty"`+ UseDataIntegrityProtections *bool `yaml:"use_data_integrity_protections,omitempty"`+ MightGzip *bool `yaml:"might_gzip,omitempty"`+ UseMultipartUploads *bool `yaml:"use_multipart_uploads,omitempty"`+ UseUnsignedPayload *bool `yaml:"use_unsigned_payload,omitempty"`+ UseXID *bool `yaml:"use_x_id,omitempty"`+ SignAcceptEncoding *bool `yaml:"sign_accept_encoding,omitempty"`+ CopyCutoff *int64 `yaml:"copy_cutoff,omitempty"`+ MaxUploadParts *int `yaml:"max_upload_parts,omitempty"`+ MinChunkSize *int64 `yaml:"min_chunk_size,omitempty"`}// Provider defines the configurable data in each provider.yamldiff --git a/backend/s3/s3.go b/backend/s3/s3.goindex 137b31adb..c12a97619 100644--- a/backend/s3/s3.go+++ b/backend/s3/s3.go@@ -39,6 +39,9 @@ import (smithyhttp "github.com/aws/smithy-go/transport/http""github.com/ncw/swift/v2"+ "golang.org/x/net/http/httpguts"+ "golang.org/x/sync/errgroup"+"github.com/rclone/rclone/fs""github.com/rclone/rclone/fs/accounting""github.com/rclone/rclone/fs/chunksize"@@ -59,8 +62,6 @@ import ("github.com/rclone/rclone/lib/readers""github.com/rclone/rclone/lib/rest""github.com/rclone/rclone/lib/version"- "golang.org/x/net/http/httpguts"- "golang.org/x/sync/errgroup")// Register with Fs@@ -574,6 +575,13 @@ circumstances or for testing.`,Default: false,Advanced: true,+ }, {+ Name: "use_data_integrity_protections",+ Help: `If true use AWS S3 data integrity protections.++See [AWS Docs on Data Integrity Protections](https://docs.aws.amazon.com/sdkref/latest/guide/feature-dataintegrity.html)`,+ Default: fs.Tristate{},+ Advanced: true,}, {Name: "versions",Help: "Include old versions in directory listings.",@@ -892,67 +900,68 @@ var systemMetadataInfo = map[string]fs.MetadataHelp{// Options defines the configuration for this backendtype Options struct {- Provider string `config:"provider"`- EnvAuth bool `config:"env_auth"`- AccessKeyID string `config:"access_key_id"`- SecretAccessKey string `config:"secret_access_key"`- Region string `config:"region"`- Endpoint string `config:"endpoint"`- STSEndpoint string `config:"sts_endpoint"`- UseDualStack bool `config:"use_dual_stack"`- LocationConstraint string `config:"location_constraint"`- ACL string `config:"acl"`- BucketACL string `config:"bucket_acl"`- RequesterPays bool `config:"requester_pays"`- ServerSideEncryption string `config:"server_side_encryption"`- SSEKMSKeyID string `config:"sse_kms_key_id"`- SSECustomerAlgorithm string `config:"sse_customer_algorithm"`- SSECustomerKey string `config:"sse_customer_key"`- SSECustomerKeyBase64 string `config:"sse_customer_key_base64"`- SSECustomerKeyMD5 string `config:"sse_customer_key_md5"`- StorageClass string `config:"storage_class"`- UploadCutoff fs.SizeSuffix `config:"upload_cutoff"`- CopyCutoff fs.SizeSuffix `config:"copy_cutoff"`- ChunkSize fs.SizeSuffix `config:"chunk_size"`- MaxUploadParts int `config:"max_upload_parts"`- DisableChecksum bool `config:"disable_checksum"`- SharedCredentialsFile string `config:"shared_credentials_file"`- Profile string `config:"profile"`- SessionToken string `config:"session_token"`- UploadConcurrency int `config:"upload_concurrency"`- ForcePathStyle bool `config:"force_path_style"`- V2Auth bool `config:"v2_auth"`- UseAccelerateEndpoint bool `config:"use_accelerate_endpoint"`- UseARNRegion bool `config:"use_arn_region"`- LeavePartsOnError bool `config:"leave_parts_on_error"`- ListChunk int32 `config:"list_chunk"`- ListVersion int `config:"list_version"`- ListURLEncode fs.Tristate `config:"list_url_encode"`- NoCheckBucket bool `config:"no_check_bucket"`- NoHead bool `config:"no_head"`- NoHeadObject bool `config:"no_head_object"`- Enc encoder.MultiEncoder `config:"encoding"`- DisableHTTP2 bool `config:"disable_http2"`- DownloadURL string `config:"download_url"`- DirectoryMarkers bool `config:"directory_markers"`- UseMultipartEtag fs.Tristate `config:"use_multipart_etag"`- UsePresignedRequest bool `config:"use_presigned_request"`- Versions bool `config:"versions"`- VersionAt fs.Time `config:"version_at"`- VersionDeleted bool `config:"version_deleted"`- Decompress bool `config:"decompress"`- MightGzip fs.Tristate `config:"might_gzip"`- UseAcceptEncodingGzip fs.Tristate `config:"use_accept_encoding_gzip"`- NoSystemMetadata bool `config:"no_system_metadata"`- UseAlreadyExists fs.Tristate `config:"use_already_exists"`- UseMultipartUploads fs.Tristate `config:"use_multipart_uploads"`- UseUnsignedPayload fs.Tristate `config:"use_unsigned_payload"`- SDKLogMode sdkLogMode `config:"sdk_log_mode"`- DirectoryBucket bool `config:"directory_bucket"`- IBMAPIKey string `config:"ibm_api_key"`- IBMInstanceID string `config:"ibm_resource_instance_id"`- UseXID fs.Tristate `config:"use_x_id"`- SignAcceptEncoding fs.Tristate `config:"sign_accept_encoding"`+ Provider string `config:"provider"`+ EnvAuth bool `config:"env_auth"`+ AccessKeyID string `config:"access_key_id"`+ SecretAccessKey string `config:"secret_access_key"`+ Region string `config:"region"`+ Endpoint string `config:"endpoint"`+ STSEndpoint string `config:"sts_endpoint"`+ UseDualStack bool `config:"use_dual_stack"`+ LocationConstraint string `config:"location_constraint"`+ ACL string `config:"acl"`+ BucketACL string `config:"bucket_acl"`+ RequesterPays bool `config:"requester_pays"`+ ServerSideEncryption string `config:"server_side_encryption"`+ SSEKMSKeyID string `config:"sse_kms_key_id"`+ SSECustomerAlgorithm string `config:"sse_customer_algorithm"`+ SSECustomerKey string `config:"sse_customer_key"`+ SSECustomerKeyBase64 string `config:"sse_customer_key_base64"`+ SSECustomerKeyMD5 string `config:"sse_customer_key_md5"`+ StorageClass string `config:"storage_class"`+ UploadCutoff fs.SizeSuffix `config:"upload_cutoff"`+ CopyCutoff fs.SizeSuffix `config:"copy_cutoff"`+ ChunkSize fs.SizeSuffix `config:"chunk_size"`+ MaxUploadParts int `config:"max_upload_parts"`+ DisableChecksum bool `config:"disable_checksum"`+ SharedCredentialsFile string `config:"shared_credentials_file"`+ Profile string `config:"profile"`+ SessionToken string `config:"session_token"`+ UploadConcurrency int `config:"upload_concurrency"`+ ForcePathStyle bool `config:"force_path_style"`+ V2Auth bool `config:"v2_auth"`+ UseAccelerateEndpoint bool `config:"use_accelerate_endpoint"`+ UseARNRegion bool `config:"use_arn_region"`+ LeavePartsOnError bool `config:"leave_parts_on_error"`+ ListChunk int32 `config:"list_chunk"`+ ListVersion int `config:"list_version"`+ ListURLEncode fs.Tristate `config:"list_url_encode"`+ NoCheckBucket bool `config:"no_check_bucket"`+ NoHead bool `config:"no_head"`+ NoHeadObject bool `config:"no_head_object"`+ Enc encoder.MultiEncoder `config:"encoding"`+ DisableHTTP2 bool `config:"disable_http2"`+ DownloadURL string `config:"download_url"`+ DirectoryMarkers bool `config:"directory_markers"`+ UseMultipartEtag fs.Tristate `config:"use_multipart_etag"`+ UsePresignedRequest bool `config:"use_presigned_request"`+ UseDataIntegrityProtections fs.Tristate `config:"use_data_integrity_protections"`+ Versions bool `config:"versions"`+ VersionAt fs.Time `config:"version_at"`+ VersionDeleted bool `config:"version_deleted"`+ Decompress bool `config:"decompress"`+ MightGzip fs.Tristate `config:"might_gzip"`+ UseAcceptEncodingGzip fs.Tristate `config:"use_accept_encoding_gzip"`+ NoSystemMetadata bool `config:"no_system_metadata"`+ UseAlreadyExists fs.Tristate `config:"use_already_exists"`+ UseMultipartUploads fs.Tristate `config:"use_multipart_uploads"`+ UseUnsignedPayload fs.Tristate `config:"use_unsigned_payload"`+ SDKLogMode sdkLogMode `config:"sdk_log_mode"`+ DirectoryBucket bool `config:"directory_bucket"`+ IBMAPIKey string `config:"ibm_api_key"`+ IBMInstanceID string `config:"ibm_resource_instance_id"`+ UseXID fs.Tristate `config:"use_x_id"`+ SignAcceptEncoding fs.Tristate `config:"sign_accept_encoding"`}// Fs represents a remote s3 server@@ -1302,6 +1311,10 @@ func s3Connection(ctx context.Context, opt *Options, client *http.Client) (s3Cli} else {s3Opt.EndpointOptions.UseDualStackEndpoint = aws.DualStackEndpointStateDisabled}+ if !opt.UseDataIntegrityProtections.Value {+ s3Opt.RequestChecksumCalculation = aws.RequestChecksumCalculationWhenRequired+ s3Opt.ResponseChecksumValidation = aws.ResponseChecksumValidationWhenRequired+ }// FIXME not ported from SDK v1 - not sure what this does// s3Opt.UsEast1RegionalEndpoint = endpoints.RegionalS3UsEast1Endpoint})@@ -1497,6 +1510,7 @@ func setQuirks(opt *Options, provider *Provider) {set(&opt.ListURLEncode, true, provider.Quirks.ListURLEncode)set(&opt.UseMultipartEtag, true, provider.Quirks.UseMultipartEtag)set(&opt.UseAcceptEncodingGzip, true, provider.Quirks.UseAcceptEncodingGzip)+ set(&opt.UseDataIntegrityProtections, false, provider.Quirks.UseDataIntegrityProtections)set(&opt.MightGzip, true, provider.Quirks.MightGzip)set(&opt.UseAlreadyExists, true, provider.Quirks.UseAlreadyExists)set(&opt.UseMultipartUploads, true, provider.Quirks.UseMultipartUploads)