"rsync for cloud storage" - Google Drive, S3, Dropbox, Backblaze B2, One Drive, Swift, Hubic, Wasabi, Google Cloud Storage, Azure Blob, Azure Files, Yandex Files
Go azure-blobazure-blob-storageazure-filesbackblaze-b2cloud-storagedropboxencryptionftpfuse-filesystemgogolanggoogle-cloud-storagegoogle-driveonedriveopenstack-swiftrclones3sftpsyncwebdav
b2: Add Server-Side encryption support
fries123494deb6bd6f3f
diff --git a/backend/b2/api/types.go b/backend/b2/api/types.goindex b221d4c2d..ab4b221ec 100644--- a/backend/b2/api/types.go+++ b/backend/b2/api/types.go@@ -48,6 +48,14 @@ type LifecycleRule struct {FileNamePrefix string `json:"fileNamePrefix"`}+// ServerSideEncryption is a configuration object for B2 Server-Side Encryption+type ServerSideEncryption struct {+ Mode string `json:"mode"`+ Algorithm string `json:"algorithm"` // Encryption algorith to use+ CustomerKey string `json:"customerKey"` // User provided Base64 encoded key that is used by the server to encrypt files+ CustomerKeyMd5 string `json:"customerKeyMd5"` // An MD5 hash of the decoded key+}+// Timestamp is a UTC time when this file was uploaded. It is a base// 10 number of milliseconds since midnight, January 1, 1970 UTC. This// fits in a 64 bit integer such as the type "long" in the programming@@ -261,21 +269,22 @@ type GetFileInfoRequest struct {//// Example: { "src_last_modified_millis" : "1452802803026", "large_file_sha1" : "a3195dc1e7b46a2ff5da4b3c179175b75671e80d", "color": "blue" }type StartLargeFileRequest struct {- BucketID string `json:"bucketId"` //The ID of the bucket that the file will go in.- Name string `json:"fileName"` // The name of the file. See Files for requirements on file names.- ContentType string `json:"contentType"` // The MIME type of the content of the file, which will be returned in the Content-Type header when downloading the file. Use the Content-Type b2/x-auto to automatically set the stored Content-Type post upload. In the case where a file extension is absent or the lookup fails, the Content-Type is set to application/octet-stream.- Info map[string]string `json:"fileInfo"` // A JSON object holding the name/value pairs for the custom file info.+ BucketID string `json:"bucketId"` // The ID of the bucket that the file will go in.+ Name string `json:"fileName"` // The name of the file. See Files for requirements on file names.+ ContentType string `json:"contentType"` // The MIME type of the content of the file, which will be returned in the Content-Type header when downloading the file. Use the Content-Type b2/x-auto to automatically set the stored Content-Type post upload. In the case where a file extension is absent or the lookup fails, the Content-Type is set to application/octet-stream.+ Info map[string]string `json:"fileInfo"` // A JSON object holding the name/value pairs for the custom file info.+ ServerSideEncryption ServerSideEncryption `json:"serverSideEncryption,omitempty"` // A JSON object holding values related to Server-Side Encryption}// StartLargeFileResponse is the response to StartLargeFileRequesttype StartLargeFileResponse struct {- ID string `json:"fileId"` // The unique identifier for this version of this file. Used with b2_get_file_info, b2_download_file_by_id, and b2_delete_file_version.- Name string `json:"fileName"` // The name of this file, which can be used with b2_download_file_by_name.- AccountID string `json:"accountId"` // The identifier for the account.- BucketID string `json:"bucketId"` // The unique ID of the bucket.- ContentType string `json:"contentType"` // The MIME type of the file.- Info map[string]string `json:"fileInfo"` // The custom information that was uploaded with the file. This is a JSON object, holding the name/value pairs that were uploaded with the file.- UploadTimestamp Timestamp `json:"uploadTimestamp"` // This is a UTC time when this file was uploaded.+ ID string `json:"fileId"` // The unique identifier for this version of this file. Used with b2_get_file_info, b2_download_file_by_id, and b2_delete_file_version.+ Name string `json:"fileName"` // The name of this file, which can be used with b2_download_file_by_name.+ AccountID string `json:"accountId"` // The identifier for the account.+ BucketID string `json:"bucketId"` // The unique ID of the bucket.+ ContentType string `json:"contentType"` // The MIME type of the file.+ Info map[string]string `json:"fileInfo"` // The custom information that was uploaded with the file. This is a JSON object, holding the name/value pairs that were uploaded with the file.+ UploadTimestamp Timestamp `json:"uploadTimestamp,omitempty"` // This is a UTC time when this file was uploaded.}// GetUploadPartURLRequest is passed to b2_get_upload_part_url@@ -325,21 +334,25 @@ type CancelLargeFileResponse struct {// CopyFileRequest is as passed to b2_copy_filetype CopyFileRequest struct {- SourceID string `json:"sourceFileId"` // The ID of the source file being copied.- Name string `json:"fileName"` // The name of the new file being created.- Range string `json:"range,omitempty"` // The range of bytes to copy. If not provided, the whole source file will be copied.- MetadataDirective string `json:"metadataDirective,omitempty"` // The strategy for how to populate metadata for the new file: COPY or REPLACE- ContentType string `json:"contentType,omitempty"` // The MIME type of the content of the file (REPLACE only)- Info map[string]string `json:"fileInfo,omitempty"` // This field stores the metadata that will be stored with the file. (REPLACE only)- DestBucketID string `json:"destinationBucketId,omitempty"` // The destination ID of the bucket if set, if not the source bucket will be used+ SourceID string `json:"sourceFileId"` // The ID of the source file being copied.+ Name string `json:"fileName"` // The name of the new file being created.+ Range string `json:"range,omitempty"` // The range of bytes to copy. If not provided, the whole source file will be copied.+ MetadataDirective string `json:"metadataDirective,omitempty"` // The strategy for how to populate metadata for the new file: COPY or REPLACE+ ContentType string `json:"contentType,omitempty"` // The MIME type of the content of the file (REPLACE only)+ Info map[string]string `json:"fileInfo,omitempty"` // This field stores the metadata that will be stored with the file. (REPLACE only)+ DestBucketID string `json:"destinationBucketId,omitempty"` // The destination ID of the bucket if set, if not the source bucket will be used+ SourceServerSideEncryption ServerSideEncryption `json:"sourceServerSideEncryption,omitempty"` // A JSON object holding values related to Server-Side Encryption for the source file+ DestinationServerSideEncryption ServerSideEncryption `json:"destinationServerSideEncryption,omitempty"` // A JSON object holding values related to Server-Side Encryption for the destination file}// CopyPartRequest is the request for b2_copy_part - the response is UploadPartResponsetype CopyPartRequest struct {- SourceID string `json:"sourceFileId"` // The ID of the source file being copied.- LargeFileID string `json:"largeFileId"` // The ID of the large file the part will belong to, as returned by b2_start_large_file.- PartNumber int64 `json:"partNumber"` // Which part this is (starting from 1)- Range string `json:"range,omitempty"` // The range of bytes to copy. If not provided, the whole source file will be copied.+ SourceID string `json:"sourceFileId"` // The ID of the source file being copied.+ LargeFileID string `json:"largeFileId"` // The ID of the large file the part will belong to, as returned by b2_start_large_file.+ PartNumber int64 `json:"partNumber"` // Which part this is (starting from 1)+ Range string `json:"range,omitempty"` // The range of bytes to copy. If not provided, the whole source file will be copied.+ SourceServerSideEncryption ServerSideEncryption `json:"sourceServerSideEncryption,omitempty"` // A JSON object holding values related to Server-Side Encryption for the source file+ DestinationServerSideEncryption ServerSideEncryption `json:"destinationServerSideEncryption,omitempty"` // A JSON object holding values related to Server-Side Encryption for the destination file}// UpdateBucketRequest describes a request to modify a B2 bucketdiff --git a/backend/b2/b2.go b/backend/b2/b2.goindex 70e4c928a..4bf4a2a5a 100644--- a/backend/b2/b2.go+++ b/backend/b2/b2.go@@ -8,7 +8,9 @@ import ("bufio""bytes""context"+ "crypto/md5""crypto/sha1"+ "encoding/base64""encoding/json""errors""fmt"@@ -53,6 +55,9 @@ const (nameHeader = "X-Bz-File-Name"timestampHeader = "X-Bz-Upload-Timestamp"retryAfterHeader = "Retry-After"+ sseAlgorithmHeader = "X-Bz-Server-Side-Encryption-Customer-Algorithm"+ sseKeyHeader = "X-Bz-Server-Side-Encryption-Customer-Key"+ sseMd5Header = "X-Bz-Server-Side-Encryption-Customer-Key-Md5"minSleep = 10 * time.MillisecondmaxSleep = 5 * time.MinutedecayConstant = 1 // bigger for slower decay, exponential@@ -252,6 +257,51 @@ See: [rclone backend lifecycle](#lifecycle) for setting lifecycles after bucketDefault: (encoder.Display |encoder.EncodeBackSlash |encoder.EncodeInvalidUtf8),+ }, {+ Name: "sse_customer_algorithm",+ Help: "If using SSE-C, the server-side encryption algorithm used when storing this object in B2.",+ Advanced: true,+ Examples: []fs.OptionExample{{+ Value: "",+ Help: "None",+ }, {+ Value: "AES256",+ Help: "Advanced Encryption Standard (256 bits key length)",+ }},+ }, {+ Name: "sse_customer_key",+ Help: `To use SSE-C, you may provide the secret encryption key encoded in a UTF-8 compatible string to encrypt/decrypt your data++Alternatively you can provide --sse-customer-key-base64.`,+ Advanced: true,+ Examples: []fs.OptionExample{{+ Value: "",+ Help: "None",+ }},+ Sensitive: true,+ }, {+ Name: "sse_customer_key_base64",+ Help: `To use SSE-C, you may provide the secret encryption key encoded in Base64 format to encrypt/decrypt your data++Alternatively you can provide --sse-customer-key.`,+ Advanced: true,+ Examples: []fs.OptionExample{{+ Value: "",+ Help: "None",+ }},+ Sensitive: true,+ }, {+ Name: "sse_customer_key_md5",+ Help: `If using SSE-C you may provide the secret encryption key MD5 checksum (optional).++If you leave it blank, this is calculated automatically from the sse_customer_key provided.+`,+ Advanced: true,+ Examples: []fs.OptionExample{{+ Value: "",+ Help: "None",+ }},+ Sensitive: true,}},})}@@ -274,6 +324,10 @@ type Options struct {DownloadAuthorizationDuration fs.Duration `config:"download_auth_duration"`Lifecycle int `config:"lifecycle"`Enc encoder.MultiEncoder `config:"encoding"`+ SSECustomerAlgorithm string `config:"sse_customer_algorithm"`+ SSECustomerKey string `config:"sse_customer_key"`+ SSECustomerKeyBase64 string `config:"sse_customer_key_base64"`+ SSECustomerKeyMD5 string `config:"sse_customer_key_md5"`}// Fs represents a remote b2 server@@ -504,6 +558,24 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, eif opt.Endpoint == "" {opt.Endpoint = defaultEndpoint}+ if opt.SSECustomerKey != "" && opt.SSECustomerKeyBase64 != "" {+ return nil, errors.New("b2: can't use both sse_customer_key and sse_customer_key_base64 at the same time")+ } else if opt.SSECustomerKeyBase64 != "" {+ // Decode the Base64-encoded key and store it in the SSECustomerKey field+ decoded, err := base64.StdEncoding.DecodeString(opt.SSECustomerKeyBase64)+ if err != nil {+ return nil, fmt.Errorf("b2: Could not decode sse_customer_key_base64: %w", err)+ }+ opt.SSECustomerKey = string(decoded)+ } else {+ // Encode the raw key as Base64+ opt.SSECustomerKeyBase64 = base64.StdEncoding.EncodeToString([]byte(opt.SSECustomerKey))+ }+ if opt.SSECustomerKey != "" && opt.SSECustomerKeyMD5 == "" {+ // Calculate CustomerKeyMd5 if not supplied+ md5sumBinary := md5.Sum([]byte(opt.SSECustomerKey))+ opt.SSECustomerKeyMD5 = base64.StdEncoding.EncodeToString(md5sumBinary[:])+ }ci := fs.GetConfig(ctx)f := &Fs{name: name,@@ -1435,6 +1507,16 @@ func (f *Fs) copy(ctx context.Context, dstObj *Object, srcObj *Object, newInfo *Name: f.opt.Enc.FromStandardPath(dstPath),DestBucketID: destBucketID,}+ if f.opt.SSECustomerKey != "" && f.opt.SSECustomerKeyMD5 != "" {+ serverSideEncryptionConfig := api.ServerSideEncryption{+ Mode: "SSE-C",+ Algorithm: f.opt.SSECustomerAlgorithm,+ CustomerKey: f.opt.SSECustomerKeyBase64,+ CustomerKeyMd5: f.opt.SSECustomerKeyMD5,+ }+ request.SourceServerSideEncryption = serverSideEncryptionConfig+ request.DestinationServerSideEncryption = serverSideEncryptionConfig+ }if newInfo == nil {request.MetadataDirective = "COPY"} else {@@ -1866,9 +1948,10 @@ var _ io.ReadCloser = &openFile{}func (o *Object) getOrHead(ctx context.Context, method string, options []fs.OpenOption) (resp *http.Response, info *api.File, err error) {opts := rest.Opts{- Method: method,- Options: options,- NoResponse: method == "HEAD",+ Method: method,+ Options: options,+ NoResponse: method == "HEAD",+ ExtraHeaders: map[string]string{},}// Use downloadUrl from backblaze if downloadUrl is not set@@ -1886,6 +1969,11 @@ func (o *Object) getOrHead(ctx context.Context, method string, options []fs.Openbucket, bucketPath := o.split()opts.Path += "/file/" + urlEncode(o.fs.opt.Enc.FromStandardName(bucket)) + "/" + urlEncode(o.fs.opt.Enc.FromStandardPath(bucketPath))}+ if o.fs.opt.SSECustomerKey != "" && o.fs.opt.SSECustomerKeyMD5 != "" {+ opts.ExtraHeaders[sseAlgorithmHeader] = o.fs.opt.SSECustomerAlgorithm+ opts.ExtraHeaders[sseKeyHeader] = o.fs.opt.SSECustomerKeyBase64+ opts.ExtraHeaders[sseMd5Header] = o.fs.opt.SSECustomerKeyMD5+ }err = o.fs.pacer.Call(func() (bool, error) {resp, err = o.fs.srv.Call(ctx, &opts)return o.fs.shouldRetry(ctx, resp, err)@@ -2150,6 +2238,11 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op},ContentLength: &size,}+ if o.fs.opt.SSECustomerKey != "" && o.fs.opt.SSECustomerKeyMD5 != "" {+ opts.ExtraHeaders[sseAlgorithmHeader] = o.fs.opt.SSECustomerAlgorithm+ opts.ExtraHeaders[sseKeyHeader] = o.fs.opt.SSECustomerKeyBase64+ opts.ExtraHeaders[sseMd5Header] = o.fs.opt.SSECustomerKeyMD5+ }var response api.FileInfo// Don't retry, return a retry error insteaderr = o.fs.pacer.CallNoRetry(func() (bool, error) {diff --git a/backend/b2/upload.go b/backend/b2/upload.goindex e2cd4a1c2..bc35ad3c5 100644--- a/backend/b2/upload.go+++ b/backend/b2/upload.go@@ -144,6 +144,14 @@ func (f *Fs) newLargeUpload(ctx context.Context, o *Object, in io.Reader, src fsrequest.ContentType = newInfo.ContentTyperequest.Info = newInfo.Info}+ if o.fs.opt.SSECustomerKey != "" && o.fs.opt.SSECustomerKeyMD5 != "" {+ request.ServerSideEncryption = api.ServerSideEncryption{+ Mode: "SSE-C",+ Algorithm: o.fs.opt.SSECustomerAlgorithm,+ CustomerKey: o.fs.opt.SSECustomerKeyBase64,+ CustomerKeyMd5: o.fs.opt.SSECustomerKeyMD5,+ }+ }opts := rest.Opts{Method: "POST",Path: "/b2_start_large_file",@@ -295,6 +303,12 @@ func (up *largeUpload) WriteChunk(ctx context.Context, chunkNumber int, reader iContentLength: &sizeWithHash,}+ if up.o.fs.opt.SSECustomerKey != "" && up.o.fs.opt.SSECustomerKeyMD5 != "" {+ opts.ExtraHeaders[sseAlgorithmHeader] = up.o.fs.opt.SSECustomerAlgorithm+ opts.ExtraHeaders[sseKeyHeader] = up.o.fs.opt.SSECustomerKeyBase64+ opts.ExtraHeaders[sseMd5Header] = up.o.fs.opt.SSECustomerKeyMD5+ }+var response api.UploadPartResponseresp, err := up.f.srv.CallJSON(ctx, &opts, nil, &response)@@ -334,6 +348,17 @@ func (up *largeUpload) copyChunk(ctx context.Context, part int, partSize int64)PartNumber: int64(part + 1),Range: fmt.Sprintf("bytes=%d-%d", offset, offset+partSize-1),}++ if up.o.fs.opt.SSECustomerKey != "" && up.o.fs.opt.SSECustomerKeyMD5 != "" {+ serverSideEncryptionConfig := api.ServerSideEncryption{+ Mode: "SSE-C",+ Algorithm: up.o.fs.opt.SSECustomerAlgorithm,+ CustomerKey: up.o.fs.opt.SSECustomerKeyBase64,+ CustomerKeyMd5: up.o.fs.opt.SSECustomerKeyMD5,+ }+ request.SourceServerSideEncryption = serverSideEncryptionConfig+ request.DestinationServerSideEncryption = serverSideEncryptionConfig+ }var response api.UploadPartResponseresp, err := up.f.srv.CallJSON(ctx, &opts, &request, &response)retry, err := up.f.shouldRetry(ctx, resp, err)