"rsync for cloud storage" - Google Drive, S3, Dropbox, Backblaze B2, One Drive, Swift, Hubic, Wasabi, Google Cloud Storage, Azure Blob, Azure Files, Yandex Files
Go azure-blobazure-blob-storageazure-filesbackblaze-b2cloud-storagedropboxencryptionftpfuse-filesystemgogolanggoogle-cloud-storagegoogle-driveonedriveopenstack-swiftrclones3sftpsyncwebdav
b2: support authentication with new bucket restricted application keys
Diana3c40238f020d
diff --git a/backend/b2/api/types.go b/backend/b2/api/types.goindex 304797cdc..13982dfc9 100644--- a/backend/b2/api/types.go+++ b/backend/b2/api/types.go@@ -133,23 +133,32 @@ type File struct {Info map[string]string `json:"fileInfo"` // The custom information that was uploaded with the file. This is a JSON object, holding the name/value pairs that were uploaded with the file.}-// AuthorizeAccountResponse is as returned from the b2_authorize_account call-type AuthorizeAccountResponse struct {+// StorageAPI is as returned from the b2_authorize_account call+type StorageAPI struct {AbsoluteMinimumPartSize int `json:"absoluteMinimumPartSize"` // The smallest possible size of a part of a large file.- AccountID string `json:"accountId"` // The identifier for the account.Allowed struct { // An object (see below) containing the capabilities of this auth token, and any restrictions on using it.- BucketID string `json:"bucketId"` // When present, access is restricted to one bucket.- BucketName string `json:"bucketName"` // When present, name of bucket - may be empty- Capabilities []string `json:"capabilities"` // A list of strings, each one naming a capability the key has.+ Buckets []struct { // When present, access is restricted to one or more buckets.+ ID string `json:"id"` // ID of bucket+ Name string `json:"name"` // When present, name of bucket - may be empty+ } `json:"buckets"`+ Capabilities []string `json:"capabilities"` // A list of strings, each one naming a capability the key has for every bucket.NamePrefix any `json:"namePrefix"` // When present, access is restricted to files whose names start with the prefix} `json:"allowed"`APIURL string `json:"apiUrl"` // The base URL to use for all API calls except for uploading and downloading files.- AuthorizationToken string `json:"authorizationToken"` // An authorization token to use with all calls, other than b2_authorize_account, that need an Authorization header.DownloadURL string `json:"downloadUrl"` // The base URL to use for downloading files.MinimumPartSize int `json:"minimumPartSize"` // DEPRECATED: This field will always have the same value as recommendedPartSize. Use recommendedPartSize instead.RecommendedPartSize int `json:"recommendedPartSize"` // The recommended size for each part of a large file. We recommend using this part size for optimal upload performance.}+// AuthorizeAccountResponse is as returned from the b2_authorize_account call+type AuthorizeAccountResponse struct {+ AccountID string `json:"accountId"` // The identifier for the account.+ AuthorizationToken string `json:"authorizationToken"` // An authorization token to use with all calls, other than b2_authorize_account, that need an Authorization header.+ APIs struct { // Supported APIs for this account / key. These are API-dependent JSON objects.+ Storage StorageAPI `json:"storageApi"`+ } `json:"apiInfo"`+}+// ListBucketsRequest is parameters for b2_list_buckets calltype ListBucketsRequest struct {AccountID string `json:"accountId"` // The identifier for the account.diff --git a/backend/b2/b2.go b/backend/b2/b2.goindex 0e409e73b..a21d63dac 100644--- a/backend/b2/b2.go+++ b/backend/b2/b2.go@@ -607,17 +607,29 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, eif err != nil {return nil, fmt.Errorf("failed to authorize account: %w", err)}- // If this is a key limited to a single bucket, it must exist already- if f.rootBucket != "" && f.info.Allowed.BucketID != "" {- allowedBucket := f.opt.Enc.ToStandardName(f.info.Allowed.BucketName)- if allowedBucket == "" {- return nil, errors.New("bucket that application key is restricted to no longer exists")+ // If this is a key limited to one or more buckets, one of them must exist+ // and be ours.+ if f.rootBucket != "" && len(f.info.APIs.Storage.Allowed.Buckets) != 0 {+ buckets := f.info.APIs.Storage.Allowed.Buckets+ var rootFound = false+ var rootID string+ for _, b := range buckets {+ allowedBucket := f.opt.Enc.ToStandardName(b.Name)+ if allowedBucket == "" {+ fs.Debugf(f, "bucket %q that application key is restricted to no longer exists", b.ID)+ continue+ }++ if allowedBucket == f.rootBucket {+ rootFound = true+ rootID = b.ID+ }}- if allowedBucket != f.rootBucket {- return nil, fmt.Errorf("you must use bucket %q with this application key", allowedBucket)+ if !rootFound {+ return nil, fmt.Errorf("you must use bucket(s) %q with this application key", buckets)}f.cache.MarkOK(f.rootBucket)- f.setBucketID(f.rootBucket, f.info.Allowed.BucketID)+ f.setBucketID(f.rootBucket, rootID)}if f.rootBucket != "" && f.rootDirectory != "" {// Check to see if the (bucket,directory) is actually an existing file@@ -643,7 +655,7 @@ func (f *Fs) authorizeAccount(ctx context.Context) error {defer f.authMu.Unlock()opts := rest.Opts{Method: "GET",- Path: "/b2api/v1/b2_authorize_account",+ Path: "/b2api/v4/b2_authorize_account",RootURL: f.opt.Endpoint,UserName: f.opt.Account,Password: f.opt.Key,@@ -656,13 +668,13 @@ func (f *Fs) authorizeAccount(ctx context.Context) error {if err != nil {return fmt.Errorf("failed to authenticate: %w", err)}- f.srv.SetRoot(f.info.APIURL+"/b2api/v1").SetHeader("Authorization", f.info.AuthorizationToken)+ f.srv.SetRoot(f.info.APIs.Storage.APIURL+"/b2api/v1").SetHeader("Authorization", f.info.AuthorizationToken)return nil}// hasPermission returns if the current AuthorizationToken has the selected permissionfunc (f *Fs) hasPermission(permission string) bool {- return slices.Contains(f.info.Allowed.Capabilities, permission)+ return slices.Contains(f.info.APIs.Storage.Allowed.Capabilities, permission)}// getUploadURL returns the upload info with the UploadURL and the AuthorizationToken@@ -1067,44 +1079,68 @@ type listBucketFn func(*api.Bucket) error// listBucketsToFn lists the buckets to the function suppliedfunc (f *Fs) listBucketsToFn(ctx context.Context, bucketName string, fn listBucketFn) error {- var account = api.ListBucketsRequest{- AccountID: f.info.AccountID,- BucketID: f.info.Allowed.BucketID,- }- if bucketName != "" && account.BucketID == "" {- account.BucketName = f.opt.Enc.FromStandardName(bucketName)- }+ responses := make([]api.ListBucketsResponse, len(f.info.APIs.Storage.Allowed.Buckets))[:0]- var response api.ListBucketsResponse- opts := rest.Opts{- Method: "POST",- Path: "/b2_list_buckets",- }- err := f.pacer.Call(func() (bool, error) {- resp, err := f.srv.CallJSON(ctx, &opts, &account, &response)- return f.shouldRetry(ctx, resp, err)- })- if err != nil {- return err+ for i := range f.info.APIs.Storage.Allowed.Buckets {+ b := &f.info.APIs.Storage.Allowed.Buckets[i]+ // Empty names indicate a bucket that no longer exists, this is non-fatal+ // for multi-bucket API keys.+ if b.Name == "" {+ continue+ }+ // When requesting a specific bucket skip over non-matching names+ if bucketName != "" && b.Name != bucketName {+ continue+ }++ var account = api.ListBucketsRequest{+ AccountID: f.info.AccountID,+ BucketID: b.ID,+ }+ if bucketName != "" && account.BucketID == "" {+ account.BucketName = f.opt.Enc.FromStandardName(bucketName)+ }++ var response api.ListBucketsResponse+ opts := rest.Opts{+ Method: "POST",+ Path: "/b2_list_buckets",+ }+ err := f.pacer.Call(func() (bool, error) {+ resp, err := f.srv.CallJSON(ctx, &opts, &account, &response)+ return f.shouldRetry(ctx, resp, err)+ })+ if err != nil {+ return err+ }+ responses = append(responses, response)}+f.bucketIDMutex.Lock()f.bucketTypeMutex.Lock()f._bucketID = make(map[string]string, 1)f._bucketType = make(map[string]string, 1)- for i := range response.Buckets {- bucket := &response.Buckets[i]- bucket.Name = f.opt.Enc.ToStandardName(bucket.Name)- f.cache.MarkOK(bucket.Name)- f._bucketID[bucket.Name] = bucket.ID- f._bucketType[bucket.Name] = bucket.Type++ for ri := range responses {+ response := &responses[ri]+ for i := range response.Buckets {+ bucket := &response.Buckets[i]+ bucket.Name = f.opt.Enc.ToStandardName(bucket.Name)+ f.cache.MarkOK(bucket.Name)+ f._bucketID[bucket.Name] = bucket.ID+ f._bucketType[bucket.Name] = bucket.Type+ }}f.bucketTypeMutex.Unlock()f.bucketIDMutex.Unlock()- for i := range response.Buckets {- bucket := &response.Buckets[i]- err = fn(bucket)- if err != nil {- return err+ for ri := range responses {+ response := &responses[ri]+ for i := range response.Buckets {+ bucket := &response.Buckets[i]+ err := fn(bucket)+ if err != nil {+ return err+ }}}return nil@@ -1606,7 +1642,7 @@ func (f *Fs) PublicLink(ctx context.Context, remote string, expire fs.Duration,bucket, bucketPath := f.split(remote)var RootURL stringif f.opt.DownloadURL == "" {- RootURL = f.info.DownloadURL+ RootURL = f.info.APIs.Storage.DownloadURL} else {RootURL = f.opt.DownloadURL}@@ -1957,7 +1993,7 @@ func (o *Object) getOrHead(ctx context.Context, method string, options []fs.Open// Use downloadUrl from backblaze if downloadUrl is not set// otherwise use the custom downloadUrlif o.fs.opt.DownloadURL == "" {- opts.RootURL = o.fs.info.DownloadURL+ opts.RootURL = o.fs.info.APIs.Storage.DownloadURL} else {opts.RootURL = o.fs.opt.DownloadURL}diff --git a/docs/content/b2.md b/docs/content/b2.mdindex eb283dff9..539bb4ec4 100644--- a/docs/content/b2.md+++ b/docs/content/b2.md@@ -283,7 +283,7 @@ It is useful to know how many requests are sent to the server in different scenaAll copy commands send the following 4 requests:```text-/b2api/v1/b2_authorize_account+/b2api/v4/b2_authorize_account/b2api/v1/b2_create_bucket/b2api/v1/b2_list_buckets/b2api/v1/b2_list_file_names