"rsync for cloud storage" - Google Drive, S3, Dropbox, Backblaze B2, One Drive, Swift, Hubic, Wasabi, Google Cloud Storage, Azure Blob, Azure Files, Yandex Files
Go azure-blobazure-blob-storageazure-filesbackblaze-b2cloud-storagedropboxencryptionftpfuse-filesystemgogolanggoogle-cloud-storagegoogle-driveonedriveopenstack-swiftrclones3sftpsyncwebdav
cloudinary: add cloudinary backend - fixes #7989
yuval-cloudinary7e4b8e33f596
diff --git a/backend/all/all.go b/backend/all/all.goindex 1abd6770f..df6a6e45b 100644--- a/backend/all/all.go+++ b/backend/all/all.go@@ -10,6 +10,7 @@ import (_ "github.com/rclone/rclone/backend/box"_ "github.com/rclone/rclone/backend/cache"_ "github.com/rclone/rclone/backend/chunker"+ _ "github.com/rclone/rclone/backend/cloudinary"_ "github.com/rclone/rclone/backend/combine"_ "github.com/rclone/rclone/backend/compress"_ "github.com/rclone/rclone/backend/crypt"diff --git a/backend/cloudinary/api/types.go b/backend/cloudinary/api/types.gonew file mode 100644index 000000000..0a635a56b--- /dev/null+++ b/backend/cloudinary/api/types.go@@ -0,0 +1,48 @@+// Package api has type definitions for cloudinary+package api++import (+ "fmt"+)++// CloudinaryEncoder extends the built-in encoder+type CloudinaryEncoder interface {+ // FromStandardPath takes a / separated path in Standard encoding+ // and converts it to a / separated path in this encoding.+ FromStandardPath(string) string+ // FromStandardName takes name in Standard encoding and converts+ // it in this encoding.+ FromStandardName(string) string+ // ToStandardPath takes a / separated path in this encoding+ // and converts it to a / separated path in Standard encoding.+ ToStandardPath(string) string+ // ToStandardName takes name in this encoding and converts+ // it in Standard encoding.+ ToStandardName(string) string+ // Encoded root of the remote (as passed into NewFs)+ FromStandardFullPath(string) string+}++// UpdateOptions was created to pass options from Update to Put+type UpdateOptions struct {+ PublicID string+ ResourceType string+ DeliveryType string+ AssetFolder string+ DisplayName string+}++// Header formats the option as a string+func (o *UpdateOptions) Header() (string, string) {+ return "UpdateOption", fmt.Sprintf("%s/%s/%s", o.ResourceType, o.DeliveryType, o.PublicID)+}++// Mandatory returns whether the option must be parsed or can be ignored+func (o *UpdateOptions) Mandatory() bool {+ return false+}++// String formats the option into human-readable form+func (o *UpdateOptions) String() string {+ return fmt.Sprintf("Fully qualified Public ID: %s/%s/%s", o.ResourceType, o.DeliveryType, o.PublicID)+}diff --git a/backend/cloudinary/cloudinary.go b/backend/cloudinary/cloudinary.gonew file mode 100644index 000000000..6af5bd370--- /dev/null+++ b/backend/cloudinary/cloudinary.go@@ -0,0 +1,711 @@+// Package cloudinary provides an interface to the Cloudinary DAM+package cloudinary++import (+ "context"+ "encoding/hex"+ "errors"+ "fmt"+ "io"+ "net/http"+ "path"+ "strconv"+ "strings"+ "time"++ "github.com/cloudinary/cloudinary-go/v2"+ SDKApi "github.com/cloudinary/cloudinary-go/v2/api"+ "github.com/cloudinary/cloudinary-go/v2/api/admin"+ "github.com/cloudinary/cloudinary-go/v2/api/admin/search"+ "github.com/cloudinary/cloudinary-go/v2/api/uploader"+ "github.com/rclone/rclone/backend/cloudinary/api"+ "github.com/rclone/rclone/fs"+ "github.com/rclone/rclone/fs/config"+ "github.com/rclone/rclone/fs/config/configmap"+ "github.com/rclone/rclone/fs/config/configstruct"+ "github.com/rclone/rclone/fs/fserrors"+ "github.com/rclone/rclone/fs/fshttp"+ "github.com/rclone/rclone/fs/hash"+ "github.com/rclone/rclone/lib/encoder"+ "github.com/rclone/rclone/lib/pacer"+ "github.com/rclone/rclone/lib/rest"+ "github.com/zeebo/blake3"+)++// Cloudinary shouldn't have a trailing dot if there is no path+func cldPathDir(somePath string) string {+ if somePath == "" || somePath == "." {+ return somePath+ }+ dir := path.Dir(somePath)+ if dir == "." {+ return ""+ }+ return dir+}++// Register with Fs+func init() {+ fs.Register(&fs.RegInfo{+ Name: "cloudinary",+ Description: "Cloudinary",+ NewFs: NewFs,+ Options: []fs.Option{+ {+ Name: "cloud_name",+ Help: "Cloudinary Environment Name",+ Required: true,+ Sensitive: true,+ },+ {+ Name: "api_key",+ Help: "Cloudinary API Key",+ Required: true,+ Sensitive: true,+ },+ {+ Name: "api_secret",+ Help: "Cloudinary API Secret",+ Required: true,+ Sensitive: true,+ },+ {+ Name: "upload_prefix",+ Help: "Specify the API endpoint for environments out of the US",+ },+ {+ Name: "upload_preset",+ Help: "Upload Preset to select asset manipulation on upload",+ },+ {+ Name: config.ConfigEncoding,+ Help: config.ConfigEncodingHelp,+ Advanced: true,+ Default: (encoder.Base | // Slash,LtGt,DoubleQuote,Question,Asterisk,Pipe,Hash,Percent,BackSlash,Del,Ctl,RightSpace,InvalidUtf8,Dot+ encoder.EncodeSlash |+ encoder.EncodeLtGt |+ encoder.EncodeDoubleQuote |+ encoder.EncodeQuestion |+ encoder.EncodeAsterisk |+ encoder.EncodePipe |+ encoder.EncodeHash |+ encoder.EncodePercent |+ encoder.EncodeBackSlash |+ encoder.EncodeDel |+ encoder.EncodeCtl |+ encoder.EncodeRightSpace |+ encoder.EncodeInvalidUtf8 |+ encoder.EncodeDot),+ },+ {+ Name: "eventually_consistent_delay",+ Default: fs.Duration(0),+ Advanced: true,+ Help: "Wait N seconds for eventual consistency of the databases that support the backend operation",+ },+ },+ })+}++// Options defines the configuration for this backend+type Options struct {+ CloudName string `config:"cloud_name"`+ APIKey string `config:"api_key"`+ APISecret string `config:"api_secret"`+ UploadPrefix string `config:"upload_prefix"`+ UploadPreset string `config:"upload_preset"`+ Enc encoder.MultiEncoder `config:"encoding"`+ EventuallyConsistentDelay fs.Duration `config:"eventually_consistent_delay"`+}++// Fs represents a remote cloudinary server+type Fs struct {+ name string+ root string+ opt Options+ features *fs.Features+ pacer *fs.Pacer+ srv *rest.Client // For downloading assets via the Cloudinary CDN+ cld *cloudinary.Cloudinary // API calls are going through the Cloudinary SDK+ lastCRUD time.Time+}++// Object describes a cloudinary object+type Object struct {+ fs *Fs+ remote string+ size int64+ modTime time.Time+ url string+ md5sum string+ publicID string+ resourceType string+ deliveryType string+}++// NewFs constructs an Fs from the path, bucket:path+func NewFs(ctx context.Context, name string, root string, m configmap.Mapper) (fs.Fs, error) {+ opt := new(Options)+ err := configstruct.Set(m, opt)+ if err != nil {+ return nil, err+ }++ // Initialize the Cloudinary client+ cld, err := cloudinary.NewFromParams(opt.CloudName, opt.APIKey, opt.APISecret)+ if err != nil {+ return nil, fmt.Errorf("failed to create Cloudinary client: %w", err)+ }+ cld.Admin.Client = *fshttp.NewClient(ctx)+ cld.Upload.Client = *fshttp.NewClient(ctx)+ if opt.UploadPrefix != "" {+ cld.Config.API.UploadPrefix = opt.UploadPrefix+ }+ client := fshttp.NewClient(ctx)+ f := &Fs{+ name: name,+ root: root,+ opt: *opt,+ cld: cld,+ pacer: fs.NewPacer(ctx, pacer.NewDefault(pacer.MinSleep(1000), pacer.MaxSleep(10000), pacer.DecayConstant(2))),+ srv: rest.NewClient(client),+ }++ f.features = (&fs.Features{+ CanHaveEmptyDirectories: true,+ }).Fill(ctx, f)++ if root != "" {+ // Check to see if the root actually an existing file+ remote := path.Base(root)+ f.root = cldPathDir(root)+ _, err := f.NewObject(ctx, remote)+ if err != nil {+ if err == fs.ErrorObjectNotFound || errors.Is(err, fs.ErrorNotAFile) {+ // File doesn't exist so return the previous root+ f.root = root+ return f, nil+ }+ return nil, err+ }+ // return an error with an fs which points to the parent+ return f, fs.ErrorIsFile+ }+ return f, nil+}++// ------------------------------------------------------------++// FromStandardPath implementation of the api.CloudinaryEncoder+func (f *Fs) FromStandardPath(s string) string {+ return strings.ReplaceAll(f.opt.Enc.FromStandardPath(s), "&", "\uFF06")+}++// FromStandardName implementation of the api.CloudinaryEncoder+func (f *Fs) FromStandardName(s string) string {+ return strings.ReplaceAll(f.opt.Enc.FromStandardName(s), "&", "\uFF06")+}++// ToStandardPath implementation of the api.CloudinaryEncoder+func (f *Fs) ToStandardPath(s string) string {+ return strings.ReplaceAll(f.opt.Enc.ToStandardPath(s), "\uFF06", "&")+}++// ToStandardName implementation of the api.CloudinaryEncoder+func (f *Fs) ToStandardName(s string) string {+ return strings.ReplaceAll(f.opt.Enc.ToStandardName(s), "\uFF06", "&")+}++// FromStandardFullPath encodes a full path to Cloudinary standard+func (f *Fs) FromStandardFullPath(dir string) string {+ return path.Join(api.CloudinaryEncoder.FromStandardPath(f, f.root), api.CloudinaryEncoder.FromStandardPath(f, dir))+}++// ToAssetFolderAPI encodes folders as expected by the Cloudinary SDK+func (f *Fs) ToAssetFolderAPI(dir string) string {+ return strings.ReplaceAll(dir, "%", "%25")+}++// ToDisplayNameElastic encodes a special case of elasticsearch+func (f *Fs) ToDisplayNameElastic(dir string) string {+ return strings.ReplaceAll(dir, "!", "\\!")+}++// Name of the remote (as passed into NewFs)+func (f *Fs) Name() string {+ return f.name+}++// Root of the remote (as passed into NewFs)+func (f *Fs) Root() string {+ return f.root+}++// WaitEventuallyConsistent waits till the FS is eventually consistent+func (f *Fs) WaitEventuallyConsistent() {+ if f.opt.EventuallyConsistentDelay == fs.Duration(0) {+ return+ }+ delay := time.Duration(f.opt.EventuallyConsistentDelay)+ timeSinceLastCRUD := time.Since(f.lastCRUD)+ if timeSinceLastCRUD < delay {+ time.Sleep(delay - timeSinceLastCRUD)+ }+}++// String converts this Fs to a string+func (f *Fs) String() string {+ return fmt.Sprintf("Cloudinary root '%s'", f.root)+}++// Features returns the optional features of this Fs+func (f *Fs) Features() *fs.Features {+ return f.features+}++// List the objects and directories in dir into entries+func (f *Fs) List(ctx context.Context, dir string) (fs.DirEntries, error) {+ remotePrefix := f.FromStandardFullPath(dir)+ if remotePrefix != "" && !strings.HasSuffix(remotePrefix, "/") {+ remotePrefix += "/"+ }++ var entries fs.DirEntries+ dirs := make(map[string]struct{})+ nextCursor := ""+ f.WaitEventuallyConsistent()+ for {+ // user the folders api to list folders.+ folderParams := admin.SubFoldersParams{+ Folder: f.ToAssetFolderAPI(remotePrefix),+ MaxResults: 500,+ }+ if nextCursor != "" {+ folderParams.NextCursor = nextCursor+ }++ results, err := f.cld.Admin.SubFolders(ctx, folderParams)+ if err != nil {+ return nil, fmt.Errorf("failed to list sub-folders: %w", err)+ }+ if results.Error.Message != "" {+ if strings.HasPrefix(results.Error.Message, "Can't find folder with path") {+ return nil, fs.ErrorDirNotFound+ }++ return nil, fmt.Errorf("failed to list sub-folders: %s", results.Error.Message)+ }++ for _, folder := range results.Folders {+ relativePath := api.CloudinaryEncoder.ToStandardPath(f, strings.TrimPrefix(folder.Path, remotePrefix))+ parts := strings.Split(relativePath, "/")++ // It's a directory+ dirName := parts[len(parts)-1]+ if _, found := dirs[dirName]; !found {+ d := fs.NewDir(path.Join(dir, dirName), time.Time{})+ entries = append(entries, d)+ dirs[dirName] = struct{}{}+ }+ }+ // Break if there are no more results+ if results.NextCursor == "" {+ break+ }+ nextCursor = results.NextCursor+ }++ for {+ // Use the assets.AssetsByAssetFolder API to list assets+ assetsParams := admin.AssetsByAssetFolderParams{+ AssetFolder: remotePrefix,+ MaxResults: 500,+ }+ if nextCursor != "" {+ assetsParams.NextCursor = nextCursor+ }++ results, err := f.cld.Admin.AssetsByAssetFolder(ctx, assetsParams)+ if err != nil {+ return nil, fmt.Errorf("failed to list assets: %w", err)+ }++ for _, asset := range results.Assets {+ remote := api.CloudinaryEncoder.ToStandardName(f, asset.DisplayName)+ if dir != "" {+ remote = path.Join(dir, api.CloudinaryEncoder.ToStandardName(f, asset.DisplayName))+ }+ o := &Object{+ fs: f,+ remote: remote,+ size: int64(asset.Bytes),+ modTime: asset.CreatedAt,+ url: asset.SecureURL,+ publicID: asset.PublicID,+ resourceType: asset.AssetType,+ deliveryType: asset.Type,+ }+ entries = append(entries, o)+ }++ // Break if there are no more results+ if results.NextCursor == "" {+ break+ }+ nextCursor = results.NextCursor+ }++ return entries, nil+}++// NewObject finds the Object at remote. If it can't be found it returns the error fs.ErrorObjectNotFound.+func (f *Fs) NewObject(ctx context.Context, remote string) (fs.Object, error) {+ searchParams := search.Query{+ Expression: fmt.Sprintf("asset_folder:\"%s\" AND display_name:\"%s\"",+ f.FromStandardFullPath(cldPathDir(remote)),+ f.ToDisplayNameElastic(api.CloudinaryEncoder.FromStandardName(f, path.Base(remote)))),+ SortBy: []search.SortByField{{"uploaded_at": "desc"}},+ MaxResults: 2,+ }+ var results *admin.SearchResult+ f.WaitEventuallyConsistent()+ err := f.pacer.Call(func() (bool, error) {+ var err1 error+ results, err1 = f.cld.Admin.Search(ctx, searchParams)+ if err1 == nil && results.TotalCount != len(results.Assets) {+ err1 = errors.New("partial response so waiting for eventual consistency")+ }+ return shouldRetry(ctx, nil, err1)+ })+ if err != nil {+ return nil, fs.ErrorObjectNotFound+ }+ if results.TotalCount == 0 || len(results.Assets) == 0 {+ return nil, fs.ErrorObjectNotFound+ }+ asset := results.Assets[0]++ o := &Object{+ fs: f,+ remote: remote,+ size: int64(asset.Bytes),+ modTime: asset.UploadedAt,+ url: asset.SecureURL,+ md5sum: asset.Etag,+ publicID: asset.PublicID,+ resourceType: asset.ResourceType,+ deliveryType: asset.Type,+ }++ return o, nil+}++func (f *Fs) getSuggestedPublicID(assetFolder string, displayName string, modTime time.Time) string {+ payload := []byte(path.Join(assetFolder, displayName))+ hash := blake3.Sum256(payload)+ return hex.EncodeToString(hash[:])+}++// Put uploads content to Cloudinary+func (f *Fs) Put(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) {+ if src.Size() == 0 {+ return nil, fs.ErrorCantUploadEmptyFiles+ }++ params := uploader.UploadParams{+ UploadPreset: f.opt.UploadPreset,+ }++ updateObject := false+ var modTime time.Time+ for _, option := range options {+ if updateOptions, ok := option.(*api.UpdateOptions); ok {+ if updateOptions.PublicID != "" {+ updateObject = true+ params.Overwrite = SDKApi.Bool(true)+ params.Invalidate = SDKApi.Bool(true)+ params.PublicID = updateOptions.PublicID+ params.ResourceType = updateOptions.ResourceType+ params.Type = SDKApi.DeliveryType(updateOptions.DeliveryType)+ params.AssetFolder = updateOptions.AssetFolder+ params.DisplayName = updateOptions.DisplayName+ modTime = src.ModTime(ctx)+ }+ }+ }+ if !updateObject {+ params.AssetFolder = f.FromStandardFullPath(cldPathDir(src.Remote()))+ params.DisplayName = api.CloudinaryEncoder.FromStandardName(f, path.Base(src.Remote()))+ // We want to conform to the unique asset ID of rclone, which is (asset_folder,display_name,last_modified).+ // We also want to enable customers to choose their own public_id, in case duplicate names are not a crucial use case.+ // Upload_presets that apply randomness to the public ID would not work well with rclone duplicate assets support.+ params.FilenameOverride = f.getSuggestedPublicID(params.AssetFolder, params.DisplayName, src.ModTime(ctx))+ }+ uploadResult, err := f.cld.Upload.Upload(ctx, in, params)+ f.lastCRUD = time.Now()+ if err != nil {+ return nil, fmt.Errorf("failed to upload to Cloudinary: %w", err)+ }+ if !updateObject {+ modTime = uploadResult.CreatedAt+ }+ if uploadResult.Error.Message != "" {+ return nil, errors.New(uploadResult.Error.Message)+ }++ o := &Object{+ fs: f,+ remote: src.Remote(),+ size: int64(uploadResult.Bytes),+ modTime: modTime,+ url: uploadResult.SecureURL,+ md5sum: uploadResult.Etag,+ publicID: uploadResult.PublicID,+ resourceType: uploadResult.ResourceType,+ deliveryType: uploadResult.Type,+ }+ return o, nil+}++// Precision of the remote+func (f *Fs) Precision() time.Duration {+ return fs.ModTimeNotSupported+}++// Hashes returns the supported hash sets+func (f *Fs) Hashes() hash.Set {+ return hash.Set(hash.MD5)+}++// Mkdir creates empty folders+func (f *Fs) Mkdir(ctx context.Context, dir string) error {+ params := admin.CreateFolderParams{Folder: f.ToAssetFolderAPI(f.FromStandardFullPath(dir))}+ res, err := f.cld.Admin.CreateFolder(ctx, params)+ f.lastCRUD = time.Now()+ if err != nil {+ return err+ }+ if res.Error.Message != "" {+ return errors.New(res.Error.Message)+ }++ return nil+}++// Rmdir deletes empty folders+func (f *Fs) Rmdir(ctx context.Context, dir string) error {+ // Additional test because Cloudinary will delete folders without+ // assets, regardless of empty sub-folders+ folder := f.ToAssetFolderAPI(f.FromStandardFullPath(dir))+ folderParams := admin.SubFoldersParams{+ Folder: folder,+ MaxResults: 1,+ }+ results, err := f.cld.Admin.SubFolders(ctx, folderParams)+ if err != nil {+ return err+ }+ if results.TotalCount > 0 {+ return fs.ErrorDirectoryNotEmpty+ }++ params := admin.DeleteFolderParams{Folder: folder}+ res, err := f.cld.Admin.DeleteFolder(ctx, params)+ f.lastCRUD = time.Now()+ if err != nil {+ return err+ }+ if res.Error.Message != "" {+ if strings.HasPrefix(res.Error.Message, "Can't find folder with path") {+ return fs.ErrorDirNotFound+ }++ return errors.New(res.Error.Message)+ }++ return nil+}++// retryErrorCodes is a slice of error codes that we will retry+var retryErrorCodes = []int{+ 420, // Too Many Requests (legacy)+ 429, // Too Many Requests+ 500, // Internal Server Error+ 502, // Bad Gateway+ 503, // Service Unavailable+ 504, // Gateway Timeout+ 509, // Bandwidth Limit Exceeded+}++// shouldRetry returns a boolean as to whether this resp and err+// deserve to be retried. It returns the err as a convenience+func shouldRetry(ctx context.Context, resp *http.Response, err error) (bool, error) {+ if fserrors.ContextError(ctx, &err) {+ return false, err+ }+ if err != nil {+ tryAgain := "Try again on "+ if idx := strings.Index(err.Error(), tryAgain); idx != -1 {+ layout := "2006-01-02 15:04:05 UTC"+ dateStr := err.Error()[idx+len(tryAgain) : idx+len(tryAgain)+len(layout)]+ timestamp, err2 := time.Parse(layout, dateStr)+ if err2 == nil {+ return true, fserrors.NewErrorRetryAfter(time.Until(timestamp))+ }+ }++ fs.Debugf(nil, "Retrying API error %v", err)+ return true, err+ }++ return fserrors.ShouldRetry(err) || fserrors.ShouldRetryHTTP(resp, retryErrorCodes), err+}++// ------------------------------------------------------------++// Hash returns the MD5 of an object+func (o *Object) Hash(ctx context.Context, ty hash.Type) (string, error) {+ if ty != hash.MD5 {+ return "", hash.ErrUnsupported+ }+ return o.md5sum, nil+}++// Return a string version+func (o *Object) String() string {+ if o == nil {+ return "<nil>"+ }+ return o.remote+}++// Fs returns the parent Fs+func (o *Object) Fs() fs.Info {+ return o.fs+}++// Remote returns the remote path+func (o *Object) Remote() string {+ return o.remote+}++// ModTime returns the modification time of the object+func (o *Object) ModTime(ctx context.Context) time.Time {+ return o.modTime+}++// Size of object in bytes+func (o *Object) Size() int64 {+ return o.size+}++// Storable returns if this object is storable+func (o *Object) Storable() bool {+ return true+}++// SetModTime sets the modification time of the local fs object+func (o *Object) SetModTime(ctx context.Context, modTime time.Time) error {+ return fs.ErrorCantSetModTime+}++// Open an object for read+func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.ReadCloser, err error) {+ var resp *http.Response+ opts := rest.Opts{+ Method: "GET",+ RootURL: o.url,+ Options: options,+ }+ var offset int64+ var count int64+ var key string+ var value string+ fs.FixRangeOption(options, o.size)+ for _, option := range options {+ switch x := option.(type) {+ case *fs.RangeOption:+ offset, count = x.Decode(o.size)+ if count < 0 {+ count = o.size - offset+ }+ key, value = option.Header()+ case *fs.SeekOption:+ offset = x.Offset+ count = o.size - offset+ key, value = option.Header()+ default:+ if option.Mandatory() {+ fs.Logf(o, "Unsupported mandatory option: %v", option)+ }+ }+ }+ if key != "" && value != "" {+ opts.ExtraHeaders = make(map[string]string)+ opts.ExtraHeaders[key] = value+ }+ // Make sure that the asset is fully available+ err = o.fs.pacer.Call(func() (bool, error) {+ resp, err = o.fs.srv.Call(ctx, &opts)+ if err == nil {+ cl, clErr := strconv.Atoi(resp.Header.Get("content-length"))+ if clErr == nil && count == int64(cl) {+ return false, nil+ }+ }+ return shouldRetry(ctx, resp, err)+ })+ if err != nil {+ return nil, fmt.Errorf("failed download of \"%s\": %w", o.url, err)+ }+ return resp.Body, err+}++// Update the object with the contents of the io.Reader+func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) error {+ options = append(options, &api.UpdateOptions{+ PublicID: o.publicID,+ ResourceType: o.resourceType,+ DeliveryType: o.deliveryType,+ DisplayName: api.CloudinaryEncoder.FromStandardName(o.fs, path.Base(o.Remote())),+ AssetFolder: o.fs.FromStandardFullPath(cldPathDir(o.Remote())),+ })+ updatedObj, err := o.fs.Put(ctx, in, src, options...)+ if err != nil {+ return err+ }+ if uo, ok := updatedObj.(*Object); ok {+ o.size = uo.size+ o.modTime = time.Now() // Skipping uo.modTime because the API returns the create time+ o.url = uo.url+ o.md5sum = uo.md5sum+ o.publicID = uo.publicID+ o.resourceType = uo.resourceType+ o.deliveryType = uo.deliveryType+ }+ return nil+}++// Remove an object+func (o *Object) Remove(ctx context.Context) error {+ params := uploader.DestroyParams{+ PublicID: o.publicID,+ ResourceType: o.resourceType,+ Type: o.deliveryType,+ }+ res, dErr := o.fs.cld.Upload.Destroy(ctx, params)+ o.fs.lastCRUD = time.Now()+ if dErr != nil {+ return dErr+ }++ if res.Error.Message != "" {+ return errors.New(res.Error.Message)+ }++ if res.Result != "ok" {+ return errors.New(res.Result)+ }++ return nil+}diff --git a/backend/cloudinary/cloudinary_test.go b/backend/cloudinary/cloudinary_test.gonew file mode 100644index 000000000..009771eaa--- /dev/null+++ b/backend/cloudinary/cloudinary_test.go@@ -0,0 +1,23 @@+// Test Cloudinary filesystem interface++package cloudinary_test++import (+ "testing"++ "github.com/rclone/rclone/backend/cloudinary"+ "github.com/rclone/rclone/fstest/fstests"+)++// TestIntegration runs integration tests against the remote+func TestIntegration(t *testing.T) {+ name := "TestCloudinary"+ fstests.Run(t, &fstests.Opt{+ RemoteName: name + ":",+ NilObject: (*cloudinary.Object)(nil),+ SkipInvalidUTF8: true,+ ExtraConfig: []fstests.ExtraConfigItem{+ {Name: name, Key: "eventually_consistent_delay", Value: "7"},+ },+ })+}diff --git a/bin/make_manual.py b/bin/make_manual.pyindex 990ab7130..d2e756457 100755--- a/bin/make_manual.py+++ b/bin/make_manual.py@@ -35,6 +35,7 @@ docs = ["box.md","cache.md","chunker.md",+ "cloudinary.md","sharefile.md","crypt.md","compress.md",diff --git a/docs/content/_index.md b/docs/content/_index.mdindex 711b55041..1a1c7fc79 100644--- a/docs/content/_index.md+++ b/docs/content/_index.md@@ -116,6 +116,7 @@ WebDAV or S3, that work out of the box.){{< provider name="Arvan Cloud Object Storage (AOS)" home="https://www.arvancloud.ir/en/products/cloud-storage" config="/s3/#arvan-cloud-object-storage-aos" >}}{{< provider name="Citrix ShareFile" home="http://sharefile.com/" config="/sharefile/" >}}{{< provider name="Cloudflare R2" home="https://blog.cloudflare.com/r2-open-beta/" config="/s3/#cloudflare-r2" >}}+{{< provider name="Cloudinary" home="https://cloudinary.com/" config="/cloudinary/" >}}{{< provider name="DigitalOcean Spaces" home="https://www.digitalocean.com/products/object-storage/" config="/s3/#digitalocean-spaces" >}}{{< provider name="Digi Storage" home="https://storage.rcs-rds.ro/" config="/koofr/#digi-storage" >}}{{< provider name="Dreamhost" home="https://www.dreamhost.com/cloud/storage/" config="/s3/#dreamhost" >}}diff --git a/docs/content/cloudinary.md b/docs/content/cloudinary.mdnew file mode 100644index 000000000..6b822d372--- /dev/null+++ b/docs/content/cloudinary.md@@ -0,0 +1,222 @@+---+title: "Cloudinary"+description: "Rclone docs for Cloudinary backend"+versionIntroduced: "v1.69"++---+# {{< icon "fa fa-cloud" >}} Cloudinary++This is a backend for the [Cloudinary](https://cloudinary.com/) platform++## About Cloudinary++[Cloudinary](https://cloudinary.com/) is an image and video API platform.+Trusted by 1.5 million developers and 10,000 enterprise and hyper-growth companies as a critical part of their tech stack to deliver visually engaging experiences.++## Accounts & Pricing++To use this backend, you need to [create a free account](https://cloudinary.com/users/register_free) on Cloudinary. Start with a free plan with generous usage limits. Then, as your requirements grow, upgrade to a plan that best fits your needs. See [the pricing details](https://cloudinary.com/pricing).++## Securing Your Credentials++Please refer to the [docs](/docs/#configuration-encryption-cheatsheet)++## Configuration++Here is an example of making a Cloudinary configuration.++First, create a [cloudinary.com](https://cloudinary.com/users/register_free) account and choose a plan.++You will need to log in and get the `API Key` and `API Secret` for your account from the developer section.++Now run++`rclone config`++Follow the interactive setup process:++```text+No remotes found, make a new one?+n) New remote+s) Set configuration password+q) Quit config+n/s/q> n++Enter the name for the new remote.+name> cloudinary-media-library++Option Storage.+Type of storage to configure.+Choose a number from below, or type in your own value.+[snip]+XX / cloudinary.com+\ (cloudinary)+[snip]+Storage> cloudinary++Option cloud_name.+You can find your cloudinary.com cloud_name in your [dashboard](https://console.cloudinary.com/pm/developer-dashboard)+Enter a value.+cloud_name> ****************************++Option api_key.+You can find your cloudinary.com api key in your [dashboard](https://console.cloudinary.com/pm/developer-dashboard)+Enter a value.+api_key> ****************************++Option api_secret.+You can find your cloudinary.com api secret in your [dashboard](https://console.cloudinary.com/pm/developer-dashboard)+This value must be a single character, one of the following: y, g.+y/g> y+Enter a value.+api_secret> ****************************++Option upload_prefix.+[Upload prefix](https://cloudinary.com/documentation/cloudinary_sdks#configuration_parameters) to specify alternative data center+Enter a value.+upload_prefix>++Option upload_preset.+[Upload presets](https://cloudinary.com/documentation/upload_presets) can be defined for different upload profiles+Enter a value.+upload_preset>++Edit advanced config?+y) Yes+n) No (default)+y/n> n++Configuration complete.+Options:+- type: cloudinary+- api_key: ****************************+- api_secret: ****************************+- cloud_name: ****************************+- upload_prefix:+- upload_preset:++Keep this "cloudinary-media-library" remote?+y) Yes this is OK (default)+e) Edit this remote+d) Delete this remote+y/e/d> y+```++List directories in the top level of your Media Library++`rclone lsd cloudinary-media-library:`++Make a new directory.++`rclone mkdir cloudinary-media-library:directory`++List the contents of a directory.++`rclone ls cloudinary-media-library:directory`++### Modified time and hashes++Cloudinary stores md5 and timestamps for any successful Put automatically and read-only.++{{< rem autogenerated options start" - DO NOT EDIT - instead edit fs.RegInfo in backend/cloudinary/cloudinary.go then run make backenddocs" >}}+### Standard options++Here are the Standard options specific to cloudinary (Cloudinary).++#### --cloudinary-cloud-name++Cloudinary Environment Name++Properties:++- Config: cloud_name+- Env Var: RCLONE_CLOUDINARY_CLOUD_NAME+- Type: string+- Required: true++#### --cloudinary-api-key++Cloudinary API Key++Properties:++- Config: api_key+- Env Var: RCLONE_CLOUDINARY_API_KEY+- Type: string+- Required: true++#### --cloudinary-api-secret++Cloudinary API Secret++**NB** Input to this must be obscured - see [rclone obscure](/commands/rclone_obscure/).++Properties:++- Config: api_secret+- Env Var: RCLONE_CLOUDINARY_API_SECRET+- Type: string+- Required: true++#### --cloudinary-upload-prefix++Specify the API endpoint for environments out of the US++Properties:++- Config: upload_prefix+- Env Var: RCLONE_CLOUDINARY_UPLOAD_PREFIX+- Type: string+- Required: false++#### --cloudinary-upload-preset++Upload Preset to select asset manipulation on upload++Properties:++- Config: upload_preset+- Env Var: RCLONE_CLOUDINARY_UPLOAD_PRESET+- Type: string+- Required: false++### Advanced options++Here are the Advanced options specific to cloudinary (Cloudinary).++#### --cloudinary-encoding++The encoding for the backend.++See the [encoding section in the overview](/overview/#encoding) for more info.++Properties:++- Config: encoding+- Env Var: RCLONE_CLOUDINARY_ENCODING+- Type: Encoding+- Default: Slash,LtGt,DoubleQuote,Question,Asterisk,Pipe,Hash,Percent,BackSlash,Del,Ctl,RightSpace,InvalidUtf8,Dot++#### --cloudinary-eventually-consistent-delay++Wait N seconds for eventual consistency of the databases that support the backend operation++Properties:++- Config: eventually_consistent_delay+- Env Var: RCLONE_CLOUDINARY_EVENTUALLY_CONSISTENT_DELAY+- Type: Duration+- Default: 0s++#### --cloudinary-description++Description of the remote.++Properties:++- Config: description+- Env Var: RCLONE_CLOUDINARY_DESCRIPTION+- Type: string+- Required: false++{{< rem autogenerated options stop >}}diff --git a/docs/content/docs.md b/docs/content/docs.mdindex cb65ed746..dccb5c5f7 100644--- a/docs/content/docs.md+++ b/docs/content/docs.md@@ -36,6 +36,7 @@ See the following for detailed instructions for* [Chunker](/chunker/) - transparently splits large files for other remotes* [Citrix ShareFile](/sharefile/)* [Compress](/compress/)+ * [Cloudinary](/cloudinary/)* [Combine](/combine/)* [Crypt](/crypt/) - to encrypt other remotes* [DigitalOcean Spaces](/s3/#digitalocean-spaces)diff --git a/docs/content/overview.md b/docs/content/overview.mdindex 6376dc253..9cf84899f 100644--- a/docs/content/overview.md+++ b/docs/content/overview.md@@ -22,6 +22,7 @@ Here is an overview of the major features of each cloud storage system.| Backblaze B2 | SHA1 | R/W | No | No | R/W | - || Box | SHA1 | R/W | Yes | No | - | - || Citrix ShareFile | MD5 | R/W | Yes | No | - | - |+| Cloudinary | MD5 | R | No | Yes | - | - || Dropbox | DBHASH ¹ | R | Yes | No | - | - || Enterprise File Fabric | - | R/W | Yes | No | R/W | - || Files.com | MD5, CRC32 | DR/W | Yes | No | R | - |@@ -502,6 +503,7 @@ upon backend-specific capabilities.| Box | Yes | Yes | Yes | Yes | Yes | No | Yes | No | Yes | Yes | Yes || Citrix ShareFile | Yes | Yes | Yes | Yes | No | No | No | No | No | No | Yes || Dropbox | Yes | Yes | Yes | Yes | No | No | Yes | No | Yes | Yes | Yes |+| Cloudinary | No | No | No | No | No | No | Yes | No | No | No | No || Enterprise File Fabric | Yes | Yes | Yes | Yes | Yes | No | No | No | No | No | Yes || Files.com | Yes | Yes | Yes | Yes | No | No | Yes | No | Yes | No | Yes || FTP | No | No | Yes | Yes | No | No | Yes | No | No | No | Yes |diff --git a/docs/layouts/chrome/navbar.html b/docs/layouts/chrome/navbar.htmlindex bc7bb0ce3..dd180a0d2 100644--- a/docs/layouts/chrome/navbar.html+++ b/docs/layouts/chrome/navbar.html@@ -58,6 +58,7 @@<a class="dropdown-item" href="/b2/"><i class="fa fa-fire fa-fw"></i> Backblaze B2</a><a class="dropdown-item" href="/box/"><i class="fa fa-archive fa-fw"></i> Box</a><a class="dropdown-item" href="/chunker/"><i class="fa fa-cut fa-fw"></i> Chunker (splits large files)</a>+ <a class="dropdown-item" href="/cloudinary/"><i class="fa fa-image fa-fw"></i> Cloudinary</a><a class="dropdown-item" href="/compress/"><i class="fas fa-compress fa-fw"></i> Compress (transparent gzip compression)</a><a class="dropdown-item" href="/combine/"><i class="fa fa-folder-plus fa-fw"></i> Combine (remotes into a directory tree)</a><a class="dropdown-item" href="/sharefile/"><i class="fas fa-share-square fa-fw"></i> Citrix ShareFile</a>diff --git a/fstest/test_all/config.yaml b/fstest/test_all/config.yamlindex 1565c7e8e..24fed9290 100644--- a/fstest/test_all/config.yaml+++ b/fstest/test_all/config.yaml@@ -102,6 +102,21 @@ backends:fastlist: truemaxfile: 1k## end chunker+ - backend: "cloudinary"+ remote: "TestCloudinary:"+ fastlist: false+ ignore:+ # fs/operations+ - TestCheckSum+ - TestCheckSumDownload+ - TestHashSums/Md5+ - TestReadFile+ - TestCopyURL+ - TestMoveFileWithIgnoreExisting+ #vfs+ - TestFileSetModTime/cache=off,open=false,write=false+ - TestFileSetModTime/cache=off,open=true,write=false+ - TestRWFileHandleWriteNoWrite- backend: "combine"remote: "TestCombine:dir1"fastlist: falsediff --git a/go.mod b/go.modindex b9b03e310..c1691b78f 100644--- a/go.mod+++ b/go.mod@@ -24,6 +24,7 @@ require (github.com/aws/aws-sdk-go-v2/service/s3 v1.71.0github.com/aws/smithy-go v1.22.1github.com/buengese/sgzip v0.1.1+ github.com/cloudinary/cloudinary-go/v2 v2.9.0github.com/cloudsoda/go-smb2 v0.0.0-20231124195312-f3ec8ae2c891github.com/colinmarc/hdfs/v2 v2.4.0github.com/coreos/go-semver v0.3.1@@ -76,6 +77,7 @@ require (github.com/xanzy/ssh-agent v0.3.3github.com/youmark/pkcs8 v0.0.0-20240726163527-a2c0da244d78github.com/yunify/qingstor-sdk-go/v3 v3.2.0+ github.com/zeebo/blake3 v0.2.3go.etcd.io/bbolt v1.3.10goftp.io/server/v2 v2.0.1golang.org/x/crypto v0.31.0@@ -128,6 +130,7 @@ require (github.com/chilts/sid v0.0.0-20190607042430-660e94789ec9 // indirectgithub.com/cloudflare/circl v1.3.7 // indirectgithub.com/cpuguy83/go-md2man/v2 v2.0.4 // indirect+ github.com/creasty/defaults v1.7.0 // indirectgithub.com/cronokirby/saferith v0.33.0 // indirectgithub.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirectgithub.com/dustin/go-humanize v1.0.1 // indirect@@ -151,6 +154,7 @@ require (github.com/googleapis/enterprise-certificate-proxy v0.3.4 // indirectgithub.com/googleapis/gax-go/v2 v2.14.0 // indirectgithub.com/gopherjs/gopherjs v0.0.0-20181103185306-d547d1d9531e // indirect+ github.com/gorilla/schema v1.4.1 // indirectgithub.com/hashicorp/errwrap v1.1.0 // indirectgithub.com/hashicorp/go-cleanhttp v0.5.2 // indirectgithub.com/hashicorp/go-multierror v1.1.1 // indirect@@ -197,7 +201,6 @@ require (github.com/tklauser/numcpus v0.7.0 // indirectgithub.com/willscott/go-nfs-client v0.0.0-20240104095149-b44639837b00 // indirectgithub.com/yusufpapurcu/wmi v1.2.4 // indirect- github.com/zeebo/blake3 v0.2.3 // indirectgithub.com/zeebo/errs v1.3.0 // indirectgo.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.54.0 // indirectgo.opentelemetry.io/otel v1.29.0 // indirectdiff --git a/go.sum b/go.sumindex 70ad47b58..6a3820d35 100644--- a/go.sum+++ b/go.sum@@ -175,6 +175,8 @@ github.com/cloudflare/circl v1.1.0/go.mod h1:prBCrKB9DV4poKZY1l9zBXg2QJY7mvgRvtMgithub.com/cloudflare/circl v1.3.3/go.mod h1:5XYMA4rFBvNIrhs50XuiBJ15vF2pZn4nnUKZrLbUZFA=github.com/cloudflare/circl v1.3.7 h1:qlCDlTPz2n9fu58M0Nh1J/JzcFpfgkFHHX3O35r5vcU=github.com/cloudflare/circl v1.3.7/go.mod h1:sRTcRWXGLrKw6yIGJ+l7amYJFfAXbZG0kBSc8r4zxgA=+github.com/cloudinary/cloudinary-go/v2 v2.9.0 h1:8C76QklmuV4qmKAC7cUnu9D68X9kCkFMuLspPikECCo=+github.com/cloudinary/cloudinary-go/v2 v2.9.0/go.mod h1:ireC4gqVetsjVhYlwjUJwKTbZuWjEIynbR9zQTlqsvo=github.com/cloudsoda/go-smb2 v0.0.0-20231124195312-f3ec8ae2c891 h1:nPP4suUiNage0vvyEBgfAnhTPwwXhNqtHmSuiCIQwKU=github.com/cloudsoda/go-smb2 v0.0.0-20231124195312-f3ec8ae2c891/go.mod h1:xFxVVe3plxwhM+6BgTTPByEgG8hggo8+gtRUkbc5W8Q=github.com/cloudwego/base64x v0.1.4 h1:jwCgWpFanWmN8xoIUHa2rtzmkd5J2plF/dnLS6Xd/0Y=@@ -190,6 +192,8 @@ github.com/coreos/go-systemd/v22 v22.5.0 h1:RrqgGjYQKalulkV8NGVIfkXQf6YYmOyiJKk8github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc=github.com/cpuguy83/go-md2man/v2 v2.0.4 h1:wfIWP927BUkWJb2NmU/kNDYIBTh/ziUX91+lVfRxZq4=github.com/cpuguy83/go-md2man/v2 v2.0.4/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=+github.com/creasty/defaults v1.7.0 h1:eNdqZvc5B509z18lD8yc212CAqJNvfT1Jq6L8WowdBA=+github.com/creasty/defaults v1.7.0/go.mod h1:iGzKe6pbEHnpMPtfDXZEr0NVxWnPTjb1bbDy08fPzYM=github.com/cronokirby/saferith v0.33.0 h1:TgoQlfsD4LIwx71+ChfRcIpjkw+RPOapDEVxa+LhwLo=github.com/cronokirby/saferith v0.33.0/go.mod h1:QKJhjoqUtBsXCAVEjw38mFqoi7DebT7kthcD7UzbnoA=github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=@@ -345,6 +349,8 @@ github.com/googleapis/gax-go/v2 v2.14.0/go.mod h1:lhBCnjdLrWRaPvLWhmc8IS24m9mr07github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=github.com/gopherjs/gopherjs v0.0.0-20181103185306-d547d1d9531e h1:JKmoR8x90Iww1ks85zJ1lfDGgIiMDuIptTOhJq+zKyg=github.com/gopherjs/gopherjs v0.0.0-20181103185306-d547d1d9531e/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=+github.com/gorilla/schema v1.4.1 h1:jUg5hUjCSDZpNGLuXQOgIWGdlgrIdYvgQ0wZtdK1M3E=+github.com/gorilla/schema v1.4.1/go.mod h1:Dg5SSm5PV60mhF2NFaTV1xuYYj8tV8NOPRo4FggUMnM=github.com/gorilla/securecookie v1.1.1 h1:miw7JPhV+b/lAHSXz4qd/nN9jRiAFV5FwjeKyCS8BvQ=github.com/gorilla/securecookie v1.1.1/go.mod h1:ra0sb63/xPlUeL+yeDciTfxMRAA+MP+HVt/4epWDjd4=github.com/gorilla/sessions v1.2.1 h1:DHd3rPN5lE3Ts3D8rKkQ8x/0kqfeNmBAaiSi+o7FsgI=