Skip to content

"rsync for cloud storage" - Google Drive, S3, Dropbox, Backblaze B2, One Drive, Swift, Hubic, Wasabi, Google Cloud Storage, Azure Blob, Azure Files, Yandex Files

cloudinary: add cloudinary backend - fixes #7989

yuval-cloudinary7e4b8e33f596
diff --git a/backend/all/all.go b/backend/all/all.go
index 1abd6770f..df6a6e45b 100644
--- a/backend/all/all.go
+++ b/backend/all/all.go
@@ -10,6 +10,7 @@ import (
_ "github.com/rclone/rclone/backend/box"
_ "github.com/rclone/rclone/backend/cache"
_ "github.com/rclone/rclone/backend/chunker"
+ _ "github.com/rclone/rclone/backend/cloudinary"
_ "github.com/rclone/rclone/backend/combine"
_ "github.com/rclone/rclone/backend/compress"
_ "github.com/rclone/rclone/backend/crypt"
diff --git a/backend/cloudinary/api/types.go b/backend/cloudinary/api/types.go
new file mode 100644
index 000000000..0a635a56b
--- /dev/null
+++ b/backend/cloudinary/api/types.go
@@ -0,0 +1,48 @@
+// Package api has type definitions for cloudinary
+package api
+
+import (
+ "fmt"
+)
+
+// CloudinaryEncoder extends the built-in encoder
+type CloudinaryEncoder interface {
+ // FromStandardPath takes a / separated path in Standard encoding
+ // and converts it to a / separated path in this encoding.
+ FromStandardPath(string) string
+ // FromStandardName takes name in Standard encoding and converts
+ // it in this encoding.
+ FromStandardName(string) string
+ // ToStandardPath takes a / separated path in this encoding
+ // and converts it to a / separated path in Standard encoding.
+ ToStandardPath(string) string
+ // ToStandardName takes name in this encoding and converts
+ // it in Standard encoding.
+ ToStandardName(string) string
+ // Encoded root of the remote (as passed into NewFs)
+ FromStandardFullPath(string) string
+}
+
+// UpdateOptions was created to pass options from Update to Put
+type UpdateOptions struct {
+ PublicID string
+ ResourceType string
+ DeliveryType string
+ AssetFolder string
+ DisplayName string
+}
+
+// Header formats the option as a string
+func (o *UpdateOptions) Header() (string, string) {
+ return "UpdateOption", fmt.Sprintf("%s/%s/%s", o.ResourceType, o.DeliveryType, o.PublicID)
+}
+
+// Mandatory returns whether the option must be parsed or can be ignored
+func (o *UpdateOptions) Mandatory() bool {
+ return false
+}
+
+// String formats the option into human-readable form
+func (o *UpdateOptions) String() string {
+ return fmt.Sprintf("Fully qualified Public ID: %s/%s/%s", o.ResourceType, o.DeliveryType, o.PublicID)
+}
diff --git a/backend/cloudinary/cloudinary.go b/backend/cloudinary/cloudinary.go
new file mode 100644
index 000000000..6af5bd370
--- /dev/null
+++ b/backend/cloudinary/cloudinary.go
@@ -0,0 +1,711 @@
+// Package cloudinary provides an interface to the Cloudinary DAM
+package cloudinary
+
+import (
+ "context"
+ "encoding/hex"
+ "errors"
+ "fmt"
+ "io"
+ "net/http"
+ "path"
+ "strconv"
+ "strings"
+ "time"
+
+ "github.com/cloudinary/cloudinary-go/v2"
+ SDKApi "github.com/cloudinary/cloudinary-go/v2/api"
+ "github.com/cloudinary/cloudinary-go/v2/api/admin"
+ "github.com/cloudinary/cloudinary-go/v2/api/admin/search"
+ "github.com/cloudinary/cloudinary-go/v2/api/uploader"
+ "github.com/rclone/rclone/backend/cloudinary/api"
+ "github.com/rclone/rclone/fs"
+ "github.com/rclone/rclone/fs/config"
+ "github.com/rclone/rclone/fs/config/configmap"
+ "github.com/rclone/rclone/fs/config/configstruct"
+ "github.com/rclone/rclone/fs/fserrors"
+ "github.com/rclone/rclone/fs/fshttp"
+ "github.com/rclone/rclone/fs/hash"
+ "github.com/rclone/rclone/lib/encoder"
+ "github.com/rclone/rclone/lib/pacer"
+ "github.com/rclone/rclone/lib/rest"
+ "github.com/zeebo/blake3"
+)
+
+// Cloudinary shouldn't have a trailing dot if there is no path
+func cldPathDir(somePath string) string {
+ if somePath == "" || somePath == "." {
+ return somePath
+ }
+ dir := path.Dir(somePath)
+ if dir == "." {
+ return ""
+ }
+ return dir
+}
+
+// Register with Fs
+func init() {
+ fs.Register(&fs.RegInfo{
+ Name: "cloudinary",
+ Description: "Cloudinary",
+ NewFs: NewFs,
+ Options: []fs.Option{
+ {
+ Name: "cloud_name",
+ Help: "Cloudinary Environment Name",
+ Required: true,
+ Sensitive: true,
+ },
+ {
+ Name: "api_key",
+ Help: "Cloudinary API Key",
+ Required: true,
+ Sensitive: true,
+ },
+ {
+ Name: "api_secret",
+ Help: "Cloudinary API Secret",
+ Required: true,
+ Sensitive: true,
+ },
+ {
+ Name: "upload_prefix",
+ Help: "Specify the API endpoint for environments out of the US",
+ },
+ {
+ Name: "upload_preset",
+ Help: "Upload Preset to select asset manipulation on upload",
+ },
+ {
+ Name: config.ConfigEncoding,
+ Help: config.ConfigEncodingHelp,
+ Advanced: true,
+ Default: (encoder.Base | // Slash,LtGt,DoubleQuote,Question,Asterisk,Pipe,Hash,Percent,BackSlash,Del,Ctl,RightSpace,InvalidUtf8,Dot
+ encoder.EncodeSlash |
+ encoder.EncodeLtGt |
+ encoder.EncodeDoubleQuote |
+ encoder.EncodeQuestion |
+ encoder.EncodeAsterisk |
+ encoder.EncodePipe |
+ encoder.EncodeHash |
+ encoder.EncodePercent |
+ encoder.EncodeBackSlash |
+ encoder.EncodeDel |
+ encoder.EncodeCtl |
+ encoder.EncodeRightSpace |
+ encoder.EncodeInvalidUtf8 |
+ encoder.EncodeDot),
+ },
+ {
+ Name: "eventually_consistent_delay",
+ Default: fs.Duration(0),
+ Advanced: true,
+ Help: "Wait N seconds for eventual consistency of the databases that support the backend operation",
+ },
+ },
+ })
+}
+
+// Options defines the configuration for this backend
+type Options struct {
+ CloudName string `config:"cloud_name"`
+ APIKey string `config:"api_key"`
+ APISecret string `config:"api_secret"`
+ UploadPrefix string `config:"upload_prefix"`
+ UploadPreset string `config:"upload_preset"`
+ Enc encoder.MultiEncoder `config:"encoding"`
+ EventuallyConsistentDelay fs.Duration `config:"eventually_consistent_delay"`
+}
+
+// Fs represents a remote cloudinary server
+type Fs struct {
+ name string
+ root string
+ opt Options
+ features *fs.Features
+ pacer *fs.Pacer
+ srv *rest.Client // For downloading assets via the Cloudinary CDN
+ cld *cloudinary.Cloudinary // API calls are going through the Cloudinary SDK
+ lastCRUD time.Time
+}
+
+// Object describes a cloudinary object
+type Object struct {
+ fs *Fs
+ remote string
+ size int64
+ modTime time.Time
+ url string
+ md5sum string
+ publicID string
+ resourceType string
+ deliveryType string
+}
+
+// NewFs constructs an Fs from the path, bucket:path
+func NewFs(ctx context.Context, name string, root string, m configmap.Mapper) (fs.Fs, error) {
+ opt := new(Options)
+ err := configstruct.Set(m, opt)
+ if err != nil {
+ return nil, err
+ }
+
+ // Initialize the Cloudinary client
+ cld, err := cloudinary.NewFromParams(opt.CloudName, opt.APIKey, opt.APISecret)
+ if err != nil {
+ return nil, fmt.Errorf("failed to create Cloudinary client: %w", err)
+ }
+ cld.Admin.Client = *fshttp.NewClient(ctx)
+ cld.Upload.Client = *fshttp.NewClient(ctx)
+ if opt.UploadPrefix != "" {
+ cld.Config.API.UploadPrefix = opt.UploadPrefix
+ }
+ client := fshttp.NewClient(ctx)
+ f := &Fs{
+ name: name,
+ root: root,
+ opt: *opt,
+ cld: cld,
+ pacer: fs.NewPacer(ctx, pacer.NewDefault(pacer.MinSleep(1000), pacer.MaxSleep(10000), pacer.DecayConstant(2))),
+ srv: rest.NewClient(client),
+ }
+
+ f.features = (&fs.Features{
+ CanHaveEmptyDirectories: true,
+ }).Fill(ctx, f)
+
+ if root != "" {
+ // Check to see if the root actually an existing file
+ remote := path.Base(root)
+ f.root = cldPathDir(root)
+ _, err := f.NewObject(ctx, remote)
+ if err != nil {
+ if err == fs.ErrorObjectNotFound || errors.Is(err, fs.ErrorNotAFile) {
+ // File doesn't exist so return the previous root
+ f.root = root
+ return f, nil
+ }
+ return nil, err
+ }
+ // return an error with an fs which points to the parent
+ return f, fs.ErrorIsFile
+ }
+ return f, nil
+}
+
+// ------------------------------------------------------------
+
+// FromStandardPath implementation of the api.CloudinaryEncoder
+func (f *Fs) FromStandardPath(s string) string {
+ return strings.ReplaceAll(f.opt.Enc.FromStandardPath(s), "&", "\uFF06")
+}
+
+// FromStandardName implementation of the api.CloudinaryEncoder
+func (f *Fs) FromStandardName(s string) string {
+ return strings.ReplaceAll(f.opt.Enc.FromStandardName(s), "&", "\uFF06")
+}
+
+// ToStandardPath implementation of the api.CloudinaryEncoder
+func (f *Fs) ToStandardPath(s string) string {
+ return strings.ReplaceAll(f.opt.Enc.ToStandardPath(s), "\uFF06", "&")
+}
+
+// ToStandardName implementation of the api.CloudinaryEncoder
+func (f *Fs) ToStandardName(s string) string {
+ return strings.ReplaceAll(f.opt.Enc.ToStandardName(s), "\uFF06", "&")
+}
+
+// FromStandardFullPath encodes a full path to Cloudinary standard
+func (f *Fs) FromStandardFullPath(dir string) string {
+ return path.Join(api.CloudinaryEncoder.FromStandardPath(f, f.root), api.CloudinaryEncoder.FromStandardPath(f, dir))
+}
+
+// ToAssetFolderAPI encodes folders as expected by the Cloudinary SDK
+func (f *Fs) ToAssetFolderAPI(dir string) string {
+ return strings.ReplaceAll(dir, "%", "%25")
+}
+
+// ToDisplayNameElastic encodes a special case of elasticsearch
+func (f *Fs) ToDisplayNameElastic(dir string) string {
+ return strings.ReplaceAll(dir, "!", "\\!")
+}
+
+// Name of the remote (as passed into NewFs)
+func (f *Fs) Name() string {
+ return f.name
+}
+
+// Root of the remote (as passed into NewFs)
+func (f *Fs) Root() string {
+ return f.root
+}
+
+// WaitEventuallyConsistent waits till the FS is eventually consistent
+func (f *Fs) WaitEventuallyConsistent() {
+ if f.opt.EventuallyConsistentDelay == fs.Duration(0) {
+ return
+ }
+ delay := time.Duration(f.opt.EventuallyConsistentDelay)
+ timeSinceLastCRUD := time.Since(f.lastCRUD)
+ if timeSinceLastCRUD < delay {
+ time.Sleep(delay - timeSinceLastCRUD)
+ }
+}
+
+// String converts this Fs to a string
+func (f *Fs) String() string {
+ return fmt.Sprintf("Cloudinary root '%s'", f.root)
+}
+
+// Features returns the optional features of this Fs
+func (f *Fs) Features() *fs.Features {
+ return f.features
+}
+
+// List the objects and directories in dir into entries
+func (f *Fs) List(ctx context.Context, dir string) (fs.DirEntries, error) {
+ remotePrefix := f.FromStandardFullPath(dir)
+ if remotePrefix != "" && !strings.HasSuffix(remotePrefix, "/") {
+ remotePrefix += "/"
+ }
+
+ var entries fs.DirEntries
+ dirs := make(map[string]struct{})
+ nextCursor := ""
+ f.WaitEventuallyConsistent()
+ for {
+ // user the folders api to list folders.
+ folderParams := admin.SubFoldersParams{
+ Folder: f.ToAssetFolderAPI(remotePrefix),
+ MaxResults: 500,
+ }
+ if nextCursor != "" {
+ folderParams.NextCursor = nextCursor
+ }
+
+ results, err := f.cld.Admin.SubFolders(ctx, folderParams)
+ if err != nil {
+ return nil, fmt.Errorf("failed to list sub-folders: %w", err)
+ }
+ if results.Error.Message != "" {
+ if strings.HasPrefix(results.Error.Message, "Can't find folder with path") {
+ return nil, fs.ErrorDirNotFound
+ }
+
+ return nil, fmt.Errorf("failed to list sub-folders: %s", results.Error.Message)
+ }
+
+ for _, folder := range results.Folders {
+ relativePath := api.CloudinaryEncoder.ToStandardPath(f, strings.TrimPrefix(folder.Path, remotePrefix))
+ parts := strings.Split(relativePath, "/")
+
+ // It's a directory
+ dirName := parts[len(parts)-1]
+ if _, found := dirs[dirName]; !found {
+ d := fs.NewDir(path.Join(dir, dirName), time.Time{})
+ entries = append(entries, d)
+ dirs[dirName] = struct{}{}
+ }
+ }
+ // Break if there are no more results
+ if results.NextCursor == "" {
+ break
+ }
+ nextCursor = results.NextCursor
+ }
+
+ for {
+ // Use the assets.AssetsByAssetFolder API to list assets
+ assetsParams := admin.AssetsByAssetFolderParams{
+ AssetFolder: remotePrefix,
+ MaxResults: 500,
+ }
+ if nextCursor != "" {
+ assetsParams.NextCursor = nextCursor
+ }
+
+ results, err := f.cld.Admin.AssetsByAssetFolder(ctx, assetsParams)
+ if err != nil {
+ return nil, fmt.Errorf("failed to list assets: %w", err)
+ }
+
+ for _, asset := range results.Assets {
+ remote := api.CloudinaryEncoder.ToStandardName(f, asset.DisplayName)
+ if dir != "" {
+ remote = path.Join(dir, api.CloudinaryEncoder.ToStandardName(f, asset.DisplayName))
+ }
+ o := &Object{
+ fs: f,
+ remote: remote,
+ size: int64(asset.Bytes),
+ modTime: asset.CreatedAt,
+ url: asset.SecureURL,
+ publicID: asset.PublicID,
+ resourceType: asset.AssetType,
+ deliveryType: asset.Type,
+ }
+ entries = append(entries, o)
+ }
+
+ // Break if there are no more results
+ if results.NextCursor == "" {
+ break
+ }
+ nextCursor = results.NextCursor
+ }
+
+ return entries, nil
+}
+
+// NewObject finds the Object at remote. If it can't be found it returns the error fs.ErrorObjectNotFound.
+func (f *Fs) NewObject(ctx context.Context, remote string) (fs.Object, error) {
+ searchParams := search.Query{
+ Expression: fmt.Sprintf("asset_folder:\"%s\" AND display_name:\"%s\"",
+ f.FromStandardFullPath(cldPathDir(remote)),
+ f.ToDisplayNameElastic(api.CloudinaryEncoder.FromStandardName(f, path.Base(remote)))),
+ SortBy: []search.SortByField{{"uploaded_at": "desc"}},
+ MaxResults: 2,
+ }
+ var results *admin.SearchResult
+ f.WaitEventuallyConsistent()
+ err := f.pacer.Call(func() (bool, error) {
+ var err1 error
+ results, err1 = f.cld.Admin.Search(ctx, searchParams)
+ if err1 == nil && results.TotalCount != len(results.Assets) {
+ err1 = errors.New("partial response so waiting for eventual consistency")
+ }
+ return shouldRetry(ctx, nil, err1)
+ })
+ if err != nil {
+ return nil, fs.ErrorObjectNotFound
+ }
+ if results.TotalCount == 0 || len(results.Assets) == 0 {
+ return nil, fs.ErrorObjectNotFound
+ }
+ asset := results.Assets[0]
+
+ o := &Object{
+ fs: f,
+ remote: remote,
+ size: int64(asset.Bytes),
+ modTime: asset.UploadedAt,
+ url: asset.SecureURL,
+ md5sum: asset.Etag,
+ publicID: asset.PublicID,
+ resourceType: asset.ResourceType,
+ deliveryType: asset.Type,
+ }
+
+ return o, nil
+}
+
+func (f *Fs) getSuggestedPublicID(assetFolder string, displayName string, modTime time.Time) string {
+ payload := []byte(path.Join(assetFolder, displayName))
+ hash := blake3.Sum256(payload)
+ return hex.EncodeToString(hash[:])
+}
+
+// Put uploads content to Cloudinary
+func (f *Fs) Put(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) {
+ if src.Size() == 0 {
+ return nil, fs.ErrorCantUploadEmptyFiles
+ }
+
+ params := uploader.UploadParams{
+ UploadPreset: f.opt.UploadPreset,
+ }
+
+ updateObject := false
+ var modTime time.Time
+ for _, option := range options {
+ if updateOptions, ok := option.(*api.UpdateOptions); ok {
+ if updateOptions.PublicID != "" {
+ updateObject = true
+ params.Overwrite = SDKApi.Bool(true)
+ params.Invalidate = SDKApi.Bool(true)
+ params.PublicID = updateOptions.PublicID
+ params.ResourceType = updateOptions.ResourceType
+ params.Type = SDKApi.DeliveryType(updateOptions.DeliveryType)
+ params.AssetFolder = updateOptions.AssetFolder
+ params.DisplayName = updateOptions.DisplayName
+ modTime = src.ModTime(ctx)
+ }
+ }
+ }
+ if !updateObject {
+ params.AssetFolder = f.FromStandardFullPath(cldPathDir(src.Remote()))
+ params.DisplayName = api.CloudinaryEncoder.FromStandardName(f, path.Base(src.Remote()))
+ // We want to conform to the unique asset ID of rclone, which is (asset_folder,display_name,last_modified).
+ // We also want to enable customers to choose their own public_id, in case duplicate names are not a crucial use case.
+ // Upload_presets that apply randomness to the public ID would not work well with rclone duplicate assets support.
+ params.FilenameOverride = f.getSuggestedPublicID(params.AssetFolder, params.DisplayName, src.ModTime(ctx))
+ }
+ uploadResult, err := f.cld.Upload.Upload(ctx, in, params)
+ f.lastCRUD = time.Now()
+ if err != nil {
+ return nil, fmt.Errorf("failed to upload to Cloudinary: %w", err)
+ }
+ if !updateObject {
+ modTime = uploadResult.CreatedAt
+ }
+ if uploadResult.Error.Message != "" {
+ return nil, errors.New(uploadResult.Error.Message)
+ }
+
+ o := &Object{
+ fs: f,
+ remote: src.Remote(),
+ size: int64(uploadResult.Bytes),
+ modTime: modTime,
+ url: uploadResult.SecureURL,
+ md5sum: uploadResult.Etag,
+ publicID: uploadResult.PublicID,
+ resourceType: uploadResult.ResourceType,
+ deliveryType: uploadResult.Type,
+ }
+ return o, nil
+}
+
+// Precision of the remote
+func (f *Fs) Precision() time.Duration {
+ return fs.ModTimeNotSupported
+}
+
+// Hashes returns the supported hash sets
+func (f *Fs) Hashes() hash.Set {
+ return hash.Set(hash.MD5)
+}
+
+// Mkdir creates empty folders
+func (f *Fs) Mkdir(ctx context.Context, dir string) error {
+ params := admin.CreateFolderParams{Folder: f.ToAssetFolderAPI(f.FromStandardFullPath(dir))}
+ res, err := f.cld.Admin.CreateFolder(ctx, params)
+ f.lastCRUD = time.Now()
+ if err != nil {
+ return err
+ }
+ if res.Error.Message != "" {
+ return errors.New(res.Error.Message)
+ }
+
+ return nil
+}
+
+// Rmdir deletes empty folders
+func (f *Fs) Rmdir(ctx context.Context, dir string) error {
+ // Additional test because Cloudinary will delete folders without
+ // assets, regardless of empty sub-folders
+ folder := f.ToAssetFolderAPI(f.FromStandardFullPath(dir))
+ folderParams := admin.SubFoldersParams{
+ Folder: folder,
+ MaxResults: 1,
+ }
+ results, err := f.cld.Admin.SubFolders(ctx, folderParams)
+ if err != nil {
+ return err
+ }
+ if results.TotalCount > 0 {
+ return fs.ErrorDirectoryNotEmpty
+ }
+
+ params := admin.DeleteFolderParams{Folder: folder}
+ res, err := f.cld.Admin.DeleteFolder(ctx, params)
+ f.lastCRUD = time.Now()
+ if err != nil {
+ return err
+ }
+ if res.Error.Message != "" {
+ if strings.HasPrefix(res.Error.Message, "Can't find folder with path") {
+ return fs.ErrorDirNotFound
+ }
+
+ return errors.New(res.Error.Message)
+ }
+
+ return nil
+}
+
+// retryErrorCodes is a slice of error codes that we will retry
+var retryErrorCodes = []int{
+ 420, // Too Many Requests (legacy)
+ 429, // Too Many Requests
+ 500, // Internal Server Error
+ 502, // Bad Gateway
+ 503, // Service Unavailable
+ 504, // Gateway Timeout
+ 509, // Bandwidth Limit Exceeded
+}
+
+// shouldRetry returns a boolean as to whether this resp and err
+// deserve to be retried. It returns the err as a convenience
+func shouldRetry(ctx context.Context, resp *http.Response, err error) (bool, error) {
+ if fserrors.ContextError(ctx, &err) {
+ return false, err
+ }
+ if err != nil {
+ tryAgain := "Try again on "
+ if idx := strings.Index(err.Error(), tryAgain); idx != -1 {
+ layout := "2006-01-02 15:04:05 UTC"
+ dateStr := err.Error()[idx+len(tryAgain) : idx+len(tryAgain)+len(layout)]
+ timestamp, err2 := time.Parse(layout, dateStr)
+ if err2 == nil {
+ return true, fserrors.NewErrorRetryAfter(time.Until(timestamp))
+ }
+ }
+
+ fs.Debugf(nil, "Retrying API error %v", err)
+ return true, err
+ }
+
+ return fserrors.ShouldRetry(err) || fserrors.ShouldRetryHTTP(resp, retryErrorCodes), err
+}
+
+// ------------------------------------------------------------
+
+// Hash returns the MD5 of an object
+func (o *Object) Hash(ctx context.Context, ty hash.Type) (string, error) {
+ if ty != hash.MD5 {
+ return "", hash.ErrUnsupported
+ }
+ return o.md5sum, nil
+}
+
+// Return a string version
+func (o *Object) String() string {
+ if o == nil {
+ return "<nil>"
+ }
+ return o.remote
+}
+
+// Fs returns the parent Fs
+func (o *Object) Fs() fs.Info {
+ return o.fs
+}
+
+// Remote returns the remote path
+func (o *Object) Remote() string {
+ return o.remote
+}
+
+// ModTime returns the modification time of the object
+func (o *Object) ModTime(ctx context.Context) time.Time {
+ return o.modTime
+}
+
+// Size of object in bytes
+func (o *Object) Size() int64 {
+ return o.size
+}
+
+// Storable returns if this object is storable
+func (o *Object) Storable() bool {
+ return true
+}
+
+// SetModTime sets the modification time of the local fs object
+func (o *Object) SetModTime(ctx context.Context, modTime time.Time) error {
+ return fs.ErrorCantSetModTime
+}
+
+// Open an object for read
+func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.ReadCloser, err error) {
+ var resp *http.Response
+ opts := rest.Opts{
+ Method: "GET",
+ RootURL: o.url,
+ Options: options,
+ }
+ var offset int64
+ var count int64
+ var key string
+ var value string
+ fs.FixRangeOption(options, o.size)
+ for _, option := range options {
+ switch x := option.(type) {
+ case *fs.RangeOption:
+ offset, count = x.Decode(o.size)
+ if count < 0 {
+ count = o.size - offset
+ }
+ key, value = option.Header()
+ case *fs.SeekOption:
+ offset = x.Offset
+ count = o.size - offset
+ key, value = option.Header()
+ default:
+ if option.Mandatory() {
+ fs.Logf(o, "Unsupported mandatory option: %v", option)
+ }
+ }
+ }
+ if key != "" && value != "" {
+ opts.ExtraHeaders = make(map[string]string)
+ opts.ExtraHeaders[key] = value
+ }
+ // Make sure that the asset is fully available
+ err = o.fs.pacer.Call(func() (bool, error) {
+ resp, err = o.fs.srv.Call(ctx, &opts)
+ if err == nil {
+ cl, clErr := strconv.Atoi(resp.Header.Get("content-length"))
+ if clErr == nil && count == int64(cl) {
+ return false, nil
+ }
+ }
+ return shouldRetry(ctx, resp, err)
+ })
+ if err != nil {
+ return nil, fmt.Errorf("failed download of \"%s\": %w", o.url, err)
+ }
+ return resp.Body, err
+}
+
+// Update the object with the contents of the io.Reader
+func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) error {
+ options = append(options, &api.UpdateOptions{
+ PublicID: o.publicID,
+ ResourceType: o.resourceType,
+ DeliveryType: o.deliveryType,
+ DisplayName: api.CloudinaryEncoder.FromStandardName(o.fs, path.Base(o.Remote())),
+ AssetFolder: o.fs.FromStandardFullPath(cldPathDir(o.Remote())),
+ })
+ updatedObj, err := o.fs.Put(ctx, in, src, options...)
+ if err != nil {
+ return err
+ }
+ if uo, ok := updatedObj.(*Object); ok {
+ o.size = uo.size
+ o.modTime = time.Now() // Skipping uo.modTime because the API returns the create time
+ o.url = uo.url
+ o.md5sum = uo.md5sum
+ o.publicID = uo.publicID
+ o.resourceType = uo.resourceType
+ o.deliveryType = uo.deliveryType
+ }
+ return nil
+}
+
+// Remove an object
+func (o *Object) Remove(ctx context.Context) error {
+ params := uploader.DestroyParams{
+ PublicID: o.publicID,
+ ResourceType: o.resourceType,
+ Type: o.deliveryType,
+ }
+ res, dErr := o.fs.cld.Upload.Destroy(ctx, params)
+ o.fs.lastCRUD = time.Now()
+ if dErr != nil {
+ return dErr
+ }
+
+ if res.Error.Message != "" {
+ return errors.New(res.Error.Message)
+ }
+
+ if res.Result != "ok" {
+ return errors.New(res.Result)
+ }
+
+ return nil
+}
diff --git a/backend/cloudinary/cloudinary_test.go b/backend/cloudinary/cloudinary_test.go
new file mode 100644
index 000000000..009771eaa
--- /dev/null
+++ b/backend/cloudinary/cloudinary_test.go
@@ -0,0 +1,23 @@
+// Test Cloudinary filesystem interface
+
+package cloudinary_test
+
+import (
+ "testing"
+
+ "github.com/rclone/rclone/backend/cloudinary"
+ "github.com/rclone/rclone/fstest/fstests"
+)
+
+// TestIntegration runs integration tests against the remote
+func TestIntegration(t *testing.T) {
+ name := "TestCloudinary"
+ fstests.Run(t, &fstests.Opt{
+ RemoteName: name + ":",
+ NilObject: (*cloudinary.Object)(nil),
+ SkipInvalidUTF8: true,
+ ExtraConfig: []fstests.ExtraConfigItem{
+ {Name: name, Key: "eventually_consistent_delay", Value: "7"},
+ },
+ })
+}
diff --git a/bin/make_manual.py b/bin/make_manual.py
index 990ab7130..d2e756457 100755
--- a/bin/make_manual.py
+++ b/bin/make_manual.py
@@ -35,6 +35,7 @@ docs = [
"box.md",
"cache.md",
"chunker.md",
+ "cloudinary.md",
"sharefile.md",
"crypt.md",
"compress.md",
diff --git a/docs/content/_index.md b/docs/content/_index.md
index 711b55041..1a1c7fc79 100644
--- a/docs/content/_index.md
+++ b/docs/content/_index.md
@@ -116,6 +116,7 @@ WebDAV or S3, that work out of the box.)
{{< provider name="Arvan Cloud Object Storage (AOS)" home="https://www.arvancloud.ir/en/products/cloud-storage" config="/s3/#arvan-cloud-object-storage-aos" >}}
{{< provider name="Citrix ShareFile" home="http://sharefile.com/" config="/sharefile/" >}}
{{< provider name="Cloudflare R2" home="https://blog.cloudflare.com/r2-open-beta/" config="/s3/#cloudflare-r2" >}}
+{{< provider name="Cloudinary" home="https://cloudinary.com/" config="/cloudinary/" >}}
{{< provider name="DigitalOcean Spaces" home="https://www.digitalocean.com/products/object-storage/" config="/s3/#digitalocean-spaces" >}}
{{< provider name="Digi Storage" home="https://storage.rcs-rds.ro/" config="/koofr/#digi-storage" >}}
{{< provider name="Dreamhost" home="https://www.dreamhost.com/cloud/storage/" config="/s3/#dreamhost" >}}
diff --git a/docs/content/cloudinary.md b/docs/content/cloudinary.md
new file mode 100644
index 000000000..6b822d372
--- /dev/null
+++ b/docs/content/cloudinary.md
@@ -0,0 +1,222 @@
+---
+title: "Cloudinary"
+description: "Rclone docs for Cloudinary backend"
+versionIntroduced: "v1.69"
+
+---
+# {{< icon "fa fa-cloud" >}} Cloudinary
+
+This is a backend for the [Cloudinary](https://cloudinary.com/) platform
+
+## About Cloudinary
+
+[Cloudinary](https://cloudinary.com/) is an image and video API platform.
+Trusted by 1.5 million developers and 10,000 enterprise and hyper-growth companies as a critical part of their tech stack to deliver visually engaging experiences.
+
+## Accounts & Pricing
+
+To use this backend, you need to [create a free account](https://cloudinary.com/users/register_free) on Cloudinary. Start with a free plan with generous usage limits. Then, as your requirements grow, upgrade to a plan that best fits your needs. See [the pricing details](https://cloudinary.com/pricing).
+
+## Securing Your Credentials
+
+Please refer to the [docs](/docs/#configuration-encryption-cheatsheet)
+
+## Configuration
+
+Here is an example of making a Cloudinary configuration.
+
+First, create a [cloudinary.com](https://cloudinary.com/users/register_free) account and choose a plan.
+
+You will need to log in and get the `API Key` and `API Secret` for your account from the developer section.
+
+Now run
+
+`rclone config`
+
+Follow the interactive setup process:
+
+```text
+No remotes found, make a new one?
+n) New remote
+s) Set configuration password
+q) Quit config
+n/s/q> n
+
+Enter the name for the new remote.
+name> cloudinary-media-library
+
+Option Storage.
+Type of storage to configure.
+Choose a number from below, or type in your own value.
+[snip]
+XX / cloudinary.com
+\ (cloudinary)
+[snip]
+Storage> cloudinary
+
+Option cloud_name.
+You can find your cloudinary.com cloud_name in your [dashboard](https://console.cloudinary.com/pm/developer-dashboard)
+Enter a value.
+cloud_name> ****************************
+
+Option api_key.
+You can find your cloudinary.com api key in your [dashboard](https://console.cloudinary.com/pm/developer-dashboard)
+Enter a value.
+api_key> ****************************
+
+Option api_secret.
+You can find your cloudinary.com api secret in your [dashboard](https://console.cloudinary.com/pm/developer-dashboard)
+This value must be a single character, one of the following: y, g.
+y/g> y
+Enter a value.
+api_secret> ****************************
+
+Option upload_prefix.
+[Upload prefix](https://cloudinary.com/documentation/cloudinary_sdks#configuration_parameters) to specify alternative data center
+Enter a value.
+upload_prefix>
+
+Option upload_preset.
+[Upload presets](https://cloudinary.com/documentation/upload_presets) can be defined for different upload profiles
+Enter a value.
+upload_preset>
+
+Edit advanced config?
+y) Yes
+n) No (default)
+y/n> n
+
+Configuration complete.
+Options:
+- type: cloudinary
+- api_key: ****************************
+- api_secret: ****************************
+- cloud_name: ****************************
+- upload_prefix:
+- upload_preset:
+
+Keep this "cloudinary-media-library" remote?
+y) Yes this is OK (default)
+e) Edit this remote
+d) Delete this remote
+y/e/d> y
+```
+
+List directories in the top level of your Media Library
+
+`rclone lsd cloudinary-media-library:`
+
+Make a new directory.
+
+`rclone mkdir cloudinary-media-library:directory`
+
+List the contents of a directory.
+
+`rclone ls cloudinary-media-library:directory`
+
+### Modified time and hashes
+
+Cloudinary stores md5 and timestamps for any successful Put automatically and read-only.
+
+{{< rem autogenerated options start" - DO NOT EDIT - instead edit fs.RegInfo in backend/cloudinary/cloudinary.go then run make backenddocs" >}}
+### Standard options
+
+Here are the Standard options specific to cloudinary (Cloudinary).
+
+#### --cloudinary-cloud-name
+
+Cloudinary Environment Name
+
+Properties:
+
+- Config: cloud_name
+- Env Var: RCLONE_CLOUDINARY_CLOUD_NAME
+- Type: string
+- Required: true
+
+#### --cloudinary-api-key
+
+Cloudinary API Key
+
+Properties:
+
+- Config: api_key
+- Env Var: RCLONE_CLOUDINARY_API_KEY
+- Type: string
+- Required: true
+
+#### --cloudinary-api-secret
+
+Cloudinary API Secret
+
+**NB** Input to this must be obscured - see [rclone obscure](/commands/rclone_obscure/).
+
+Properties:
+
+- Config: api_secret
+- Env Var: RCLONE_CLOUDINARY_API_SECRET
+- Type: string
+- Required: true
+
+#### --cloudinary-upload-prefix
+
+Specify the API endpoint for environments out of the US
+
+Properties:
+
+- Config: upload_prefix
+- Env Var: RCLONE_CLOUDINARY_UPLOAD_PREFIX
+- Type: string
+- Required: false
+
+#### --cloudinary-upload-preset
+
+Upload Preset to select asset manipulation on upload
+
+Properties:
+
+- Config: upload_preset
+- Env Var: RCLONE_CLOUDINARY_UPLOAD_PRESET
+- Type: string
+- Required: false
+
+### Advanced options
+
+Here are the Advanced options specific to cloudinary (Cloudinary).
+
+#### --cloudinary-encoding
+
+The encoding for the backend.
+
+See the [encoding section in the overview](/overview/#encoding) for more info.
+
+Properties:
+
+- Config: encoding
+- Env Var: RCLONE_CLOUDINARY_ENCODING
+- Type: Encoding
+- Default: Slash,LtGt,DoubleQuote,Question,Asterisk,Pipe,Hash,Percent,BackSlash,Del,Ctl,RightSpace,InvalidUtf8,Dot
+
+#### --cloudinary-eventually-consistent-delay
+
+Wait N seconds for eventual consistency of the databases that support the backend operation
+
+Properties:
+
+- Config: eventually_consistent_delay
+- Env Var: RCLONE_CLOUDINARY_EVENTUALLY_CONSISTENT_DELAY
+- Type: Duration
+- Default: 0s
+
+#### --cloudinary-description
+
+Description of the remote.
+
+Properties:
+
+- Config: description
+- Env Var: RCLONE_CLOUDINARY_DESCRIPTION
+- Type: string
+- Required: false
+
+{{< rem autogenerated options stop >}}
diff --git a/docs/content/docs.md b/docs/content/docs.md
index cb65ed746..dccb5c5f7 100644
--- a/docs/content/docs.md
+++ b/docs/content/docs.md
@@ -36,6 +36,7 @@ See the following for detailed instructions for
* [Chunker](/chunker/) - transparently splits large files for other remotes
* [Citrix ShareFile](/sharefile/)
* [Compress](/compress/)
+ * [Cloudinary](/cloudinary/)
* [Combine](/combine/)
* [Crypt](/crypt/) - to encrypt other remotes
* [DigitalOcean Spaces](/s3/#digitalocean-spaces)
diff --git a/docs/content/overview.md b/docs/content/overview.md
index 6376dc253..9cf84899f 100644
--- a/docs/content/overview.md
+++ b/docs/content/overview.md
@@ -22,6 +22,7 @@ Here is an overview of the major features of each cloud storage system.
| Backblaze B2 | SHA1 | R/W | No | No | R/W | - |
| Box | SHA1 | R/W | Yes | No | - | - |
| Citrix ShareFile | MD5 | R/W | Yes | No | - | - |
+| Cloudinary | MD5 | R | No | Yes | - | - |
| Dropbox | DBHASH ¹ | R | Yes | No | - | - |
| Enterprise File Fabric | - | R/W | Yes | No | R/W | - |
| Files.com | MD5, CRC32 | DR/W | Yes | No | R | - |
@@ -502,6 +503,7 @@ upon backend-specific capabilities.
| Box | Yes | Yes | Yes | Yes | Yes | No | Yes | No | Yes | Yes | Yes |
| Citrix ShareFile | Yes | Yes | Yes | Yes | No | No | No | No | No | No | Yes |
| Dropbox | Yes | Yes | Yes | Yes | No | No | Yes | No | Yes | Yes | Yes |
+| Cloudinary | No | No | No | No | No | No | Yes | No | No | No | No |
| Enterprise File Fabric | Yes | Yes | Yes | Yes | Yes | No | No | No | No | No | Yes |
| Files.com | Yes | Yes | Yes | Yes | No | No | Yes | No | Yes | No | Yes |
| FTP | No | No | Yes | Yes | No | No | Yes | No | No | No | Yes |
diff --git a/docs/layouts/chrome/navbar.html b/docs/layouts/chrome/navbar.html
index bc7bb0ce3..dd180a0d2 100644
--- a/docs/layouts/chrome/navbar.html
+++ b/docs/layouts/chrome/navbar.html
@@ -58,6 +58,7 @@
<a class="dropdown-item" href="/b2/"><i class="fa fa-fire fa-fw"></i> Backblaze B2</a>
<a class="dropdown-item" href="/box/"><i class="fa fa-archive fa-fw"></i> Box</a>
<a class="dropdown-item" href="/chunker/"><i class="fa fa-cut fa-fw"></i> Chunker (splits large files)</a>
+ <a class="dropdown-item" href="/cloudinary/"><i class="fa fa-image fa-fw"></i> Cloudinary</a>
<a class="dropdown-item" href="/compress/"><i class="fas fa-compress fa-fw"></i> Compress (transparent gzip compression)</a>
<a class="dropdown-item" href="/combine/"><i class="fa fa-folder-plus fa-fw"></i> Combine (remotes into a directory tree)</a>
<a class="dropdown-item" href="/sharefile/"><i class="fas fa-share-square fa-fw"></i> Citrix ShareFile</a>
diff --git a/fstest/test_all/config.yaml b/fstest/test_all/config.yaml
index 1565c7e8e..24fed9290 100644
--- a/fstest/test_all/config.yaml
+++ b/fstest/test_all/config.yaml
@@ -102,6 +102,21 @@ backends:
fastlist: true
maxfile: 1k
## end chunker
+ - backend: "cloudinary"
+ remote: "TestCloudinary:"
+ fastlist: false
+ ignore:
+ # fs/operations
+ - TestCheckSum
+ - TestCheckSumDownload
+ - TestHashSums/Md5
+ - TestReadFile
+ - TestCopyURL
+ - TestMoveFileWithIgnoreExisting
+ #vfs
+ - TestFileSetModTime/cache=off,open=false,write=false
+ - TestFileSetModTime/cache=off,open=true,write=false
+ - TestRWFileHandleWriteNoWrite
- backend: "combine"
remote: "TestCombine:dir1"
fastlist: false
diff --git a/go.mod b/go.mod
index b9b03e310..c1691b78f 100644
--- a/go.mod
+++ b/go.mod
@@ -24,6 +24,7 @@ require (
github.com/aws/aws-sdk-go-v2/service/s3 v1.71.0
github.com/aws/smithy-go v1.22.1
github.com/buengese/sgzip v0.1.1
+ github.com/cloudinary/cloudinary-go/v2 v2.9.0
github.com/cloudsoda/go-smb2 v0.0.0-20231124195312-f3ec8ae2c891
github.com/colinmarc/hdfs/v2 v2.4.0
github.com/coreos/go-semver v0.3.1
@@ -76,6 +77,7 @@ require (
github.com/xanzy/ssh-agent v0.3.3
github.com/youmark/pkcs8 v0.0.0-20240726163527-a2c0da244d78
github.com/yunify/qingstor-sdk-go/v3 v3.2.0
+ github.com/zeebo/blake3 v0.2.3
go.etcd.io/bbolt v1.3.10
goftp.io/server/v2 v2.0.1
golang.org/x/crypto v0.31.0
@@ -128,6 +130,7 @@ require (
github.com/chilts/sid v0.0.0-20190607042430-660e94789ec9 // indirect
github.com/cloudflare/circl v1.3.7 // indirect
github.com/cpuguy83/go-md2man/v2 v2.0.4 // indirect
+ github.com/creasty/defaults v1.7.0 // indirect
github.com/cronokirby/saferith v0.33.0 // indirect
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
github.com/dustin/go-humanize v1.0.1 // indirect
@@ -151,6 +154,7 @@ require (
github.com/googleapis/enterprise-certificate-proxy v0.3.4 // indirect
github.com/googleapis/gax-go/v2 v2.14.0 // indirect
github.com/gopherjs/gopherjs v0.0.0-20181103185306-d547d1d9531e // indirect
+ github.com/gorilla/schema v1.4.1 // indirect
github.com/hashicorp/errwrap v1.1.0 // indirect
github.com/hashicorp/go-cleanhttp v0.5.2 // indirect
github.com/hashicorp/go-multierror v1.1.1 // indirect
@@ -197,7 +201,6 @@ require (
github.com/tklauser/numcpus v0.7.0 // indirect
github.com/willscott/go-nfs-client v0.0.0-20240104095149-b44639837b00 // indirect
github.com/yusufpapurcu/wmi v1.2.4 // indirect
- github.com/zeebo/blake3 v0.2.3 // indirect
github.com/zeebo/errs v1.3.0 // indirect
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.54.0 // indirect
go.opentelemetry.io/otel v1.29.0 // indirect
diff --git a/go.sum b/go.sum
index 70ad47b58..6a3820d35 100644
--- a/go.sum
+++ b/go.sum
@@ -175,6 +175,8 @@ github.com/cloudflare/circl v1.1.0/go.mod h1:prBCrKB9DV4poKZY1l9zBXg2QJY7mvgRvtM
github.com/cloudflare/circl v1.3.3/go.mod h1:5XYMA4rFBvNIrhs50XuiBJ15vF2pZn4nnUKZrLbUZFA=
github.com/cloudflare/circl v1.3.7 h1:qlCDlTPz2n9fu58M0Nh1J/JzcFpfgkFHHX3O35r5vcU=
github.com/cloudflare/circl v1.3.7/go.mod h1:sRTcRWXGLrKw6yIGJ+l7amYJFfAXbZG0kBSc8r4zxgA=
+github.com/cloudinary/cloudinary-go/v2 v2.9.0 h1:8C76QklmuV4qmKAC7cUnu9D68X9kCkFMuLspPikECCo=
+github.com/cloudinary/cloudinary-go/v2 v2.9.0/go.mod h1:ireC4gqVetsjVhYlwjUJwKTbZuWjEIynbR9zQTlqsvo=
github.com/cloudsoda/go-smb2 v0.0.0-20231124195312-f3ec8ae2c891 h1:nPP4suUiNage0vvyEBgfAnhTPwwXhNqtHmSuiCIQwKU=
github.com/cloudsoda/go-smb2 v0.0.0-20231124195312-f3ec8ae2c891/go.mod h1:xFxVVe3plxwhM+6BgTTPByEgG8hggo8+gtRUkbc5W8Q=
github.com/cloudwego/base64x v0.1.4 h1:jwCgWpFanWmN8xoIUHa2rtzmkd5J2plF/dnLS6Xd/0Y=
@@ -190,6 +192,8 @@ github.com/coreos/go-systemd/v22 v22.5.0 h1:RrqgGjYQKalulkV8NGVIfkXQf6YYmOyiJKk8
github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc=
github.com/cpuguy83/go-md2man/v2 v2.0.4 h1:wfIWP927BUkWJb2NmU/kNDYIBTh/ziUX91+lVfRxZq4=
github.com/cpuguy83/go-md2man/v2 v2.0.4/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
+github.com/creasty/defaults v1.7.0 h1:eNdqZvc5B509z18lD8yc212CAqJNvfT1Jq6L8WowdBA=
+github.com/creasty/defaults v1.7.0/go.mod h1:iGzKe6pbEHnpMPtfDXZEr0NVxWnPTjb1bbDy08fPzYM=
github.com/cronokirby/saferith v0.33.0 h1:TgoQlfsD4LIwx71+ChfRcIpjkw+RPOapDEVxa+LhwLo=
github.com/cronokirby/saferith v0.33.0/go.mod h1:QKJhjoqUtBsXCAVEjw38mFqoi7DebT7kthcD7UzbnoA=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
@@ -345,6 +349,8 @@ github.com/googleapis/gax-go/v2 v2.14.0/go.mod h1:lhBCnjdLrWRaPvLWhmc8IS24m9mr07
github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
github.com/gopherjs/gopherjs v0.0.0-20181103185306-d547d1d9531e h1:JKmoR8x90Iww1ks85zJ1lfDGgIiMDuIptTOhJq+zKyg=
github.com/gopherjs/gopherjs v0.0.0-20181103185306-d547d1d9531e/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
+github.com/gorilla/schema v1.4.1 h1:jUg5hUjCSDZpNGLuXQOgIWGdlgrIdYvgQ0wZtdK1M3E=
+github.com/gorilla/schema v1.4.1/go.mod h1:Dg5SSm5PV60mhF2NFaTV1xuYYj8tV8NOPRo4FggUMnM=
github.com/gorilla/securecookie v1.1.1 h1:miw7JPhV+b/lAHSXz4qd/nN9jRiAFV5FwjeKyCS8BvQ=
github.com/gorilla/securecookie v1.1.1/go.mod h1:ra0sb63/xPlUeL+yeDciTfxMRAA+MP+HVt/4epWDjd4=
github.com/gorilla/sessions v1.2.1 h1:DHd3rPN5lE3Ts3D8rKkQ8x/0kqfeNmBAaiSi+o7FsgI=