Skip to content

"rsync for cloud storage" - Google Drive, S3, Dropbox, Backblaze B2, One Drive, Swift, Hubic, Wasabi, Google Cloud Storage, Azure Blob, Azure Files, Yandex Files

Add Drime backend

dougal6529d2cd8ffb
diff --git a/README.md b/README.md
index 8733afe54..2c99a640b 100644
--- a/README.md
+++ b/README.md
@@ -40,6 +40,7 @@ directories to and from different cloud storage providers.
- DigitalOcean Spaces [:page_facing_up:](https://rclone.org/s3/#digitalocean-spaces)
- Digi Storage [:page_facing_up:](https://rclone.org/koofr/#digi-storage)
- Dreamhost [:page_facing_up:](https://rclone.org/s3/#dreamhost)
+- Drime [:page_facing_up:](https://rclone.org/s3/#drime)
- Dropbox [:page_facing_up:](https://rclone.org/dropbox/)
- Enterprise File Fabric [:page_facing_up:](https://rclone.org/filefabric/)
- Exaba [:page_facing_up:](https://rclone.org/s3/#exaba)
diff --git a/backend/all/all.go b/backend/all/all.go
index aed11e189..7ad63c271 100644
--- a/backend/all/all.go
+++ b/backend/all/all.go
@@ -16,6 +16,7 @@ import (
_ "github.com/rclone/rclone/backend/compress"
_ "github.com/rclone/rclone/backend/crypt"
_ "github.com/rclone/rclone/backend/doi"
+ _ "github.com/rclone/rclone/backend/drime"
_ "github.com/rclone/rclone/backend/drive"
_ "github.com/rclone/rclone/backend/dropbox"
_ "github.com/rclone/rclone/backend/fichier"
diff --git a/backend/drime/api/types.go b/backend/drime/api/types.go
new file mode 100644
index 000000000..0a1a8bab2
--- /dev/null
+++ b/backend/drime/api/types.go
@@ -0,0 +1,237 @@
+// Package api has type definitions for drime
+//
+// Converted from the API docs with help from https://mholt.github.io/json-to-go/
+package api
+
+import (
+ "encoding/json"
+ "fmt"
+ "time"
+)
+
+// Types of things in Item
+const (
+ ItemTypeFolder = "folder"
+)
+
+// User information
+type User struct {
+ Email string `json:"email"`
+ ID json.Number `json:"id"`
+ Avatar string `json:"avatar"`
+ ModelType string `json:"model_type"`
+ OwnsEntry bool `json:"owns_entry"`
+ EntryPermissions []any `json:"entry_permissions"`
+ DisplayName string `json:"display_name"`
+}
+
+// Permissions for a file
+type Permissions struct {
+ FilesUpdate bool `json:"files.update"`
+ FilesCreate bool `json:"files.create"`
+ FilesDownload bool `json:"files.download"`
+ FilesDelete bool `json:"files.delete"`
+}
+
+// Item describes a folder or a file as returned by /drive/file-entries
+type Item struct {
+ ID json.Number `json:"id"`
+ Name string `json:"name"`
+ Description any `json:"description"`
+ FileName string `json:"file_name"`
+ Mime string `json:"mime"`
+ Color any `json:"color"`
+ Backup bool `json:"backup"`
+ Tracked int `json:"tracked"`
+ FileSize int64 `json:"file_size"`
+ UserID json.Number `json:"user_id"`
+ ParentID json.Number `json:"parent_id"`
+ CreatedAt time.Time `json:"created_at"`
+ UpdatedAt time.Time `json:"updated_at"`
+ DeletedAt any `json:"deleted_at"`
+ IsDeleted int `json:"is_deleted"`
+ Path string `json:"path"`
+ DiskPrefix any `json:"disk_prefix"`
+ Type string `json:"type"`
+ Extension any `json:"extension"`
+ FileHash any `json:"file_hash"`
+ Public bool `json:"public"`
+ Thumbnail bool `json:"thumbnail"`
+ MuxStatus any `json:"mux_status"`
+ ThumbnailURL any `json:"thumbnail_url"`
+ WorkspaceID int `json:"workspace_id"`
+ IsEncrypted int `json:"is_encrypted"`
+ Iv any `json:"iv"`
+ VaultID any `json:"vault_id"`
+ OwnerID int `json:"owner_id"`
+ Hash string `json:"hash"`
+ URL string `json:"url"`
+ Users []User `json:"users"`
+ Tags []any `json:"tags"`
+ Permissions Permissions `json:"permissions"`
+}
+
+// Listing response
+type Listing struct {
+ CurrentPage int `json:"current_page"`
+ Data []Item `json:"data"`
+ From int `json:"from"`
+ LastPage int `json:"last_page"`
+ NextPage int `json:"next_page"`
+ PerPage int `json:"per_page"`
+ PrevPage int `json:"prev_page"`
+ To int `json:"to"`
+ Total int `json:"total"`
+}
+
+// UploadResponse for a file
+type UploadResponse struct {
+ Status string `json:"status"`
+ FileEntry Item `json:"fileEntry"`
+}
+
+// CreateFolderRequest for a folder
+type CreateFolderRequest struct {
+ Name string `json:"name"`
+ ParentID json.Number `json:"parentId,omitempty"`
+}
+
+// CreateFolderResponse for a folder
+type CreateFolderResponse struct {
+ Status string `json:"status"`
+ Folder Item `json:"folder"`
+}
+
+// Error is returned from drime when things go wrong
+type Error struct {
+ Message string `json:"message"`
+}
+
+// Error returns a string for the error and satisfies the error interface
+func (e Error) Error() string {
+ out := fmt.Sprintf("Error %q", e.Message)
+ return out
+}
+
+// Check Error satisfies the error interface
+var _ error = (*Error)(nil)
+
+// DeleteRequest is the input to DELETE /file-entries
+type DeleteRequest struct {
+ EntryIDs []string `json:"entryIds"`
+ DeleteForever bool `json:"deleteForever"`
+}
+
+// DeleteResponse is the input to DELETE /file-entries
+type DeleteResponse struct {
+ Status string `json:"status"`
+ Message string `json:"message"`
+ Errors map[string]string `json:"errors"`
+}
+
+// UpdateItemRequest describes the updates to be done to an item for PUT /file-entries/{id}/
+type UpdateItemRequest struct {
+ Name string `json:"name,omitempty"`
+ Description string `json:"description,omitempty"`
+}
+
+// UpdateItemResponse is returned by PUT /file-entries/{id}/
+type UpdateItemResponse struct {
+ Status string `json:"status"`
+ FileEntry Item `json:"fileEntry"`
+}
+
+// MoveRequest is the input to /file-entries/move
+type MoveRequest struct {
+ EntryIDs []string `json:"entryIds"`
+ DestinationID string `json:"destinationId"`
+}
+
+// MoveResponse is returned by POST /file-entries/move
+type MoveResponse struct {
+ Status string `json:"status"`
+ Entries []Item `json:"entries"`
+}
+
+// CopyRequest is the input to /file-entries/duplicate
+type CopyRequest struct {
+ EntryIDs []string `json:"entryIds"`
+ DestinationID string `json:"destinationId"`
+}
+
+// CopyResponse is returned by POST /file-entries/duplicate
+type CopyResponse struct {
+ Status string `json:"status"`
+ Entries []Item `json:"entries"`
+}
+
+// MultiPartCreateRequest is the input of POST /s3/multipart/create
+type MultiPartCreateRequest struct {
+ Filename string `json:"filename"`
+ Mime string `json:"mime"`
+ Size int64 `json:"size"`
+ Extension string `json:"extension"`
+ ParentID json.Number `json:"parent_id"`
+ RelativePath string `json:"relativePath"`
+}
+
+// MultiPartCreateResponse is returned by POST /s3/multipart/create
+type MultiPartCreateResponse struct {
+ UploadID string `json:"uploadId"`
+ Key string `json:"key"`
+}
+
+// CompletedPart Type for completed parts when making a multipart upload.
+type CompletedPart struct {
+ ETag string `json:"ETag"`
+ PartNumber int32 `json:"PartNumber"`
+}
+
+// MultiPartGetURLsRequest is the input of POST /s3/multipart/batch-sign-part-urls
+type MultiPartGetURLsRequest struct {
+ UploadID string `json:"uploadId"`
+ Key string `json:"key"`
+ PartNumbers []int `json:"partNumbers"`
+}
+
+// MultiPartGetURLsResponse is the result of POST /s3/multipart/batch-sign-part-urls
+type MultiPartGetURLsResponse struct {
+ URLs []struct {
+ URL string `json:"url"`
+ PartNumber int32 `json:"partNumber"`
+ } `json:"urls"`
+}
+
+// MultiPartCompleteRequest is the input to POST /s3/multipart/complete
+type MultiPartCompleteRequest struct {
+ UploadID string `json:"uploadId"`
+ Key string `json:"key"`
+ Parts []CompletedPart `json:"parts"`
+}
+
+// MultiPartCompleteResponse is the result of POST /s3/multipart/complete
+type MultiPartCompleteResponse struct {
+ Location string `json:"location"`
+}
+
+// MultiPartEntriesRequest is the input to POST /s3/entries
+type MultiPartEntriesRequest struct {
+ ClientMime string `json:"clientMime"`
+ ClientName string `json:"clientName"`
+ Filename string `json:"filename"`
+ Size int64 `json:"size"`
+ ClientExtension string `json:"clientExtension"`
+ ParentID json.Number `json:"parent_id"`
+ RelativePath string `json:"relativePath"`
+}
+
+// MultiPartEntriesResponse is the result of POST /s3/entries
+type MultiPartEntriesResponse struct {
+ FileEntry Item `json:"fileEntry"`
+}
+
+// MultiPartAbort is the input of POST /s3/multipart/abort
+type MultiPartAbort struct {
+ UploadID string `json:"uploadId"`
+ Key string `json:"key"`
+}
diff --git a/backend/drime/drime.go b/backend/drime/drime.go
new file mode 100644
index 000000000..8a199ab11
--- /dev/null
+++ b/backend/drime/drime.go
@@ -0,0 +1,1563 @@
+// Package drime provides an interface to the Drime
+// object storage system.
+package drime
+
+/*
+Return results give
+
+ X-Ratelimit-Limit: 2000
+ X-Ratelimit-Remaining: 1999
+
+The rate limit headers indicate the number of allowed API requests per
+minute. The limit is two thousand requests per minute, and rclone
+should stay under that.
+*/
+
+import (
+ "context"
+ "encoding/json"
+ "errors"
+ "fmt"
+ "io"
+ "net/http"
+ "net/url"
+ "path"
+ "sort"
+ "strconv"
+ "strings"
+ "sync"
+ "sync/atomic"
+ "time"
+
+ "github.com/rclone/rclone/backend/drime/api"
+ "github.com/rclone/rclone/fs"
+ "github.com/rclone/rclone/fs/chunksize"
+ "github.com/rclone/rclone/fs/config"
+ "github.com/rclone/rclone/fs/config/configmap"
+ "github.com/rclone/rclone/fs/config/configstruct"
+ "github.com/rclone/rclone/fs/fserrors"
+ "github.com/rclone/rclone/fs/fshttp"
+ "github.com/rclone/rclone/fs/hash"
+ "github.com/rclone/rclone/lib/dircache"
+ "github.com/rclone/rclone/lib/encoder"
+ "github.com/rclone/rclone/lib/multipart"
+ "github.com/rclone/rclone/lib/pacer"
+ "github.com/rclone/rclone/lib/random"
+ "github.com/rclone/rclone/lib/rest"
+)
+
+const (
+ minSleep = 10 * time.Millisecond
+ maxSleep = 20 * time.Second
+ decayConstant = 1 // bigger for slower decay, exponential
+ baseURL = "https://app.drime.cloud/"
+ rootURL = baseURL + "api/v1"
+ maxUploadParts = 10000 // maximum allowed number of parts in a multi-part upload
+ minChunkSize = fs.SizeSuffix(1024 * 1024 * 5)
+ defaultUploadCutoff = fs.SizeSuffix(200 * 1024 * 1024)
+)
+
+// Register with Fs
+func init() {
+ fs.Register(&fs.RegInfo{
+ Name: "drime",
+ Description: "Drime",
+ NewFs: NewFs,
+ Options: []fs.Option{{
+ Name: "access_token",
+ Help: `API Access token
+
+You can get this from the web control panel.`,
+ Sensitive: true,
+ }, {
+ Name: "root_folder_id",
+ Help: `ID of the root folder
+
+Leave this blank normally, rclone will fill it in automatically.
+
+If you want rclone to be restricted to a particular folder you can
+fill it in - see the docs for more info.
+`,
+ Default: "",
+ Advanced: true,
+ Sensitive: true,
+ }, {
+ Name: "workspace_id",
+ Help: `Account ID
+
+Leave this blank normally unless you wish to specify a Workspace ID.
+`,
+ Default: "",
+ Advanced: true,
+ Sensitive: true,
+ }, {
+ Name: "list_chunk",
+ Help: `Number of items to list in each call`,
+ Default: 1000,
+ Advanced: true,
+ }, {
+ Name: "hard_delete",
+ Help: "Delete files permanently rather than putting them into the trash.",
+ Default: false,
+ Advanced: true,
+ }, {
+ Name: "upload_cutoff",
+ Help: `Cutoff for switching to chunked upload.
+
+Any files larger than this will be uploaded in chunks of chunk_size.
+The minimum is 0 and the maximum is 5 GiB.`,
+ Default: defaultUploadCutoff,
+ Advanced: true,
+ }, {
+ Name: "chunk_size",
+ Help: `Chunk size to use for uploading.
+
+When uploading files larger than upload_cutoff or files with unknown
+size (e.g. from "rclone rcat" or uploaded with "rclone mount" or google
+photos or google docs) they will be uploaded as multipart uploads
+using this chunk size.
+
+Note that "--drime-upload-concurrency" chunks of this size are buffered
+in memory per transfer.
+
+If you are transferring large files over high-speed links and you have
+enough memory, then increasing this will speed up the transfers.
+
+Rclone will automatically increase the chunk size when uploading a
+large file of known size to stay below the 10,000 chunks limit.
+
+Files of unknown size are uploaded with the configured
+chunk_size. Since the default chunk size is 5 MiB and there can be at
+most 10,000 chunks, this means that by default the maximum size of
+a file you can stream upload is 48 GiB. If you wish to stream upload
+larger files then you will need to increase chunk_size.
+`,
+ Default: minChunkSize,
+ Advanced: true,
+ }, {
+ Name: "upload_concurrency",
+ Help: `Concurrency for multipart uploads and copies.
+
+This is the number of chunks of the same file that are uploaded
+concurrently for multipart uploads and copies.
+
+If you are uploading small numbers of large files over high-speed links
+and these uploads do not fully utilize your bandwidth, then increasing
+this may help to speed up the transfers.`,
+ Default: 4,
+ Advanced: true,
+ }, {
+ Name: "upload_cutoff",
+ Help: `Cutoff for switching to chunked upload.
+
+Any files larger than this will be uploaded in chunks of chunk_size.
+The minimum is 0 and the maximum is 5 GiB.`,
+ Default: defaultUploadCutoff,
+ Advanced: true,
+ }, {
+ Name: config.ConfigEncoding,
+ Help: config.ConfigEncodingHelp,
+ Advanced: true,
+ Default: (encoder.Display | // Slash Control Delete Dot
+ encoder.EncodeLeftSpace |
+ encoder.EncodeBackSlash |
+ encoder.EncodeRightSpace |
+ encoder.EncodeInvalidUtf8),
+ }},
+ })
+}
+
+/*
+TestDrime{sb0-v}
+stringNeedsEscaping = []rune{
+ '/', '\\', '\a', '\b', '\f', '\n', '\r', '\t', '\v', '\x00', '\x01', '\x02', '\x03', '\x04', '\x05', '\x06', '\x0e', '\x0f', '\x10', '\x11', '\x12', '\x13', '\x14',
+ '\x15', '\x16', '\x17', '\x18', '\x19', '\x1a', '\x1b', '\x1c', '\x1d', '\x1e', '\x1f', '\x7f', '\xbf', '\xfe'
+}
+maxFileLength = 255 // for 1 byte unicode characters
+maxFileLength = 127 // for 2 byte unicode characters
+maxFileLength = 85 // for 3 byte unicode characters
+maxFileLength = 63 // for 4 byte unicode characters
+canWriteUnnormalized = true
+canReadUnnormalized = true
+canReadRenormalized = false
+canStream = true
+base32768isOK = true // make sure maxFileLength for 2 byte unicode chars is the same as for 1 byte characters
+*/
+
+// Options defines the configuration for this backend
+type Options struct {
+ AccessToken string `config:"access_token"`
+ RootFolderID string `config:"root_folder_id"`
+ WorkspaceID string `config:"workspace_id"`
+ UploadConcurrency int `config:"upload_concurrency"`
+ ChunkSize fs.SizeSuffix `config:"chunk_size"`
+ HardDelete bool `config:"hard_delete"`
+ UploadCutoff fs.SizeSuffix `config:"upload_cutoff"`
+ ListChunk int `config:"list_chunk"`
+ Enc encoder.MultiEncoder `config:"encoding"`
+}
+
+// Fs represents a remote drime
+type Fs struct {
+ name string // name of this remote
+ root string // the path we are working on
+ opt Options // parsed options
+ features *fs.Features // optional features
+ srv *rest.Client // the connection to the server
+ dirCache *dircache.DirCache // Map of directory path to directory id
+ pacer *fs.Pacer // pacer for API calls
+}
+
+// Object describes a drime object
+//
+// The full set of metadata will always be present
+type Object struct {
+ fs *Fs // what this object is part of
+ remote string // The remote path
+ size int64 // size of the object
+ modTime time.Time // modification time of the object
+ id string // ID of the object
+ dirID string // ID of the object's directory
+ mimeType string // mime type of the object
+ url string // where to download this object
+}
+
+// ------------------------------------------------------------
+
+func checkUploadChunkSize(cs fs.SizeSuffix) error {
+ if cs < minChunkSize {
+ return fmt.Errorf("%s is less than %s", cs, minChunkSize)
+ }
+ return nil
+}
+
+func (f *Fs) setUploadChunkSize(cs fs.SizeSuffix) (old fs.SizeSuffix, err error) {
+ err = checkUploadChunkSize(cs)
+ if err == nil {
+ old, f.opt.ChunkSize = f.opt.ChunkSize, cs
+ }
+ return
+}
+
+func (f *Fs) setUploadCutoff(cs fs.SizeSuffix) (old fs.SizeSuffix, err error) {
+ old, f.opt.UploadCutoff = f.opt.UploadCutoff, cs
+ return
+}
+
+// Name of the remote (as passed into NewFs)
+func (f *Fs) Name() string {
+ return f.name
+}
+
+// Root of the remote (as passed into NewFs)
+func (f *Fs) Root() string {
+ return f.root
+}
+
+// String converts this Fs to a string
+func (f *Fs) String() string {
+ return fmt.Sprintf("drime root '%s'", f.root)
+}
+
+// Features returns the optional features of this Fs
+func (f *Fs) Features() *fs.Features {
+ return f.features
+}
+
+// parsePath parses a drime 'url'
+func parsePath(path string) (root string) {
+ root = strings.Trim(path, "/")
+ return
+}
+
+// retryErrorCodes is a slice of error codes that we will retry
+var retryErrorCodes = []int{
+ 429, // Too Many Requests.
+ 500, // Internal Server Error
+ 502, // Bad Gateway
+ 503, // Service Unavailable
+ 504, // Gateway Timeout
+ 509, // Bandwidth Limit Exceeded
+}
+
+// shouldRetry returns a boolean as to whether this resp and err
+// deserve to be retried. It returns the err as a convenience
+func shouldRetry(ctx context.Context, resp *http.Response, err error) (bool, error) {
+ if fserrors.ContextError(ctx, &err) {
+ return false, err
+ }
+ return fserrors.ShouldRetry(err) || fserrors.ShouldRetryHTTP(resp, retryErrorCodes), err
+}
+
+// readMetaDataForPath reads the metadata from the path
+func (f *Fs) readMetaDataForPath(ctx context.Context, path string) (info *api.Item, err error) {
+ leaf, directoryID, err := f.dirCache.FindPath(ctx, path, false)
+ if err != nil {
+ if err == fs.ErrorDirNotFound {
+ return nil, fs.ErrorObjectNotFound
+ }
+ return nil, err
+ }
+
+ found, err := f.listAll(ctx, directoryID, false, true, leaf, func(item *api.Item) bool {
+ if item.Name == leaf {
+ info = item
+ return true
+ }
+ return false
+ })
+ if err != nil {
+ return nil, err
+ }
+ if !found {
+ return nil, fs.ErrorObjectNotFound
+ }
+ return info, nil
+}
+
+// getItem reads item for ID given
+func (f *Fs) getItem(ctx context.Context, id string, dirID string, leaf string) (info *api.Item, err error) {
+ found, err := f.listAll(ctx, dirID, false, true, leaf, func(item *api.Item) bool {
+ if item.ID.String() == id {
+ info = item
+ return true
+ }
+ return false
+ })
+ if !found {
+ return nil, fs.ErrorObjectNotFound
+ }
+ return info, err
+}
+
+// errorHandler parses a non 2xx error response into an error
+func errorHandler(resp *http.Response) error {
+ body, err := rest.ReadBody(resp)
+ if err != nil {
+ fs.Debugf(nil, "Couldn't read error out of body: %v", err)
+ body = nil
+ }
+ // Decode error response if there was one - they can be blank
+ var errResponse api.Error
+ if len(body) > 0 {
+ err = json.Unmarshal(body, &errResponse)
+ if err != nil {
+ fs.Debugf(nil, "Couldn't decode error response: %v", err)
+ }
+ }
+ if errResponse.Message == "" {
+ errResponse.Message = fmt.Sprintf("%s (%d): %s", resp.Status, resp.StatusCode, string(body))
+ }
+ return errResponse
+}
+
+// NewFs constructs an Fs from the path, container:path
+func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, error) {
+ // Parse config into Options struct
+ opt := new(Options)
+ err := configstruct.Set(m, opt)
+ if err != nil {
+ return nil, err
+ }
+
+ err = checkUploadChunkSize(opt.ChunkSize)
+ if err != nil {
+ return nil, fmt.Errorf("drime: chunk size: %w", err)
+ }
+
+ root = parsePath(root)
+
+ client := fshttp.NewClient(ctx)
+
+ f := &Fs{
+ name: name,
+ root: root,
+ opt: *opt,
+ srv: rest.NewClient(client).SetRoot(rootURL),
+ pacer: fs.NewPacer(ctx, pacer.NewDefault(pacer.MinSleep(minSleep), pacer.MaxSleep(maxSleep), pacer.DecayConstant(decayConstant))),
+ }
+ f.features = (&fs.Features{
+ CanHaveEmptyDirectories: true,
+ ReadMimeType: true,
+ WriteMimeType: true,
+ }).Fill(ctx, f)
+ f.srv.SetErrorHandler(errorHandler)
+ f.srv.SetHeader("Authorization", "Bearer "+f.opt.AccessToken)
+ f.srv.SetHeader("Accept", "application/json")
+
+ // Get rootFolderID
+ rootID := f.opt.RootFolderID
+ f.dirCache = dircache.New(root, rootID, f)
+
+ // Find the current root
+ err = f.dirCache.FindRoot(ctx, false)
+ if err != nil {
+ // Assume it is a file
+ newRoot, remote := dircache.SplitPath(root)
+ tempF := *f
+ tempF.dirCache = dircache.New(newRoot, rootID, &tempF)
+ tempF.root = newRoot
+ // Make new Fs which is the parent
+ err = tempF.dirCache.FindRoot(ctx, false)
+ if err != nil {
+ // No root so return old f
+ return f, nil
+ }
+ _, err := tempF.newObjectWithInfo(ctx, remote, nil)
+ if err != nil {
+ if err == fs.ErrorObjectNotFound {
+ // File doesn't exist so return old f
+ return f, nil
+ }
+ return nil, err
+ }
+ f.features.Fill(ctx, &tempF)
+ // XXX: update the old f here instead of returning tempF, since
+ // `features` were already filled with functions having *f as a receiver.
+ // See https://github.com/rclone/rclone/issues/2182
+ f.dirCache = tempF.dirCache
+ f.root = tempF.root
+ // return an error with an fs which points to the parent
+ return f, fs.ErrorIsFile
+ }
+ return f, nil
+}
+
+// rootSlash returns root with a slash on if it is empty, otherwise empty string
+func (f *Fs) rootSlash() string {
+ if f.root == "" {
+ return f.root
+ }
+ return f.root + "/"
+}
+
+// Return an Object from a path
+//
+// If it can't be found it returns the error fs.ErrorObjectNotFound.
+func (f *Fs) newObjectWithInfo(ctx context.Context, remote string, info *api.Item) (fs.Object, error) {
+ o := &Object{
+ fs: f,
+ remote: remote,
+ }
+ var err error
+ if info != nil {
+ // Set info
+ err = o.setMetaData(info)
+ } else {
+ err = o.readMetaData(ctx) // reads info and meta, returning an error
+ }
+ if err != nil {
+ return nil, err
+ }
+ return o, nil
+}
+
+// NewObject finds the Object at remote. If it can't be found
+// it returns the error fs.ErrorObjectNotFound.
+func (f *Fs) NewObject(ctx context.Context, remote string) (fs.Object, error) {
+ return f.newObjectWithInfo(ctx, remote, nil)
+}
+
+// FindLeaf finds a directory of name leaf in the folder with ID pathID
+func (f *Fs) FindLeaf(ctx context.Context, pathID, leaf string) (pathIDOut string, found bool, err error) {
+ // Find the leaf in pathID
+ found, err = f.listAll(ctx, pathID, true, false, leaf, func(item *api.Item) bool {
+ if item.Name == leaf {
+ pathIDOut = item.ID.String()
+ return true
+ }
+ return false
+ })
+ return pathIDOut, found, err
+}
+
+// createDir makes a directory with pathID as parent and name leaf and modTime
+func (f *Fs) createDir(ctx context.Context, pathID, leaf string, modTime time.Time) (item *api.Item, err error) {
+ var resp *http.Response
+ var result api.CreateFolderResponse
+ opts := rest.Opts{
+ Method: "POST",
+ Path: "/folders",
+ }
+ mkdir := api.CreateFolderRequest{
+ Name: f.opt.Enc.FromStandardName(leaf),
+ ParentID: json.Number(pathID),
+ }
+ err = f.pacer.Call(func() (bool, error) {
+ resp, err = f.srv.CallJSON(ctx, &opts, &mkdir, &result)
+ return shouldRetry(ctx, resp, err)
+ })
+ if err != nil {
+ return nil, fmt.Errorf("failed to create folder: %w", err)
+ }
+ return &result.Folder, nil
+}
+
+// CreateDir makes a directory with pathID as parent and name leaf
+func (f *Fs) CreateDir(ctx context.Context, pathID, leaf string) (newID string, err error) {
+ item, err := f.createDir(ctx, pathID, leaf, time.Now())
+ if err != nil {
+ return "", err
+ }
+ return item.ID.String(), nil
+}
+
+// list the objects into the function supplied
+//
+// If directories is set it only sends directories
+// User function to process a File item from listAll
+//
+// Should return true to finish processing
+type listAllFn func(*api.Item) bool
+
+// Lists the directory required calling the user function on each item found
+//
+// If name is set then the server will limit the returned items to those
+// with that name.
+//
+// If the user fn ever returns true then it early exits with found = true
+func (f *Fs) listAll(ctx context.Context, dirID string, directoriesOnly bool, filesOnly bool, name string, fn listAllFn) (found bool, err error) {
+ opts := rest.Opts{
+ Method: "GET",
+ Path: "/drive/file-entries",
+ Parameters: url.Values{},
+ }
+ if dirID != "" {
+ opts.Parameters.Add("parentIds", dirID)
+ }
+ if directoriesOnly {
+ opts.Parameters.Add("type", api.ItemTypeFolder)
+ }
+ if f.opt.WorkspaceID != "" {
+ opts.Parameters.Set("workspaceId", f.opt.WorkspaceID)
+ }
+ opts.Parameters.Set("perPage", strconv.Itoa(f.opt.ListChunk))
+ page := 1
+OUTER:
+ for {
+ opts.Parameters.Set("page", strconv.Itoa(page))
+ var result api.Listing
+ var resp *http.Response
+ err = f.pacer.Call(func() (bool, error) {
+ resp, err = f.srv.CallJSON(ctx, &opts, nil, &result)
+ return shouldRetry(ctx, resp, err)
+ })
+ if err != nil {
+ return found, fmt.Errorf("couldn't list files: %w", err)
+ }
+ for _, item := range result.Data {
+ if item.Type == api.ItemTypeFolder {
+ if filesOnly {
+ continue
+ }
+ } else {
+ if directoriesOnly {
+ continue
+ }
+ }
+ item.Name = f.opt.Enc.ToStandardName(item.Name)
+ if fn(&item) {
+ found = true
+ break OUTER
+ }
+ }
+ if result.NextPage == 0 {
+ break
+ }
+ page = result.NextPage
+ }
+ return found, err
+}
+
+// Convert a list item into a DirEntry
+func (f *Fs) itemToDirEntry(ctx context.Context, remote string, info *api.Item) (entry fs.DirEntry, err error) {
+ if info.Type == api.ItemTypeFolder {
+ // cache the directory ID for later lookups
+ f.dirCache.Put(remote, info.ID.String())
+ entry = fs.NewDir(remote, info.UpdatedAt).
+ SetSize(info.FileSize).
+ SetID(info.ID.String()).
+ SetParentID(info.ParentID.String())
+ } else {
+ entry, err = f.newObjectWithInfo(ctx, remote, info)
+ if err != nil {
+ return nil, err
+ }
+ }
+ return entry, nil
+}
+
+// List the objects and directories in dir into entries. The
+// entries can be returned in any order but should be for a
+// complete directory.
+//
+// dir should be "" to list the root, and should not have
+// trailing slashes.
+//
+// This should return ErrDirNotFound if the directory isn't
+// found.
+func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err error) {
+ directoryID, err := f.dirCache.FindDir(ctx, dir, false)
+ if err != nil {
+ return nil, err
+ }
+ var iErr error
+ _, err = f.listAll(ctx, directoryID, false, false, "", func(info *api.Item) bool {
+ remote := path.Join(dir, info.Name)
+ entry, err := f.itemToDirEntry(ctx, remote, info)
+ if err != nil {
+ iErr = err
+ return true
+ }
+ entries = append(entries, entry)
+ return false
+ })
+ if err != nil {
+ return nil, err
+ }
+ if iErr != nil {
+ return nil, iErr
+ }
+ return entries, nil
+}
+
+// Creates from the parameters passed in a half finished Object which
+// must have setMetaData called on it
+//
+// Returns the object, leaf, directoryID and error.
+//
+// Used to create new objects
+func (f *Fs) createObject(ctx context.Context, remote string, modTime time.Time, size int64) (o *Object, leaf string, directoryID string, err error) {
+ // Create the directory for the object if it doesn't exist
+ leaf, directoryID, err = f.dirCache.FindPath(ctx, remote, true)
+ if err != nil {
+ return
+ }
+ // Temporary Object under construction
+ o = &Object{
+ fs: f,
+ remote: remote,
+ }
+ return o, leaf, directoryID, nil
+}
+
+// Put the object
+//
+// Copy the reader in to the new object which is returned.
+//
+// The new object may have been created if an error is returned
+func (f *Fs) Put(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) {
+ existingObj, err := f.NewObject(ctx, src.Remote())
+ switch err {
+ case nil:
+ return existingObj, existingObj.Update(ctx, in, src, options...)
+ case fs.ErrorObjectNotFound:
+ // Not found so create it
+ return f.PutUnchecked(ctx, in, src, options...)
+ default:
+ return nil, err
+ }
+}
+
+// PutStream uploads to the remote path with the modTime given of indeterminate size
+func (f *Fs) PutStream(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) {
+ return f.Put(ctx, in, src, options...)
+}
+
+// PutUnchecked the object into the container
+//
+// This will produce a duplicate if the object already exists.
+//
+// Copy the reader in to the new object which is returned.
+//
+// The new object may have been created if an error is returned
+func (f *Fs) PutUnchecked(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) {
+ remote := src.Remote()
+ size := src.Size()
+ modTime := src.ModTime(ctx)
+
+ o, _, _, err := f.createObject(ctx, remote, modTime, size)
+ if err != nil {
+ return nil, err
+ }
+ return o, o.Update(ctx, in, src, options...)
+}
+
+// Mkdir creates the container if it doesn't exist
+func (f *Fs) Mkdir(ctx context.Context, dir string) error {
+ _, err := f.dirCache.FindDir(ctx, dir, true)
+ return err
+}
+
+// deleteObject removes an object by ID
+func (f *Fs) deleteObject(ctx context.Context, id string) error {
+ opts := rest.Opts{
+ Method: "POST",
+ Path: "/file-entries/delete",
+ }
+ request := api.DeleteRequest{
+ EntryIDs: []string{id},
+ DeleteForever: f.opt.HardDelete,
+ }
+ var result api.DeleteResponse
+ err := f.pacer.Call(func() (bool, error) {
+ resp, err := f.srv.CallJSON(ctx, &opts, &request, &result)
+ return shouldRetry(ctx, resp, err)
+ })
+ if err != nil {
+ return fmt.Errorf("failed to delete item: %w", err)
+ }
+ // Check the individual result codes also
+ for name, errstring := range result.Errors {
+ return fmt.Errorf("failed to delete item %q: %s", name, errstring)
+ }
+ return nil
+}
+
+// purgeCheck removes the root directory, if check is set then it
+// refuses to do so if it has anything in
+func (f *Fs) purgeCheck(ctx context.Context, dir string, check bool) error {
+ root := path.Join(f.root, dir)
+ if root == "" {
+ return errors.New("can't purge root directory")
+ }
+ dc := f.dirCache
+ rootID, err := dc.FindDir(ctx, dir, false)
+ if err != nil {
+ return err
+ }
+
+ // Check to see if there is contents in the directory
+ if check {
+ found, err := f.listAll(ctx, rootID, false, false, "", func(item *api.Item) bool {
+ return true
+ })
+ if err != nil {
+ return err
+ }
+ if found {
+ return fs.ErrorDirectoryNotEmpty
+ }
+ }
+
+ // Delete the directory
+ err = f.deleteObject(ctx, rootID)
+ if err != nil {
+ return err
+ }
+
+ f.dirCache.FlushDir(dir)
+ return nil
+}
+
+// Rmdir deletes the root folder
+//
+// Returns an error if it isn't empty
+func (f *Fs) Rmdir(ctx context.Context, dir string) error {
+ return f.purgeCheck(ctx, dir, true)
+}
+
+// Precision return the precision of this Fs
+func (f *Fs) Precision() time.Duration {
+ return fs.ModTimeNotSupported
+}
+
+// Purge deletes all the files and the container
+//
+// Optional interface: Only implement this if you have a way of
+// deleting all the files quicker than just running Remove() on the
+// result of List()
+func (f *Fs) Purge(ctx context.Context, dir string) error {
+ return f.purgeCheck(ctx, dir, false)
+}
+
+// patch an attribute on an object to value
+func (f *Fs) patch(ctx context.Context, id, attribute string, value string) (item *api.Item, err error) {
+ var resp *http.Response
+ var request = api.UpdateItemRequest{
+ Name: value,
+ }
+ var result api.UpdateItemResponse
+ opts := rest.Opts{
+ Method: "PUT",
+ Path: "/file-entries/" + id,
+ }
+ err = f.pacer.Call(func() (bool, error) {
+ resp, err = f.srv.CallJSON(ctx, &opts, &request, &result)
+ return shouldRetry(ctx, resp, err)
+ })
+ if err != nil {
+ return nil, fmt.Errorf("failed to patch item %q to %v: %w", attribute, value, err)
+ }
+ return &result.FileEntry, nil
+}
+
+// rename a file or a folder
+func (f *Fs) rename(ctx context.Context, id, newLeaf string) (item *api.Item, err error) {
+ return f.patch(ctx, id, "name", f.opt.Enc.FromStandardName(newLeaf))
+}
+
+// move a file or a folder to a new directory
+// func (f *Fs) move(ctx context.Context, id, newDirID string, dstLeaf string) (item *api.Item, err error) {
+func (f *Fs) move(ctx context.Context, id, newDirID string) (err error) {
+ var resp *http.Response
+ var request = api.MoveRequest{
+ EntryIDs: []string{id},
+ DestinationID: newDirID,
+ }
+ var result api.MoveResponse
+ opts := rest.Opts{
+ Method: "POST",
+ Path: "/file-entries/move",
+ }
+ err = f.pacer.Call(func() (bool, error) {
+ resp, err = f.srv.CallJSON(ctx, &opts, &request, &result)
+ return shouldRetry(ctx, resp, err)
+ })
+ if err != nil {
+ return fmt.Errorf("failed to move item: %w", err)
+ }
+
+ return nil
+}
+
+// move and rename a file or folder to directoryID with leaf
+func (f *Fs) moveTo(ctx context.Context, id, srcLeaf, dstLeaf, srcDirectoryID, dstDirectoryID string) (info *api.Item, err error) {
+ newLeaf := f.opt.Enc.FromStandardName(dstLeaf)
+ oldLeaf := f.opt.Enc.FromStandardName(srcLeaf)
+ doRenameLeaf := oldLeaf != newLeaf
+ doMove := srcDirectoryID != dstDirectoryID
+
+ // Now rename the leaf to a temporary name if we are moving to
+ // another directory to make sure we don't overwrite something
+ // in the destination directory by accident
+ if doRenameLeaf && doMove {
+ tmpLeaf := newLeaf + "." + random.String(8)
+ info, err = f.rename(ctx, id, tmpLeaf)
+ if err != nil {
+ return nil, fmt.Errorf("Move rename leaf: %w", err)
+ }
+ }
+
+ // Move the object to a new directory (with the existing name)
+ // if required
+ if doMove {
+ err = f.move(ctx, id, dstDirectoryID)
+ if err != nil {
+ return nil, err
+ }
+ }
+
+ // Rename the leaf to its final name if required
+ if doRenameLeaf {
+ info, err = f.rename(ctx, id, newLeaf)
+ if err != nil {
+ return nil, fmt.Errorf("Move rename leaf: %w", err)
+ }
+ }
+
+ if info == nil {
+ info, err = f.getItem(ctx, id, dstDirectoryID, dstLeaf)
+ if err != nil {
+ return nil, err
+ }
+ }
+
+ return info, nil
+}
+
+// Move src to this remote using server-side move operations.
+//
+// This is stored with the remote path given.
+//
+// It returns the destination Object and a possible error.
+//
+// Will only be called if src.Fs().Name() == f.Name()
+//
+// If it isn't possible then return fs.ErrorCantMove
+func (f *Fs) Move(ctx context.Context, src fs.Object, remote string) (fs.Object, error) {
+ srcObj, ok := src.(*Object)
+ if !ok {
+ fs.Debugf(src, "Can't move - not same remote type")
+ return nil, fs.ErrorCantMove
+ }
+
+ // Find existing object
+ srcLeaf, srcDirectoryID, err := srcObj.fs.dirCache.FindPath(ctx, srcObj.remote, false)
+ if err != nil {
+ return nil, err
+ }
+
+ // Create temporary object
+ dstObj, dstLeaf, dstDirectoryID, err := f.createObject(ctx, remote, srcObj.modTime, srcObj.size)
+ if err != nil {
+ return nil, err
+ }
+
+ // Do the move
+ info, err := f.moveTo(ctx, srcObj.id, srcLeaf, dstLeaf, srcDirectoryID, dstDirectoryID)
+ if err != nil {
+ return nil, err
+ }
+
+ err = dstObj.setMetaData(info)
+ if err != nil {
+ return nil, err
+ }
+ return dstObj, nil
+}
+
+// DirMove moves src, srcRemote to this remote at dstRemote
+// using server-side move operations.
+//
+// Will only be called if src.Fs().Name() == f.Name()
+//
+// If it isn't possible then return fs.ErrorCantDirMove
+//
+// If destination exists then return fs.ErrorDirExists
+func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string) error {
+ srcFs, ok := src.(*Fs)
+ if !ok {
+ fs.Debugf(srcFs, "Can't move directory - not same remote type")
+ return fs.ErrorCantDirMove
+ }
+
+ srcID, srcDirectoryID, srcLeaf, dstDirectoryID, dstLeaf, err := f.dirCache.DirMove(ctx, srcFs.dirCache, srcFs.root, srcRemote, f.root, dstRemote)
+ if err != nil {
+ return err
+ }
+
+ // Do the move
+ _, err = f.moveTo(ctx, srcID, srcLeaf, dstLeaf, srcDirectoryID, dstDirectoryID)
+ if err != nil {
+ return err
+ }
+ srcFs.dirCache.FlushDir(srcRemote)
+ return nil
+}
+
+// copy a file or a folder to a new directory
+func (f *Fs) copy(ctx context.Context, id, newDirID string) (item *api.Item, err error) {
+ var resp *http.Response
+ var request = api.CopyRequest{
+ EntryIDs: []string{id},
+ DestinationID: newDirID,
+ }
+ var result api.CopyResponse
+ opts := rest.Opts{
+ Method: "POST",
+ Path: "/file-entries/duplicate",
+ }
+ err = f.pacer.Call(func() (bool, error) {
+ resp, err = f.srv.CallJSON(ctx, &opts, &request, &result)
+ return shouldRetry(ctx, resp, err)
+ })
+ if err != nil {
+ return nil, fmt.Errorf("failed to copy item: %w", err)
+ }
+ itemResult := result.Entries[0]
+ return &itemResult, nil
+}
+
+// copy and rename a file or folder to directoryID with leaf
+func (f *Fs) copyTo(ctx context.Context, srcID, srcLeaf, dstLeaf, dstDirectoryID string) (info *api.Item, err error) {
+ // Can have duplicates so don't have to be careful here
+
+ // Copy to dstDirectoryID first
+ info, err = f.copy(ctx, srcID, dstDirectoryID)
+ if err != nil {
+ return nil, err
+ }
+
+ // Rename if required
+ if srcLeaf != dstLeaf {
+ info, err = f.rename(ctx, info.ID.String(), dstLeaf)
+ if err != nil {
+ return nil, err
+ }
+ }
+ return info, nil
+}
+
+// Copy src to this remote using server-side copy operations.
+//
+// This is stored with the remote path given.
+//
+// It returns the destination Object and a possible error.
+//
+// Will only be called if src.Fs().Name() == f.Name()
+//
+// If it isn't possible then return fs.ErrorCantCopy
+func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (dst fs.Object, err error) {
+ srcObj, ok := src.(*Object)
+ if !ok {
+ fs.Debugf(src, "Can't copy - not same remote type")
+ return nil, fs.ErrorCantCopy
+ }
+ srcLeaf := path.Base(srcObj.remote)
+
+ srcPath := srcObj.fs.rootSlash() + srcObj.remote
+ dstPath := f.rootSlash() + remote
+ if srcPath == dstPath {
+ return nil, fmt.Errorf("can't copy %q -> %q as are same name", srcPath, dstPath)
+ }
+
+ // Find existing object
+ existingObj, err := f.NewObject(ctx, remote)
+ if err == nil {
+ defer func() {
+ // Don't remove existing object if returning an error
+ if err != nil {
+ return
+ }
+ fs.Debugf(existingObj, "Server side copy: removing existing object after successful copy")
+ err = existingObj.Remove(ctx)
+ }()
+ }
+
+ // Create temporary object
+ dstObj, dstLeaf, dstDirectoryID, err := f.createObject(ctx, remote, srcObj.modTime, srcObj.size)
+ if err != nil {
+ return nil, err
+ }
+
+ // Copy the object
+ info, err := f.copyTo(ctx, srcObj.id, srcLeaf, dstLeaf, dstDirectoryID)
+ if err != nil {
+ return nil, err
+ }
+ err = dstObj.setMetaData(info)
+ if err != nil {
+ return nil, err
+ }
+
+ return dstObj, nil
+}
+
+// DirCacheFlush resets the directory cache - used in testing as an
+// optional interface
+func (f *Fs) DirCacheFlush() {
+ f.dirCache.ResetRoot()
+}
+
+// Hashes returns the supported hash sets.
+func (f *Fs) Hashes() hash.Set {
+ return hash.Set(hash.None)
+}
+
+var warnStreamUpload sync.Once
+
+// Status of open chunked upload
+type drimeChunkWriter struct {
+ uploadID string
+ key string
+ chunkSize int64
+ size int64
+ f *Fs
+ o *Object
+ written atomic.Int64
+ fileEntry api.Item
+
+ uploadName string
+ leaf string
+ mime string
+ extension string
+ parentID json.Number
+ relativePath string
+
+ completedPartsMu sync.Mutex
+ completedParts []api.CompletedPart
+}
+
+// OpenChunkWriter returns the chunk size and a ChunkWriter
+//
+// Pass in the remote and the src object
+// You can also use options to hint at the desired chunk size
+func (f *Fs) OpenChunkWriter(ctx context.Context, remote string, src fs.ObjectInfo, options ...fs.OpenOption) (info fs.ChunkWriterInfo, writer fs.ChunkWriter, err error) {
+ // Create the directory for the object if it doesn't exist
+ leaf, directoryID, err := f.dirCache.FindPath(ctx, remote, true)
+ if err != nil {
+ return info, nil, err
+ }
+
+ // Temporary Object under construction
+ o := &Object{
+ fs: f,
+ remote: remote,
+ }
+
+ size := src.Size()
+ fs.FixRangeOption(options, size)
+
+ // calculate size of parts
+ chunkSize := f.opt.ChunkSize
+
+ // size can be -1 here meaning we don't know the size of the incoming file. We use ChunkSize
+ // buffers here (default 5 MB). With a maximum number of parts (10,000) this will be a file of
+ // 48 GB.
+ if size == -1 {
+ warnStreamUpload.Do(func() {
+ fs.Logf(f, "Streaming uploads using chunk size %v will have maximum file size of %v",
+ chunkSize, fs.SizeSuffix(int64(chunkSize)*int64(maxUploadParts)))
+ })
+ } else {
+ chunkSize = chunksize.Calculator(src, size, maxUploadParts, chunkSize)
+ }
+
+ createSize := max(0, size)
+
+ // Initiate multipart upload
+ req := api.MultiPartCreateRequest{
+ Filename: leaf,
+ Mime: fs.MimeType(ctx, src),
+ Size: createSize,
+ Extension: strings.TrimPrefix(path.Ext(leaf), `.`),
+ ParentID: json.Number(directoryID),
+ RelativePath: f.opt.Enc.FromStandardPath(path.Join(f.root, remote)),
+ }
+
+ var resp api.MultiPartCreateResponse
+
+ opts := rest.Opts{
+ Method: "POST",
+ Path: "/s3/multipart/create",
+ Options: options,
+ }
+
+ err = o.fs.pacer.Call(func() (bool, error) {
+ res, err := o.fs.srv.CallJSON(ctx, &opts, req, &resp)
+ return shouldRetry(ctx, res, err)
+ })
+
+ if err != nil {
+ return info, nil, fmt.Errorf("failed to initiate multipart upload: %w", err)
+ }
+
+ mime := fs.MimeType(ctx, src)
+ ext := strings.TrimPrefix(path.Ext(leaf), ".")
+ // must have file extension for multipart upload
+ if ext == "" {
+ ext = "bin"
+ }
+ rel := f.opt.Enc.FromStandardPath(path.Join(f.root, remote))
+
+ chunkWriter := &drimeChunkWriter{
+ uploadID: resp.UploadID,
+ key: resp.Key,
+ chunkSize: int64(chunkSize),
+ size: size,
+ f: f,
+ o: o,
+ uploadName: path.Base(resp.Key),
+ leaf: leaf,
+ mime: mime,
+ extension: ext,
+ parentID: json.Number(directoryID),
+ relativePath: rel,
+ }
+ info = fs.ChunkWriterInfo{
+ ChunkSize: int64(chunkSize),
+ Concurrency: f.opt.UploadConcurrency,
+ LeavePartsOnError: false,
+ }
+ return info, chunkWriter, err
+}
+
+// WriteChunk will write chunk number with reader bytes, where chunk number >= 0
+func (s *drimeChunkWriter) WriteChunk(ctx context.Context, chunkNumber int, reader io.ReadSeeker) (bytesWritten int64, err error) {
+ // chunk numbers between 1 and 100000
+ chunkNumber++
+
+ // find size of chunk
+ chunkSize, err := reader.Seek(0, io.SeekEnd)
+ if err != nil {
+ return 0, fmt.Errorf("failed to seek chunk: %w", err)
+ }
+
+ if chunkSize == 0 && chunkNumber != 1 {
+ return 0, nil
+ }
+
+ partOpts := rest.Opts{
+ Method: "POST",
+ Path: "/s3/multipart/batch-sign-part-urls",
+ }
+
+ req := api.MultiPartGetURLsRequest{
+ UploadID: s.uploadID,
+ Key: s.key,
+ PartNumbers: []int{
+ chunkNumber,
+ },
+ }
+
+ var resp api.MultiPartGetURLsResponse
+
+ err = s.f.pacer.Call(func() (bool, error) {
+ res, err := s.f.srv.CallJSON(ctx, &partOpts, req, &resp)
+ return shouldRetry(ctx, res, err)
+ })
+
+ if err != nil {
+ return 0, fmt.Errorf("failed to get part URL: %w", err)
+ }
+
+ if len(resp.URLs) != 1 {
+ return 0, fmt.Errorf("expecting 1 URL but got %d", len(resp.URLs))
+ }
+ partURL := resp.URLs[0].URL
+
+ opts := rest.Opts{
+ Method: "PUT",
+ RootURL: partURL,
+ Body: reader,
+ ContentType: "application/octet-stream",
+ ContentLength: &chunkSize,
+ NoResponse: true,
+ ExtraHeaders: map[string]string{
+ "Authorization": "", // clear the default auth
+ },
+ }
+
+ var uploadRes *http.Response
+
+ err = s.f.pacer.Call(func() (bool, error) {
+ _, err = reader.Seek(0, io.SeekStart)
+ if err != nil {
+ return false, fmt.Errorf("failed to seek chunk: %w", err)
+ }
+ uploadRes, err = s.f.srv.Call(ctx, &opts)
+ return shouldRetry(ctx, uploadRes, err)
+ })
+
+ if err != nil {
+ return 0, fmt.Errorf("failed to upload part %d: %w", chunkNumber, err)
+ }
+
+ // Get ETag from response
+ etag := uploadRes.Header.Get("ETag")
+ fs.CheckClose(uploadRes.Body, &err)
+
+ s.completedPartsMu.Lock()
+ defer s.completedPartsMu.Unlock()
+ s.completedParts = append(s.completedParts, api.CompletedPart{
+ PartNumber: int32(chunkNumber),
+ ETag: etag,
+ })
+
+ // Count size written for unknown file sizes
+ s.written.Add(chunkSize)
+
+ return chunkSize, nil
+}
+
+// Close complete chunked writer finalising the file.
+func (s *drimeChunkWriter) Close(ctx context.Context) error {
+ s.completedPartsMu.Lock()
+ defer s.completedPartsMu.Unlock()
+
+ // Complete multipart upload
+ sort.Slice(s.completedParts, func(i, j int) bool {
+ return s.completedParts[i].PartNumber < s.completedParts[j].PartNumber
+ })
+
+ completeBody := api.MultiPartCompleteRequest{
+ UploadID: s.uploadID,
+ Key: s.key,
+ Parts: s.completedParts,
+ }
+
+ completeOpts := rest.Opts{
+ Method: "POST",
+ Path: "/s3/multipart/complete",
+ }
+
+ var response api.MultiPartCompleteResponse
+
+ err := s.f.pacer.Call(func() (bool, error) {
+ res, err := s.f.srv.CallJSON(ctx, &completeOpts, completeBody, &response)
+ return shouldRetry(ctx, res, err)
+ })
+
+ if err != nil {
+ return fmt.Errorf("failed to complete multipart upload: %w", err)
+ }
+
+ finalSize := s.size
+ if finalSize < 0 {
+ finalSize = s.written.Load()
+ }
+
+ // s3/entries request to create drime object from multipart upload
+ req := api.MultiPartEntriesRequest{
+ ClientMime: s.mime,
+ ClientName: s.leaf,
+ Filename: s.uploadName,
+ Size: finalSize,
+ ClientExtension: s.extension,
+ ParentID: s.parentID,
+ RelativePath: s.relativePath,
+ }
+
+ entriesOpts := rest.Opts{
+ Method: "POST",
+ Path: "/s3/entries",
+ }
+
+ var res api.MultiPartEntriesResponse
+ err = s.f.pacer.Call(func() (bool, error) {
+ res, err := s.f.srv.CallJSON(ctx, &entriesOpts, req, &res)
+ return shouldRetry(ctx, res, err)
+ })
+ if err != nil {
+ return fmt.Errorf("failed to create entry after multipart upload: %w", err)
+ }
+ s.fileEntry = res.FileEntry
+
+ return nil
+}
+
+// Abort chunk write
+//
+// You can and should call Abort without calling Close.
+func (s *drimeChunkWriter) Abort(ctx context.Context) error {
+ opts := rest.Opts{
+ Method: "POST",
+ Path: "/s3/multipart/abort",
+ NoResponse: true,
+ }
+
+ req := api.MultiPartAbort{
+ UploadID: s.uploadID,
+ Key: s.key,
+ }
+
+ err := s.f.pacer.Call(func() (bool, error) {
+ res, err := s.f.srv.CallJSON(ctx, &opts, req, nil)
+ return shouldRetry(ctx, res, err)
+ })
+
+ if err != nil {
+ return fmt.Errorf("failed to abort multipart upload: %w", err)
+ }
+
+ return nil
+}
+
+// ------------------------------------------------------------
+
+// Fs returns the parent Fs
+func (o *Object) Fs() fs.Info {
+ return o.fs
+}
+
+// String returns a string version
+func (o *Object) String() string {
+ if o == nil {
+ return "<nil>"
+ }
+ return o.remote
+}
+
+// Remote returns the remote path
+func (o *Object) Remote() string {
+ return o.remote
+}
+
+// Hash returns the hash of an object returning a lowercase hex string
+func (o *Object) Hash(ctx context.Context, t hash.Type) (string, error) {
+ return "", hash.ErrUnsupported
+}
+
+// Size returns the size of an object in bytes
+func (o *Object) Size() int64 {
+ return o.size
+}
+
+// setMetaDataAny sets the metadata from info but doesn't check the type
+func (o *Object) setMetaDataAny(info *api.Item) {
+ o.size = info.FileSize
+ o.modTime = info.UpdatedAt
+ o.id = info.ID.String()
+ o.dirID = info.ParentID.String()
+ o.mimeType = info.Mime
+ o.url = info.URL
+}
+
+// setMetaData sets the metadata from info
+func (o *Object) setMetaData(info *api.Item) (err error) {
+ if info.Type == api.ItemTypeFolder {
+ return fs.ErrorIsDir
+ }
+ if info.ID == "" {
+ return fmt.Errorf("ID not found in response")
+ }
+ o.setMetaDataAny(info)
+ return nil
+}
+
+// readMetaData gets the metadata unconditionally as we expect Object
+// to always have the full set of metadata
+func (o *Object) readMetaData(ctx context.Context) (err error) {
+ var info *api.Item
+ info, err = o.fs.readMetaDataForPath(ctx, o.remote)
+ if err != nil {
+ return err
+ }
+ return o.setMetaData(info)
+}
+
+// ModTime returns the modification time of the object
+//
+// It attempts to read the objects mtime and if that isn't present the
+// LastModified returned in the http headers
+func (o *Object) ModTime(ctx context.Context) time.Time {
+ return o.modTime
+}
+
+// SetModTime sets the modification time of the local fs object
+func (o *Object) SetModTime(ctx context.Context, modTime time.Time) error {
+ return fs.ErrorCantSetModTime
+}
+
+// Storable returns a boolean showing whether this object storable
+func (o *Object) Storable() bool {
+ return true
+}
+
+// Open an object for read
+func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.ReadCloser, err error) {
+ if o.id == "" {
+ return nil, errors.New("can't download - no id")
+ }
+ if o.url == "" {
+ // On upload an Object is returned with no url, so fetch it here if needed
+ err = o.readMetaData(ctx)
+ if err != nil {
+ return nil, fmt.Errorf("read metadata: %w", err)
+ }
+ }
+ fs.FixRangeOption(options, o.size)
+ var resp *http.Response
+ opts := rest.Opts{
+ Method: "GET",
+ RootURL: baseURL + o.url,
+ Options: options,
+ }
+
+ err = o.fs.pacer.Call(func() (bool, error) {
+ resp, err = o.fs.srv.Call(ctx, &opts)
+ return shouldRetry(ctx, resp, err)
+ })
+ if err != nil {
+ return nil, err
+ }
+ return resp.Body, err
+}
+
+// Update the object with the contents of the io.Reader, modTime and size
+//
+// If existing is set then it updates the object rather than creating a new one.
+//
+// The new object may have been created if an error is returned.
+func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (err error) {
+ remote := o.Remote()
+ size := src.Size()
+
+ // Create the directory for the object if it doesn't exist
+ leaf, directoryID, err := o.fs.dirCache.FindPath(ctx, remote, true)
+ if err != nil {
+ return err
+ }
+
+ // If the file exists, delete it after a successful upload
+ if o.id != "" {
+ id := o.id
+ o.id = ""
+ defer func() {
+ if err != nil {
+ return
+ }
+ fs.Debugf(o, "Removing old object on successful upload")
+ deleteErr := o.fs.deleteObject(ctx, id)
+ if deleteErr != nil {
+ err = fmt.Errorf("failed to delete existing object: %w", deleteErr)
+ }
+ }()
+ }
+
+ if size < 0 || size > int64(o.fs.opt.UploadCutoff) {
+ chunkWriter, err := multipart.UploadMultipart(ctx, src, in, multipart.UploadMultipartOptions{
+ Open: o.fs,
+ OpenOptions: options,
+ })
+ if err != nil {
+ return err
+ }
+ s := chunkWriter.(*drimeChunkWriter)
+
+ return o.setMetaData(&s.fileEntry)
+ }
+
+ // Do the upload
+ var resp *http.Response
+ var result api.UploadResponse
+ var encodedLeaf = o.fs.opt.Enc.FromStandardName(leaf)
+ opts := rest.Opts{
+ Method: "POST",
+ Body: in,
+ MultipartParams: url.Values{
+ "parentId": {directoryID},
+ "relativePath": {encodedLeaf},
+ },
+ MultipartContentName: "file",
+ MultipartFileName: encodedLeaf,
+ MultipartContentType: fs.MimeType(ctx, src),
+ Path: "/uploads",
+ Options: options,
+ }
+ err = o.fs.pacer.CallNoRetry(func() (bool, error) {
+ resp, err = o.fs.srv.CallJSON(ctx, &opts, nil, &result)
+ return shouldRetry(ctx, resp, err)
+ })
+ if err != nil {
+ return fmt.Errorf("failed to upload file: %w", err)
+ }
+ return o.setMetaData(&result.FileEntry)
+}
+
+// Remove an object
+func (o *Object) Remove(ctx context.Context) error {
+ return o.fs.deleteObject(ctx, o.id)
+}
+
+// ID returns the ID of the Object if known, or "" if not
+func (o *Object) ID() string {
+ return o.id
+}
+
+// MimeType returns the content type of the Object if known, or "" if not
+func (o *Object) MimeType(ctx context.Context) string {
+ return o.mimeType
+}
+
+// ParentID returns the ID of the Object parent if known, or "" if not
+func (o *Object) ParentID() string {
+ return o.dirID
+}
+
+// Check the interfaces are satisfied
+var (
+ _ fs.Fs = (*Fs)(nil)
+ _ fs.Purger = (*Fs)(nil)
+ _ fs.PutStreamer = (*Fs)(nil)
+ _ fs.Copier = (*Fs)(nil)
+ _ fs.Mover = (*Fs)(nil)
+ _ fs.DirMover = (*Fs)(nil)
+ _ fs.DirCacheFlusher = (*Fs)(nil)
+ _ fs.OpenChunkWriter = (*Fs)(nil)
+ _ fs.Object = (*Object)(nil)
+ _ fs.IDer = (*Object)(nil)
+ _ fs.ParentIDer = (*Object)(nil)
+ _ fs.MimeTyper = (*Object)(nil)
+)
diff --git a/backend/drime/drime_test.go b/backend/drime/drime_test.go
new file mode 100644
index 000000000..37b72d6c7
--- /dev/null
+++ b/backend/drime/drime_test.go
@@ -0,0 +1,33 @@
+// Drime filesystem interface
+package drime
+
+import (
+ "testing"
+
+ "github.com/rclone/rclone/fs"
+ "github.com/rclone/rclone/fstest/fstests"
+)
+
+// TestIntegration runs integration tests against the remote
+func TestIntegration(t *testing.T) {
+ fstests.Run(t, &fstests.Opt{
+ RemoteName: "TestDrime:",
+ NilObject: (*Object)(nil),
+ ChunkedUpload: fstests.ChunkedUploadConfig{
+ MinChunkSize: minChunkSize,
+ },
+ })
+}
+
+func (f *Fs) SetUploadChunkSize(cs fs.SizeSuffix) (fs.SizeSuffix, error) {
+ return f.setUploadChunkSize(cs)
+}
+
+func (f *Fs) SetUploadCutoff(cs fs.SizeSuffix) (fs.SizeSuffix, error) {
+ return f.setUploadCutoff(cs)
+}
+
+var (
+ _ fstests.SetUploadChunkSizer = (*Fs)(nil)
+ _ fstests.SetUploadCutoffer = (*Fs)(nil)
+)
diff --git a/bin/make_manual.py b/bin/make_manual.py
index 437363857..e4593b213 100755
--- a/bin/make_manual.py
+++ b/bin/make_manual.py
@@ -43,6 +43,7 @@ docs = [
"compress.md",
"combine.md",
"doi.md",
+ "drime.md"
"dropbox.md",
"filefabric.md",
"filelu.md",
diff --git a/docs/content/_index.md b/docs/content/_index.md
index 04b668bd4..16cc1d9e4 100644
--- a/docs/content/_index.md
+++ b/docs/content/_index.md
@@ -129,6 +129,7 @@ WebDAV or S3, that work out of the box.)
{{< provider name="DigitalOcean Spaces" home="https://www.digitalocean.com/products/object-storage/" config="/s3/#digitalocean-spaces" >}}
{{< provider name="Digi Storage" home="https://storage.rcs-rds.ro/" config="/koofr/#digi-storage" >}}
{{< provider name="Dreamhost" home="https://www.dreamhost.com/cloud/storage/" config="/s3/#dreamhost" >}}
+{{< provider name="Drime" home="https://www.drime.cloud/" config="/drime/" >}}
{{< provider name="Dropbox" home="https://www.dropbox.com/" config="/dropbox/" >}}
{{< provider name="Enterprise File Fabric" home="https://storagemadeeasy.com/about/" config="/filefabric/" >}}
{{< provider name="Exaba" home="https://exaba.com/" config="/s3/#exaba" >}}
diff --git a/docs/content/docs.md b/docs/content/docs.md
index f3269ac61..780573626 100644
--- a/docs/content/docs.md
+++ b/docs/content/docs.md
@@ -43,6 +43,7 @@ See the following for detailed instructions for
- [Crypt](/crypt/) - to encrypt other remotes
- [DigitalOcean Spaces](/s3/#digitalocean-spaces)
- [Digi Storage](/koofr/#digi-storage)
+- [Drime](/drime/)
- [Dropbox](/dropbox/)
- [Enterprise File Fabric](/filefabric/)
- [FileLu Cloud Storage](/filelu/)
diff --git a/docs/content/drime.md b/docs/content/drime.md
new file mode 100644
index 000000000..7b8055e2a
--- /dev/null
+++ b/docs/content/drime.md
@@ -0,0 +1,244 @@
+---
+title: "Drime"
+description: "Rclone docs for Drime"
+versionIntroduced: "v1.73"
+---
+
+# {{< icon "fa fa-cloud" >}} Drime
+
+[Drime](https://drime.cloud/) is a cloud storage and transfer service focused
+on fast, resilient file delivery. It offers both free and paid tiers with
+emphasis on high-speed uploads and link sharing.
+
+To setup Drime you need to log in, navigate to Settings, Developer, and create a
+token to use as an API access key. Give it a sensible name and copy the token
+for use in the config.
+
+## Configuration
+
+Here is a run through of `rclone config` to make a remote called `remote`.
+
+Firstly run:
+
+
+```console
+rclone config
+```
+
+Then follow through the interactive setup:
+
+
+```text
+No remotes found, make a new one?
+n) New remote
+s) Set configuration password
+q) Quit config
+n/s/q> n
+
+Enter name for new remote.
+name> remote
+
+Option Storage.
+Type of storage to configure.
+Choose a number from below, or type in your own value.
+XX / Drime
+ \ (drime)
+Storage> drime
+
+Option access_token.
+API Access token
+You can get this from the web control panel.
+Enter a value. Press Enter to leave empty.
+access_token> YOUR_API_ACCESS_TOKEN
+
+Edit advanced config?
+y) Yes
+n) No (default)
+y/n> n
+
+Configuration complete.
+Options:
+- type: drime
+- access_token: YOUR_API_ACCESS_TOKEN
+Keep this "remote" remote?
+y) Yes this is OK (default)
+e) Edit this remote
+d) Delete this remote
+y/e/d> y
+```
+
+Once configured you can then use `rclone` like this (replace `remote` with the
+name you gave your remote):
+
+List directories and files in the top level of your Drime
+
+```console
+rclone lsf remote:
+```
+
+To copy a local directory to a Drime directory called backup
+
+```console
+rclone copy /home/source remote:backup
+```
+
+
+### Modification times and hashes
+
+Drime does not support modification times or hashes.
+
+This means that by default syncs will only use the size of the file to determine
+if it needs updating.
+
+You can use the `--update` flag which will use the time the object was uploaded.
+For many operations this is sufficient to determine if it has changed. However
+files created with timestamps in the past will be missed by the sync if using
+`--update`.
+
+
+### Restricted filename characters
+
+In addition to the [default restricted characters set](/overview/#restricted-characters)
+the following characters are also replaced:
+
+| Character | Value | Replacement |
+| --------- |:-----:|:-----------:|
+| \ | 0x5C | \ |
+
+File names can also not start or end with the following characters. These only
+get replaced if they are the first or last character in the name:
+
+| Character | Value | Replacement |
+| --------- |:-----:|:-----------:|
+| SP | 0x20 | ␠ |
+
+Invalid UTF-8 bytes will also be [replaced](/overview/#invalid-utf8),
+as they can't be used in JSON strings.
+
+### Root folder ID
+
+You can set the `root_folder_id` for rclone. This is the directory
+(identified by its `Folder ID`) that rclone considers to be the root
+of your Drime drive.
+
+Normally you will leave this blank and rclone will determine the
+correct root to use itself and fill in the value in the config file.
+
+However you can set this to restrict rclone to a specific folder
+hierarchy.
+
+In order to do this you will have to find the `Folder ID` of the
+directory you wish rclone to display.
+
+You can do this with rclone
+
+```console
+$ rclone lsf -Fip --dirs-only remote:
+d6341f53-ee65-4f29-9f59-d11e8070b2a0;Files/
+f4f5c9b8-6ece-478b-b03e-4538edfe5a1c;Photos/
+d50e356c-29ca-4b27-a3a7-494d91026e04;Videos/
+```
+
+The ID to use is the part before the `;` so you could set
+
+```text
+root_folder_id = d6341f53-ee65-4f29-9f59-d11e8070b2a0
+```
+
+To restrict rclone to the `Files` directory.
+
+<!-- autogenerated options start - DO NOT EDIT - instead edit fs.RegInfo in backend/drime/drime.go and run make backenddocs to verify --> <!-- markdownlint-disable-line line-length -->
+### Standard options
+
+Here are the Standard options specific to drime (Drime).
+
+#### --drime-access-token
+
+API Access token
+
+You can get this from the web control panel.
+
+Properties:
+
+- Config: access_token
+- Env Var: RCLONE_DRIME_ACCESS_TOKEN
+- Type: string
+- Required: false
+
+### Advanced options
+
+Here are the Advanced options specific to drime (Drime).
+
+#### --drime-root-folder-id
+
+ID of the root folder
+
+Leave this blank normally, rclone will fill it in automatically.
+
+If you want rclone to be restricted to a particular folder you can
+fill it in - see the docs for more info.
+
+
+Properties:
+
+- Config: root_folder_id
+- Env Var: RCLONE_DRIME_ROOT_FOLDER_ID
+- Type: string
+- Required: false
+
+#### --drime-workspace-id
+
+Account ID
+
+Leave this blank normally, rclone will fill it in automatically.
+
+
+Properties:
+
+- Config: workspace_id
+- Env Var: RCLONE_DRIME_WORKSPACE_ID
+- Type: string
+- Required: false
+
+#### --drime-list-chunk
+
+Number of items to list in each call
+
+Properties:
+
+- Config: list_chunk
+- Env Var: RCLONE_DRIME_LIST_CHUNK
+- Type: int
+- Default: 1000
+
+#### --drime-encoding
+
+The encoding for the backend.
+
+See the [encoding section in the overview](/overview/#encoding) for more info.
+
+Properties:
+
+- Config: encoding
+- Env Var: RCLONE_DRIME_ENCODING
+- Type: Encoding
+- Default: Slash,BackSlash,Del,Ctl,LeftSpace,RightSpace,InvalidUtf8,Dot
+
+#### --drime-description
+
+Description of the remote.
+
+Properties:
+
+- Config: description
+- Env Var: RCLONE_DRIME_DESCRIPTION
+- Type: string
+- Required: false
+
+<!-- autogenerated options stop -->
+
+## Limitations
+
+Drime only supports filenames up to 255 bytes in length, where filenames are
+encoded in UTF8.
+
diff --git a/docs/content/overview.md b/docs/content/overview.md
index 35789f4ae..52a244e91 100644
--- a/docs/content/overview.md
+++ b/docs/content/overview.md
@@ -23,6 +23,7 @@ Here is an overview of the major features of each cloud storage system.
| Box | SHA1 | R/W | Yes | No | - | - |
| Citrix ShareFile | MD5 | R/W | Yes | No | - | - |
| Cloudinary | MD5 | R | No | Yes | - | - |
+| Drime | - | - | No | No | R/W | - |
| Dropbox | DBHASH ¹ | R | Yes | No | - | - |
| Enterprise File Fabric | - | R/W | Yes | No | R/W | - |
| FileLu Cloud Storage | MD5 | R/W | No | Yes | R | - |
@@ -515,6 +516,7 @@ upon backend-specific capabilities.
| Backblaze B2 | No | Yes | No | No | Yes | Yes | Yes | Yes | Yes | No | No |
| Box | Yes | Yes | Yes | Yes | Yes | No | Yes | No | Yes | Yes | Yes |
| Citrix ShareFile | Yes | Yes | Yes | Yes | No | No | No | No | No | No | Yes |
+| Drime | Yes | Yes | Yes | Yes | No | No | Yes | Yes | No | No | Yes |
| Dropbox | Yes | Yes | Yes | Yes | No | No | Yes | No | Yes | Yes | Yes |
| Cloudinary | No | No | No | No | No | No | Yes | No | No | No | No |
| Enterprise File Fabric | Yes | Yes | Yes | Yes | Yes | No | No | No | No | No | Yes |
diff --git a/docs/layouts/chrome/navbar.html b/docs/layouts/chrome/navbar.html
index 3f0e7da84..b6b617ea3 100644
--- a/docs/layouts/chrome/navbar.html
+++ b/docs/layouts/chrome/navbar.html
@@ -66,6 +66,7 @@
<a class="dropdown-item" href="/sharefile/"><i class="fas fa-share-square fa-fw"></i> Citrix ShareFile</a>
<a class="dropdown-item" href="/crypt/"><i class="fa fa-lock fa-fw"></i> Crypt (encrypts the others)</a>
<a class="dropdown-item" href="/koofr/#digi-storage"><i class="fa fa-cloud fa-fw"></i> Digi Storage</a>
+ <a class="dropdown-item" href="/drime/"><i class="fab fa-cloud fa-fw"></i> Drime</a>
<a class="dropdown-item" href="/dropbox/"><i class="fab fa-dropbox fa-fw"></i> Dropbox</a>
<a class="dropdown-item" href="/filefabric/"><i class="fa fa-cloud fa-fw"></i> Enterprise File Fabric</a>
<a class="dropdown-item" href="/filelu/"><i class="fa fa-folder fa-fw"></i> FileLu Cloud Storage</a>
diff --git a/fstest/test_all/config.yaml b/fstest/test_all/config.yaml
index c36f6b3e7..d0ddcbd10 100644
--- a/fstest/test_all/config.yaml
+++ b/fstest/test_all/config.yaml
@@ -677,3 +677,9 @@ backends:
# with the parent backend having a different precision.
- TestServerSideCopyOverSelf
- TestServerSideMoveOverSelf
+ - backend: "drime"
+ remote: "TestDrime:"
+ ignoretests:
+ # The TestBisyncRemoteLocal/check_access_filters tests fail due to duplicated objects
+ - cmd/bisync
+ fastlist: false