"rsync for cloud storage" - Google Drive, S3, Dropbox, Backblaze B2, One Drive, Swift, Hubic, Wasabi, Google Cloud Storage, Azure Blob, Azure Files, Yandex Files
Go azure-blobazure-blob-storageazure-filesbackblaze-b2cloud-storagedropboxencryptionftpfuse-filesystemgogolanggoogle-cloud-storagegoogle-driveonedriveopenstack-swiftrclones3sftpsyncwebdav
// Package kdrive provides an interface to the kDrive
// object storage system.
package kdrive
import (
"context"
"errors"
"fmt"
"io"
"net/http"
"net/url"
"path"
"slices"
"strconv"
"strings"
"time"
"github.com/rclone/rclone/backend/kdrive/api"
"github.com/rclone/rclone/backend/kdrive/khash"
"github.com/rclone/rclone/fs"
"github.com/rclone/rclone/fs/config"
"github.com/rclone/rclone/fs/config/configmap"
"github.com/rclone/rclone/fs/config/configstruct"
"github.com/rclone/rclone/fs/fserrors"
"github.com/rclone/rclone/fs/hash"
"github.com/rclone/rclone/fs/list"
"github.com/rclone/rclone/lib/dircache"
"github.com/rclone/rclone/lib/encoder"
"github.com/rclone/rclone/lib/multipart"
"github.com/rclone/rclone/lib/oauthutil"
"github.com/rclone/rclone/lib/pacer"
"github.com/rclone/rclone/lib/rest"
"golang.org/x/text/unicode/norm"
)
const (
rcloneClientID = "EB9BE55E-C6EA-43BB-8719-262EE2BAED0E"
defaultEndpoint = "https://api.infomaniak.com"
minSleep = 10 * time.Millisecond
maxSleep = 2 * time.Second
decayConstant = 2 // bigger for slower decay, exponential
// userRootDirID is the ID of the user's private root directory
userRootDirID = "1"
)
var (
// Base config for how to auth
oauthConfig = &oauthutil.Config{
Scopes: []string{
"drive",
},
AuthURL: "https://login.infomaniak.com/authorize",
TokenURL: "https://login.infomaniak.com/token",
ClientID: rcloneClientID,
RedirectURL: oauthutil.RedirectLocalhostURL,
UsePKCE: true,
}
)
// Register with Fs
func init() {
fs.Register(&fs.RegInfo{
Name: "kdrive",
Description: "Infomaniak kDrive",
NewFs: NewFs,
Config: func(ctx context.Context, name string, m configmap.Mapper, config fs.ConfigIn) (*fs.ConfigOut, error) {
return oauthutil.ConfigOut("", &oauthutil.Options{
OAuth2Config: oauthConfig,
})
},
Options: append(oauthutil.SharedOptionsPKCE, []fs.Option{{
Name: config.ConfigEncoding,
Help: config.ConfigEncodingHelp,
Advanced: true,
Default: encoder.Display | encoder.EncodeLeftSpace | encoder.EncodeRightSpace | encoder.EncodeInvalidUtf8,
}, {
Name: "root_folder_id",
Help: `The folder to use as root.
You may use either one of the convenience shortcuts [private or common]
or an explicit folder ID.
- private: your user private directory
- common: the kDrive common directory, shared among users`,
Advanced: true,
Sensitive: true,
}, {
Name: "drive_id",
Help: `ID of the kDrive to use.
When showing a folder on kdrive, you can find the drive_id here:
https://ksuite.infomaniak.com/{account_id}/kdrive/app/drive/{drive_id}/files/...`,
Default: "",
}, {
Name: "chunk_size",
Help: `Files above this size are uploaded in chunks of this size.
The kDrive API accepts chunks between 1 MB and 1 GB, up to 10,000
chunks per file and a total file size of 1 TiB. The chunk size is
raised automatically for very large files to stay within the 10,000
chunks limit.`,
Default: fs.SizeSuffix(20 * 1024 * 1024),
Advanced: true,
}, {
Name: "upload_concurrency",
Help: `Concurrency for chunked uploads.`,
Default: 4,
Advanced: true,
}, {
Name: "endpoint",
Help: `The API endpoint to use.
Leave blank normally. There is no reason to change the endpoint except for internal use or beta-testing.`,
Advanced: true,
}}...),
})
}
// Options defines the configuration for this backend
type Options struct {
Enc encoder.MultiEncoder `config:"encoding"`
RootFolderID string `config:"root_folder_id"`
DriveID string `config:"drive_id"`
ChunkSize fs.SizeSuffix `config:"chunk_size"`
UploadConcurrency int `config:"upload_concurrency"`
Endpoint string `config:"endpoint"`
}
// Fs represents a remote kdrive
type Fs struct {
name string // name of this remote
root string // the path we are working on
opt Options // parsed options
features *fs.Features // optional features
srv *rest.Client // the connection to the server
dirCache *dircache.DirCache // Map of directory path to directory id
pacer *fs.Pacer // pacer for API calls
}
// Object describes a kdrive object
//
// Will definitely have info but maybe not meta
type Object struct {
fs *Fs // what this object is part of
remote string // The remote path
hasMetaData bool // whether info below has been set
size int64 // size of the object
modTime time.Time // modification time of the object
id string // ID of the object
xxh3 string // XXH3 if known
nestedHash bool // if XXH3 is a nested hash
}
// ------------------------------------------------------------
// Name of the remote (as passed into NewFs)
func (f *Fs) Name() string {
return f.name
}
// Root of the remote (as passed into NewFs)
func (f *Fs) Root() string {
return f.root
}
// String converts this Fs to a string
func (f *Fs) String() string {
return fmt.Sprintf("kdrive root '%s'", f.root)
}
// Features returns the optional features of this Fs
func (f *Fs) Features() *fs.Features {
return f.features
}
// parsePath parses a kdrive 'url'
func parsePath(path string) (root string) {
root = strings.Trim(path, "/")
return
}
// retryErrorCodes is a slice of error codes that we will retry
var retryErrorCodes = []int{
408, // Request Timeout
429, // Too Many Requests
500, // Internal Server Error
502, // Bad Gateway
503, // Service Unavailable
504, // Gateway Timeout
509, // Bandwidth Limit Exceeded
}
// shouldRetry returns a boolean as to whether this resp and err
// deserve to be retried. It returns the err as a convenience
func shouldRetry(ctx context.Context, resp *http.Response, err error) (bool, error) {
if fserrors.ContextError(ctx, &err) {
return false, err
}
// Server side copies hold a transient lock on the source and
// destination: retrying gives the concurrent copy time to finish.
if isLockedByInternalCopy(err) {
return true, err
}
return fserrors.ShouldRetry(err) || fserrors.ShouldRetryHTTP(resp, retryErrorCodes), err
}
// isLockedByInternalCopy checks if the error is a kDrive "lock_error"
// raised because the file/directory is locked by a server side copy.
// These locks are transient: the API releases them as soon as the copy
// completes, so the operation can be retried.
func isLockedByInternalCopy(err error) bool {
var apiErr *api.ResultStatus
return errors.As(err, &apiErr) &&
apiErr.ErrorDetail.Result == "lock_error" &&
slices.Contains(apiErr.ErrorDetail.Context.Values, "internal_copy")
}
// errorHandler parses a non 2xx error response into an error
func errorHandler(resp *http.Response) error {
// Decode error response
errResponse := new(api.ResultStatus)
err := rest.DecodeJSON(resp, &errResponse)
if err != nil {
fs.Debugf(nil, "Couldn't decode error response: %v", err)
}
if errResponse.ErrorDetail.ErrorString == "" {
errResponse.ErrorDetail.ErrorString = resp.Status
}
if errResponse.ErrorDetail.Result == "" {
errResponse.ErrorDetail.Result = resp.Status
}
return errResponse
}
// isNotFoundError checks if the given error is a standard kDrive API "Not Found" error.
func isNotFoundError(err error) bool {
var apiErr *api.ResultStatus
return errors.As(err, &apiErr) && apiErr.ErrorDetail.Result == "object_not_found"
}
// NewFs constructs an Fs from the path, container:path
func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, error) {
// Parse config into Options struct
opt := new(Options)
err := configstruct.Set(m, opt)
if err != nil {
return nil, err
}
// All API calls are scoped to the drive_id, so it is required
if opt.DriveID == "" {
return nil, errors.New("drive_id is required for kDrive")
}
// Validate the chunk size: an out of range value breaks uploads
if err := checkChunkSize(opt.ChunkSize); err != nil {
return nil, err
}
// Use the default API endpoint unless explicitly defined;
// not defined as option default to keep it hidden from end user,
// since there is little to no reason to change it.
if opt.Endpoint == "" {
opt.Endpoint = defaultEndpoint
}
oAuthClient, _, err := oauthutil.NewClient(ctx, name, m, oauthConfig)
if err != nil {
return nil, fmt.Errorf("failed to configure kDrive: %w", err)
}
root = parsePath(root)
root = norm.NFC.String(root)
f := &Fs{
name: name,
root: root,
opt: *opt,
srv: rest.NewClient(oAuthClient).SetRoot(opt.Endpoint),
pacer: fs.NewPacer(ctx, pacer.NewDefault(pacer.MinSleep(minSleep), pacer.MaxSleep(maxSleep), pacer.DecayConstant(decayConstant))),
}
f.features = (&fs.Features{
CaseInsensitive: false,
CanHaveEmptyDirectories: true,
PartialUploads: false,
}).Fill(ctx, f)
f.srv.SetErrorHandler(errorHandler)
rootID, err := f.computeRootID(ctx)
if err != nil {
return nil, err
}
f.dirCache = dircache.New(root, rootID, f)
// Find the current root
err = f.dirCache.FindRoot(ctx, false)
if err != nil {
// Assume it is a file
newRoot, remote := dircache.SplitPath(root)
tempF := *f
tempF.dirCache = dircache.New(newRoot, rootID, &tempF)
tempF.root = newRoot
// Make new Fs which is the parent
err = tempF.dirCache.FindRoot(ctx, false)
if err != nil {
// No root so return old f
return f, nil
}
_, err = tempF.newObjectWithInfo(ctx, remote, nil)
if err != nil {
if errors.Is(err, fs.ErrorObjectNotFound) {
// File doesn't exist so return old f
return f, nil
}
return nil, err
}
// XXX: update the old f here instead of returning tempF, since
// `features` were already filled with functions having *f as a receiver.
// See https://github.com/rclone/rclone/issues/2182
f.dirCache = tempF.dirCache
f.root = tempF.root
// return an error with an fs which points to the parent
return f, fs.ErrorIsFile
}
return f, nil
}
// computeRootID finds the real RootId of the configured RootFolderID.
func (f *Fs) computeRootID(ctx context.Context) (rootID string, err error) {
// A numeric root_folder_id is the ID of the folder itself
if _, err := strconv.Atoi(f.opt.RootFolderID); err == nil {
return f.opt.RootFolderID, nil
}
var leaf string
switch f.opt.RootFolderID {
case "private", "":
leaf = "Private"
case "common":
leaf = "Common documents"
default:
leaf = f.opt.RootFolderID
}
var found bool
rootID, found, err = f.FindLeaf(ctx, userRootDirID, leaf)
if err != nil {
return "", fmt.Errorf("failed to find root folder %q: %w", leaf, err)
}
if !found {
return "", fmt.Errorf("root folder %q not found", leaf)
}
return rootID, nil
}
// getItem retrieves a file or directory by its ID.
func (f *Fs) getItem(ctx context.Context, id string) (*api.Item, error) {
// https://developer.infomaniak.com/docs/api/get/2/drive/%7Bdrive_id%7D/files/%7Bfile_id%7D
opts := rest.Opts{
Method: "GET",
Path: fmt.Sprintf("/3/drive/%s/files/%s", f.opt.DriveID, id),
Parameters: url.Values{},
}
opts.Parameters.Set("with", "path")
var result api.ItemResult
var resp *http.Response
var err error
err = f.pacer.Call(func() (bool, error) {
resp, err = f.srv.CallJSON(ctx, &opts, nil, &result)
err = result.ResultStatus.Update(err)
return shouldRetry(ctx, resp, err)
})
if err != nil {
if isNotFoundError(err) {
return nil, fs.ErrorObjectNotFound
}
return nil, fmt.Errorf("couldn't get item: %w", err)
}
item := result.Data
if item.ID == 0 {
return nil, fs.ErrorObjectNotFound
}
item.Name = f.opt.Enc.ToStandardName(norm.NFC.String(item.Name))
return &item, nil
}
// findItemInDir retrieves a file or directory by its name in a specific directory using the API.
func (f *Fs) findItemInDir(ctx context.Context, directoryID string, leaf string) (*api.Item, error) {
// https://developer.infomaniak.com/docs/api/get/3/drive/%7Bdrive_id%7D/files/%7Bfile_id%7D/name
opts := rest.Opts{
Method: "GET",
Path: fmt.Sprintf("/3/drive/%s/files/%s/name", f.opt.DriveID, directoryID),
Parameters: url.Values{},
}
opts.Parameters.Set("name", f.opt.Enc.FromStandardName(norm.NFC.String(leaf)))
opts.Parameters.Set("with", "path,hash")
var result api.ItemResult
var resp *http.Response
var err error
err = f.pacer.Call(func() (bool, error) {
resp, err = f.srv.CallJSON(ctx, &opts, nil, &result)
err = result.ResultStatus.Update(err)
return shouldRetry(ctx, resp, err)
})
if err != nil {
if isNotFoundError(err) {
return nil, fs.ErrorObjectNotFound
}
return nil, fmt.Errorf("couldn't find item in dir: %w", err)
}
item := result.Data
// Check if item is valid (has an ID)
if item.ID == 0 {
return nil, fs.ErrorObjectNotFound
}
// Normalize the name
item.Name = f.opt.Enc.ToStandardName(norm.NFC.String(item.Name))
return &item, nil
}
// findItemByPath retrieves a file or directory by its path
func (f *Fs) findItemByPath(ctx context.Context, remote string) (*api.Item, error) {
if remote == "" || remote == "." {
rootID, err := f.dirCache.FindDir(ctx, "", false)
if err != nil {
return nil, err
}
return f.getItem(ctx, rootID)
}
// Get the directoryID of the parent directory
directory, leaf := path.Split(remote)
directory = strings.TrimSuffix(directory, "/")
directoryID, err := f.dirCache.FindDir(ctx, directory, false)
if err != nil {
if errors.Is(err, fs.ErrorDirNotFound) {
return nil, fs.ErrorObjectNotFound
}
return nil, err
}
return f.findItemInDir(ctx, directoryID, leaf)
}
// Return an Object from a path
//
// If it can't be found it returns the error fs.ErrorObjectNotFound.
func (f *Fs) newObjectWithInfo(ctx context.Context, remote string, info *api.Item) (fs.Object, error) {
o := &Object{
fs: f,
remote: remote,
}
var err error
if info != nil {
// Set info
err = o.setMetaData(info)
} else {
err = o.readMetaData(ctx) // reads info and meta, returning an error
}
if err != nil {
return nil, err
}
return o, nil
}
// NewObject finds the Object at remote. If it can't be found
// it returns the error fs.ErrorObjectNotFound.
func (f *Fs) NewObject(ctx context.Context, remote string) (fs.Object, error) {
return f.newObjectWithInfo(ctx, remote, nil)
}
// FindLeaf finds a directory of name leaf in the folder with ID pathID
func (f *Fs) FindLeaf(ctx context.Context, pathID, leaf string) (pathIDOut string, found bool, err error) {
// Use API endpoint to find item directly by name
item, err := f.findItemInDir(ctx, pathID, leaf)
if err != nil {
if errors.Is(err, fs.ErrorObjectNotFound) {
return "", false, nil
}
return "", false, err
}
if item.Type == "dir" {
return strconv.Itoa(item.ID), true, nil
}
// Not a directory
return "", false, nil
}
// CreateDir makes a directory with pathID as parent and name leaf
func (f *Fs) CreateDir(ctx context.Context, pathID, leaf string) (newID string, err error) {
// fs.Debugf(f, "CreateDir(%q, %q)\n", pathID, leaf)
var resp *http.Response
var result api.CreateDirResult
// https://developer.infomaniak.com/docs/api/post/3/drive/%7Bdrive_id%7D/files/%7Bfile_id%7D/directory
opts := rest.Opts{
Method: "POST",
Path: fmt.Sprintf("/3/drive/%s/files/%s/directory", f.opt.DriveID, pathID),
Parameters: url.Values{},
}
opts.Parameters.Set("name", f.opt.Enc.FromStandardName(norm.NFC.String(leaf)))
err = f.pacer.Call(func() (bool, error) {
resp, err = f.srv.CallJSON(ctx, &opts, nil, &result)
err = result.ResultStatus.Update(err)
return shouldRetry(ctx, resp, err)
})
if err != nil {
return "", err
}
return strconv.Itoa(result.Data.ID), nil
}
// list the objects into the function supplied
//
// If directories is set it only sends directories
// User function to process a File item from listAll
//
// Should return true to finish processing
type listAllFn func(*api.Item) bool
// Lists the directory required calling the user function on each item found.
//
// If the user fn ever returns true then it early exits with found = true.
func (f *Fs) listAll(ctx context.Context, dirID string, directoriesOnly bool, filesOnly bool, recursive bool, fn listAllFn) (found bool, err error) {
var rootPath string
if recursive {
// Only recursive listings need the root item to build
// relative paths from the returned FullPaths
rootItem, err := f.getItem(ctx, dirID)
if err != nil {
return false, err
}
rootPath = rootItem.FullPath + "/"
}
listSomeFiles := func(currentDirID string, fromCursor string) (api.SearchResult, error) {
// https://developer.infomaniak.com/docs/api/get/3/drive/%7Bdrive_id%7D/files/%7Bfile_id%7D/files
opts := rest.Opts{
Method: "GET",
Path: fmt.Sprintf("/3/drive/%s/files/%s/files", f.opt.DriveID, currentDirID),
Parameters: url.Values{},
}
opts.Parameters.Set("limit", "1000")
opts.Parameters.Set("with", "path,hash")
if recursive {
opts.Parameters.Set("depth", "unlimited")
}
if len(fromCursor) > 0 {
opts.Parameters.Set("cursor", fromCursor)
}
var result api.SearchResult
var resp *http.Response
err = f.pacer.Call(func() (bool, error) {
resp, err = f.srv.CallJSON(ctx, &opts, nil, &result)
err = result.ResultStatus.Update(err)
return shouldRetry(ctx, resp, err)
})
if err != nil {
return result, fmt.Errorf("couldn't list files: %w", err)
}
return result, nil
}
var listErr error
var recursiveContents func(currentDirID string, currentSubDir string, fromCursor string)
recursiveContents = func(currentDirID string, currentSubDir string, fromCursor string) {
if listErr != nil || found {
return
}
result, err := listSomeFiles(currentDirID, fromCursor)
if err != nil {
listErr = err
return
}
// First, analyze what has been returned, and go in-depth if required
for i := range result.Data {
item := &result.Data[i]
if item.Type == "dir" {
if filesOnly {
continue
}
} else {
if directoriesOnly {
continue
}
}
item.Name = f.opt.Enc.ToStandardName(norm.NFC.String(item.Name))
if recursive {
item.FullPath = f.opt.Enc.ToStandardPath(norm.NFC.String(strings.TrimPrefix(item.FullPath, rootPath)))
} else {
// Without depth=unlimited the API returns direct
// children only, so the remote is just the name
item.FullPath = item.Name
}
if fn(item) {
found = true
return
}
// The API refuses depth=unlimited on the root directory
// (userRootDirID), so recurse into each child manually
if recursive && currentDirID == userRootDirID && item.Type == "dir" {
recursiveContents(strconv.Itoa(item.ID), path.Join(currentSubDir, item.Name), "" /*reset cursor*/)
if found {
return
}
}
}
// Then load the rest of the files in that folder and apply the same logic
if result.HasMore {
recursiveContents(currentDirID, currentSubDir, result.Cursor)
}
}
recursiveContents(dirID, "", "")
if listErr != nil {
return found, listErr
}
return
}
// listHelper iterates over all items from the directory
// and calls the callback for each element.
func (f *Fs) listHelper(ctx context.Context, dir string, recursive bool, callback func(entries fs.DirEntry) error) (err error) {
directoryID, err := f.dirCache.FindDir(ctx, dir, false)
//fs.Debugf(ctx, "listHelper: root=%s dir=%s directoryID=%s", f.root, dir, directoryID)
if err != nil {
return err
}
var iErr error
_, err = f.listAll(ctx, directoryID, false, false, recursive, func(info *api.Item) bool {
remote := path.Join(dir, info.FullPath)
if info.Type == "dir" {
// cache the directory ID for later lookups
f.dirCache.Put(remote, strconv.Itoa(info.ID))
d := fs.NewDir(remote, info.ModTime()).SetID(strconv.Itoa(info.ID))
d.SetParentID(strconv.Itoa(info.ParentID))
d.SetSize(info.Size)
iErr = callback(d)
} else {
o, err := f.newObjectWithInfo(ctx, remote, info)
if err != nil {
iErr = err
return true
}
iErr = callback(o)
}
if iErr != nil {
return true
}
return false
})
if err != nil {
return err
}
if iErr != nil {
return iErr
}
return nil
}
// List the objects and directories in dir into entries. The
// entries can be returned in any order but should be for a
// complete directory.
//
// dir should be "" to list the root, and should not have
// trailing slashes.
//
// This should return ErrDirNotFound if the directory isn't
// found.
func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err error) {
return list.WithListP(ctx, dir, f)
}
// ListP lists the objects and directories of the Fs starting
// from dir non recursively into out.
//
// dir should be "" to start from the root, and should not
// have trailing slashes.
//
// This should return ErrDirNotFound if the directory isn't
// found.
//
// It should call callback for each tranche of entries read.
// These need not be returned in any particular order. If
// callback returns an error then the listing will stop
// immediately.
func (f *Fs) ListP(ctx context.Context, dir string, callback fs.ListRCallback) (err error) {
l := list.NewHelper(callback)
err = f.listHelper(ctx, dir, false, func(o fs.DirEntry) error {
return l.Add(o)
})
if err != nil {
return err
}
return l.Flush()
}
// ListR lists the objects and directories of the Fs starting
// from dir recursively into out.
func (f *Fs) ListR(ctx context.Context, dir string, callback fs.ListRCallback) (err error) {
l := list.NewHelper(callback)
err = f.listHelper(ctx, dir, true, func(o fs.DirEntry) error {
// fs.Debugf(nil, "ADD OBJECT %s", o.Remote())
return l.Add(o)
})
if err != nil {
return err
}
return l.Flush()
}
// Creates from the parameters passed in a half finished Object which
// must have setMetaData called on it
//
// Returns the object, leaf, directoryID and error.
//
// Used to create new objects
func (f *Fs) createObject(ctx context.Context, remote string, modTime time.Time, size int64) (o *Object, leaf string, directoryID string, err error) {
// Create the directory for the object if it doesn't exist
// fs.Debugf(ctx, "createObject: remote = %s", remote)
leaf, directoryID, err = f.dirCache.FindPath(ctx, remote, true)
if err != nil {
return
}
// Temporary Object under construction
o = &Object{
fs: f,
remote: remote,
size: size,
modTime: modTime,
}
return o, leaf, directoryID, nil
}
// Put the object into the container
//
// Copy the reader in to the new object which is returned.
//
// The new object may have been created if an error is returned
func (f *Fs) Put(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) {
remote := src.Remote()
size := src.Size()
modTime := src.ModTime(ctx)
o, leaf, directoryID, err := f.createObject(ctx, remote, modTime, size)
if err != nil {
return nil, err
}
return o, o.update(ctx, in, src, directoryID, leaf, options...)
}
// Mkdir creates the container if it doesn't exist
func (f *Fs) Mkdir(ctx context.Context, dir string) error {
_, err := f.dirCache.FindDir(ctx, dir, true)
return err
}
// purgeCheck removes the root directory, if check is set then it
// refuses to do so if it has anything in
func (f *Fs) purgeCheck(ctx context.Context, dir string, check bool) error {
root := path.Join(f.root, dir)
if root == "" {
return errors.New("can't purge root directory")
}
dc := f.dirCache
rootID, err := dc.FindDir(ctx, dir, false)
if err != nil {
return err
}
if check {
nonEmpty, err := f.listAll(ctx, rootID, false, false, false, func(i *api.Item) bool {
return true
})
if err != nil {
return err
}
if nonEmpty {
return fs.ErrorDirectoryNotEmpty
}
}
// https://developer.infomaniak.com/docs/api/delete/2/drive/%7Bdrive_id%7D/files/%7Bfile_id%7D
opts := rest.Opts{
Method: "DELETE",
Path: fmt.Sprintf("/2/drive/%s/files/%s", f.opt.DriveID, rootID),
Parameters: url.Values{},
}
var resp *http.Response
var result api.CancellableResponse
err = f.pacer.Call(func() (bool, error) {
resp, err = f.srv.CallJSON(ctx, &opts, nil, &result)
err = result.ResultStatus.Update(err)
return shouldRetry(ctx, resp, err)
})
if err != nil {
return fmt.Errorf("rmdir failed: %w", err)
}
f.dirCache.FlushDir(dir)
return nil
}
// Rmdir deletes the root folder
//
// Returns an error if it isn't empty
func (f *Fs) Rmdir(ctx context.Context, dir string) error {
return f.purgeCheck(ctx, dir, true)
}
// Precision return the precision of this Fs
func (f *Fs) Precision() time.Duration {
return time.Second
}
// Copy src to this remote using server-side copy operations.
//
// This is stored with the remote path given.
//
// It returns the destination Object and a possible error.
//
// Will only be called if src.Fs().Name() == f.Name()
//
// If it isn't possible then return fs.ErrorCantCopy
func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object, error) {
srcObj, ok := src.(*Object)
if !ok {
fs.Debugf(src, "Can't copy - not same remote type")
return nil, fs.ErrorCantCopy
}
err := srcObj.readMetaData(ctx)
if err != nil {
return nil, err
}
// Create temporary object
dstObj, leaf, directoryID, err := f.createObject(ctx, remote, srcObj.modTime, srcObj.size)
if err != nil {
return nil, err
}
// https://developer.infomaniak.com/docs/api/post/3/drive/%7Bdrive_id%7D/files/%7Bfile_id%7D/copy/%7Bdestination_directory_id%7D
opts := rest.Opts{
Method: "POST",
Path: fmt.Sprintf("/3/drive/%s/files/%s/copy/%s", f.opt.DriveID, srcObj.id, directoryID),
Parameters: url.Values{},
}
opts.Parameters.Set("name", f.opt.Enc.FromStandardName(norm.NFC.String(leaf)))
opts.Parameters.Set("conflict", "version")
var resp *http.Response
var result api.FileCopyResponse
err = f.pacer.Call(func() (bool, error) {
resp, err = f.srv.CallJSON(ctx, &opts, nil, &result)
err = result.ResultStatus.Update(err)
return shouldRetry(ctx, resp, err)
})
if err != nil {
return nil, err
}
err = dstObj.setMetaData(&result.Data)
if err != nil {
return nil, err
}
return dstObj, nil
}
// Purge deletes all the files in the directory
//
// Optional interface: Only implement this if you have a way of
// deleting all the files quicker than just running Remove() on the
// result of List()
func (f *Fs) Purge(ctx context.Context, dir string) error {
return f.purgeCheck(ctx, dir, false)
}
// CleanUp empties the trash
func (f *Fs) CleanUp(ctx context.Context) error {
// https://developer.infomaniak.com/docs/api/delete/2/drive/%7Bdrive_id%7D/trash
opts := rest.Opts{
Method: "DELETE",
Path: fmt.Sprintf("/2/drive/%s/trash", f.opt.DriveID),
Parameters: url.Values{},
}
var resp *http.Response
var result api.ResultStatus
var err error
return f.pacer.Call(func() (bool, error) {
resp, err = f.srv.CallJSON(ctx, &opts, nil, &result)
err = result.Update(err)
return shouldRetry(ctx, resp, err)
})
}
// Move src to this remote using server-side move operations.
//
// This is stored with the remote path given.
//
// It returns the destination Object and a possible error.
//
// Will only be called if src.Fs().Name() == f.Name()
//
// If it isn't possible then return fs.ErrorCantMove
func (f *Fs) Move(ctx context.Context, src fs.Object, remote string) (fs.Object, error) {
srcObj, ok := src.(*Object)
if !ok {
fs.Debugf(src, "Can't move - not same remote type")
return nil, fs.ErrorCantMove
}
// Create temporary object
_, leaf, directoryID, err := f.createObject(ctx, remote, srcObj.modTime, srcObj.size)
if err != nil {
return nil, err
}
// https://developer.infomaniak.com/docs/api/post/3/drive/%7Bdrive_id%7D/files/%7Bfile_id%7D/move/%7Bdestination_directory_id%7D
opts := rest.Opts{
Method: "POST",
Path: fmt.Sprintf("/3/drive/%s/files/%s/move/%s", f.opt.DriveID, srcObj.id, directoryID),
Parameters: url.Values{},
}
opts.Parameters.Set("name", f.opt.Enc.FromStandardName(norm.NFC.String(leaf)))
var resp *http.Response
var result api.CancellableResponse
// The API fails a move onto an occupied destination by default
// (conflict=error): that is what we want, rclone removes the
// destination itself before calling Move
err = f.pacer.Call(func() (bool, error) {
resp, err = f.srv.CallJSON(ctx, &opts, nil, &result)
err = result.ResultStatus.Update(err)
return shouldRetry(ctx, resp, err)
})
if err != nil {
return nil, err
}
dstObj, err := f.NewObject(ctx, remote)
if err != nil {
return nil, err
}
// The move API carries no info about the destination and conflicts
// are resolved above, so make sure the file at the destination
// really is the moved one before reporting success
if moved, ok := dstObj.(*Object); !ok || moved.id != srcObj.id {
return nil, fmt.Errorf("move failed: %q at destination is not the moved file", remote)
}
return dstObj, nil
}
// DirMove moves src, srcRemote to this remote at dstRemote
// using server-side move operations.
//
// Will only be called if src.Fs().Name() == f.Name()
//
// If it isn't possible then return fs.ErrorCantDirMove
//
// If destination exists then return fs.ErrorDirExists
func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string) error {
srcFs, ok := src.(*Fs)
if !ok {
fs.Debugf(srcFs, "Can't move directory - not same remote type")
return fs.ErrorCantDirMove
}
srcID, _, _, dstDirectoryID, dstLeaf, err := f.dirCache.DirMove(ctx, srcFs.dirCache, srcFs.root, srcRemote, f.root, dstRemote)
if err != nil {
return err
}
// https://developer.infomaniak.com/docs/api/post/3/drive/%7Bdrive_id%7D/files/%7Bfile_id%7D/move/%7Bdestination_directory_id%7D
opts := rest.Opts{
Method: "POST",
Path: fmt.Sprintf("/3/drive/%s/files/%s/move/%s", f.opt.DriveID, srcID, dstDirectoryID),
Parameters: url.Values{},
}
opts.Parameters.Set("name", f.opt.Enc.FromStandardName(norm.NFC.String(dstLeaf)))
var resp *http.Response
var result api.CancellableResponse
err = f.pacer.Call(func() (bool, error) {
resp, err = f.srv.CallJSON(ctx, &opts, nil, &result)
err = result.ResultStatus.Update(err)
return shouldRetry(ctx, resp, err)
})
if err != nil {
return err
}
srcFs.dirCache.FlushDir(srcRemote)
f.dirCache.FlushDir(dstRemote)
return nil
}
// DirCacheFlush resets the directory cache - used in testing as an
// optional interface
func (f *Fs) DirCacheFlush() {
f.dirCache.ResetRoot()
}
func (f *Fs) getPublicLink(ctx context.Context, fileID int) (string, bool, error) {
// https://developer.infomaniak.com/docs/api/get/2/drive/%7Bdrive_id%7D/files/%7Bfile_id%7D/link
opts := rest.Opts{
Method: "GET",
Path: fmt.Sprintf("/2/drive/%s/files/%d/link", f.opt.DriveID, fileID),
}
var result api.PubLinkResult
err := f.pacer.Call(func() (bool, error) {
resp, err := f.srv.CallJSON(ctx, &opts, nil, &result)
err = result.Update(err)
return shouldRetry(ctx, resp, err)
})
expired := false
if err != nil {
return "", expired, fmt.Errorf("failed to get public link: %w", err)
}
// check if the shared link is blocked
if result.Data.AccessBlocked {
return "", expired, nil
}
// check if the shared link is expired
if result.Data.ValidUntil > 0 {
// ValidUntil is a Unix timestamp (UTC)
if time.Now().Unix() > int64(result.Data.ValidUntil) {
// link is expired
expired = true
}
}
return result.Data.URL, expired, nil
}
func (f *Fs) deletePublicLink(ctx context.Context, fileID int) error {
// https://developer.infomaniak.com/docs/api/delete/2/drive/%7Bdrive_id%7D/files/%7Bfile_id%7D/link
opts := rest.Opts{
Method: "DELETE",
Path: fmt.Sprintf("/2/drive/%s/files/%d/link", f.opt.DriveID, fileID),
}
var result api.ResultStatus
err := f.pacer.Call(func() (bool, error) {
resp, err := f.srv.CallJSON(ctx, &opts, nil, &result)
err = result.Update(err)
return shouldRetry(ctx, resp, err)
})
if err != nil {
return fmt.Errorf("failed to remove public link: %w", err)
}
return nil
}
func (f *Fs) createPublicLink(ctx context.Context, fileID int, expire fs.Duration) (string, error) {
createReq := struct {
Right string `json:"right"` // true for read access
ValidUntil int `json:"valid_until,omitempty"`
}{
Right: "public",
}
// Set expiry if provided
if expire != fs.DurationOff {
createReq.ValidUntil = int(time.Now().Add(time.Duration(expire)).Unix())
}
// https://developer.infomaniak.com/docs/api/post/2/drive/%7Bdrive_id%7D/files/%7Bfile_id%7D/link
opts := rest.Opts{
Method: "POST",
Path: fmt.Sprintf("/2/drive/%s/files/%d/link", f.opt.DriveID, fileID),
}
var result api.PubLinkResult
err := f.pacer.Call(func() (bool, error) {
resp, err := f.srv.CallJSON(ctx, &opts, createReq, &result)
err = result.ResultStatus.Update(err)
return shouldRetry(ctx, resp, err)
})
if err != nil {
return "", fmt.Errorf("failed to create public link: %w", err)
}
return result.Data.URL, nil
}
func (f *Fs) updatePublicLink(ctx context.Context, fileID int, expire fs.Duration) error {
createReq := struct {
ValidUntil int `json:"valid_until,omitempty"`
}{}
// Set expiry if provided
if expire != fs.DurationOff {
createReq.ValidUntil = int(time.Now().Add(time.Duration(expire)).Unix())
}
// https://developer.infomaniak.com/docs/api/put/2/drive/%7Bdrive_id%7D/files/%7Bfile_id%7D/link
opts := rest.Opts{
Method: "PUT",
Path: fmt.Sprintf("/2/drive/%s/files/%d/link", f.opt.DriveID, fileID),
}
var result api.PubLinkUpdateResult
err := f.pacer.Call(func() (bool, error) {
resp, err := f.srv.CallJSON(ctx, &opts, createReq, &result)
err = result.ResultStatus.Update(err)
return shouldRetry(ctx, resp, err)
})
if err != nil {
return fmt.Errorf("failed to update public link: %w", err)
}
return nil
}
// PublicLink adds a "readable by anyone with link" permission on the given file or folder.
// If unlink is true, it removes the existing public link.
func (f *Fs) PublicLink(ctx context.Context, remote string, expire fs.Duration, unlink bool) (string, error) {
item, err := f.findItemByPath(ctx, remote)
if err != nil {
return "", err
}
if unlink {
// Remove existing public link
err = f.deletePublicLink(ctx, item.ID)
return "", err
}
// Check if link exists
link, expired, _ := f.getPublicLink(ctx, item.ID)
if link != "" {
// link exist
if expire != fs.DurationOff {
// update valid until and return the same link
err = f.updatePublicLink(ctx, item.ID, expire)
if err != nil {
return "", err
}
return link, nil
} else if expired {
// delete and recreate after
_ = f.deletePublicLink(ctx, item.ID)
} else {
// return link
return link, nil
}
}
// Create or get existing public link
return f.createPublicLink(ctx, item.ID, expire)
}
// About gets quota information
func (f *Fs) About(ctx context.Context) (usage *fs.Usage, err error) {
// https://developer.infomaniak.com/docs/api/get/2/drive/%7Bdrive_id%7D
opts := rest.Opts{
Method: "GET",
Path: fmt.Sprintf("/2/drive/%s", f.opt.DriveID),
Parameters: url.Values{},
}
opts.Parameters.Set("only", "used_size,size")
var resp *http.Response
var q api.QuotaInfo
err = f.pacer.Call(func() (bool, error) {
resp, err = f.srv.CallJSON(ctx, &opts, nil, &q)
err = q.ResultStatus.Update(err)
return shouldRetry(ctx, resp, err)
})
if err != nil {
return nil, err
}
free := max(q.Data.Size-q.Data.UsedSize, 0)
usage = &fs.Usage{
Total: fs.NewUsageValue(q.Data.Size), // quota of bytes that can be used
Used: fs.NewUsageValue(q.Data.UsedSize), // bytes in use
Free: fs.NewUsageValue(free), // bytes which can be uploaded before reaching the quota
}
return usage, nil
}
// Hashes returns the supported hash sets.
func (f *Fs) Hashes() hash.Set {
return hash.NewHashSet(hash.XXH3)
}
// ------------------------------------------------------------
// Fs returns the parent Fs
func (o *Object) Fs() fs.Info {
return o.fs
}
// Return a string version
func (o *Object) String() string {
if o == nil {
return "<nil>"
}
return o.remote
}
// Remote returns the remote path
func (o *Object) Remote() string {
return o.remote
}
// get file hash
func (o *Object) retrieveHash(ctx context.Context) (hash string, err error) {
var resp *http.Response
var result api.ChecksumFileResult
// https://developer.infomaniak.com/docs/api/get/2/drive/%7Bdrive_id%7D/files/%7Bfile_id%7D/hash
opts := rest.Opts{
Method: "GET",
Path: fmt.Sprintf("/2/drive/%s/files/%s/hash", o.fs.opt.DriveID, o.id),
}
err = o.fs.pacer.Call(func() (bool, error) {
resp, err = o.fs.srv.CallJSON(ctx, &opts, nil, &result)
err = result.ResultStatus.Update(err)
return shouldRetry(ctx, resp, err)
})
if err != nil {
return "", err
}
return result.Data.Hash, nil
}
// Hash returns the XXH3 hash of an object as a lowercase hex string.
//
// Files which kDrive stores as a nested hash (files uploaded in chunks,
// either by rclone or by the kDrive apps which use their own chunk
// sizes) can't be compared with a plain XXH3 of the contents, so no
// hash is returned for those.
func (o *Object) Hash(ctx context.Context, t hash.Type) (string, error) {
if t != hash.XXH3 {
return "", hash.ErrUnsupported
}
if o.xxh3 == "" {
remoteHash, err := o.retrieveHash(ctx)
if err != nil {
return "", fmt.Errorf("failed to get hash: %w", err)
}
o.setHash(remoteHash)
}
if o.nestedHash {
return "", nil
}
return o.xxh3, nil
}
// Size returns the size of an object in bytes
func (o *Object) Size() int64 {
err := o.readMetaData(context.Background())
if err != nil {
fs.Logf(o, "Failed to read metadata: %v", err)
return 0
}
return o.size
}
// setMetaData sets the metadata from info
func (o *Object) setMetaData(info *api.Item) (err error) {
if info.Type == "dir" {
return fs.ErrorIsDir
}
o.hasMetaData = true
o.size = info.Size
o.modTime = info.ModTime()
o.id = strconv.Itoa(info.ID)
// The hash is always replaced: a response without one must not
// leave the previous version's hash in place
o.xxh3 = ""
o.nestedHash = false
o.setHash(info.Hash)
return nil
}
// setHash sets the hashes from that passed in.
// Handles both simple format ("xxh3:...") and nested format ("N:xxh3:...")
func (o *Object) setHash(hashStr string) {
// Use khash.ParseHash to handle both simple and nested formats
parsedHash, isNested, err := khash.ParseHash(hashStr)
if err == nil && parsedHash != "" {
o.xxh3 = parsedHash
o.nestedHash = isNested
}
}
// readMetaData gets the metadata if it hasn't already been fetched
//
// it also sets the info
func (o *Object) readMetaData(ctx context.Context) (err error) {
if o.hasMetaData {
return nil
}
info, err := o.fs.findItemByPath(ctx, o.remote)
if err != nil {
return err
}
return o.setMetaData(info)
}
// ModTime returns the modification time of the object
//
// It attempts to read the objects mtime and if that isn't present the
// LastModified returned in the http headers
func (o *Object) ModTime(ctx context.Context) time.Time {
err := o.readMetaData(ctx)
if err != nil {
fs.Logf(o, "Failed to read metadata: %v", err)
return time.Now()
}
return o.modTime
}
// SetModTime sets the modification time of the object
func (o *Object) SetModTime(ctx context.Context, modTime time.Time) error {
var result api.ResultStatus
modTimeReq := struct {
LastModifiedAt int64 `json:"last_modified_at"`
}{
LastModifiedAt: modTime.Unix(),
}
// https://developer.infomaniak.com/docs/api/post/3/drive/%7Bdrive_id%7D/files/%7Bfile_id%7D/last-modified
opts := rest.Opts{
Method: "POST",
Path: fmt.Sprintf("/3/drive/%s/files/%s/last-modified", o.fs.opt.DriveID, o.id),
}
err := o.fs.pacer.Call(func() (bool, error) {
resp, err := o.fs.srv.CallJSON(ctx, &opts, modTimeReq, &result)
err = result.Update(err)
return shouldRetry(ctx, resp, err)
})
if err != nil {
return err
}
// Update Object modTime: the server stores modtimes with second precision so truncate
// here too to keep the in-memory modtime identical to the one a
// fresh listing returns.
o.modTime = modTime.Truncate(time.Second)
return nil
}
// Storable returns a boolean showing whether this object storable
func (o *Object) Storable() bool {
return true
}
// Open an object for read
func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.ReadCloser, err error) {
fs.FixRangeOption(options, o.Size())
var resp *http.Response
// https://developer.infomaniak.com/docs/api/get/2/drive/%7Bdrive_id%7D/files/%7Bfile_id%7D/download
opts := rest.Opts{
Method: "GET",
Path: fmt.Sprintf("/2/drive/%s/files/%s/download", o.fs.opt.DriveID, o.id),
Options: options,
}
err = o.fs.pacer.Call(func() (bool, error) {
resp, err = o.fs.srv.Call(ctx, &opts)
return shouldRetry(ctx, resp, err)
})
if err != nil {
return nil, err
}
return resp.Body, err
}
// Update the object with the contents of the io.Reader, modTime and size
//
// If existing is set then it updates the object rather than creating a new one.
//
// The new object may have been created if an error is returned
func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (err error) {
size := src.Size() // NB can upload without size
remote := o.Remote()
if size < 0 {
return errors.New("can't upload unknown sizes objects")
}
// Create the directory for the object if it doesn't exist
leaf, directoryID, err := o.fs.dirCache.FindPath(ctx, remote, true)
if err != nil {
return err
}
return o.update(ctx, in, src, directoryID, leaf, options...)
}
func (o *Object) update(ctx context.Context, in io.Reader, src fs.ObjectInfo, directoryID, leaf string, options ...fs.OpenOption) (err error) {
size := src.Size() // NB can upload without size
if size < 0 {
return errors.New("can't upload unknown sizes objects")
}
// if file size is less than the chunk size, upload direct
if size <= int64(o.fs.opt.ChunkSize) {
return o.updateDirect(ctx, in, directoryID, leaf, src, options...)
}
// else, use multipart upload with parallelism
return o.updateMultipart(ctx, in, src, options...)
}
func (o *Object) updateDirect(ctx context.Context, in io.Reader, directoryID, leaf string, src fs.ObjectInfo, options ...fs.OpenOption) (err error) {
size := src.Size()
// The body is not buffered: on transient failures the higher
// layers retry the whole transfer
opts := rest.Opts{
Method: "POST",
Path: fmt.Sprintf("/3/drive/%s/upload", o.fs.opt.DriveID),
Body: in,
ContentType: fs.MimeType(ctx, src),
ContentLength: &size,
Parameters: url.Values{},
TransferEncoding: []string{"identity"}, // kdrive doesn't like chunked encoding
Options: options,
}
leaf = o.fs.opt.Enc.FromStandardName(norm.NFC.String(leaf))
opts.Parameters.Set("file_name", leaf)
opts.Parameters.Set("directory_id", directoryID)
opts.Parameters.Set("total_size", fmt.Sprintf("%d", size))
opts.Parameters.Set("last_modified_at", formatModTime(src.ModTime(ctx)))
opts.Parameters.Set("conflict", "version")
opts.Parameters.Set("with", "hash")
// The server verifies the content against total_chunk_hash; it is
// optional so only sent when the source hash is known
if totalHash, hashErr := src.Hash(ctx, hash.XXH3); hashErr == nil && totalHash != "" {
opts.Parameters.Set("total_chunk_hash", "xxh3:"+totalHash)
}
// CallNoRetry: leave transient failures to the higher layers
var resp *http.Response
var result api.UploadFileResponse
err = o.fs.pacer.CallNoRetry(func() (bool, error) {
resp, err = o.fs.srv.CallJSON(ctx, &opts, nil, &result)
err = result.ResultStatus.Update(err)
return shouldRetry(ctx, resp, err)
})
if err != nil {
return err
}
return o.setMetaData(&result.Data)
}
// updateMultipart uploads large files using chunked upload with parallel streaming
func (o *Object) updateMultipart(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) error {
chunkWriter, err := multipart.UploadMultipart(ctx, src, in, multipart.UploadMultipartOptions{
Open: o.fs,
OpenOptions: options,
})
if err != nil {
return err
}
// Extract the file info from the chunk writer
session := chunkWriter.(*uploadSession)
if session.fileInfo == nil {
return errors.New("upload failed: no file info returned")
}
return o.setMetaData(session.fileInfo)
}
// Remove an object
func (o *Object) Remove(ctx context.Context) error {
// https://developer.infomaniak.com/docs/api/delete/2/drive/%7Bdrive_id%7D/files/%7Bfile_id%7D
opts := rest.Opts{
Method: "DELETE",
Path: fmt.Sprintf("/2/drive/%s/files/%s", o.fs.opt.DriveID, o.id),
}
var result api.CancellableResponse
return o.fs.pacer.Call(func() (bool, error) {
resp, err := o.fs.srv.CallJSON(ctx, &opts, nil, &result)
err = result.ResultStatus.Update(err)
return shouldRetry(ctx, resp, err)
})
}
// ID returns the ID of the Object if known, or "" if not
func (o *Object) ID() string {
return o.id
}
// Check the interfaces are satisfied
var (
_ fs.Fs = (*Fs)(nil)
_ fs.Purger = (*Fs)(nil)
_ fs.CleanUpper = (*Fs)(nil)
_ fs.Copier = (*Fs)(nil)
_ fs.Mover = (*Fs)(nil)
_ fs.DirMover = (*Fs)(nil)
_ fs.DirCacheFlusher = (*Fs)(nil)
_ fs.ListRer = (*Fs)(nil)
_ fs.ListPer = (*Fs)(nil)
_ fs.Abouter = (*Fs)(nil)
_ fs.PublicLinker = (*Fs)(nil)
_ fs.OpenChunkWriter = (*Fs)(nil)
_ fs.Object = (*Object)(nil)
_ fs.IDer = (*Object)(nil)
)