"rsync for cloud storage" - Google Drive, S3, Dropbox, Backblaze B2, One Drive, Swift, Hubic, Wasabi, Google Cloud Storage, Azure Blob, Azure Files, Yandex Files
Go azure-blobazure-blob-storageazure-filesbackblaze-b2cloud-storagedropboxencryptionftpfuse-filesystemgogolanggoogle-cloud-storagegoogle-driveonedriveopenstack-swiftrclones3sftpsyncwebdav
Add pixeldrain backend
Fornax3b3625037c4b
diff --git a/README.md b/README.mdindex 0c17cb6a8..833902ebc 100644--- a/README.md+++ b/README.md@@ -93,6 +93,7 @@ Rclone *("rsync for cloud storage")* is a command-line program to sync files and* pCloud [:page_facing_up:](https://rclone.org/pcloud/)* Petabox [:page_facing_up:](https://rclone.org/s3/#petabox)* PikPak [:page_facing_up:](https://rclone.org/pikpak/)+ * Pixeldrain [:page_facing_up:](https://rclone.org/pixeldrain/)* premiumize.me [:page_facing_up:](https://rclone.org/premiumizeme/)* put.io [:page_facing_up:](https://rclone.org/putio/)* Proton Drive [:page_facing_up:](https://rclone.org/protondrive/)diff --git a/backend/all/all.go b/backend/all/all.goindex f097cb261..9800df87b 100644--- a/backend/all/all.go+++ b/backend/all/all.go@@ -39,6 +39,7 @@ import (_ "github.com/rclone/rclone/backend/oracleobjectstorage"_ "github.com/rclone/rclone/backend/pcloud"_ "github.com/rclone/rclone/backend/pikpak"+ _ "github.com/rclone/rclone/backend/pixeldrain"_ "github.com/rclone/rclone/backend/premiumizeme"_ "github.com/rclone/rclone/backend/protondrive"_ "github.com/rclone/rclone/backend/putio"diff --git a/backend/pixeldrain/api_client.go b/backend/pixeldrain/api_client.gonew file mode 100644index 000000000..e422f61c4--- /dev/null+++ b/backend/pixeldrain/api_client.go@@ -0,0 +1,397 @@+package pixeldrain++import (+ "context"+ "encoding/json"+ "errors"+ "fmt"+ "io"+ "net/http"+ "net/url"+ "strings"+ "time"++ "github.com/rclone/rclone/fs"+ "github.com/rclone/rclone/fs/fserrors"+ "github.com/rclone/rclone/lib/rest"+)++// FilesystemPath is the object which is returned from the pixeldrain API when+// running the stat command on a path. It includes the node information for all+// the members of the path and for all the children of the requested directory.+type FilesystemPath struct {+ Path []FilesystemNode `json:"path"`+ BaseIndex int `json:"base_index"`+ Children []FilesystemNode `json:"children"`+}++// Base returns the base node of the path, this is the node that the path points+// to+func (fsp *FilesystemPath) Base() FilesystemNode {+ return fsp.Path[fsp.BaseIndex]+}++// FilesystemNode is a single node in the pixeldrain filesystem. Usually part of+// a Path or Children slice. The Node is also returned as response from update+// commands, if requested+type FilesystemNode struct {+ Type string `json:"type"`+ Path string `json:"path"`+ Name string `json:"name"`+ Created time.Time `json:"created"`+ Modified time.Time `json:"modified"`+ ModeOctal string `json:"mode_octal"`++ // File params+ FileSize int64 `json:"file_size"`+ FileType string `json:"file_type"`+ SHA256Sum string `json:"sha256_sum"`++ // ID is only filled in when the file/directory is publicly shared+ ID string `json:"id,omitempty"`+}++// ChangeLog is a log of changes that happened in a filesystem. Changes returned+// from the API are on chronological order from old to new. A change log can be+// requested for any directory or file, but change logging needs to be enabled+// with the update API before any log entries will be made. Changes are logged+// for 24 hours after logging was enabled. Each time a change log is requested+// the timer is reset to 24 hours.+type ChangeLog []ChangeLogEntry++// ChangeLogEntry is a single entry in a directory's change log. It contains the+// time at which the change occurred. The path relative to the requested+// directory and the action that was performend (update, move or delete). In+// case of a move operation the new path of the file is stored in the path_new+// field+type ChangeLogEntry struct {+ Time time.Time `json:"time"`+ Path string `json:"path"`+ PathNew string `json:"path_new"`+ Action string `json:"action"`+ Type string `json:"type"`+}++// UserInfo contains information about the logged in user+type UserInfo struct {+ Username string `json:"username"`+ Subscription SubscriptionType `json:"subscription"`+ StorageSpaceUsed int64 `json:"storage_space_used"`+}++// SubscriptionType contains information about a subscription type. It's not the+// active subscription itself, only the properties of the subscription. Like the+// perks and cost+type SubscriptionType struct {+ Name string `json:"name"`+ StorageSpace int64 `json:"storage_space"`+}++// APIError is the error type returned by the pixeldrain API+type APIError struct {+ StatusCode string `json:"value"`+ Message string `json:"message"`+}++func (e APIError) Error() string { return e.StatusCode }++// Generalized errors which are caught in our own handlers and translated to+// more specific errors from the fs package.+var (+ errNotFound = errors.New("pd api: path not found")+ errExists = errors.New("pd api: node already exists")+ errAuthenticationFailed = errors.New("pd api: authentication failed")+)++func apiErrorHandler(resp *http.Response) (err error) {+ var e APIError+ if err = json.NewDecoder(resp.Body).Decode(&e); err != nil {+ return fmt.Errorf("failed to parse error json: %w", err)+ }++ // We close the body here so that the API handlers can be sure that the+ // response body is not still open when an error was returned+ if err = resp.Body.Close(); err != nil {+ return fmt.Errorf("failed to close resp body: %w", err)+ }++ if e.StatusCode == "path_not_found" {+ return errNotFound+ } else if e.StatusCode == "directory_not_empty" {+ return fs.ErrorDirectoryNotEmpty+ } else if e.StatusCode == "node_already_exists" {+ return errExists+ } else if e.StatusCode == "authentication_failed" {+ return errAuthenticationFailed+ } else if e.StatusCode == "permission_denied" {+ return fs.ErrorPermissionDenied+ }++ return e+}++var retryErrorCodes = []int{+ 429, // Too Many Requests.+ 500, // Internal Server Error+ 502, // Bad Gateway+ 503, // Service Unavailable+ 504, // Gateway Timeout+}++// shouldRetry returns a boolean as to whether this resp and err deserve to be+// retried. It returns the err as a convenience so it can be used as the return+// value in the pacer function+func shouldRetry(ctx context.Context, resp *http.Response, err error) (bool, error) {+ if fserrors.ContextError(ctx, &err) {+ return false, err+ }+ return fserrors.ShouldRetry(err) || fserrors.ShouldRetryHTTP(resp, retryErrorCodes), err+}++// paramsFromMetadata turns the fs.Metadata into instructions the pixeldrain API+// can understand.+func paramsFromMetadata(meta fs.Metadata) (params url.Values) {+ params = make(url.Values)++ if modified, ok := meta["mtime"]; ok {+ params.Set("modified", modified)+ }+ if created, ok := meta["btime"]; ok {+ params.Set("created", created)+ }+ if mode, ok := meta["mode"]; ok {+ params.Set("mode", mode)+ }+ if shared, ok := meta["shared"]; ok {+ params.Set("shared", shared)+ }+ if loggingEnabled, ok := meta["logging_enabled"]; ok {+ params.Set("logging_enabled", loggingEnabled)+ }++ return params+}++// nodeToObject converts a single FilesystemNode API response to an object. The+// node is usually a single element from a directory listing+func (f *Fs) nodeToObject(node FilesystemNode) (o *Object) {+ // Trim the path prefix. The path prefix is hidden from rclone during all+ // operations. Saving it here would confuse rclone a lot. So instead we+ // strip it here and add it back for every API request we need to perform+ node.Path = strings.TrimPrefix(node.Path, f.pathPrefix)+ return &Object{fs: f, base: node}+}++func (f *Fs) nodeToDirectory(node FilesystemNode) fs.DirEntry {+ return fs.NewDir(strings.TrimPrefix(node.Path, f.pathPrefix), node.Modified).SetID(node.ID)+}++func (f *Fs) escapePath(p string) (out string) {+ // Add the path prefix, encode all the parts and combine them together+ var parts = strings.Split(f.pathPrefix+p, "/")+ for i := range parts {+ parts[i] = url.PathEscape(parts[i])+ }+ return strings.Join(parts, "/")+}++func (f *Fs) put(+ ctx context.Context,+ path string,+ body io.Reader,+ meta fs.Metadata,+ options []fs.OpenOption,+) (node FilesystemNode, err error) {+ var params = paramsFromMetadata(meta)++ // Tell the server to automatically create parent directories if they don't+ // exist yet+ params.Set("make_parents", "true")++ return node, f.pacer.Call(func() (bool, error) {+ resp, err := f.srv.CallJSON(+ ctx,+ &rest.Opts{+ Method: "PUT",+ Path: f.escapePath(path),+ Body: body,+ Parameters: params,+ Options: options,+ },+ nil,+ &node,+ )+ return shouldRetry(ctx, resp, err)+ })+}++func (f *Fs) read(ctx context.Context, path string, options []fs.OpenOption) (in io.ReadCloser, err error) {+ var resp *http.Response+ err = f.pacer.Call(func() (bool, error) {+ resp, err = f.srv.Call(ctx, &rest.Opts{+ Method: "GET",+ Path: f.escapePath(path),+ Options: options,+ })+ return shouldRetry(ctx, resp, err)+ })+ if err != nil {+ return nil, err+ }+ return resp.Body, err+}++func (f *Fs) stat(ctx context.Context, path string) (fsp FilesystemPath, err error) {+ return fsp, f.pacer.Call(func() (bool, error) {+ resp, err := f.srv.CallJSON(+ ctx,+ &rest.Opts{+ Method: "GET",+ Path: f.escapePath(path),+ // To receive node info from the pixeldrain API you need to add the+ // ?stat query. Without it pixeldrain will return the file contents+ // in the URL points to a file+ Parameters: url.Values{"stat": []string{""}},+ },+ nil,+ &fsp,+ )+ return shouldRetry(ctx, resp, err)+ })+}++func (f *Fs) changeLog(ctx context.Context, start, end time.Time) (changeLog ChangeLog, err error) {+ return changeLog, f.pacer.Call(func() (bool, error) {+ resp, err := f.srv.CallJSON(+ ctx,+ &rest.Opts{+ Method: "GET",+ Path: f.escapePath(""),+ Parameters: url.Values{+ "change_log": []string{""},+ "start": []string{start.Format(time.RFC3339Nano)},+ "end": []string{end.Format(time.RFC3339Nano)},+ },+ },+ nil,+ &changeLog,+ )+ return shouldRetry(ctx, resp, err)+ })+}++func (f *Fs) update(ctx context.Context, path string, fields fs.Metadata) (node FilesystemNode, err error) {+ var params = paramsFromMetadata(fields)+ params.Set("action", "update")++ return node, f.pacer.Call(func() (bool, error) {+ resp, err := f.srv.CallJSON(+ ctx,+ &rest.Opts{+ Method: "POST",+ Path: f.escapePath(path),+ MultipartParams: params,+ },+ nil,+ &node,+ )+ return shouldRetry(ctx, resp, err)+ })+}++func (f *Fs) mkdir(ctx context.Context, dir string) (err error) {+ return f.pacer.Call(func() (bool, error) {+ resp, err := f.srv.CallJSON(+ ctx,+ &rest.Opts{+ Method: "POST",+ Path: f.escapePath(dir),+ MultipartParams: url.Values{"action": []string{"mkdirall"}},+ NoResponse: true,+ },+ nil,+ nil,+ )+ return shouldRetry(ctx, resp, err)+ })+}++var errIncompatibleSourceFS = errors.New("source filesystem is not the same as target")++// Renames a file on the server side. Can be used for both directories and files+func (f *Fs) rename(ctx context.Context, src fs.Fs, from, to string, meta fs.Metadata) (node FilesystemNode, err error) {+ srcFs, ok := src.(*Fs)+ if !ok {+ // This is not a pixeldrain FS, can't move+ return node, errIncompatibleSourceFS+ } else if srcFs.opt.RootFolderID != f.opt.RootFolderID {+ // Path is not in the same root dir, can't move+ return node, errIncompatibleSourceFS+ }++ var params = paramsFromMetadata(meta)+ params.Set("action", "rename")++ // The target is always in our own filesystem so here we use our+ // own pathPrefix+ params.Set("target", f.pathPrefix+to)++ // Create parent directories if the parent directory of the file+ // does not exist yet+ params.Set("make_parents", "true")++ return node, f.pacer.Call(func() (bool, error) {+ resp, err := f.srv.CallJSON(+ ctx,+ &rest.Opts{+ Method: "POST",+ // Important: We use the source FS path prefix here+ Path: srcFs.escapePath(from),+ MultipartParams: params,+ },+ nil,+ &node,+ )+ return shouldRetry(ctx, resp, err)+ })+}++func (f *Fs) delete(ctx context.Context, path string, recursive bool) (err error) {+ var params url.Values+ if recursive {+ // Tell the server to recursively delete all child files+ params = url.Values{"recursive": []string{"true"}}+ }++ return f.pacer.Call(func() (bool, error) {+ resp, err := f.srv.CallJSON(+ ctx,+ &rest.Opts{+ Method: "DELETE",+ Path: f.escapePath(path),+ Parameters: params,+ NoResponse: true,+ },+ nil, nil,+ )+ return shouldRetry(ctx, resp, err)+ })+}++func (f *Fs) userInfo(ctx context.Context) (user UserInfo, err error) {+ return user, f.pacer.Call(func() (bool, error) {+ resp, err := f.srv.CallJSON(+ ctx,+ &rest.Opts{+ Method: "GET",+ // The default RootURL points at the filesystem endpoint. We can't+ // use that to request user information. So here we override it to+ // the user endpoint+ RootURL: f.opt.APIURL + "/user",+ },+ nil,+ &user,+ )+ return shouldRetry(ctx, resp, err)+ })+}diff --git a/backend/pixeldrain/pixeldrain.go b/backend/pixeldrain/pixeldrain.gonew file mode 100644index 000000000..abf6c1f57--- /dev/null+++ b/backend/pixeldrain/pixeldrain.go@@ -0,0 +1,567 @@+// Package pixeldrain provides an interface to the Pixeldrain object storage+// system.+package pixeldrain++import (+ "context"+ "errors"+ "fmt"+ "io"+ "path"+ "strconv"+ "strings"+ "time"++ "github.com/rclone/rclone/fs"+ "github.com/rclone/rclone/fs/config/configmap"+ "github.com/rclone/rclone/fs/config/configstruct"+ "github.com/rclone/rclone/fs/fshttp"+ "github.com/rclone/rclone/fs/hash"+ "github.com/rclone/rclone/lib/pacer"+ "github.com/rclone/rclone/lib/rest"+)++const (+ timeFormat = time.RFC3339Nano+ minSleep = pacer.MinSleep(10 * time.Millisecond)+ maxSleep = pacer.MaxSleep(1 * time.Second)+ decayConstant = pacer.DecayConstant(2) // bigger for slower decay, exponential+)++// Register with Fs+func init() {+ fs.Register(&fs.RegInfo{+ Name: "pixeldrain",+ Description: "Pixeldrain Filesystem",+ NewFs: NewFs,+ Config: nil,+ Options: []fs.Option{{+ Name: "api_key",+ Help: "API key for your pixeldrain account.\n" ++ "Found on https://pixeldrain.com/user/api_keys.",+ Sensitive: true,+ }, {+ Name: "root_folder_id",+ Help: "Root of the filesystem to use.\n\n" ++ "Set to 'me' to use your personal filesystem. " ++ "Set to a shared directory ID to use a shared directory.",+ Default: "me",+ }, {+ Name: "api_url",+ Help: "The API endpoint to connect to. In the vast majority of cases it's fine to leave\n" ++ "this at default. It is only intended to be changed for testing purposes.",+ Default: "https://pixeldrain.com/api",+ Advanced: true,+ Required: true,+ }},+ MetadataInfo: &fs.MetadataInfo{+ System: map[string]fs.MetadataHelp{+ "mode": {+ Help: "File mode",+ Type: "octal, unix style",+ Example: "755",+ },+ "mtime": {+ Help: "Time of last modification",+ Type: "RFC 3339",+ Example: timeFormat,+ },+ "btime": {+ Help: "Time of file birth (creation)",+ Type: "RFC 3339",+ Example: timeFormat,+ },+ },+ Help: "Pixeldrain supports file modes and creation times.",+ },+ })+}++// Options defines the configuration for this backend+type Options struct {+ APIKey string `config:"api_key"`+ RootFolderID string `config:"root_folder_id"`+ APIURL string `config:"api_url"`+}++// Fs represents a remote box+type Fs struct {+ name string // name of this remote, as given to NewFS+ root string // the path we are working on, as given to NewFS+ opt Options // parsed options+ features *fs.Features // optional features+ srv *rest.Client // the connection to the server+ pacer *fs.Pacer+ loggedIn bool // if the user is authenticated++ // Pathprefix is the directory we're working in. The pathPrefix is stripped+ // from every API response containing a path. The pathPrefix always begins+ // and ends with a slash for concatenation convenience+ pathPrefix string+}++// Object describes a pixeldrain file+type Object struct {+ fs *Fs // what this object is part of+ base FilesystemNode // the node this object references+}++// NewFs constructs an Fs from the path, container:path+func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, error) {+ // Parse config into Options struct+ opt := new(Options)+ err := configstruct.Set(m, opt)+ if err != nil {+ return nil, err+ }++ f := &Fs{+ name: name,+ root: root,+ opt: *opt,+ srv: rest.NewClient(fshttp.NewClient(ctx)).SetErrorHandler(apiErrorHandler),+ pacer: fs.NewPacer(ctx, pacer.NewDefault(minSleep, maxSleep, decayConstant)),+ }+ f.features = (&fs.Features{+ ReadMimeType: true,+ CanHaveEmptyDirectories: true,+ ReadMetadata: true,+ WriteMetadata: true,+ }).Fill(ctx, f)++ // Set the path prefix. This is the path to the root directory on the+ // server. We add it to each request and strip it from each response because+ // rclone does not want to see it+ f.pathPrefix = "/" + path.Join(opt.RootFolderID, f.root) + "/"++ // The root URL equates to https://pixeldrain.com/api/filesystem during+ // normal operation. API handlers need to manually add the pathPrefix to+ // each request+ f.srv.SetRoot(opt.APIURL + "/filesystem")++ // If using an APIKey, set the Authorization header+ if len(opt.APIKey) > 0 {+ f.srv.SetUserPass("", opt.APIKey)++ // Check if credentials are correct+ user, err := f.userInfo(ctx)+ if err != nil {+ return nil, fmt.Errorf("failed to get user data: %w", err)+ }++ f.loggedIn = true++ fs.Infof(f,+ "Logged in as '%s', subscription '%s', storage limit %d",+ user.Username, user.Subscription.Name, user.Subscription.StorageSpace,+ )+ }++ if !f.loggedIn && opt.RootFolderID == "me" {+ return nil, errors.New("authentication required: the 'me' directory can only be accessed while logged in")+ }++ // Satisfy TestFsIsFile. This test expects that we throw an error if the+ // filesystem root is a file+ fsp, err := f.stat(ctx, "")+ if err != errNotFound && err != nil {+ // It doesn't matter if the root directory does not exist, as long as it+ // is not a file. This is what the test dictates+ return f, err+ } else if err == nil && fsp.Base().Type == "file" {+ // The filesystem root is a file, rclone wants us to set the root to the+ // parent directory+ f.root = path.Dir(f.root)+ f.pathPrefix = "/" + path.Join(opt.RootFolderID, f.root) + "/"+ return f, fs.ErrorIsFile+ }++ return f, nil+}++// List the objects and directories in dir into entries. The+// entries can be returned in any order but should be for a+// complete directory.+//+// dir should be "" to list the root, and should not have+// trailing slashes.+//+// This should return ErrDirNotFound if the directory isn't+// found.+func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err error) {+ fsp, err := f.stat(ctx, dir)+ if err == errNotFound {+ return nil, fs.ErrorDirNotFound+ } else if err != nil {+ return nil, err+ } else if fsp.Base().Type == "file" {+ return nil, fs.ErrorIsFile+ }++ entries = make(fs.DirEntries, len(fsp.Children))+ for i := range fsp.Children {+ if fsp.Children[i].Type == "dir" {+ entries[i] = f.nodeToDirectory(fsp.Children[i])+ } else {+ entries[i] = f.nodeToObject(fsp.Children[i])+ }+ }++ return entries, nil+}++// NewObject finds the Object at remote. If it can't be found+// it returns the error fs.ErrorObjectNotFound.+func (f *Fs) NewObject(ctx context.Context, remote string) (fs.Object, error) {+ fsp, err := f.stat(ctx, remote)+ if err == errNotFound {+ return nil, fs.ErrorObjectNotFound+ } else if err != nil {+ return nil, err+ } else if fsp.Base().Type == "dir" {+ return nil, fs.ErrorIsDir+ }+ return f.nodeToObject(fsp.Base()), nil+}++// Put the object+//+// Copy the reader in to the new object which is returned.+//+// The new object may have been created if an error is returned+func (f *Fs) Put(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) {+ meta, err := fs.GetMetadataOptions(ctx, f, src, options)+ if err != nil {+ return nil, fmt.Errorf("failed to get object metadata")+ }++ // Overwrite the mtime if it was not already set in the metadata+ if _, ok := meta["mtime"]; !ok {+ if meta == nil {+ meta = make(fs.Metadata)+ }+ meta["mtime"] = src.ModTime(ctx).Format(timeFormat)+ }++ node, err := f.put(ctx, src.Remote(), in, meta, options)+ if err != nil {+ return nil, fmt.Errorf("failed to put object: %w", err)+ }++ return f.nodeToObject(node), nil+}++// Mkdir creates the container if it doesn't exist+func (f *Fs) Mkdir(ctx context.Context, dir string) (err error) {+ err = f.mkdir(ctx, dir)+ if err == errNotFound {+ return fs.ErrorDirNotFound+ } else if err == errExists {+ // Spec says we do not return an error if the directory already exists+ return nil+ }+ return err+}++// Rmdir deletes the root folder+//+// Returns an error if it isn't empty+func (f *Fs) Rmdir(ctx context.Context, dir string) (err error) {+ err = f.delete(ctx, dir, false)+ if err == errNotFound {+ return fs.ErrorDirNotFound+ }+ return err+}++// Name of the remote (as passed into NewFs)+func (f *Fs) Name() string { return f.name }++// Root of the remote (as passed into NewFs)+func (f *Fs) Root() string { return f.root }++// String converts this Fs to a string+func (f *Fs) String() string { return fmt.Sprintf("pixeldrain root '%s'", f.root) }++// Precision return the precision of this Fs+func (f *Fs) Precision() time.Duration { return time.Millisecond }++// Hashes returns the supported hash sets.+func (f *Fs) Hashes() hash.Set { return hash.Set(hash.SHA256) }++// Features returns the optional features of this Fs+func (f *Fs) Features() *fs.Features { return f.features }++// Purge all files in the directory specified+//+// Implement this if you have a way of deleting all the files+// quicker than just running Remove() on the result of List()+//+// Return an error if it doesn't exist+func (f *Fs) Purge(ctx context.Context, dir string) (err error) {+ err = f.delete(ctx, dir, true)+ if err == errNotFound {+ return fs.ErrorDirNotFound+ }+ return err+}++// Move src to this remote using server-side move operations.+//+// This is stored with the remote path given.+//+// It returns the destination Object and a possible error.+//+// Will only be called if src.Fs().Name() == f.Name()+//+// If it isn't possible then return fs.ErrorCantMove+func (f *Fs) Move(ctx context.Context, src fs.Object, remote string) (fs.Object, error) {+ srcObj, ok := src.(*Object)+ if !ok {+ // This is not a pixeldrain object. Can't move+ return nil, fs.ErrorCantMove+ }++ node, err := f.rename(ctx, srcObj.fs, srcObj.base.Path, remote, fs.GetConfig(ctx).MetadataSet)+ if err == errIncompatibleSourceFS {+ return nil, fs.ErrorCantMove+ } else if err == errNotFound {+ return nil, fs.ErrorObjectNotFound+ }++ return f.nodeToObject(node), nil+}++// DirMove moves src, srcRemote to this remote at dstRemote+// using server-side move operations.+//+// Will only be called if src.Fs().Name() == f.Name()+//+// If it isn't possible then return fs.ErrorCantDirMove+//+// If destination exists then return fs.ErrorDirExists+func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string) (err error) {+ _, err = f.rename(ctx, src, srcRemote, dstRemote, nil)+ if err == errIncompatibleSourceFS {+ return fs.ErrorCantDirMove+ } else if err == errNotFound {+ return fs.ErrorDirNotFound+ } else if err == errExists {+ return fs.ErrorDirExists+ }+ return err+}++// ChangeNotify calls the passed function with a path+// that has had changes. If the implementation+// uses polling, it should adhere to the given interval.+// At least one value will be written to the channel,+// specifying the initial value and updated values might+// follow. A 0 Duration should pause the polling.+// The ChangeNotify implementation must empty the channel+// regularly. When the channel gets closed, the implementation+// should stop polling and release resources.+func (f *Fs) ChangeNotify(ctx context.Context, notify func(string, fs.EntryType), newInterval <-chan time.Duration) {+ // If the bucket ID is not /me we need to explicitly enable change logging+ // for this directory or file+ if f.pathPrefix != "/me/" {+ _, err := f.update(ctx, "", fs.Metadata{"logging_enabled": "true"})+ if err != nil {+ fs.Errorf(f, "Failed to set up change logging for path '%s': %s", f.pathPrefix, err)+ }+ }++ go f.changeNotify(ctx, notify, newInterval)+}+func (f *Fs) changeNotify(ctx context.Context, notify func(string, fs.EntryType), newInterval <-chan time.Duration) {+ var ticker = time.NewTicker(<-newInterval)+ var lastPoll = time.Now()++ for {+ select {+ case dur, ok := <-newInterval:+ if !ok {+ ticker.Stop()+ return+ }++ fs.Debugf(f, "Polling changes at an interval of %s", dur)+ ticker.Reset(dur)++ case t := <-ticker.C:+ clog, err := f.changeLog(ctx, lastPoll, t)+ if err != nil {+ fs.Errorf(f, "Failed to get change log for path '%s': %s", f.pathPrefix, err)+ continue+ }++ for i := range clog {+ fs.Debugf(f, "Path '%s' (%s) changed (%s) in directory '%s'",+ clog[i].Path, clog[i].Type, clog[i].Action, f.pathPrefix)++ if clog[i].Type == "dir" {+ notify(strings.TrimPrefix(clog[i].Path, "/"), fs.EntryDirectory)+ } else if clog[i].Type == "file" {+ notify(strings.TrimPrefix(clog[i].Path, "/"), fs.EntryObject)+ }+ }++ lastPoll = t+ }+ }+}++// PutStream uploads to the remote path with the modTime given of indeterminate size+//+// May create the object even if it returns an error - if so+// will return the object and the error, otherwise will return+// nil and the error+func (f *Fs) PutStream(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) {+ // Put already supports streaming so we just use that+ return f.Put(ctx, in, src, options...)+}++// DirSetModTime sets the mtime metadata on a directory+func (f *Fs) DirSetModTime(ctx context.Context, dir string, modTime time.Time) (err error) {+ _, err = f.update(ctx, dir, fs.Metadata{"mtime": modTime.Format(timeFormat)})+ return err+}++// PublicLink generates a public link to the remote path (usually readable by anyone)+func (f *Fs) PublicLink(ctx context.Context, remote string, expire fs.Duration, unlink bool) (string, error) {+ fsn, err := f.update(ctx, remote, fs.Metadata{"shared": strconv.FormatBool(!unlink)})+ if err != nil {+ return "", err+ }+ if fsn.ID != "" {+ return strings.Replace(f.opt.APIURL, "/api", "/d/", 1) + fsn.ID, nil+ }+ return "", nil+}++// About gets quota information+func (f *Fs) About(ctx context.Context) (usage *fs.Usage, err error) {+ user, err := f.userInfo(ctx)+ if err != nil {+ return nil, fmt.Errorf("failed to read user info: %w", err)+ }++ usage = &fs.Usage{Used: fs.NewUsageValue(user.StorageSpaceUsed)}++ if user.Subscription.StorageSpace > -1 {+ usage.Total = fs.NewUsageValue(user.Subscription.StorageSpace)+ }++ return usage, nil+}++// SetModTime sets the modification time of the local fs object+func (o *Object) SetModTime(ctx context.Context, modTime time.Time) (err error) {+ _, err = o.fs.update(ctx, o.base.Path, fs.Metadata{"mtime": modTime.Format(timeFormat)})+ if err == nil {+ o.base.Modified = modTime+ }+ return err+}++// Open an object for read+func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.ReadCloser, err error) {+ return o.fs.read(ctx, o.base.Path, options)+}++// Update the object with the contents of the io.Reader, modTime and size+//+// If existing is set then it updates the object rather than creating a new one.+//+// The new object may have been created if an error is returned.+func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (err error) {+ // Copy the parameters and update the object+ o.base.Modified = src.ModTime(ctx)+ o.base.FileSize = src.Size()+ o.base.SHA256Sum, _ = src.Hash(ctx, hash.SHA256)+ _, err = o.fs.Put(ctx, in, o, options...)+ return err+}++// Remove an object+func (o *Object) Remove(ctx context.Context) error {+ return o.fs.delete(ctx, o.base.Path, false)+}++// Fs returns the parent Fs+func (o *Object) Fs() fs.Info {+ return o.fs+}++// Hash returns the SHA-256 of an object returning a lowercase hex string+func (o *Object) Hash(ctx context.Context, t hash.Type) (string, error) {+ if t != hash.SHA256 {+ return "", hash.ErrUnsupported+ }+ return o.base.SHA256Sum, nil+}++// Storable returns a boolean showing whether this object storable+func (o *Object) Storable() bool {+ return true+}++// Return a string version+func (o *Object) String() string {+ if o == nil {+ return "<nil>"+ }+ return o.base.Path+}++// Remote returns the remote path+func (o *Object) Remote() string {+ return o.base.Path+}++// ModTime returns the modification time of the object+//+// It attempts to read the objects mtime and if that isn't present the+// LastModified returned in the http headers+func (o *Object) ModTime(ctx context.Context) time.Time {+ return o.base.Modified+}++// Size returns the size of an object in bytes+func (o *Object) Size() int64 {+ return o.base.FileSize+}++// MimeType returns the content type of the Object if known, or "" if not+func (o *Object) MimeType(ctx context.Context) string {+ return o.base.FileType+}++// Metadata returns metadata for an object+//+// It should return nil if there is no Metadata+func (o *Object) Metadata(ctx context.Context) (fs.Metadata, error) {+ return fs.Metadata{+ "mode": o.base.ModeOctal,+ "mtime": o.base.Modified.Format(timeFormat),+ "btime": o.base.Created.Format(timeFormat),+ }, nil+}++// Verify that all the interfaces are implemented correctly+var (+ _ fs.Fs = (*Fs)(nil)+ _ fs.Info = (*Fs)(nil)+ _ fs.Purger = (*Fs)(nil)+ _ fs.Mover = (*Fs)(nil)+ _ fs.DirMover = (*Fs)(nil)+ _ fs.ChangeNotifier = (*Fs)(nil)+ _ fs.PutStreamer = (*Fs)(nil)+ _ fs.DirSetModTimer = (*Fs)(nil)+ _ fs.PublicLinker = (*Fs)(nil)+ _ fs.Abouter = (*Fs)(nil)+ _ fs.Object = (*Object)(nil)+ _ fs.DirEntry = (*Object)(nil)+ _ fs.MimeTyper = (*Object)(nil)+ _ fs.Metadataer = (*Object)(nil)+)diff --git a/backend/pixeldrain/pixeldrain_test.go b/backend/pixeldrain/pixeldrain_test.gonew file mode 100644index 000000000..f36975243--- /dev/null+++ b/backend/pixeldrain/pixeldrain_test.go@@ -0,0 +1,18 @@+// Test pixeldrain filesystem interface+package pixeldrain_test++import (+ "testing"++ "github.com/rclone/rclone/backend/pixeldrain"+ "github.com/rclone/rclone/fstest/fstests"+)++// TestIntegration runs integration tests against the remote+func TestIntegration(t *testing.T) {+ fstests.Run(t, &fstests.Opt{+ RemoteName: "TestPixeldrain:",+ NilObject: (*pixeldrain.Object)(nil),+ SkipInvalidUTF8: true, // Pixeldrain throws an error on invalid utf-8+ })+}diff --git a/bin/make_manual.py b/bin/make_manual.pyindex 2ebeb207f..41ddccf7c 100755--- a/bin/make_manual.py+++ b/bin/make_manual.py@@ -69,6 +69,7 @@ docs = ["swift.md","pcloud.md","pikpak.md",+ "pixeldrain.md","premiumizeme.md","protondrive.md","putio.md",diff --git a/docs/content/_index.md b/docs/content/_index.mdindex 1a535220f..714cce51a 100644--- a/docs/content/_index.md+++ b/docs/content/_index.md@@ -161,6 +161,7 @@ WebDAV or S3, that work out of the box.){{< provider name="pCloud" home="https://www.pcloud.com/" config="/pcloud/" >}}{{< provider name="Petabox" home="https://petabox.io/" config="/s3/#petabox" >}}{{< provider name="PikPak" home="https://mypikpak.com/" config="/pikpak/" >}}+{{< provider name="Pixeldrain" home="https://pixeldrain.com/" config="/pixeldrain/" >}}{{< provider name="premiumize.me" home="https://premiumize.me/" config="/premiumizeme/" >}}{{< provider name="put.io" home="https://put.io/" config="/putio/" >}}{{< provider name="Proton Drive" home="https://proton.me/drive" config="/protondrive/" >}}diff --git a/docs/content/docs.md b/docs/content/docs.mdindex 861ee963b..5eaaee51e 100644--- a/docs/content/docs.md+++ b/docs/content/docs.md@@ -65,6 +65,7 @@ See the following for detailed instructions for* [Oracle Object Storage](/oracleobjectstorage/)* [Pcloud](/pcloud/)* [PikPak](/pikpak/)+ * [Pixeldrain](/pixeldrain/)* [premiumize.me](/premiumizeme/)* [put.io](/putio/)* [Proton Drive](/protondrive/)diff --git a/docs/content/overview.md b/docs/content/overview.mdindex 2b4ecd111..b869b1035 100644--- a/docs/content/overview.md+++ b/docs/content/overview.md@@ -46,6 +46,7 @@ Here is an overview of the major features of each cloud storage system.| Oracle Object Storage | MD5 | R/W | No | No | R/W | - || pCloud | MD5, SHA1 ⁷ | R | No | No | W | - || PikPak | MD5 | R | No | No | R | - |+| Pixeldrain | SHA256 | R/W | No | No | R | RW || premiumize.me | - | - | Yes | No | R | - || put.io | CRC-32 | R/W | No | Yes | R | - || Proton Drive | SHA1 | R/W | No | No | R | - |@@ -520,6 +521,7 @@ upon backend-specific capabilities.| Oracle Object Storage | No | Yes | No | No | Yes | Yes | Yes | Yes | No | No | No || pCloud | Yes | Yes | Yes | Yes | Yes | No | No | No | Yes | Yes | Yes || PikPak | Yes | Yes | Yes | Yes | Yes | No | No | No | Yes | Yes | Yes |+| Pixeldrain | Yes | No | Yes | Yes | No | No | Yes | No | Yes | Yes | Yes || premiumize.me | Yes | No | Yes | Yes | No | No | No | No | Yes | Yes | Yes || put.io | Yes | No | Yes | Yes | Yes | No | Yes | No | No | Yes | Yes || Proton Drive | Yes | No | Yes | Yes | Yes | No | No | No | No | Yes | Yes |diff --git a/docs/content/pixeldrain.md b/docs/content/pixeldrain.mdnew file mode 100644index 000000000..853ad55cc--- /dev/null+++ b/docs/content/pixeldrain.md@@ -0,0 +1,181 @@+---+title: "Pixeldrain"+description: "Rclone docs for Pixeldrain"+versionIntroduced: "v1.68"+---++# {{< icon "fa fa-circle" >}} Pixeldrain++This is the backend for Pixeldrain's premium filesystem feature. This is not the+same as pixeldrain's free file sharing product. The filesystem requires either a+Pro subscription or the Prepaid plan. [More information on+subscriptions](https://pixeldrain.com/#pro).++An overview of the filesystem's features and limitations is available in the+[filesystem guide](https://pixeldrain.com/filesystem) on pixeldrain.++### Usage with account++To use the personal filesystem you will need a [pixeldrain+account](https://pixeldrain.com/register) and either the Prepaid plan or one of+the Patreon-based subscriptions. After registering and subscribing, your+personal filesystem will be available at this link: https://pixeldrain.com/d/me.++Go to the [API keys page](https://pixeldrain.com/user/api_keys) on your account+and generate a new API key for rclone. Then run `rclone config` and use the API+key to create a new backend.++Example:++```+No remotes found, make a new one?+n) New remote+d) Delete remote+c) Copy remote+s) Set configuration password+q) Quit config+n/d/c/s/q> n++Enter name for new remote.+name> pixeldrainfs++Option Storage.+Type of storage to configure.+Choose a number from below, or type in your own value.+...+XX / Pixeldrain Filesystem+ \ (pixeldrain)+...+Storage> pixeldrain++Option api_key.+API key for your pixeldrain account.+Found on https://pixeldrain.com/user/api_keys.+Enter a value. Press Enter to leave empty.+api_key> b1bb1e81-9b7b-406b-986a-c9b20be76e15++Option directory_id.+Root of the filesystem to use. Set to 'me' to use your personal filesystem.+Set to a shared directory ID to use a shared directory.+Enter a string value. Press Enter for the default (me).+directory_id>++Edit advanced config?+y) Yes+n) No (default)+y/n>++Configuration complete.+Options:+- type: pixeldrain+- api_key: b1bb1e81-9b7b-406b-986a-c9b20be76e15+Keep this "pixeldrainfs" remote?+y) Yes this is OK (default)+e) Edit this remote+d) Delete this remote+y/e/d>++Current remotes:++Name Type+==== ====+pixeldrainfs pixeldrain++e) Edit existing remote+n) New remote+d) Delete remote+r) Rename remote+c) Copy remote+s) Set configuration password+q) Quit config+e/n/d/r/c/s/q> q+```++### Usage without account++It is possible to gain read-only access to publicly shared directories through+rclone. For this you only need a directory ID. The directory ID can be found in+the URL of a shared directory, the URL will look like this+`https://pixeldrain.com/d/abcd1234` where `abcd1234` is the directory ID.+Directory IDs in your own filesystem can also be listed with the `lsf` command:++`rclone lsf Pixeldrain: --dirs-only -Fpi`++This will print directories in your `Pixeldrain` home directory and their public+IDs.++Enter this directory ID in the rclone config and you will be able to access the+directory.++{{< rem autogenerated options start" - DO NOT EDIT - instead edit fs.RegInfo in backend/pixeldrain/pixeldrain.go then run make backenddocs" >}}+### Standard options++Here are the Standard options specific to pixeldrain (Pixeldrain Filesystem).++#### --pixeldrain-api-key++API key for your pixeldrain account.+Found on https://pixeldrain.com/user/api_keys.++Properties:++- Config: api_key+- Env Var: RCLONE_PIXELDRAIN_API_KEY+- Type: string+- Required: false++#### --pixeldrain-root-folder-id++Root of the filesystem to use.++Set to 'me' to use your personal filesystem. Set to a shared directory ID to use a shared directory.++Properties:++- Config: root_folder_id+- Env Var: RCLONE_PIXELDRAIN_ROOT_FOLDER_ID+- Type: string+- Default: "me"++### Advanced options++Here are the Advanced options specific to pixeldrain (Pixeldrain Filesystem).++#### --pixeldrain-api-url++The API endpoint to connect to. In the vast majority of cases it's fine to leave+this at default. It is only intended to be changed for testing purposes.++Properties:++- Config: api_url+- Env Var: RCLONE_PIXELDRAIN_API_URL+- Type: string+- Default: "https://pixeldrain.com/api"++#### --pixeldrain-description++Description of the remote.++Properties:++- Config: description+- Env Var: RCLONE_PIXELDRAIN_DESCRIPTION+- Type: string+- Required: false++### Metadata++Pixeldrain supports file modes and creation times.++Here are the possible system metadata items for the pixeldrain backend.++| Name | Help | Type | Example | Read Only |+|------|------|------|---------|-----------|+| btime | Time of file birth (creation) | RFC 3339 | 2006-01-02T15:04:05.999999999Z07:00 | N |+| mode | File mode | octal, unix style | 755 | N |+| mtime | Time of last modification | RFC 3339 | 2006-01-02T15:04:05.999999999Z07:00 | N |++See the [metadata](/docs/#metadata) docs for more info.++{{< rem autogenerated options stop >}}diff --git a/docs/layouts/chrome/navbar.html b/docs/layouts/chrome/navbar.htmlindex 71d5174d1..1c67b098b 100644--- a/docs/layouts/chrome/navbar.html+++ b/docs/layouts/chrome/navbar.html@@ -90,6 +90,7 @@<a class="dropdown-item" href="/oracleobjectstorage/"><i class="fa fa-cloud fa-fw"></i> Oracle Object Storage</a><a class="dropdown-item" href="/pcloud/"><i class="fa fa-cloud fa-fw"></i> pCloud</a><a class="dropdown-item" href="/pikpak/"><i class="fa fa-cloud fa-fw"></i> PikPak</a>+ <a class="dropdown-item" href="/pixeldrain/"><i class="fa fa-circle fa-fw"></i> Pixeldrain</a><a class="dropdown-item" href="/premiumizeme/"><i class="fa fa-user fa-fw"></i> premiumize.me</a><a class="dropdown-item" href="/putio/"><i class="fas fa-parking fa-fw"></i> put.io</a><a class="dropdown-item" href="/protondrive/"><i class="fas fa-folder fa-fw"></i> Proton Drive</a>diff --git a/fstest/test_all/config.yaml b/fstest/test_all/config.yamlindex 37a561b81..155f3093f 100644--- a/fstest/test_all/config.yaml+++ b/fstest/test_all/config.yaml@@ -364,6 +364,11 @@ backends:# This test fails with message# "share_status_prohibited" (9): Sorry, the sharing service is under maintenance in the current region.- TestIntegration/FsMkdir/FsPutFiles/PublicLink+ - backend: "pixeldrain"+ remote: "TestPixeldrain:"+ ignore:+ - TestIntegration/FsMkdir/FsEncoding/invalid_UTF-8+ fastlist: false- backend: "webdav"remote: "TestWebdavNextcloud:"ignore: